diff --git a/tierb_study_20260826_144644/STATUS.txt b/tierb_study_20260826_144644/STATUS.txt index c6ce21253c327cc251bd5028146d79fe0a14f339..7da3fa4df198a0fefe876e83313aa6e4b4aa3415 100644 --- a/tierb_study_20260826_144644/STATUS.txt +++ b/tierb_study_20260826_144644/STATUS.txt @@ -1,6 +1,6 @@ study_root: /workspace/tierb_study_20260826_144644 gbs: 8 -updated: 2026-08-26T15:22:49+00:00 +updated: 2026-08-26T15:32:33+00:00 current: echo__tierbprobe__gbs8 completed steps: diff --git a/tierb_study_20260826_144644/echo/tierbprobe/manifest.txt b/tierb_study_20260826_144644/echo/tierbprobe/manifest.txt index d9e47cde374dda55960e70d3dd5851cc213c128d..44f873a53d6864aff316fa96da0ebaa8719fcec2 100644 --- a/tierb_study_20260826_144644/echo/tierbprobe/manifest.txt +++ b/tierb_study_20260826_144644/echo/tierbprobe/manifest.txt @@ -1,9 +1,9 @@ -timestamp: 2026-08-26T14:46:59+00:00 +timestamp: 2026-08-26T15:23:04+00:00 host: sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 gpu: NVIDIA B200 x8 driver: 580.95.05 study_scripts_branch: granularity-echo -study_scripts_commit: 6b6ef3d +study_scripts_commit: 7ec1185 campaign: echo backend: echo phase: tierbprobe diff --git a/tierb_study_20260826_144644/echo/tierbprobe/provenance.txt b/tierb_study_20260826_144644/echo/tierbprobe/provenance.txt index 497ec58ff0ee9ee6ee8bcd8e7757cfd2c8c420d1..9bd9a0169d4cf9311e598d8ed2f9dcd2f3ad7d85 100644 --- a/tierb_study_20260826_144644/echo/tierbprobe/provenance.txt +++ b/tierb_study_20260826_144644/echo/tierbprobe/provenance.txt @@ -1,7 +1,7 @@ ============================================================== PROVENANCE: echo__tierbprobe__gbs8 ============================================================== -captured_at : 2026-08-26T15:22:49+00:00 +captured_at : 2026-08-26T15:32:33+00:00 hostname : sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 kernel : 6.8.0-1043-nvidia cwd : /workspace/blackwell_run_scripts @@ -18,7 +18,7 @@ command : run_ladder.sh --phase tierbprobe --gbs 8 --out /workspace/tierb_ 7, NVIDIA B200, 10.0, 183359 MiB, 580.95.05 persistence/clocks: 0, 120 MHz, 1965 MHz, 1000.00 W, 28 - 1, 120 MHz, 1965 MHz, 1000.00 W, 32 + 1, 120 MHz, 1965 MHz, 1000.00 W, 33 2, 120 MHz, 1965 MHz, 1000.00 W, 32 3, 120 MHz, 1965 MHz, 1000.00 W, 29 4, 120 MHz, 1965 MHz, 1000.00 W, 28 @@ -27,3 +27,22 @@ command : run_ladder.sh --phase tierbprobe --gbs 8 --out /workspace/tierb_ 7, 120 MHz, 1965 MHz, 1000.00 W, 28 NVLink topology: + GPU0 GPU1 GPU2 GPU3 GPU4 GPU5 GPU6 GPU7 NIC0 NIC1 NIC2 NIC3 NIC4 NIC5 NIC6 NIC7 NIC8 CPU Affinity NUMA Affinity GPU NUMA ID + GPU0 X NV18 NV18 NV18 NV18 NV18 NV18 NV18 PIX NODE SYS SYS SYS SYS SYS SYS NODE 0-27 0 N/A + GPU1 NV18 X NV18 NV18 NV18 NV18 NV18 NV18 NODE PIX SYS SYS SYS SYS SYS SYS NODE 0-27 0 N/A + GPU2 NV18 NV18 X NV18 NV18 NV18 NV18 NV18 SYS SYS PIX NODE SYS SYS SYS SYS SYS 28-55 1 N/A + GPU3 NV18 NV18 NV18 X NV18 NV18 NV18 NV18 SYS SYS NODE PIX SYS SYS SYS SYS SYS 28-55 1 N/A + GPU4 NV18 NV18 NV18 NV18 X NV18 NV18 NV18 SYS SYS SYS SYS PIX NODE SYS SYS SYS 56-83 2 N/A + GPU5 NV18 NV18 NV18 NV18 NV18 X NV18 NV18 SYS SYS SYS SYS NODE PIX SYS SYS SYS 56-83 2 N/A + GPU6 NV18 NV18 NV18 NV18 NV18 NV18 X NV18 SYS SYS SYS SYS SYS SYS PIX NODE SYS 84-111 3 N/A + GPU7 NV18 NV18 NV18 NV18 NV18 NV18 NV18 X SYS SYS SYS SYS SYS SYS NODE PIX SYS 84-111 3 N/A + NIC0 PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS SYS SYS SYS NODE + NIC1 NODE PIX SYS SYS SYS SYS SYS SYS NODE X SYS SYS SYS SYS SYS SYS NODE + NIC2 SYS SYS PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS SYS SYS + NIC3 SYS SYS NODE PIX SYS SYS SYS SYS SYS SYS NODE X SYS SYS SYS SYS SYS + NIC4 SYS SYS SYS SYS PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS + unavailable + +---- toolchain ----------------------------------------------- + nvcc : release 12.9, V12.9.86 + gcc : gcc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0 diff --git a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1/mem_samples.csv b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1/mem_samples.csv index 7de13aef3bfb46e27735060451ff501226dcf89f..085b71431e089791771c1f66476641a7b742b098 100644 --- a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1/mem_samples.csv +++ b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1/mem_samples.csv @@ -38,235 +38,11 @@ 5, 4 6, 4 7, 4 -0, 4 -1, 4 -2, 4 -3, 4 -4, 4 -5, 4 -6, 4 -7, 4 -0, 821 -1, 821 -2, 821 -3, 821 -4, 821 -5, 821 +0, 819 +1, 795 +2, 801 +3, 809 +4, 819 +5, 801 6, 819 -7, 821 -0, 1537 -1, 1537 -2, 1537 -3, 1537 -4, 1537 -5, 1537 -6, 1537 -7, 1537 -0, 2555 -1, 2555 -2, 2555 -3, 2555 -4, 2555 -5, 2555 -6, 2555 -7, 2555 -0, 26289 -1, 26377 -2, 26223 -3, 26223 -4, 26223 -5, 26223 -6, 26355 -7, 26377 -0, 32143 -1, 32229 -2, 32079 -3, 32077 -4, 32077 -5, 32075 -6, 32209 -7, 32229 -0, 36477 -1, 36593 -2, 36443 -3, 36441 -4, 36441 -5, 36429 -6, 36547 -7, 36579 -0, 37783 -1, 37867 -2, 37717 -3, 37715 -4, 37715 -5, 37713 -6, 37849 -7, 37867 -0, 37871 -1, 37955 -2, 37805 -3, 37803 -4, 37803 -5, 37801 -6, 37937 -7, 37955 -0, 37871 -1, 37955 -2, 37805 -3, 37803 -4, 37803 -5, 37801 -6, 37937 -7, 37955 -0, 38519 -1, 38603 -2, 38453 -3, 38451 -4, 38451 -5, 38449 -6, 38585 -7, 38603 -0, 38819 -1, 38903 -2, 38753 -3, 38753 -4, 38753 -5, 38749 -6, 38885 -7, 38903 -0, 38819 -1, 38905 -2, 38753 -3, 38753 -4, 38753 -5, 38751 -6, 38885 -7, 38905 -0, 38827 -1, 38913 -2, 38763 -3, 38761 -4, 38761 -5, 38759 -6, 38893 -7, 38913 -0, 38831 -1, 38915 -2, 38765 -3, 38763 -4, 38763 -5, 38761 -6, 38897 -7, 38915 -0, 121381 -1, 121673 -2, 121511 -3, 121513 -4, 121471 -5, 121563 -6, 121637 -7, 121651 -0, 121857 -1, 122153 -2, 121983 -3, 121983 -4, 121983 -5, 122063 -6, 122133 -7, 122153 -0, 121857 -1, 122153 -2, 121983 -3, 121983 -4, 121983 -5, 122063 -6, 122133 -7, 122153 -0, 121857 -1, 122153 -2, 121983 -3, 121983 -4, 121983 -5, 122063 -6, 122133 -7, 122153 -0, 121857 -1, 122153 -2, 121983 -3, 121983 -4, 121983 -5, 122063 -6, 122133 -7, 122153 -0, 122307 -1, 122615 -2, 122403 -3, 122425 -4, 122385 -5, 122475 -6, 122541 -7, 122587 -0, 123267 -1, 125593 -2, 125483 -3, 125433 -4, 125453 -5, 125541 -6, 125603 -7, 125603 -0, 124109 -1, 126453 -2, 126283 -3, 126283 -4, 126283 -5, 126363 -6, 126433 -7, 126453 -0, 139007 -1, 138657 -2, 138497 -3, 138493 -4, 138493 -5, 138567 -6, 140755 -7, 140961 -0, 139007 -1, 138657 -2, 138497 -3, 138493 -4, 138493 -5, 138567 -6, 140755 -7, 140961 -0, 139007 -1, 138657 -2, 138497 -3, 138493 -4, 138493 -5, 138567 -6, 140755 -7, 140961 -0, 139007 -1, 138657 -2, 138497 -3, 138493 -4, 138493 -5, 138567 -6, 140755 -7, 140961 -0, 139007 -1, 138657 -2, 138497 -3, 138493 -4, 138493 -5, 138567 -6, 140755 -7, 140961 -0, 775 -1, 778 -2, 778 -3, 1098 -4, 138191 -5, 2378 -6, 778 -7, 778 +7, 809 diff --git a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1/train.log b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1/train.log index b9c65dcc584081f5aa67359b7c277cfbd2c6d9a9..f477078d4614c92d42df6b3c88effef3e5e265c7 100644 --- a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1/train.log +++ b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1/train.log @@ -1,4 +1,4 @@ -[14:52:33] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +[15:28:32] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) ============================================================== DSV2_LITE_L24 / echo (slots/rank=8 policy=matched) E=64 K=6 H'=1408 m=8 @@ -7,10 +7,10 @@ memory_dump=none snapshots=/workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1/memory_snapshots/snapshot.pickle repo=/workspace/Megatron-MoE-Echo @ b099fd84 ============================================================== -W0826 14:52:34.770000 27578 torch/distributed/run.py:766] -W0826 14:52:34.770000 27578 torch/distributed/run.py:766] ***************************************** -W0826 14:52:34.770000 27578 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -W0826 14:52:34.770000 27578 torch/distributed/run.py:766] ***************************************** +W0826 15:28:33.569000 93676 torch/distributed/run.py:766] +W0826 15:28:33.569000 93676 torch/distributed/run.py:766] ***************************************** +W0826 15:28:33.569000 93676 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 15:28:33.569000 93676 torch/distributed/run.py:766] ***************************************** using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer Number of virtual stages per pipeline stage: None @@ -558,8 +558,6 @@ using torch.bfloat16 for parameters ... -------------------- end of arguments --------------------- INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 > building GPT2BPETokenizer tokenizer ... -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS > initializing torch distributed ... @@ -567,6 +565,9 @@ WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled @@ -575,41 +576,40 @@ WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it [Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank 3465 is connected to is connected to 2 is connected to is connected to 77 is connected to 77 peer ranks. peer ranks. 7 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77Expected number of connected peer ranks is : 77 +[Gloo] Rank [Gloo] Rank 2 is connected to 47 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 7 is connected to 67 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : 7 - - -[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 2 is connected to 17 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 [Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank 357 is connected to 6 is connected to is connected to 7 is connected to 77 peer ranks. 7 peer ranks. peer ranks. Expected number of connected peer ranks is : peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 7Expected number of connected peer ranks is : +77 +7 + [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 > initialized tensor model parallel with size 1 > initialized pipeline model parallel with size 1 > setting random seeds to 1234 ... -[Gloo] Rank 0 is connected to 0 peer ranks. [Gloo] Rank Expected number of connected peer ranks is : [Gloo] Rank [Gloo] Rank 0000 is connected to is connected to - is connected to 000 peer ranks. peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 -0 - [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[rank4]:[W826 14:52:52.967964758 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank3]:[W826 14:52:52.968112568 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank1]:[W826 14:52:52.968125289 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank5]:[W826 14:52:52.968146714 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank6]:[W826 14:52:52.969375088 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank7]:[W826 14:52:52.969491722 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank2]:[W826 14:52:52.969677566 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank4]:[W826 15:28:51.489979107 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 15:28:51.491562568 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 15:28:51.491595213 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 15:28:51.491712157 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 15:28:51.496604036 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 15:28:51.496678451 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 15:28:51.501207527 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. > compiling dataset index builder ... make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' make: Nothing to be done for 'default'. @@ -617,10 +617,10 @@ make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' >>> done with dataset index builder. Compilation time: 0.033 seconds WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. > compiling and loading fused kernels ... -[rank0]:[W826 14:52:53.128320646 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. ->>> done with compiling and loading fused kernels. Compilation time: 3.601 seconds -time to initialize megatron (seconds): 7.861 -[after megatron is initialized] datetime: 2026-08-26 14:52:59 +[rank0]:[W826 15:28:51.650644079 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.709 seconds +time to initialize megatron (seconds): 7.069 +[after megatron is initialized] datetime: 2026-08-26 15:28:57 building GPT model ... /workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1454: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. warnings.warn( @@ -660,21 +660,23 @@ INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradi Params for bucket 1 (103022592 elements, 103022592 padded size): module.output_layer.weight Params for bucket 2 (48503296 elements, 48503296 padded size): + module.decoder.layers.23.mlp.router.weight module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight module.decoder.layers.23.self_attention.linear_kv_up_proj.weight module.decoder.layers.23.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.23.self_attention.linear_kv_down_proj.weight - module.decoder.layers.23.self_attention.linear_q_proj.weight module.decoder.layers.23.self_attention.linear_proj.weight + module.decoder.layers.23.input_layernorm.weight + module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight module.decoder.final_layernorm.weight - module.decoder.layers.23.mlp.router.weight - module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight module.decoder.layers.23.pre_mlp_layernorm.weight - module.decoder.layers.23.input_layernorm.weight - module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.23.self_attention.linear_q_proj.weight Params for bucket 3 (40901632 elements, 40901632 padded size): - module.decoder.layers.22.mlp.router.weight + module.decoder.layers.22.self_attention.linear_kv_up_proj.weight + module.decoder.layers.22.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.22.self_attention.linear_kv_down_proj.weight module.decoder.layers.22.self_attention.linear_q_proj.weight module.decoder.layers.22.self_attention.linear_proj.weight module.decoder.layers.22.input_layernorm.weight @@ -682,283 +684,281 @@ Params for bucket 3 (40901632 elements, 40901632 padded size): module.decoder.layers.21.self_attention.linear_kv_up_proj.weight module.decoder.layers.21.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.21.self_attention.linear_kv_down_proj.weight + module.decoder.layers.22.mlp.router.weight module.decoder.layers.22.pre_mlp_layernorm.weight - module.decoder.layers.22.self_attention.linear_kv_up_proj.weight - module.decoder.layers.22.self_attention.linear_kv_down_proj.weight - module.decoder.layers.22.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight module.decoder.layers.21.mlp.router.weight module.decoder.layers.21.pre_mlp_layernorm.weight module.decoder.layers.21.self_attention.linear_q_proj.weight Params for bucket 4 (41163264 elements, 41163264 padded size): - module.decoder.layers.21.self_attention.linear_proj.weight module.decoder.layers.21.input_layernorm.weight - module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.20.mlp.router.weight + module.decoder.layers.20.pre_mlp_layernorm.weight + module.decoder.layers.20.self_attention.linear_proj.weight + module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.21.self_attention.linear_proj.weight + module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight module.decoder.layers.20.self_attention.linear_kv_up_proj.weight module.decoder.layers.20.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.20.self_attention.linear_kv_down_proj.weight - module.decoder.layers.20.self_attention.linear_proj.weight - module.decoder.layers.20.input_layernorm.weight - module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.20.mlp.router.weight - module.decoder.layers.20.pre_mlp_layernorm.weight module.decoder.layers.20.self_attention.linear_q_proj.weight + module.decoder.layers.20.input_layernorm.weight Params for bucket 5 (42734080 elements, 42734080 padded size): - module.decoder.layers.19.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.19.self_attention.linear_kv_up_proj.weight - module.decoder.layers.19.self_attention.linear_kv_down_proj.weight - module.decoder.layers.19.input_layernorm.weight - module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight module.decoder.layers.19.pre_mlp_layernorm.weight module.decoder.layers.19.mlp.router.weight module.decoder.layers.19.self_attention.linear_q_proj.weight module.decoder.layers.19.self_attention.linear_proj.weight module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.19.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.19.self_attention.linear_kv_up_proj.weight + module.decoder.layers.19.self_attention.linear_kv_down_proj.weight + module.decoder.layers.19.input_layernorm.weight + module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight Params for bucket 6 (40901632 elements, 40901632 padded size): - module.decoder.layers.18.mlp.router.weight - module.decoder.layers.18.pre_mlp_layernorm.weight - module.decoder.layers.18.self_attention.linear_q_proj.weight + module.decoder.layers.18.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.18.self_attention.linear_kv_down_proj.weight + module.decoder.layers.18.self_attention.linear_kv_up_proj.weight module.decoder.layers.18.self_attention.linear_proj.weight - module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight module.decoder.layers.17.mlp.router.weight module.decoder.layers.17.pre_mlp_layernorm.weight - module.decoder.layers.17.self_attention.linear_q_proj.weight - module.decoder.layers.18.self_attention.linear_kv_up_proj.weight - module.decoder.layers.18.self_attention.linear_kv_down_proj.weight - module.decoder.layers.18.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.18.mlp.router.weight + module.decoder.layers.18.pre_mlp_layernorm.weight + module.decoder.layers.18.self_attention.linear_q_proj.weight module.decoder.layers.18.input_layernorm.weight - module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight module.decoder.layers.17.self_attention.linear_kv_up_proj.weight module.decoder.layers.17.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.17.self_attention.linear_kv_down_proj.weight + module.decoder.layers.17.self_attention.linear_q_proj.weight Params for bucket 7 (41163264 elements, 41163264 padded size): module.decoder.layers.16.mlp.router.weight + module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight module.decoder.layers.16.pre_mlp_layernorm.weight module.decoder.layers.16.self_attention.linear_q_proj.weight - module.decoder.layers.16.self_attention.linear_proj.weight - module.decoder.layers.16.input_layernorm.weight module.decoder.layers.17.self_attention.linear_proj.weight - module.decoder.layers.17.input_layernorm.weight - module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.17.input_layernorm.weight module.decoder.layers.16.self_attention.linear_kv_up_proj.weight module.decoder.layers.16.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.16.self_attention.linear_kv_down_proj.weight + module.decoder.layers.16.self_attention.linear_proj.weight + module.decoder.layers.16.input_layernorm.weight module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight Params for bucket 8 (42734080 elements, 42734080 padded size): module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.15.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.15.self_attention.linear_kv_up_proj.weight - module.decoder.layers.15.self_attention.linear_kv_down_proj.weight + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.15.mlp.router.weight module.decoder.layers.15.self_attention.linear_q_proj.weight module.decoder.layers.15.self_attention.linear_proj.weight - module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.15.pre_mlp_layernorm.weight - module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.15.self_attention.linear_kv_up_proj.weight + module.decoder.layers.15.self_attention.linear_kv_down_proj.weight module.decoder.layers.15.input_layernorm.weight + module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight Params for bucket 9 (40901632 elements, 40901632 padded size): - module.decoder.layers.14.self_attention.linear_kv_up_proj.weight - module.decoder.layers.14.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.14.self_attention.linear_kv_down_proj.weight + module.decoder.layers.14.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.14.self_attention.linear_kv_up_proj.weight + module.decoder.layers.14.self_attention.linear_q_proj.weight module.decoder.layers.14.self_attention.linear_proj.weight module.decoder.layers.14.input_layernorm.weight - module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight module.decoder.layers.13.self_attention.linear_kv_up_proj.weight module.decoder.layers.13.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.13.self_attention.linear_kv_down_proj.weight module.decoder.layers.13.self_attention.linear_q_proj.weight module.decoder.layers.14.mlp.router.weight module.decoder.layers.14.pre_mlp_layernorm.weight - module.decoder.layers.14.self_attention.linear_q_proj.weight - module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight module.decoder.layers.13.mlp.router.weight module.decoder.layers.13.pre_mlp_layernorm.weight Params for bucket 10 (41163264 elements, 41163264 padded size): - module.decoder.layers.13.self_attention.linear_proj.weight - module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight module.decoder.layers.13.input_layernorm.weight - module.decoder.layers.12.pre_mlp_layernorm.weight - module.decoder.layers.12.self_attention.linear_q_proj.weight - module.decoder.layers.12.self_attention.linear_proj.weight - module.decoder.layers.12.input_layernorm.weight - module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight module.decoder.layers.12.self_attention.linear_kv_up_proj.weight module.decoder.layers.12.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.12.self_attention.linear_kv_down_proj.weight + module.decoder.layers.12.self_attention.linear_proj.weight module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.12.self_attention.linear_q_proj.weight + module.decoder.layers.12.input_layernorm.weight Params for bucket 11 (42734080 elements, 42734080 padded size): - module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.11.self_attention.linear_kv_up_proj.weight - module.decoder.layers.11.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.11.self_attention.linear_kv_down_proj.weight module.decoder.layers.11.pre_mlp_layernorm.weight module.decoder.layers.11.mlp.router.weight module.decoder.layers.11.self_attention.linear_q_proj.weight + module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.11.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.11.self_attention.linear_kv_up_proj.weight + module.decoder.layers.11.self_attention.linear_kv_down_proj.weight module.decoder.layers.11.self_attention.linear_proj.weight module.decoder.layers.11.input_layernorm.weight module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight Params for bucket 12 (40901632 elements, 40901632 padded size): module.decoder.layers.10.mlp.router.weight - module.decoder.layers.10.self_attention.linear_kv_up_proj.weight - module.decoder.layers.10.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.10.self_attention.linear_kv_down_proj.weight + module.decoder.layers.10.pre_mlp_layernorm.weight module.decoder.layers.10.self_attention.linear_q_proj.weight module.decoder.layers.10.self_attention.linear_proj.weight module.decoder.layers.10.input_layernorm.weight module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.9.pre_mlp_layernorm.weight module.decoder.layers.9.self_attention.linear_kv_up_proj.weight - module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.9.self_attention.linear_kv_down_proj.weight + module.decoder.layers.10.self_attention.linear_kv_up_proj.weight + module.decoder.layers.10.self_attention.linear_kv_down_proj.weight + module.decoder.layers.10.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight module.decoder.layers.9.mlp.router.weight - module.decoder.layers.9.self_attention.linear_kv_down_proj.weight + module.decoder.layers.9.pre_mlp_layernorm.weight module.decoder.layers.9.self_attention.linear_q_proj.weight - module.decoder.layers.9.self_attention.linear_kv_up_proj.layer_norm_weight Params for bucket 13 (41163264 elements, 41163264 padded size): module.decoder.layers.9.self_attention.linear_proj.weight module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.9.input_layernorm.weight + module.decoder.layers.8.mlp.router.weight module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.input_layernorm.weight + module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.9.input_layernorm.weight + module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight module.decoder.layers.8.self_attention.linear_kv_up_proj.weight module.decoder.layers.8.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.8.self_attention.linear_kv_down_proj.weight module.decoder.layers.8.self_attention.linear_q_proj.weight module.decoder.layers.8.self_attention.linear_proj.weight - module.decoder.layers.8.input_layernorm.weight - module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.8.mlp.router.weight - module.decoder.layers.8.self_attention.linear_kv_down_proj.weight - module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight Params for bucket 14 (42734080 elements, 42734080 padded size): - module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.7.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.7.self_attention.linear_kv_down_proj.weight - module.decoder.layers.7.self_attention.linear_proj.weight - module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight module.decoder.layers.7.self_attention.linear_kv_up_proj.weight module.decoder.layers.7.pre_mlp_layernorm.weight module.decoder.layers.7.mlp.router.weight module.decoder.layers.7.self_attention.linear_q_proj.weight + module.decoder.layers.7.self_attention.linear_proj.weight module.decoder.layers.7.input_layernorm.weight module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.7.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.7.self_attention.linear_kv_down_proj.weight + module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight Params for bucket 15 (40901632 elements, 40901632 padded size): - module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.6.self_attention.linear_kv_down_proj.weight + module.decoder.layers.6.self_attention.linear_kv_up_proj.weight + module.decoder.layers.6.self_attention.linear_proj.weight module.decoder.layers.6.input_layernorm.weight module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight module.decoder.layers.5.mlp.router.weight module.decoder.layers.5.pre_mlp_layernorm.weight module.decoder.layers.5.self_attention.linear_q_proj.weight - module.decoder.layers.6.self_attention.linear_kv_up_proj.weight - module.decoder.layers.6.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.6.mlp.router.weight module.decoder.layers.6.pre_mlp_layernorm.weight module.decoder.layers.6.self_attention.linear_q_proj.weight - module.decoder.layers.6.self_attention.linear_proj.weight module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight module.decoder.layers.5.self_attention.linear_kv_up_proj.weight module.decoder.layers.5.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.5.self_attention.linear_kv_down_proj.weight Params for bucket 16 (41163264 elements, 41163264 padded size): - module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.4.mlp.router.weight - module.decoder.layers.4.pre_mlp_layernorm.weight - module.decoder.layers.4.self_attention.linear_q_proj.weight - module.decoder.layers.4.self_attention.linear_proj.weight module.decoder.layers.5.self_attention.linear_proj.weight - module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight module.decoder.layers.5.input_layernorm.weight module.decoder.layers.4.self_attention.linear_kv_up_proj.weight module.decoder.layers.4.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.4.self_attention.linear_kv_down_proj.weight + module.decoder.layers.4.self_attention.linear_q_proj.weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.pre_mlp_layernorm.weight module.decoder.layers.4.input_layernorm.weight module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight Params for bucket 17 (42734080 elements, 42734080 padded size): + module.decoder.layers.3.mlp.router.weight module.decoder.layers.3.pre_mlp_layernorm.weight module.decoder.layers.3.self_attention.linear_proj.weight module.decoder.layers.3.input_layernorm.weight module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.3.mlp.router.weight - module.decoder.layers.3.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.3.self_attention.linear_kv_up_proj.weight + module.decoder.layers.3.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.3.self_attention.linear_kv_down_proj.weight module.decoder.layers.3.self_attention.linear_q_proj.weight module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight Params for bucket 18 (40901632 elements, 40901632 padded size): module.decoder.layers.2.self_attention.linear_kv_up_proj.weight - module.decoder.layers.2.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.2.self_attention.linear_kv_down_proj.weight + module.decoder.layers.2.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.2.self_attention.linear_q_proj.weight + module.decoder.layers.2.self_attention.linear_proj.weight module.decoder.layers.1.mlp.router.weight - module.decoder.layers.1.self_attention.linear_kv_down_proj.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_kv_up_proj.weight + module.decoder.layers.1.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.1.self_attention.linear_q_proj.weight module.decoder.layers.2.mlp.router.weight module.decoder.layers.2.pre_mlp_layernorm.weight - module.decoder.layers.2.self_attention.linear_proj.weight module.decoder.layers.2.input_layernorm.weight module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.1.pre_mlp_layernorm.weight - module.decoder.layers.1.self_attention.linear_kv_up_proj.weight - module.decoder.layers.1.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.1.self_attention.linear_q_proj.weight + module.decoder.layers.1.self_attention.linear_kv_down_proj.weight Params for bucket 19 (71436288 elements, 71436288 padded size): module.decoder.layers.1.self_attention.linear_proj.weight - module.decoder.layers.0.mlp.linear_fc1.weight - module.decoder.layers.1.input_layernorm.weight module.decoder.layers.0.mlp.linear_fc2.weight + module.decoder.layers.1.input_layernorm.weight + module.decoder.layers.0.mlp.linear_fc1.weight Params for bucket 20 (116789760 elements, 116789760 padded size): - module.decoder.layers.0.mlp.linear_fc1.layer_norm_weight module.decoder.layers.0.self_attention.linear_kv_up_proj.weight - module.decoder.layers.0.self_attention.linear_q_proj.weight module.decoder.layers.0.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.0.input_layernorm.weight module.decoder.layers.0.self_attention.linear_kv_down_proj.weight + module.decoder.layers.0.input_layernorm.weight + module.decoder.layers.0.mlp.linear_fc1.layer_norm_weight + module.decoder.layers.0.self_attention.linear_q_proj.weight module.decoder.layers.0.self_attention.linear_proj.weight module.embedding.word_embeddings.weight INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 40 Params for bucket 1 (40370176 elements, 40370176 padded size): - module.decoder.layers.23.mlp.experts.linear_fc2.weight7 - module.decoder.layers.23.mlp.experts.linear_fc2.weight5 - module.decoder.layers.23.mlp.experts.linear_fc2.weight3 - module.decoder.layers.23.mlp.experts.linear_fc2.weight2 - module.decoder.layers.23.mlp.experts.linear_fc2.weight0 - module.decoder.layers.23.mlp.experts.linear_fc1.weight6 module.decoder.layers.23.mlp.experts.linear_fc2.weight4 module.decoder.layers.23.mlp.experts.linear_fc2.weight6 module.decoder.layers.23.mlp.experts.linear_fc2.weight1 module.decoder.layers.23.mlp.experts.linear_fc1.weight7 module.decoder.layers.23.mlp.experts.linear_fc1.weight5 + module.decoder.layers.23.mlp.experts.linear_fc2.weight7 + module.decoder.layers.23.mlp.experts.linear_fc2.weight3 + module.decoder.layers.23.mlp.experts.linear_fc2.weight5 + module.decoder.layers.23.mlp.experts.linear_fc2.weight2 + module.decoder.layers.23.mlp.experts.linear_fc2.weight0 + module.decoder.layers.23.mlp.experts.linear_fc1.weight6 Params for bucket 2 (40370176 elements, 40370176 padded size): + module.decoder.layers.23.mlp.experts.linear_fc1.weight2 + module.decoder.layers.23.mlp.experts.linear_fc1.weight1 + module.decoder.layers.22.mlp.experts.linear_fc2.weight6 + module.decoder.layers.22.mlp.experts.linear_fc2.weight4 module.decoder.layers.23.mlp.experts.linear_fc1.weight4 module.decoder.layers.23.mlp.experts.linear_fc1.weight0 module.decoder.layers.23.mlp.experts.linear_fc1.weight3 module.decoder.layers.22.mlp.experts.linear_fc2.weight7 module.decoder.layers.22.mlp.experts.linear_fc2.weight5 - module.decoder.layers.23.mlp.experts.linear_fc1.weight2 - module.decoder.layers.23.mlp.experts.linear_fc1.weight1 - module.decoder.layers.22.mlp.experts.linear_fc2.weight6 - module.decoder.layers.22.mlp.experts.linear_fc2.weight4 Params for bucket 3 (40370176 elements, 40370176 padded size): + module.decoder.layers.22.mlp.experts.linear_fc2.weight1 + module.decoder.layers.22.mlp.experts.linear_fc1.weight7 + module.decoder.layers.22.mlp.experts.linear_fc1.weight5 module.decoder.layers.22.mlp.experts.linear_fc2.weight3 module.decoder.layers.22.mlp.experts.linear_fc2.weight0 module.decoder.layers.22.mlp.experts.linear_fc2.weight2 module.decoder.layers.22.mlp.experts.linear_fc1.weight6 module.decoder.layers.22.mlp.experts.linear_fc1.weight4 module.decoder.layers.22.mlp.experts.linear_fc1.weight3 - module.decoder.layers.22.mlp.experts.linear_fc1.weight7 - module.decoder.layers.22.mlp.experts.linear_fc2.weight1 - module.decoder.layers.22.mlp.experts.linear_fc1.weight5 Params for bucket 4 (40370176 elements, 40370176 padded size): - module.decoder.layers.22.mlp.experts.linear_fc1.weight0 + module.decoder.layers.22.mlp.experts.linear_fc1.weight2 module.decoder.layers.21.mlp.experts.linear_fc2.weight6 + module.decoder.layers.22.mlp.experts.linear_fc1.weight1 module.decoder.layers.21.mlp.experts.linear_fc2.weight4 module.decoder.layers.21.mlp.experts.linear_fc2.weight1 - module.decoder.layers.22.mlp.experts.linear_fc1.weight2 - module.decoder.layers.22.mlp.experts.linear_fc1.weight1 + module.decoder.layers.22.mlp.experts.linear_fc1.weight0 module.decoder.layers.21.mlp.experts.linear_fc2.weight7 module.decoder.layers.21.mlp.experts.linear_fc2.weight5 module.decoder.layers.21.mlp.experts.linear_fc2.weight3 @@ -973,65 +973,65 @@ Params for bucket 5 (40370176 elements, 40370176 padded size): module.decoder.layers.21.mlp.experts.linear_fc1.weight6 module.decoder.layers.21.mlp.experts.linear_fc1.weight3 Params for bucket 6 (40370176 elements, 40370176 padded size): - module.decoder.layers.20.mlp.experts.linear_fc2.weight6 - module.decoder.layers.20.mlp.experts.linear_fc2.weight4 - module.decoder.layers.20.mlp.experts.linear_fc2.weight1 - module.decoder.layers.20.mlp.experts.linear_fc1.weight7 - module.decoder.layers.21.mlp.experts.linear_fc1.weight0 - module.decoder.layers.20.mlp.experts.linear_fc2.weight7 module.decoder.layers.20.mlp.experts.linear_fc2.weight5 + module.decoder.layers.20.mlp.experts.linear_fc2.weight7 module.decoder.layers.20.mlp.experts.linear_fc2.weight3 module.decoder.layers.20.mlp.experts.linear_fc2.weight2 module.decoder.layers.20.mlp.experts.linear_fc2.weight0 module.decoder.layers.20.mlp.experts.linear_fc1.weight6 + module.decoder.layers.21.mlp.experts.linear_fc1.weight0 + module.decoder.layers.20.mlp.experts.linear_fc2.weight6 + module.decoder.layers.20.mlp.experts.linear_fc2.weight4 + module.decoder.layers.20.mlp.experts.linear_fc2.weight1 + module.decoder.layers.20.mlp.experts.linear_fc1.weight7 Params for bucket 7 (40370176 elements, 40370176 padded size): - module.decoder.layers.20.mlp.experts.linear_fc1.weight5 - module.decoder.layers.20.mlp.experts.linear_fc1.weight1 - module.decoder.layers.20.mlp.experts.linear_fc1.weight2 - module.decoder.layers.19.mlp.experts.linear_fc2.weight6 - module.decoder.layers.20.mlp.experts.linear_fc1.weight3 module.decoder.layers.20.mlp.experts.linear_fc1.weight4 + module.decoder.layers.20.mlp.experts.linear_fc1.weight3 module.decoder.layers.20.mlp.experts.linear_fc1.weight0 + module.decoder.layers.19.mlp.experts.linear_fc2.weight6 + module.decoder.layers.20.mlp.experts.linear_fc1.weight5 + module.decoder.layers.20.mlp.experts.linear_fc1.weight2 + module.decoder.layers.20.mlp.experts.linear_fc1.weight1 module.decoder.layers.19.mlp.experts.linear_fc2.weight7 Params for bucket 8 (40370176 elements, 40370176 padded size): - module.decoder.layers.19.mlp.experts.linear_fc2.weight4 module.decoder.layers.19.mlp.experts.linear_fc2.weight1 - module.decoder.layers.19.mlp.experts.linear_fc1.weight7 - module.decoder.layers.19.mlp.experts.linear_fc1.weight5 - module.decoder.layers.19.mlp.experts.linear_fc2.weight5 module.decoder.layers.19.mlp.experts.linear_fc2.weight2 - module.decoder.layers.19.mlp.experts.linear_fc2.weight3 - module.decoder.layers.19.mlp.experts.linear_fc2.weight0 + module.decoder.layers.19.mlp.experts.linear_fc2.weight4 module.decoder.layers.19.mlp.experts.linear_fc1.weight6 module.decoder.layers.19.mlp.experts.linear_fc1.weight4 + module.decoder.layers.19.mlp.experts.linear_fc2.weight5 + module.decoder.layers.19.mlp.experts.linear_fc2.weight3 + module.decoder.layers.19.mlp.experts.linear_fc2.weight0 + module.decoder.layers.19.mlp.experts.linear_fc1.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight5 Params for bucket 9 (40370176 elements, 40370176 padded size): + module.decoder.layers.19.mlp.experts.linear_fc1.weight3 + module.decoder.layers.19.mlp.experts.linear_fc1.weight0 + module.decoder.layers.18.mlp.experts.linear_fc2.weight6 + module.decoder.layers.18.mlp.experts.linear_fc2.weight4 module.decoder.layers.19.mlp.experts.linear_fc1.weight2 module.decoder.layers.18.mlp.experts.linear_fc2.weight7 module.decoder.layers.19.mlp.experts.linear_fc1.weight1 module.decoder.layers.18.mlp.experts.linear_fc2.weight5 module.decoder.layers.18.mlp.experts.linear_fc2.weight3 module.decoder.layers.18.mlp.experts.linear_fc2.weight2 - module.decoder.layers.19.mlp.experts.linear_fc1.weight3 - module.decoder.layers.19.mlp.experts.linear_fc1.weight0 - module.decoder.layers.18.mlp.experts.linear_fc2.weight6 - module.decoder.layers.18.mlp.experts.linear_fc2.weight4 Params for bucket 10 (40370176 elements, 40370176 padded size): - module.decoder.layers.18.mlp.experts.linear_fc2.weight0 - module.decoder.layers.18.mlp.experts.linear_fc1.weight6 - module.decoder.layers.18.mlp.experts.linear_fc1.weight4 - module.decoder.layers.18.mlp.experts.linear_fc1.weight3 module.decoder.layers.18.mlp.experts.linear_fc2.weight1 module.decoder.layers.18.mlp.experts.linear_fc1.weight7 module.decoder.layers.18.mlp.experts.linear_fc1.weight5 module.decoder.layers.18.mlp.experts.linear_fc1.weight2 + module.decoder.layers.18.mlp.experts.linear_fc1.weight6 + module.decoder.layers.18.mlp.experts.linear_fc2.weight0 + module.decoder.layers.18.mlp.experts.linear_fc1.weight4 + module.decoder.layers.18.mlp.experts.linear_fc1.weight3 Params for bucket 11 (40370176 elements, 40370176 padded size): + module.decoder.layers.18.mlp.experts.linear_fc1.weight1 module.decoder.layers.17.mlp.experts.linear_fc2.weight7 module.decoder.layers.17.mlp.experts.linear_fc2.weight5 - module.decoder.layers.18.mlp.experts.linear_fc1.weight0 module.decoder.layers.17.mlp.experts.linear_fc2.weight3 module.decoder.layers.17.mlp.experts.linear_fc2.weight2 module.decoder.layers.17.mlp.experts.linear_fc2.weight0 - module.decoder.layers.18.mlp.experts.linear_fc1.weight1 + module.decoder.layers.18.mlp.experts.linear_fc1.weight0 module.decoder.layers.17.mlp.experts.linear_fc2.weight6 module.decoder.layers.17.mlp.experts.linear_fc2.weight4 module.decoder.layers.17.mlp.experts.linear_fc2.weight1 @@ -1051,25 +1051,25 @@ Params for bucket 13 (40370176 elements, 40370176 padded size): module.decoder.layers.16.mlp.experts.linear_fc2.weight2 module.decoder.layers.16.mlp.experts.linear_fc2.weight0 module.decoder.layers.16.mlp.experts.linear_fc1.weight6 - module.decoder.layers.16.mlp.experts.linear_fc1.weight5 module.decoder.layers.16.mlp.experts.linear_fc2.weight6 module.decoder.layers.16.mlp.experts.linear_fc2.weight4 module.decoder.layers.16.mlp.experts.linear_fc2.weight1 module.decoder.layers.16.mlp.experts.linear_fc1.weight7 + module.decoder.layers.16.mlp.experts.linear_fc1.weight5 Params for bucket 14 (40370176 elements, 40370176 padded size): module.decoder.layers.16.mlp.experts.linear_fc1.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight3 module.decoder.layers.16.mlp.experts.linear_fc1.weight0 module.decoder.layers.15.mlp.experts.linear_fc2.weight7 module.decoder.layers.15.mlp.experts.linear_fc2.weight5 - module.decoder.layers.16.mlp.experts.linear_fc1.weight3 module.decoder.layers.16.mlp.experts.linear_fc1.weight1 module.decoder.layers.16.mlp.experts.linear_fc1.weight2 module.decoder.layers.15.mlp.experts.linear_fc2.weight6 module.decoder.layers.15.mlp.experts.linear_fc2.weight4 Params for bucket 15 (40370176 elements, 40370176 padded size): module.decoder.layers.15.mlp.experts.linear_fc2.weight3 - module.decoder.layers.15.mlp.experts.linear_fc2.weight0 module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 module.decoder.layers.15.mlp.experts.linear_fc1.weight6 module.decoder.layers.15.mlp.experts.linear_fc1.weight4 module.decoder.layers.15.mlp.experts.linear_fc1.weight3 @@ -1077,28 +1077,28 @@ Params for bucket 15 (40370176 elements, 40370176 padded size): module.decoder.layers.15.mlp.experts.linear_fc2.weight1 module.decoder.layers.15.mlp.experts.linear_fc1.weight5 Params for bucket 16 (40370176 elements, 40370176 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 module.decoder.layers.14.mlp.experts.linear_fc2.weight7 module.decoder.layers.14.mlp.experts.linear_fc2.weight5 module.decoder.layers.14.mlp.experts.linear_fc2.weight3 - module.decoder.layers.15.mlp.experts.linear_fc1.weight2 - module.decoder.layers.15.mlp.experts.linear_fc1.weight1 - module.decoder.layers.14.mlp.experts.linear_fc2.weight6 - module.decoder.layers.14.mlp.experts.linear_fc2.weight4 module.decoder.layers.14.mlp.experts.linear_fc2.weight2 - module.decoder.layers.14.mlp.experts.linear_fc2.weight1 module.decoder.layers.14.mlp.experts.linear_fc2.weight0 Params for bucket 17 (40370176 elements, 40370176 padded size): module.decoder.layers.14.mlp.experts.linear_fc1.weight7 module.decoder.layers.14.mlp.experts.linear_fc1.weight5 module.decoder.layers.14.mlp.experts.linear_fc1.weight2 module.decoder.layers.14.mlp.experts.linear_fc1.weight1 - module.decoder.layers.14.mlp.experts.linear_fc1.weight6 module.decoder.layers.14.mlp.experts.linear_fc1.weight3 module.decoder.layers.14.mlp.experts.linear_fc1.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 Params for bucket 18 (40370176 elements, 40370176 padded size): - module.decoder.layers.13.mlp.experts.linear_fc2.weight4 module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 module.decoder.layers.13.mlp.experts.linear_fc2.weight1 module.decoder.layers.13.mlp.experts.linear_fc1.weight7 module.decoder.layers.14.mlp.experts.linear_fc1.weight0 @@ -1112,36 +1112,36 @@ Params for bucket 19 (40370176 elements, 40370176 padded size): module.decoder.layers.13.mlp.experts.linear_fc1.weight5 module.decoder.layers.13.mlp.experts.linear_fc1.weight1 module.decoder.layers.13.mlp.experts.linear_fc1.weight2 - module.decoder.layers.12.mlp.experts.linear_fc2.weight6 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 module.decoder.layers.13.mlp.experts.linear_fc1.weight4 module.decoder.layers.13.mlp.experts.linear_fc1.weight3 module.decoder.layers.13.mlp.experts.linear_fc1.weight0 - module.decoder.layers.12.mlp.experts.linear_fc2.weight7 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 Params for bucket 20 (40370176 elements, 40370176 padded size): - module.decoder.layers.12.mlp.experts.linear_fc2.weight2 - module.decoder.layers.12.mlp.experts.linear_fc2.weight1 - module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 module.decoder.layers.12.mlp.experts.linear_fc2.weight0 module.decoder.layers.12.mlp.experts.linear_fc1.weight6 module.decoder.layers.12.mlp.experts.linear_fc1.weight4 - module.decoder.layers.12.mlp.experts.linear_fc2.weight5 - module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 module.decoder.layers.12.mlp.experts.linear_fc1.weight7 module.decoder.layers.12.mlp.experts.linear_fc1.weight5 Params for bucket 21 (40370176 elements, 40370176 padded size): module.decoder.layers.12.mlp.experts.linear_fc1.weight3 - module.decoder.layers.11.mlp.experts.linear_fc2.weight7 module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 module.decoder.layers.11.mlp.experts.linear_fc2.weight5 module.decoder.layers.11.mlp.experts.linear_fc2.weight3 module.decoder.layers.11.mlp.experts.linear_fc2.weight2 - module.decoder.layers.12.mlp.experts.linear_fc1.weight1 module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 module.decoder.layers.11.mlp.experts.linear_fc2.weight6 module.decoder.layers.11.mlp.experts.linear_fc2.weight4 Params for bucket 22 (40370176 elements, 40370176 padded size): - module.decoder.layers.11.mlp.experts.linear_fc2.weight0 module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 module.decoder.layers.11.mlp.experts.linear_fc1.weight4 module.decoder.layers.11.mlp.experts.linear_fc1.weight3 module.decoder.layers.11.mlp.experts.linear_fc2.weight1 @@ -1149,15 +1149,15 @@ Params for bucket 22 (40370176 elements, 40370176 padded size): module.decoder.layers.11.mlp.experts.linear_fc1.weight5 module.decoder.layers.11.mlp.experts.linear_fc1.weight2 Params for bucket 23 (40370176 elements, 40370176 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 module.decoder.layers.11.mlp.experts.linear_fc1.weight0 module.decoder.layers.10.mlp.experts.linear_fc2.weight7 - module.decoder.layers.10.mlp.experts.linear_fc2.weight5 module.decoder.layers.10.mlp.experts.linear_fc2.weight3 - module.decoder.layers.10.mlp.experts.linear_fc2.weight2 module.decoder.layers.10.mlp.experts.linear_fc2.weight0 module.decoder.layers.11.mlp.experts.linear_fc1.weight1 module.decoder.layers.10.mlp.experts.linear_fc2.weight6 module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 module.decoder.layers.10.mlp.experts.linear_fc2.weight1 module.decoder.layers.10.mlp.experts.linear_fc1.weight7 Params for bucket 24 (40370176 elements, 40370176 padded size): @@ -1165,12 +1165,12 @@ Params for bucket 24 (40370176 elements, 40370176 padded size): module.decoder.layers.10.mlp.experts.linear_fc1.weight4 module.decoder.layers.10.mlp.experts.linear_fc1.weight3 module.decoder.layers.10.mlp.experts.linear_fc1.weight0 - module.decoder.layers.10.mlp.experts.linear_fc1.weight2 module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 module.decoder.layers.10.mlp.experts.linear_fc1.weight1 Params for bucket 25 (40370176 elements, 40370176 padded size): - module.decoder.layers.9.mlp.experts.linear_fc2.weight6 module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 module.decoder.layers.9.mlp.experts.linear_fc2.weight1 module.decoder.layers.9.mlp.experts.linear_fc1.weight7 module.decoder.layers.9.mlp.experts.linear_fc1.weight5 @@ -1181,109 +1181,109 @@ Params for bucket 25 (40370176 elements, 40370176 padded size): module.decoder.layers.9.mlp.experts.linear_fc2.weight0 module.decoder.layers.9.mlp.experts.linear_fc1.weight6 Params for bucket 26 (40370176 elements, 40370176 padded size): - module.decoder.layers.9.mlp.experts.linear_fc1.weight1 module.decoder.layers.9.mlp.experts.linear_fc1.weight2 - module.decoder.layers.8.mlp.experts.linear_fc2.weight6 - module.decoder.layers.8.mlp.experts.linear_fc2.weight4 - module.decoder.layers.9.mlp.experts.linear_fc1.weight4 - module.decoder.layers.9.mlp.experts.linear_fc1.weight0 - module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 module.decoder.layers.8.mlp.experts.linear_fc2.weight7 module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 Params for bucket 27 (40370176 elements, 40370176 padded size): - module.decoder.layers.8.mlp.experts.linear_fc1.weight7 - module.decoder.layers.8.mlp.experts.linear_fc2.weight1 - module.decoder.layers.8.mlp.experts.linear_fc1.weight5 module.decoder.layers.8.mlp.experts.linear_fc2.weight3 - module.decoder.layers.8.mlp.experts.linear_fc2.weight0 module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 module.decoder.layers.8.mlp.experts.linear_fc1.weight6 module.decoder.layers.8.mlp.experts.linear_fc1.weight4 module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 Params for bucket 28 (40370176 elements, 40370176 padded size): - module.decoder.layers.8.mlp.experts.linear_fc1.weight2 - module.decoder.layers.7.mlp.experts.linear_fc2.weight6 - module.decoder.layers.8.mlp.experts.linear_fc1.weight1 - module.decoder.layers.7.mlp.experts.linear_fc2.weight4 - module.decoder.layers.7.mlp.experts.linear_fc2.weight1 - module.decoder.layers.7.mlp.experts.linear_fc2.weight7 module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 module.decoder.layers.7.mlp.experts.linear_fc2.weight5 module.decoder.layers.7.mlp.experts.linear_fc2.weight3 module.decoder.layers.7.mlp.experts.linear_fc2.weight2 module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 Params for bucket 29 (40370176 elements, 40370176 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 module.decoder.layers.7.mlp.experts.linear_fc1.weight7 module.decoder.layers.7.mlp.experts.linear_fc1.weight5 module.decoder.layers.7.mlp.experts.linear_fc1.weight2 module.decoder.layers.7.mlp.experts.linear_fc1.weight1 - module.decoder.layers.7.mlp.experts.linear_fc1.weight4 - module.decoder.layers.7.mlp.experts.linear_fc1.weight6 - module.decoder.layers.7.mlp.experts.linear_fc1.weight3 Params for bucket 30 (40370176 elements, 40370176 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 module.decoder.layers.6.mlp.experts.linear_fc2.weight7 module.decoder.layers.6.mlp.experts.linear_fc2.weight5 module.decoder.layers.6.mlp.experts.linear_fc2.weight3 module.decoder.layers.6.mlp.experts.linear_fc2.weight2 module.decoder.layers.6.mlp.experts.linear_fc2.weight0 module.decoder.layers.6.mlp.experts.linear_fc1.weight6 - module.decoder.layers.7.mlp.experts.linear_fc1.weight0 - module.decoder.layers.6.mlp.experts.linear_fc2.weight6 - module.decoder.layers.6.mlp.experts.linear_fc2.weight4 - module.decoder.layers.6.mlp.experts.linear_fc2.weight1 - module.decoder.layers.6.mlp.experts.linear_fc1.weight7 Params for bucket 31 (40370176 elements, 40370176 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 module.decoder.layers.6.mlp.experts.linear_fc1.weight3 module.decoder.layers.6.mlp.experts.linear_fc1.weight4 module.decoder.layers.6.mlp.experts.linear_fc1.weight0 - module.decoder.layers.5.mlp.experts.linear_fc2.weight7 - module.decoder.layers.6.mlp.experts.linear_fc1.weight5 - module.decoder.layers.6.mlp.experts.linear_fc1.weight2 - module.decoder.layers.6.mlp.experts.linear_fc1.weight1 module.decoder.layers.5.mlp.experts.linear_fc2.weight6 Params for bucket 32 (40370176 elements, 40370176 padded size): module.decoder.layers.5.mlp.experts.linear_fc2.weight5 - module.decoder.layers.5.mlp.experts.linear_fc2.weight2 module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 module.decoder.layers.5.mlp.experts.linear_fc2.weight0 module.decoder.layers.5.mlp.experts.linear_fc1.weight6 module.decoder.layers.5.mlp.experts.linear_fc1.weight4 - module.decoder.layers.5.mlp.experts.linear_fc2.weight1 module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 module.decoder.layers.5.mlp.experts.linear_fc1.weight7 module.decoder.layers.5.mlp.experts.linear_fc1.weight5 Params for bucket 33 (40370176 elements, 40370176 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight3 module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 module.decoder.layers.4.mlp.experts.linear_fc2.weight7 module.decoder.layers.4.mlp.experts.linear_fc2.weight5 module.decoder.layers.4.mlp.experts.linear_fc2.weight3 module.decoder.layers.4.mlp.experts.linear_fc2.weight2 - module.decoder.layers.5.mlp.experts.linear_fc1.weight2 - module.decoder.layers.5.mlp.experts.linear_fc1.weight1 - module.decoder.layers.4.mlp.experts.linear_fc2.weight6 - module.decoder.layers.4.mlp.experts.linear_fc2.weight4 Params for bucket 34 (40370176 elements, 40370176 padded size): - module.decoder.layers.4.mlp.experts.linear_fc1.weight6 - module.decoder.layers.4.mlp.experts.linear_fc2.weight0 - module.decoder.layers.4.mlp.experts.linear_fc1.weight4 - module.decoder.layers.4.mlp.experts.linear_fc1.weight3 module.decoder.layers.4.mlp.experts.linear_fc2.weight1 module.decoder.layers.4.mlp.experts.linear_fc1.weight5 module.decoder.layers.4.mlp.experts.linear_fc1.weight7 module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 Params for bucket 35 (40370176 elements, 40370176 padded size): - module.decoder.layers.3.mlp.experts.linear_fc2.weight6 - module.decoder.layers.4.mlp.experts.linear_fc1.weight0 - module.decoder.layers.3.mlp.experts.linear_fc2.weight4 - module.decoder.layers.3.mlp.experts.linear_fc2.weight1 - module.decoder.layers.3.mlp.experts.linear_fc1.weight7 module.decoder.layers.4.mlp.experts.linear_fc1.weight1 - module.decoder.layers.3.mlp.experts.linear_fc2.weight7 module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 module.decoder.layers.3.mlp.experts.linear_fc2.weight3 - module.decoder.layers.3.mlp.experts.linear_fc2.weight2 module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 Params for bucket 36 (40370176 elements, 40370176 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight2 module.decoder.layers.3.mlp.experts.linear_fc1.weight5 @@ -1310,14 +1310,14 @@ Params for bucket 38 (40370176 elements, 40370176 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight3 module.decoder.layers.1.mlp.experts.linear_fc2.weight7 module.decoder.layers.1.mlp.experts.linear_fc2.weight5 - module.decoder.layers.2.mlp.experts.linear_fc1.weight2 module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 module.decoder.layers.1.mlp.experts.linear_fc2.weight6 module.decoder.layers.1.mlp.experts.linear_fc2.weight4 Params for bucket 39 (40370176 elements, 40370176 padded size): module.decoder.layers.1.mlp.experts.linear_fc2.weight3 - module.decoder.layers.1.mlp.experts.linear_fc2.weight0 module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 module.decoder.layers.1.mlp.experts.linear_fc1.weight5 module.decoder.layers.1.mlp.experts.linear_fc2.weight2 module.decoder.layers.1.mlp.experts.linear_fc2.weight1 @@ -1326,11 +1326,11 @@ Params for bucket 39 (40370176 elements, 40370176 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight3 Params for bucket 40 (17301504 elements, 17301504 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight2 - module.decoder.layers.1.mlp.experts.linear_fc1.weight1 module.decoder.layers.1.mlp.experts.linear_fc1.weight0 -INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine -[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 14:53:00 +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 15:28:57 > building train, validation, and test datasets ... > datasets target sizes (minimum size): train: 96 @@ -1338,7 +1338,7 @@ INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine test: 0 INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] > building train, validation, and test datasets for GPT ... -INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers @@ -1351,134 +1351,30 @@ INFO:megatron.core.datasets.gpt_dataset: Load the sample index from eadc3756447e INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-shuffle_index.npy INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536 > finished creating GPT datasets ... -[after dataloaders are built] datetime: 2026-08-26 14:53:00 +[after dataloaders are built] datetime: 2026-08-26 15:28:58 done with setup ... -(min, max) time across ranks (ms): - model-and-optimizer-setup ......................: (666.93, 677.90) - train/valid/test-data-iterators-setup ..........: (59.55, 72.35) training ... Overwriting rerun_state_machine.current_iteration from -1 to 0... -[before the start of training step] datetime: 2026-08-26 14:53:00 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1136, 806, 598, 801, 1057, 568, 1425, 551, 493, 1125, 847, 618, 233, 1609, 616, 1132, 762, 670, 669, 798, 511, 444, 1000, 765, 684, 990, 497, 949, 788, 750, 780, 1019, 499, 640, 538, 572, 447, 407, 968, 1245, 693, 755, 504, 528, 1388, 964, 869, 440, 760, 548, 821, 911, 1227, 906, 363, 573, 687, 588, 1239, 1058, 648, 389, 763, 523] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1008, 980, 480, 787, 1021, 627, 1422, 531, 447, 1331, 1035, 557, 255, 1588, 698, 1124, 613, 744, 668, 696, 515, 444, 1012, 839, 732, 896, 495, 1026, 865, 625, 921, 953, 625, 553, 603, 562, 485, 408, 1035, 1137, 696, 665, 505, 606, 1408, 884, 694, 459, 682, 511, 959, 848, 1512, 896, 351, 679, 662, 542, 1304, 850, 619, 405, 754, 318] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1124, 897, 526, 734, 821, 523, 1439, 540, 551, 1080, 1080, 439, 272, 1690, 666, 1058, 619, 646, 736, 804, 541, 471, 991, 820, 721, 795, 515, 973, 860, 658, 906, 990, 484, 663, 690, 629, 554, 401, 948, 1234, 639, 695, 447, 537, 1381, 959, 688, 433, 726, 571, 951, 932, 1468, 883, 324, 662, 696, 540, 1335, 941, 616, 440, 805, 394] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1110, 698, 519, 864, 620, 586, 1478, 595, 527, 1414, 847, 607, 240, 1699, 627, 976, 686, 621, 761, 779, 499, 443, 934, 901, 760, 752, 519, 978, 928, 788, 889, 1055, 501, 903, 648, 599, 486, 341, 1025, 1248, 767, 658, 451, 591, 1401, 897, 677, 373, 882, 517, 662, 936, 1459, 801, 356, 633, 721, 532, 1234, 868, 625, 359, 827, 474] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (638.65, 654.69) + train/valid/test-data-iterators-setup ..........: (60.92, 74.84) +[before the start of training step] datetime: 2026-08-26 15:28:58 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1111, 1011, 510, 748, 707, 617, 1460, 549, 543, 1122, 956, 616, 234, 1644, 711, 1065, 634, 668, 714, 732, 547, 398, 915, 857, 703, 1209, 525, 963, 1003, 688, 811, 1045, 474, 741, 608, 628, 502, 426, 971, 1228, 792, 720, 513, 499, 1437, 837, 690, 403, 705, 538, 1037, 852, 997, 925, 344, 598, 687, 518, 1237, 991, 701, 369, 742, 426] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1124, 897, 526, 734, 821, 523, 1439, 540, 551, 1080, 1080, 439, 272, 1690, 666, 1058, 619, 646, 736, 804, 541, 471, 991, 820, 721, 795, 515, 973, 860, 658, 906, 990, 484, 663, 690, 629, 554, 401, 948, 1234, 639, 695, 447, 537, 1381, 959, 688, 433, 726, 571, 951, 932, 1468, 883, 324, 662, 696, 540, 1335, 941, 616, 440, 805, 394] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1152, 1076, 467, 783, 655, 617, 1506, 614, 593, 1015, 1041, 649, 226, 1745, 575, 996, 666, 617, 772, 759, 572, 392, 1031, 930, 644, 838, 537, 1019, 882, 603, 978, 1053, 622, 917, 658, 586, 539, 343, 837, 1149, 698, 686, 443, 661, 1395, 896, 570, 452, 704, 557, 949, 753, 1397, 828, 329, 609, 736, 559, 1295, 955, 603, 346, 747, 330] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1069, 744, 547, 742, 950, 510, 1462, 546, 490, 1225, 1037, 548, 259, 1619, 553, 1003, 672, 646, 672, 758, 488, 461, 940, 820, 772, 849, 548, 922, 847, 702, 926, 1006, 742, 755, 563, 591, 477, 373, 911, 1248, 717, 692, 544, 485, 1420, 882, 707, 455, 794, 527, 786, 867, 1430, 1018, 314, 669, 750, 450, 1336, 918, 738, 446, 750, 464] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1086, 878, 503, 728, 694, 613, 1438, 549, 566, 1102, 864, 620, 276, 1613, 630, 990, 697, 655, 793, 721, 542, 452, 1006, 868, 744, 1048, 528, 1083, 906, 771, 907, 1038, 539, 556, 671, 610, 471, 390, 1052, 1227, 714, 723, 458, 681, 1424, 853, 682, 457, 838, 495, 879, 891, 1317, 797, 367, 638, 748, 533, 1163, 909, 714, 346, 765, 335] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1110, 698, 519, 864, 620, 586, 925, 595, 0, 0, 0, 0, 0, 0, 0, 0, 527, 1414, 847, 607, 240, 998, 627, 976, 0, 0, 0, 0, 0, 0, 0, 0, 686, 621, 761, 779, 499, 443, 934, 901, 453, 101, 0, 0, 0, 0, 0, 0, 760, 752, 519, 978, 928, 788, 889, 538, 0, 0, 0, 0, 0, 0, 0, 0, 501, 903, 648, 599, 486, 341, 1025, 1248, 600, 0, 0, 0, 0, 0, 0, 0, 767, 658, 451, 591, 1401, 897, 677, 373, 130, 122, 0, 0, 0, 0, 0, 0, 882, 517, 662, 936, 1337, 801, 356, 633, 0, 0, 0, 0, 0, 0, 0, 0, 721, 532, 1234, 868, 625, 359, 827, 474, 387, 100, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5917, 6236, 6178, 6152, 6351, 6067, 6124, 6127] avg=6144.0000 max=6351 max_rank=4 min=5917 min_rank=0 imbalance_ratio=1.033691 max_load_reduction=42801 max_load_reduction_pct=0.870789 moved_to_echo=24669 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1152, 1076, 467, 783, 655, 617, 942, 614, 0, 0, 0, 0, 0, 0, 0, 0, 593, 1015, 1041, 649, 226, 1026, 575, 996, 0, 0, 0, 0, 0, 0, 0, 0, 666, 617, 772, 759, 572, 392, 1031, 930, 462, 103, 0, 0, 0, 0, 0, 0, 644, 838, 537, 1019, 882, 603, 978, 538, 0, 0, 0, 0, 0, 0, 0, 0, 622, 917, 658, 586, 539, 343, 837, 1149, 616, 0, 0, 0, 0, 0, 0, 0, 698, 686, 443, 661, 1395, 896, 570, 452, 129, 117, 0, 0, 0, 0, 0, 0, 704, 557, 949, 753, 1280, 828, 329, 609, 0, 0, 0, 0, 0, 0, 0, 0, 736, 559, 1295, 955, 603, 346, 747, 330, 386, 102, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6306, 6121, 6304, 6039, 6267, 6047, 6009, 6059] avg=6144.0000 max=6306 max_rank=0 min=6009 min_rank=6 imbalance_ratio=1.026367 max_load_reduction=42846 max_load_reduction_pct=0.871704 moved_to_echo=24382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1008, 980, 480, 787, 1021, 627, 890, 531, 0, 0, 0, 0, 0, 0, 0, 0, 447, 1331, 1035, 557, 255, 933, 698, 1124, 0, 0, 0, 0, 0, 0, 0, 0, 613, 744, 668, 696, 515, 444, 1012, 839, 436, 94, 0, 0, 0, 0, 0, 0, 732, 896, 495, 1026, 865, 625, 921, 487, 0, 0, 0, 0, 0, 0, 0, 0, 625, 553, 603, 562, 485, 408, 1035, 1137, 561, 0, 0, 0, 0, 0, 0, 0, 696, 665, 505, 606, 1408, 884, 694, 459, 117, 126, 0, 0, 0, 0, 0, 0, 682, 511, 959, 848, 1386, 896, 351, 679, 0, 0, 0, 0, 0, 0, 0, 0, 662, 542, 1304, 850, 619, 405, 754, 318, 349, 96, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6324, 6380, 6061, 6047, 5969, 6160, 6312, 5899] avg=6144.0000 max=6380 max_rank=1 min=5899 min_rank=7 imbalance_ratio=1.038411 max_load_reduction=42772 max_load_reduction_pct=0.870199 moved_to_echo=24340 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1069, 744, 547, 742, 950, 510, 915, 546, 0, 0, 0, 0, 0, 0, 0, 0, 490, 1225, 1037, 548, 259, 951, 553, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 672, 646, 672, 758, 488, 461, 940, 820, 448, 96, 0, 0, 0, 0, 0, 0, 772, 849, 548, 922, 847, 702, 926, 513, 0, 0, 0, 0, 0, 0, 0, 0, 742, 755, 563, 591, 477, 373, 911, 1248, 572, 0, 0, 0, 0, 0, 0, 0, 717, 692, 544, 485, 1420, 882, 707, 455, 124, 120, 0, 0, 0, 0, 0, 0, 794, 527, 786, 867, 1310, 1018, 314, 669, 0, 0, 0, 0, 0, 0, 0, 0, 750, 450, 1336, 918, 738, 446, 750, 464, 369, 99, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6023, 6066, 6001, 6079, 6232, 6146, 6285, 6320] avg=6144.0000 max=6320 max_rank=7 min=6001 min_rank=2 imbalance_ratio=1.028646 max_load_reduction=42832 max_load_reduction_pct=0.871419 moved_to_echo=24983 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1136, 806, 598, 801, 1057, 568, 884, 551, 0, 0, 0, 0, 0, 0, 0, 0, 493, 1125, 847, 618, 233, 937, 616, 1132, 0, 0, 0, 0, 0, 0, 0, 0, 762, 670, 669, 798, 511, 444, 1000, 765, 440, 100, 0, 0, 0, 0, 0, 0, 684, 990, 497, 949, 788, 750, 780, 512, 0, 0, 0, 0, 0, 0, 0, 0, 499, 640, 538, 572, 447, 407, 968, 1245, 572, 0, 0, 0, 0, 0, 0, 0, 693, 755, 504, 528, 1388, 964, 869, 440, 129, 106, 0, 0, 0, 0, 0, 0, 760, 548, 821, 911, 1121, 906, 363, 573, 0, 0, 0, 0, 0, 0, 0, 0, 687, 588, 1239, 1058, 648, 389, 763, 523, 378, 101, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6401, 6001, 6159, 5950, 5888, 6376, 6003, 6374] avg=6144.0000 max=6401 max_rank=0 min=5888 min_rank=4 imbalance_ratio=1.041829 max_load_reduction=42751 max_load_reduction_pct=0.869771 moved_to_echo=24641 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1008, 980, 480, 787, 1021, 627, 1422, 531, 447, 1331, 1035, 557, 255, 1588, 698, 1124, 613, 744, 668, 696, 515, 444, 1012, 839, 732, 896, 495, 1026, 865, 625, 921, 953, 625, 553, 603, 562, 485, 408, 1035, 1137, 696, 665, 505, 606, 1408, 884, 694, 459, 682, 511, 959, 848, 1512, 896, 351, 679, 662, 542, 1304, 850, 619, 405, 754, 318] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1136, 806, 598, 801, 1057, 568, 1425, 551, 493, 1125, 847, 618, 233, 1609, 616, 1132, 762, 670, 669, 798, 511, 444, 1000, 765, 684, 990, 497, 949, 788, 750, 780, 1019, 499, 640, 538, 572, 447, 407, 968, 1245, 693, 755, 504, 528, 1388, 964, 869, 440, 760, 548, 821, 911, 1227, 906, 363, 573, 687, 588, 1239, 1058, 648, 389, 763, 523] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1110, 698, 519, 864, 620, 586, 1478, 595, 527, 1414, 847, 607, 240, 1699, 627, 976, 686, 621, 761, 779, 499, 443, 934, 901, 760, 752, 519, 978, 928, 788, 889, 1055, 501, 903, 648, 599, 486, 341, 1025, 1248, 767, 658, 451, 591, 1401, 897, 677, 373, 882, 517, 662, 936, 1459, 801, 356, 633, 721, 532, 1234, 868, 625, 359, 827, 474] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1069, 744, 547, 742, 950, 510, 1462, 546, 490, 1225, 1037, 548, 259, 1619, 553, 1003, 672, 646, 672, 758, 488, 461, 940, 820, 772, 849, 548, 922, 847, 702, 926, 1006, 742, 755, 563, 591, 477, 373, 911, 1248, 717, 692, 544, 485, 1420, 882, 707, 455, 794, 527, 786, 867, 1430, 1018, 314, 669, 750, 450, 1336, 918, 738, 446, 750, 464] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1111, 1011, 510, 748, 707, 617, 913, 549, 0, 0, 0, 0, 0, 0, 0, 0, 543, 1122, 956, 616, 234, 966, 711, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 634, 668, 714, 732, 547, 398, 915, 857, 448, 97, 0, 0, 0, 0, 0, 0, 703, 1209, 525, 963, 1003, 688, 811, 534, 0, 0, 0, 0, 0, 0, 0, 0, 474, 741, 608, 628, 502, 426, 971, 1228, 581, 0, 0, 0, 0, 0, 0, 0, 792, 720, 513, 499, 1437, 837, 690, 403, 128, 83, 0, 0, 0, 0, 0, 0, 705, 538, 1037, 852, 914, 925, 344, 598, 0, 0, 0, 0, 0, 0, 0, 0, 687, 518, 1237, 991, 701, 369, 742, 426, 383, 99, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6166, 6213, 6010, 6436, 6159, 6102, 5913, 6153] avg=6144.0000 max=6436 max_rank=3 min=5913 min_rank=6 imbalance_ratio=1.047526 max_load_reduction=42716 max_load_reduction_pct=0.869059 moved_to_echo=24327 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1008, 980, 480, 787, 1021, 627, 890, 531, 0, 0, 0, 0, 0, 0, 0, 0, 447, 1331, 1035, 557, 255, 933, 698, 1124, 0, 0, 0, 0, 0, 0, 0, 0, 613, 744, 668, 696, 515, 444, 1012, 839, 436, 94, 0, 0, 0, 0, 0, 0, 732, 896, 495, 1026, 865, 625, 921, 487, 0, 0, 0, 0, 0, 0, 0, 0, 625, 553, 603, 562, 485, 408, 1035, 1137, 561, 0, 0, 0, 0, 0, 0, 0, 696, 665, 505, 606, 1408, 884, 694, 459, 117, 126, 0, 0, 0, 0, 0, 0, 682, 511, 959, 848, 1386, 896, 351, 679, 0, 0, 0, 0, 0, 0, 0, 0, 662, 542, 1304, 850, 619, 405, 754, 318, 349, 96, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6324, 6380, 6061, 6047, 5969, 6160, 6312, 5899] avg=6144.0000 max=6380 max_rank=1 min=5899 min_rank=7 imbalance_ratio=1.038411 max_load_reduction=42772 max_load_reduction_pct=0.870199 moved_to_echo=24340 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1086, 878, 503, 728, 694, 613, 899, 549, 0, 0, 0, 0, 0, 0, 0, 0, 566, 1102, 864, 620, 276, 948, 630, 990, 0, 0, 0, 0, 0, 0, 0, 0, 697, 655, 793, 721, 542, 452, 1006, 868, 441, 95, 0, 0, 0, 0, 0, 0, 744, 1048, 528, 1083, 906, 771, 907, 529, 0, 0, 0, 0, 0, 0, 0, 0, 539, 556, 671, 610, 471, 390, 1052, 1227, 570, 0, 0, 0, 0, 0, 0, 0, 714, 723, 458, 681, 1424, 853, 682, 457, 128, 110, 0, 0, 0, 0, 0, 0, 838, 495, 879, 891, 1207, 797, 367, 638, 0, 0, 0, 0, 0, 0, 0, 0, 748, 533, 1163, 909, 714, 346, 765, 335, 381, 98, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5950, 5996, 6270, 6516, 6086, 6230, 6112, 5992] avg=6144.0000 max=6516 max_rank=3 min=5950 min_rank=0 imbalance_ratio=1.060547 max_load_reduction=42636 max_load_reduction_pct=0.867432 moved_to_echo=24420 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1136, 806, 598, 801, 1057, 568, 884, 551, 0, 0, 0, 0, 0, 0, 0, 0, 493, 1125, 847, 618, 233, 937, 616, 1132, 0, 0, 0, 0, 0, 0, 0, 0, 762, 670, 669, 798, 511, 444, 1000, 765, 440, 100, 0, 0, 0, 0, 0, 0, 684, 990, 497, 949, 788, 750, 780, 512, 0, 0, 0, 0, 0, 0, 0, 0, 499, 640, 538, 572, 447, 407, 968, 1245, 572, 0, 0, 0, 0, 0, 0, 0, 693, 755, 504, 528, 1388, 964, 869, 440, 129, 106, 0, 0, 0, 0, 0, 0, 760, 548, 821, 911, 1121, 906, 363, 573, 0, 0, 0, 0, 0, 0, 0, 0, 687, 588, 1239, 1058, 648, 389, 763, 523, 378, 101, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6401, 6001, 6159, 5950, 5888, 6376, 6003, 6374] avg=6144.0000 max=6401 max_rank=0 min=5888 min_rank=4 imbalance_ratio=1.041829 max_load_reduction=42751 max_load_reduction_pct=0.869771 moved_to_echo=24641 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1124, 897, 526, 734, 821, 523, 900, 540, 0, 0, 0, 0, 0, 0, 0, 0, 551, 1080, 1080, 439, 272, 993, 666, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 619, 646, 736, 804, 541, 471, 991, 820, 441, 100, 0, 0, 0, 0, 0, 0, 721, 795, 515, 973, 860, 658, 906, 505, 0, 0, 0, 0, 0, 0, 0, 0, 484, 663, 690, 629, 554, 401, 948, 1234, 597, 0, 0, 0, 0, 0, 0, 0, 639, 695, 447, 537, 1381, 959, 688, 433, 122, 123, 0, 0, 0, 0, 0, 0, 726, 571, 951, 932, 1345, 883, 324, 662, 0, 0, 0, 0, 0, 0, 0, 0, 696, 540, 1335, 941, 616, 440, 805, 394, 363, 98, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6065, 6139, 6169, 5933, 6200, 6024, 6394, 6228] avg=6144.0000 max=6394 max_rank=6 min=5933 min_rank=3 imbalance_ratio=1.040690 max_load_reduction=42758 max_load_reduction_pct=0.869914 moved_to_echo=24846 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. - warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. - warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. - warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. - warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. - warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. - warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. - warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. - warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[8832, 7168, 4224, 6272, 6528, 4736, 7296, 4480, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[5376, 5376, 5888, 6144, 4224, 3584, 7936, 6912, 3584, 896, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6144, 4352, 7168, 7040, 9984, 7168, 2816, 5120, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5760, 5632, 3968, 4608, 11264, 7296, 5632, 3584, 1024, 1024, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[5760, 7424, 4224, 7936, 7168, 5632, 7168, 4224, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4608, 5760, 4992, 4864, 3968, 3200, 7808, 9728, 4736, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4224, 9472, 7808, 4736, 2048, 7808, 5120, 8448, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[5760, 4352, 10240, 7552, 5376, 3200, 6272, 3328, 3072, 896, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[449, 563, 694, 949, 814, 753, 630, 764, 375, 877, 1387, 634, 1079, 853, 1180, 1009, 751, 531, 731, 771, 995, 782, 1165, 639, 791, 680, 600, 548, 582, 667, 1306, 429, 1276, 461, 698, 496, 742, 480, 416, 997, 919, 432, 443, 788, 531, 915, 306, 324, 2016, 1176, 847, 359, 790, 759, 655, 1160, 1167, 556, 802, 602, 743, 924, 720, 674] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[596, 615, 717, 744, 1206, 610, 599, 762, 466, 922, 1314, 532, 1008, 823, 1038, 1024, 699, 447, 750, 818, 1093, 874, 1205, 528, 835, 1008, 648, 483, 752, 664, 1403, 317, 1126, 526, 590, 468, 785, 419, 338, 972, 1077, 411, 382, 759, 474, 864, 244, 346, 2091, 1075, 984, 383, 605, 784, 661, 967, 882, 555, 867, 616, 883, 854, 928, 736] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[408, 488, 720, 671, 895, 708, 668, 776, 405, 941, 1409, 635, 1246, 856, 1000, 1041, 715, 511, 746, 617, 1059, 980, 1036, 639, 944, 811, 710, 495, 747, 647, 1404, 397, 1063, 482, 668, 436, 767, 504, 354, 926, 1197, 478, 496, 695, 512, 809, 343, 399, 2038, 1074, 930, 308, 638, 874, 603, 977, 924, 582, 733, 650, 628, 962, 1019, 758] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[462, 370, 844, 668, 817, 869, 742, 715, 330, 845, 1407, 575, 1301, 1140, 926, 1042, 731, 555, 759, 488, 900, 840, 1057, 506, 841, 913, 594, 593, 631, 724, 1526, 495, 1166, 537, 656, 338, 720, 524, 391, 1069, 1276, 421, 330, 820, 561, 867, 294, 425, 2259, 957, 879, 561, 571, 759, 748, 1082, 865, 519, 773, 598, 725, 873, 774, 608] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[506, 566, 687, 613, 927, 713, 649, 802, 417, 850, 1357, 592, 1199, 924, 1030, 1008, 671, 716, 696, 631, 1088, 871, 1065, 699, 847, 818, 579, 464, 770, 621, 1312, 430, 1236, 377, 642, 500, 681, 384, 460, 1045, 1033, 389, 456, 697, 683, 880, 320, 347, 1988, 1344, 1001, 441, 572, 818, 619, 894, 948, 621, 770, 606, 806, 860, 931, 685] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[523, 658, 723, 635, 808, 728, 741, 782, 359, 964, 1419, 523, 1182, 852, 950, 1099, 633, 574, 684, 788, 1050, 700, 1160, 745, 789, 744, 573, 532, 749, 718, 1486, 464, 1222, 435, 679, 475, 823, 453, 391, 997, 1335, 475, 439, 727, 615, 847, 294, 424, 1933, 1194, 830, 435, 536, 829, 653, 1007, 866, 503, 818, 699, 640, 801, 895, 547] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[359, 597, 622, 761, 941, 650, 575, 969, 436, 921, 1336, 533, 961, 830, 1091, 1147, 862, 427, 683, 553, 1150, 902, 942, 918, 805, 750, 610, 456, 746, 656, 1327, 369, 1143, 518, 735, 428, 607, 485, 517, 886, 1062, 509, 260, 658, 615, 939, 410, 306, 2189, 1342, 976, 344, 548, 870, 702, 913, 976, 602, 811, 728, 627, 932, 915, 714] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[470, 523, 603, 587, 938, 629, 697, 741, 407, 939, 1397, 549, 1252, 885, 960, 1234, 703, 613, 652, 848, 1001, 826, 1098, 599, 862, 792, 659, 553, 892, 648, 1416, 393, 1058, 522, 632, 607, 780, 381, 382, 1047, 1285, 434, 519, 823, 592, 881, 287, 482, 2023, 1024, 888, 299, 486, 884, 637, 1066, 859, 486, 749, 713, 709, 739, 961, 551] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[408, 488, 720, 671, 895, 708, 668, 776, 657, 0, 0, 0, 0, 0, 0, 0, 405, 941, 123, 635, 1246, 856, 1000, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 715, 511, 746, 617, 1059, 980, 881, 639, 0, 0, 0, 0, 0, 0, 0, 0, 944, 811, 710, 495, 747, 647, 1404, 397, 86, 52, 0, 0, 0, 0, 0, 0, 1063, 482, 668, 436, 767, 504, 354, 926, 577, 210, 0, 0, 0, 0, 0, 0, 1197, 478, 496, 695, 512, 809, 343, 399, 1231, 0, 0, 0, 0, 0, 0, 0, 597, 1074, 930, 308, 638, 874, 603, 977, 0, 0, 0, 0, 0, 0, 0, 0, 924, 582, 733, 650, 628, 962, 1019, 758, 69, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5991, 6247, 6148, 6293, 5987, 6160, 6001, 6325] avg=6144.0000 max=6325 max_rank=7 min=5987 min_rank=4 imbalance_ratio=1.029460 max_load_reduction=42827 max_load_reduction_pct=0.871318 moved_to_echo=24473 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[523, 658, 723, 635, 808, 728, 741, 782, 661, 0, 0, 0, 0, 0, 0, 0, 359, 964, 125, 523, 1182, 852, 950, 1099, 0, 0, 0, 0, 0, 0, 0, 0, 633, 574, 684, 788, 1050, 700, 986, 745, 0, 0, 0, 0, 0, 0, 0, 0, 789, 744, 573, 532, 749, 718, 1486, 464, 97, 52, 0, 0, 0, 0, 0, 0, 1222, 435, 679, 475, 823, 453, 391, 997, 581, 200, 0, 0, 0, 0, 0, 0, 1335, 475, 439, 727, 615, 847, 294, 424, 1167, 0, 0, 0, 0, 0, 0, 0, 566, 1194, 830, 435, 536, 829, 653, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 866, 503, 818, 699, 640, 801, 895, 547, 77, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6259, 6054, 6160, 6204, 6256, 6323, 6050, 5846] avg=6144.0000 max=6323 max_rank=5 min=5846 min_rank=7 imbalance_ratio=1.029134 max_load_reduction=42829 max_load_reduction_pct=0.871358 moved_to_echo=24475 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[506, 566, 687, 613, 927, 713, 649, 802, 637, 0, 0, 0, 0, 0, 0, 0, 417, 850, 105, 592, 1199, 924, 1030, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 671, 716, 696, 631, 1088, 871, 897, 699, 0, 0, 0, 0, 0, 0, 0, 0, 847, 818, 579, 464, 770, 621, 1312, 430, 93, 53, 0, 0, 0, 0, 0, 0, 1236, 377, 642, 500, 681, 384, 460, 1045, 562, 210, 0, 0, 0, 0, 0, 0, 1033, 389, 456, 697, 683, 880, 320, 347, 1205, 0, 0, 0, 0, 0, 0, 0, 573, 1344, 1001, 441, 572, 818, 619, 894, 0, 0, 0, 0, 0, 0, 0, 0, 948, 621, 770, 606, 806, 860, 931, 685, 75, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6100, 6125, 6269, 5987, 6097, 6010, 6262, 6302] avg=6144.0000 max=6302 max_rank=7 min=5987 min_rank=3 imbalance_ratio=1.025716 max_load_reduction=42850 max_load_reduction_pct=0.871785 moved_to_echo=24671 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[449, 563, 694, 949, 814, 753, 630, 764, 646, 0, 0, 0, 0, 0, 0, 0, 375, 877, 122, 634, 1079, 853, 1180, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 751, 531, 731, 771, 995, 782, 991, 639, 0, 0, 0, 0, 0, 0, 0, 0, 791, 680, 600, 548, 582, 667, 1306, 429, 97, 51, 0, 0, 0, 0, 0, 0, 1276, 461, 698, 496, 742, 480, 416, 997, 568, 208, 0, 0, 0, 0, 0, 0, 919, 432, 443, 788, 531, 915, 306, 324, 1217, 0, 0, 0, 0, 0, 0, 0, 591, 1176, 847, 359, 790, 759, 655, 1160, 0, 0, 0, 0, 0, 0, 0, 0, 1167, 556, 802, 602, 743, 924, 720, 674, 77, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6262, 6129, 6191, 5751, 6342, 5875, 6337, 6265] avg=6144.0000 max=6342 max_rank=4 min=5751 min_rank=3 imbalance_ratio=1.032227 max_load_reduction=42810 max_load_reduction_pct=0.870972 moved_to_echo=24819 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[596, 615, 717, 744, 1206, 610, 599, 762, 612, 0, 0, 0, 0, 0, 0, 0, 466, 922, 116, 532, 1008, 823, 1038, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 699, 447, 750, 818, 1093, 874, 1025, 528, 0, 0, 0, 0, 0, 0, 0, 0, 835, 1008, 648, 483, 752, 664, 1403, 317, 100, 48, 0, 0, 0, 0, 0, 0, 1126, 526, 590, 468, 785, 419, 338, 972, 538, 216, 0, 0, 0, 0, 0, 0, 1077, 411, 382, 759, 474, 864, 244, 346, 1263, 0, 0, 0, 0, 0, 0, 0, 612, 1075, 984, 383, 605, 784, 661, 967, 0, 0, 0, 0, 0, 0, 0, 0, 882, 555, 867, 616, 883, 854, 928, 736, 80, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6461, 5929, 6234, 6258, 5978, 5820, 6071, 6401] avg=6144.0000 max=6461 max_rank=0 min=5820 min_rank=5 imbalance_ratio=1.051595 max_load_reduction=42691 max_load_reduction_pct=0.868551 moved_to_echo=24270 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[359, 597, 622, 761, 941, 650, 575, 969, 623, 0, 0, 0, 0, 0, 0, 0, 436, 921, 117, 533, 961, 830, 1091, 1147, 0, 0, 0, 0, 0, 0, 0, 0, 862, 427, 683, 553, 1150, 902, 801, 918, 0, 0, 0, 0, 0, 0, 0, 0, 805, 750, 610, 456, 746, 656, 1327, 369, 78, 49, 0, 0, 0, 0, 0, 0, 1143, 518, 735, 428, 607, 485, 517, 886, 547, 226, 0, 0, 0, 0, 0, 0, 1062, 509, 260, 658, 615, 939, 410, 306, 1322, 0, 0, 0, 0, 0, 0, 0, 641, 1342, 976, 344, 548, 870, 702, 913, 0, 0, 0, 0, 0, 0, 0, 0, 976, 602, 811, 728, 627, 932, 915, 714, 63, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6097, 6036, 6296, 5846, 6092, 6081, 6336, 6368] avg=6144.0000 max=6368 max_rank=7 min=5846 min_rank=3 imbalance_ratio=1.036458 max_load_reduction=42784 max_load_reduction_pct=0.870443 moved_to_echo=24877 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[462, 370, 844, 668, 817, 869, 742, 715, 656, 0, 0, 0, 0, 0, 0, 0, 330, 845, 124, 575, 1301, 1140, 926, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 731, 555, 759, 488, 900, 840, 899, 506, 0, 0, 0, 0, 0, 0, 0, 0, 841, 913, 594, 593, 631, 724, 1526, 495, 88, 51, 0, 0, 0, 0, 0, 0, 1166, 537, 656, 338, 720, 524, 391, 1069, 576, 233, 0, 0, 0, 0, 0, 0, 1276, 421, 330, 820, 561, 867, 294, 425, 1364, 0, 0, 0, 0, 0, 0, 0, 662, 957, 879, 561, 571, 759, 748, 1082, 0, 0, 0, 0, 0, 0, 0, 0, 865, 519, 773, 598, 725, 873, 774, 608, 70, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6283, 5678, 6456, 6210, 6358, 6219, 5805] avg=6144.0000 max=6456 max_rank=3 min=5678 min_rank=2 imbalance_ratio=1.050781 max_load_reduction=42696 max_load_reduction_pct=0.868652 moved_to_echo=24592 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[470, 523, 603, 587, 938, 629, 697, 741, 651, 0, 0, 0, 0, 0, 0, 0, 407, 939, 123, 549, 1252, 885, 960, 1234, 0, 0, 0, 0, 0, 0, 0, 0, 703, 613, 652, 848, 1001, 826, 934, 599, 0, 0, 0, 0, 0, 0, 0, 0, 862, 792, 659, 553, 892, 648, 1416, 393, 91, 51, 0, 0, 0, 0, 0, 0, 1058, 522, 632, 607, 780, 381, 382, 1047, 572, 209, 0, 0, 0, 0, 0, 0, 1285, 434, 519, 823, 592, 881, 287, 482, 1222, 0, 0, 0, 0, 0, 0, 0, 592, 1024, 888, 299, 486, 884, 637, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 859, 486, 749, 713, 709, 739, 961, 551, 73, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5839, 6349, 6176, 6357, 6190, 6525, 5876, 5840] avg=6144.0000 max=6525 max_rank=5 min=5839 min_rank=0 imbalance_ratio=1.062012 max_load_reduction=42627 max_load_reduction_pct=0.867249 moved_to_echo=24431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. - warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. - warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. - warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. - warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1152, 1076, 467, 783, 655, 617, 942, 614, 0, 0, 0, 0, 0, 0, 0, 0, 593, 1015, 1041, 649, 226, 1026, 575, 996, 0, 0, 0, 0, 0, 0, 0, 0, 666, 617, 772, 759, 572, 392, 1031, 930, 462, 103, 0, 0, 0, 0, 0, 0, 644, 838, 537, 1019, 882, 603, 978, 538, 0, 0, 0, 0, 0, 0, 0, 0, 622, 917, 658, 586, 539, 343, 837, 1149, 616, 0, 0, 0, 0, 0, 0, 0, 698, 686, 443, 661, 1395, 896, 570, 452, 129, 117, 0, 0, 0, 0, 0, 0, 704, 557, 949, 753, 1280, 828, 329, 609, 0, 0, 0, 0, 0, 0, 0, 0, 736, 559, 1295, 955, 603, 346, 747, 330, 386, 102, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6306, 6121, 6304, 6039, 6267, 6047, 6009, 6059] avg=6144.0000 max=6306 max_rank=0 min=6009 min_rank=6 imbalance_ratio=1.026367 max_load_reduction=42846 max_load_reduction_pct=0.871704 moved_to_echo=24382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1110, 698, 519, 864, 620, 586, 925, 595, 0, 0, 0, 0, 0, 0, 0, 0, 527, 1414, 847, 607, 240, 998, 627, 976, 0, 0, 0, 0, 0, 0, 0, 0, 686, 621, 761, 779, 499, 443, 934, 901, 453, 101, 0, 0, 0, 0, 0, 0, 760, 752, 519, 978, 928, 788, 889, 538, 0, 0, 0, 0, 0, 0, 0, 0, 501, 903, 648, 599, 486, 341, 1025, 1248, 600, 0, 0, 0, 0, 0, 0, 0, 767, 658, 451, 591, 1401, 897, 677, 373, 130, 122, 0, 0, 0, 0, 0, 0, 882, 517, 662, 936, 1337, 801, 356, 633, 0, 0, 0, 0, 0, 0, 0, 0, 721, 532, 1234, 868, 625, 359, 827, 474, 387, 100, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5917, 6236, 6178, 6152, 6351, 6067, 6124, 6127] avg=6144.0000 max=6351 max_rank=4 min=5917 min_rank=0 imbalance_ratio=1.033691 max_load_reduction=42801 max_load_reduction_pct=0.870789 moved_to_echo=24669 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1069, 744, 547, 742, 950, 510, 915, 546, 0, 0, 0, 0, 0, 0, 0, 0, 490, 1225, 1037, 548, 259, 951, 553, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 672, 646, 672, 758, 488, 461, 940, 820, 448, 96, 0, 0, 0, 0, 0, 0, 772, 849, 548, 922, 847, 702, 926, 513, 0, 0, 0, 0, 0, 0, 0, 0, 742, 755, 563, 591, 477, 373, 911, 1248, 572, 0, 0, 0, 0, 0, 0, 0, 717, 692, 544, 485, 1420, 882, 707, 455, 124, 120, 0, 0, 0, 0, 0, 0, 794, 527, 786, 867, 1310, 1018, 314, 669, 0, 0, 0, 0, 0, 0, 0, 0, 750, 450, 1336, 918, 738, 446, 750, 464, 369, 99, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6023, 6066, 6001, 6079, 6232, 6146, 6285, 6320] avg=6144.0000 max=6320 max_rank=7 min=6001 min_rank=2 imbalance_ratio=1.028646 max_load_reduction=42832 max_load_reduction_pct=0.871419 moved_to_echo=24983 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. @@ -1495,974 +1391,14 @@ WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] glob warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6784, 6528, 4992, 4224, 5888, 5376, 11264, 3328, 768, 512, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4864, 9216, 7424, 3200, 4864, 6656, 5376, 8192, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3840, 4480, 5632, 5632, 7424, 5760, 5376, 6400, 5248, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[7552, 4480, 6400, 5248, 5888, 7040, 7168, 5376, 640, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[9344, 3968, 5376, 3840, 6016, 3712, 3328, 8064, 4608, 1792, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[9216, 3584, 3328, 6016, 4608, 7040, 2560, 3072, 10112, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[3200, 7296, 1024, 4608, 9344, 7168, 8192, 8704, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5888, 4480, 5760, 5632, 8448, 6784, 7424, 5376, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[694, 958, 444, 505, 606, 397, 472, 692, 580, 876, 296, 1504, 1210, 823, 973, 862, 993, 563, 952, 1248, 798, 500, 729, 951, 554, 604, 1168, 608, 671, 312, 753, 291, 1404, 788, 1176, 608, 1080, 665, 688, 719, 838, 475, 350, 1282, 912, 1272, 577, 700, 362, 979, 1558, 707, 818, 721, 332, 545, 1174, 918, 587, 485, 429, 902, 737, 777] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[535, 856, 467, 449, 537, 457, 353, 815, 454, 982, 340, 1476, 1021, 871, 1005, 679, 1213, 634, 874, 1114, 882, 490, 901, 808, 622, 597, 1568, 727, 788, 328, 750, 206, 1277, 822, 1183, 934, 1082, 770, 784, 768, 875, 529, 278, 1197, 1095, 1010, 706, 554, 420, 1105, 1486, 545, 809, 608, 301, 648, 1143, 842, 208, 535, 540, 967, 644, 688] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[550, 958, 316, 467, 575, 360, 433, 517, 513, 1060, 298, 1484, 962, 1076, 1056, 680, 1195, 661, 977, 1033, 726, 354, 909, 774, 561, 668, 1227, 789, 732, 410, 661, 195, 1471, 797, 1339, 755, 1313, 878, 777, 726, 837, 515, 363, 1267, 1041, 1207, 480, 788, 383, 973, 1160, 710, 898, 623, 388, 604, 1029, 911, 314, 507, 504, 870, 750, 797] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[647, 855, 372, 525, 551, 573, 600, 831, 483, 947, 341, 1486, 1008, 1044, 982, 564, 1159, 565, 1074, 1130, 664, 345, 1073, 893, 543, 717, 1194, 781, 973, 255, 708, 310, 1399, 862, 1145, 860, 955, 819, 608, 948, 902, 461, 249, 1109, 939, 912, 495, 692, 468, 1107, 1555, 576, 904, 515, 351, 535, 847, 830, 370, 663, 483, 1006, 713, 681] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[592, 883, 436, 547, 467, 501, 401, 679, 478, 1210, 383, 1621, 980, 429, 1046, 718, 1255, 608, 998, 906, 748, 420, 1029, 810, 670, 760, 1237, 1094, 844, 279, 800, 132, 1354, 810, 1334, 495, 1328, 830, 499, 787, 896, 454, 309, 1248, 1069, 1237, 555, 537, 378, 1011, 1353, 584, 793, 678, 397, 657, 1314, 814, 247, 390, 473, 905, 735, 720] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[637, 849, 305, 566, 655, 316, 390, 702, 558, 973, 438, 1507, 1006, 353, 970, 734, 1115, 654, 635, 951, 954, 413, 1366, 809, 772, 752, 1019, 1152, 616, 271, 1100, 136, 1398, 714, 1087, 774, 1052, 897, 708, 777, 801, 480, 431, 1003, 828, 1098, 639, 566, 415, 1303, 1505, 648, 832, 719, 371, 591, 1293, 919, 244, 478, 425, 885, 780, 817] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[654, 876, 367, 385, 453, 370, 534, 642, 476, 1039, 410, 1513, 1072, 923, 864, 712, 1049, 650, 1077, 866, 751, 588, 860, 926, 607, 775, 1185, 594, 752, 298, 643, 377, 1406, 1008, 1350, 776, 1227, 672, 691, 852, 706, 532, 354, 1213, 940, 848, 473, 954, 527, 919, 1525, 620, 927, 588, 438, 503, 899, 831, 348, 656, 547, 1008, 825, 701] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[591, 991, 296, 428, 595, 433, 577, 478, 560, 1066, 278, 1567, 965, 556, 948, 637, 1334, 609, 978, 1121, 837, 443, 950, 843, 620, 642, 1367, 1015, 880, 246, 532, 289, 1392, 772, 1494, 430, 1178, 718, 652, 1024, 807, 446, 350, 1597, 746, 825, 743, 908, 363, 866, 1487, 802, 967, 548, 375, 536, 1025, 837, 383, 411, 565, 832, 727, 674] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[535, 856, 467, 449, 537, 457, 353, 815, 1277, 253, 0, 0, 0, 0, 0, 0, 454, 982, 340, 776, 1021, 871, 1005, 679, 0, 0, 0, 0, 0, 0, 0, 0, 488, 634, 874, 1114, 882, 490, 901, 808, 0, 0, 0, 0, 0, 0, 0, 0, 622, 597, 1568, 727, 788, 328, 750, 206, 447, 226, 0, 0, 0, 0, 0, 0, 0, 822, 1109, 934, 1082, 770, 784, 768, 0, 0, 0, 0, 0, 0, 0, 0, 875, 529, 278, 1155, 1095, 1010, 706, 554, 0, 0, 0, 0, 0, 0, 0, 0, 420, 1105, 1486, 545, 809, 608, 301, 648, 74, 42, 32, 0, 0, 0, 0, 0, 1143, 842, 208, 535, 540, 967, 644, 688, 467, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5999, 6128, 6191, 6259, 6269, 6202, 6070, 6034] avg=6144.0000 max=6269 max_rank=4 min=5999 min_rank=0 imbalance_ratio=1.020345 max_load_reduction=42883 max_load_reduction_pct=0.872457 moved_to_echo=24575 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[694, 958, 444, 505, 606, 397, 472, 692, 1404, 258, 0, 0, 0, 0, 0, 0, 580, 876, 296, 791, 1210, 823, 973, 862, 0, 0, 0, 0, 0, 0, 0, 0, 399, 563, 952, 1248, 798, 500, 729, 951, 0, 0, 0, 0, 0, 0, 0, 0, 554, 604, 1168, 608, 671, 312, 753, 291, 455, 185, 0, 0, 0, 0, 0, 0, 0, 788, 1103, 608, 1080, 665, 688, 719, 0, 0, 0, 0, 0, 0, 0, 0, 838, 475, 350, 1238, 912, 1272, 577, 700, 0, 0, 0, 0, 0, 0, 0, 0, 362, 979, 1558, 707, 818, 721, 332, 545, 73, 44, 26, 0, 0, 0, 0, 0, 1174, 918, 587, 485, 429, 902, 737, 777, 383, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6430, 6411, 6140, 5601, 5651, 6362, 6165, 6392] avg=6144.0000 max=6430 max_rank=0 min=5601 min_rank=3 imbalance_ratio=1.046549 max_load_reduction=42722 max_load_reduction_pct=0.869181 moved_to_echo=24570 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[592, 883, 436, 547, 467, 501, 401, 679, 1354, 278, 0, 0, 0, 0, 0, 0, 478, 1210, 383, 852, 980, 429, 1046, 718, 0, 0, 0, 0, 0, 0, 0, 0, 504, 608, 998, 906, 748, 420, 1029, 810, 0, 0, 0, 0, 0, 0, 0, 0, 670, 760, 1237, 1094, 844, 279, 800, 132, 491, 234, 0, 0, 0, 0, 0, 0, 0, 810, 1251, 495, 1328, 830, 499, 787, 0, 0, 0, 0, 0, 0, 0, 0, 896, 454, 309, 1205, 1069, 1237, 555, 537, 0, 0, 0, 0, 0, 0, 0, 0, 378, 1011, 1353, 584, 793, 678, 397, 657, 83, 43, 33, 0, 0, 0, 0, 0, 1314, 814, 247, 390, 473, 905, 735, 720, 484, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6138, 6096, 6023, 6541, 6000, 6262, 6010, 6082] avg=6144.0000 max=6541 max_rank=3 min=6000 min_rank=4 imbalance_ratio=1.064616 max_load_reduction=42611 max_load_reduction_pct=0.866923 moved_to_echo=24354 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[550, 958, 316, 467, 575, 360, 433, 517, 1471, 258, 0, 0, 0, 0, 0, 0, 513, 1060, 298, 772, 962, 1076, 1056, 680, 0, 0, 0, 0, 0, 0, 0, 0, 469, 661, 977, 1033, 726, 354, 909, 774, 0, 0, 0, 0, 0, 0, 0, 0, 561, 668, 1227, 789, 732, 410, 661, 195, 454, 226, 0, 0, 0, 0, 0, 0, 0, 797, 1252, 755, 1313, 878, 777, 726, 0, 0, 0, 0, 0, 0, 0, 0, 837, 515, 363, 1219, 1041, 1207, 480, 788, 0, 0, 0, 0, 0, 0, 0, 0, 383, 973, 1160, 710, 898, 623, 388, 604, 87, 48, 36, 0, 0, 0, 0, 0, 1029, 911, 314, 507, 504, 870, 750, 797, 464, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5905, 6417, 5903, 5923, 6498, 6450, 5910, 6146] avg=6144.0000 max=6498 max_rank=4 min=5903 min_rank=2 imbalance_ratio=1.057617 max_load_reduction=42654 max_load_reduction_pct=0.867798 moved_to_echo=25004 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[647, 855, 372, 525, 551, 573, 600, 831, 1399, 255, 0, 0, 0, 0, 0, 0, 483, 947, 341, 781, 1008, 1044, 982, 564, 0, 0, 0, 0, 0, 0, 0, 0, 465, 565, 1074, 1130, 664, 345, 1073, 893, 0, 0, 0, 0, 0, 0, 0, 0, 543, 717, 1194, 781, 973, 255, 708, 310, 450, 216, 0, 0, 0, 0, 0, 0, 0, 862, 1074, 860, 955, 819, 608, 948, 0, 0, 0, 0, 0, 0, 0, 0, 902, 461, 249, 1071, 939, 912, 495, 692, 0, 0, 0, 0, 0, 0, 0, 0, 468, 1107, 1555, 576, 904, 515, 351, 535, 71, 38, 31, 0, 0, 0, 0, 0, 847, 830, 370, 663, 483, 1006, 713, 681, 447, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6608, 6150, 6209, 6147, 6126, 5721, 6151, 6040] avg=6144.0000 max=6608 max_rank=0 min=5721 min_rank=5 imbalance_ratio=1.075521 max_load_reduction=42544 max_load_reduction_pct=0.865560 moved_to_echo=24038 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[637, 849, 305, 566, 655, 316, 390, 702, 1398, 258, 0, 0, 0, 0, 0, 0, 558, 973, 438, 793, 1006, 353, 970, 734, 0, 0, 0, 0, 0, 0, 0, 0, 447, 654, 635, 951, 954, 413, 1366, 809, 0, 0, 0, 0, 0, 0, 0, 0, 772, 752, 1019, 1152, 616, 271, 1100, 136, 456, 208, 0, 0, 0, 0, 0, 0, 0, 714, 1019, 774, 1052, 897, 708, 777, 0, 0, 0, 0, 0, 0, 0, 0, 801, 480, 431, 968, 828, 1098, 639, 566, 0, 0, 0, 0, 0, 0, 0, 0, 415, 1303, 1505, 648, 832, 719, 371, 591, 68, 35, 30, 0, 0, 0, 0, 0, 1293, 919, 244, 478, 425, 885, 780, 817, 430, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6076, 5825, 6229, 6482, 5941, 5811, 6517, 6271] avg=6144.0000 max=6517 max_rank=6 min=5811 min_rank=5 imbalance_ratio=1.060710 max_load_reduction=42635 max_load_reduction_pct=0.867411 moved_to_echo=24540 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[654, 876, 367, 385, 453, 370, 534, 642, 1406, 259, 0, 0, 0, 0, 0, 0, 476, 1039, 410, 796, 1072, 923, 864, 712, 0, 0, 0, 0, 0, 0, 0, 0, 422, 650, 1077, 866, 751, 588, 860, 926, 0, 0, 0, 0, 0, 0, 0, 0, 607, 775, 1185, 594, 752, 298, 643, 377, 458, 195, 0, 0, 0, 0, 0, 0, 0, 1008, 1266, 776, 1227, 672, 691, 852, 0, 0, 0, 0, 0, 0, 0, 0, 706, 532, 354, 1171, 940, 848, 473, 954, 0, 0, 0, 0, 0, 0, 0, 0, 527, 919, 1525, 620, 927, 588, 438, 503, 84, 42, 28, 0, 0, 0, 0, 0, 899, 831, 348, 656, 547, 1008, 825, 701, 404, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5946, 6292, 6140, 5884, 6492, 5978, 6201, 6219] avg=6144.0000 max=6492 max_rank=4 min=5884 min_rank=3 imbalance_ratio=1.056641 max_load_reduction=42660 max_load_reduction_pct=0.867920 moved_to_echo=24890 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[591, 991, 296, 428, 595, 433, 577, 478, 1392, 269, 0, 0, 0, 0, 0, 0, 560, 1066, 278, 823, 965, 556, 948, 637, 0, 0, 0, 0, 0, 0, 0, 0, 536, 609, 978, 1121, 837, 443, 950, 843, 0, 0, 0, 0, 0, 0, 0, 0, 620, 642, 1367, 1015, 880, 246, 532, 289, 475, 249, 0, 0, 0, 0, 0, 0, 0, 772, 1401, 430, 1178, 718, 652, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 807, 446, 350, 1541, 746, 825, 743, 908, 0, 0, 0, 0, 0, 0, 0, 0, 363, 866, 1487, 802, 967, 548, 375, 536, 93, 56, 35, 0, 0, 0, 0, 0, 1025, 837, 383, 411, 565, 832, 727, 674, 514, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6050, 5833, 6317, 6315, 6175, 6366, 6128, 5968] avg=6144.0000 max=6366 max_rank=5 min=5833 min_rank=1 imbalance_ratio=1.036133 max_load_reduction=42786 max_load_reduction_pct=0.870483 moved_to_echo=24637 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4992, 7296, 3072, 3968, 4480, 3456, 3840, 5376, 11136, 2176, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3328, 8320, 11648, 5248, 7040, 5120, 3072, 4736, 640, 384, 256, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4992, 5632, 9984, 6784, 6272, 2432, 6016, 2048, 3712, 1792, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 6656, 9600, 5632, 9216, 6272, 5504, 6656, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[8832, 6912, 2816, 4224, 3968, 7424, 6016, 5888, 3712, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6784, 3968, 2688, 9600, 7680, 8448, 4736, 5760, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3840, 4992, 7680, 8448, 6400, 3584, 7936, 6912, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4224, 8192, 2816, 6400, 8320, 6144, 7936, 5632, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[735, 687, 587, 548, 1425, 690, 1158, 1235, 901, 297, 257, 364, 790, 813, 650, 739, 1186, 1316, 720, 838, 556, 1085, 587, 759, 818, 288, 839, 1195, 545, 1407, 955, 378, 361, 828, 502, 1238, 771, 479, 511, 361, 402, 722, 902, 517, 717, 669, 884, 692, 1081, 960, 2456, 720, 116, 552, 750, 597, 1346, 413, 438, 299, 736, 1502, 845, 437] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[730, 705, 911, 453, 1050, 537, 1115, 1366, 925, 376, 285, 321, 909, 841, 460, 573, 982, 1001, 651, 914, 707, 919, 764, 1160, 675, 469, 968, 1042, 669, 1350, 809, 360, 364, 974, 504, 1212, 371, 500, 419, 386, 437, 964, 1160, 570, 962, 672, 767, 638, 879, 1005, 2291, 478, 214, 621, 646, 620, 1053, 387, 594, 390, 871, 1860, 813, 533] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[690, 712, 1004, 447, 1242, 676, 1031, 1329, 791, 402, 368, 265, 876, 769, 637, 485, 1262, 1096, 821, 941, 586, 932, 655, 829, 559, 510, 1130, 990, 679, 1478, 749, 427, 320, 659, 477, 1070, 645, 629, 427, 375, 590, 738, 970, 701, 840, 532, 777, 910, 1420, 850, 1614, 605, 253, 440, 771, 822, 1170, 404, 510, 332, 1079, 1588, 744, 522] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[777, 597, 1189, 426, 1118, 479, 1162, 1673, 652, 395, 295, 209, 954, 762, 619, 537, 1154, 1099, 781, 689, 811, 707, 635, 1008, 475, 461, 1244, 1125, 958, 1439, 878, 452, 287, 794, 427, 1385, 654, 705, 612, 287, 359, 708, 909, 665, 842, 586, 827, 715, 981, 951, 2133, 448, 211, 663, 807, 893, 1057, 274, 592, 271, 727, 1575, 531, 516] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[582, 607, 748, 312, 933, 637, 1068, 1484, 896, 530, 339, 220, 916, 862, 514, 498, 954, 1138, 779, 618, 591, 819, 526, 549, 523, 395, 1081, 1225, 1018, 1327, 810, 621, 576, 843, 492, 1278, 509, 401, 470, 402, 428, 1150, 770, 766, 870, 783, 795, 816, 663, 1058, 2230, 439, 207, 727, 1061, 801, 1108, 327, 670, 439, 535, 1787, 999, 632] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[779, 511, 886, 543, 1176, 639, 1127, 1422, 819, 495, 326, 643, 918, 738, 492, 583, 1009, 1008, 926, 816, 551, 1004, 708, 1039, 609, 297, 1155, 1085, 605, 1639, 854, 761, 633, 832, 504, 1228, 430, 458, 397, 422, 473, 1061, 860, 490, 712, 739, 904, 654, 935, 909, 1473, 592, 219, 704, 741, 557, 1150, 395, 466, 348, 937, 1442, 890, 434] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[669, 592, 787, 461, 967, 530, 1131, 1580, 783, 383, 428, 317, 856, 914, 753, 639, 1466, 827, 699, 669, 523, 929, 431, 702, 524, 453, 1120, 1052, 722, 1316, 877, 579, 305, 932, 435, 1533, 485, 401, 519, 539, 444, 805, 586, 818, 777, 771, 808, 659, 897, 1106, 2598, 615, 257, 652, 750, 671, 1015, 272, 508, 317, 790, 1768, 870, 570] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[684, 653, 734, 522, 1447, 319, 1121, 1310, 507, 429, 316, 297, 929, 848, 987, 488, 1039, 873, 1141, 617, 674, 918, 623, 382, 796, 452, 1102, 1004, 559, 1524, 793, 852, 515, 845, 487, 955, 735, 491, 496, 354, 507, 883, 777, 639, 940, 728, 925, 771, 1220, 851, 1963, 713, 245, 473, 638, 800, 973, 437, 546, 183, 1071, 1536, 1043, 472] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[735, 687, 587, 548, 1425, 690, 1158, 554, 0, 0, 0, 0, 0, 0, 0, 0, 901, 297, 257, 364, 790, 813, 650, 739, 946, 488, 0, 0, 0, 0, 0, 0, 591, 1316, 720, 838, 556, 1085, 587, 759, 0, 0, 0, 0, 0, 0, 0, 0, 818, 288, 839, 1195, 545, 807, 955, 378, 0, 0, 0, 0, 0, 0, 0, 0, 361, 828, 502, 1238, 771, 479, 511, 361, 600, 397, 193, 0, 0, 0, 0, 0, 402, 722, 902, 517, 717, 669, 884, 692, 198, 21, 0, 0, 0, 0, 0, 0, 1081, 960, 1510, 720, 116, 552, 750, 597, 0, 0, 0, 0, 0, 0, 0, 0, 1346, 413, 438, 299, 736, 1481, 845, 437, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6384, 6245, 6452, 5825, 6241, 5724, 6286, 5995] avg=6144.0000 max=6452 max_rank=2 min=5724 min_rank=5 imbalance_ratio=1.050130 max_load_reduction=42700 max_load_reduction_pct=0.868734 moved_to_echo=24246 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[730, 705, 911, 453, 1050, 537, 1115, 613, 0, 0, 0, 0, 0, 0, 0, 0, 925, 376, 285, 321, 909, 841, 460, 573, 883, 540, 0, 0, 0, 0, 0, 0, 489, 1001, 651, 914, 707, 919, 764, 1160, 0, 0, 0, 0, 0, 0, 0, 0, 675, 469, 968, 1042, 669, 774, 809, 360, 0, 0, 0, 0, 0, 0, 0, 0, 364, 974, 504, 1212, 371, 500, 419, 386, 576, 329, 213, 0, 0, 0, 0, 0, 437, 964, 1160, 570, 962, 672, 767, 638, 164, 26, 0, 0, 0, 0, 0, 0, 879, 1005, 1408, 478, 214, 621, 646, 620, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 387, 594, 390, 871, 1834, 813, 533, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6114, 6113, 6605, 5766, 5848, 6360, 5871, 6475] avg=6144.0000 max=6605 max_rank=2 min=5766 min_rank=3 imbalance_ratio=1.075033 max_load_reduction=42547 max_load_reduction_pct=0.865621 moved_to_echo=24554 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[690, 712, 1004, 447, 1242, 676, 1031, 597, 0, 0, 0, 0, 0, 0, 0, 0, 791, 402, 368, 265, 876, 769, 637, 485, 622, 525, 0, 0, 0, 0, 0, 0, 629, 1096, 821, 941, 586, 932, 655, 829, 0, 0, 0, 0, 0, 0, 0, 0, 559, 510, 1130, 990, 679, 848, 749, 427, 0, 0, 0, 0, 0, 0, 0, 0, 320, 659, 477, 1070, 645, 629, 427, 375, 630, 423, 207, 0, 0, 0, 0, 0, 590, 738, 970, 701, 840, 532, 777, 910, 210, 22, 0, 0, 0, 0, 0, 0, 1420, 850, 992, 605, 253, 440, 771, 822, 0, 0, 0, 0, 0, 0, 0, 0, 1170, 404, 510, 332, 1079, 1566, 744, 522, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6399, 5740, 6489, 5892, 5862, 6290, 6153, 6327] avg=6144.0000 max=6489 max_rank=2 min=5740 min_rank=1 imbalance_ratio=1.056152 max_load_reduction=42663 max_load_reduction_pct=0.867981 moved_to_echo=24632 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[777, 597, 1189, 426, 1118, 479, 1162, 750, 0, 0, 0, 0, 0, 0, 0, 0, 652, 395, 295, 209, 954, 762, 619, 537, 822, 662, 0, 0, 0, 0, 0, 0, 576, 1099, 781, 689, 811, 707, 635, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 475, 461, 1244, 1125, 958, 825, 878, 452, 0, 0, 0, 0, 0, 0, 0, 0, 287, 794, 427, 1385, 654, 705, 612, 287, 614, 386, 261, 0, 0, 0, 0, 0, 359, 708, 909, 665, 842, 586, 827, 715, 192, 22, 0, 0, 0, 0, 0, 0, 981, 951, 1311, 448, 211, 663, 807, 893, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 274, 592, 271, 727, 1553, 531, 516, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6498, 5907, 6306, 6418, 6412, 5825, 6265, 5521] avg=6144.0000 max=6498 max_rank=0 min=5521 min_rank=7 imbalance_ratio=1.057617 max_load_reduction=42654 max_load_reduction_pct=0.867798 moved_to_echo=24023 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[779, 511, 886, 543, 1176, 639, 1127, 630, 0, 0, 0, 0, 0, 0, 0, 0, 819, 495, 326, 643, 918, 738, 492, 583, 572, 566, 0, 0, 0, 0, 0, 0, 495, 1008, 926, 816, 551, 1004, 708, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 609, 297, 1155, 1085, 605, 937, 854, 761, 0, 0, 0, 0, 0, 0, 0, 0, 633, 832, 504, 1228, 430, 458, 397, 422, 702, 342, 226, 0, 0, 0, 0, 0, 473, 1061, 860, 490, 712, 739, 904, 654, 172, 25, 0, 0, 0, 0, 0, 0, 935, 909, 901, 592, 219, 704, 741, 557, 0, 0, 0, 0, 0, 0, 0, 0, 1150, 395, 466, 348, 937, 1417, 890, 434, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6291, 6152, 6547, 6303, 6174, 6090, 5558, 6037] avg=6144.0000 max=6547 max_rank=2 min=5558 min_rank=6 imbalance_ratio=1.065592 max_load_reduction=42605 max_load_reduction_pct=0.866801 moved_to_echo=23859 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[582, 607, 748, 312, 933, 637, 1068, 666, 0, 0, 0, 0, 0, 0, 0, 0, 896, 530, 339, 220, 916, 862, 514, 498, 859, 587, 0, 0, 0, 0, 0, 0, 476, 1138, 779, 618, 591, 819, 526, 549, 0, 0, 0, 0, 0, 0, 0, 0, 523, 395, 1081, 1225, 1018, 761, 810, 621, 0, 0, 0, 0, 0, 0, 0, 0, 576, 843, 492, 1278, 509, 401, 470, 402, 566, 319, 231, 0, 0, 0, 0, 0, 428, 1150, 770, 766, 870, 783, 795, 816, 159, 25, 0, 0, 0, 0, 0, 0, 663, 1058, 1371, 439, 207, 727, 1061, 801, 0, 0, 0, 0, 0, 0, 0, 0, 1108, 327, 670, 439, 535, 1762, 999, 632, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5553, 6221, 5496, 6434, 6087, 6562, 6327, 6472] avg=6144.0000 max=6562 max_rank=5 min=5496 min_rank=2 imbalance_ratio=1.068034 max_load_reduction=42590 max_load_reduction_pct=0.866496 moved_to_echo=25448 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[684, 653, 734, 522, 1447, 319, 1121, 588, 0, 0, 0, 0, 0, 0, 0, 0, 507, 429, 316, 297, 929, 848, 987, 488, 756, 518, 0, 0, 0, 0, 0, 0, 518, 873, 1141, 617, 674, 918, 623, 382, 0, 0, 0, 0, 0, 0, 0, 0, 796, 452, 1102, 1004, 559, 874, 793, 852, 0, 0, 0, 0, 0, 0, 0, 0, 515, 845, 487, 955, 735, 491, 496, 354, 650, 348, 204, 0, 0, 0, 0, 0, 507, 883, 777, 639, 940, 728, 925, 771, 173, 21, 0, 0, 0, 0, 0, 0, 1220, 851, 1207, 713, 245, 473, 638, 800, 0, 0, 0, 0, 0, 0, 0, 0, 973, 437, 546, 183, 1071, 1515, 1043, 472, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6068, 6075, 5746, 6432, 6080, 6364, 6147, 6240] avg=6144.0000 max=6432 max_rank=3 min=5746 min_rank=2 imbalance_ratio=1.046875 max_load_reduction=42720 max_load_reduction_pct=0.869141 moved_to_echo=24831 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[669, 592, 787, 461, 967, 530, 1131, 708, 0, 0, 0, 0, 0, 0, 0, 0, 783, 383, 428, 317, 856, 914, 753, 639, 1001, 625, 0, 0, 0, 0, 0, 0, 731, 827, 699, 669, 523, 929, 431, 702, 0, 0, 0, 0, 0, 0, 0, 0, 524, 453, 1120, 1052, 722, 755, 877, 579, 0, 0, 0, 0, 0, 0, 0, 0, 305, 932, 435, 1533, 485, 401, 519, 539, 561, 491, 247, 0, 0, 0, 0, 0, 444, 805, 586, 818, 777, 771, 808, 659, 244, 25, 0, 0, 0, 0, 0, 0, 897, 1106, 1597, 615, 257, 652, 750, 671, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 272, 508, 317, 790, 1743, 870, 570, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5845, 6699, 5511, 6082, 6448, 5937, 6545, 6085] avg=6144.0000 max=6699 max_rank=1 min=5511 min_rank=2 imbalance_ratio=1.090332 max_load_reduction=42453 max_load_reduction_pct=0.863708 moved_to_echo=25015 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4992, 3328, 8704, 8832, 5760, 6656, 6784, 4480, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3712, 7040, 7040, 5248, 6784, 5504, 6784, 5888, 1536, 256, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[8192, 7808, 10368, 4736, 1792, 4864, 6272, 5888, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3456, 6784, 3840, 9984, 4608, 4096, 3968, 3200, 4992, 3072, 1792, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5760, 5120, 6912, 3712, 9472, 4608, 8960, 5120, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[8960, 2944, 4352, 2688, 6784, 12928, 6784, 4224, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4608, 8448, 6528, 6144, 5120, 7424, 4992, 6528, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6400, 3328, 2688, 2688, 7168, 6656, 5120, 4608, 6528, 4608, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[1182, 1219, 759, 462, 542, 1779, 868, 661, 624, 693, 599, 679, 600, 639, 493, 642, 577, 1229, 303, 1607, 832, 703, 860, 864, 659, 375, 975, 1072, 653, 2073, 452, 724, 327, 428, 694, 1321, 941, 325, 504, 783, 640, 564, 162, 661, 503, 473, 783, 424, 382, 1616, 1556, 354, 496, 1685, 1119, 707, 240, 1072, 520, 1277, 472, 961, 455, 308] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[926, 1081, 786, 301, 632, 1952, 1096, 890, 811, 637, 713, 502, 612, 588, 488, 371, 406, 1108, 346, 1358, 680, 801, 524, 1129, 680, 258, 654, 1165, 562, 1723, 808, 731, 605, 325, 987, 733, 1011, 337, 455, 795, 783, 631, 373, 1025, 1070, 476, 376, 536, 317, 1565, 1507, 326, 574, 1230, 1183, 682, 456, 1706, 458, 984, 489, 717, 785, 336] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[1301, 1146, 924, 503, 797, 1690, 978, 552, 740, 626, 829, 622, 647, 411, 472, 336, 394, 1358, 304, 1755, 1030, 629, 568, 731, 673, 377, 964, 977, 591, 1916, 699, 826, 339, 519, 430, 1135, 712, 491, 356, 575, 745, 713, 168, 531, 602, 463, 802, 697, 311, 1560, 1199, 316, 658, 1341, 1581, 709, 409, 1602, 511, 995, 658, 581, 768, 309] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1005, 972, 772, 473, 795, 1632, 873, 615, 690, 685, 566, 695, 535, 494, 335, 558, 445, 1179, 171, 1829, 669, 382, 945, 725, 505, 484, 418, 1051, 754, 1964, 492, 626, 547, 264, 420, 1800, 1169, 324, 425, 981, 754, 1050, 146, 509, 524, 635, 1071, 480, 366, 1441, 1677, 344, 442, 1426, 1394, 700, 348, 1117, 476, 1037, 531, 1242, 682, 496] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[1263, 1117, 1148, 541, 440, 1662, 937, 655, 615, 652, 525, 984, 691, 579, 481, 315, 466, 1405, 275, 1981, 887, 907, 895, 849, 578, 380, 1171, 1263, 331, 1903, 484, 525, 272, 377, 500, 1602, 837, 296, 366, 527, 574, 535, 141, 593, 456, 509, 969, 250, 412, 1130, 1635, 263, 893, 1421, 1125, 720, 405, 1430, 411, 1524, 509, 482, 665, 418] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[1314, 1238, 905, 345, 561, 1746, 1134, 465, 391, 912, 1130, 638, 619, 532, 302, 611, 401, 1161, 302, 1829, 607, 634, 701, 639, 500, 298, 1007, 1690, 420, 1582, 441, 780, 367, 520, 432, 1375, 577, 454, 566, 643, 466, 657, 308, 654, 690, 556, 1020, 313, 426, 1291, 1352, 331, 745, 1317, 1087, 766, 444, 1465, 495, 1461, 609, 658, 557, 715] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1212, 1284, 1010, 436, 692, 1968, 988, 597, 672, 568, 1045, 694, 669, 531, 378, 414, 363, 1204, 332, 1669, 528, 601, 673, 717, 507, 368, 672, 1450, 715, 2000, 423, 796, 459, 467, 392, 1609, 707, 444, 420, 928, 654, 877, 356, 736, 672, 533, 936, 537, 377, 1100, 1389, 261, 742, 929, 1679, 713, 331, 1074, 452, 915, 769, 507, 586, 425] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1213, 993, 810, 383, 491, 2001, 1106, 848, 739, 660, 662, 634, 502, 683, 607, 468, 525, 1264, 373, 1593, 1069, 621, 790, 506, 503, 405, 1030, 1056, 470, 2214, 576, 692, 635, 300, 539, 1098, 944, 504, 555, 693, 541, 795, 224, 669, 597, 665, 719, 307, 324, 1249, 1326, 352, 480, 1198, 1074, 724, 467, 1131, 512, 1573, 533, 672, 642, 623] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1301, 1146, 924, 503, 797, 225, 978, 552, 0, 0, 0, 0, 0, 0, 0, 0, 740, 626, 829, 622, 647, 411, 472, 336, 934, 145, 0, 0, 0, 0, 0, 0, 394, 1358, 304, 1233, 1030, 629, 568, 731, 0, 0, 0, 0, 0, 0, 0, 0, 673, 377, 964, 977, 591, 1299, 699, 826, 0, 0, 0, 0, 0, 0, 0, 0, 339, 519, 430, 1135, 712, 491, 356, 575, 617, 140, 175, 0, 0, 0, 0, 0, 745, 713, 168, 531, 602, 463, 802, 697, 1320, 0, 0, 0, 0, 0, 0, 0, 311, 1560, 125, 316, 658, 1341, 1581, 709, 0, 0, 0, 0, 0, 0, 0, 0, 409, 1602, 511, 995, 658, 581, 768, 309, 347, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6426, 5762, 6247, 6406, 5489, 6041, 6601, 6180] avg=6144.0000 max=6601 max_rank=6 min=5489 min_rank=4 imbalance_ratio=1.074382 max_load_reduction=42551 max_load_reduction_pct=0.865702 moved_to_echo=24311 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1005, 972, 772, 473, 795, 218, 873, 615, 0, 0, 0, 0, 0, 0, 0, 0, 690, 685, 566, 695, 535, 494, 335, 558, 1307, 140, 0, 0, 0, 0, 0, 0, 445, 1179, 171, 1286, 669, 382, 945, 725, 0, 0, 0, 0, 0, 0, 0, 0, 505, 484, 418, 1051, 754, 1331, 492, 626, 0, 0, 0, 0, 0, 0, 0, 0, 547, 264, 420, 1800, 1169, 324, 425, 981, 633, 197, 182, 0, 0, 0, 0, 0, 754, 1050, 146, 509, 524, 635, 1071, 480, 1274, 0, 0, 0, 0, 0, 0, 0, 366, 1441, 173, 344, 442, 1426, 1394, 700, 0, 0, 0, 0, 0, 0, 0, 0, 348, 1117, 476, 1037, 531, 1242, 682, 496, 361, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5723, 6005, 5802, 5661, 6942, 6443, 6286, 6290] avg=6144.0000 max=6942 max_rank=4 min=5661 min_rank=3 imbalance_ratio=1.129883 max_load_reduction=42210 max_load_reduction_pct=0.858765 moved_to_echo=25961 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1182, 1219, 759, 462, 542, 237, 868, 661, 0, 0, 0, 0, 0, 0, 0, 0, 624, 693, 599, 679, 600, 639, 493, 642, 1213, 153, 0, 0, 0, 0, 0, 0, 577, 1229, 303, 1130, 832, 703, 860, 864, 0, 0, 0, 0, 0, 0, 0, 0, 659, 375, 975, 1072, 653, 1405, 452, 724, 0, 0, 0, 0, 0, 0, 0, 0, 327, 428, 694, 1321, 941, 325, 504, 783, 668, 182, 160, 0, 0, 0, 0, 0, 640, 564, 162, 661, 503, 473, 783, 424, 1389, 0, 0, 0, 0, 0, 0, 0, 382, 1616, 161, 354, 496, 1685, 1119, 707, 0, 0, 0, 0, 0, 0, 0, 0, 240, 1072, 520, 1277, 472, 961, 455, 308, 317, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5930, 6335, 6498, 6315, 6333, 5599, 6520, 5622] avg=6144.0000 max=6520 max_rank=6 min=5599 min_rank=5 imbalance_ratio=1.061198 max_load_reduction=42632 max_load_reduction_pct=0.867350 moved_to_echo=24074 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[926, 1081, 786, 301, 632, 260, 1096, 890, 0, 0, 0, 0, 0, 0, 0, 0, 811, 637, 713, 502, 612, 588, 488, 371, 1174, 168, 0, 0, 0, 0, 0, 0, 406, 1108, 346, 955, 680, 801, 524, 1129, 0, 0, 0, 0, 0, 0, 0, 0, 680, 258, 654, 1165, 562, 1168, 808, 731, 0, 0, 0, 0, 0, 0, 0, 0, 605, 325, 987, 733, 1011, 337, 455, 795, 555, 177, 135, 0, 0, 0, 0, 0, 783, 631, 373, 1025, 1070, 476, 376, 536, 1524, 0, 0, 0, 0, 0, 0, 0, 317, 1565, 156, 326, 574, 1230, 1183, 682, 0, 0, 0, 0, 0, 0, 0, 0, 456, 1706, 458, 984, 489, 717, 785, 336, 268, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5972, 6064, 5949, 6026, 6115, 6794, 6033, 6199] avg=6144.0000 max=6794 max_rank=5 min=5949 min_rank=2 imbalance_ratio=1.105794 max_load_reduction=42358 max_load_reduction_pct=0.861776 moved_to_echo=25141 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1263, 1117, 1148, 541, 440, 221, 937, 655, 0, 0, 0, 0, 0, 0, 0, 0, 615, 652, 525, 984, 691, 579, 481, 315, 1274, 143, 0, 0, 0, 0, 0, 0, 466, 1405, 275, 1392, 887, 907, 895, 849, 0, 0, 0, 0, 0, 0, 0, 0, 578, 380, 1171, 1263, 331, 1290, 484, 525, 0, 0, 0, 0, 0, 0, 0, 0, 272, 377, 500, 1602, 837, 296, 366, 527, 613, 192, 198, 0, 0, 0, 0, 0, 574, 535, 141, 593, 456, 509, 969, 250, 1298, 0, 0, 0, 0, 0, 0, 0, 412, 1130, 169, 263, 893, 1421, 1125, 720, 0, 0, 0, 0, 0, 0, 0, 0, 405, 1430, 411, 1524, 509, 482, 665, 418, 391, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6322, 6259, 7076, 6022, 5780, 5325, 6133, 6235] avg=6144.0000 max=7076 max_rank=2 min=5325 min_rank=5 imbalance_ratio=1.151693 max_load_reduction=42076 max_load_reduction_pct=0.856038 moved_to_echo=23473 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1314, 1238, 905, 345, 561, 232, 1134, 465, 0, 0, 0, 0, 0, 0, 0, 0, 391, 912, 1130, 638, 619, 532, 302, 611, 1053, 150, 0, 0, 0, 0, 0, 0, 401, 1161, 302, 1286, 607, 634, 701, 639, 0, 0, 0, 0, 0, 0, 0, 0, 500, 298, 1007, 1690, 420, 1072, 441, 780, 0, 0, 0, 0, 0, 0, 0, 0, 367, 520, 432, 1375, 577, 454, 566, 643, 510, 158, 182, 0, 0, 0, 0, 0, 466, 657, 308, 654, 690, 556, 1020, 313, 1364, 0, 0, 0, 0, 0, 0, 0, 426, 1291, 141, 331, 745, 1317, 1087, 766, 0, 0, 0, 0, 0, 0, 0, 0, 444, 1465, 495, 1461, 609, 658, 557, 715, 361, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6194, 6338, 5731, 6208, 5784, 6028, 6104, 6765] avg=6144.0000 max=6765 max_rank=7 min=5731 min_rank=2 imbalance_ratio=1.101074 max_load_reduction=42387 max_load_reduction_pct=0.862366 moved_to_echo=24681 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1212, 1284, 1010, 436, 692, 256, 988, 597, 0, 0, 0, 0, 0, 0, 0, 0, 672, 568, 1045, 694, 669, 531, 378, 414, 1087, 171, 0, 0, 0, 0, 0, 0, 363, 1204, 332, 1164, 528, 601, 673, 717, 0, 0, 0, 0, 0, 0, 0, 0, 507, 368, 672, 1450, 715, 1351, 423, 796, 0, 0, 0, 0, 0, 0, 0, 0, 459, 467, 392, 1609, 707, 444, 420, 928, 649, 167, 171, 0, 0, 0, 0, 0, 654, 877, 356, 736, 672, 533, 936, 537, 1541, 0, 0, 0, 0, 0, 0, 0, 377, 1100, 135, 261, 742, 929, 1679, 713, 0, 0, 0, 0, 0, 0, 0, 0, 331, 1074, 452, 915, 769, 507, 586, 425, 334, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6475, 6229, 5582, 6282, 6413, 6842, 5936, 5393] avg=6144.0000 max=6842 max_rank=5 min=5393 min_rank=7 imbalance_ratio=1.113607 max_load_reduction=42310 max_load_reduction_pct=0.860799 moved_to_echo=24584 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1213, 993, 810, 383, 491, 266, 1106, 848, 0, 0, 0, 0, 0, 0, 0, 0, 739, 660, 662, 634, 502, 683, 607, 468, 1033, 172, 0, 0, 0, 0, 0, 0, 525, 1264, 373, 1119, 1069, 621, 790, 506, 0, 0, 0, 0, 0, 0, 0, 0, 503, 405, 1030, 1056, 470, 1500, 576, 692, 0, 0, 0, 0, 0, 0, 0, 0, 635, 300, 539, 1098, 944, 504, 555, 693, 714, 155, 159, 0, 0, 0, 0, 0, 541, 795, 224, 669, 597, 665, 719, 307, 1563, 0, 0, 0, 0, 0, 0, 0, 324, 1249, 138, 352, 480, 1198, 1074, 724, 0, 0, 0, 0, 0, 0, 0, 0, 467, 1131, 512, 1573, 533, 672, 642, 623, 315, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6110, 6160, 6267, 6232, 6296, 6080, 5539, 6468] avg=6144.0000 max=6468 max_rank=7 min=5539 min_rank=6 imbalance_ratio=1.052734 max_load_reduction=42684 max_load_reduction_pct=0.868408 moved_to_echo=24383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2944, 11008, 1280, 2560, 5120, 10624, 10368, 5760, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3584, 9984, 2432, 9600, 6400, 5376, 6016, 6272, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3200, 10624, 3840, 9856, 4608, 5888, 5248, 3712, 2816, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5376, 5504, 6144, 5504, 4992, 4480, 3584, 3840, 9088, 1280, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3584, 3200, 4480, 10752, 6912, 3200, 3712, 6016, 4992, 1408, 1408, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[9472, 9088, 7168, 3456, 4992, 1920, 8064, 5376, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4608, 3072, 6912, 9728, 4608, 10496, 4480, 5760, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5248, 5888, 1920, 5504, 5120, 4352, 6784, 3584, 11392, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[597, 1117, 236, 295, 403, 1185, 1227, 401, 703, 346, 420, 597, 225, 356, 622, 89, 1855, 1108, 835, 1011, 706, 2416, 1055, 546, 904, 681, 660, 494, 740, 1182, 348, 368, 491, 152, 558, 651, 561, 1504, 1424, 1375, 1017, 408, 567, 1439, 242, 506, 1745, 865, 633, 1134, 1057, 1060, 1145, 304, 195, 420, 1097, 1196, 531, 339, 871, 248, 1343, 346] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1050, 617, 373, 182, 250, 1514, 1301, 459, 789, 510, 596, 258, 341, 632, 591, 406, 2320, 459, 1080, 819, 372, 2072, 774, 276, 508, 629, 920, 721, 794, 470, 400, 293, 602, 628, 634, 998, 390, 1500, 537, 713, 1227, 748, 1124, 1439, 559, 431, 1311, 681, 573, 579, 1303, 600, 1120, 502, 195, 763, 1686, 1341, 1242, 552, 910, 332, 919, 237] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[749, 974, 208, 221, 417, 931, 2067, 1107, 1038, 352, 804, 546, 300, 519, 745, 199, 1776, 1088, 850, 468, 937, 2037, 688, 272, 868, 588, 626, 308, 550, 546, 424, 352, 493, 263, 832, 849, 634, 1441, 1127, 944, 790, 411, 1065, 1572, 315, 547, 2203, 587, 776, 1576, 798, 682, 810, 442, 199, 699, 1081, 1146, 541, 440, 720, 254, 885, 475] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[580, 1093, 283, 204, 523, 1072, 1152, 586, 439, 380, 783, 409, 362, 609, 930, 291, 1816, 1124, 954, 532, 682, 2021, 672, 345, 1103, 946, 863, 512, 651, 803, 465, 281, 463, 180, 1162, 623, 746, 1214, 1523, 1064, 970, 769, 1103, 1115, 356, 504, 2246, 597, 761, 698, 1115, 644, 917, 490, 169, 496, 972, 1516, 616, 440, 915, 278, 677, 347] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[692, 938, 208, 188, 306, 1407, 1138, 674, 593, 281, 982, 251, 359, 423, 666, 246, 1812, 1708, 1189, 657, 964, 1995, 883, 396, 588, 833, 684, 422, 698, 1193, 509, 372, 740, 254, 1564, 875, 551, 1148, 1091, 1286, 729, 507, 870, 1470, 303, 602, 1064, 575, 647, 1176, 1100, 968, 746, 491, 283, 391, 864, 1232, 572, 478, 388, 348, 1138, 446] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[400, 1053, 307, 232, 210, 1564, 1330, 689, 1051, 376, 592, 314, 225, 458, 801, 242, 2106, 938, 1035, 790, 643, 2398, 754, 429, 728, 671, 843, 407, 667, 602, 348, 347, 663, 373, 934, 571, 660, 1287, 1223, 1024, 1013, 678, 913, 1168, 437, 562, 1585, 830, 613, 1397, 888, 848, 479, 433, 136, 625, 1114, 1428, 1102, 633, 637, 224, 783, 341] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[925, 1172, 315, 191, 363, 1410, 1226, 678, 819, 270, 704, 338, 210, 330, 911, 179, 2160, 1205, 1014, 598, 1151, 2046, 799, 961, 1097, 1194, 675, 377, 620, 803, 403, 538, 422, 371, 655, 531, 662, 1113, 1105, 996, 726, 320, 751, 1064, 452, 577, 1799, 1119, 625, 869, 1149, 691, 625, 580, 143, 472, 769, 1066, 713, 562, 804, 247, 906, 586] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[656, 935, 197, 294, 402, 1145, 1729, 511, 564, 337, 908, 488, 398, 430, 928, 376, 2005, 879, 916, 584, 649, 1980, 1010, 388, 905, 589, 842, 384, 756, 548, 353, 465, 423, 169, 630, 691, 628, 1425, 1151, 819, 1227, 308, 774, 1284, 632, 509, 2387, 902, 571, 979, 1073, 690, 1103, 793, 208, 755, 752, 1123, 652, 555, 659, 557, 677, 495] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[749, 974, 208, 221, 417, 931, 2067, 1107, 183, 105, 0, 0, 0, 0, 0, 0, 1038, 352, 804, 546, 300, 519, 745, 199, 2030, 0, 0, 0, 0, 0, 0, 0, 1229, 1088, 850, 468, 937, 0, 688, 272, 0, 0, 0, 0, 0, 0, 0, 0, 868, 588, 626, 308, 550, 546, 424, 352, 1297, 22, 7, 0, 0, 0, 0, 0, 493, 263, 832, 849, 634, 1013, 1127, 944, 0, 0, 0, 0, 0, 0, 0, 0, 790, 411, 1065, 1572, 315, 547, 906, 587, 0, 0, 0, 0, 0, 0, 0, 0, 776, 1576, 798, 682, 810, 442, 199, 699, 420, 0, 0, 0, 0, 0, 0, 0, 1081, 1146, 541, 440, 720, 254, 885, 475, 245, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6962, 6533, 5532, 5588, 6155, 6193, 6402, 5787] avg=6144.0000 max=6962 max_rank=0 min=5532 min_rank=2 imbalance_ratio=1.133138 max_load_reduction=42190 max_load_reduction_pct=0.858358 moved_to_echo=24537 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[580, 1093, 283, 204, 523, 1072, 1152, 586, 154, 107, 0, 0, 0, 0, 0, 0, 439, 380, 783, 409, 362, 609, 930, 291, 2015, 0, 0, 0, 0, 0, 0, 0, 1258, 1124, 954, 532, 682, 0, 672, 345, 0, 0, 0, 0, 0, 0, 0, 0, 1103, 946, 863, 512, 651, 803, 465, 281, 1322, 22, 6, 0, 0, 0, 0, 0, 463, 180, 1162, 623, 746, 854, 1523, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 970, 769, 1103, 1115, 356, 504, 924, 597, 0, 0, 0, 0, 0, 0, 0, 0, 761, 698, 1115, 644, 917, 490, 169, 496, 429, 0, 0, 0, 0, 0, 0, 0, 972, 1516, 616, 440, 915, 278, 677, 347, 206, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5754, 6218, 5567, 6974, 6615, 6338, 5719, 5967] avg=6144.0000 max=6974 max_rank=3 min=5567 min_rank=2 imbalance_ratio=1.135091 max_load_reduction=42178 max_load_reduction_pct=0.858114 moved_to_echo=24639 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[597, 1117, 236, 295, 403, 1185, 1227, 401, 191, 110, 0, 0, 0, 0, 0, 0, 703, 346, 420, 597, 225, 356, 622, 89, 2409, 0, 0, 0, 0, 0, 0, 0, 1284, 1108, 835, 1011, 706, 0, 1055, 546, 0, 0, 0, 0, 0, 0, 0, 0, 904, 681, 660, 494, 740, 1182, 348, 368, 1027, 23, 7, 0, 0, 0, 0, 0, 491, 152, 558, 651, 561, 1058, 1424, 1375, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 408, 567, 1439, 242, 506, 718, 865, 0, 0, 0, 0, 0, 0, 0, 0, 633, 1134, 1057, 1060, 1145, 304, 195, 420, 438, 0, 0, 0, 0, 0, 0, 0, 1097, 1196, 531, 339, 871, 248, 1343, 346, 255, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5762, 5767, 6545, 6434, 6270, 5762, 6386, 6226] avg=6144.0000 max=6545 max_rank=2 min=5762 min_rank=0 imbalance_ratio=1.065267 max_load_reduction=42607 max_load_reduction_pct=0.866842 moved_to_echo=24644 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[400, 1053, 307, 232, 210, 1564, 1330, 689, 163, 125, 0, 0, 0, 0, 0, 0, 1051, 376, 592, 314, 225, 458, 801, 242, 2390, 0, 0, 0, 0, 0, 0, 0, 1457, 938, 1035, 790, 643, 0, 754, 429, 0, 0, 0, 0, 0, 0, 0, 0, 728, 671, 843, 407, 667, 602, 348, 347, 933, 26, 8, 0, 0, 0, 0, 0, 663, 373, 934, 571, 660, 906, 1223, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 678, 913, 1168, 437, 562, 652, 830, 0, 0, 0, 0, 0, 0, 0, 0, 613, 1397, 888, 848, 479, 433, 136, 625, 498, 0, 0, 0, 0, 0, 0, 0, 1114, 1428, 1102, 633, 637, 224, 783, 341, 218, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6073, 6449, 6046, 5580, 6354, 6253, 5917, 6480] avg=6144.0000 max=6480 max_rank=7 min=5580 min_rank=3 imbalance_ratio=1.054688 max_load_reduction=42672 max_load_reduction_pct=0.868164 moved_to_echo=25004 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1050, 617, 373, 182, 250, 1514, 1301, 459, 190, 137, 0, 0, 0, 0, 0, 0, 789, 510, 596, 258, 341, 632, 591, 406, 2066, 0, 0, 0, 0, 0, 0, 0, 1606, 459, 1080, 819, 372, 0, 774, 276, 0, 0, 0, 0, 0, 0, 0, 0, 508, 629, 920, 721, 794, 470, 400, 293, 771, 29, 6, 0, 0, 0, 0, 0, 602, 628, 634, 998, 390, 1055, 537, 713, 0, 0, 0, 0, 0, 0, 0, 0, 1227, 748, 1124, 1439, 559, 431, 540, 681, 0, 0, 0, 0, 0, 0, 0, 0, 573, 579, 1303, 600, 1120, 502, 195, 763, 548, 0, 0, 0, 0, 0, 0, 0, 1686, 1341, 1242, 552, 910, 332, 919, 237, 255, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6073, 6189, 5386, 5541, 5557, 6749, 6183, 7474] avg=6144.0000 max=7474 max_rank=7 min=5386 min_rank=2 imbalance_ratio=1.216471 max_load_reduction=41678 max_load_reduction_pct=0.847941 moved_to_echo=25963 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[925, 1172, 315, 191, 363, 1410, 1226, 678, 141, 128, 0, 0, 0, 0, 0, 0, 819, 270, 704, 338, 210, 330, 911, 179, 2040, 0, 0, 0, 0, 0, 0, 0, 1494, 1205, 1014, 598, 1151, 0, 799, 961, 0, 0, 0, 0, 0, 0, 0, 0, 1097, 1194, 675, 377, 620, 803, 403, 538, 1059, 27, 6, 0, 0, 0, 0, 0, 422, 371, 655, 531, 662, 783, 1105, 996, 0, 0, 0, 0, 0, 0, 0, 0, 726, 320, 751, 1064, 452, 577, 740, 1119, 0, 0, 0, 0, 0, 0, 0, 0, 625, 869, 1149, 691, 625, 580, 143, 472, 511, 0, 0, 0, 0, 0, 0, 0, 769, 1066, 713, 562, 804, 247, 906, 586, 189, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6549, 5801, 7222, 6799, 5525, 5749, 5665, 5842] avg=6144.0000 max=7222 max_rank=2 min=5525 min_rank=4 imbalance_ratio=1.175456 max_load_reduction=41930 max_load_reduction_pct=0.853068 moved_to_echo=22781 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[656, 935, 197, 294, 402, 1145, 1729, 511, 184, 123, 0, 0, 0, 0, 0, 0, 564, 337, 908, 488, 398, 430, 928, 376, 1975, 0, 0, 0, 0, 0, 0, 0, 1376, 879, 916, 584, 649, 0, 1010, 388, 0, 0, 0, 0, 0, 0, 0, 0, 905, 589, 842, 384, 756, 548, 353, 465, 1409, 28, 5, 0, 0, 0, 0, 0, 423, 169, 630, 691, 628, 996, 1151, 819, 0, 0, 0, 0, 0, 0, 0, 0, 1227, 308, 774, 1284, 632, 509, 978, 902, 0, 0, 0, 0, 0, 0, 0, 0, 571, 979, 1073, 690, 1103, 793, 208, 755, 478, 0, 0, 0, 0, 0, 0, 0, 752, 1123, 652, 555, 659, 557, 677, 495, 245, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6176, 6404, 5802, 6284, 5507, 6614, 6650, 5715] avg=6144.0000 max=6650 max_rank=6 min=5507 min_rank=4 imbalance_ratio=1.082357 max_load_reduction=42502 max_load_reduction_pct=0.864705 moved_to_echo=24486 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[692, 938, 208, 188, 306, 1407, 1138, 674, 145, 107, 0, 0, 0, 0, 0, 0, 593, 281, 982, 251, 359, 423, 666, 246, 1990, 0, 0, 0, 0, 0, 0, 0, 1255, 1708, 1189, 657, 964, 0, 883, 396, 0, 0, 0, 0, 0, 0, 0, 0, 588, 833, 684, 422, 698, 1193, 509, 372, 626, 22, 5, 0, 0, 0, 0, 0, 740, 254, 1564, 875, 551, 808, 1091, 1286, 0, 0, 0, 0, 0, 0, 0, 0, 729, 507, 870, 1470, 303, 602, 438, 575, 0, 0, 0, 0, 0, 0, 0, 0, 647, 1176, 1100, 968, 746, 491, 283, 391, 428, 0, 0, 0, 0, 0, 0, 0, 864, 1232, 572, 478, 388, 348, 1138, 446, 195, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5803, 5791, 7052, 5952, 7169, 5494, 6230, 5661] avg=6144.0000 max=7169 max_rank=4 min=5494 min_rank=5 imbalance_ratio=1.166829 max_load_reduction=41983 max_load_reduction_pct=0.854146 moved_to_echo=24554 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6016, 2944, 5888, 3328, 2432, 3840, 6272, 2048, 17024, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[8448, 10112, 6016, 4096, 6016, 2560, 7424, 3328, 1920, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5248, 8448, 8576, 6272, 7040, 4096, 1536, 4736, 3840, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5760, 7936, 2176, 1920, 2944, 10240, 11264, 5120, 1408, 1024, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[7808, 4224, 7168, 10624, 3328, 4352, 6016, 6272, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6784, 6144, 6144, 3712, 5504, 6272, 3328, 3072, 8448, 256, 128, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[11008, 8576, 7936, 5504, 6144, 0, 6656, 3712, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4352, 2432, 7040, 5888, 4864, 7552, 9216, 8320, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1367, 308, 2513, 401, 358, 825, 696, 1439, 617, 1555, 716, 685, 965, 383, 885, 364, 969, 709, 1547, 1058, 236, 1359, 981, 1583, 428, 295, 782, 382, 728, 489, 1840, 269, 589, 765, 893, 653, 787, 881, 383, 655, 234, 1399, 182, 177, 649, 375, 467, 402, 743, 48, 541, 921, 1296, 923, 303, 260, 310, 1681, 595, 263, 618, 1252, 1402, 773] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[2113, 473, 2218, 326, 308, 806, 308, 1348, 1223, 927, 829, 1225, 740, 490, 305, 380, 1182, 1161, 1190, 707, 363, 2461, 1617, 1949, 439, 321, 822, 256, 778, 372, 1216, 283, 604, 386, 1087, 390, 697, 481, 626, 435, 407, 928, 231, 165, 518, 278, 401, 864, 737, 223, 550, 468, 1563, 311, 360, 323, 840, 767, 389, 571, 1453, 1065, 1475, 423] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1766, 312, 2267, 678, 375, 855, 570, 1400, 934, 907, 866, 1000, 635, 502, 304, 475, 1118, 1017, 1326, 669, 395, 1724, 1169, 1781, 357, 329, 532, 403, 1024, 750, 1324, 246, 832, 774, 1296, 244, 1218, 933, 625, 671, 591, 1666, 164, 259, 390, 491, 217, 619, 620, 207, 490, 452, 795, 1013, 300, 257, 465, 533, 190, 472, 855, 857, 2105, 541] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1293, 645, 2066, 712, 215, 1181, 327, 1662, 1071, 693, 885, 1589, 654, 357, 787, 444, 734, 872, 1380, 457, 686, 987, 1108, 1085, 495, 229, 836, 197, 922, 501, 1249, 120, 441, 846, 742, 319, 783, 687, 613, 694, 528, 1624, 642, 200, 671, 829, 355, 411, 652, 221, 477, 1153, 1444, 758, 369, 219, 1139, 625, 357, 877, 1180, 1141, 1353, 363] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1443, 508, 1849, 1029, 336, 989, 435, 1446, 1154, 790, 1174, 729, 948, 421, 423, 508, 820, 384, 1066, 748, 501, 913, 953, 1201, 353, 889, 878, 323, 748, 407, 1245, 232, 1093, 601, 938, 315, 844, 1035, 706, 1027, 498, 1317, 541, 300, 495, 444, 570, 577, 624, 444, 526, 1030, 904, 911, 381, 298, 713, 1469, 214, 682, 966, 1333, 1056, 457] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[2070, 507, 1893, 766, 336, 922, 837, 1694, 1198, 987, 670, 1019, 704, 469, 748, 221, 828, 678, 1081, 663, 687, 827, 1195, 1202, 216, 278, 334, 458, 788, 384, 1993, 280, 547, 522, 864, 309, 528, 836, 624, 1009, 712, 1850, 388, 149, 588, 462, 271, 503, 1146, 402, 791, 1008, 1349, 695, 727, 259, 482, 657, 494, 899, 743, 739, 1246, 420] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1287, 477, 2069, 608, 371, 831, 1031, 1390, 1083, 701, 811, 1088, 1048, 340, 618, 775, 988, 317, 934, 949, 560, 1097, 1076, 1219, 451, 387, 578, 429, 639, 452, 1140, 229, 537, 684, 877, 240, 476, 793, 590, 904, 228, 857, 650, 248, 938, 445, 496, 689, 701, 484, 887, 1163, 1280, 1351, 510, 581, 737, 980, 266, 775, 482, 1454, 1273, 603] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1784, 543, 2085, 588, 263, 737, 772, 1366, 831, 845, 707, 1143, 907, 480, 629, 483, 1588, 337, 1316, 347, 767, 1795, 985, 1678, 482, 184, 652, 416, 776, 403, 1284, 151, 447, 636, 1260, 162, 614, 691, 588, 743, 896, 1139, 271, 146, 660, 132, 300, 988, 936, 210, 385, 903, 1366, 849, 520, 349, 592, 635, 445, 591, 748, 1326, 1896, 404] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1766, 312, 97, 678, 375, 855, 570, 1400, 0, 0, 0, 0, 0, 0, 0, 0, 934, 907, 866, 1000, 635, 502, 304, 475, 19, 0, 0, 0, 0, 0, 0, 0, 1118, 1017, 1326, 669, 395, 1041, 1169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 357, 329, 532, 403, 1024, 750, 1324, 246, 1264, 423, 0, 0, 0, 0, 0, 0, 832, 774, 1296, 244, 1218, 933, 625, 671, 614, 252, 0, 0, 0, 0, 0, 0, 591, 1666, 164, 259, 390, 491, 217, 619, 1747, 0, 0, 0, 0, 0, 0, 0, 620, 207, 490, 452, 795, 1013, 300, 257, 517, 431, 0, 0, 0, 0, 0, 0, 465, 533, 190, 472, 855, 857, 1472, 541, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6053, 5642, 6735, 6652, 7459, 6144, 5082, 5385] avg=6144.0000 max=7459 max_rank=4 min=5082 min_rank=6 imbalance_ratio=1.214030 max_load_reduction=41693 max_load_reduction_pct=0.848246 moved_to_echo=24070 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2070, 507, 82, 766, 336, 922, 837, 1694, 0, 0, 0, 0, 0, 0, 0, 0, 1198, 987, 670, 1019, 704, 469, 748, 221, 11, 0, 0, 0, 0, 0, 0, 0, 828, 678, 1081, 663, 687, 499, 1195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 278, 334, 458, 788, 384, 1993, 280, 853, 353, 0, 0, 0, 0, 0, 0, 547, 522, 864, 309, 528, 836, 624, 1009, 363, 121, 0, 0, 0, 0, 0, 0, 712, 1850, 388, 149, 588, 462, 271, 503, 1458, 0, 0, 0, 0, 0, 0, 0, 1146, 402, 791, 1008, 1349, 695, 727, 259, 349, 207, 0, 0, 0, 0, 0, 0, 482, 657, 494, 899, 743, 739, 872, 420, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7214, 6027, 5631, 5937, 5723, 6381, 6933, 5306] avg=6144.0000 max=7214 max_rank=0 min=5306 min_rank=7 imbalance_ratio=1.174154 max_load_reduction=41938 max_load_reduction_pct=0.853231 moved_to_echo=24343 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1367, 308, 108, 401, 358, 825, 696, 1439, 0, 0, 0, 0, 0, 0, 0, 0, 617, 1555, 716, 685, 965, 383, 885, 364, 12, 0, 0, 0, 0, 0, 0, 0, 969, 709, 1547, 1058, 236, 820, 981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 428, 295, 782, 382, 728, 489, 1840, 269, 1125, 469, 0, 0, 0, 0, 0, 0, 589, 765, 893, 653, 787, 881, 383, 655, 409, 199, 0, 0, 0, 0, 0, 0, 234, 1399, 182, 177, 649, 375, 467, 402, 1936, 0, 0, 0, 0, 0, 0, 0, 743, 48, 541, 921, 1296, 923, 303, 260, 458, 340, 0, 0, 0, 0, 0, 0, 310, 1681, 595, 263, 618, 1252, 981, 773, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5502, 6182, 6320, 6807, 6214, 5821, 5833, 6473] avg=6144.0000 max=6807 max_rank=3 min=5502 min_rank=0 imbalance_ratio=1.107910 max_load_reduction=42345 max_load_reduction_pct=0.861511 moved_to_echo=24341 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2113, 473, 95, 326, 308, 806, 308, 1348, 0, 0, 0, 0, 0, 0, 0, 0, 1223, 927, 829, 1225, 740, 490, 305, 380, 13, 0, 0, 0, 0, 0, 0, 0, 1182, 1161, 1190, 707, 363, 1485, 1617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 439, 321, 822, 256, 778, 372, 1216, 283, 1385, 414, 0, 0, 0, 0, 0, 0, 604, 386, 1087, 390, 697, 481, 626, 435, 430, 360, 0, 0, 0, 0, 0, 0, 407, 928, 231, 165, 518, 278, 401, 864, 1709, 0, 0, 0, 0, 0, 0, 0, 737, 223, 550, 468, 1563, 311, 360, 323, 564, 616, 0, 0, 0, 0, 0, 0, 840, 767, 389, 571, 1453, 1065, 1032, 423, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5777, 6132, 7705, 6286, 5496, 5501, 5715, 6540] avg=6144.0000 max=7705 max_rank=2 min=5496 min_rank=4 imbalance_ratio=1.254069 max_load_reduction=41447 max_load_reduction_pct=0.843241 moved_to_echo=23252 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1287, 477, 89, 608, 371, 831, 1031, 1390, 0, 0, 0, 0, 0, 0, 0, 0, 1083, 701, 811, 1088, 1048, 340, 618, 775, 11, 0, 0, 0, 0, 0, 0, 0, 988, 317, 934, 949, 560, 663, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 451, 387, 578, 429, 639, 452, 1140, 229, 865, 386, 0, 0, 0, 0, 0, 0, 537, 684, 877, 240, 476, 793, 590, 904, 371, 160, 0, 0, 0, 0, 0, 0, 228, 857, 650, 248, 938, 445, 496, 689, 1594, 0, 0, 0, 0, 0, 0, 0, 701, 484, 887, 1163, 1280, 1351, 510, 581, 354, 274, 0, 0, 0, 0, 0, 0, 737, 980, 266, 775, 482, 1454, 891, 603, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6084, 6475, 5487, 5556, 5632, 6145, 7585, 6188] avg=6144.0000 max=7585 max_rank=6 min=5487 min_rank=2 imbalance_ratio=1.234538 max_load_reduction=41567 max_load_reduction_pct=0.845683 moved_to_echo=25550 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1443, 508, 70, 1029, 336, 989, 435, 1446, 0, 0, 0, 0, 0, 0, 0, 0, 1154, 790, 1174, 729, 948, 421, 423, 508, 13, 0, 0, 0, 0, 0, 0, 0, 820, 384, 1066, 748, 501, 545, 953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 353, 889, 878, 323, 748, 407, 1245, 232, 853, 351, 0, 0, 0, 0, 0, 0, 1093, 601, 938, 315, 844, 1035, 706, 1027, 312, 137, 0, 0, 0, 0, 0, 0, 498, 1317, 541, 300, 495, 444, 570, 577, 1428, 0, 0, 0, 0, 0, 0, 0, 624, 444, 526, 1030, 904, 911, 381, 298, 348, 231, 0, 0, 0, 0, 0, 0, 713, 1469, 214, 682, 966, 1333, 731, 457, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6256, 6160, 5017, 6279, 7008, 6170, 5697, 6565] avg=6144.0000 max=7008 max_rank=4 min=5017 min_rank=2 imbalance_ratio=1.140625 max_load_reduction=42144 max_load_reduction_pct=0.857422 moved_to_echo=25440 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1293, 645, 88, 712, 215, 1181, 327, 1662, 0, 0, 0, 0, 0, 0, 0, 0, 1071, 693, 885, 1589, 654, 357, 787, 444, 12, 0, 0, 0, 0, 0, 0, 0, 734, 872, 1380, 457, 686, 596, 1108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 229, 836, 197, 922, 501, 1249, 120, 771, 386, 0, 0, 0, 0, 0, 0, 441, 846, 742, 319, 783, 687, 613, 694, 395, 144, 0, 0, 0, 0, 0, 0, 528, 1624, 642, 200, 671, 829, 355, 411, 1592, 0, 0, 0, 0, 0, 0, 0, 652, 221, 477, 1153, 1444, 758, 369, 219, 314, 247, 0, 0, 0, 0, 0, 0, 1139, 625, 357, 877, 1180, 1141, 946, 363, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6123, 6492, 5833, 5706, 5664, 6852, 5854, 6628] avg=6144.0000 max=6852 max_rank=5 min=5664 min_rank=4 imbalance_ratio=1.115234 max_load_reduction=42300 max_load_reduction_pct=0.860596 moved_to_echo=24998 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1784, 543, 90, 588, 263, 737, 772, 1366, 0, 0, 0, 0, 0, 0, 0, 0, 831, 845, 707, 1143, 907, 480, 629, 483, 17, 0, 0, 0, 0, 0, 0, 0, 1588, 337, 1316, 347, 767, 1084, 985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 184, 652, 416, 776, 403, 1284, 151, 1192, 389, 0, 0, 0, 0, 0, 0, 447, 636, 1260, 162, 614, 691, 588, 743, 553, 262, 0, 0, 0, 0, 0, 0, 896, 1139, 271, 146, 660, 132, 300, 988, 1606, 0, 0, 0, 0, 0, 0, 0, 936, 210, 385, 903, 1366, 849, 520, 349, 486, 449, 0, 0, 0, 0, 0, 0, 592, 635, 445, 591, 748, 1326, 1326, 404, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6042, 6424, 5929, 5956, 6138, 6453, 6067] avg=6144.0000 max=6453 max_rank=6 min=5929 min_rank=3 imbalance_ratio=1.050293 max_load_reduction=42699 max_load_reduction_pct=0.868713 moved_to_echo=24614 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5120, 5248, 8064, 2688, 6016, 6400, 4864, 6144, 3456, 1664, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[8320, 5504, 9856, 5632, 4224, 6784, 9088, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[13184, 3840, 768, 5120, 2688, 7168, 4992, 11776, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4096, 10880, 3072, 1664, 4992, 3456, 3200, 5120, 13184, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[5376, 7424, 3072, 5248, 7168, 9216, 8320, 4096, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[6272, 2304, 4736, 7168, 10112, 6912, 3584, 2560, 3456, 2816, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[8192, 7424, 6784, 8576, 6656, 3456, 4736, 3712, 128, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3328, 2944, 5504, 2944, 6528, 3840, 11392, 1920, 8320, 3200, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[827, 1647, 286, 658, 389, 409, 162, 1286, 429, 186, 1436, 485, 496, 1115, 1051, 930, 263, 286, 269, 500, 1111, 1156, 454, 721, 399, 606, 1092, 1243, 945, 1421, 885, 452, 1561, 496, 1007, 312, 879, 2096, 360, 1250, 404, 657, 541, 1415, 392, 612, 652, 955, 265, 1083, 60, 1106, 955, 814, 1337, 634, 613, 663, 1766, 706, 668, 763, 170, 365] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[546, 712, 164, 553, 232, 241, 138, 464, 506, 237, 1550, 455, 883, 1661, 1463, 214, 264, 254, 414, 472, 1224, 1334, 308, 511, 520, 181, 1550, 1189, 1113, 1644, 537, 320, 1792, 238, 2205, 365, 695, 1613, 346, 1599, 286, 1386, 505, 917, 577, 519, 1155, 511, 186, 598, 69, 1258, 1818, 851, 2126, 123, 623, 127, 1785, 616, 981, 793, 151, 484] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[920, 781, 289, 765, 321, 402, 227, 576, 647, 138, 1518, 398, 686, 1296, 1110, 867, 356, 275, 217, 651, 1187, 1034, 278, 1026, 767, 453, 1150, 1101, 1081, 827, 1750, 411, 831, 613, 295, 94, 805, 1507, 745, 1476, 246, 1553, 540, 1119, 751, 343, 619, 377, 255, 904, 112, 1188, 1356, 1446, 1464, 513, 671, 154, 2209, 623, 940, 741, 292, 865] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[731, 1181, 310, 489, 429, 270, 303, 1237, 370, 227, 876, 346, 1388, 1182, 1316, 438, 674, 343, 757, 490, 907, 969, 362, 682, 498, 629, 1042, 857, 961, 1905, 1568, 513, 921, 654, 923, 255, 697, 1624, 650, 918, 676, 1775, 596, 647, 570, 399, 907, 384, 236, 881, 126, 1083, 1251, 447, 2059, 255, 538, 191, 1957, 418, 632, 996, 362, 874] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[605, 481, 390, 725, 378, 166, 288, 1584, 770, 94, 1607, 227, 604, 963, 1573, 389, 324, 328, 384, 388, 775, 1589, 354, 608, 429, 670, 1699, 1397, 942, 1088, 1130, 517, 1002, 496, 1189, 189, 1179, 1869, 1942, 1052, 284, 1460, 427, 1170, 619, 422, 1019, 591, 621, 652, 142, 447, 806, 860, 802, 636, 765, 177, 1513, 617, 1169, 985, 160, 424] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1156, 1475, 723, 613, 292, 334, 868, 1027, 1042, 98, 1201, 328, 662, 1650, 2076, 369, 358, 554, 597, 187, 1052, 880, 415, 1084, 548, 470, 849, 582, 1029, 1219, 1112, 694, 657, 227, 801, 294, 864, 1657, 461, 1018, 240, 970, 539, 1754, 307, 757, 813, 578, 280, 814, 46, 1364, 879, 574, 1803, 448, 793, 277, 1196, 838, 454, 795, 639, 471] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[675, 927, 862, 696, 374, 340, 95, 1135, 498, 254, 1524, 467, 361, 991, 1131, 402, 213, 371, 246, 426, 1559, 1064, 132, 391, 447, 739, 1903, 1377, 1066, 1043, 1004, 407, 1714, 408, 1085, 275, 1551, 2124, 522, 1895, 190, 346, 351, 2286, 645, 1039, 705, 476, 233, 487, 42, 1279, 1174, 737, 1266, 827, 410, 303, 1575, 505, 359, 443, 155, 625] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[772, 842, 652, 540, 443, 316, 122, 792, 481, 134, 819, 294, 441, 1046, 1618, 495, 404, 399, 442, 331, 1870, 1101, 430, 320, 433, 680, 1210, 1101, 1020, 1752, 1380, 504, 1374, 1160, 1510, 341, 707, 1785, 589, 1139, 407, 1038, 471, 1003, 505, 932, 997, 292, 320, 887, 60, 990, 461, 504, 1690, 1032, 692, 274, 1412, 804, 855, 722, 379, 636] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[827, 1647, 286, 658, 389, 409, 162, 1286, 1544, 0, 0, 0, 0, 0, 0, 0, 429, 186, 1436, 485, 496, 1115, 926, 930, 0, 0, 0, 0, 0, 0, 0, 0, 263, 286, 269, 500, 1111, 1156, 454, 721, 860, 473, 0, 0, 0, 0, 0, 0, 399, 606, 1092, 1243, 945, 0, 885, 452, 0, 0, 0, 0, 0, 0, 0, 0, 1561, 496, 1007, 312, 879, 79, 360, 1250, 0, 0, 0, 0, 0, 0, 0, 0, 404, 657, 541, 1415, 392, 612, 652, 955, 125, 92, 39, 0, 0, 0, 0, 0, 265, 1083, 60, 1106, 955, 814, 1245, 634, 0, 0, 0, 0, 0, 0, 0, 0, 613, 663, 1766, 706, 668, 763, 170, 365, 522, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7208, 6003, 6093, 5622, 5944, 5884, 6162, 6236] avg=6144.0000 max=7208 max_rank=0 min=5622 min_rank=3 imbalance_ratio=1.173177 max_load_reduction=41944 max_load_reduction_pct=0.853353 moved_to_echo=24226 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[546, 712, 164, 553, 232, 241, 138, 464, 1188, 0, 0, 0, 0, 0, 0, 0, 506, 237, 1550, 455, 883, 1661, 1288, 214, 0, 0, 0, 0, 0, 0, 0, 0, 264, 254, 414, 472, 1224, 1334, 308, 511, 995, 364, 0, 0, 0, 0, 0, 0, 520, 181, 1550, 1189, 1113, 0, 537, 320, 0, 0, 0, 0, 0, 0, 0, 0, 1792, 238, 2205, 365, 695, 61, 346, 1599, 0, 0, 0, 0, 0, 0, 0, 0, 286, 1386, 505, 917, 577, 519, 1155, 511, 175, 147, 46, 0, 0, 0, 0, 0, 186, 598, 69, 1258, 1818, 851, 1979, 123, 0, 0, 0, 0, 0, 0, 0, 0, 623, 127, 1785, 616, 981, 793, 151, 484, 603, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4238, 6794, 6140, 5410, 7301, 6224, 6882, 6163] avg=6144.0000 max=7301 max_rank=4 min=4238 min_rank=0 imbalance_ratio=1.188314 max_load_reduction=41851 max_load_reduction_pct=0.851461 moved_to_echo=26570 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[920, 781, 289, 765, 321, 402, 227, 576, 1110, 0, 0, 0, 0, 0, 0, 0, 647, 138, 1518, 398, 686, 1296, 978, 867, 0, 0, 0, 0, 0, 0, 0, 0, 356, 275, 217, 651, 1187, 1034, 278, 1026, 500, 340, 0, 0, 0, 0, 0, 0, 767, 453, 1150, 1101, 1081, 0, 1750, 411, 0, 0, 0, 0, 0, 0, 0, 0, 831, 613, 295, 94, 805, 57, 745, 1476, 0, 0, 0, 0, 0, 0, 0, 0, 246, 1553, 540, 1119, 751, 343, 619, 377, 132, 101, 23, 0, 0, 0, 0, 0, 255, 904, 112, 1188, 1356, 1446, 1363, 513, 0, 0, 0, 0, 0, 0, 0, 0, 671, 154, 2209, 623, 940, 741, 292, 865, 304, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5391, 6528, 5864, 6713, 4916, 5804, 7137, 6799] avg=6144.0000 max=7137 max_rank=6 min=4916 min_rank=4 imbalance_ratio=1.161621 max_load_reduction=42015 max_load_reduction_pct=0.854797 moved_to_echo=24656 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[605, 481, 390, 725, 378, 166, 288, 1584, 1377, 0, 0, 0, 0, 0, 0, 0, 770, 94, 1607, 227, 604, 963, 1385, 389, 0, 0, 0, 0, 0, 0, 0, 0, 324, 328, 384, 388, 775, 1589, 354, 608, 660, 422, 0, 0, 0, 0, 0, 0, 429, 670, 1699, 1397, 942, 0, 1130, 517, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 496, 1189, 189, 1179, 70, 1942, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 284, 1460, 427, 1170, 619, 422, 1019, 591, 188, 55, 30, 0, 0, 0, 0, 0, 621, 652, 142, 447, 806, 860, 747, 636, 0, 0, 0, 0, 0, 0, 0, 0, 765, 177, 1513, 617, 1169, 985, 160, 424, 398, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5994, 6039, 5832, 6784, 7119, 6265, 4911, 6208] avg=6144.0000 max=7119 max_rank=4 min=4911 min_rank=6 imbalance_ratio=1.158691 max_load_reduction=42033 max_load_reduction_pct=0.855164 moved_to_echo=24503 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[731, 1181, 310, 489, 429, 270, 303, 1237, 1199, 0, 0, 0, 0, 0, 0, 0, 370, 227, 876, 346, 1388, 1182, 1155, 438, 0, 0, 0, 0, 0, 0, 0, 0, 674, 343, 757, 490, 907, 969, 362, 682, 1155, 371, 0, 0, 0, 0, 0, 0, 498, 629, 1042, 857, 961, 0, 1568, 513, 0, 0, 0, 0, 0, 0, 0, 0, 921, 654, 923, 255, 697, 54, 650, 918, 0, 0, 0, 0, 0, 0, 0, 0, 676, 1775, 596, 647, 570, 399, 907, 384, 161, 146, 53, 0, 0, 0, 0, 0, 236, 881, 126, 1083, 1251, 447, 1913, 255, 0, 0, 0, 0, 0, 0, 0, 0, 538, 191, 1957, 418, 632, 996, 362, 874, 697, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6149, 5982, 6710, 6068, 5072, 6314, 6192, 6665] avg=6144.0000 max=6710 max_rank=2 min=5072 min_rank=4 imbalance_ratio=1.092122 max_load_reduction=42442 max_load_reduction_pct=0.863485 moved_to_echo=24243 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1156, 1475, 723, 613, 292, 334, 868, 1027, 1221, 0, 0, 0, 0, 0, 0, 0, 1042, 98, 1201, 328, 662, 1650, 1828, 369, 0, 0, 0, 0, 0, 0, 0, 0, 358, 554, 597, 187, 1052, 880, 415, 1084, 739, 374, 0, 0, 0, 0, 0, 0, 548, 470, 849, 582, 1029, 0, 1112, 694, 0, 0, 0, 0, 0, 0, 0, 0, 657, 227, 801, 294, 864, 62, 461, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 240, 970, 539, 1754, 307, 757, 813, 578, 248, 125, 34, 0, 0, 0, 0, 0, 280, 814, 46, 1364, 879, 574, 1678, 448, 0, 0, 0, 0, 0, 0, 0, 0, 793, 277, 1196, 838, 454, 795, 639, 471, 446, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7709, 7178, 6240, 5284, 4384, 6365, 6083, 5909] avg=6144.0000 max=7709 max_rank=0 min=4384 min_rank=4 imbalance_ratio=1.254720 max_load_reduction=41443 max_load_reduction_pct=0.843160 moved_to_echo=22741 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[675, 927, 862, 696, 374, 340, 95, 1135, 1565, 0, 0, 0, 0, 0, 0, 0, 498, 254, 1524, 467, 361, 991, 996, 402, 0, 0, 0, 0, 0, 0, 0, 0, 213, 371, 246, 426, 1559, 1064, 132, 391, 631, 480, 0, 0, 0, 0, 0, 0, 447, 739, 1903, 1377, 1066, 0, 1004, 407, 0, 0, 0, 0, 0, 0, 0, 0, 1714, 408, 1085, 275, 1551, 79, 522, 1895, 0, 0, 0, 0, 0, 0, 0, 0, 190, 346, 351, 2286, 645, 1039, 705, 476, 135, 87, 30, 0, 0, 0, 0, 0, 233, 487, 42, 1279, 1174, 737, 1179, 827, 0, 0, 0, 0, 0, 0, 0, 0, 410, 303, 1575, 505, 359, 443, 155, 625, 382, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6669, 5493, 5513, 6943, 7529, 6290, 5958, 4757] avg=6144.0000 max=7529 max_rank=4 min=4757 min_rank=7 imbalance_ratio=1.225423 max_load_reduction=41623 max_load_reduction_pct=0.846822 moved_to_echo=24534 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[772, 842, 652, 540, 443, 316, 122, 792, 1315, 0, 0, 0, 0, 0, 0, 0, 481, 134, 819, 294, 441, 1046, 1425, 495, 0, 0, 0, 0, 0, 0, 0, 0, 404, 399, 442, 331, 1870, 1101, 430, 320, 1060, 403, 0, 0, 0, 0, 0, 0, 433, 680, 1210, 1101, 1020, 0, 1380, 504, 0, 0, 0, 0, 0, 0, 0, 0, 1374, 1160, 1510, 341, 707, 67, 589, 1139, 0, 0, 0, 0, 0, 0, 0, 0, 407, 1038, 471, 1003, 505, 932, 997, 292, 193, 117, 51, 0, 0, 0, 0, 0, 320, 887, 60, 990, 461, 504, 1573, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 692, 274, 1412, 804, 855, 722, 379, 636, 641, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5794, 5135, 6760, 6328, 6887, 6006, 5827, 6415] avg=6144.0000 max=6887 max_rank=4 min=5135 min_rank=1 imbalance_ratio=1.120931 max_load_reduction=42265 max_load_reduction_pct=0.859884 moved_to_echo=25135 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4096, 4480, 10496, 8960, 8192, 0, 9472, 3840, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[2816, 9216, 4096, 10368, 4480, 5120, 6912, 4224, 1408, 896, 384, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4864, 1408, 10624, 3072, 5632, 9984, 9984, 4224, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5120, 2176, 13440, 5248, 6144, 6272, 2432, 4864, 4096, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2944, 2816, 3328, 3456, 9728, 9216, 2816, 5376, 6656, 3328, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6272, 8064, 3712, 5120, 2944, 2560, 2304, 8192, 10624, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[9856, 4352, 9088, 2176, 7424, 640, 5632, 10368, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2432, 6400, 768, 8832, 8704, 6272, 11776, 4480, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1140, 620, 1237, 429, 2229, 229, 1576, 350, 384, 789, 393, 407, 655, 900, 603, 600, 633, 2057, 151, 387, 1533, 612, 1324, 582, 271, 249, 326, 613, 616, 1358, 255, 1482, 342, 557, 688, 492, 1242, 584, 701, 684, 676, 1255, 1179, 1392, 380, 557, 382, 983, 460, 611, 571, 486, 1383, 1165, 573, 879, 676, 449, 548, 469, 798, 191, 701, 2108] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[429, 368, 665, 272, 1600, 185, 920, 578, 434, 548, 133, 275, 648, 850, 789, 744, 819, 1731, 495, 251, 1864, 239, 923, 552, 398, 323, 61, 542, 1503, 1502, 336, 2610, 325, 557, 342, 310, 1025, 703, 670, 645, 487, 2412, 1866, 1101, 612, 201, 450, 711, 380, 168, 855, 508, 1428, 1713, 367, 557, 488, 355, 462, 689, 638, 1270, 1055, 2215] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[922, 742, 2241, 547, 329, 111, 756, 232, 272, 416, 350, 325, 1134, 655, 543, 682, 734, 1948, 59, 174, 1518, 379, 1668, 737, 670, 346, 147, 715, 970, 1509, 261, 1276, 465, 365, 632, 267, 790, 446, 382, 691, 1089, 1931, 1403, 1196, 611, 803, 432, 834, 460, 587, 596, 792, 1281, 1667, 228, 831, 1028, 1120, 637, 325, 471, 584, 1006, 1834] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[1888, 662, 1094, 788, 591, 160, 857, 494, 224, 582, 270, 335, 2024, 127, 1770, 1029, 1419, 1328, 81, 380, 1260, 320, 941, 924, 330, 467, 319, 340, 958, 751, 412, 1160, 227, 334, 486, 585, 836, 508, 199, 1751, 286, 2098, 866, 847, 439, 742, 356, 1126, 359, 773, 1251, 610, 1063, 1455, 245, 819, 1358, 900, 1073, 146, 392, 153, 635, 1949] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[241, 513, 882, 890, 1427, 103, 994, 738, 310, 795, 257, 333, 663, 139, 1761, 692, 1292, 1795, 84, 470, 2363, 272, 1048, 692, 440, 307, 95, 1026, 494, 901, 311, 1374, 352, 746, 860, 178, 855, 744, 1219, 588, 1186, 1271, 1094, 1444, 1034, 513, 620, 583, 337, 186, 210, 1145, 1782, 1386, 657, 539, 414, 454, 448, 309, 645, 729, 999, 1923] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[685, 410, 1262, 342, 1498, 138, 1088, 585, 498, 519, 389, 340, 477, 223, 675, 1372, 868, 2369, 107, 333, 1915, 390, 1895, 851, 345, 342, 286, 678, 1320, 925, 281, 1943, 500, 540, 659, 308, 811, 798, 319, 381, 991, 808, 1341, 840, 452, 409, 761, 528, 415, 677, 531, 939, 1483, 1562, 416, 798, 765, 775, 675, 215, 534, 730, 1015, 1827] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[486, 612, 1942, 440, 2199, 151, 1751, 507, 313, 667, 245, 436, 193, 599, 835, 1163, 715, 2081, 145, 735, 1741, 423, 891, 556, 477, 379, 138, 980, 876, 1307, 343, 1775, 246, 691, 424, 321, 681, 896, 648, 328, 912, 1329, 1013, 1256, 509, 273, 709, 760, 399, 581, 601, 370, 1580, 1734, 200, 856, 821, 370, 666, 228, 292, 843, 808, 1706] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1385, 398, 1036, 449, 1997, 154, 1551, 578, 593, 592, 330, 333, 1053, 400, 585, 1053, 941, 1567, 113, 345, 1776, 551, 1380, 476, 626, 335, 210, 614, 575, 1237, 278, 1030, 206, 558, 683, 556, 350, 655, 448, 582, 927, 1255, 1755, 731, 504, 510, 682, 786, 252, 562, 1285, 915, 1019, 1648, 595, 518, 1763, 641, 369, 263, 582, 141, 1371, 1499] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[922, 742, 2241, 547, 232, 111, 756, 232, 0, 0, 0, 0, 0, 0, 0, 0, 272, 416, 350, 325, 1134, 655, 543, 682, 968, 97, 86, 0, 0, 0, 0, 0, 734, 632, 59, 174, 1518, 379, 1668, 737, 0, 0, 0, 0, 0, 0, 0, 0, 670, 346, 147, 715, 970, 1509, 261, 1276, 290, 204, 0, 0, 0, 0, 0, 0, 465, 365, 632, 267, 790, 446, 382, 691, 1316, 334, 0, 0, 0, 0, 0, 0, 1089, 629, 1403, 1196, 611, 803, 432, 834, 0, 0, 0, 0, 0, 0, 0, 0, 460, 587, 596, 792, 1281, 1377, 228, 831, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1120, 637, 325, 471, 584, 1006, 1544, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5783, 5528, 5901, 6388, 5688, 6997, 6152, 6715] avg=6144.0000 max=6997 max_rank=5 min=5528 min_rank=1 imbalance_ratio=1.138835 max_load_reduction=42155 max_load_reduction_pct=0.857646 moved_to_echo=25552 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1140, 620, 1237, 429, 1568, 229, 1576, 350, 0, 0, 0, 0, 0, 0, 0, 0, 384, 789, 393, 407, 655, 900, 603, 600, 629, 661, 99, 0, 0, 0, 0, 0, 633, 668, 151, 387, 1533, 612, 1324, 582, 0, 0, 0, 0, 0, 0, 0, 0, 271, 249, 326, 613, 616, 1358, 255, 1482, 202, 235, 0, 0, 0, 0, 0, 0, 342, 557, 688, 492, 1242, 584, 701, 684, 1389, 217, 0, 0, 0, 0, 0, 0, 676, 409, 1179, 1392, 380, 557, 382, 983, 0, 0, 0, 0, 0, 0, 0, 0, 460, 611, 571, 486, 1383, 963, 573, 879, 0, 0, 0, 0, 0, 0, 0, 0, 676, 449, 548, 469, 798, 191, 701, 1774, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7149, 6120, 5890, 5607, 6896, 5958, 5926, 5606] avg=6144.0000 max=7149 max_rank=0 min=5606 min_rank=7 imbalance_ratio=1.163574 max_load_reduction=42003 max_load_reduction_pct=0.854553 moved_to_echo=24386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[685, 410, 1262, 342, 1054, 138, 1088, 585, 0, 0, 0, 0, 0, 0, 0, 0, 498, 519, 389, 340, 477, 223, 675, 1372, 405, 444, 86, 0, 0, 0, 0, 0, 868, 769, 107, 333, 1915, 390, 1895, 851, 0, 0, 0, 0, 0, 0, 0, 0, 345, 342, 286, 678, 1320, 925, 281, 1943, 271, 203, 0, 0, 0, 0, 0, 0, 500, 540, 659, 308, 811, 798, 319, 381, 1600, 139, 0, 0, 0, 0, 0, 0, 991, 264, 1341, 840, 452, 409, 761, 528, 0, 0, 0, 0, 0, 0, 0, 0, 415, 677, 531, 939, 1483, 1291, 416, 798, 0, 0, 0, 0, 0, 0, 0, 0, 765, 775, 675, 215, 534, 730, 1015, 1538, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5564, 5428, 7128, 6594, 6055, 5586, 6550, 6247] avg=6144.0000 max=7128 max_rank=2 min=5428 min_rank=1 imbalance_ratio=1.160156 max_load_reduction=42024 max_load_reduction_pct=0.854980 moved_to_echo=24438 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1888, 662, 1094, 788, 416, 160, 857, 494, 0, 0, 0, 0, 0, 0, 0, 0, 224, 582, 270, 335, 2024, 127, 1770, 1029, 1052, 175, 91, 0, 0, 0, 0, 0, 1419, 431, 81, 380, 1260, 320, 941, 924, 0, 0, 0, 0, 0, 0, 0, 0, 330, 467, 319, 340, 958, 751, 412, 1160, 253, 217, 0, 0, 0, 0, 0, 0, 227, 334, 486, 585, 836, 508, 199, 1751, 897, 363, 0, 0, 0, 0, 0, 0, 286, 683, 866, 847, 439, 742, 356, 1126, 0, 0, 0, 0, 0, 0, 0, 0, 359, 773, 1251, 610, 1063, 1202, 245, 819, 0, 0, 0, 0, 0, 0, 0, 0, 1358, 900, 1073, 146, 392, 153, 635, 1641, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6359, 7679, 5756, 5207, 6186, 5345, 6322, 6298] avg=6144.0000 max=7679 max_rank=1 min=5207 min_rank=3 imbalance_ratio=1.249837 max_load_reduction=41473 max_load_reduction_pct=0.843770 moved_to_echo=24151 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[486, 612, 1942, 440, 1547, 151, 1751, 507, 0, 0, 0, 0, 0, 0, 0, 0, 313, 667, 245, 436, 193, 599, 835, 1163, 666, 652, 80, 0, 0, 0, 0, 0, 715, 675, 145, 735, 1741, 423, 891, 556, 0, 0, 0, 0, 0, 0, 0, 0, 477, 379, 138, 980, 876, 1307, 343, 1775, 301, 190, 0, 0, 0, 0, 0, 0, 246, 691, 424, 321, 681, 896, 648, 328, 1406, 230, 0, 0, 0, 0, 0, 0, 912, 433, 1013, 1256, 509, 273, 709, 760, 0, 0, 0, 0, 0, 0, 0, 0, 399, 581, 601, 370, 1580, 1433, 200, 856, 0, 0, 0, 0, 0, 0, 0, 0, 821, 370, 666, 228, 292, 843, 808, 1436, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7436, 5849, 5881, 6766, 5871, 5865, 6020, 5464] avg=6144.0000 max=7436 max_rank=0 min=5464 min_rank=7 imbalance_ratio=1.210286 max_load_reduction=41716 max_load_reduction_pct=0.848714 moved_to_echo=23220 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[241, 513, 882, 890, 1004, 103, 994, 738, 0, 0, 0, 0, 0, 0, 0, 0, 310, 795, 257, 333, 663, 139, 1761, 692, 637, 423, 90, 0, 0, 0, 0, 0, 1292, 583, 84, 470, 2363, 272, 1048, 692, 0, 0, 0, 0, 0, 0, 0, 0, 440, 307, 95, 1026, 494, 901, 311, 1374, 241, 214, 0, 0, 0, 0, 0, 0, 352, 746, 860, 178, 855, 744, 1219, 588, 1212, 219, 0, 0, 0, 0, 0, 0, 1186, 415, 1094, 1444, 1034, 513, 620, 583, 0, 0, 0, 0, 0, 0, 0, 0, 337, 186, 210, 1145, 1782, 1145, 657, 539, 0, 0, 0, 0, 0, 0, 0, 0, 414, 454, 448, 309, 645, 729, 999, 1619, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5365, 6100, 6804, 5403, 6973, 6889, 6001, 5617] avg=6144.0000 max=6973 max_rank=4 min=5365 min_rank=0 imbalance_ratio=1.134928 max_load_reduction=42179 max_load_reduction_pct=0.858134 moved_to_echo=25480 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1385, 398, 1036, 449, 1402, 154, 1551, 578, 0, 0, 0, 0, 0, 0, 0, 0, 593, 592, 330, 333, 1053, 400, 585, 1053, 632, 595, 74, 0, 0, 0, 0, 0, 941, 505, 113, 345, 1776, 551, 1380, 476, 0, 0, 0, 0, 0, 0, 0, 0, 626, 335, 210, 614, 575, 1237, 278, 1030, 291, 171, 0, 0, 0, 0, 0, 0, 206, 558, 683, 556, 350, 655, 448, 582, 1062, 220, 0, 0, 0, 0, 0, 0, 927, 403, 1755, 731, 504, 510, 682, 786, 0, 0, 0, 0, 0, 0, 0, 0, 252, 562, 1285, 915, 1019, 1357, 595, 518, 0, 0, 0, 0, 0, 0, 0, 0, 1763, 641, 369, 263, 582, 141, 1371, 1254, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6953, 6240, 6087, 5367, 5320, 6298, 6503, 6384] avg=6144.0000 max=6953 max_rank=0 min=5320 min_rank=4 imbalance_ratio=1.131673 max_load_reduction=42199 max_load_reduction_pct=0.858541 moved_to_echo=24505 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[429, 368, 665, 272, 1126, 185, 920, 578, 0, 0, 0, 0, 0, 0, 0, 0, 434, 548, 133, 275, 648, 850, 789, 744, 1209, 474, 104, 0, 0, 0, 0, 0, 819, 562, 495, 251, 1864, 239, 923, 552, 0, 0, 0, 0, 0, 0, 0, 0, 398, 323, 61, 542, 1503, 1502, 336, 2610, 298, 247, 0, 0, 0, 0, 0, 0, 325, 557, 342, 310, 1025, 703, 670, 645, 1169, 417, 0, 0, 0, 0, 0, 0, 487, 786, 1866, 1101, 612, 201, 450, 711, 0, 0, 0, 0, 0, 0, 0, 0, 380, 168, 855, 508, 1428, 1415, 367, 557, 0, 0, 0, 0, 0, 0, 0, 0, 488, 355, 462, 689, 638, 1270, 1055, 1864, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4543, 6208, 5705, 7820, 6163, 6214, 5678, 6821] avg=6144.0000 max=7820 max_rank=3 min=4543 min_rank=0 imbalance_ratio=1.272786 max_load_reduction=41332 max_load_reduction_pct=0.840902 moved_to_echo=24876 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7424, 5120, 4992, 2688, 4352, 4736, 7680, 12672, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6656, 4096, 10624, 8832, 4608, 4096, 4480, 6400, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3072, 4992, 2432, 2816, 6912, 3968, 7680, 7424, 6272, 3584, 768, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3584, 2816, 1664, 5632, 7424, 9600, 2560, 12672, 2176, 1792, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[7424, 4864, 1280, 3200, 14080, 3200, 10112, 5376, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2688, 4352, 4864, 3072, 6656, 5376, 4608, 5760, 10112, 2176, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3072, 4224, 6016, 5888, 11136, 10240, 3328, 5888, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7296, 4352, 10368, 4224, 8448, 1280, 9600, 4096, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[767, 860, 1299, 305, 507, 310, 1043, 115, 749, 269, 387, 194, 674, 1403, 584, 248, 104, 1047, 156, 876, 570, 178, 981, 1958, 905, 343, 878, 503, 1346, 460, 623, 699, 491, 2904, 606, 1050, 1378, 1721, 1175, 1044, 836, 876, 739, 292, 595, 202, 189, 701, 1309, 677, 223, 789, 1111, 1151, 500, 289, 683, 1508, 1181, 494, 831, 1438, 441, 387] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[861, 1844, 559, 789, 443, 239, 2072, 250, 1362, 600, 1165, 265, 399, 967, 410, 292, 122, 661, 373, 880, 491, 376, 277, 1361, 145, 529, 484, 1184, 802, 248, 635, 469, 464, 1935, 840, 916, 1359, 1250, 1101, 799, 1183, 319, 629, 324, 789, 254, 408, 1293, 364, 152, 303, 951, 489, 1716, 710, 1047, 372, 2199, 1091, 569, 1160, 1611, 435, 566] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[100, 1576, 1710, 283, 498, 90, 1127, 198, 1949, 741, 982, 255, 756, 815, 415, 489, 69, 1931, 504, 618, 390, 62, 1021, 1136, 567, 549, 1148, 620, 2138, 41, 552, 467, 1238, 2279, 874, 1403, 785, 1554, 1330, 271, 1151, 233, 591, 201, 294, 662, 178, 566, 655, 342, 342, 1430, 442, 267, 502, 505, 440, 1127, 924, 1471, 862, 1392, 122, 922] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[1203, 2182, 843, 415, 231, 78, 1637, 127, 1114, 307, 103, 308, 606, 303, 409, 230, 58, 509, 285, 806, 977, 88, 352, 2377, 668, 297, 799, 424, 1748, 836, 523, 619, 1698, 2460, 1371, 757, 1312, 1356, 1833, 790, 1706, 486, 314, 207, 347, 256, 76, 660, 988, 336, 349, 921, 503, 1251, 735, 319, 901, 613, 1265, 667, 681, 1196, 745, 591] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[431, 1229, 1182, 417, 470, 51, 2013, 219, 954, 428, 274, 340, 561, 1260, 307, 250, 273, 1875, 440, 614, 1106, 106, 408, 1659, 358, 216, 1215, 501, 2079, 143, 918, 809, 1110, 1693, 604, 1537, 1133, 1050, 1619, 208, 1549, 526, 609, 131, 564, 742, 158, 635, 1044, 563, 331, 1163, 217, 572, 1150, 531, 440, 454, 931, 1057, 852, 2078, 482, 313] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[294, 1224, 1065, 1503, 298, 599, 2386, 265, 853, 266, 899, 475, 649, 637, 444, 207, 168, 893, 298, 819, 537, 268, 871, 1404, 565, 1298, 984, 575, 861, 229, 492, 713, 620, 2519, 517, 452, 1349, 1053, 1112, 1383, 1123, 753, 503, 276, 518, 104, 54, 587, 1249, 373, 264, 278, 450, 2125, 882, 1069, 287, 1843, 587, 562, 783, 1568, 250, 620] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[162, 1684, 883, 836, 323, 273, 1249, 224, 975, 700, 281, 372, 1467, 340, 975, 862, 85, 580, 148, 1427, 678, 291, 439, 460, 539, 719, 615, 148, 1336, 689, 199, 534, 1387, 2797, 1061, 1053, 828, 1122, 1347, 773, 1123, 545, 390, 280, 639, 198, 82, 521, 1072, 216, 286, 878, 644, 2082, 577, 312, 908, 1329, 506, 387, 1494, 2767, 395, 660] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[353, 1886, 1265, 608, 468, 981, 1365, 179, 1367, 343, 769, 296, 576, 343, 273, 498, 179, 1047, 256, 1382, 726, 61, 306, 2382, 626, 304, 782, 503, 2296, 278, 561, 548, 797, 1186, 1092, 1714, 1425, 1541, 690, 385, 1254, 858, 509, 377, 329, 80, 271, 485, 847, 253, 219, 725, 1051, 331, 1694, 399, 468, 1332, 1197, 876, 620, 1208, 893, 239] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[861, 1844, 559, 789, 443, 239, 1784, 250, 0, 0, 0, 0, 0, 0, 0, 0, 1362, 600, 1165, 265, 399, 967, 410, 292, 846, 321, 0, 0, 0, 0, 0, 0, 122, 661, 373, 880, 491, 376, 277, 1361, 395, 243, 0, 0, 0, 0, 0, 0, 145, 529, 484, 1184, 802, 248, 635, 469, 518, 0, 0, 0, 0, 0, 0, 0, 464, 0, 840, 916, 1359, 9, 1101, 799, 0, 0, 0, 0, 0, 0, 0, 0, 1183, 319, 629, 324, 789, 254, 408, 1293, 1614, 0, 0, 0, 0, 0, 0, 0, 364, 152, 303, 951, 489, 1716, 710, 1047, 223, 288, 0, 0, 0, 0, 0, 0, 372, 2199, 1091, 569, 1160, 627, 435, 566, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6769, 6627, 5179, 5014, 5488, 6813, 6243, 7019] avg=6144.0000 max=7019 max_rank=7 min=5014 min_rank=3 imbalance_ratio=1.142415 max_load_reduction=42133 max_load_reduction_pct=0.857198 moved_to_echo=25563 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[767, 860, 1299, 305, 507, 310, 898, 115, 0, 0, 0, 0, 0, 0, 0, 0, 749, 269, 387, 194, 674, 1403, 584, 248, 1165, 482, 0, 0, 0, 0, 0, 0, 104, 1047, 156, 876, 570, 178, 981, 1958, 544, 217, 0, 0, 0, 0, 0, 0, 905, 343, 878, 503, 1346, 460, 623, 699, 463, 0, 0, 0, 0, 0, 0, 0, 491, 0, 606, 1050, 1378, 12, 1175, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 836, 876, 739, 292, 595, 202, 189, 701, 2422, 0, 0, 0, 0, 0, 0, 0, 1309, 677, 223, 789, 1111, 1151, 500, 289, 199, 145, 0, 0, 0, 0, 0, 0, 683, 1508, 1181, 494, 831, 559, 441, 387, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5061, 6155, 6631, 6220, 5756, 6852, 6393, 6084] avg=6144.0000 max=6852 max_rank=5 min=5061 min_rank=0 imbalance_ratio=1.115234 max_load_reduction=42300 max_load_reduction_pct=0.860596 moved_to_echo=25085 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[431, 1229, 1182, 417, 470, 51, 1733, 219, 0, 0, 0, 0, 0, 0, 0, 0, 954, 428, 274, 340, 561, 1260, 307, 250, 711, 281, 0, 0, 0, 0, 0, 0, 273, 1875, 440, 614, 1106, 106, 408, 1659, 331, 314, 0, 0, 0, 0, 0, 0, 358, 216, 1215, 501, 2079, 143, 918, 809, 669, 0, 0, 0, 0, 0, 0, 0, 1110, 0, 604, 1537, 1133, 8, 1619, 208, 0, 0, 0, 0, 0, 0, 0, 0, 1549, 526, 609, 131, 564, 742, 158, 635, 1412, 0, 0, 0, 0, 0, 0, 0, 1044, 563, 331, 1163, 217, 572, 1150, 531, 288, 280, 0, 0, 0, 0, 0, 0, 440, 454, 931, 1057, 852, 807, 482, 313, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5732, 5366, 7126, 6908, 6219, 6326, 6139, 5336] avg=6144.0000 max=7126 max_rank=2 min=5336 min_rank=7 imbalance_ratio=1.159831 max_load_reduction=42026 max_load_reduction_pct=0.855021 moved_to_echo=24020 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[100, 1576, 1710, 283, 498, 90, 971, 198, 0, 0, 0, 0, 0, 0, 0, 0, 1949, 741, 982, 255, 756, 815, 415, 489, 1052, 379, 0, 0, 0, 0, 0, 0, 69, 1931, 504, 618, 390, 62, 1021, 1136, 491, 210, 0, 0, 0, 0, 0, 0, 567, 549, 1148, 620, 2138, 41, 552, 467, 448, 0, 0, 0, 0, 0, 0, 0, 1238, 0, 874, 1403, 785, 11, 1330, 271, 0, 0, 0, 0, 0, 0, 0, 0, 1151, 233, 591, 201, 294, 662, 178, 566, 1900, 0, 0, 0, 0, 0, 0, 0, 655, 342, 342, 1430, 442, 267, 502, 505, 193, 156, 0, 0, 0, 0, 0, 0, 440, 1127, 924, 1471, 862, 541, 122, 922, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5426, 7833, 6432, 6530, 5912, 5776, 4834, 6409] avg=6144.0000 max=7833 max_rank=1 min=4834 min_rank=6 imbalance_ratio=1.274902 max_load_reduction=41319 max_load_reduction_pct=0.840637 moved_to_echo=22931 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[294, 1224, 1065, 1503, 298, 599, 2050, 265, 0, 0, 0, 0, 0, 0, 0, 0, 853, 266, 899, 475, 649, 637, 444, 207, 716, 419, 0, 0, 0, 0, 0, 0, 168, 893, 298, 819, 537, 268, 871, 1404, 336, 242, 0, 0, 0, 0, 0, 0, 565, 1298, 984, 575, 861, 229, 492, 713, 507, 0, 0, 0, 0, 0, 0, 0, 620, 0, 517, 452, 1349, 1, 1112, 1383, 0, 0, 0, 0, 0, 0, 0, 0, 1123, 753, 503, 276, 518, 104, 54, 587, 2100, 0, 0, 0, 0, 0, 0, 0, 1249, 373, 264, 278, 450, 2125, 882, 1069, 223, 336, 0, 0, 0, 0, 0, 0, 287, 1843, 587, 562, 783, 596, 250, 620, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7298, 5565, 5836, 6224, 5434, 6018, 7249, 5528] avg=6144.0000 max=7298 max_rank=0 min=5434 min_rank=4 imbalance_ratio=1.187826 max_load_reduction=41854 max_load_reduction_pct=0.851522 moved_to_echo=24229 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[162, 1684, 883, 836, 323, 273, 1076, 224, 0, 0, 0, 0, 0, 0, 0, 0, 975, 700, 281, 372, 1467, 340, 975, 862, 759, 465, 0, 0, 0, 0, 0, 0, 85, 580, 148, 1427, 678, 291, 439, 460, 354, 418, 0, 0, 0, 0, 0, 0, 539, 719, 615, 148, 1336, 689, 199, 534, 891, 0, 0, 0, 0, 0, 0, 0, 1387, 0, 1061, 1053, 828, 9, 1347, 773, 0, 0, 0, 0, 0, 0, 0, 0, 1123, 545, 390, 280, 639, 198, 82, 521, 2332, 0, 0, 0, 0, 0, 0, 0, 1072, 216, 286, 878, 644, 2082, 577, 312, 384, 173, 0, 0, 0, 0, 0, 0, 908, 1329, 506, 387, 1494, 1074, 395, 660, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5461, 7196, 4880, 5670, 6458, 6110, 6624, 6753] avg=6144.0000 max=7196 max_rank=1 min=4880 min_rank=2 imbalance_ratio=1.171224 max_load_reduction=41956 max_load_reduction_pct=0.853597 moved_to_echo=25945 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1203, 2182, 843, 415, 231, 78, 1410, 127, 0, 0, 0, 0, 0, 0, 0, 0, 1114, 307, 103, 308, 606, 303, 409, 230, 918, 408, 0, 0, 0, 0, 0, 0, 58, 509, 285, 806, 977, 88, 352, 2377, 428, 180, 0, 0, 0, 0, 0, 0, 668, 297, 799, 424, 1748, 836, 523, 619, 385, 0, 0, 0, 0, 0, 0, 0, 1698, 0, 1371, 757, 1312, 10, 1833, 790, 0, 0, 0, 0, 0, 0, 0, 0, 1706, 486, 314, 207, 347, 256, 76, 660, 2052, 0, 0, 0, 0, 0, 0, 0, 988, 336, 349, 921, 503, 1251, 735, 319, 166, 227, 0, 0, 0, 0, 0, 0, 901, 613, 1265, 667, 681, 465, 745, 591, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6489, 4706, 6060, 6299, 7771, 6104, 5795, 5928] avg=6144.0000 max=7771 max_rank=4 min=4706 min_rank=1 imbalance_ratio=1.264811 max_load_reduction=41381 max_load_reduction_pct=0.841899 moved_to_echo=25598 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[353, 1886, 1265, 608, 468, 981, 1176, 179, 0, 0, 0, 0, 0, 0, 0, 0, 1367, 343, 769, 296, 576, 343, 273, 498, 1043, 196, 0, 0, 0, 0, 0, 0, 179, 1047, 256, 1382, 726, 61, 306, 2382, 487, 182, 0, 0, 0, 0, 0, 0, 626, 304, 782, 503, 2296, 278, 561, 548, 389, 0, 0, 0, 0, 0, 0, 0, 797, 0, 1092, 1714, 1425, 11, 690, 385, 0, 0, 0, 0, 0, 0, 0, 0, 1254, 858, 509, 377, 329, 80, 271, 485, 990, 0, 0, 0, 0, 0, 0, 0, 847, 253, 219, 725, 1051, 331, 1694, 399, 167, 189, 0, 0, 0, 0, 0, 0, 468, 1332, 1197, 876, 620, 470, 893, 239, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6916, 5704, 7008, 6287, 6114, 5153, 5875, 6095] avg=6144.0000 max=7008 max_rank=2 min=5153 min_rank=5 imbalance_ratio=1.140625 max_load_reduction=42144 max_load_reduction_pct=0.857422 moved_to_echo=23237 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[9344, 3712, 4864, 2560, 5760, 6144, 3840, 3200, 7296, 3072, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[7552, 2944, 2432, 7168, 4992, 9600, 6784, 4480, 1920, 1920, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4224, 12544, 8832, 5248, 3328, 2688, 11136, 1664, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4608, 10496, 7808, 6144, 7296, 5248, 3840, 4352, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[9984, 4608, 4352, 2176, 4096, 2560, 1536, 5504, 14848, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1152, 8576, 2560, 7424, 5504, 1536, 4736, 12800, 3456, 2048, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4480, 4352, 6912, 4480, 12672, 2944, 4608, 4864, 4352, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[7808, 0, 7040, 8960, 9600, 128, 10240, 5760, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[215, 340, 422, 1422, 1103, 622, 313, 1261, 833, 204, 187, 1337, 1251, 667, 1062, 1224, 667, 372, 277, 388, 361, 183, 1082, 2490, 258, 951, 1253, 905, 419, 139, 955, 124, 2605, 1029, 1152, 1521, 1931, 668, 180, 514, 1786, 1535, 535, 643, 528, 1739, 437, 788, 464, 226, 160, 842, 73, 998, 568, 377, 417, 372, 1479, 134, 1105, 703, 237, 119] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[332, 359, 168, 1261, 384, 420, 169, 441, 930, 291, 520, 304, 1836, 711, 986, 2805, 449, 284, 644, 538, 1043, 522, 795, 546, 141, 648, 2766, 398, 146, 206, 1429, 79, 1380, 791, 1422, 1086, 2378, 1430, 224, 1660, 1185, 1687, 1311, 426, 1014, 919, 63, 532, 933, 72, 391, 210, 483, 673, 486, 990, 306, 776, 613, 262, 1509, 389, 968, 32] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[276, 393, 339, 2148, 662, 516, 523, 787, 814, 457, 616, 457, 1451, 968, 1135, 840, 626, 592, 632, 691, 974, 522, 618, 657, 350, 758, 1005, 1070, 210, 712, 1070, 766, 598, 1104, 1715, 1105, 2194, 1226, 176, 2383, 895, 1546, 606, 374, 419, 2575, 111, 422, 202, 616, 608, 257, 81, 964, 557, 606, 191, 748, 1122, 260, 516, 1023, 109, 208] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[371, 66, 258, 2232, 1083, 501, 288, 730, 414, 374, 386, 1349, 1317, 1009, 1240, 705, 820, 1197, 768, 287, 496, 505, 742, 649, 131, 515, 1669, 1002, 213, 978, 2392, 223, 457, 698, 1180, 1064, 3093, 536, 106, 468, 906, 884, 879, 726, 823, 1874, 89, 533, 1047, 113, 67, 569, 366, 1210, 329, 661, 567, 807, 1604, 297, 879, 1145, 183, 82] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[295, 590, 255, 1549, 1627, 377, 511, 722, 1100, 282, 393, 237, 2271, 486, 429, 385, 901, 322, 1260, 678, 635, 597, 732, 529, 379, 1046, 410, 1578, 48, 726, 405, 971, 576, 856, 640, 918, 2761, 356, 73, 1892, 499, 1403, 380, 2874, 480, 1202, 377, 502, 216, 355, 31, 655, 240, 810, 1833, 1055, 67, 1200, 619, 599, 540, 2233, 61, 123] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[653, 526, 265, 965, 1507, 586, 348, 1644, 903, 513, 600, 441, 1742, 890, 1386, 2407, 507, 510, 427, 347, 509, 382, 1415, 586, 94, 571, 960, 522, 350, 181, 1138, 92, 1117, 830, 334, 931, 2328, 519, 252, 400, 516, 1341, 761, 1923, 785, 1741, 244, 612, 1147, 188, 141, 477, 770, 1261, 819, 749, 466, 1487, 950, 154, 1342, 404, 102, 94] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[310, 212, 394, 2086, 820, 508, 113, 755, 828, 190, 334, 1201, 1740, 254, 1108, 764, 870, 1457, 424, 726, 299, 437, 989, 730, 144, 749, 1265, 1726, 440, 304, 1666, 917, 883, 852, 1383, 1371, 2384, 527, 791, 1461, 1205, 1449, 1025, 263, 350, 1288, 233, 553, 372, 538, 522, 615, 57, 1259, 370, 291, 77, 373, 1869, 447, 580, 707, 222, 75] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[129, 124, 573, 2707, 362, 526, 130, 112, 549, 396, 724, 1286, 1809, 405, 601, 1708, 1606, 88, 484, 326, 405, 234, 1320, 777, 834, 657, 1221, 608, 82, 422, 1876, 416, 912, 688, 1440, 1101, 3858, 614, 271, 2174, 1298, 1517, 510, 272, 1044, 915, 145, 76, 541, 225, 478, 265, 297, 954, 562, 1080, 842, 346, 405, 117, 1594, 333, 669, 112] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[215, 340, 422, 1422, 1103, 622, 313, 1261, 396, 313, 0, 0, 0, 0, 0, 0, 833, 204, 187, 1337, 517, 667, 1062, 1224, 0, 0, 0, 0, 0, 0, 0, 0, 667, 372, 277, 388, 361, 183, 1082, 2490, 739, 47, 0, 0, 0, 0, 0, 0, 258, 951, 1253, 905, 419, 139, 955, 124, 116, 0, 0, 0, 0, 0, 0, 0, 2605, 1029, 1152, 1521, 0, 668, 180, 351, 0, 0, 0, 0, 0, 0, 0, 0, 1786, 1535, 535, 643, 528, 687, 437, 788, 0, 0, 0, 0, 0, 0, 0, 0, 464, 226, 160, 842, 73, 998, 568, 377, 1271, 0, 0, 0, 0, 0, 0, 0, 417, 372, 1479, 134, 1105, 703, 237, 119, 660, 338, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6407, 6031, 6606, 5120, 7506, 6939, 4979, 5564] avg=6144.0000 max=7506 max_rank=4 min=4979 min_rank=6 imbalance_ratio=1.221680 max_load_reduction=41646 max_load_reduction_pct=0.847290 moved_to_echo=24988 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[332, 359, 168, 1261, 384, 420, 169, 441, 582, 165, 0, 0, 0, 0, 0, 0, 930, 291, 520, 304, 757, 711, 986, 2805, 0, 0, 0, 0, 0, 0, 0, 0, 449, 284, 644, 538, 1043, 522, 795, 546, 390, 153, 0, 0, 0, 0, 0, 0, 141, 648, 2766, 398, 146, 206, 1429, 79, 374, 0, 0, 0, 0, 0, 0, 0, 1380, 791, 1422, 1086, 0, 1430, 224, 1133, 0, 0, 0, 0, 0, 0, 0, 0, 1185, 1687, 1311, 426, 1014, 364, 63, 532, 0, 0, 0, 0, 0, 0, 0, 0, 933, 72, 391, 210, 483, 673, 486, 990, 1565, 0, 0, 0, 0, 0, 0, 0, 306, 776, 613, 262, 1509, 389, 968, 32, 813, 497, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4281, 7304, 5364, 6187, 7466, 6582, 5803, 6165] avg=6144.0000 max=7466 max_rank=4 min=4281 min_rank=0 imbalance_ratio=1.215169 max_load_reduction=41686 max_load_reduction_pct=0.848104 moved_to_echo=26016 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[276, 393, 339, 2148, 662, 516, 523, 787, 460, 464, 0, 0, 0, 0, 0, 0, 814, 457, 616, 457, 598, 968, 1135, 840, 0, 0, 0, 0, 0, 0, 0, 0, 626, 592, 632, 691, 974, 522, 618, 657, 1095, 220, 0, 0, 0, 0, 0, 0, 350, 758, 1005, 1070, 210, 712, 1070, 766, 538, 0, 0, 0, 0, 0, 0, 0, 598, 1104, 1715, 1105, 0, 1226, 176, 1625, 0, 0, 0, 0, 0, 0, 0, 0, 895, 1546, 606, 374, 419, 1016, 111, 422, 0, 0, 0, 0, 0, 0, 0, 0, 202, 616, 608, 257, 81, 964, 557, 606, 1444, 0, 0, 0, 0, 0, 0, 0, 191, 748, 1122, 260, 516, 1023, 109, 208, 750, 393, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6568, 5885, 6627, 6479, 7549, 5389, 5335, 5320] avg=6144.0000 max=7549 max_rank=4 min=5320 min_rank=7 imbalance_ratio=1.228678 max_load_reduction=41603 max_load_reduction_pct=0.846415 moved_to_echo=23593 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[295, 590, 255, 1549, 1627, 377, 511, 722, 720, 216, 0, 0, 0, 0, 0, 0, 1100, 282, 393, 237, 936, 486, 429, 385, 0, 0, 0, 0, 0, 0, 0, 0, 901, 322, 1260, 678, 635, 597, 732, 529, 511, 175, 0, 0, 0, 0, 0, 0, 379, 1046, 410, 1578, 48, 726, 405, 971, 427, 0, 0, 0, 0, 0, 0, 0, 576, 856, 640, 918, 0, 356, 73, 1290, 0, 0, 0, 0, 0, 0, 0, 0, 499, 1403, 380, 2874, 480, 475, 377, 502, 0, 0, 0, 0, 0, 0, 0, 0, 216, 355, 31, 655, 240, 810, 1833, 1055, 1819, 0, 0, 0, 0, 0, 0, 0, 67, 1200, 619, 599, 540, 2233, 61, 123, 942, 615, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6862, 4248, 6340, 5990, 4709, 6990, 7014, 6999] avg=6144.0000 max=7014 max_rank=6 min=4248 min_rank=1 imbalance_ratio=1.141602 max_load_reduction=42138 max_load_reduction_pct=0.857300 moved_to_echo=25712 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[371, 66, 258, 2232, 1083, 501, 288, 730, 417, 338, 0, 0, 0, 0, 0, 0, 414, 374, 386, 1349, 544, 1009, 1240, 705, 0, 0, 0, 0, 0, 0, 0, 0, 820, 1197, 768, 287, 496, 505, 742, 649, 796, 43, 0, 0, 0, 0, 0, 0, 131, 515, 1669, 1002, 213, 978, 2392, 223, 105, 0, 0, 0, 0, 0, 0, 0, 457, 698, 1180, 1064, 0, 536, 106, 320, 0, 0, 0, 0, 0, 0, 0, 0, 906, 884, 879, 726, 823, 740, 89, 533, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 113, 67, 569, 366, 1210, 329, 661, 2037, 0, 0, 0, 0, 0, 0, 0, 567, 807, 1604, 297, 879, 1145, 183, 82, 1056, 356, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6284, 6021, 6303, 7228, 4361, 5580, 6399, 6976] avg=6144.0000 max=7228 max_rank=3 min=4361 min_rank=4 imbalance_ratio=1.176432 max_load_reduction=41924 max_load_reduction_pct=0.852946 moved_to_echo=23316 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[310, 212, 394, 2086, 820, 508, 113, 755, 551, 232, 0, 0, 0, 0, 0, 0, 828, 190, 334, 1201, 718, 254, 1108, 764, 0, 0, 0, 0, 0, 0, 0, 0, 870, 1457, 424, 726, 299, 437, 989, 730, 547, 135, 0, 0, 0, 0, 0, 0, 144, 749, 1265, 1726, 440, 304, 1666, 917, 329, 0, 0, 0, 0, 0, 0, 0, 883, 852, 1383, 1371, 0, 527, 791, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1205, 1449, 1025, 263, 350, 509, 233, 553, 0, 0, 0, 0, 0, 0, 0, 0, 372, 538, 522, 615, 57, 1259, 370, 291, 1570, 0, 0, 0, 0, 0, 0, 0, 77, 373, 1869, 447, 580, 707, 222, 75, 814, 471, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5981, 5397, 6614, 7540, 6804, 5587, 5594, 5635] avg=6144.0000 max=7540 max_rank=3 min=5397 min_rank=1 imbalance_ratio=1.227214 max_load_reduction=41612 max_load_reduction_pct=0.846598 moved_to_echo=23620 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[653, 526, 265, 965, 1507, 586, 348, 1644, 557, 317, 0, 0, 0, 0, 0, 0, 903, 513, 600, 441, 710, 890, 1386, 2407, 0, 0, 0, 0, 0, 0, 0, 0, 507, 510, 427, 347, 509, 382, 1415, 586, 744, 41, 0, 0, 0, 0, 0, 0, 94, 571, 960, 522, 350, 181, 1138, 92, 94, 0, 0, 0, 0, 0, 0, 0, 1117, 830, 334, 931, 0, 519, 252, 265, 0, 0, 0, 0, 0, 0, 0, 0, 516, 1341, 761, 1923, 785, 680, 244, 612, 0, 0, 0, 0, 0, 0, 0, 0, 1147, 188, 141, 477, 770, 1261, 819, 749, 1533, 0, 0, 0, 0, 0, 0, 0, 466, 1487, 950, 154, 1342, 404, 102, 94, 795, 475, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7368, 7850, 5468, 4002, 4248, 6862, 7085, 6269] avg=6144.0000 max=7850 max_rank=1 min=4002 min_rank=3 imbalance_ratio=1.277669 max_load_reduction=41302 max_load_reduction_pct=0.840291 moved_to_echo=24464 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[129, 124, 573, 2707, 362, 526, 130, 112, 573, 165, 0, 0, 0, 0, 0, 0, 549, 396, 724, 1286, 747, 405, 601, 1708, 0, 0, 0, 0, 0, 0, 0, 0, 1606, 88, 484, 326, 405, 234, 1320, 777, 389, 201, 0, 0, 0, 0, 0, 0, 834, 657, 1221, 608, 82, 422, 1876, 416, 490, 0, 0, 0, 0, 0, 0, 0, 912, 688, 1440, 1101, 0, 614, 271, 1483, 0, 0, 0, 0, 0, 0, 0, 0, 1298, 1517, 510, 272, 1044, 361, 145, 76, 0, 0, 0, 0, 0, 0, 0, 0, 541, 225, 478, 265, 297, 954, 562, 1080, 2541, 0, 0, 0, 0, 0, 0, 0, 842, 346, 405, 117, 1594, 333, 669, 112, 1317, 489, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5401, 6416, 5830, 6606, 6509, 5223, 6943, 6224] avg=6144.0000 max=6943 max_rank=6 min=5223 min_rank=5 imbalance_ratio=1.130046 max_load_reduction=42209 max_load_reduction_pct=0.858744 moved_to_echo=24899 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2688, 2688, 2688, 14464, 7552, 4096, 2432, 6528, 4352, 2304, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4992, 2432, 2432, 3968, 2432, 8192, 5632, 5888, 13824, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6400, 2816, 3840, 6656, 5632, 5504, 8064, 10880, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[8320, 11392, 6016, 7552, 5504, 4864, 1792, 4096, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6528, 4864, 4992, 4096, 4736, 3456, 7808, 7040, 5248, 1024, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2944, 6144, 8704, 2304, 8192, 7040, 2560, 896, 7168, 3712, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[8576, 6912, 9344, 9216, 0, 5888, 2176, 7552, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[2432, 6016, 10624, 7936, 1920, 3712, 11008, 3712, 2560, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[113, 642, 1083, 716, 934, 669, 437, 348, 789, 1140, 112, 861, 618, 534, 688, 209, 933, 74, 1212, 2056, 1899, 695, 639, 1231, 488, 557, 431, 1476, 826, 237, 169, 1327, 204, 148, 1776, 485, 117, 135, 259, 1025, 1009, 896, 564, 2109, 99, 228, 168, 1512, 956, 837, 169, 725, 1113, 298, 1666, 159, 483, 986, 2609, 1607, 693, 1011, 855, 108] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[270, 779, 574, 642, 822, 464, 100, 273, 530, 665, 1044, 979, 211, 651, 867, 198, 914, 87, 1369, 1412, 1116, 320, 379, 968, 355, 712, 325, 1243, 663, 423, 119, 2479, 209, 215, 2248, 585, 201, 291, 578, 560, 767, 523, 278, 1556, 385, 712, 867, 2141, 569, 1136, 172, 332, 581, 486, 1276, 1380, 980, 832, 2030, 2129, 1624, 1152, 189, 215] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[536, 862, 1291, 606, 111, 576, 530, 635, 746, 1281, 282, 954, 400, 846, 620, 263, 564, 137, 1078, 1980, 460, 556, 243, 2100, 152, 615, 193, 1143, 832, 556, 294, 1510, 353, 314, 2329, 576, 436, 243, 266, 1243, 1120, 547, 1545, 1178, 426, 463, 272, 826, 1149, 347, 127, 492, 334, 323, 2368, 229, 295, 568, 1221, 2606, 1349, 1357, 959, 339] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[88, 572, 883, 1003, 686, 651, 181, 787, 735, 1102, 134, 713, 616, 822, 827, 143, 413, 238, 1038, 3299, 413, 947, 243, 468, 451, 75, 180, 1261, 1005, 176, 379, 1545, 144, 1040, 2091, 536, 150, 639, 326, 1406, 1069, 556, 1238, 3039, 261, 174, 13, 1345, 834, 1212, 250, 525, 891, 340, 1641, 173, 359, 863, 899, 1806, 117, 701, 1147, 1293] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[137, 369, 374, 706, 459, 1015, 137, 1464, 662, 1849, 319, 927, 623, 104, 1218, 157, 129, 290, 2455, 1752, 301, 425, 884, 4114, 148, 152, 209, 974, 223, 802, 285, 753, 253, 241, 2032, 489, 752, 197, 467, 694, 2397, 67, 510, 1010, 110, 159, 135, 1105, 907, 513, 130, 319, 487, 257, 1176, 107, 753, 870, 2832, 1797, 1869, 683, 567, 851] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[258, 899, 615, 507, 707, 728, 153, 990, 531, 601, 1742, 519, 286, 1210, 1312, 202, 122, 60, 637, 1749, 692, 507, 518, 2000, 287, 525, 534, 1818, 1021, 456, 737, 1665, 270, 372, 796, 612, 713, 379, 276, 1496, 423, 949, 840, 2407, 206, 62, 585, 1211, 1077, 303, 285, 410, 407, 913, 2174, 176, 547, 1245, 1502, 1394, 438, 1055, 539, 502] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[112, 1010, 1762, 1143, 471, 733, 183, 775, 660, 772, 424, 511, 217, 803, 1009, 209, 1052, 134, 479, 952, 608, 287, 875, 915, 313, 428, 335, 1706, 424, 576, 520, 1604, 455, 212, 2119, 584, 442, 388, 626, 1312, 1368, 1930, 968, 1190, 333, 426, 501, 1592, 480, 1140, 230, 381, 216, 694, 1648, 467, 379, 651, 629, 2177, 1165, 1298, 753, 396] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[331, 524, 629, 222, 3034, 1020, 41, 690, 911, 620, 180, 345, 376, 1075, 625, 168, 372, 170, 601, 709, 585, 497, 130, 1111, 504, 629, 83, 1143, 999, 147, 162, 2272, 344, 858, 2112, 646, 439, 285, 203, 647, 471, 852, 833, 3059, 336, 190, 200, 1115, 620, 725, 330, 257, 698, 507, 2781, 223, 455, 1511, 2584, 550, 867, 1841, 559, 1149] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[113, 642, 1083, 716, 934, 669, 437, 348, 1475, 0, 0, 0, 0, 0, 0, 0, 789, 1140, 112, 861, 618, 534, 688, 209, 1134, 175, 0, 0, 0, 0, 0, 0, 933, 74, 1212, 1105, 1899, 695, 639, 1231, 0, 0, 0, 0, 0, 0, 0, 0, 488, 557, 431, 1476, 826, 237, 169, 1327, 494, 25, 0, 0, 0, 0, 0, 0, 204, 148, 1776, 485, 117, 135, 259, 1025, 776, 143, 0, 0, 0, 0, 0, 0, 1009, 896, 564, 1260, 99, 228, 168, 1512, 0, 0, 0, 0, 0, 0, 0, 0, 956, 837, 169, 725, 1113, 298, 1666, 159, 681, 0, 0, 0, 0, 0, 0, 0, 483, 986, 0, 1113, 693, 1011, 855, 108, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6417, 6260, 7788, 6030, 5068, 5736, 6604, 5249] avg=6144.0000 max=7788 max_rank=2 min=5068 min_rank=4 imbalance_ratio=1.267578 max_load_reduction=41364 max_load_reduction_pct=0.841553 moved_to_echo=22657 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[536, 862, 1291, 606, 111, 576, 530, 635, 690, 0, 0, 0, 0, 0, 0, 0, 746, 1281, 282, 954, 400, 846, 620, 263, 531, 169, 0, 0, 0, 0, 0, 0, 564, 137, 1078, 1064, 460, 556, 243, 2100, 0, 0, 0, 0, 0, 0, 0, 0, 152, 615, 193, 1143, 832, 556, 294, 1510, 802, 14, 0, 0, 0, 0, 0, 0, 353, 314, 2329, 576, 436, 243, 266, 1243, 747, 80, 0, 0, 0, 0, 0, 0, 1120, 547, 1545, 704, 426, 463, 272, 826, 0, 0, 0, 0, 0, 0, 0, 0, 1149, 347, 127, 492, 334, 323, 2368, 229, 380, 0, 0, 0, 0, 0, 0, 0, 295, 568, 0, 1804, 1349, 1357, 959, 339, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5837, 6092, 6202, 6111, 6587, 5903, 5749, 6671] avg=6144.0000 max=6671 max_rank=7 min=5749 min_rank=6 imbalance_ratio=1.085775 max_load_reduction=42481 max_load_reduction_pct=0.864278 moved_to_echo=24910 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[112, 1010, 1762, 1143, 471, 733, 183, 775, 355, 0, 0, 0, 0, 0, 0, 0, 660, 772, 424, 511, 217, 803, 1009, 209, 274, 81, 0, 0, 0, 0, 0, 0, 1052, 134, 479, 512, 608, 287, 875, 915, 0, 0, 0, 0, 0, 0, 0, 0, 313, 428, 335, 1706, 424, 576, 520, 1604, 670, 14, 0, 0, 0, 0, 0, 0, 455, 212, 2119, 584, 442, 388, 626, 1312, 359, 81, 0, 0, 0, 0, 0, 0, 1368, 1930, 968, 711, 333, 426, 501, 1592, 0, 0, 0, 0, 0, 0, 0, 0, 480, 1140, 230, 381, 216, 694, 1648, 467, 384, 0, 0, 0, 0, 0, 0, 0, 379, 651, 0, 1507, 1165, 1298, 753, 396, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6544, 4960, 4862, 6590, 6578, 7829, 5640, 6149] avg=6144.0000 max=7829 max_rank=5 min=4862 min_rank=2 imbalance_ratio=1.274251 max_load_reduction=41323 max_load_reduction_pct=0.840719 moved_to_echo=26196 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[270, 779, 574, 642, 822, 464, 100, 273, 1148, 0, 0, 0, 0, 0, 0, 0, 530, 665, 1044, 979, 211, 651, 867, 198, 882, 124, 0, 0, 0, 0, 0, 0, 914, 87, 1369, 751, 1116, 320, 379, 968, 0, 0, 0, 0, 0, 0, 0, 0, 355, 712, 325, 1243, 663, 423, 119, 2479, 659, 22, 0, 0, 0, 0, 0, 0, 209, 215, 2248, 585, 201, 291, 578, 560, 537, 110, 0, 0, 0, 0, 0, 0, 767, 523, 278, 917, 385, 712, 867, 2141, 0, 0, 0, 0, 0, 0, 0, 0, 569, 1136, 172, 332, 581, 486, 1276, 1380, 507, 0, 0, 0, 0, 0, 0, 0, 980, 832, 0, 1470, 1624, 1152, 189, 215, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5072, 6151, 5904, 7000, 5534, 6590, 6439, 6462] avg=6144.0000 max=7000 max_rank=3 min=5072 min_rank=0 imbalance_ratio=1.139323 max_load_reduction=42152 max_load_reduction_pct=0.857585 moved_to_echo=25025 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[88, 572, 883, 1003, 686, 651, 181, 787, 509, 0, 0, 0, 0, 0, 0, 0, 735, 1102, 134, 713, 616, 822, 827, 143, 390, 282, 0, 0, 0, 0, 0, 0, 413, 238, 1038, 1772, 413, 947, 243, 468, 0, 0, 0, 0, 0, 0, 0, 0, 451, 75, 180, 1261, 1005, 176, 379, 1545, 556, 36, 0, 0, 0, 0, 0, 0, 144, 1040, 2091, 536, 150, 639, 326, 1406, 1245, 207, 0, 0, 0, 0, 0, 0, 1069, 556, 1238, 1815, 261, 174, 13, 1345, 0, 0, 0, 0, 0, 0, 0, 0, 834, 1212, 250, 525, 891, 340, 1641, 173, 981, 0, 0, 0, 0, 0, 0, 0, 359, 863, 0, 1250, 117, 701, 1147, 1293, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5360, 5764, 5532, 5664, 7784, 6471, 6847, 5730] avg=6144.0000 max=7784 max_rank=4 min=5360 min_rank=0 imbalance_ratio=1.266927 max_load_reduction=41368 max_load_reduction_pct=0.841634 moved_to_echo=26832 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[137, 369, 374, 706, 459, 1015, 137, 1464, 1602, 0, 0, 0, 0, 0, 0, 0, 662, 1849, 319, 927, 623, 104, 1218, 157, 1230, 149, 0, 0, 0, 0, 0, 0, 129, 290, 2455, 942, 301, 425, 884, 4114, 0, 0, 0, 0, 0, 0, 0, 0, 148, 152, 209, 974, 223, 802, 285, 753, 553, 12, 0, 0, 0, 0, 0, 0, 253, 241, 2032, 489, 752, 197, 467, 694, 661, 68, 0, 0, 0, 0, 0, 0, 2397, 67, 510, 604, 110, 159, 135, 1105, 0, 0, 0, 0, 0, 0, 0, 0, 907, 513, 130, 319, 487, 257, 1176, 107, 326, 0, 0, 0, 0, 0, 0, 0, 753, 870, 0, 1244, 1869, 683, 567, 851, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6263, 7238, 9540, 4111, 5854, 5087, 4222, 6837] avg=6144.0000 max=9540 max_rank=2 min=4111 min_rank=3 imbalance_ratio=1.552734 max_load_reduction=39612 max_load_reduction_pct=0.805908 moved_to_echo=22000 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[331, 524, 629, 222, 3034, 1020, 41, 690, 1462, 0, 0, 0, 0, 0, 0, 0, 911, 620, 180, 345, 376, 1075, 625, 168, 1122, 60, 0, 0, 0, 0, 0, 0, 372, 170, 601, 382, 585, 497, 130, 1111, 0, 0, 0, 0, 0, 0, 0, 0, 504, 629, 83, 1143, 999, 147, 162, 2272, 169, 37, 0, 0, 0, 0, 0, 0, 344, 858, 2112, 646, 439, 285, 203, 647, 267, 208, 0, 0, 0, 0, 0, 0, 471, 852, 833, 1826, 336, 190, 200, 1115, 0, 0, 0, 0, 0, 0, 0, 0, 620, 725, 330, 257, 698, 507, 2781, 223, 988, 0, 0, 0, 0, 0, 0, 0, 455, 1511, 0, 381, 867, 1841, 559, 1149, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7953, 5482, 3848, 6145, 6009, 5823, 7129, 6763] avg=6144.0000 max=7953 max_rank=0 min=3848 min_rank=2 imbalance_ratio=1.294434 max_load_reduction=41199 max_load_reduction_pct=0.838196 moved_to_echo=25724 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[258, 899, 615, 507, 707, 728, 153, 990, 849, 0, 0, 0, 0, 0, 0, 0, 531, 601, 1742, 519, 286, 1210, 1312, 202, 653, 149, 0, 0, 0, 0, 0, 0, 122, 60, 637, 940, 692, 507, 518, 2000, 0, 0, 0, 0, 0, 0, 0, 0, 287, 525, 534, 1818, 1021, 456, 737, 1665, 429, 29, 0, 0, 0, 0, 0, 0, 270, 372, 796, 612, 713, 379, 276, 1496, 660, 164, 0, 0, 0, 0, 0, 0, 423, 949, 840, 1437, 206, 62, 585, 1211, 0, 0, 0, 0, 0, 0, 0, 0, 1077, 303, 285, 410, 407, 913, 2174, 176, 777, 0, 0, 0, 0, 0, 0, 0, 547, 1245, 0, 965, 438, 1055, 539, 502, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5706, 7205, 5476, 7501, 5738, 5713, 6522, 5291] avg=6144.0000 max=7501 max_rank=3 min=5291 min_rank=7 imbalance_ratio=1.220866 max_load_reduction=41651 max_load_reduction_pct=0.847392 moved_to_echo=23264 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[5632, 8064, 4352, 5888, 3456, 6144, 7168, 1664, 6272, 1280, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4608, 1280, 8960, 7552, 6144, 4352, 3968, 12928, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4352, 7552, 0, 9856, 8192, 9216, 5632, 4864, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6656, 6272, 1792, 3456, 4736, 3840, 14848, 2944, 5120, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1920, 5760, 7296, 5632, 7296, 5888, 1792, 6016, 8192, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[8704, 6400, 6784, 9344, 2176, 2432, 2816, 10880, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[2304, 3456, 15616, 4608, 3328, 2560, 3072, 8448, 5376, 1152, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2816, 3712, 2304, 10880, 6016, 3456, 2688, 13184, 4352, 256, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[701, 26, 799, 322, 309, 765, 31, 853, 69, 1184, 1011, 936, 792, 399, 1039, 869, 506, 500, 539, 1991, 247, 641, 513, 168, 323, 348, 256, 189, 79, 1227, 401, 746, 672, 794, 636, 767, 1715, 605, 336, 1602, 3184, 1085, 1577, 216, 1334, 343, 273, 67, 780, 1021, 336, 3246, 412, 1767, 204, 809, 217, 46, 770, 561, 930, 1175, 2300, 593] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[630, 20, 125, 829, 73, 410, 35, 316, 17, 869, 404, 1034, 552, 235, 2347, 944, 954, 1916, 1281, 484, 189, 135, 410, 277, 594, 109, 556, 8, 330, 790, 941, 125, 933, 723, 632, 896, 2678, 284, 1991, 820, 2867, 930, 833, 162, 628, 740, 269, 148, 768, 679, 363, 2478, 415, 3232, 273, 892, 736, 155, 523, 596, 2400, 906, 1039, 224] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[1026, 38, 519, 649, 165, 290, 44, 568, 454, 1126, 765, 1371, 734, 489, 268, 842, 1410, 486, 420, 700, 705, 1426, 961, 799, 377, 308, 246, 223, 256, 1485, 431, 321, 1230, 286, 965, 1598, 1553, 563, 1192, 818, 1734, 742, 906, 608, 1196, 884, 280, 476, 814, 732, 713, 2996, 381, 1710, 496, 553, 419, 200, 744, 1168, 507, 733, 1398, 655] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[226, 15, 1220, 389, 185, 961, 68, 373, 22, 634, 388, 710, 824, 335, 1894, 1456, 1812, 991, 579, 1353, 203, 359, 285, 323, 168, 326, 224, 177, 95, 1608, 1454, 61, 572, 512, 1537, 949, 1703, 290, 1311, 2610, 2174, 942, 601, 381, 1206, 1260, 72, 160, 540, 1802, 617, 1898, 94, 1375, 189, 811, 303, 274, 835, 1382, 1348, 1001, 619, 66] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[1909, 18, 55, 1351, 79, 353, 16, 304, 33, 1213, 525, 716, 724, 1299, 1190, 1286, 670, 947, 163, 994, 289, 342, 392, 434, 470, 436, 212, 88, 169, 1390, 694, 429, 1117, 308, 1426, 1523, 1896, 768, 1455, 540, 1656, 356, 1450, 74, 1877, 1120, 272, 397, 410, 491, 257, 3807, 120, 2463, 225, 606, 140, 312, 587, 525, 1025, 1290, 799, 670] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[604, 59, 347, 1273, 54, 494, 84, 940, 93, 1285, 585, 1153, 950, 126, 2342, 762, 834, 1439, 552, 1240, 193, 908, 442, 755, 615, 240, 751, 345, 113, 423, 342, 645, 784, 2030, 503, 226, 1211, 478, 2086, 355, 563, 1922, 923, 191, 687, 2456, 126, 129, 1308, 919, 786, 2039, 269, 1464, 138, 34, 593, 364, 770, 1424, 1611, 928, 650, 197] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[712, 15, 729, 845, 353, 455, 14, 601, 28, 463, 763, 1524, 977, 592, 1343, 1007, 305, 1587, 293, 736, 1731, 307, 1519, 365, 475, 88, 334, 96, 204, 1826, 345, 541, 1040, 611, 297, 652, 1968, 629, 377, 939, 2164, 177, 993, 175, 2202, 1162, 1119, 596, 1283, 397, 235, 3024, 150, 1337, 580, 567, 151, 200, 1291, 816, 749, 731, 1080, 287] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[167, 24, 825, 888, 228, 499, 219, 133, 571, 3907, 501, 1150, 1145, 866, 1671, 1683, 1249, 931, 563, 605, 703, 403, 192, 320, 641, 80, 267, 65, 643, 575, 1059, 555, 1694, 455, 848, 797, 627, 540, 432, 2017, 1253, 706, 1160, 69, 2563, 1195, 394, 61, 404, 937, 554, 2546, 266, 1346, 636, 394, 89, 175, 807, 1054, 270, 232, 1234, 69] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[701, 26, 799, 322, 309, 765, 31, 853, 2097, 936, 0, 0, 0, 0, 0, 0, 69, 1184, 1011, 936, 792, 399, 325, 869, 0, 0, 0, 0, 0, 0, 0, 0, 506, 500, 539, 1991, 247, 641, 513, 168, 313, 110, 0, 0, 0, 0, 0, 0, 323, 348, 256, 189, 79, 1227, 401, 746, 1901, 779, 329, 0, 0, 0, 0, 0, 672, 794, 636, 767, 0, 605, 336, 1289, 0, 0, 0, 0, 0, 0, 0, 0, 1283, 1085, 1577, 216, 1334, 343, 273, 67, 0, 0, 0, 0, 0, 0, 0, 0, 780, 1021, 336, 1149, 412, 1767, 204, 809, 0, 0, 0, 0, 0, 0, 0, 0, 217, 46, 770, 561, 930, 1175, 2300, 593, 275, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6839, 5585, 5528, 6578, 5099, 6178, 6478, 6867] avg=6144.0000 max=6867 max_rank=7 min=5099 min_rank=4 imbalance_ratio=1.117676 max_load_reduction=42285 max_load_reduction_pct=0.860291 moved_to_echo=24622 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1909, 18, 55, 1351, 79, 353, 16, 304, 2459, 1035, 0, 0, 0, 0, 0, 0, 33, 1213, 525, 716, 724, 1299, 372, 1286, 0, 0, 0, 0, 0, 0, 0, 0, 670, 947, 163, 994, 289, 342, 392, 434, 105, 126, 0, 0, 0, 0, 0, 0, 470, 436, 212, 88, 169, 1390, 694, 429, 989, 861, 377, 0, 0, 0, 0, 0, 1117, 308, 1426, 1523, 0, 768, 1455, 435, 0, 0, 0, 0, 0, 0, 0, 0, 667, 356, 1450, 74, 1877, 1120, 272, 397, 0, 0, 0, 0, 0, 0, 0, 0, 410, 491, 257, 1348, 120, 2463, 225, 606, 0, 0, 0, 0, 0, 0, 0, 0, 140, 312, 587, 525, 1025, 1290, 799, 670, 315, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7579, 6168, 4462, 6115, 7032, 6213, 5920, 5663] avg=6144.0000 max=7579 max_rank=0 min=4462 min_rank=2 imbalance_ratio=1.233561 max_load_reduction=41573 max_load_reduction_pct=0.845805 moved_to_echo=24828 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[226, 15, 1220, 389, 185, 961, 68, 373, 1226, 929, 0, 0, 0, 0, 0, 0, 22, 634, 388, 710, 824, 335, 590, 1456, 0, 0, 0, 0, 0, 0, 0, 0, 1812, 991, 579, 1353, 203, 359, 285, 323, 510, 201, 0, 0, 0, 0, 0, 0, 168, 326, 224, 177, 95, 1608, 1454, 61, 1298, 774, 601, 0, 0, 0, 0, 0, 572, 512, 1537, 949, 0, 290, 1311, 2100, 0, 0, 0, 0, 0, 0, 0, 0, 876, 942, 601, 381, 1206, 1260, 72, 160, 0, 0, 0, 0, 0, 0, 0, 0, 540, 1802, 617, 672, 94, 1375, 189, 811, 0, 0, 0, 0, 0, 0, 0, 0, 303, 274, 835, 1382, 1348, 1001, 619, 66, 502, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5592, 4959, 6616, 6786, 7271, 5498, 6100, 6330] avg=6144.0000 max=7271 max_rank=4 min=4959 min_rank=1 imbalance_ratio=1.183431 max_load_reduction=41881 max_load_reduction_pct=0.852071 moved_to_echo=25199 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[630, 20, 125, 829, 73, 410, 35, 316, 1601, 1462, 0, 0, 0, 0, 0, 0, 17, 869, 404, 1034, 552, 235, 732, 944, 0, 0, 0, 0, 0, 0, 0, 0, 954, 1916, 1281, 484, 189, 135, 410, 277, 160, 249, 0, 0, 0, 0, 0, 0, 594, 109, 556, 8, 330, 790, 941, 125, 1712, 1216, 744, 0, 0, 0, 0, 0, 933, 723, 632, 896, 0, 284, 1991, 660, 0, 0, 0, 0, 0, 0, 0, 0, 1155, 930, 833, 162, 628, 740, 269, 148, 0, 0, 0, 0, 0, 0, 0, 0, 768, 679, 363, 877, 415, 3232, 273, 892, 0, 0, 0, 0, 0, 0, 0, 0, 736, 155, 523, 596, 2400, 906, 1039, 224, 622, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5501, 4787, 6055, 7125, 6119, 4865, 7499, 7201] avg=6144.0000 max=7499 max_rank=6 min=4787 min_rank=1 imbalance_ratio=1.220540 max_load_reduction=41653 max_load_reduction_pct=0.847432 moved_to_echo=25684 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1026, 38, 519, 649, 165, 290, 44, 568, 1939, 848, 0, 0, 0, 0, 0, 0, 454, 1126, 765, 1371, 734, 489, 72, 842, 0, 0, 0, 0, 0, 0, 0, 0, 1410, 486, 420, 700, 705, 1426, 961, 799, 165, 33, 0, 0, 0, 0, 0, 0, 377, 308, 246, 223, 256, 1485, 431, 321, 1038, 705, 88, 0, 0, 0, 0, 0, 1230, 286, 965, 1598, 0, 563, 1192, 653, 0, 0, 0, 0, 0, 0, 0, 0, 696, 742, 906, 608, 1196, 884, 280, 476, 0, 0, 0, 0, 0, 0, 0, 0, 814, 732, 713, 1057, 381, 1710, 496, 553, 0, 0, 0, 0, 0, 0, 0, 0, 419, 200, 744, 1168, 507, 733, 1398, 655, 75, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6086, 5853, 7105, 5478, 6487, 5788, 6456, 5899] avg=6144.0000 max=7105 max_rank=2 min=5478 min_rank=3 imbalance_ratio=1.156413 max_load_reduction=42047 max_load_reduction_pct=0.855448 moved_to_echo=24630 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[167, 24, 825, 888, 228, 499, 219, 133, 1644, 343, 0, 0, 0, 0, 0, 0, 571, 3907, 501, 1150, 1145, 866, 521, 1683, 0, 0, 0, 0, 0, 0, 0, 0, 1249, 931, 563, 605, 703, 403, 192, 320, 394, 177, 0, 0, 0, 0, 0, 0, 641, 80, 267, 65, 643, 575, 1059, 555, 748, 284, 530, 0, 0, 0, 0, 0, 1694, 455, 848, 797, 0, 540, 432, 1623, 0, 0, 0, 0, 0, 0, 0, 0, 505, 706, 1160, 69, 2563, 1195, 394, 61, 0, 0, 0, 0, 0, 0, 0, 0, 404, 937, 554, 902, 266, 1346, 636, 394, 0, 0, 0, 0, 0, 0, 0, 0, 89, 175, 807, 1054, 270, 232, 1234, 69, 443, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4970, 10344, 5537, 5447, 6389, 6653, 5439, 4373] avg=6144.0000 max=10344 max_rank=1 min=4373 min_rank=7 imbalance_ratio=1.683594 max_load_reduction=38808 max_load_reduction_pct=0.789551 moved_to_echo=22854 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[604, 59, 347, 1273, 54, 494, 84, 940, 1317, 662, 0, 0, 0, 0, 0, 0, 93, 1285, 585, 1153, 950, 126, 729, 762, 0, 0, 0, 0, 0, 0, 0, 0, 834, 1439, 552, 1240, 193, 908, 442, 755, 69, 249, 0, 0, 0, 0, 0, 0, 615, 240, 751, 345, 113, 423, 342, 645, 336, 549, 743, 0, 0, 0, 0, 0, 784, 2030, 503, 226, 0, 478, 2086, 286, 0, 0, 0, 0, 0, 0, 0, 0, 227, 1922, 923, 191, 687, 2456, 126, 129, 0, 0, 0, 0, 0, 0, 0, 0, 1308, 919, 786, 722, 269, 1464, 138, 34, 0, 0, 0, 0, 0, 0, 0, 0, 593, 364, 770, 1424, 1611, 928, 650, 197, 621, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5834, 5683, 6681, 5102, 6393, 6661, 5640, 7158] avg=6144.0000 max=7158 max_rank=7 min=5102 min_rank=3 imbalance_ratio=1.165039 max_load_reduction=41994 max_load_reduction_pct=0.854370 moved_to_echo=25852 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[712, 15, 729, 845, 353, 455, 14, 601, 1953, 1074, 0, 0, 0, 0, 0, 0, 28, 463, 763, 1524, 977, 592, 419, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 305, 1587, 293, 736, 1731, 307, 1519, 365, 183, 142, 0, 0, 0, 0, 0, 0, 475, 88, 334, 96, 204, 1826, 345, 541, 1292, 894, 426, 0, 0, 0, 0, 0, 1040, 611, 297, 652, 0, 629, 377, 756, 0, 0, 0, 0, 0, 0, 0, 0, 872, 177, 993, 175, 2202, 1162, 1119, 596, 0, 0, 0, 0, 0, 0, 0, 0, 1283, 397, 235, 1071, 150, 1337, 580, 567, 0, 0, 0, 0, 0, 0, 0, 0, 151, 200, 1291, 816, 749, 731, 1080, 287, 356, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6751, 5773, 7168, 6521, 4362, 7296, 5620, 5661] avg=6144.0000 max=7296 max_rank=5 min=4362 min_rank=4 imbalance_ratio=1.187500 max_load_reduction=41856 max_load_reduction_pct=0.851562 moved_to_echo=22939 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6016, 256, 4736, 6656, 1536, 4352, 512, 4096, 14336, 7296, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[7808, 8832, 4480, 8192, 4352, 4608, 4736, 3456, 1920, 1408, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[8064, 5760, 6912, 7424, 0, 4224, 9216, 7808, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6400, 7040, 3968, 7808, 2176, 14720, 2816, 4736, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3712, 2048, 2944, 1280, 1920, 9344, 5760, 3456, 9344, 6144, 3840, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2688, 1792, 6400, 7552, 8960, 7040, 9216, 2816, 3328, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1408, 10752, 4992, 8704, 6784, 4352, 3840, 8960, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[6400, 6912, 8448, 1920, 11776, 9216, 2816, 2048, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[118, 419, 147, 650, 435, 274, 251, 266, 304, 518, 1523, 683, 351, 508, 712, 743, 949, 722, 447, 797, 555, 938, 110, 700, 226, 114, 554, 1383, 2312, 684, 157, 95, 198, 515, 1835, 626, 1110, 260, 804, 467, 3523, 1229, 983, 528, 232, 1022, 422, 111, 96, 495, 253, 558, 3082, 2430, 1549, 363, 726, 3595, 813, 424, 609, 833, 594, 222] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[413, 234, 235, 802, 297, 136, 204, 228, 796, 1053, 800, 509, 617, 316, 386, 512, 2639, 737, 65, 1994, 943, 2232, 106, 1750, 1002, 20, 408, 1370, 1094, 197, 562, 106, 27, 61, 911, 515, 1415, 894, 387, 500, 3757, 613, 1156, 552, 224, 1191, 76, 241, 449, 1416, 166, 411, 1477, 1493, 788, 726, 1406, 1419, 318, 93, 1175, 621, 1304, 607] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[286, 63, 240, 846, 369, 455, 972, 416, 569, 766, 963, 449, 949, 844, 948, 253, 550, 930, 200, 993, 1337, 739, 206, 886, 961, 130, 329, 955, 1134, 657, 465, 167, 193, 425, 177, 356, 883, 1002, 827, 662, 3261, 438, 1563, 733, 81, 1169, 141, 548, 515, 799, 182, 475, 3122, 2531, 944, 1029, 606, 2103, 782, 406, 210, 933, 997, 1032] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[323, 804, 273, 676, 445, 452, 357, 368, 242, 475, 2557, 420, 1090, 484, 1002, 371, 689, 635, 203, 781, 1064, 2126, 374, 948, 147, 258, 708, 1148, 869, 358, 532, 248, 293, 574, 550, 459, 1202, 175, 1641, 553, 4051, 990, 1074, 502, 131, 384, 221, 907, 317, 892, 425, 176, 2885, 2176, 730, 807, 1482, 763, 527, 230, 1406, 1013, 112, 77] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[394, 631, 162, 1167, 569, 257, 103, 309, 366, 407, 1744, 416, 460, 1119, 385, 529, 1453, 252, 303, 743, 714, 459, 1033, 1248, 519, 47, 597, 1885, 242, 164, 843, 73, 18, 287, 768, 141, 497, 141, 842, 1013, 2608, 2075, 1832, 232, 210, 683, 181, 469, 100, 1177, 871, 276, 2611, 2771, 516, 640, 1094, 2973, 1520, 485, 799, 1407, 61, 261] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[109, 538, 186, 954, 470, 25, 226, 235, 576, 408, 1478, 396, 712, 607, 588, 1548, 2087, 332, 556, 1472, 1008, 764, 230, 116, 294, 46, 72, 973, 867, 307, 571, 439, 59, 19, 907, 114, 313, 376, 163, 421, 4767, 721, 1270, 224, 82, 1794, 933, 72, 470, 952, 307, 291, 1552, 3173, 672, 1234, 1901, 2402, 175, 95, 1269, 1054, 2030, 150] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[359, 572, 140, 1022, 503, 712, 51, 327, 306, 1278, 1675, 537, 501, 370, 659, 298, 1049, 912, 83, 539, 1490, 564, 415, 2021, 792, 131, 232, 1635, 1407, 533, 1068, 607, 35, 84, 529, 737, 549, 210, 446, 300, 6033, 2130, 2282, 507, 847, 341, 443, 712, 13, 1190, 395, 492, 624, 2197, 404, 525, 1757, 611, 348, 181, 121, 633, 405, 283] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[244, 47, 159, 1113, 248, 643, 161, 710, 358, 542, 1193, 767, 925, 1490, 1058, 362, 1383, 465, 66, 781, 717, 1415, 839, 1267, 810, 150, 186, 1855, 850, 119, 1192, 53, 10, 33, 2154, 571, 1657, 82, 518, 311, 4571, 1158, 1808, 621, 162, 474, 114, 92, 29, 859, 1551, 65, 3308, 2081, 206, 469, 897, 1201, 577, 502, 204, 447, 60, 192] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[413, 234, 235, 802, 297, 136, 204, 228, 2693, 0, 0, 0, 0, 0, 0, 0, 796, 1053, 800, 509, 617, 316, 386, 512, 224, 0, 0, 0, 0, 0, 0, 0, 1181, 737, 65, 1994, 943, 2232, 106, 1750, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 20, 408, 1370, 1094, 197, 562, 106, 1458, 180, 188, 0, 0, 0, 0, 0, 27, 61, 911, 515, 1415, 894, 387, 500, 1136, 0, 0, 0, 0, 0, 0, 0, 1064, 613, 1156, 552, 224, 1191, 76, 241, 0, 0, 0, 0, 0, 0, 0, 0, 449, 1416, 166, 411, 1477, 177, 788, 726, 0, 0, 0, 0, 0, 0, 0, 0, 1406, 1007, 318, 93, 1175, 621, 1304, 607, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5242, 5213, 9008, 6585, 5846, 5117, 5610, 6531] avg=6144.0000 max=9008 max_rank=2 min=5117 min_rank=5 imbalance_ratio=1.466146 max_load_reduction=40144 max_load_reduction_pct=0.816732 moved_to_echo=23104 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[394, 631, 162, 1167, 569, 257, 103, 309, 1869, 0, 0, 0, 0, 0, 0, 0, 366, 407, 1744, 416, 460, 1119, 385, 529, 469, 0, 0, 0, 0, 0, 0, 0, 650, 252, 303, 743, 714, 459, 1033, 1248, 0, 0, 0, 0, 0, 0, 0, 0, 519, 47, 597, 1885, 242, 164, 843, 73, 803, 334, 394, 0, 0, 0, 0, 0, 18, 287, 768, 141, 497, 141, 842, 1013, 2109, 0, 0, 0, 0, 0, 0, 0, 739, 2075, 1832, 232, 210, 683, 181, 469, 0, 0, 0, 0, 0, 0, 0, 0, 100, 1177, 871, 276, 2611, 328, 516, 640, 0, 0, 0, 0, 0, 0, 0, 0, 1094, 2110, 1520, 485, 799, 1407, 61, 261, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5461, 5895, 5402, 5901, 5816, 6421, 6519, 7737] avg=6144.0000 max=7737 max_rank=7 min=5402 min_rank=2 imbalance_ratio=1.259277 max_load_reduction=41415 max_load_reduction_pct=0.842590 moved_to_echo=26493 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[118, 419, 147, 650, 435, 274, 251, 266, 2525, 0, 0, 0, 0, 0, 0, 0, 304, 518, 1523, 683, 351, 508, 712, 743, 568, 0, 0, 0, 0, 0, 0, 0, 425, 722, 447, 797, 555, 938, 110, 700, 0, 0, 0, 0, 0, 0, 0, 0, 226, 114, 554, 1383, 2312, 684, 157, 95, 524, 293, 477, 0, 0, 0, 0, 0, 198, 515, 1835, 626, 1110, 260, 804, 467, 1849, 0, 0, 0, 0, 0, 0, 0, 998, 1229, 983, 528, 232, 1022, 422, 111, 0, 0, 0, 0, 0, 0, 0, 0, 96, 495, 253, 558, 3082, 288, 1549, 363, 0, 0, 0, 0, 0, 0, 0, 0, 726, 2550, 813, 424, 609, 833, 594, 222, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5085, 5910, 4694, 6819, 7664, 5525, 6684, 6771] avg=6144.0000 max=7664 max_rank=4 min=4694 min_rank=2 imbalance_ratio=1.247396 max_load_reduction=41488 max_load_reduction_pct=0.844076 moved_to_echo=26644 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[109, 538, 186, 954, 470, 25, 226, 235, 3417, 0, 0, 0, 0, 0, 0, 0, 576, 408, 1478, 396, 712, 607, 588, 1548, 379, 0, 0, 0, 0, 0, 0, 0, 934, 332, 556, 1472, 1008, 764, 230, 116, 0, 0, 0, 0, 0, 0, 0, 0, 294, 46, 72, 973, 867, 307, 571, 439, 1153, 382, 318, 0, 0, 0, 0, 0, 59, 19, 907, 114, 313, 376, 163, 421, 2415, 0, 0, 0, 0, 0, 0, 0, 1350, 721, 1270, 224, 82, 1794, 933, 72, 0, 0, 0, 0, 0, 0, 0, 0, 470, 952, 307, 291, 1552, 376, 672, 1234, 0, 0, 0, 0, 0, 0, 0, 0, 1901, 1705, 175, 95, 1269, 1054, 2030, 150, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6160, 6692, 5412, 5422, 4787, 6446, 5854, 8379] avg=6144.0000 max=8379 max_rank=7 min=4787 min_rank=4 imbalance_ratio=1.363770 max_load_reduction=40773 max_load_reduction_pct=0.829529 moved_to_echo=25466 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[323, 804, 273, 676, 445, 452, 357, 368, 2904, 0, 0, 0, 0, 0, 0, 0, 242, 475, 2557, 420, 1090, 484, 1002, 371, 120, 0, 0, 0, 0, 0, 0, 0, 309, 635, 203, 781, 1064, 2126, 374, 948, 0, 0, 0, 0, 0, 0, 0, 0, 147, 258, 708, 1148, 869, 358, 532, 248, 380, 262, 101, 0, 0, 0, 0, 0, 293, 574, 550, 459, 1202, 175, 1641, 553, 1656, 0, 0, 0, 0, 0, 0, 0, 1147, 990, 1074, 502, 131, 384, 221, 907, 0, 0, 0, 0, 0, 0, 0, 0, 317, 892, 425, 176, 2885, 258, 730, 807, 0, 0, 0, 0, 0, 0, 0, 0, 1482, 542, 527, 230, 1406, 1013, 112, 77, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6602, 6761, 6440, 5011, 7103, 5356, 6490, 5389] avg=6144.0000 max=7103 max_rank=4 min=5011 min_rank=3 imbalance_ratio=1.156087 max_load_reduction=42049 max_load_reduction_pct=0.855489 moved_to_echo=24338 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[286, 63, 240, 846, 369, 455, 972, 416, 2341, 2, 0, 0, 0, 0, 0, 0, 569, 766, 963, 449, 949, 844, 948, 253, 336, 0, 0, 0, 0, 0, 0, 0, 242, 930, 200, 993, 1337, 739, 206, 886, 0, 0, 0, 0, 0, 0, 0, 0, 961, 130, 329, 955, 1134, 657, 465, 167, 308, 307, 282, 0, 0, 0, 0, 0, 193, 425, 177, 356, 883, 1002, 827, 662, 1929, 0, 0, 0, 0, 0, 0, 0, 920, 438, 1563, 733, 81, 1169, 141, 548, 0, 0, 0, 0, 0, 0, 0, 0, 515, 799, 182, 475, 3122, 293, 944, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 606, 1485, 782, 406, 210, 933, 997, 1032, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5990, 6077, 5533, 5695, 6454, 5593, 7359, 6451] avg=6144.0000 max=7359 max_rank=6 min=5533 min_rank=2 imbalance_ratio=1.197754 max_load_reduction=41793 max_load_reduction_pct=0.850281 moved_to_echo=25857 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[244, 47, 159, 1113, 248, 643, 161, 710, 3276, 0, 0, 0, 0, 0, 0, 0, 358, 542, 1193, 767, 925, 1490, 1058, 362, 189, 0, 0, 0, 0, 0, 0, 0, 619, 465, 66, 781, 717, 1415, 839, 1267, 0, 0, 0, 0, 0, 0, 0, 0, 810, 150, 186, 1855, 850, 119, 1192, 53, 764, 251, 159, 0, 0, 0, 0, 0, 10, 33, 2154, 571, 1657, 82, 518, 311, 1584, 0, 0, 0, 0, 0, 0, 0, 1295, 1158, 1808, 621, 162, 474, 114, 92, 0, 0, 0, 0, 0, 0, 0, 0, 29, 859, 1551, 65, 3308, 246, 206, 469, 0, 0, 0, 0, 0, 0, 0, 0, 897, 853, 577, 502, 204, 447, 60, 192, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6601, 6884, 6169, 6389, 6920, 5724, 6733, 3732] avg=6144.0000 max=6920 max_rank=4 min=3732 min_rank=7 imbalance_ratio=1.126302 max_load_reduction=42232 max_load_reduction_pct=0.859212 moved_to_echo=23109 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[359, 572, 140, 1022, 503, 712, 51, 327, 4325, 0, 0, 0, 0, 0, 0, 0, 306, 1278, 1675, 537, 501, 370, 659, 298, 96, 0, 0, 0, 0, 0, 0, 0, 470, 912, 83, 539, 1490, 564, 415, 2021, 0, 0, 0, 0, 0, 0, 0, 0, 792, 131, 232, 1635, 1407, 533, 1068, 607, 579, 265, 81, 0, 0, 0, 0, 0, 35, 84, 529, 737, 549, 210, 446, 300, 1672, 0, 0, 0, 0, 0, 0, 0, 1708, 2130, 2282, 507, 847, 341, 443, 712, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1190, 395, 492, 624, 260, 404, 525, 0, 0, 0, 0, 0, 0, 0, 0, 1757, 434, 348, 181, 121, 633, 405, 283, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8011, 5720, 6494, 7330, 4562, 8970, 3903, 4162] avg=6144.0000 max=8970 max_rank=5 min=3903 min_rank=6 imbalance_ratio=1.459961 max_load_reduction=40182 max_load_reduction_pct=0.817505 moved_to_echo=21597 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4864, 4992, 2048, 8192, 7936, 9344, 3328, 8960, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2048, 7808, 4224, 2816, 18688, 2304, 5888, 5888, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4864, 896, 3200, 11264, 8832, 3072, 5504, 1792, 6016, 2304, 2048, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[2304, 3328, 1664, 7296, 3456, 3072, 2432, 2944, 23424, 128, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[896, 2048, 7936, 3584, 7680, 3200, 5632, 4352, 14464, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[9984, 10752, 5120, 2432, 5888, 7040, 5632, 2944, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[9344, 9472, 12032, 3968, 2048, 7168, 2560, 3200, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3584, 5504, 12032, 4224, 5632, 5760, 5760, 4736, 2432, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[378, 1021, 290, 336, 528, 134, 2087, 1154, 2124, 263, 821, 780, 2125, 365, 267, 39, 1267, 1589, 260, 432, 278, 1194, 14, 402, 65, 1465, 665, 192, 403, 891, 1284, 13, 297, 58, 125, 403, 3076, 60, 832, 827, 448, 223, 1166, 2542, 1780, 2061, 2304, 99, 930, 338, 414, 507, 264, 27, 133, 2927, 390, 305, 826, 684, 790, 305, 542, 343] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[364, 2338, 327, 28, 626, 171, 1407, 790, 1451, 649, 93, 414, 1041, 583, 202, 52, 898, 1273, 1213, 94, 552, 748, 294, 1463, 106, 1484, 713, 317, 395, 1365, 1010, 230, 1405, 6, 766, 638, 593, 399, 1926, 1101, 235, 455, 533, 2980, 83, 1361, 2802, 16, 311, 1410, 1869, 480, 480, 159, 173, 1649, 580, 700, 1146, 530, 268, 358, 930, 119] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1363, 1468, 93, 360, 830, 448, 1188, 948, 1176, 1165, 295, 641, 182, 437, 159, 351, 1617, 892, 1429, 170, 417, 980, 346, 1056, 204, 944, 1220, 97, 129, 102, 862, 1049, 411, 115, 1028, 774, 906, 218, 891, 1357, 685, 638, 854, 2713, 154, 1064, 3509, 78, 311, 465, 1014, 544, 357, 60, 547, 1665, 64, 626, 926, 1046, 520, 348, 2326, 320] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[520, 1015, 418, 900, 462, 210, 3138, 1082, 1260, 570, 439, 460, 1106, 417, 83, 27, 1139, 900, 663, 502, 202, 1453, 59, 1677, 22, 3467, 725, 81, 492, 320, 800, 202, 373, 87, 682, 265, 475, 197, 737, 2281, 501, 234, 655, 1673, 372, 1586, 1758, 61, 954, 199, 1683, 566, 85, 139, 640, 2852, 62, 160, 817, 692, 1380, 304, 1648, 223] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[279, 1423, 308, 689, 347, 220, 1402, 1399, 1725, 556, 347, 720, 423, 1662, 237, 567, 549, 1155, 378, 246, 125, 457, 367, 853, 74, 1865, 1321, 101, 386, 372, 263, 483, 85, 190, 1376, 1074, 1017, 55, 1105, 1397, 426, 178, 1046, 1395, 631, 1456, 923, 69, 1055, 1902, 769, 514, 1344, 55, 425, 2385, 83, 286, 2292, 813, 1310, 1589, 467, 141] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[471, 544, 859, 208, 560, 344, 2174, 1525, 1704, 307, 1656, 286, 781, 250, 33, 293, 1010, 1137, 926, 436, 556, 1214, 743, 1530, 184, 758, 850, 114, 209, 245, 1367, 8, 367, 331, 824, 776, 1814, 217, 2118, 1913, 742, 469, 298, 2143, 742, 1079, 1346, 251, 87, 320, 1618, 607, 181, 328, 54, 1473, 189, 705, 1535, 349, 509, 278, 1959, 248] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[605, 1765, 62, 801, 518, 141, 1573, 1956, 970, 445, 944, 420, 2944, 967, 119, 166, 1411, 219, 349, 276, 214, 938, 63, 519, 50, 908, 1195, 337, 231, 355, 3537, 75, 812, 66, 521, 574, 698, 73, 2249, 862, 1130, 205, 1401, 1415, 276, 196, 2255, 57, 199, 1077, 882, 442, 391, 21, 985, 1729, 86, 443, 2306, 316, 682, 506, 1167, 57] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[542, 2306, 258, 923, 476, 119, 1126, 1330, 1090, 637, 1502, 850, 1066, 856, 247, 1181, 2199, 821, 796, 470, 557, 1322, 42, 1794, 85, 1630, 642, 144, 211, 876, 620, 69, 60, 119, 288, 609, 1245, 52, 426, 255, 661, 199, 1101, 1843, 282, 1826, 1857, 56, 241, 558, 391, 310, 681, 395, 109, 2295, 294, 761, 1158, 1179, 1145, 862, 862, 245] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[364, 2338, 327, 28, 626, 171, 957, 790, 0, 0, 0, 0, 0, 0, 0, 0, 1451, 649, 93, 414, 1041, 583, 202, 52, 351, 106, 0, 0, 0, 0, 0, 0, 898, 1273, 1213, 94, 552, 748, 294, 1463, 0, 0, 0, 0, 0, 0, 0, 0, 106, 1484, 713, 317, 395, 1365, 1010, 230, 1383, 0, 0, 0, 0, 0, 0, 0, 1405, 6, 766, 638, 593, 399, 1926, 1101, 227, 0, 0, 0, 0, 0, 0, 0, 235, 455, 533, 2980, 83, 1361, 391, 16, 0, 0, 0, 0, 0, 0, 0, 0, 311, 1410, 1869, 480, 480, 159, 173, 1649, 117, 0, 0, 0, 0, 0, 0, 0, 580, 700, 1146, 530, 268, 358, 930, 119, 677, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5601, 4942, 6535, 7003, 7061, 6054, 6648, 5308] avg=6144.0000 max=7061 max_rank=4 min=4942 min_rank=1 imbalance_ratio=1.149251 max_load_reduction=42091 max_load_reduction_pct=0.856344 moved_to_echo=25071 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[520, 1015, 418, 900, 462, 210, 2133, 1082, 0, 0, 0, 0, 0, 0, 0, 0, 1260, 570, 439, 460, 1106, 417, 83, 27, 220, 236, 0, 0, 0, 0, 0, 0, 1139, 900, 663, 502, 202, 1453, 59, 1677, 1, 0, 0, 0, 0, 0, 0, 0, 22, 3467, 725, 81, 492, 320, 800, 202, 868, 0, 0, 0, 0, 0, 0, 0, 373, 87, 682, 265, 475, 197, 737, 2281, 506, 0, 0, 0, 0, 0, 0, 0, 501, 234, 655, 1673, 372, 1586, 245, 61, 0, 0, 0, 0, 0, 0, 0, 0, 954, 199, 1683, 566, 85, 139, 640, 2852, 262, 0, 0, 0, 0, 0, 0, 0, 62, 160, 817, 692, 1380, 304, 1648, 223, 425, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6740, 4818, 6596, 6977, 5603, 5327, 7380, 5711] avg=6144.0000 max=7380 max_rank=6 min=4818 min_rank=1 imbalance_ratio=1.201172 max_load_reduction=41772 max_load_reduction_pct=0.849854 moved_to_echo=24021 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[378, 1021, 290, 336, 528, 134, 1419, 1154, 0, 0, 0, 0, 0, 0, 0, 0, 2124, 263, 821, 780, 2125, 365, 267, 39, 288, 157, 0, 0, 0, 0, 0, 0, 1267, 1589, 260, 432, 278, 1194, 14, 402, 1, 0, 0, 0, 0, 0, 0, 0, 65, 1465, 665, 192, 403, 891, 1284, 13, 1137, 0, 0, 0, 0, 0, 0, 0, 297, 58, 125, 403, 3076, 60, 832, 827, 336, 0, 0, 0, 0, 0, 0, 0, 448, 223, 1166, 2542, 1780, 2061, 322, 99, 0, 0, 0, 0, 0, 0, 0, 0, 930, 338, 414, 507, 264, 27, 133, 2927, 174, 0, 0, 0, 0, 0, 0, 0, 390, 305, 826, 684, 790, 305, 542, 343, 557, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5260, 7229, 5437, 6115, 6014, 8641, 5714, 4742] avg=6144.0000 max=8641 max_rank=5 min=4742 min_rank=7 imbalance_ratio=1.406413 max_load_reduction=40511 max_load_reduction_pct=0.824198 moved_to_echo=25111 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[279, 1423, 308, 689, 347, 220, 954, 1399, 0, 0, 0, 0, 0, 0, 0, 0, 1725, 556, 347, 720, 423, 1662, 237, 567, 115, 105, 0, 0, 0, 0, 0, 0, 549, 1155, 378, 246, 125, 457, 367, 853, 0, 0, 0, 0, 0, 0, 0, 0, 74, 1865, 1321, 101, 386, 372, 263, 483, 455, 0, 0, 0, 0, 0, 0, 0, 85, 190, 1376, 1074, 1017, 55, 1105, 1397, 226, 0, 0, 0, 0, 0, 0, 0, 426, 178, 1046, 1395, 631, 1456, 130, 69, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1902, 769, 514, 1344, 55, 425, 2385, 117, 0, 0, 0, 0, 0, 0, 0, 83, 286, 2292, 813, 1310, 1589, 467, 141, 223, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5619, 6457, 4130, 5320, 6525, 5331, 8566, 7204] avg=6144.0000 max=8566 max_rank=6 min=4130 min_rank=2 imbalance_ratio=1.394206 max_load_reduction=40586 max_load_reduction_pct=0.825724 moved_to_echo=27626 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1363, 1468, 93, 360, 830, 448, 793, 948, 0, 0, 0, 0, 0, 0, 0, 0, 1176, 1165, 295, 641, 182, 437, 159, 351, 444, 94, 0, 0, 0, 0, 0, 0, 1617, 892, 1429, 170, 417, 980, 346, 1056, 4, 0, 0, 0, 0, 0, 0, 0, 204, 944, 1220, 97, 129, 102, 862, 1049, 1737, 0, 0, 0, 0, 0, 0, 0, 411, 115, 1028, 774, 906, 218, 891, 1357, 196, 0, 0, 0, 0, 0, 0, 0, 685, 638, 854, 2713, 154, 1064, 477, 78, 0, 0, 0, 0, 0, 0, 0, 0, 311, 465, 1014, 544, 357, 60, 547, 1665, 101, 0, 0, 0, 0, 0, 0, 0, 64, 626, 926, 1046, 520, 348, 2326, 320, 851, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6303, 4944, 6911, 6344, 5896, 6663, 5064, 7027] avg=6144.0000 max=7027 max_rank=7 min=4944 min_rank=1 imbalance_ratio=1.143717 max_load_reduction=42125 max_load_reduction_pct=0.857035 moved_to_echo=24650 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[471, 544, 859, 208, 560, 344, 1478, 1525, 0, 0, 0, 0, 0, 0, 0, 0, 1704, 307, 1656, 286, 781, 250, 33, 293, 168, 164, 0, 0, 0, 0, 0, 0, 1010, 1137, 926, 436, 556, 1214, 743, 1530, 1, 0, 0, 0, 0, 0, 0, 0, 184, 758, 850, 114, 209, 245, 1367, 8, 664, 0, 0, 0, 0, 0, 0, 0, 367, 331, 824, 776, 1814, 217, 2118, 1913, 350, 0, 0, 0, 0, 0, 0, 0, 742, 469, 298, 2143, 742, 1079, 189, 251, 0, 0, 0, 0, 0, 0, 0, 0, 87, 320, 1618, 607, 181, 328, 54, 1473, 181, 0, 0, 0, 0, 0, 0, 0, 189, 705, 1535, 349, 509, 278, 1959, 248, 325, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5989, 5642, 7553, 4399, 8710, 5913, 4849, 6097] avg=6144.0000 max=8710 max_rank=4 min=4399 min_rank=3 imbalance_ratio=1.417643 max_load_reduction=40442 max_load_reduction_pct=0.822795 moved_to_echo=25569 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[542, 2306, 258, 923, 476, 119, 767, 1330, 0, 0, 0, 0, 0, 0, 0, 0, 1090, 637, 1502, 850, 1066, 856, 247, 1181, 232, 84, 0, 0, 0, 0, 0, 0, 2199, 821, 796, 470, 557, 1322, 42, 1794, 0, 0, 0, 0, 0, 0, 0, 0, 85, 1630, 642, 144, 211, 876, 620, 69, 916, 0, 0, 0, 0, 0, 0, 0, 60, 119, 288, 609, 1245, 52, 426, 255, 181, 0, 0, 0, 0, 0, 0, 0, 661, 199, 1101, 1843, 282, 1826, 260, 56, 0, 0, 0, 0, 0, 0, 0, 0, 241, 558, 391, 310, 681, 395, 109, 2295, 94, 0, 0, 0, 0, 0, 0, 0, 294, 761, 1158, 1179, 1145, 862, 862, 245, 449, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6721, 7745, 8001, 5193, 3235, 6228, 5074, 6955] avg=6144.0000 max=8001 max_rank=2 min=3235 min_rank=4 imbalance_ratio=1.302246 max_load_reduction=41151 max_load_reduction_pct=0.837219 moved_to_echo=21492 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[605, 1765, 62, 801, 518, 141, 1071, 1956, 0, 0, 0, 0, 0, 0, 0, 0, 970, 445, 944, 420, 2944, 967, 119, 166, 282, 118, 0, 0, 0, 0, 0, 0, 1411, 219, 349, 276, 214, 938, 63, 519, 0, 0, 0, 0, 0, 0, 0, 0, 50, 908, 1195, 337, 231, 355, 3537, 75, 1113, 0, 0, 0, 0, 0, 0, 0, 812, 66, 521, 574, 698, 73, 2249, 862, 253, 0, 0, 0, 0, 0, 0, 0, 1130, 205, 1401, 1415, 276, 196, 315, 57, 0, 0, 0, 0, 0, 0, 0, 0, 199, 1077, 882, 442, 391, 21, 985, 1729, 131, 0, 0, 0, 0, 0, 0, 0, 86, 443, 2306, 316, 682, 506, 1167, 57, 545, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6919, 7375, 3989, 7801, 6108, 4995, 5857, 6108] avg=6144.0000 max=7801 max_rank=3 min=3989 min_rank=2 imbalance_ratio=1.269694 max_load_reduction=41351 max_load_reduction_pct=0.841288 moved_to_echo=23068 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4096, 6272, 8704, 4096, 3840, 1280, 3072, 17024, 1280, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4608, 11904, 2688, 4352, 4352, 1792, 9600, 10240, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[3840, 1024, 5632, 5120, 9856, 1280, 10368, 10112, 2304, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[896, 12544, 7424, 1408, 2560, 4608, 9856, 2176, 8320, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[11520, 4608, 6144, 4608, 9728, 5632, 1408, 2688, 2176, 1152, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[10112, 8064, 6016, 2688, 2944, 8320, 2048, 9344, 128, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1792, 4096, 11008, 5632, 6656, 4608, 9984, 1792, 4096, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4864, 2688, 7168, 16768, 4352, 10752, 2432, 768, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=17 tokens_per_expert_current_ep_rank=[4, 214, 170, 291, 1867, 523, 545, 378, 365, 2823, 180, 712, 346, 1995, 53, 1112, 801, 184, 1201, 259, 393, 165, 931, 1439, 471, 535, 162, 1153, 184, 133, 2267, 709, 765, 241, 1018, 1348, 117, 624, 686, 41, 748, 2202, 392, 263, 2206, 2327, 1074, 317, 598, 858, 188, 2083, 591, 669, 321, 790, 522, 430, 788, 207, 1170, 526, 2279, 198] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=17 tokens_per_expert_current_ep_rank=[19, 246, 43, 103, 3878, 1018, 338, 391, 891, 564, 294, 807, 179, 1726, 589, 1243, 1674, 1377, 439, 408, 852, 126, 398, 67, 1515, 358, 555, 361, 92, 606, 1677, 710, 870, 696, 1081, 1197, 440, 1254, 1765, 10, 227, 1637, 211, 256, 2438, 1045, 342, 1049, 639, 207, 146, 850, 1523, 190, 68, 616, 110, 102, 201, 543, 2125, 312, 2263, 1195] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=17 tokens_per_expert_current_ep_rank=[18, 221, 118, 501, 320, 87, 410, 343, 622, 1561, 531, 255, 366, 1724, 1518, 2373, 1648, 261, 417, 24, 375, 394, 400, 762, 1853, 224, 1392, 755, 601, 633, 2513, 253, 1660, 57, 1573, 1758, 523, 742, 176, 89, 1010, 905, 260, 166, 3850, 458, 807, 764, 269, 925, 831, 551, 555, 946, 654, 445, 447, 524, 310, 516, 1965, 460, 1208, 275] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=17 tokens_per_expert_current_ep_rank=[19, 66, 120, 448, 1456, 617, 756, 1074, 654, 1280, 785, 1851, 329, 2904, 814, 1387, 2601, 1141, 422, 68, 370, 518, 479, 272, 1517, 261, 736, 502, 812, 512, 1777, 120, 671, 111, 153, 497, 228, 628, 1089, 127, 117, 1030, 159, 1537, 2368, 983, 1434, 313, 822, 575, 55, 2560, 125, 155, 339, 951, 92, 422, 433, 1706, 432, 708, 1222, 442] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=17 tokens_per_expert_current_ep_rank=[45, 136, 251, 290, 872, 909, 46, 783, 423, 2633, 520, 416, 1068, 1700, 570, 1003, 1583, 273, 727, 470, 613, 427, 793, 2115, 1782, 935, 1403, 370, 272, 686, 1330, 137, 522, 289, 1072, 801, 175, 818, 682, 105, 418, 1185, 429, 332, 2282, 1898, 919, 1304, 134, 641, 425, 1256, 149, 184, 536, 475, 1374, 248, 397, 440, 1364, 1134, 1072, 511] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=17 tokens_per_expert_current_ep_rank=[28, 167, 125, 319, 1390, 417, 738, 903, 1463, 168, 648, 1609, 279, 270, 57, 1307, 2206, 669, 342, 117, 735, 349, 1372, 1789, 906, 588, 408, 196, 51, 333, 2055, 161, 1262, 341, 1162, 1376, 118, 2351, 723, 10, 253, 2617, 83, 160, 2604, 1792, 1485, 391, 202, 95, 154, 2901, 246, 96, 226, 1685, 88, 821, 31, 1120, 1347, 265, 833, 149] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=17 tokens_per_expert_current_ep_rank=[196, 707, 534, 505, 899, 1721, 655, 1316, 756, 1058, 612, 701, 222, 1967, 289, 738, 2151, 567, 190, 535, 167, 1022, 869, 898, 1594, 235, 356, 812, 78, 651, 1810, 199, 200, 198, 2302, 285, 535, 232, 434, 213, 314, 2147, 1170, 363, 1239, 2377, 2547, 368, 69, 454, 145, 537, 39, 586, 452, 454, 257, 851, 164, 1264, 1933, 1186, 610, 217] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=17 tokens_per_expert_current_ep_rank=[3, 290, 88, 249, 1523, 1045, 1404, 614, 487, 1580, 909, 2133, 146, 1322, 92, 613, 1456, 1151, 420, 217, 250, 168, 1237, 418, 752, 1146, 1305, 528, 178, 140, 1729, 91, 743, 525, 703, 1766, 650, 815, 803, 31, 233, 1561, 368, 977, 1582, 2156, 1526, 911, 625, 128, 186, 2012, 288, 210, 135, 708, 132, 335, 397, 1187, 1880, 656, 578, 661] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=17 rerouted_tokens_per_expert_current_ep_rank=[19, 246, 43, 103, 3878, 1018, 338, 391, 1756, 0, 0, 0, 0, 0, 0, 0, 891, 564, 294, 807, 179, 277, 589, 1243, 0, 0, 0, 0, 0, 0, 0, 0, 1674, 1377, 439, 408, 852, 126, 398, 67, 204, 41, 0, 0, 0, 0, 0, 0, 1515, 358, 555, 361, 92, 606, 1676, 710, 0, 0, 0, 0, 0, 0, 0, 0, 870, 696, 1081, 1197, 440, 1254, 1765, 10, 592, 0, 0, 0, 0, 0, 0, 0, 227, 1248, 211, 256, 0, 1045, 342, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 639, 207, 146, 850, 1523, 190, 68, 616, 816, 682, 0, 0, 0, 0, 0, 0, 110, 102, 201, 543, 2125, 312, 2263, 1195, 185, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7792, 4844, 5586, 5873, 7905, 4378, 5737, 7037] avg=6144.0000 max=7905 max_rank=4 min=4378 min_rank=5 imbalance_ratio=1.286621 max_load_reduction=41247 max_load_reduction_pct=0.839172 moved_to_echo=25057 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=17 rerouted_tokens_per_expert_current_ep_rank=[4, 214, 170, 291, 1867, 523, 545, 378, 1589, 0, 0, 0, 0, 0, 0, 0, 365, 2823, 180, 712, 346, 319, 53, 1112, 0, 0, 0, 0, 0, 0, 0, 0, 801, 184, 1201, 259, 393, 165, 931, 1439, 275, 48, 0, 0, 0, 0, 0, 0, 471, 535, 162, 1153, 184, 133, 2265, 709, 0, 0, 0, 0, 0, 0, 0, 0, 765, 241, 1018, 1348, 117, 624, 686, 41, 685, 0, 0, 0, 0, 0, 0, 0, 748, 1678, 392, 263, 0, 2327, 1074, 317, 0, 0, 0, 0, 0, 0, 0, 0, 598, 858, 188, 2083, 591, 669, 321, 790, 943, 617, 0, 0, 0, 0, 0, 0, 522, 430, 788, 207, 1170, 526, 2279, 198, 249, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5581, 5910, 5696, 5612, 5525, 6799, 7658, 6371] avg=6144.0000 max=7658 max_rank=6 min=5525 min_rank=4 imbalance_ratio=1.246419 max_load_reduction=41494 max_load_reduction_pct=0.844198 moved_to_echo=26353 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=17 rerouted_tokens_per_expert_current_ep_rank=[19, 66, 120, 448, 1456, 617, 756, 1074, 1706, 0, 0, 0, 0, 0, 0, 0, 654, 1280, 785, 1851, 329, 465, 814, 1387, 0, 0, 0, 0, 0, 0, 0, 0, 2601, 1141, 422, 68, 370, 518, 479, 272, 128, 69, 0, 0, 0, 0, 0, 0, 1517, 261, 736, 502, 812, 512, 1776, 120, 0, 0, 0, 0, 0, 0, 0, 0, 671, 111, 153, 497, 228, 628, 1089, 127, 997, 0, 0, 0, 0, 0, 0, 0, 117, 786, 159, 1537, 0, 983, 1434, 313, 0, 0, 0, 0, 0, 0, 0, 0, 822, 575, 55, 2560, 125, 155, 339, 951, 1373, 662, 0, 0, 0, 0, 0, 0, 92, 422, 433, 1706, 432, 708, 1222, 442, 116, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6262, 7565, 6068, 6236, 4501, 5329, 7617, 5574] avg=6144.0000 max=7617 max_rank=6 min=4501 min_rank=4 imbalance_ratio=1.239746 max_load_reduction=41535 max_load_reduction_pct=0.845032 moved_to_echo=23021 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=17 rerouted_tokens_per_expert_current_ep_rank=[18, 221, 118, 501, 320, 87, 410, 343, 2775, 0, 0, 0, 0, 0, 0, 0, 622, 1561, 531, 255, 366, 276, 1518, 2373, 0, 0, 0, 0, 0, 0, 0, 0, 1648, 261, 417, 24, 375, 394, 400, 762, 113, 41, 0, 0, 0, 0, 0, 0, 1853, 224, 1392, 755, 601, 633, 2511, 253, 0, 0, 0, 0, 0, 0, 0, 0, 1660, 57, 1573, 1758, 523, 742, 176, 89, 592, 0, 0, 0, 0, 0, 0, 0, 1010, 690, 260, 166, 0, 458, 807, 764, 0, 0, 0, 0, 0, 0, 0, 0, 269, 925, 831, 551, 555, 946, 654, 445, 815, 1075, 0, 0, 0, 0, 0, 0, 447, 524, 310, 516, 1965, 460, 1208, 275, 102, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4793, 7502, 4435, 8222, 7170, 4155, 7066, 5809] avg=6144.0000 max=8222 max_rank=3 min=4155 min_rank=5 imbalance_ratio=1.338216 max_load_reduction=40930 max_load_reduction_pct=0.832723 moved_to_echo=24200 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=17 rerouted_tokens_per_expert_current_ep_rank=[45, 136, 251, 290, 872, 909, 46, 783, 1645, 0, 0, 0, 0, 0, 0, 0, 423, 2633, 520, 416, 1068, 259, 570, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1583, 273, 727, 470, 613, 427, 793, 2115, 152, 45, 0, 0, 0, 0, 0, 0, 1782, 935, 1403, 370, 272, 686, 1325, 137, 0, 0, 0, 0, 0, 0, 0, 0, 522, 289, 1072, 801, 175, 818, 682, 105, 588, 0, 0, 0, 0, 0, 0, 0, 418, 896, 429, 332, 0, 1898, 919, 1304, 0, 0, 0, 0, 0, 0, 0, 0, 134, 641, 425, 1256, 149, 184, 536, 475, 808, 637, 0, 0, 0, 0, 0, 0, 1374, 248, 397, 440, 1364, 1134, 1072, 511, 137, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4977, 6892, 7198, 6910, 5052, 6196, 5245, 6682] avg=6144.0000 max=7198 max_rank=2 min=4977 min_rank=0 imbalance_ratio=1.171549 max_load_reduction=41954 max_load_reduction_pct=0.853556 moved_to_echo=23175 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=17 rerouted_tokens_per_expert_current_ep_rank=[3, 290, 88, 249, 1523, 1045, 1404, 614, 1141, 0, 0, 0, 0, 0, 0, 0, 487, 1580, 909, 2133, 146, 212, 92, 613, 0, 0, 0, 0, 0, 0, 0, 0, 1456, 1151, 420, 217, 250, 168, 1237, 418, 195, 31, 0, 0, 0, 0, 0, 0, 752, 1146, 1305, 528, 178, 140, 1728, 91, 0, 0, 0, 0, 0, 0, 0, 0, 743, 525, 703, 1766, 650, 815, 803, 31, 454, 0, 0, 0, 0, 0, 0, 0, 233, 1190, 368, 977, 0, 2156, 1526, 911, 0, 0, 0, 0, 0, 0, 0, 0, 625, 128, 186, 2012, 288, 210, 135, 708, 625, 441, 0, 0, 0, 0, 0, 0, 132, 335, 397, 1187, 1880, 656, 578, 661, 176, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6357, 6172, 5543, 5868, 6490, 7361, 5358, 6003] avg=6144.0000 max=7361 max_rank=5 min=5358 min_rank=6 imbalance_ratio=1.198079 max_load_reduction=41791 max_load_reduction_pct=0.850240 moved_to_echo=25212 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=17 rerouted_tokens_per_expert_current_ep_rank=[196, 707, 534, 505, 899, 1721, 655, 1316, 893, 0, 0, 0, 0, 0, 0, 0, 756, 1058, 612, 701, 222, 315, 289, 738, 0, 0, 0, 0, 0, 0, 0, 0, 2151, 567, 190, 535, 167, 1022, 869, 898, 268, 47, 0, 0, 0, 0, 0, 0, 1594, 235, 356, 812, 78, 651, 1809, 199, 0, 0, 0, 0, 0, 0, 0, 0, 200, 198, 2302, 285, 535, 232, 434, 213, 675, 0, 0, 0, 0, 0, 0, 0, 314, 1637, 1170, 363, 0, 2377, 2547, 368, 0, 0, 0, 0, 0, 0, 0, 0, 69, 454, 145, 537, 39, 586, 452, 454, 930, 346, 0, 0, 0, 0, 0, 0, 257, 851, 164, 1264, 1933, 1186, 610, 217, 242, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7426, 4691, 6714, 5734, 5074, 8776, 4012, 6725] avg=6144.0000 max=8776 max_rank=5 min=4012 min_rank=6 imbalance_ratio=1.428385 max_load_reduction=40376 max_load_reduction_pct=0.821452 moved_to_echo=24587 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=17 rerouted_tokens_per_expert_current_ep_rank=[28, 167, 125, 319, 1390, 417, 738, 903, 1877, 0, 0, 0, 0, 0, 0, 0, 1463, 168, 648, 1609, 279, 45, 57, 1307, 0, 0, 0, 0, 0, 0, 0, 0, 2206, 669, 342, 117, 735, 349, 1372, 1789, 327, 6, 0, 0, 0, 0, 0, 0, 906, 588, 408, 196, 51, 333, 2054, 161, 0, 0, 0, 0, 0, 0, 0, 0, 1262, 341, 1162, 1376, 118, 2351, 723, 10, 92, 0, 0, 0, 0, 0, 0, 0, 253, 1994, 83, 160, 0, 1792, 1485, 391, 0, 0, 0, 0, 0, 0, 0, 0, 202, 95, 154, 2901, 246, 96, 226, 1685, 127, 727, 0, 0, 0, 0, 0, 0, 88, 821, 31, 1120, 1347, 265, 833, 149, 296, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5964, 5576, 7912, 4697, 7435, 6158, 6459, 4951] avg=6144.0000 max=7912 max_rank=2 min=4697 min_rank=3 imbalance_ratio=1.287760 max_load_reduction=41240 max_load_reduction_pct=0.839030 moved_to_echo=25003 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5760, 11776, 4480, 8576, 2944, 2176, 4096, 9856, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[10496, 4352, 6400, 4736, 2304, 3712, 15232, 2432, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3072, 3840, 2816, 7040, 12288, 5248, 10112, 3712, 1536, 128, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6784, 2560, 9088, 9088, 2816, 7552, 6400, 640, 4736, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3456, 3968, 2176, 12800, 3584, 3072, 2816, 6144, 6528, 5248, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[3328, 10240, 3072, 4096, 0, 13056, 10240, 5504, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[384, 2048, 1536, 2816, 12288, 6400, 4992, 5888, 13440, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[14208, 5632, 4224, 2176, 3840, 3200, 6528, 7808, 1664, 384, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=18 tokens_per_expert_current_ep_rank=[619, 631, 3329, 816, 273, 1020, 922, 1363, 739, 2213, 387, 71, 687, 192, 689, 778, 226, 274, 543, 144, 2239, 1991, 2019, 300, 329, 49, 356, 2712, 673, 642, 585, 1160, 1662, 151, 3, 5, 231, 1004, 49, 3171, 179, 258, 442, 35, 1744, 103, 678, 239, 283, 442, 164, 221, 338, 5, 1891, 730, 29, 441, 785, 3316, 224, 429, 418, 511] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=18 tokens_per_expert_current_ep_rank=[1292, 1398, 4306, 1064, 15, 1545, 1516, 1097, 44, 2354, 869, 7, 241, 776, 237, 516, 73, 1246, 879, 434, 747, 639, 2149, 980, 157, 5, 726, 1871, 1226, 189, 137, 744, 1013, 443, 44, 31, 24, 344, 16, 2852, 35, 96, 715, 306, 663, 56, 2269, 102, 404, 1732, 30, 1606, 140, 34, 2163, 654, 49, 42, 810, 1635, 137, 593, 208, 427] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=18 tokens_per_expert_current_ep_rank=[774, 399, 1942, 2779, 763, 150, 804, 2587, 109, 1850, 183, 164, 1799, 190, 1568, 1474, 35, 391, 1714, 260, 678, 902, 318, 245, 627, 38, 220, 1539, 343, 751, 390, 523, 2281, 264, 160, 156, 577, 867, 9, 2060, 542, 625, 284, 66, 1036, 218, 755, 547, 356, 1070, 87, 537, 666, 13, 2014, 1016, 101, 358, 104, 1966, 62, 94, 1650, 2102] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=18 tokens_per_expert_current_ep_rank=[463, 470, 2682, 911, 224, 1392, 2190, 861, 89, 1843, 1114, 115, 318, 841, 856, 1321, 331, 308, 359, 267, 1330, 2640, 1607, 240, 155, 53, 55, 1503, 323, 390, 108, 512, 1814, 1739, 16, 932, 166, 988, 97, 1697, 191, 404, 669, 9, 1203, 668, 811, 649, 273, 858, 95, 1435, 997, 18, 1427, 1480, 58, 93, 292, 3218, 83, 310, 84, 507] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=18 tokens_per_expert_current_ep_rank=[788, 2534, 4631, 1148, 847, 338, 763, 447, 365, 1263, 680, 496, 423, 279, 119, 538, 21, 646, 674, 972, 1402, 1107, 933, 405, 249, 39, 69, 3161, 325, 233, 538, 880, 1123, 2541, 50, 149, 52, 1084, 57, 1625, 486, 439, 642, 12, 951, 149, 1649, 189, 205, 1213, 42, 1599, 89, 32, 1744, 606, 125, 403, 146, 2652, 613, 934, 171, 67] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=18 tokens_per_expert_current_ep_rank=[305, 2006, 1080, 838, 902, 635, 1370, 2339, 612, 2109, 788, 54, 333, 341, 710, 1481, 1021, 271, 1411, 146, 478, 685, 577, 238, 449, 169, 234, 962, 370, 1162, 313, 179, 978, 2311, 796, 443, 373, 433, 690, 1198, 1105, 272, 194, 73, 749, 32, 1583, 604, 1992, 774, 510, 188, 580, 9, 810, 752, 472, 1106, 230, 1897, 894, 2117, 122, 297] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=18 tokens_per_expert_current_ep_rank=[1190, 527, 3546, 1032, 292, 546, 2702, 1249, 186, 2121, 751, 194, 262, 516, 273, 184, 33, 518, 775, 48, 593, 1192, 241, 776, 450, 79, 967, 1216, 883, 286, 296, 279, 679, 1281, 63, 315, 851, 995, 212, 4183, 549, 766, 1685, 164, 904, 215, 578, 200, 30, 813, 176, 1073, 807, 474, 1679, 547, 32, 192, 987, 1609, 983, 757, 1007, 143] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=18 tokens_per_expert_current_ep_rank=[897, 836, 1547, 1178, 1184, 280, 2509, 963, 206, 2031, 1307, 45, 276, 228, 1084, 808, 216, 485, 2224, 15, 2224, 1122, 1387, 145, 322, 293, 177, 642, 306, 437, 120, 332, 911, 200, 387, 604, 516, 557, 103, 887, 90, 176, 353, 13, 1382, 46, 1413, 672, 564, 1184, 233, 497, 500, 23, 2158, 561, 1075, 1727, 110, 3601, 896, 621, 1127, 139] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=18 rerouted_tokens_per_expert_current_ep_rank=[619, 631, 0, 816, 273, 1020, 214, 1363, 0, 0, 0, 0, 0, 0, 0, 0, 739, 2213, 387, 71, 687, 192, 689, 778, 347, 109, 0, 0, 0, 0, 0, 0, 226, 274, 543, 144, 2239, 1926, 2019, 300, 0, 0, 0, 0, 0, 0, 0, 0, 329, 49, 356, 2712, 673, 642, 585, 1160, 1276, 346, 0, 0, 0, 0, 0, 0, 1662, 151, 3, 5, 231, 1004, 49, 2748, 0, 0, 0, 0, 0, 0, 0, 0, 179, 258, 442, 35, 1744, 103, 678, 239, 2053, 0, 0, 0, 0, 0, 0, 0, 283, 442, 164, 221, 338, 5, 1891, 730, 253, 0, 0, 0, 0, 0, 0, 0, 29, 441, 785, 3316, 224, 429, 418, 511, 76, 65, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4936, 6212, 7671, 8128, 5853, 5731, 4327, 6294] avg=6144.0000 max=8128 max_rank=3 min=4327 min_rank=6 imbalance_ratio=1.322917 max_load_reduction=41024 max_load_reduction_pct=0.834635 moved_to_echo=22205 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=18 rerouted_tokens_per_expert_current_ep_rank=[1292, 1398, 0, 1064, 15, 1545, 351, 1097, 0, 0, 0, 0, 0, 0, 0, 0, 44, 2354, 869, 7, 241, 776, 237, 516, 312, 180, 0, 0, 0, 0, 0, 0, 73, 1246, 879, 434, 747, 619, 2149, 980, 0, 0, 0, 0, 0, 0, 0, 0, 157, 5, 726, 1871, 1226, 189, 137, 744, 1651, 569, 0, 0, 0, 0, 0, 0, 1013, 443, 44, 31, 24, 344, 16, 2472, 0, 0, 0, 0, 0, 0, 0, 0, 35, 96, 715, 306, 663, 56, 2269, 102, 2655, 0, 0, 0, 0, 0, 0, 0, 404, 1732, 30, 1606, 140, 34, 2163, 654, 416, 0, 0, 0, 0, 0, 0, 0, 49, 42, 810, 1635, 137, 593, 208, 427, 68, 20, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6762, 5536, 7127, 7275, 4387, 6897, 7179, 3989] avg=6144.0000 max=7275 max_rank=3 min=3989 min_rank=7 imbalance_ratio=1.184082 max_load_reduction=41877 max_load_reduction_pct=0.851990 moved_to_echo=22452 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=18 rerouted_tokens_per_expert_current_ep_rank=[897, 836, 0, 1178, 1184, 280, 580, 963, 0, 0, 0, 0, 0, 0, 0, 0, 206, 2031, 1307, 45, 276, 228, 1084, 808, 97, 298, 0, 0, 0, 0, 0, 0, 216, 485, 2224, 15, 2224, 1086, 1387, 145, 0, 0, 0, 0, 0, 0, 0, 0, 322, 293, 177, 642, 306, 437, 120, 332, 593, 942, 0, 0, 0, 0, 0, 0, 911, 200, 387, 604, 516, 557, 103, 769, 0, 0, 0, 0, 0, 0, 0, 0, 90, 176, 353, 13, 1382, 46, 1413, 672, 954, 0, 0, 0, 0, 0, 0, 0, 564, 1184, 233, 497, 500, 23, 2158, 561, 689, 0, 0, 0, 0, 0, 0, 0, 1075, 1727, 110, 3601, 896, 621, 1127, 139, 21, 36, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5918, 6380, 7782, 4164, 4047, 5099, 6409, 9353] avg=6144.0000 max=9353 max_rank=7 min=4047 min_rank=4 imbalance_ratio=1.522298 max_load_reduction=39799 max_load_reduction_pct=0.809713 moved_to_echo=24908 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=18 rerouted_tokens_per_expert_current_ep_rank=[463, 470, 0, 911, 224, 1392, 507, 861, 0, 0, 0, 0, 0, 0, 0, 0, 89, 1843, 1114, 115, 318, 841, 856, 1321, 185, 260, 0, 0, 0, 0, 0, 0, 331, 308, 359, 267, 1330, 2554, 1607, 240, 0, 0, 0, 0, 0, 0, 0, 0, 155, 53, 55, 1503, 323, 390, 108, 512, 1028, 822, 0, 0, 0, 0, 0, 0, 1814, 1739, 16, 932, 166, 988, 97, 1471, 0, 0, 0, 0, 0, 0, 0, 0, 191, 404, 669, 9, 1203, 668, 811, 649, 1654, 0, 0, 0, 0, 0, 0, 0, 273, 858, 95, 1435, 997, 18, 1427, 1480, 601, 0, 0, 0, 0, 0, 0, 0, 58, 93, 292, 3218, 83, 310, 84, 507, 41, 86, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4828, 6942, 6996, 4949, 7223, 6258, 7184, 4772] avg=6144.0000 max=7223 max_rank=4 min=4772 min_rank=7 imbalance_ratio=1.175618 max_load_reduction=41929 max_load_reduction_pct=0.853048 moved_to_echo=25437 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=18 rerouted_tokens_per_expert_current_ep_rank=[774, 399, 0, 2779, 763, 150, 187, 2587, 0, 0, 0, 0, 0, 0, 0, 0, 109, 1850, 183, 164, 1799, 190, 1568, 1474, 225, 95, 0, 0, 0, 0, 0, 0, 35, 391, 1714, 260, 678, 873, 318, 245, 0, 0, 0, 0, 0, 0, 0, 0, 627, 38, 220, 1539, 343, 751, 390, 523, 745, 302, 0, 0, 0, 0, 0, 0, 2281, 264, 160, 156, 577, 867, 9, 1786, 0, 0, 0, 0, 0, 0, 0, 0, 542, 625, 284, 66, 1036, 218, 755, 547, 1197, 0, 0, 0, 0, 0, 0, 0, 356, 1070, 87, 537, 666, 13, 2014, 1016, 220, 0, 0, 0, 0, 0, 0, 0, 101, 358, 104, 1966, 62, 94, 1650, 2102, 49, 29, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7639, 7657, 4514, 5478, 6100, 5270, 5979, 6515] avg=6144.0000 max=7657 max_rank=1 min=4514 min_rank=2 imbalance_ratio=1.246257 max_load_reduction=41495 max_load_reduction_pct=0.844218 moved_to_echo=23864 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=18 rerouted_tokens_per_expert_current_ep_rank=[788, 2534, 0, 1148, 847, 338, 178, 447, 0, 0, 0, 0, 0, 0, 0, 0, 365, 1263, 680, 496, 423, 279, 119, 538, 177, 90, 0, 0, 0, 0, 0, 0, 21, 646, 674, 972, 1402, 1071, 933, 405, 0, 0, 0, 0, 0, 0, 0, 0, 249, 39, 69, 3161, 325, 233, 538, 880, 1776, 286, 0, 0, 0, 0, 0, 0, 1123, 2541, 50, 149, 52, 1084, 57, 1409, 0, 0, 0, 0, 0, 0, 0, 0, 486, 439, 642, 12, 951, 149, 1649, 189, 2855, 0, 0, 0, 0, 0, 0, 0, 205, 1213, 42, 1599, 89, 32, 1744, 606, 209, 0, 0, 0, 0, 0, 0, 0, 125, 403, 146, 2652, 613, 934, 171, 67, 39, 36, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6280, 4430, 6124, 7556, 6465, 7372, 5739, 5186] avg=6144.0000 max=7556 max_rank=3 min=4430 min_rank=1 imbalance_ratio=1.229818 max_load_reduction=41596 max_load_reduction_pct=0.846273 moved_to_echo=24762 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=18 rerouted_tokens_per_expert_current_ep_rank=[305, 2006, 0, 838, 902, 635, 304, 2339, 0, 0, 0, 0, 0, 0, 0, 0, 612, 2109, 788, 54, 333, 341, 710, 1481, 134, 168, 0, 0, 0, 0, 0, 0, 1021, 271, 1411, 146, 478, 659, 577, 238, 0, 0, 0, 0, 0, 0, 0, 0, 449, 169, 234, 962, 370, 1162, 313, 179, 415, 519, 0, 0, 0, 0, 0, 0, 978, 2311, 796, 443, 373, 433, 690, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1105, 272, 194, 73, 749, 32, 1583, 604, 665, 0, 0, 0, 0, 0, 0, 0, 1992, 774, 510, 188, 580, 9, 810, 752, 379, 0, 0, 0, 0, 0, 0, 0, 472, 1106, 230, 1897, 894, 2117, 122, 297, 32, 26, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7329, 6730, 4801, 4772, 7056, 5277, 5994, 7193] avg=6144.0000 max=7329 max_rank=0 min=4772 min_rank=3 imbalance_ratio=1.192871 max_load_reduction=41823 max_load_reduction_pct=0.850891 moved_to_echo=25520 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=18 rerouted_tokens_per_expert_current_ep_rank=[1190, 527, 0, 1032, 292, 546, 624, 1249, 0, 0, 0, 0, 0, 0, 0, 0, 186, 2121, 751, 194, 262, 516, 273, 184, 457, 321, 0, 0, 0, 0, 0, 0, 33, 518, 775, 48, 593, 1153, 241, 776, 0, 0, 0, 0, 0, 0, 0, 0, 450, 79, 967, 1216, 883, 286, 296, 279, 1359, 1015, 0, 0, 0, 0, 0, 0, 679, 1281, 63, 315, 851, 995, 212, 3625, 0, 0, 0, 0, 0, 0, 0, 0, 549, 766, 1685, 164, 904, 215, 578, 200, 2187, 0, 0, 0, 0, 0, 0, 0, 30, 813, 176, 1073, 807, 474, 1679, 547, 742, 0, 0, 0, 0, 0, 0, 0, 32, 192, 987, 1609, 983, 757, 1007, 143, 101, 39, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5460, 5265, 4137, 6830, 8021, 7248, 6341, 5850] avg=6144.0000 max=8021 max_rank=4 min=4137 min_rank=2 imbalance_ratio=1.305501 max_load_reduction=41131 max_load_reduction_pct=0.836812 moved_to_echo=27460 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2816, 768, 2816, 13696, 4480, 4096, 2560, 4736, 8960, 4864, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4224, 8192, 1408, 7168, 4224, 640, 13952, 6400, 3584, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3200, 3072, 4992, 768, 8704, 1536, 9856, 3328, 14336, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[2048, 4480, 3584, 19968, 3968, 5888, 4864, 4224, 512, 384, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6400, 8832, 0, 9856, 4608, 6016, 3072, 11008, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2432, 15872, 6144, 1152, 4352, 3456, 5632, 7168, 2048, 1536, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[10496, 8960, 1536, 2688, 2816, 6272, 1280, 15360, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2048, 4224, 8704, 2304, 9728, 9984, 9344, 3456, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=19 tokens_per_expert_current_ep_rank=[428, 610, 132, 73, 1049, 82, 195, 2150, 534, 1369, 875, 252, 508, 1735, 1558, 715, 110, 907, 259, 647, 1625, 902, 964, 1559, 689, 219, 1924, 393, 481, 57, 137, 615, 192, 480, 482, 1226, 710, 1263, 3434, 1128, 480, 284, 536, 678, 2396, 418, 41, 1549, 916, 27, 780, 800, 473, 115, 147, 1196, 2838, 588, 1151, 111, 87, 291, 304, 278] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=19 tokens_per_expert_current_ep_rank=[864, 1466, 189, 950, 41, 38, 1822, 2309, 418, 1590, 1019, 174, 860, 1303, 1298, 175, 80, 2815, 72, 105, 1923, 854, 740, 2223, 2419, 525, 608, 229, 951, 288, 165, 273, 759, 346, 457, 1212, 847, 230, 2343, 401, 164, 611, 192, 272, 2644, 894, 687, 2224, 230, 54, 635, 313, 123, 158, 8, 773, 1435, 753, 195, 298, 12, 46, 47, 1003] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=19 tokens_per_expert_current_ep_rank=[1104, 3380, 124, 351, 239, 140, 194, 714, 742, 1021, 662, 600, 2061, 2759, 2504, 3718, 30, 1615, 373, 135, 1188, 90, 325, 1475, 1120, 595, 1238, 488, 444, 735, 320, 170, 464, 386, 1354, 929, 482, 444, 1835, 1860, 360, 496, 477, 28, 2183, 492, 159, 112, 266, 164, 482, 293, 94, 221, 239, 140, 875, 463, 317, 700, 117, 356, 419, 1291] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=19 tokens_per_expert_current_ep_rank=[449, 803, 72, 266, 43, 23, 878, 870, 179, 2839, 1939, 1631, 1304, 2403, 989, 1019, 160, 3960, 71, 607, 1073, 175, 321, 472, 719, 1154, 529, 157, 227, 355, 220, 313, 977, 551, 821, 784, 181, 705, 3642, 1296, 378, 37, 331, 265, 934, 976, 595, 1318, 1564, 39, 2054, 1733, 67, 153, 17, 1546, 342, 678, 564, 130, 12, 24, 208, 10] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=19 tokens_per_expert_current_ep_rank=[498, 2405, 168, 213, 527, 197, 1980, 1841, 366, 1443, 430, 1068, 1154, 2587, 1551, 1553, 53, 974, 157, 524, 943, 221, 610, 289, 454, 650, 3069, 200, 846, 68, 2263, 272, 1279, 9, 343, 1365, 138, 1143, 499, 279, 38, 463, 401, 197, 709, 320, 1064, 270, 1702, 494, 1141, 494, 149, 106, 73, 1724, 1253, 1431, 1306, 196, 13, 360, 317, 302] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=19 tokens_per_expert_current_ep_rank=[428, 873, 118, 723, 435, 172, 2110, 1749, 576, 903, 2075, 1257, 437, 2702, 3189, 329, 396, 1280, 143, 96, 1785, 227, 707, 2257, 101, 212, 1032, 204, 41, 276, 19, 466, 1632, 267, 779, 1009, 10, 385, 650, 373, 222, 291, 462, 183, 880, 665, 571, 1140, 1500, 120, 1614, 2047, 979, 723, 412, 1384, 1095, 609, 487, 437, 38, 59, 626, 185] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=19 tokens_per_expert_current_ep_rank=[696, 509, 262, 133, 893, 199, 450, 359, 88, 3902, 314, 289, 1234, 391, 1513, 2538, 202, 2930, 415, 490, 2300, 113, 620, 2167, 1222, 983, 1930, 317, 246, 147, 67, 254, 829, 423, 1302, 1373, 301, 1063, 463, 1597, 479, 42, 106, 716, 1126, 1145, 94, 1159, 835, 9, 2604, 768, 417, 955, 15, 748, 368, 836, 207, 67, 49, 97, 266, 520] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=19 tokens_per_expert_current_ep_rank=[626, 719, 273, 122, 61, 63, 1159, 1771, 562, 415, 2707, 592, 1890, 2541, 1566, 1502, 261, 3199, 47, 1078, 2551, 164, 411, 1466, 655, 33, 1230, 352, 582, 706, 583, 121, 629, 195, 808, 336, 60, 605, 775, 815, 1193, 729, 197, 94, 2025, 1317, 72, 2464, 1319, 114, 173, 165, 30, 124, 98, 613, 1335, 1168, 148, 232, 81, 71, 500, 659] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=19 rerouted_tokens_per_expert_current_ep_rank=[428, 610, 132, 73, 1049, 82, 195, 2150, 443, 0, 0, 0, 0, 0, 0, 0, 534, 896, 875, 252, 508, 0, 0, 715, 0, 0, 0, 0, 0, 0, 0, 0, 110, 508, 259, 647, 1625, 902, 964, 1559, 0, 0, 0, 0, 0, 0, 0, 0, 689, 219, 1924, 393, 481, 57, 137, 615, 1186, 0, 0, 0, 0, 0, 0, 0, 192, 480, 482, 1226, 710, 1263, 2225, 1128, 0, 0, 0, 0, 0, 0, 0, 0, 480, 284, 536, 678, 2396, 418, 41, 1549, 1209, 30, 0, 0, 0, 0, 0, 0, 916, 27, 780, 800, 473, 115, 147, 1196, 399, 209, 0, 0, 0, 0, 0, 0, 2838, 588, 1151, 111, 87, 291, 304, 278, 1735, 163, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5162, 3780, 6574, 5701, 7706, 7621, 5062, 7546] avg=6144.0000 max=7706 max_rank=4 min=3780 min_rank=1 imbalance_ratio=1.254232 max_load_reduction=41446 max_load_reduction_pct=0.843221 moved_to_echo=27935 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=19 rerouted_tokens_per_expert_current_ep_rank=[1104, 3380, 124, 351, 239, 140, 194, 714, 331, 0, 0, 0, 0, 0, 0, 0, 742, 667, 662, 600, 2061, 0, 0, 3718, 0, 0, 0, 0, 0, 0, 0, 0, 30, 905, 373, 135, 1188, 90, 325, 1475, 0, 0, 0, 0, 0, 0, 0, 0, 1120, 595, 1238, 488, 444, 735, 320, 170, 1907, 0, 0, 0, 0, 0, 0, 0, 464, 386, 1354, 929, 482, 444, 1189, 1860, 0, 0, 0, 0, 0, 0, 0, 0, 360, 496, 477, 28, 2183, 492, 159, 112, 646, 23, 0, 0, 0, 0, 0, 0, 266, 164, 482, 293, 94, 221, 239, 140, 710, 336, 0, 0, 0, 0, 0, 0, 875, 463, 317, 700, 117, 356, 419, 1291, 2759, 261, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6577, 8450, 4521, 7017, 7108, 4976, 2945, 7558] avg=6144.0000 max=8450 max_rank=1 min=2945 min_rank=6 imbalance_ratio=1.375326 max_load_reduction=40702 max_load_reduction_pct=0.828084 moved_to_echo=22587 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=19 rerouted_tokens_per_expert_current_ep_rank=[449, 803, 72, 266, 43, 23, 878, 870, 920, 0, 0, 0, 0, 0, 0, 0, 179, 1855, 1939, 1631, 1304, 0, 0, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 160, 2218, 71, 607, 1073, 175, 321, 472, 0, 0, 0, 0, 0, 0, 0, 0, 719, 1154, 529, 157, 227, 355, 220, 313, 753, 0, 0, 0, 0, 0, 0, 0, 977, 551, 821, 784, 181, 705, 2360, 1296, 0, 0, 0, 0, 0, 0, 0, 0, 378, 37, 331, 265, 934, 976, 595, 1318, 1282, 64, 0, 0, 0, 0, 0, 0, 1564, 39, 2054, 1733, 67, 153, 17, 1546, 1742, 132, 0, 0, 0, 0, 0, 0, 342, 678, 564, 130, 12, 24, 208, 10, 2403, 104, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4324, 7927, 5097, 4427, 7675, 6180, 9047, 4475] avg=6144.0000 max=9047 max_rank=6 min=4324 min_rank=0 imbalance_ratio=1.472493 max_load_reduction=40105 max_load_reduction_pct=0.815938 moved_to_echo=27377 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=19 rerouted_tokens_per_expert_current_ep_rank=[864, 1466, 189, 950, 41, 38, 1822, 2309, 515, 0, 0, 0, 0, 0, 0, 0, 418, 1040, 1019, 174, 860, 0, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 80, 1577, 72, 105, 1923, 854, 740, 2223, 0, 0, 0, 0, 0, 0, 0, 0, 2419, 525, 608, 229, 951, 288, 165, 273, 988, 0, 0, 0, 0, 0, 0, 0, 759, 346, 457, 1212, 847, 230, 1519, 401, 0, 0, 0, 0, 0, 0, 0, 0, 164, 611, 192, 272, 2644, 894, 687, 2224, 824, 35, 0, 0, 0, 0, 0, 0, 230, 54, 635, 313, 123, 158, 8, 773, 1238, 176, 0, 0, 0, 0, 0, 0, 1435, 753, 195, 298, 12, 46, 47, 1003, 1303, 134, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 3686, 7574, 6446, 5771, 8547, 3708, 5226] avg=6144.0000 max=8547 max_rank=5 min=3686 min_rank=1 imbalance_ratio=1.391113 max_load_reduction=40605 max_load_reduction_pct=0.826111 moved_to_echo=23252 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=19 rerouted_tokens_per_expert_current_ep_rank=[498, 2405, 168, 213, 527, 197, 1980, 1841, 472, 0, 0, 0, 0, 0, 0, 0, 366, 935, 430, 1068, 1154, 0, 0, 1553, 0, 0, 0, 0, 0, 0, 0, 0, 53, 543, 157, 524, 943, 221, 610, 289, 0, 0, 0, 0, 0, 0, 0, 0, 454, 650, 3069, 200, 846, 68, 2263, 272, 1183, 0, 0, 0, 0, 0, 0, 0, 1279, 9, 343, 1365, 138, 1143, 320, 279, 0, 0, 0, 0, 0, 0, 0, 0, 38, 463, 401, 197, 709, 320, 1064, 270, 179, 36, 0, 0, 0, 0, 0, 0, 1702, 494, 1141, 494, 149, 106, 73, 1724, 431, 208, 0, 0, 0, 0, 0, 0, 1253, 1431, 1306, 196, 13, 360, 317, 302, 2587, 160, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8301, 5506, 3340, 9005, 4876, 3677, 6522, 7925] avg=6144.0000 max=9005 max_rank=3 min=3340 min_rank=2 imbalance_ratio=1.465658 max_load_reduction=40147 max_load_reduction_pct=0.816793 moved_to_echo=23000 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=19 rerouted_tokens_per_expert_current_ep_rank=[428, 873, 118, 723, 435, 172, 2110, 1749, 292, 0, 0, 0, 0, 0, 0, 0, 576, 591, 2075, 1257, 437, 0, 0, 329, 0, 0, 0, 0, 0, 0, 0, 0, 396, 717, 143, 96, 1785, 227, 707, 2257, 0, 0, 0, 0, 0, 0, 0, 0, 101, 212, 1032, 204, 41, 276, 19, 466, 2431, 0, 0, 0, 0, 0, 0, 0, 1632, 267, 779, 1009, 10, 385, 422, 373, 0, 0, 0, 0, 0, 0, 0, 0, 222, 291, 462, 183, 880, 665, 571, 1140, 228, 20, 0, 0, 0, 0, 0, 0, 1500, 120, 1614, 2047, 979, 723, 412, 1384, 563, 428, 0, 0, 0, 0, 0, 0, 1095, 609, 487, 437, 38, 59, 626, 185, 2702, 330, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6900, 5265, 6328, 4782, 4877, 4662, 9770, 6568] avg=6144.0000 max=9770 max_rank=6 min=4662 min_rank=5 imbalance_ratio=1.590169 max_load_reduction=39382 max_load_reduction_pct=0.801229 moved_to_echo=25877 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=19 rerouted_tokens_per_expert_current_ep_rank=[696, 509, 262, 133, 893, 199, 450, 359, 1265, 0, 0, 0, 0, 0, 0, 0, 88, 2550, 314, 289, 1234, 0, 0, 2538, 0, 0, 0, 0, 0, 0, 0, 0, 202, 1641, 415, 490, 2300, 113, 620, 2167, 0, 0, 0, 0, 0, 0, 0, 0, 1222, 983, 1930, 317, 246, 147, 67, 254, 1153, 0, 0, 0, 0, 0, 0, 0, 829, 423, 1302, 1373, 301, 1063, 300, 1597, 0, 0, 0, 0, 0, 0, 0, 0, 479, 42, 106, 716, 1126, 1145, 94, 1159, 163, 87, 0, 0, 0, 0, 0, 0, 835, 9, 2604, 768, 417, 955, 15, 748, 1289, 204, 0, 0, 0, 0, 0, 0, 368, 836, 207, 67, 49, 97, 266, 520, 391, 156, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4766, 7013, 7948, 6319, 7188, 5117, 7844, 2957] avg=6144.0000 max=7948 max_rank=2 min=2957 min_rank=7 imbalance_ratio=1.293620 max_load_reduction=41204 max_load_reduction_pct=0.838298 moved_to_echo=23106 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=19 rerouted_tokens_per_expert_current_ep_rank=[626, 719, 273, 122, 61, 63, 1159, 1771, 134, 0, 0, 0, 0, 0, 0, 0, 562, 272, 2707, 592, 1890, 0, 0, 1502, 0, 0, 0, 0, 0, 0, 0, 0, 261, 1792, 47, 1078, 2551, 164, 411, 1466, 0, 0, 0, 0, 0, 0, 0, 0, 655, 33, 1230, 352, 582, 706, 583, 121, 1193, 0, 0, 0, 0, 0, 0, 0, 629, 195, 808, 336, 60, 605, 503, 815, 0, 0, 0, 0, 0, 0, 0, 0, 1193, 729, 197, 94, 2025, 1317, 72, 2464, 272, 9, 0, 0, 0, 0, 0, 0, 1319, 114, 173, 165, 30, 124, 98, 613, 1407, 211, 0, 0, 0, 0, 0, 0, 1335, 1168, 148, 232, 81, 71, 500, 659, 2541, 162, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4928, 7525, 7770, 5455, 3951, 8372, 4254, 6897] avg=6144.0000 max=8372 max_rank=5 min=3951 min_rank=4 imbalance_ratio=1.362630 max_load_reduction=40780 max_load_reduction_pct=0.829671 moved_to_echo=23474 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[8448, 1024, 9600, 6656, 2432, 2560, 1024, 8192, 7808, 1920, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3328, 3072, 2816, 2560, 12928, 6272, 3328, 10240, 4864, 384, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[9600, 6528, 4480, 2176, 512, 1408, 2688, 4352, 16512, 1536, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6784, 2688, 6400, 8320, 2816, 5888, 8960, 7808, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1408, 9984, 1664, 3712, 13440, 2816, 4736, 12032, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5120, 10880, 1408, 2944, 3328, 1024, 8832, 11776, 4480, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[7424, 4480, 11648, 2432, 3840, 2688, 3840, 2560, 10880, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[3584, 8832, 10112, 5888, 9472, 0, 0, 11648, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=20 tokens_per_expert_current_ep_rank=[2414, 228, 82, 71, 2595, 1650, 369, 215, 258, 14, 299, 397, 44, 15, 139, 712, 1252, 297, 433, 290, 1631, 254, 275, 796, 1429, 1124, 934, 318, 886, 427, 111, 906, 786, 144, 30, 183, 76, 438, 2310, 1135, 246, 351, 125, 390, 1018, 15, 183, 1132, 702, 65, 3230, 609, 674, 389, 4366, 2166, 73, 932, 1630, 1147, 447, 848, 2246, 201] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=20 tokens_per_expert_current_ep_rank=[2074, 483, 687, 1096, 1564, 1182, 314, 128, 109, 874, 287, 561, 1249, 528, 308, 1401, 852, 164, 857, 447, 433, 74, 287, 194, 45, 2239, 283, 1457, 347, 281, 126, 470, 650, 1460, 261, 273, 333, 295, 669, 901, 123, 425, 137, 785, 1329, 1517, 235, 214, 1757, 191, 1591, 1554, 451, 757, 1031, 631, 1325, 1461, 2253, 1321, 341, 1053, 1946, 481] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=20 tokens_per_expert_current_ep_rank=[781, 97, 49, 962, 2854, 1143, 194, 72, 598, 212, 30, 815, 1156, 287, 3134, 3325, 729, 188, 399, 173, 1409, 335, 370, 1931, 408, 1779, 920, 1679, 265, 452, 195, 879, 306, 221, 20, 334, 98, 285, 1447, 46, 5, 2731, 479, 109, 1255, 225, 424, 1397, 1003, 1013, 1339, 1291, 744, 1746, 233, 1303, 80, 403, 1209, 249, 74, 196, 413, 654] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=20 tokens_per_expert_current_ep_rank=[1834, 414, 97, 565, 484, 2421, 963, 280, 50, 243, 104, 330, 623, 291, 133, 2020, 142, 990, 3680, 274, 398, 56, 108, 205, 355, 319, 1511, 1166, 338, 169, 45, 234, 1464, 262, 380, 383, 416, 1012, 658, 2662, 115, 720, 361, 17, 540, 122, 723, 946, 666, 371, 2089, 252, 221, 3065, 2983, 1175, 251, 1655, 1546, 41, 1538, 587, 196, 893] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=20 tokens_per_expert_current_ep_rank=[537, 263, 2, 423, 3395, 89, 65, 47, 118, 27, 29, 573, 1793, 44, 191, 1224, 221, 854, 532, 619, 1380, 461, 589, 3893, 395, 2345, 1566, 1199, 182, 144, 657, 1786, 562, 123, 248, 115, 519, 667, 1967, 479, 118, 1318, 350, 160, 614, 395, 365, 538, 1994, 26, 1243, 1866, 897, 577, 441, 1568, 297, 491, 1180, 992, 258, 1496, 1324, 321] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=20 tokens_per_expert_current_ep_rank=[488, 497, 242, 713, 2004, 312, 187, 199, 646, 30, 51, 697, 767, 152, 1380, 1157, 955, 340, 263, 266, 3349, 647, 446, 1087, 76, 1662, 2854, 2959, 1796, 5, 164, 32, 764, 48, 12, 549, 603, 2224, 335, 542, 47, 685, 299, 419, 480, 217, 226, 1065, 2092, 419, 1341, 1049, 608, 1488, 2103, 1282, 1171, 93, 563, 274, 372, 656, 549, 154] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=20 tokens_per_expert_current_ep_rank=[3343, 741, 7, 1122, 1987, 2297, 239, 110, 373, 269, 201, 343, 2186, 62, 22, 2797, 470, 991, 734, 259, 443, 21, 273, 801, 890, 2107, 1382, 1494, 28, 442, 206, 277, 418, 122, 244, 339, 82, 384, 2318, 2858, 457, 602, 86, 6, 1163, 259, 19, 46, 873, 54, 3138, 489, 314, 1177, 729, 1352, 25, 542, 2015, 338, 61, 272, 1311, 142] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=20 tokens_per_expert_current_ep_rank=[2069, 28, 16, 1050, 1613, 531, 441, 48, 653, 95, 179, 596, 371, 101, 236, 3594, 716, 223, 1751, 101, 566, 155, 2173, 290, 203, 1539, 940, 1682, 755, 857, 21, 335, 560, 440, 894, 91, 108, 273, 718, 1086, 383, 709, 207, 560, 383, 1162, 72, 291, 953, 117, 929, 1300, 390, 1235, 1815, 2106, 870, 21, 4511, 371, 202, 426, 1704, 337] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=20 rerouted_tokens_per_expert_current_ep_rank=[2414, 228, 82, 71, 1917, 1650, 369, 215, 0, 0, 0, 0, 0, 0, 0, 0, 258, 14, 299, 397, 44, 15, 139, 712, 678, 209, 284, 0, 0, 0, 0, 0, 1252, 297, 433, 290, 1631, 254, 275, 796, 205, 127, 0, 0, 0, 0, 0, 0, 1429, 788, 934, 318, 886, 427, 111, 906, 0, 0, 0, 0, 0, 0, 0, 0, 786, 144, 30, 183, 76, 438, 2310, 1135, 2715, 0, 0, 0, 0, 0, 0, 0, 246, 351, 125, 390, 1018, 15, 183, 1132, 3230, 687, 0, 0, 0, 0, 0, 0, 702, 65, 0, 609, 674, 389, 680, 2166, 0, 0, 0, 0, 0, 0, 0, 0, 73, 932, 1425, 1147, 447, 848, 2246, 201, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6946, 3049, 5560, 5799, 7817, 7377, 5285, 7319] avg=6144.0000 max=7817 max_rank=4 min=3049 min_rank=1 imbalance_ratio=1.272298 max_load_reduction=41335 max_load_reduction_pct=0.840963 moved_to_echo=27798 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=20 rerouted_tokens_per_expert_current_ep_rank=[781, 97, 49, 962, 2108, 1143, 194, 72, 0, 0, 0, 0, 0, 0, 0, 0, 598, 212, 30, 815, 1156, 287, 3134, 3325, 746, 331, 15, 0, 0, 0, 0, 0, 729, 188, 399, 173, 1409, 335, 370, 1931, 152, 201, 0, 0, 0, 0, 0, 0, 408, 1247, 920, 1679, 265, 452, 195, 879, 0, 0, 0, 0, 0, 0, 0, 0, 306, 221, 20, 334, 98, 285, 1447, 46, 144, 0, 0, 0, 0, 0, 0, 0, 5, 2731, 479, 109, 1255, 225, 424, 1397, 1339, 36, 0, 0, 0, 0, 0, 0, 1003, 1013, 0, 1291, 744, 1746, 38, 1303, 0, 0, 0, 0, 0, 0, 0, 0, 80, 403, 1057, 249, 74, 196, 413, 654, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5406, 10649, 5887, 6045, 2901, 8000, 7138, 3126] avg=6144.0000 max=10649 max_rank=1 min=2901 min_rank=4 imbalance_ratio=1.733236 max_load_reduction=38503 max_load_reduction_pct=0.783346 moved_to_echo=21165 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=20 rerouted_tokens_per_expert_current_ep_rank=[2074, 483, 687, 1096, 1155, 1182, 314, 128, 0, 0, 0, 0, 0, 0, 0, 0, 109, 874, 287, 561, 1249, 528, 308, 1401, 409, 416, 67, 0, 0, 0, 0, 0, 852, 164, 857, 447, 433, 74, 287, 194, 283, 253, 0, 0, 0, 0, 0, 0, 45, 1570, 283, 1457, 347, 281, 126, 470, 0, 0, 0, 0, 0, 0, 0, 0, 650, 1460, 261, 273, 333, 295, 669, 901, 641, 0, 0, 0, 0, 0, 0, 0, 123, 425, 137, 785, 1329, 1517, 235, 214, 1591, 162, 0, 0, 0, 0, 0, 0, 1757, 191, 0, 1554, 451, 757, 161, 631, 0, 0, 0, 0, 0, 0, 0, 0, 1325, 1461, 1970, 1321, 341, 1053, 1946, 481, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7119, 6209, 3844, 4579, 5483, 6518, 5502, 9898] avg=6144.0000 max=9898 max_rank=7 min=3844 min_rank=2 imbalance_ratio=1.611003 max_load_reduction=39254 max_load_reduction_pct=0.798625 moved_to_echo=27401 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=20 rerouted_tokens_per_expert_current_ep_rank=[3343, 741, 7, 1122, 1468, 2297, 239, 110, 0, 0, 0, 0, 0, 0, 0, 0, 373, 269, 201, 343, 2186, 62, 22, 2797, 519, 392, 47, 0, 0, 0, 0, 0, 470, 991, 734, 259, 443, 21, 273, 801, 253, 238, 0, 0, 0, 0, 0, 0, 890, 1477, 1382, 1494, 28, 442, 206, 277, 0, 0, 0, 0, 0, 0, 0, 0, 418, 122, 244, 339, 82, 384, 2318, 2858, 453, 0, 0, 0, 0, 0, 0, 0, 457, 602, 86, 6, 1163, 259, 19, 46, 3138, 114, 0, 0, 0, 0, 0, 0, 873, 54, 0, 489, 314, 1177, 115, 1352, 0, 0, 0, 0, 0, 0, 0, 0, 25, 542, 1762, 338, 61, 272, 1311, 142, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9327, 7211, 4483, 6196, 7218, 5890, 4374, 4453] avg=6144.0000 max=9327 max_rank=0 min=4374 min_rank=6 imbalance_ratio=1.518066 max_load_reduction=39825 max_load_reduction_pct=0.810242 moved_to_echo=21935 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=20 rerouted_tokens_per_expert_current_ep_rank=[537, 263, 2, 423, 2507, 89, 65, 47, 0, 0, 0, 0, 0, 0, 0, 0, 118, 27, 29, 573, 1793, 44, 191, 1224, 888, 436, 28, 0, 0, 0, 0, 0, 221, 854, 532, 619, 1380, 461, 589, 3893, 148, 265, 0, 0, 0, 0, 0, 0, 395, 1644, 1566, 1199, 182, 144, 657, 1786, 0, 0, 0, 0, 0, 0, 0, 0, 562, 123, 248, 115, 519, 667, 1967, 479, 274, 0, 0, 0, 0, 0, 0, 0, 118, 1318, 350, 160, 614, 395, 365, 538, 1243, 69, 0, 0, 0, 0, 0, 0, 1994, 26, 0, 1866, 897, 577, 70, 1568, 0, 0, 0, 0, 0, 0, 0, 0, 297, 491, 1032, 992, 258, 1496, 1324, 321, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3933, 5351, 8962, 7573, 4954, 5170, 6998, 6211] avg=6144.0000 max=8962 max_rank=2 min=3933 min_rank=0 imbalance_ratio=1.458659 max_load_reduction=40190 max_load_reduction_pct=0.817668 moved_to_echo=23333 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=20 rerouted_tokens_per_expert_current_ep_rank=[1834, 414, 97, 565, 358, 2421, 963, 280, 0, 0, 0, 0, 0, 0, 0, 0, 50, 243, 104, 330, 623, 291, 133, 2020, 126, 59, 194, 0, 0, 0, 0, 0, 142, 990, 3680, 274, 398, 56, 108, 205, 194, 36, 0, 0, 0, 0, 0, 0, 355, 224, 1511, 1166, 338, 169, 45, 234, 0, 0, 0, 0, 0, 0, 0, 0, 1464, 262, 380, 383, 416, 1012, 658, 2662, 1855, 0, 0, 0, 0, 0, 0, 0, 115, 720, 361, 17, 540, 122, 723, 946, 2089, 469, 0, 0, 0, 0, 0, 0, 666, 371, 0, 252, 221, 3065, 465, 1175, 0, 0, 0, 0, 0, 0, 0, 0, 251, 1655, 1352, 41, 1538, 587, 196, 893, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6932, 4173, 6083, 4042, 9092, 6102, 6215, 6513] avg=6144.0000 max=9092 max_rank=4 min=4042 min_rank=3 imbalance_ratio=1.479818 max_load_reduction=40060 max_load_reduction_pct=0.815023 moved_to_echo=27922 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=20 rerouted_tokens_per_expert_current_ep_rank=[488, 497, 242, 713, 1476, 312, 187, 199, 0, 0, 0, 0, 0, 0, 0, 0, 646, 30, 51, 697, 767, 152, 1380, 1157, 528, 313, 139, 0, 0, 0, 0, 0, 955, 340, 263, 266, 3349, 647, 446, 1087, 75, 189, 0, 0, 0, 0, 0, 0, 76, 1160, 2854, 2959, 1796, 5, 164, 32, 0, 0, 0, 0, 0, 0, 0, 0, 764, 48, 12, 549, 603, 2224, 335, 542, 1312, 0, 0, 0, 0, 0, 0, 0, 47, 685, 299, 419, 480, 217, 226, 1065, 1341, 335, 0, 0, 0, 0, 0, 0, 2092, 419, 0, 1049, 608, 1488, 317, 1282, 0, 0, 0, 0, 0, 0, 0, 0, 1171, 93, 488, 274, 372, 656, 549, 154, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 5860, 7617, 9046, 6389, 5114, 7255, 3757] avg=6144.0000 max=9046 max_rank=3 min=3757 min_rank=7 imbalance_ratio=1.472331 max_load_reduction=40106 max_load_reduction_pct=0.815959 moved_to_echo=22515 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=20 rerouted_tokens_per_expert_current_ep_rank=[2069, 28, 16, 1050, 1192, 531, 441, 48, 0, 0, 0, 0, 0, 0, 0, 0, 653, 95, 179, 596, 371, 101, 236, 3594, 421, 286, 118, 0, 0, 0, 0, 0, 716, 223, 1751, 101, 566, 155, 2173, 290, 567, 174, 0, 0, 0, 0, 0, 0, 203, 1079, 940, 1682, 755, 857, 21, 335, 0, 0, 0, 0, 0, 0, 0, 0, 560, 440, 894, 91, 108, 273, 718, 1086, 1128, 0, 0, 0, 0, 0, 0, 0, 383, 709, 207, 560, 383, 1162, 72, 291, 929, 285, 0, 0, 0, 0, 0, 0, 953, 117, 0, 1300, 390, 1235, 284, 2106, 0, 0, 0, 0, 0, 0, 0, 0, 870, 21, 3944, 371, 202, 426, 1704, 337, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5375, 6650, 6716, 5872, 5298, 4981, 6385, 7875] avg=6144.0000 max=7875 max_rank=7 min=4981 min_rank=5 imbalance_ratio=1.281738 max_load_reduction=41277 max_load_reduction_pct=0.839783 moved_to_echo=24539 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2816, 1792, 1280, 4352, 8192, 1536, 5632, 16256, 4352, 2560, 896, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[4096, 5632, 13056, 4736, 3328, 5632, 9728, 3200, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5376, 4096, 8704, 2432, 9728, 2048, 4608, 9216, 1920, 1536, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[13568, 2816, 1280, 6016, 12288, 9728, 2816, 1152, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[1536, 7552, 2048, 2560, 6784, 3968, 2304, 5632, 14976, 2176, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[3840, 9216, 10496, 12032, 4608, 2816, 1536, 4992, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[10112, 2304, 0, 8448, 4352, 10496, 2176, 11648, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5632, 2944, 2176, 2304, 2304, 5632, 10496, 9728, 8576, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=21 tokens_per_expert_current_ep_rank=[317, 515, 53, 780, 32, 50, 511, 1806, 512, 1038, 3057, 1010, 39, 174, 359, 600, 776, 49, 620, 299, 2205, 847, 152, 973, 105, 743, 549, 270, 5273, 98, 59, 149, 23, 615, 320, 11, 1233, 250, 1297, 116, 4291, 1560, 977, 582, 406, 390, 1143, 2443, 835, 301, 777, 136, 97, 1141, 185, 16, 215, 163, 2191, 908, 264, 1064, 1174, 8] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=21 tokens_per_expert_current_ep_rank=[567, 45, 1741, 232, 267, 101, 195, 546, 336, 2327, 1052, 151, 60, 1278, 57, 500, 2143, 234, 693, 489, 2555, 2088, 241, 526, 149, 924, 127, 102, 3622, 804, 107, 259, 13, 2882, 323, 67, 769, 427, 1026, 605, 2735, 961, 238, 122, 2101, 293, 1013, 1077, 3080, 227, 823, 148, 613, 1034, 254, 28, 59, 22, 460, 84, 570, 494, 1425, 661] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=21 tokens_per_expert_current_ep_rank=[42, 661, 1115, 688, 262, 458, 787, 3795, 65, 868, 1894, 109, 27, 465, 47, 834, 1337, 30, 269, 31, 2047, 1066, 169, 772, 309, 1080, 498, 185, 6768, 159, 29, 1019, 21, 337, 98, 52, 682, 59, 1377, 113, 1720, 815, 94, 37, 934, 76, 1413, 505, 664, 304, 2936, 153, 348, 2033, 384, 28, 281, 202, 2018, 79, 715, 203, 2426, 160] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=21 tokens_per_expert_current_ep_rank=[169, 328, 638, 756, 81, 34, 643, 331, 628, 854, 3026, 433, 125, 871, 159, 479, 1063, 30, 600, 811, 1949, 385, 888, 616, 712, 2781, 989, 81, 4117, 205, 46, 461, 28, 673, 192, 322, 865, 361, 261, 334, 3258, 1983, 83, 572, 351, 141, 1874, 164, 1917, 476, 1235, 595, 334, 1795, 480, 20, 245, 71, 2439, 134, 96, 683, 1752, 129] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=21 tokens_per_expert_current_ep_rank=[2605, 649, 514, 351, 102, 142, 199, 2093, 24, 323, 580, 554, 214, 309, 30, 82, 1885, 123, 491, 1230, 4280, 863, 661, 1130, 32, 921, 1427, 786, 1472, 360, 28, 10, 23, 234, 88, 513, 460, 29, 384, 993, 2139, 1244, 582, 236, 688, 268, 630, 923, 2162, 2647, 2658, 825, 94, 1553, 53, 86, 920, 575, 909, 75, 4, 286, 2220, 181] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=21 tokens_per_expert_current_ep_rank=[354, 204, 413, 94, 438, 256, 1375, 1507, 329, 497, 3001, 689, 91, 456, 14, 180, 1220, 306, 182, 90, 1206, 974, 550, 422, 11, 3359, 793, 297, 3237, 309, 22, 97, 66, 1761, 28, 29, 535, 163, 1088, 706, 3698, 1789, 482, 21, 388, 102, 2098, 1736, 1782, 329, 1496, 389, 166, 800, 894, 428, 1236, 204, 1101, 7, 239, 986, 983, 449] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=21 tokens_per_expert_current_ep_rank=[236, 308, 78, 950, 102, 167, 696, 864, 410, 2912, 5039, 291, 10, 384, 174, 628, 422, 39, 248, 65, 4267, 744, 537, 798, 322, 1425, 143, 696, 1922, 135, 57, 516, 5, 169, 1382, 17, 848, 107, 891, 76, 2653, 225, 779, 426, 402, 266, 1095, 3625, 86, 849, 1224, 246, 767, 2910, 751, 40, 466, 835, 894, 323, 49, 431, 154, 576] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=21 tokens_per_expert_current_ep_rank=[1697, 538, 45, 2209, 594, 97, 172, 1286, 535, 237, 939, 65, 18, 1438, 37, 527, 122, 157, 305, 547, 3841, 2137, 487, 355, 455, 2376, 206, 161, 6311, 275, 9, 2527, 7, 1680, 12, 94, 17, 15, 844, 90, 411, 204, 1021, 363, 1936, 597, 542, 496, 1410, 974, 415, 340, 1675, 742, 139, 59, 1216, 198, 794, 200, 302, 84, 253, 1317] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=21 rerouted_tokens_per_expert_current_ep_rank=[42, 661, 1115, 688, 262, 458, 787, 3795, 642, 465, 0, 0, 0, 0, 0, 0, 65, 868, 1894, 109, 27, 465, 47, 834, 547, 134, 0, 0, 0, 0, 0, 0, 1337, 30, 269, 31, 1271, 1066, 169, 772, 0, 0, 0, 0, 0, 0, 0, 0, 309, 1080, 498, 185, 3807, 159, 29, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 21, 337, 98, 52, 682, 59, 1377, 113, 1420, 575, 0, 0, 0, 0, 0, 0, 300, 815, 94, 37, 934, 76, 1413, 505, 0, 0, 0, 0, 0, 0, 0, 0, 664, 304, 2936, 153, 348, 1486, 384, 28, 0, 0, 0, 0, 0, 0, 0, 0, 281, 202, 2018, 79, 715, 203, 2426, 160, 1921, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8915, 4990, 4945, 7086, 4734, 4174, 6303, 8005] avg=6144.0000 max=8915 max_rank=0 min=4174 min_rank=5 imbalance_ratio=1.451009 max_load_reduction=40237 max_load_reduction_pct=0.818624 moved_to_echo=23216 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=21 rerouted_tokens_per_expert_current_ep_rank=[317, 515, 53, 780, 32, 50, 511, 1806, 692, 363, 0, 0, 0, 0, 0, 0, 512, 1038, 3057, 1010, 39, 174, 359, 600, 307, 144, 0, 0, 0, 0, 0, 0, 776, 49, 620, 299, 1369, 847, 152, 973, 0, 0, 0, 0, 0, 0, 0, 0, 105, 743, 549, 270, 2965, 98, 59, 149, 0, 0, 0, 0, 0, 0, 0, 0, 23, 615, 320, 11, 1233, 250, 1297, 116, 3543, 448, 0, 0, 0, 0, 0, 0, 748, 1560, 977, 582, 406, 390, 1143, 2443, 0, 0, 0, 0, 0, 0, 0, 0, 835, 301, 777, 136, 97, 834, 185, 16, 0, 0, 0, 0, 0, 0, 0, 0, 215, 163, 2191, 908, 264, 1064, 1174, 8, 1497, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5119, 7240, 5085, 4938, 7856, 8249, 3181, 7484] avg=6144.0000 max=8249 max_rank=5 min=3181 min_rank=6 imbalance_ratio=1.342611 max_load_reduction=40903 max_load_reduction_pct=0.832174 moved_to_echo=26770 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=21 rerouted_tokens_per_expert_current_ep_rank=[567, 45, 1741, 232, 267, 101, 195, 546, 802, 249, 0, 0, 0, 0, 0, 0, 336, 2327, 1052, 151, 60, 1278, 57, 500, 278, 167, 0, 0, 0, 0, 0, 0, 2143, 234, 693, 489, 1586, 2088, 241, 526, 0, 0, 0, 0, 0, 0, 0, 0, 149, 924, 127, 102, 2038, 804, 107, 259, 0, 0, 0, 0, 0, 0, 0, 0, 13, 2882, 323, 67, 769, 427, 1026, 605, 2258, 307, 0, 0, 0, 0, 0, 0, 477, 961, 238, 122, 2101, 293, 1013, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 3080, 227, 823, 148, 613, 756, 254, 28, 0, 0, 0, 0, 0, 0, 0, 0, 59, 22, 460, 84, 570, 494, 1425, 661, 1028, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4745, 6206, 8000, 4510, 8677, 6282, 5929, 4803] avg=6144.0000 max=8677 max_rank=4 min=4510 min_rank=3 imbalance_ratio=1.412272 max_load_reduction=40475 max_load_reduction_pct=0.823466 moved_to_echo=25691 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=21 rerouted_tokens_per_expert_current_ep_rank=[354, 204, 413, 94, 438, 256, 1375, 1507, 378, 222, 0, 0, 0, 0, 0, 0, 329, 497, 3001, 689, 91, 456, 14, 180, 215, 79, 0, 0, 0, 0, 0, 0, 1220, 306, 182, 90, 749, 974, 550, 422, 0, 0, 0, 0, 0, 0, 0, 0, 11, 3359, 793, 297, 1821, 309, 22, 97, 0, 0, 0, 0, 0, 0, 0, 0, 66, 1761, 28, 29, 535, 163, 1088, 706, 3054, 275, 0, 0, 0, 0, 0, 0, 644, 1789, 482, 21, 388, 102, 2098, 1736, 0, 0, 0, 0, 0, 0, 0, 0, 1782, 329, 1496, 389, 166, 585, 894, 428, 0, 0, 0, 0, 0, 0, 0, 0, 1236, 204, 1101, 7, 239, 986, 983, 449, 919, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5241, 5551, 4493, 6709, 7705, 7260, 6069, 6124] avg=6144.0000 max=7705 max_rank=4 min=4493 min_rank=2 imbalance_ratio=1.254069 max_load_reduction=41447 max_load_reduction_pct=0.843241 moved_to_echo=27158 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=21 rerouted_tokens_per_expert_current_ep_rank=[236, 308, 78, 950, 102, 167, 696, 864, 1344, 136, 0, 0, 0, 0, 0, 0, 410, 2912, 5039, 291, 10, 384, 174, 628, 787, 284, 0, 0, 0, 0, 0, 0, 422, 39, 248, 65, 2639, 744, 537, 798, 0, 0, 0, 0, 0, 0, 0, 0, 322, 1425, 143, 696, 1071, 135, 57, 516, 0, 0, 0, 0, 0, 0, 0, 0, 5, 169, 1382, 17, 848, 107, 891, 76, 2195, 166, 0, 0, 0, 0, 0, 0, 458, 225, 779, 426, 402, 266, 1095, 3625, 0, 0, 0, 0, 0, 0, 0, 0, 86, 849, 1224, 246, 767, 2123, 751, 40, 0, 0, 0, 0, 0, 0, 0, 0, 466, 835, 894, 323, 49, 431, 154, 576, 549, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4881, 10919, 5492, 4365, 5856, 7276, 6086, 4277] avg=6144.0000 max=10919 max_rank=1 min=4277 min_rank=7 imbalance_ratio=1.777181 max_load_reduction=38233 max_load_reduction_pct=0.777852 moved_to_echo=23495 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=21 rerouted_tokens_per_expert_current_ep_rank=[2605, 649, 514, 351, 102, 142, 199, 2093, 1343, 101, 0, 0, 0, 0, 0, 0, 24, 323, 580, 554, 214, 309, 30, 82, 418, 280, 0, 0, 0, 0, 0, 0, 1885, 123, 491, 1230, 2657, 863, 661, 1130, 0, 0, 0, 0, 0, 0, 0, 0, 32, 921, 1427, 786, 829, 360, 28, 10, 0, 0, 0, 0, 0, 0, 0, 0, 23, 234, 88, 513, 460, 29, 384, 993, 1766, 125, 0, 0, 0, 0, 0, 0, 373, 1244, 582, 236, 688, 268, 630, 923, 0, 0, 0, 0, 0, 0, 0, 0, 2162, 2647, 2658, 825, 94, 1135, 53, 86, 0, 0, 0, 0, 0, 0, 0, 0, 920, 575, 909, 75, 4, 286, 2220, 181, 417, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8099, 2814, 9040, 4393, 4615, 4944, 9660, 5587] avg=6144.0000 max=9660 max_rank=6 min=2814 min_rank=1 imbalance_ratio=1.572266 max_load_reduction=39492 max_load_reduction_pct=0.803467 moved_to_echo=24806 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=21 rerouted_tokens_per_expert_current_ep_rank=[1697, 538, 45, 2209, 594, 97, 172, 1286, 1206, 434, 0, 0, 0, 0, 0, 0, 535, 237, 939, 65, 18, 1438, 37, 527, 199, 252, 0, 0, 0, 0, 0, 0, 122, 157, 305, 547, 2383, 2137, 487, 355, 0, 0, 0, 0, 0, 0, 0, 0, 455, 2376, 206, 161, 3550, 275, 9, 2527, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1680, 12, 94, 17, 15, 844, 90, 339, 536, 0, 0, 0, 0, 0, 0, 72, 204, 1021, 363, 1936, 597, 542, 496, 0, 0, 0, 0, 0, 0, 0, 0, 1410, 974, 415, 340, 1675, 543, 139, 59, 0, 0, 0, 0, 0, 0, 0, 0, 1216, 198, 794, 200, 302, 84, 253, 1317, 1791, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8278, 4247, 6493, 9559, 3634, 5231, 5555, 6155] avg=6144.0000 max=9559 max_rank=3 min=3634 min_rank=4 imbalance_ratio=1.555827 max_load_reduction=39593 max_load_reduction_pct=0.805522 moved_to_echo=20575 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=21 rerouted_tokens_per_expert_current_ep_rank=[169, 328, 638, 756, 81, 34, 643, 331, 611, 283, 0, 0, 0, 0, 0, 0, 628, 854, 3026, 433, 125, 871, 159, 479, 483, 127, 0, 0, 0, 0, 0, 0, 1063, 30, 600, 811, 1211, 385, 888, 616, 0, 0, 0, 0, 0, 0, 0, 0, 712, 2781, 989, 81, 2317, 205, 46, 461, 0, 0, 0, 0, 0, 0, 0, 0, 28, 673, 192, 322, 865, 361, 261, 334, 2690, 349, 0, 0, 0, 0, 0, 0, 568, 1983, 83, 572, 351, 141, 1874, 164, 0, 0, 0, 0, 0, 0, 0, 0, 1917, 476, 1235, 595, 334, 1312, 480, 20, 0, 0, 0, 0, 0, 0, 0, 0, 245, 71, 2439, 134, 96, 683, 1752, 129, 1168, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3874, 7185, 5604, 7592, 6075, 5736, 6369, 6717] avg=6144.0000 max=7592 max_rank=3 min=3874 min_rank=0 imbalance_ratio=1.235677 max_load_reduction=41560 max_load_reduction_pct=0.845540 moved_to_echo=24897 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[256, 8448, 2560, 1152, 5504, 1536, 7168, 3072, 17280, 2816, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3712, 8832, 4352, 2432, 7296, 2176, 9856, 11008, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[9088, 1024, 3456, 3584, 13952, 9216, 3712, 5632, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2944, 9088, 18688, 3328, 640, 5376, 896, 3840, 3328, 1536, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6016, 3328, 4608, 6144, 1920, 1408, 4608, 12288, 7040, 2304, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[4736, 2304, 10880, 1920, 2304, 4352, 10496, 3584, 9344, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2176, 13696, 4736, 2688, 18432, 2432, 384, 5120, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[12032, 6144, 11648, 2944, 4096, 8832, 3200, 768, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=22 tokens_per_expert_current_ep_rank=[297, 231, 509, 406, 1335, 1636, 2667, 1577, 3863, 0, 195, 570, 1120, 611, 801, 1419, 1521, 2114, 308, 1142, 107, 1242, 901, 1173, 250, 842, 4, 756, 128, 1739, 6, 605, 35, 4181, 759, 227, 198, 1935, 316, 483, 640, 369, 108, 19, 418, 2236, 168, 327, 169, 259, 47, 1154, 97, 1459, 42, 13, 582, 437, 790, 280, 37, 1181, 19, 92] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=22 tokens_per_expert_current_ep_rank=[359, 315, 120, 631, 455, 46, 1024, 604, 3937, 35, 444, 578, 36, 358, 1828, 2205, 36, 554, 548, 1032, 490, 1781, 4492, 2266, 441, 1682, 96, 664, 2140, 1424, 413, 215, 96, 3321, 116, 523, 178, 449, 622, 148, 336, 743, 424, 437, 49, 233, 67, 505, 956, 506, 251, 356, 21, 1341, 333, 3, 1636, 146, 161, 1790, 766, 869, 275, 246] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=22 tokens_per_expert_current_ep_rank=[558, 81, 38, 596, 40, 19, 1177, 2150, 3426, 267, 25, 161, 26, 510, 1125, 994, 1177, 4735, 1405, 1273, 111, 4360, 4711, 249, 970, 1206, 0, 126, 1172, 766, 11, 167, 159, 295, 1357, 122, 631, 583, 2992, 224, 1048, 1354, 80, 242, 11, 276, 73, 560, 190, 905, 292, 342, 29, 277, 556, 19, 245, 34, 230, 210, 51, 1540, 95, 498] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=22 tokens_per_expert_current_ep_rank=[228, 109, 584, 536, 401, 487, 1613, 643, 1891, 147, 77, 234, 999, 2417, 801, 558, 695, 1124, 857, 480, 126, 1097, 2524, 1416, 58, 1565, 305, 175, 1992, 1189, 79, 533, 115, 2537, 1198, 398, 176, 1150, 3301, 868, 1003, 1622, 57, 255, 33, 965, 388, 638, 87, 808, 434, 288, 348, 804, 446, 824, 1807, 605, 209, 206, 489, 374, 228, 551] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=22 tokens_per_expert_current_ep_rank=[440, 332, 14, 133, 821, 56, 1924, 165, 1845, 69, 2276, 132, 79, 505, 17, 1443, 544, 2416, 3536, 98, 25, 592, 2713, 1604, 123, 875, 64, 29, 868, 1576, 5, 376, 107, 3575, 1757, 1022, 164, 496, 3840, 1266, 137, 252, 21, 31, 10, 113, 42, 199, 343, 1362, 39, 977, 44, 1192, 197, 69, 138, 1175, 222, 1567, 63, 1408, 311, 1318] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=22 tokens_per_expert_current_ep_rank=[232, 136, 394, 787, 1588, 129, 1398, 105, 2598, 414, 1291, 504, 363, 511, 2466, 372, 213, 278, 385, 311, 76, 2192, 4554, 24, 570, 1796, 7, 338, 2034, 1018, 53, 122, 1, 1811, 189, 407, 500, 736, 245, 154, 654, 463, 432, 326, 1484, 390, 1320, 283, 1233, 1187, 534, 550, 204, 3437, 554, 34, 509, 230, 365, 478, 1121, 517, 127, 1418] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=22 tokens_per_expert_current_ep_rank=[131, 82, 291, 1167, 199, 28, 669, 75, 4657, 15, 705, 360, 202, 434, 381, 581, 854, 94, 1083, 536, 64, 1826, 3464, 653, 409, 2003, 11, 188, 385, 2706, 12, 696, 109, 3075, 1675, 553, 417, 1155, 1482, 614, 535, 172, 250, 1083, 19, 182, 549, 1141, 437, 1239, 729, 1048, 41, 1313, 879, 6, 263, 155, 346, 402, 692, 1483, 173, 1974] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=22 tokens_per_expert_current_ep_rank=[106, 53, 381, 686, 549, 139, 2408, 440, 969, 127, 1426, 490, 1688, 615, 668, 586, 568, 367, 917, 488, 235, 1217, 2866, 1496, 416, 169, 306, 50, 1243, 1695, 7, 556, 30, 6703, 769, 146, 79, 273, 962, 1563, 1030, 1912, 63, 106, 201, 81, 756, 261, 147, 1147, 490, 624, 197, 829, 37, 102, 962, 1693, 219, 292, 1047, 571, 1297, 636] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=22 rerouted_tokens_per_expert_current_ep_rank=[359, 315, 120, 631, 455, 46, 1024, 604, 1223, 377, 0, 0, 0, 0, 0, 0, 2021, 35, 444, 578, 36, 358, 1828, 2205, 0, 0, 0, 0, 0, 0, 0, 0, 36, 554, 548, 1032, 490, 916, 0, 2266, 0, 0, 0, 0, 0, 0, 0, 0, 441, 1682, 96, 664, 2140, 1424, 413, 215, 837, 0, 0, 0, 0, 0, 0, 0, 96, 1184, 116, 523, 178, 449, 622, 148, 0, 0, 0, 0, 0, 0, 0, 0, 336, 743, 424, 437, 49, 233, 67, 505, 3249, 0, 0, 0, 0, 0, 0, 0, 956, 506, 251, 356, 21, 1341, 333, 3, 1243, 914, 0, 0, 0, 0, 0, 0, 1636, 146, 161, 1790, 766, 869, 275, 246, 1539, 28, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5154, 7505, 5842, 7912, 3316, 6043, 5924, 7456] avg=6144.0000 max=7912 max_rank=3 min=3316 min_rank=4 imbalance_ratio=1.287760 max_load_reduction=41240 max_load_reduction_pct=0.839030 moved_to_echo=22739 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=22 rerouted_tokens_per_expert_current_ep_rank=[228, 109, 584, 536, 401, 487, 1613, 643, 934, 181, 0, 0, 0, 0, 0, 0, 971, 147, 77, 234, 999, 2417, 801, 558, 0, 0, 0, 0, 0, 0, 0, 0, 695, 1124, 857, 480, 126, 565, 0, 1416, 0, 0, 0, 0, 0, 0, 0, 0, 58, 1565, 305, 175, 1992, 1189, 79, 533, 515, 0, 0, 0, 0, 0, 0, 0, 115, 905, 1198, 398, 176, 1150, 3301, 868, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1622, 57, 255, 33, 965, 388, 638, 1825, 0, 0, 0, 0, 0, 0, 0, 87, 808, 434, 288, 348, 804, 446, 824, 699, 698, 0, 0, 0, 0, 0, 0, 1807, 605, 209, 206, 489, 374, 228, 551, 739, 17, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5716, 6204, 5263, 6411, 8111, 6786, 5436, 5225] avg=6144.0000 max=8111 max_rank=4 min=5225 min_rank=7 imbalance_ratio=1.320150 max_load_reduction=41041 max_load_reduction_pct=0.834981 moved_to_echo=25558 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=22 rerouted_tokens_per_expert_current_ep_rank=[131, 82, 291, 1167, 199, 28, 669, 75, 1133, 447, 0, 0, 0, 0, 0, 0, 2389, 15, 705, 360, 202, 434, 381, 581, 0, 0, 0, 0, 0, 0, 0, 0, 854, 94, 1083, 536, 64, 939, 0, 653, 0, 0, 0, 0, 0, 0, 0, 0, 409, 2003, 11, 188, 385, 2706, 12, 696, 858, 0, 0, 0, 0, 0, 0, 0, 109, 1096, 1675, 553, 417, 1155, 1482, 614, 0, 0, 0, 0, 0, 0, 0, 0, 535, 172, 250, 1083, 19, 182, 549, 1141, 2505, 0, 0, 0, 0, 0, 0, 0, 437, 1239, 729, 1048, 41, 1313, 879, 6, 959, 846, 0, 0, 0, 0, 0, 0, 263, 155, 346, 402, 692, 1483, 173, 1974, 1821, 29, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4222, 5067, 4223, 7268, 7101, 6436, 7497, 7338] avg=6144.0000 max=7497 max_rank=6 min=4222 min_rank=0 imbalance_ratio=1.220215 max_load_reduction=41655 max_load_reduction_pct=0.847473 moved_to_echo=28372 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=22 rerouted_tokens_per_expert_current_ep_rank=[440, 332, 14, 133, 821, 56, 1924, 165, 1317, 177, 0, 0, 0, 0, 0, 0, 947, 69, 2276, 132, 79, 505, 17, 1443, 0, 0, 0, 0, 0, 0, 0, 0, 544, 2416, 3536, 98, 25, 305, 0, 1604, 0, 0, 0, 0, 0, 0, 0, 0, 123, 875, 64, 29, 868, 1576, 5, 376, 278, 0, 0, 0, 0, 0, 0, 0, 107, 1274, 1757, 1022, 164, 496, 3840, 1266, 0, 0, 0, 0, 0, 0, 0, 0, 137, 252, 21, 31, 10, 113, 42, 199, 1963, 0, 0, 0, 0, 0, 0, 0, 343, 1362, 39, 977, 44, 1192, 197, 69, 750, 984, 0, 0, 0, 0, 0, 0, 138, 1175, 222, 1567, 63, 1408, 311, 1318, 721, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5379, 5468, 8528, 4194, 9926, 2768, 5957, 6932] avg=6144.0000 max=9926 max_rank=4 min=2768 min_rank=5 imbalance_ratio=1.615560 max_load_reduction=39226 max_load_reduction_pct=0.798055 moved_to_echo=25583 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=22 rerouted_tokens_per_expert_current_ep_rank=[297, 231, 509, 406, 1335, 1636, 2667, 1577, 1540, 370, 0, 0, 0, 0, 0, 0, 1983, 0, 195, 570, 1120, 611, 801, 1419, 0, 0, 0, 0, 0, 0, 0, 0, 1521, 2114, 308, 1142, 107, 639, 0, 1173, 0, 0, 0, 0, 0, 0, 0, 0, 250, 842, 4, 756, 128, 1739, 6, 605, 583, 0, 0, 0, 0, 0, 0, 0, 35, 1490, 759, 227, 198, 1935, 316, 483, 0, 0, 0, 0, 0, 0, 0, 0, 640, 369, 108, 19, 418, 2236, 168, 327, 652, 0, 0, 0, 0, 0, 0, 0, 169, 259, 47, 1154, 97, 1459, 42, 13, 249, 1151, 0, 0, 0, 0, 0, 0, 582, 437, 790, 280, 37, 1181, 19, 92, 1510, 20, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10568, 6699, 7004, 4913, 5443, 4937, 4640, 4948] avg=6144.0000 max=10568 max_rank=0 min=4640 min_rank=6 imbalance_ratio=1.720052 max_load_reduction=38584 max_load_reduction_pct=0.784993 moved_to_echo=19968 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=22 rerouted_tokens_per_expert_current_ep_rank=[558, 81, 38, 596, 40, 19, 1177, 2150, 108, 328, 0, 0, 0, 0, 0, 0, 1758, 267, 25, 161, 26, 510, 1125, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1177, 4735, 1405, 1273, 111, 2240, 0, 249, 0, 0, 0, 0, 0, 0, 0, 0, 970, 1206, 0, 126, 1172, 766, 11, 167, 2050, 0, 0, 0, 0, 0, 0, 0, 159, 106, 1357, 122, 631, 583, 2992, 224, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1354, 80, 242, 11, 276, 73, 560, 3408, 0, 0, 0, 0, 0, 0, 0, 190, 905, 292, 342, 29, 277, 556, 19, 1303, 81, 0, 0, 0, 0, 0, 0, 245, 34, 230, 210, 51, 1540, 95, 498, 1340, 70, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5095, 4866, 11190, 6468, 6174, 7052, 3994, 4313] avg=6144.0000 max=11190 max_rank=2 min=3994 min_rank=6 imbalance_ratio=1.821289 max_load_reduction=37962 max_load_reduction_pct=0.772339 moved_to_echo=21533 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=22 rerouted_tokens_per_expert_current_ep_rank=[232, 136, 394, 787, 1588, 129, 1398, 105, 671, 253, 0, 0, 0, 0, 0, 0, 1325, 414, 1291, 504, 363, 511, 2466, 372, 0, 0, 0, 0, 0, 0, 0, 0, 213, 278, 385, 311, 76, 1118, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 570, 1796, 7, 338, 2034, 1018, 53, 122, 1034, 0, 0, 0, 0, 0, 0, 0, 1, 639, 189, 407, 500, 736, 245, 154, 0, 0, 0, 0, 0, 0, 0, 0, 654, 463, 432, 326, 1484, 390, 1320, 283, 3295, 0, 0, 0, 0, 0, 0, 0, 1233, 1187, 534, 550, 204, 3437, 554, 34, 1259, 501, 0, 0, 0, 0, 0, 0, 509, 230, 365, 478, 1121, 517, 127, 1418, 1020, 40, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5693, 7246, 2405, 6972, 2871, 8647, 9493, 5825] avg=6144.0000 max=9493 max_rank=6 min=2405 min_rank=2 imbalance_ratio=1.545085 max_load_reduction=39659 max_load_reduction_pct=0.806864 moved_to_echo=26836 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=22 rerouted_tokens_per_expert_current_ep_rank=[106, 53, 381, 686, 549, 139, 2408, 440, 2470, 93, 0, 0, 0, 0, 0, 0, 497, 127, 1426, 490, 1688, 615, 668, 586, 0, 0, 0, 0, 0, 0, 0, 0, 568, 367, 917, 488, 235, 626, 0, 1496, 0, 0, 0, 0, 0, 0, 0, 0, 416, 169, 306, 50, 1243, 1695, 7, 556, 572, 0, 0, 0, 0, 0, 0, 0, 30, 2388, 769, 146, 79, 273, 962, 1563, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1912, 63, 106, 201, 81, 756, 261, 2073, 0, 0, 0, 0, 0, 0, 0, 147, 1147, 490, 624, 197, 829, 37, 102, 793, 1845, 0, 0, 0, 0, 0, 0, 962, 1693, 219, 292, 1047, 571, 1297, 636, 379, 19, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7325, 6097, 4697, 5014, 6210, 6483, 6211, 7115] avg=6144.0000 max=7325 max_rank=0 min=4697 min_rank=2 imbalance_ratio=1.192220 max_load_reduction=41827 max_load_reduction_pct=0.850972 moved_to_echo=26019 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[5632, 11776, 9088, 5376, 1280, 7424, 0, 8960, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[11904, 1152, 6528, 3072, 4608, 6016, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[6144, 4480, 2560, 5248, 4352, 8064, 2560, 6784, 9088, 256, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5504, 6912, 1536, 2560, 2304, 4480, 3456, 3968, 19072, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[768, 9088, 7936, 3456, 2432, 6784, 13824, 5376, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3328, 10240, 896, 2432, 9984, 12160, 640, 3328, 6784, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2432, 1408, 2432, 4992, 5504, 2560, 12928, 5760, 9472, 2304, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[3584, 7424, 2816, 5376, 1024, 10752, 3072, 1152, 7296, 7040, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=23 tokens_per_expert_current_ep_rank=[1632, 42, 863, 1089, 847, 36, 78, 346, 22, 573, 72, 1043, 2324, 2092, 169, 193, 1024, 1632, 588, 863, 2818, 226, 4, 254, 1572, 874, 25, 252, 13, 2574, 495, 398, 982, 35, 56, 1260, 228, 735, 896, 264, 92, 519, 1024, 371, 812, 225, 1383, 620, 80, 772, 63, 275, 192, 1009, 1463, 2879, 200, 564, 10, 3898, 534, 551, 1211, 916] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=23 tokens_per_expert_current_ep_rank=[272, 15, 204, 1029, 1686, 184, 178, 247, 157, 688, 141, 1660, 902, 175, 61, 415, 1208, 367, 680, 287, 420, 42, 37, 1528, 1266, 1875, 31, 496, 45, 2202, 363, 2808, 77, 39, 56, 1401, 56, 517, 211, 674, 1001, 1060, 305, 230, 1907, 209, 2628, 1570, 269, 725, 78, 2404, 9, 2736, 1875, 2406, 269, 162, 41, 309, 585, 662, 333, 2679] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=23 tokens_per_expert_current_ep_rank=[2384, 209, 2204, 484, 1765, 376, 255, 554, 147, 911, 522, 528, 1691, 1546, 104, 175, 379, 954, 1729, 866, 62, 139, 695, 518, 1250, 1494, 37, 1411, 283, 847, 511, 583, 984, 570, 1994, 926, 189, 246, 251, 174, 45, 78, 1946, 281, 2221, 432, 1102, 406, 10, 46, 31, 2856, 485, 1201, 2323, 1621, 77, 392, 95, 108, 729, 504, 721, 495] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=23 tokens_per_expert_current_ep_rank=[153, 84, 3422, 892, 1914, 36, 100, 2189, 60, 550, 27, 319, 2226, 1602, 70, 438, 729, 1087, 4533, 2120, 632, 74, 505, 544, 641, 778, 593, 647, 195, 399, 1749, 2846, 90, 448, 645, 603, 47, 2003, 245, 27, 83, 95, 1477, 365, 476, 37, 120, 125, 15, 636, 97, 1621, 423, 642, 1076, 1065, 334, 979, 3, 741, 33, 152, 1132, 1163] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=23 tokens_per_expert_current_ep_rank=[1091, 256, 412, 2243, 1073, 315, 351, 1048, 101, 78, 15, 1284, 357, 1959, 2, 1466, 2625, 2226, 616, 1153, 347, 213, 78, 991, 146, 1295, 109, 228, 16, 1748, 1305, 1418, 361, 1268, 108, 901, 106, 844, 549, 63, 26, 103, 256, 1971, 1550, 21, 483, 159, 68, 371, 67, 2193, 2, 2522, 773, 1100, 424, 983, 918, 111, 37, 1666, 2243, 340] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=23 tokens_per_expert_current_ep_rank=[1276, 236, 734, 969, 1443, 75, 174, 744, 213, 647, 83, 1438, 1676, 2098, 20, 646, 63, 1500, 1721, 463, 890, 56, 159, 1292, 1150, 3088, 10, 1954, 56, 783, 136, 1056, 24, 411, 144, 688, 979, 638, 55, 50, 715, 111, 564, 1033, 1164, 740, 1867, 962, 107, 213, 633, 676, 750, 2548, 774, 1655, 133, 83, 39, 765, 48, 738, 2150, 846] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=23 tokens_per_expert_current_ep_rank=[924, 281, 665, 2179, 639, 505, 203, 168, 2, 857, 41, 974, 1171, 676, 16, 658, 290, 1714, 4141, 503, 219, 425, 2, 2589, 907, 2159, 633, 290, 5, 1814, 609, 1030, 24, 67, 37, 155, 553, 1584, 114, 1056, 708, 971, 91, 596, 454, 164, 489, 50, 43, 553, 310, 778, 26, 705, 849, 3464, 202, 872, 13, 873, 247, 1412, 3562, 841] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=23 tokens_per_expert_current_ep_rank=[1767, 713, 1359, 419, 2341, 202, 57, 1415, 170, 1801, 41, 724, 1227, 1076, 752, 142, 764, 1683, 551, 546, 793, 65, 73, 4125, 958, 786, 440, 273, 86, 2112, 196, 1876, 196, 31, 687, 459, 0, 1777, 638, 1228, 1187, 189, 961, 86, 294, 513, 811, 582, 191, 749, 57, 371, 51, 1688, 207, 2952, 551, 348, 181, 293, 101, 403, 1423, 414] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=23 rerouted_tokens_per_expert_current_ep_rank=[1632, 42, 863, 1089, 638, 36, 78, 346, 0, 0, 0, 0, 0, 0, 0, 0, 22, 573, 72, 1043, 2324, 2092, 169, 193, 209, 462, 0, 0, 0, 0, 0, 0, 1024, 1632, 133, 863, 2818, 226, 4, 254, 0, 0, 0, 0, 0, 0, 0, 0, 1572, 874, 25, 252, 13, 704, 495, 398, 0, 0, 0, 0, 0, 0, 0, 0, 982, 35, 56, 1260, 228, 735, 896, 264, 455, 866, 0, 0, 0, 0, 0, 0, 92, 519, 1024, 371, 812, 225, 1383, 620, 758, 313, 0, 0, 0, 0, 0, 0, 80, 772, 63, 275, 192, 1009, 1463, 1255, 0, 0, 0, 0, 0, 0, 0, 0, 200, 564, 10, 3898, 534, 551, 1211, 916, 1095, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4724, 7159, 6954, 4333, 5777, 6117, 5109, 8979] avg=6144.0000 max=8979 max_rank=7 min=4333 min_rank=3 imbalance_ratio=1.461426 max_load_reduction=40173 max_load_reduction_pct=0.817322 moved_to_echo=25982 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=23 rerouted_tokens_per_expert_current_ep_rank=[153, 84, 3422, 892, 1441, 36, 100, 2189, 0, 0, 0, 0, 0, 0, 0, 0, 60, 550, 27, 319, 2226, 1602, 70, 438, 473, 71, 0, 0, 0, 0, 0, 0, 729, 1087, 1025, 2120, 632, 74, 505, 544, 0, 0, 0, 0, 0, 0, 0, 0, 641, 778, 593, 647, 195, 111, 1749, 2846, 0, 0, 0, 0, 0, 0, 0, 0, 90, 448, 645, 603, 47, 2003, 245, 27, 3508, 320, 0, 0, 0, 0, 0, 0, 83, 95, 1477, 365, 476, 37, 120, 125, 280, 48, 0, 0, 0, 0, 0, 0, 15, 636, 97, 1621, 423, 642, 1076, 465, 0, 0, 0, 0, 0, 0, 0, 0, 334, 979, 3, 741, 33, 152, 1132, 1163, 169, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8317, 5836, 6716, 7560, 7936, 3106, 4975, 4706] avg=6144.0000 max=8317 max_rank=0 min=3106 min_rank=5 imbalance_ratio=1.353678 max_load_reduction=40835 max_load_reduction_pct=0.830790 moved_to_echo=20723 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=23 rerouted_tokens_per_expert_current_ep_rank=[272, 15, 204, 1029, 1270, 184, 178, 247, 0, 0, 0, 0, 0, 0, 0, 0, 157, 688, 141, 1660, 902, 175, 61, 415, 416, 395, 0, 0, 0, 0, 0, 0, 1208, 367, 154, 287, 420, 42, 37, 1528, 0, 0, 0, 0, 0, 0, 0, 0, 1266, 1875, 31, 496, 45, 603, 363, 2808, 0, 0, 0, 0, 0, 0, 0, 0, 77, 39, 56, 1401, 56, 517, 211, 674, 526, 724, 0, 0, 0, 0, 0, 0, 1001, 1060, 305, 230, 1907, 209, 2628, 1570, 633, 268, 0, 0, 0, 0, 0, 0, 269, 725, 78, 2404, 9, 2736, 1875, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 269, 162, 41, 309, 585, 662, 333, 2679, 936, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3399, 5010, 4043, 7487, 4281, 9811, 9145, 5976] avg=6144.0000 max=9811 max_rank=5 min=3399 min_rank=0 imbalance_ratio=1.596842 max_load_reduction=39341 max_load_reduction_pct=0.800395 moved_to_echo=29213 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=23 rerouted_tokens_per_expert_current_ep_rank=[2384, 209, 2204, 484, 1325, 376, 255, 554, 0, 0, 0, 0, 0, 0, 0, 0, 147, 911, 522, 528, 1691, 1546, 104, 175, 440, 157, 0, 0, 0, 0, 0, 0, 379, 954, 387, 866, 62, 139, 695, 518, 0, 0, 0, 0, 0, 0, 0, 0, 1250, 1494, 37, 1411, 283, 219, 511, 583, 0, 0, 0, 0, 0, 0, 0, 0, 984, 570, 1994, 926, 189, 246, 251, 174, 1342, 491, 0, 0, 0, 0, 0, 0, 45, 78, 1946, 281, 2221, 432, 1102, 406, 432, 106, 0, 0, 0, 0, 0, 0, 10, 46, 31, 2856, 485, 1201, 2323, 698, 0, 0, 0, 0, 0, 0, 0, 0, 77, 392, 95, 108, 729, 504, 721, 495, 365, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7791, 6221, 4000, 5788, 7167, 7049, 7650, 3486] avg=6144.0000 max=7791 max_rank=0 min=3486 min_rank=7 imbalance_ratio=1.268066 max_load_reduction=41361 max_load_reduction_pct=0.841492 moved_to_echo=25352 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=23 rerouted_tokens_per_expert_current_ep_rank=[1091, 256, 412, 2243, 808, 315, 351, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 101, 78, 15, 1284, 357, 1959, 2, 1466, 265, 314, 0, 0, 0, 0, 0, 0, 2625, 2226, 140, 1153, 347, 213, 78, 991, 0, 0, 0, 0, 0, 0, 0, 0, 146, 1295, 109, 228, 16, 479, 1305, 1418, 0, 0, 0, 0, 0, 0, 0, 0, 361, 1268, 108, 901, 106, 844, 549, 63, 476, 331, 0, 0, 0, 0, 0, 0, 26, 103, 256, 1971, 1550, 21, 483, 159, 289, 212, 0, 0, 0, 0, 0, 0, 68, 371, 67, 2193, 2, 2522, 773, 480, 0, 0, 0, 0, 0, 0, 0, 0, 424, 983, 918, 111, 37, 1666, 2243, 340, 743, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6524, 5841, 7773, 4996, 5007, 5070, 6476, 7465] avg=6144.0000 max=7773 max_rank=2 min=4996 min_rank=3 imbalance_ratio=1.265137 max_load_reduction=41379 max_load_reduction_pct=0.841858 moved_to_echo=24018 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=23 rerouted_tokens_per_expert_current_ep_rank=[1276, 236, 734, 969, 1087, 75, 174, 744, 0, 0, 0, 0, 0, 0, 0, 0, 213, 647, 83, 1438, 1676, 2098, 20, 646, 356, 140, 0, 0, 0, 0, 0, 0, 63, 1500, 390, 463, 890, 56, 159, 1292, 0, 0, 0, 0, 0, 0, 0, 0, 1150, 3088, 10, 1954, 56, 215, 136, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 24, 411, 144, 688, 979, 638, 55, 50, 1331, 498, 0, 0, 0, 0, 0, 0, 715, 111, 564, 1033, 1164, 740, 1867, 962, 436, 95, 0, 0, 0, 0, 0, 0, 107, 213, 633, 676, 750, 2548, 774, 721, 0, 0, 0, 0, 0, 0, 0, 0, 133, 83, 39, 765, 48, 738, 2150, 846, 333, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5295, 7317, 4813, 7665, 4818, 7687, 6422, 5135] avg=6144.0000 max=7687 max_rank=5 min=4813 min_rank=2 imbalance_ratio=1.251139 max_load_reduction=41465 max_load_reduction_pct=0.843608 moved_to_echo=24062 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=23 rerouted_tokens_per_expert_current_ep_rank=[924, 281, 665, 2179, 482, 505, 203, 168, 0, 0, 0, 0, 0, 0, 0, 0, 2, 857, 41, 974, 1171, 676, 16, 658, 157, 326, 0, 0, 0, 0, 0, 0, 290, 1714, 937, 503, 219, 425, 2, 2589, 0, 0, 0, 0, 0, 0, 0, 0, 907, 2159, 633, 290, 5, 497, 609, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 24, 67, 37, 155, 553, 1584, 114, 1056, 3204, 1042, 0, 0, 0, 0, 0, 0, 708, 971, 91, 596, 454, 164, 489, 50, 912, 220, 0, 0, 0, 0, 0, 0, 43, 553, 310, 778, 26, 705, 849, 1510, 0, 0, 0, 0, 0, 0, 0, 0, 202, 872, 13, 873, 247, 1412, 3562, 841, 771, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5407, 4878, 6679, 6130, 7836, 4655, 4774, 8793] avg=6144.0000 max=8793 max_rank=7 min=4655 min_rank=5 imbalance_ratio=1.431152 max_load_reduction=40359 max_load_reduction_pct=0.821106 moved_to_echo=26058 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=23 rerouted_tokens_per_expert_current_ep_rank=[1767, 713, 1359, 419, 1763, 202, 57, 1415, 0, 0, 0, 0, 0, 0, 0, 0, 170, 1801, 41, 724, 1227, 1076, 752, 142, 578, 379, 0, 0, 0, 0, 0, 0, 764, 1683, 125, 546, 793, 65, 73, 4125, 0, 0, 0, 0, 0, 0, 0, 0, 958, 786, 440, 273, 86, 578, 196, 1876, 0, 0, 0, 0, 0, 0, 0, 0, 196, 31, 687, 459, 0, 1777, 638, 1228, 426, 888, 0, 0, 0, 0, 0, 0, 1187, 189, 961, 86, 294, 513, 811, 582, 777, 257, 0, 0, 0, 0, 0, 0, 191, 749, 57, 371, 51, 1688, 207, 1287, 0, 0, 0, 0, 0, 0, 0, 0, 551, 348, 181, 293, 101, 403, 1423, 414, 898, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7695, 6890, 8174, 5193, 6330, 5657, 4601, 4612] avg=6144.0000 max=8174 max_rank=2 min=4601 min_rank=6 imbalance_ratio=1.330404 max_load_reduction=40978 max_load_reduction_pct=0.833700 moved_to_echo=21200 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3968, 3200, 6656, 4992, 8960, 2432, 8960, 4480, 4608, 1536, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[2304, 4480, 1408, 7168, 2432, 6144, 12800, 7808, 5376, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[896, 6144, 1024, 8064, 11648, 11264, 1280, 4224, 2944, 2304, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[2816, 2944, 3840, 6400, 2176, 8448, 3072, 3584, 11392, 5248, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[7168, 11264, 3328, 6912, 6272, 1280, 1664, 11904, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[7936, 12416, 1920, 5632, 768, 3456, 5376, 12032, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[896, 4096, 1408, 11264, 2048, 13056, 9344, 7552, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[9600, 1920, 9984, 9344, 8832, 1792, 1408, 6784, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=24 tokens_per_expert_current_ep_rank=[434, 174, 184, 8, 1538, 49, 136, 160, 1634, 299, 2771, 1851, 535, 668, 297, 405, 405, 1972, 436, 245, 394, 1508, 254, 326, 274, 1916, 625, 404, 117, 1073, 1262, 1659, 65, 79, 182, 2114, 3291, 10, 526, 2469, 1181, 900, 19, 268, 295, 3519, 1775, 63, 574, 1627, 772, 439, 614, 162, 506, 15, 109, 6, 289, 801, 455, 11, 112, 1891] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=24 tokens_per_expert_current_ep_rank=[322, 230, 511, 19, 1198, 107, 136, 1439, 1069, 1226, 207, 1431, 723, 601, 121, 1045, 937, 1594, 766, 192, 164, 2698, 782, 249, 28, 2079, 521, 362, 1006, 1274, 1620, 891, 59, 856, 60, 969, 2633, 66, 286, 1888, 2486, 895, 24, 1679, 63, 2350, 1347, 103, 186, 1026, 1249, 278, 150, 395, 1048, 168, 14, 377, 560, 506, 1186, 0, 372, 325] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=24 tokens_per_expert_current_ep_rank=[147, 1790, 1657, 148, 877, 189, 369, 133, 652, 386, 357, 2044, 991, 2488, 145, 1613, 255, 1647, 186, 2187, 724, 716, 1385, 11, 1095, 744, 602, 591, 2446, 736, 3458, 2858, 51, 654, 242, 218, 2418, 244, 126, 823, 1590, 194, 4, 38, 90, 2370, 270, 15, 907, 993, 350, 440, 78, 271, 306, 98, 1181, 391, 35, 136, 524, 132, 91, 245] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=24 tokens_per_expert_current_ep_rank=[171, 181, 741, 1047, 2152, 503, 406, 183, 650, 1497, 189, 681, 1133, 716, 473, 859, 6, 2289, 3452, 400, 1781, 91, 1194, 3, 140, 883, 593, 476, 2568, 1906, 2195, 733, 84, 244, 497, 926, 2067, 11, 316, 2632, 315, 359, 40, 112, 545, 948, 2805, 236, 526, 289, 119, 149, 415, 178, 473, 1739, 462, 165, 476, 450, 505, 29, 404, 344] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=24 tokens_per_expert_current_ep_rank=[516, 56, 824, 11, 935, 46, 248, 888, 2282, 39, 1388, 900, 110, 23, 1266, 425, 450, 134, 1002, 504, 415, 575, 373, 6, 150, 1468, 961, 431, 8, 473, 2733, 1973, 10, 205, 2, 393, 4757, 157, 14, 1056, 893, 1555, 40, 1502, 488, 3191, 2629, 1912, 1212, 53, 1449, 261, 217, 953, 938, 24, 147, 144, 365, 1739, 735, 17, 238, 243] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=24 tokens_per_expert_current_ep_rank=[1832, 290, 474, 313, 1911, 249, 232, 242, 101, 74, 217, 1570, 116, 313, 382, 907, 3573, 3364, 364, 3577, 810, 712, 1259, 267, 669, 415, 113, 287, 83, 32, 1501, 3316, 24, 348, 329, 136, 2197, 178, 128, 1280, 70, 436, 58, 620, 395, 3301, 622, 15, 1722, 441, 467, 318, 192, 1636, 1421, 655, 442, 7, 85, 811, 964, 122, 28, 139] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=24 tokens_per_expert_current_ep_rank=[1020, 241, 270, 12, 65, 11, 527, 976, 519, 151, 1639, 1378, 1391, 3106, 1526, 654, 1875, 2030, 474, 492, 2193, 1038, 276, 805, 169, 923, 28, 343, 3, 840, 1564, 173, 117, 614, 996, 127, 1064, 26, 730, 3001, 224, 1742, 4, 921, 1812, 1160, 17, 48, 54, 2380, 981, 161, 81, 491, 76, 4, 275, 1412, 113, 215, 2549, 19, 44, 982] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=24 tokens_per_expert_current_ep_rank=[59, 102, 870, 15, 2290, 24, 416, 211, 674, 324, 1114, 1748, 203, 120, 652, 349, 370, 403, 561, 102, 1672, 156, 2030, 315, 611, 366, 277, 353, 576, 1143, 2322, 2637, 0, 290, 21, 2124, 3896, 32, 1826, 209, 2174, 188, 25, 768, 198, 1301, 999, 474, 706, 938, 877, 65, 2583, 1395, 1248, 95, 93, 89, 749, 846, 87, 85, 1470, 236] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=24 rerouted_tokens_per_expert_current_ep_rank=[147, 1790, 1657, 148, 877, 189, 369, 133, 983, 750, 350, 0, 0, 0, 0, 0, 652, 386, 357, 941, 991, 2488, 145, 1613, 0, 0, 0, 0, 0, 0, 0, 0, 255, 143, 186, 2187, 724, 716, 1385, 11, 0, 0, 0, 0, 0, 0, 0, 0, 1095, 744, 602, 591, 2446, 736, 360, 2858, 0, 0, 0, 0, 0, 0, 0, 0, 51, 654, 242, 218, 1959, 244, 126, 823, 0, 0, 0, 0, 0, 0, 0, 0, 1590, 194, 4, 38, 90, 1387, 270, 15, 0, 0, 0, 0, 0, 0, 0, 0, 907, 993, 350, 440, 78, 271, 306, 98, 753, 459, 0, 0, 0, 0, 0, 0, 1181, 391, 35, 136, 524, 132, 91, 245, 3098, 754, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7393, 7573, 5607, 9432, 4317, 3588, 4655, 6587] avg=6144.0000 max=9432 max_rank=3 min=3588 min_rank=5 imbalance_ratio=1.535156 max_load_reduction=39720 max_load_reduction_pct=0.808105 moved_to_echo=19147 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=24 rerouted_tokens_per_expert_current_ep_rank=[434, 174, 184, 8, 1538, 49, 136, 160, 1460, 898, 317, 0, 0, 0, 0, 0, 1634, 299, 2771, 852, 535, 668, 297, 405, 0, 0, 0, 0, 0, 0, 0, 0, 405, 171, 436, 245, 394, 1508, 254, 326, 0, 0, 0, 0, 0, 0, 0, 0, 274, 1916, 625, 404, 117, 1073, 132, 1659, 0, 0, 0, 0, 0, 0, 0, 0, 65, 79, 182, 2114, 2666, 10, 526, 2469, 0, 0, 0, 0, 0, 0, 0, 0, 1181, 900, 19, 268, 295, 2059, 1775, 63, 0, 0, 0, 0, 0, 0, 0, 0, 574, 1627, 772, 439, 614, 162, 506, 15, 682, 625, 0, 0, 0, 0, 0, 0, 109, 6, 289, 801, 455, 11, 112, 1891, 1130, 903, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5358, 7461, 3739, 6200, 8111, 6560, 6016, 5707] avg=6144.0000 max=8111 max_rank=4 min=3739 min_rank=2 imbalance_ratio=1.320150 max_load_reduction=41041 max_load_reduction_pct=0.834981 moved_to_echo=26394 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=24 rerouted_tokens_per_expert_current_ep_rank=[516, 56, 824, 11, 935, 46, 248, 888, 1324, 61, 154, 0, 0, 0, 0, 0, 2282, 39, 1388, 415, 110, 23, 1266, 425, 0, 0, 0, 0, 0, 0, 0, 0, 450, 12, 1002, 504, 415, 575, 373, 6, 0, 0, 0, 0, 0, 0, 0, 0, 150, 1468, 961, 431, 8, 473, 285, 1973, 0, 0, 0, 0, 0, 0, 0, 0, 10, 205, 2, 393, 3854, 157, 14, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 893, 1555, 40, 1502, 488, 1867, 2629, 1912, 0, 0, 0, 0, 0, 0, 0, 0, 1212, 53, 1449, 261, 217, 953, 938, 24, 331, 903, 0, 0, 0, 0, 0, 0, 147, 144, 365, 1739, 735, 17, 238, 243, 2448, 61, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5063, 5948, 3337, 5749, 5691, 10886, 6341, 6137] avg=6144.0000 max=10886 max_rank=5 min=3337 min_rank=2 imbalance_ratio=1.771810 max_load_reduction=38266 max_load_reduction_pct=0.778524 moved_to_echo=29055 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=24 rerouted_tokens_per_expert_current_ep_rank=[322, 230, 511, 19, 1198, 107, 136, 1439, 975, 726, 245, 0, 0, 0, 0, 0, 1069, 1226, 207, 659, 723, 601, 121, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 937, 138, 766, 192, 164, 2698, 782, 249, 0, 0, 0, 0, 0, 0, 0, 0, 28, 2079, 521, 362, 1006, 1274, 169, 891, 0, 0, 0, 0, 0, 0, 0, 0, 59, 856, 60, 969, 2133, 66, 286, 1888, 0, 0, 0, 0, 0, 0, 0, 0, 2486, 895, 24, 1679, 63, 1375, 1347, 103, 0, 0, 0, 0, 0, 0, 0, 0, 186, 1026, 1249, 278, 150, 395, 1048, 168, 527, 500, 0, 0, 0, 0, 0, 0, 14, 377, 560, 506, 1186, 0, 372, 325, 1451, 730, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5908, 5651, 5926, 6330, 6317, 7972, 5527, 5521] avg=6144.0000 max=7972 max_rank=5 min=5521 min_rank=7 imbalance_ratio=1.297526 max_load_reduction=41180 max_load_reduction_pct=0.837809 moved_to_echo=25337 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=24 rerouted_tokens_per_expert_current_ep_rank=[1020, 241, 270, 12, 65, 11, 527, 976, 481, 924, 236, 0, 0, 0, 0, 0, 519, 151, 1639, 635, 1391, 3106, 1526, 654, 0, 0, 0, 0, 0, 0, 0, 0, 1875, 177, 474, 492, 2193, 1038, 276, 805, 0, 0, 0, 0, 0, 0, 0, 0, 169, 923, 28, 343, 3, 840, 163, 173, 0, 0, 0, 0, 0, 0, 0, 0, 117, 614, 996, 127, 862, 26, 730, 3001, 0, 0, 0, 0, 0, 0, 0, 0, 224, 1742, 4, 921, 1812, 679, 17, 48, 0, 0, 0, 0, 0, 0, 0, 0, 54, 2380, 981, 161, 81, 491, 76, 4, 507, 202, 0, 0, 0, 0, 0, 0, 275, 1412, 113, 215, 2549, 19, 44, 982, 1401, 929, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4763, 9621, 7330, 2642, 6473, 5447, 4937, 7939] avg=6144.0000 max=9621 max_rank=1 min=2642 min_rank=3 imbalance_ratio=1.565918 max_load_reduction=39531 max_load_reduction_pct=0.804260 moved_to_echo=24796 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=24 rerouted_tokens_per_expert_current_ep_rank=[1832, 290, 474, 313, 1911, 249, 232, 242, 1373, 1536, 272, 0, 0, 0, 0, 0, 101, 74, 217, 716, 116, 313, 382, 907, 0, 0, 0, 0, 0, 0, 0, 0, 3573, 283, 364, 3577, 810, 712, 1259, 267, 0, 0, 0, 0, 0, 0, 0, 0, 669, 415, 113, 287, 83, 32, 151, 3316, 0, 0, 0, 0, 0, 0, 0, 0, 24, 348, 329, 136, 1777, 178, 128, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 70, 436, 58, 620, 395, 1928, 622, 15, 0, 0, 0, 0, 0, 0, 0, 0, 1722, 441, 467, 318, 192, 1636, 1421, 655, 582, 420, 0, 0, 0, 0, 0, 0, 442, 7, 85, 811, 964, 122, 28, 139, 1350, 1545, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8724, 2826, 10845, 5066, 4200, 4144, 7854, 5493] avg=6144.0000 max=10845 max_rank=2 min=2826 min_rank=1 imbalance_ratio=1.765137 max_load_reduction=38307 max_load_reduction_pct=0.779358 moved_to_echo=21691 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=24 rerouted_tokens_per_expert_current_ep_rank=[171, 181, 741, 1047, 2152, 503, 406, 183, 393, 1042, 116, 0, 0, 0, 0, 0, 650, 1497, 189, 314, 1133, 716, 473, 859, 0, 0, 0, 0, 0, 0, 0, 0, 6, 199, 3452, 400, 1781, 91, 1194, 3, 0, 0, 0, 0, 0, 0, 0, 0, 140, 883, 593, 476, 2568, 1906, 229, 733, 0, 0, 0, 0, 0, 0, 0, 0, 84, 244, 497, 926, 1675, 11, 316, 2632, 0, 0, 0, 0, 0, 0, 0, 0, 315, 359, 40, 112, 545, 555, 2805, 236, 0, 0, 0, 0, 0, 0, 0, 0, 526, 289, 119, 149, 415, 178, 473, 1739, 251, 392, 0, 0, 0, 0, 0, 0, 462, 165, 476, 450, 505, 29, 404, 344, 1966, 1048, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6935, 5831, 7126, 7528, 6385, 4967, 4531, 5849] avg=6144.0000 max=7528 max_rank=3 min=4531 min_rank=6 imbalance_ratio=1.225260 max_load_reduction=41624 max_load_reduction_pct=0.846842 moved_to_echo=21732 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=24 rerouted_tokens_per_expert_current_ep_rank=[59, 102, 870, 15, 2290, 24, 416, 211, 539, 183, 299, 0, 0, 0, 0, 0, 674, 324, 1114, 805, 203, 120, 652, 349, 0, 0, 0, 0, 0, 0, 0, 0, 370, 36, 561, 102, 1672, 156, 2030, 315, 0, 0, 0, 0, 0, 0, 0, 0, 611, 366, 277, 353, 576, 1143, 242, 2637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 290, 21, 2124, 3156, 32, 1826, 209, 0, 0, 0, 0, 0, 0, 0, 0, 2174, 188, 25, 768, 198, 762, 999, 474, 0, 0, 0, 0, 0, 0, 0, 0, 706, 938, 877, 65, 2583, 1395, 1248, 95, 644, 740, 0, 0, 0, 0, 0, 0, 93, 89, 749, 846, 87, 85, 1470, 236, 2080, 184, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5008, 4241, 5242, 6205, 7658, 5588, 9291, 5919] avg=6144.0000 max=9291 max_rank=6 min=4241 min_rank=1 imbalance_ratio=1.512207 max_load_reduction=39861 max_load_reduction_pct=0.810974 moved_to_echo=28456 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[7680, 4096, 7936, 5376, 5248, 8064, 4864, 6272, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[4608, 3072, 5632, 1664, 11008, 1280, 2560, 4352, 7552, 6144, 2048, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[8960, 6272, 256, 6016, 3968, 10624, 10496, 2944, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[7936, 1280, 7296, 7808, 8192, 7552, 7680, 2048, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2816, 2688, 2688, 5504, 7040, 512, 2816, 4480, 14976, 6272, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[5888, 7808, 6272, 2176, 4352, 5504, 6016, 2816, 4352, 4352, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3200, 8832, 3840, 3328, 6912, 7552, 1792, 14336, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[512, 3328, 2432, 7040, 18176, 768, 3968, 13440, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.rerun_state_machine:Result validation enabled /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") - [2026-08-26 14:53:42] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 42135.0 | throughput per GPU (TFLOP/s/GPU): 3.0 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.092139E+01 | loss scale: 1.0 | grad norm: 30.620 | number of skipped iterations: 0 | number of nan iterations: 0 | -Number of parameters in transformer block in billions: 13.53 -Number of parameters in embedding layers in billions: 0.21 -Total number of parameters in billions: 13.74 -Number of parameters in most loaded shard in billions: 13.7356 -Theoretical memory footprints: weight and optimizer=98244.96 MB -[Rank 0] (after 1 iterations) memory (MB) | allocated: 40462.88671875 | max allocated: 115427.2763671875 | reserved: 117354.0 | max reserved: 117354.0 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1178, 975, 539, 742, 911, 595, 1579, 602, 601, 1101, 914, 551, 254, 1767, 621, 998, 666, 703, 718, 653, 533, 440, 1034, 860, 743, 873, 473, 1061, 853, 621, 923, 1031, 661, 680, 645, 590, 543, 412, 1000, 1177, 680, 613, 426, 665, 1453, 835, 667, 358, 711, 519, 735, 850, 1367, 976, 375, 592, 716, 509, 1106, 911, 704, 432, 792, 339] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1128, 1018, 531, 821, 911, 587, 1428, 610, 506, 1038, 1063, 577, 228, 1915, 670, 1029, 670, 545, 719, 711, 526, 490, 944, 838, 791, 844, 534, 993, 859, 695, 954, 959, 552, 598, 696, 613, 582, 443, 975, 1202, 717, 621, 506, 538, 1337, 895, 581, 415, 714, 562, 805, 801, 1411, 1120, 322, 602, 752, 527, 1126, 831, 720, 428, 753, 275] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1064, 1053, 564, 770, 623, 564, 1547, 571, 588, 962, 933, 535, 270, 1761, 730, 986, 641, 624, 624, 817, 519, 439, 948, 825, 726, 971, 478, 951, 855, 671, 947, 915, 577, 857, 678, 541, 523, 396, 1029, 1131, 700, 663, 493, 486, 1376, 868, 721, 428, 708, 523, 1096, 753, 1417, 935, 378, 677, 693, 563, 1305, 965, 704, 281, 862, 353] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1134, 902, 449, 782, 562, 622, 1492, 514, 530, 1127, 1047, 544, 256, 1684, 552, 1023, 626, 588, 615, 765, 555, 454, 956, 768, 775, 1040, 460, 912, 867, 647, 910, 1017, 552, 612, 610, 555, 510, 391, 881, 1308, 720, 710, 490, 554, 1458, 943, 722, 438, 710, 606, 1010, 807, 1437, 1078, 325, 767, 707, 604, 1370, 977, 708, 353, 770, 294] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1197, 852, 487, 795, 646, 538, 1453, 605, 627, 1191, 1074, 638, 249, 1723, 583, 1056, 706, 548, 663, 869, 531, 473, 971, 775, 730, 889, 487, 977, 833, 746, 807, 1044, 540, 851, 609, 615, 467, 378, 908, 1214, 730, 627, 516, 383, 1447, 1038, 625, 473, 877, 593, 813, 824, 1380, 835, 305, 597, 746, 592, 1283, 878, 652, 340, 801, 452] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1194, 901, 462, 748, 746, 601, 1560, 573, 615, 1079, 972, 627, 266, 1569, 656, 1145, 630, 684, 627, 805, 509, 422, 1013, 799, 727, 1086, 484, 1083, 935, 585, 892, 967, 531, 914, 648, 581, 476, 426, 958, 1227, 728, 635, 473, 453, 1435, 897, 626, 420, 721, 507, 837, 827, 1408, 921, 340, 709, 725, 640, 1153, 924, 629, 363, 753, 305] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1199, 984, 518, 796, 841, 597, 1402, 657, 512, 1097, 1013, 616, 245, 1618, 628, 1110, 607, 666, 600, 755, 472, 462, 927, 796, 762, 814, 488, 1188, 897, 662, 934, 972, 556, 778, 652, 563, 465, 402, 1031, 1138, 759, 756, 426, 532, 1279, 918, 594, 392, 738, 506, 799, 864, 1310, 1030, 387, 631, 747, 536, 1294, 927, 686, 342, 888, 391] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1145, 932, 521, 814, 1011, 599, 1530, 574, 612, 1109, 836, 622, 241, 1742, 673, 1086, 743, 667, 703, 717, 544, 377, 963, 796, 699, 989, 441, 916, 838, 588, 777, 1019, 518, 593, 620, 566, 468, 335, 891, 1255, 716, 631, 524, 708, 1409, 894, 754, 430, 736, 632, 953, 893, 1382, 971, 368, 680, 744, 568, 1085, 951, 553, 403, 747, 350] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1197, 852, 487, 795, 646, 538, 764, 605, 0, 0, 0, 0, 0, 0, 0, 0, 627, 1191, 1074, 638, 249, 968, 583, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 706, 548, 663, 869, 531, 473, 971, 775, 690, 0, 0, 0, 0, 0, 0, 0, 730, 889, 487, 560, 833, 746, 807, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 540, 851, 609, 615, 467, 378, 908, 1214, 477, 65, 0, 0, 0, 0, 0, 0, 730, 627, 516, 383, 1447, 1038, 625, 473, 237, 113, 0, 0, 0, 0, 0, 0, 877, 593, 813, 824, 1143, 835, 305, 597, 0, 0, 0, 0, 0, 0, 0, 0, 746, 592, 1283, 878, 652, 340, 801, 452, 304, 212, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5884, 6386, 6226, 6096, 6124, 6189, 5987, 6260] avg=6144.0000 max=6386 max_rank=1 min=5884 min_rank=0 imbalance_ratio=1.039388 max_load_reduction=42766 max_load_reduction_pct=0.870076 moved_to_echo=24560 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1178, 975, 539, 742, 911, 595, 830, 602, 0, 0, 0, 0, 0, 0, 0, 0, 601, 1101, 914, 551, 254, 994, 621, 998, 0, 0, 0, 0, 0, 0, 0, 0, 666, 703, 718, 653, 533, 440, 1034, 860, 707, 0, 0, 0, 0, 0, 0, 0, 743, 873, 473, 609, 853, 621, 923, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 661, 680, 645, 590, 543, 412, 1000, 1177, 518, 66, 0, 0, 0, 0, 0, 0, 680, 613, 426, 665, 1453, 835, 667, 358, 235, 122, 0, 0, 0, 0, 0, 0, 711, 519, 735, 850, 1132, 976, 375, 592, 0, 0, 0, 0, 0, 0, 0, 0, 716, 509, 1106, 911, 704, 432, 792, 339, 330, 231, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6372, 6034, 6314, 6126, 6292, 6054, 5890, 6070] avg=6144.0000 max=6372 max_rank=0 min=5890 min_rank=6 imbalance_ratio=1.037109 max_load_reduction=42780 max_load_reduction_pct=0.870361 moved_to_echo=24306 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1128, 1018, 531, 821, 911, 587, 751, 610, 0, 0, 0, 0, 0, 0, 0, 0, 506, 1038, 1063, 577, 228, 1077, 670, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 670, 545, 719, 711, 526, 490, 944, 838, 766, 0, 0, 0, 0, 0, 0, 0, 791, 844, 534, 569, 859, 695, 954, 959, 0, 0, 0, 0, 0, 0, 0, 0, 552, 598, 696, 613, 582, 443, 975, 1202, 468, 72, 0, 0, 0, 0, 0, 0, 717, 621, 506, 538, 1337, 895, 581, 415, 242, 115, 0, 0, 0, 0, 0, 0, 714, 562, 805, 801, 1169, 1120, 322, 602, 0, 0, 0, 0, 0, 0, 0, 0, 752, 527, 1126, 831, 720, 428, 753, 275, 309, 209, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6357, 6188, 6209, 6205, 6201, 5967, 6095, 5930] avg=6144.0000 max=6357 max_rank=0 min=5930 min_rank=7 imbalance_ratio=1.034668 max_load_reduction=42795 max_load_reduction_pct=0.870667 moved_to_echo=24193 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1064, 1053, 564, 770, 623, 564, 814, 571, 0, 0, 0, 0, 0, 0, 0, 0, 588, 962, 933, 535, 270, 990, 730, 986, 0, 0, 0, 0, 0, 0, 0, 0, 641, 624, 624, 817, 519, 439, 948, 825, 705, 0, 0, 0, 0, 0, 0, 0, 726, 971, 478, 545, 855, 671, 947, 915, 0, 0, 0, 0, 0, 0, 0, 0, 577, 857, 678, 541, 523, 396, 1029, 1131, 507, 66, 0, 0, 0, 0, 0, 0, 700, 663, 493, 486, 1376, 868, 721, 428, 243, 110, 0, 0, 0, 0, 0, 0, 708, 523, 1096, 753, 1174, 935, 378, 677, 0, 0, 0, 0, 0, 0, 0, 0, 693, 563, 1305, 965, 704, 281, 862, 353, 296, 226, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6023, 5994, 6142, 6108, 6305, 6088, 6244, 6248] avg=6144.0000 max=6305 max_rank=4 min=5994 min_rank=1 imbalance_ratio=1.026204 max_load_reduction=42847 max_load_reduction_pct=0.871724 moved_to_echo=24885 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1134, 902, 449, 782, 562, 622, 777, 514, 0, 0, 0, 0, 0, 0, 0, 0, 530, 1127, 1047, 544, 256, 939, 552, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 626, 588, 615, 765, 555, 454, 956, 768, 678, 0, 0, 0, 0, 0, 0, 0, 775, 1040, 460, 518, 867, 647, 910, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 552, 612, 610, 555, 510, 391, 881, 1308, 493, 67, 0, 0, 0, 0, 0, 0, 720, 710, 490, 554, 1458, 943, 722, 438, 250, 108, 0, 0, 0, 0, 0, 0, 710, 606, 1010, 807, 1187, 1078, 325, 767, 0, 0, 0, 0, 0, 0, 0, 0, 707, 604, 1370, 977, 708, 353, 770, 294, 286, 222, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5742, 6018, 6005, 6234, 5979, 6393, 6490, 6291] avg=6144.0000 max=6490 max_rank=6 min=5742 min_rank=0 imbalance_ratio=1.056315 max_load_reduction=42662 max_load_reduction_pct=0.867961 moved_to_echo=25153 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1145, 932, 521, 814, 1011, 599, 804, 574, 0, 0, 0, 0, 0, 0, 0, 0, 612, 1109, 836, 622, 241, 980, 673, 1086, 0, 0, 0, 0, 0, 0, 0, 0, 743, 667, 703, 717, 544, 377, 963, 796, 697, 0, 0, 0, 0, 0, 0, 0, 699, 989, 441, 525, 838, 588, 777, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 518, 593, 620, 566, 468, 335, 891, 1255, 502, 65, 0, 0, 0, 0, 0, 0, 716, 631, 524, 708, 1409, 894, 754, 430, 237, 106, 0, 0, 0, 0, 0, 0, 736, 632, 953, 893, 1145, 971, 368, 680, 0, 0, 0, 0, 0, 0, 0, 0, 744, 568, 1085, 951, 553, 403, 747, 350, 285, 224, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6400, 6159, 6207, 5876, 5813, 6409, 6378, 5910] avg=6144.0000 max=6409 max_rank=5 min=5813 min_rank=4 imbalance_ratio=1.043132 max_load_reduction=42743 max_load_reduction_pct=0.869609 moved_to_echo=24510 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. - warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1199, 984, 518, 796, 841, 597, 737, 657, 0, 0, 0, 0, 0, 0, 0, 0, 512, 1097, 1013, 616, 245, 910, 628, 1110, 0, 0, 0, 0, 0, 0, 0, 0, 607, 666, 600, 755, 472, 462, 927, 796, 647, 0, 0, 0, 0, 0, 0, 0, 762, 814, 488, 681, 897, 662, 934, 972, 0, 0, 0, 0, 0, 0, 0, 0, 556, 778, 652, 563, 465, 402, 1031, 1138, 460, 61, 0, 0, 0, 0, 0, 0, 759, 756, 426, 532, 1279, 918, 594, 392, 225, 137, 0, 0, 0, 0, 0, 0, 738, 506, 799, 864, 1085, 1030, 387, 631, 0, 0, 0, 0, 0, 0, 0, 0, 747, 536, 1294, 927, 686, 342, 888, 391, 370, 205, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6329, 6131, 5932, 6210, 6106, 6018, 6040, 6386] avg=6144.0000 max=6386 max_rank=7 min=5932 min_rank=2 imbalance_ratio=1.039388 max_load_reduction=42766 max_load_reduction_pct=0.870076 moved_to_echo=24550 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. - warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1194, 901, 462, 748, 746, 601, 820, 573, 0, 0, 0, 0, 0, 0, 0, 0, 615, 1079, 972, 627, 266, 882, 656, 1145, 0, 0, 0, 0, 0, 0, 0, 0, 630, 684, 627, 805, 509, 422, 1013, 799, 628, 0, 0, 0, 0, 0, 0, 0, 727, 1086, 484, 621, 935, 585, 892, 967, 0, 0, 0, 0, 0, 0, 0, 0, 531, 914, 648, 581, 476, 426, 958, 1227, 512, 59, 0, 0, 0, 0, 0, 0, 728, 635, 473, 453, 1435, 897, 626, 420, 242, 125, 0, 0, 0, 0, 0, 0, 721, 507, 837, 827, 1166, 921, 340, 709, 0, 0, 0, 0, 0, 0, 0, 0, 725, 640, 1153, 924, 629, 363, 753, 305, 337, 228, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6045, 6242, 6117, 6297, 6332, 6034, 6028, 6057] avg=6144.0000 max=6332 max_rank=4 min=6028 min_rank=6 imbalance_ratio=1.030599 max_load_reduction=42820 max_load_reduction_pct=0.871175 moved_to_echo=24451 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. @@ -2471,10156 +1407,7 @@ WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] glob warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. - warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. - warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6016, 7552, 3968, 4736, 7040, 5248, 7168, 7936, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5376, 5120, 5376, 6144, 4224, 3584, 7808, 6528, 5632, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[4608, 8704, 7936, 4736, 2048, 7808, 5120, 8448, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[9344, 7680, 4096, 6272, 6272, 4736, 6400, 4736, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6016, 4480, 7168, 6656, 9216, 7936, 2816, 5376, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5760, 5376, 3968, 4352, 11264, 7296, 5376, 3456, 1920, 1024, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4608, 5888, 5248, 4736, 4096, 3200, 7680, 9728, 3968, 640, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[5888, 4608, 9728, 7424, 5376, 2944, 6400, 2816, 2560, 1792, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[522, 466, 587, 644, 907, 687, 689, 811, 408, 901, 1496, 642, 1044, 881, 1137, 1157, 819, 562, 691, 714, 1031, 928, 1114, 787, 780, 831, 626, 511, 664, 737, 1311, 338, 1171, 479, 599, 457, 621, 402, 488, 1052, 1310, 460, 415, 733, 476, 837, 209, 396, 2000, 1146, 1015, 422, 628, 846, 592, 921, 859, 598, 866, 677, 792, 862, 759, 641] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[409, 568, 700, 635, 842, 760, 616, 758, 375, 1010, 1309, 487, 1172, 844, 924, 1081, 776, 416, 638, 958, 1082, 995, 1086, 541, 798, 834, 651, 569, 696, 745, 1321, 455, 1092, 570, 655, 390, 740, 469, 353, 978, 1364, 458, 376, 775, 464, 911, 296, 387, 1857, 1150, 808, 586, 716, 795, 636, 1117, 908, 522, 765, 685, 722, 1004, 990, 562] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[437, 435, 673, 728, 865, 718, 720, 775, 396, 1016, 1178, 526, 1192, 765, 1075, 1081, 737, 599, 636, 905, 1078, 965, 1040, 739, 866, 880, 561, 523, 642, 714, 1303, 437, 1097, 518, 683, 448, 757, 398, 390, 1110, 1236, 446, 237, 835, 523, 820, 269, 449, 1845, 1193, 885, 551, 675, 936, 573, 1055, 724, 679, 753, 587, 737, 1083, 830, 625] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[502, 404, 781, 686, 920, 616, 676, 743, 396, 1066, 1303, 607, 1192, 806, 936, 1107, 680, 498, 690, 758, 981, 848, 1147, 614, 710, 891, 688, 577, 828, 654, 1403, 386, 1109, 525, 694, 395, 756, 458, 417, 1034, 1215, 454, 541, 792, 556, 809, 289, 492, 2156, 1219, 824, 366, 569, 790, 648, 1041, 789, 517, 756, 703, 676, 831, 940, 697] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[518, 385, 699, 718, 886, 743, 675, 728, 407, 928, 1340, 630, 1163, 966, 1075, 1095, 660, 721, 656, 695, 940, 912, 1128, 535, 818, 815, 637, 609, 746, 611, 1359, 491, 1173, 471, 624, 418, 760, 508, 532, 1123, 1310, 470, 342, 834, 625, 767, 268, 388, 1974, 1120, 792, 449, 555, 799, 689, 940, 843, 553, 806, 667, 591, 971, 786, 715] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[433, 443, 633, 712, 1031, 943, 757, 714, 436, 947, 1306, 394, 1401, 753, 1155, 993, 754, 461, 668, 784, 1144, 895, 1118, 459, 780, 788, 543, 513, 612, 602, 1359, 468, 1218, 400, 731, 459, 712, 405, 419, 1151, 1198, 461, 259, 792, 556, 743, 317, 382, 2198, 1060, 890, 574, 689, 838, 662, 1019, 880, 605, 792, 636, 708, 871, 845, 683] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[454, 480, 628, 680, 875, 804, 788, 757, 352, 978, 1364, 501, 1230, 777, 965, 999, 731, 738, 650, 802, 1151, 915, 1238, 640, 712, 1053, 633, 486, 654, 704, 1439, 376, 1204, 489, 634, 473, 729, 466, 383, 1048, 1233, 356, 374, 750, 448, 826, 260, 422, 2085, 1115, 943, 406, 602, 778, 735, 914, 763, 573, 797, 636, 816, 934, 739, 667] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[446, 564, 553, 613, 817, 798, 721, 835, 419, 919, 1267, 517, 1151, 750, 1005, 1104, 719, 590, 715, 766, 939, 979, 1168, 579, 810, 879, 593, 478, 802, 692, 1410, 426, 1170, 484, 665, 373, 826, 328, 427, 1048, 993, 349, 461, 750, 473, 874, 321, 420, 2175, 1273, 910, 398, 864, 928, 669, 1016, 923, 556, 746, 575, 692, 1015, 829, 597] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[409, 568, 700, 635, 842, 760, 616, 758, 479, 247, 0, 0, 0, 0, 0, 0, 375, 1010, 115, 487, 1172, 844, 924, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 776, 416, 638, 958, 1082, 995, 757, 541, 0, 0, 0, 0, 0, 0, 0, 0, 798, 834, 651, 569, 696, 745, 1321, 455, 159, 0, 0, 0, 0, 0, 0, 0, 1092, 570, 655, 390, 740, 469, 353, 978, 715, 26, 0, 0, 0, 0, 0, 0, 1364, 458, 376, 775, 464, 911, 296, 387, 1174, 0, 0, 0, 0, 0, 0, 0, 436, 1150, 808, 586, 716, 795, 636, 1117, 0, 0, 0, 0, 0, 0, 0, 0, 908, 522, 765, 685, 722, 1004, 990, 562, 144, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6014, 6008, 6163, 6228, 5988, 6205, 6244, 6302] avg=6144.0000 max=6302 max_rank=7 min=5988 min_rank=4 imbalance_ratio=1.025716 max_load_reduction=42850 max_load_reduction_pct=0.871785 moved_to_echo=24739 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[522, 466, 587, 644, 907, 687, 689, 811, 548, 266, 0, 0, 0, 0, 0, 0, 408, 901, 131, 642, 1044, 881, 1137, 1157, 0, 0, 0, 0, 0, 0, 0, 0, 819, 562, 691, 714, 1031, 928, 776, 787, 0, 0, 0, 0, 0, 0, 0, 0, 780, 831, 626, 511, 664, 737, 1311, 338, 163, 0, 0, 0, 0, 0, 0, 0, 1171, 479, 599, 457, 621, 402, 488, 1052, 817, 27, 0, 0, 0, 0, 0, 0, 1310, 460, 415, 733, 476, 837, 209, 396, 1264, 0, 0, 0, 0, 0, 0, 0, 470, 1146, 1015, 422, 628, 846, 592, 921, 0, 0, 0, 0, 0, 0, 0, 0, 859, 598, 866, 677, 792, 862, 759, 641, 148, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6127, 6301, 6308, 5961, 6113, 6100, 6040, 6202] avg=6144.0000 max=6308 max_rank=2 min=5961 min_rank=3 imbalance_ratio=1.026693 max_load_reduction=42844 max_load_reduction_pct=0.871663 moved_to_echo=24455 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[502, 404, 781, 686, 920, 616, 676, 743, 477, 286, 0, 0, 0, 0, 0, 0, 396, 1066, 114, 607, 1192, 806, 936, 1107, 0, 0, 0, 0, 0, 0, 0, 0, 680, 498, 690, 758, 981, 848, 799, 614, 0, 0, 0, 0, 0, 0, 0, 0, 710, 891, 688, 577, 828, 654, 1403, 386, 168, 0, 0, 0, 0, 0, 0, 0, 1109, 525, 694, 395, 756, 458, 417, 1034, 712, 28, 0, 0, 0, 0, 0, 0, 1215, 454, 541, 792, 556, 809, 289, 492, 1363, 0, 0, 0, 0, 0, 0, 0, 507, 1219, 824, 366, 569, 790, 648, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 789, 517, 756, 703, 676, 831, 940, 697, 152, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6091, 6224, 5868, 6305, 6128, 6511, 5964, 6061] avg=6144.0000 max=6511 max_rank=5 min=5868 min_rank=2 imbalance_ratio=1.059733 max_load_reduction=42641 max_load_reduction_pct=0.867533 moved_to_echo=24664 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[437, 435, 673, 728, 865, 718, 720, 775, 431, 245, 0, 0, 0, 0, 0, 0, 396, 1016, 104, 526, 1192, 765, 1075, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 737, 599, 636, 905, 1078, 965, 725, 739, 0, 0, 0, 0, 0, 0, 0, 0, 866, 880, 561, 523, 642, 714, 1303, 437, 152, 0, 0, 0, 0, 0, 0, 0, 1097, 518, 683, 448, 757, 398, 390, 1110, 643, 25, 0, 0, 0, 0, 0, 0, 1236, 446, 237, 835, 523, 820, 269, 449, 1166, 0, 0, 0, 0, 0, 0, 0, 434, 1193, 885, 551, 675, 936, 573, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 724, 679, 753, 587, 737, 1083, 830, 625, 138, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6027, 6155, 6384, 6078, 6069, 5981, 6302, 6156] avg=6144.0000 max=6384 max_rank=2 min=5981 min_rank=5 imbalance_ratio=1.039062 max_load_reduction=42768 max_load_reduction_pct=0.870117 moved_to_echo=24508 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[518, 385, 699, 718, 886, 743, 675, 728, 491, 262, 0, 0, 0, 0, 0, 0, 407, 928, 117, 630, 1163, 966, 1075, 1095, 0, 0, 0, 0, 0, 0, 0, 0, 660, 721, 656, 695, 940, 912, 786, 535, 0, 0, 0, 0, 0, 0, 0, 0, 818, 815, 637, 609, 746, 611, 1359, 491, 165, 0, 0, 0, 0, 0, 0, 0, 1173, 471, 624, 418, 760, 508, 532, 1123, 732, 27, 0, 0, 0, 0, 0, 0, 1310, 470, 342, 834, 625, 767, 268, 388, 1248, 0, 0, 0, 0, 0, 0, 0, 464, 1120, 792, 449, 555, 799, 689, 940, 0, 0, 0, 0, 0, 0, 0, 0, 843, 553, 806, 667, 591, 971, 786, 715, 150, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6105, 6381, 5905, 6251, 6368, 6252, 5808, 6082] avg=6144.0000 max=6381 max_rank=1 min=5808 min_rank=6 imbalance_ratio=1.038574 max_load_reduction=42771 max_load_reduction_pct=0.870178 moved_to_echo=24510 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[433, 443, 633, 712, 1031, 943, 757, 714, 483, 296, 0, 0, 0, 0, 0, 0, 436, 947, 107, 394, 1401, 753, 1155, 993, 0, 0, 0, 0, 0, 0, 0, 0, 754, 461, 668, 784, 1144, 895, 768, 459, 0, 0, 0, 0, 0, 0, 0, 0, 780, 788, 543, 513, 612, 602, 1359, 468, 165, 0, 0, 0, 0, 0, 0, 0, 1218, 400, 731, 459, 712, 405, 419, 1151, 716, 33, 0, 0, 0, 0, 0, 0, 1198, 461, 259, 792, 556, 743, 317, 382, 1392, 0, 0, 0, 0, 0, 0, 0, 510, 1060, 890, 574, 689, 838, 662, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 880, 605, 792, 636, 708, 871, 845, 683, 152, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6445, 6186, 5933, 5830, 6244, 6100, 6242, 6172] avg=6144.0000 max=6445 max_rank=0 min=5830 min_rank=3 imbalance_ratio=1.048991 max_load_reduction=42707 max_load_reduction_pct=0.868876 moved_to_echo=24758 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[454, 480, 628, 680, 875, 804, 788, 757, 500, 277, 0, 0, 0, 0, 0, 0, 352, 978, 119, 501, 1230, 777, 965, 999, 0, 0, 0, 0, 0, 0, 0, 0, 731, 738, 650, 802, 1151, 915, 863, 640, 0, 0, 0, 0, 0, 0, 0, 0, 712, 1053, 633, 486, 654, 704, 1439, 376, 181, 0, 0, 0, 0, 0, 0, 0, 1204, 489, 634, 473, 729, 466, 383, 1048, 745, 30, 0, 0, 0, 0, 0, 0, 1233, 356, 374, 750, 448, 826, 260, 422, 1318, 0, 0, 0, 0, 0, 0, 0, 490, 1115, 943, 406, 602, 778, 735, 914, 0, 0, 0, 0, 0, 0, 0, 0, 763, 573, 797, 636, 816, 934, 739, 667, 164, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6243, 5921, 6490, 6238, 6201, 5987, 5983, 6089] avg=6144.0000 max=6490 max_rank=2 min=5921 min_rank=1 imbalance_ratio=1.056315 max_load_reduction=42662 max_load_reduction_pct=0.867961 moved_to_echo=24260 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[446, 564, 553, 613, 817, 798, 721, 835, 464, 289, 0, 0, 0, 0, 0, 0, 419, 919, 111, 517, 1151, 750, 1005, 1104, 0, 0, 0, 0, 0, 0, 0, 0, 719, 590, 715, 766, 939, 979, 814, 579, 0, 0, 0, 0, 0, 0, 0, 0, 810, 879, 593, 478, 802, 692, 1410, 426, 171, 0, 0, 0, 0, 0, 0, 0, 1170, 484, 665, 373, 826, 328, 427, 1048, 692, 28, 0, 0, 0, 0, 0, 0, 993, 349, 461, 750, 473, 874, 321, 420, 1375, 0, 0, 0, 0, 0, 0, 0, 511, 1273, 910, 398, 864, 928, 669, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 923, 556, 746, 575, 692, 1015, 829, 597, 155, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6100, 5976, 6101, 6261, 6041, 6016, 6569, 6088] avg=6144.0000 max=6569 max_rank=6 min=5976 min_rank=1 imbalance_ratio=1.069173 max_load_reduction=42583 max_load_reduction_pct=0.866353 moved_to_echo=24714 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3840, 3840, 5376, 5504, 7168, 6144, 5760, 6144, 3968, 2176, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[6400, 7040, 4992, 4352, 5760, 5504, 11008, 3456, 1408, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3840, 9344, 7168, 3840, 5376, 6784, 5248, 8064, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6784, 4608, 6400, 5248, 5760, 7680, 6784, 5248, 1280, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5888, 4608, 5376, 6400, 8448, 7552, 6400, 4992, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[9344, 3968, 5376, 3456, 6016, 3456, 3456, 8576, 5888, 256, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[9984, 3456, 3072, 6272, 4224, 6656, 2304, 3456, 10368, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3200, 7808, 1024, 4352, 9600, 6656, 8320, 8704, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[573, 740, 502, 487, 818, 293, 676, 590, 524, 1148, 305, 1434, 842, 524, 938, 687, 1269, 653, 889, 1188, 726, 418, 844, 911, 594, 652, 935, 795, 756, 284, 787, 217, 1451, 801, 1316, 867, 1114, 747, 656, 765, 954, 569, 318, 1051, 793, 1287, 572, 609, 394, 1050, 1591, 733, 942, 666, 328, 370, 1699, 960, 162, 415, 539, 1020, 632, 782] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[579, 1006, 370, 483, 565, 526, 437, 663, 508, 936, 316, 1581, 870, 610, 1084, 719, 1186, 712, 967, 900, 900, 384, 950, 848, 655, 803, 1277, 861, 780, 320, 634, 265, 1348, 785, 1361, 689, 1088, 748, 734, 835, 792, 446, 378, 1437, 777, 871, 530, 973, 439, 1012, 1303, 595, 980, 766, 374, 543, 884, 872, 342, 570, 590, 927, 707, 761] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[757, 1039, 442, 448, 656, 481, 487, 688, 413, 1050, 322, 1699, 1056, 536, 951, 679, 1128, 512, 917, 936, 756, 368, 1021, 808, 618, 647, 1129, 1123, 624, 269, 625, 178, 1511, 983, 1715, 529, 1100, 751, 771, 721, 866, 515, 336, 1306, 879, 1059, 505, 601, 313, 922, 1556, 691, 823, 729, 384, 524, 1109, 988, 254, 527, 410, 954, 722, 735] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[682, 980, 442, 417, 640, 438, 424, 527, 469, 1215, 319, 1662, 896, 943, 897, 608, 1230, 550, 869, 1081, 718, 383, 1144, 883, 561, 821, 1205, 658, 785, 322, 563, 218, 1452, 1016, 1183, 925, 913, 744, 855, 743, 853, 452, 275, 1332, 906, 1093, 606, 730, 357, 899, 1470, 621, 958, 728, 320, 468, 1275, 828, 218, 521, 405, 831, 824, 801] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[620, 989, 356, 422, 588, 333, 529, 815, 444, 1093, 354, 1649, 1081, 507, 939, 764, 1271, 499, 943, 1003, 729, 414, 877, 911, 458, 876, 1236, 773, 971, 259, 603, 240, 1428, 1082, 1320, 682, 1338, 687, 699, 913, 874, 558, 301, 1141, 886, 904, 546, 755, 170, 872, 1665, 670, 930, 685, 336, 522, 978, 892, 210, 534, 638, 1081, 620, 689] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[719, 913, 457, 422, 635, 494, 480, 738, 476, 805, 282, 1551, 769, 662, 959, 770, 1127, 571, 1209, 1072, 756, 436, 780, 902, 602, 772, 1195, 872, 783, 312, 647, 154, 1447, 966, 1622, 543, 1174, 731, 623, 1015, 815, 555, 320, 1295, 857, 972, 431, 837, 315, 957, 1527, 800, 785, 729, 370, 599, 1095, 877, 232, 442, 452, 967, 771, 709] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[643, 990, 336, 568, 615, 348, 532, 595, 536, 1090, 300, 1485, 868, 1049, 935, 594, 1339, 720, 879, 1023, 826, 341, 935, 894, 577, 652, 1243, 702, 752, 302, 654, 96, 1453, 810, 1254, 851, 1050, 855, 560, 1036, 871, 567, 417, 1375, 1054, 870, 516, 705, 307, 1008, 1448, 729, 1074, 697, 269, 447, 1054, 896, 279, 517, 387, 866, 735, 776] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[745, 973, 393, 483, 604, 409, 517, 700, 417, 887, 280, 1638, 1006, 724, 1000, 634, 1103, 591, 1040, 1006, 852, 424, 973, 779, 629, 787, 1283, 805, 788, 308, 607, 125, 1497, 847, 1397, 781, 1249, 836, 659, 708, 940, 461, 321, 1266, 823, 972, 580, 642, 318, 989, 1558, 575, 972, 739, 380, 473, 1056, 930, 225, 430, 467, 973, 827, 751] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[579, 1006, 370, 483, 565, 526, 437, 663, 1318, 0, 0, 0, 0, 0, 0, 0, 508, 936, 316, 1067, 870, 610, 1084, 719, 0, 0, 0, 0, 0, 0, 0, 0, 557, 712, 967, 900, 900, 384, 950, 848, 0, 0, 0, 0, 0, 0, 0, 0, 655, 803, 1277, 861, 780, 320, 634, 265, 629, 218, 30, 0, 0, 0, 0, 0, 0, 785, 1046, 689, 1088, 748, 734, 835, 0, 0, 0, 0, 0, 0, 0, 0, 792, 446, 378, 1437, 777, 871, 530, 973, 6, 0, 0, 0, 0, 0, 0, 0, 439, 1012, 1303, 595, 980, 766, 374, 543, 169, 0, 0, 0, 0, 0, 0, 0, 884, 872, 342, 570, 590, 927, 707, 761, 296, 140, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5947, 6110, 6218, 6472, 5925, 6210, 6181, 6089] avg=6144.0000 max=6472 max_rank=3 min=5925 min_rank=4 imbalance_ratio=1.053385 max_load_reduction=42680 max_load_reduction_pct=0.868327 moved_to_echo=24405 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[573, 740, 502, 487, 818, 293, 676, 590, 1419, 0, 0, 0, 0, 0, 0, 0, 524, 1148, 305, 967, 842, 524, 938, 687, 0, 0, 0, 0, 0, 0, 0, 0, 596, 653, 889, 1188, 726, 418, 844, 911, 0, 0, 0, 0, 0, 0, 0, 0, 594, 652, 935, 795, 756, 284, 787, 217, 673, 198, 32, 0, 0, 0, 0, 0, 0, 801, 1012, 867, 1114, 747, 656, 765, 0, 0, 0, 0, 0, 0, 0, 0, 954, 569, 318, 1051, 793, 1287, 572, 609, 6, 0, 0, 0, 0, 0, 0, 0, 394, 1050, 1591, 733, 942, 666, 328, 370, 163, 0, 0, 0, 0, 0, 0, 0, 1699, 960, 162, 415, 539, 1020, 632, 782, 269, 135, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6098, 5935, 6225, 5923, 5962, 6159, 6237, 6613] avg=6144.0000 max=6613 max_rank=7 min=5923 min_rank=3 imbalance_ratio=1.076335 max_load_reduction=42539 max_load_reduction_pct=0.865458 moved_to_echo=24971 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[757, 1039, 442, 448, 656, 481, 487, 688, 1479, 0, 0, 0, 0, 0, 0, 0, 413, 1050, 322, 1147, 1056, 536, 951, 679, 0, 0, 0, 0, 0, 0, 0, 0, 530, 512, 917, 936, 756, 368, 1021, 808, 0, 0, 0, 0, 0, 0, 0, 0, 618, 647, 1129, 1123, 624, 269, 625, 178, 598, 234, 32, 0, 0, 0, 0, 0, 0, 983, 1318, 529, 1100, 751, 771, 721, 0, 0, 0, 0, 0, 0, 0, 0, 866, 515, 336, 1306, 879, 1059, 505, 601, 8, 0, 0, 0, 0, 0, 0, 0, 313, 922, 1556, 691, 823, 729, 384, 524, 213, 0, 0, 0, 0, 0, 0, 0, 1109, 988, 254, 527, 410, 954, 722, 735, 318, 176, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6477, 6154, 5848, 6077, 6173, 6075, 6155, 6193] avg=6144.0000 max=6477 max_rank=0 min=5848 min_rank=2 imbalance_ratio=1.054199 max_load_reduction=42675 max_load_reduction_pct=0.868225 moved_to_echo=24596 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[682, 980, 442, 417, 640, 438, 424, 527, 1421, 0, 0, 0, 0, 0, 0, 0, 469, 1215, 319, 1115, 896, 943, 897, 608, 0, 0, 0, 0, 0, 0, 0, 0, 575, 550, 869, 1081, 718, 383, 1144, 883, 0, 0, 0, 0, 0, 0, 0, 0, 561, 821, 1205, 658, 785, 322, 563, 218, 655, 232, 31, 0, 0, 0, 0, 0, 0, 1016, 899, 925, 913, 744, 855, 743, 0, 0, 0, 0, 0, 0, 0, 0, 853, 452, 275, 1332, 906, 1093, 606, 730, 9, 0, 0, 0, 0, 0, 0, 0, 357, 899, 1470, 621, 958, 728, 320, 468, 150, 0, 0, 0, 0, 0, 0, 0, 1275, 828, 218, 521, 405, 831, 824, 801, 315, 125, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5971, 6462, 6203, 6051, 6095, 6256, 5971, 6143] avg=6144.0000 max=6462 max_rank=1 min=5971 min_rank=0 imbalance_ratio=1.051758 max_load_reduction=42690 max_load_reduction_pct=0.868530 moved_to_echo=24465 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[620, 989, 356, 422, 588, 333, 529, 815, 1396, 0, 0, 0, 0, 0, 0, 0, 444, 1093, 354, 1113, 1081, 507, 939, 764, 0, 0, 0, 0, 0, 0, 0, 0, 597, 499, 943, 1003, 729, 414, 877, 911, 0, 0, 0, 0, 0, 0, 0, 0, 458, 876, 1236, 773, 971, 259, 603, 240, 674, 227, 32, 0, 0, 0, 0, 0, 0, 1082, 1015, 682, 1338, 687, 699, 913, 0, 0, 0, 0, 0, 0, 0, 0, 874, 558, 301, 1141, 886, 904, 546, 755, 6, 0, 0, 0, 0, 0, 0, 0, 170, 872, 1665, 670, 930, 685, 336, 522, 164, 0, 0, 0, 0, 0, 0, 0, 978, 892, 210, 534, 638, 1081, 620, 689, 309, 135, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6048, 6295, 5973, 6349, 6416, 5971, 6014, 6086] avg=6144.0000 max=6416 max_rank=4 min=5971 min_rank=5 imbalance_ratio=1.044271 max_load_reduction=42736 max_load_reduction_pct=0.869466 moved_to_echo=24487 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[643, 990, 336, 568, 615, 348, 532, 595, 1422, 0, 0, 0, 0, 0, 0, 0, 536, 1090, 300, 1002, 868, 1049, 935, 594, 0, 0, 0, 0, 0, 0, 0, 0, 629, 720, 879, 1023, 826, 341, 935, 894, 0, 0, 0, 0, 0, 0, 0, 0, 577, 652, 1243, 702, 752, 302, 654, 96, 710, 205, 31, 0, 0, 0, 0, 0, 0, 810, 964, 851, 1050, 855, 560, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 871, 567, 417, 1375, 1054, 870, 516, 705, 6, 0, 0, 0, 0, 0, 0, 0, 307, 1008, 1448, 729, 1074, 697, 269, 447, 155, 0, 0, 0, 0, 0, 0, 0, 1054, 896, 279, 517, 387, 866, 735, 776, 278, 129, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6049, 6374, 6247, 5924, 6126, 6381, 6134, 5917] avg=6144.0000 max=6381 max_rank=5 min=5917 min_rank=7 imbalance_ratio=1.038574 max_load_reduction=42771 max_load_reduction_pct=0.870178 moved_to_echo=24558 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[719, 913, 457, 422, 635, 494, 480, 738, 1416, 0, 0, 0, 0, 0, 0, 0, 476, 805, 282, 1046, 769, 662, 959, 770, 0, 0, 0, 0, 0, 0, 0, 0, 529, 571, 1209, 1072, 756, 436, 780, 902, 0, 0, 0, 0, 0, 0, 0, 0, 602, 772, 1195, 872, 783, 312, 647, 154, 598, 214, 31, 0, 0, 0, 0, 0, 0, 966, 1248, 543, 1174, 731, 623, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 815, 555, 320, 1295, 857, 972, 431, 837, 7, 0, 0, 0, 0, 0, 0, 0, 315, 957, 1527, 800, 785, 729, 370, 599, 201, 0, 0, 0, 0, 0, 0, 0, 1095, 877, 232, 442, 452, 967, 771, 709, 291, 166, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6274, 5769, 6255, 6180, 6300, 6089, 6283, 6002] avg=6144.0000 max=6300 max_rank=4 min=5769 min_rank=1 imbalance_ratio=1.025391 max_load_reduction=42852 max_load_reduction_pct=0.871826 moved_to_echo=24674 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[745, 973, 393, 483, 604, 409, 517, 700, 1464, 0, 0, 0, 0, 0, 0, 0, 417, 887, 280, 1105, 1006, 724, 1000, 634, 0, 0, 0, 0, 0, 0, 0, 0, 518, 591, 1040, 1006, 852, 424, 973, 779, 0, 0, 0, 0, 0, 0, 0, 0, 629, 787, 1283, 805, 788, 308, 607, 125, 585, 226, 33, 0, 0, 0, 0, 0, 0, 847, 1075, 781, 1249, 836, 659, 708, 0, 0, 0, 0, 0, 0, 0, 0, 940, 461, 321, 1266, 823, 972, 580, 642, 6, 0, 0, 0, 0, 0, 0, 0, 318, 989, 1558, 575, 972, 739, 380, 473, 173, 0, 0, 0, 0, 0, 0, 0, 1056, 930, 225, 430, 467, 973, 827, 751, 307, 143, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6288, 6053, 6183, 6176, 6155, 6011, 6177, 6109] avg=6144.0000 max=6288 max_rank=0 min=6011 min_rank=5 imbalance_ratio=1.023438 max_load_reduction=42864 max_load_reduction_pct=0.872070 moved_to_echo=24452 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5376, 7680, 3328, 3840, 5248, 3328, 4096, 5376, 11392, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[9216, 7296, 2048, 3968, 3968, 7680, 5888, 6016, 2432, 1152, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7040, 4224, 2688, 10240, 7040, 8064, 4352, 5888, 128, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4736, 6016, 9600, 6656, 6272, 2432, 5120, 1536, 5248, 1792, 256, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4608, 4864, 7808, 8320, 6272, 3200, 7552, 7040, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2688, 7808, 12160, 5504, 7552, 5760, 2816, 3968, 1408, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3840, 8320, 2560, 8576, 7424, 5632, 7808, 5504, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 7296, 8704, 5888, 9088, 6144, 5632, 6784, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[634, 506, 1026, 406, 1186, 682, 1217, 1445, 808, 308, 334, 267, 926, 805, 630, 584, 1265, 1045, 580, 865, 737, 982, 538, 676, 641, 230, 1207, 1146, 734, 1424, 929, 558, 420, 1059, 468, 1078, 471, 443, 509, 316, 421, 830, 807, 661, 750, 771, 672, 680, 896, 1129, 2408, 652, 217, 532, 769, 740, 1200, 309, 472, 364, 704, 1571, 1078, 434] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[834, 638, 799, 481, 1581, 495, 1192, 1370, 681, 274, 347, 386, 789, 710, 683, 400, 1490, 735, 1003, 792, 702, 593, 569, 656, 534, 504, 985, 920, 571, 1424, 864, 559, 412, 775, 424, 1140, 564, 381, 494, 361, 957, 902, 1228, 734, 761, 780, 897, 769, 871, 982, 2193, 545, 267, 441, 1046, 544, 1124, 324, 465, 252, 1038, 1676, 758, 486] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[727, 649, 818, 433, 960, 556, 1170, 1534, 799, 577, 288, 331, 1012, 771, 734, 1000, 1231, 1010, 705, 807, 477, 505, 670, 943, 567, 361, 997, 938, 622, 1480, 861, 476, 309, 823, 477, 1171, 461, 708, 456, 299, 412, 741, 958, 563, 861, 824, 839, 861, 951, 1009, 2368, 444, 290, 612, 951, 642, 966, 373, 571, 337, 518, 1824, 954, 570] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[733, 672, 782, 573, 1153, 695, 1188, 1293, 774, 333, 376, 360, 883, 795, 574, 648, 1236, 918, 826, 859, 831, 702, 667, 998, 663, 309, 1082, 1010, 499, 1391, 738, 453, 390, 850, 464, 1089, 732, 427, 505, 364, 693, 868, 891, 622, 755, 642, 767, 818, 893, 995, 2119, 533, 319, 399, 946, 755, 1246, 459, 539, 273, 797, 1528, 1061, 399] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[714, 701, 787, 378, 1025, 599, 1078, 1469, 556, 364, 374, 335, 918, 847, 669, 565, 1347, 1171, 635, 696, 561, 1026, 550, 1172, 627, 321, 1196, 1140, 721, 1366, 825, 585, 360, 869, 441, 1187, 493, 540, 466, 570, 379, 813, 902, 611, 663, 699, 910, 888, 808, 1035, 2368, 642, 269, 657, 793, 623, 1105, 329, 561, 249, 677, 1798, 610, 519] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[782, 651, 922, 397, 1059, 725, 1079, 1460, 839, 277, 310, 341, 862, 742, 809, 701, 1283, 986, 615, 767, 781, 656, 609, 1000, 744, 353, 932, 895, 976, 1328, 853, 358, 349, 932, 504, 1169, 561, 342, 441, 314, 787, 1106, 875, 791, 752, 679, 671, 725, 669, 1305, 1732, 500, 159, 549, 885, 787, 1084, 243, 658, 294, 851, 1752, 1094, 500] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[910, 590, 811, 454, 1303, 705, 1108, 1510, 636, 428, 343, 503, 862, 867, 457, 437, 1306, 1001, 788, 698, 604, 1002, 623, 1371, 547, 337, 1166, 1139, 697, 1484, 781, 501, 441, 899, 450, 1306, 599, 512, 470, 378, 681, 874, 933, 698, 797, 723, 853, 621, 735, 999, 1557, 504, 203, 466, 697, 531, 1298, 284, 480, 464, 863, 1635, 730, 502] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[640, 716, 952, 550, 1438, 694, 1089, 1406, 841, 456, 303, 166, 885, 885, 727, 847, 1223, 905, 662, 754, 836, 839, 787, 1050, 630, 495, 1354, 865, 798, 1587, 817, 285, 361, 900, 427, 1188, 710, 574, 431, 258, 591, 544, 1025, 555, 606, 711, 837, 677, 944, 931, 1695, 520, 173, 634, 807, 850, 1148, 348, 657, 304, 872, 945, 953, 494] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[834, 638, 799, 481, 1581, 495, 1192, 419, 0, 0, 0, 0, 0, 0, 0, 0, 681, 274, 347, 386, 789, 710, 683, 400, 711, 412, 323, 0, 0, 0, 0, 0, 663, 735, 1003, 792, 702, 593, 569, 656, 0, 0, 0, 0, 0, 0, 0, 0, 534, 504, 985, 920, 571, 1025, 864, 559, 0, 0, 0, 0, 0, 0, 0, 0, 412, 775, 424, 1140, 564, 381, 494, 361, 951, 415, 0, 0, 0, 0, 0, 0, 957, 902, 1228, 734, 761, 780, 897, 769, 54, 0, 0, 0, 0, 0, 0, 0, 871, 982, 1482, 545, 267, 441, 1046, 544, 0, 0, 0, 0, 0, 0, 0, 0, 1124, 324, 465, 252, 1038, 1676, 758, 486, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6439, 5716, 5713, 5962, 5917, 7082, 6178, 6145] avg=6144.0000 max=7082 max_rank=5 min=5713 min_rank=2 imbalance_ratio=1.152669 max_load_reduction=42070 max_load_reduction_pct=0.855916 moved_to_echo=25322 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[714, 701, 787, 378, 1025, 599, 1078, 449, 0, 0, 0, 0, 0, 0, 0, 0, 556, 364, 374, 335, 918, 847, 669, 565, 768, 372, 310, 0, 0, 0, 0, 0, 600, 1171, 635, 696, 561, 1026, 550, 1172, 0, 0, 0, 0, 0, 0, 0, 0, 627, 321, 1196, 1140, 721, 983, 825, 585, 0, 0, 0, 0, 0, 0, 0, 0, 360, 869, 441, 1187, 493, 540, 466, 570, 1020, 375, 0, 0, 0, 0, 0, 0, 379, 813, 902, 611, 663, 699, 910, 888, 52, 0, 0, 0, 0, 0, 0, 0, 808, 1035, 1600, 642, 269, 657, 793, 623, 0, 0, 0, 0, 0, 0, 0, 0, 1105, 329, 561, 249, 677, 1798, 610, 519, 21, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5731, 6078, 6411, 6398, 6321, 5917, 6427, 5869] avg=6144.0000 max=6427 max_rank=6 min=5731 min_rank=0 imbalance_ratio=1.046061 max_load_reduction=42725 max_load_reduction_pct=0.869242 moved_to_echo=24534 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[733, 672, 782, 573, 1153, 695, 1188, 395, 0, 0, 0, 0, 0, 0, 0, 0, 774, 333, 376, 360, 883, 795, 574, 648, 687, 342, 315, 0, 0, 0, 0, 0, 550, 918, 826, 859, 831, 702, 667, 998, 0, 0, 0, 0, 0, 0, 0, 0, 663, 309, 1082, 1010, 499, 1002, 738, 453, 0, 0, 0, 0, 0, 0, 0, 0, 390, 850, 464, 1089, 732, 427, 505, 364, 898, 344, 0, 0, 0, 0, 0, 0, 693, 868, 891, 622, 755, 642, 767, 818, 53, 0, 0, 0, 0, 0, 0, 0, 893, 995, 1432, 533, 319, 399, 946, 755, 0, 0, 0, 0, 0, 0, 0, 0, 1246, 459, 539, 273, 797, 1528, 1061, 399, 21, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6191, 6087, 6351, 5756, 6063, 6109, 6272, 6323] avg=6144.0000 max=6351 max_rank=2 min=5756 min_rank=3 imbalance_ratio=1.033691 max_load_reduction=42801 max_load_reduction_pct=0.870789 moved_to_echo=24767 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[727, 649, 818, 433, 960, 556, 1170, 466, 0, 0, 0, 0, 0, 0, 0, 0, 799, 577, 288, 331, 1012, 771, 734, 1000, 771, 343, 339, 0, 0, 0, 0, 0, 542, 1010, 705, 807, 477, 505, 670, 943, 0, 0, 0, 0, 0, 0, 0, 0, 567, 361, 997, 938, 622, 1053, 861, 476, 0, 0, 0, 0, 0, 0, 0, 0, 309, 823, 477, 1171, 461, 708, 456, 299, 1068, 346, 0, 0, 0, 0, 0, 0, 412, 741, 958, 563, 861, 824, 839, 861, 61, 0, 0, 0, 0, 0, 0, 0, 951, 1009, 1597, 444, 290, 612, 951, 642, 0, 0, 0, 0, 0, 0, 0, 0, 966, 373, 571, 337, 518, 1824, 954, 570, 27, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5779, 6965, 5659, 5875, 6118, 6120, 6496, 6140] avg=6144.0000 max=6965 max_rank=1 min=5659 min_rank=2 imbalance_ratio=1.133626 max_load_reduction=42187 max_load_reduction_pct=0.858297 moved_to_echo=24874 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[634, 506, 1026, 406, 1186, 682, 1217, 442, 0, 0, 0, 0, 0, 0, 0, 0, 808, 308, 334, 267, 926, 805, 630, 584, 781, 350, 323, 0, 0, 0, 0, 0, 563, 1045, 580, 865, 737, 982, 538, 676, 0, 0, 0, 0, 0, 0, 0, 0, 641, 230, 1207, 1146, 734, 1025, 929, 558, 0, 0, 0, 0, 0, 0, 0, 0, 420, 1059, 468, 1078, 471, 443, 509, 316, 1003, 352, 0, 0, 0, 0, 0, 0, 421, 830, 807, 661, 750, 771, 672, 680, 54, 0, 0, 0, 0, 0, 0, 0, 896, 1129, 1627, 652, 217, 532, 769, 740, 0, 0, 0, 0, 0, 0, 0, 0, 1200, 309, 472, 364, 704, 1571, 1078, 434, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6099, 6116, 5986, 6470, 6119, 5646, 6562, 6154] avg=6144.0000 max=6562 max_rank=6 min=5646 min_rank=5 imbalance_ratio=1.068034 max_load_reduction=42590 max_load_reduction_pct=0.866496 moved_to_echo=24481 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[910, 590, 811, 454, 1303, 705, 1108, 462, 0, 0, 0, 0, 0, 0, 0, 0, 636, 428, 343, 503, 862, 867, 457, 437, 505, 361, 336, 0, 0, 0, 0, 0, 582, 1001, 788, 698, 604, 1002, 623, 1371, 0, 0, 0, 0, 0, 0, 0, 0, 547, 337, 1166, 1139, 697, 1069, 781, 501, 0, 0, 0, 0, 0, 0, 0, 0, 441, 899, 450, 1306, 599, 512, 470, 378, 1048, 363, 0, 0, 0, 0, 0, 0, 681, 874, 933, 698, 797, 723, 853, 621, 56, 0, 0, 0, 0, 0, 0, 0, 735, 999, 1052, 504, 203, 466, 697, 531, 0, 0, 0, 0, 0, 0, 0, 0, 1298, 284, 480, 464, 863, 1635, 730, 502, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6343, 5735, 6669, 6237, 6466, 6236, 5187, 6279] avg=6144.0000 max=6669 max_rank=2 min=5187 min_rank=6 imbalance_ratio=1.085449 max_load_reduction=42483 max_load_reduction_pct=0.864319 moved_to_echo=24168 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[782, 651, 922, 397, 1059, 725, 1079, 446, 0, 0, 0, 0, 0, 0, 0, 0, 839, 277, 310, 341, 862, 742, 809, 701, 561, 355, 301, 0, 0, 0, 0, 0, 571, 986, 615, 767, 781, 656, 609, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 744, 353, 932, 895, 976, 957, 853, 358, 0, 0, 0, 0, 0, 0, 0, 0, 349, 932, 504, 1169, 561, 342, 441, 314, 1014, 357, 0, 0, 0, 0, 0, 0, 787, 1106, 875, 791, 752, 679, 671, 725, 50, 0, 0, 0, 0, 0, 0, 0, 669, 1305, 1171, 500, 159, 549, 885, 787, 0, 0, 0, 0, 0, 0, 0, 0, 1084, 243, 658, 294, 851, 1752, 1094, 500, 20, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6061, 6098, 5985, 6068, 5983, 6436, 6025, 6496] avg=6144.0000 max=6496 max_rank=7 min=5983 min_rank=4 imbalance_ratio=1.057292 max_load_reduction=42656 max_load_reduction_pct=0.867839 moved_to_echo=24940 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[640, 716, 952, 550, 1438, 694, 1089, 430, 0, 0, 0, 0, 0, 0, 0, 0, 841, 456, 303, 166, 885, 885, 727, 847, 549, 338, 360, 0, 0, 0, 0, 0, 545, 905, 662, 754, 836, 839, 787, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 630, 495, 1354, 865, 798, 1142, 817, 285, 0, 0, 0, 0, 0, 0, 0, 0, 361, 900, 427, 1188, 710, 574, 431, 258, 976, 340, 0, 0, 0, 0, 0, 0, 591, 544, 1025, 555, 606, 711, 837, 677, 60, 0, 0, 0, 0, 0, 0, 0, 944, 931, 1146, 520, 173, 634, 807, 850, 0, 0, 0, 0, 0, 0, 0, 0, 1148, 348, 657, 304, 872, 945, 953, 494, 25, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6509, 6357, 6378, 6386, 6165, 5606, 6005, 5746] avg=6144.0000 max=6509 max_rank=0 min=5606 min_rank=5 imbalance_ratio=1.059408 max_load_reduction=42643 max_load_reduction_pct=0.867574 moved_to_echo=23522 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3072, 7168, 3712, 9344, 4608, 3968, 3840, 2944, 8064, 2944, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4992, 2944, 8960, 8064, 5632, 8320, 6784, 3840, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[6016, 3072, 2688, 2816, 7168, 6528, 5376, 5248, 5376, 2944, 2688, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4736, 7808, 5888, 6272, 5632, 6400, 5120, 7936, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4992, 6784, 7680, 5248, 6016, 5888, 6528, 6144, 512, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[6784, 8448, 11136, 4352, 1920, 4352, 6912, 5504, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[9216, 2688, 4480, 2560, 6400, 12800, 7296, 3968, 256, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6016, 5248, 6912, 3712, 9728, 5248, 9216, 3584, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[993, 873, 742, 464, 507, 2191, 1013, 770, 545, 1029, 710, 625, 624, 534, 448, 600, 529, 1397, 445, 1229, 1027, 590, 830, 1146, 490, 197, 702, 1066, 401, 1992, 619, 484, 433, 393, 638, 1032, 965, 298, 546, 740, 504, 752, 257, 709, 564, 689, 817, 366, 249, 1521, 1385, 467, 504, 1021, 1228, 770, 523, 1188, 463, 1601, 591, 863, 696, 567] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[914, 1212, 813, 472, 356, 2304, 987, 692, 586, 557, 560, 391, 663, 1167, 343, 709, 435, 1295, 206, 2092, 1016, 763, 556, 752, 602, 466, 772, 1180, 523, 1615, 421, 818, 545, 473, 466, 1299, 1025, 267, 511, 1073, 539, 824, 183, 791, 384, 339, 651, 656, 439, 1212, 1824, 185, 592, 1257, 1620, 511, 604, 994, 465, 1121, 616, 453, 675, 320] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1325, 1221, 893, 588, 626, 2032, 975, 720, 636, 602, 595, 649, 823, 467, 405, 868, 399, 1408, 196, 1381, 758, 519, 973, 618, 517, 454, 1007, 1386, 388, 1957, 541, 913, 333, 333, 683, 1273, 906, 397, 414, 549, 548, 527, 180, 435, 549, 518, 1160, 554, 385, 1369, 1514, 194, 537, 1385, 988, 746, 354, 1404, 507, 1247, 592, 650, 536, 545] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[1000, 1235, 1039, 498, 480, 2096, 964, 1083, 609, 854, 746, 646, 576, 933, 557, 592, 430, 1264, 339, 1478, 835, 559, 743, 1232, 541, 341, 814, 1141, 622, 1908, 347, 750, 559, 321, 746, 1272, 1048, 325, 439, 683, 547, 529, 186, 906, 455, 538, 868, 349, 419, 1280, 1258, 212, 605, 814, 1406, 709, 662, 1255, 380, 1149, 566, 451, 559, 404] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[1129, 1339, 743, 512, 552, 2005, 813, 744, 706, 856, 1101, 544, 664, 801, 511, 589, 346, 1328, 260, 1897, 762, 475, 880, 805, 546, 577, 826, 1207, 526, 1850, 582, 742, 356, 266, 584, 1232, 810, 435, 432, 786, 593, 799, 209, 857, 486, 404, 946, 420, 400, 1439, 1182, 232, 502, 1194, 1309, 634, 466, 1041, 449, 1282, 603, 592, 561, 433] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[1119, 1199, 1065, 536, 296, 1651, 997, 866, 714, 639, 628, 599, 597, 822, 455, 1043, 549, 1137, 375, 1950, 859, 554, 861, 660, 717, 473, 1064, 1300, 338, 1505, 441, 1042, 479, 378, 559, 1474, 907, 502, 564, 596, 608, 729, 333, 506, 398, 478, 982, 392, 204, 1804, 1350, 185, 391, 1208, 1375, 682, 670, 792, 331, 1271, 478, 418, 608, 449] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[1074, 853, 717, 546, 420, 2235, 916, 696, 633, 863, 1017, 687, 651, 580, 649, 454, 469, 1508, 223, 2006, 733, 537, 872, 1089, 442, 323, 797, 1068, 565, 1336, 497, 761, 326, 229, 401, 1499, 1008, 421, 286, 738, 304, 863, 198, 573, 645, 456, 1563, 280, 361, 1347, 1029, 366, 605, 1299, 1181, 858, 442, 1380, 435, 1285, 515, 943, 687, 412] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1247, 1531, 952, 462, 435, 1841, 996, 830, 610, 666, 644, 732, 760, 676, 426, 333, 403, 1190, 320, 1709, 984, 515, 941, 1033, 415, 456, 611, 1499, 459, 1759, 673, 810, 461, 355, 501, 1368, 1162, 264, 467, 1059, 533, 961, 191, 576, 568, 491, 1081, 393, 491, 1239, 1307, 233, 401, 1431, 1042, 688, 397, 1248, 386, 1252, 472, 357, 520, 339] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[914, 1212, 813, 472, 356, 299, 987, 692, 0, 0, 0, 0, 0, 0, 0, 0, 586, 557, 560, 391, 663, 1167, 343, 709, 747, 209, 0, 0, 0, 0, 0, 0, 435, 1295, 206, 1101, 1016, 763, 556, 752, 0, 0, 0, 0, 0, 0, 0, 0, 602, 466, 772, 1180, 523, 1264, 421, 818, 0, 0, 0, 0, 0, 0, 0, 0, 545, 473, 466, 1299, 1025, 267, 511, 1073, 592, 250, 0, 0, 0, 0, 0, 0, 539, 824, 183, 791, 384, 339, 651, 656, 1755, 0, 0, 0, 0, 0, 0, 0, 439, 411, 1824, 185, 592, 1257, 1620, 511, 0, 0, 0, 0, 0, 0, 0, 0, 604, 994, 465, 1121, 616, 453, 675, 320, 351, 244, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5745, 5932, 6124, 6046, 6501, 6122, 6839, 5843] avg=6144.0000 max=6839 max_rank=6 min=5745 min_rank=0 imbalance_ratio=1.113118 max_load_reduction=42313 max_load_reduction_pct=0.860860 moved_to_echo=25305 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[993, 873, 742, 464, 507, 284, 1013, 770, 0, 0, 0, 0, 0, 0, 0, 0, 545, 1029, 710, 625, 624, 534, 448, 600, 439, 263, 0, 0, 0, 0, 0, 0, 529, 1397, 445, 647, 1027, 590, 830, 1146, 0, 0, 0, 0, 0, 0, 0, 0, 490, 197, 702, 1066, 401, 1559, 619, 484, 0, 0, 0, 0, 0, 0, 0, 0, 433, 393, 638, 1032, 965, 298, 546, 740, 743, 238, 0, 0, 0, 0, 0, 0, 504, 752, 257, 709, 564, 689, 817, 366, 1669, 0, 0, 0, 0, 0, 0, 0, 249, 515, 1385, 467, 504, 1021, 1228, 770, 0, 0, 0, 0, 0, 0, 0, 0, 523, 1188, 463, 1601, 591, 863, 696, 567, 433, 143, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5646, 5817, 6611, 5518, 6026, 6327, 6139, 7068] avg=6144.0000 max=7068 max_rank=7 min=5518 min_rank=3 imbalance_ratio=1.150391 max_load_reduction=42084 max_load_reduction_pct=0.856201 moved_to_echo=25560 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1074, 853, 717, 546, 420, 289, 916, 696, 0, 0, 0, 0, 0, 0, 0, 0, 633, 863, 1017, 687, 651, 580, 649, 454, 716, 233, 0, 0, 0, 0, 0, 0, 469, 1508, 223, 1056, 733, 537, 872, 1089, 0, 0, 0, 0, 0, 0, 0, 0, 442, 323, 797, 1068, 565, 1046, 497, 761, 0, 0, 0, 0, 0, 0, 0, 0, 326, 229, 401, 1499, 1008, 421, 286, 738, 658, 243, 0, 0, 0, 0, 0, 0, 304, 863, 198, 573, 645, 456, 1563, 280, 1703, 0, 0, 0, 0, 0, 0, 0, 361, 456, 1029, 366, 605, 1299, 1181, 858, 0, 0, 0, 0, 0, 0, 0, 0, 442, 1380, 435, 1285, 515, 943, 687, 412, 290, 234, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5511, 6483, 6487, 5499, 5809, 6585, 6155, 6623] avg=6144.0000 max=6623 max_rank=7 min=5499 min_rank=3 imbalance_ratio=1.077962 max_load_reduction=42529 max_load_reduction_pct=0.865255 moved_to_echo=25172 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1325, 1221, 893, 588, 626, 256, 975, 720, 0, 0, 0, 0, 0, 0, 0, 0, 636, 602, 595, 649, 823, 467, 405, 868, 498, 241, 0, 0, 0, 0, 0, 0, 399, 1408, 196, 717, 758, 519, 973, 618, 0, 0, 0, 0, 0, 0, 0, 0, 517, 454, 1007, 1386, 388, 1527, 541, 913, 0, 0, 0, 0, 0, 0, 0, 0, 333, 333, 683, 1273, 906, 397, 414, 549, 674, 225, 0, 0, 0, 0, 0, 0, 548, 527, 180, 435, 549, 518, 1160, 554, 1551, 0, 0, 0, 0, 0, 0, 0, 385, 454, 1514, 194, 537, 1385, 988, 746, 0, 0, 0, 0, 0, 0, 0, 0, 354, 1404, 507, 1247, 592, 650, 536, 545, 430, 166, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6604, 5784, 5588, 6733, 5787, 6022, 6203, 6431] avg=6144.0000 max=6733 max_rank=3 min=5588 min_rank=2 imbalance_ratio=1.095866 max_load_reduction=42419 max_load_reduction_pct=0.863017 moved_to_echo=24443 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1000, 1235, 1039, 498, 480, 271, 964, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 609, 854, 746, 646, 576, 933, 557, 592, 528, 221, 0, 0, 0, 0, 0, 0, 430, 1264, 339, 778, 835, 559, 743, 1232, 0, 0, 0, 0, 0, 0, 0, 0, 541, 341, 814, 1141, 622, 1493, 347, 750, 0, 0, 0, 0, 0, 0, 0, 0, 559, 321, 746, 1272, 1048, 325, 439, 683, 625, 228, 0, 0, 0, 0, 0, 0, 547, 529, 186, 906, 455, 538, 868, 349, 1597, 0, 0, 0, 0, 0, 0, 0, 419, 434, 1258, 212, 605, 814, 1406, 709, 0, 0, 0, 0, 0, 0, 0, 0, 662, 1255, 380, 1149, 566, 451, 559, 404, 415, 172, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6570, 6262, 6180, 6049, 6246, 5975, 5857, 6013] avg=6144.0000 max=6570 max_rank=0 min=5857 min_rank=6 imbalance_ratio=1.069336 max_load_reduction=42582 max_load_reduction_pct=0.866333 moved_to_echo=24091 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1129, 1339, 743, 512, 552, 260, 813, 744, 0, 0, 0, 0, 0, 0, 0, 0, 706, 856, 1101, 544, 664, 801, 511, 589, 677, 249, 0, 0, 0, 0, 0, 0, 346, 1328, 260, 999, 762, 475, 880, 805, 0, 0, 0, 0, 0, 0, 0, 0, 546, 577, 826, 1207, 526, 1448, 582, 742, 0, 0, 0, 0, 0, 0, 0, 0, 356, 266, 584, 1232, 810, 435, 432, 786, 703, 218, 0, 0, 0, 0, 0, 0, 593, 799, 209, 857, 486, 404, 946, 420, 1527, 0, 0, 0, 0, 0, 0, 0, 400, 487, 1182, 232, 502, 1194, 1309, 634, 0, 0, 0, 0, 0, 0, 0, 0, 466, 1041, 449, 1282, 603, 592, 561, 433, 402, 221, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6092, 6698, 5855, 6454, 5822, 6241, 5940, 6050] avg=6144.0000 max=6698 max_rank=1 min=5822 min_rank=4 imbalance_ratio=1.090169 max_load_reduction=42454 max_load_reduction_pct=0.863729 moved_to_echo=24053 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1119, 1199, 1065, 536, 296, 214, 997, 866, 0, 0, 0, 0, 0, 0, 0, 0, 714, 639, 628, 599, 597, 822, 455, 1043, 696, 312, 0, 0, 0, 0, 0, 0, 549, 1137, 375, 1026, 859, 554, 861, 660, 0, 0, 0, 0, 0, 0, 0, 0, 717, 473, 1064, 1300, 338, 1178, 441, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 479, 378, 559, 1474, 907, 502, 564, 596, 881, 179, 0, 0, 0, 0, 0, 0, 608, 729, 333, 506, 398, 478, 982, 392, 1258, 0, 0, 0, 0, 0, 0, 0, 204, 611, 1350, 185, 391, 1208, 1375, 682, 0, 0, 0, 0, 0, 0, 0, 0, 670, 792, 331, 1271, 478, 418, 608, 449, 327, 228, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6292, 6505, 6021, 6553, 6519, 5684, 6006, 5572] avg=6144.0000 max=6553 max_rank=3 min=5572 min_rank=7 imbalance_ratio=1.066569 max_load_reduction=42599 max_load_reduction_pct=0.866679 moved_to_echo=23781 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1247, 1531, 952, 462, 435, 239, 996, 830, 0, 0, 0, 0, 0, 0, 0, 0, 610, 666, 644, 732, 760, 676, 426, 333, 610, 214, 0, 0, 0, 0, 0, 0, 403, 1190, 320, 900, 984, 515, 941, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 415, 456, 611, 1499, 459, 1377, 673, 810, 0, 0, 0, 0, 0, 0, 0, 0, 461, 355, 501, 1368, 1162, 264, 467, 1059, 605, 200, 0, 0, 0, 0, 0, 0, 533, 961, 191, 576, 568, 491, 1081, 393, 1402, 0, 0, 0, 0, 0, 0, 0, 491, 420, 1307, 233, 401, 1431, 1042, 688, 0, 0, 0, 0, 0, 0, 0, 0, 397, 1248, 386, 1252, 472, 357, 520, 339, 382, 199, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6692, 5671, 6286, 6300, 6442, 6196, 6013, 5552] avg=6144.0000 max=6692 max_rank=0 min=5552 min_rank=7 imbalance_ratio=1.089193 max_load_reduction=42460 max_load_reduction_pct=0.863851 moved_to_echo=24203 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3072, 3840, 10880, 2176, 4224, 9728, 10240, 5632, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5120, 6144, 6016, 4992, 5376, 6016, 3840, 5248, 4992, 2048, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4352, 3328, 6656, 9856, 3840, 11008, 4224, 6400, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4224, 9344, 3456, 10240, 4480, 4736, 4864, 3584, 3072, 1664, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[8832, 9472, 7040, 4096, 3712, 2176, 7680, 6528, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3584, 10624, 2432, 7296, 7040, 4608, 6656, 7424, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3584, 2816, 4608, 10496, 7936, 2944, 3712, 6272, 5504, 1792, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4224, 6016, 1792, 5376, 4096, 3968, 8192, 3456, 12544, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[489, 517, 193, 381, 348, 1181, 1204, 859, 763, 331, 736, 451, 381, 461, 898, 289, 2306, 1035, 792, 1058, 901, 3171, 1042, 382, 1076, 949, 1088, 411, 549, 1114, 362, 271, 372, 699, 911, 544, 486, 1281, 1058, 1081, 639, 519, 920, 824, 326, 605, 1054, 1040, 797, 625, 1049, 1329, 452, 396, 163, 624, 969, 809, 553, 623, 779, 272, 923, 441] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[438, 391, 104, 233, 233, 1207, 1643, 591, 1150, 262, 435, 611, 361, 369, 1043, 230, 2360, 850, 1032, 618, 932, 1838, 870, 300, 424, 599, 764, 543, 729, 565, 367, 358, 573, 280, 810, 862, 511, 1840, 1588, 1017, 1419, 548, 1131, 1629, 471, 391, 1234, 885, 601, 844, 1245, 1194, 953, 430, 178, 509, 1477, 871, 551, 643, 587, 192, 747, 491] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[586, 604, 187, 340, 239, 988, 1721, 517, 710, 439, 642, 207, 478, 482, 1167, 129, 2178, 931, 983, 564, 983, 2566, 762, 352, 537, 942, 728, 344, 488, 1194, 667, 309, 556, 552, 551, 530, 436, 1220, 1292, 1172, 906, 788, 1765, 1265, 477, 537, 1545, 743, 848, 433, 1249, 740, 593, 509, 152, 437, 1222, 1207, 855, 606, 864, 308, 363, 467] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[741, 690, 318, 264, 290, 1396, 901, 769, 932, 442, 1207, 336, 479, 632, 718, 332, 1878, 978, 1029, 825, 674, 1666, 686, 381, 858, 515, 825, 565, 849, 488, 523, 383, 654, 265, 885, 794, 507, 1248, 1014, 884, 1017, 473, 914, 1464, 351, 396, 1790, 730, 525, 1351, 1157, 777, 792, 516, 340, 478, 831, 1509, 733, 597, 696, 319, 950, 625] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[533, 780, 421, 227, 318, 738, 1300, 768, 463, 267, 800, 530, 370, 534, 860, 190, 2321, 1336, 1039, 938, 716, 1936, 1129, 382, 310, 988, 684, 381, 360, 1242, 504, 284, 396, 426, 425, 709, 436, 1463, 1665, 1187, 780, 583, 1370, 1459, 415, 564, 1787, 1307, 668, 646, 878, 765, 600, 303, 133, 603, 727, 1671, 576, 740, 1057, 245, 565, 354] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[527, 796, 148, 237, 688, 1151, 1477, 523, 636, 232, 596, 410, 305, 698, 931, 140, 1272, 964, 1468, 604, 686, 2150, 1131, 431, 1168, 1065, 728, 444, 830, 623, 478, 540, 350, 315, 581, 536, 439, 1400, 1216, 1042, 1011, 679, 1265, 1342, 245, 536, 2175, 1044, 539, 549, 1034, 836, 486, 395, 179, 621, 779, 1753, 731, 809, 845, 263, 656, 424] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[885, 625, 184, 334, 377, 1544, 1566, 497, 643, 282, 722, 538, 355, 517, 771, 216, 1632, 1468, 961, 709, 1032, 1855, 858, 535, 667, 692, 876, 381, 546, 1114, 310, 432, 401, 439, 970, 384, 617, 1102, 1397, 810, 1132, 1043, 822, 1129, 380, 486, 1665, 835, 667, 853, 935, 1137, 937, 304, 214, 573, 834, 852, 1002, 543, 491, 347, 1123, 604] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[852, 567, 263, 281, 420, 1576, 1279, 569, 963, 282, 589, 447, 392, 416, 580, 190, 1923, 1366, 988, 1224, 547, 2099, 818, 341, 558, 739, 735, 413, 621, 1255, 367, 428, 342, 262, 526, 553, 734, 1639, 828, 826, 1518, 622, 919, 1343, 348, 610, 1618, 898, 511, 1269, 840, 937, 689, 413, 204, 727, 880, 1301, 606, 546, 915, 402, 982, 256] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[438, 391, 104, 233, 233, 1207, 1643, 591, 912, 0, 0, 0, 0, 0, 0, 0, 1150, 262, 435, 611, 361, 369, 1043, 230, 1614, 0, 0, 0, 0, 0, 0, 0, 1269, 850, 1032, 618, 932, 0, 870, 300, 0, 0, 0, 0, 0, 0, 0, 0, 424, 599, 764, 543, 729, 565, 367, 358, 566, 224, 0, 0, 0, 0, 0, 0, 573, 280, 810, 862, 511, 1555, 1588, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1419, 548, 1131, 1629, 471, 391, 125, 885, 0, 0, 0, 0, 0, 0, 0, 0, 601, 844, 1245, 1194, 953, 430, 178, 509, 543, 112, 0, 0, 0, 0, 0, 0, 1477, 871, 551, 643, 587, 192, 747, 491, 285, 67, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5752, 6075, 5871, 5139, 7196, 6599, 6609, 5911] avg=6144.0000 max=7196 max_rank=4 min=5139 min_rank=3 imbalance_ratio=1.171224 max_load_reduction=41956 max_load_reduction_pct=0.853597 moved_to_echo=26315 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[489, 517, 193, 381, 348, 1181, 1204, 859, 891, 0, 0, 0, 0, 0, 0, 0, 763, 331, 736, 451, 381, 461, 898, 289, 2786, 0, 0, 0, 0, 0, 0, 0, 1240, 1035, 792, 1058, 901, 0, 1042, 382, 0, 0, 0, 0, 0, 0, 0, 0, 1076, 949, 1088, 411, 549, 1114, 362, 271, 483, 385, 0, 0, 0, 0, 0, 0, 372, 699, 911, 544, 486, 1083, 1058, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 639, 519, 920, 824, 326, 605, 107, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 797, 625, 1049, 1329, 452, 396, 163, 624, 464, 109, 0, 0, 0, 0, 0, 0, 969, 809, 553, 623, 779, 272, 923, 441, 198, 66, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6063, 7096, 6450, 6688, 6234, 4980, 6008, 5633] avg=6144.0000 max=7096 max_rank=1 min=4980 min_rank=5 imbalance_ratio=1.154948 max_load_reduction=42056 max_load_reduction_pct=0.855632 moved_to_echo=22855 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[741, 690, 318, 264, 290, 1396, 901, 769, 726, 0, 0, 0, 0, 0, 0, 0, 932, 442, 1207, 336, 479, 632, 718, 332, 1463, 0, 0, 0, 0, 0, 0, 0, 1010, 978, 1029, 825, 674, 0, 686, 381, 0, 0, 0, 0, 0, 0, 0, 0, 858, 515, 825, 565, 849, 488, 523, 383, 821, 203, 0, 0, 0, 0, 0, 0, 654, 265, 885, 794, 507, 1055, 1014, 884, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 473, 914, 1464, 351, 396, 181, 730, 0, 0, 0, 0, 0, 0, 0, 0, 525, 1351, 1157, 777, 792, 516, 340, 478, 788, 89, 0, 0, 0, 0, 0, 0, 831, 1509, 733, 597, 696, 319, 950, 625, 193, 53, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6095, 6541, 5583, 6030, 6058, 5526, 6813, 6506] avg=6144.0000 max=6813 max_rank=6 min=5526 min_rank=5 imbalance_ratio=1.108887 max_load_reduction=42339 max_load_reduction_pct=0.861389 moved_to_echo=24903 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[586, 604, 187, 340, 239, 988, 1721, 517, 842, 0, 0, 0, 0, 0, 0, 0, 710, 439, 642, 207, 478, 482, 1167, 129, 2255, 0, 0, 0, 0, 0, 0, 0, 1171, 931, 983, 564, 983, 0, 762, 352, 0, 0, 0, 0, 0, 0, 0, 0, 537, 942, 728, 344, 488, 1194, 667, 309, 709, 311, 0, 0, 0, 0, 0, 0, 556, 552, 551, 530, 436, 1031, 1292, 1172, 0, 0, 0, 0, 0, 0, 0, 0, 906, 788, 1765, 1265, 477, 537, 156, 743, 0, 0, 0, 0, 0, 0, 0, 0, 848, 433, 1249, 740, 593, 509, 152, 437, 680, 103, 0, 0, 0, 0, 0, 0, 1222, 1207, 855, 606, 864, 308, 363, 467, 189, 62, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6024, 6509, 5746, 6229, 6120, 6637, 5744, 6143] avg=6144.0000 max=6637 max_rank=5 min=5744 min_rank=6 imbalance_ratio=1.080241 max_load_reduction=42515 max_load_reduction_pct=0.864970 moved_to_echo=24644 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[527, 796, 148, 237, 688, 1151, 1477, 523, 496, 0, 0, 0, 0, 0, 0, 0, 636, 232, 596, 410, 305, 698, 931, 140, 1890, 0, 0, 0, 0, 0, 0, 0, 671, 964, 1468, 604, 686, 0, 1131, 431, 0, 0, 0, 0, 0, 0, 0, 0, 1168, 1065, 728, 444, 830, 623, 478, 540, 1003, 260, 0, 0, 0, 0, 0, 0, 350, 315, 581, 536, 439, 1180, 1216, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 679, 1265, 1342, 245, 536, 212, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 539, 549, 1034, 836, 486, 395, 179, 621, 960, 64, 0, 0, 0, 0, 0, 0, 779, 1753, 731, 809, 845, 263, 656, 424, 220, 41, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6043, 5838, 5955, 7139, 5659, 6334, 5663, 6521] avg=6144.0000 max=7139 max_rank=3 min=5659 min_rank=4 imbalance_ratio=1.161947 max_load_reduction=42013 max_load_reduction_pct=0.854757 moved_to_echo=24177 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[885, 625, 184, 334, 377, 1544, 1566, 497, 631, 0, 0, 0, 0, 0, 0, 0, 643, 282, 722, 538, 355, 517, 771, 216, 1630, 0, 0, 0, 0, 0, 0, 0, 878, 1468, 961, 709, 1032, 0, 858, 535, 0, 0, 0, 0, 0, 0, 0, 0, 667, 692, 876, 381, 546, 1114, 310, 432, 764, 225, 0, 0, 0, 0, 0, 0, 401, 439, 970, 384, 617, 931, 1397, 810, 0, 0, 0, 0, 0, 0, 0, 0, 1132, 1043, 822, 1129, 380, 486, 168, 835, 0, 0, 0, 0, 0, 0, 0, 0, 667, 853, 935, 1137, 937, 304, 214, 573, 733, 77, 0, 0, 0, 0, 0, 0, 834, 852, 1002, 543, 491, 347, 1123, 604, 171, 46, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6643, 5674, 6441, 6007, 5949, 5995, 6430, 6013] avg=6144.0000 max=6643 max_rank=0 min=5674 min_rank=1 imbalance_ratio=1.081217 max_load_reduction=42509 max_load_reduction_pct=0.864848 moved_to_echo=24387 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[533, 780, 421, 227, 318, 738, 1300, 768, 897, 0, 0, 0, 0, 0, 0, 0, 463, 267, 800, 530, 370, 534, 860, 190, 1702, 0, 0, 0, 0, 0, 0, 0, 1248, 1336, 1039, 938, 716, 0, 1129, 382, 0, 0, 0, 0, 0, 0, 0, 0, 310, 988, 684, 381, 360, 1242, 504, 284, 820, 234, 0, 0, 0, 0, 0, 0, 396, 426, 425, 709, 436, 1236, 1665, 1187, 0, 0, 0, 0, 0, 0, 0, 0, 780, 583, 1370, 1459, 415, 564, 181, 1307, 0, 0, 0, 0, 0, 0, 0, 0, 668, 646, 878, 765, 600, 303, 133, 603, 786, 110, 0, 0, 0, 0, 0, 0, 727, 1671, 576, 740, 1057, 245, 565, 354, 227, 66, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5982, 5716, 6788, 5807, 6480, 6659, 5492, 6228] avg=6144.0000 max=6788 max_rank=2 min=5492 min_rank=6 imbalance_ratio=1.104818 max_load_reduction=42364 max_load_reduction_pct=0.861898 moved_to_echo=24859 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[852, 567, 263, 281, 420, 1576, 1279, 569, 743, 0, 0, 0, 0, 0, 0, 0, 963, 282, 589, 447, 392, 416, 580, 190, 1844, 0, 0, 0, 0, 0, 0, 0, 1034, 1366, 988, 1224, 547, 0, 818, 341, 0, 0, 0, 0, 0, 0, 0, 0, 558, 739, 735, 413, 621, 1255, 367, 428, 742, 255, 0, 0, 0, 0, 0, 0, 342, 262, 526, 553, 734, 1385, 828, 826, 0, 0, 0, 0, 0, 0, 0, 0, 1518, 622, 919, 1343, 348, 610, 164, 898, 0, 0, 0, 0, 0, 0, 0, 0, 511, 1269, 840, 937, 689, 413, 204, 727, 712, 91, 0, 0, 0, 0, 0, 0, 880, 1301, 606, 546, 915, 402, 982, 256, 254, 55, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6550, 5703, 6318, 6113, 5456, 6422, 6393, 6197] avg=6144.0000 max=6550 max_rank=0 min=5456 min_rank=4 imbalance_ratio=1.066081 max_load_reduction=42602 max_load_reduction_pct=0.866740 moved_to_echo=24468 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5248, 6656, 8448, 7808, 5504, 3328, 1664, 4608, 5760, 768, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6272, 2560, 5760, 3584, 3200, 4224, 7040, 1792, 15232, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[8576, 8960, 8320, 6656, 6528, 0, 7296, 3200, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[8448, 5376, 9216, 10496, 3072, 4224, 1408, 7552, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7808, 9984, 5632, 5120, 6272, 2432, 6400, 3712, 1792, 512, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[5120, 4992, 1920, 2304, 2944, 9856, 11136, 5120, 6144, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3712, 3328, 5760, 4992, 4224, 9472, 10112, 8064, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5632, 6528, 6528, 3584, 4992, 7680, 3584, 3072, 6016, 2176, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1972, 560, 1669, 917, 360, 1297, 376, 1320, 1026, 991, 748, 622, 807, 473, 449, 424, 786, 1620, 1276, 766, 769, 733, 1375, 897, 279, 427, 280, 269, 1717, 395, 1497, 124, 521, 473, 735, 363, 624, 850, 903, 543, 365, 1030, 552, 260, 716, 411, 348, 480, 641, 272, 520, 935, 1004, 733, 480, 276, 813, 1384, 414, 934, 992, 1325, 1301, 733] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1178, 648, 2087, 752, 231, 992, 797, 1385, 1104, 1132, 691, 809, 1152, 363, 627, 486, 477, 665, 625, 691, 557, 2043, 1244, 1301, 740, 436, 524, 709, 695, 371, 1685, 212, 494, 631, 863, 584, 689, 636, 562, 1056, 734, 1469, 105, 213, 1102, 264, 305, 371, 824, 334, 545, 919, 1445, 602, 297, 300, 688, 460, 444, 251, 542, 1088, 2356, 570] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1922, 311, 2346, 691, 273, 1164, 463, 1594, 1006, 721, 858, 903, 1331, 667, 730, 319, 1551, 1086, 1655, 1129, 270, 1159, 1231, 1865, 219, 347, 652, 176, 473, 480, 899, 217, 767, 224, 1004, 318, 759, 1068, 533, 521, 302, 1580, 362, 246, 343, 322, 246, 342, 513, 199, 631, 707, 659, 1114, 378, 255, 511, 2148, 94, 894, 1117, 817, 957, 513] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1582, 406, 1703, 682, 336, 1035, 573, 1231, 815, 695, 1018, 1049, 905, 537, 566, 698, 1119, 905, 1554, 705, 689, 1288, 1143, 989, 928, 137, 583, 189, 938, 609, 1381, 198, 434, 299, 1343, 331, 905, 444, 695, 853, 1059, 1009, 403, 161, 408, 357, 335, 591, 757, 125, 504, 990, 1109, 1013, 503, 258, 673, 684, 228, 716, 1370, 1420, 1522, 467] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1826, 652, 1625, 748, 357, 1311, 515, 1850, 1071, 872, 973, 348, 1076, 508, 670, 505, 1101, 1278, 900, 1062, 654, 1034, 984, 1291, 321, 897, 286, 290, 893, 477, 1214, 147, 662, 629, 764, 386, 707, 1099, 555, 908, 233, 1430, 240, 239, 1014, 424, 328, 357, 688, 235, 325, 636, 403, 673, 527, 389, 432, 2033, 281, 578, 1101, 1265, 1078, 797] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1866, 605, 1454, 999, 264, 772, 484, 1174, 809, 831, 810, 472, 771, 619, 830, 760, 1084, 575, 1186, 1200, 553, 620, 1253, 1672, 537, 485, 861, 508, 704, 253, 1507, 191, 632, 728, 928, 840, 655, 825, 881, 904, 508, 1497, 209, 217, 685, 166, 489, 391, 705, 173, 552, 1004, 858, 555, 239, 525, 547, 1173, 202, 419, 1084, 1572, 1533, 747] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1595, 925, 1851, 412, 570, 1078, 494, 1486, 1031, 974, 741, 840, 713, 596, 1326, 224, 1158, 851, 1718, 757, 540, 1661, 1091, 1138, 482, 331, 709, 422, 552, 185, 1414, 196, 616, 361, 1079, 428, 505, 911, 639, 1030, 349, 1400, 357, 137, 333, 134, 158, 674, 1443, 268, 707, 707, 1055, 402, 491, 339, 777, 821, 543, 498, 724, 1227, 1321, 657] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1983, 662, 1939, 558, 358, 441, 429, 1629, 1339, 714, 1031, 618, 670, 373, 757, 544, 954, 1154, 1347, 896, 457, 772, 1175, 2364, 509, 172, 841, 404, 707, 444, 1624, 136, 633, 278, 684, 229, 614, 891, 784, 1128, 690, 696, 416, 278, 408, 266, 184, 829, 485, 187, 681, 698, 1089, 1215, 574, 403, 543, 620, 407, 859, 1582, 1188, 791, 821] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1656, 311, 0, 691, 273, 1164, 463, 1594, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 721, 858, 903, 1331, 667, 730, 319, 1, 0, 0, 0, 0, 0, 0, 0, 1551, 1086, 370, 1129, 270, 1159, 1231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 219, 347, 652, 176, 473, 480, 899, 217, 1549, 338, 0, 0, 0, 0, 0, 0, 767, 224, 1004, 318, 759, 1068, 533, 521, 289, 265, 0, 0, 0, 0, 0, 0, 302, 1580, 362, 246, 343, 322, 246, 342, 2346, 316, 0, 0, 0, 0, 0, 0, 513, 199, 631, 707, 659, 1114, 378, 255, 947, 370, 0, 0, 0, 0, 0, 0, 511, 2148, 94, 894, 1117, 817, 298, 513, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6152, 6536, 6796, 5350, 5748, 6405, 5773, 6392] avg=6144.0000 max=6796 max_rank=2 min=5350 min_rank=3 imbalance_ratio=1.106120 max_load_reduction=42356 max_load_reduction_pct=0.861735 moved_to_echo=24318 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1699, 560, 0, 917, 360, 1297, 376, 1320, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 991, 748, 622, 807, 473, 449, 424, 1, 0, 0, 0, 0, 0, 0, 0, 786, 1620, 285, 766, 769, 733, 1375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 279, 427, 280, 269, 1717, 395, 1497, 124, 745, 261, 0, 0, 0, 0, 0, 0, 521, 473, 735, 363, 624, 850, 903, 543, 394, 272, 0, 0, 0, 0, 0, 0, 365, 1030, 552, 260, 716, 411, 348, 480, 1669, 152, 0, 0, 0, 0, 0, 0, 641, 272, 520, 935, 1004, 733, 480, 276, 730, 503, 0, 0, 0, 0, 0, 0, 813, 1384, 414, 934, 992, 1325, 404, 733, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6529, 5541, 6334, 5994, 5678, 5983, 6094, 6999] avg=6144.0000 max=6999 max_rank=7 min=5541 min_rank=1 imbalance_ratio=1.139160 max_load_reduction=42153 max_load_reduction_pct=0.857605 moved_to_echo=24754 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1015, 648, 0, 752, 231, 992, 797, 1385, 0, 0, 0, 0, 0, 0, 0, 0, 1104, 1132, 691, 809, 1152, 363, 627, 486, 1, 0, 0, 0, 0, 0, 0, 0, 477, 665, 141, 691, 557, 2043, 1244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 740, 436, 524, 709, 695, 371, 1685, 212, 1080, 127, 0, 0, 0, 0, 0, 0, 494, 631, 863, 584, 689, 636, 562, 1056, 713, 162, 0, 0, 0, 0, 0, 0, 734, 1469, 105, 213, 1102, 264, 305, 371, 2087, 221, 0, 0, 0, 0, 0, 0, 824, 334, 545, 919, 1445, 602, 297, 300, 357, 912, 0, 0, 0, 0, 0, 0, 688, 460, 444, 251, 542, 1088, 731, 570, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5820, 6365, 5818, 6579, 6390, 6871, 6535, 4774] avg=6144.0000 max=6871 max_rank=5 min=4774 min_rank=7 imbalance_ratio=1.118327 max_load_reduction=42281 max_load_reduction_pct=0.860209 moved_to_echo=24570 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1352, 406, 0, 682, 336, 1035, 573, 1231, 0, 0, 0, 0, 0, 0, 0, 0, 815, 695, 1018, 1049, 905, 537, 566, 698, 7, 0, 0, 0, 0, 0, 0, 0, 1119, 905, 338, 705, 689, 1288, 1143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 928, 137, 583, 189, 938, 609, 1381, 198, 822, 323, 0, 0, 0, 0, 0, 0, 434, 299, 1343, 331, 905, 444, 695, 853, 464, 223, 0, 0, 0, 0, 0, 0, 1059, 1009, 403, 161, 408, 357, 335, 591, 1703, 167, 0, 0, 0, 0, 0, 0, 757, 125, 504, 990, 1109, 1013, 503, 258, 893, 594, 0, 0, 0, 0, 0, 0, 673, 684, 228, 716, 1370, 1420, 464, 467, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5615, 6290, 6187, 6108, 5991, 6193, 6746, 6022] avg=6144.0000 max=6746 max_rank=6 min=5615 min_rank=0 imbalance_ratio=1.097982 max_load_reduction=42406 max_load_reduction_pct=0.862752 moved_to_echo=24952 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1573, 652, 0, 748, 357, 1311, 515, 1850, 0, 0, 0, 0, 0, 0, 0, 0, 1071, 872, 973, 348, 1076, 508, 670, 505, 1, 0, 0, 0, 0, 0, 0, 0, 1101, 1278, 201, 1062, 654, 1034, 984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 321, 897, 286, 290, 893, 477, 1214, 147, 1072, 184, 0, 0, 0, 0, 0, 0, 662, 629, 764, 386, 707, 1099, 555, 908, 326, 252, 0, 0, 0, 0, 0, 0, 233, 1430, 240, 239, 1014, 424, 328, 357, 1625, 219, 0, 0, 0, 0, 0, 0, 688, 235, 325, 636, 403, 673, 527, 389, 515, 417, 0, 0, 0, 0, 0, 0, 432, 2033, 281, 578, 1101, 1265, 335, 797, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7006, 6024, 6314, 5781, 6288, 6109, 4808, 6822] avg=6144.0000 max=7006 max_rank=0 min=4808 min_rank=6 imbalance_ratio=1.140299 max_load_reduction=42146 max_load_reduction_pct=0.857463 moved_to_echo=24027 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1608, 605, 0, 999, 264, 772, 484, 1174, 0, 0, 0, 0, 0, 0, 0, 0, 809, 831, 810, 472, 771, 619, 830, 760, 1, 0, 0, 0, 0, 0, 0, 0, 1084, 575, 266, 1200, 553, 620, 1253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 485, 861, 508, 704, 253, 1507, 191, 1389, 242, 0, 0, 0, 0, 0, 0, 632, 728, 928, 840, 655, 825, 881, 904, 464, 257, 0, 0, 0, 0, 0, 0, 508, 1497, 209, 217, 685, 166, 489, 391, 1454, 283, 0, 0, 0, 0, 0, 0, 705, 173, 552, 1004, 858, 555, 239, 525, 678, 593, 0, 0, 0, 0, 0, 0, 547, 1173, 202, 419, 1084, 1572, 476, 747, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5906, 5903, 5551, 6677, 7114, 5899, 5882, 6220] avg=6144.0000 max=7114 max_rank=4 min=5551 min_rank=2 imbalance_ratio=1.157878 max_load_reduction=42038 max_load_reduction_pct=0.855265 moved_to_echo=25115 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1708, 662, 0, 558, 358, 441, 429, 1629, 0, 0, 0, 0, 0, 0, 0, 0, 1339, 714, 1031, 618, 670, 373, 757, 544, 1, 0, 0, 0, 0, 0, 0, 0, 954, 1154, 302, 896, 457, 772, 1175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 172, 841, 404, 707, 444, 1624, 136, 1964, 275, 0, 0, 0, 0, 0, 0, 633, 278, 684, 229, 614, 891, 784, 1128, 239, 274, 0, 0, 0, 0, 0, 0, 690, 696, 416, 278, 408, 266, 184, 829, 1939, 400, 0, 0, 0, 0, 0, 0, 485, 187, 681, 698, 1089, 1215, 574, 403, 770, 306, 0, 0, 0, 0, 0, 0, 543, 620, 407, 859, 1582, 1188, 246, 821, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5785, 6047, 5710, 7076, 5754, 6106, 6408, 6266] avg=6144.0000 max=7076 max_rank=3 min=5710 min_rank=2 imbalance_ratio=1.151693 max_load_reduction=42076 max_load_reduction_pct=0.856038 moved_to_echo=24534 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1374, 925, 0, 412, 570, 1078, 494, 1486, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 974, 741, 840, 713, 596, 1326, 224, 1, 0, 0, 0, 0, 0, 0, 0, 1158, 851, 384, 757, 540, 1661, 1091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 331, 709, 422, 552, 185, 1414, 196, 945, 351, 0, 0, 0, 0, 0, 0, 616, 361, 1079, 428, 505, 911, 639, 1030, 400, 220, 0, 0, 0, 0, 0, 0, 349, 1400, 357, 137, 333, 134, 158, 674, 1851, 193, 0, 0, 0, 0, 0, 0, 1443, 268, 707, 707, 1055, 402, 491, 339, 983, 511, 0, 0, 0, 0, 0, 0, 777, 821, 543, 498, 724, 1227, 410, 657, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6339, 6446, 6442, 5587, 6189, 5586, 6906, 5657] avg=6144.0000 max=6906 max_rank=6 min=5586 min_rank=5 imbalance_ratio=1.124023 max_load_reduction=42246 max_load_reduction_pct=0.859497 moved_to_echo=24338 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[12032, 4864, 0, 5760, 2816, 8192, 4224, 11776, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[4096, 3328, 4736, 3072, 6784, 3328, 11264, 1536, 9600, 2176, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6144, 1920, 4480, 6656, 7680, 6400, 3584, 2816, 5888, 4224, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4352, 10112, 2688, 1792, 5120, 2432, 2432, 4096, 14720, 2048, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4992, 9344, 2688, 5248, 8576, 9984, 3456, 5376, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[4864, 3712, 7424, 3584, 5504, 6784, 5632, 7040, 3328, 2048, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[8320, 7040, 6912, 5760, 7552, 4224, 6016, 3968, 128, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[8320, 8192, 2304, 7296, 4608, 9344, 9600, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[457, 1357, 334, 830, 233, 288, 259, 319, 340, 113, 730, 407, 1054, 2015, 1087, 836, 626, 229, 285, 306, 1287, 1040, 130, 706, 555, 854, 1206, 1374, 1042, 1835, 1232, 725, 1644, 945, 1065, 270, 471, 1396, 692, 1542, 681, 472, 419, 759, 448, 589, 933, 832, 385, 541, 89, 738, 1121, 865, 1005, 539, 938, 304, 2237, 265, 84, 1462, 395, 935] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1225, 895, 345, 565, 268, 177, 299, 294, 566, 97, 760, 239, 1134, 712, 1106, 707, 451, 535, 373, 406, 1102, 1177, 321, 956, 421, 969, 1598, 1169, 1547, 1262, 1264, 430, 2083, 582, 595, 422, 437, 1346, 565, 1062, 536, 1469, 542, 911, 700, 444, 735, 379, 333, 465, 179, 1089, 1327, 799, 1767, 777, 597, 399, 2642, 433, 233, 908, 466, 560] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1236, 976, 390, 566, 370, 137, 179, 1527, 715, 148, 1328, 317, 546, 1103, 1575, 575, 296, 371, 650, 365, 919, 1388, 465, 492, 821, 450, 1862, 765, 840, 1514, 1012, 347, 1554, 506, 920, 288, 897, 1558, 1399, 1231, 904, 1757, 237, 991, 339, 728, 823, 578, 315, 824, 55, 976, 624, 527, 809, 598, 677, 113, 1940, 460, 838, 463, 470, 508] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[549, 808, 571, 660, 281, 98, 385, 662, 386, 122, 879, 291, 899, 1367, 1612, 504, 343, 586, 560, 428, 1161, 716, 260, 407, 814, 1223, 1306, 1111, 1021, 1505, 1065, 605, 1887, 755, 1112, 353, 768, 1536, 1339, 1194, 587, 1470, 451, 662, 318, 861, 831, 663, 409, 1099, 61, 978, 741, 311, 854, 713, 586, 228, 1840, 545, 557, 861, 396, 1001] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[812, 568, 699, 477, 210, 147, 90, 934, 575, 178, 1347, 545, 362, 1038, 1662, 331, 248, 441, 184, 288, 1422, 1765, 357, 601, 434, 984, 946, 1399, 1340, 1652, 1230, 752, 1663, 251, 1943, 776, 1087, 1735, 476, 1155, 314, 738, 274, 650, 1091, 386, 874, 707, 163, 640, 75, 923, 1606, 381, 1098, 519, 1111, 243, 1737, 524, 518, 480, 275, 721] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1463, 1361, 549, 554, 182, 166, 498, 381, 515, 255, 796, 336, 717, 1286, 1729, 840, 295, 533, 195, 560, 1041, 802, 399, 923, 514, 506, 920, 1040, 1154, 1543, 1471, 949, 938, 650, 1135, 373, 613, 2146, 847, 1261, 414, 1681, 476, 676, 336, 536, 635, 570, 517, 848, 193, 1173, 959, 622, 1164, 457, 648, 601, 1273, 365, 432, 890, 599, 651] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[514, 1592, 368, 855, 383, 435, 173, 649, 682, 152, 1324, 487, 785, 928, 1194, 878, 389, 389, 540, 379, 1869, 1447, 208, 638, 807, 553, 1087, 892, 1133, 996, 708, 494, 989, 284, 1023, 154, 854, 1918, 315, 1564, 291, 744, 493, 1294, 629, 298, 557, 258, 205, 1031, 32, 1867, 1801, 571, 1759, 411, 765, 837, 1791, 753, 304, 459, 304, 669] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1010, 880, 267, 707, 379, 178, 293, 930, 695, 136, 1676, 286, 1197, 1340, 1387, 677, 479, 511, 568, 287, 1764, 915, 175, 790, 482, 914, 1585, 877, 1076, 1133, 1261, 687, 1095, 675, 397, 430, 876, 1842, 594, 841, 181, 1041, 567, 848, 599, 618, 631, 233, 292, 645, 54, 923, 1760, 256, 1861, 633, 772, 267, 1229, 966, 549, 599, 447, 889] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1236, 976, 390, 566, 370, 137, 179, 1527, 1492, 0, 0, 0, 0, 0, 0, 0, 715, 148, 1328, 317, 546, 1103, 1374, 575, 0, 0, 0, 0, 0, 0, 0, 0, 296, 371, 650, 365, 919, 1388, 465, 492, 1038, 66, 0, 0, 0, 0, 0, 0, 821, 450, 1036, 765, 840, 0, 1012, 347, 0, 0, 0, 0, 0, 0, 0, 0, 1465, 506, 920, 288, 897, 0, 1399, 1231, 0, 0, 0, 0, 0, 0, 0, 0, 904, 1757, 237, 991, 339, 728, 823, 578, 476, 507, 0, 0, 0, 0, 0, 0, 315, 824, 55, 976, 624, 527, 809, 598, 154, 89, 0, 0, 0, 0, 0, 0, 677, 113, 1940, 460, 838, 463, 470, 508, 319, 47, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6873, 6106, 6050, 5271, 6706, 7340, 4971, 5835] avg=6144.0000 max=7340 max_rank=5 min=4971 min_rank=6 imbalance_ratio=1.194661 max_load_reduction=41812 max_load_reduction_pct=0.850667 moved_to_echo=24852 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[457, 1357, 334, 830, 233, 288, 259, 319, 1337, 0, 0, 0, 0, 0, 0, 0, 340, 113, 730, 407, 1054, 2015, 949, 836, 0, 0, 0, 0, 0, 0, 0, 0, 626, 229, 285, 306, 1287, 1040, 130, 706, 1258, 59, 0, 0, 0, 0, 0, 0, 555, 854, 671, 1374, 1042, 0, 1232, 725, 0, 0, 0, 0, 0, 0, 0, 0, 1549, 945, 1065, 270, 471, 0, 692, 1542, 0, 0, 0, 0, 0, 0, 0, 0, 681, 472, 419, 759, 448, 589, 933, 832, 577, 328, 0, 0, 0, 0, 0, 0, 385, 541, 89, 738, 1121, 865, 1005, 539, 106, 95, 0, 0, 0, 0, 0, 0, 938, 304, 2237, 265, 84, 1462, 395, 935, 207, 32, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5414, 6444, 5926, 6453, 6534, 6038, 5484, 6859] avg=6144.0000 max=6859 max_rank=7 min=5414 min_rank=0 imbalance_ratio=1.116374 max_load_reduction=42293 max_load_reduction_pct=0.860453 moved_to_echo=24915 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1463, 1361, 549, 554, 182, 166, 498, 381, 2055, 0, 0, 0, 0, 0, 0, 0, 515, 255, 796, 336, 717, 1286, 1509, 840, 0, 0, 0, 0, 0, 0, 0, 0, 295, 533, 195, 560, 1041, 802, 399, 923, 1057, 91, 0, 0, 0, 0, 0, 0, 514, 506, 513, 1040, 1154, 0, 1471, 949, 0, 0, 0, 0, 0, 0, 0, 0, 884, 650, 1135, 373, 613, 0, 847, 1261, 0, 0, 0, 0, 0, 0, 0, 0, 414, 1681, 476, 676, 336, 536, 635, 570, 486, 250, 0, 0, 0, 0, 0, 0, 517, 848, 193, 1173, 959, 622, 1164, 457, 169, 54, 0, 0, 0, 0, 0, 0, 648, 601, 1273, 365, 432, 890, 599, 651, 157, 51, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7209, 6254, 5896, 6147, 5763, 6060, 6156, 5667] avg=6144.0000 max=7209 max_rank=0 min=5667 min_rank=7 imbalance_ratio=1.173340 max_load_reduction=41943 max_load_reduction_pct=0.853333 moved_to_echo=23646 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1225, 895, 345, 565, 268, 177, 299, 294, 1289, 0, 0, 0, 0, 0, 0, 0, 566, 97, 760, 239, 1134, 712, 965, 707, 0, 0, 0, 0, 0, 0, 0, 0, 451, 535, 373, 406, 1102, 1177, 321, 956, 866, 57, 0, 0, 0, 0, 0, 0, 421, 969, 889, 1169, 1547, 0, 1264, 430, 0, 0, 0, 0, 0, 0, 0, 0, 1963, 582, 595, 422, 437, 0, 565, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 536, 1469, 542, 911, 700, 444, 735, 379, 396, 435, 0, 0, 0, 0, 0, 0, 333, 465, 179, 1089, 1327, 799, 1767, 777, 108, 120, 0, 0, 0, 0, 0, 0, 597, 399, 2642, 433, 233, 908, 466, 560, 274, 33, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5357, 5180, 6244, 6689, 5626, 6547, 6964, 6545] avg=6144.0000 max=6964 max_rank=6 min=5180 min_rank=1 imbalance_ratio=1.133464 max_load_reduction=42188 max_load_reduction_pct=0.858317 moved_to_echo=25682 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[549, 808, 571, 660, 281, 98, 385, 662, 1472, 0, 0, 0, 0, 0, 0, 0, 386, 122, 879, 291, 899, 1367, 1406, 504, 0, 0, 0, 0, 0, 0, 0, 0, 343, 586, 560, 428, 1161, 716, 260, 407, 1032, 64, 0, 0, 0, 0, 0, 0, 814, 1223, 726, 1111, 1021, 0, 1065, 605, 0, 0, 0, 0, 0, 0, 0, 0, 1778, 755, 1112, 353, 768, 0, 1339, 1194, 0, 0, 0, 0, 0, 0, 0, 0, 587, 1470, 451, 662, 318, 861, 831, 663, 473, 356, 0, 0, 0, 0, 0, 0, 409, 1099, 61, 978, 741, 311, 854, 713, 158, 109, 0, 0, 0, 0, 0, 0, 586, 228, 1840, 545, 557, 861, 396, 1001, 224, 48, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5486, 5854, 5557, 6565, 7299, 6672, 5433, 6286] avg=6144.0000 max=7299 max_rank=4 min=5433 min_rank=6 imbalance_ratio=1.187988 max_load_reduction=41853 max_load_reduction_pct=0.851501 moved_to_echo=25690 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[514, 1592, 368, 855, 383, 435, 173, 649, 1838, 0, 0, 0, 0, 0, 0, 0, 682, 152, 1324, 487, 785, 928, 1035, 878, 0, 0, 0, 0, 0, 0, 0, 0, 389, 389, 540, 379, 1869, 1447, 208, 638, 683, 80, 0, 0, 0, 0, 0, 0, 807, 553, 597, 892, 1133, 0, 708, 494, 0, 0, 0, 0, 0, 0, 0, 0, 930, 284, 1023, 154, 854, 0, 315, 1564, 0, 0, 0, 0, 0, 0, 0, 0, 291, 744, 493, 1294, 629, 298, 557, 258, 313, 301, 0, 0, 0, 0, 0, 0, 205, 1031, 32, 1867, 1801, 571, 1759, 411, 121, 59, 0, 0, 0, 0, 0, 0, 765, 837, 1791, 753, 304, 459, 304, 669, 189, 38, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6807, 6271, 6622, 5184, 5124, 5178, 7857, 6109] avg=6144.0000 max=7857 max_rank=6 min=5124 min_rank=4 imbalance_ratio=1.278809 max_load_reduction=41295 max_load_reduction_pct=0.840149 moved_to_echo=24268 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1010, 880, 267, 707, 379, 178, 293, 930, 1764, 0, 0, 0, 0, 0, 0, 0, 695, 136, 1676, 286, 1197, 1340, 1210, 677, 0, 0, 0, 0, 0, 0, 0, 0, 479, 511, 568, 287, 1764, 915, 175, 790, 777, 78, 0, 0, 0, 0, 0, 0, 482, 914, 881, 877, 1076, 0, 1261, 687, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 675, 397, 430, 876, 0, 594, 841, 0, 0, 0, 0, 0, 0, 0, 0, 181, 1041, 567, 848, 599, 618, 631, 233, 356, 432, 0, 0, 0, 0, 0, 0, 292, 645, 54, 923, 1760, 256, 1861, 633, 136, 63, 0, 0, 0, 0, 0, 0, 772, 267, 1229, 966, 549, 599, 447, 889, 272, 41, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6408, 7217, 6344, 6178, 4845, 5506, 6623, 6031] avg=6144.0000 max=7217 max_rank=1 min=4845 min_rank=4 imbalance_ratio=1.174642 max_load_reduction=41935 max_load_reduction_pct=0.853170 moved_to_echo=23005 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[812, 568, 699, 477, 210, 147, 90, 934, 1661, 0, 0, 0, 0, 0, 0, 0, 575, 178, 1347, 545, 362, 1038, 1450, 331, 0, 0, 0, 0, 0, 0, 0, 0, 248, 441, 184, 288, 1422, 1765, 357, 601, 1133, 74, 0, 0, 0, 0, 0, 0, 434, 984, 526, 1399, 1340, 0, 1230, 752, 0, 0, 0, 0, 0, 0, 0, 0, 1567, 251, 1943, 776, 1087, 0, 476, 1155, 0, 0, 0, 0, 0, 0, 0, 0, 314, 738, 274, 650, 1091, 386, 874, 707, 519, 258, 0, 0, 0, 0, 0, 0, 163, 640, 75, 923, 1606, 381, 1098, 519, 163, 96, 0, 0, 0, 0, 0, 0, 1111, 243, 1737, 524, 518, 480, 275, 721, 162, 49, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5598, 5826, 6513, 6665, 7255, 5811, 5664, 5820] avg=6144.0000 max=7255 max_rank=4 min=5598 min_rank=0 imbalance_ratio=1.180827 max_load_reduction=41897 max_load_reduction_pct=0.852397 moved_to_echo=24550 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4864, 6528, 5888, 8704, 9216, 0, 9344, 4992, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3968, 9472, 3584, 6912, 4480, 4480, 6144, 4224, 3712, 2944, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[3200, 3712, 3456, 3072, 10624, 9344, 2432, 5632, 7936, 640, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[11264, 4736, 8192, 3072, 6016, 0, 6272, 9856, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4480, 1280, 8960, 2944, 6784, 9856, 9984, 5376, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2688, 6144, 768, 8704, 9984, 4352, 10368, 4736, 1152, 768, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[7296, 8448, 3584, 5248, 2432, 1664, 2176, 5760, 12928, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6144, 3072, 14720, 4352, 3584, 6144, 3456, 6016, 1920, 384, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[965, 321, 1382, 297, 928, 170, 1150, 839, 527, 582, 371, 285, 1132, 219, 534, 721, 704, 755, 115, 238, 2212, 568, 1207, 207, 729, 444, 372, 537, 390, 1617, 320, 1713, 936, 709, 870, 255, 720, 350, 1045, 658, 780, 2254, 724, 1027, 239, 769, 653, 626, 161, 705, 756, 699, 1493, 1445, 218, 825, 964, 892, 260, 673, 693, 841, 896, 2465] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[1091, 631, 747, 310, 1154, 84, 1719, 476, 735, 215, 195, 204, 1093, 138, 1117, 665, 916, 1562, 113, 340, 2362, 210, 1151, 485, 597, 617, 174, 314, 732, 2457, 1220, 1468, 577, 581, 801, 393, 281, 294, 306, 795, 471, 1581, 922, 584, 347, 722, 439, 738, 333, 263, 902, 648, 1675, 1758, 673, 844, 1297, 1097, 683, 203, 379, 94, 870, 2309] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[1583, 422, 1053, 365, 1244, 271, 566, 77, 571, 433, 291, 422, 732, 829, 774, 999, 860, 935, 90, 189, 1751, 249, 2415, 646, 411, 539, 350, 164, 617, 1688, 318, 1677, 981, 464, 928, 578, 1138, 683, 420, 482, 750, 1460, 1011, 998, 413, 334, 418, 869, 537, 235, 1643, 764, 1011, 1432, 383, 803, 1100, 622, 259, 856, 276, 558, 913, 2302] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[730, 321, 711, 340, 2035, 193, 1830, 399, 292, 279, 238, 578, 563, 212, 593, 726, 942, 2689, 81, 253, 1916, 495, 981, 524, 487, 322, 215, 764, 1076, 961, 548, 2219, 484, 515, 1122, 165, 614, 800, 883, 1452, 681, 1754, 897, 1551, 461, 535, 448, 447, 272, 470, 312, 505, 1878, 1379, 247, 838, 753, 744, 422, 471, 442, 683, 729, 1685] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[814, 391, 1290, 828, 1434, 95, 398, 681, 382, 399, 469, 233, 446, 1087, 929, 178, 644, 2255, 183, 234, 1609, 531, 774, 689, 1085, 719, 70, 779, 898, 746, 317, 1742, 315, 988, 398, 399, 692, 536, 223, 444, 258, 1367, 2137, 1883, 461, 601, 462, 733, 258, 853, 485, 1071, 1097, 1008, 351, 944, 1210, 484, 1000, 255, 637, 970, 802, 2501] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[531, 727, 752, 727, 1438, 92, 998, 377, 813, 1181, 565, 352, 617, 644, 820, 1604, 861, 1107, 150, 259, 1843, 484, 819, 1003, 768, 293, 186, 506, 738, 889, 271, 1449, 130, 580, 478, 577, 695, 752, 286, 704, 586, 1536, 1124, 522, 396, 434, 351, 1279, 501, 254, 938, 882, 1022, 2179, 273, 922, 1146, 1198, 728, 179, 662, 677, 960, 2337] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[1426, 365, 1116, 431, 1855, 99, 956, 610, 638, 675, 317, 301, 878, 425, 656, 1154, 728, 756, 204, 427, 2075, 820, 1546, 453, 875, 499, 275, 730, 1196, 1430, 216, 1332, 341, 593, 534, 485, 624, 541, 832, 1306, 544, 942, 748, 931, 271, 836, 463, 1231, 156, 289, 1283, 871, 944, 1619, 375, 824, 641, 531, 285, 254, 894, 367, 834, 2299] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[1078, 703, 713, 649, 1322, 193, 1258, 792, 366, 477, 258, 326, 618, 720, 1043, 1025, 247, 1082, 178, 248, 2077, 227, 2459, 568, 625, 345, 166, 1053, 672, 1081, 383, 1417, 373, 778, 408, 361, 940, 682, 507, 401, 568, 1482, 1382, 897, 250, 696, 755, 1382, 178, 336, 956, 933, 1564, 1466, 142, 653, 832, 797, 504, 377, 555, 749, 592, 2287] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1091, 631, 747, 310, 1115, 84, 1719, 476, 0, 0, 0, 0, 0, 0, 0, 0, 735, 215, 195, 204, 1093, 138, 1117, 665, 866, 666, 0, 0, 0, 0, 0, 0, 916, 1562, 113, 340, 1386, 210, 1151, 485, 0, 0, 0, 0, 0, 0, 0, 0, 597, 617, 174, 314, 732, 2457, 1220, 1395, 0, 0, 0, 0, 0, 0, 0, 0, 577, 581, 801, 393, 281, 294, 306, 795, 544, 369, 310, 73, 39, 0, 0, 0, 471, 1037, 922, 584, 347, 722, 439, 738, 0, 0, 0, 0, 0, 0, 0, 0, 333, 263, 902, 648, 1675, 1389, 673, 844, 0, 0, 0, 0, 0, 0, 0, 0, 1297, 1097, 683, 203, 379, 94, 870, 1443, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6173, 5894, 6163, 7506, 5363, 5260, 6727, 6066] avg=6144.0000 max=7506 max_rank=3 min=5260 min_rank=5 imbalance_ratio=1.221680 max_load_reduction=41646 max_load_reduction_pct=0.847290 moved_to_echo=23416 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[965, 321, 1382, 297, 897, 170, 1150, 839, 0, 0, 0, 0, 0, 0, 0, 0, 527, 582, 371, 285, 1132, 219, 534, 721, 924, 623, 0, 0, 0, 0, 0, 0, 704, 755, 115, 238, 1299, 568, 1207, 207, 0, 0, 0, 0, 0, 0, 0, 0, 729, 444, 372, 537, 390, 1617, 320, 1627, 0, 0, 0, 0, 0, 0, 0, 0, 936, 709, 870, 255, 720, 350, 1045, 658, 775, 303, 290, 86, 31, 0, 0, 0, 780, 1479, 724, 1027, 239, 769, 653, 626, 0, 0, 0, 0, 0, 0, 0, 0, 161, 705, 756, 699, 1493, 1142, 218, 825, 0, 0, 0, 0, 0, 0, 0, 0, 964, 892, 260, 673, 693, 841, 896, 1541, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6021, 5918, 5093, 6036, 7028, 6297, 5999, 6760] avg=6144.0000 max=7028 max_rank=4 min=5093 min_rank=2 imbalance_ratio=1.143880 max_load_reduction=42124 max_load_reduction_pct=0.857015 moved_to_echo=26084 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[531, 727, 752, 727, 1389, 92, 998, 377, 0, 0, 0, 0, 0, 0, 0, 0, 813, 1181, 565, 352, 617, 644, 820, 1604, 876, 519, 0, 0, 0, 0, 0, 0, 861, 1107, 150, 259, 1082, 484, 819, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 768, 293, 186, 506, 738, 889, 271, 1376, 0, 0, 0, 0, 0, 0, 0, 0, 130, 580, 478, 577, 695, 752, 286, 704, 528, 457, 242, 73, 49, 0, 0, 0, 586, 1008, 1124, 522, 396, 434, 351, 1279, 0, 0, 0, 0, 0, 0, 0, 0, 501, 254, 938, 882, 1022, 1722, 273, 922, 0, 0, 0, 0, 0, 0, 0, 0, 1146, 1198, 728, 179, 662, 677, 960, 1461, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5593, 7991, 5765, 5027, 5551, 5700, 6514, 7011] avg=6144.0000 max=7991 max_rank=1 min=5027 min_rank=3 imbalance_ratio=1.300618 max_load_reduction=41161 max_load_reduction_pct=0.837423 moved_to_echo=24776 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1583, 422, 1053, 365, 1202, 271, 566, 77, 0, 0, 0, 0, 0, 0, 0, 0, 571, 433, 291, 422, 732, 829, 774, 999, 863, 493, 0, 0, 0, 0, 0, 0, 860, 935, 90, 189, 1029, 249, 2415, 646, 0, 0, 0, 0, 0, 0, 0, 0, 411, 539, 350, 164, 617, 1688, 318, 1593, 0, 0, 0, 0, 0, 0, 0, 0, 981, 464, 928, 578, 1138, 683, 420, 482, 502, 300, 229, 84, 42, 0, 0, 0, 750, 958, 1011, 998, 413, 334, 418, 869, 0, 0, 0, 0, 0, 0, 0, 0, 537, 235, 1643, 764, 1011, 1132, 383, 803, 0, 0, 0, 0, 0, 0, 0, 0, 1100, 622, 259, 856, 276, 558, 913, 1439, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5539, 6407, 6413, 5680, 6831, 5751, 6508, 6023] avg=6144.0000 max=6831 max_rank=4 min=5539 min_rank=0 imbalance_ratio=1.111816 max_load_reduction=42321 max_load_reduction_pct=0.861023 moved_to_echo=25113 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[730, 321, 711, 340, 1962, 193, 1830, 399, 0, 0, 0, 0, 0, 0, 0, 0, 292, 279, 238, 578, 563, 212, 593, 726, 636, 544, 0, 0, 0, 0, 0, 0, 942, 2689, 81, 253, 1117, 495, 981, 524, 0, 0, 0, 0, 0, 0, 0, 0, 487, 322, 215, 764, 1076, 961, 548, 2104, 0, 0, 0, 0, 0, 0, 0, 0, 484, 515, 1122, 165, 614, 800, 883, 1452, 606, 294, 255, 115, 73, 0, 0, 0, 681, 1148, 897, 1551, 461, 535, 448, 447, 0, 0, 0, 0, 0, 0, 0, 0, 272, 470, 312, 505, 1878, 1085, 247, 838, 0, 0, 0, 0, 0, 0, 0, 0, 753, 744, 422, 471, 442, 683, 729, 1049, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6486, 4661, 7082, 6477, 7378, 6168, 5607, 5293] avg=6144.0000 max=7378 max_rank=4 min=4661 min_rank=1 imbalance_ratio=1.200846 max_load_reduction=41774 max_load_reduction_pct=0.849894 moved_to_echo=24446 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1426, 365, 1116, 431, 1792, 99, 956, 610, 0, 0, 0, 0, 0, 0, 0, 0, 638, 675, 317, 301, 878, 425, 656, 1154, 862, 585, 0, 0, 0, 0, 0, 0, 728, 756, 204, 427, 1218, 820, 1546, 453, 0, 0, 0, 0, 0, 0, 0, 0, 875, 499, 275, 730, 1196, 1430, 216, 1265, 0, 0, 0, 0, 0, 0, 0, 0, 341, 593, 534, 485, 624, 541, 832, 1306, 324, 340, 272, 67, 63, 0, 0, 0, 544, 618, 748, 931, 271, 836, 463, 1231, 0, 0, 0, 0, 0, 0, 0, 0, 156, 289, 1283, 871, 944, 1279, 375, 824, 0, 0, 0, 0, 0, 0, 0, 0, 641, 531, 285, 254, 894, 367, 834, 1437, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6795, 6491, 6152, 6486, 6322, 5642, 6021, 5243] avg=6144.0000 max=6795 max_rank=0 min=5243 min_rank=7 imbalance_ratio=1.105957 max_load_reduction=42357 max_load_reduction_pct=0.861755 moved_to_echo=23228 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1078, 703, 713, 649, 1277, 193, 1258, 792, 0, 0, 0, 0, 0, 0, 0, 0, 366, 477, 258, 326, 618, 720, 1043, 1025, 858, 585, 0, 0, 0, 0, 0, 0, 247, 1082, 178, 248, 1220, 227, 2459, 568, 0, 0, 0, 0, 0, 0, 0, 0, 625, 345, 166, 1053, 672, 1081, 383, 1346, 0, 0, 0, 0, 0, 0, 0, 0, 373, 778, 408, 361, 940, 682, 507, 401, 510, 308, 272, 71, 45, 0, 0, 0, 568, 972, 1382, 897, 250, 696, 755, 1382, 0, 0, 0, 0, 0, 0, 0, 0, 178, 336, 956, 933, 1564, 1158, 142, 653, 0, 0, 0, 0, 0, 0, 0, 0, 832, 797, 504, 377, 555, 749, 592, 1429, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6663, 6276, 6229, 5671, 5656, 6902, 5920, 5835] avg=6144.0000 max=6902 max_rank=5 min=5656 min_rank=4 imbalance_ratio=1.123372 max_load_reduction=42250 max_load_reduction_pct=0.859578 moved_to_echo=24313 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[814, 391, 1290, 828, 1385, 95, 398, 681, 0, 0, 0, 0, 0, 0, 0, 0, 382, 399, 469, 233, 446, 1087, 929, 178, 938, 453, 0, 0, 0, 0, 0, 0, 644, 2255, 183, 234, 945, 531, 774, 689, 0, 0, 0, 0, 0, 0, 0, 0, 1085, 719, 70, 779, 898, 746, 317, 1655, 0, 0, 0, 0, 0, 0, 0, 0, 315, 988, 398, 399, 692, 536, 223, 444, 470, 211, 211, 87, 49, 0, 0, 0, 258, 897, 2137, 1883, 461, 601, 462, 733, 0, 0, 0, 0, 0, 0, 0, 0, 258, 853, 485, 1071, 1097, 797, 351, 944, 0, 0, 0, 0, 0, 0, 0, 0, 1210, 484, 1000, 255, 637, 970, 802, 1563, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5882, 5514, 6255, 6269, 5023, 7432, 5856, 6921] avg=6144.0000 max=7432 max_rank=5 min=5023 min_rank=4 imbalance_ratio=1.209635 max_load_reduction=41720 max_load_reduction_pct=0.848796 moved_to_echo=25232 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[4352, 4352, 2816, 2816, 6144, 4352, 6528, 7168, 6912, 4480, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[8064, 6400, 4224, 3328, 4608, 4992, 6656, 11392, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4736, 8192, 8960, 8448, 2944, 4992, 4096, 7424, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=50304 observed_loss=-1152 tokens_per_expert=[4224, 5248, 5632, 3328, 5760, 4736, 4608, 6272, 4352, 2688, 2176, 768, 512, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5632, 3840, 1920, 4864, 6400, 10880, 3712, 12416, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[8320, 3968, 7808, 3968, 11136, 1280, 8960, 4352, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[2432, 3456, 7296, 6400, 10752, 9728, 2688, 6656, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6016, 11264, 1152, 2304, 9344, 3584, 11392, 4608, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=50304 slack_slots=48000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[889, 1373, 1094, 1484, 792, 145, 1099, 168, 572, 224, 229, 472, 408, 533, 359, 127, 283, 1809, 179, 1268, 495, 66, 544, 3208, 368, 394, 873, 1032, 2157, 81, 532, 464, 223, 2927, 287, 593, 805, 697, 2421, 1400, 951, 317, 526, 46, 551, 228, 419, 1006, 1762, 351, 355, 826, 687, 763, 632, 434, 952, 1064, 915, 793, 199, 1597, 492, 212] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[1686, 2173, 1405, 1592, 460, 168, 1190, 126, 1583, 409, 555, 1048, 442, 519, 548, 298, 290, 581, 318, 932, 894, 611, 506, 2402, 267, 440, 1236, 389, 1215, 337, 938, 442, 416, 1219, 442, 925, 975, 914, 1903, 562, 1635, 424, 616, 312, 185, 188, 849, 668, 1631, 128, 512, 630, 224, 825, 701, 435, 622, 799, 1071, 307, 463, 1887, 432, 252] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[444, 1388, 1742, 513, 246, 275, 1103, 199, 1021, 1004, 354, 817, 866, 1381, 623, 618, 101, 693, 112, 701, 1008, 266, 1034, 841, 528, 551, 1186, 489, 2211, 672, 626, 672, 1194, 1425, 1059, 1168, 1253, 1001, 1769, 326, 1057, 670, 834, 189, 313, 191, 92, 498, 1032, 378, 318, 925, 591, 357, 409, 341, 395, 936, 520, 1282, 1250, 2128, 438, 528] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[924, 1888, 2111, 1004, 465, 146, 1108, 88, 876, 800, 478, 243, 909, 521, 469, 220, 197, 710, 177, 1049, 796, 285, 354, 2056, 1216, 325, 1704, 674, 1100, 484, 492, 550, 326, 1148, 594, 1051, 514, 863, 1880, 1126, 1935, 523, 663, 499, 228, 332, 159, 490, 952, 249, 242, 340, 964, 772, 803, 314, 784, 1332, 918, 96, 1703, 2308, 273, 352] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1308, 1568, 1453, 1456, 1432, 224, 1754, 169, 1056, 310, 518, 152, 381, 362, 116, 390, 199, 1403, 247, 445, 1430, 99, 804, 1521, 259, 369, 804, 672, 1445, 311, 904, 443, 539, 961, 1079, 1737, 1510, 959, 1062, 1157, 1051, 748, 361, 242, 410, 219, 613, 592, 1224, 195, 128, 337, 264, 646, 763, 750, 124, 1676, 1314, 1038, 1137, 1477, 310, 525] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1637, 894, 751, 1155, 806, 106, 2024, 211, 1802, 308, 475, 368, 253, 649, 143, 329, 314, 573, 215, 733, 415, 136, 760, 1689, 269, 476, 1290, 1431, 1856, 211, 180, 565, 1749, 1420, 477, 1328, 1417, 1708, 1364, 581, 1172, 519, 468, 237, 325, 334, 282, 565, 734, 334, 185, 949, 474, 1370, 930, 797, 722, 1090, 1301, 336, 1098, 841, 485, 536] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[668, 1534, 1932, 432, 185, 128, 1479, 580, 1265, 496, 824, 205, 667, 459, 213, 346, 557, 1098, 340, 917, 2046, 283, 613, 2501, 598, 377, 902, 911, 1382, 360, 379, 457, 606, 1349, 990, 1276, 1928, 1020, 1017, 370, 532, 826, 638, 594, 459, 358, 202, 619, 1448, 200, 218, 156, 335, 880, 531, 1179, 650, 631, 1321, 596, 639, 1241, 934, 275] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[502, 2424, 1104, 1402, 231, 105, 1260, 116, 582, 235, 468, 217, 364, 400, 706, 436, 322, 617, 126, 1406, 1339, 260, 207, 2285, 383, 487, 692, 632, 1250, 646, 312, 628, 560, 2245, 787, 656, 979, 928, 2426, 859, 978, 932, 353, 105, 784, 58, 294, 471, 835, 159, 241, 718, 1275, 596, 1438, 182, 730, 2126, 702, 653, 1172, 1442, 904, 420] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[889, 196, 1094, 1484, 792, 145, 1099, 168, 0, 0, 0, 0, 0, 0, 0, 0, 572, 224, 229, 472, 408, 533, 359, 127, 1035, 954, 0, 0, 0, 0, 0, 0, 283, 1809, 179, 1268, 495, 66, 544, 2914, 0, 0, 0, 0, 0, 0, 0, 0, 368, 394, 873, 1032, 2157, 81, 532, 464, 294, 137, 0, 0, 0, 0, 0, 0, 223, 1209, 287, 593, 805, 697, 0, 1400, 0, 0, 0, 0, 0, 0, 0, 0, 951, 317, 526, 46, 551, 228, 419, 1006, 2421, 142, 0, 0, 0, 0, 0, 0, 1762, 351, 355, 826, 687, 763, 632, 434, 677, 764, 0, 0, 0, 0, 0, 0, 952, 1064, 915, 793, 199, 783, 492, 212, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5867, 4913, 7558, 6332, 5214, 6607, 7251, 5410] avg=6144.0000 max=7558 max_rank=2 min=4913 min_rank=1 imbalance_ratio=1.230143 max_load_reduction=41594 max_load_reduction_pct=0.846232 moved_to_echo=24482 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[924, 268, 2111, 1004, 465, 146, 1108, 88, 0, 0, 0, 0, 0, 0, 0, 0, 876, 800, 478, 243, 909, 521, 469, 220, 1424, 374, 0, 0, 0, 0, 0, 0, 197, 710, 177, 1049, 796, 285, 354, 1868, 0, 0, 0, 0, 0, 0, 0, 0, 1216, 325, 1704, 674, 1100, 484, 492, 550, 188, 198, 0, 0, 0, 0, 0, 0, 326, 475, 594, 1051, 514, 863, 0, 1126, 0, 0, 0, 0, 0, 0, 0, 0, 1935, 523, 663, 499, 228, 332, 159, 490, 1880, 196, 0, 0, 0, 0, 0, 0, 952, 249, 242, 340, 964, 772, 803, 314, 979, 299, 0, 0, 0, 0, 0, 0, 784, 1332, 918, 96, 1703, 1131, 273, 352, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6114, 6314, 5436, 6931, 4949, 6905, 5914, 6589] avg=6144.0000 max=6931 max_rank=3 min=4949 min_rank=4 imbalance_ratio=1.128092 max_load_reduction=42221 max_load_reduction_pct=0.858988 moved_to_echo=24357 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1686, 309, 1405, 1592, 460, 168, 1190, 126, 0, 0, 0, 0, 0, 0, 0, 0, 1583, 409, 555, 1048, 442, 519, 548, 298, 1639, 397, 0, 0, 0, 0, 0, 0, 290, 581, 318, 932, 894, 611, 506, 2182, 0, 0, 0, 0, 0, 0, 0, 0, 267, 440, 1236, 389, 1215, 337, 938, 442, 220, 162, 0, 0, 0, 0, 0, 0, 416, 504, 442, 925, 975, 914, 0, 562, 0, 0, 0, 0, 0, 0, 0, 0, 1635, 424, 616, 312, 185, 188, 849, 668, 1903, 225, 0, 0, 0, 0, 0, 0, 1631, 128, 512, 630, 224, 825, 701, 435, 800, 318, 0, 0, 0, 0, 0, 0, 622, 799, 1071, 307, 463, 925, 432, 252, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6936, 7438, 6314, 5646, 4738, 7005, 6204, 4871] avg=6144.0000 max=7438 max_rank=1 min=4738 min_rank=4 imbalance_ratio=1.210612 max_load_reduction=41714 max_load_reduction_pct=0.848674 moved_to_echo=22818 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[444, 197, 1742, 513, 246, 275, 1103, 199, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1004, 354, 817, 866, 1381, 623, 618, 1047, 464, 0, 0, 0, 0, 0, 0, 101, 693, 112, 701, 1008, 266, 1034, 764, 0, 0, 0, 0, 0, 0, 0, 0, 528, 551, 1186, 489, 2211, 672, 626, 672, 77, 183, 0, 0, 0, 0, 0, 0, 1194, 589, 1059, 1168, 1253, 1001, 0, 326, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 670, 834, 189, 313, 191, 92, 498, 1769, 144, 0, 0, 0, 0, 0, 0, 1032, 378, 318, 925, 591, 357, 409, 341, 902, 372, 0, 0, 0, 0, 0, 0, 395, 936, 520, 1282, 1250, 1043, 438, 528, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4719, 8195, 4679, 7195, 6590, 5757, 5625, 6392] avg=6144.0000 max=8195 max_rank=1 min=4679 min_rank=2 imbalance_ratio=1.333822 max_load_reduction=40957 max_load_reduction_pct=0.833272 moved_to_echo=24364 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1637, 120, 751, 1155, 806, 106, 2024, 211, 0, 0, 0, 0, 0, 0, 0, 0, 1802, 308, 475, 368, 253, 649, 143, 329, 678, 467, 0, 0, 0, 0, 0, 0, 314, 573, 215, 733, 415, 136, 760, 1531, 0, 0, 0, 0, 0, 0, 0, 0, 269, 476, 1290, 1431, 1856, 211, 180, 565, 158, 75, 0, 0, 0, 0, 0, 0, 1749, 579, 477, 1328, 1417, 1708, 0, 581, 0, 0, 0, 0, 0, 0, 0, 0, 1172, 519, 468, 237, 325, 334, 282, 565, 1364, 96, 0, 0, 0, 0, 0, 0, 734, 334, 185, 949, 474, 1370, 930, 797, 361, 374, 0, 0, 0, 0, 0, 0, 722, 1090, 1301, 336, 1098, 405, 485, 536, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6810, 5472, 4677, 6511, 7839, 5362, 6508, 5973] avg=6144.0000 max=7839 max_rank=4 min=4677 min_rank=2 imbalance_ratio=1.275879 max_load_reduction=41313 max_load_reduction_pct=0.840515 moved_to_echo=25682 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1308, 223, 1453, 1456, 1432, 224, 1754, 169, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 310, 518, 152, 381, 362, 116, 390, 1183, 313, 0, 0, 0, 0, 0, 0, 199, 1403, 247, 445, 1430, 99, 804, 1382, 0, 0, 0, 0, 0, 0, 0, 0, 259, 369, 804, 672, 1445, 311, 904, 443, 139, 127, 0, 0, 0, 0, 0, 0, 539, 397, 1079, 1737, 1510, 959, 0, 1157, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 748, 361, 242, 410, 219, 613, 592, 1062, 162, 0, 0, 0, 0, 0, 0, 1224, 195, 128, 337, 264, 646, 763, 750, 626, 251, 0, 0, 0, 0, 0, 0, 124, 1676, 1314, 1038, 1137, 724, 310, 525, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8019, 4781, 6009, 5473, 7378, 5460, 5184, 6848] avg=6144.0000 max=8019 max_rank=0 min=4781 min_rank=1 imbalance_ratio=1.305176 max_load_reduction=41133 max_load_reduction_pct=0.836853 moved_to_echo=24870 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[502, 345, 1104, 1402, 231, 105, 1260, 116, 0, 0, 0, 0, 0, 0, 0, 0, 582, 235, 468, 217, 364, 400, 706, 436, 1828, 732, 0, 0, 0, 0, 0, 0, 322, 617, 126, 1406, 1339, 260, 207, 2076, 0, 0, 0, 0, 0, 0, 0, 0, 383, 487, 692, 632, 1250, 646, 312, 628, 209, 124, 0, 0, 0, 0, 0, 0, 560, 927, 787, 656, 979, 928, 0, 859, 0, 0, 0, 0, 0, 0, 0, 0, 978, 932, 353, 105, 784, 58, 294, 471, 2426, 251, 0, 0, 0, 0, 0, 0, 835, 159, 241, 718, 1275, 596, 1438, 182, 611, 586, 0, 0, 0, 0, 0, 0, 730, 2126, 702, 653, 1172, 707, 904, 420, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5065, 5968, 6353, 5363, 5696, 6652, 6641, 7414] avg=6144.0000 max=7414 max_rank=7 min=5065 min_rank=0 imbalance_ratio=1.206706 max_load_reduction=41738 max_load_reduction_pct=0.849162 moved_to_echo=26403 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[668, 218, 1932, 432, 185, 128, 1479, 580, 0, 0, 0, 0, 0, 0, 0, 0, 1265, 496, 824, 205, 667, 459, 213, 346, 1157, 439, 0, 0, 0, 0, 0, 0, 557, 1098, 340, 917, 2046, 283, 613, 2272, 0, 0, 0, 0, 0, 0, 0, 0, 598, 377, 902, 911, 1382, 360, 379, 457, 229, 106, 0, 0, 0, 0, 0, 0, 606, 558, 990, 1276, 1928, 1020, 0, 370, 0, 0, 0, 0, 0, 0, 0, 0, 532, 826, 638, 594, 459, 358, 202, 619, 1017, 159, 0, 0, 0, 0, 0, 0, 1448, 200, 218, 156, 335, 880, 531, 1179, 526, 352, 0, 0, 0, 0, 0, 0, 650, 631, 1321, 596, 639, 609, 934, 275, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5622, 6071, 8126, 5701, 6748, 5404, 5825, 5655] avg=6144.0000 max=8126 max_rank=2 min=5404 min_rank=5 imbalance_ratio=1.322591 max_load_reduction=41026 max_load_reduction_pct=0.834676 moved_to_echo=23632 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3968, 3456, 8704, 6272, 12672, 3200, 4480, 4224, 1536, 1152, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[5632, 5248, 5760, 8832, 9472, 8192, 0, 6400, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4992, 9728, 8064, 5120, 7680, 6400, 4352, 3200, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[8064, 1920, 11648, 9088, 4736, 1408, 11136, 1664, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[9728, 2048, 2304, 4992, 4864, 6272, 6272, 4480, 5504, 3328, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2304, 7552, 1792, 7552, 8448, 2048, 4864, 15104, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[8832, 3840, 3968, 3584, 4352, 4864, 3200, 2816, 10112, 4224, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[9344, 4992, 4480, 2304, 3328, 1920, 2944, 4992, 13952, 1408, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[622, 467, 318, 1068, 871, 548, 260, 1039, 943, 368, 336, 121, 1187, 667, 1072, 630, 1369, 464, 547, 399, 980, 418, 1587, 314, 153, 539, 1082, 2021, 426, 1044, 1261, 531, 1107, 773, 755, 1209, 1945, 370, 427, 2755, 1740, 869, 629, 1402, 1244, 799, 112, 671, 405, 432, 32, 611, 404, 1179, 833, 1038, 485, 667, 1190, 296, 509, 358, 121, 133] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[221, 638, 535, 2439, 538, 450, 332, 1251, 752, 195, 260, 647, 2400, 1866, 460, 1156, 822, 1179, 195, 291, 859, 866, 1130, 815, 489, 749, 801, 647, 232, 666, 483, 256, 734, 1245, 1672, 1860, 1913, 368, 208, 2200, 1042, 839, 491, 799, 584, 1545, 358, 1042, 401, 214, 135, 454, 61, 1453, 267, 484, 139, 291, 1166, 360, 583, 1045, 305, 274] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[347, 437, 392, 1855, 2940, 464, 351, 579, 501, 446, 698, 843, 2857, 675, 1158, 1710, 178, 629, 812, 647, 167, 808, 631, 778, 981, 507, 569, 490, 112, 64, 1429, 157, 405, 1494, 276, 1449, 2904, 1233, 89, 1350, 585, 1817, 1459, 423, 115, 1281, 69, 1186, 280, 228, 326, 304, 15, 1063, 469, 493, 303, 690, 1346, 69, 1200, 874, 74, 71] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[314, 285, 419, 1839, 786, 379, 562, 686, 914, 601, 309, 894, 1202, 1159, 1301, 603, 1557, 1919, 970, 277, 519, 568, 857, 595, 138, 1089, 1095, 1096, 893, 479, 1368, 653, 257, 1151, 1084, 1233, 3409, 1255, 156, 751, 1123, 905, 675, 423, 603, 1407, 125, 421, 1035, 316, 255, 596, 154, 955, 421, 763, 513, 415, 972, 327, 497, 379, 121, 129] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[317, 259, 163, 2289, 2206, 456, 173, 1597, 618, 515, 625, 483, 2322, 746, 1012, 586, 1487, 442, 289, 359, 196, 295, 1044, 801, 142, 1082, 970, 1478, 64, 414, 1313, 178, 789, 1872, 166, 1283, 1511, 566, 327, 1502, 1201, 1125, 932, 1175, 391, 1416, 166, 1238, 481, 341, 70, 798, 649, 1168, 182, 764, 706, 515, 563, 724, 673, 184, 449, 304] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[720, 282, 490, 2140, 824, 591, 523, 618, 1252, 402, 441, 806, 1306, 678, 1888, 957, 249, 1898, 491, 357, 512, 570, 745, 454, 397, 1044, 522, 805, 124, 239, 1274, 391, 878, 882, 481, 1582, 2576, 1242, 224, 1303, 1335, 978, 948, 375, 495, 1577, 196, 208, 326, 528, 340, 794, 23, 1401, 640, 602, 600, 572, 1637, 204, 530, 1017, 314, 354] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[270, 313, 322, 1576, 1123, 445, 555, 1575, 448, 393, 1239, 458, 1174, 1118, 909, 484, 631, 764, 683, 1060, 580, 282, 982, 609, 209, 1158, 759, 1361, 316, 335, 2366, 427, 458, 946, 646, 1508, 870, 551, 417, 988, 1387, 1646, 1883, 275, 695, 1262, 65, 605, 213, 942, 318, 542, 128, 989, 518, 1312, 163, 1022, 1146, 535, 960, 254, 938, 46] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[421, 111, 532, 1090, 2529, 1370, 201, 870, 932, 101, 619, 495, 1118, 772, 666, 710, 1174, 706, 494, 396, 615, 395, 848, 1001, 560, 494, 1617, 1445, 199, 1356, 1379, 155, 483, 638, 830, 1422, 3256, 853, 604, 785, 1560, 1029, 1496, 751, 791, 888, 69, 718, 185, 270, 119, 234, 268, 1017, 233, 984, 504, 605, 872, 178, 1013, 811, 51, 264] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[347, 437, 392, 1592, 2940, 464, 351, 579, 0, 0, 0, 0, 0, 0, 0, 0, 501, 446, 698, 843, 1583, 675, 1158, 1710, 0, 0, 0, 0, 0, 0, 0, 0, 178, 629, 812, 647, 167, 808, 631, 778, 335, 148, 0, 0, 0, 0, 0, 0, 981, 507, 569, 490, 112, 64, 1429, 157, 263, 5, 0, 0, 0, 0, 0, 0, 405, 1494, 276, 1449, 0, 1233, 89, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 585, 1817, 1459, 423, 115, 462, 69, 1186, 0, 0, 0, 0, 0, 0, 0, 0, 280, 228, 326, 304, 15, 1063, 469, 493, 2479, 0, 0, 0, 0, 0, 0, 0, 303, 690, 1346, 69, 1200, 874, 74, 71, 819, 1126, 425, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7102, 7614, 5133, 4577, 5956, 6116, 5657, 6997] avg=6144.0000 max=7614 max_rank=1 min=4577 min_rank=3 imbalance_ratio=1.239258 max_load_reduction=41538 max_load_reduction_pct=0.845093 moved_to_echo=24726 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[622, 467, 318, 917, 871, 548, 260, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 943, 368, 336, 121, 659, 667, 1072, 630, 0, 0, 0, 0, 0, 0, 0, 0, 1369, 464, 547, 399, 980, 418, 1587, 314, 684, 61, 0, 0, 0, 0, 0, 0, 153, 539, 1082, 2021, 426, 1044, 1261, 531, 151, 11, 0, 0, 0, 0, 0, 0, 1107, 773, 755, 1209, 0, 370, 427, 2060, 0, 0, 0, 0, 0, 0, 0, 0, 1740, 869, 629, 1402, 1244, 289, 112, 671, 0, 0, 0, 0, 0, 0, 0, 0, 405, 432, 32, 611, 404, 1179, 833, 1038, 1660, 0, 0, 0, 0, 0, 0, 0, 485, 667, 1190, 296, 509, 358, 121, 133, 510, 467, 285, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5042, 4796, 6823, 7219, 6701, 6956, 6594, 5021] avg=6144.0000 max=7219 max_rank=3 min=4796 min_rank=1 imbalance_ratio=1.174967 max_load_reduction=41933 max_load_reduction_pct=0.853129 moved_to_echo=25272 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[314, 285, 419, 1578, 786, 379, 562, 686, 0, 0, 0, 0, 0, 0, 0, 0, 914, 601, 309, 894, 667, 1159, 1301, 603, 0, 0, 0, 0, 0, 0, 0, 0, 1557, 1919, 970, 277, 519, 568, 857, 595, 186, 62, 0, 0, 0, 0, 0, 0, 138, 1089, 1095, 1096, 893, 479, 1368, 653, 261, 3, 0, 0, 0, 0, 0, 0, 257, 1151, 1084, 1233, 0, 1255, 156, 562, 0, 0, 0, 0, 0, 0, 0, 0, 1123, 905, 675, 423, 603, 508, 125, 421, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 316, 255, 596, 154, 955, 421, 763, 2910, 0, 0, 0, 0, 0, 0, 0, 513, 415, 972, 327, 497, 379, 121, 129, 899, 473, 499, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5009, 6448, 7510, 7075, 5698, 4783, 7405, 5224] avg=6144.0000 max=7510 max_rank=2 min=4783 min_rank=5 imbalance_ratio=1.222331 max_load_reduction=41642 max_load_reduction_pct=0.847209 moved_to_echo=23110 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[221, 638, 535, 2089, 538, 450, 332, 1251, 0, 0, 0, 0, 0, 0, 0, 0, 752, 195, 260, 647, 1321, 1866, 460, 1156, 0, 0, 0, 0, 0, 0, 0, 0, 822, 1179, 195, 291, 859, 866, 1130, 815, 551, 128, 0, 0, 0, 0, 0, 0, 489, 749, 801, 647, 232, 666, 483, 256, 350, 11, 0, 0, 0, 0, 0, 0, 734, 1245, 1672, 1860, 0, 368, 208, 1638, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 839, 491, 799, 584, 553, 358, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 401, 214, 135, 454, 61, 1453, 267, 484, 1634, 0, 0, 0, 0, 0, 0, 0, 139, 291, 1166, 360, 583, 1045, 305, 274, 992, 951, 279, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6054, 6657, 6836, 4684, 7725, 5708, 5103, 6385] avg=6144.0000 max=7725 max_rank=4 min=4684 min_rank=3 imbalance_ratio=1.257324 max_load_reduction=41427 max_load_reduction_pct=0.842834 moved_to_echo=24921 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[720, 282, 490, 1836, 824, 591, 523, 618, 0, 0, 0, 0, 0, 0, 0, 0, 1252, 402, 441, 806, 725, 678, 1888, 957, 0, 0, 0, 0, 0, 0, 0, 0, 249, 1898, 491, 357, 512, 570, 745, 454, 323, 67, 0, 0, 0, 0, 0, 0, 397, 1044, 522, 805, 124, 239, 1274, 391, 304, 5, 0, 0, 0, 0, 0, 0, 878, 882, 481, 1582, 0, 1242, 224, 975, 0, 0, 0, 0, 0, 0, 0, 0, 1335, 978, 948, 375, 495, 569, 196, 208, 0, 0, 0, 0, 0, 0, 0, 0, 326, 528, 340, 794, 23, 1401, 640, 602, 2200, 0, 0, 0, 0, 0, 0, 0, 600, 572, 1637, 204, 530, 1017, 314, 354, 1008, 514, 376, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5884, 7149, 5666, 5105, 6264, 5104, 6854, 7126] avg=6144.0000 max=7149 max_rank=1 min=5104 min_rank=5 imbalance_ratio=1.163574 max_load_reduction=42003 max_load_reduction_pct=0.854553 moved_to_echo=25348 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[317, 259, 163, 1964, 2206, 456, 173, 1597, 0, 0, 0, 0, 0, 0, 0, 0, 618, 515, 625, 483, 1287, 746, 1012, 586, 0, 0, 0, 0, 0, 0, 0, 0, 1487, 442, 289, 359, 196, 295, 1044, 801, 373, 120, 0, 0, 0, 0, 0, 0, 142, 1082, 970, 1478, 64, 414, 1313, 178, 325, 6, 0, 0, 0, 0, 0, 0, 789, 1872, 166, 1283, 0, 566, 327, 1123, 0, 0, 0, 0, 0, 0, 0, 0, 1201, 1125, 932, 1175, 391, 511, 166, 1238, 0, 0, 0, 0, 0, 0, 0, 0, 481, 341, 70, 798, 649, 1168, 182, 764, 1291, 0, 0, 0, 0, 0, 0, 0, 706, 515, 563, 724, 673, 184, 449, 304, 905, 915, 220, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7135, 5872, 5406, 5972, 6126, 6739, 5744, 6158] avg=6144.0000 max=7135 max_rank=0 min=5406 min_rank=2 imbalance_ratio=1.161296 max_load_reduction=42017 max_load_reduction_pct=0.854838 moved_to_echo=24767 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[421, 111, 532, 936, 2529, 1370, 201, 870, 0, 0, 0, 0, 0, 0, 0, 0, 932, 101, 619, 495, 620, 772, 666, 710, 0, 0, 0, 0, 0, 0, 0, 0, 1174, 706, 494, 396, 615, 395, 848, 1001, 195, 58, 0, 0, 0, 0, 0, 0, 560, 494, 1617, 1445, 199, 1356, 1379, 155, 154, 3, 0, 0, 0, 0, 0, 0, 483, 638, 830, 1422, 0, 853, 604, 587, 0, 0, 0, 0, 0, 0, 0, 0, 1560, 1029, 1496, 751, 791, 321, 69, 718, 0, 0, 0, 0, 0, 0, 0, 0, 185, 270, 119, 234, 268, 1017, 233, 984, 2781, 0, 0, 0, 0, 0, 0, 0, 504, 605, 872, 178, 1013, 811, 51, 264, 567, 440, 475, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6970, 4915, 5882, 7362, 5417, 6735, 6091, 5780] avg=6144.0000 max=7362 max_rank=3 min=4915 min_rank=1 imbalance_ratio=1.198242 max_load_reduction=41790 max_load_reduction_pct=0.850220 moved_to_echo=24023 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[270, 313, 322, 1353, 1123, 445, 555, 1575, 0, 0, 0, 0, 0, 0, 0, 0, 448, 393, 1239, 458, 652, 1118, 909, 484, 0, 0, 0, 0, 0, 0, 0, 0, 631, 764, 683, 1060, 580, 282, 982, 609, 245, 60, 0, 0, 0, 0, 0, 0, 209, 1158, 759, 1361, 316, 335, 2366, 427, 223, 4, 0, 0, 0, 0, 0, 0, 458, 946, 646, 1508, 0, 551, 417, 739, 0, 0, 0, 0, 0, 0, 0, 0, 1387, 1646, 1883, 275, 695, 455, 65, 605, 0, 0, 0, 0, 0, 0, 0, 0, 213, 942, 318, 542, 128, 989, 518, 1312, 742, 0, 0, 0, 0, 0, 0, 0, 163, 1022, 1146, 535, 960, 254, 938, 46, 807, 462, 128, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5956, 5701, 5896, 7158, 5265, 7011, 5704, 6461] avg=6144.0000 max=7158 max_rank=3 min=5265 min_rank=4 imbalance_ratio=1.165039 max_load_reduction=41994 max_load_reduction_pct=0.854370 moved_to_echo=24441 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3328, 2816, 3200, 12288, 11904, 4736, 3072, 8320, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[7552, 8064, 4608, 3840, 4480, 4224, 7936, 5376, 2944, 768, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[9984, 9216, 8576, 5632, 4992, 3712, 1280, 6144, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3456, 4864, 8960, 2816, 6016, 4992, 2432, 1664, 6528, 5376, 2688, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5120, 9088, 6016, 11648, 0, 6528, 2560, 8704, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6400, 3072, 4608, 4864, 7552, 7808, 8576, 6912, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3328, 3328, 1664, 4352, 1792, 9344, 3584, 6528, 15744, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[3072, 6784, 7424, 9344, 2432, 4608, 10880, 2816, 2048, 128, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[521, 438, 455, 380, 1165, 636, 592, 1777, 658, 495, 192, 555, 367, 534, 965, 242, 439, 375, 412, 774, 1223, 536, 913, 1198, 208, 697, 277, 1923, 443, 205, 407, 1415, 384, 480, 1892, 410, 642, 344, 837, 1869, 1432, 292, 311, 2253, 265, 365, 204, 1903, 745, 339, 393, 225, 703, 566, 2041, 252, 739, 616, 1721, 2388, 945, 1409, 522, 248] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[831, 387, 1281, 1319, 116, 111, 183, 556, 957, 1832, 1525, 1110, 225, 478, 1037, 75, 200, 152, 870, 1066, 642, 654, 486, 654, 140, 856, 49, 551, 1574, 312, 209, 1112, 80, 438, 1876, 334, 1081, 60, 190, 1639, 776, 1069, 340, 1367, 194, 391, 556, 1137, 1153, 384, 249, 648, 717, 326, 1034, 529, 1012, 1435, 2019, 3270, 555, 1872, 535, 336] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[529, 1057, 979, 321, 1300, 868, 245, 552, 974, 748, 560, 509, 381, 865, 774, 177, 643, 183, 1436, 2474, 541, 413, 688, 1747, 200, 374, 132, 964, 511, 148, 295, 2024, 150, 266, 2238, 115, 1083, 244, 509, 940, 239, 453, 234, 2325, 88, 459, 549, 1829, 2446, 491, 203, 186, 713, 279, 2114, 281, 596, 884, 1238, 583, 954, 1975, 394, 482] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[273, 337, 314, 470, 504, 698, 264, 541, 378, 406, 1216, 355, 213, 503, 571, 693, 92, 64, 1479, 1708, 1474, 382, 419, 551, 385, 113, 220, 1808, 1055, 357, 255, 1901, 105, 778, 3019, 837, 925, 306, 888, 904, 357, 638, 236, 2548, 304, 180, 336, 2217, 648, 841, 251, 226, 687, 544, 1769, 733, 2474, 1269, 2260, 769, 453, 801, 488, 362] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[229, 906, 1207, 953, 171, 341, 316, 397, 822, 382, 660, 956, 724, 1259, 755, 480, 1188, 122, 1353, 2481, 597, 463, 273, 422, 210, 492, 268, 1619, 1151, 633, 309, 2255, 128, 972, 1582, 469, 561, 180, 233, 559, 396, 637, 1366, 1326, 100, 250, 773, 1544, 1385, 1077, 162, 378, 395, 760, 1671, 257, 425, 887, 521, 2101, 671, 1639, 969, 384] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1162, 1263, 1629, 323, 385, 177, 355, 1609, 705, 915, 130, 454, 273, 359, 1264, 285, 352, 107, 1724, 988, 331, 513, 450, 514, 237, 279, 439, 958, 1526, 535, 227, 1426, 206, 402, 2495, 147, 1298, 352, 376, 793, 772, 308, 765, 1350, 106, 530, 634, 1505, 767, 380, 466, 426, 852, 258, 2463, 869, 787, 1228, 1021, 1419, 1810, 1540, 324, 609] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1225, 772, 2595, 516, 579, 925, 372, 606, 601, 720, 263, 1302, 242, 1338, 748, 179, 131, 198, 1912, 1168, 358, 570, 95, 544, 207, 620, 347, 1424, 1827, 457, 286, 2493, 93, 826, 1641, 641, 547, 574, 288, 499, 624, 488, 1751, 1759, 208, 669, 620, 1555, 1079, 290, 337, 297, 243, 387, 2232, 164, 293, 450, 1558, 1489, 165, 1188, 281, 296] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[822, 840, 405, 480, 222, 700, 510, 636, 813, 307, 155, 825, 664, 605, 1339, 387, 214, 131, 1220, 2753, 542, 1067, 803, 93, 427, 158, 259, 1353, 1004, 582, 681, 1137, 297, 694, 3320, 632, 249, 32, 589, 1590, 1572, 275, 767, 2350, 239, 175, 238, 1313, 917, 1651, 77, 429, 736, 464, 1792, 640, 938, 600, 473, 1246, 398, 797, 638, 890] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[831, 387, 1281, 1319, 116, 111, 183, 556, 1361, 0, 0, 0, 0, 0, 0, 0, 957, 1832, 1525, 1110, 225, 478, 1037, 75, 1892, 0, 0, 0, 0, 0, 0, 0, 200, 152, 870, 1066, 642, 654, 486, 654, 97, 0, 0, 0, 0, 0, 0, 0, 140, 856, 49, 551, 1574, 312, 209, 1112, 191, 17, 0, 0, 0, 0, 0, 0, 80, 438, 1876, 334, 1081, 60, 190, 1639, 4, 0, 0, 0, 0, 0, 0, 0, 776, 1069, 340, 1040, 194, 391, 556, 1137, 0, 0, 0, 0, 0, 0, 0, 0, 1153, 384, 249, 648, 717, 326, 1034, 529, 136, 101, 0, 0, 0, 0, 0, 0, 1012, 1435, 2019, 0, 555, 1670, 535, 336, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 9131, 4821, 5011, 5702, 5503, 5277, 7562] avg=6144.0000 max=9131 max_rank=1 min=4821 min_rank=2 imbalance_ratio=1.486165 max_load_reduction=40021 max_load_reduction_pct=0.814229 moved_to_echo=24044 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[521, 438, 455, 380, 1165, 636, 592, 1777, 994, 0, 0, 0, 0, 0, 0, 0, 658, 495, 192, 555, 367, 534, 965, 242, 1380, 0, 0, 0, 0, 0, 0, 0, 439, 375, 412, 774, 1223, 536, 913, 1198, 73, 0, 0, 0, 0, 0, 0, 0, 208, 697, 277, 1923, 443, 205, 407, 1415, 314, 14, 0, 0, 0, 0, 0, 0, 384, 480, 1892, 410, 642, 344, 837, 1869, 3, 0, 0, 0, 0, 0, 0, 0, 1432, 292, 311, 1715, 265, 365, 204, 1903, 0, 0, 0, 0, 0, 0, 0, 0, 745, 339, 393, 225, 703, 566, 2041, 252, 224, 76, 0, 0, 0, 0, 0, 0, 739, 616, 1721, 0, 945, 1257, 522, 248, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6958, 5388, 5943, 5903, 6861, 6487, 5564, 6048] avg=6144.0000 max=6958 max_rank=0 min=5388 min_rank=1 imbalance_ratio=1.132487 max_load_reduction=42194 max_load_reduction_pct=0.858439 moved_to_echo=24960 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[529, 1057, 979, 321, 1300, 868, 245, 552, 242, 0, 0, 0, 0, 0, 0, 0, 974, 748, 560, 509, 381, 865, 774, 177, 337, 0, 0, 0, 0, 0, 0, 0, 643, 183, 1436, 2474, 541, 413, 688, 1747, 103, 0, 0, 0, 0, 0, 0, 0, 200, 374, 132, 964, 511, 148, 295, 2024, 325, 4, 0, 0, 0, 0, 0, 0, 150, 266, 2238, 115, 1083, 244, 509, 940, 4, 0, 0, 0, 0, 0, 0, 0, 239, 453, 234, 1769, 88, 459, 549, 1829, 0, 0, 0, 0, 0, 0, 0, 0, 2446, 491, 203, 186, 713, 279, 2114, 281, 231, 107, 0, 0, 0, 0, 0, 0, 596, 884, 1238, 0, 954, 1761, 394, 482, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6093, 5325, 8228, 4977, 5549, 5620, 7051, 6309] avg=6144.0000 max=8228 max_rank=2 min=4977 min_rank=3 imbalance_ratio=1.339193 max_load_reduction=40924 max_load_reduction_pct=0.832601 moved_to_echo=24529 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1162, 1263, 1629, 323, 385, 177, 355, 1609, 591, 0, 0, 0, 0, 0, 0, 0, 705, 915, 130, 454, 273, 359, 1264, 285, 821, 0, 0, 0, 0, 0, 0, 0, 352, 107, 1724, 988, 331, 513, 450, 514, 80, 0, 0, 0, 0, 0, 0, 0, 237, 279, 439, 958, 1526, 535, 227, 1426, 188, 7, 0, 0, 0, 0, 0, 0, 206, 402, 2495, 147, 1298, 352, 376, 793, 3, 0, 0, 0, 0, 0, 0, 0, 772, 308, 765, 1028, 106, 530, 634, 1505, 0, 0, 0, 0, 0, 0, 0, 0, 767, 380, 466, 426, 852, 258, 2463, 869, 134, 83, 0, 0, 0, 0, 0, 0, 787, 1228, 1021, 0, 1810, 1374, 324, 609, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7494, 5206, 5059, 5822, 6072, 5648, 6698, 7153] avg=6144.0000 max=7494 max_rank=0 min=5059 min_rank=2 imbalance_ratio=1.219727 max_load_reduction=41658 max_load_reduction_pct=0.847534 moved_to_echo=25571 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[229, 906, 1207, 953, 171, 341, 316, 397, 876, 0, 0, 0, 0, 0, 0, 0, 822, 382, 660, 956, 724, 1259, 755, 480, 1214, 0, 0, 0, 0, 0, 0, 0, 1188, 122, 1353, 2481, 597, 463, 273, 422, 90, 0, 0, 0, 0, 0, 0, 0, 210, 492, 268, 1619, 1151, 633, 309, 2255, 189, 11, 0, 0, 0, 0, 0, 0, 128, 972, 1582, 469, 561, 180, 233, 559, 6, 0, 0, 0, 0, 0, 0, 0, 396, 637, 1366, 1001, 100, 250, 773, 1544, 0, 0, 0, 0, 0, 0, 0, 0, 1385, 1077, 162, 378, 395, 760, 1671, 257, 136, 94, 0, 0, 0, 0, 0, 0, 425, 887, 521, 0, 671, 1449, 969, 384, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5396, 7252, 6989, 7137, 4690, 6067, 6315, 5306] avg=6144.0000 max=7252 max_rank=1 min=4690 min_rank=4 imbalance_ratio=1.180339 max_load_reduction=41900 max_load_reduction_pct=0.852458 moved_to_echo=22378 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[273, 337, 314, 470, 504, 698, 264, 541, 321, 0, 0, 0, 0, 0, 0, 0, 378, 406, 1216, 355, 213, 503, 571, 693, 444, 0, 0, 0, 0, 0, 0, 0, 92, 64, 1479, 1708, 1474, 382, 419, 551, 41, 0, 0, 0, 0, 0, 0, 0, 385, 113, 220, 1808, 1055, 357, 255, 1901, 356, 4, 0, 0, 0, 0, 0, 0, 105, 778, 3019, 837, 925, 306, 888, 904, 1, 0, 0, 0, 0, 0, 0, 0, 357, 638, 236, 1938, 304, 180, 336, 2217, 0, 0, 0, 0, 0, 0, 0, 0, 648, 841, 251, 226, 687, 544, 1769, 733, 254, 43, 0, 0, 0, 0, 0, 0, 2474, 1269, 2260, 0, 453, 716, 488, 362, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3722, 4779, 6210, 6454, 7763, 6206, 5996, 8022] avg=6144.0000 max=8022 max_rank=7 min=3722 min_rank=0 imbalance_ratio=1.305664 max_load_reduction=41130 max_load_reduction_pct=0.836792 moved_to_echo=27987 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1225, 772, 2595, 516, 579, 925, 372, 606, 621, 0, 0, 0, 0, 0, 0, 0, 601, 720, 263, 1302, 242, 1338, 748, 179, 861, 0, 0, 0, 0, 0, 0, 0, 131, 198, 1912, 1168, 358, 570, 95, 544, 62, 0, 0, 0, 0, 0, 0, 0, 207, 620, 347, 1424, 1827, 457, 286, 2493, 245, 7, 0, 0, 0, 0, 0, 0, 93, 826, 1641, 641, 547, 574, 288, 499, 2, 0, 0, 0, 0, 0, 0, 0, 624, 488, 1751, 1339, 208, 669, 620, 1555, 0, 0, 0, 0, 0, 0, 0, 0, 1079, 290, 337, 297, 243, 387, 2232, 164, 175, 64, 0, 0, 0, 0, 0, 0, 293, 450, 1558, 0, 165, 1060, 281, 296, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8211, 6254, 5038, 7913, 5111, 7254, 5268, 4103] avg=6144.0000 max=8211 max_rank=0 min=4103 min_rank=7 imbalance_ratio=1.336426 max_load_reduction=40941 max_load_reduction_pct=0.832947 moved_to_echo=21736 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[822, 840, 405, 480, 222, 700, 510, 636, 518, 0, 0, 0, 0, 0, 0, 0, 813, 307, 155, 825, 664, 605, 1339, 387, 722, 0, 0, 0, 0, 0, 0, 0, 214, 131, 1220, 2753, 542, 1067, 803, 93, 41, 0, 0, 0, 0, 0, 0, 0, 427, 158, 259, 1353, 1004, 582, 681, 1137, 328, 6, 0, 0, 0, 0, 0, 0, 297, 694, 3320, 632, 249, 32, 589, 1590, 1, 0, 0, 0, 0, 0, 0, 0, 1572, 275, 767, 1788, 239, 175, 238, 1313, 0, 0, 0, 0, 0, 0, 0, 0, 917, 1651, 77, 429, 736, 464, 1792, 640, 234, 43, 0, 0, 0, 0, 0, 0, 938, 600, 473, 0, 398, 712, 638, 890, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5133, 5817, 6864, 5935, 7404, 6367, 6983, 4649] avg=6144.0000 max=7404 max_rank=4 min=4649 min_rank=7 imbalance_ratio=1.205078 max_load_reduction=41748 max_load_reduction_pct=0.849365 moved_to_echo=25403 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6016, 5888, 4736, 6144, 3200, 6016, 7552, 2560, 7680, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[9216, 5504, 2176, 2816, 5120, 3584, 15232, 3840, 1536, 640, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1536, 4864, 18176, 3712, 6400, 2176, 3968, 8832, 128, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2048, 3712, 2048, 10624, 9216, 3328, 2688, 13824, 2176, 128, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5632, 6016, 8960, 4864, 4480, 4480, 2944, 6784, 5632, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3328, 1408, 10496, 13440, 5760, 4608, 4224, 5760, 640, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6272, 4224, 5888, 11648, 1536, 3072, 3968, 13056, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7296, 7424, 10880, 0, 6016, 10112, 4224, 3712, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[372, 27, 369, 664, 209, 445, 3, 666, 3, 1302, 1077, 838, 840, 912, 1195, 1691, 793, 1338, 555, 1618, 726, 625, 1259, 1067, 64, 192, 111, 64, 145, 2040, 551, 183, 1110, 433, 1339, 1099, 2163, 377, 1883, 1235, 3396, 618, 185, 464, 525, 706, 1026, 254, 881, 923, 152, 1293, 28, 1517, 328, 364, 154, 385, 316, 1800, 626, 507, 914, 207] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[658, 20, 763, 400, 173, 627, 78, 868, 365, 2127, 649, 1495, 1059, 173, 1243, 1878, 896, 814, 586, 1943, 1134, 554, 394, 930, 560, 64, 611, 420, 361, 1055, 194, 1154, 1295, 803, 274, 702, 1613, 940, 217, 775, 2862, 1076, 1071, 220, 674, 1241, 68, 210, 405, 649, 890, 1495, 272, 1473, 952, 534, 313, 144, 1007, 292, 406, 461, 1359, 213] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[652, 92, 1306, 887, 528, 228, 101, 516, 164, 532, 850, 1717, 1173, 879, 1095, 468, 372, 2126, 153, 510, 1456, 75, 653, 729, 729, 155, 209, 174, 102, 1271, 986, 274, 1212, 722, 215, 301, 1973, 452, 339, 1442, 2235, 246, 491, 301, 981, 561, 753, 261, 588, 433, 1360, 2279, 83, 1721, 795, 1273, 810, 297, 944, 1601, 418, 756, 1670, 477] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[245, 15, 1382, 289, 123, 645, 196, 882, 13, 1296, 699, 1027, 888, 197, 887, 1033, 1178, 672, 604, 1101, 340, 175, 1298, 857, 549, 1100, 37, 15, 306, 1356, 1462, 174, 481, 775, 1216, 917, 2723, 592, 1367, 1195, 3917, 428, 626, 762, 394, 337, 189, 556, 941, 783, 382, 2367, 386, 1054, 230, 399, 350, 743, 610, 321, 650, 1604, 526, 290] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[401, 210, 884, 1241, 190, 644, 23, 275, 85, 373, 595, 979, 996, 486, 1033, 795, 1312, 1282, 531, 814, 818, 247, 833, 940, 370, 856, 280, 252, 81, 528, 381, 1038, 975, 793, 346, 1239, 2779, 456, 1127, 663, 1173, 943, 1097, 512, 377, 1094, 118, 419, 1814, 1336, 799, 2592, 321, 1676, 114, 498, 479, 413, 702, 853, 1149, 538, 1603, 381] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[319, 71, 571, 366, 298, 289, 117, 377, 44, 967, 895, 1329, 818, 273, 1688, 704, 806, 2206, 1114, 949, 187, 186, 366, 233, 521, 53, 38, 85, 139, 693, 1897, 588, 589, 694, 700, 766, 2966, 379, 1800, 1484, 1537, 525, 1044, 478, 1602, 943, 229, 81, 1092, 876, 436, 1725, 209, 3152, 492, 367, 787, 123, 247, 461, 2129, 1037, 858, 157] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[386, 8, 698, 196, 214, 564, 5, 782, 147, 342, 1362, 1039, 610, 605, 987, 1461, 919, 2507, 1576, 862, 407, 166, 473, 285, 853, 98, 522, 81, 385, 982, 740, 254, 485, 639, 935, 877, 763, 533, 1152, 1697, 1293, 753, 804, 132, 1040, 160, 824, 573, 1049, 459, 700, 1998, 135, 4347, 336, 1358, 967, 153, 182, 849, 1245, 782, 1059, 357] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[1085, 17, 1125, 516, 65, 1195, 9, 636, 36, 722, 1016, 834, 800, 241, 2731, 660, 413, 1156, 695, 1021, 587, 240, 1470, 227, 731, 69, 736, 57, 59, 1284, 127, 297, 687, 927, 1012, 657, 1985, 510, 1346, 600, 2040, 723, 733, 316, 1024, 2146, 242, 817, 1698, 1045, 865, 2143, 129, 838, 54, 61, 372, 138, 680, 1393, 604, 1136, 959, 415] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[372, 27, 369, 664, 209, 445, 3, 666, 1987, 435, 0, 0, 0, 0, 0, 0, 3, 1302, 1077, 838, 840, 912, 506, 1691, 0, 0, 0, 0, 0, 0, 0, 0, 793, 873, 555, 1618, 726, 625, 1259, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 64, 192, 111, 64, 145, 2040, 551, 183, 604, 689, 682, 0, 0, 0, 0, 0, 1110, 433, 1339, 1099, 176, 377, 1883, 1235, 0, 0, 0, 0, 0, 0, 0, 0, 2610, 618, 185, 464, 525, 706, 1026, 254, 0, 0, 0, 0, 0, 0, 0, 0, 881, 923, 152, 254, 28, 1517, 328, 364, 0, 0, 0, 0, 0, 0, 0, 0, 154, 385, 316, 1800, 626, 507, 914, 207, 465, 104, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5177, 7169, 7516, 5325, 7652, 6388, 4447, 5478] avg=6144.0000 max=7652 max_rank=4 min=4447 min_rank=6 imbalance_ratio=1.245443 max_load_reduction=41500 max_load_reduction_pct=0.844320 moved_to_echo=23965 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[658, 20, 763, 400, 173, 627, 78, 868, 1482, 503, 0, 0, 0, 0, 0, 0, 365, 2127, 649, 1495, 1059, 173, 526, 1878, 0, 0, 0, 0, 0, 0, 0, 0, 896, 532, 586, 1943, 1134, 554, 394, 930, 0, 0, 0, 0, 0, 0, 0, 0, 560, 64, 611, 420, 361, 1055, 194, 1154, 699, 717, 575, 0, 0, 0, 0, 0, 1295, 803, 274, 702, 131, 940, 217, 775, 0, 0, 0, 0, 0, 0, 0, 0, 2200, 1076, 1071, 220, 674, 1241, 68, 210, 0, 0, 0, 0, 0, 0, 0, 0, 405, 649, 890, 293, 272, 1473, 952, 534, 0, 0, 0, 0, 0, 0, 0, 0, 313, 144, 1007, 292, 406, 461, 1359, 213, 282, 87, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5572, 8272, 6969, 6410, 5137, 6760, 5468, 4564] avg=6144.0000 max=8272 max_rank=1 min=4564 min_rank=7 imbalance_ratio=1.346354 max_load_reduction=40880 max_load_reduction_pct=0.831706 moved_to_echo=21929 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[401, 210, 884, 1241, 190, 644, 23, 275, 2553, 873, 0, 0, 0, 0, 0, 0, 85, 373, 595, 979, 996, 486, 437, 795, 0, 0, 0, 0, 0, 0, 0, 0, 1312, 837, 531, 814, 818, 247, 833, 940, 0, 0, 0, 0, 0, 0, 0, 0, 370, 856, 280, 252, 81, 528, 381, 1038, 1212, 596, 235, 0, 0, 0, 0, 0, 975, 793, 346, 1239, 226, 456, 1127, 663, 0, 0, 0, 0, 0, 0, 0, 0, 903, 943, 1097, 512, 377, 1094, 118, 419, 0, 0, 0, 0, 0, 0, 0, 0, 1814, 1336, 799, 507, 321, 1676, 114, 498, 0, 0, 0, 0, 0, 0, 0, 0, 479, 413, 702, 853, 1149, 538, 1603, 381, 445, 35, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7294, 4746, 6332, 5829, 5825, 5463, 7065, 6598] avg=6144.0000 max=7294 max_rank=0 min=4746 min_rank=1 imbalance_ratio=1.187174 max_load_reduction=41858 max_load_reduction_pct=0.851603 moved_to_echo=24951 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[245, 15, 1382, 289, 123, 645, 196, 882, 2502, 797, 0, 0, 0, 0, 0, 0, 13, 1296, 699, 1027, 888, 197, 376, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1178, 439, 604, 1101, 340, 175, 1298, 857, 0, 0, 0, 0, 0, 0, 0, 0, 549, 1100, 37, 15, 306, 1356, 1462, 174, 1107, 511, 787, 0, 0, 0, 0, 0, 481, 775, 1216, 917, 221, 592, 1367, 1195, 0, 0, 0, 0, 0, 0, 0, 0, 3010, 428, 626, 762, 394, 337, 189, 556, 0, 0, 0, 0, 0, 0, 0, 0, 941, 783, 382, 463, 386, 1054, 230, 399, 0, 0, 0, 0, 0, 0, 0, 0, 350, 743, 610, 321, 650, 1604, 526, 290, 233, 120, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7076, 5529, 5992, 7404, 6764, 6302, 4638, 5447] avg=6144.0000 max=7404 max_rank=3 min=4638 min_rank=6 imbalance_ratio=1.205078 max_load_reduction=41748 max_load_reduction_pct=0.849365 moved_to_echo=23151 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[652, 92, 1306, 887, 528, 228, 101, 516, 1812, 768, 0, 0, 0, 0, 0, 0, 164, 532, 850, 1717, 1173, 879, 464, 468, 0, 0, 0, 0, 0, 0, 0, 0, 372, 1387, 153, 510, 1456, 75, 653, 729, 0, 0, 0, 0, 0, 0, 0, 0, 729, 155, 209, 174, 102, 1271, 986, 274, 1066, 631, 449, 0, 0, 0, 0, 0, 1212, 722, 215, 301, 161, 452, 339, 1442, 0, 0, 0, 0, 0, 0, 0, 0, 1718, 246, 491, 301, 981, 561, 753, 261, 0, 0, 0, 0, 0, 0, 0, 0, 588, 433, 1360, 445, 83, 1721, 795, 1273, 0, 0, 0, 0, 0, 0, 0, 0, 810, 297, 944, 1601, 418, 756, 1670, 477, 739, 68, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6890, 6247, 5335, 6046, 4844, 5312, 6698, 7780] avg=6144.0000 max=7780 max_rank=7 min=4844 min_rank=4 imbalance_ratio=1.266276 max_load_reduction=41372 max_load_reduction_pct=0.841715 moved_to_echo=24634 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[319, 71, 571, 366, 298, 289, 117, 377, 2728, 586, 0, 0, 0, 0, 0, 0, 44, 967, 895, 1329, 818, 273, 710, 704, 0, 0, 0, 0, 0, 0, 0, 0, 806, 1435, 1114, 949, 187, 186, 366, 233, 0, 0, 0, 0, 0, 0, 0, 0, 521, 53, 38, 85, 139, 693, 1897, 588, 811, 978, 312, 0, 0, 0, 0, 0, 589, 694, 700, 766, 238, 379, 1800, 1484, 0, 0, 0, 0, 0, 0, 0, 0, 1174, 525, 1044, 478, 1602, 943, 229, 81, 0, 0, 0, 0, 0, 0, 0, 0, 1092, 876, 436, 328, 209, 3152, 492, 367, 0, 0, 0, 0, 0, 0, 0, 0, 787, 123, 247, 461, 2129, 1037, 858, 157, 771, 51, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5722, 5740, 5276, 6115, 6650, 6076, 6952, 6621] avg=6144.0000 max=6952 max_rank=6 min=5276 min_rank=2 imbalance_ratio=1.131510 max_load_reduction=42200 max_load_reduction_pct=0.858561 moved_to_echo=26299 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1085, 17, 1125, 516, 65, 1195, 9, 636, 1824, 722, 0, 0, 0, 0, 0, 0, 36, 722, 1016, 834, 800, 241, 1156, 660, 0, 0, 0, 0, 0, 0, 0, 0, 413, 755, 695, 1021, 587, 240, 1470, 227, 0, 0, 0, 0, 0, 0, 0, 0, 731, 69, 736, 57, 59, 1284, 127, 297, 1002, 1575, 409, 0, 0, 0, 0, 0, 687, 927, 1012, 657, 161, 510, 1346, 600, 0, 0, 0, 0, 0, 0, 0, 0, 1569, 723, 733, 316, 1024, 2146, 242, 817, 0, 0, 0, 0, 0, 0, 0, 0, 1698, 1045, 865, 419, 129, 838, 54, 61, 0, 0, 0, 0, 0, 0, 0, 0, 372, 138, 680, 1393, 604, 1136, 959, 415, 401, 62, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7194, 5465, 5408, 6346, 5900, 7570, 5109, 6160] avg=6144.0000 max=7570 max_rank=5 min=5109 min_rank=6 imbalance_ratio=1.232096 max_load_reduction=41582 max_load_reduction_pct=0.845988 moved_to_echo=24739 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[386, 8, 698, 196, 214, 564, 5, 782, 701, 673, 0, 0, 0, 0, 0, 0, 147, 342, 1362, 1039, 610, 605, 418, 1461, 0, 0, 0, 0, 0, 0, 0, 0, 919, 1636, 1576, 862, 407, 166, 473, 285, 0, 0, 0, 0, 0, 0, 0, 0, 853, 98, 522, 81, 385, 982, 740, 254, 934, 569, 259, 0, 0, 0, 0, 0, 485, 639, 935, 877, 62, 533, 1152, 1697, 0, 0, 0, 0, 0, 0, 0, 0, 995, 753, 804, 132, 1040, 160, 824, 573, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 459, 700, 391, 135, 4347, 336, 1358, 0, 0, 0, 0, 0, 0, 0, 0, 967, 153, 182, 849, 1245, 782, 1059, 357, 871, 39, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4227, 5984, 6324, 5677, 6380, 5281, 8775, 6504] avg=6144.0000 max=8775 max_rank=6 min=4227 min_rank=0 imbalance_ratio=1.428223 max_load_reduction=40377 max_load_reduction_pct=0.821472 moved_to_echo=26940 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[14208, 5376, 6144, 3200, 6656, 7296, 3456, 3200, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[4224, 512, 7168, 4608, 1920, 4736, 640, 5120, 15616, 5376, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4352, 2432, 4736, 7680, 7296, 6912, 8960, 2560, 4224, 640, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[896, 7680, 7168, 9344, 7296, 3840, 4608, 8704, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[6912, 5888, 6144, 6656, 1408, 4352, 9344, 9216, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4480, 2688, 2560, 1152, 1664, 9216, 6400, 3968, 7552, 6272, 3712, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6784, 7936, 5888, 8832, 5760, 2304, 6784, 5376, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[8576, 6528, 5632, 3200, 1664, 15872, 3328, 4864, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[23, 510, 280, 894, 365, 48, 235, 490, 334, 1369, 1623, 459, 1189, 283, 290, 557, 668, 413, 857, 972, 1678, 420, 126, 318, 1501, 133, 402, 1409, 507, 455, 437, 263, 110, 10, 741, 30, 1134, 903, 436, 178, 3445, 838, 1693, 180, 146, 1358, 162, 129, 410, 3281, 672, 189, 2084, 2968, 637, 1405, 2417, 1046, 294, 237, 511, 1238, 645, 117] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[81, 287, 131, 1482, 534, 302, 226, 210, 534, 473, 584, 388, 327, 874, 452, 367, 1624, 143, 36, 505, 503, 685, 569, 1278, 578, 85, 545, 1462, 1017, 412, 108, 330, 146, 210, 151, 182, 452, 944, 946, 932, 3589, 733, 1421, 425, 208, 1290, 102, 754, 496, 2135, 691, 1499, 2653, 1951, 773, 847, 2586, 1438, 278, 673, 909, 1617, 774, 215] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[449, 138, 43, 848, 474, 281, 312, 443, 681, 667, 352, 473, 275, 866, 392, 372, 1913, 595, 67, 285, 957, 1649, 216, 1654, 1741, 35, 98, 581, 387, 757, 439, 52, 15, 31, 130, 207, 269, 943, 1014, 269, 4922, 2320, 1521, 633, 145, 865, 568, 727, 282, 579, 271, 274, 3691, 2609, 148, 722, 1039, 2941, 239, 605, 482, 1549, 262, 388] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[53, 432, 140, 1170, 759, 133, 109, 870, 886, 1413, 320, 528, 854, 233, 796, 381, 1501, 1022, 200, 730, 1281, 1302, 464, 853, 247, 75, 283, 933, 544, 66, 1531, 187, 45, 69, 926, 220, 862, 738, 1428, 595, 4210, 1562, 1346, 1055, 119, 897, 48, 121, 560, 2458, 537, 145, 2565, 1249, 128, 690, 2559, 1574, 165, 416, 46, 458, 800, 265] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[269, 861, 308, 595, 321, 366, 324, 471, 459, 2056, 1501, 462, 621, 812, 813, 141, 996, 676, 159, 1065, 1870, 2883, 809, 1011, 175, 345, 453, 867, 613, 597, 317, 620, 89, 242, 716, 640, 622, 174, 1000, 919, 3091, 1835, 1758, 282, 58, 1018, 61, 747, 73, 801, 800, 228, 1683, 2351, 922, 774, 713, 746, 550, 475, 810, 912, 85, 141] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[297, 500, 152, 1330, 294, 1077, 262, 421, 234, 1274, 1712, 264, 1154, 600, 302, 503, 1260, 819, 317, 359, 543, 768, 280, 316, 563, 895, 170, 1026, 1090, 201, 670, 395, 20, 271, 314, 392, 717, 530, 441, 286, 5589, 2769, 1173, 488, 339, 1378, 278, 832, 234, 1170, 136, 292, 2205, 2091, 1106, 1298, 1076, 1163, 342, 178, 350, 362, 1062, 222] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[215, 196, 89, 2132, 758, 609, 388, 155, 282, 481, 784, 274, 276, 1240, 573, 88, 796, 498, 361, 957, 348, 2790, 218, 1311, 212, 67, 206, 1670, 633, 674, 192, 31, 109, 119, 1233, 1163, 326, 163, 1112, 421, 4039, 1430, 1409, 829, 242, 1222, 903, 559, 145, 540, 1090, 190, 2770, 1790, 1142, 519, 1049, 1250, 701, 497, 1377, 472, 151, 686] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[39, 242, 165, 1001, 499, 607, 198, 337, 148, 823, 542, 207, 686, 527, 1798, 317, 1032, 156, 294, 1614, 636, 844, 459, 892, 1157, 25, 538, 498, 714, 220, 223, 53, 246, 86, 639, 611, 1323, 1216, 947, 676, 5280, 1467, 1714, 638, 125, 165, 95, 108, 264, 755, 1264, 132, 3557, 3122, 588, 640, 1361, 1187, 554, 509, 235, 1773, 56, 328] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[449, 138, 43, 848, 474, 281, 312, 443, 3014, 0, 0, 0, 0, 0, 0, 0, 681, 667, 352, 473, 275, 866, 392, 372, 533, 422, 81, 0, 0, 0, 0, 0, 1913, 595, 67, 285, 957, 1116, 216, 1654, 0, 0, 0, 0, 0, 0, 0, 0, 1741, 35, 98, 581, 387, 757, 439, 52, 2517, 140, 0, 0, 0, 0, 0, 0, 15, 31, 130, 207, 269, 943, 1014, 269, 1351, 1174, 0, 0, 0, 0, 0, 0, 557, 2320, 1521, 633, 145, 865, 568, 727, 0, 0, 0, 0, 0, 0, 0, 0, 282, 579, 271, 274, 0, 2047, 148, 722, 0, 0, 0, 0, 0, 0, 0, 0, 958, 2941, 239, 605, 482, 1549, 262, 388, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6002, 5114, 6803, 6747, 5403, 7336, 4323, 7424] avg=6144.0000 max=7424 max_rank=7 min=4323 min_rank=6 imbalance_ratio=1.208333 max_load_reduction=41728 max_load_reduction_pct=0.848958 moved_to_echo=24486 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[269, 861, 308, 595, 321, 366, 324, 471, 1892, 0, 0, 0, 0, 0, 0, 0, 459, 2056, 1501, 462, 621, 812, 813, 141, 932, 380, 55, 0, 0, 0, 0, 0, 996, 676, 159, 1065, 1870, 1951, 809, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 175, 345, 453, 867, 613, 597, 317, 620, 1147, 126, 0, 0, 0, 0, 0, 0, 89, 242, 716, 640, 622, 174, 1000, 919, 848, 536, 0, 0, 0, 0, 0, 0, 351, 1835, 1758, 282, 58, 1018, 61, 747, 0, 0, 0, 0, 0, 0, 0, 0, 73, 801, 800, 228, 0, 1845, 922, 774, 0, 0, 0, 0, 0, 0, 0, 0, 658, 746, 550, 475, 810, 912, 85, 141, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5407, 8232, 8537, 5260, 5786, 6110, 5443, 4377] avg=6144.0000 max=8537 max_rank=2 min=4377 min_rank=7 imbalance_ratio=1.389486 max_load_reduction=40615 max_load_reduction_pct=0.826314 moved_to_echo=21716 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[23, 510, 280, 894, 365, 48, 235, 490, 2109, 0, 0, 0, 0, 0, 0, 0, 334, 1369, 1623, 459, 1189, 283, 290, 557, 135, 480, 189, 0, 0, 0, 0, 0, 668, 413, 857, 972, 1678, 285, 126, 318, 0, 0, 0, 0, 0, 0, 0, 0, 1501, 133, 402, 1409, 507, 455, 437, 263, 1421, 160, 0, 0, 0, 0, 0, 0, 110, 10, 741, 30, 1134, 903, 436, 178, 945, 663, 0, 0, 0, 0, 0, 0, 391, 838, 1693, 180, 146, 1358, 162, 129, 0, 0, 0, 0, 0, 0, 0, 0, 410, 3281, 672, 189, 0, 2328, 637, 1405, 0, 0, 0, 0, 0, 0, 0, 0, 2228, 1046, 294, 237, 511, 1238, 645, 117, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4954, 6908, 5317, 6688, 5150, 4897, 8922, 6316] avg=6144.0000 max=8922 max_rank=6 min=4897 min_rank=5 imbalance_ratio=1.452148 max_load_reduction=40230 max_load_reduction_pct=0.818481 moved_to_echo=25285 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[81, 287, 131, 1482, 534, 302, 226, 210, 2197, 0, 0, 0, 0, 0, 0, 0, 534, 473, 584, 388, 327, 874, 452, 367, 221, 315, 202, 0, 0, 0, 0, 0, 1624, 143, 36, 505, 503, 464, 569, 1278, 0, 0, 0, 0, 0, 0, 0, 0, 578, 85, 545, 1462, 1017, 412, 108, 330, 1810, 105, 0, 0, 0, 0, 0, 0, 146, 210, 151, 182, 452, 944, 946, 932, 985, 843, 0, 0, 0, 0, 0, 0, 407, 733, 1421, 425, 208, 1290, 102, 754, 0, 0, 0, 0, 0, 0, 0, 0, 496, 2135, 691, 1499, 0, 1531, 773, 847, 0, 0, 0, 0, 0, 0, 0, 0, 2384, 1438, 278, 673, 909, 1617, 774, 215, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5450, 4737, 5122, 6452, 5791, 5340, 7972, 8288] avg=6144.0000 max=8288 max_rank=7 min=4737 min_rank=1 imbalance_ratio=1.348958 max_load_reduction=40864 max_load_reduction_pct=0.831380 moved_to_echo=27391 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[297, 500, 152, 1330, 294, 1077, 262, 421, 3422, 0, 0, 0, 0, 0, 0, 0, 234, 1274, 1712, 264, 1154, 600, 302, 503, 248, 338, 84, 0, 0, 0, 0, 0, 1260, 819, 317, 359, 543, 520, 280, 316, 0, 0, 0, 0, 0, 0, 0, 0, 563, 895, 170, 1026, 1090, 201, 670, 395, 1504, 112, 0, 0, 0, 0, 0, 0, 20, 271, 314, 392, 717, 530, 441, 286, 1534, 701, 0, 0, 0, 0, 0, 0, 633, 2769, 1173, 488, 339, 1378, 278, 832, 0, 0, 0, 0, 0, 0, 0, 0, 234, 1170, 136, 292, 0, 1641, 1106, 1298, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1163, 342, 178, 350, 362, 1062, 222, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7755, 6713, 4414, 6626, 5206, 7890, 5877, 4671] avg=6144.0000 max=7890 max_rank=5 min=4414 min_rank=2 imbalance_ratio=1.284180 max_load_reduction=41262 max_load_reduction_pct=0.839478 moved_to_echo=23644 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[215, 196, 89, 2132, 758, 609, 388, 155, 2477, 0, 0, 0, 0, 0, 0, 0, 282, 481, 784, 274, 276, 1240, 573, 88, 906, 291, 87, 0, 0, 0, 0, 0, 796, 498, 361, 957, 348, 1884, 218, 1311, 0, 0, 0, 0, 0, 0, 0, 0, 212, 67, 206, 1670, 633, 674, 192, 31, 1890, 100, 0, 0, 0, 0, 0, 0, 109, 119, 1233, 1163, 326, 163, 1112, 421, 1112, 880, 0, 0, 0, 0, 0, 0, 450, 1430, 1409, 829, 242, 1222, 903, 559, 0, 0, 0, 0, 0, 0, 0, 0, 145, 540, 1090, 190, 0, 1399, 1142, 519, 0, 0, 0, 0, 0, 0, 0, 0, 962, 1250, 701, 497, 1377, 472, 151, 686, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7019, 5282, 6373, 5675, 6638, 7044, 5025, 6096] avg=6144.0000 max=7044 max_rank=5 min=5025 min_rank=6 imbalance_ratio=1.146484 max_load_reduction=42108 max_load_reduction_pct=0.856689 moved_to_echo=24803 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[39, 242, 165, 1001, 499, 607, 198, 337, 3233, 0, 0, 0, 0, 0, 0, 0, 148, 823, 542, 207, 686, 527, 1798, 317, 273, 505, 106, 0, 0, 0, 0, 0, 1032, 156, 294, 1614, 636, 571, 459, 892, 0, 0, 0, 0, 0, 0, 0, 0, 1157, 25, 538, 498, 714, 220, 223, 53, 2426, 168, 0, 0, 0, 0, 0, 0, 246, 86, 639, 611, 1323, 1216, 947, 676, 1449, 1131, 0, 0, 0, 0, 0, 0, 598, 1467, 1714, 638, 125, 165, 95, 108, 0, 0, 0, 0, 0, 0, 0, 0, 264, 755, 1264, 132, 0, 2449, 588, 640, 0, 0, 0, 0, 0, 0, 0, 0, 1255, 1187, 554, 509, 235, 1773, 56, 328, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6321, 5932, 5654, 6022, 8324, 4910, 6092, 5897] avg=6144.0000 max=8324 max_rank=4 min=4910 min_rank=5 imbalance_ratio=1.354818 max_load_reduction=40828 max_load_reduction_pct=0.830648 moved_to_echo=25223 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[53, 432, 140, 1170, 759, 133, 109, 870, 2578, 0, 0, 0, 0, 0, 0, 0, 886, 1413, 320, 528, 854, 233, 796, 381, 421, 202, 200, 0, 0, 0, 0, 0, 1501, 1022, 200, 730, 1281, 881, 464, 853, 0, 0, 0, 0, 0, 0, 0, 0, 247, 75, 283, 933, 544, 66, 1531, 187, 1749, 67, 0, 0, 0, 0, 0, 0, 45, 69, 926, 220, 862, 738, 1428, 595, 1155, 816, 0, 0, 0, 0, 0, 0, 477, 1562, 1346, 1055, 119, 897, 48, 121, 0, 0, 0, 0, 0, 0, 0, 0, 560, 2458, 537, 145, 0, 980, 128, 690, 0, 0, 0, 0, 0, 0, 0, 0, 2359, 1574, 165, 416, 46, 458, 800, 265, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6244, 6234, 6932, 5682, 6854, 5625, 5498, 6083] avg=6144.0000 max=6932 max_rank=2 min=5498 min_rank=6 imbalance_ratio=1.128255 max_load_reduction=42220 max_load_reduction_pct=0.858968 moved_to_echo=24060 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[9856, 4352, 2304, 6528, 7936, 7680, 3200, 7680, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1536, 3200, 1408, 9472, 4096, 3456, 2176, 3456, 20992, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3584, 8576, 7424, 3072, 5504, 5504, 5504, 2816, 3712, 2944, 1024, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[896, 1152, 4864, 3456, 5760, 5632, 7424, 4352, 9472, 6784, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2560, 11776, 5504, 3072, 0, 14336, 5504, 6912, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6272, 1664, 2816, 8448, 5632, 3456, 3968, 2048, 14464, 1024, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3968, 13056, 12160, 4608, 1408, 8320, 2304, 4096, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[11904, 11392, 3200, 3712, 4736, 8448, 3840, 2432, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[391, 756, 169, 615, 576, 629, 1525, 1010, 1760, 53, 168, 483, 3216, 311, 37, 200, 1948, 777, 518, 58, 378, 544, 45, 361, 79, 2026, 1054, 56, 159, 404, 1602, 671, 149, 68, 942, 268, 1660, 227, 1929, 1182, 1014, 51, 1863, 1412, 320, 473, 4637, 88, 185, 328, 1313, 1841, 458, 4, 85, 2006, 116, 562, 281, 1216, 902, 301, 652, 40] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[133, 1250, 330, 455, 330, 982, 1030, 1994, 1652, 711, 226, 937, 2337, 180, 72, 382, 1392, 779, 616, 178, 154, 352, 126, 836, 305, 1594, 466, 601, 245, 264, 1051, 134, 387, 638, 1135, 520, 1897, 585, 1587, 2030, 618, 1097, 608, 789, 96, 1604, 2371, 72, 255, 175, 1579, 455, 688, 87, 46, 1039, 627, 524, 1132, 294, 1705, 548, 834, 1036] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[908, 539, 474, 1134, 472, 427, 881, 1343, 1995, 530, 80, 909, 898, 585, 20, 111, 1414, 614, 506, 306, 174, 2192, 144, 1319, 91, 1777, 324, 227, 60, 470, 575, 851, 29, 202, 1467, 648, 1871, 101, 1214, 1633, 823, 319, 541, 1503, 323, 178, 1518, 48, 374, 491, 1778, 1027, 1236, 247, 145, 2390, 55, 467, 3299, 477, 965, 378, 776, 279] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[418, 1341, 391, 641, 922, 200, 1648, 778, 1934, 878, 526, 1208, 227, 1133, 38, 292, 1815, 625, 730, 153, 207, 677, 170, 781, 265, 1936, 2955, 72, 167, 417, 398, 549, 106, 256, 1667, 535, 684, 153, 1921, 915, 1427, 279, 764, 882, 922, 719, 2042, 94, 540, 441, 1006, 220, 935, 893, 448, 1159, 48, 300, 1047, 1209, 1026, 608, 988, 426] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[711, 1238, 317, 378, 333, 151, 2275, 2307, 1655, 649, 478, 1232, 2207, 369, 33, 301, 1971, 770, 653, 455, 435, 517, 32, 768, 108, 1727, 667, 259, 221, 582, 1374, 95, 123, 44, 310, 311, 1148, 543, 808, 1413, 712, 247, 1782, 2480, 369, 843, 887, 31, 349, 461, 536, 383, 217, 189, 124, 2169, 302, 503, 1146, 551, 1629, 654, 2319, 301] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[398, 1169, 571, 911, 349, 825, 688, 613, 1590, 586, 54, 572, 401, 502, 46, 170, 1958, 726, 1546, 268, 83, 400, 499, 546, 261, 1869, 840, 67, 159, 278, 2718, 108, 260, 98, 1333, 166, 2098, 310, 1166, 1972, 550, 245, 1490, 896, 466, 437, 1667, 30, 622, 342, 1670, 322, 354, 134, 269, 3050, 107, 1191, 1642, 316, 737, 542, 1819, 1080] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[337, 546, 214, 860, 393, 121, 3443, 1545, 1386, 809, 136, 976, 1316, 204, 45, 119, 2069, 217, 1002, 416, 335, 724, 2138, 865, 126, 1655, 900, 15, 219, 121, 1158, 116, 183, 125, 772, 531, 897, 263, 429, 1002, 514, 652, 1128, 1953, 227, 195, 2703, 20, 389, 302, 1424, 879, 142, 36, 155, 3645, 620, 1037, 1595, 488, 926, 36, 1045, 313] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[188, 1164, 25, 827, 360, 262, 1872, 2315, 1007, 879, 365, 1413, 1544, 283, 135, 224, 1037, 187, 1060, 69, 410, 505, 612, 1242, 19, 1307, 1184, 113, 269, 438, 577, 19, 279, 35, 922, 957, 1006, 93, 3803, 1157, 433, 537, 494, 2505, 618, 648, 4456, 57, 661, 370, 1040, 127, 339, 46, 79, 1617, 236, 909, 847, 253, 492, 228, 705, 1292] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[391, 756, 169, 615, 576, 629, 1156, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1760, 53, 168, 483, 3216, 311, 37, 200, 471, 27, 0, 0, 0, 0, 0, 0, 1948, 777, 518, 58, 378, 544, 45, 361, 657, 130, 0, 0, 0, 0, 0, 0, 79, 2026, 1054, 56, 159, 404, 1602, 671, 1769, 0, 0, 0, 0, 0, 0, 0, 149, 68, 942, 268, 1660, 227, 1328, 1182, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 51, 1863, 1412, 320, 473, 2211, 88, 0, 0, 0, 0, 0, 0, 0, 0, 185, 328, 1313, 1841, 458, 4, 85, 2006, 323, 0, 0, 0, 0, 0, 0, 0, 116, 562, 281, 1216, 902, 301, 652, 40, 19, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5302, 6726, 5416, 7820, 5824, 7432, 6543, 4089] avg=6144.0000 max=7820 max_rank=3 min=4089 min_rank=7 imbalance_ratio=1.272786 max_load_reduction=41332 max_load_reduction_pct=0.840902 moved_to_echo=23888 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[133, 1250, 330, 455, 330, 982, 781, 1994, 0, 0, 0, 0, 0, 0, 0, 0, 1652, 711, 226, 937, 2337, 180, 72, 382, 387, 18, 0, 0, 0, 0, 0, 0, 1392, 779, 616, 178, 154, 352, 126, 836, 336, 107, 0, 0, 0, 0, 0, 0, 305, 1594, 466, 601, 245, 264, 1051, 134, 904, 0, 0, 0, 0, 0, 0, 0, 387, 638, 1135, 520, 1897, 585, 1093, 2030, 0, 0, 0, 0, 0, 0, 0, 0, 618, 1097, 608, 789, 96, 1604, 1131, 72, 0, 0, 0, 0, 0, 0, 0, 0, 255, 175, 1579, 455, 688, 87, 46, 1039, 218, 0, 0, 0, 0, 0, 0, 0, 627, 524, 1132, 294, 1705, 548, 834, 1036, 13, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6255, 6902, 4876, 5564, 8285, 6015, 4542, 6713] avg=6144.0000 max=8285 max_rank=4 min=4542 min_rank=6 imbalance_ratio=1.348470 max_load_reduction=40867 max_load_reduction_pct=0.831441 moved_to_echo=25555 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[418, 1341, 391, 641, 922, 200, 1249, 778, 0, 0, 0, 0, 0, 0, 0, 0, 1934, 878, 526, 1208, 227, 1133, 38, 292, 469, 29, 0, 0, 0, 0, 0, 0, 1815, 625, 730, 153, 207, 677, 170, 781, 289, 130, 0, 0, 0, 0, 0, 0, 265, 1936, 2955, 72, 167, 417, 398, 549, 779, 0, 0, 0, 0, 0, 0, 0, 106, 256, 1667, 535, 684, 153, 1322, 915, 0, 0, 0, 0, 0, 0, 0, 0, 1427, 279, 764, 882, 922, 719, 974, 94, 0, 0, 0, 0, 0, 0, 0, 0, 540, 441, 1006, 220, 935, 893, 448, 1159, 349, 0, 0, 0, 0, 0, 0, 0, 48, 300, 1047, 1209, 1026, 608, 988, 426, 21, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5940, 6734, 5577, 7538, 5638, 6061, 5991, 5673] avg=6144.0000 max=7538 max_rank=3 min=5577 min_rank=2 imbalance_ratio=1.226888 max_load_reduction=41614 max_load_reduction_pct=0.846639 moved_to_echo=23363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[908, 539, 474, 1134, 472, 427, 669, 1343, 0, 0, 0, 0, 0, 0, 0, 0, 1995, 530, 80, 909, 898, 585, 20, 111, 296, 15, 0, 0, 0, 0, 0, 0, 1414, 614, 506, 306, 174, 2192, 144, 1319, 215, 82, 0, 0, 0, 0, 0, 0, 91, 1777, 324, 227, 60, 470, 575, 851, 579, 0, 0, 0, 0, 0, 0, 0, 29, 202, 1467, 648, 1871, 101, 836, 1633, 0, 0, 0, 0, 0, 0, 0, 0, 823, 319, 541, 1503, 323, 178, 724, 48, 0, 0, 0, 0, 0, 0, 0, 0, 374, 491, 1778, 1027, 1236, 247, 145, 2390, 186, 0, 0, 0, 0, 0, 0, 0, 55, 467, 3299, 477, 965, 378, 776, 279, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5966, 5439, 6966, 4954, 6787, 4459, 7874, 6707] avg=6144.0000 max=7874 max_rank=6 min=4459 min_rank=5 imbalance_ratio=1.281576 max_load_reduction=41278 max_load_reduction_pct=0.839803 moved_to_echo=25827 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[398, 1169, 571, 911, 349, 825, 522, 613, 0, 0, 0, 0, 0, 0, 0, 0, 1590, 586, 54, 572, 401, 502, 46, 170, 284, 12, 0, 0, 0, 0, 0, 0, 1958, 726, 1546, 268, 83, 400, 499, 546, 236, 78, 0, 0, 0, 0, 0, 0, 261, 1869, 840, 67, 159, 278, 2718, 108, 636, 0, 0, 0, 0, 0, 0, 0, 260, 98, 1333, 166, 2098, 310, 804, 1972, 0, 0, 0, 0, 0, 0, 0, 0, 550, 245, 1490, 896, 466, 437, 795, 30, 0, 0, 0, 0, 0, 0, 0, 0, 622, 342, 1670, 322, 354, 134, 269, 3050, 146, 0, 0, 0, 0, 0, 0, 0, 107, 1191, 1642, 316, 737, 542, 1819, 1080, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5358, 4217, 6340, 6936, 7041, 4909, 6909, 7442] avg=6144.0000 max=7442 max_rank=7 min=4217 min_rank=1 imbalance_ratio=1.211263 max_load_reduction=41710 max_load_reduction_pct=0.848592 moved_to_echo=26301 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[711, 1238, 317, 378, 333, 151, 1713, 2307, 0, 0, 0, 0, 0, 0, 0, 0, 1655, 649, 478, 1232, 2207, 369, 33, 301, 201, 42, 0, 0, 0, 0, 0, 0, 1971, 770, 653, 455, 435, 517, 32, 768, 129, 58, 0, 0, 0, 0, 0, 0, 108, 1727, 667, 259, 221, 582, 1374, 95, 340, 0, 0, 0, 0, 0, 0, 0, 123, 44, 310, 311, 1148, 543, 549, 1413, 0, 0, 0, 0, 0, 0, 0, 0, 712, 247, 1782, 2480, 369, 843, 418, 31, 0, 0, 0, 0, 0, 0, 0, 0, 349, 461, 536, 383, 217, 189, 124, 2169, 487, 0, 0, 0, 0, 0, 0, 0, 302, 503, 1146, 551, 1629, 654, 2319, 301, 33, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7148, 7167, 5788, 5373, 4441, 6882, 4915, 7438] avg=6144.0000 max=7438 max_rank=7 min=4441 min_rank=4 imbalance_ratio=1.210612 max_load_reduction=41714 max_load_reduction_pct=0.848674 moved_to_echo=23676 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[337, 546, 214, 860, 393, 121, 2608, 1545, 0, 0, 0, 0, 0, 0, 0, 0, 1386, 809, 136, 976, 1316, 204, 45, 119, 104, 61, 0, 0, 0, 0, 0, 0, 2069, 217, 1002, 416, 335, 724, 2138, 865, 383, 29, 0, 0, 0, 0, 0, 0, 126, 1655, 900, 15, 219, 121, 1158, 116, 1031, 0, 0, 0, 0, 0, 0, 0, 183, 125, 772, 531, 897, 263, 296, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 514, 652, 1128, 1953, 227, 195, 1289, 20, 0, 0, 0, 0, 0, 0, 0, 0, 389, 302, 1424, 879, 142, 36, 155, 3645, 730, 0, 0, 0, 0, 0, 0, 0, 620, 1037, 1595, 488, 926, 36, 1045, 313, 44, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6624, 5156, 8178, 5341, 4069, 5978, 7702, 6104] avg=6144.0000 max=8178 max_rank=2 min=4069 min_rank=4 imbalance_ratio=1.331055 max_load_reduction=40974 max_load_reduction_pct=0.833618 moved_to_echo=23853 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[188, 1164, 25, 827, 360, 262, 1418, 2315, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 879, 365, 1413, 1544, 283, 135, 224, 928, 33, 0, 0, 0, 0, 0, 0, 1037, 187, 1060, 69, 410, 505, 612, 1242, 632, 257, 0, 0, 0, 0, 0, 0, 19, 1307, 1184, 113, 269, 438, 577, 19, 1700, 0, 0, 0, 0, 0, 0, 0, 279, 35, 922, 957, 1006, 93, 2618, 1157, 0, 0, 0, 0, 0, 0, 0, 0, 433, 537, 494, 2505, 618, 648, 2124, 57, 0, 0, 0, 0, 0, 0, 0, 0, 661, 370, 1040, 127, 339, 46, 79, 1617, 397, 0, 0, 0, 0, 0, 0, 0, 236, 909, 847, 253, 492, 228, 705, 1292, 24, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6559, 6811, 6011, 5626, 7067, 7416, 4676, 4986] avg=6144.0000 max=7416 max_rank=5 min=4676 min_rank=6 imbalance_ratio=1.207031 max_load_reduction=41736 max_load_reduction_pct=0.849121 moved_to_echo=24145 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[13056, 5120, 2048, 7808, 12160, 3584, 512, 1920, 3200, 256, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2176, 5504, 11008, 4864, 8448, 3328, 9216, 4864, 256, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[1536, 1536, 8576, 3968, 11264, 2304, 8960, 11392, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3584, 8064, 2560, 5888, 3840, 3712, 10240, 12032, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6144, 3456, 8704, 12544, 3456, 5120, 9728, 512, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3456, 2944, 10368, 5376, 4480, 1664, 1408, 17152, 2944, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[13696, 4736, 6656, 1920, 2176, 6016, 3840, 6784, 2944, 896, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1280, 13952, 8448, 1536, 1536, 3072, 9472, 2560, 7808, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=17 tokens_per_expert_current_ep_rank=[113, 167, 35, 623, 129, 910, 1168, 476, 90, 1182, 787, 626, 434, 1193, 540, 1012, 952, 569, 1148, 197, 606, 376, 957, 712, 1857, 557, 269, 2928, 432, 202, 1149, 335, 377, 722, 565, 1895, 1124, 541, 2272, 3, 126, 1336, 260, 225, 2171, 1709, 1327, 122, 666, 1469, 95, 1466, 297, 472, 111, 1323, 1200, 664, 587, 265, 1097, 535, 1136, 263] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=17 tokens_per_expert_current_ep_rank=[21, 60, 56, 471, 1730, 199, 705, 567, 217, 440, 612, 46, 723, 1918, 125, 2963, 1032, 154, 434, 462, 500, 284, 1531, 1396, 576, 1226, 552, 749, 165, 578, 1610, 361, 507, 1088, 1362, 951, 377, 947, 1280, 10, 697, 1552, 173, 95, 3549, 1251, 1252, 178, 739, 618, 307, 2246, 84, 75, 8, 1527, 96, 291, 995, 194, 856, 223, 2528, 633] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=17 tokens_per_expert_current_ep_rank=[331, 168, 575, 34, 445, 346, 426, 226, 448, 2205, 994, 244, 105, 2205, 298, 1105, 259, 1098, 426, 58, 460, 352, 917, 1362, 3765, 474, 37, 341, 160, 329, 1832, 306, 943, 69, 979, 2646, 1156, 824, 351, 145, 393, 933, 536, 320, 2785, 836, 692, 288, 464, 494, 214, 781, 210, 945, 225, 1660, 1229, 575, 1162, 459, 1176, 734, 1506, 1091] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=17 tokens_per_expert_current_ep_rank=[13, 52, 32, 224, 2157, 670, 516, 698, 624, 725, 539, 171, 651, 2275, 386, 1533, 453, 374, 985, 399, 1047, 612, 1200, 558, 986, 493, 117, 597, 118, 598, 1390, 176, 642, 1094, 459, 711, 186, 1606, 1555, 54, 87, 1699, 608, 186, 2273, 1371, 712, 160, 397, 447, 216, 4173, 242, 342, 98, 1002, 647, 283, 1167, 315, 2114, 551, 1790, 596] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=17 tokens_per_expert_current_ep_rank=[72, 98, 69, 154, 1354, 879, 322, 315, 709, 2284, 1235, 373, 162, 2143, 429, 1824, 1728, 885, 399, 297, 129, 432, 863, 819, 1410, 389, 1405, 661, 25, 306, 2807, 68, 482, 107, 1631, 1429, 307, 438, 1034, 30, 85, 1259, 131, 243, 1533, 190, 1480, 575, 86, 1106, 706, 893, 299, 510, 133, 2783, 528, 678, 816, 908, 1496, 331, 1594, 286] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=17 tokens_per_expert_current_ep_rank=[12, 490, 8, 53, 1098, 2258, 2110, 171, 380, 1424, 352, 236, 190, 1689, 582, 2303, 155, 147, 620, 401, 485, 20, 1329, 1432, 1195, 97, 288, 396, 181, 614, 976, 160, 816, 591, 1076, 1795, 774, 2009, 1925, 163, 201, 1235, 104, 432, 1973, 742, 545, 2417, 435, 887, 397, 554, 319, 247, 108, 1550, 1313, 348, 853, 321, 2455, 159, 398, 158] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=17 tokens_per_expert_current_ep_rank=[15, 257, 251, 767, 1322, 1048, 2388, 690, 1035, 975, 621, 1253, 277, 2457, 445, 1672, 1211, 514, 1116, 281, 349, 279, 742, 300, 271, 480, 1337, 246, 115, 612, 1583, 96, 437, 1047, 1113, 576, 725, 2053, 592, 77, 172, 1626, 235, 144, 1087, 2359, 631, 2188, 113, 158, 66, 1661, 171, 150, 144, 1416, 115, 631, 496, 827, 1570, 279, 979, 309] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=17 tokens_per_expert_current_ep_rank=[68, 62, 11, 271, 565, 625, 107, 237, 240, 328, 846, 511, 1133, 1455, 287, 1528, 1526, 1405, 1458, 878, 468, 411, 1348, 313, 891, 312, 507, 147, 149, 306, 1786, 311, 367, 1187, 353, 1645, 155, 2059, 1645, 258, 153, 1907, 258, 248, 2556, 1310, 1478, 469, 171, 858, 284, 3119, 57, 249, 69, 1509, 141, 971, 506, 436, 1074, 390, 2014, 766] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=17 rerouted_tokens_per_expert_current_ep_rank=[21, 60, 56, 471, 1730, 199, 705, 567, 2121, 743, 0, 0, 0, 0, 0, 0, 217, 440, 612, 46, 723, 713, 125, 2963, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 154, 434, 462, 500, 284, 1531, 1396, 462, 69, 0, 0, 0, 0, 0, 0, 576, 1226, 552, 749, 165, 578, 1610, 361, 413, 147, 0, 0, 0, 0, 0, 0, 507, 1088, 1362, 366, 377, 947, 1280, 10, 0, 0, 0, 0, 0, 0, 0, 0, 697, 1552, 173, 95, 1428, 1251, 1252, 178, 0, 0, 0, 0, 0, 0, 0, 0, 739, 618, 307, 2246, 84, 75, 8, 1527, 369, 0, 0, 0, 0, 0, 0, 0, 96, 291, 995, 194, 856, 223, 2115, 633, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6673, 5839, 6324, 6377, 5937, 6626, 5973, 5403] avg=6144.0000 max=6673 max_rank=0 min=5403 min_rank=7 imbalance_ratio=1.086100 max_load_reduction=42479 max_load_reduction_pct=0.864237 moved_to_echo=23939 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=17 rerouted_tokens_per_expert_current_ep_rank=[113, 167, 35, 623, 129, 910, 1168, 476, 1297, 462, 0, 0, 0, 0, 0, 0, 90, 1182, 787, 626, 434, 444, 540, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 952, 569, 1148, 197, 606, 376, 957, 712, 287, 137, 0, 0, 0, 0, 0, 0, 1857, 557, 269, 2928, 432, 202, 1149, 335, 185, 294, 0, 0, 0, 0, 0, 0, 377, 722, 565, 727, 1124, 541, 2272, 3, 0, 0, 0, 0, 0, 0, 0, 0, 126, 1336, 260, 225, 874, 1709, 1327, 122, 0, 0, 0, 0, 0, 0, 0, 0, 666, 1469, 95, 1466, 297, 472, 111, 1323, 737, 0, 0, 0, 0, 0, 0, 0, 1200, 664, 587, 265, 1097, 535, 951, 263, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5380, 5115, 5941, 8208, 6331, 5979, 6636, 5562] avg=6144.0000 max=8208 max_rank=3 min=5115 min_rank=1 imbalance_ratio=1.335938 max_load_reduction=40944 max_load_reduction_pct=0.833008 moved_to_echo=24508 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=17 rerouted_tokens_per_expert_current_ep_rank=[331, 168, 575, 34, 445, 346, 426, 226, 1664, 854, 0, 0, 0, 0, 0, 0, 448, 2205, 994, 244, 105, 820, 298, 1105, 0, 0, 0, 0, 0, 0, 0, 0, 259, 1098, 426, 58, 460, 352, 917, 1362, 531, 192, 0, 0, 0, 0, 0, 0, 3765, 474, 37, 341, 160, 329, 1832, 306, 246, 410, 0, 0, 0, 0, 0, 0, 943, 69, 979, 1015, 1156, 824, 351, 145, 0, 0, 0, 0, 0, 0, 0, 0, 393, 933, 536, 320, 1121, 836, 692, 288, 0, 0, 0, 0, 0, 0, 0, 0, 464, 494, 214, 781, 210, 945, 225, 1660, 1029, 0, 0, 0, 0, 0, 0, 0, 1229, 575, 1162, 459, 1176, 734, 1260, 1091, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5069, 6219, 5655, 7900, 5482, 5119, 6022, 7686] avg=6144.0000 max=7900 max_rank=3 min=5069 min_rank=0 imbalance_ratio=1.285807 max_load_reduction=41252 max_load_reduction_pct=0.839274 moved_to_echo=24309 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=17 rerouted_tokens_per_expert_current_ep_rank=[13, 52, 32, 224, 2157, 670, 516, 698, 1358, 882, 0, 0, 0, 0, 0, 0, 624, 725, 539, 171, 651, 845, 386, 1533, 0, 0, 0, 0, 0, 0, 0, 0, 453, 374, 985, 399, 1047, 612, 1200, 558, 548, 51, 0, 0, 0, 0, 0, 0, 986, 493, 117, 597, 118, 598, 1390, 176, 292, 110, 0, 0, 0, 0, 0, 0, 642, 1094, 459, 274, 186, 1606, 1555, 54, 0, 0, 0, 0, 0, 0, 0, 0, 87, 1699, 608, 186, 915, 1371, 712, 160, 0, 0, 0, 0, 0, 0, 0, 0, 397, 447, 216, 4173, 242, 342, 98, 1002, 276, 0, 0, 0, 0, 0, 0, 0, 647, 283, 1167, 315, 2114, 551, 1498, 596, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6602, 5474, 6227, 4877, 5870, 5738, 7193, 7171] avg=6144.0000 max=7193 max_rank=6 min=4877 min_rank=3 imbalance_ratio=1.170736 max_load_reduction=41959 max_load_reduction_pct=0.853658 moved_to_echo=25972 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=17 rerouted_tokens_per_expert_current_ep_rank=[15, 257, 251, 767, 1322, 1048, 2388, 690, 654, 957, 0, 0, 0, 0, 0, 0, 1035, 975, 621, 1253, 277, 905, 445, 1672, 0, 0, 0, 0, 0, 0, 0, 0, 1211, 514, 1116, 281, 349, 279, 742, 300, 595, 45, 0, 0, 0, 0, 0, 0, 271, 480, 1337, 246, 115, 612, 1583, 96, 162, 93, 0, 0, 0, 0, 0, 0, 437, 1047, 1113, 210, 725, 2053, 592, 77, 0, 0, 0, 0, 0, 0, 0, 0, 172, 1626, 235, 144, 433, 2359, 631, 2188, 0, 0, 0, 0, 0, 0, 0, 0, 113, 158, 66, 1661, 171, 150, 144, 1416, 228, 0, 0, 0, 0, 0, 0, 0, 115, 631, 496, 827, 1570, 279, 817, 309, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8349, 7183, 5432, 4995, 6254, 7788, 4107, 5044] avg=6144.0000 max=8349 max_rank=0 min=4107 min_rank=6 imbalance_ratio=1.358887 max_load_reduction=40803 max_load_reduction_pct=0.830139 moved_to_echo=23193 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=17 rerouted_tokens_per_expert_current_ep_rank=[72, 98, 69, 154, 1354, 879, 322, 315, 916, 830, 0, 0, 0, 0, 0, 0, 709, 2284, 1235, 373, 162, 797, 429, 1824, 0, 0, 0, 0, 0, 0, 0, 0, 1728, 885, 399, 297, 129, 432, 863, 819, 516, 103, 0, 0, 0, 0, 0, 0, 1410, 389, 1405, 661, 25, 306, 2807, 68, 260, 221, 0, 0, 0, 0, 0, 0, 482, 107, 1631, 550, 307, 438, 1034, 30, 0, 0, 0, 0, 0, 0, 0, 0, 85, 1259, 131, 243, 617, 190, 1480, 575, 0, 0, 0, 0, 0, 0, 0, 0, 86, 1106, 706, 893, 299, 510, 133, 2783, 555, 0, 0, 0, 0, 0, 0, 0, 528, 678, 816, 908, 1496, 331, 1334, 286, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5009, 7813, 6171, 7552, 4579, 4580, 7071, 6377] avg=6144.0000 max=7813 max_rank=1 min=4579 min_rank=4 imbalance_ratio=1.271647 max_load_reduction=41339 max_load_reduction_pct=0.841044 moved_to_echo=22607 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=17 rerouted_tokens_per_expert_current_ep_rank=[68, 62, 11, 271, 565, 625, 107, 237, 1527, 564, 0, 0, 0, 0, 0, 0, 240, 328, 846, 511, 1133, 541, 287, 1528, 0, 0, 0, 0, 0, 0, 0, 0, 1526, 1405, 1458, 878, 468, 411, 1348, 313, 350, 119, 0, 0, 0, 0, 0, 0, 891, 312, 507, 147, 149, 306, 1786, 311, 329, 255, 0, 0, 0, 0, 0, 0, 367, 1187, 353, 632, 155, 2059, 1645, 258, 0, 0, 0, 0, 0, 0, 0, 0, 153, 1907, 258, 248, 1029, 1310, 1478, 469, 0, 0, 0, 0, 0, 0, 0, 0, 171, 858, 284, 3119, 57, 249, 69, 1509, 639, 0, 0, 0, 0, 0, 0, 0, 141, 971, 506, 436, 1074, 390, 1685, 766, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4037, 5414, 8276, 4993, 6656, 6852, 6955, 5969] avg=6144.0000 max=8276 max_rank=2 min=4037 min_rank=0 imbalance_ratio=1.347005 max_load_reduction=40876 max_load_reduction_pct=0.831624 moved_to_echo=26432 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=17 rerouted_tokens_per_expert_current_ep_rank=[12, 490, 8, 53, 1098, 2258, 2110, 171, 1179, 654, 0, 0, 0, 0, 0, 0, 380, 1424, 352, 236, 190, 628, 582, 2303, 0, 0, 0, 0, 0, 0, 0, 0, 155, 147, 620, 401, 485, 20, 1329, 1432, 407, 130, 0, 0, 0, 0, 0, 0, 1195, 97, 288, 396, 181, 614, 976, 160, 65, 278, 0, 0, 0, 0, 0, 0, 816, 591, 1076, 689, 774, 2009, 1925, 163, 0, 0, 0, 0, 0, 0, 0, 0, 201, 1235, 104, 432, 794, 742, 545, 2417, 0, 0, 0, 0, 0, 0, 0, 0, 435, 887, 397, 554, 319, 247, 108, 1550, 698, 0, 0, 0, 0, 0, 0, 0, 1313, 348, 853, 321, 2455, 159, 333, 158, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8033, 6095, 5126, 4250, 8043, 6470, 5195, 5940] avg=6144.0000 max=8043 max_rank=4 min=4250 min_rank=3 imbalance_ratio=1.309082 max_load_reduction=41109 max_load_reduction_pct=0.836365 moved_to_echo=25648 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1920, 11648, 2432, 1920, 7296, 9856, 8192, 6400, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[11008, 4096, 4608, 6144, 1408, 3584, 13184, 1920, 2048, 1920, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[768, 1408, 1152, 2688, 8832, 7040, 7808, 3456, 10752, 6016, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[5376, 4480, 6656, 3840, 11904, 3328, 10112, 4224, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3840, 9600, 6016, 3584, 3712, 5760, 3200, 13952, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4608, 6016, 7552, 4480, 4864, 10496, 10752, 768, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[7424, 5248, 6656, 3072, 4096, 2816, 8960, 6912, 3712, 896, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3072, 6144, 2304, 14976, 1792, 3072, 896, 12800, 4608, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=18 tokens_per_expert_current_ep_rank=[52, 632, 2136, 728, 269, 31, 641, 1509, 67, 2441, 770, 15, 126, 546, 853, 1094, 818, 126, 3148, 108, 261, 1324, 1603, 139, 739, 122, 60, 1812, 202, 1149, 355, 1849, 2270, 660, 75, 689, 259, 1243, 422, 2133, 115, 459, 1379, 1, 588, 74, 821, 63, 117, 2032, 47, 980, 257, 24, 1505, 3135, 431, 37, 1079, 825, 344, 1085, 151, 127] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=18 tokens_per_expert_current_ep_rank=[703, 1806, 1004, 2599, 534, 994, 2518, 677, 46, 1332, 605, 190, 374, 720, 233, 610, 31, 353, 611, 729, 638, 1423, 2334, 44, 464, 113, 171, 2350, 283, 100, 64, 1379, 1112, 666, 93, 253, 97, 816, 261, 2538, 575, 447, 3244, 101, 731, 390, 830, 338, 554, 585, 20, 1282, 204, 19, 998, 413, 14, 99, 454, 3969, 243, 1422, 189, 163] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=18 tokens_per_expert_current_ep_rank=[736, 1193, 2388, 2089, 606, 175, 3149, 2192, 554, 1124, 525, 781, 399, 668, 1633, 132, 10, 581, 541, 123, 894, 967, 296, 93, 745, 35, 141, 634, 618, 662, 36, 395, 1358, 675, 165, 51, 698, 2684, 84, 1352, 276, 195, 208, 809, 341, 142, 1269, 482, 2229, 675, 112, 347, 369, 193, 2018, 450, 53, 672, 384, 3655, 359, 826, 477, 429] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=18 tokens_per_expert_current_ep_rank=[692, 1924, 2896, 2322, 1068, 1224, 2645, 2985, 528, 3528, 448, 176, 55, 229, 175, 431, 104, 350, 1040, 448, 995, 225, 1824, 165, 928, 17, 300, 1069, 555, 252, 147, 569, 1506, 1993, 178, 165, 39, 2126, 628, 1148, 135, 125, 1261, 20, 608, 660, 1112, 50, 675, 893, 366, 196, 194, 16, 947, 559, 59, 42, 409, 820, 181, 1203, 116, 408] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=18 tokens_per_expert_current_ep_rank=[710, 1287, 2512, 335, 1290, 400, 2017, 578, 12, 1484, 225, 137, 311, 734, 1043, 689, 38, 490, 756, 462, 2356, 2379, 2376, 123, 221, 20, 143, 1488, 208, 208, 957, 971, 1934, 955, 5, 534, 182, 1554, 57, 1137, 141, 216, 2033, 2, 717, 308, 2045, 63, 328, 538, 65, 663, 12, 71, 1440, 874, 32, 25, 619, 3667, 712, 306, 319, 638] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=18 tokens_per_expert_current_ep_rank=[163, 617, 3007, 1967, 314, 723, 1158, 520, 32, 1296, 106, 89, 631, 730, 1155, 402, 766, 314, 1515, 375, 1293, 742, 1577, 56, 171, 90, 95, 1492, 1497, 322, 1489, 1191, 585, 746, 55, 168, 181, 559, 588, 1770, 216, 756, 2473, 60, 281, 60, 363, 111, 433, 812, 57, 925, 23, 203, 1705, 715, 95, 125, 897, 3822, 206, 3962, 84, 221] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=18 tokens_per_expert_current_ep_rank=[796, 2774, 3042, 2337, 1095, 368, 2314, 1389, 8, 4131, 544, 50, 386, 720, 1190, 572, 34, 142, 1044, 119, 587, 474, 831, 907, 590, 36, 207, 1066, 1270, 214, 330, 515, 161, 441, 45, 535, 81, 705, 102, 1280, 331, 202, 1108, 630, 488, 198, 855, 107, 959, 1039, 218, 647, 46, 93, 2073, 1020, 166, 77, 357, 514, 610, 2805, 1090, 87] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=18 tokens_per_expert_current_ep_rank=[574, 1433, 3912, 846, 698, 161, 3793, 660, 724, 3060, 611, 72, 485, 357, 888, 931, 279, 482, 321, 335, 1224, 1651, 606, 660, 289, 113, 208, 600, 1217, 488, 120, 1235, 1767, 1775, 52, 261, 124, 755, 1035, 941, 266, 90, 953, 266, 213, 257, 1086, 111, 1111, 928, 58, 592, 147, 96, 1082, 1293, 67, 27, 1738, 1569, 55, 848, 410, 146] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=18 rerouted_tokens_per_expert_current_ep_rank=[52, 632, 0, 694, 269, 31, 0, 1509, 0, 0, 0, 0, 0, 0, 0, 0, 67, 2441, 770, 15, 126, 546, 853, 1094, 138, 0, 0, 0, 0, 0, 0, 0, 818, 126, 3148, 108, 261, 1324, 1603, 139, 34, 18, 61, 0, 0, 0, 0, 0, 739, 122, 60, 1812, 202, 1149, 355, 1849, 674, 183, 0, 0, 0, 0, 0, 0, 2270, 660, 75, 689, 259, 1243, 422, 2072, 0, 0, 0, 0, 0, 0, 0, 0, 115, 459, 1379, 1, 588, 74, 821, 63, 1462, 0, 0, 0, 0, 0, 0, 0, 117, 2032, 47, 980, 257, 24, 1505, 3135, 228, 0, 0, 0, 0, 0, 0, 0, 431, 37, 1079, 825, 344, 1085, 151, 127, 74, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3187, 6050, 7640, 7145, 7690, 4962, 8325, 4153] avg=6144.0000 max=8325 max_rank=6 min=3187 min_rank=0 imbalance_ratio=1.354980 max_load_reduction=40827 max_load_reduction_pct=0.830627 moved_to_echo=25130 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=18 rerouted_tokens_per_expert_current_ep_rank=[703, 1806, 0, 2477, 534, 994, 0, 677, 0, 0, 0, 0, 0, 0, 0, 0, 46, 1332, 605, 190, 374, 720, 233, 610, 543, 0, 0, 0, 0, 0, 0, 0, 31, 353, 611, 729, 638, 1423, 2334, 44, 122, 70, 73, 0, 0, 0, 0, 0, 464, 113, 171, 2350, 283, 100, 64, 1379, 317, 714, 0, 0, 0, 0, 0, 0, 1112, 666, 93, 253, 97, 816, 261, 2465, 0, 0, 0, 0, 0, 0, 0, 0, 575, 447, 3244, 101, 731, 390, 830, 338, 687, 0, 0, 0, 0, 0, 0, 0, 554, 585, 20, 1282, 204, 19, 998, 413, 894, 0, 0, 0, 0, 0, 0, 0, 14, 99, 454, 3969, 243, 1422, 189, 163, 297, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7191, 4653, 6428, 5955, 5763, 7343, 4969, 6850] avg=6144.0000 max=7343 max_rank=5 min=4653 min_rank=1 imbalance_ratio=1.195150 max_load_reduction=41809 max_load_reduction_pct=0.850606 moved_to_echo=24925 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=18 rerouted_tokens_per_expert_current_ep_rank=[736, 1193, 0, 1991, 606, 175, 0, 2192, 0, 0, 0, 0, 0, 0, 0, 0, 554, 1124, 525, 781, 399, 668, 1633, 132, 680, 0, 0, 0, 0, 0, 0, 0, 10, 581, 541, 123, 894, 967, 296, 93, 98, 88, 39, 0, 0, 0, 0, 0, 745, 35, 141, 634, 618, 662, 36, 395, 754, 893, 0, 0, 0, 0, 0, 0, 1358, 675, 165, 51, 698, 2684, 84, 1313, 0, 0, 0, 0, 0, 0, 0, 0, 276, 195, 208, 809, 341, 142, 1269, 482, 1634, 0, 0, 0, 0, 0, 0, 0, 2229, 675, 112, 347, 369, 193, 2018, 450, 1120, 0, 0, 0, 0, 0, 0, 0, 53, 672, 384, 3655, 359, 826, 477, 429, 368, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6893, 6496, 3730, 4913, 7028, 5356, 7513, 7223] avg=6144.0000 max=7513 max_rank=6 min=3730 min_rank=2 imbalance_ratio=1.222819 max_load_reduction=41639 max_load_reduction_pct=0.847148 moved_to_echo=27120 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=18 rerouted_tokens_per_expert_current_ep_rank=[710, 1287, 0, 316, 1290, 400, 0, 578, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1484, 225, 137, 311, 734, 1043, 689, 438, 0, 0, 0, 0, 0, 0, 0, 38, 490, 756, 462, 2356, 2379, 2376, 123, 19, 56, 35, 0, 0, 0, 0, 0, 221, 20, 143, 1488, 208, 208, 957, 971, 794, 572, 0, 0, 0, 0, 0, 0, 1934, 955, 5, 534, 182, 1554, 57, 1102, 0, 0, 0, 0, 0, 0, 0, 0, 141, 216, 2033, 2, 717, 308, 2045, 63, 1718, 0, 0, 0, 0, 0, 0, 0, 328, 538, 65, 663, 12, 71, 1440, 874, 716, 0, 0, 0, 0, 0, 0, 0, 32, 25, 619, 3667, 712, 306, 319, 638, 235, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4581, 5073, 9090, 5582, 6323, 7243, 4707, 6553] avg=6144.0000 max=9090 max_rank=2 min=4581 min_rank=0 imbalance_ratio=1.479492 max_load_reduction=40062 max_load_reduction_pct=0.815063 moved_to_echo=24826 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=18 rerouted_tokens_per_expert_current_ep_rank=[692, 1924, 0, 2213, 1068, 1224, 0, 2985, 0, 0, 0, 0, 0, 0, 0, 0, 528, 3528, 448, 176, 55, 229, 175, 431, 572, 0, 0, 0, 0, 0, 0, 0, 104, 350, 1040, 448, 995, 225, 1824, 165, 109, 75, 33, 0, 0, 0, 0, 0, 928, 17, 300, 1069, 555, 252, 147, 569, 915, 750, 0, 0, 0, 0, 0, 0, 1506, 1993, 178, 165, 39, 2126, 628, 1115, 0, 0, 0, 0, 0, 0, 0, 0, 135, 125, 1261, 20, 608, 660, 1112, 50, 1981, 0, 0, 0, 0, 0, 0, 0, 675, 893, 366, 196, 194, 16, 947, 559, 939, 0, 0, 0, 0, 0, 0, 0, 59, 42, 409, 820, 181, 1203, 116, 408, 309, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10106, 6142, 5368, 5502, 7750, 5952, 4785, 3547] avg=6144.0000 max=10106 max_rank=0 min=3547 min_rank=7 imbalance_ratio=1.644857 max_load_reduction=39046 max_load_reduction_pct=0.794393 moved_to_echo=22034 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=18 rerouted_tokens_per_expert_current_ep_rank=[163, 617, 0, 1875, 314, 723, 0, 520, 0, 0, 0, 0, 0, 0, 0, 0, 32, 1296, 106, 89, 631, 730, 1155, 402, 250, 0, 0, 0, 0, 0, 0, 0, 766, 314, 1515, 375, 1293, 742, 1577, 56, 92, 33, 51, 0, 0, 0, 0, 0, 171, 90, 95, 1492, 1497, 322, 1489, 1191, 950, 329, 0, 0, 0, 0, 0, 0, 585, 746, 55, 168, 181, 559, 588, 1719, 0, 0, 0, 0, 0, 0, 0, 0, 216, 756, 2473, 60, 281, 60, 363, 111, 2057, 0, 0, 0, 0, 0, 0, 0, 433, 812, 57, 925, 23, 203, 1705, 715, 411, 0, 0, 0, 0, 0, 0, 0, 95, 125, 897, 3822, 206, 3962, 84, 221, 135, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4212, 4691, 6814, 7626, 4601, 6377, 5284, 9547] avg=6144.0000 max=9547 max_rank=7 min=4212 min_rank=0 imbalance_ratio=1.553874 max_load_reduction=39605 max_load_reduction_pct=0.805766 moved_to_echo=25809 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=18 rerouted_tokens_per_expert_current_ep_rank=[574, 1433, 0, 807, 698, 161, 0, 660, 0, 0, 0, 0, 0, 0, 0, 0, 724, 3060, 611, 72, 485, 357, 888, 931, 819, 0, 0, 0, 0, 0, 0, 0, 279, 482, 321, 335, 1224, 1651, 606, 660, 39, 109, 27, 0, 0, 0, 0, 0, 289, 113, 208, 600, 1217, 488, 120, 1235, 1236, 1076, 0, 0, 0, 0, 0, 0, 1767, 1775, 52, 261, 124, 755, 1035, 914, 0, 0, 0, 0, 0, 0, 0, 0, 266, 90, 953, 266, 213, 257, 1086, 111, 2676, 0, 0, 0, 0, 0, 0, 0, 1111, 928, 58, 592, 147, 96, 1082, 1293, 1346, 0, 0, 0, 0, 0, 0, 0, 67, 27, 1738, 1569, 55, 848, 410, 146, 443, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4333, 7947, 5733, 6582, 6683, 5918, 6653, 5303] avg=6144.0000 max=7947 max_rank=1 min=4333 min_rank=0 imbalance_ratio=1.293457 max_load_reduction=41205 max_load_reduction_pct=0.838318 moved_to_echo=24557 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=18 rerouted_tokens_per_expert_current_ep_rank=[796, 2774, 0, 2227, 1095, 368, 0, 1389, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4131, 544, 50, 386, 720, 1190, 572, 499, 0, 0, 0, 0, 0, 0, 0, 34, 142, 1044, 119, 587, 474, 831, 907, 110, 65, 36, 0, 0, 0, 0, 0, 590, 36, 207, 1066, 1270, 214, 330, 515, 960, 659, 0, 0, 0, 0, 0, 0, 161, 441, 45, 535, 81, 705, 102, 1244, 0, 0, 0, 0, 0, 0, 0, 0, 331, 202, 1108, 630, 488, 198, 855, 107, 2082, 0, 0, 0, 0, 0, 0, 0, 959, 1039, 218, 647, 46, 93, 2073, 1020, 821, 0, 0, 0, 0, 0, 0, 0, 166, 77, 357, 514, 610, 2805, 1090, 87, 270, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8649, 8100, 4349, 5847, 3314, 6001, 6916, 5976] avg=6144.0000 max=8649 max_rank=0 min=3314 min_rank=4 imbalance_ratio=1.407715 max_load_reduction=40503 max_load_reduction_pct=0.824036 moved_to_echo=22207 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[4224, 640, 1408, 10624, 5888, 3456, 3584, 8192, 6656, 5248, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[2176, 2944, 9088, 2816, 8320, 9216, 11520, 2304, 640, 640, 384, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4480, 11776, 0, 12672, 5888, 4096, 0, 10624, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[2048, 18432, 3840, 1536, 2816, 4736, 7296, 4864, 3968, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2176, 2560, 12672, 1920, 3968, 2176, 8448, 1408, 14336, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1024, 1152, 6016, 18944, 2816, 12544, 2944, 2304, 2176, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[10752, 7936, 768, 2688, 1664, 10496, 3200, 12032, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6528, 7552, 1024, 5632, 1280, 768, 11776, 8576, 6528, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=19 tokens_per_expert_current_ep_rank=[1720, 352, 25, 106, 287, 127, 1787, 3373, 144, 1252, 1896, 505, 670, 1602, 1598, 860, 132, 272, 178, 230, 1996, 319, 324, 1576, 770, 570, 1792, 37, 1004, 82, 103, 475, 2808, 329, 2518, 679, 679, 744, 685, 727, 29, 121, 361, 184, 1230, 1392, 23, 962, 372, 252, 794, 1666, 206, 94, 725, 1296, 447, 1406, 885, 433, 37, 142, 81, 681] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=19 tokens_per_expert_current_ep_rank=[1679, 726, 729, 495, 223, 70, 311, 220, 116, 874, 1401, 189, 2479, 1368, 2026, 2019, 741, 804, 327, 1132, 2372, 392, 629, 2392, 829, 396, 1394, 706, 888, 432, 140, 922, 763, 501, 1085, 1062, 195, 957, 1353, 1363, 506, 582, 122, 502, 1502, 1968, 476, 211, 96, 728, 772, 701, 518, 630, 31, 344, 496, 293, 266, 154, 144, 248, 205, 957] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=19 tokens_per_expert_current_ep_rank=[875, 454, 288, 710, 270, 11, 618, 1833, 116, 640, 2240, 229, 2211, 3733, 449, 4019, 97, 747, 285, 243, 1458, 30, 297, 929, 311, 517, 1161, 1093, 440, 59, 378, 16, 609, 183, 553, 2103, 163, 273, 1438, 912, 9, 192, 159, 912, 868, 42, 1139, 1510, 1259, 28, 347, 781, 376, 235, 66, 2580, 554, 611, 2402, 121, 7, 300, 1448, 215] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=19 tokens_per_expert_current_ep_rank=[906, 182, 111, 90, 44, 115, 161, 2373, 493, 460, 1483, 8, 1009, 1779, 3004, 3124, 86, 427, 236, 1916, 1584, 17, 278, 2260, 201, 571, 2773, 857, 499, 64, 421, 102, 334, 415, 1702, 1981, 479, 1045, 1743, 1586, 7, 781, 91, 810, 1303, 342, 87, 621, 768, 71, 441, 181, 327, 84, 54, 671, 209, 982, 2680, 944, 8, 128, 271, 372] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=19 tokens_per_expert_current_ep_rank=[1983, 731, 131, 891, 571, 12, 448, 377, 37, 1918, 1013, 56, 680, 3938, 1906, 518, 583, 1633, 54, 335, 2030, 822, 262, 831, 1459, 345, 666, 719, 1861, 236, 522, 265, 534, 74, 682, 1245, 640, 479, 1007, 678, 233, 970, 850, 64, 1536, 2058, 343, 1207, 320, 164, 1769, 532, 744, 455, 34, 700, 355, 926, 121, 157, 40, 644, 65, 1693] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=19 tokens_per_expert_current_ep_rank=[829, 2123, 897, 33, 72, 41, 1097, 1477, 514, 555, 525, 724, 2052, 1684, 1298, 375, 39, 2158, 25, 318, 863, 387, 362, 823, 1456, 360, 1197, 294, 848, 88, 376, 410, 618, 152, 497, 737, 940, 2834, 1375, 2883, 166, 323, 82, 317, 1786, 308, 224, 1312, 862, 40, 2230, 655, 468, 490, 16, 938, 719, 821, 634, 4, 438, 727, 398, 858] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=19 tokens_per_expert_current_ep_rank=[1848, 449, 302, 75, 582, 67, 1511, 1935, 345, 2433, 1104, 251, 1456, 4609, 989, 502, 500, 850, 252, 107, 1227, 230, 409, 2163, 1016, 399, 1382, 88, 1174, 32, 28, 68, 1127, 154, 749, 1203, 609, 1389, 1037, 188, 477, 488, 442, 37, 2752, 224, 462, 579, 785, 77, 591, 1666, 444, 123, 487, 977, 731, 611, 422, 496, 339, 74, 34, 995] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=19 tokens_per_expert_current_ep_rank=[1736, 63, 322, 87, 23, 57, 905, 443, 131, 508, 1428, 558, 2103, 1726, 1685, 1735, 122, 1583, 46, 166, 656, 688, 178, 1771, 1723, 189, 948, 19, 412, 724, 294, 650, 722, 302, 1594, 492, 1949, 639, 2101, 288, 225, 135, 52, 77, 1121, 527, 616, 5612, 172, 125, 1599, 1573, 293, 319, 236, 810, 273, 1123, 640, 939, 131, 191, 154, 443] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=19 rerouted_tokens_per_expert_current_ep_rank=[875, 454, 288, 710, 270, 11, 618, 1833, 771, 0, 0, 0, 0, 0, 0, 0, 116, 640, 2240, 229, 2211, 0, 428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 747, 285, 243, 1458, 30, 297, 929, 301, 21, 0, 0, 0, 0, 0, 0, 311, 517, 1161, 1093, 440, 59, 378, 16, 2382, 147, 0, 0, 0, 0, 0, 0, 609, 183, 553, 1663, 163, 273, 0, 912, 0, 0, 0, 0, 0, 0, 0, 0, 9, 192, 159, 912, 868, 42, 1139, 1510, 520, 139, 0, 0, 0, 0, 0, 0, 1259, 28, 347, 781, 376, 235, 66, 2580, 1024, 1637, 0, 0, 0, 0, 0, 0, 554, 611, 2402, 121, 7, 300, 1448, 215, 2709, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5830, 5864, 4408, 6504, 4356, 5490, 8333, 8367] avg=6144.0000 max=8367 max_rank=7 min=4356 min_rank=4 imbalance_ratio=1.361816 max_load_reduction=40785 max_load_reduction_pct=0.829773 moved_to_echo=26546 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=19 rerouted_tokens_per_expert_current_ep_rank=[1720, 352, 25, 106, 287, 127, 1787, 3373, 367, 0, 0, 0, 0, 0, 0, 0, 144, 1252, 1896, 505, 670, 0, 1523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 272, 178, 230, 1996, 319, 324, 1576, 97, 75, 0, 0, 0, 0, 0, 0, 770, 570, 1792, 37, 1004, 82, 103, 475, 509, 70, 0, 0, 0, 0, 0, 0, 2808, 329, 2518, 537, 679, 744, 0, 727, 0, 0, 0, 0, 0, 0, 0, 0, 29, 121, 361, 184, 1230, 1392, 23, 962, 248, 45, 0, 0, 0, 0, 0, 0, 372, 252, 794, 1666, 206, 94, 725, 1296, 439, 351, 0, 0, 0, 0, 0, 0, 447, 1406, 885, 433, 37, 142, 81, 681, 1163, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8144, 5990, 5199, 5412, 8342, 4595, 6195, 5275] avg=6144.0000 max=8342 max_rank=4 min=4595 min_rank=5 imbalance_ratio=1.357747 max_load_reduction=40810 max_load_reduction_pct=0.830282 moved_to_echo=24407 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=19 rerouted_tokens_per_expert_current_ep_rank=[1679, 726, 729, 495, 223, 70, 311, 220, 726, 0, 0, 0, 0, 0, 0, 0, 116, 874, 1401, 189, 2479, 0, 1931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 741, 804, 327, 1132, 2372, 392, 629, 2392, 152, 95, 0, 0, 0, 0, 0, 0, 829, 396, 1394, 706, 888, 432, 140, 922, 1197, 138, 0, 0, 0, 0, 0, 0, 763, 501, 1085, 840, 195, 957, 0, 1363, 0, 0, 0, 0, 0, 0, 0, 0, 506, 582, 122, 502, 1502, 1968, 476, 211, 489, 70, 0, 0, 0, 0, 0, 0, 96, 728, 772, 701, 518, 630, 31, 344, 375, 822, 0, 0, 0, 0, 0, 0, 496, 293, 266, 154, 144, 248, 205, 957, 993, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5179, 6990, 9036, 7042, 5704, 6428, 5017, 3756] avg=6144.0000 max=9036 max_rank=2 min=3756 min_rank=7 imbalance_ratio=1.470703 max_load_reduction=40116 max_load_reduction_pct=0.816162 moved_to_echo=20905 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=19 rerouted_tokens_per_expert_current_ep_rank=[906, 182, 111, 90, 44, 115, 161, 2373, 935, 0, 0, 0, 0, 0, 0, 0, 493, 460, 1483, 8, 1009, 0, 2863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 427, 236, 1916, 1584, 17, 278, 2260, 284, 141, 0, 0, 0, 0, 0, 0, 201, 571, 2773, 857, 499, 64, 421, 102, 1853, 180, 0, 0, 0, 0, 0, 0, 334, 415, 1702, 1566, 479, 1045, 0, 1586, 0, 0, 0, 0, 0, 0, 0, 0, 7, 781, 91, 810, 1303, 342, 87, 621, 628, 131, 0, 0, 0, 0, 0, 0, 768, 71, 441, 181, 327, 84, 54, 671, 489, 1271, 0, 0, 0, 0, 0, 0, 209, 982, 2680, 944, 8, 128, 271, 372, 1290, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4917, 6316, 7229, 7521, 7127, 4801, 4357, 6884] avg=6144.0000 max=7521 max_rank=3 min=4357 min_rank=6 imbalance_ratio=1.224121 max_load_reduction=41631 max_load_reduction_pct=0.846985 moved_to_echo=23169 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=19 rerouted_tokens_per_expert_current_ep_rank=[1983, 731, 131, 891, 571, 12, 448, 377, 541, 0, 0, 0, 0, 0, 0, 0, 37, 1918, 1013, 56, 680, 0, 1815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 583, 1633, 54, 335, 2030, 822, 262, 831, 183, 91, 0, 0, 0, 0, 0, 0, 1459, 345, 666, 719, 1861, 236, 522, 265, 308, 103, 0, 0, 0, 0, 0, 0, 534, 74, 682, 976, 640, 479, 0, 678, 0, 0, 0, 0, 0, 0, 0, 0, 233, 970, 850, 64, 1536, 2058, 343, 1207, 363, 86, 0, 0, 0, 0, 0, 0, 320, 164, 1769, 532, 744, 455, 34, 700, 1081, 210, 0, 0, 0, 0, 0, 0, 355, 926, 121, 157, 40, 644, 65, 1693, 2857, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5685, 5519, 6824, 6484, 4063, 7710, 6009, 6858] avg=6144.0000 max=7710 max_rank=5 min=4063 min_rank=4 imbalance_ratio=1.254883 max_load_reduction=41442 max_load_reduction_pct=0.843140 moved_to_echo=24640 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=19 rerouted_tokens_per_expert_current_ep_rank=[1848, 449, 302, 75, 582, 67, 1511, 1935, 558, 0, 0, 0, 0, 0, 0, 0, 345, 2433, 1104, 251, 1456, 0, 943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 850, 252, 107, 1227, 230, 409, 2163, 172, 46, 0, 0, 0, 0, 0, 0, 1016, 399, 1382, 88, 1174, 32, 28, 68, 298, 106, 0, 0, 0, 0, 0, 0, 1127, 154, 749, 952, 609, 1389, 0, 188, 0, 0, 0, 0, 0, 0, 0, 0, 477, 488, 442, 37, 2752, 224, 462, 579, 373, 79, 0, 0, 0, 0, 0, 0, 785, 77, 591, 1666, 444, 123, 487, 977, 1266, 204, 0, 0, 0, 0, 0, 0, 731, 611, 422, 496, 339, 74, 34, 995, 3343, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7327, 6532, 5956, 4591, 5168, 5913, 6620, 7045] avg=6144.0000 max=7327 max_rank=0 min=4591 min_rank=3 imbalance_ratio=1.192546 max_load_reduction=41825 max_load_reduction_pct=0.850932 moved_to_echo=24746 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=19 rerouted_tokens_per_expert_current_ep_rank=[829, 2123, 897, 33, 72, 41, 1097, 1477, 738, 0, 0, 0, 0, 0, 0, 0, 514, 555, 525, 724, 2052, 0, 1237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 2158, 25, 318, 863, 387, 362, 823, 105, 61, 0, 0, 0, 0, 0, 0, 1456, 360, 1197, 294, 848, 88, 376, 410, 223, 142, 0, 0, 0, 0, 0, 0, 618, 152, 497, 584, 940, 2834, 0, 2883, 0, 0, 0, 0, 0, 0, 0, 0, 166, 323, 82, 317, 1786, 308, 224, 1312, 495, 48, 0, 0, 0, 0, 0, 0, 862, 40, 2230, 655, 468, 490, 16, 938, 463, 152, 0, 0, 0, 0, 0, 0, 719, 821, 634, 4, 438, 727, 398, 858, 1221, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7307, 5607, 5141, 5394, 8508, 5061, 6314, 5820] avg=6144.0000 max=8508 max_rank=4 min=5061 min_rank=5 imbalance_ratio=1.384766 max_load_reduction=40644 max_load_reduction_pct=0.826904 moved_to_echo=25703 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=19 rerouted_tokens_per_expert_current_ep_rank=[1736, 63, 322, 87, 23, 57, 905, 443, 1127, 0, 0, 0, 0, 0, 0, 0, 131, 508, 1428, 558, 2103, 0, 1606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 1583, 46, 166, 656, 688, 178, 1771, 70, 79, 0, 0, 0, 0, 0, 0, 1723, 189, 948, 19, 412, 724, 294, 650, 1028, 217, 0, 0, 0, 0, 0, 0, 722, 302, 1594, 390, 1949, 639, 0, 288, 0, 0, 0, 0, 0, 0, 0, 0, 225, 135, 52, 77, 1121, 527, 616, 5612, 757, 32, 0, 0, 0, 0, 0, 0, 172, 125, 1599, 1573, 293, 319, 236, 810, 473, 707, 0, 0, 0, 0, 0, 0, 273, 1123, 640, 939, 131, 191, 154, 443, 1253, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4763, 6334, 5359, 6204, 5884, 9154, 6307, 5147] avg=6144.0000 max=9154 max_rank=5 min=4763 min_rank=0 imbalance_ratio=1.489909 max_load_reduction=39998 max_load_reduction_pct=0.813761 moved_to_echo=26492 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[3840, 6784, 8064, 3328, 1152, 2560, 2688, 6272, 14848, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7808, 3456, 11392, 3840, 7168, 1792, 2304, 2944, 7808, 1152, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4736, 1536, 8576, 7808, 3456, 2432, 1664, 8320, 5632, 5376, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1664, 3712, 2176, 2944, 12160, 6912, 3456, 12032, 3968, 640, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[1920, 8704, 11136, 2560, 12672, 0, 12416, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[11648, 5120, 2816, 2560, 2176, 512, 6912, 12032, 5888, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7552, 2176, 9472, 7552, 5760, 8448, 0, 8704, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2304, 8576, 1408, 4480, 12288, 2944, 2816, 12800, 1408, 640, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=20 tokens_per_expert_current_ep_rank=[550, 714, 733, 297, 2090, 196, 324, 319, 264, 216, 59, 237, 2622, 36, 240, 2549, 548, 888, 968, 54, 2340, 346, 103, 1792, 111, 1481, 1020, 1547, 884, 211, 12, 547, 602, 288, 370, 340, 511, 423, 1333, 386, 6, 963, 736, 49, 2373, 291, 417, 302, 1535, 310, 1009, 746, 1432, 2520, 1207, 1647, 457, 336, 778, 2229, 319, 521, 201, 217] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=20 tokens_per_expert_current_ep_rank=[2014, 200, 169, 568, 2433, 362, 207, 114, 116, 520, 119, 259, 707, 250, 825, 472, 451, 363, 280, 322, 2524, 84, 230, 807, 11, 553, 404, 794, 517, 748, 895, 50, 292, 575, 517, 450, 119, 237, 2102, 639, 800, 275, 1146, 1010, 1137, 274, 1549, 2484, 1844, 1085, 715, 754, 434, 1742, 1065, 1606, 59, 124, 3653, 681, 506, 1564, 1079, 267] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=20 tokens_per_expert_current_ep_rank=[806, 359, 56, 1455, 966, 826, 483, 145, 75, 502, 313, 459, 529, 161, 58, 2345, 939, 462, 2652, 483, 342, 210, 143, 275, 378, 567, 1420, 847, 1592, 120, 63, 776, 648, 591, 233, 89, 109, 320, 561, 1540, 150, 2750, 248, 303, 1791, 475, 784, 480, 1159, 1043, 887, 1068, 301, 795, 3300, 1343, 275, 1860, 1818, 768, 686, 734, 1010, 226] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=20 tokens_per_expert_current_ep_rank=[718, 386, 120, 371, 1218, 336, 466, 106, 699, 55, 490, 318, 1653, 43, 50, 2150, 355, 395, 979, 105, 1217, 583, 631, 1257, 27, 1395, 342, 2416, 2744, 668, 546, 358, 1636, 1123, 1852, 694, 1329, 768, 324, 566, 251, 1751, 214, 337, 651, 312, 160, 163, 295, 750, 1076, 1568, 1557, 602, 975, 1782, 166, 262, 1979, 877, 310, 640, 767, 218] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=20 tokens_per_expert_current_ep_rank=[1551, 77, 5, 320, 1445, 933, 80, 99, 946, 1210, 60, 42, 1311, 5, 1613, 1480, 662, 155, 623, 398, 1128, 437, 206, 400, 215, 3737, 583, 525, 1738, 140, 884, 1869, 950, 204, 2313, 297, 780, 881, 1780, 480, 203, 149, 551, 136, 1622, 26, 196, 598, 815, 485, 1813, 2186, 179, 330, 189, 1069, 807, 1157, 904, 986, 168, 286, 959, 776] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=20 tokens_per_expert_current_ep_rank=[1179, 951, 653, 320, 141, 272, 344, 141, 50, 381, 100, 249, 658, 51, 498, 1393, 775, 630, 215, 588, 1364, 91, 253, 1279, 24, 1935, 824, 738, 259, 1159, 222, 157, 298, 592, 71, 663, 882, 194, 1489, 2736, 470, 313, 166, 972, 654, 38, 2610, 89, 853, 563, 1037, 1434, 593, 2151, 3428, 1604, 938, 557, 2714, 1117, 306, 1286, 215, 225] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=20 tokens_per_expert_current_ep_rank=[2452, 1395, 94, 796, 1137, 2626, 707, 702, 228, 48, 41, 485, 69, 208, 185, 548, 434, 176, 684, 826, 1250, 55, 479, 139, 824, 2878, 2633, 1019, 197, 55, 819, 788, 473, 245, 396, 274, 143, 1257, 1020, 1409, 293, 97, 326, 66, 503, 61, 101, 173, 786, 252, 2477, 760, 199, 387, 1251, 2370, 77, 2261, 2806, 616, 587, 633, 1660, 216] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=20 tokens_per_expert_current_ep_rank=[738, 109, 593, 801, 2930, 633, 1017, 51, 424, 591, 240, 337, 1164, 36, 127, 1564, 260, 788, 3328, 221, 519, 98, 291, 28, 66, 2195, 533, 1703, 338, 945, 57, 34, 788, 1175, 2169, 1078, 237, 138, 1131, 1193, 5, 725, 753, 274, 1067, 34, 925, 334, 1012, 72, 1557, 418, 54, 2084, 3219, 641, 160, 754, 2164, 319, 656, 361, 849, 47] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=20 rerouted_tokens_per_expert_current_ep_rank=[2014, 200, 169, 568, 2433, 362, 207, 114, 782, 32, 0, 0, 0, 0, 0, 0, 116, 520, 119, 259, 707, 250, 825, 472, 976, 0, 0, 0, 0, 0, 0, 0, 451, 363, 280, 322, 2524, 84, 230, 807, 187, 247, 86, 0, 0, 0, 0, 0, 11, 366, 404, 794, 517, 748, 895, 50, 0, 0, 0, 0, 0, 0, 0, 0, 292, 575, 517, 450, 119, 237, 2070, 639, 0, 0, 0, 0, 0, 0, 0, 0, 800, 275, 1146, 1010, 1137, 274, 1549, 2484, 1168, 89, 0, 0, 0, 0, 0, 0, 1844, 1085, 715, 754, 434, 1742, 0, 191, 0, 0, 0, 0, 0, 0, 0, 0, 59, 124, 2785, 681, 506, 1564, 1079, 267, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6881, 4244, 5581, 3785, 4899, 9932, 6765, 7065] avg=6144.0000 max=9932 max_rank=5 min=3785 min_rank=3 imbalance_ratio=1.616536 max_load_reduction=39220 max_load_reduction_pct=0.797933 moved_to_echo=28661 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=20 rerouted_tokens_per_expert_current_ep_rank=[806, 359, 56, 1455, 966, 826, 483, 145, 389, 8, 0, 0, 0, 0, 0, 0, 75, 502, 313, 459, 529, 161, 58, 2345, 3026, 0, 0, 0, 0, 0, 0, 0, 939, 462, 2652, 483, 342, 210, 143, 275, 191, 206, 43, 0, 0, 0, 0, 0, 378, 376, 1420, 847, 1592, 120, 63, 776, 0, 0, 0, 0, 0, 0, 0, 0, 648, 591, 233, 89, 109, 320, 553, 1540, 0, 0, 0, 0, 0, 0, 0, 0, 150, 2750, 248, 303, 1791, 475, 784, 480, 977, 274, 0, 0, 0, 0, 0, 0, 1159, 1043, 887, 1068, 301, 795, 0, 160, 0, 0, 0, 0, 0, 0, 0, 0, 275, 1860, 1386, 768, 686, 734, 1010, 226, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5493, 7468, 5946, 5572, 4083, 8232, 5413, 6945] avg=6144.0000 max=8232 max_rank=5 min=4083 min_rank=4 imbalance_ratio=1.339844 max_load_reduction=40920 max_load_reduction_pct=0.832520 moved_to_echo=24673 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=20 rerouted_tokens_per_expert_current_ep_rank=[550, 714, 733, 297, 2090, 196, 324, 319, 166, 20, 0, 0, 0, 0, 0, 0, 264, 216, 59, 237, 2622, 36, 240, 2549, 1106, 0, 0, 0, 0, 0, 0, 0, 548, 888, 968, 54, 2340, 346, 103, 1792, 500, 253, 18, 0, 0, 0, 0, 0, 111, 981, 1020, 1547, 884, 211, 12, 547, 0, 0, 0, 0, 0, 0, 0, 0, 602, 288, 370, 340, 511, 423, 1313, 386, 0, 0, 0, 0, 0, 0, 0, 0, 6, 963, 736, 49, 2373, 291, 417, 302, 1198, 101, 0, 0, 0, 0, 0, 0, 1535, 310, 1009, 746, 1432, 2520, 0, 196, 0, 0, 0, 0, 0, 0, 0, 0, 457, 336, 594, 2229, 319, 521, 201, 217, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5409, 7329, 7810, 5313, 4233, 6436, 7748, 4874] avg=6144.0000 max=7810 max_rank=2 min=4233 min_rank=4 imbalance_ratio=1.271159 max_load_reduction=41342 max_load_reduction_pct=0.841105 moved_to_echo=23291 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=20 rerouted_tokens_per_expert_current_ep_rank=[1179, 951, 653, 320, 141, 272, 344, 141, 581, 23, 0, 0, 0, 0, 0, 0, 50, 381, 100, 249, 658, 51, 498, 1393, 3143, 0, 0, 0, 0, 0, 0, 0, 775, 630, 215, 588, 1364, 91, 253, 1279, 654, 247, 64, 0, 0, 0, 0, 0, 24, 1281, 824, 738, 259, 1159, 222, 157, 0, 0, 0, 0, 0, 0, 0, 0, 298, 592, 71, 663, 882, 194, 1466, 2736, 0, 0, 0, 0, 0, 0, 0, 0, 470, 313, 166, 972, 654, 38, 2610, 89, 1166, 285, 0, 0, 0, 0, 0, 0, 853, 563, 1037, 1434, 593, 2151, 0, 191, 0, 0, 0, 0, 0, 0, 0, 0, 938, 557, 2069, 1117, 306, 1286, 215, 225, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4605, 6523, 6160, 4664, 6902, 6763, 6822, 6713] avg=6144.0000 max=6902 max_rank=4 min=4605 min_rank=0 imbalance_ratio=1.123372 max_load_reduction=42250 max_load_reduction_pct=0.859578 moved_to_echo=27200 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=20 rerouted_tokens_per_expert_current_ep_rank=[718, 386, 120, 371, 1218, 336, 466, 106, 423, 5, 0, 0, 0, 0, 0, 0, 699, 55, 490, 318, 1653, 43, 50, 2150, 895, 0, 0, 0, 0, 0, 0, 0, 355, 395, 979, 105, 1217, 583, 631, 1257, 471, 274, 47, 0, 0, 0, 0, 0, 27, 924, 342, 2416, 2744, 668, 546, 358, 0, 0, 0, 0, 0, 0, 0, 0, 1636, 1123, 1852, 694, 1329, 768, 319, 566, 0, 0, 0, 0, 0, 0, 0, 0, 251, 1751, 214, 337, 651, 312, 160, 163, 1296, 80, 0, 0, 0, 0, 0, 0, 295, 750, 1076, 1568, 1557, 602, 0, 212, 0, 0, 0, 0, 0, 0, 0, 0, 166, 262, 1509, 877, 310, 640, 767, 218, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4149, 6353, 6314, 8025, 8287, 5215, 6060, 4749] avg=6144.0000 max=8287 max_rank=4 min=4149 min_rank=0 imbalance_ratio=1.348796 max_load_reduction=40865 max_load_reduction_pct=0.831401 moved_to_echo=24311 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=20 rerouted_tokens_per_expert_current_ep_rank=[2452, 1395, 94, 796, 1137, 2626, 707, 702, 605, 19, 0, 0, 0, 0, 0, 0, 228, 48, 41, 485, 69, 208, 185, 548, 1148, 0, 0, 0, 0, 0, 0, 0, 434, 176, 684, 826, 1250, 55, 479, 139, 977, 370, 70, 0, 0, 0, 0, 0, 824, 1901, 2633, 1019, 197, 55, 819, 788, 0, 0, 0, 0, 0, 0, 0, 0, 473, 245, 396, 274, 143, 1257, 1001, 1409, 0, 0, 0, 0, 0, 0, 0, 0, 293, 97, 326, 66, 503, 61, 101, 173, 1727, 103, 0, 0, 0, 0, 0, 0, 786, 252, 2477, 760, 199, 387, 0, 273, 0, 0, 0, 0, 0, 0, 0, 0, 77, 2261, 2131, 616, 587, 633, 1660, 216, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10533, 2960, 5460, 8236, 5198, 3450, 5134, 8181] avg=6144.0000 max=10533 max_rank=0 min=2960 min_rank=1 imbalance_ratio=1.714355 max_load_reduction=38619 max_load_reduction_pct=0.785706 moved_to_echo=21963 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=20 rerouted_tokens_per_expert_current_ep_rank=[738, 109, 593, 801, 2930, 633, 1017, 51, 463, 17, 0, 0, 0, 0, 0, 0, 424, 591, 240, 337, 1164, 36, 127, 1564, 2952, 0, 0, 0, 0, 0, 0, 0, 260, 788, 3328, 221, 519, 98, 291, 28, 742, 98, 51, 0, 0, 0, 0, 0, 66, 1453, 533, 1703, 338, 945, 57, 34, 0, 0, 0, 0, 0, 0, 0, 0, 788, 1175, 2169, 1078, 237, 138, 1114, 1193, 0, 0, 0, 0, 0, 0, 0, 0, 5, 725, 753, 274, 1067, 34, 925, 334, 466, 267, 0, 0, 0, 0, 0, 0, 1012, 72, 1557, 418, 54, 2084, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 160, 754, 1650, 319, 656, 361, 849, 47, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7352, 7435, 6424, 5129, 7892, 4850, 5274, 4796] avg=6144.0000 max=7892 max_rank=4 min=4796 min_rank=7 imbalance_ratio=1.284505 max_load_reduction=41260 max_load_reduction_pct=0.839437 moved_to_echo=22812 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=20 rerouted_tokens_per_expert_current_ep_rank=[1551, 77, 5, 320, 1445, 933, 80, 99, 193, 27, 0, 0, 0, 0, 0, 0, 946, 1210, 60, 42, 1311, 5, 1613, 1480, 173, 0, 0, 0, 0, 0, 0, 0, 662, 155, 623, 398, 1128, 437, 206, 400, 1263, 164, 21, 0, 0, 0, 0, 0, 215, 2474, 583, 525, 1738, 140, 884, 1869, 0, 0, 0, 0, 0, 0, 0, 0, 950, 204, 2313, 297, 780, 881, 1753, 480, 0, 0, 0, 0, 0, 0, 0, 0, 203, 149, 551, 136, 1622, 26, 196, 598, 777, 16, 0, 0, 0, 0, 0, 0, 815, 485, 1813, 2186, 179, 330, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 807, 1157, 690, 986, 168, 286, 959, 776, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4730, 6840, 5457, 8428, 7658, 4274, 5936, 5829] avg=6144.0000 max=8428 max_rank=3 min=4274 min_rank=5 imbalance_ratio=1.371745 max_load_reduction=40724 max_load_reduction_pct=0.828532 moved_to_echo=23697 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2304, 7040, 4224, 3200, 9856, 1536, 6784, 4736, 8832, 1280, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2944, 7424, 12928, 7680, 3584, 6144, 6784, 2304, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[5760, 4864, 7936, 3968, 4224, 4224, 9600, 8960, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[8320, 4608, 10624, 8960, 4864, 10624, 0, 1536, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2816, 3584, 1536, 2432, 8832, 896, 3712, 12544, 13440, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[10112, 4224, 2432, 4992, 12416, 6272, 3712, 1792, 3712, 256, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[4480, 3968, 9856, 3072, 10752, 1920, 2432, 6016, 4992, 1920, 512, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[1664, 9856, 7808, 9600, 8320, 4096, 3584, 4608, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=21 tokens_per_expert_current_ep_rank=[509, 1313, 1344, 701, 30, 226, 830, 1671, 220, 5187, 1236, 3627, 189, 922, 208, 860, 1309, 689, 244, 618, 2019, 277, 83, 187, 66, 327, 1116, 59, 647, 192, 35, 92, 54, 604, 230, 335, 1836, 447, 876, 713, 2310, 1375, 21, 256, 2389, 930, 2571, 820, 199, 880, 241, 277, 380, 605, 338, 653, 19, 212, 160, 367, 488, 831, 693, 9] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=21 tokens_per_expert_current_ep_rank=[1377, 765, 305, 136, 239, 730, 215, 319, 143, 1863, 3393, 1428, 52, 495, 186, 638, 696, 183, 320, 83, 2178, 1025, 598, 790, 202, 1737, 1073, 587, 3310, 97, 1, 94, 37, 681, 1323, 46, 201, 30, 646, 795, 3154, 207, 395, 638, 2688, 157, 673, 1193, 745, 1536, 1120, 1038, 44, 591, 1468, 791, 35, 564, 907, 490, 41, 1319, 293, 48] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=21 tokens_per_expert_current_ep_rank=[658, 575, 15, 1180, 503, 243, 1886, 637, 16, 1294, 404, 153, 73, 2015, 257, 1988, 100, 56, 39, 1672, 3142, 1279, 654, 704, 531, 1985, 889, 211, 2849, 178, 15, 1268, 13, 844, 151, 547, 248, 36, 1346, 574, 1856, 1643, 309, 501, 2906, 37, 643, 587, 1110, 552, 1295, 121, 221, 463, 754, 637, 1656, 165, 556, 13, 36, 149, 576, 1138] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=21 tokens_per_expert_current_ep_rank=[57, 174, 1141, 139, 24, 233, 583, 1733, 296, 276, 282, 1007, 152, 341, 653, 482, 1094, 255, 471, 42, 4168, 1075, 141, 340, 237, 455, 1015, 1274, 1282, 73, 16, 7, 14, 427, 105, 1168, 1286, 13, 1372, 801, 4104, 2119, 517, 169, 915, 280, 1781, 3020, 528, 1318, 1432, 364, 213, 1967, 228, 52, 199, 80, 2424, 425, 294, 751, 967, 301] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=21 tokens_per_expert_current_ep_rank=[326, 146, 1413, 255, 34, 61, 1133, 225, 55, 3113, 3119, 512, 467, 1772, 91, 486, 2352, 235, 30, 40, 4907, 919, 507, 710, 197, 612, 823, 7, 3964, 557, 11, 47, 80, 2006, 31, 34, 1731, 352, 306, 337, 2565, 1003, 44, 8, 531, 610, 1284, 772, 269, 1443, 238, 340, 1309, 671, 87, 180, 21, 267, 304, 850, 482, 624, 1226, 21] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=21 tokens_per_expert_current_ep_rank=[173, 184, 365, 113, 80, 54, 523, 1332, 115, 3231, 2198, 119, 411, 956, 25, 64, 2606, 102, 16, 400, 859, 135, 246, 889, 16, 1233, 1001, 112, 4868, 13, 10, 205, 100, 974, 268, 135, 471, 145, 1075, 1046, 5615, 1991, 753, 282, 491, 100, 1216, 2421, 1361, 116, 1149, 115, 890, 642, 107, 135, 486, 181, 496, 141, 954, 1007, 1346, 289] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=21 tokens_per_expert_current_ep_rank=[246, 258, 315, 239, 470, 29, 31, 213, 194, 1113, 1656, 1672, 26, 615, 731, 513, 2828, 327, 886, 1718, 2365, 694, 247, 420, 226, 1068, 490, 275, 1461, 609, 144, 314, 112, 847, 197, 322, 237, 23, 829, 462, 5101, 1578, 905, 155, 1303, 7, 2101, 852, 747, 197, 3729, 110, 16, 1174, 87, 363, 908, 310, 768, 541, 76, 1550, 241, 911] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=21 tokens_per_expert_current_ep_rank=[92, 534, 504, 98, 221, 159, 54, 1424, 507, 1943, 2349, 335, 238, 744, 428, 465, 1736, 4598, 666, 186, 1556, 891, 347, 100, 52, 807, 917, 65, 2662, 76, 54, 170, 93, 841, 71, 446, 429, 457, 1487, 94, 2063, 2847, 348, 90, 193, 30, 1110, 1205, 1722, 625, 1866, 197, 12, 727, 228, 212, 181, 231, 1833, 840, 2244, 771, 669, 12] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=21 rerouted_tokens_per_expert_current_ep_rank=[658, 575, 15, 1180, 503, 243, 1886, 637, 1203, 0, 0, 0, 0, 0, 0, 0, 16, 473, 404, 153, 73, 2015, 257, 1988, 0, 0, 0, 0, 0, 0, 0, 0, 100, 56, 39, 1672, 1379, 1279, 654, 704, 0, 0, 0, 0, 0, 0, 0, 0, 531, 1985, 889, 211, 2849, 178, 15, 1268, 536, 0, 0, 0, 0, 0, 0, 0, 13, 844, 151, 547, 248, 36, 1346, 574, 653, 875, 0, 0, 0, 0, 0, 0, 0, 1024, 309, 501, 2906, 37, 643, 587, 0, 0, 0, 0, 0, 0, 0, 0, 1110, 552, 1295, 121, 221, 463, 754, 637, 378, 83, 0, 0, 0, 0, 0, 0, 1656, 165, 556, 13, 36, 149, 576, 1138, 443, 888, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6900, 5379, 5883, 8462, 5287, 6007, 5614, 5620] avg=6144.0000 max=8462 max_rank=3 min=5287 min_rank=4 imbalance_ratio=1.377279 max_load_reduction=40690 max_load_reduction_pct=0.827840 moved_to_echo=22528 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=21 rerouted_tokens_per_expert_current_ep_rank=[57, 174, 1141, 139, 24, 233, 583, 1733, 2661, 0, 0, 0, 0, 0, 0, 0, 296, 102, 282, 1007, 152, 341, 653, 482, 0, 0, 0, 0, 0, 0, 0, 0, 1094, 255, 471, 42, 1829, 1075, 141, 340, 0, 0, 0, 0, 0, 0, 0, 0, 237, 455, 1015, 1274, 1282, 73, 16, 7, 691, 0, 0, 0, 0, 0, 0, 0, 14, 427, 105, 1168, 1286, 13, 1372, 801, 1443, 1161, 0, 0, 0, 0, 0, 0, 0, 1320, 517, 169, 915, 280, 1781, 3020, 0, 0, 0, 0, 0, 0, 0, 0, 528, 1318, 1432, 364, 213, 1967, 228, 52, 80, 108, 0, 0, 0, 0, 0, 0, 199, 80, 2424, 425, 294, 751, 967, 301, 94, 1178, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6745, 3315, 5247, 5050, 7790, 8002, 6290, 6713] avg=6144.0000 max=8002 max_rank=5 min=3315 min_rank=1 imbalance_ratio=1.302409 max_load_reduction=41150 max_load_reduction_pct=0.837199 moved_to_echo=28795 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=21 rerouted_tokens_per_expert_current_ep_rank=[509, 1313, 1344, 701, 30, 226, 830, 1671, 1498, 0, 0, 0, 0, 0, 0, 0, 220, 1893, 1236, 3627, 189, 922, 208, 860, 0, 0, 0, 0, 0, 0, 0, 0, 1309, 689, 244, 618, 887, 277, 83, 187, 0, 0, 0, 0, 0, 0, 0, 0, 66, 327, 1116, 59, 647, 192, 35, 92, 448, 0, 0, 0, 0, 0, 0, 0, 54, 604, 230, 335, 1836, 447, 876, 713, 812, 562, 0, 0, 0, 0, 0, 0, 0, 857, 21, 256, 2389, 930, 2571, 820, 0, 0, 0, 0, 0, 0, 0, 0, 199, 880, 241, 277, 380, 605, 338, 653, 1518, 70, 0, 0, 0, 0, 0, 0, 19, 212, 160, 367, 488, 831, 693, 9, 1776, 570, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8122, 9155, 4294, 2982, 6469, 7844, 5161, 5125] avg=6144.0000 max=9155 max_rank=1 min=2982 min_rank=3 imbalance_ratio=1.490072 max_load_reduction=39997 max_load_reduction_pct=0.813741 moved_to_echo=24599 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=21 rerouted_tokens_per_expert_current_ep_rank=[326, 146, 1413, 255, 34, 61, 1133, 225, 1663, 0, 0, 0, 0, 0, 0, 0, 55, 1136, 3119, 512, 467, 1772, 91, 486, 0, 0, 0, 0, 0, 0, 0, 0, 2352, 235, 30, 40, 2154, 919, 507, 710, 0, 0, 0, 0, 0, 0, 0, 0, 197, 612, 823, 7, 3964, 557, 11, 47, 327, 0, 0, 0, 0, 0, 0, 0, 80, 2006, 31, 34, 1731, 352, 306, 337, 902, 1366, 0, 0, 0, 0, 0, 0, 0, 625, 44, 8, 531, 610, 1284, 772, 0, 0, 0, 0, 0, 0, 0, 0, 269, 1443, 238, 340, 1309, 671, 87, 180, 911, 51, 0, 0, 0, 0, 0, 0, 21, 267, 304, 850, 482, 624, 1226, 21, 1066, 1387, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5256, 7638, 6947, 6545, 7145, 3874, 5499, 6248] avg=6144.0000 max=7638 max_rank=1 min=3874 min_rank=5 imbalance_ratio=1.243164 max_load_reduction=41514 max_load_reduction_pct=0.844604 moved_to_echo=22766 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=21 rerouted_tokens_per_expert_current_ep_rank=[173, 184, 365, 113, 80, 54, 523, 1332, 3641, 0, 0, 0, 0, 0, 0, 0, 115, 1172, 2198, 119, 411, 956, 25, 64, 0, 0, 0, 0, 0, 0, 0, 0, 2606, 102, 16, 400, 368, 135, 246, 889, 0, 0, 0, 0, 0, 0, 0, 0, 16, 1233, 1001, 112, 4868, 13, 10, 205, 653, 0, 0, 0, 0, 0, 0, 0, 100, 974, 268, 135, 471, 145, 1075, 1046, 1974, 243, 0, 0, 0, 0, 0, 0, 0, 1233, 753, 282, 491, 100, 1216, 2421, 0, 0, 0, 0, 0, 0, 0, 0, 1361, 116, 1149, 115, 890, 642, 107, 135, 950, 105, 0, 0, 0, 0, 0, 0, 486, 181, 496, 141, 954, 1007, 1346, 289, 1109, 248, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6465, 5060, 4762, 8111, 6431, 6496, 5570, 6257] avg=6144.0000 max=8111 max_rank=3 min=4762 min_rank=2 imbalance_ratio=1.320150 max_load_reduction=41041 max_load_reduction_pct=0.834981 moved_to_echo=24754 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=21 rerouted_tokens_per_expert_current_ep_rank=[1377, 765, 305, 136, 239, 730, 215, 319, 2046, 0, 0, 0, 0, 0, 0, 0, 143, 680, 3393, 1428, 52, 495, 186, 638, 0, 0, 0, 0, 0, 0, 0, 0, 696, 183, 320, 83, 957, 1025, 598, 790, 0, 0, 0, 0, 0, 0, 0, 0, 202, 1737, 1073, 587, 3310, 97, 1, 94, 67, 0, 0, 0, 0, 0, 0, 0, 37, 681, 1323, 46, 201, 30, 646, 795, 1108, 606, 0, 0, 0, 0, 0, 0, 0, 130, 395, 638, 2688, 157, 673, 1193, 0, 0, 0, 0, 0, 0, 0, 0, 745, 1536, 1120, 1038, 44, 591, 1468, 791, 545, 10, 0, 0, 0, 0, 0, 0, 35, 564, 907, 490, 41, 1319, 293, 48, 638, 615, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6132, 7015, 4652, 7168, 5473, 5874, 7888, 4950] avg=6144.0000 max=7888 max_rank=6 min=4652 min_rank=2 imbalance_ratio=1.283854 max_load_reduction=41264 max_load_reduction_pct=0.839518 moved_to_echo=24185 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=21 rerouted_tokens_per_expert_current_ep_rank=[246, 258, 315, 239, 470, 29, 31, 213, 3308, 0, 0, 0, 0, 0, 0, 0, 194, 407, 1656, 1672, 26, 615, 731, 513, 0, 0, 0, 0, 0, 0, 0, 0, 2828, 327, 886, 1718, 1039, 694, 247, 420, 0, 0, 0, 0, 0, 0, 0, 0, 226, 1068, 490, 275, 1461, 609, 144, 314, 515, 0, 0, 0, 0, 0, 0, 0, 112, 847, 197, 322, 237, 23, 829, 462, 1793, 658, 0, 0, 0, 0, 0, 0, 0, 983, 905, 155, 1303, 7, 2101, 852, 0, 0, 0, 0, 0, 0, 0, 0, 747, 197, 3729, 110, 16, 1174, 87, 363, 325, 80, 0, 0, 0, 0, 0, 0, 908, 310, 768, 541, 76, 1550, 241, 911, 381, 668, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5109, 5814, 8159, 5102, 5480, 6306, 6828, 6354] avg=6144.0000 max=8159 max_rank=2 min=5102 min_rank=3 imbalance_ratio=1.327962 max_load_reduction=40993 max_load_reduction_pct=0.834005 moved_to_echo=24968 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=21 rerouted_tokens_per_expert_current_ep_rank=[92, 534, 504, 98, 221, 159, 54, 1424, 1337, 0, 0, 0, 0, 0, 0, 0, 507, 710, 2349, 335, 238, 744, 428, 465, 0, 0, 0, 0, 0, 0, 0, 0, 1736, 4598, 666, 186, 684, 891, 347, 100, 0, 0, 0, 0, 0, 0, 0, 0, 52, 807, 917, 65, 2662, 76, 54, 170, 929, 0, 0, 0, 0, 0, 0, 0, 93, 841, 71, 446, 429, 457, 1487, 94, 726, 433, 0, 0, 0, 0, 0, 0, 0, 1773, 348, 90, 193, 30, 1110, 1205, 0, 0, 0, 0, 0, 0, 0, 0, 1722, 625, 1866, 197, 12, 727, 228, 212, 568, 145, 0, 0, 0, 0, 0, 0, 181, 231, 1833, 840, 2244, 771, 669, 12, 665, 439, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4423, 5776, 9208, 5732, 5077, 4749, 6302, 7885] avg=6144.0000 max=9208 max_rank=2 min=4423 min_rank=0 imbalance_ratio=1.498698 max_load_reduction=39944 max_load_reduction_pct=0.812663 moved_to_echo=24013 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3584, 2048, 7552, 3712, 4736, 7040, 6016, 2816, 6272, 6016, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[6784, 6784, 11136, 2688, 3200, 6912, 3328, 3072, 5376, 768, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3456, 3968, 5504, 2944, 1664, 1792, 5376, 7680, 17408, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1664, 6656, 14720, 8960, 1664, 7936, 2688, 5504, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[12800, 6528, 2688, 4864, 9344, 6400, 2944, 4224, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1536, 8320, 7424, 2688, 21120, 1920, 384, 2304, 4224, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 8064, 3328, 2176, 11520, 2176, 11392, 10880, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[512, 7296, 2432, 3072, 6528, 1536, 8064, 4864, 9472, 6016, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=22 tokens_per_expert_current_ep_rank=[853, 599, 378, 505, 1339, 58, 1181, 274, 1933, 50, 417, 1358, 968, 421, 1177, 1188, 375, 2639, 2459, 998, 534, 1629, 919, 1244, 335, 2398, 22, 97, 1590, 1047, 179, 838, 45, 659, 534, 243, 2959, 387, 3416, 446, 68, 241, 119, 549, 49, 458, 510, 600, 874, 113, 592, 959, 31, 708, 14, 36, 492, 190, 537, 1130, 988, 210, 83, 910] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=22 tokens_per_expert_current_ep_rank=[227, 50, 67, 220, 1788, 44, 3380, 32, 1211, 850, 128, 262, 305, 1486, 46, 466, 270, 1597, 2001, 1499, 1200, 969, 3869, 1435, 95, 2390, 587, 730, 3143, 472, 102, 301, 103, 1596, 222, 671, 1371, 562, 2610, 145, 111, 392, 4, 169, 29, 275, 274, 105, 288, 688, 796, 417, 94, 2376, 172, 67, 1178, 246, 215, 563, 1199, 79, 486, 427] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=22 tokens_per_expert_current_ep_rank=[87, 70, 102, 577, 1094, 287, 1856, 553, 2014, 95, 230, 255, 922, 774, 423, 1913, 2530, 696, 643, 766, 417, 2093, 837, 967, 168, 891, 552, 227, 943, 1568, 2, 1123, 42, 1721, 341, 225, 697, 618, 1196, 543, 458, 164, 82, 101, 386, 418, 682, 105, 1974, 1258, 1098, 1930, 1009, 690, 238, 16, 1059, 399, 873, 1473, 136, 558, 2480, 507] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=22 tokens_per_expert_current_ep_rank=[1869, 69, 906, 240, 748, 108, 906, 289, 666, 121, 95, 1602, 1630, 908, 401, 855, 817, 830, 794, 1248, 243, 1282, 978, 2254, 158, 5176, 127, 252, 1445, 1741, 77, 305, 166, 2083, 667, 233, 50, 733, 1180, 656, 643, 838, 23, 223, 40, 95, 308, 1045, 349, 1104, 144, 245, 24, 3156, 119, 44, 1953, 397, 281, 651, 1050, 272, 465, 775] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=22 tokens_per_expert_current_ep_rank=[438, 1123, 456, 360, 148, 40, 2315, 375, 2013, 184, 2338, 185, 159, 829, 405, 502, 1275, 809, 767, 197, 14, 614, 2123, 1251, 210, 966, 18, 84, 350, 2284, 10, 157, 43, 3521, 1126, 2412, 396, 799, 1251, 1386, 523, 382, 905, 83, 89, 816, 31, 380, 414, 2055, 514, 792, 25, 1535, 793, 31, 858, 918, 1129, 786, 248, 776, 547, 589] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=22 tokens_per_expert_current_ep_rank=[545, 76, 515, 1048, 169, 12, 2468, 253, 1159, 76, 427, 1216, 974, 131, 62, 518, 540, 3167, 374, 990, 160, 2397, 6031, 147, 300, 567, 211, 63, 1940, 2379, 7, 2376, 116, 414, 1011, 244, 176, 464, 2966, 2393, 720, 591, 17, 13, 246, 453, 337, 314, 197, 329, 408, 396, 792, 501, 68, 163, 540, 29, 306, 251, 797, 463, 1700, 439] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=22 tokens_per_expert_current_ep_rank=[3585, 751, 18, 604, 932, 51, 621, 322, 3503, 227, 1559, 266, 1297, 119, 47, 2127, 170, 647, 1648, 33, 129, 522, 2988, 214, 82, 610, 464, 154, 4226, 936, 18, 803, 428, 1729, 969, 2416, 159, 117, 1130, 1866, 418, 106, 212, 11, 3, 832, 513, 349, 229, 252, 160, 449, 310, 872, 1043, 39, 321, 627, 397, 218, 68, 160, 2476, 600] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=22 tokens_per_expert_current_ep_rank=[57, 204, 1388, 398, 1998, 48, 453, 800, 1751, 7, 739, 333, 1354, 3014, 221, 972, 760, 1869, 1225, 113, 1097, 4395, 1443, 175, 382, 1141, 204, 1025, 3327, 1174, 304, 399, 129, 1336, 163, 203, 62, 1234, 960, 452, 349, 40, 26, 245, 93, 1335, 466, 500, 111, 392, 116, 1027, 291, 483, 256, 520, 447, 651, 771, 350, 39, 435, 156, 2744] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=22 rerouted_tokens_per_expert_current_ep_rank=[87, 70, 102, 577, 1094, 287, 1856, 553, 668, 60, 0, 0, 0, 0, 0, 0, 1346, 95, 230, 255, 922, 774, 423, 1913, 0, 0, 0, 0, 0, 0, 0, 0, 2530, 696, 643, 766, 417, 441, 0, 967, 0, 0, 0, 0, 0, 0, 0, 0, 168, 891, 552, 227, 550, 1568, 2, 1123, 0, 0, 0, 0, 0, 0, 0, 0, 42, 1721, 341, 225, 697, 618, 381, 543, 0, 0, 0, 0, 0, 0, 0, 0, 458, 164, 82, 101, 386, 418, 682, 105, 837, 1355, 0, 0, 0, 0, 0, 0, 1974, 1258, 1098, 1930, 1009, 690, 238, 16, 812, 297, 0, 0, 0, 0, 0, 0, 1059, 399, 873, 1473, 136, 558, 2480, 507, 333, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5354, 5958, 6460, 5081, 4568, 4588, 9322, 7821] avg=6144.0000 max=9322 max_rank=6 min=4568 min_rank=4 imbalance_ratio=1.517253 max_load_reduction=39830 max_load_reduction_pct=0.810343 moved_to_echo=26299 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=22 rerouted_tokens_per_expert_current_ep_rank=[853, 599, 378, 505, 1339, 58, 1181, 274, 641, 102, 0, 0, 0, 0, 0, 0, 1292, 50, 417, 1358, 968, 421, 1177, 1188, 0, 0, 0, 0, 0, 0, 0, 0, 375, 2639, 2459, 998, 534, 344, 0, 1244, 0, 0, 0, 0, 0, 0, 0, 0, 335, 2398, 22, 97, 925, 1047, 179, 838, 0, 0, 0, 0, 0, 0, 0, 0, 45, 659, 534, 243, 2959, 387, 1086, 446, 0, 0, 0, 0, 0, 0, 0, 0, 68, 241, 119, 549, 49, 458, 510, 600, 919, 1054, 0, 0, 0, 0, 0, 0, 874, 113, 592, 959, 31, 708, 14, 36, 2320, 231, 0, 0, 0, 0, 0, 0, 492, 190, 537, 1130, 988, 210, 83, 910, 563, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5930, 6871, 8593, 5841, 6359, 4567, 5878, 5113] avg=6144.0000 max=8593 max_rank=2 min=4567 min_rank=5 imbalance_ratio=1.398600 max_load_reduction=40559 max_load_reduction_pct=0.825175 moved_to_echo=21917 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=22 rerouted_tokens_per_expert_current_ep_rank=[227, 50, 67, 220, 1788, 44, 3380, 32, 402, 202, 0, 0, 0, 0, 0, 0, 809, 850, 128, 262, 305, 1486, 46, 466, 0, 0, 0, 0, 0, 0, 0, 0, 270, 1597, 2001, 1499, 1200, 205, 0, 1435, 0, 0, 0, 0, 0, 0, 0, 0, 95, 2390, 587, 730, 1829, 472, 102, 301, 0, 0, 0, 0, 0, 0, 0, 0, 103, 1596, 222, 671, 1371, 562, 829, 145, 0, 0, 0, 0, 0, 0, 0, 0, 111, 392, 4, 169, 29, 275, 274, 105, 3869, 627, 0, 0, 0, 0, 0, 0, 288, 688, 796, 417, 94, 2376, 172, 67, 1773, 137, 0, 0, 0, 0, 0, 0, 1178, 246, 215, 563, 1199, 79, 486, 427, 1112, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6412, 4352, 8207, 6506, 5499, 5855, 6808, 5513] avg=6144.0000 max=8207 max_rank=2 min=4352 min_rank=1 imbalance_ratio=1.335775 max_load_reduction=40945 max_load_reduction_pct=0.833028 moved_to_echo=23675 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=22 rerouted_tokens_per_expert_current_ep_rank=[1869, 69, 906, 240, 748, 108, 906, 289, 221, 93, 0, 0, 0, 0, 0, 0, 445, 121, 95, 1602, 1630, 908, 401, 855, 0, 0, 0, 0, 0, 0, 0, 0, 817, 830, 794, 1248, 243, 270, 0, 2254, 0, 0, 0, 0, 0, 0, 0, 0, 158, 5176, 127, 252, 841, 1741, 77, 305, 0, 0, 0, 0, 0, 0, 0, 0, 166, 2083, 667, 233, 50, 733, 376, 656, 0, 0, 0, 0, 0, 0, 0, 0, 643, 838, 23, 223, 40, 95, 308, 1045, 978, 830, 0, 0, 0, 0, 0, 0, 349, 1104, 144, 245, 24, 3156, 119, 44, 801, 182, 0, 0, 0, 0, 0, 0, 1953, 397, 281, 651, 1050, 272, 465, 775, 511, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5449, 6057, 6456, 8677, 4964, 5023, 6168, 6358] avg=6144.0000 max=8677 max_rank=3 min=4964 min_rank=4 imbalance_ratio=1.412272 max_load_reduction=40475 max_load_reduction_pct=0.823466 moved_to_echo=22513 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=22 rerouted_tokens_per_expert_current_ep_rank=[438, 1123, 456, 360, 148, 40, 2315, 375, 672, 26, 0, 0, 0, 0, 0, 0, 1341, 184, 2338, 185, 159, 829, 405, 502, 0, 0, 0, 0, 0, 0, 0, 0, 1275, 809, 767, 197, 14, 125, 0, 1251, 0, 0, 0, 0, 0, 0, 0, 0, 210, 966, 18, 84, 196, 2284, 10, 157, 0, 0, 0, 0, 0, 0, 0, 0, 43, 3521, 1126, 2412, 396, 799, 390, 1386, 0, 0, 0, 0, 0, 0, 0, 0, 523, 382, 905, 83, 89, 816, 31, 380, 2123, 400, 0, 0, 0, 0, 0, 0, 414, 2055, 514, 792, 25, 1535, 793, 31, 853, 89, 0, 0, 0, 0, 0, 0, 858, 918, 1129, 786, 248, 776, 547, 589, 128, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5953, 5943, 4438, 3925, 10073, 5732, 7101, 5987] avg=6144.0000 max=10073 max_rank=4 min=3925 min_rank=3 imbalance_ratio=1.639486 max_load_reduction=39079 max_load_reduction_pct=0.795064 moved_to_echo=28893 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=22 rerouted_tokens_per_expert_current_ep_rank=[545, 76, 515, 1048, 169, 12, 2468, 253, 384, 124, 0, 0, 0, 0, 0, 0, 775, 76, 427, 1216, 974, 131, 62, 518, 0, 0, 0, 0, 0, 0, 0, 0, 540, 3167, 374, 990, 160, 505, 0, 147, 0, 0, 0, 0, 0, 0, 0, 0, 300, 567, 211, 63, 1130, 2379, 7, 2376, 0, 0, 0, 0, 0, 0, 0, 0, 116, 414, 1011, 244, 176, 464, 943, 2393, 0, 0, 0, 0, 0, 0, 0, 0, 720, 591, 17, 13, 246, 453, 337, 314, 6031, 1552, 0, 0, 0, 0, 0, 0, 197, 329, 408, 396, 792, 501, 68, 163, 2014, 340, 0, 0, 0, 0, 0, 0, 540, 29, 306, 251, 797, 463, 1700, 439, 686, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5594, 4179, 5883, 7033, 5761, 10274, 5208, 5220] avg=6144.0000 max=10274 max_rank=5 min=4179 min_rank=1 imbalance_ratio=1.672201 max_load_reduction=38878 max_load_reduction_pct=0.790975 moved_to_echo=26463 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=22 rerouted_tokens_per_expert_current_ep_rank=[57, 204, 1388, 398, 1998, 48, 453, 800, 581, 214, 0, 0, 0, 0, 0, 0, 1170, 7, 739, 333, 1354, 3014, 221, 972, 0, 0, 0, 0, 0, 0, 0, 0, 760, 1869, 1225, 113, 1097, 925, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 382, 1141, 204, 1025, 1935, 1174, 304, 399, 0, 0, 0, 0, 0, 0, 0, 0, 129, 1336, 163, 203, 62, 1234, 305, 452, 0, 0, 0, 0, 0, 0, 0, 0, 349, 40, 26, 245, 93, 1335, 466, 500, 1443, 2846, 0, 0, 0, 0, 0, 0, 111, 392, 116, 1027, 291, 483, 256, 520, 652, 624, 0, 0, 0, 0, 0, 0, 447, 651, 771, 350, 39, 435, 156, 2744, 1178, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6141, 7810, 6164, 6564, 3884, 7343, 4472, 6774] avg=6144.0000 max=7810 max_rank=1 min=3884 min_rank=4 imbalance_ratio=1.271159 max_load_reduction=41342 max_load_reduction_pct=0.841105 moved_to_echo=22473 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=22 rerouted_tokens_per_expert_current_ep_rank=[3585, 751, 18, 604, 932, 51, 621, 322, 1163, 272, 0, 0, 0, 0, 0, 0, 2340, 227, 1559, 266, 1297, 119, 47, 2127, 0, 0, 0, 0, 0, 0, 0, 0, 170, 647, 1648, 33, 129, 110, 0, 214, 0, 0, 0, 0, 0, 0, 0, 0, 82, 610, 464, 154, 2458, 936, 18, 803, 0, 0, 0, 0, 0, 0, 0, 0, 428, 1729, 969, 2416, 159, 117, 360, 1866, 0, 0, 0, 0, 0, 0, 0, 0, 418, 106, 212, 11, 3, 832, 513, 349, 2988, 338, 0, 0, 0, 0, 0, 0, 229, 252, 160, 449, 310, 872, 1043, 39, 767, 74, 0, 0, 0, 0, 0, 0, 321, 627, 397, 218, 68, 160, 2476, 600, 1496, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8319, 7982, 2951, 5525, 8044, 5770, 4195, 6366] avg=6144.0000 max=8319 max_rank=0 min=2951 min_rank=2 imbalance_ratio=1.354004 max_load_reduction=40833 max_load_reduction_pct=0.830750 moved_to_echo=24375 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[9600, 1664, 6016, 5504, 7680, 7808, 2816, 8576, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3328, 2816, 1408, 1408, 1024, 4736, 3200, 3456, 19200, 9088, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[4480, 6272, 3840, 6272, 2688, 10368, 2816, 1024, 10112, 2048, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[6912, 3584, 4608, 5504, 4608, 3072, 8448, 7040, 6016, 128, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1792, 14208, 2304, 2688, 9984, 11648, 768, 6400, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[6784, 12288, 9984, 5888, 3840, 2944, 0, 7808, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1152, 13184, 5120, 6656, 5888, 4992, 4736, 7936, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[7680, 2944, 3840, 3968, 8320, 768, 13184, 2944, 4736, 1152, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=23 tokens_per_expert_current_ep_rank=[1972, 813, 1209, 255, 919, 333, 367, 221, 71, 299, 336, 2423, 190, 2142, 6, 587, 333, 4070, 1247, 799, 743, 92, 88, 326, 848, 2360, 44, 439, 14, 2511, 299, 223, 462, 636, 650, 1704, 378, 2271, 680, 91, 81, 356, 4808, 454, 541, 236, 94, 449, 43, 513, 264, 1978, 768, 922, 1225, 1700, 165, 128, 125, 33, 112, 199, 360, 147] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=23 tokens_per_expert_current_ep_rank=[1179, 38, 2050, 760, 3378, 338, 55, 149, 5, 453, 56, 677, 1506, 2401, 42, 102, 62, 463, 327, 454, 631, 245, 261, 24, 1542, 714, 59, 137, 28, 1538, 437, 2757, 124, 731, 342, 1994, 190, 26, 180, 330, 77, 206, 1222, 2507, 2073, 179, 2032, 927, 332, 53, 17, 2719, 0, 2227, 1362, 1513, 377, 78, 34, 2947, 188, 239, 124, 934] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=23 tokens_per_expert_current_ep_rank=[2066, 118, 340, 229, 755, 550, 52, 1056, 6, 1042, 67, 490, 1496, 3560, 41, 623, 453, 1088, 704, 337, 1643, 242, 100, 513, 1370, 1072, 27, 1027, 71, 610, 451, 265, 192, 779, 554, 325, 122, 154, 878, 672, 37, 310, 4260, 303, 212, 385, 4182, 249, 56, 50, 10, 3163, 214, 953, 250, 3447, 43, 196, 637, 1026, 614, 20, 626, 1769] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=23 tokens_per_expert_current_ep_rank=[330, 21, 136, 996, 4063, 518, 333, 1263, 75, 747, 174, 3251, 380, 858, 193, 213, 113, 1427, 1320, 846, 162, 241, 87, 529, 1027, 475, 10, 711, 12, 1932, 59, 1915, 231, 78, 159, 510, 83, 513, 1427, 67, 770, 566, 1193, 1074, 1461, 141, 2394, 818, 452, 231, 233, 328, 268, 959, 2185, 2106, 41, 13, 72, 961, 3147, 676, 244, 1334] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=23 tokens_per_expert_current_ep_rank=[538, 200, 318, 850, 1491, 255, 38, 262, 104, 352, 355, 937, 813, 1169, 39, 289, 109, 3043, 1078, 1156, 1193, 78, 171, 1326, 2160, 2573, 132, 499, 18, 1373, 133, 347, 80, 934, 268, 1693, 1686, 778, 411, 623, 275, 117, 2252, 444, 467, 53, 803, 668, 144, 484, 94, 624, 487, 1647, 1168, 1528, 216, 471, 40, 1716, 1734, 90, 2304, 1454] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=23 tokens_per_expert_current_ep_rank=[149, 438, 1849, 762, 1863, 1165, 12, 257, 8, 43, 6, 343, 580, 2473, 41, 635, 1021, 1423, 1152, 886, 1740, 14, 441, 215, 1785, 870, 303, 657, 34, 2118, 787, 1234, 299, 95, 245, 2345, 2, 1512, 32, 124, 141, 455, 598, 429, 403, 73, 388, 713, 251, 209, 148, 1545, 12, 4271, 309, 902, 441, 461, 2, 4576, 561, 517, 424, 1365] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=23 tokens_per_expert_current_ep_rank=[1447, 36, 268, 403, 579, 44, 358, 614, 2, 818, 253, 1860, 1345, 1915, 103, 239, 37, 1298, 345, 715, 1153, 813, 16, 498, 1471, 1506, 202, 668, 29, 1053, 405, 2977, 151, 145, 24, 525, 508, 473, 745, 656, 1008, 150, 602, 130, 1093, 758, 2825, 387, 106, 614, 303, 25, 619, 2165, 1627, 3168, 224, 110, 105, 2367, 856, 730, 522, 1961] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=23 tokens_per_expert_current_ep_rank=[947, 997, 3045, 876, 829, 116, 82, 1806, 51, 150, 501, 156, 1688, 1376, 118, 407, 949, 1038, 2241, 2645, 108, 527, 389, 607, 1773, 866, 900, 1182, 32, 1655, 415, 2156, 197, 212, 56, 497, 103, 795, 177, 94, 137, 399, 173, 266, 266, 136, 1473, 36, 239, 1656, 39, 722, 114, 2491, 1209, 2720, 204, 854, 1008, 494, 56, 105, 83, 1513] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=23 rerouted_tokens_per_expert_current_ep_rank=[1179, 38, 2050, 760, 3231, 338, 55, 149, 0, 0, 0, 0, 0, 0, 0, 0, 5, 453, 56, 677, 1506, 2401, 42, 102, 657, 0, 0, 0, 0, 0, 0, 0, 62, 463, 327, 454, 631, 245, 261, 24, 147, 12, 0, 0, 0, 0, 0, 0, 1542, 714, 59, 137, 28, 560, 437, 2757, 0, 0, 0, 0, 0, 0, 0, 0, 124, 731, 342, 1994, 190, 26, 180, 330, 1153, 253, 0, 0, 0, 0, 0, 0, 77, 206, 935, 2507, 2073, 179, 2032, 927, 0, 0, 0, 0, 0, 0, 0, 0, 332, 53, 17, 2719, 0, 2227, 1362, 360, 0, 0, 0, 0, 0, 0, 0, 0, 377, 78, 34, 2947, 188, 239, 124, 934, 275, 68, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7800, 5899, 2626, 6234, 5323, 8936, 7070, 5264] avg=6144.0000 max=8936 max_rank=5 min=2626 min_rank=2 imbalance_ratio=1.454427 max_load_reduction=40216 max_load_reduction_pct=0.818197 moved_to_echo=26593 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=23 rerouted_tokens_per_expert_current_ep_rank=[1972, 813, 1209, 255, 880, 333, 367, 221, 0, 0, 0, 0, 0, 0, 0, 0, 71, 299, 336, 2423, 190, 2142, 6, 587, 1074, 0, 0, 0, 0, 0, 0, 0, 333, 4070, 1247, 799, 743, 92, 88, 326, 39, 48, 0, 0, 0, 0, 0, 0, 848, 2360, 44, 439, 14, 913, 299, 223, 0, 0, 0, 0, 0, 0, 0, 0, 462, 636, 650, 1704, 378, 2271, 680, 91, 1296, 413, 0, 0, 0, 0, 0, 0, 81, 356, 3675, 454, 541, 236, 94, 449, 0, 0, 0, 0, 0, 0, 0, 0, 43, 513, 264, 1978, 768, 922, 1225, 404, 0, 0, 0, 0, 0, 0, 0, 0, 165, 128, 125, 33, 112, 199, 360, 147, 1085, 111, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6050, 7128, 7785, 5140, 8581, 5886, 6117, 2465] avg=6144.0000 max=8581 max_rank=4 min=2465 min_rank=7 imbalance_ratio=1.396647 max_load_reduction=40571 max_load_reduction_pct=0.825419 moved_to_echo=23049 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=23 rerouted_tokens_per_expert_current_ep_rank=[538, 200, 318, 850, 1427, 255, 38, 262, 0, 0, 0, 0, 0, 0, 0, 0, 104, 352, 355, 937, 813, 1169, 39, 289, 587, 0, 0, 0, 0, 0, 0, 0, 109, 3043, 1078, 1156, 1193, 78, 171, 1326, 64, 22, 0, 0, 0, 0, 0, 0, 2160, 2573, 132, 499, 18, 500, 133, 347, 0, 0, 0, 0, 0, 0, 0, 0, 80, 934, 268, 1693, 1686, 778, 411, 623, 1165, 225, 0, 0, 0, 0, 0, 0, 275, 117, 1722, 444, 467, 53, 803, 668, 0, 0, 0, 0, 0, 0, 0, 0, 144, 484, 94, 624, 487, 1647, 1168, 363, 0, 0, 0, 0, 0, 0, 0, 0, 216, 471, 40, 1716, 1734, 90, 2304, 1454, 508, 61, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3888, 4645, 8240, 6362, 7863, 4549, 5011, 8594] avg=6144.0000 max=8594 max_rank=7 min=3888 min_rank=0 imbalance_ratio=1.398763 max_load_reduction=40558 max_load_reduction_pct=0.825155 moved_to_echo=26017 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=23 rerouted_tokens_per_expert_current_ep_rank=[330, 21, 136, 996, 3887, 518, 333, 1263, 0, 0, 0, 0, 0, 0, 0, 0, 75, 747, 174, 3251, 380, 858, 193, 213, 826, 0, 0, 0, 0, 0, 0, 0, 113, 1427, 1320, 846, 162, 241, 87, 529, 176, 12, 0, 0, 0, 0, 0, 0, 1027, 475, 10, 711, 12, 704, 59, 1915, 0, 0, 0, 0, 0, 0, 0, 0, 231, 78, 159, 510, 83, 513, 1427, 67, 1606, 317, 0, 0, 0, 0, 0, 0, 770, 566, 912, 1074, 1461, 141, 2394, 818, 0, 0, 0, 0, 0, 0, 0, 0, 452, 231, 233, 328, 268, 959, 2185, 500, 0, 0, 0, 0, 0, 0, 0, 0, 41, 13, 72, 961, 3147, 676, 244, 1334, 269, 85, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7484, 6717, 4913, 4913, 4991, 8136, 5156, 6842] avg=6144.0000 max=8136 max_rank=5 min=4913 min_rank=2 imbalance_ratio=1.324219 max_load_reduction=41016 max_load_reduction_pct=0.834473 moved_to_echo=25125 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=23 rerouted_tokens_per_expert_current_ep_rank=[2066, 118, 340, 229, 723, 550, 52, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1042, 67, 490, 1496, 3560, 41, 623, 260, 0, 0, 0, 0, 0, 0, 0, 453, 1088, 704, 337, 1643, 242, 100, 513, 32, 43, 0, 0, 0, 0, 0, 0, 1370, 1072, 27, 1027, 71, 223, 451, 265, 0, 0, 0, 0, 0, 0, 0, 0, 192, 779, 554, 325, 122, 154, 878, 672, 2628, 100, 0, 0, 0, 0, 0, 0, 37, 310, 3256, 303, 212, 385, 4182, 249, 0, 0, 0, 0, 0, 0, 0, 0, 56, 50, 10, 3163, 214, 953, 250, 819, 0, 0, 0, 0, 0, 0, 0, 0, 43, 196, 637, 1026, 614, 20, 626, 1769, 961, 27, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5134, 7585, 5155, 4506, 6404, 8934, 5515, 5919] avg=6144.0000 max=8934 max_rank=5 min=4506 min_rank=3 imbalance_ratio=1.454102 max_load_reduction=40218 max_load_reduction_pct=0.818237 moved_to_echo=26772 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=23 rerouted_tokens_per_expert_current_ep_rank=[1447, 36, 268, 403, 550, 44, 358, 614, 0, 0, 0, 0, 0, 0, 0, 0, 2, 818, 253, 1860, 1345, 1915, 103, 239, 455, 0, 0, 0, 0, 0, 0, 0, 37, 1298, 345, 715, 1153, 813, 16, 498, 29, 9, 0, 0, 0, 0, 0, 0, 1471, 1506, 202, 668, 29, 371, 405, 2977, 0, 0, 0, 0, 0, 0, 0, 0, 151, 145, 24, 525, 508, 473, 745, 656, 2420, 177, 0, 0, 0, 0, 0, 0, 1008, 150, 454, 130, 1093, 758, 2825, 387, 0, 0, 0, 0, 0, 0, 0, 0, 106, 614, 303, 25, 619, 2165, 1627, 748, 0, 0, 0, 0, 0, 0, 0, 0, 224, 110, 105, 2367, 856, 730, 522, 1961, 139, 50, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3720, 6990, 4913, 7629, 5824, 6805, 6207, 7064] avg=6144.0000 max=7629 max_rank=3 min=3720 min_rank=0 imbalance_ratio=1.241699 max_load_reduction=41523 max_load_reduction_pct=0.844788 moved_to_echo=25900 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=23 rerouted_tokens_per_expert_current_ep_rank=[149, 438, 1849, 762, 1782, 1165, 12, 257, 0, 0, 0, 0, 0, 0, 0, 0, 8, 43, 6, 343, 580, 2473, 41, 635, 905, 0, 0, 0, 0, 0, 0, 0, 1021, 1423, 1152, 886, 1740, 14, 441, 215, 81, 6, 0, 0, 0, 0, 0, 0, 1785, 870, 303, 657, 34, 771, 787, 1234, 0, 0, 0, 0, 0, 0, 0, 0, 299, 95, 245, 2345, 2, 1512, 32, 124, 687, 348, 0, 0, 0, 0, 0, 0, 141, 455, 458, 429, 403, 73, 388, 713, 0, 0, 0, 0, 0, 0, 0, 0, 251, 209, 148, 1545, 12, 4271, 309, 215, 0, 0, 0, 0, 0, 0, 0, 0, 441, 461, 2, 4576, 561, 517, 424, 1365, 134, 94, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6414, 5034, 6979, 6441, 5689, 3060, 6960, 8575] avg=6144.0000 max=8575 max_rank=7 min=3060 min_rank=5 imbalance_ratio=1.395671 max_load_reduction=40577 max_load_reduction_pct=0.825541 moved_to_echo=24284 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=23 rerouted_tokens_per_expert_current_ep_rank=[947, 997, 3045, 876, 793, 116, 82, 1806, 0, 0, 0, 0, 0, 0, 0, 0, 51, 150, 501, 156, 1688, 1376, 118, 407, 707, 0, 0, 0, 0, 0, 0, 0, 949, 1038, 2241, 2645, 108, 527, 389, 607, 36, 1, 0, 0, 0, 0, 0, 0, 1773, 866, 900, 1182, 32, 603, 415, 2156, 0, 0, 0, 0, 0, 0, 0, 0, 197, 212, 56, 497, 103, 795, 177, 94, 2074, 272, 0, 0, 0, 0, 0, 0, 137, 399, 133, 266, 266, 136, 1473, 36, 0, 0, 0, 0, 0, 0, 0, 0, 239, 1656, 39, 722, 114, 2491, 1209, 646, 0, 0, 0, 0, 0, 0, 0, 0, 204, 854, 1008, 494, 56, 105, 83, 1513, 39, 73, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8662, 5154, 8541, 7927, 4477, 2846, 7116, 4429] avg=6144.0000 max=8662 max_rank=0 min=2846 min_rank=5 imbalance_ratio=1.409831 max_load_reduction=40490 max_load_reduction_pct=0.823771 moved_to_echo=18868 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[8704, 2688, 9216, 5248, 13312, 3328, 1408, 5632, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[2560, 2560, 11648, 5632, 6528, 2048, 14208, 4352, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[384, 3968, 1792, 10240, 8064, 16000, 640, 3200, 5504, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[1792, 3712, 2304, 9600, 3072, 6528, 4608, 2688, 13056, 2176, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1792, 2432, 2048, 14208, 7296, 2688, 4736, 10496, 3456, 640, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[12032, 10496, 1792, 5376, 256, 4736, 3072, 11904, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3200, 13952, 8448, 7936, 7424, 2304, 1664, 4096, 640, 256, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[1664, 3840, 1152, 11136, 2560, 15744, 9344, 4096, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=24 tokens_per_expert_current_ep_rank=[4104, 118, 281, 7, 1334, 197, 65, 755, 172, 451, 411, 1596, 955, 341, 1325, 760, 527, 802, 184, 551, 1150, 216, 1479, 30, 2015, 1769, 28, 750, 47, 47, 1900, 769, 8, 414, 424, 479, 2414, 94, 62, 2072, 914, 213, 9, 676, 1161, 960, 692, 0, 540, 1102, 1345, 1476, 704, 1468, 4333, 48, 190, 105, 553, 77, 1077, 247, 23, 136] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=24 tokens_per_expert_current_ep_rank=[5246, 281, 107, 8, 2192, 22, 229, 133, 209, 112, 609, 5299, 165, 1590, 375, 238, 411, 3534, 794, 369, 383, 1230, 383, 276, 901, 373, 66, 886, 741, 230, 1185, 802, 152, 570, 207, 189, 2224, 52, 527, 990, 1660, 374, 7, 793, 2173, 1393, 516, 31, 365, 16, 726, 34, 1013, 3042, 33, 732, 93, 59, 326, 351, 954, 103, 23, 45] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=24 tokens_per_expert_current_ep_rank=[2329, 259, 493, 19, 1156, 761, 139, 85, 385, 92, 1412, 1520, 156, 1342, 977, 454, 849, 686, 993, 89, 437, 220, 1240, 141, 136, 388, 20, 1213, 57, 7, 2885, 1921, 1, 703, 668, 377, 2756, 532, 105, 3833, 294, 291, 4, 252, 158, 1255, 2092, 1, 131, 2372, 655, 365, 3365, 1290, 639, 289, 75, 225, 774, 796, 753, 15, 237, 988] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=24 tokens_per_expert_current_ep_rank=[156, 48, 232, 156, 2087, 267, 102, 306, 379, 199, 1640, 1364, 748, 2551, 133, 581, 14, 3019, 1157, 178, 1203, 458, 2401, 506, 212, 937, 222, 717, 1157, 631, 2332, 722, 72, 1146, 20, 3037, 1978, 80, 98, 1122, 452, 279, 61, 552, 419, 3012, 2929, 183, 319, 109, 449, 144, 2242, 78, 692, 429, 293, 471, 628, 580, 60, 14, 119, 270] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=24 tokens_per_expert_current_ep_rank=[3638, 83, 41, 42, 806, 406, 73, 676, 1465, 1083, 881, 1430, 1266, 1023, 2593, 296, 268, 1432, 269, 130, 941, 1210, 254, 555, 613, 1226, 132, 285, 121, 254, 1139, 559, 215, 43, 559, 1272, 1431, 29, 107, 1649, 2078, 364, 4, 742, 1145, 2322, 707, 22, 1374, 219, 586, 1843, 248, 857, 889, 160, 76, 21, 639, 784, 2169, 584, 354, 470] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=24 tokens_per_expert_current_ep_rank=[1865, 23, 1336, 4, 1061, 149, 249, 130, 2345, 1714, 1093, 1091, 545, 87, 143, 898, 470, 291, 457, 177, 63, 998, 952, 758, 58, 1089, 439, 135, 43, 980, 1523, 2063, 17, 178, 89, 2075, 1386, 23, 1, 1891, 782, 1240, 6, 109, 240, 919, 3846, 86, 899, 560, 1214, 2608, 224, 220, 830, 57, 647, 243, 1494, 523, 2792, 11, 389, 324] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=24 tokens_per_expert_current_ep_rank=[520, 1741, 101, 9, 1704, 449, 123, 558, 409, 69, 2117, 1264, 869, 564, 1130, 437, 69, 1624, 764, 526, 771, 1405, 1127, 4, 852, 1691, 238, 970, 201, 692, 4453, 400, 156, 709, 8, 1309, 674, 572, 605, 743, 1317, 1128, 67, 298, 946, 3450, 532, 26, 1307, 106, 441, 728, 3418, 86, 677, 458, 178, 118, 364, 3, 137, 3, 528, 209] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=24 tokens_per_expert_current_ep_rank=[2962, 790, 242, 68, 1419, 67, 412, 670, 245, 304, 409, 1144, 372, 2226, 2180, 434, 151, 1276, 99, 343, 80, 312, 528, 406, 77, 2327, 525, 327, 1583, 41, 3465, 440, 1, 914, 34, 152, 3992, 53, 59, 1256, 3240, 197, 5, 19, 60, 1063, 195, 197, 448, 1675, 1023, 842, 2660, 1445, 231, 7, 881, 410, 581, 641, 545, 19, 54, 329] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=24 rerouted_tokens_per_expert_current_ep_rank=[4104, 118, 281, 7, 1334, 197, 65, 755, 133, 60, 65, 0, 0, 0, 0, 0, 172, 451, 411, 347, 955, 341, 1325, 760, 0, 0, 0, 0, 0, 0, 0, 0, 527, 802, 184, 551, 1150, 216, 1479, 30, 456, 0, 0, 0, 0, 0, 0, 0, 2015, 1769, 28, 750, 47, 47, 1312, 769, 0, 0, 0, 0, 0, 0, 0, 0, 8, 414, 424, 479, 2349, 94, 62, 2072, 0, 0, 0, 0, 0, 0, 0, 0, 914, 213, 9, 676, 1161, 827, 692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 1102, 1345, 1476, 211, 1468, 4333, 48, 0, 0, 0, 0, 0, 0, 0, 0, 190, 105, 553, 77, 1077, 247, 23, 136, 1249, 493, 72, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7119, 4762, 5395, 6737, 5902, 4492, 10523, 4222] avg=6144.0000 max=10523 max_rank=6 min=4222 min_rank=7 imbalance_ratio=1.712728 max_load_reduction=38629 max_load_reduction_pct=0.785909 moved_to_echo=25139 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=24 rerouted_tokens_per_expert_current_ep_rank=[5246, 281, 107, 8, 2192, 22, 229, 133, 194, 37, 60, 0, 0, 0, 0, 0, 209, 112, 609, 1151, 165, 1590, 375, 238, 0, 0, 0, 0, 0, 0, 0, 0, 411, 3534, 794, 369, 383, 1230, 383, 276, 284, 0, 0, 0, 0, 0, 0, 0, 901, 373, 66, 886, 741, 230, 819, 802, 0, 0, 0, 0, 0, 0, 0, 0, 152, 570, 207, 189, 2164, 52, 527, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1660, 374, 7, 793, 2173, 1199, 516, 31, 0, 0, 0, 0, 0, 0, 0, 0, 365, 16, 726, 34, 303, 3042, 33, 732, 0, 0, 0, 0, 0, 0, 0, 0, 93, 59, 326, 351, 954, 103, 23, 45, 4148, 710, 45, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8509, 4449, 7664, 4818, 4851, 6753, 5251, 6857] avg=6144.0000 max=8509 max_rank=0 min=4449 min_rank=1 imbalance_ratio=1.384928 max_load_reduction=40643 max_load_reduction_pct=0.826884 moved_to_echo=23712 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=24 rerouted_tokens_per_expert_current_ep_rank=[156, 48, 232, 156, 2087, 267, 102, 306, 420, 73, 53, 0, 0, 0, 0, 0, 379, 199, 1640, 297, 748, 2551, 133, 581, 0, 0, 0, 0, 0, 0, 0, 0, 14, 3019, 1157, 178, 1203, 458, 2401, 506, 559, 0, 0, 0, 0, 0, 0, 0, 212, 937, 222, 717, 1157, 631, 1611, 722, 0, 0, 0, 0, 0, 0, 0, 0, 72, 1146, 20, 3037, 1925, 80, 98, 1122, 0, 0, 0, 0, 0, 0, 0, 0, 452, 279, 61, 552, 419, 2592, 2929, 183, 0, 0, 0, 0, 0, 0, 0, 0, 319, 109, 449, 144, 670, 78, 692, 429, 0, 0, 0, 0, 0, 0, 0, 0, 293, 471, 628, 580, 60, 14, 119, 270, 1067, 1572, 89, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3900, 6528, 9495, 6209, 7500, 7467, 2890, 5163] avg=6144.0000 max=9495 max_rank=2 min=2890 min_rank=6 imbalance_ratio=1.545410 max_load_reduction=39657 max_load_reduction_pct=0.806824 moved_to_echo=23020 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=24 rerouted_tokens_per_expert_current_ep_rank=[3638, 83, 41, 42, 806, 406, 73, 676, 324, 36, 38, 0, 0, 0, 0, 0, 1465, 1083, 881, 311, 1266, 1023, 2593, 296, 0, 0, 0, 0, 0, 0, 0, 0, 268, 1432, 269, 130, 941, 1210, 254, 555, 273, 0, 0, 0, 0, 0, 0, 0, 613, 1226, 132, 285, 121, 254, 787, 559, 0, 0, 0, 0, 0, 0, 0, 0, 215, 43, 559, 1272, 1393, 29, 107, 1649, 0, 0, 0, 0, 0, 0, 0, 0, 2078, 364, 4, 742, 1145, 1998, 707, 22, 0, 0, 0, 0, 0, 0, 0, 0, 1374, 219, 586, 1843, 75, 857, 889, 160, 0, 0, 0, 0, 0, 0, 0, 0, 76, 21, 639, 784, 2169, 584, 354, 470, 1119, 173, 43, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6163, 8918, 5332, 3977, 5267, 7060, 6003, 6432] avg=6144.0000 max=8918 max_rank=1 min=3977 min_rank=3 imbalance_ratio=1.451497 max_load_reduction=40234 max_load_reduction_pct=0.818563 moved_to_echo=24762 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=24 rerouted_tokens_per_expert_current_ep_rank=[2329, 259, 493, 19, 1156, 761, 139, 85, 175, 91, 74, 0, 0, 0, 0, 0, 385, 92, 1412, 330, 156, 1342, 977, 454, 0, 0, 0, 0, 0, 0, 0, 0, 849, 686, 993, 89, 437, 220, 1240, 141, 692, 0, 0, 0, 0, 0, 0, 0, 136, 388, 20, 1213, 57, 7, 1992, 1921, 0, 0, 0, 0, 0, 0, 0, 0, 1, 703, 668, 377, 2682, 532, 105, 3833, 0, 0, 0, 0, 0, 0, 0, 0, 294, 291, 4, 252, 158, 1080, 2092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 131, 2372, 655, 365, 1005, 1290, 639, 289, 0, 0, 0, 0, 0, 0, 0, 0, 75, 225, 774, 796, 753, 15, 237, 988, 1190, 2360, 110, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5581, 5148, 5347, 5734, 8901, 4172, 6746, 7523] avg=6144.0000 max=8901 max_rank=4 min=4172 min_rank=5 imbalance_ratio=1.448730 max_load_reduction=40251 max_load_reduction_pct=0.818909 moved_to_echo=27342 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=24 rerouted_tokens_per_expert_current_ep_rank=[1865, 23, 1336, 4, 1061, 149, 249, 130, 131, 52, 40, 0, 0, 0, 0, 0, 2345, 1714, 1093, 233, 545, 87, 143, 898, 0, 0, 0, 0, 0, 0, 0, 0, 470, 291, 457, 177, 63, 998, 952, 758, 369, 0, 0, 0, 0, 0, 0, 0, 58, 1089, 439, 135, 43, 980, 1039, 2063, 0, 0, 0, 0, 0, 0, 0, 0, 17, 178, 89, 2075, 1346, 23, 1, 1891, 0, 0, 0, 0, 0, 0, 0, 0, 782, 1240, 6, 109, 240, 788, 3846, 86, 0, 0, 0, 0, 0, 0, 0, 0, 899, 560, 1214, 2608, 62, 220, 830, 57, 0, 0, 0, 0, 0, 0, 0, 0, 647, 243, 1494, 523, 2792, 11, 389, 324, 858, 162, 63, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5040, 7058, 4535, 5846, 5620, 7097, 6450, 7506] avg=6144.0000 max=7506 max_rank=7 min=4535 min_rank=2 imbalance_ratio=1.221680 max_load_reduction=41646 max_load_reduction_pct=0.847290 moved_to_echo=26673 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=24 rerouted_tokens_per_expert_current_ep_rank=[520, 1741, 101, 9, 1704, 449, 123, 558, 481, 141, 18, 0, 0, 0, 0, 0, 409, 69, 2117, 275, 869, 564, 1130, 437, 0, 0, 0, 0, 0, 0, 0, 0, 69, 1624, 764, 526, 771, 1405, 1127, 4, 1068, 0, 0, 0, 0, 0, 0, 0, 852, 1691, 238, 970, 201, 692, 3074, 400, 0, 0, 0, 0, 0, 0, 0, 0, 156, 709, 8, 1309, 656, 572, 605, 743, 0, 0, 0, 0, 0, 0, 0, 0, 1317, 1128, 67, 298, 946, 2969, 532, 26, 0, 0, 0, 0, 0, 0, 0, 0, 1307, 106, 441, 728, 1021, 86, 677, 458, 0, 0, 0, 0, 0, 0, 0, 0, 178, 118, 364, 3, 137, 3, 528, 209, 989, 2397, 170, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5845, 5870, 7358, 8118, 4758, 7283, 4824, 5096] avg=6144.0000 max=8118 max_rank=3 min=4758 min_rank=4 imbalance_ratio=1.321289 max_load_reduction=41034 max_load_reduction_pct=0.834839 moved_to_echo=21961 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=24 rerouted_tokens_per_expert_current_ep_rank=[2962, 790, 242, 68, 1419, 67, 412, 670, 148, 109, 108, 0, 0, 0, 0, 0, 245, 304, 409, 249, 372, 2226, 2180, 434, 0, 0, 0, 0, 0, 0, 0, 0, 151, 1276, 99, 343, 80, 312, 528, 406, 831, 0, 0, 0, 0, 0, 0, 0, 77, 2327, 525, 327, 1583, 41, 2393, 440, 0, 0, 0, 0, 0, 0, 0, 0, 1, 914, 34, 152, 3884, 53, 59, 1256, 0, 0, 0, 0, 0, 0, 0, 0, 3240, 197, 5, 19, 60, 915, 195, 197, 0, 0, 0, 0, 0, 0, 0, 0, 448, 1675, 1023, 842, 794, 1445, 231, 7, 0, 0, 0, 0, 0, 0, 0, 0, 881, 410, 581, 641, 545, 19, 54, 329, 895, 1866, 132, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6995, 6419, 4026, 7713, 6353, 4828, 6465, 6353] avg=6144.0000 max=7713 max_rank=3 min=4026 min_rank=2 imbalance_ratio=1.255371 max_load_reduction=41439 max_load_reduction_pct=0.843079 moved_to_echo=23999 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[5632, 4096, 8576, 3200, 5120, 9728, 8960, 4224, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4864, 9856, 1792, 5376, 3968, 2944, 13056, 7680, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[5504, 6272, 6528, 8064, 4224, 8576, 8448, 2304, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2560, 1664, 5376, 3840, 8576, 1024, 1792, 2816, 11520, 9856, 768, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[20864, 3456, 2944, 384, 11776, 2432, 1408, 3328, 2048, 640, 512, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[10752, 4096, 256, 3456, 6400, 12416, 11520, 640, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[640, 4736, 2048, 8960, 16512, 1536, 1664, 13568, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2816, 12672, 4736, 2432, 5120, 6144, 8448, 2688, 4608, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True - [2026-08-26 14:53:43] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 848.2 | throughput per GPU (TFLOP/s/GPU): 150.4 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.092003E+01 | loss scale: 1.0 | grad norm: 30.566 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1283, 718, 575, 639, 614, 892, 1358, 163, 716, 774, 1282, 454, 125, 1376, 446, 938, 1041, 848, 689, 806, 625, 539, 1059, 548, 815, 783, 294, 368, 889, 1041, 478, 936, 1152, 442, 919, 815, 499, 385, 883, 774, 496, 590, 362, 774, 1388, 1550, 809, 321, 474, 716, 681, 852, 1173, 1668, 839, 574, 573, 981, 1217, 795, 563, 372, 1111, 292] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1403, 671, 540, 598, 471, 850, 1298, 236, 786, 1231, 1294, 434, 118, 1434, 423, 986, 1203, 812, 809, 832, 605, 506, 850, 571, 806, 847, 311, 476, 803, 1034, 466, 902, 867, 442, 737, 909, 417, 366, 851, 919, 500, 541, 384, 448, 1525, 1542, 742, 315, 611, 761, 738, 844, 1124, 1603, 820, 544, 617, 924, 1305, 755, 488, 436, 1210, 261] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1452, 729, 512, 511, 592, 911, 1263, 261, 634, 1194, 1207, 410, 170, 1281, 449, 1021, 1334, 852, 737, 842, 628, 490, 990, 543, 894, 742, 311, 447, 826, 1059, 435, 911, 768, 439, 769, 835, 431, 391, 932, 780, 411, 538, 424, 797, 1405, 1548, 805, 350, 631, 644, 783, 797, 1029, 1925, 732, 554, 495, 860, 1311, 733, 559, 416, 1176, 246] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1464, 694, 528, 606, 525, 864, 1325, 194, 825, 1133, 1294, 418, 152, 1278, 465, 985, 1017, 889, 750, 781, 649, 468, 1025, 572, 788, 814, 261, 488, 871, 1011, 376, 934, 984, 441, 873, 872, 533, 398, 947, 787, 456, 501, 353, 477, 1539, 1544, 829, 360, 548, 694, 765, 786, 1157, 1550, 828, 544, 473, 1025, 1366, 764, 520, 393, 1146, 255] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1402, 812, 527, 574, 544, 933, 1303, 251, 842, 1195, 1287, 527, 146, 1388, 434, 1020, 1115, 865, 750, 830, 585, 413, 979, 601, 842, 820, 304, 469, 876, 806, 457, 918, 863, 503, 788, 883, 513, 383, 876, 717, 456, 572, 329, 521, 1400, 1610, 828, 317, 528, 707, 867, 850, 1116, 1600, 746, 625, 522, 854, 1268, 789, 462, 427, 1164, 253] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1445, 737, 565, 530, 662, 808, 1413, 259, 792, 1060, 1291, 390, 142, 1423, 497, 914, 1017, 813, 798, 798, 652, 507, 1064, 664, 732, 845, 397, 483, 885, 1033, 458, 913, 783, 487, 738, 748, 552, 379, 879, 745, 512, 552, 383, 675, 1436, 1477, 623, 340, 655, 668, 725, 939, 964, 1593, 699, 452, 573, 971, 1309, 793, 649, 465, 1107, 294] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1377, 781, 691, 594, 607, 835, 1284, 180, 679, 861, 1238, 560, 157, 1400, 516, 1006, 1141, 908, 698, 861, 614, 476, 1001, 532, 709, 804, 290, 409, 882, 961, 381, 973, 863, 505, 758, 910, 478, 381, 1006, 849, 482, 543, 379, 489, 1595, 1589, 795, 353, 467, 642, 741, 884, 1066, 1574, 717, 534, 456, 1077, 1388, 804, 512, 415, 1182, 312] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1471, 689, 494, 636, 630, 927, 1365, 188, 644, 1199, 1198, 459, 137, 1317, 483, 961, 1160, 888, 750, 864, 589, 484, 951, 594, 789, 685, 318, 394, 836, 1115, 404, 950, 1014, 435, 725, 887, 437, 375, 980, 685, 454, 565, 380, 511, 1373, 1494, 748, 356, 607, 745, 731, 844, 1001, 1704, 800, 552, 551, 1035, 1341, 851, 501, 390, 1189, 322] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1267, 671, 540, 598, 471, 850, 1298, 236, 0, 0, 0, 0, 0, 0, 0, 0, 786, 1231, 1294, 434, 118, 1073, 423, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1108, 812, 809, 832, 605, 506, 850, 571, 0, 0, 0, 0, 0, 0, 0, 0, 806, 847, 311, 476, 803, 1034, 466, 902, 557, 0, 0, 0, 0, 0, 0, 0, 867, 442, 737, 909, 417, 366, 851, 919, 361, 191, 17, 0, 0, 0, 0, 0, 500, 541, 384, 448, 1525, 1542, 742, 315, 52, 0, 0, 0, 0, 0, 0, 0, 611, 761, 738, 844, 1124, 855, 820, 544, 0, 0, 0, 0, 0, 0, 0, 0, 617, 924, 1305, 755, 488, 436, 1210, 261, 119, 43, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5931, 6345, 6093, 6202, 6077, 6049, 6297, 6158] avg=6144.0000 max=6345 max_rank=1 min=5931 min_rank=0 imbalance_ratio=1.032715 max_load_reduction=42807 max_load_reduction_pct=0.870911 moved_to_echo=24581 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1311, 729, 512, 511, 592, 911, 1263, 261, 0, 0, 0, 0, 0, 0, 0, 0, 634, 1194, 1207, 410, 170, 959, 449, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1229, 852, 737, 842, 628, 490, 990, 543, 0, 0, 0, 0, 0, 0, 0, 0, 894, 742, 311, 447, 826, 1059, 435, 911, 669, 0, 0, 0, 0, 0, 0, 0, 768, 439, 769, 835, 431, 391, 932, 780, 322, 230, 18, 0, 0, 0, 0, 0, 411, 538, 424, 797, 1405, 1548, 805, 350, 57, 0, 0, 0, 0, 0, 0, 0, 631, 644, 783, 797, 1029, 1026, 732, 554, 0, 0, 0, 0, 0, 0, 0, 0, 495, 860, 1311, 733, 559, 416, 1176, 246, 123, 48, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6090, 6044, 6311, 6294, 5915, 6335, 6196, 5967] avg=6144.0000 max=6335 max_rank=5 min=5915 min_rank=4 imbalance_ratio=1.031087 max_load_reduction=42817 max_load_reduction_pct=0.871114 moved_to_echo=24413 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1159, 718, 575, 639, 614, 892, 1358, 163, 0, 0, 0, 0, 0, 0, 0, 0, 716, 774, 1282, 454, 125, 1030, 446, 938, 0, 0, 0, 0, 0, 0, 0, 0, 959, 848, 689, 806, 625, 539, 1059, 548, 0, 0, 0, 0, 0, 0, 0, 0, 815, 783, 294, 368, 889, 1041, 478, 936, 580, 0, 0, 0, 0, 0, 0, 0, 1152, 442, 919, 815, 499, 385, 883, 774, 346, 199, 16, 0, 0, 0, 0, 0, 496, 590, 362, 774, 1388, 1550, 809, 321, 45, 0, 0, 0, 0, 0, 0, 0, 474, 716, 681, 852, 1173, 889, 839, 574, 0, 0, 0, 0, 0, 0, 0, 0, 573, 981, 1217, 795, 563, 372, 1111, 292, 108, 37, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6118, 5765, 6073, 6184, 6430, 6335, 6198, 6049] avg=6144.0000 max=6430 max_rank=4 min=5765 min_rank=1 imbalance_ratio=1.046549 max_load_reduction=42722 max_load_reduction_pct=0.869181 moved_to_echo=25012 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1322, 694, 528, 606, 525, 864, 1325, 194, 0, 0, 0, 0, 0, 0, 0, 0, 825, 1133, 1294, 418, 152, 957, 465, 985, 0, 0, 0, 0, 0, 0, 0, 0, 936, 889, 750, 781, 649, 468, 1025, 572, 0, 0, 0, 0, 0, 0, 0, 0, 788, 814, 261, 488, 871, 1011, 376, 934, 539, 0, 0, 0, 0, 0, 0, 0, 984, 441, 873, 872, 533, 398, 947, 787, 321, 185, 18, 0, 0, 0, 0, 0, 456, 501, 353, 477, 1539, 1544, 829, 360, 44, 0, 0, 0, 0, 0, 0, 0, 548, 694, 765, 786, 1157, 826, 828, 544, 0, 0, 0, 0, 0, 0, 0, 0, 473, 1025, 1366, 764, 520, 393, 1146, 255, 124, 37, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6058, 6229, 6070, 6082, 6359, 6103, 6148, 6103] avg=6144.0000 max=6359 max_rank=4 min=6058 min_rank=0 imbalance_ratio=1.034993 max_load_reduction=42793 max_load_reduction_pct=0.870626 moved_to_echo=24713 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1266, 812, 527, 574, 544, 933, 1303, 251, 0, 0, 0, 0, 0, 0, 0, 0, 842, 1195, 1287, 527, 146, 1039, 434, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 865, 750, 830, 585, 413, 979, 601, 0, 0, 0, 0, 0, 0, 0, 0, 842, 820, 304, 469, 876, 806, 457, 918, 556, 0, 0, 0, 0, 0, 0, 0, 863, 503, 788, 883, 513, 383, 876, 717, 349, 191, 17, 0, 0, 0, 0, 0, 456, 572, 329, 521, 1400, 1610, 828, 317, 48, 0, 0, 0, 0, 0, 0, 0, 528, 707, 867, 850, 1116, 853, 746, 625, 0, 0, 0, 0, 0, 0, 0, 0, 522, 854, 1268, 789, 462, 427, 1164, 253, 119, 40, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6210, 6490, 6050, 6048, 6083, 6081, 6292, 5898] avg=6144.0000 max=6490 max_rank=1 min=5898 min_rank=7 imbalance_ratio=1.056315 max_load_reduction=42662 max_load_reduction_pct=0.867961 moved_to_echo=24354 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1236, 781, 691, 594, 607, 835, 1284, 180, 0, 0, 0, 0, 0, 0, 0, 0, 679, 861, 1238, 560, 157, 1044, 516, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 908, 698, 861, 614, 476, 1001, 532, 0, 0, 0, 0, 0, 0, 0, 0, 709, 804, 290, 409, 882, 961, 381, 973, 550, 0, 0, 0, 0, 0, 0, 0, 863, 505, 758, 910, 478, 381, 1006, 849, 356, 193, 21, 0, 0, 0, 0, 0, 482, 543, 379, 489, 1595, 1589, 795, 353, 51, 0, 0, 0, 0, 0, 0, 0, 467, 642, 741, 884, 1066, 831, 717, 534, 0, 0, 0, 0, 0, 0, 0, 0, 456, 1077, 1388, 804, 512, 415, 1182, 312, 120, 45, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6208, 6061, 6135, 5959, 6320, 6276, 5882, 6311] avg=6144.0000 max=6320 max_rank=4 min=5882 min_rank=6 imbalance_ratio=1.028646 max_load_reduction=42832 max_load_reduction_pct=0.871419 moved_to_echo=24789 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1305, 737, 565, 530, 662, 808, 1413, 259, 0, 0, 0, 0, 0, 0, 0, 0, 792, 1060, 1291, 390, 142, 1065, 497, 914, 0, 0, 0, 0, 0, 0, 0, 0, 936, 813, 798, 798, 652, 507, 1064, 664, 0, 0, 0, 0, 0, 0, 0, 0, 732, 845, 397, 483, 885, 1033, 458, 913, 554, 0, 0, 0, 0, 0, 0, 0, 783, 487, 738, 748, 552, 379, 879, 745, 358, 190, 18, 0, 0, 0, 0, 0, 512, 552, 383, 675, 1436, 1477, 623, 340, 44, 0, 0, 0, 0, 0, 0, 0, 655, 668, 725, 939, 964, 849, 699, 452, 0, 0, 0, 0, 0, 0, 0, 0, 573, 971, 1309, 793, 649, 465, 1107, 294, 122, 37, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6279, 6151, 6232, 6300, 5877, 6042, 5951, 6320] avg=6144.0000 max=6320 max_rank=7 min=5877 min_rank=4 imbalance_ratio=1.028646 max_load_reduction=42832 max_load_reduction_pct=0.871419 moved_to_echo=24190 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1329, 689, 494, 636, 630, 927, 1365, 188, 0, 0, 0, 0, 0, 0, 0, 0, 644, 1199, 1198, 459, 137, 986, 483, 961, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 888, 750, 864, 589, 484, 951, 594, 0, 0, 0, 0, 0, 0, 0, 0, 789, 685, 318, 394, 836, 1115, 404, 950, 592, 0, 0, 0, 0, 0, 0, 0, 1014, 435, 725, 887, 437, 375, 980, 685, 331, 204, 18, 0, 0, 0, 0, 0, 454, 565, 380, 511, 1373, 1494, 748, 356, 50, 0, 0, 0, 0, 0, 0, 0, 607, 745, 731, 844, 1001, 908, 800, 552, 0, 0, 0, 0, 0, 0, 0, 0, 551, 1035, 1341, 851, 501, 390, 1189, 322, 124, 42, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6258, 6067, 6188, 6083, 6091, 5931, 6188, 6346] avg=6144.0000 max=6346 max_rank=7 min=5931 min_rank=5 imbalance_ratio=1.032878 max_load_reduction=42806 max_load_reduction_pct=0.870890 moved_to_echo=24556 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6016, 8704, 10112, 3712, 1152, 8192, 3840, 7936, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[6400, 6400, 2560, 3584, 6912, 8064, 3456, 7552, 4608, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[7296, 3712, 6400, 6912, 3968, 3072, 7424, 6272, 2816, 1664, 256, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3840, 4480, 3072, 4736, 11776, 12416, 6272, 2816, 512, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[10240, 5888, 4480, 4736, 4736, 7040, 10624, 1792, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4608, 5632, 6144, 6912, 8704, 7040, 6272, 4480, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[8320, 6912, 6016, 6656, 4992, 3968, 7936, 4736, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[4352, 7808, 10624, 6400, 4352, 3328, 9344, 2304, 1024, 384, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[262, 440, 354, 616, 1291, 654, 846, 600, 267, 348, 1178, 242, 1126, 573, 947, 2065, 933, 1206, 985, 716, 872, 938, 1501, 624, 622, 771, 963, 1811, 252, 825, 502, 628, 353, 318, 414, 548, 909, 321, 390, 898, 1683, 609, 51, 749, 395, 470, 38, 640, 3113, 389, 1458, 121, 765, 1008, 783, 800, 589, 930, 756, 508, 1648, 401, 682, 457] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[257, 531, 731, 567, 1250, 581, 974, 644, 298, 431, 1140, 293, 1085, 560, 715, 1927, 892, 1207, 870, 306, 916, 1224, 1403, 554, 679, 807, 974, 1784, 221, 859, 523, 651, 267, 341, 405, 704, 1210, 282, 300, 978, 1519, 808, 41, 661, 259, 505, 64, 618, 3212, 514, 1451, 128, 830, 1113, 774, 896, 408, 816, 785, 564, 1501, 396, 506, 442] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[183, 606, 195, 603, 1337, 676, 892, 634, 306, 312, 1153, 221, 1078, 547, 857, 2278, 946, 1245, 888, 424, 948, 1044, 1283, 773, 622, 767, 781, 1852, 163, 786, 440, 720, 353, 310, 436, 558, 1092, 279, 401, 808, 1576, 824, 29, 836, 440, 513, 52, 622, 3293, 460, 1322, 92, 723, 1146, 794, 855, 560, 862, 715, 490, 1844, 371, 464, 472] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[203, 424, 332, 530, 1281, 642, 950, 590, 260, 267, 1238, 317, 1153, 576, 852, 2286, 810, 1127, 895, 566, 908, 1030, 1468, 699, 602, 729, 838, 1837, 170, 882, 517, 691, 287, 326, 491, 601, 962, 252, 392, 921, 1569, 793, 37, 733, 343, 458, 41, 799, 3221, 441, 1388, 109, 790, 1143, 842, 720, 619, 902, 674, 502, 1714, 437, 516, 459] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[252, 369, 687, 537, 1216, 572, 933, 533, 290, 327, 859, 254, 1193, 463, 901, 2218, 849, 1285, 970, 744, 882, 900, 1341, 691, 648, 777, 893, 1900, 263, 878, 536, 573, 319, 339, 463, 569, 1061, 224, 329, 983, 1650, 569, 20, 832, 385, 507, 39, 878, 3274, 393, 1395, 63, 591, 1150, 758, 788, 565, 960, 567, 585, 1730, 384, 613, 435] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[212, 442, 462, 498, 1236, 753, 931, 604, 247, 386, 1235, 204, 1256, 630, 913, 2239, 832, 1197, 899, 460, 905, 917, 1216, 664, 594, 810, 925, 1894, 183, 843, 497, 552, 392, 395, 486, 604, 1087, 362, 387, 928, 1529, 784, 39, 769, 333, 486, 52, 758, 3054, 342, 1380, 97, 707, 1085, 792, 889, 502, 954, 497, 637, 1822, 446, 532, 389] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[280, 522, 341, 550, 1233, 562, 935, 485, 268, 360, 1322, 244, 1199, 587, 863, 2263, 838, 1356, 930, 519, 997, 962, 1429, 703, 656, 842, 903, 1721, 224, 873, 532, 649, 367, 322, 484, 655, 1020, 335, 329, 927, 1472, 798, 34, 722, 323, 449, 48, 857, 3002, 433, 1221, 152, 615, 1037, 812, 944, 534, 932, 483, 485, 1849, 373, 540, 450] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[228, 423, 488, 586, 1282, 521, 794, 533, 314, 402, 1239, 321, 1236, 614, 895, 2013, 1008, 1263, 1012, 966, 1008, 1079, 1305, 591, 595, 797, 862, 1996, 236, 859, 552, 463, 392, 342, 468, 509, 1009, 260, 374, 982, 1618, 618, 55, 772, 298, 432, 26, 599, 3010, 434, 1415, 133, 675, 981, 708, 870, 607, 878, 660, 502, 1546, 390, 621, 487] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[262, 440, 354, 616, 1291, 654, 846, 600, 568, 510, 0, 0, 0, 0, 0, 0, 267, 348, 1178, 242, 1126, 573, 947, 1409, 0, 0, 0, 0, 0, 0, 0, 0, 933, 1106, 985, 716, 872, 938, 0, 624, 0, 0, 0, 0, 0, 0, 0, 0, 622, 771, 963, 1610, 252, 825, 502, 628, 0, 0, 0, 0, 0, 0, 0, 0, 353, 318, 414, 548, 909, 321, 390, 898, 1773, 0, 0, 0, 0, 0, 0, 0, 1683, 609, 51, 749, 395, 470, 38, 640, 991, 543, 0, 0, 0, 0, 0, 0, 797, 389, 1458, 121, 765, 1008, 783, 800, 0, 0, 0, 0, 0, 0, 0, 0, 589, 930, 756, 508, 1648, 401, 682, 457, 201, 100, 88, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6141, 6090, 6174, 6173, 5924, 6169, 6121, 6360] avg=6144.0000 max=6360 max_rank=7 min=5924 min_rank=4 imbalance_ratio=1.035156 max_load_reduction=42792 max_load_reduction_pct=0.870605 moved_to_echo=24574 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[257, 531, 731, 567, 1250, 581, 974, 644, 530, 477, 0, 0, 0, 0, 0, 0, 298, 431, 1140, 293, 1085, 560, 715, 1315, 0, 0, 0, 0, 0, 0, 0, 0, 892, 1107, 870, 306, 916, 1224, 0, 554, 0, 0, 0, 0, 0, 0, 0, 0, 679, 807, 974, 1586, 221, 859, 523, 651, 0, 0, 0, 0, 0, 0, 0, 0, 267, 341, 405, 704, 1210, 282, 300, 978, 1829, 0, 0, 0, 0, 0, 0, 0, 1519, 808, 41, 661, 259, 505, 64, 618, 926, 561, 0, 0, 0, 0, 0, 0, 822, 514, 1451, 128, 830, 1113, 774, 896, 0, 0, 0, 0, 0, 0, 0, 0, 408, 816, 785, 564, 1501, 396, 506, 442, 198, 100, 82, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6542, 5837, 5869, 6300, 6316, 5962, 6528, 5798] avg=6144.0000 max=6542 max_rank=0 min=5798 min_rank=7 imbalance_ratio=1.064779 max_load_reduction=42610 max_load_reduction_pct=0.866903 moved_to_echo=24604 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[203, 424, 332, 530, 1281, 642, 950, 590, 628, 498, 0, 0, 0, 0, 0, 0, 260, 267, 1238, 317, 1153, 576, 852, 1560, 0, 0, 0, 0, 0, 0, 0, 0, 810, 1034, 895, 566, 908, 1030, 0, 699, 0, 0, 0, 0, 0, 0, 0, 0, 602, 729, 838, 1633, 170, 882, 517, 691, 0, 0, 0, 0, 0, 0, 0, 0, 287, 326, 491, 601, 962, 252, 392, 921, 1834, 0, 0, 0, 0, 0, 0, 0, 1569, 793, 37, 733, 343, 458, 41, 799, 970, 562, 0, 0, 0, 0, 0, 0, 825, 441, 1388, 109, 790, 1143, 842, 720, 0, 0, 0, 0, 0, 0, 0, 0, 619, 902, 674, 502, 1714, 437, 516, 459, 204, 93, 98, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6078, 6223, 5942, 6062, 6066, 6305, 6258, 6218] avg=6144.0000 max=6305 max_rank=5 min=5942 min_rank=2 imbalance_ratio=1.026204 max_load_reduction=42847 max_load_reduction_pct=0.871724 moved_to_echo=24847 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[252, 369, 687, 537, 1216, 572, 933, 533, 610, 455, 0, 0, 0, 0, 0, 0, 290, 327, 859, 254, 1193, 463, 901, 1513, 0, 0, 0, 0, 0, 0, 0, 0, 849, 1179, 970, 744, 882, 900, 0, 691, 0, 0, 0, 0, 0, 0, 0, 0, 648, 777, 893, 1690, 263, 878, 536, 573, 0, 0, 0, 0, 0, 0, 0, 0, 319, 339, 463, 569, 1061, 224, 329, 983, 1864, 0, 0, 0, 0, 0, 0, 0, 1650, 569, 20, 832, 385, 507, 39, 878, 886, 572, 0, 0, 0, 0, 0, 0, 838, 393, 1395, 63, 591, 1150, 758, 788, 0, 0, 0, 0, 0, 0, 0, 0, 565, 960, 567, 585, 1730, 384, 613, 435, 210, 106, 95, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6164, 5800, 6215, 6258, 6151, 6338, 5976, 6250] avg=6144.0000 max=6338 max_rank=5 min=5800 min_rank=1 imbalance_ratio=1.031576 max_load_reduction=42814 max_load_reduction_pct=0.871053 moved_to_echo=24715 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[212, 442, 462, 498, 1236, 753, 931, 604, 615, 414, 0, 0, 0, 0, 0, 0, 247, 386, 1235, 204, 1256, 630, 913, 1528, 0, 0, 0, 0, 0, 0, 0, 0, 832, 1098, 899, 460, 905, 917, 0, 664, 0, 0, 0, 0, 0, 0, 0, 0, 594, 810, 925, 1684, 183, 843, 497, 552, 0, 0, 0, 0, 0, 0, 0, 0, 392, 395, 486, 604, 1087, 362, 387, 928, 1739, 0, 0, 0, 0, 0, 0, 0, 1529, 784, 39, 769, 333, 486, 52, 758, 802, 533, 0, 0, 0, 0, 0, 0, 782, 342, 1380, 97, 707, 1085, 792, 889, 0, 0, 0, 0, 0, 0, 0, 0, 502, 954, 497, 637, 1822, 446, 532, 389, 210, 99, 96, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6167, 6399, 5775, 6088, 6380, 6085, 6074, 6184] avg=6144.0000 max=6399 max_rank=1 min=5775 min_rank=2 imbalance_ratio=1.041504 max_load_reduction=42753 max_load_reduction_pct=0.869812 moved_to_echo=24723 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[280, 522, 341, 550, 1233, 562, 935, 485, 626, 486, 0, 0, 0, 0, 0, 0, 268, 360, 1322, 244, 1199, 587, 863, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 838, 1240, 930, 519, 997, 962, 0, 703, 0, 0, 0, 0, 0, 0, 0, 0, 656, 842, 903, 1527, 224, 873, 532, 649, 0, 0, 0, 0, 0, 0, 0, 0, 367, 322, 484, 655, 1020, 335, 329, 927, 1714, 0, 0, 0, 0, 0, 0, 0, 1472, 798, 34, 722, 323, 449, 48, 857, 943, 529, 0, 0, 0, 0, 0, 0, 759, 433, 1221, 152, 615, 1037, 812, 944, 0, 0, 0, 0, 0, 0, 0, 0, 534, 932, 483, 485, 1849, 373, 540, 450, 194, 116, 101, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6020, 6379, 6189, 6206, 6153, 6175, 5973, 6057] avg=6144.0000 max=6379 max_rank=1 min=5973 min_rank=6 imbalance_ratio=1.038249 max_load_reduction=42773 max_load_reduction_pct=0.870219 moved_to_echo=24358 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[228, 423, 488, 586, 1282, 521, 794, 533, 553, 444, 0, 0, 0, 0, 0, 0, 314, 402, 1239, 321, 1236, 614, 895, 1374, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1159, 1012, 966, 1008, 1079, 0, 591, 0, 0, 0, 0, 0, 0, 0, 0, 595, 797, 862, 1775, 236, 859, 552, 463, 0, 0, 0, 0, 0, 0, 0, 0, 392, 342, 468, 509, 1009, 260, 374, 982, 1714, 0, 0, 0, 0, 0, 0, 0, 1618, 618, 55, 772, 298, 432, 26, 599, 861, 525, 0, 0, 0, 0, 0, 0, 771, 434, 1415, 133, 675, 981, 708, 870, 0, 0, 0, 0, 0, 0, 0, 0, 607, 878, 660, 502, 1546, 390, 621, 487, 221, 104, 86, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5852, 6395, 6823, 6139, 6050, 5804, 5987, 6102] avg=6144.0000 max=6823 max_rank=2 min=5804 min_rank=5 imbalance_ratio=1.110514 max_load_reduction=42329 max_load_reduction_pct=0.861186 moved_to_echo=23943 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[183, 606, 195, 603, 1337, 676, 892, 634, 626, 436, 0, 0, 0, 0, 0, 0, 306, 312, 1153, 221, 1078, 547, 857, 1555, 0, 0, 0, 0, 0, 0, 0, 0, 946, 1142, 888, 424, 948, 1044, 0, 773, 0, 0, 0, 0, 0, 0, 0, 0, 622, 767, 781, 1647, 163, 786, 440, 720, 0, 0, 0, 0, 0, 0, 0, 0, 353, 310, 436, 558, 1092, 279, 401, 808, 1875, 0, 0, 0, 0, 0, 0, 0, 1576, 824, 29, 836, 440, 513, 52, 622, 847, 575, 0, 0, 0, 0, 0, 0, 843, 460, 1322, 92, 723, 1146, 794, 855, 0, 0, 0, 0, 0, 0, 0, 0, 560, 862, 715, 490, 1844, 371, 464, 472, 205, 103, 97, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6188, 6029, 6165, 5926, 6112, 6314, 6235, 6183] avg=6144.0000 max=6314 max_rank=5 min=5926 min_rank=3 imbalance_ratio=1.027669 max_load_reduction=42838 max_load_reduction_pct=0.871541 moved_to_echo=24844 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5120, 6400, 7168, 13184, 1792, 6912, 4224, 4992, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[6528, 3456, 11136, 896, 5760, 8704, 6272, 6784, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7168, 9088, 7552, 4736, 7552, 8192, 0, 5376, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[12672, 5888, 384, 6144, 2816, 3840, 384, 5888, 7296, 4480, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[1920, 3840, 3712, 4608, 10240, 4992, 7296, 4736, 4864, 3840, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2304, 2944, 9472, 2176, 9344, 4608, 7040, 11904, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4480, 7296, 5248, 4352, 13696, 3200, 4480, 3712, 1664, 896, 768, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[2816, 2816, 3712, 4864, 8448, 2432, 2944, 7552, 14464, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[149, 807, 112, 444, 282, 533, 284, 1960, 36, 1187, 259, 1745, 1035, 1252, 285, 676, 1913, 609, 649, 813, 538, 380, 1373, 206, 509, 1689, 2857, 1161, 668, 534, 353, 93, 561, 684, 1709, 1291, 1618, 1070, 1822, 417, 317, 985, 121, 127, 1432, 1715, 139, 572, 962, 963, 704, 327, 564, 552, 180, 135, 1369, 543, 295, 326, 256, 734, 590, 681] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[142, 802, 98, 368, 260, 878, 252, 1839, 26, 1114, 230, 1895, 668, 1375, 310, 563, 1807, 516, 608, 1125, 420, 296, 1782, 173, 431, 1535, 2775, 1092, 692, 596, 216, 107, 918, 632, 1971, 1209, 1914, 1015, 1863, 429, 373, 768, 117, 123, 1320, 1681, 107, 730, 946, 775, 637, 299, 488, 351, 176, 187, 1522, 622, 297, 413, 274, 764, 545, 695] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[101, 733, 86, 433, 308, 540, 245, 2080, 25, 1204, 224, 2005, 734, 1258, 411, 650, 2104, 504, 519, 1016, 528, 307, 1627, 160, 510, 1644, 2999, 1268, 670, 521, 411, 97, 563, 585, 1887, 1253, 1596, 732, 1885, 434, 318, 784, 136, 168, 1134, 1918, 201, 464, 727, 988, 635, 338, 534, 599, 196, 130, 1502, 593, 262, 406, 292, 746, 522, 702] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[107, 744, 138, 454, 323, 437, 252, 1374, 33, 983, 258, 1794, 541, 1699, 358, 759, 1927, 362, 730, 1041, 342, 289, 1608, 203, 454, 1411, 2786, 1033, 607, 663, 294, 156, 691, 633, 2036, 1174, 1809, 960, 1940, 557, 245, 967, 174, 113, 1529, 1713, 203, 409, 947, 888, 630, 456, 643, 543, 216, 225, 1680, 444, 298, 337, 381, 687, 705, 759] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[119, 861, 127, 380, 256, 568, 248, 1710, 30, 1402, 304, 1853, 660, 1268, 369, 796, 2059, 518, 676, 969, 489, 306, 1329, 203, 532, 1487, 2972, 1272, 747, 402, 371, 91, 726, 654, 1782, 1093, 1771, 1061, 1693, 501, 318, 974, 122, 132, 1431, 1831, 172, 465, 721, 1034, 758, 383, 550, 293, 178, 199, 1414, 572, 265, 384, 351, 771, 536, 643] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[91, 931, 122, 432, 316, 499, 282, 1496, 58, 1172, 204, 1834, 570, 1400, 374, 656, 1877, 542, 648, 952, 551, 319, 1402, 172, 481, 1668, 2881, 1112, 921, 494, 352, 118, 865, 723, 1926, 1069, 1769, 784, 1901, 531, 338, 911, 128, 171, 1284, 1652, 174, 447, 830, 912, 663, 462, 514, 452, 198, 163, 1462, 538, 306, 485, 459, 725, 565, 818] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[121, 827, 74, 413, 265, 525, 231, 2101, 32, 1242, 282, 1732, 732, 1109, 400, 574, 2327, 440, 561, 1029, 454, 271, 1569, 151, 553, 1628, 3054, 1237, 762, 488, 348, 134, 675, 723, 1773, 1374, 1476, 955, 1983, 428, 309, 915, 116, 137, 1360, 1707, 129, 439, 775, 1037, 714, 300, 584, 442, 196, 184, 1316, 554, 256, 317, 381, 796, 535, 600] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[83, 711, 78, 433, 260, 513, 272, 1981, 28, 960, 244, 2024, 650, 1280, 352, 600, 1902, 547, 655, 929, 367, 305, 1475, 123, 511, 1799, 3072, 1158, 647, 428, 341, 107, 772, 709, 1725, 1191, 1931, 714, 1791, 460, 262, 1055, 155, 136, 1427, 1934, 107, 529, 912, 965, 769, 296, 477, 426, 153, 111, 1673, 558, 266, 437, 306, 772, 547, 751] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[149, 807, 112, 444, 282, 533, 284, 1960, 1607, 126, 0, 0, 0, 0, 0, 0, 36, 1187, 259, 1554, 1035, 1252, 285, 676, 0, 0, 0, 0, 0, 0, 0, 0, 1500, 609, 649, 813, 538, 380, 1373, 206, 0, 0, 0, 0, 0, 0, 0, 0, 509, 1689, 1167, 1161, 668, 534, 353, 93, 0, 0, 0, 0, 0, 0, 0, 0, 561, 684, 400, 1291, 1618, 1070, 0, 417, 0, 0, 0, 0, 0, 0, 0, 0, 317, 985, 121, 127, 1432, 1715, 139, 572, 413, 197, 191, 0, 0, 0, 0, 0, 962, 963, 704, 327, 564, 552, 180, 135, 1822, 83, 0, 0, 0, 0, 0, 0, 1369, 543, 295, 326, 256, 734, 590, 681, 986, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6304, 6284, 6068, 6174, 6041, 6209, 6292, 5780] avg=6144.0000 max=6304 max_rank=0 min=5780 min_rank=7 imbalance_ratio=1.026042 max_load_reduction=42848 max_load_reduction_pct=0.871745 moved_to_echo=24322 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[142, 802, 98, 368, 260, 878, 252, 1839, 1561, 145, 0, 0, 0, 0, 0, 0, 26, 1114, 230, 1688, 668, 1375, 310, 563, 0, 0, 0, 0, 0, 0, 0, 0, 1417, 516, 608, 1125, 420, 296, 1782, 173, 0, 0, 0, 0, 0, 0, 0, 0, 431, 1535, 1134, 1092, 692, 596, 216, 107, 0, 0, 0, 0, 0, 0, 0, 0, 918, 632, 461, 1209, 1914, 1015, 0, 429, 0, 0, 0, 0, 0, 0, 0, 0, 373, 768, 117, 123, 1320, 1681, 107, 730, 390, 227, 207, 0, 0, 0, 0, 0, 946, 775, 637, 299, 488, 351, 176, 187, 1863, 80, 0, 0, 0, 0, 0, 0, 1522, 622, 297, 413, 274, 764, 545, 695, 1138, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6345, 5974, 6337, 5803, 6578, 6043, 5802, 6270] avg=6144.0000 max=6578 max_rank=4 min=5802 min_rank=6 imbalance_ratio=1.070638 max_load_reduction=42574 max_load_reduction_pct=0.866170 moved_to_echo=24693 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[107, 744, 138, 454, 323, 437, 252, 1374, 1567, 150, 0, 0, 0, 0, 0, 0, 33, 983, 258, 1598, 541, 1699, 358, 759, 0, 0, 0, 0, 0, 0, 0, 0, 1511, 362, 730, 1041, 342, 289, 1608, 203, 0, 0, 0, 0, 0, 0, 0, 0, 454, 1411, 1138, 1033, 607, 663, 294, 156, 0, 0, 0, 0, 0, 0, 0, 0, 691, 633, 476, 1174, 1809, 960, 0, 557, 0, 0, 0, 0, 0, 0, 0, 0, 245, 967, 174, 113, 1529, 1713, 203, 409, 416, 235, 196, 0, 0, 0, 0, 0, 947, 888, 630, 456, 643, 543, 216, 225, 1940, 81, 0, 0, 0, 0, 0, 0, 1680, 444, 298, 337, 381, 687, 705, 759, 1175, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5546, 6229, 6086, 5756, 6300, 6200, 6569, 6466] avg=6144.0000 max=6569 max_rank=6 min=5546 min_rank=0 imbalance_ratio=1.069173 max_load_reduction=42583 max_load_reduction_pct=0.866353 moved_to_echo=25535 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[101, 733, 86, 433, 308, 540, 245, 2080, 1687, 139, 0, 0, 0, 0, 0, 0, 25, 1204, 224, 1786, 734, 1258, 411, 650, 0, 0, 0, 0, 0, 0, 0, 0, 1650, 504, 519, 1016, 528, 307, 1627, 160, 0, 0, 0, 0, 0, 0, 0, 0, 510, 1644, 1225, 1268, 670, 521, 411, 97, 0, 0, 0, 0, 0, 0, 0, 0, 563, 585, 441, 1253, 1596, 732, 0, 434, 0, 0, 0, 0, 0, 0, 0, 0, 318, 784, 136, 168, 1134, 1918, 201, 464, 454, 218, 219, 0, 0, 0, 0, 0, 727, 988, 635, 338, 534, 599, 196, 130, 1885, 87, 0, 0, 0, 0, 0, 0, 1502, 593, 262, 406, 292, 746, 522, 702, 1089, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6352, 6292, 6311, 6346, 5604, 6014, 6119, 6114] avg=6144.0000 max=6352 max_rank=0 min=5604 min_rank=4 imbalance_ratio=1.033854 max_load_reduction=42800 max_load_reduction_pct=0.870768 moved_to_echo=23851 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[121, 827, 74, 413, 265, 525, 231, 2101, 1718, 131, 0, 0, 0, 0, 0, 0, 32, 1242, 282, 1543, 732, 1109, 400, 574, 0, 0, 0, 0, 0, 0, 0, 0, 1825, 440, 561, 1029, 454, 271, 1569, 151, 0, 0, 0, 0, 0, 0, 0, 0, 553, 1628, 1247, 1237, 762, 488, 348, 134, 0, 0, 0, 0, 0, 0, 0, 0, 675, 723, 415, 1374, 1476, 955, 0, 428, 0, 0, 0, 0, 0, 0, 0, 0, 309, 915, 116, 137, 1360, 1707, 129, 439, 502, 204, 189, 0, 0, 0, 0, 0, 775, 1037, 714, 300, 584, 442, 196, 184, 1983, 89, 0, 0, 0, 0, 0, 0, 1316, 554, 256, 317, 381, 796, 535, 600, 1023, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6406, 5914, 6300, 6397, 6046, 6007, 6304, 5778] avg=6144.0000 max=6406 max_rank=0 min=5778 min_rank=7 imbalance_ratio=1.042643 max_load_reduction=42746 max_load_reduction_pct=0.869670 moved_to_echo=24135 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[91, 931, 122, 432, 316, 499, 282, 1496, 1626, 146, 0, 0, 0, 0, 0, 0, 58, 1172, 204, 1629, 570, 1400, 374, 656, 0, 0, 0, 0, 0, 0, 0, 0, 1468, 542, 648, 952, 551, 319, 1402, 172, 0, 0, 0, 0, 0, 0, 0, 0, 481, 1668, 1168, 1112, 921, 494, 352, 118, 0, 0, 0, 0, 0, 0, 0, 0, 865, 723, 438, 1069, 1769, 784, 0, 531, 0, 0, 0, 0, 0, 0, 0, 0, 338, 911, 128, 171, 1284, 1652, 174, 447, 409, 226, 205, 0, 0, 0, 0, 0, 830, 912, 663, 462, 514, 452, 198, 163, 1901, 87, 0, 0, 0, 0, 0, 0, 1462, 538, 306, 485, 459, 725, 565, 818, 1116, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5941, 6063, 6054, 6314, 6179, 5945, 6182, 6474] avg=6144.0000 max=6474 max_rank=7 min=5941 min_rank=0 imbalance_ratio=1.053711 max_load_reduction=42678 max_load_reduction_pct=0.868286 moved_to_echo=24780 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[119, 861, 127, 380, 256, 568, 248, 1710, 1672, 131, 0, 0, 0, 0, 0, 0, 30, 1402, 304, 1650, 660, 1268, 369, 796, 0, 0, 0, 0, 0, 0, 0, 0, 1614, 518, 676, 969, 489, 306, 1329, 203, 0, 0, 0, 0, 0, 0, 0, 0, 532, 1487, 1214, 1272, 747, 402, 371, 91, 0, 0, 0, 0, 0, 0, 0, 0, 726, 654, 418, 1093, 1771, 1061, 0, 501, 0, 0, 0, 0, 0, 0, 0, 0, 318, 974, 122, 132, 1431, 1831, 172, 465, 445, 205, 203, 0, 0, 0, 0, 0, 721, 1034, 758, 383, 550, 293, 178, 199, 1693, 86, 0, 0, 0, 0, 0, 0, 1414, 572, 265, 384, 351, 771, 536, 643, 1028, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6072, 6479, 6104, 6116, 6224, 6298, 5895, 5964] avg=6144.0000 max=6479 max_rank=1 min=5895 min_rank=6 imbalance_ratio=1.054525 max_load_reduction=42673 max_load_reduction_pct=0.868184 moved_to_echo=24381 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[83, 711, 78, 433, 260, 513, 272, 1981, 1728, 127, 0, 0, 0, 0, 0, 0, 28, 960, 244, 1803, 650, 1280, 352, 600, 0, 0, 0, 0, 0, 0, 0, 0, 1491, 547, 655, 929, 367, 305, 1475, 123, 0, 0, 0, 0, 0, 0, 0, 0, 511, 1799, 1255, 1158, 647, 428, 341, 107, 0, 0, 0, 0, 0, 0, 0, 0, 772, 709, 403, 1191, 1931, 714, 0, 460, 0, 0, 0, 0, 0, 0, 0, 0, 262, 1055, 155, 136, 1427, 1934, 107, 529, 411, 199, 221, 0, 0, 0, 0, 0, 912, 965, 769, 296, 477, 426, 153, 111, 1791, 89, 0, 0, 0, 0, 0, 0, 1673, 558, 266, 437, 306, 772, 547, 751, 996, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6186, 5917, 5892, 6246, 6180, 6436, 5989, 6306] avg=6144.0000 max=6436 max_rank=5 min=5892 min_rank=2 imbalance_ratio=1.047526 max_load_reduction=42716 max_load_reduction_pct=0.869059 moved_to_echo=24911 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[12032, 4480, 2304, 3200, 2816, 6016, 4608, 5760, 8576, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4096, 12928, 9600, 9344, 5760, 4224, 2688, 1024, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[12544, 4096, 5120, 7936, 3712, 2560, 12288, 1408, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1024, 6528, 896, 3456, 2304, 4608, 2176, 14592, 13184, 1152, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2560, 7424, 1152, 1152, 11008, 14208, 1280, 4096, 3456, 1792, 1664, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[5888, 5376, 3456, 9728, 13952, 7296, 0, 3840, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[384, 9344, 2048, 13312, 5632, 10752, 2944, 5376, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[6912, 7680, 5632, 2944, 4480, 3712, 1536, 1408, 14976, 768, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[2000, 679, 814, 77, 121, 275, 1977, 1495, 125, 193, 744, 298, 784, 720, 341, 1814, 2557, 317, 719, 1182, 500, 859, 93, 920, 300, 334, 1178, 234, 794, 1273, 2363, 871, 1385, 756, 249, 145, 564, 623, 301, 361, 1100, 2734, 501, 526, 230, 495, 481, 548, 1199, 881, 1007, 315, 79, 891, 438, 2418, 853, 49, 173, 304, 905, 636, 798, 256] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1656, 835, 1034, 45, 63, 296, 1969, 2005, 137, 301, 570, 352, 894, 715, 347, 1689, 2606, 184, 902, 921, 616, 905, 154, 1307, 182, 312, 973, 329, 580, 1182, 2459, 638, 1721, 802, 169, 141, 558, 735, 127, 478, 1116, 2368, 621, 743, 289, 479, 551, 613, 1303, 699, 604, 301, 125, 1097, 354, 2002, 747, 84, 209, 509, 1003, 670, 511, 265] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1710, 676, 780, 58, 96, 226, 1870, 1724, 146, 228, 757, 295, 960, 723, 275, 1681, 2347, 341, 846, 1195, 604, 890, 83, 1130, 174, 270, 1268, 210, 685, 1300, 2678, 801, 1473, 1158, 189, 109, 576, 629, 163, 364, 953, 2744, 577, 755, 246, 444, 475, 385, 1006, 1086, 606, 237, 104, 1289, 320, 2550, 837, 30, 231, 299, 845, 750, 527, 168] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[2123, 732, 646, 91, 101, 269, 1720, 1607, 118, 221, 597, 393, 1087, 648, 246, 1900, 2506, 273, 716, 1236, 556, 1053, 105, 1204, 169, 386, 1189, 224, 723, 1241, 2506, 771, 1354, 723, 158, 183, 294, 589, 264, 442, 1130, 2361, 686, 633, 301, 731, 504, 488, 1166, 977, 539, 254, 138, 1072, 468, 2166, 1068, 41, 223, 313, 1024, 693, 615, 197] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1950, 900, 739, 79, 90, 249, 2113, 1519, 125, 266, 759, 316, 855, 538, 343, 1581, 2599, 234, 951, 1157, 632, 863, 71, 1164, 233, 375, 1085, 214, 792, 1258, 2804, 662, 1209, 718, 227, 161, 558, 692, 151, 421, 1209, 2231, 632, 608, 326, 443, 433, 801, 1245, 919, 677, 237, 123, 1291, 346, 2229, 825, 30, 239, 327, 954, 618, 490, 266] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[1889, 795, 1033, 53, 99, 183, 1895, 1891, 78, 202, 663, 322, 899, 645, 381, 1511, 3176, 212, 680, 1248, 458, 1087, 112, 1396, 126, 296, 819, 229, 863, 1167, 2910, 640, 1569, 802, 199, 118, 487, 740, 218, 377, 1009, 2152, 329, 700, 334, 394, 444, 587, 1272, 872, 662, 332, 98, 1188, 223, 2324, 907, 10, 180, 298, 1053, 559, 536, 221] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1852, 775, 886, 56, 112, 290, 1918, 1858, 115, 263, 479, 343, 1067, 730, 529, 1471, 2770, 266, 764, 1122, 526, 1192, 104, 1089, 203, 430, 1155, 248, 816, 1322, 2488, 1007, 1087, 825, 218, 210, 547, 544, 240, 609, 1092, 2049, 311, 724, 383, 415, 454, 482, 1233, 816, 639, 210, 148, 1039, 349, 2105, 792, 98, 198, 440, 1081, 656, 604, 308] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1728, 951, 793, 49, 170, 264, 1822, 1894, 145, 301, 692, 270, 1068, 556, 291, 1390, 2355, 284, 505, 1188, 506, 908, 47, 1037, 166, 379, 1241, 204, 795, 1293, 2929, 1144, 1297, 707, 198, 184, 285, 574, 220, 450, 1547, 2422, 522, 646, 272, 476, 381, 339, 1029, 954, 805, 207, 109, 889, 345, 2582, 817, 28, 353, 318, 992, 932, 693, 214] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2000, 679, 814, 77, 121, 275, 488, 1495, 0, 0, 0, 0, 0, 0, 0, 0, 125, 193, 744, 298, 784, 720, 341, 1814, 794, 402, 91, 0, 0, 0, 0, 0, 1167, 317, 719, 1182, 500, 859, 93, 920, 0, 0, 0, 0, 0, 0, 0, 0, 300, 334, 1178, 234, 794, 1273, 1276, 871, 0, 0, 0, 0, 0, 0, 0, 0, 1385, 756, 249, 145, 564, 623, 301, 361, 996, 685, 0, 0, 0, 0, 0, 0, 1100, 2332, 501, 526, 230, 495, 481, 548, 0, 0, 0, 0, 0, 0, 0, 0, 1199, 881, 1007, 315, 79, 891, 438, 1624, 0, 0, 0, 0, 0, 0, 0, 0, 853, 49, 173, 304, 905, 636, 798, 256, 1489, 705, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5949, 6306, 5757, 6260, 6065, 6213, 6434, 6168] avg=6144.0000 max=6434 max_rank=6 min=5757 min_rank=2 imbalance_ratio=1.047201 max_load_reduction=42718 max_load_reduction_pct=0.869100 moved_to_echo=24880 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1710, 676, 780, 58, 96, 226, 462, 1724, 0, 0, 0, 0, 0, 0, 0, 0, 146, 228, 757, 295, 960, 723, 275, 1681, 837, 403, 103, 0, 0, 0, 0, 0, 1071, 341, 846, 1195, 604, 890, 83, 1130, 0, 0, 0, 0, 0, 0, 0, 0, 174, 270, 1268, 210, 685, 1300, 1446, 801, 0, 0, 0, 0, 0, 0, 0, 0, 1473, 1158, 189, 109, 576, 629, 163, 364, 1129, 629, 0, 0, 0, 0, 0, 0, 953, 2341, 577, 755, 246, 444, 475, 385, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1086, 606, 237, 104, 1289, 320, 1713, 0, 0, 0, 0, 0, 0, 0, 0, 837, 30, 231, 299, 845, 750, 527, 168, 1408, 647, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5732, 6408, 6160, 6154, 6419, 6176, 6361, 5742] avg=6144.0000 max=6419 max_rank=4 min=5732 min_rank=0 imbalance_ratio=1.044759 max_load_reduction=42733 max_load_reduction_pct=0.869405 moved_to_echo=24698 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1656, 835, 1034, 45, 63, 296, 486, 2005, 0, 0, 0, 0, 0, 0, 0, 0, 137, 301, 570, 352, 894, 715, 347, 1689, 657, 348, 95, 0, 0, 0, 0, 0, 1190, 184, 902, 921, 616, 905, 154, 1307, 0, 0, 0, 0, 0, 0, 0, 0, 182, 312, 973, 329, 580, 1182, 1327, 638, 0, 0, 0, 0, 0, 0, 0, 0, 1721, 802, 169, 141, 558, 735, 127, 478, 1037, 698, 0, 0, 0, 0, 0, 0, 1116, 2020, 621, 743, 289, 479, 551, 613, 0, 0, 0, 0, 0, 0, 0, 0, 1303, 699, 604, 301, 125, 1097, 354, 1345, 0, 0, 0, 0, 0, 0, 0, 0, 747, 84, 209, 509, 1003, 670, 511, 265, 1483, 718, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6420, 6105, 6179, 5523, 6466, 6432, 5828, 6199] avg=6144.0000 max=6466 max_rank=4 min=5523 min_rank=3 imbalance_ratio=1.052409 max_load_reduction=42686 max_load_reduction_pct=0.868449 moved_to_echo=24925 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1950, 900, 739, 79, 90, 249, 522, 1519, 0, 0, 0, 0, 0, 0, 0, 0, 125, 266, 759, 316, 855, 538, 343, 1581, 732, 328, 108, 0, 0, 0, 0, 0, 1187, 234, 951, 1157, 632, 863, 71, 1164, 0, 0, 0, 0, 0, 0, 0, 0, 233, 375, 1085, 214, 792, 1258, 1513, 662, 0, 0, 0, 0, 0, 0, 0, 0, 1209, 718, 227, 161, 558, 692, 151, 421, 1183, 696, 0, 0, 0, 0, 0, 0, 1209, 1903, 632, 608, 326, 443, 433, 801, 0, 0, 0, 0, 0, 0, 0, 0, 1245, 919, 677, 237, 123, 1291, 346, 1497, 0, 0, 0, 0, 0, 0, 0, 0, 825, 30, 239, 327, 954, 618, 490, 266, 1591, 716, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6048, 5951, 6259, 6132, 6016, 6355, 6335, 6056] avg=6144.0000 max=6355 max_rank=5 min=5951 min_rank=1 imbalance_ratio=1.034342 max_load_reduction=42797 max_load_reduction_pct=0.870707 moved_to_echo=24762 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1889, 795, 1033, 53, 99, 183, 468, 1891, 0, 0, 0, 0, 0, 0, 0, 0, 78, 202, 663, 322, 899, 645, 381, 1511, 763, 316, 112, 0, 0, 0, 0, 0, 1450, 212, 680, 1248, 458, 1087, 112, 1396, 0, 0, 0, 0, 0, 0, 0, 0, 126, 296, 819, 229, 863, 1167, 1571, 640, 0, 0, 0, 0, 0, 0, 0, 0, 1569, 802, 199, 118, 487, 740, 218, 377, 1227, 851, 0, 0, 0, 0, 0, 0, 1009, 1836, 329, 700, 334, 394, 444, 587, 0, 0, 0, 0, 0, 0, 0, 0, 1272, 872, 662, 332, 98, 1188, 223, 1561, 0, 0, 0, 0, 0, 0, 0, 0, 907, 10, 180, 298, 1053, 559, 536, 221, 1427, 875, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6411, 5892, 6643, 5711, 6588, 5633, 6208, 6066] avg=6144.0000 max=6643 max_rank=2 min=5633 min_rank=5 imbalance_ratio=1.081217 max_load_reduction=42509 max_load_reduction_pct=0.864848 moved_to_echo=24495 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2123, 732, 646, 91, 101, 269, 421, 1607, 0, 0, 0, 0, 0, 0, 0, 0, 118, 221, 597, 393, 1087, 648, 246, 1900, 714, 349, 102, 0, 0, 0, 0, 0, 1134, 273, 716, 1236, 556, 1053, 105, 1204, 0, 0, 0, 0, 0, 0, 0, 0, 169, 386, 1189, 224, 723, 1241, 1342, 771, 0, 0, 0, 0, 0, 0, 0, 0, 1354, 723, 158, 183, 294, 589, 264, 442, 1062, 677, 0, 0, 0, 0, 0, 0, 1130, 2012, 686, 633, 301, 731, 504, 488, 0, 0, 0, 0, 0, 0, 0, 0, 1166, 977, 539, 254, 138, 1072, 468, 1452, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 41, 223, 313, 1024, 693, 615, 197, 1299, 695, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5990, 6375, 6277, 6045, 5746, 6485, 6066, 6168] avg=6144.0000 max=6485 max_rank=5 min=5746 min_rank=4 imbalance_ratio=1.055501 max_load_reduction=42667 max_load_reduction_pct=0.868062 moved_to_echo=24465 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1852, 775, 886, 56, 112, 290, 474, 1858, 0, 0, 0, 0, 0, 0, 0, 0, 115, 263, 479, 343, 1067, 730, 529, 1471, 691, 301, 96, 0, 0, 0, 0, 0, 1264, 266, 764, 1122, 526, 1192, 104, 1089, 0, 0, 0, 0, 0, 0, 0, 0, 203, 430, 1155, 248, 816, 1322, 1343, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1087, 825, 218, 210, 547, 544, 240, 609, 1049, 742, 0, 0, 0, 0, 0, 0, 1092, 1748, 311, 724, 383, 415, 454, 482, 0, 0, 0, 0, 0, 0, 0, 0, 1233, 816, 639, 210, 148, 1039, 349, 1414, 0, 0, 0, 0, 0, 0, 0, 0, 792, 98, 198, 440, 1081, 656, 604, 308, 1444, 764, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6303, 6085, 6327, 6524, 6071, 5609, 5848, 6385] avg=6144.0000 max=6524 max_rank=3 min=5609 min_rank=5 imbalance_ratio=1.061849 max_load_reduction=42628 max_load_reduction_pct=0.867269 moved_to_echo=23913 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1728, 951, 793, 49, 170, 264, 450, 1894, 0, 0, 0, 0, 0, 0, 0, 0, 145, 301, 692, 270, 1068, 556, 291, 1390, 848, 356, 113, 0, 0, 0, 0, 0, 1075, 284, 505, 1188, 506, 908, 47, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 166, 379, 1241, 204, 795, 1293, 1581, 1144, 0, 0, 0, 0, 0, 0, 0, 0, 1297, 707, 198, 184, 285, 574, 220, 450, 1235, 631, 0, 0, 0, 0, 0, 0, 1547, 2066, 522, 646, 272, 476, 381, 339, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 954, 805, 207, 109, 889, 345, 1734, 0, 0, 0, 0, 0, 0, 0, 0, 817, 28, 353, 318, 992, 932, 693, 214, 1372, 649, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6299, 6030, 5550, 6803, 5781, 6249, 6072, 6368] avg=6144.0000 max=6803 max_rank=3 min=5550 min_rank=2 imbalance_ratio=1.107259 max_load_reduction=42349 max_load_reduction_pct=0.861593 moved_to_echo=24470 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[11136, 6528, 1664, 1280, 3968, 5248, 1792, 3584, 8960, 5632, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[9600, 2176, 6144, 9344, 4480, 7808, 896, 9344, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[9216, 16384, 4224, 5376, 2432, 3968, 3840, 4352, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6912, 384, 1920, 2816, 7936, 5632, 4864, 1920, 11520, 5888, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[9472, 7296, 5632, 2176, 1024, 8832, 2944, 12416, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1664, 2816, 8960, 1920, 6144, 10112, 11520, 6656, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1024, 2048, 5376, 2688, 7680, 5376, 2816, 13056, 6144, 2816, 896, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[14976, 6400, 6784, 512, 896, 2176, 3840, 14080, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[409, 850, 967, 470, 662, 388, 706, 1129, 198, 727, 535, 480, 282, 1153, 2401, 112, 288, 3965, 37, 820, 64, 542, 2401, 255, 668, 396, 1864, 1874, 1536, 2481, 870, 1, 29, 186, 4, 2116, 387, 366, 422, 450, 259, 149, 190, 1222, 1876, 28, 1610, 1210, 267, 404, 834, 1154, 818, 794, 458, 934, 129, 1245, 55, 163, 1275, 85, 94, 408] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[416, 605, 1434, 441, 318, 348, 601, 1232, 124, 1114, 916, 447, 208, 1119, 2263, 128, 230, 4171, 29, 890, 58, 786, 2685, 215, 609, 465, 2342, 1626, 1515, 2497, 668, 0, 53, 107, 10, 1790, 317, 348, 474, 595, 207, 141, 201, 1162, 2094, 16, 1057, 590, 289, 353, 540, 1292, 1082, 581, 557, 590, 92, 1327, 81, 132, 1950, 36, 48, 540] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[486, 704, 1179, 283, 590, 334, 714, 1061, 140, 965, 757, 430, 286, 1273, 2369, 114, 217, 3862, 27, 951, 60, 476, 3551, 344, 494, 90, 2219, 1796, 1325, 2132, 532, 0, 40, 166, 8, 2023, 337, 411, 390, 520, 206, 150, 166, 1445, 1981, 33, 1558, 779, 338, 356, 421, 1624, 1082, 711, 591, 775, 234, 975, 98, 204, 1263, 29, 100, 377] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[260, 796, 1328, 505, 434, 415, 548, 1154, 132, 863, 678, 491, 310, 1150, 2095, 86, 242, 4326, 22, 921, 50, 550, 2430, 286, 401, 540, 2297, 1918, 1478, 2545, 777, 0, 40, 73, 15, 2089, 314, 313, 449, 443, 182, 114, 247, 1177, 2200, 21, 1029, 705, 312, 368, 597, 1674, 1096, 639, 488, 669, 65, 1470, 99, 408, 1420, 32, 52, 324] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[942, 996, 1056, 315, 608, 556, 899, 744, 117, 672, 846, 635, 523, 1213, 1874, 172, 281, 3508, 61, 960, 33, 862, 2252, 255, 435, 422, 2109, 1504, 1277, 2021, 771, 3, 65, 246, 19, 1722, 364, 359, 479, 624, 286, 184, 284, 1102, 2391, 20, 889, 1024, 353, 429, 731, 1235, 1070, 894, 493, 808, 170, 905, 95, 292, 2114, 87, 120, 376] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[425, 702, 1032, 352, 793, 520, 729, 1151, 193, 525, 934, 541, 252, 1258, 2158, 168, 248, 3989, 34, 959, 30, 446, 2879, 274, 571, 541, 2467, 1626, 1762, 2110, 619, 0, 40, 153, 8, 2217, 461, 406, 448, 409, 248, 122, 184, 1111, 2174, 24, 1115, 786, 345, 468, 543, 1623, 963, 687, 317, 703, 118, 1159, 68, 199, 1220, 89, 85, 371] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[307, 736, 1243, 387, 384, 704, 695, 1149, 223, 903, 782, 451, 308, 1113, 1857, 77, 246, 4230, 29, 891, 61, 626, 2352, 309, 519, 124, 2302, 1930, 1507, 2299, 519, 4, 37, 97, 20, 2002, 356, 379, 316, 544, 141, 119, 332, 1365, 2094, 29, 1309, 1147, 376, 383, 772, 1126, 1131, 816, 367, 613, 94, 1454, 135, 233, 1311, 97, 76, 614] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[375, 743, 1192, 389, 392, 526, 800, 957, 157, 929, 720, 445, 265, 1404, 1796, 93, 236, 3578, 34, 1078, 54, 456, 2414, 323, 443, 70, 2331, 1932, 2019, 2203, 565, 1, 65, 112, 9, 2294, 420, 298, 395, 593, 252, 193, 249, 1833, 1894, 43, 1437, 1107, 391, 368, 563, 1279, 957, 1086, 592, 731, 156, 601, 161, 228, 1466, 51, 57, 351] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[416, 605, 1434, 441, 318, 348, 601, 1232, 445, 160, 0, 0, 0, 0, 0, 0, 124, 1114, 916, 447, 208, 1119, 2263, 128, 112, 42, 0, 0, 0, 0, 0, 0, 230, 1457, 29, 890, 58, 786, 2685, 215, 0, 0, 0, 0, 0, 0, 0, 0, 609, 465, 1361, 1626, 1515, 0, 668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 107, 10, 1790, 317, 348, 474, 595, 2337, 105, 0, 0, 0, 0, 0, 0, 207, 141, 201, 1162, 2052, 16, 1057, 590, 0, 0, 0, 0, 0, 0, 0, 0, 289, 353, 540, 1292, 1082, 581, 557, 590, 424, 0, 0, 0, 0, 0, 0, 0, 92, 1327, 81, 132, 1950, 36, 48, 540, 2609, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6000, 6473, 6350, 6244, 6136, 5426, 5708, 6815] avg=6144.0000 max=6815 max_rank=7 min=5426 min_rank=5 imbalance_ratio=1.109212 max_load_reduction=42337 max_load_reduction_pct=0.861348 moved_to_echo=24085 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[409, 850, 967, 470, 662, 388, 706, 1129, 354, 159, 0, 0, 0, 0, 0, 0, 198, 727, 535, 480, 282, 1153, 2401, 112, 89, 38, 0, 0, 0, 0, 0, 0, 288, 1385, 37, 820, 64, 542, 2401, 255, 0, 0, 0, 0, 0, 0, 0, 0, 668, 396, 1084, 1874, 1536, 0, 870, 1, 0, 0, 0, 0, 0, 0, 0, 0, 29, 186, 4, 2116, 387, 366, 422, 450, 2322, 100, 0, 0, 0, 0, 0, 0, 259, 149, 190, 1222, 1838, 28, 1610, 1210, 0, 0, 0, 0, 0, 0, 0, 0, 267, 404, 834, 1154, 818, 794, 458, 934, 337, 0, 0, 0, 0, 0, 0, 0, 129, 1245, 55, 163, 1275, 85, 94, 408, 2480, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6094, 6015, 5792, 6429, 6382, 6506, 6000, 5934] avg=6144.0000 max=6506 max_rank=5 min=5792 min_rank=2 imbalance_ratio=1.058919 max_load_reduction=42646 max_load_reduction_pct=0.867635 moved_to_echo=24822 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[486, 704, 1179, 283, 590, 334, 714, 1061, 421, 137, 0, 0, 0, 0, 0, 0, 140, 965, 757, 430, 286, 1273, 2369, 114, 106, 40, 0, 0, 0, 0, 0, 0, 217, 1349, 27, 951, 60, 476, 3551, 344, 0, 0, 0, 0, 0, 0, 0, 0, 494, 90, 1290, 1796, 1325, 0, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 166, 8, 2023, 337, 411, 390, 520, 1995, 97, 0, 0, 0, 0, 0, 0, 206, 150, 166, 1445, 1941, 33, 1558, 779, 0, 0, 0, 0, 0, 0, 0, 0, 338, 356, 421, 1624, 1082, 711, 591, 775, 402, 0, 0, 0, 0, 0, 0, 0, 234, 975, 98, 204, 1263, 29, 100, 377, 2416, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5909, 6480, 6975, 5527, 5987, 6278, 6300, 5696] avg=6144.0000 max=6975 max_rank=2 min=5527 min_rank=3 imbalance_ratio=1.135254 max_load_reduction=42177 max_load_reduction_pct=0.858093 moved_to_echo=24261 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[942, 996, 1056, 315, 608, 556, 899, 744, 401, 130, 0, 0, 0, 0, 0, 0, 117, 672, 846, 635, 523, 1213, 1874, 172, 101, 48, 0, 0, 0, 0, 0, 0, 281, 1226, 61, 960, 33, 862, 2252, 255, 0, 0, 0, 0, 0, 0, 0, 0, 435, 422, 1225, 1504, 1277, 0, 771, 3, 0, 0, 0, 0, 0, 0, 0, 0, 65, 246, 19, 1722, 364, 359, 479, 624, 1891, 88, 0, 0, 0, 0, 0, 0, 286, 184, 284, 1102, 2343, 20, 889, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 353, 429, 731, 1235, 1070, 894, 493, 808, 382, 0, 0, 0, 0, 0, 0, 0, 170, 905, 95, 292, 2114, 87, 120, 376, 2194, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6647, 6201, 5930, 5637, 5857, 6132, 6395, 6353] avg=6144.0000 max=6647 max_rank=0 min=5637 min_rank=3 imbalance_ratio=1.081868 max_load_reduction=42505 max_load_reduction_pct=0.864766 moved_to_echo=24737 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[260, 796, 1328, 505, 434, 415, 548, 1154, 436, 164, 0, 0, 0, 0, 0, 0, 132, 863, 678, 491, 310, 1150, 2095, 86, 110, 44, 0, 0, 0, 0, 0, 0, 242, 1511, 22, 921, 50, 550, 2430, 286, 0, 0, 0, 0, 0, 0, 0, 0, 401, 540, 1335, 1918, 1478, 0, 777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 73, 15, 2089, 314, 313, 449, 443, 2381, 109, 0, 0, 0, 0, 0, 0, 182, 114, 247, 1177, 2156, 21, 1029, 705, 0, 0, 0, 0, 0, 0, 0, 0, 312, 368, 597, 1674, 1096, 639, 488, 669, 416, 0, 0, 0, 0, 0, 0, 0, 65, 1470, 99, 408, 1420, 32, 52, 324, 2706, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6040, 5959, 6012, 6449, 6226, 5631, 6259, 6576] avg=6144.0000 max=6576 max_rank=7 min=5631 min_rank=5 imbalance_ratio=1.070312 max_load_reduction=42576 max_load_reduction_pct=0.866211 moved_to_echo=24692 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[425, 702, 1032, 352, 793, 520, 729, 1151, 473, 136, 0, 0, 0, 0, 0, 0, 193, 525, 934, 541, 252, 1258, 2158, 168, 122, 49, 0, 0, 0, 0, 0, 0, 248, 1386, 34, 959, 30, 446, 2879, 274, 0, 0, 0, 0, 0, 0, 0, 0, 571, 541, 1421, 1626, 1762, 0, 619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 153, 8, 2217, 461, 406, 448, 409, 1974, 103, 0, 0, 0, 0, 0, 0, 248, 122, 184, 1111, 2125, 24, 1115, 786, 0, 0, 0, 0, 0, 0, 0, 0, 345, 468, 543, 1623, 963, 687, 317, 703, 451, 0, 0, 0, 0, 0, 0, 0, 118, 1159, 68, 199, 1220, 89, 85, 371, 2500, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6313, 6200, 6256, 6540, 6219, 5715, 6100, 5809] avg=6144.0000 max=6540 max_rank=3 min=5715 min_rank=5 imbalance_ratio=1.064453 max_load_reduction=42612 max_load_reduction_pct=0.866943 moved_to_echo=23843 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[375, 743, 1192, 389, 392, 526, 800, 957, 443, 142, 0, 0, 0, 0, 0, 0, 157, 929, 720, 445, 265, 1404, 1796, 93, 111, 38, 0, 0, 0, 0, 0, 0, 236, 1250, 34, 1078, 54, 456, 2414, 323, 0, 0, 0, 0, 0, 0, 0, 0, 443, 70, 1355, 1932, 2019, 0, 565, 1, 0, 0, 0, 0, 0, 0, 0, 0, 65, 112, 9, 2294, 420, 298, 395, 593, 2061, 90, 0, 0, 0, 0, 0, 0, 252, 193, 249, 1833, 1856, 43, 1437, 1107, 0, 0, 0, 0, 0, 0, 0, 0, 391, 368, 563, 1279, 957, 1086, 592, 731, 422, 0, 0, 0, 0, 0, 0, 0, 156, 601, 161, 228, 1466, 51, 57, 351, 2238, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5959, 5958, 5845, 6385, 6337, 6970, 6389, 5309] avg=6144.0000 max=6970 max_rank=5 min=5309 min_rank=7 imbalance_ratio=1.134440 max_load_reduction=42182 max_load_reduction_pct=0.858195 moved_to_echo=25005 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[307, 736, 1243, 387, 384, 704, 695, 1149, 437, 148, 0, 0, 0, 0, 0, 0, 223, 903, 782, 451, 308, 1113, 1857, 77, 110, 42, 0, 0, 0, 0, 0, 0, 246, 1478, 29, 891, 61, 626, 2352, 309, 0, 0, 0, 0, 0, 0, 0, 0, 519, 124, 1338, 1930, 1507, 0, 519, 4, 0, 0, 0, 0, 0, 0, 0, 0, 37, 97, 20, 2002, 356, 379, 316, 544, 2151, 106, 0, 0, 0, 0, 0, 0, 141, 119, 332, 1365, 2052, 29, 1309, 1147, 0, 0, 0, 0, 0, 0, 0, 0, 376, 383, 772, 1126, 1131, 816, 367, 613, 417, 0, 0, 0, 0, 0, 0, 0, 94, 1454, 135, 233, 1311, 97, 76, 614, 2646, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6190, 5866, 5992, 5941, 6008, 6494, 6001, 6660] avg=6144.0000 max=6660 max_rank=7 min=5866 min_rank=1 imbalance_ratio=1.083984 max_load_reduction=42492 max_load_reduction_pct=0.864502 moved_to_echo=25163 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2688, 3200, 5120, 11008, 8320, 6272, 3968, 5888, 3328, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1408, 6784, 6272, 3968, 2560, 9728, 16896, 1024, 896, 384, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1792, 1280, 1920, 10496, 16384, 256, 10112, 7424, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1152, 9216, 896, 1920, 12032, 512, 640, 3456, 19840, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2048, 11136, 384, 7552, 512, 4864, 20992, 2304, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4224, 2688, 10496, 14208, 12544, 0, 5376, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3712, 6144, 9472, 3200, 4224, 3840, 5760, 8704, 3456, 1280, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[384, 1152, 128, 16256, 3072, 2944, 3456, 4224, 17152, 896, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[245, 990, 597, 51, 550, 711, 199, 3698, 609, 140, 201, 2100, 272, 935, 1263, 219, 3608, 3778, 69, 312, 1470, 34, 1660, 14, 139, 70, 568, 481, 0, 625, 1661, 49, 656, 15, 132, 827, 5, 77, 3505, 2603, 1672, 145, 2, 235, 6, 529, 951, 237, 124, 525, 1814, 2655, 669, 852, 36, 224, 145, 724, 1048, 28, 25, 644, 6, 718] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[256, 865, 668, 56, 584, 750, 176, 4013, 517, 84, 196, 2120, 368, 947, 1141, 557, 3040, 2772, 71, 232, 1335, 21, 1765, 42, 103, 30, 793, 847, 1, 838, 1452, 36, 692, 46, 372, 752, 23, 56, 3814, 2913, 1805, 159, 29, 286, 13, 408, 1269, 201, 151, 695, 1992, 2131, 679, 970, 44, 330, 140, 499, 768, 61, 60, 565, 14, 539] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[108, 839, 819, 75, 629, 1031, 312, 3521, 553, 123, 161, 2257, 370, 1053, 1613, 508, 3267, 2383, 45, 223, 1921, 55, 1475, 66, 99, 109, 800, 798, 2, 440, 1545, 78, 699, 33, 218, 527, 25, 101, 3823, 2197, 1666, 168, 5, 369, 5, 437, 1357, 403, 288, 388, 1533, 2515, 645, 1027, 109, 418, 143, 484, 687, 60, 72, 767, 11, 694] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[274, 1243, 544, 36, 578, 735, 228, 3353, 673, 123, 182, 2183, 399, 607, 1630, 395, 2868, 2821, 65, 160, 1669, 38, 1372, 46, 86, 52, 609, 941, 1, 559, 1500, 55, 651, 35, 391, 845, 10, 66, 3895, 2278, 1657, 156, 39, 174, 15, 475, 1160, 276, 124, 327, 2055, 2567, 667, 1089, 66, 498, 116, 831, 1136, 64, 58, 703, 8, 695] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[206, 1051, 854, 34, 626, 916, 264, 3784, 327, 87, 230, 2358, 255, 1470, 1362, 359, 3507, 2546, 48, 130, 1166, 62, 1779, 66, 100, 62, 649, 593, 4, 805, 1354, 58, 830, 31, 242, 483, 25, 120, 3762, 2585, 1545, 117, 35, 167, 12, 372, 784, 469, 166, 805, 1642, 2446, 445, 810, 68, 446, 133, 647, 1194, 42, 37, 721, 8, 851] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[210, 944, 773, 33, 661, 1015, 100, 3927, 472, 110, 133, 1800, 302, 737, 1214, 465, 3371, 2948, 75, 203, 1634, 47, 1809, 53, 161, 48, 733, 1008, 0, 496, 1255, 55, 748, 26, 398, 653, 39, 97, 3605, 2371, 1640, 128, 17, 361, 5, 579, 939, 249, 187, 717, 1925, 2659, 440, 788, 93, 619, 93, 739, 920, 43, 33, 599, 19, 631] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[128, 875, 562, 69, 783, 692, 265, 3970, 440, 182, 134, 2123, 352, 562, 1752, 480, 3427, 3050, 62, 156, 1639, 61, 1830, 53, 140, 56, 678, 593, 0, 905, 1780, 41, 712, 37, 264, 626, 19, 92, 3647, 2573, 1611, 230, 17, 244, 22, 367, 999, 188, 188, 271, 1943, 2380, 425, 919, 109, 330, 149, 535, 1115, 108, 30, 618, 12, 532] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[249, 876, 710, 72, 443, 881, 288, 3709, 505, 125, 95, 1795, 363, 836, 1337, 755, 3461, 2403, 60, 387, 2361, 85, 1461, 154, 94, 60, 870, 745, 8, 573, 1229, 124, 824, 79, 337, 488, 26, 142, 3688, 1932, 1676, 130, 24, 383, 34, 653, 1064, 688, 403, 509, 1258, 2898, 288, 855, 126, 566, 71, 781, 669, 48, 29, 823, 33, 513] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[245, 990, 597, 51, 550, 711, 199, 2521, 0, 0, 0, 0, 0, 0, 0, 0, 609, 140, 201, 2100, 272, 935, 1263, 219, 189, 0, 0, 0, 0, 0, 0, 0, 0, 3264, 69, 312, 1470, 34, 1660, 14, 0, 0, 0, 0, 0, 0, 0, 0, 139, 70, 568, 481, 0, 625, 1661, 49, 1724, 365, 146, 0, 0, 0, 0, 0, 656, 15, 132, 827, 5, 77, 1781, 2603, 0, 0, 0, 0, 0, 0, 0, 0, 1672, 145, 2, 235, 6, 529, 951, 237, 1031, 888, 325, 0, 0, 0, 0, 0, 124, 525, 1814, 1767, 669, 852, 36, 224, 0, 0, 0, 0, 0, 0, 0, 0, 145, 724, 1048, 28, 25, 644, 6, 718, 3243, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5864, 5928, 6823, 5828, 6096, 6021, 6011, 6581] avg=6144.0000 max=6823 max_rank=2 min=5828 min_rank=3 imbalance_ratio=1.110514 max_load_reduction=42329 max_load_reduction_pct=0.861186 moved_to_echo=24709 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[108, 839, 819, 75, 629, 1031, 312, 2400, 0, 0, 0, 0, 0, 0, 0, 0, 553, 123, 161, 2257, 370, 1053, 1613, 508, 119, 0, 0, 0, 0, 0, 0, 0, 0, 2059, 45, 223, 1921, 55, 1475, 66, 0, 0, 0, 0, 0, 0, 0, 0, 99, 109, 800, 798, 2, 440, 1545, 78, 1880, 330, 139, 0, 0, 0, 0, 0, 699, 33, 218, 527, 25, 101, 1943, 2197, 0, 0, 0, 0, 0, 0, 0, 0, 1666, 168, 5, 369, 5, 437, 1357, 403, 982, 841, 205, 0, 0, 0, 0, 0, 288, 388, 1533, 1674, 645, 1027, 109, 418, 0, 0, 0, 0, 0, 0, 0, 0, 143, 484, 687, 60, 72, 767, 11, 694, 2937, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6213, 6757, 5844, 6220, 5743, 6438, 6082, 5855] avg=6144.0000 max=6757 max_rank=1 min=5743 min_rank=4 imbalance_ratio=1.099772 max_load_reduction=42395 max_load_reduction_pct=0.862528 moved_to_echo=24118 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[256, 865, 668, 56, 584, 750, 176, 2735, 0, 0, 0, 0, 0, 0, 0, 0, 517, 84, 196, 2120, 368, 947, 1141, 557, 138, 0, 0, 0, 0, 0, 0, 0, 0, 2396, 71, 232, 1335, 21, 1765, 42, 0, 0, 0, 0, 0, 0, 0, 0, 103, 30, 793, 847, 1, 838, 1452, 36, 1876, 307, 159, 0, 0, 0, 0, 0, 692, 46, 372, 752, 23, 56, 1938, 2913, 0, 0, 0, 0, 0, 0, 0, 0, 1805, 159, 29, 286, 13, 408, 1269, 201, 1119, 713, 238, 0, 0, 0, 0, 0, 151, 695, 1992, 1418, 679, 970, 44, 330, 0, 0, 0, 0, 0, 0, 0, 0, 140, 499, 768, 61, 60, 565, 14, 539, 2733, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6090, 6068, 5862, 6442, 6792, 6240, 6279, 5379] avg=6144.0000 max=6792 max_rank=4 min=5379 min_rank=7 imbalance_ratio=1.105469 max_load_reduction=42360 max_load_reduction_pct=0.861816 moved_to_echo=24690 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[274, 1243, 544, 36, 578, 735, 228, 2285, 0, 0, 0, 0, 0, 0, 0, 0, 673, 123, 182, 2183, 399, 607, 1630, 395, 141, 0, 0, 0, 0, 0, 0, 0, 0, 2438, 65, 160, 1669, 38, 1372, 46, 0, 0, 0, 0, 0, 0, 0, 0, 86, 52, 609, 941, 1, 559, 1500, 55, 1916, 290, 133, 0, 0, 0, 0, 0, 651, 35, 391, 845, 10, 66, 1979, 2278, 0, 0, 0, 0, 0, 0, 0, 0, 1657, 156, 39, 174, 15, 475, 1160, 276, 935, 859, 242, 0, 0, 0, 0, 0, 124, 327, 2055, 1708, 667, 1089, 66, 498, 0, 0, 0, 0, 0, 0, 0, 0, 116, 831, 1136, 64, 58, 703, 8, 695, 2578, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5923, 6333, 5788, 6142, 6255, 5988, 6534, 6189] avg=6144.0000 max=6534 max_rank=6 min=5788 min_rank=2 imbalance_ratio=1.063477 max_load_reduction=42618 max_load_reduction_pct=0.867065 moved_to_echo=24966 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[128, 875, 562, 69, 783, 692, 265, 2699, 0, 0, 0, 0, 0, 0, 0, 0, 440, 182, 134, 2123, 352, 562, 1752, 480, 155, 0, 0, 0, 0, 0, 0, 0, 0, 2629, 62, 156, 1639, 61, 1830, 53, 0, 0, 0, 0, 0, 0, 0, 0, 140, 56, 678, 593, 0, 905, 1780, 41, 1798, 347, 161, 0, 0, 0, 0, 0, 712, 37, 264, 626, 19, 92, 1849, 2573, 0, 0, 0, 0, 0, 0, 0, 0, 1611, 230, 17, 244, 22, 367, 999, 188, 1110, 800, 266, 0, 0, 0, 0, 0, 188, 271, 1943, 1580, 425, 919, 109, 330, 0, 0, 0, 0, 0, 0, 0, 0, 149, 535, 1115, 108, 30, 618, 12, 532, 3080, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6073, 6180, 6430, 6499, 6172, 5854, 5765, 6179] avg=6144.0000 max=6499 max_rank=3 min=5765 min_rank=6 imbalance_ratio=1.057780 max_load_reduction=42653 max_load_reduction_pct=0.867778 moved_to_echo=23970 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[210, 944, 773, 33, 661, 1015, 100, 2676, 0, 0, 0, 0, 0, 0, 0, 0, 472, 110, 133, 1800, 302, 737, 1214, 465, 147, 0, 0, 0, 0, 0, 0, 0, 0, 2548, 75, 203, 1634, 47, 1809, 53, 0, 0, 0, 0, 0, 0, 0, 0, 161, 48, 733, 1008, 0, 496, 1255, 55, 1773, 342, 156, 0, 0, 0, 0, 0, 748, 26, 398, 653, 39, 97, 1832, 2371, 0, 0, 0, 0, 0, 0, 0, 0, 1640, 128, 17, 361, 5, 579, 939, 249, 1095, 890, 253, 0, 0, 0, 0, 0, 187, 717, 1925, 1769, 440, 788, 93, 619, 0, 0, 0, 0, 0, 0, 0, 0, 93, 739, 920, 43, 33, 599, 19, 631, 3029, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6412, 5380, 6369, 6027, 6164, 6156, 6538, 6106] avg=6144.0000 max=6538 max_rank=6 min=5380 min_rank=1 imbalance_ratio=1.064128 max_load_reduction=42614 max_load_reduction_pct=0.866984 moved_to_echo=24964 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[249, 876, 710, 72, 443, 881, 288, 2528, 0, 0, 0, 0, 0, 0, 0, 0, 505, 125, 95, 1795, 363, 836, 1337, 755, 120, 0, 0, 0, 0, 0, 0, 0, 0, 2077, 60, 387, 2361, 85, 1461, 154, 0, 0, 0, 0, 0, 0, 0, 0, 94, 60, 870, 745, 8, 573, 1229, 124, 1814, 351, 147, 0, 0, 0, 0, 0, 824, 79, 337, 488, 26, 142, 1874, 1932, 0, 0, 0, 0, 0, 0, 0, 0, 1676, 130, 24, 383, 34, 653, 1064, 688, 1034, 970, 206, 0, 0, 0, 0, 0, 403, 509, 1258, 1928, 288, 855, 126, 566, 0, 0, 0, 0, 0, 0, 0, 0, 71, 781, 669, 48, 29, 823, 33, 513, 3110, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6047, 5931, 6585, 6015, 5702, 6862, 5933, 6077] avg=6144.0000 max=6862 max_rank=5 min=5702 min_rank=4 imbalance_ratio=1.116862 max_load_reduction=42290 max_load_reduction_pct=0.860392 moved_to_echo=24574 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[206, 1051, 854, 34, 626, 916, 264, 2579, 0, 0, 0, 0, 0, 0, 0, 0, 327, 87, 230, 2358, 255, 1470, 1362, 359, 127, 0, 0, 0, 0, 0, 0, 0, 0, 2200, 48, 130, 1166, 62, 1779, 66, 0, 0, 0, 0, 0, 0, 0, 0, 100, 62, 649, 593, 4, 805, 1354, 58, 1850, 354, 150, 0, 0, 0, 0, 0, 830, 31, 242, 483, 25, 120, 1912, 2585, 0, 0, 0, 0, 0, 0, 0, 0, 1545, 117, 35, 167, 12, 372, 784, 469, 1055, 818, 219, 0, 0, 0, 0, 0, 166, 805, 1642, 1628, 445, 810, 68, 446, 0, 0, 0, 0, 0, 0, 0, 0, 133, 647, 1194, 42, 37, 721, 8, 851, 3153, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6530, 6575, 5451, 5979, 6228, 5593, 6010, 6786] avg=6144.0000 max=6786 max_rank=7 min=5451 min_rank=2 imbalance_ratio=1.104492 max_load_reduction=42366 max_load_reduction_pct=0.861938 moved_to_echo=24617 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[13312, 1280, 256, 2304, 128, 3840, 8576, 2816, 8448, 6784, 2048, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1664, 4352, 14208, 13568, 4352, 7424, 768, 3456, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4096, 1024, 1408, 16768, 2688, 7168, 11392, 3840, 1152, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1024, 512, 5760, 6016, 128, 5248, 11776, 512, 14720, 2688, 1280, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5888, 384, 2432, 5248, 256, 768, 15232, 19456, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 19712, 512, 1920, 13312, 512, 13184, 512, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1792, 7808, 5632, 512, 4864, 6784, 1920, 20480, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[1024, 5248, 7552, 512, 384, 5504, 128, 5248, 23936, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1454, 59, 262, 4398, 1143, 5, 198, 1062, 508, 715, 847, 562, 4101, 1805, 67, 1606, 780, 47, 76, 1122, 26, 874, 831, 820, 39, 897, 341, 944, 279, 25, 49, 24, 51, 319, 876, 269, 780, 2, 160, 307, 742, 5104, 20, 2590, 2504, 86, 1927, 253, 261, 621, 135, 818, 2083, 1027, 838, 49, 257, 86, 8, 38, 76, 7, 881, 11] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1485, 93, 373, 3839, 1153, 32, 117, 992, 347, 1041, 833, 390, 3987, 1879, 28, 1825, 624, 99, 115, 897, 31, 795, 762, 1019, 14, 821, 340, 1171, 371, 65, 34, 50, 33, 187, 829, 514, 751, 4, 110, 732, 1042, 6016, 16, 2626, 2841, 61, 1217, 259, 150, 542, 226, 786, 2023, 686, 341, 26, 309, 85, 2, 25, 71, 3, 989, 28] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1728, 37, 96, 4776, 864, 5, 198, 1049, 533, 831, 765, 414, 5066, 2678, 4, 1478, 713, 21, 40, 970, 31, 609, 904, 645, 22, 747, 168, 1043, 562, 24, 36, 10, 11, 231, 872, 181, 373, 0, 31, 372, 913, 5668, 8, 1996, 3165, 14, 1531, 236, 97, 639, 181, 804, 2500, 715, 315, 3, 228, 41, 29, 31, 34, 2, 847, 17] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1178, 54, 240, 4608, 307, 2, 124, 1264, 560, 899, 987, 352, 4345, 1900, 31, 1366, 663, 16, 24, 964, 35, 418, 938, 556, 16, 776, 126, 713, 260, 135, 23, 98, 19, 273, 937, 391, 723, 1, 140, 415, 575, 5270, 34, 2284, 2398, 42, 2262, 285, 233, 1780, 153, 913, 2415, 1742, 914, 70, 255, 59, 14, 38, 28, 3, 485, 23] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1501, 89, 197, 4370, 813, 33, 120, 890, 344, 719, 864, 635, 4681, 2779, 48, 1834, 727, 40, 26, 1479, 47, 523, 625, 708, 42, 846, 448, 1378, 517, 65, 65, 46, 18, 335, 937, 91, 674, 2, 66, 295, 263, 4771, 16, 2310, 2250, 65, 1664, 254, 116, 835, 58, 579, 1730, 2403, 953, 23, 201, 168, 7, 90, 52, 2, 417, 8] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1294, 66, 347, 4342, 729, 4, 161, 906, 371, 1171, 802, 444, 4202, 2054, 22, 1831, 709, 95, 26, 1023, 27, 623, 911, 1043, 26, 734, 337, 1176, 396, 26, 49, 44, 26, 276, 1272, 330, 777, 6, 80, 388, 807, 5294, 37, 2046, 2992, 54, 1695, 210, 160, 578, 131, 703, 2427, 646, 858, 20, 237, 32, 32, 61, 58, 2, 908, 18] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1811, 33, 319, 3768, 665, 2, 209, 1238, 452, 716, 830, 253, 4973, 1785, 19, 786, 635, 24, 20, 1069, 30, 565, 882, 1172, 11, 850, 263, 928, 337, 49, 54, 50, 27, 450, 1521, 187, 661, 0, 33, 499, 762, 5528, 11, 2051, 2417, 41, 1863, 311, 179, 1741, 179, 711, 2179, 1194, 718, 35, 228, 74, 7, 47, 18, 4, 664, 14] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1426, 63, 177, 4118, 701, 9, 131, 1399, 369, 680, 791, 358, 5180, 2367, 15, 1438, 737, 37, 15, 1470, 49, 1083, 623, 846, 15, 858, 364, 916, 273, 42, 30, 35, 26, 353, 806, 171, 584, 1, 107, 377, 796, 5965, 22, 2124, 2417, 62, 1996, 246, 128, 364, 98, 944, 1693, 1539, 257, 34, 207, 137, 13, 74, 59, 1, 927, 9] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1454, 59, 262, 2346, 1143, 5, 198, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 508, 715, 847, 562, 0, 1714, 67, 1606, 0, 0, 0, 0, 0, 0, 0, 0, 780, 47, 76, 1122, 26, 874, 831, 820, 1498, 285, 2, 0, 0, 0, 0, 0, 39, 897, 341, 944, 279, 25, 49, 24, 2627, 497, 0, 0, 0, 0, 0, 0, 51, 319, 876, 269, 780, 2, 160, 307, 1767, 1474, 0, 0, 0, 0, 0, 0, 742, 0, 20, 2590, 1006, 86, 1927, 253, 0, 0, 0, 0, 0, 0, 0, 0, 261, 621, 135, 818, 2083, 1027, 838, 49, 89, 0, 0, 0, 0, 0, 0, 0, 257, 86, 8, 38, 76, 7, 881, 11, 4607, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6529, 6019, 6361, 5722, 6005, 6624, 5921, 5971] avg=6144.0000 max=6624 max_rank=5 min=5722 min_rank=3 imbalance_ratio=1.078125 max_load_reduction=42528 max_load_reduction_pct=0.865234 moved_to_echo=24521 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1728, 37, 96, 2548, 864, 5, 198, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 533, 831, 765, 414, 0, 2543, 4, 1478, 0, 0, 0, 0, 0, 0, 0, 0, 713, 21, 40, 970, 31, 609, 904, 645, 1893, 309, 3, 0, 0, 0, 0, 0, 22, 747, 168, 1043, 562, 24, 36, 10, 3245, 552, 0, 0, 0, 0, 0, 0, 11, 231, 872, 181, 373, 0, 31, 372, 1919, 1821, 0, 0, 0, 0, 0, 0, 913, 0, 8, 1996, 1272, 14, 1531, 236, 0, 0, 0, 0, 0, 0, 0, 0, 97, 639, 181, 804, 2500, 715, 315, 3, 132, 0, 0, 0, 0, 0, 0, 0, 228, 41, 29, 31, 34, 2, 847, 17, 5116, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6525, 6568, 6138, 6409, 5811, 5970, 5386, 6345] avg=6144.0000 max=6568 max_rank=1 min=5386 min_rank=6 imbalance_ratio=1.069010 max_load_reduction=42584 max_load_reduction_pct=0.866374 moved_to_echo=23512 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1178, 54, 240, 2458, 307, 2, 124, 1264, 0, 0, 0, 0, 0, 0, 0, 0, 560, 899, 987, 352, 0, 1805, 31, 1366, 0, 0, 0, 0, 0, 0, 0, 0, 663, 16, 24, 964, 35, 418, 938, 556, 1434, 298, 2, 0, 0, 0, 0, 0, 16, 776, 126, 713, 260, 135, 23, 98, 2784, 514, 0, 0, 0, 0, 0, 0, 19, 273, 937, 391, 723, 1, 140, 415, 1852, 1561, 0, 0, 0, 0, 0, 0, 575, 0, 34, 2284, 964, 42, 2262, 285, 0, 0, 0, 0, 0, 0, 0, 0, 233, 1780, 153, 913, 2415, 1742, 914, 70, 93, 0, 0, 0, 0, 0, 0, 0, 255, 59, 14, 38, 28, 3, 485, 23, 4756, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5627, 6000, 5348, 5445, 6312, 6446, 8313, 5661] avg=6144.0000 max=8313 max_rank=6 min=5348 min_rank=2 imbalance_ratio=1.353027 max_load_reduction=40839 max_load_reduction_pct=0.830872 moved_to_echo=26732 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1811, 33, 319, 2010, 665, 2, 209, 1238, 0, 0, 0, 0, 0, 0, 0, 0, 452, 716, 830, 253, 0, 1695, 19, 786, 0, 0, 0, 0, 0, 0, 0, 0, 635, 24, 20, 1069, 30, 565, 882, 1172, 1446, 244, 2, 0, 0, 0, 0, 0, 11, 850, 263, 928, 337, 49, 54, 50, 3186, 538, 0, 0, 0, 0, 0, 0, 27, 450, 1521, 187, 661, 0, 33, 499, 1514, 1787, 0, 0, 0, 0, 0, 0, 762, 0, 11, 2051, 971, 41, 1863, 311, 0, 0, 0, 0, 0, 0, 0, 0, 179, 1741, 179, 711, 2179, 1194, 718, 35, 88, 0, 0, 0, 0, 0, 0, 0, 228, 74, 7, 47, 18, 4, 664, 14, 4990, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6287, 4751, 6089, 6266, 6679, 6010, 7024, 6046] avg=6144.0000 max=7024 max_rank=6 min=4751 min_rank=1 imbalance_ratio=1.143229 max_load_reduction=42128 max_load_reduction_pct=0.857096 moved_to_echo=25759 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1501, 89, 197, 2322, 813, 33, 120, 890, 0, 0, 0, 0, 0, 0, 0, 0, 344, 719, 864, 635, 0, 2631, 48, 1834, 0, 0, 0, 0, 0, 0, 0, 0, 727, 40, 26, 1479, 47, 523, 625, 708, 1350, 287, 7, 0, 0, 0, 0, 0, 42, 846, 448, 1378, 517, 65, 65, 46, 3000, 465, 0, 0, 0, 0, 0, 0, 18, 335, 937, 91, 674, 2, 66, 295, 1761, 1681, 0, 0, 0, 0, 0, 0, 263, 0, 16, 2310, 900, 65, 1664, 254, 0, 0, 0, 0, 0, 0, 0, 0, 116, 835, 58, 579, 1730, 2403, 953, 23, 141, 0, 0, 0, 0, 0, 0, 0, 201, 168, 7, 90, 52, 2, 417, 8, 4306, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5965, 7075, 5819, 6872, 5860, 5472, 6838, 5251] avg=6144.0000 max=7075 max_rank=1 min=5251 min_rank=7 imbalance_ratio=1.151530 max_load_reduction=42077 max_load_reduction_pct=0.856059 moved_to_echo=23421 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1294, 66, 347, 2316, 729, 4, 161, 906, 0, 0, 0, 0, 0, 0, 0, 0, 371, 1171, 802, 444, 0, 1951, 22, 1831, 0, 0, 0, 0, 0, 0, 0, 0, 709, 95, 26, 1023, 27, 623, 911, 1043, 1790, 281, 2, 0, 0, 0, 0, 0, 26, 734, 337, 1176, 396, 26, 49, 44, 2693, 516, 0, 0, 0, 0, 0, 0, 26, 276, 1272, 330, 777, 6, 80, 388, 1745, 1509, 0, 0, 0, 0, 0, 0, 807, 0, 37, 2046, 1202, 54, 1695, 210, 0, 0, 0, 0, 0, 0, 0, 0, 160, 578, 131, 703, 2427, 646, 858, 20, 101, 0, 0, 0, 0, 0, 0, 0, 237, 32, 32, 61, 58, 2, 908, 18, 4778, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5823, 6592, 6530, 5997, 6409, 6051, 5624, 6126] avg=6144.0000 max=6592 max_rank=1 min=5624 min_rank=6 imbalance_ratio=1.072917 max_load_reduction=42560 max_load_reduction_pct=0.865885 moved_to_echo=24210 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1485, 93, 373, 2048, 1153, 32, 117, 992, 0, 0, 0, 0, 0, 0, 0, 0, 347, 1041, 833, 390, 0, 1785, 28, 1825, 0, 0, 0, 0, 0, 0, 0, 0, 624, 99, 115, 897, 31, 795, 762, 1019, 1700, 248, 2, 0, 0, 0, 0, 0, 14, 821, 340, 1171, 371, 65, 34, 50, 2555, 587, 0, 0, 0, 0, 0, 0, 33, 187, 829, 514, 751, 4, 110, 732, 1543, 1432, 0, 0, 0, 0, 0, 0, 1042, 0, 16, 2626, 1141, 61, 1217, 259, 0, 0, 0, 0, 0, 0, 0, 0, 150, 542, 226, 786, 2023, 686, 341, 26, 92, 0, 0, 0, 0, 0, 0, 0, 309, 85, 2, 25, 71, 3, 989, 28, 5429, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6293, 6249, 6292, 6008, 6135, 6362, 4872, 6941] avg=6144.0000 max=6941 max_rank=7 min=4872 min_rank=6 imbalance_ratio=1.129720 max_load_reduction=42211 max_load_reduction_pct=0.858785 moved_to_echo=24310 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1426, 63, 177, 2197, 701, 9, 131, 1399, 0, 0, 0, 0, 0, 0, 0, 0, 369, 680, 791, 358, 0, 2247, 15, 1438, 0, 0, 0, 0, 0, 0, 0, 0, 737, 37, 15, 1470, 49, 1083, 623, 846, 1446, 266, 3, 0, 0, 0, 0, 0, 15, 858, 364, 916, 273, 42, 30, 35, 3319, 581, 0, 0, 0, 0, 0, 0, 26, 353, 806, 171, 584, 1, 107, 377, 1655, 1861, 0, 0, 0, 0, 0, 0, 796, 0, 22, 2124, 971, 62, 1996, 246, 0, 0, 0, 0, 0, 0, 0, 0, 128, 364, 98, 944, 1693, 1539, 257, 34, 117, 0, 0, 0, 0, 0, 0, 0, 207, 137, 13, 74, 59, 1, 927, 9, 5384, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6103, 5898, 6575, 6433, 5941, 6217, 5174, 6811] avg=6144.0000 max=6811 max_rank=7 min=5174 min_rank=6 imbalance_ratio=1.108561 max_load_reduction=42341 max_load_reduction_pct=0.861430 moved_to_echo=24143 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[5632, 384, 384, 9088, 384, 5504, 6528, 6912, 12672, 2304, 128, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[11904, 512, 2048, 18304, 6400, 128, 1280, 8832, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[256, 6656, 2432, 8320, 3072, 512, 384, 384, 23424, 4352, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[3584, 6784, 6784, 3456, 0, 16384, 256, 12288, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1408, 7168, 1280, 6272, 17152, 9984, 5248, 384, 896, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6016, 0, 256, 18048, 8448, 512, 14208, 2176, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[256, 2432, 8064, 2176, 5376, 128, 768, 3456, 13824, 13184, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2048, 768, 128, 512, 512, 128, 6144, 128, 39424, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1811, 578, 20, 50, 2091, 38, 486, 155, 43, 335, 4754, 39, 121, 1464, 563, 1011, 2103, 41, 108, 96, 133, 646, 9, 641, 231, 1696, 672, 790, 298, 6771, 2452, 3, 357, 60, 1378, 320, 1760, 355, 245, 3, 1394, 510, 3, 62, 132, 564, 145, 14, 2509, 281, 19, 152, 345, 331, 3079, 814, 111, 769, 17, 628, 14, 767, 1150, 615] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1151, 715, 70, 31, 2298, 16, 278, 320, 16, 153, 5130, 48, 154, 1586, 440, 1311, 2930, 77, 60, 86, 291, 282, 8, 407, 287, 1017, 406, 765, 375, 7273, 2117, 20, 258, 50, 1671, 456, 1698, 255, 83, 9, 1398, 652, 1, 31, 110, 741, 127, 12, 2506, 343, 6, 253, 453, 338, 2833, 1214, 157, 518, 7, 613, 2, 608, 1249, 382] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1500, 461, 19, 74, 2027, 24, 233, 561, 11, 162, 5472, 17, 72, 1396, 469, 1574, 2694, 50, 83, 44, 292, 507, 17, 287, 330, 1072, 610, 856, 210, 6853, 1785, 8, 431, 49, 1431, 553, 1552, 261, 78, 11, 1336, 770, 0, 48, 167, 620, 258, 7, 2500, 338, 13, 59, 285, 351, 2578, 1232, 96, 850, 35, 968, 5, 533, 1317, 650] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1282, 369, 17, 67, 2395, 39, 397, 605, 60, 277, 4833, 40, 164, 1572, 675, 1122, 2234, 66, 155, 49, 280, 819, 11, 299, 127, 1302, 436, 478, 508, 7130, 1573, 19, 580, 28, 869, 394, 1225, 275, 162, 26, 1534, 956, 0, 76, 162, 631, 284, 24, 2766, 282, 32, 159, 723, 260, 2141, 1607, 261, 816, 35, 579, 13, 590, 1517, 745] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1699, 364, 29, 58, 1159, 28, 687, 152, 16, 264, 5338, 20, 122, 1313, 659, 1404, 3381, 65, 71, 103, 201, 655, 11, 264, 161, 1488, 315, 811, 283, 7102, 2033, 15, 293, 48, 1275, 373, 1258, 428, 211, 11, 1166, 1571, 2, 66, 53, 597, 89, 9, 2437, 509, 39, 22, 433, 279, 2547, 1116, 124, 903, 21, 618, 5, 609, 1334, 435] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[2332, 685, 21, 80, 1714, 25, 625, 231, 15, 159, 5242, 15, 45, 1410, 737, 1202, 2696, 90, 76, 54, 390, 382, 8, 163, 312, 1351, 329, 445, 214, 7308, 2249, 5, 298, 56, 844, 391, 1226, 274, 288, 21, 1244, 851, 5, 53, 115, 516, 601, 15, 3110, 280, 3, 22, 131, 296, 2030, 1163, 102, 865, 11, 1106, 26, 502, 1623, 474] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1711, 700, 61, 66, 1421, 46, 215, 294, 14, 234, 5761, 29, 38, 1490, 482, 1635, 1975, 49, 164, 81, 238, 582, 13, 656, 207, 1441, 654, 604, 293, 7191, 1944, 15, 405, 49, 1142, 411, 1923, 318, 97, 10, 1130, 882, 3, 122, 105, 789, 130, 18, 2779, 261, 8, 31, 145, 439, 2159, 1341, 75, 756, 15, 1076, 9, 458, 1143, 619] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1294, 675, 59, 41, 1248, 32, 421, 145, 40, 394, 4649, 28, 25, 1866, 499, 944, 2181, 117, 122, 100, 422, 417, 10, 271, 311, 1490, 278, 683, 457, 7263, 2596, 87, 320, 265, 1090, 350, 1539, 496, 290, 19, 2147, 724, 8, 28, 190, 353, 92, 32, 3059, 818, 26, 203, 254, 405, 2256, 964, 177, 809, 8, 500, 7, 484, 1426, 648] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1151, 715, 70, 31, 2298, 16, 278, 320, 1232, 0, 0, 0, 0, 0, 0, 0, 16, 153, 2388, 48, 154, 1586, 440, 1311, 0, 0, 0, 0, 0, 0, 0, 0, 2930, 77, 60, 86, 291, 282, 8, 407, 2216, 0, 0, 0, 0, 0, 0, 0, 287, 1017, 406, 765, 375, 978, 2117, 20, 0, 0, 0, 0, 0, 0, 0, 0, 258, 50, 1671, 456, 1698, 255, 83, 9, 1180, 873, 0, 0, 0, 0, 0, 0, 1398, 652, 1, 31, 110, 741, 127, 12, 2899, 0, 0, 0, 0, 0, 0, 0, 1210, 343, 6, 253, 453, 338, 2833, 1214, 0, 0, 0, 0, 0, 0, 0, 0, 157, 518, 7, 613, 2, 608, 1249, 382, 1869, 64, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6111, 6096, 6357, 5965, 6533, 5971, 6650, 5469] avg=6144.0000 max=6650 max_rank=6 min=5469 min_rank=7 imbalance_ratio=1.082357 max_load_reduction=42502 max_load_reduction_pct=0.864705 moved_to_echo=24623 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1811, 578, 20, 50, 2091, 38, 486, 155, 1234, 0, 0, 0, 0, 0, 0, 0, 43, 335, 2213, 39, 121, 1464, 563, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 2103, 41, 108, 96, 133, 646, 9, 641, 2063, 0, 0, 0, 0, 0, 0, 0, 231, 1696, 672, 790, 298, 911, 2452, 3, 0, 0, 0, 0, 0, 0, 0, 0, 357, 60, 1378, 320, 1760, 355, 245, 3, 1099, 809, 0, 0, 0, 0, 0, 0, 1394, 510, 3, 62, 132, 564, 145, 14, 2698, 0, 0, 0, 0, 0, 0, 0, 1211, 281, 19, 152, 345, 331, 3079, 814, 0, 0, 0, 0, 0, 0, 0, 0, 111, 769, 17, 628, 14, 767, 1150, 615, 1732, 64, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6463, 5789, 5840, 7053, 6386, 5522, 6232, 5867] avg=6144.0000 max=7053 max_rank=3 min=5522 min_rank=5 imbalance_ratio=1.147949 max_load_reduction=42099 max_load_reduction_pct=0.856506 moved_to_echo=24007 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1500, 461, 19, 74, 2027, 24, 233, 561, 1229, 0, 0, 0, 0, 0, 0, 0, 11, 162, 2547, 17, 72, 1396, 469, 1574, 0, 0, 0, 0, 0, 0, 0, 0, 2694, 50, 83, 44, 292, 507, 17, 287, 2088, 0, 0, 0, 0, 0, 0, 0, 330, 1072, 610, 856, 210, 922, 1785, 8, 0, 0, 0, 0, 0, 0, 0, 0, 431, 49, 1431, 553, 1552, 261, 78, 11, 1112, 931, 0, 0, 0, 0, 0, 0, 1336, 770, 0, 48, 167, 620, 258, 7, 2731, 0, 0, 0, 0, 0, 0, 0, 1207, 338, 13, 59, 285, 351, 2578, 1232, 0, 0, 0, 0, 0, 0, 0, 0, 96, 850, 35, 968, 5, 533, 1317, 650, 1994, 64, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6128, 6248, 6062, 5793, 6409, 5937, 6063, 6512] avg=6144.0000 max=6512 max_rank=7 min=5793 min_rank=3 imbalance_ratio=1.059896 max_load_reduction=42640 max_load_reduction_pct=0.867513 moved_to_echo=24921 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1282, 369, 17, 67, 2395, 39, 397, 605, 1360, 0, 0, 0, 0, 0, 0, 0, 60, 277, 2249, 40, 164, 1572, 675, 1122, 0, 0, 0, 0, 0, 0, 0, 0, 2234, 66, 155, 49, 280, 819, 11, 299, 2173, 0, 0, 0, 0, 0, 0, 0, 127, 1302, 436, 478, 508, 958, 1573, 19, 0, 0, 0, 0, 0, 0, 0, 0, 580, 28, 869, 394, 1225, 275, 162, 26, 1157, 823, 0, 0, 0, 0, 0, 0, 1534, 956, 0, 76, 162, 631, 284, 24, 2842, 0, 0, 0, 0, 0, 0, 0, 1335, 282, 32, 159, 723, 260, 2141, 1607, 0, 0, 0, 0, 0, 0, 0, 0, 261, 816, 35, 579, 13, 590, 1517, 745, 1761, 71, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6531, 6159, 6086, 5401, 5539, 6509, 6539, 6388] avg=6144.0000 max=6539 max_rank=6 min=5401 min_rank=3 imbalance_ratio=1.064290 max_load_reduction=42613 max_load_reduction_pct=0.866964 moved_to_echo=24975 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1699, 364, 29, 58, 1159, 28, 687, 152, 1202, 0, 0, 0, 0, 0, 0, 0, 16, 264, 2476, 20, 122, 1313, 659, 1404, 0, 0, 0, 0, 0, 0, 0, 0, 3381, 65, 71, 103, 201, 655, 11, 264, 2168, 0, 0, 0, 0, 0, 0, 0, 161, 1488, 315, 811, 283, 944, 2033, 15, 0, 0, 0, 0, 0, 0, 0, 0, 293, 48, 1275, 373, 1258, 428, 211, 11, 1157, 913, 0, 0, 0, 0, 0, 0, 1166, 1571, 2, 66, 53, 597, 89, 9, 2833, 0, 0, 0, 0, 0, 0, 0, 1168, 509, 39, 22, 433, 279, 2547, 1116, 0, 0, 0, 0, 0, 0, 0, 0, 124, 903, 21, 618, 5, 609, 1334, 435, 1949, 67, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5378, 6274, 6919, 6050, 5967, 6386, 6113, 6065] avg=6144.0000 max=6919 max_rank=2 min=5378 min_rank=0 imbalance_ratio=1.126139 max_load_reduction=42233 max_load_reduction_pct=0.859233 moved_to_echo=24531 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1711, 700, 61, 66, 1421, 46, 215, 294, 1366, 0, 0, 0, 0, 0, 0, 0, 14, 234, 2681, 29, 38, 1490, 482, 1635, 0, 0, 0, 0, 0, 0, 0, 0, 1975, 49, 164, 81, 238, 582, 13, 656, 2191, 0, 0, 0, 0, 0, 0, 0, 207, 1441, 654, 604, 293, 967, 1944, 15, 0, 0, 0, 0, 0, 0, 0, 0, 405, 49, 1142, 411, 1923, 318, 97, 10, 1167, 981, 0, 0, 0, 0, 0, 0, 1130, 882, 3, 122, 105, 789, 130, 18, 2866, 0, 0, 0, 0, 0, 0, 0, 1342, 261, 8, 31, 145, 439, 2159, 1341, 0, 0, 0, 0, 0, 0, 0, 0, 75, 756, 15, 1076, 9, 458, 1143, 619, 2099, 71, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5880, 6603, 5949, 6125, 6503, 6045, 5726, 6321] avg=6144.0000 max=6603 max_rank=1 min=5726 min_rank=6 imbalance_ratio=1.074707 max_load_reduction=42549 max_load_reduction_pct=0.865662 moved_to_echo=24595 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1294, 675, 59, 41, 1248, 32, 421, 145, 1504, 0, 0, 0, 0, 0, 0, 0, 40, 394, 2164, 28, 25, 1866, 499, 944, 0, 0, 0, 0, 0, 0, 0, 0, 2181, 117, 122, 100, 422, 417, 10, 271, 2213, 0, 0, 0, 0, 0, 0, 0, 311, 1490, 278, 683, 457, 976, 2596, 87, 0, 0, 0, 0, 0, 0, 0, 0, 320, 265, 1090, 350, 1539, 496, 290, 19, 1179, 791, 0, 0, 0, 0, 0, 0, 2147, 724, 8, 28, 190, 353, 92, 32, 2895, 0, 0, 0, 0, 0, 0, 0, 1477, 818, 26, 203, 254, 405, 2256, 964, 0, 0, 0, 0, 0, 0, 0, 0, 177, 809, 8, 500, 7, 484, 1426, 648, 1694, 78, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5419, 5960, 5853, 6878, 6339, 6469, 6403, 5831] avg=6144.0000 max=6878 max_rank=3 min=5419 min_rank=0 imbalance_ratio=1.119466 max_load_reduction=42274 max_load_reduction_pct=0.860067 moved_to_echo=25042 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2332, 685, 21, 80, 1714, 25, 625, 231, 1529, 0, 0, 0, 0, 0, 0, 0, 15, 159, 2440, 15, 45, 1410, 737, 1202, 0, 0, 0, 0, 0, 0, 0, 0, 2696, 90, 76, 54, 390, 382, 8, 163, 2227, 0, 0, 0, 0, 0, 0, 0, 312, 1351, 329, 445, 214, 982, 2249, 5, 0, 0, 0, 0, 0, 0, 0, 0, 298, 56, 844, 391, 1226, 274, 288, 21, 1186, 892, 0, 0, 0, 0, 0, 0, 1244, 851, 5, 53, 115, 516, 601, 15, 2913, 0, 0, 0, 0, 0, 0, 0, 1501, 280, 3, 22, 131, 296, 2030, 1163, 0, 0, 0, 0, 0, 0, 0, 0, 102, 865, 11, 1106, 26, 502, 1623, 474, 1910, 80, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7242, 6023, 6086, 5887, 5476, 6313, 5426, 6699] avg=6144.0000 max=7242 max_rank=0 min=5426 min_rank=6 imbalance_ratio=1.178711 max_load_reduction=41910 max_load_reduction_pct=0.852661 moved_to_echo=23914 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[12800, 4608, 384, 512, 14464, 256, 3456, 2560, 10752, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[256, 2048, 19200, 256, 768, 12160, 4608, 10240, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[10496, 3200, 256, 1024, 2816, 2816, 19712, 9472, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1152, 6400, 256, 6144, 128, 4608, 10880, 4608, 15104, 640, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[2048, 10880, 3712, 5504, 2688, 7680, 16768, 256, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[11392, 7040, 128, 512, 1152, 4864, 1792, 256, 22784, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[20224, 640, 896, 640, 2304, 4352, 128, 3072, 17408, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2944, 640, 9728, 3328, 12288, 2688, 1536, 128, 9344, 7040, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[4217, 342, 326, 639, 21, 244, 903, 1021, 51, 0, 0, 1630, 1, 1796, 1, 778, 56, 1, 1436, 25, 466, 40, 12, 223, 712, 141, 2707, 1, 8, 3648, 1246, 63, 132, 1164, 54, 1798, 192, 794, 485, 1, 16, 4397, 30, 7, 4116, 145, 8, 2128, 209, 12, 9, 241, 346, 1143, 30, 3, 116, 1676, 204, 666, 1, 510, 4876, 888] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[4433, 274, 663, 735, 10, 453, 665, 724, 22, 0, 0, 1584, 2, 1543, 3, 715, 13, 9, 1413, 11, 434, 72, 96, 106, 764, 180, 2876, 4, 6, 3876, 1012, 33, 218, 1107, 79, 1614, 106, 1128, 819, 0, 42, 3235, 47, 18, 3548, 202, 15, 2231, 234, 38, 10, 444, 414, 711, 14, 5, 218, 1558, 392, 506, 4, 627, 5905, 932] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[4546, 259, 177, 442, 27, 417, 782, 1472, 54, 0, 0, 1850, 4, 1000, 1, 326, 60, 21, 872, 10, 405, 51, 83, 175, 611, 178, 2702, 4, 6, 4184, 813, 30, 389, 1578, 137, 1625, 267, 320, 666, 0, 29, 3922, 46, 24, 3994, 235, 6, 2374, 276, 25, 80, 259, 691, 659, 205, 9, 212, 1302, 520, 597, 1, 268, 6122, 752] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[4455, 279, 158, 691, 4, 512, 603, 1707, 61, 0, 0, 1787, 1, 2099, 0, 736, 32, 2, 1712, 7, 233, 36, 54, 96, 1059, 58, 2208, 7, 1, 4045, 757, 16, 88, 1065, 34, 2172, 26, 359, 725, 0, 51, 3425, 39, 8, 4073, 238, 4, 1846, 165, 15, 20, 213, 187, 708, 9, 1, 42, 1995, 309, 509, 1, 988, 5755, 666] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[4288, 412, 545, 570, 34, 318, 762, 1821, 42, 0, 0, 1174, 2, 635, 1, 616, 16, 4, 1170, 36, 449, 53, 44, 132, 975, 71, 2749, 0, 4, 4422, 1056, 20, 98, 1297, 30, 1685, 260, 1111, 585, 0, 44, 3463, 83, 18, 3889, 176, 17, 1990, 215, 10, 14, 351, 670, 971, 31, 6, 199, 1212, 293, 578, 0, 719, 5809, 907] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[4006, 337, 402, 514, 27, 342, 1406, 1033, 98, 0, 0, 1742, 1, 917, 0, 395, 60, 8, 1269, 69, 196, 16, 71, 172, 741, 187, 3014, 6, 9, 3910, 859, 26, 213, 1586, 110, 2193, 296, 658, 695, 0, 36, 3064, 74, 3, 3522, 317, 18, 2639, 410, 51, 13, 446, 490, 939, 22, 14, 111, 2050, 441, 775, 15, 483, 5367, 268] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[4480, 430, 109, 800, 53, 339, 1400, 1888, 60, 0, 0, 2148, 1, 1228, 3, 549, 47, 16, 815, 17, 592, 17, 71, 145, 586, 69, 2463, 5, 0, 3427, 966, 35, 207, 2044, 82, 1624, 113, 447, 696, 1, 28, 4055, 35, 18, 3836, 146, 11, 1939, 183, 8, 31, 266, 285, 707, 124, 1, 135, 1554, 255, 321, 4, 523, 5865, 849] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[4139, 352, 409, 695, 14, 285, 981, 2404, 70, 5, 3, 931, 41, 869, 7, 495, 10, 12, 2070, 26, 472, 51, 55, 151, 1023, 276, 2088, 1, 2, 4004, 1143, 30, 243, 1175, 229, 2055, 187, 839, 643, 7, 61, 2703, 209, 7, 3531, 237, 7, 1862, 179, 31, 90, 399, 460, 801, 111, 6, 320, 2099, 428, 460, 7, 327, 5388, 937] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1943, 342, 326, 639, 21, 244, 903, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 1630, 1, 1796, 1, 778, 1276, 1225, 0, 0, 0, 0, 0, 0, 56, 1, 1436, 25, 466, 40, 12, 223, 2662, 919, 0, 0, 0, 0, 0, 0, 712, 141, 2707, 1, 8, 1453, 1246, 63, 0, 0, 0, 0, 0, 0, 0, 0, 132, 1164, 54, 1798, 192, 794, 485, 1, 1049, 0, 0, 0, 0, 0, 0, 0, 16, 4397, 30, 7, 152, 145, 8, 2128, 0, 0, 0, 0, 0, 0, 0, 0, 209, 12, 9, 241, 346, 1143, 30, 3, 3964, 437, 0, 0, 0, 0, 0, 0, 116, 1676, 204, 666, 1, 510, 1777, 888, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5439, 6758, 5840, 6331, 5669, 6883, 6394, 5838] avg=6144.0000 max=6883 max_rank=5 min=5439 min_rank=0 imbalance_ratio=1.120280 max_load_reduction=42269 max_load_reduction_pct=0.859965 moved_to_echo=24784 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2042, 274, 663, 735, 10, 453, 665, 724, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 1584, 2, 1543, 3, 715, 1356, 1288, 0, 0, 0, 0, 0, 0, 13, 9, 1413, 11, 434, 72, 96, 106, 3224, 976, 0, 0, 0, 0, 0, 0, 764, 180, 2876, 4, 6, 1544, 1012, 33, 0, 0, 0, 0, 0, 0, 0, 0, 218, 1107, 79, 1614, 106, 1128, 819, 0, 1103, 0, 0, 0, 0, 0, 0, 0, 42, 3235, 47, 18, 131, 202, 15, 2231, 0, 0, 0, 0, 0, 0, 0, 0, 234, 38, 10, 444, 414, 711, 14, 5, 3417, 530, 0, 0, 0, 0, 0, 0, 218, 1558, 392, 506, 4, 627, 2151, 932, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5566, 6513, 6354, 6419, 6174, 5921, 5817, 6388] avg=6144.0000 max=6513 max_rank=1 min=5566 min_rank=0 imbalance_ratio=1.060059 max_load_reduction=42639 max_load_reduction_pct=0.867493 moved_to_echo=24300 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2053, 279, 158, 691, 4, 512, 603, 1707, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 1787, 1, 2099, 0, 736, 1415, 1294, 0, 0, 0, 0, 0, 0, 32, 2, 1712, 7, 233, 36, 54, 96, 3142, 1019, 0, 0, 0, 0, 0, 0, 1059, 58, 2208, 7, 1, 1611, 757, 16, 0, 0, 0, 0, 0, 0, 0, 0, 88, 1065, 34, 2172, 26, 359, 725, 0, 1108, 0, 0, 0, 0, 0, 0, 0, 51, 3425, 39, 8, 151, 238, 4, 1846, 0, 0, 0, 0, 0, 0, 0, 0, 165, 15, 20, 213, 187, 708, 9, 1, 3922, 516, 0, 0, 0, 0, 0, 0, 42, 1995, 309, 509, 1, 988, 2097, 666, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6007, 7393, 6333, 5717, 5577, 5762, 5756, 6607] avg=6144.0000 max=7393 max_rank=1 min=5577 min_rank=4 imbalance_ratio=1.203288 max_load_reduction=41759 max_load_reduction_pct=0.849589 moved_to_echo=23702 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2086, 259, 177, 442, 27, 417, 782, 1472, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 1850, 4, 1000, 1, 326, 1469, 1325, 0, 0, 0, 0, 0, 0, 60, 21, 872, 10, 405, 51, 83, 175, 3346, 1056, 0, 0, 0, 0, 0, 0, 611, 178, 2702, 4, 6, 1659, 813, 30, 0, 0, 0, 0, 0, 0, 0, 0, 389, 1578, 137, 1625, 267, 320, 666, 0, 1135, 0, 0, 0, 0, 0, 0, 0, 29, 3922, 46, 24, 143, 235, 6, 2374, 0, 0, 0, 0, 0, 0, 0, 0, 276, 25, 80, 259, 691, 659, 205, 9, 3851, 553, 0, 0, 0, 0, 0, 0, 212, 1302, 520, 597, 1, 268, 2223, 752, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5662, 6029, 6079, 6003, 6117, 6779, 6608, 5875] avg=6144.0000 max=6779 max_rank=5 min=5662 min_rank=0 imbalance_ratio=1.103353 max_load_reduction=42373 max_load_reduction_pct=0.862081 moved_to_echo=25379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1847, 337, 402, 514, 27, 342, 1406, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 1742, 1, 917, 0, 395, 1368, 1163, 0, 0, 0, 0, 0, 0, 60, 8, 1269, 69, 196, 16, 71, 172, 2930, 985, 0, 0, 0, 0, 0, 0, 741, 187, 3014, 6, 9, 1557, 859, 26, 0, 0, 0, 0, 0, 0, 0, 0, 213, 1586, 110, 2193, 296, 658, 695, 0, 996, 0, 0, 0, 0, 0, 0, 0, 36, 3064, 74, 3, 130, 317, 18, 2639, 0, 0, 0, 0, 0, 0, 0, 0, 410, 51, 13, 446, 490, 939, 22, 14, 3392, 481, 0, 0, 0, 0, 0, 0, 111, 2050, 441, 775, 15, 483, 1956, 268, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5908, 5684, 5776, 6399, 6747, 6281, 6258, 6099] avg=6144.0000 max=6747 max_rank=4 min=5684 min_rank=1 imbalance_ratio=1.098145 max_load_reduction=42405 max_load_reduction_pct=0.862732 moved_to_echo=25385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1976, 412, 545, 570, 34, 318, 762, 1821, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 1174, 2, 635, 1, 616, 1547, 1245, 0, 0, 0, 0, 0, 0, 16, 4, 1170, 36, 449, 53, 44, 132, 3171, 1114, 0, 0, 0, 0, 0, 0, 975, 71, 2749, 0, 4, 1761, 1056, 20, 0, 0, 0, 0, 0, 0, 0, 0, 98, 1297, 30, 1685, 260, 1111, 585, 0, 1067, 0, 0, 0, 0, 0, 0, 0, 44, 3463, 83, 18, 144, 176, 17, 1990, 0, 0, 0, 0, 0, 0, 0, 0, 215, 10, 14, 351, 670, 971, 31, 6, 3745, 521, 0, 0, 0, 0, 0, 0, 199, 1212, 293, 578, 0, 719, 2117, 907, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6438, 5262, 6189, 6636, 6133, 5935, 6534, 6025] avg=6144.0000 max=6636 max_rank=3 min=5262 min_rank=1 imbalance_ratio=1.080078 max_load_reduction=42516 max_load_reduction_pct=0.864990 moved_to_echo=24627 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1908, 352, 409, 695, 14, 285, 981, 2404, 0, 0, 0, 0, 0, 0, 0, 0, 70, 5, 3, 931, 41, 869, 7, 495, 1401, 1202, 0, 0, 0, 0, 0, 0, 10, 12, 2070, 26, 472, 51, 55, 151, 2941, 1008, 0, 0, 0, 0, 0, 0, 1023, 276, 2088, 1, 2, 1595, 1143, 30, 0, 0, 0, 0, 0, 0, 0, 0, 243, 1175, 229, 2055, 187, 839, 643, 7, 1029, 0, 0, 0, 0, 0, 0, 0, 61, 2703, 209, 7, 131, 237, 7, 1862, 0, 0, 0, 0, 0, 0, 0, 0, 179, 31, 90, 399, 460, 801, 111, 6, 3400, 483, 0, 0, 0, 0, 0, 0, 320, 2099, 428, 460, 7, 327, 1964, 937, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7048, 5024, 6796, 6158, 6407, 5217, 5960, 6542] avg=6144.0000 max=7048 max_rank=0 min=5024 min_rank=1 imbalance_ratio=1.147135 max_load_reduction=42104 max_load_reduction_pct=0.856608 moved_to_echo=24126 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2065, 430, 109, 800, 53, 339, 1400, 1888, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 2148, 1, 1228, 3, 549, 1199, 1301, 0, 0, 0, 0, 0, 0, 47, 16, 815, 17, 592, 17, 71, 145, 3202, 863, 0, 0, 0, 0, 0, 0, 586, 69, 2463, 5, 0, 1365, 966, 35, 0, 0, 0, 0, 0, 0, 0, 0, 207, 2044, 82, 1624, 113, 447, 696, 1, 1114, 0, 0, 0, 0, 0, 0, 0, 28, 4055, 35, 18, 142, 146, 11, 1939, 0, 0, 0, 0, 0, 0, 0, 0, 183, 8, 31, 266, 285, 707, 124, 1, 3694, 526, 0, 0, 0, 0, 0, 0, 135, 1554, 255, 321, 4, 523, 2137, 849, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7084, 6489, 5785, 5489, 6328, 6374, 5825, 5778] avg=6144.0000 max=7084 max_rank=0 min=5489 min_rank=3 imbalance_ratio=1.152995 max_load_reduction=42068 max_load_reduction_pct=0.855876 moved_to_echo=24305 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1408, 13568, 2944, 4480, 128, 4480, 16512, 6272, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[512, 128, 128, 12928, 128, 10112, 128, 4736, 11136, 10112, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6528, 1280, 20864, 128, 128, 12672, 7936, 256, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[16000, 2688, 2816, 5120, 256, 2944, 7552, 12160, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[384, 28288, 640, 128, 1152, 1792, 128, 17024, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[384, 128, 10880, 256, 3328, 384, 512, 1280, 24704, 8064, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1664, 11136, 768, 14848, 1536, 5760, 5376, 128, 8704, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1920, 256, 384, 2688, 3584, 6656, 640, 128, 29440, 4096, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[554, 183, 313, 541, 160, 2048, 808, 9, 55, 352, 515, 9, 45, 1344, 63, 267, 41, 19, 4, 1009, 5, 1, 497, 1682, 765, 4068, 667, 559, 2908, 2906, 3210, 506, 78, 1520, 899, 254, 1998, 537, 34, 118, 112, 2011, 97, 3, 602, 7, 0, 0, 4506, 370, 43, 3386, 56, 22, 28, 210, 11, 2994, 502, 320, 2212, 0, 0, 109] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[238, 116, 168, 1187, 156, 957, 1091, 4, 4, 377, 698, 76, 72, 672, 33, 242, 31, 13, 4, 661, 8, 1, 277, 1097, 940, 4282, 539, 869, 2480, 4327, 3382, 329, 10, 1794, 793, 285, 1833, 151, 30, 84, 49, 1642, 72, 1, 915, 0, 0, 1, 5845, 523, 22, 2910, 57, 19, 9, 185, 2, 2820, 301, 52, 3343, 0, 0, 73] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[817, 189, 243, 1077, 174, 933, 1071, 34, 33, 651, 605, 46, 42, 1080, 53, 218, 169, 26, 3, 900, 8, 12, 907, 1461, 1147, 4276, 834, 341, 1958, 3561, 2368, 275, 25, 2123, 824, 245, 1256, 1071, 146, 90, 158, 1795, 57, 23, 1048, 0, 0, 0, 5681, 477, 41, 2570, 54, 75, 36, 222, 5, 2743, 265, 102, 2390, 1, 2, 115] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[700, 112, 598, 1015, 186, 2170, 659, 41, 100, 507, 583, 54, 156, 1003, 126, 92, 35, 19, 10, 1103, 12, 2, 389, 1147, 1231, 3466, 1710, 181, 2748, 2420, 3202, 168, 37, 1796, 843, 258, 1674, 254, 63, 116, 127, 1829, 98, 11, 759, 0, 0, 0, 5489, 648, 21, 2615, 108, 58, 47, 186, 7, 3139, 324, 283, 2292, 0, 1, 124] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[626, 157, 193, 448, 115, 973, 715, 9, 14, 376, 649, 56, 76, 883, 58, 153, 17, 19, 1, 1059, 19, 5, 580, 1822, 1937, 3638, 1107, 319, 2537, 3562, 2942, 384, 59, 1683, 1018, 265, 2153, 269, 46, 80, 106, 2488, 26, 7, 719, 0, 0, 2, 4726, 540, 20, 2571, 32, 2, 45, 205, 11, 2771, 446, 144, 3126, 1, 0, 142] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[495, 203, 147, 1688, 370, 1295, 895, 37, 27, 292, 747, 49, 30, 285, 49, 272, 40, 16, 4, 713, 4, 2, 452, 1194, 578, 3851, 834, 741, 2459, 3152, 3377, 252, 28, 1041, 1110, 238, 1438, 1076, 61, 147, 92, 1787, 126, 5, 1294, 5, 0, 5, 5238, 466, 15, 2825, 57, 21, 22, 243, 11, 3250, 56, 672, 3149, 0, 1, 123] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[464, 330, 242, 2236, 588, 1560, 924, 26, 34, 93, 553, 57, 60, 352, 61, 140, 73, 13, 3, 1189, 6, 7, 339, 1632, 1168, 3140, 941, 797, 1673, 3710, 2698, 382, 152, 1392, 550, 240, 2790, 230, 83, 126, 200, 1789, 61, 25, 1400, 0, 0, 0, 5027, 539, 67, 3403, 47, 20, 25, 159, 12, 2631, 215, 46, 2333, 6, 0, 93] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[921, 126, 467, 1236, 242, 1724, 925, 20, 18, 699, 1051, 64, 44, 823, 60, 125, 132, 112, 5, 1499, 67, 4, 579, 1822, 1104, 3476, 1173, 698, 2690, 2945, 2719, 504, 81, 1700, 1048, 244, 995, 186, 55, 99, 149, 1807, 48, 44, 976, 2, 21, 3, 5213, 630, 93, 1700, 71, 29, 50, 258, 5, 2953, 253, 389, 1855, 3, 2, 116] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[238, 116, 168, 1187, 156, 957, 1091, 4, 1443, 0, 0, 0, 0, 0, 0, 0, 4, 377, 698, 76, 72, 672, 33, 242, 4282, 86, 0, 0, 0, 0, 0, 0, 31, 13, 4, 661, 8, 1, 277, 1097, 3043, 244, 133, 0, 0, 0, 0, 0, 940, 0, 539, 869, 2480, 0, 811, 329, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1794, 793, 285, 1833, 151, 30, 84, 995, 23, 0, 0, 0, 0, 0, 0, 49, 1642, 72, 1, 915, 0, 0, 1, 3997, 0, 0, 0, 0, 0, 0, 0, 2802, 523, 22, 2910, 57, 19, 9, 185, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2797, 301, 52, 3343, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5360, 6542, 5512, 5968, 5998, 6677, 6527, 6568] avg=6144.0000 max=6677 max_rank=5 min=5360 min_rank=0 imbalance_ratio=1.086751 max_load_reduction=42475 max_load_reduction_pct=0.864156 moved_to_echo=25770 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[554, 183, 313, 541, 160, 2048, 808, 9, 1370, 0, 0, 0, 0, 0, 0, 0, 55, 352, 515, 9, 45, 1344, 63, 267, 4068, 58, 0, 0, 0, 0, 0, 0, 41, 19, 4, 1009, 5, 1, 497, 1682, 2346, 164, 126, 0, 0, 0, 0, 0, 765, 0, 667, 559, 2908, 0, 770, 506, 0, 0, 0, 0, 0, 0, 0, 0, 78, 1520, 899, 254, 1998, 537, 34, 118, 944, 25, 0, 0, 0, 0, 0, 0, 112, 2011, 97, 3, 602, 7, 0, 0, 2684, 0, 0, 0, 0, 0, 0, 0, 2160, 370, 43, 3386, 56, 22, 28, 210, 0, 0, 0, 0, 0, 0, 0, 0, 11, 2969, 502, 320, 2212, 0, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5986, 6776, 5894, 6175, 6407, 5516, 6275, 6123] avg=6144.0000 max=6776 max_rank=1 min=5516 min_rank=5 imbalance_ratio=1.102865 max_load_reduction=42376 max_load_reduction_pct=0.862142 moved_to_echo=24321 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[817, 189, 243, 1077, 174, 933, 1071, 34, 1010, 0, 0, 0, 0, 0, 0, 0, 33, 651, 605, 46, 42, 1080, 53, 218, 4276, 71, 0, 0, 0, 0, 0, 0, 169, 26, 3, 900, 8, 12, 907, 1461, 2958, 201, 93, 0, 0, 0, 0, 0, 1147, 0, 834, 341, 1958, 0, 568, 275, 0, 0, 0, 0, 0, 0, 0, 0, 25, 2123, 824, 245, 1256, 1071, 146, 90, 697, 22, 0, 0, 0, 0, 0, 0, 158, 1795, 57, 23, 1048, 0, 0, 0, 3289, 0, 0, 0, 0, 0, 0, 0, 2723, 477, 41, 2570, 54, 75, 36, 222, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2721, 265, 102, 2390, 1, 2, 115, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5548, 7075, 6738, 5123, 6499, 6370, 6198, 5601] avg=6144.0000 max=7075 max_rank=1 min=5123 min_rank=3 imbalance_ratio=1.151530 max_load_reduction=42077 max_load_reduction_pct=0.856059 moved_to_echo=24668 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[626, 157, 193, 448, 115, 973, 715, 9, 1256, 0, 0, 0, 0, 0, 0, 0, 14, 376, 649, 56, 76, 883, 58, 153, 3638, 72, 0, 0, 0, 0, 0, 0, 17, 19, 1, 1059, 19, 5, 580, 1822, 2460, 201, 115, 0, 0, 0, 0, 0, 1937, 0, 1107, 319, 2537, 0, 705, 384, 0, 0, 0, 0, 0, 0, 0, 0, 59, 1683, 1018, 265, 2153, 269, 46, 80, 866, 23, 0, 0, 0, 0, 0, 0, 106, 2488, 26, 7, 719, 0, 0, 2, 3289, 0, 0, 0, 0, 0, 0, 0, 2266, 540, 20, 2571, 32, 2, 45, 205, 0, 0, 0, 0, 0, 0, 0, 0, 11, 2748, 446, 144, 3126, 1, 0, 142, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4492, 5975, 6298, 6989, 6462, 6637, 5681, 6618] avg=6144.0000 max=6989 max_rank=3 min=4492 min_rank=0 imbalance_ratio=1.137533 max_load_reduction=42163 max_load_reduction_pct=0.857808 moved_to_echo=25398 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[495, 203, 147, 1688, 370, 1295, 895, 37, 1441, 0, 0, 0, 0, 0, 0, 0, 27, 292, 747, 49, 30, 285, 49, 272, 3851, 63, 0, 0, 0, 0, 0, 0, 40, 16, 4, 713, 4, 2, 452, 1194, 2727, 179, 132, 0, 0, 0, 0, 0, 578, 0, 834, 741, 2459, 0, 810, 252, 0, 0, 0, 0, 0, 0, 0, 0, 28, 1041, 1110, 238, 1438, 1076, 61, 147, 994, 27, 0, 0, 0, 0, 0, 0, 92, 1787, 126, 5, 1294, 5, 0, 5, 2910, 0, 0, 0, 0, 0, 0, 0, 2511, 466, 15, 2825, 57, 21, 22, 243, 0, 0, 0, 0, 0, 0, 0, 0, 11, 3223, 56, 672, 3149, 0, 1, 123, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6571, 5665, 5463, 5674, 6160, 6224, 6160, 7235] avg=6144.0000 max=7235 max_rank=7 min=5463 min_rank=2 imbalance_ratio=1.177572 max_load_reduction=41917 max_load_reduction_pct=0.852804 moved_to_echo=25779 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[464, 330, 242, 2236, 588, 1560, 924, 26, 1156, 0, 0, 0, 0, 0, 0, 0, 34, 93, 553, 57, 60, 352, 61, 140, 3140, 76, 0, 0, 0, 0, 0, 0, 73, 13, 3, 1189, 6, 7, 339, 1632, 2620, 209, 109, 0, 0, 0, 0, 0, 1168, 0, 941, 797, 1673, 0, 636, 382, 0, 0, 0, 0, 0, 0, 0, 0, 152, 1392, 550, 240, 2790, 230, 83, 126, 797, 25, 0, 0, 0, 0, 0, 0, 200, 1789, 61, 25, 1400, 0, 0, 0, 3425, 0, 0, 0, 0, 0, 0, 0, 2407, 539, 67, 3403, 47, 20, 25, 159, 0, 0, 0, 0, 0, 0, 0, 0, 12, 2606, 215, 46, 2333, 6, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7526, 4566, 6200, 5597, 6385, 6900, 6667, 5311] avg=6144.0000 max=7526 max_rank=0 min=4566 min_rank=1 imbalance_ratio=1.224935 max_load_reduction=41626 max_load_reduction_pct=0.846883 moved_to_echo=25263 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[921, 126, 467, 1236, 242, 1724, 925, 20, 1160, 0, 0, 0, 0, 0, 0, 0, 18, 699, 1051, 64, 44, 823, 60, 125, 3476, 58, 0, 0, 0, 0, 0, 0, 132, 112, 5, 1499, 67, 4, 579, 1822, 2714, 168, 107, 0, 0, 0, 0, 0, 1104, 0, 1173, 698, 2690, 0, 652, 504, 0, 0, 0, 0, 0, 0, 0, 0, 81, 1700, 1048, 244, 995, 186, 55, 99, 800, 24, 0, 0, 0, 0, 0, 0, 149, 1807, 48, 44, 976, 2, 21, 3, 2719, 0, 0, 0, 0, 0, 0, 0, 2499, 630, 93, 1700, 71, 29, 50, 258, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2929, 253, 389, 1855, 3, 2, 116, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6821, 6418, 7209, 6821, 5232, 5769, 5330, 5552] avg=6144.0000 max=7209 max_rank=2 min=5232 min_rank=4 imbalance_ratio=1.173340 max_load_reduction=41943 max_load_reduction_pct=0.853333 moved_to_echo=21883 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[700, 112, 598, 1015, 186, 2170, 659, 41, 1367, 0, 0, 0, 0, 0, 0, 0, 100, 507, 583, 54, 156, 1003, 126, 92, 3466, 48, 0, 0, 0, 0, 0, 0, 35, 19, 10, 1103, 12, 2, 389, 1147, 2858, 137, 126, 0, 0, 0, 0, 0, 1231, 0, 1710, 181, 2748, 0, 767, 168, 0, 0, 0, 0, 0, 0, 0, 0, 37, 1796, 843, 258, 1674, 254, 63, 116, 942, 26, 0, 0, 0, 0, 0, 0, 127, 1829, 98, 11, 759, 0, 0, 0, 2235, 0, 0, 0, 0, 0, 0, 0, 2631, 648, 21, 2615, 108, 58, 47, 186, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3113, 324, 283, 2292, 0, 1, 124, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6848, 6135, 5838, 6805, 6009, 5059, 6314, 6144] avg=6144.0000 max=6848 max_rank=0 min=5059 min_rank=5 imbalance_ratio=1.114583 max_load_reduction=42304 max_load_reduction_pct=0.860677 moved_to_echo=23526 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[8960, 0, 7808, 4608, 19456, 0, 5760, 2816, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4864, 1536, 2432, 9472, 2048, 11776, 7168, 256, 10240, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[640, 256, 128, 8192, 256, 128, 4096, 11904, 21760, 1536, 1024, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[20096, 4224, 384, 22016, 512, 256, 384, 1792, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[512, 13056, 7168, 2048, 14208, 3840, 640, 896, 7040, 256, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 23168, 2432, 2048, 20736, 128, 128, 896, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1024, 15232, 640, 128, 7808, 128, 128, 128, 24576, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[384, 3456, 5504, 512, 640, 6528, 512, 1536, 30208, 640, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[141, 622, 246, 335, 1737, 4076, 72, 2, 171, 1053, 436, 11, 500, 229, 2283, 85, 118, 136, 119, 3779, 0, 324, 361, 0, 165, 142, 238, 279, 5876, 195, 18, 2213, 26, 149, 57, 18, 225, 938, 1, 0, 1169, 1668, 0, 5561, 1015, 12, 301, 1773, 0, 3599, 43, 493, 16, 0, 170, 2718, 19, 28, 647, 2233, 215, 11, 60, 25] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[59, 786, 285, 289, 1949, 4084, 36, 8, 116, 1823, 177, 21, 454, 355, 2444, 141, 49, 67, 27, 3493, 0, 98, 282, 0, 273, 15, 168, 188, 5634, 420, 27, 2886, 45, 96, 63, 41, 324, 1026, 0, 1, 1464, 1484, 0, 4819, 1219, 8, 24, 1207, 2, 4312, 45, 653, 32, 0, 22, 2013, 94, 13, 1207, 1919, 210, 5, 120, 30] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[189, 1046, 340, 282, 1059, 4816, 12, 23, 451, 992, 254, 12, 295, 707, 2613, 150, 31, 96, 126, 1862, 0, 311, 338, 8, 323, 197, 313, 163, 4821, 374, 76, 2655, 43, 174, 100, 62, 237, 988, 0, 4, 1991, 2453, 3, 4901, 1155, 19, 52, 2041, 0, 3824, 34, 677, 24, 1, 49, 1407, 108, 103, 1346, 2208, 108, 30, 37, 38] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[263, 550, 314, 302, 1439, 4324, 102, 4, 304, 2054, 519, 30, 647, 731, 1655, 207, 38, 16, 180, 2337, 0, 250, 403, 0, 317, 136, 130, 113, 5017, 378, 11, 3126, 26, 168, 107, 59, 567, 1290, 0, 0, 1472, 1957, 2, 5359, 1006, 29, 89, 1470, 1, 3961, 33, 525, 27, 1, 114, 1819, 61, 34, 842, 2001, 135, 47, 36, 17] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[105, 731, 183, 280, 2263, 3926, 17, 1, 199, 2013, 1219, 43, 223, 250, 2059, 116, 39, 85, 146, 3047, 0, 266, 196, 2, 449, 238, 216, 35, 4935, 155, 4, 2791, 13, 115, 17, 52, 475, 2630, 0, 0, 1507, 1163, 0, 5533, 722, 49, 25, 890, 0, 3955, 33, 668, 13, 0, 70, 1630, 102, 8, 728, 2257, 188, 5, 39, 33] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[87, 1030, 202, 300, 1927, 4691, 11, 3, 216, 2097, 585, 21, 325, 469, 2188, 215, 107, 38, 100, 2183, 0, 204, 649, 1, 250, 125, 106, 104, 5150, 278, 4, 3320, 11, 108, 43, 3, 636, 1090, 0, 0, 1550, 1361, 1, 5114, 786, 53, 94, 1013, 0, 4250, 36, 382, 181, 1, 59, 1798, 38, 78, 734, 2563, 143, 11, 22, 7] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[241, 761, 482, 294, 1776, 4474, 32, 21, 120, 1448, 544, 27, 618, 388, 1782, 224, 152, 31, 199, 2408, 0, 400, 168, 13, 403, 219, 132, 318, 5259, 518, 4, 2277, 16, 141, 87, 0, 545, 858, 1, 0, 2044, 1486, 1, 5427, 863, 51, 64, 1586, 1, 3729, 26, 737, 31, 3, 79, 2460, 65, 6, 789, 1992, 248, 10, 58, 15] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[123, 1055, 672, 491, 2189, 4495, 47, 9, 321, 1973, 602, 78, 380, 430, 1537, 321, 384, 93, 184, 1977, 16, 161, 235, 0, 314, 173, 106, 289, 3173, 397, 7, 3216, 16, 167, 54, 48, 1056, 2109, 3, 33, 1003, 1210, 0, 4803, 1060, 9, 117, 1361, 0, 4191, 27, 1252, 275, 2, 48, 2202, 119, 30, 589, 1594, 237, 4, 73, 12] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[141, 622, 246, 335, 1737, 2492, 72, 2, 0, 0, 0, 0, 0, 0, 0, 0, 171, 1053, 436, 11, 500, 229, 2283, 85, 475, 0, 0, 0, 0, 0, 0, 0, 118, 136, 119, 3779, 0, 324, 361, 0, 1584, 863, 158, 0, 0, 0, 0, 0, 165, 142, 238, 279, 2531, 195, 18, 2213, 0, 0, 0, 0, 0, 0, 0, 0, 26, 149, 57, 18, 225, 938, 1, 0, 4284, 0, 0, 0, 0, 0, 0, 0, 1169, 1668, 0, 537, 1015, 12, 301, 1773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2966, 43, 493, 16, 0, 170, 2718, 0, 0, 0, 0, 0, 0, 0, 0, 19, 28, 647, 2233, 215, 11, 60, 25, 2482, 740, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5647, 5243, 7442, 5781, 5698, 6475, 6406, 6460] avg=6144.0000 max=7442 max_rank=2 min=5243 min_rank=1 imbalance_ratio=1.211263 max_load_reduction=41710 max_load_reduction_pct=0.848592 moved_to_echo=25039 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[59, 786, 285, 289, 1949, 2497, 36, 8, 0, 0, 0, 0, 0, 0, 0, 0, 116, 1823, 177, 21, 454, 355, 2444, 141, 569, 0, 0, 0, 0, 0, 0, 0, 49, 67, 27, 3493, 0, 98, 282, 0, 1587, 828, 189, 0, 0, 0, 0, 0, 273, 15, 168, 188, 2426, 420, 27, 2886, 0, 0, 0, 0, 0, 0, 0, 0, 45, 96, 63, 41, 324, 1026, 0, 1, 3713, 0, 0, 0, 0, 0, 0, 0, 1464, 1484, 0, 465, 1219, 8, 24, 1207, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3554, 45, 653, 32, 0, 22, 2013, 0, 0, 0, 0, 0, 0, 0, 0, 94, 13, 1207, 1919, 210, 5, 120, 30, 2380, 641, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5909, 6100, 6620, 6403, 5309, 5871, 6321, 6619] avg=6144.0000 max=6620 max_rank=2 min=5309 min_rank=4 imbalance_ratio=1.077474 max_load_reduction=42532 max_load_reduction_pct=0.865316 moved_to_echo=24120 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[263, 550, 314, 302, 1439, 2644, 102, 4, 0, 0, 0, 0, 0, 0, 0, 0, 304, 2054, 519, 30, 647, 731, 1655, 207, 522, 0, 0, 0, 0, 0, 0, 0, 38, 16, 180, 2337, 0, 250, 403, 0, 1680, 737, 174, 0, 0, 0, 0, 0, 317, 136, 130, 113, 2161, 378, 11, 3126, 0, 0, 0, 0, 0, 0, 0, 0, 26, 168, 107, 59, 567, 1290, 0, 0, 4129, 0, 0, 0, 0, 0, 0, 0, 1472, 1957, 2, 517, 1006, 29, 89, 1470, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3265, 33, 525, 27, 1, 114, 1819, 0, 0, 0, 0, 0, 0, 0, 0, 61, 34, 842, 2001, 135, 47, 36, 17, 2119, 713, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5618, 6669, 5815, 6372, 6346, 6542, 5785, 6005] avg=6144.0000 max=6669 max_rank=1 min=5618 min_rank=0 imbalance_ratio=1.085449 max_load_reduction=42483 max_load_reduction_pct=0.864319 moved_to_echo=24678 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[189, 1046, 340, 282, 1059, 2945, 12, 23, 0, 0, 0, 0, 0, 0, 0, 0, 451, 992, 254, 12, 295, 707, 2613, 150, 504, 0, 0, 0, 0, 0, 0, 0, 31, 96, 126, 1862, 0, 311, 338, 8, 1871, 708, 168, 0, 0, 0, 0, 0, 323, 197, 313, 163, 2077, 374, 76, 2655, 0, 0, 0, 0, 0, 0, 0, 0, 43, 174, 100, 62, 237, 988, 0, 4, 3776, 0, 0, 0, 0, 0, 0, 0, 1991, 2453, 3, 473, 1155, 19, 52, 2041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3152, 34, 677, 24, 1, 49, 1407, 0, 0, 0, 0, 0, 0, 0, 0, 108, 103, 1346, 2208, 108, 30, 37, 38, 2036, 652, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5896, 5978, 5519, 6178, 5384, 8187, 5344, 6666] avg=6144.0000 max=8187 max_rank=5 min=5344 min_rank=6 imbalance_ratio=1.332520 max_load_reduction=40965 max_load_reduction_pct=0.833435 moved_to_echo=25581 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[87, 1030, 202, 300, 1927, 2868, 11, 3, 0, 0, 0, 0, 0, 0, 0, 0, 216, 2097, 585, 21, 325, 469, 2188, 215, 561, 0, 0, 0, 0, 0, 0, 0, 107, 38, 100, 2183, 0, 204, 649, 1, 1823, 756, 187, 0, 0, 0, 0, 0, 250, 125, 106, 104, 2219, 278, 4, 3320, 0, 0, 0, 0, 0, 0, 0, 0, 11, 108, 43, 3, 636, 1090, 0, 0, 3940, 0, 0, 0, 0, 0, 0, 0, 1550, 1361, 1, 494, 786, 53, 94, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3502, 36, 382, 181, 1, 59, 1798, 0, 0, 0, 0, 0, 0, 0, 0, 38, 78, 734, 2563, 143, 11, 22, 7, 2175, 680, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6428, 6677, 6048, 6406, 5831, 5352, 5959, 6451] avg=6144.0000 max=6677 max_rank=1 min=5352 min_rank=5 imbalance_ratio=1.086751 max_load_reduction=42475 max_load_reduction_pct=0.864156 moved_to_echo=23593 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[105, 731, 183, 280, 2263, 2397, 17, 1, 0, 0, 0, 0, 0, 0, 0, 0, 199, 2013, 1219, 43, 223, 250, 2059, 116, 525, 0, 0, 0, 0, 0, 0, 0, 39, 85, 146, 3047, 0, 266, 196, 2, 1529, 729, 177, 0, 0, 0, 0, 0, 449, 238, 216, 35, 2118, 155, 4, 2791, 0, 0, 0, 0, 0, 0, 0, 0, 13, 115, 17, 52, 475, 2630, 0, 0, 4267, 0, 0, 0, 0, 0, 0, 0, 1507, 1163, 0, 526, 722, 49, 25, 890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3253, 33, 668, 13, 0, 70, 1630, 0, 0, 0, 0, 0, 0, 0, 0, 102, 8, 728, 2257, 188, 5, 39, 33, 2088, 740, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5977, 6647, 6216, 6006, 7569, 4882, 5667, 6188] avg=6144.0000 max=7569 max_rank=4 min=4882 min_rank=5 imbalance_ratio=1.231934 max_load_reduction=41583 max_load_reduction_pct=0.846008 moved_to_echo=24306 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[123, 1055, 672, 491, 2189, 2748, 47, 9, 0, 0, 0, 0, 0, 0, 0, 0, 321, 1973, 602, 78, 380, 430, 1537, 321, 553, 0, 0, 0, 0, 0, 0, 0, 384, 93, 184, 1977, 16, 161, 235, 0, 1747, 466, 184, 0, 0, 0, 0, 0, 314, 173, 106, 289, 1367, 397, 7, 3216, 0, 0, 0, 0, 0, 0, 0, 0, 16, 167, 54, 48, 1056, 2109, 3, 33, 3700, 0, 0, 0, 0, 0, 0, 0, 1003, 1210, 0, 464, 1060, 9, 117, 1361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3454, 27, 1252, 275, 2, 48, 2202, 0, 0, 0, 0, 0, 0, 0, 0, 119, 30, 589, 1594, 237, 4, 73, 12, 1340, 639, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7334, 6195, 5447, 5869, 7186, 5224, 7260, 4637] avg=6144.0000 max=7334 max_rank=0 min=4637 min_rank=7 imbalance_ratio=1.193685 max_load_reduction=41818 max_load_reduction_pct=0.850789 moved_to_echo=24307 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[241, 761, 482, 294, 1776, 2736, 32, 21, 0, 0, 0, 0, 0, 0, 0, 0, 120, 1448, 544, 27, 618, 388, 1782, 224, 492, 0, 0, 0, 0, 0, 0, 0, 152, 31, 199, 2408, 0, 400, 168, 13, 1738, 772, 164, 0, 0, 0, 0, 0, 403, 219, 132, 318, 2266, 518, 4, 2277, 0, 0, 0, 0, 0, 0, 0, 0, 16, 141, 87, 0, 545, 858, 1, 0, 4181, 0, 0, 0, 0, 0, 0, 0, 2044, 1486, 1, 524, 863, 51, 64, 1586, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3073, 26, 737, 31, 3, 79, 2460, 0, 0, 0, 0, 0, 0, 0, 0, 65, 6, 789, 1992, 248, 10, 58, 15, 2221, 722, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6343, 5643, 6045, 6137, 5829, 6619, 6410, 6126] avg=6144.0000 max=6619 max_rank=5 min=5643 min_rank=1 imbalance_ratio=1.077311 max_load_reduction=42533 max_load_reduction_pct=0.865336 moved_to_echo=24984 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1024, 640, 1152, 21120, 128, 2048, 2688, 128, 13568, 5888, 1408, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[1920, 13568, 4352, 256, 3456, 3584, 16640, 1536, 4224, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2560, 1280, 1536, 1536, 17280, 2816, 256, 22528, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[12288, 12800, 128, 4096, 7936, 256, 768, 11392, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1280, 6656, 2816, 2688, 14464, 21376, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 26240, 384, 5504, 640, 128, 640, 16128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[640, 384, 6912, 16768, 1536, 128, 512, 256, 16896, 5632, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[256, 1152, 640, 384, 4096, 11008, 128, 128, 32000, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[2, 2231, 840, 4202, 107, 31, 367, 0, 90, 13, 97, 75, 413, 0, 1624, 49, 1604, 2, 563, 32, 660, 285, 46, 930, 197, 2244, 13, 75, 70, 1130, 441, 326, 4222, 99, 271, 378, 407, 0, 874, 1800, 145, 10, 1242, 133, 0, 4, 4, 149, 443, 295, 5596, 1056, 6680, 0, 773, 0, 591, 63, 0, 3654, 432, 34, 295, 743] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[2, 2499, 471, 4308, 128, 39, 254, 10, 115, 8, 147, 20, 394, 1, 1528, 48, 843, 1, 451, 18, 522, 453, 33, 1718, 193, 1822, 13, 133, 67, 1449, 450, 248, 3512, 175, 305, 403, 605, 5, 543, 1122, 349, 19, 760, 363, 10, 62, 4, 158, 270, 219, 4167, 1382, 7030, 36, 1741, 0, 1076, 63, 0, 4191, 624, 124, 362, 1086] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[14, 3224, 599, 4553, 188, 90, 238, 4, 142, 10, 90, 122, 1163, 0, 1873, 14, 1203, 0, 654, 79, 687, 335, 106, 1926, 597, 628, 47, 57, 86, 1644, 447, 352, 4026, 198, 243, 465, 468, 4, 1117, 709, 131, 3, 1325, 211, 6, 10, 6, 259, 267, 589, 3967, 1625, 5889, 1, 1174, 5, 442, 22, 3, 2316, 414, 84, 494, 1507] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 1738, 752, 4700, 66, 28, 264, 14, 158, 9, 92, 109, 799, 1, 2402, 18, 1700, 9, 747, 44, 384, 1017, 82, 2321, 222, 1392, 6, 129, 453, 1572, 249, 258, 2752, 157, 94, 328, 268, 8, 914, 2184, 248, 16, 845, 200, 11, 4, 2, 169, 413, 120, 3210, 1729, 6601, 2, 841, 0, 472, 70, 0, 3464, 471, 95, 394, 1335] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[7, 3509, 729, 4422, 108, 55, 74, 12, 150, 11, 129, 53, 623, 0, 2364, 68, 814, 0, 459, 122, 311, 330, 26, 2488, 367, 1043, 25, 50, 208, 1395, 346, 645, 2587, 111, 107, 367, 333, 1, 1001, 1441, 106, 7, 1320, 293, 3, 46, 22, 97, 219, 212, 5070, 1639, 6301, 3, 979, 2, 594, 150, 1, 3350, 479, 179, 163, 1026] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[2, 1701, 117, 5125, 171, 41, 158, 10, 198, 27, 291, 152, 898, 0, 1401, 90, 1379, 4, 393, 48, 345, 434, 33, 2718, 224, 1086, 5, 69, 190, 1065, 304, 254, 3066, 145, 324, 257, 282, 5, 2274, 2483, 93, 9, 1255, 355, 8, 8, 0, 150, 894, 191, 4342, 1345, 6796, 0, 604, 1, 1007, 37, 0, 2546, 437, 126, 223, 956] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[3, 2031, 497, 3759, 85, 127, 61, 1, 135, 6, 380, 59, 605, 17, 2288, 7, 2518, 3, 756, 132, 415, 487, 23, 2436, 265, 1081, 10, 58, 556, 1903, 172, 535, 3708, 96, 214, 173, 220, 18, 1524, 1795, 119, 14, 1450, 183, 15, 26, 31, 147, 141, 106, 4141, 1022, 5847, 0, 1089, 0, 483, 75, 0, 2778, 437, 129, 143, 1617] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[32, 2287, 216, 3327, 167, 108, 164, 32, 156, 10, 1208, 96, 386, 5, 2360, 67, 667, 35, 653, 86, 903, 709, 55, 2430, 356, 793, 43, 48, 210, 1108, 458, 336, 3524, 279, 261, 405, 462, 3, 1720, 2468, 101, 10, 495, 364, 8, 14, 20, 414, 843, 262, 3890, 1633, 6440, 6, 898, 2, 575, 55, 0, 2355, 385, 145, 149, 1455] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2, 2231, 840, 2743, 107, 31, 367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 13, 97, 75, 413, 0, 1624, 49, 2133, 832, 0, 0, 0, 0, 0, 0, 1604, 2, 563, 32, 660, 285, 46, 930, 1218, 0, 0, 0, 0, 0, 0, 0, 197, 2244, 13, 75, 70, 1130, 441, 326, 1719, 627, 104, 0, 0, 0, 0, 0, 2503, 99, 271, 378, 407, 0, 874, 1800, 0, 0, 0, 0, 0, 0, 0, 0, 145, 10, 1242, 133, 0, 4, 4, 149, 4547, 0, 0, 0, 0, 0, 0, 0, 443, 295, 3920, 1056, 0, 0, 773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 591, 63, 0, 3654, 432, 34, 295, 743, 354, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6321, 5326, 5340, 6946, 6332, 6234, 6487, 6166] avg=6144.0000 max=6946 max_rank=3 min=5326 min_rank=1 imbalance_ratio=1.130534 max_load_reduction=42206 max_load_reduction_pct=0.858683 moved_to_echo=25219 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[14, 3224, 599, 2972, 188, 90, 238, 4, 0, 0, 0, 0, 0, 0, 0, 0, 142, 10, 90, 122, 1163, 0, 1873, 14, 1880, 902, 0, 0, 0, 0, 0, 0, 1203, 0, 654, 79, 687, 335, 106, 1926, 863, 0, 0, 0, 0, 0, 0, 0, 597, 628, 47, 57, 86, 1644, 447, 352, 1639, 679, 74, 0, 0, 0, 0, 0, 2387, 198, 243, 465, 468, 4, 1117, 709, 0, 0, 0, 0, 0, 0, 0, 0, 131, 3, 1325, 211, 6, 10, 6, 259, 4009, 0, 0, 0, 0, 0, 0, 0, 267, 589, 2779, 1625, 0, 1, 1174, 5, 0, 0, 0, 0, 0, 0, 0, 0, 442, 22, 3, 2316, 414, 84, 494, 1507, 251, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7329, 6196, 5853, 6250, 5591, 5960, 6440, 5533] avg=6144.0000 max=7329 max_rank=0 min=5533 min_rank=7 imbalance_ratio=1.192871 max_load_reduction=41823 max_load_reduction_pct=0.850891 moved_to_echo=23524 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2, 2499, 471, 2812, 128, 39, 254, 10, 0, 0, 0, 0, 0, 0, 0, 0, 115, 8, 147, 20, 394, 1, 1528, 48, 2245, 853, 0, 0, 0, 0, 0, 0, 843, 1, 451, 18, 522, 453, 33, 1718, 907, 0, 0, 0, 0, 0, 0, 0, 193, 1822, 13, 133, 67, 1449, 450, 248, 1430, 643, 77, 0, 0, 0, 0, 0, 2082, 175, 305, 403, 605, 5, 543, 1122, 0, 0, 0, 0, 0, 0, 0, 0, 349, 19, 760, 363, 10, 62, 4, 158, 4785, 0, 0, 0, 0, 0, 0, 0, 270, 219, 2920, 1382, 0, 36, 1741, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1076, 63, 0, 4191, 624, 124, 362, 1086, 263, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6215, 5359, 4946, 6525, 5240, 6510, 6568, 7789] avg=6144.0000 max=7789 max_rank=7 min=4946 min_rank=2 imbalance_ratio=1.267741 max_load_reduction=41363 max_load_reduction_pct=0.841532 moved_to_echo=26107 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1738, 752, 3068, 66, 28, 264, 14, 0, 0, 0, 0, 0, 0, 0, 0, 158, 9, 92, 109, 799, 1, 2402, 18, 2107, 931, 0, 0, 0, 0, 0, 0, 1700, 9, 747, 44, 384, 1017, 82, 2321, 698, 0, 0, 0, 0, 0, 0, 0, 222, 1392, 6, 129, 453, 1572, 249, 258, 1120, 701, 59, 0, 0, 0, 0, 0, 1632, 157, 94, 328, 268, 8, 914, 2184, 0, 0, 0, 0, 0, 0, 0, 0, 248, 16, 845, 200, 11, 4, 2, 169, 4494, 0, 0, 0, 0, 0, 0, 0, 413, 120, 2250, 1729, 0, 2, 841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 70, 0, 3464, 471, 95, 394, 1335, 203, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5930, 6626, 7002, 6161, 5585, 5989, 5355, 6504] avg=6144.0000 max=7002 max_rank=2 min=5355 min_rank=6 imbalance_ratio=1.139648 max_load_reduction=42150 max_load_reduction_pct=0.857544 moved_to_echo=23433 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[7, 3509, 729, 2880, 108, 55, 74, 12, 0, 0, 0, 0, 0, 0, 0, 0, 150, 11, 129, 53, 623, 0, 2364, 68, 2013, 879, 0, 0, 0, 0, 0, 0, 814, 0, 459, 122, 311, 330, 26, 2488, 1107, 0, 0, 0, 0, 0, 0, 0, 367, 1043, 25, 50, 208, 1395, 346, 645, 1057, 663, 98, 0, 0, 0, 0, 0, 1530, 111, 107, 367, 333, 1, 1001, 1441, 0, 0, 0, 0, 0, 0, 0, 0, 106, 7, 1320, 293, 3, 46, 22, 97, 4288, 0, 0, 0, 0, 0, 0, 0, 219, 212, 3542, 1639, 0, 3, 979, 2, 0, 0, 0, 0, 0, 0, 0, 0, 594, 150, 1, 3350, 479, 179, 163, 1026, 323, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7374, 6290, 5657, 5897, 4891, 6182, 6596, 6265] avg=6144.0000 max=7374 max_rank=0 min=4891 min_rank=4 imbalance_ratio=1.200195 max_load_reduction=41778 max_load_reduction_pct=0.849976 moved_to_echo=23934 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2, 1701, 117, 3346, 171, 41, 158, 10, 0, 0, 0, 0, 0, 0, 0, 0, 198, 27, 291, 152, 898, 0, 1401, 90, 2171, 1015, 0, 0, 0, 0, 0, 0, 1379, 4, 393, 48, 345, 434, 33, 2718, 945, 0, 0, 0, 0, 0, 0, 0, 224, 1086, 5, 69, 190, 1065, 304, 254, 1248, 764, 81, 0, 0, 0, 0, 0, 1818, 145, 324, 257, 282, 5, 2274, 2483, 0, 0, 0, 0, 0, 0, 0, 0, 93, 9, 1255, 355, 8, 8, 0, 150, 4625, 0, 0, 0, 0, 0, 0, 0, 894, 191, 3042, 1345, 0, 0, 604, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 37, 0, 2546, 437, 126, 223, 956, 274, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5546, 6243, 6299, 5290, 7588, 6503, 6077, 5606] avg=6144.0000 max=7588 max_rank=4 min=5290 min_rank=3 imbalance_ratio=1.235026 max_load_reduction=41564 max_load_reduction_pct=0.845622 moved_to_echo=25774 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[32, 2287, 216, 2172, 167, 108, 164, 32, 0, 0, 0, 0, 0, 0, 0, 0, 156, 10, 1208, 96, 386, 5, 2360, 67, 2057, 659, 0, 0, 0, 0, 0, 0, 667, 35, 653, 86, 903, 709, 55, 2430, 846, 0, 0, 0, 0, 0, 0, 0, 356, 793, 43, 48, 210, 1108, 458, 336, 1435, 496, 72, 0, 0, 0, 0, 0, 2089, 279, 261, 405, 462, 3, 1720, 2468, 0, 0, 0, 0, 0, 0, 0, 0, 101, 10, 495, 364, 8, 14, 20, 414, 4383, 0, 0, 0, 0, 0, 0, 0, 843, 262, 2726, 1633, 0, 6, 898, 2, 0, 0, 0, 0, 0, 0, 0, 0, 575, 55, 0, 2355, 385, 145, 149, 1455, 246, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5178, 7004, 6384, 5355, 7687, 5809, 6370, 5365] avg=6144.0000 max=7687 max_rank=4 min=5178 min_rank=0 imbalance_ratio=1.251139 max_load_reduction=41465 max_load_reduction_pct=0.843608 moved_to_echo=25231 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3, 2031, 497, 2454, 85, 127, 61, 1, 0, 0, 0, 0, 0, 0, 0, 0, 135, 6, 380, 59, 605, 17, 2288, 7, 1867, 744, 0, 0, 0, 0, 0, 0, 2518, 3, 756, 132, 415, 487, 23, 2436, 901, 0, 0, 0, 0, 0, 0, 0, 265, 1081, 10, 58, 556, 1903, 172, 535, 1510, 561, 77, 0, 0, 0, 0, 0, 2198, 96, 214, 173, 220, 18, 1524, 1795, 0, 0, 0, 0, 0, 0, 0, 0, 119, 14, 1450, 183, 15, 26, 31, 147, 3980, 0, 0, 0, 0, 0, 0, 0, 141, 106, 2901, 1022, 0, 0, 1089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 75, 0, 2778, 437, 129, 143, 1617, 262, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5259, 6108, 7671, 6728, 6238, 5965, 5259, 5924] avg=6144.0000 max=7671 max_rank=2 min=5259 min_rank=0 imbalance_ratio=1.248535 max_load_reduction=41481 max_load_reduction_pct=0.843933 moved_to_echo=23386 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[2432, 10112, 256, 640, 1920, 11392, 2944, 3072, 11264, 5248, 768, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1152, 128, 2560, 768, 5376, 128, 15872, 384, 16512, 6912, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[16256, 1280, 1920, 2816, 3072, 128, 9984, 14080, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3584, 2048, 24192, 11520, 0, 128, 8192, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[10752, 128, 4736, 640, 4352, 4096, 512, 17024, 7552, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5248, 640, 128, 24704, 3712, 1024, 2304, 9728, 2176, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 19328, 4224, 22528, 1024, 640, 1664, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1408, 128, 8704, 2176, 128, 256, 128, 1664, 35200, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[1311, 23, 1, 0, 42, 720, 0, 44, 2360, 771, 4, 0, 742, 3510, 397, 485, 2879, 3837, 0, 239, 1868, 1723, 40, 490, 30, 0, 5803, 113, 129, 347, 29, 1086, 465, 258, 431, 216, 21, 0, 1362, 14, 974, 440, 112, 52, 11, 5548, 88, 12, 3, 1929, 0, 39, 501, 1232, 1664, 0, 30, 93, 4252, 114, 86, 150, 4, 28] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[974, 31, 9, 7, 59, 425, 0, 22, 2836, 955, 2, 1, 753, 3367, 765, 342, 1631, 4028, 0, 360, 1044, 1212, 108, 666, 141, 0, 4738, 158, 270, 156, 99, 922, 419, 470, 449, 242, 139, 0, 1865, 31, 1457, 1022, 278, 25, 6, 5313, 889, 120, 16, 1826, 2, 130, 243, 1279, 2361, 0, 82, 96, 3783, 176, 86, 232, 4, 30] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[898, 256, 0, 4, 50, 876, 0, 13, 2701, 1255, 43, 0, 1197, 3006, 813, 527, 3144, 3552, 0, 302, 798, 1105, 20, 351, 134, 0, 5956, 126, 113, 510, 54, 1097, 566, 209, 282, 254, 146, 0, 914, 34, 1145, 527, 631, 45, 18, 5502, 80, 26, 155, 1651, 1, 78, 397, 1099, 2008, 5, 19, 85, 3822, 110, 117, 312, 12, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[713, 109, 26, 1, 39, 654, 0, 16, 2212, 790, 13, 0, 596, 2845, 805, 177, 3218, 5767, 0, 355, 1475, 838, 56, 586, 114, 1, 6034, 135, 57, 369, 59, 912, 602, 482, 213, 181, 51, 1, 922, 122, 1606, 642, 334, 61, 43, 5822, 230, 35, 225, 1816, 1, 39, 300, 1109, 1346, 0, 44, 59, 3381, 133, 58, 275, 42, 5] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[981, 76, 16, 0, 4, 871, 0, 25, 885, 468, 12, 0, 442, 2598, 780, 228, 3527, 3748, 1, 473, 954, 963, 26, 460, 58, 0, 6182, 111, 167, 520, 54, 1044, 302, 324, 553, 141, 47, 0, 1636, 36, 2425, 390, 281, 107, 24, 6162, 517, 40, 403, 1854, 2, 36, 43, 1040, 1426, 0, 36, 120, 4786, 134, 283, 304, 12, 14] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[1150, 336, 3, 1, 40, 643, 0, 23, 2183, 973, 31, 0, 677, 3224, 801, 422, 2364, 3851, 0, 284, 856, 1210, 63, 1184, 54, 2, 5596, 67, 316, 351, 21, 1306, 617, 237, 395, 147, 220, 0, 1308, 5, 1398, 376, 159, 65, 26, 6749, 389, 58, 146, 1450, 1, 75, 53, 1027, 1587, 0, 68, 82, 3922, 138, 150, 255, 5, 12] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[1906, 94, 4, 0, 15, 778, 0, 19, 1815, 776, 71, 0, 636, 2442, 597, 526, 1708, 3064, 0, 828, 2009, 1985, 25, 669, 49, 0, 5472, 67, 110, 682, 94, 1567, 316, 141, 559, 195, 207, 0, 941, 37, 1965, 784, 97, 177, 38, 5843, 41, 34, 39, 1875, 3, 125, 270, 840, 1825, 0, 56, 52, 4085, 147, 135, 298, 13, 6] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[1021, 125, 31, 5, 79, 805, 0, 8, 1670, 523, 152, 2, 609, 2682, 1023, 707, 2699, 4857, 27, 315, 1131, 689, 48, 1405, 47, 11, 5805, 166, 200, 215, 33, 761, 335, 220, 89, 272, 49, 6, 1392, 26, 2069, 443, 101, 43, 41, 6456, 391, 79, 95, 1509, 4, 21, 44, 730, 2273, 3, 119, 112, 3604, 217, 204, 299, 25, 30] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1311, 23, 1, 0, 42, 720, 0, 44, 3837, 7, 0, 0, 0, 0, 0, 0, 2360, 771, 4, 0, 742, 1573, 397, 485, 0, 0, 0, 0, 0, 0, 0, 0, 2708, 0, 0, 239, 1868, 1723, 40, 490, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 4319, 113, 129, 347, 29, 1086, 0, 0, 0, 0, 0, 0, 0, 0, 465, 258, 431, 216, 21, 0, 1362, 14, 2533, 799, 0, 0, 0, 0, 0, 0, 974, 440, 112, 52, 11, 3008, 88, 12, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1929, 0, 39, 501, 1232, 1664, 0, 972, 171, 0, 0, 0, 0, 0, 0, 30, 93, 4252, 114, 86, 150, 4, 28, 1138, 512, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5985, 6332, 7068, 6053, 6099, 4697, 6511, 6407] avg=6144.0000 max=7068 max_rank=2 min=4697 min_rank=5 imbalance_ratio=1.150391 max_load_reduction=42084 max_load_reduction_pct=0.856201 moved_to_echo=23714 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[974, 31, 9, 7, 59, 425, 0, 22, 4028, 7, 0, 0, 0, 0, 0, 0, 2836, 955, 2, 1, 753, 1508, 765, 342, 0, 0, 0, 0, 0, 0, 0, 0, 1535, 0, 0, 360, 1044, 1212, 108, 666, 0, 0, 0, 0, 0, 0, 0, 0, 141, 0, 3527, 158, 270, 156, 99, 922, 0, 0, 0, 0, 0, 0, 0, 0, 419, 470, 449, 242, 139, 0, 1865, 31, 2426, 767, 0, 0, 0, 0, 0, 0, 1457, 1022, 278, 25, 6, 2880, 889, 120, 0, 0, 0, 0, 0, 0, 0, 0, 16, 1826, 2, 130, 243, 1279, 2361, 0, 793, 96, 0, 0, 0, 0, 0, 0, 82, 96, 3783, 176, 86, 232, 4, 30, 1092, 418, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5562, 7162, 4925, 5273, 6808, 6677, 6746, 5999] avg=6144.0000 max=7162 max_rank=1 min=4925 min_rank=2 imbalance_ratio=1.165690 max_load_reduction=41990 max_load_reduction_pct=0.854289 moved_to_echo=26230 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[898, 256, 0, 4, 50, 876, 0, 13, 3552, 7, 0, 0, 0, 0, 0, 0, 2701, 1255, 43, 0, 1197, 1347, 813, 527, 0, 0, 0, 0, 0, 0, 0, 0, 2958, 0, 0, 302, 798, 1105, 20, 351, 0, 0, 0, 0, 0, 0, 0, 0, 134, 0, 4432, 126, 113, 510, 54, 1097, 0, 0, 0, 0, 0, 0, 0, 0, 566, 209, 282, 254, 146, 0, 914, 34, 2512, 684, 0, 0, 0, 0, 0, 0, 1145, 527, 631, 45, 18, 2983, 80, 26, 0, 0, 0, 0, 0, 0, 0, 0, 155, 1651, 1, 78, 397, 1099, 2008, 5, 998, 186, 0, 0, 0, 0, 0, 0, 19, 85, 3822, 110, 117, 312, 12, 1, 975, 526, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5656, 7883, 5534, 6466, 5601, 5455, 6578, 5979] avg=6144.0000 max=7883 max_rank=1 min=5455 min_rank=5 imbalance_ratio=1.283040 max_load_reduction=41269 max_load_reduction_pct=0.839620 moved_to_echo=23613 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1150, 336, 3, 1, 40, 643, 0, 23, 3851, 9, 0, 0, 0, 0, 0, 0, 2183, 973, 31, 0, 677, 1445, 801, 422, 0, 0, 0, 0, 0, 0, 0, 0, 2224, 0, 0, 284, 856, 1210, 63, 1184, 0, 0, 0, 0, 0, 0, 0, 0, 54, 2, 4165, 67, 316, 351, 21, 1306, 0, 0, 0, 0, 0, 0, 0, 0, 617, 237, 395, 147, 220, 0, 1308, 5, 3082, 734, 0, 0, 0, 0, 0, 0, 1398, 376, 159, 65, 26, 3658, 389, 58, 0, 0, 0, 0, 0, 0, 0, 0, 146, 1450, 1, 75, 53, 1027, 1587, 0, 937, 140, 0, 0, 0, 0, 0, 0, 68, 82, 3922, 138, 150, 255, 5, 12, 1045, 494, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6056, 6532, 5821, 6282, 6745, 6129, 5416, 6171] avg=6144.0000 max=6745 max_rank=4 min=5416 min_rank=6 imbalance_ratio=1.097819 max_load_reduction=42407 max_load_reduction_pct=0.862773 moved_to_echo=24461 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1906, 94, 4, 0, 15, 778, 0, 19, 3064, 8, 0, 0, 0, 0, 0, 0, 1815, 776, 71, 0, 636, 1094, 597, 526, 0, 0, 0, 0, 0, 0, 0, 0, 1607, 0, 0, 828, 2009, 1985, 25, 669, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 4073, 67, 110, 682, 94, 1567, 0, 0, 0, 0, 0, 0, 0, 0, 316, 141, 559, 195, 207, 0, 941, 37, 2668, 556, 0, 0, 0, 0, 0, 0, 1965, 784, 97, 177, 38, 3167, 41, 34, 0, 0, 0, 0, 0, 0, 0, 0, 39, 1875, 3, 125, 270, 840, 1825, 0, 916, 101, 0, 0, 0, 0, 0, 0, 56, 52, 4085, 147, 135, 298, 13, 6, 792, 483, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5888, 5515, 7123, 6642, 5620, 6303, 5994, 6067] avg=6144.0000 max=7123 max_rank=2 min=5515 min_rank=1 imbalance_ratio=1.159342 max_load_reduction=42029 max_load_reduction_pct=0.855082 moved_to_echo=23984 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[981, 76, 16, 0, 4, 871, 0, 25, 3748, 12, 0, 0, 0, 0, 0, 0, 885, 468, 12, 0, 442, 1157, 780, 228, 0, 0, 0, 0, 0, 0, 0, 0, 3313, 0, 1, 473, 954, 963, 26, 460, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 4592, 111, 167, 520, 54, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 302, 324, 553, 141, 47, 0, 1636, 36, 2817, 595, 0, 0, 0, 0, 0, 0, 2425, 390, 281, 107, 24, 3333, 517, 40, 0, 0, 0, 0, 0, 0, 0, 0, 403, 1854, 2, 36, 43, 1040, 1426, 0, 1040, 214, 0, 0, 0, 0, 0, 0, 36, 120, 4786, 134, 283, 304, 12, 14, 846, 550, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5733, 3972, 6190, 6546, 6451, 7117, 6058, 7085] avg=6144.0000 max=7117 max_rank=5 min=3972 min_rank=1 imbalance_ratio=1.158366 max_load_reduction=42035 max_load_reduction_pct=0.855204 moved_to_echo=26711 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1021, 125, 31, 5, 79, 805, 0, 8, 4857, 8, 0, 0, 0, 0, 0, 0, 1670, 523, 152, 2, 609, 1202, 1023, 707, 0, 0, 0, 0, 0, 0, 0, 0, 2539, 0, 27, 315, 1131, 689, 48, 1405, 0, 0, 0, 0, 0, 0, 0, 0, 47, 11, 4320, 166, 200, 215, 33, 761, 0, 0, 0, 0, 0, 0, 0, 0, 335, 220, 89, 272, 49, 6, 1392, 26, 2948, 611, 0, 0, 0, 0, 0, 0, 2069, 443, 101, 43, 41, 3500, 391, 79, 0, 0, 0, 0, 0, 0, 0, 0, 95, 1509, 4, 21, 44, 730, 2273, 3, 972, 160, 0, 0, 0, 0, 0, 0, 119, 112, 3604, 217, 204, 299, 25, 30, 869, 513, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6939, 5888, 6154, 5753, 5948, 6667, 5811, 5992] avg=6144.0000 max=6939 max_rank=0 min=5753 min_rank=3 imbalance_ratio=1.129395 max_load_reduction=42213 max_load_reduction_pct=0.858826 moved_to_echo=24418 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[713, 109, 26, 1, 39, 654, 0, 16, 5767, 8, 0, 0, 0, 0, 0, 0, 2212, 790, 13, 0, 596, 1275, 805, 177, 0, 0, 0, 0, 0, 0, 0, 0, 3027, 0, 0, 355, 1475, 838, 56, 586, 0, 0, 0, 0, 0, 0, 0, 0, 114, 1, 4490, 135, 57, 369, 59, 912, 0, 0, 0, 0, 0, 0, 0, 0, 602, 482, 213, 181, 51, 1, 922, 122, 2658, 648, 0, 0, 0, 0, 0, 0, 1606, 642, 334, 61, 43, 3156, 230, 35, 0, 0, 0, 0, 0, 0, 0, 0, 225, 1816, 1, 39, 300, 1109, 1346, 0, 1011, 191, 0, 0, 0, 0, 0, 0, 44, 59, 3381, 133, 58, 275, 42, 5, 922, 533, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7333, 5868, 6337, 6137, 5880, 6107, 6038, 5452] avg=6144.0000 max=7333 max_rank=0 min=5452 min_rank=7 imbalance_ratio=1.193522 max_load_reduction=41819 max_load_reduction_pct=0.850810 moved_to_echo=23477 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[16768, 6528, 384, 128, 5760, 10624, 6016, 3456, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3712, 2432, 3072, 1664, 896, 128, 10368, 384, 21760, 5504, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1152, 13952, 128, 640, 1920, 8448, 14592, 128, 7680, 1280, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[19968, 0, 128, 3200, 10240, 9728, 512, 5888, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[13056, 4736, 2048, 640, 256, 25728, 2688, 512, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[512, 768, 31744, 1280, 1152, 2176, 128, 128, 7680, 4096, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[8960, 1152, 128, 128, 384, 5888, 0, 256, 32768, 128, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[640, 128, 33920, 1024, 1408, 3200, 512, 8704, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[99, 14, 9, 2, 6, 2620, 1093, 771, 77, 14, 119, 108, 1, 300, 7, 0, 921, 26, 890, 2536, 6755, 247, 84, 40, 2858, 0, 69, 1117, 1, 5353, 36, 607, 7, 439, 1743, 2, 3, 1809, 215, 0, 53, 13, 1814, 46, 1, 39, 1007, 2, 727, 224, 4717, 533, 3875, 212, 58, 385, 3136, 5, 0, 184, 1072, 0, 51, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[31, 15, 12, 0, 2, 2367, 1065, 1242, 4, 8, 260, 74, 13, 96, 10, 0, 2137, 20, 2169, 1807, 6465, 1145, 216, 22, 1007, 0, 153, 1191, 3, 6001, 56, 418, 23, 488, 885, 2, 0, 449, 415, 0, 27, 36, 2427, 185, 5, 105, 859, 0, 610, 228, 4660, 496, 3831, 182, 190, 274, 3059, 11, 5, 120, 1362, 164, 45, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[195, 4, 29, 5, 1, 2577, 708, 2049, 63, 9, 223, 354, 3, 20, 44, 3, 1744, 36, 867, 3095, 6061, 806, 399, 59, 1102, 0, 206, 1268, 18, 4034, 68, 822, 7, 584, 1433, 15, 0, 384, 252, 1, 172, 12, 2544, 272, 6, 98, 1041, 0, 493, 76, 4484, 280, 4055, 230, 306, 421, 3337, 19, 1, 307, 1312, 1, 136, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[28, 5, 11, 0, 1, 3435, 733, 1045, 150, 0, 224, 137, 0, 21, 1, 0, 1158, 120, 948, 2013, 6731, 375, 89, 174, 2067, 0, 98, 1218, 1, 4506, 41, 478, 5, 396, 826, 0, 0, 1122, 131, 0, 20, 19, 3076, 86, 0, 148, 623, 0, 923, 477, 4391, 413, 5224, 151, 96, 603, 3256, 31, 0, 128, 1112, 1, 86, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[39, 35, 3, 0, 0, 2921, 1208, 1508, 31, 6, 110, 169, 2, 40, 19, 1, 857, 37, 1232, 2980, 6488, 736, 170, 48, 1905, 0, 112, 1091, 1, 4721, 24, 949, 12, 600, 1328, 0, 1, 523, 216, 0, 55, 16, 2459, 178, 1, 72, 1438, 0, 569, 356, 4275, 360, 4336, 182, 28, 352, 2460, 19, 0, 132, 1672, 11, 57, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[94, 18, 37, 3, 1, 2462, 1611, 567, 20, 2, 272, 294, 1, 19, 22, 0, 952, 21, 1618, 2111, 5308, 872, 29, 10, 1343, 0, 225, 1093, 6, 5903, 84, 532, 9, 1274, 1661, 5, 0, 743, 95, 0, 21, 22, 2137, 36, 2, 57, 1024, 0, 323, 126, 4683, 547, 4045, 177, 47, 742, 3971, 26, 1, 162, 1533, 3, 149, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[27, 245, 13, 9, 4, 4346, 858, 1432, 26, 189, 248, 171, 5, 76, 2, 2, 948, 73, 2177, 1307, 6336, 650, 134, 42, 1388, 0, 53, 870, 29, 3521, 109, 587, 61, 480, 443, 4, 14, 1320, 268, 1, 62, 222, 2174, 399, 102, 67, 593, 0, 778, 418, 4524, 674, 4838, 227, 434, 437, 2867, 11, 2, 318, 1445, 24, 63, 5] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[20, 150, 17, 0, 0, 3339, 955, 889, 3, 25, 274, 209, 4, 61, 60, 1, 1154, 15, 1173, 2063, 6415, 1296, 148, 32, 834, 1, 24, 1403, 2, 5228, 44, 808, 58, 1009, 1151, 1, 0, 591, 301, 2, 44, 28, 2598, 108, 1, 305, 966, 0, 667, 268, 5845, 356, 3240, 174, 125, 620, 3124, 42, 0, 119, 714, 5, 42, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[99, 14, 9, 2, 6, 2620, 1093, 771, 586, 216, 0, 0, 0, 0, 0, 0, 77, 14, 119, 108, 1, 300, 7, 0, 5938, 0, 0, 0, 0, 0, 0, 0, 921, 26, 890, 2536, 284, 247, 84, 40, 0, 0, 0, 0, 0, 0, 0, 0, 2858, 0, 69, 1117, 1, 2828, 36, 607, 0, 0, 0, 0, 0, 0, 0, 0, 7, 439, 1743, 2, 3, 1809, 215, 0, 2680, 533, 0, 0, 0, 0, 0, 0, 53, 13, 1814, 46, 1, 39, 1007, 2, 2037, 409, 0, 0, 0, 0, 0, 0, 727, 224, 0, 533, 3659, 212, 58, 385, 0, 0, 0, 0, 0, 0, 0, 0, 3136, 5, 0, 184, 1072, 0, 51, 0, 1530, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5416, 6564, 5028, 7516, 7431, 5421, 5798, 5978] avg=6144.0000 max=7516 max_rank=3 min=5028 min_rank=2 imbalance_ratio=1.223307 max_load_reduction=41636 max_load_reduction_pct=0.847087 moved_to_echo=24628 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[28, 5, 11, 0, 1, 3435, 733, 1045, 493, 291, 0, 0, 0, 0, 0, 0, 150, 0, 224, 137, 0, 21, 1, 0, 5917, 0, 0, 0, 0, 0, 0, 0, 1158, 120, 948, 2013, 283, 375, 89, 174, 0, 0, 0, 0, 0, 0, 0, 0, 2067, 0, 98, 1218, 1, 2381, 41, 478, 0, 0, 0, 0, 0, 0, 0, 0, 5, 396, 826, 0, 0, 1122, 131, 0, 2495, 531, 0, 0, 0, 0, 0, 0, 20, 19, 3076, 86, 0, 148, 623, 0, 1896, 344, 0, 0, 0, 0, 0, 0, 923, 477, 0, 413, 4933, 151, 96, 603, 0, 0, 0, 0, 0, 0, 0, 0, 3256, 31, 0, 128, 1112, 1, 86, 0, 1288, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6042, 6450, 5160, 6284, 5506, 6212, 7596, 5902] avg=6144.0000 max=7596 max_rank=6 min=5160 min_rank=2 imbalance_ratio=1.236328 max_load_reduction=41556 max_load_reduction_pct=0.845459 moved_to_echo=25216 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[195, 4, 29, 5, 1, 2577, 708, 2049, 441, 226, 0, 0, 0, 0, 0, 0, 63, 9, 223, 354, 3, 20, 44, 3, 5328, 0, 0, 0, 0, 0, 0, 0, 1744, 36, 867, 3095, 255, 806, 399, 59, 0, 0, 0, 0, 0, 0, 0, 0, 1102, 0, 206, 1268, 18, 2132, 68, 822, 0, 0, 0, 0, 0, 0, 0, 0, 7, 584, 1433, 15, 0, 384, 252, 1, 2548, 478, 0, 0, 0, 0, 0, 0, 172, 12, 2544, 272, 6, 98, 1041, 0, 1936, 308, 0, 0, 0, 0, 0, 0, 493, 76, 0, 280, 3829, 230, 306, 421, 0, 0, 0, 0, 0, 0, 0, 0, 3337, 19, 1, 307, 1312, 1, 136, 1, 1153, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6235, 6047, 7261, 5616, 5702, 6389, 5635, 6267] avg=6144.0000 max=7261 max_rank=2 min=5616 min_rank=3 imbalance_ratio=1.181803 max_load_reduction=41891 max_load_reduction_pct=0.852275 moved_to_echo=23993 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[31, 15, 12, 0, 2, 2367, 1065, 1242, 661, 217, 0, 0, 0, 0, 0, 0, 4, 8, 260, 74, 13, 96, 10, 0, 5685, 0, 0, 0, 0, 0, 0, 0, 2137, 20, 2169, 1807, 268, 1145, 216, 22, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 0, 153, 1191, 3, 3157, 56, 418, 0, 0, 0, 0, 0, 0, 0, 0, 23, 488, 885, 2, 0, 449, 415, 0, 2649, 512, 0, 0, 0, 0, 0, 0, 27, 36, 2427, 185, 5, 105, 859, 0, 2011, 463, 0, 0, 0, 0, 0, 0, 610, 228, 0, 496, 3614, 182, 190, 274, 0, 0, 0, 0, 0, 0, 0, 0, 3059, 11, 5, 120, 1362, 164, 45, 0, 1720, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5612, 6150, 7784, 5985, 5423, 6118, 5594, 6486] avg=6144.0000 max=7784 max_rank=2 min=5423 min_rank=4 imbalance_ratio=1.266927 max_load_reduction=41368 max_load_reduction_pct=0.841634 moved_to_echo=23621 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[94, 18, 37, 3, 1, 2462, 1611, 567, 646, 225, 0, 0, 0, 0, 0, 0, 20, 2, 272, 294, 1, 19, 22, 0, 4666, 0, 0, 0, 0, 0, 0, 0, 952, 21, 1618, 2111, 224, 872, 29, 10, 0, 0, 0, 0, 0, 0, 0, 0, 1343, 0, 225, 1093, 6, 3118, 84, 532, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1274, 1661, 5, 0, 743, 95, 0, 2661, 418, 0, 0, 0, 0, 0, 0, 21, 22, 2137, 36, 2, 57, 1024, 0, 2022, 451, 0, 0, 0, 0, 0, 0, 323, 126, 0, 547, 3820, 177, 47, 742, 0, 0, 0, 0, 0, 0, 0, 0, 3971, 26, 1, 162, 1533, 3, 149, 1, 1688, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5664, 5296, 5837, 6401, 6866, 5772, 5782, 7534] avg=6144.0000 max=7534 max_rank=7 min=5296 min_rank=1 imbalance_ratio=1.226237 max_load_reduction=41618 max_load_reduction_pct=0.846720 moved_to_echo=25954 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[39, 35, 3, 0, 0, 2921, 1208, 1508, 516, 242, 0, 0, 0, 0, 0, 0, 31, 6, 110, 169, 2, 40, 19, 1, 5703, 0, 0, 0, 0, 0, 0, 0, 857, 37, 1232, 2980, 273, 736, 170, 48, 0, 0, 0, 0, 0, 0, 0, 0, 1905, 0, 112, 1091, 1, 2494, 24, 949, 0, 0, 0, 0, 0, 0, 0, 0, 12, 600, 1328, 0, 1, 523, 216, 0, 2430, 512, 0, 0, 0, 0, 0, 0, 55, 16, 2459, 178, 1, 72, 1438, 0, 1845, 361, 0, 0, 0, 0, 0, 0, 569, 356, 0, 360, 4094, 182, 28, 352, 0, 0, 0, 0, 0, 0, 0, 0, 2460, 19, 0, 132, 1672, 11, 57, 1, 1350, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6472, 6081, 6333, 6576, 5622, 6425, 5941, 5702] avg=6144.0000 max=6576 max_rank=3 min=5622 min_rank=4 imbalance_ratio=1.070312 max_load_reduction=42576 max_load_reduction_pct=0.866211 moved_to_echo=23690 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[27, 245, 13, 9, 4, 4346, 858, 1432, 385, 270, 0, 0, 0, 0, 0, 0, 26, 189, 248, 171, 5, 76, 2, 2, 5569, 0, 0, 0, 0, 0, 0, 0, 948, 73, 2177, 1307, 267, 650, 134, 42, 0, 0, 0, 0, 0, 0, 0, 0, 1388, 0, 53, 870, 29, 1861, 109, 587, 0, 0, 0, 0, 0, 0, 0, 0, 61, 480, 443, 4, 14, 1320, 268, 1, 2571, 500, 0, 0, 0, 0, 0, 0, 62, 222, 2174, 399, 102, 67, 593, 0, 1953, 269, 0, 0, 0, 0, 0, 0, 778, 418, 0, 674, 4568, 227, 434, 437, 0, 0, 0, 0, 0, 0, 0, 0, 2867, 11, 2, 318, 1445, 24, 63, 5, 1006, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7589, 6288, 5598, 4897, 5662, 5841, 7536, 5741] avg=6144.0000 max=7589 max_rank=0 min=4897 min_rank=3 imbalance_ratio=1.235189 max_load_reduction=41563 max_load_reduction_pct=0.845601 moved_to_echo=24780 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[20, 150, 17, 0, 0, 3339, 955, 889, 572, 180, 0, 0, 0, 0, 0, 0, 3, 25, 274, 209, 4, 61, 60, 1, 5639, 0, 0, 0, 0, 0, 0, 0, 1154, 15, 1173, 2063, 270, 1296, 148, 32, 0, 0, 0, 0, 0, 0, 0, 0, 834, 1, 24, 1403, 2, 2761, 44, 808, 0, 0, 0, 0, 0, 0, 0, 0, 58, 1009, 1151, 1, 0, 591, 301, 2, 3321, 506, 0, 0, 0, 0, 0, 0, 44, 28, 2598, 108, 1, 305, 966, 0, 2524, 400, 0, 0, 0, 0, 0, 0, 667, 268, 0, 356, 3060, 174, 125, 620, 0, 0, 0, 0, 0, 0, 0, 0, 3124, 42, 0, 119, 714, 5, 42, 1, 1495, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6122, 6276, 6151, 5877, 6940, 6974, 5270, 5542] avg=6144.0000 max=6974 max_rank=5 min=5270 min_rank=6 imbalance_ratio=1.135091 max_load_reduction=42178 max_load_reduction_pct=0.858114 moved_to_echo=24726 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[640, 512, 256, 128, 128, 24192, 8320, 9600, 4352, 1920, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[9984, 384, 11136, 17920, 2176, 6144, 1280, 512, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[12544, 128, 1024, 9344, 128, 20736, 512, 5248, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[512, 384, 19328, 1408, 128, 896, 7552, 128, 16256, 3072, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[256, 5376, 9472, 128, 128, 7040, 1920, 128, 21376, 4096, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[25216, 256, 128, 1536, 10240, 256, 640, 128, 11264, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[5120, 2176, 0, 3712, 31616, 1536, 1408, 3840, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[384, 256, 1792, 1536, 128, 640, 256, 128, 44544, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[11, 206, 0, 0, 621, 288, 67, 0, 61, 55, 3593, 0, 1503, 1899, 1169, 0, 161, 0, 349, 33, 0, 0, 0, 1822, 785, 742, 4388, 0, 0, 62, 1, 13, 254, 0, 251, 779, 0, 0, 2000, 33, 2471, 2307, 254, 7324, 37, 0, 1860, 3751, 1715, 22, 241, 103, 350, 4, 377, 1483, 1451, 1993, 70, 1748, 10, 0, 380, 55] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[4, 177, 0, 0, 772, 646, 170, 5, 24, 59, 4448, 0, 997, 1455, 1014, 0, 265, 5, 281, 83, 14, 0, 0, 1647, 290, 570, 6495, 0, 0, 6, 3, 10, 315, 0, 859, 946, 0, 39, 1450, 152, 1295, 1903, 885, 7304, 24, 0, 1392, 4388, 804, 2, 50, 36, 530, 2, 177, 2266, 885, 1292, 141, 1802, 141, 0, 331, 301] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[60, 68, 0, 7, 490, 114, 60, 6, 21, 151, 2769, 2, 1600, 2237, 1170, 0, 96, 15, 153, 56, 13, 3, 0, 2134, 392, 578, 6036, 3, 1, 26, 9, 4, 278, 0, 412, 855, 0, 22, 1327, 194, 1876, 3065, 610, 6859, 6, 0, 1123, 5191, 1079, 19, 269, 165, 408, 4, 134, 1839, 1741, 1036, 217, 1773, 5, 0, 177, 194] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[31, 219, 0, 2, 586, 124, 299, 5, 11, 99, 3458, 0, 1581, 1791, 1853, 0, 168, 1, 92, 17, 3, 1, 0, 1545, 700, 746, 5836, 0, 1, 11, 8, 31, 251, 0, 652, 678, 0, 1, 1562, 4, 1261, 2552, 683, 7277, 29, 0, 1780, 4376, 1132, 37, 47, 172, 344, 0, 453, 2128, 1058, 735, 102, 2207, 9, 1, 286, 116] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[100, 121, 0, 0, 609, 206, 132, 0, 66, 132, 3281, 8, 1199, 2580, 788, 1, 243, 6, 763, 138, 0, 4, 0, 2686, 448, 491, 5725, 0, 0, 26, 0, 3, 214, 3, 338, 669, 0, 0, 1169, 14, 2605, 1948, 139, 7352, 16, 0, 936, 4056, 1828, 11, 137, 139, 146, 4, 542, 2960, 1001, 974, 33, 1831, 3, 2, 257, 69] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[12, 454, 0, 1, 447, 89, 82, 2, 24, 42, 4163, 0, 1408, 3226, 1706, 0, 102, 0, 268, 68, 0, 0, 0, 1327, 574, 630, 5175, 1, 7, 37, 0, 22, 282, 1, 201, 575, 0, 2, 1070, 40, 1514, 1724, 96, 7206, 30, 0, 1575, 5806, 746, 18, 144, 189, 152, 5, 227, 2592, 1497, 768, 51, 2088, 14, 0, 468, 204] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[116, 138, 6, 8, 604, 465, 104, 2, 36, 74, 4136, 27, 806, 1732, 553, 2, 190, 0, 247, 76, 4, 1, 1, 2381, 419, 428, 5831, 0, 0, 6, 0, 19, 260, 2, 648, 756, 0, 1, 1222, 25, 3401, 4347, 552, 6564, 126, 0, 1351, 3777, 878, 1, 180, 135, 563, 3, 68, 1467, 1724, 1489, 141, 629, 4, 0, 318, 108] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[56, 451, 3, 10, 1603, 695, 63, 73, 182, 70, 3677, 2, 1260, 1837, 781, 4, 255, 3, 941, 94, 8, 0, 0, 1502, 663, 925, 5899, 5, 2, 61, 0, 10, 314, 5, 534, 686, 2, 21, 1057, 124, 2316, 1999, 374, 6570, 70, 0, 2254, 4347, 1134, 12, 248, 145, 232, 13, 288, 1250, 1218, 953, 33, 1310, 15, 7, 336, 150] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[11, 206, 0, 0, 621, 288, 67, 0, 4859, 0, 0, 0, 0, 0, 0, 0, 61, 55, 1443, 0, 1503, 1899, 1169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 0, 349, 33, 0, 0, 0, 1822, 2465, 1039, 0, 0, 0, 0, 0, 0, 785, 742, 3808, 0, 0, 62, 1, 13, 0, 0, 0, 0, 0, 0, 0, 0, 254, 0, 251, 779, 0, 0, 2000, 33, 2691, 0, 0, 0, 0, 0, 0, 0, 2471, 1885, 254, 0, 37, 0, 1860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1715, 22, 241, 103, 350, 4, 377, 1483, 1960, 21, 0, 0, 0, 0, 0, 0, 1451, 1993, 70, 1748, 10, 0, 380, 55, 580, 422, 190, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6052, 6130, 5869, 5411, 6008, 6507, 6276, 6899] avg=6144.0000 max=6899 max_rank=7 min=5411 min_rank=3 imbalance_ratio=1.122884 max_load_reduction=42253 max_load_reduction_pct=0.859639 moved_to_echo=25690 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[60, 68, 0, 7, 490, 114, 60, 6, 4551, 0, 0, 0, 0, 0, 0, 0, 21, 151, 1113, 2, 1600, 2237, 1170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 15, 153, 56, 13, 3, 0, 2134, 2308, 1438, 0, 0, 0, 0, 0, 0, 392, 578, 5238, 3, 1, 26, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 278, 0, 412, 855, 0, 22, 1327, 194, 3723, 0, 0, 0, 0, 0, 0, 0, 1876, 2504, 610, 0, 6, 0, 1123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1079, 19, 269, 165, 408, 4, 134, 1839, 1510, 30, 0, 0, 0, 0, 0, 0, 1741, 1036, 217, 1773, 5, 0, 177, 194, 798, 561, 146, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5356, 6294, 6216, 6251, 6811, 6119, 5457, 6648] avg=6144.0000 max=6811 max_rank=4 min=5356 min_rank=0 imbalance_ratio=1.108561 max_load_reduction=42341 max_load_reduction_pct=0.861430 moved_to_echo=25035 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[31, 219, 0, 2, 586, 124, 299, 5, 4828, 0, 0, 0, 0, 0, 0, 0, 11, 99, 1389, 0, 1581, 1791, 1853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 1, 92, 17, 3, 1, 0, 1545, 2449, 1212, 0, 0, 0, 0, 0, 0, 700, 746, 5065, 0, 1, 11, 8, 31, 0, 0, 0, 0, 0, 0, 0, 0, 251, 0, 652, 678, 0, 1, 1562, 4, 3139, 0, 0, 0, 0, 0, 0, 0, 1261, 2085, 683, 0, 29, 0, 1780, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1132, 37, 47, 172, 344, 0, 453, 2128, 1886, 25, 0, 0, 0, 0, 0, 0, 1058, 735, 102, 2207, 9, 1, 286, 116, 771, 467, 183, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6094, 6724, 5488, 6562, 6287, 5838, 6224, 5935] avg=6144.0000 max=6724 max_rank=1 min=5488 min_rank=2 imbalance_ratio=1.094401 max_load_reduction=42428 max_load_reduction_pct=0.863200 moved_to_echo=24284 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[100, 121, 0, 0, 609, 206, 132, 0, 4879, 0, 0, 0, 0, 0, 0, 0, 66, 132, 1319, 8, 1199, 2580, 788, 1, 0, 0, 0, 0, 0, 0, 0, 0, 243, 6, 763, 138, 0, 4, 0, 2686, 2473, 1124, 0, 0, 0, 0, 0, 0, 448, 491, 4968, 0, 0, 26, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 214, 3, 338, 669, 0, 0, 1169, 14, 2910, 0, 0, 0, 0, 0, 0, 0, 2605, 1592, 139, 0, 16, 0, 936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1828, 11, 137, 139, 146, 4, 542, 2960, 1789, 22, 0, 0, 0, 0, 0, 0, 1001, 974, 33, 1831, 3, 2, 257, 69, 757, 356, 173, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6047, 6093, 7437, 5936, 5317, 5288, 7578, 5456] avg=6144.0000 max=7578 max_rank=6 min=5288 min_rank=5 imbalance_ratio=1.233398 max_load_reduction=41574 max_load_reduction_pct=0.845825 moved_to_echo=23639 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[12, 454, 0, 1, 447, 89, 82, 2, 4782, 0, 0, 0, 0, 0, 0, 0, 24, 42, 1672, 0, 1408, 3226, 1706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 0, 268, 68, 0, 0, 0, 1327, 2424, 1611, 0, 0, 0, 0, 0, 0, 574, 630, 4491, 1, 7, 37, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 282, 1, 201, 575, 0, 2, 1070, 40, 4164, 0, 0, 0, 0, 0, 0, 0, 1514, 1409, 96, 0, 30, 0, 1575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 746, 18, 144, 189, 152, 5, 227, 2592, 2271, 31, 0, 0, 0, 0, 0, 0, 1497, 768, 51, 2088, 14, 0, 468, 204, 684, 315, 220, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5869, 8078, 5800, 5762, 6335, 4624, 6375, 6309] avg=6144.0000 max=8078 max_rank=1 min=4624 min_rank=5 imbalance_ratio=1.314779 max_load_reduction=41074 max_load_reduction_pct=0.835653 moved_to_echo=23643 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[116, 138, 6, 8, 604, 465, 104, 2, 4356, 0, 0, 0, 0, 0, 0, 0, 36, 74, 1653, 27, 806, 1732, 553, 2, 0, 0, 0, 0, 0, 0, 0, 0, 190, 0, 247, 76, 4, 1, 1, 2381, 2208, 1048, 0, 0, 0, 0, 0, 0, 419, 428, 5057, 0, 0, 6, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 260, 2, 648, 756, 0, 1, 1222, 25, 2709, 0, 0, 0, 0, 0, 0, 0, 3401, 3548, 552, 0, 126, 0, 1351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 878, 1, 180, 135, 563, 3, 68, 1467, 2260, 20, 0, 0, 0, 0, 0, 0, 1724, 1489, 141, 629, 4, 0, 318, 108, 774, 799, 223, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5799, 4883, 6156, 5929, 5623, 8978, 5575, 6209] avg=6144.0000 max=8978 max_rank=5 min=4883 min_rank=1 imbalance_ratio=1.461263 max_load_reduction=40174 max_load_reduction_pct=0.817342 moved_to_echo=26385 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[56, 451, 3, 10, 1603, 695, 63, 73, 4360, 0, 0, 0, 0, 0, 0, 0, 182, 70, 1478, 2, 1260, 1837, 781, 4, 0, 0, 0, 0, 0, 0, 0, 0, 255, 3, 941, 94, 8, 0, 0, 1502, 2210, 1205, 0, 0, 0, 0, 0, 0, 663, 925, 5119, 5, 2, 61, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 314, 5, 534, 686, 2, 21, 1057, 124, 3119, 0, 0, 0, 0, 0, 0, 0, 2316, 1633, 374, 0, 70, 0, 2254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1134, 12, 248, 145, 232, 13, 288, 1250, 2005, 23, 0, 0, 0, 0, 0, 0, 1218, 953, 33, 1310, 15, 7, 336, 150, 780, 366, 194, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7314, 5614, 6218, 6785, 5862, 6647, 5350, 5362] avg=6144.0000 max=7314 max_rank=0 min=5350 min_rank=6 imbalance_ratio=1.190430 max_load_reduction=41838 max_load_reduction_pct=0.851196 moved_to_echo=23221 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4, 177, 0, 0, 772, 646, 170, 5, 4847, 0, 0, 0, 0, 0, 0, 0, 24, 59, 1787, 0, 997, 1455, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 265, 5, 281, 83, 14, 0, 0, 1647, 2457, 1216, 0, 0, 0, 0, 0, 0, 290, 570, 5637, 0, 0, 6, 3, 10, 0, 0, 0, 0, 0, 0, 0, 0, 315, 0, 859, 946, 0, 39, 1450, 152, 3148, 0, 0, 0, 0, 0, 0, 0, 1295, 1555, 885, 0, 24, 0, 1392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 804, 2, 50, 36, 530, 2, 177, 2266, 2426, 24, 0, 0, 0, 0, 0, 0, 885, 1292, 141, 1802, 141, 0, 331, 301, 858, 348, 235, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6621, 5336, 5968, 6516, 6909, 5151, 6317, 6334] avg=6144.0000 max=6909 max_rank=4 min=5151 min_rank=5 imbalance_ratio=1.124512 max_load_reduction=42243 max_load_reduction_pct=0.859436 moved_to_echo=24711 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[16768, 16256, 3712, 0, 384, 0, 12288, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[10624, 9344, 896, 13440, 256, 128, 2560, 1280, 6016, 3712, 1664, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[9344, 128, 1408, 1152, 2816, 128, 2304, 16000, 16128, 256, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[512, 768, 11904, 128, 10368, 16768, 9088, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[1536, 128, 3200, 640, 128, 128, 128, 15104, 19072, 9984, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[512, 1920, 128, 128, 5760, 2688, 1024, 128, 37504, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2176, 128, 3968, 6016, 128, 128, 10880, 640, 25728, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4352, 5120, 39424, 128, 128, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=17 tokens_per_expert_current_ep_rank=[2717, 7, 9, 645, 80, 14, 4438, 906, 4784, 7271, 106, 6, 0, 57, 109, 80, 224, 2, 1451, 443, 305, 200, 50, 0, 480, 1270, 35, 0, 140, 9, 104, 972, 0, 7, 172, 1115, 8, 105, 21, 1, 465, 396, 181, 1696, 2623, 738, 181, 456, 4182, 4, 1, 46, 107, 129, 0, 3, 731, 76, 54, 2, 20, 1982, 3168, 3568] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=17 tokens_per_expert_current_ep_rank=[3025, 2, 4, 676, 79, 8, 2335, 475, 5359, 7369, 126, 11, 0, 149, 42, 140, 207, 7, 868, 213, 485, 407, 27, 4, 421, 1241, 37, 32, 538, 0, 73, 644, 0, 3, 60, 195, 0, 443, 1, 0, 661, 526, 295, 1296, 3462, 1335, 200, 1136, 4863, 0, 14, 30, 7, 228, 0, 2, 790, 93, 37, 0, 9, 2071, 2664, 3727] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=17 tokens_per_expert_current_ep_rank=[2567, 1, 0, 590, 47, 1, 3863, 466, 5615, 8116, 97, 0, 0, 274, 19, 176, 193, 3, 1937, 232, 517, 876, 51, 1, 1338, 1379, 49, 7, 81, 0, 60, 509, 0, 2, 64, 440, 1, 219, 4, 1, 431, 542, 249, 695, 1858, 918, 404, 229, 2617, 0, 2, 8, 7, 267, 0, 1, 944, 50, 59, 1, 4, 2100, 3792, 4178] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=17 tokens_per_expert_current_ep_rank=[2804, 3, 1, 727, 20, 49, 3607, 371, 4728, 7734, 84, 6, 0, 126, 8, 298, 185, 20, 1785, 437, 810, 222, 213, 3, 331, 1754, 3, 9, 77, 1, 43, 455, 0, 3, 64, 570, 1, 230, 46, 1, 327, 204, 241, 1249, 3848, 1521, 263, 159, 2346, 4, 1, 90, 22, 283, 1, 4, 1176, 73, 52, 1, 38, 1225, 4516, 3679] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=17 tokens_per_expert_current_ep_rank=[2952, 1, 5, 1514, 24, 1, 3409, 1202, 6596, 7561, 123, 19, 0, 113, 13, 266, 398, 15, 1216, 145, 106, 744, 23, 1, 804, 957, 12, 1, 441, 2, 106, 2691, 0, 2, 66, 184, 12, 167, 8, 1, 105, 355, 428, 625, 1967, 501, 83, 308, 2310, 10, 1, 166, 5, 510, 0, 18, 551, 90, 31, 1, 5, 2158, 3605, 3418] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=17 tokens_per_expert_current_ep_rank=[3174, 15, 0, 856, 26, 2, 3611, 450, 5510, 7660, 76, 4, 0, 67, 3, 130, 278, 4, 1807, 382, 239, 199, 37, 5, 474, 1618, 32, 17, 239, 0, 53, 394, 0, 15, 270, 497, 1, 625, 15, 0, 257, 661, 297, 842, 2280, 1096, 366, 407, 4363, 0, 5, 10, 3, 196, 0, 5, 700, 41, 92, 10, 3, 1361, 3829, 3543] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=17 tokens_per_expert_current_ep_rank=[3587, 0, 2, 714, 17, 2, 3047, 344, 5981, 7361, 144, 4, 15, 235, 7, 150, 396, 7, 1499, 137, 422, 221, 226, 13, 640, 729, 25, 31, 44, 2, 88, 268, 0, 23, 198, 186, 4, 309, 7, 2, 279, 371, 102, 750, 2928, 1602, 223, 545, 3026, 10, 6, 11, 11, 283, 0, 5, 1133, 264, 44, 2, 11, 2291, 3999, 4169] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=17 tokens_per_expert_current_ep_rank=[2882, 19, 28, 998, 454, 25, 3965, 311, 5184, 7048, 177, 0, 2, 49, 31, 346, 328, 11, 1304, 355, 326, 138, 15, 3, 1173, 1005, 41, 17, 114, 24, 58, 474, 0, 25, 390, 580, 14, 610, 10, 1, 747, 992, 166, 705, 1985, 1888, 458, 653, 2822, 220, 2, 1, 48, 477, 0, 54, 1011, 637, 93, 0, 89, 1319, 3442, 2808] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=17 rerouted_tokens_per_expert_current_ep_rank=[2717, 7, 9, 645, 80, 14, 2081, 906, 0, 0, 0, 0, 0, 0, 0, 0, 4784, 183, 106, 6, 0, 57, 109, 80, 0, 0, 0, 0, 0, 0, 0, 0, 224, 2, 1451, 443, 305, 200, 50, 0, 2111, 1019, 0, 0, 0, 0, 0, 0, 480, 1270, 35, 0, 140, 9, 104, 972, 2549, 582, 0, 0, 0, 0, 0, 0, 0, 7, 172, 1115, 8, 105, 21, 1, 4977, 0, 0, 0, 0, 0, 0, 0, 465, 396, 181, 1696, 2047, 738, 181, 456, 0, 0, 0, 0, 0, 0, 0, 0, 4182, 4, 1, 46, 107, 129, 0, 3, 1775, 576, 369, 0, 0, 0, 0, 0, 731, 76, 54, 2, 20, 1982, 2799, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6459, 5325, 5805, 6141, 6406, 6160, 7192, 5664] avg=6144.0000 max=7192 max_rank=6 min=5325 min_rank=1 imbalance_ratio=1.170573 max_load_reduction=41960 max_load_reduction_pct=0.853678 moved_to_echo=25422 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=17 rerouted_tokens_per_expert_current_ep_rank=[3025, 2, 4, 676, 79, 8, 1095, 475, 0, 0, 0, 0, 0, 0, 0, 0, 5359, 185, 126, 11, 0, 149, 42, 140, 0, 0, 0, 0, 0, 0, 0, 0, 207, 7, 868, 213, 485, 407, 27, 4, 2140, 1065, 0, 0, 0, 0, 0, 0, 421, 1241, 37, 32, 538, 0, 73, 644, 2662, 306, 0, 0, 0, 0, 0, 0, 0, 3, 60, 195, 0, 443, 1, 0, 5044, 0, 0, 0, 0, 0, 0, 0, 661, 526, 295, 1296, 2702, 1335, 200, 1136, 0, 0, 0, 0, 0, 0, 0, 0, 4863, 0, 14, 30, 7, 228, 0, 2, 934, 760, 311, 0, 0, 0, 0, 0, 790, 93, 37, 0, 9, 2071, 2353, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5364, 6012, 5423, 5954, 5746, 8151, 7149, 5353] avg=6144.0000 max=8151 max_rank=5 min=5353 min_rank=7 imbalance_ratio=1.326660 max_load_reduction=41001 max_load_reduction_pct=0.834167 moved_to_echo=26399 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=17 rerouted_tokens_per_expert_current_ep_rank=[2567, 1, 0, 590, 47, 1, 1812, 466, 0, 0, 0, 0, 0, 0, 0, 0, 5615, 204, 97, 0, 0, 274, 19, 176, 0, 0, 0, 0, 0, 0, 0, 0, 193, 3, 1937, 232, 517, 876, 51, 1, 2357, 1194, 0, 0, 0, 0, 0, 0, 1338, 1379, 49, 7, 81, 0, 60, 509, 2984, 506, 0, 0, 0, 0, 0, 0, 0, 2, 64, 440, 1, 219, 4, 1, 5555, 0, 0, 0, 0, 0, 0, 0, 431, 542, 249, 695, 1450, 918, 404, 229, 0, 0, 0, 0, 0, 0, 0, 0, 2617, 0, 2, 8, 7, 267, 0, 1, 1545, 408, 442, 0, 0, 0, 0, 0, 944, 50, 59, 1, 4, 2100, 3350, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5484, 6385, 7361, 6913, 6286, 4918, 5297, 6508] avg=6144.0000 max=7361 max_rank=2 min=4918 min_rank=5 imbalance_ratio=1.198079 max_load_reduction=41791 max_load_reduction_pct=0.850240 moved_to_echo=23009 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=17 rerouted_tokens_per_expert_current_ep_rank=[2952, 1, 5, 1514, 24, 1, 1599, 1202, 0, 0, 0, 0, 0, 0, 0, 0, 6596, 191, 123, 19, 0, 113, 13, 266, 0, 0, 0, 0, 0, 0, 0, 0, 398, 15, 1216, 145, 106, 744, 23, 1, 2195, 977, 0, 0, 0, 0, 0, 0, 804, 957, 12, 1, 441, 2, 106, 2691, 2441, 447, 0, 0, 0, 0, 0, 0, 0, 2, 66, 184, 12, 167, 8, 1, 5175, 0, 0, 0, 0, 0, 0, 0, 105, 355, 428, 625, 1535, 501, 83, 308, 0, 0, 0, 0, 0, 0, 0, 0, 2310, 10, 1, 166, 5, 510, 0, 18, 1363, 432, 420, 0, 0, 0, 0, 0, 551, 90, 31, 1, 5, 2158, 3185, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7298, 7321, 5820, 7902, 5615, 3940, 5235, 6021] avg=6144.0000 max=7902 max_rank=3 min=3940 min_rank=5 imbalance_ratio=1.286133 max_load_reduction=41250 max_load_reduction_pct=0.839233 moved_to_echo=20811 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=17 rerouted_tokens_per_expert_current_ep_rank=[2804, 3, 1, 727, 20, 49, 1684, 371, 0, 0, 0, 0, 0, 0, 0, 0, 4728, 187, 84, 6, 0, 126, 8, 298, 0, 0, 0, 0, 0, 0, 0, 0, 185, 20, 1785, 437, 810, 222, 213, 3, 2250, 1052, 0, 0, 0, 0, 0, 0, 331, 1754, 3, 9, 77, 1, 43, 455, 2627, 477, 0, 0, 0, 0, 0, 0, 0, 3, 64, 570, 1, 230, 46, 1, 5297, 0, 0, 0, 0, 0, 0, 0, 327, 204, 241, 1249, 2999, 1521, 263, 159, 0, 0, 0, 0, 0, 0, 0, 0, 2346, 4, 1, 90, 22, 283, 1, 4, 1446, 849, 532, 0, 0, 0, 0, 0, 1176, 73, 52, 1, 38, 1225, 3984, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5659, 5437, 6977, 5777, 6212, 6963, 5578, 6549] avg=6144.0000 max=6977 max_rank=2 min=5437 min_rank=1 imbalance_ratio=1.135579 max_load_reduction=42175 max_load_reduction_pct=0.858053 moved_to_echo=25302 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=17 rerouted_tokens_per_expert_current_ep_rank=[3174, 15, 0, 856, 26, 2, 1694, 450, 0, 0, 0, 0, 0, 0, 0, 0, 5510, 193, 76, 4, 0, 67, 3, 130, 0, 0, 0, 0, 0, 0, 0, 0, 278, 4, 1807, 382, 239, 199, 37, 5, 2224, 1013, 0, 0, 0, 0, 0, 0, 474, 1618, 32, 17, 239, 0, 53, 394, 2530, 473, 0, 0, 0, 0, 0, 0, 0, 15, 270, 497, 1, 625, 15, 0, 5243, 0, 0, 0, 0, 0, 0, 0, 257, 661, 297, 842, 1779, 1096, 366, 407, 0, 0, 0, 0, 0, 0, 0, 0, 4363, 0, 5, 10, 3, 196, 0, 5, 1444, 501, 447, 0, 0, 0, 0, 0, 700, 41, 92, 10, 3, 1361, 3382, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6217, 5983, 6188, 5830, 6666, 5705, 6974, 5589] avg=6144.0000 max=6974 max_rank=6 min=5589 min_rank=7 imbalance_ratio=1.135091 max_load_reduction=42178 max_load_reduction_pct=0.858114 moved_to_echo=24934 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=17 rerouted_tokens_per_expert_current_ep_rank=[2882, 19, 28, 998, 454, 25, 1859, 311, 0, 0, 0, 0, 0, 0, 0, 0, 5184, 178, 177, 0, 2, 49, 31, 346, 0, 0, 0, 0, 0, 0, 0, 0, 328, 11, 1304, 355, 326, 138, 15, 3, 2046, 803, 0, 0, 0, 0, 0, 0, 1173, 1005, 41, 17, 114, 24, 58, 474, 2005, 520, 0, 0, 0, 0, 0, 0, 0, 25, 390, 580, 14, 610, 10, 1, 4824, 0, 0, 0, 0, 0, 0, 0, 747, 992, 166, 705, 1549, 1888, 458, 653, 0, 0, 0, 0, 0, 0, 0, 0, 2822, 220, 2, 1, 48, 477, 0, 54, 1586, 436, 401, 0, 0, 0, 0, 0, 1011, 637, 93, 0, 89, 1319, 3041, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6576, 5967, 5329, 5431, 6454, 7158, 6047, 6190] avg=6144.0000 max=7158 max_rank=5 min=5329 min_rank=2 imbalance_ratio=1.165039 max_load_reduction=41994 max_load_reduction_pct=0.854370 moved_to_echo=25849 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=17 rerouted_tokens_per_expert_current_ep_rank=[3587, 0, 2, 714, 17, 2, 1429, 344, 0, 0, 0, 0, 0, 0, 0, 0, 5981, 186, 144, 4, 15, 235, 7, 150, 0, 0, 0, 0, 0, 0, 0, 0, 396, 7, 1499, 137, 422, 221, 226, 13, 2137, 1191, 0, 0, 0, 0, 0, 0, 640, 729, 25, 31, 44, 2, 88, 268, 2978, 399, 0, 0, 0, 0, 0, 0, 0, 23, 198, 186, 4, 309, 7, 2, 5038, 0, 0, 0, 0, 0, 0, 0, 279, 371, 102, 750, 2285, 1602, 223, 545, 0, 0, 0, 0, 0, 0, 0, 0, 3026, 10, 6, 11, 11, 283, 0, 5, 1219, 643, 466, 0, 0, 0, 0, 0, 1133, 264, 44, 2, 11, 2291, 3533, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6095, 6722, 6249, 5204, 5767, 6157, 5680, 7278] avg=6144.0000 max=7278 max_rank=7 min=5204 min_rank=3 imbalance_ratio=1.184570 max_load_reduction=41874 max_load_reduction_pct=0.851929 moved_to_echo=24882 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[2304, 128, 11904, 2432, 3328, 3072, 768, 128, 17536, 8320, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3328, 4096, 2048, 7936, 16384, 9600, 2304, 3968, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5760, 9984, 256, 128, 1792, 128, 640, 6528, 20864, 3712, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7040, 1408, 512, 128, 256, 14592, 25728, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[23808, 128, 128, 6784, 768, 128, 13312, 4608, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[26624, 256, 128, 384, 256, 2432, 128, 128, 11392, 4608, 3456, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[43776, 1536, 1024, 128, 128, 1152, 256, 1664, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 128, 1408, 3840, 128, 2816, 128, 128, 41216, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=18 tokens_per_expert_current_ep_rank=[0, 180, 867, 2529, 766, 1931, 0, 6, 0, 11, 0, 11, 1620, 215, 2439, 1678, 15, 3159, 0, 0, 6, 17, 50, 1, 140, 318, 438, 293, 35, 105, 689, 0, 243, 140, 848, 1094, 101, 4258, 0, 0, 411, 0, 266, 3101, 1, 0, 1673, 754, 108, 3, 161, 2395, 1158, 4020, 446, 0, 1, 311, 0, 19, 4561, 5554, 3, 3] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=18 tokens_per_expert_current_ep_rank=[1, 242, 2444, 2706, 255, 2571, 0, 27, 4, 19, 0, 74, 1718, 234, 2830, 1664, 13, 3666, 0, 0, 40, 5, 22, 1, 74, 255, 482, 130, 42, 119, 743, 0, 313, 266, 813, 1404, 208, 4077, 0, 0, 243, 5, 62, 1917, 7, 3, 1930, 487, 7, 3, 245, 2100, 987, 4520, 497, 8, 0, 207, 0, 4, 5358, 3024, 15, 61] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=18 tokens_per_expert_current_ep_rank=[0, 103, 1137, 2563, 398, 1113, 0, 69, 0, 12, 0, 43, 2023, 174, 3647, 2550, 5, 2966, 0, 0, 14, 2, 96, 0, 126, 332, 569, 98, 61, 170, 995, 0, 299, 222, 1536, 1954, 124, 3696, 0, 0, 399, 0, 77, 1116, 4, 0, 2044, 341, 262, 5, 99, 2225, 1996, 4574, 484, 1, 0, 231, 0, 11, 3982, 4145, 34, 25] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=18 tokens_per_expert_current_ep_rank=[0, 289, 1141, 3349, 154, 2812, 0, 53, 0, 16, 0, 7, 1258, 274, 2277, 1844, 1, 3124, 0, 0, 3, 3, 32, 4, 91, 188, 470, 68, 41, 77, 854, 0, 176, 138, 1030, 574, 17, 5175, 0, 3, 298, 0, 207, 2705, 0, 0, 2311, 602, 204, 31, 150, 2821, 1083, 4055, 489, 5, 0, 295, 0, 4, 4952, 3385, 12, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=18 tokens_per_expert_current_ep_rank=[0, 152, 1717, 2816, 587, 1393, 0, 14, 0, 44, 0, 14, 2123, 74, 2038, 2305, 11, 1603, 0, 0, 6, 3, 153, 0, 246, 131, 432, 270, 23, 130, 749, 0, 131, 162, 1619, 515, 99, 5633, 0, 0, 347, 6, 290, 1738, 4, 6, 2398, 253, 205, 2, 311, 1680, 637, 4160, 419, 76, 0, 220, 0, 5, 6249, 4872, 74, 7] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=18 tokens_per_expert_current_ep_rank=[10, 267, 1414, 3148, 1001, 1748, 0, 145, 18, 81, 0, 105, 1088, 25, 2746, 1163, 0, 1945, 0, 8, 5, 4, 48, 35, 100, 284, 510, 730, 15, 67, 653, 5, 472, 49, 1220, 608, 174, 4035, 0, 0, 410, 0, 74, 3302, 38, 8, 2053, 674, 270, 15, 376, 2380, 437, 4944, 450, 9, 1, 249, 0, 23, 4684, 4692, 129, 8] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=18 tokens_per_expert_current_ep_rank=[1, 202, 1060, 1758, 582, 1626, 0, 71, 0, 3, 0, 49, 1127, 155, 2388, 1767, 6, 2952, 0, 0, 2, 3, 30, 1, 196, 260, 665, 98, 20, 348, 557, 0, 206, 81, 1269, 2934, 121, 4567, 1, 0, 296, 2, 96, 1293, 8, 0, 1571, 750, 158, 0, 856, 2477, 1263, 4382, 540, 0, 0, 290, 0, 11, 5575, 4458, 15, 5] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=18 tokens_per_expert_current_ep_rank=[1, 461, 1716, 2598, 506, 1906, 0, 4, 0, 30, 0, 84, 1635, 166, 2071, 1330, 7, 2272, 0, 0, 10, 0, 61, 0, 59, 609, 413, 199, 6, 107, 767, 0, 574, 634, 752, 1371, 31, 4091, 0, 2, 698, 1, 228, 2322, 10, 0, 4179, 495, 212, 3, 352, 2446, 373, 3596, 445, 3, 1, 83, 0, 16, 3752, 5447, 17, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=18 rerouted_tokens_per_expert_current_ep_rank=[1, 242, 2444, 2019, 255, 2571, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 4, 19, 0, 74, 1718, 234, 2814, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 13, 3666, 0, 0, 40, 5, 22, 1, 1965, 1251, 118, 0, 0, 0, 0, 0, 74, 255, 482, 130, 42, 119, 743, 0, 3825, 603, 0, 0, 0, 0, 0, 0, 313, 266, 813, 1404, 208, 2826, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 5, 62, 1917, 7, 3, 1930, 487, 569, 16, 0, 0, 0, 0, 0, 0, 7, 3, 245, 2100, 987, 1952, 497, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 207, 0, 4, 1533, 3024, 15, 61, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7559, 6527, 7081, 6273, 5830, 5239, 5799, 4844] avg=6144.0000 max=7559 max_rank=0 min=4844 min_rank=7 imbalance_ratio=1.230306 max_load_reduction=41593 max_load_reduction_pct=0.846212 moved_to_echo=21712 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 180, 867, 1887, 766, 1931, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 11, 1620, 215, 2425, 1678, 0, 0, 0, 0, 0, 0, 0, 0, 15, 3159, 0, 0, 6, 17, 50, 1, 1748, 1307, 110, 0, 0, 0, 0, 0, 140, 318, 438, 293, 35, 105, 689, 0, 3256, 536, 0, 0, 0, 0, 0, 0, 243, 140, 848, 1094, 101, 2951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 411, 0, 266, 3101, 1, 0, 1673, 754, 532, 14, 0, 0, 0, 0, 0, 0, 108, 3, 161, 2395, 1158, 1736, 446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 311, 0, 19, 1305, 5554, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5637, 5960, 6413, 5810, 5377, 6752, 6007, 7196] avg=6144.0000 max=7196 max_rank=7 min=5377 min_rank=4 imbalance_ratio=1.171224 max_load_reduction=41956 max_load_reduction_pct=0.853597 moved_to_echo=25332 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 103, 1137, 1912, 398, 1113, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 43, 2023, 174, 3626, 2550, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2966, 0, 0, 14, 2, 96, 0, 1989, 1134, 112, 0, 0, 0, 0, 0, 126, 332, 569, 98, 61, 170, 995, 0, 2843, 610, 0, 0, 0, 0, 0, 0, 299, 222, 1536, 1954, 124, 2562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 399, 0, 77, 1116, 4, 0, 2044, 341, 539, 21, 0, 0, 0, 0, 0, 0, 262, 5, 99, 2225, 1996, 1975, 484, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 231, 0, 11, 1139, 4145, 34, 25, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4732, 8428, 6318, 5804, 6697, 4541, 7047, 5585] avg=6144.0000 max=8428 max_rank=1 min=4541 min_rank=5 imbalance_ratio=1.371745 max_load_reduction=40724 max_load_reduction_pct=0.828532 moved_to_echo=23870 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=18 rerouted_tokens_per_expert_current_ep_rank=[1, 202, 1060, 1312, 582, 1626, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 49, 1127, 155, 2375, 1767, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2952, 0, 0, 2, 3, 30, 1, 1905, 1402, 76, 0, 0, 0, 0, 0, 196, 260, 665, 98, 20, 348, 557, 0, 3980, 584, 0, 0, 0, 0, 0, 0, 206, 81, 1269, 2934, 121, 3165, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 296, 2, 96, 1293, 8, 0, 1571, 750, 370, 13, 0, 0, 0, 0, 0, 0, 158, 0, 856, 2477, 1263, 1893, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 290, 0, 11, 1595, 4458, 15, 5, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4854, 5476, 6377, 6708, 7777, 4399, 7187, 6374] avg=6144.0000 max=7777 max_rank=4 min=4399 min_rank=5 imbalance_ratio=1.265788 max_load_reduction=41375 max_load_reduction_pct=0.841777 moved_to_echo=25737 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 152, 1717, 2092, 587, 1393, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 14, 2123, 74, 2023, 2305, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1603, 0, 0, 6, 3, 153, 0, 1812, 1733, 127, 0, 0, 0, 0, 0, 246, 131, 432, 270, 23, 130, 749, 0, 4466, 559, 0, 0, 0, 0, 0, 0, 131, 162, 1619, 515, 99, 3900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 347, 6, 290, 1738, 4, 6, 2398, 253, 597, 15, 0, 0, 0, 0, 0, 0, 205, 2, 311, 1680, 637, 1789, 419, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 220, 0, 5, 1783, 4872, 74, 7, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5955, 6583, 5448, 7006, 6426, 5654, 5119, 6961] avg=6144.0000 max=7006 max_rank=3 min=5119 min_rank=6 imbalance_ratio=1.140299 max_load_reduction=42146 max_load_reduction_pct=0.857463 moved_to_echo=24160 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 289, 1141, 2499, 154, 2812, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 7, 1258, 274, 2264, 1844, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3124, 0, 0, 3, 3, 32, 4, 1763, 1588, 146, 0, 0, 0, 0, 0, 91, 188, 470, 68, 41, 77, 854, 0, 3535, 541, 0, 0, 0, 0, 0, 0, 176, 138, 1030, 574, 17, 3587, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 298, 0, 207, 2705, 0, 0, 2311, 602, 704, 13, 0, 0, 0, 0, 0, 0, 204, 31, 150, 2821, 1083, 1751, 489, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 295, 0, 4, 1417, 3385, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6948, 5663, 6664, 5865, 5525, 6840, 6534, 5113] avg=6144.0000 max=6948 max_rank=0 min=5113 min_rank=7 imbalance_ratio=1.130859 max_load_reduction=42204 max_load_reduction_pct=0.858643 moved_to_echo=24012 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=18 rerouted_tokens_per_expert_current_ep_rank=[10, 267, 1414, 2349, 1001, 1748, 0, 145, 0, 0, 0, 0, 0, 0, 0, 0, 18, 81, 0, 105, 1088, 25, 2731, 1163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1945, 0, 8, 5, 4, 48, 35, 2150, 1238, 137, 0, 0, 0, 0, 0, 100, 284, 510, 730, 15, 67, 653, 5, 3344, 659, 0, 0, 0, 0, 0, 0, 472, 49, 1220, 608, 174, 2797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 410, 0, 74, 3302, 38, 8, 2053, 674, 662, 15, 0, 0, 0, 0, 0, 0, 270, 15, 376, 2380, 437, 2135, 450, 9, 0, 0, 0, 0, 0, 0, 0, 0, 1, 249, 0, 23, 1340, 4692, 129, 8, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6934, 5211, 5570, 6367, 5320, 7236, 6072, 6442] avg=6144.0000 max=7236 max_rank=5 min=5211 min_rank=1 imbalance_ratio=1.177734 max_load_reduction=41916 max_load_reduction_pct=0.852783 moved_to_echo=25070 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=18 rerouted_tokens_per_expert_current_ep_rank=[1, 461, 1716, 1939, 506, 1906, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 84, 1635, 166, 2059, 1330, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2272, 0, 0, 10, 0, 61, 0, 1563, 1255, 113, 0, 0, 0, 0, 0, 59, 609, 413, 199, 6, 107, 767, 0, 2679, 480, 0, 0, 0, 0, 0, 0, 574, 634, 752, 1371, 31, 2836, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 698, 1, 228, 2322, 10, 0, 4179, 495, 546, 12, 0, 0, 0, 0, 0, 0, 212, 3, 352, 2446, 373, 1553, 445, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 83, 0, 16, 1073, 5447, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6533, 5304, 5281, 5319, 6200, 8491, 5387, 6637] avg=6144.0000 max=8491 max_rank=5 min=5281 min_rank=2 imbalance_ratio=1.381999 max_load_reduction=40661 max_load_reduction_pct=0.827250 moved_to_echo=26715 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 21760, 0, 128, 128, 128, 512, 128, 14976, 11008, 1024, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 1920, 11520, 16128, 4352, 15104, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 256, 0, 512, 12672, 1408, 20352, 14336, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3200, 128, 1408, 17536, 128, 128, 18176, 4480, 4608, 128, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[1536, 128, 2560, 18560, 7936, 14848, 3840, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1152, 2432, 4096, 1920, 256, 1152, 6016, 128, 28032, 4608, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2432, 1792, 9088, 10496, 896, 24704, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 1920, 0, 128, 11264, 35584, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=19 tokens_per_expert_current_ep_rank=[351, 225, 160, 13, 0, 183, 318, 647, 0, 3, 6, 4319, 1, 2130, 6328, 8, 130, 25, 303, 679, 3636, 4998, 230, 8, 1, 4094, 505, 172, 9, 0, 386, 21, 0, 0, 1, 787, 0, 2748, 2254, 0, 0, 478, 34, 692, 1503, 3460, 9, 567, 2443, 0, 0, 141, 0, 4, 1441, 136, 206, 540, 153, 371, 12, 0, 0, 1283] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=19 tokens_per_expert_current_ep_rank=[422, 239, 117, 291, 0, 59, 108, 208, 0, 4, 4, 4823, 0, 2003, 6652, 10, 99, 126, 466, 556, 3218, 5858, 54, 66, 0, 5176, 561, 443, 25, 1, 407, 28, 0, 0, 1, 1315, 1, 2537, 1125, 0, 0, 387, 14, 357, 1451, 1833, 26, 310, 2958, 11, 0, 65, 1, 1, 1147, 353, 155, 466, 248, 1171, 362, 1, 2, 830] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=19 tokens_per_expert_current_ep_rank=[438, 264, 265, 118, 0, 29, 460, 391, 0, 5, 88, 3491, 0, 1246, 6902, 20, 338, 4, 401, 635, 1593, 5902, 369, 25, 14, 6432, 255, 612, 19, 35, 508, 13, 0, 0, 4, 1265, 10, 3173, 758, 1, 0, 255, 66, 871, 1471, 1756, 128, 1171, 3086, 4, 0, 419, 0, 13, 1477, 325, 102, 54, 139, 453, 106, 0, 8, 1165] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=19 tokens_per_expert_current_ep_rank=[349, 501, 173, 110, 0, 175, 421, 269, 3, 18, 145, 3651, 0, 1854, 7269, 2, 49, 6, 318, 236, 3304, 6237, 93, 6, 5, 5972, 236, 260, 40, 2, 932, 1, 0, 0, 0, 1008, 1, 3028, 1746, 0, 0, 377, 34, 377, 1175, 1989, 41, 734, 3172, 10, 1, 198, 0, 11, 796, 135, 131, 269, 200, 331, 57, 0, 5, 689] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=19 tokens_per_expert_current_ep_rank=[402, 208, 184, 12, 0, 27, 998, 104, 0, 2, 24, 4656, 8, 1295, 5868, 0, 72, 3, 181, 343, 3238, 5539, 122, 79, 6, 5199, 370, 406, 15, 0, 488, 1, 0, 0, 2, 819, 0, 2144, 628, 0, 0, 361, 14, 1261, 1971, 1869, 67, 1741, 3583, 14, 1, 280, 0, 7, 1908, 61, 409, 238, 267, 772, 161, 0, 0, 724] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=19 tokens_per_expert_current_ep_rank=[245, 228, 156, 4, 0, 205, 227, 110, 0, 2, 63, 4221, 0, 1917, 5404, 34, 129, 32, 588, 603, 3097, 3445, 22, 9, 23, 3372, 810, 1218, 14, 0, 1190, 2, 0, 0, 6, 1638, 286, 2475, 2377, 1, 0, 379, 4, 1335, 1172, 2535, 61, 773, 3642, 11, 6, 233, 0, 32, 694, 451, 311, 531, 622, 822, 44, 0, 4, 1337] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=19 tokens_per_expert_current_ep_rank=[295, 235, 169, 16, 0, 32, 185, 83, 0, 2, 66, 3781, 0, 2629, 5837, 1, 72, 2, 194, 347, 2737, 4869, 361, 28, 78, 5343, 747, 397, 4, 1, 737, 1, 0, 0, 1, 1356, 16, 4262, 928, 0, 0, 536, 19, 357, 797, 2236, 95, 794, 3665, 16, 6, 84, 0, 2, 1262, 112, 131, 155, 568, 305, 789, 0, 0, 1411] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=19 tokens_per_expert_current_ep_rank=[534, 161, 412, 15, 7, 73, 649, 620, 47, 3, 24, 3939, 35, 1110, 6044, 2, 280, 4, 324, 617, 2204, 5135, 83, 222, 22, 5537, 321, 276, 4, 26, 364, 6, 3, 2, 1, 235, 52, 3239, 1126, 0, 4, 416, 0, 863, 1428, 1362, 69, 1970, 4085, 2, 7, 133, 0, 54, 1691, 124, 452, 669, 134, 1018, 253, 0, 1, 659] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=19 rerouted_tokens_per_expert_current_ep_rank=[349, 501, 173, 110, 0, 175, 421, 269, 5093, 0, 0, 0, 0, 0, 0, 0, 3, 18, 145, 3651, 0, 1854, 212, 2, 0, 0, 0, 0, 0, 0, 0, 0, 49, 6, 318, 236, 3304, 2405, 93, 6, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5248, 236, 260, 40, 2, 932, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1, 3028, 1746, 0, 579, 275, 0, 0, 0, 0, 0, 0, 0, 377, 34, 377, 1175, 1989, 41, 734, 449, 0, 0, 0, 0, 0, 0, 0, 3172, 10, 1, 198, 0, 11, 796, 135, 1283, 0, 0, 0, 0, 0, 0, 0, 131, 269, 200, 331, 57, 0, 5, 689, 1964, 1970, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7091, 5885, 6417, 6724, 6637, 5176, 5606, 5616] avg=6144.0000 max=7091 max_rank=0 min=5176 min_rank=5 imbalance_ratio=1.154134 max_load_reduction=42061 max_load_reduction_pct=0.855733 moved_to_echo=23035 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=19 rerouted_tokens_per_expert_current_ep_rank=[351, 225, 160, 13, 0, 183, 318, 647, 4434, 0, 0, 0, 0, 0, 0, 0, 0, 3, 6, 4319, 1, 2130, 184, 8, 0, 0, 0, 0, 0, 0, 0, 0, 130, 25, 303, 679, 3636, 1928, 230, 8, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3598, 505, 172, 9, 0, 386, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 787, 0, 2748, 2254, 0, 464, 188, 0, 0, 0, 0, 0, 0, 0, 478, 34, 692, 1503, 3460, 9, 567, 308, 0, 0, 0, 0, 0, 0, 0, 2443, 0, 0, 141, 0, 4, 1441, 136, 1028, 0, 0, 0, 0, 0, 0, 0, 206, 540, 153, 371, 12, 0, 0, 1283, 1710, 1578, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6331, 6651, 6939, 4692, 6442, 7051, 5193, 5853] avg=6144.0000 max=7051 max_rank=5 min=4692 min_rank=3 imbalance_ratio=1.147624 max_load_reduction=42101 max_load_reduction_pct=0.856547 moved_to_echo=24539 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=19 rerouted_tokens_per_expert_current_ep_rank=[438, 264, 265, 118, 0, 29, 460, 391, 4836, 0, 0, 0, 0, 0, 0, 0, 0, 5, 88, 3491, 0, 1246, 201, 20, 0, 0, 0, 0, 0, 0, 0, 0, 338, 4, 401, 635, 1593, 2277, 369, 25, 0, 0, 0, 0, 0, 0, 0, 0, 14, 5652, 255, 612, 19, 35, 508, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1265, 10, 3173, 758, 1, 547, 296, 0, 0, 0, 0, 0, 0, 0, 255, 66, 871, 1471, 1756, 128, 1171, 484, 0, 0, 0, 0, 0, 0, 0, 3086, 4, 0, 419, 0, 13, 1477, 325, 1214, 0, 0, 0, 0, 0, 0, 0, 102, 54, 139, 453, 106, 0, 8, 1165, 1865, 1864, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6801, 5051, 5642, 7108, 6054, 6202, 6538, 5756] avg=6144.0000 max=7108 max_rank=3 min=5051 min_rank=1 imbalance_ratio=1.156901 max_load_reduction=42044 max_load_reduction_pct=0.855387 moved_to_echo=24550 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=19 rerouted_tokens_per_expert_current_ep_rank=[422, 239, 117, 291, 0, 59, 108, 208, 4661, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4823, 0, 2003, 194, 10, 0, 0, 0, 0, 0, 0, 0, 0, 99, 126, 466, 556, 3218, 2260, 54, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4549, 561, 443, 25, 1, 407, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1315, 1, 2537, 1125, 0, 543, 238, 0, 0, 0, 0, 0, 0, 0, 387, 14, 357, 1451, 1833, 26, 310, 389, 0, 0, 0, 0, 0, 0, 0, 2958, 11, 0, 65, 1, 1, 1147, 353, 1205, 0, 0, 0, 0, 0, 0, 0, 155, 466, 248, 1171, 362, 1, 2, 830, 1797, 1850, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6105, 7038, 6845, 6014, 5760, 4767, 5741, 6882] avg=6144.0000 max=7038 max_rank=1 min=4767 min_rank=5 imbalance_ratio=1.145508 max_load_reduction=42114 max_load_reduction_pct=0.856812 moved_to_echo=23150 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=19 rerouted_tokens_per_expert_current_ep_rank=[402, 208, 184, 12, 0, 27, 998, 104, 4112, 0, 0, 0, 0, 0, 0, 0, 0, 2, 24, 4656, 8, 1295, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 3, 181, 343, 3238, 2137, 122, 79, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4569, 370, 406, 15, 0, 488, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 819, 0, 2144, 628, 0, 514, 239, 0, 0, 0, 0, 0, 0, 0, 361, 14, 1261, 1971, 1869, 67, 1741, 391, 0, 0, 0, 0, 0, 0, 0, 3583, 14, 1, 280, 0, 7, 1908, 61, 1139, 0, 0, 0, 0, 0, 0, 0, 409, 238, 267, 772, 161, 0, 0, 724, 1585, 1749, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6047, 6156, 6175, 5855, 4346, 7675, 6993, 5905] avg=6144.0000 max=7675 max_rank=5 min=4346 min_rank=4 imbalance_ratio=1.249186 max_load_reduction=41477 max_load_reduction_pct=0.843852 moved_to_echo=24919 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=19 rerouted_tokens_per_expert_current_ep_rank=[245, 228, 156, 4, 0, 205, 227, 110, 3791, 0, 0, 0, 0, 0, 0, 0, 0, 2, 63, 4221, 0, 1917, 149, 34, 0, 0, 0, 0, 0, 0, 0, 0, 129, 32, 588, 603, 3097, 1318, 22, 9, 0, 0, 0, 0, 0, 0, 0, 0, 23, 2957, 810, 1218, 14, 0, 1190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1638, 286, 2475, 2377, 1, 323, 157, 0, 0, 0, 0, 0, 0, 0, 379, 4, 1335, 1172, 2535, 61, 773, 258, 0, 0, 0, 0, 0, 0, 0, 3642, 11, 6, 233, 0, 32, 694, 451, 712, 0, 0, 0, 0, 0, 0, 0, 311, 531, 622, 822, 44, 0, 4, 1337, 1464, 1092, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4966, 6386, 5798, 6214, 7263, 6517, 5781, 6227] avg=6144.0000 max=7263 max_rank=4 min=4966 min_rank=0 imbalance_ratio=1.182129 max_load_reduction=41889 max_load_reduction_pct=0.852234 moved_to_echo=25788 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=19 rerouted_tokens_per_expert_current_ep_rank=[534, 161, 412, 15, 7, 73, 649, 620, 4235, 0, 0, 0, 0, 0, 0, 0, 47, 3, 24, 3939, 35, 1110, 176, 2, 0, 0, 0, 0, 0, 0, 0, 0, 280, 4, 324, 617, 2204, 1982, 83, 222, 0, 0, 0, 0, 0, 0, 0, 0, 22, 4866, 321, 276, 4, 26, 364, 6, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 1, 235, 52, 3239, 1126, 0, 476, 255, 0, 0, 0, 0, 0, 0, 4, 416, 0, 863, 1428, 1362, 69, 1970, 416, 0, 0, 0, 0, 0, 0, 0, 4085, 2, 7, 133, 0, 54, 1691, 124, 1056, 0, 0, 0, 0, 0, 0, 0, 452, 669, 134, 1018, 253, 0, 1, 659, 1633, 1621, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6706, 5336, 5716, 5885, 5389, 6528, 7152, 6440] avg=6144.0000 max=7152 max_rank=6 min=5336 min_rank=1 imbalance_ratio=1.164062 max_load_reduction=42000 max_load_reduction_pct=0.854492 moved_to_echo=25509 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=19 rerouted_tokens_per_expert_current_ep_rank=[295, 235, 169, 16, 0, 32, 185, 83, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 2, 66, 3781, 0, 2629, 170, 1, 0, 0, 0, 0, 0, 0, 0, 0, 72, 2, 194, 347, 2737, 1879, 361, 28, 0, 0, 0, 0, 0, 0, 0, 0, 78, 4695, 747, 397, 4, 1, 737, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1356, 16, 4262, 928, 0, 452, 246, 0, 0, 0, 0, 0, 0, 0, 536, 19, 357, 797, 2236, 95, 794, 402, 0, 0, 0, 0, 0, 0, 0, 3665, 16, 6, 84, 0, 2, 1262, 112, 1001, 0, 0, 0, 0, 0, 0, 0, 131, 155, 568, 305, 789, 0, 0, 1411, 1577, 1537, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5105, 6649, 5620, 6660, 7261, 5236, 6148, 6473] avg=6144.0000 max=7261 max_rank=4 min=5105 min_rank=0 imbalance_ratio=1.181803 max_load_reduction=41891 max_load_reduction_pct=0.852275 moved_to_echo=25118 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[26752, 128, 128, 1664, 128, 128, 10496, 1792, 8704, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 128, 8448, 384, 23680, 11008, 128, 3968, 1920, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1920, 2944, 2432, 5248, 1792, 128, 128, 8192, 13696, 13312, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1280, 256, 2816, 4096, 23040, 16256, 1408, 512, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 3200, 256, 6144, 11008, 17152, 512, 8064, 3200, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 512, 32896, 128, 14208, 1536, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3072, 2176, 1664, 640, 128, 896, 3456, 2432, 35328, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[256, 36224, 3840, 3840, 256, 128, 5120, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 2025, 282, 17, 0, 4, 706, 238, 1, 0, 2, 43, 3081, 1914, 40, 0, 88, 204, 5019, 2760, 0, 1, 70, 2078, 116, 345, 4047, 2418, 1, 1830, 188, 0, 69, 1096, 145, 85, 468, 4, 0, 0, 126, 5, 0, 0, 19, 0, 275, 0, 5957, 3586, 565, 0, 145, 2318, 331, 1295, 3454, 0, 0, 0, 739, 952] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=20 tokens_per_expert_current_ep_rank=[0, 1, 0, 2637, 410, 4, 1, 14, 260, 997, 4, 0, 0, 55, 1898, 1258, 24, 2, 91, 411, 3304, 3334, 5, 0, 366, 676, 195, 1024, 3135, 2087, 0, 1618, 183, 0, 96, 1323, 154, 129, 1171, 39, 0, 17, 114, 10, 0, 4, 11, 1, 461, 5, 5695, 4365, 782, 0, 328, 1261, 1123, 1823, 4842, 4, 1, 4, 653, 742] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=20 tokens_per_expert_current_ep_rank=[1, 79, 0, 2116, 469, 24, 11, 32, 1014, 1090, 0, 0, 4, 63, 1043, 1375, 8, 17, 76, 154, 4274, 2155, 2, 4, 174, 1501, 111, 563, 2909, 3644, 0, 1838, 358, 0, 88, 2339, 269, 127, 1277, 2, 4, 0, 378, 17, 1, 0, 19, 0, 326, 0, 4213, 3911, 276, 1, 499, 2209, 746, 2381, 2937, 1, 1, 0, 1196, 825] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=20 tokens_per_expert_current_ep_rank=[0, 30, 0, 2145, 217, 47, 5, 16, 75, 732, 1, 5, 0, 49, 3276, 1840, 33, 0, 507, 527, 2746, 3226, 1, 2, 284, 635, 674, 698, 2204, 3693, 29, 2078, 158, 1, 70, 561, 209, 238, 1292, 27, 0, 2, 100, 25, 0, 0, 43, 0, 860, 1, 5492, 2977, 422, 0, 72, 2307, 575, 1897, 4968, 9, 6, 3, 692, 370] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=20 tokens_per_expert_current_ep_rank=[0, 7, 0, 2910, 439, 8, 2, 111, 590, 1120, 15, 0, 2, 110, 1020, 904, 159, 17, 398, 139, 4511, 2001, 19, 0, 306, 1325, 559, 1489, 5231, 1921, 3, 462, 361, 0, 90, 475, 278, 84, 859, 16, 1, 0, 191, 2, 0, 1, 3, 0, 346, 1, 4440, 2169, 447, 1, 834, 1272, 1753, 2111, 5906, 1, 16, 13, 727, 976] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=20 tokens_per_expert_current_ep_rank=[0, 6, 0, 2839, 358, 81, 0, 15, 1198, 1152, 12, 0, 4, 210, 1328, 1783, 47, 7, 120, 633, 3746, 853, 3, 3, 330, 659, 84, 1008, 2499, 3229, 5, 1588, 170, 0, 55, 680, 21, 61, 934, 4, 2, 0, 105, 12, 0, 1, 21, 3, 635, 1, 5327, 3982, 969, 0, 578, 1631, 1352, 804, 5203, 0, 0, 0, 1527, 1274] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=20 tokens_per_expert_current_ep_rank=[0, 16, 0, 2369, 407, 3, 8, 24, 335, 606, 1, 0, 4, 98, 2186, 897, 161, 12, 73, 213, 3950, 2412, 0, 0, 13, 1351, 109, 838, 3728, 3827, 0, 1152, 140, 1, 67, 519, 317, 327, 1060, 95, 6, 0, 41, 8, 0, 20, 107, 0, 351, 2, 5766, 3478, 703, 0, 437, 1455, 1184, 1964, 4685, 2, 0, 5, 395, 1224] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=20 tokens_per_expert_current_ep_rank=[0, 43, 7, 2953, 353, 77, 2, 367, 897, 1330, 7, 0, 0, 140, 2713, 2444, 115, 10, 196, 205, 2425, 731, 15, 3, 179, 1868, 173, 885, 3794, 1849, 12, 2451, 357, 10, 182, 476, 141, 170, 754, 35, 25, 2, 157, 68, 0, 0, 40, 2, 490, 1, 5621, 1681, 197, 0, 595, 1960, 981, 1629, 4721, 4, 67, 5, 1334, 1203] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 2637, 410, 4, 1, 14, 2486, 580, 0, 0, 0, 0, 0, 0, 260, 997, 4, 0, 0, 55, 1898, 1258, 335, 503, 88, 0, 0, 0, 0, 0, 24, 2, 91, 411, 2969, 3334, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 366, 676, 195, 1024, 0, 1419, 0, 1618, 0, 0, 0, 0, 0, 0, 0, 0, 183, 0, 96, 1323, 154, 129, 1171, 39, 2574, 742, 0, 0, 0, 0, 0, 0, 0, 17, 114, 10, 0, 4, 11, 1, 5695, 561, 0, 0, 0, 0, 0, 0, 461, 5, 0, 3862, 782, 0, 328, 1261, 0, 0, 0, 0, 0, 0, 0, 0, 1123, 1823, 1614, 4, 1, 4, 653, 742, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6133, 5398, 6836, 5298, 6411, 6413, 6699, 5964] avg=6144.0000 max=6836 max_rank=2 min=5298 min_rank=3 imbalance_ratio=1.112630 max_load_reduction=42316 max_load_reduction_pct=0.860921 moved_to_echo=25487 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2025, 282, 17, 0, 4, 1773, 672, 0, 0, 0, 0, 0, 0, 706, 238, 1, 0, 2, 43, 3081, 1914, 509, 413, 103, 0, 0, 0, 0, 0, 40, 0, 88, 204, 4510, 2760, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 70, 2078, 116, 345, 0, 1643, 1, 1830, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 69, 1096, 145, 85, 468, 4, 3323, 529, 0, 0, 0, 0, 0, 0, 0, 0, 126, 5, 0, 0, 19, 0, 5957, 724, 0, 0, 0, 0, 0, 0, 275, 0, 0, 3173, 565, 0, 145, 2318, 0, 0, 0, 0, 0, 0, 0, 0, 331, 1295, 1152, 0, 0, 0, 739, 952, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4773, 7010, 7603, 6083, 5907, 6831, 6476, 4469] avg=6144.0000 max=7603 max_rank=2 min=4469 min_rank=7 imbalance_ratio=1.237467 max_load_reduction=41549 max_load_reduction_pct=0.845317 moved_to_echo=23683 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 30, 0, 2145, 217, 47, 5, 16, 2551, 1026, 0, 0, 0, 0, 0, 0, 75, 732, 1, 5, 0, 49, 3276, 1840, 278, 343, 157, 0, 0, 0, 0, 0, 33, 0, 507, 527, 2468, 3226, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 284, 635, 674, 698, 0, 2510, 29, 2078, 0, 0, 0, 0, 0, 0, 0, 0, 158, 1, 70, 561, 209, 238, 1292, 27, 1809, 761, 0, 0, 0, 0, 0, 0, 0, 2, 100, 25, 0, 0, 43, 0, 5492, 395, 0, 0, 0, 0, 0, 0, 860, 1, 0, 2634, 422, 0, 72, 2307, 0, 0, 0, 0, 0, 0, 0, 0, 575, 1897, 1656, 9, 6, 3, 692, 370, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6037, 6756, 6764, 6908, 5126, 6057, 6296, 5208] avg=6144.0000 max=6908 max_rank=3 min=5126 min_rank=4 imbalance_ratio=1.124349 max_load_reduction=42244 max_load_reduction_pct=0.859456 moved_to_echo=22687 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=20 rerouted_tokens_per_expert_current_ep_rank=[1, 79, 0, 2116, 469, 24, 11, 32, 1508, 1013, 0, 0, 0, 0, 0, 0, 1014, 1090, 0, 0, 4, 63, 1043, 1375, 433, 451, 155, 0, 0, 0, 0, 0, 8, 17, 76, 154, 3841, 2155, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 174, 1501, 111, 563, 0, 2476, 0, 1838, 0, 0, 0, 0, 0, 0, 0, 0, 358, 0, 88, 2339, 269, 127, 1277, 2, 2388, 450, 0, 0, 0, 0, 0, 0, 4, 0, 378, 17, 1, 0, 19, 0, 4213, 521, 0, 0, 0, 0, 0, 0, 326, 0, 0, 3460, 276, 1, 499, 2209, 0, 0, 0, 0, 0, 0, 0, 0, 746, 2381, 979, 1, 1, 0, 1196, 825, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5253, 5628, 6257, 6663, 7298, 5153, 6771, 6129] avg=6144.0000 max=7298 max_rank=4 min=5153 min_rank=5 imbalance_ratio=1.187826 max_load_reduction=41854 max_load_reduction_pct=0.851522 moved_to_echo=25351 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 0, 2839, 358, 81, 0, 15, 2672, 897, 0, 0, 0, 0, 0, 0, 1198, 1152, 12, 0, 4, 210, 1328, 1783, 379, 459, 137, 0, 0, 0, 0, 0, 47, 7, 120, 633, 3367, 853, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 330, 659, 84, 1008, 0, 2195, 5, 1588, 0, 0, 0, 0, 0, 0, 0, 0, 170, 0, 55, 680, 21, 61, 934, 4, 2053, 797, 0, 0, 0, 0, 0, 0, 2, 0, 105, 12, 0, 1, 21, 3, 5327, 446, 0, 0, 0, 0, 0, 0, 635, 1, 0, 3523, 969, 0, 578, 1631, 0, 0, 0, 0, 0, 0, 0, 0, 1352, 804, 1734, 0, 0, 0, 1527, 1274, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6868, 6662, 5033, 5869, 4775, 5917, 7337, 6691] avg=6144.0000 max=7337 max_rank=6 min=4775 min_rank=4 imbalance_ratio=1.194173 max_load_reduction=41815 max_load_reduction_pct=0.850728 moved_to_echo=24720 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 16, 0, 2369, 407, 3, 8, 24, 2410, 1067, 0, 0, 0, 0, 0, 0, 335, 606, 1, 0, 4, 98, 2186, 897, 404, 404, 167, 0, 0, 0, 0, 0, 161, 12, 73, 213, 3546, 2412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1351, 109, 838, 0, 2593, 0, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 140, 1, 67, 519, 317, 327, 1060, 95, 3062, 720, 0, 0, 0, 0, 0, 0, 6, 0, 41, 8, 0, 20, 107, 0, 5766, 666, 0, 0, 0, 0, 0, 0, 351, 2, 0, 3074, 703, 0, 437, 1455, 0, 0, 0, 0, 0, 0, 0, 0, 1184, 1964, 1555, 2, 0, 5, 395, 1224, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6304, 5102, 6417, 6056, 6308, 6614, 6022, 6329] avg=6144.0000 max=6614 max_rank=5 min=5102 min_rank=1 imbalance_ratio=1.076497 max_load_reduction=42538 max_load_reduction_pct=0.865438 moved_to_echo=25273 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 43, 7, 2953, 353, 77, 2, 367, 2424, 514, 0, 0, 0, 0, 0, 0, 897, 1330, 7, 0, 0, 140, 2713, 2444, 245, 193, 78, 0, 0, 0, 0, 0, 115, 10, 196, 205, 2180, 731, 15, 3, 0, 0, 0, 0, 0, 0, 0, 0, 179, 1868, 173, 885, 0, 1257, 12, 2451, 0, 0, 0, 0, 0, 0, 0, 0, 357, 10, 182, 476, 141, 170, 754, 35, 3116, 723, 0, 0, 0, 0, 0, 0, 25, 2, 157, 68, 0, 0, 40, 2, 5621, 678, 0, 0, 0, 0, 0, 0, 490, 1, 0, 1488, 197, 0, 595, 1960, 0, 0, 0, 0, 0, 0, 0, 0, 981, 1629, 1574, 4, 67, 5, 1334, 1203, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6740, 8047, 3455, 6825, 5964, 6593, 4731, 6797] avg=6144.0000 max=8047 max_rank=1 min=3455 min_rank=2 imbalance_ratio=1.309733 max_load_reduction=41105 max_load_reduction_pct=0.836283 moved_to_echo=24085 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 0, 2910, 439, 8, 2, 111, 3033, 534, 0, 0, 0, 0, 0, 0, 590, 1120, 15, 0, 2, 110, 1020, 904, 457, 250, 81, 0, 0, 0, 0, 0, 159, 17, 398, 139, 4054, 2001, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 306, 1325, 559, 1489, 0, 1306, 3, 462, 0, 0, 0, 0, 0, 0, 0, 0, 361, 0, 90, 475, 278, 84, 859, 16, 4295, 905, 0, 0, 0, 0, 0, 0, 1, 0, 191, 2, 0, 1, 3, 0, 4440, 936, 0, 0, 0, 0, 0, 0, 346, 1, 0, 1919, 447, 1, 834, 1272, 0, 0, 0, 0, 0, 0, 0, 0, 1753, 2111, 1968, 1, 16, 13, 727, 976, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7044, 4549, 6787, 5450, 7363, 5574, 4820, 7565] avg=6144.0000 max=7565 max_rank=7 min=4549 min_rank=1 imbalance_ratio=1.231283 max_load_reduction=41587 max_load_reduction_pct=0.846090 moved_to_echo=25322 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[8064, 13952, 12288, 128, 128, 128, 7296, 7680, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 256, 128, 20096, 2944, 384, 128, 640, 18944, 6400, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5120, 7296, 128, 128, 128, 768, 16640, 12416, 3072, 3072, 1024, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[1792, 10112, 2048, 6912, 0, 15488, 128, 13056, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3840, 128, 0, 23168, 4480, 128, 3584, 14464, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=50176 observed_loss=-1024 tokens_per_expert=[128, 128, 1280, 256, 128, 128, 384, 128, 42624, 4992, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1920, 128, 768, 7552, 1536, 1280, 7936, 256, 22656, 5632, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[640, 128, 1664, 2560, 27008, 17536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=50176 slack_slots=48128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=21 tokens_per_expert_current_ep_rank=[0, 103, 9, 112, 155, 1750, 8, 0, 0, 5498, 0, 633, 3, 473, 17, 432, 0, 1123, 2662, 0, 1802, 630, 0, 13, 5, 6037, 1, 92, 6978, 170, 0, 0, 872, 0, 45, 0, 324, 55, 12, 2, 33, 587, 628, 3544, 0, 0, 1474, 8, 0, 0, 323, 535, 0, 0, 600, 2264, 4368, 3075, 47, 0, 0, 1, 1649, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=21 tokens_per_expert_current_ep_rank=[0, 267, 213, 106, 150, 2116, 5, 0, 11, 5387, 3, 620, 25, 502, 47, 353, 0, 1750, 2055, 8, 1415, 447, 5, 89, 7, 3909, 13, 351, 6473, 262, 0, 0, 607, 1, 302, 0, 652, 277, 117, 124, 318, 1228, 755, 3155, 0, 12, 356, 51, 0, 6, 380, 333, 3, 0, 408, 4639, 4085, 3242, 294, 6, 0, 26, 1186, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=21 tokens_per_expert_current_ep_rank=[5, 55, 396, 249, 35, 2691, 15, 0, 0, 5228, 1, 892, 13, 1627, 3, 195, 0, 2459, 2019, 0, 4289, 299, 0, 2, 0, 3242, 11, 83, 4181, 366, 0, 0, 358, 1, 288, 0, 251, 323, 283, 21, 326, 794, 545, 3770, 2, 0, 327, 23, 2, 0, 730, 404, 9, 0, 571, 4860, 2119, 3386, 18, 0, 0, 6, 1379, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=21 tokens_per_expert_current_ep_rank=[3, 92, 291, 73, 346, 1523, 40, 1, 6, 6117, 0, 415, 13, 412, 3, 469, 0, 1078, 2133, 5, 2032, 845, 1, 10, 0, 6107, 21, 161, 7704, 398, 0, 0, 432, 3, 34, 0, 142, 15, 75, 104, 294, 116, 637, 3377, 1, 0, 302, 11, 0, 0, 357, 551, 0, 5, 304, 3427, 4306, 3002, 86, 1, 1, 0, 1270, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=21 tokens_per_expert_current_ep_rank=[0, 132, 769, 521, 327, 1386, 15, 1, 0, 5931, 0, 845, 32, 471, 4, 928, 0, 1295, 2754, 3, 2470, 1297, 0, 1, 1, 4472, 1, 14, 6550, 505, 0, 0, 612, 0, 35, 0, 117, 17, 64, 82, 136, 490, 716, 2546, 0, 2, 797, 20, 0, 0, 222, 199, 0, 0, 1415, 2760, 5251, 1907, 59, 1, 1, 0, 978, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=21 tokens_per_expert_current_ep_rank=[0, 250, 84, 240, 248, 1884, 5, 1, 0, 4564, 0, 504, 22, 726, 10, 690, 0, 1068, 2405, 1, 1441, 627, 0, 33, 0, 6632, 2, 96, 7025, 166, 0, 0, 535, 3, 54, 0, 144, 46, 30, 20, 54, 224, 779, 3643, 1, 0, 1283, 12, 1, 1, 230, 429, 0, 1, 849, 4561, 4394, 2223, 90, 0, 0, 6, 807, 8] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=21 tokens_per_expert_current_ep_rank=[2, 179, 194, 196, 57, 1100, 19, 1, 36, 5339, 0, 486, 77, 993, 2, 556, 5, 1144, 2623, 21, 4167, 1673, 9, 11, 0, 4136, 13, 142, 5313, 456, 0, 2, 285, 15, 12, 37, 496, 73, 53, 69, 311, 533, 1855, 5547, 4, 7, 2052, 13, 1, 3, 449, 364, 3, 0, 396, 1756, 1791, 844, 36, 2, 0, 5, 3122, 66] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=21 tokens_per_expert_current_ep_rank=[0, 90, 299, 319, 121, 816, 1, 0, 1, 3515, 3, 331, 56, 428, 23, 122, 0, 2393, 2975, 5, 2328, 1046, 2, 12, 0, 4806, 9, 1045, 6033, 282, 0, 0, 438, 41, 119, 0, 235, 32, 236, 86, 231, 1676, 933, 4311, 1, 0, 692, 24, 0, 0, 387, 1095, 0, 0, 421, 2814, 2697, 3119, 30, 0, 1, 0, 2470, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 267, 213, 106, 150, 2116, 5, 0, 2015, 645, 730, 0, 0, 0, 0, 0, 11, 4488, 3, 620, 25, 502, 47, 353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1750, 2055, 8, 719, 447, 5, 89, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3909, 13, 351, 663, 262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 607, 1, 302, 0, 652, 277, 117, 124, 5080, 0, 0, 0, 0, 0, 0, 0, 318, 1228, 755, 2901, 0, 12, 356, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 380, 333, 3, 0, 408, 4639, 899, 254, 51, 0, 0, 0, 0, 0, 2070, 3242, 294, 6, 0, 26, 1186, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6247, 6049, 5073, 5205, 7160, 5621, 6973, 6824] avg=6144.0000 max=7160 max_rank=4 min=5073 min_rank=2 imbalance_ratio=1.165365 max_load_reduction=41992 max_load_reduction_pct=0.854329 moved_to_echo=26578 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=21 rerouted_tokens_per_expert_current_ep_rank=[3, 92, 291, 73, 346, 1523, 40, 1, 2124, 927, 869, 0, 0, 0, 0, 0, 6, 5096, 0, 415, 13, 412, 3, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1078, 2133, 5, 1031, 845, 1, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6107, 21, 161, 789, 398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 432, 3, 34, 0, 142, 15, 75, 104, 6046, 0, 0, 0, 0, 0, 0, 0, 294, 116, 637, 3105, 1, 0, 302, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 357, 551, 0, 5, 304, 3427, 1021, 272, 74, 0, 0, 0, 0, 0, 2182, 3002, 86, 1, 1, 0, 1270, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6289, 6414, 5103, 7476, 6851, 4466, 6011, 6542] avg=6144.0000 max=7476 max_rank=3 min=4466 min_rank=5 imbalance_ratio=1.216797 max_load_reduction=41676 max_load_reduction_pct=0.847900 moved_to_echo=23870 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 103, 9, 112, 155, 1750, 8, 0, 2154, 822, 787, 0, 0, 0, 0, 0, 0, 4581, 0, 633, 3, 473, 17, 432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1123, 2662, 0, 915, 630, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 5, 6037, 1, 92, 715, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 872, 0, 45, 0, 324, 55, 12, 2, 5476, 0, 0, 0, 0, 0, 0, 0, 33, 587, 628, 3258, 0, 0, 1474, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 323, 535, 0, 0, 600, 2264, 917, 286, 65, 0, 0, 0, 0, 0, 2214, 3075, 47, 0, 0, 1, 1649, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5900, 6139, 5343, 7020, 6786, 5988, 4990, 6986] avg=6144.0000 max=7020 max_rank=3 min=4990 min_rank=6 imbalance_ratio=1.142578 max_load_reduction=42132 max_load_reduction_pct=0.857178 moved_to_echo=24750 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 132, 769, 521, 327, 1386, 15, 1, 2590, 1127, 739, 0, 0, 0, 0, 0, 0, 4941, 0, 845, 32, 471, 4, 928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1295, 2754, 3, 1253, 1297, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4472, 1, 14, 670, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 612, 0, 35, 0, 117, 17, 64, 82, 5141, 0, 0, 0, 0, 0, 0, 0, 136, 490, 716, 2341, 0, 2, 797, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 222, 199, 0, 0, 1415, 2760, 990, 205, 90, 0, 0, 0, 0, 0, 2661, 1907, 59, 1, 1, 0, 978, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7607, 7221, 6603, 5663, 6068, 4502, 5881, 5607] avg=6144.0000 max=7607 max_rank=0 min=4502 min_rank=5 imbalance_ratio=1.238118 max_load_reduction=41545 max_load_reduction_pct=0.845235 moved_to_echo=22058 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=21 rerouted_tokens_per_expert_current_ep_rank=[5, 55, 396, 249, 35, 2691, 15, 0, 1048, 1959, 477, 0, 0, 0, 0, 0, 0, 4352, 1, 892, 13, 1627, 3, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2459, 2019, 0, 2170, 299, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3242, 11, 83, 419, 366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 358, 1, 288, 0, 251, 323, 283, 21, 3285, 0, 0, 0, 0, 0, 0, 0, 326, 794, 545, 3462, 2, 0, 327, 23, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 730, 404, 9, 0, 571, 4860, 876, 308, 160, 0, 0, 0, 0, 0, 1071, 3386, 18, 0, 0, 6, 1379, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6930, 7083, 6949, 4121, 4810, 5479, 7920, 5860] avg=6144.0000 max=7920 max_rank=6 min=4121 min_rank=3 imbalance_ratio=1.289062 max_load_reduction=41232 max_load_reduction_pct=0.838867 moved_to_echo=24069 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 250, 84, 240, 248, 1884, 5, 1, 2167, 657, 793, 0, 0, 0, 0, 0, 0, 3803, 0, 504, 22, 726, 10, 690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 2405, 1, 732, 627, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6632, 2, 96, 719, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 3, 54, 0, 144, 46, 30, 20, 5513, 0, 0, 0, 0, 0, 0, 0, 54, 224, 779, 3349, 1, 0, 1283, 12, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 230, 429, 0, 1, 849, 4561, 761, 294, 52, 0, 0, 0, 0, 0, 2227, 2223, 90, 0, 0, 6, 807, 8, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6329, 5755, 4866, 7615, 6345, 5702, 7179, 5361] avg=6144.0000 max=7615 max_rank=3 min=4866 min_rank=2 imbalance_ratio=1.239421 max_load_reduction=41537 max_load_reduction_pct=0.845072 moved_to_echo=24587 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=21 rerouted_tokens_per_expert_current_ep_rank=[2, 179, 194, 196, 57, 1100, 19, 1, 883, 1901, 599, 0, 0, 0, 0, 0, 36, 4448, 0, 486, 77, 993, 2, 556, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1144, 2623, 21, 2114, 1673, 9, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4136, 13, 142, 544, 456, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 285, 15, 12, 37, 496, 73, 53, 69, 4170, 0, 0, 0, 0, 0, 0, 0, 311, 533, 1855, 5099, 4, 7, 2052, 13, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 449, 364, 3, 0, 396, 1756, 891, 448, 152, 0, 0, 0, 0, 0, 908, 844, 36, 2, 0, 5, 3122, 66, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5131, 6598, 7600, 5293, 5210, 9874, 4463, 4983] avg=6144.0000 max=9874 max_rank=5 min=4463 min_rank=6 imbalance_ratio=1.607096 max_load_reduction=39278 max_load_reduction_pct=0.799113 moved_to_echo=24530 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 90, 299, 319, 121, 816, 1, 0, 1330, 1062, 681, 0, 0, 0, 0, 0, 1, 2929, 3, 331, 56, 428, 23, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2393, 2975, 5, 1182, 1046, 2, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4806, 9, 1045, 617, 282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 438, 41, 119, 0, 235, 32, 236, 86, 4735, 0, 0, 0, 0, 0, 0, 0, 231, 1676, 933, 3963, 1, 0, 692, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 387, 1095, 0, 0, 421, 2814, 586, 348, 84, 0, 0, 0, 0, 0, 1367, 3119, 30, 0, 1, 0, 2470, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4719, 3893, 7615, 6759, 5922, 7520, 5735, 6989] avg=6144.0000 max=7615 max_rank=2 min=3893 min_rank=1 imbalance_ratio=1.239421 max_load_reduction=41537 max_load_reduction_pct=0.845072 moved_to_echo=26166 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 1280, 2304, 1920, 1536, 13312, 128, 128, 14336, 9216, 5760, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1792, 5760, 6912, 27520, 128, 128, 7296, 256, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[14720, 20864, 768, 128, 128, 128, 12928, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 128, 3200, 3968, 128, 128, 4992, 27136, 7040, 2432, 768, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 34688, 128, 4736, 256, 5632, 128, 3840, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 12416, 19712, 128, 10240, 6912, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 39424, 128, 2048, 5248, 2688, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4224, 128, 896, 128, 2432, 896, 896, 512, 39552, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=22 tokens_per_expert_current_ep_rank=[1331, 39, 9, 5914, 0, 8, 316, 525, 5, 261, 67, 1, 3, 0, 75, 2248, 0, 5, 56, 0, 0, 94, 38, 5446, 1, 0, 1, 2, 1539, 3, 1, 0, 51, 2079, 1054, 31, 5430, 0, 1468, 4721, 4, 0, 8, 843, 0, 0, 7885, 40, 13, 438, 0, 7, 0, 16, 6, 0, 319, 4, 0, 2979, 12, 1588, 79, 2089] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=22 tokens_per_expert_current_ep_rank=[740, 67, 8, 4985, 1, 16, 454, 446, 3, 351, 83, 48, 1, 0, 66, 3349, 0, 2, 54, 1, 0, 7, 9, 5196, 7, 0, 15, 0, 2133, 3, 1, 0, 49, 1576, 108, 793, 6266, 5, 1926, 4660, 0, 0, 0, 581, 4, 0, 7556, 6, 27, 334, 34, 2, 0, 53, 27, 0, 390, 7, 0, 3204, 11, 2622, 95, 770] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=22 tokens_per_expert_current_ep_rank=[2114, 187, 0, 5991, 0, 6, 201, 262, 0, 56, 180, 5, 0, 0, 104, 2211, 0, 1, 6, 0, 0, 26, 185, 4638, 4, 0, 1, 0, 2670, 0, 0, 0, 21, 1627, 407, 51, 5870, 0, 1713, 4785, 0, 0, 3, 948, 17, 0, 8039, 0, 44, 111, 0, 0, 0, 30, 0, 0, 266, 0, 0, 3285, 5, 1351, 43, 1688] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=22 tokens_per_expert_current_ep_rank=[998, 240, 8, 5525, 59, 18, 892, 327, 3, 528, 31, 87, 35, 0, 18, 1231, 2, 16, 46, 1, 22, 22, 34, 2698, 3, 3, 281, 0, 3593, 1, 4, 7, 21, 1918, 262, 28, 6567, 3, 3573, 6066, 1, 8, 0, 219, 120, 68, 7942, 0, 14, 968, 0, 1, 0, 223, 1, 21, 442, 9, 6, 1292, 7, 1614, 93, 932] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=22 tokens_per_expert_current_ep_rank=[2011, 467, 18, 5528, 0, 33, 213, 224, 8, 162, 58, 11, 0, 0, 13, 1328, 0, 3, 49, 0, 0, 180, 53, 4128, 2, 0, 1, 0, 2522, 0, 0, 0, 71, 1729, 1387, 56, 5220, 0, 2663, 5869, 0, 0, 9, 1115, 8, 0, 8066, 0, 58, 164, 0, 0, 0, 69, 17, 0, 476, 7, 0, 2599, 1, 1065, 44, 1447] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=22 tokens_per_expert_current_ep_rank=[1103, 64, 2, 6689, 2, 10, 262, 575, 8, 242, 85, 2, 0, 0, 28, 1331, 0, 0, 27, 0, 0, 16, 18, 4946, 3, 0, 0, 7, 2999, 0, 0, 0, 42, 943, 609, 80, 7064, 1, 1915, 5117, 0, 0, 1, 359, 7, 0, 8051, 2, 41, 363, 0, 0, 1, 12, 35, 1, 411, 0, 0, 2920, 2, 1461, 35, 1260] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=22 tokens_per_expert_current_ep_rank=[1999, 55, 1, 6123, 0, 3, 335, 194, 0, 351, 189, 28, 6, 0, 50, 1181, 0, 2, 13, 15, 0, 67, 27, 3689, 0, 0, 3, 0, 3013, 0, 0, 0, 11, 1803, 229, 51, 5578, 0, 4403, 5705, 0, 0, 0, 809, 3, 0, 7598, 0, 95, 192, 0, 19, 0, 9, 4, 3, 443, 1, 17, 2579, 1, 1394, 87, 774] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=22 tokens_per_expert_current_ep_rank=[880, 145, 39, 5804, 35, 48, 522, 186, 36, 252, 25, 5, 2, 0, 9, 1519, 1, 1, 8, 0, 11, 240, 4, 3807, 1, 8, 6, 3, 2594, 0, 2, 5, 1, 2505, 651, 7, 5625, 0, 2285, 5859, 0, 0, 0, 301, 41, 0, 7865, 2, 22, 413, 1, 0, 0, 107, 0, 0, 315, 0, 1, 2515, 12, 1578, 73, 2775] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=22 rerouted_tokens_per_expert_current_ep_rank=[740, 67, 8, 3261, 1, 16, 454, 446, 0, 0, 0, 0, 0, 0, 0, 0, 3, 351, 83, 48, 1, 0, 66, 3349, 3127, 324, 0, 0, 0, 0, 0, 0, 0, 2, 54, 1, 0, 7, 9, 5196, 1381, 51, 0, 0, 0, 0, 0, 0, 7, 0, 15, 0, 2133, 3, 1, 0, 2323, 558, 343, 0, 0, 0, 0, 0, 49, 1576, 108, 793, 0, 5, 1926, 975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 581, 4, 0, 5233, 6, 0, 0, 0, 0, 0, 0, 0, 0, 27, 334, 34, 2, 0, 53, 27, 0, 5942, 0, 0, 0, 0, 0, 0, 0, 390, 7, 0, 3153, 11, 2622, 95, 770, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4993, 7352, 6701, 5383, 5432, 5824, 6419, 7048] avg=6144.0000 max=7352 max_rank=1 min=4993 min_rank=0 imbalance_ratio=1.196615 max_load_reduction=41800 max_load_reduction_pct=0.850423 moved_to_echo=24723 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=22 rerouted_tokens_per_expert_current_ep_rank=[1331, 39, 9, 3869, 0, 8, 316, 525, 0, 0, 0, 0, 0, 0, 0, 0, 5, 261, 67, 1, 3, 0, 75, 2248, 3168, 280, 0, 0, 0, 0, 0, 0, 0, 5, 56, 0, 0, 94, 38, 5446, 1638, 47, 0, 0, 0, 0, 0, 0, 1, 0, 1, 2, 1539, 3, 1, 0, 2425, 565, 407, 0, 0, 0, 0, 0, 51, 2079, 1054, 31, 0, 0, 1468, 988, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8, 843, 0, 0, 5460, 40, 0, 0, 0, 0, 0, 0, 0, 0, 13, 438, 0, 7, 0, 16, 6, 0, 5150, 0, 0, 0, 0, 0, 0, 0, 319, 4, 0, 2932, 12, 1588, 79, 2089, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6097, 6108, 7324, 4944, 5671, 6355, 5630, 7023] avg=6144.0000 max=7324 max_rank=2 min=4944 min_rank=3 imbalance_ratio=1.192057 max_load_reduction=41828 max_load_reduction_pct=0.850993 moved_to_echo=24679 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=22 rerouted_tokens_per_expert_current_ep_rank=[2114, 187, 0, 3920, 0, 6, 201, 262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 180, 5, 0, 0, 104, 2211, 3210, 303, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 0, 26, 185, 4638, 1659, 52, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 2670, 0, 0, 0, 2472, 573, 412, 0, 0, 0, 0, 0, 21, 1627, 407, 51, 0, 0, 1713, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 948, 17, 0, 5567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 111, 0, 0, 0, 30, 0, 0, 5567, 0, 0, 0, 0, 0, 0, 0, 266, 0, 0, 3233, 5, 1351, 43, 1688, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6690, 6069, 6567, 6132, 4821, 6535, 5752, 6586] avg=6144.0000 max=6690 max_rank=0 min=4821 min_rank=4 imbalance_ratio=1.088867 max_load_reduction=42462 max_load_reduction_pct=0.863892 moved_to_echo=23694 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=22 rerouted_tokens_per_expert_current_ep_rank=[2011, 467, 18, 3617, 0, 33, 213, 224, 0, 0, 0, 0, 0, 0, 0, 0, 8, 162, 58, 11, 0, 0, 13, 1328, 3938, 271, 0, 0, 0, 0, 0, 0, 0, 3, 49, 0, 0, 180, 53, 4128, 1531, 41, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 2522, 0, 0, 0, 2480, 703, 380, 0, 0, 0, 0, 0, 71, 1729, 1387, 56, 0, 0, 2663, 1228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1115, 8, 0, 5586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 164, 0, 0, 0, 69, 17, 0, 4949, 0, 0, 0, 0, 0, 0, 0, 476, 7, 0, 2558, 1, 1065, 44, 1447, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6583, 5789, 5985, 6088, 7134, 6718, 5257, 5598] avg=6144.0000 max=7134 max_rank=4 min=5257 min_rank=6 imbalance_ratio=1.161133 max_load_reduction=42018 max_load_reduction_pct=0.854858 moved_to_echo=24707 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=22 rerouted_tokens_per_expert_current_ep_rank=[1999, 55, 1, 3998, 0, 3, 335, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 351, 189, 28, 6, 0, 50, 1181, 3832, 289, 0, 0, 0, 0, 0, 0, 0, 2, 13, 15, 0, 67, 27, 3689, 1700, 46, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3013, 0, 0, 0, 2341, 687, 425, 0, 0, 0, 0, 0, 11, 1803, 229, 51, 0, 0, 4403, 1186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 809, 3, 0, 5257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 192, 0, 19, 0, 9, 4, 3, 5289, 0, 0, 0, 0, 0, 0, 0, 443, 1, 17, 2533, 1, 1394, 87, 774, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6585, 5926, 5559, 6469, 7683, 6069, 5611, 5250] avg=6144.0000 max=7683 max_rank=4 min=5250 min_rank=7 imbalance_ratio=1.250488 max_load_reduction=41469 max_load_reduction_pct=0.843689 moved_to_echo=24613 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=22 rerouted_tokens_per_expert_current_ep_rank=[1103, 64, 2, 4376, 2, 10, 262, 575, 0, 0, 0, 0, 0, 0, 0, 0, 8, 242, 85, 2, 0, 0, 28, 1331, 3433, 366, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 16, 18, 4946, 1853, 46, 0, 0, 0, 0, 0, 0, 3, 0, 0, 7, 2999, 0, 0, 0, 2476, 613, 460, 0, 0, 0, 0, 0, 42, 943, 609, 80, 0, 1, 1915, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 359, 7, 0, 5575, 2, 0, 0, 0, 0, 0, 0, 0, 0, 41, 363, 0, 0, 1, 12, 35, 1, 6698, 0, 0, 0, 0, 0, 0, 0, 411, 0, 0, 2874, 2, 1461, 35, 1260, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6394, 5495, 6906, 6558, 4661, 5944, 7151, 6043] avg=6144.0000 max=7151 max_rank=6 min=4661 min_rank=4 imbalance_ratio=1.163900 max_load_reduction=42001 max_load_reduction_pct=0.854513 moved_to_echo=23799 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=22 rerouted_tokens_per_expert_current_ep_rank=[998, 240, 8, 3615, 59, 18, 892, 327, 0, 0, 0, 0, 0, 0, 0, 0, 3, 528, 31, 87, 35, 0, 18, 1231, 4070, 340, 0, 0, 0, 0, 0, 0, 2, 16, 46, 1, 22, 22, 34, 2698, 1530, 20, 0, 0, 0, 0, 0, 0, 3, 3, 281, 0, 3593, 1, 4, 7, 2442, 726, 380, 0, 0, 0, 0, 0, 21, 1918, 262, 28, 0, 3, 3573, 1270, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8, 0, 219, 120, 68, 5500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 968, 0, 1, 0, 223, 1, 21, 6227, 0, 0, 0, 0, 0, 0, 0, 442, 9, 6, 1272, 7, 1614, 93, 932, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6157, 6343, 4391, 7440, 7075, 5916, 7455, 4375] avg=6144.0000 max=7455 max_rank=6 min=4375 min_rank=7 imbalance_ratio=1.213379 max_load_reduction=41697 max_load_reduction_pct=0.848328 moved_to_echo=24821 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=22 rerouted_tokens_per_expert_current_ep_rank=[880, 145, 39, 3798, 35, 48, 522, 186, 0, 0, 0, 0, 0, 0, 0, 0, 36, 252, 25, 5, 2, 0, 9, 1519, 3931, 291, 0, 0, 0, 0, 0, 0, 1, 1, 8, 0, 11, 240, 4, 3807, 1607, 40, 0, 0, 0, 0, 0, 0, 1, 8, 6, 3, 2594, 0, 2, 5, 2418, 702, 399, 0, 0, 0, 0, 0, 1, 2505, 651, 7, 0, 0, 2285, 1226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 301, 41, 0, 5447, 2, 0, 0, 0, 0, 0, 0, 0, 0, 22, 413, 1, 0, 0, 107, 0, 0, 5334, 0, 0, 0, 0, 0, 0, 0, 315, 0, 1, 2475, 12, 1578, 73, 2775, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5653, 6070, 5719, 6138, 6675, 5791, 5877, 7229] avg=6144.0000 max=7229 max_rank=7 min=5653 min_rank=0 imbalance_ratio=1.176595 max_load_reduction=41923 max_load_reduction_pct=0.852926 moved_to_echo=25572 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[384, 14208, 4736, 1152, 0, 128, 19968, 8960, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[11264, 1280, 128, 30464, 128, 256, 3200, 2816, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 2304, 768, 256, 128, 0, 384, 14464, 28800, 2560, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=50304 observed_loss=-1152 tokens_per_expert=[128, 128, 384, 128, 21120, 128, 128, 128, 19456, 5248, 3328, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3072, 128, 128, 21120, 128, 12800, 640, 11776, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 384, 128, 128, 768, 384, 34560, 12928, 384, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 128, 5248, 256, 128, 43648, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[384, 3072, 128, 128, 128, 640, 128, 128, 45184, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=50304 slack_slots=48000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=23 tokens_per_expert_current_ep_rank=[0, 51, 49, 1021, 6, 47, 7434, 4, 338, 738, 2, 0, 3819, 0, 1338, 2, 6, 7593, 793, 0, 1, 1, 2, 171, 31, 0, 0, 0, 0, 8, 11, 0, 1301, 728, 5, 19, 0, 39, 1071, 0, 15, 7, 2778, 0, 1, 1011, 4518, 0, 13, 6475, 1, 1227, 10, 1, 5355, 6, 0, 13, 1089, 0, 0, 1, 0, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 119, 901, 13, 17, 7510, 5, 70, 102, 0, 2, 1752, 0, 1544, 2, 29, 7283, 824, 0, 0, 0, 0, 112, 29, 8, 0, 0, 0, 39, 112, 127, 1275, 512, 0, 0, 2, 71, 924, 0, 52, 7, 2587, 0, 0, 2347, 3499, 1, 20, 7016, 0, 2379, 12, 0, 6258, 15, 0, 3, 1557, 3, 6, 3, 3, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=23 tokens_per_expert_current_ep_rank=[0, 73, 82, 1671, 14, 128, 7444, 27, 170, 241, 4, 15, 3360, 0, 641, 41, 68, 7621, 704, 4, 0, 1, 13, 174, 279, 1, 0, 0, 2, 68, 7, 22, 943, 2118, 6, 3, 2, 35, 1012, 73, 55, 29, 3141, 1, 0, 1171, 3235, 0, 58, 6745, 0, 1823, 55, 5, 4497, 22, 1, 3, 1206, 1, 7, 1, 0, 29] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=23 tokens_per_expert_current_ep_rank=[0, 1, 160, 1260, 5, 10, 6324, 1, 110, 225, 0, 0, 3510, 0, 1498, 1, 5, 7353, 1230, 0, 0, 1, 1, 55, 67, 1, 0, 1, 0, 15, 179, 9, 982, 711, 0, 4, 0, 35, 998, 0, 20, 18, 4218, 0, 0, 2366, 3011, 0, 17, 7081, 0, 1633, 548, 0, 4530, 17, 0, 2, 925, 0, 12, 0, 0, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=23 tokens_per_expert_current_ep_rank=[1, 1, 115, 856, 2, 6, 6278, 6, 106, 192, 0, 0, 2952, 0, 1557, 2, 1, 7329, 975, 0, 0, 4, 1, 216, 197, 0, 0, 2, 0, 13, 6, 9, 1641, 1457, 3, 0, 0, 26, 2232, 1, 67, 8, 4080, 0, 0, 1107, 3990, 2, 85, 6469, 1, 767, 30, 1, 5274, 16, 0, 4, 1052, 2, 3, 0, 0, 7] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=23 tokens_per_expert_current_ep_rank=[0, 31, 18, 1883, 1, 15, 4664, 5, 40, 101, 3, 0, 2053, 0, 1061, 3, 13, 6904, 503, 0, 0, 0, 15, 55, 33, 7, 0, 0, 0, 24, 77, 13, 2137, 735, 14, 13, 2, 9, 1561, 5, 23, 53, 5385, 0, 0, 2255, 4920, 3, 22, 6304, 0, 1042, 11, 0, 6086, 36, 0, 1, 1008, 0, 4, 1, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=23 tokens_per_expert_current_ep_rank=[66, 101, 50, 2283, 6, 26, 6456, 9, 640, 463, 25, 0, 1192, 1, 910, 173, 0, 6603, 1009, 6, 0, 9, 38, 269, 98, 61, 0, 28, 0, 16, 91, 35, 1637, 841, 2, 4, 43, 30, 1851, 0, 74, 69, 5320, 0, 3, 756, 4555, 9, 82, 6725, 0, 1346, 286, 14, 2994, 164, 5, 6, 1568, 1, 4, 73, 2, 24] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=23 tokens_per_expert_current_ep_rank=[2, 0, 120, 1424, 2, 68, 6672, 2, 71, 92, 10, 0, 4435, 0, 762, 14, 1, 6201, 666, 0, 1, 1, 12, 146, 165, 6, 0, 0, 2, 24, 4, 11, 1602, 580, 0, 9, 0, 1, 1922, 0, 44, 100, 3449, 0, 0, 1184, 4894, 0, 18, 5304, 0, 1068, 7, 7, 6769, 5, 0, 0, 1227, 0, 25, 9, 0, 14] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 119, 901, 13, 17, 5178, 5, 0, 0, 0, 0, 0, 0, 0, 0, 70, 102, 0, 2, 1752, 0, 1544, 2, 933, 821, 0, 0, 0, 0, 0, 0, 29, 5252, 824, 0, 0, 0, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 29, 8, 0, 0, 0, 39, 112, 127, 6355, 0, 0, 0, 0, 0, 0, 0, 1275, 512, 0, 0, 2, 71, 924, 0, 1329, 1098, 0, 0, 0, 0, 0, 0, 52, 7, 2587, 0, 0, 2347, 573, 1, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 2379, 12, 0, 5437, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1557, 3, 6, 3, 3, 0, 2926, 1003, 661, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6233, 5226, 6217, 6670, 5211, 5567, 7863, 6165] avg=6144.0000 max=7863 max_rank=6 min=5211 min_rank=4 imbalance_ratio=1.279785 max_load_reduction=41289 max_load_reduction_pct=0.840027 moved_to_echo=24806 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 160, 1260, 5, 10, 4361, 1, 0, 0, 0, 0, 0, 0, 0, 0, 110, 225, 0, 0, 3510, 0, 1498, 1, 942, 594, 0, 0, 0, 0, 0, 0, 5, 5302, 1230, 0, 0, 1, 1, 55, 0, 0, 0, 0, 0, 0, 0, 0, 67, 1, 0, 1, 0, 15, 179, 9, 6414, 0, 0, 0, 0, 0, 0, 0, 982, 711, 0, 4, 0, 35, 998, 0, 1119, 1109, 0, 0, 0, 0, 0, 0, 20, 18, 4218, 0, 0, 2366, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 1633, 548, 0, 3936, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 925, 0, 12, 0, 0, 2, 2518, 844, 667, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5798, 6880, 6594, 6686, 4958, 7115, 6151, 4970] avg=6144.0000 max=7115 max_rank=5 min=4958 min_rank=4 imbalance_ratio=1.158040 max_load_reduction=42037 max_load_reduction_pct=0.855245 moved_to_echo=23194 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 73, 82, 1671, 14, 128, 5133, 27, 0, 0, 0, 0, 0, 0, 0, 0, 170, 241, 4, 15, 3360, 0, 641, 41, 976, 590, 0, 0, 0, 0, 0, 0, 68, 5496, 704, 4, 0, 1, 13, 174, 0, 0, 0, 0, 0, 0, 0, 0, 279, 1, 0, 0, 2, 68, 7, 22, 6110, 0, 0, 0, 0, 0, 0, 0, 943, 2118, 6, 3, 2, 35, 1012, 73, 1317, 1149, 0, 0, 0, 0, 0, 0, 55, 29, 3141, 1, 0, 1171, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 1823, 55, 5, 3907, 22, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 1206, 1, 7, 1, 0, 29, 2705, 994, 635, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7128, 6038, 6460, 6489, 6658, 4927, 5870, 5582] avg=6144.0000 max=7128 max_rank=0 min=4927 min_rank=5 imbalance_ratio=1.160156 max_load_reduction=42024 max_load_reduction_pct=0.854980 moved_to_echo=23037 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 51, 49, 1021, 6, 47, 5126, 4, 0, 0, 0, 0, 0, 0, 0, 0, 338, 738, 2, 0, 3819, 0, 1338, 2, 972, 702, 0, 0, 0, 0, 0, 0, 6, 5476, 793, 0, 1, 1, 2, 171, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 8, 11, 0, 5866, 0, 0, 0, 0, 0, 0, 0, 1301, 728, 5, 19, 0, 39, 1071, 0, 1315, 1145, 0, 0, 0, 0, 0, 0, 15, 7, 2778, 0, 1, 1011, 740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 1, 1227, 10, 1, 4653, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1089, 0, 0, 1, 0, 2, 3778, 993, 609, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6304, 7911, 6450, 5916, 5623, 4552, 5911, 6485] avg=6144.0000 max=7911 max_rank=1 min=4552 min_rank=5 imbalance_ratio=1.287598 max_load_reduction=41241 max_load_reduction_pct=0.839050 moved_to_echo=22571 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=23 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 115, 856, 2, 6, 4329, 6, 0, 0, 0, 0, 0, 0, 0, 0, 106, 192, 0, 0, 2952, 0, 1557, 2, 938, 692, 0, 0, 0, 0, 0, 0, 1, 5286, 975, 0, 0, 4, 1, 216, 0, 0, 0, 0, 0, 0, 0, 0, 197, 0, 0, 2, 0, 13, 6, 9, 5861, 0, 0, 0, 0, 0, 0, 0, 1641, 1457, 3, 0, 0, 26, 2232, 1, 1111, 1105, 0, 0, 0, 0, 0, 0, 67, 8, 4080, 0, 0, 1107, 653, 2, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 1, 767, 30, 1, 4582, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1052, 2, 3, 0, 0, 7, 3337, 838, 608, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5316, 6439, 6483, 6088, 7576, 5917, 5482, 5851] avg=6144.0000 max=7576 max_rank=4 min=5316 min_rank=0 imbalance_ratio=1.233073 max_load_reduction=41576 max_load_reduction_pct=0.845866 moved_to_echo=24826 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=23 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 120, 1424, 2, 68, 4593, 2, 0, 0, 0, 0, 0, 0, 0, 0, 71, 92, 10, 0, 4435, 0, 762, 14, 798, 892, 0, 0, 0, 0, 0, 0, 1, 4463, 666, 0, 1, 1, 12, 146, 0, 0, 0, 0, 0, 0, 0, 0, 165, 6, 0, 0, 2, 24, 4, 11, 4805, 0, 0, 0, 0, 0, 0, 0, 1602, 580, 0, 9, 0, 1, 1922, 0, 1184, 940, 0, 0, 0, 0, 0, 0, 44, 100, 3449, 0, 0, 1184, 796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 1068, 7, 7, 5877, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1227, 0, 25, 9, 0, 14, 4098, 895, 499, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6211, 7074, 5290, 5017, 6238, 5573, 6982, 6767] avg=6144.0000 max=7074 max_rank=1 min=5017 min_rank=3 imbalance_ratio=1.151367 max_load_reduction=42078 max_load_reduction_pct=0.856079 moved_to_echo=25560 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=23 rerouted_tokens_per_expert_current_ep_rank=[66, 101, 50, 2283, 6, 26, 4452, 9, 0, 0, 0, 0, 0, 0, 0, 0, 640, 463, 25, 0, 1192, 1, 910, 173, 845, 392, 0, 0, 0, 0, 0, 0, 0, 4762, 1009, 6, 0, 9, 38, 269, 0, 0, 0, 0, 0, 0, 0, 0, 98, 61, 0, 28, 0, 16, 91, 35, 6093, 0, 0, 0, 0, 0, 0, 0, 1637, 841, 2, 4, 43, 30, 1851, 0, 1142, 996, 0, 0, 0, 0, 0, 0, 74, 69, 5320, 0, 3, 756, 746, 9, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 1346, 286, 14, 2602, 164, 0, 0, 0, 0, 0, 0, 0, 0, 5, 6, 1568, 1, 4, 73, 2, 24, 3809, 862, 632, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6993, 4641, 6093, 6422, 6546, 6977, 4494, 6986] avg=6144.0000 max=6993 max_rank=0 min=4494 min_rank=6 imbalance_ratio=1.138184 max_load_reduction=42159 max_load_reduction_pct=0.857727 moved_to_echo=25003 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 31, 18, 1883, 1, 15, 3216, 5, 0, 0, 0, 0, 0, 0, 0, 0, 40, 101, 3, 0, 2053, 0, 1061, 3, 884, 798, 0, 0, 0, 0, 0, 0, 13, 4979, 503, 0, 0, 0, 15, 55, 0, 0, 0, 0, 0, 0, 0, 0, 33, 7, 0, 0, 0, 24, 77, 13, 5710, 0, 0, 0, 0, 0, 0, 0, 2137, 735, 14, 13, 2, 9, 1561, 5, 825, 1041, 0, 0, 0, 0, 0, 0, 23, 53, 5385, 0, 0, 2255, 805, 3, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 1042, 11, 0, 5288, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1008, 0, 4, 1, 0, 0, 4115, 623, 594, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5169, 4943, 5565, 5864, 6342, 8524, 6399, 6346] avg=6144.0000 max=8524 max_rank=5 min=4943 min_rank=1 imbalance_ratio=1.387370 max_load_reduction=40628 max_load_reduction_pct=0.826579 moved_to_echo=27611 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1664, 2176, 128, 128, 23168, 128, 9344, 256, 7296, 5504, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[11520, 7808, 128, 128, 128, 256, 11648, 128, 9344, 8704, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=50176 observed_loss=-1024 tokens_per_expert=[128, 128, 9728, 128, 128, 128, 128, 128, 27392, 7168, 4992, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 384, 768, 11392, 128, 384, 36480, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[384, 384, 30976, 128, 128, 12288, 5376, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[384, 0, 128, 11392, 1024, 128, 36352, 384, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 41088, 6784, 128, 128, 128, 128, 1280, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1024, 128, 0, 128, 128, 256, 512, 256, 47232, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=50176 slack_slots=48128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=24 tokens_per_expert_current_ep_rank=[35, 0, 7307, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 1958, 1240, 54, 2750, 1158, 171, 3, 287, 2358, 0, 84, 19, 0, 0, 0, 561, 1, 2521, 40, 0, 0, 0, 0, 886, 9, 2, 2645, 7, 0, 0, 2229, 0, 0, 6821, 0, 2107, 0, 589, 4017, 194, 1364, 280, 0, 0, 428, 40, 6974, 4, 3, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=24 tokens_per_expert_current_ep_rank=[45, 0, 6418, 0, 2, 0, 7, 0, 1, 5, 0, 0, 0, 2530, 874, 24, 3112, 723, 76, 2, 216, 2760, 0, 949, 76, 0, 0, 0, 125, 3, 1893, 65, 0, 0, 4, 10, 758, 22, 6, 1836, 404, 0, 0, 1630, 3, 0, 6517, 0, 1845, 0, 1851, 4720, 112, 1649, 843, 0, 0, 309, 61, 6631, 1, 33, 1, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=24 tokens_per_expert_current_ep_rank=[2, 0, 5963, 20, 21, 0, 3, 0, 0, 0, 13, 0, 0, 2571, 2459, 157, 4868, 1034, 370, 8, 729, 2383, 1, 435, 0, 0, 0, 53, 986, 85, 1533, 67, 11, 1, 4, 5, 1036, 6, 4, 1065, 380, 12, 1, 2797, 4, 1, 5268, 3, 1040, 0, 743, 4499, 69, 909, 362, 7, 2, 329, 53, 6744, 1, 19, 2, 14] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=24 tokens_per_expert_current_ep_rank=[106, 0, 7135, 0, 3, 0, 0, 2, 0, 2, 1, 0, 2, 2714, 1214, 12, 2721, 1326, 88, 8, 232, 2184, 0, 297, 58, 0, 0, 3, 26, 24, 1105, 137, 3, 0, 0, 1, 2045, 10, 4, 933, 8, 25, 0, 683, 0, 1, 7010, 3, 2120, 12, 1937, 4857, 910, 1517, 258, 0, 4, 322, 43, 7031, 7, 4, 0, 4] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=24 tokens_per_expert_current_ep_rank=[6, 1, 6357, 0, 11, 0, 0, 1, 0, 31, 3, 0, 4, 2298, 1011, 140, 4115, 665, 317, 10, 273, 3737, 0, 564, 201, 0, 2, 9, 729, 4, 2441, 34, 0, 1, 25, 1, 770, 15, 5, 2943, 112, 2, 0, 1339, 0, 0, 5964, 1, 1224, 3, 855, 3558, 261, 1022, 1131, 5, 0, 164, 52, 6718, 9, 4, 3, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=24 tokens_per_expert_current_ep_rank=[84, 0, 7024, 0, 2, 0, 0, 12, 0, 3, 65, 0, 5, 2991, 2520, 266, 1678, 773, 621, 65, 285, 3390, 0, 741, 441, 0, 1, 15, 292, 0, 2161, 120, 0, 0, 1, 22, 1105, 26, 0, 581, 146, 9, 2, 728, 79, 0, 7055, 0, 1759, 0, 851, 4352, 311, 999, 743, 4, 2, 39, 106, 6578, 69, 27, 2, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=24 tokens_per_expert_current_ep_rank=[19, 5, 7244, 0, 70, 0, 0, 28, 0, 1, 0, 0, 11, 2357, 1139, 188, 2666, 1479, 149, 5, 323, 2301, 0, 174, 346, 3, 0, 12, 385, 15, 2922, 14, 6, 0, 22, 39, 850, 31, 11, 455, 12, 1, 13, 965, 4, 0, 6864, 0, 1148, 13, 2089, 2090, 2296, 3347, 603, 52, 14, 326, 292, 5732, 2, 3, 3, 13] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=24 tokens_per_expert_current_ep_rank=[160, 0, 5990, 0, 3, 0, 0, 6, 0, 2, 1, 0, 71, 1255, 2606, 20, 3212, 566, 437, 0, 181, 1656, 2, 1030, 7, 0, 0, 0, 930, 4, 2080, 40, 2, 0, 1, 22, 1273, 1, 0, 1002, 49, 0, 0, 708, 12, 0, 6627, 0, 4156, 1, 641, 5451, 154, 835, 772, 5, 3, 62, 159, 6940, 5, 8, 4, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=24 rerouted_tokens_per_expert_current_ep_rank=[35, 0, 6631, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1958, 1240, 54, 948, 676, 452, 0, 0, 0, 0, 0, 1263, 1158, 171, 3, 287, 2358, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 561, 1, 2521, 40, 2008, 1035, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 886, 9, 2, 2645, 3429, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 2229, 0, 0, 4813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2107, 0, 589, 378, 194, 1364, 280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 428, 40, 6026, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6672, 5328, 5324, 6395, 6971, 7049, 4912, 6501] avg=6144.0000 max=7049 max_rank=5 min=4912 min_rank=6 imbalance_ratio=1.147298 max_load_reduction=42103 max_load_reduction_pct=0.856588 moved_to_echo=25433 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=24 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 5412, 20, 21, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 2571, 2459, 157, 916, 551, 801, 0, 0, 0, 0, 0, 2234, 1034, 370, 8, 729, 2383, 1, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 986, 85, 1533, 67, 1551, 1833, 235, 0, 0, 0, 0, 0, 11, 1, 4, 5, 1036, 6, 4, 1065, 3840, 0, 0, 0, 0, 0, 0, 0, 380, 12, 1, 2797, 4, 1, 3717, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 0, 743, 424, 69, 909, 362, 7, 0, 0, 0, 0, 0, 0, 0, 0, 2, 329, 53, 5828, 1, 19, 2, 14, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5458, 7468, 7194, 6343, 5972, 6915, 3554, 6248] avg=6144.0000 max=7468 max_rank=1 min=3554 min_rank=6 imbalance_ratio=1.215495 max_load_reduction=41684 max_load_reduction_pct=0.848063 moved_to_echo=22689 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=24 rerouted_tokens_per_expert_current_ep_rank=[45, 0, 5824, 0, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 0, 0, 0, 2530, 874, 24, 901, 594, 512, 0, 0, 0, 0, 0, 1428, 723, 76, 2, 216, 2760, 0, 949, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 125, 3, 1893, 65, 1918, 1172, 246, 0, 0, 0, 0, 0, 0, 0, 4, 10, 758, 22, 6, 1836, 4029, 0, 0, 0, 0, 0, 0, 0, 404, 0, 0, 1630, 3, 0, 4599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1845, 0, 1851, 445, 112, 1649, 843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 309, 61, 5730, 1, 33, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5878, 5441, 6154, 5498, 6665, 6636, 6745, 6135] avg=6144.0000 max=6745 max_rank=6 min=5441 min_rank=1 imbalance_ratio=1.097819 max_load_reduction=42407 max_load_reduction_pct=0.862773 moved_to_echo=26181 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=24 rerouted_tokens_per_expert_current_ep_rank=[106, 0, 6475, 0, 3, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 2, 2714, 1214, 12, 955, 660, 447, 0, 0, 0, 0, 0, 1250, 1326, 88, 8, 232, 2184, 0, 297, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 3, 26, 24, 1105, 137, 2063, 1024, 253, 0, 0, 0, 0, 0, 3, 0, 0, 1, 2045, 10, 4, 933, 4146, 0, 0, 0, 0, 0, 0, 0, 8, 25, 0, 683, 0, 1, 4947, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2120, 12, 1937, 458, 910, 1517, 258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 322, 43, 6076, 7, 4, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6586, 6007, 5385, 4693, 7142, 5667, 7212, 6460] avg=6144.0000 max=7212 max_rank=6 min=4693 min_rank=3 imbalance_ratio=1.173828 max_load_reduction=41940 max_load_reduction_pct=0.853271 moved_to_echo=26481 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=24 rerouted_tokens_per_expert_current_ep_rank=[6, 1, 5769, 0, 11, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 3, 0, 4, 2298, 1011, 140, 913, 588, 677, 0, 0, 0, 0, 0, 1889, 665, 317, 10, 273, 3737, 0, 564, 0, 0, 0, 0, 0, 0, 0, 0, 201, 0, 2, 9, 729, 4, 2441, 34, 1755, 1549, 186, 0, 0, 0, 0, 0, 0, 1, 25, 1, 770, 15, 5, 2943, 3037, 0, 0, 0, 0, 0, 0, 0, 112, 2, 0, 1339, 0, 0, 4209, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1224, 3, 855, 335, 261, 1022, 1131, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 52, 5805, 9, 4, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5788, 5665, 7455, 6910, 6797, 5663, 4836, 6038] avg=6144.0000 max=7455 max_rank=2 min=4836 min_rank=6 imbalance_ratio=1.213379 max_load_reduction=41697 max_load_reduction_pct=0.848328 moved_to_echo=23334 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=24 rerouted_tokens_per_expert_current_ep_rank=[19, 5, 6574, 0, 70, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 11, 2357, 1139, 188, 779, 670, 438, 0, 0, 0, 0, 0, 1224, 1479, 149, 5, 323, 2301, 0, 174, 0, 0, 0, 0, 0, 0, 0, 0, 346, 3, 0, 12, 385, 15, 2922, 14, 2020, 1004, 109, 0, 0, 0, 0, 0, 6, 0, 22, 39, 850, 31, 11, 455, 1784, 0, 0, 0, 0, 0, 0, 0, 12, 1, 13, 965, 4, 0, 4844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1148, 13, 2089, 197, 2296, 3347, 603, 52, 0, 0, 0, 0, 0, 0, 0, 0, 14, 326, 292, 4953, 2, 3, 3, 13, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6696, 5583, 5655, 6830, 3198, 5839, 9745, 5606] avg=6144.0000 max=9745 max_rank=6 min=3198 min_rank=4 imbalance_ratio=1.586100 max_load_reduction=39407 max_load_reduction_pct=0.801737 moved_to_echo=24388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=24 rerouted_tokens_per_expert_current_ep_rank=[160, 0, 5433, 0, 3, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 71, 1255, 2606, 20, 947, 557, 531, 0, 0, 0, 0, 0, 1468, 566, 437, 0, 181, 1656, 2, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 930, 4, 2080, 40, 1955, 1213, 288, 0, 0, 0, 0, 0, 2, 0, 1, 22, 1273, 1, 0, 1002, 4656, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 708, 12, 0, 4672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4156, 1, 641, 507, 154, 835, 772, 5, 0, 0, 0, 0, 0, 0, 0, 0, 3, 62, 159, 5993, 5, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5602, 5990, 5340, 6517, 6957, 5441, 7071, 6234] avg=6144.0000 max=7071 max_rank=6 min=5340 min_rank=2 imbalance_ratio=1.150879 max_load_reduction=42081 max_load_reduction_pct=0.856140 moved_to_echo=25703 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=24 rerouted_tokens_per_expert_current_ep_rank=[84, 0, 6374, 0, 2, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 65, 0, 5, 2991, 2520, 266, 894, 650, 276, 0, 0, 0, 0, 0, 770, 773, 621, 65, 285, 3390, 0, 741, 0, 0, 0, 0, 0, 0, 0, 0, 441, 0, 1, 15, 292, 0, 2161, 120, 2077, 632, 227, 0, 0, 0, 0, 0, 0, 0, 1, 22, 1105, 26, 0, 581, 3715, 0, 0, 0, 0, 0, 0, 0, 146, 9, 2, 728, 79, 0, 4978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1759, 0, 851, 410, 311, 999, 743, 4, 0, 0, 0, 0, 0, 0, 0, 0, 2, 39, 106, 5684, 69, 27, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6472, 7670, 6645, 5966, 5450, 5942, 5077, 5930] avg=6144.0000 max=7670 max_rank=1 min=5077 min_rank=6 imbalance_ratio=1.248372 max_load_reduction=41482 max_load_reduction_pct=0.843953 moved_to_echo=22399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1152, 128, 128, 128, 4096, 256, 16768, 640, 15360, 9472, 1792, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 128, 18688, 13184, 896, 7296, 4992, 4224, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[15488, 128, 9600, 3200, 4352, 11648, 4992, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[11648, 7808, 2304, 128, 2560, 20864, 128, 4352, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 8832, 128, 128, 11520, 28672, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1152, 128, 128, 11136, 128, 128, 36864, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 2048, 896, 46208, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[512, 128, 48512, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True - [2026-08-26 14:53:44] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 821.3 | throughput per GPU (TFLOP/s/GPU): 155.3 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.952471E+00 | loss scale: 1.0 | grad norm: 42.237 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[2641, 499, 368, 727, 361, 1383, 1012, 95, 2763, 593, 1185, 567, 22, 1204, 260, 582, 1250, 1154, 521, 768, 296, 308, 439, 578, 556, 901, 100, 377, 939, 793, 402, 1427, 1232, 314, 920, 1830, 650, 408, 994, 525, 215, 421, 399, 171, 1369, 1877, 722, 132, 319, 583, 550, 834, 1020, 1593, 729, 338, 388, 830, 1461, 604, 338, 468, 1554, 293] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[2544, 459, 385, 724, 308, 1396, 1035, 113, 2578, 673, 1201, 548, 18, 1130, 279, 447, 1164, 1215, 487, 913, 366, 411, 391, 605, 551, 788, 136, 388, 960, 791, 444, 1248, 1315, 369, 934, 1801, 725, 395, 1069, 535, 187, 423, 421, 237, 1298, 1957, 675, 162, 276, 609, 615, 764, 1021, 1931, 656, 424, 409, 832, 1405, 504, 428, 425, 1415, 239] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[2791, 551, 331, 678, 352, 1258, 1149, 105, 2462, 616, 1134, 553, 21, 1206, 279, 647, 1308, 1076, 552, 898, 306, 350, 398, 571, 636, 787, 109, 424, 859, 896, 384, 1092, 1200, 362, 981, 1701, 665, 390, 1109, 550, 202, 429, 404, 226, 1321, 1868, 709, 165, 345, 531, 657, 718, 971, 1651, 619, 276, 374, 994, 1461, 502, 491, 514, 1509, 478] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[2845, 514, 424, 592, 306, 1569, 1020, 88, 2767, 624, 1197, 489, 34, 1185, 239, 507, 1191, 1149, 463, 893, 316, 339, 427, 628, 545, 794, 136, 385, 918, 843, 372, 1138, 1196, 353, 915, 1909, 587, 406, 1090, 492, 151, 422, 449, 195, 1383, 1976, 750, 132, 292, 620, 590, 775, 823, 1864, 724, 279, 444, 906, 1403, 536, 407, 441, 1467, 238] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[2829, 567, 401, 699, 289, 1412, 998, 111, 2585, 638, 1217, 535, 17, 1195, 284, 440, 1283, 1195, 485, 768, 305, 363, 460, 609, 527, 764, 135, 343, 946, 821, 440, 1421, 1332, 369, 949, 1934, 623, 352, 998, 525, 176, 444, 442, 247, 1300, 1913, 760, 144, 313, 575, 615, 780, 981, 1556, 602, 366, 414, 922, 1368, 527, 392, 370, 1464, 317] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[2718, 539, 432, 734, 328, 1375, 976, 112, 2729, 592, 1263, 525, 27, 1201, 269, 510, 1253, 1116, 407, 835, 283, 337, 418, 579, 503, 762, 131, 371, 959, 861, 366, 1014, 1276, 362, 941, 1823, 652, 373, 1058, 530, 185, 448, 412, 235, 1295, 1970, 689, 175, 319, 642, 593, 786, 940, 2016, 780, 392, 398, 755, 1517, 499, 396, 394, 1463, 313] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[2737, 628, 438, 676, 340, 1198, 980, 131, 2529, 603, 1148, 518, 24, 1149, 285, 574, 1235, 1145, 478, 790, 340, 331, 508, 580, 573, 874, 127, 378, 1009, 769, 383, 1196, 1330, 428, 893, 1780, 576, 366, 1152, 544, 178, 499, 443, 245, 1376, 1849, 728, 183, 292, 515, 526, 861, 1000, 1905, 695, 363, 347, 760, 1396, 583, 395, 390, 1545, 335] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[2626, 522, 371, 762, 326, 1431, 1045, 88, 2496, 616, 1233, 496, 16, 1212, 310, 537, 1360, 1183, 496, 831, 263, 340, 452, 618, 508, 822, 119, 392, 940, 765, 414, 1326, 1276, 331, 1011, 1595, 583, 375, 996, 587, 182, 463, 458, 227, 1374, 1805, 726, 207, 313, 596, 637, 914, 999, 1651, 643, 399, 426, 761, 1393, 559, 413, 471, 1503, 362] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1635, 499, 368, 727, 361, 1383, 1012, 95, 0, 0, 0, 0, 0, 0, 0, 0, 1889, 593, 1185, 567, 22, 1204, 260, 582, 0, 0, 0, 0, 0, 0, 0, 0, 1250, 1154, 521, 768, 296, 308, 439, 578, 501, 252, 0, 0, 0, 0, 0, 0, 556, 901, 100, 377, 939, 793, 402, 1427, 859, 0, 0, 0, 0, 0, 0, 0, 1232, 314, 920, 979, 650, 408, 994, 525, 0, 0, 0, 0, 0, 0, 0, 0, 215, 421, 399, 171, 1369, 1877, 722, 132, 599, 147, 0, 0, 0, 0, 0, 0, 319, 583, 550, 834, 1020, 1593, 729, 338, 77, 0, 0, 0, 0, 0, 0, 0, 388, 830, 1461, 604, 338, 468, 1554, 293, 296, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6080, 6302, 6067, 6354, 6022, 6052, 6043, 6232] avg=6144.0000 max=6354 max_rank=3 min=6022 min_rank=4 imbalance_ratio=1.034180 max_load_reduction=42798 max_load_reduction_pct=0.870728 moved_to_echo=24349 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1574, 459, 385, 724, 308, 1396, 1035, 113, 0, 0, 0, 0, 0, 0, 0, 0, 1763, 673, 1201, 548, 18, 1130, 279, 447, 0, 0, 0, 0, 0, 0, 0, 0, 1164, 1215, 487, 913, 366, 411, 391, 605, 467, 248, 0, 0, 0, 0, 0, 0, 551, 788, 136, 388, 960, 791, 444, 1248, 828, 0, 0, 0, 0, 0, 0, 0, 1315, 369, 934, 963, 725, 395, 1069, 535, 0, 0, 0, 0, 0, 0, 0, 0, 187, 423, 421, 237, 1298, 1957, 675, 162, 590, 142, 0, 0, 0, 0, 0, 0, 276, 609, 615, 764, 1021, 1931, 656, 424, 72, 0, 0, 0, 0, 0, 0, 0, 409, 832, 1405, 504, 428, 425, 1415, 239, 276, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5994, 6059, 6267, 6134, 6305, 6092, 6368, 5933] avg=6144.0000 max=6368 max_rank=6 min=5933 min_rank=7 imbalance_ratio=1.036458 max_load_reduction=42784 max_load_reduction_pct=0.870443 moved_to_echo=24698 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1751, 567, 401, 699, 289, 1412, 998, 111, 0, 0, 0, 0, 0, 0, 0, 0, 1767, 638, 1217, 535, 17, 1195, 284, 440, 0, 0, 0, 0, 0, 0, 0, 0, 1283, 1195, 485, 768, 305, 363, 460, 609, 469, 266, 0, 0, 0, 0, 0, 0, 527, 764, 135, 343, 946, 821, 440, 1421, 920, 0, 0, 0, 0, 0, 0, 0, 1332, 369, 949, 1035, 623, 352, 998, 525, 0, 0, 0, 0, 0, 0, 0, 0, 176, 444, 442, 247, 1300, 1913, 760, 144, 633, 158, 0, 0, 0, 0, 0, 0, 313, 575, 615, 780, 981, 1556, 602, 366, 72, 0, 0, 0, 0, 0, 0, 0, 414, 922, 1368, 527, 392, 370, 1464, 317, 277, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6228, 6093, 6203, 6317, 6183, 6217, 5860, 6051] avg=6144.0000 max=6317 max_rank=3 min=5860 min_rank=6 imbalance_ratio=1.028158 max_load_reduction=42835 max_load_reduction_pct=0.871480 moved_to_echo=24311 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1694, 628, 438, 676, 340, 1198, 980, 131, 0, 0, 0, 0, 0, 0, 0, 0, 1728, 603, 1148, 518, 24, 1149, 285, 574, 0, 0, 0, 0, 0, 0, 0, 0, 1235, 1145, 478, 790, 340, 331, 508, 580, 459, 245, 0, 0, 0, 0, 0, 0, 573, 874, 127, 378, 1009, 769, 383, 1196, 890, 0, 0, 0, 0, 0, 0, 0, 1330, 428, 893, 952, 576, 366, 1152, 544, 0, 0, 0, 0, 0, 0, 0, 0, 178, 499, 443, 245, 1376, 1849, 728, 183, 583, 153, 0, 0, 0, 0, 0, 0, 292, 515, 526, 861, 1000, 1905, 695, 363, 71, 0, 0, 0, 0, 0, 0, 0, 347, 760, 1396, 583, 395, 390, 1545, 335, 271, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6085, 6029, 6111, 6199, 6241, 6237, 6228, 6022] avg=6144.0000 max=6241 max_rank=4 min=6022 min_rank=7 imbalance_ratio=1.015788 max_load_reduction=42911 max_load_reduction_pct=0.873027 moved_to_echo=24728 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1727, 551, 331, 678, 352, 1258, 1149, 105, 0, 0, 0, 0, 0, 0, 0, 0, 1683, 616, 1134, 553, 21, 1206, 279, 647, 0, 0, 0, 0, 0, 0, 0, 0, 1308, 1076, 552, 898, 306, 350, 398, 571, 446, 234, 0, 0, 0, 0, 0, 0, 636, 787, 109, 424, 859, 896, 384, 1092, 908, 0, 0, 0, 0, 0, 0, 0, 1200, 362, 981, 910, 665, 390, 1109, 550, 0, 0, 0, 0, 0, 0, 0, 0, 202, 429, 404, 226, 1321, 1868, 709, 165, 557, 156, 0, 0, 0, 0, 0, 0, 345, 531, 657, 718, 971, 1651, 619, 276, 69, 0, 0, 0, 0, 0, 0, 0, 374, 994, 1461, 502, 491, 514, 1509, 478, 264, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6151, 6139, 6139, 6095, 6167, 6037, 5837, 6587] avg=6144.0000 max=6587 max_rank=7 min=5837 min_rank=6 imbalance_ratio=1.072103 max_load_reduction=42565 max_load_reduction_pct=0.865987 moved_to_echo=24628 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1760, 514, 424, 592, 306, 1569, 1020, 88, 0, 0, 0, 0, 0, 0, 0, 0, 1890, 624, 1197, 489, 34, 1185, 239, 507, 0, 0, 0, 0, 0, 0, 0, 0, 1191, 1149, 463, 893, 316, 339, 427, 628, 502, 262, 0, 0, 0, 0, 0, 0, 545, 794, 136, 385, 918, 843, 372, 1138, 926, 0, 0, 0, 0, 0, 0, 0, 1196, 353, 915, 1022, 587, 406, 1090, 492, 0, 0, 0, 0, 0, 0, 0, 0, 151, 422, 449, 195, 1383, 1976, 750, 132, 625, 159, 0, 0, 0, 0, 0, 0, 292, 620, 590, 775, 823, 1864, 724, 279, 78, 0, 0, 0, 0, 0, 0, 0, 444, 906, 1403, 536, 407, 441, 1467, 238, 297, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6273, 6165, 6170, 6057, 6061, 6242, 6045, 6139] avg=6144.0000 max=6273 max_rank=0 min=6045 min_rank=6 imbalance_ratio=1.020996 max_load_reduction=42879 max_load_reduction_pct=0.872375 moved_to_echo=24487 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1618, 522, 371, 762, 326, 1431, 1045, 88, 0, 0, 0, 0, 0, 0, 0, 0, 1694, 616, 1233, 496, 16, 1212, 310, 537, 0, 0, 0, 0, 0, 0, 0, 0, 1360, 1183, 496, 831, 263, 340, 452, 618, 456, 222, 0, 0, 0, 0, 0, 0, 508, 822, 119, 392, 940, 765, 414, 1326, 859, 0, 0, 0, 0, 0, 0, 0, 1276, 331, 1011, 847, 583, 375, 996, 587, 0, 0, 0, 0, 0, 0, 0, 0, 182, 463, 458, 227, 1374, 1805, 726, 207, 526, 149, 0, 0, 0, 0, 0, 0, 313, 596, 637, 914, 999, 1651, 643, 399, 74, 0, 0, 0, 0, 0, 0, 0, 426, 761, 1393, 559, 413, 471, 1503, 362, 272, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6163, 6114, 6221, 6145, 6006, 6117, 6226, 6160] avg=6144.0000 max=6226 max_rank=6 min=6006 min_rank=4 imbalance_ratio=1.013346 max_load_reduction=42926 max_load_reduction_pct=0.873332 moved_to_echo=24509 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1682, 539, 432, 734, 328, 1375, 976, 112, 0, 0, 0, 0, 0, 0, 0, 0, 1864, 592, 1263, 525, 27, 1201, 269, 510, 0, 0, 0, 0, 0, 0, 0, 0, 1253, 1116, 407, 835, 283, 337, 418, 579, 495, 251, 0, 0, 0, 0, 0, 0, 503, 762, 131, 371, 959, 861, 366, 1014, 884, 0, 0, 0, 0, 0, 0, 0, 1276, 362, 941, 975, 652, 373, 1058, 530, 0, 0, 0, 0, 0, 0, 0, 0, 185, 448, 412, 235, 1295, 1970, 689, 175, 597, 152, 0, 0, 0, 0, 0, 0, 319, 642, 593, 786, 940, 2016, 780, 392, 77, 0, 0, 0, 0, 0, 0, 0, 398, 755, 1517, 499, 396, 394, 1463, 313, 293, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6178, 6251, 5974, 5851, 6167, 6158, 6545, 6028] avg=6144.0000 max=6545 max_rank=6 min=5851 min_rank=3 imbalance_ratio=1.065267 max_load_reduction=42607 max_load_reduction_pct=0.866842 moved_to_echo=24898 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2560, 4736, 4864, 6528, 7808, 14208, 5504, 2944, 640, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[14336, 4992, 9600, 4352, 256, 9600, 2304, 4352, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[10240, 2944, 7552, 7808, 5120, 3072, 8576, 4352, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[10112, 9344, 3968, 6784, 2560, 2816, 3584, 4864, 3840, 2048, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[1536, 3584, 3456, 1792, 10752, 15232, 5760, 1408, 4736, 1280, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[13568, 4352, 3200, 5632, 2688, 11136, 8320, 896, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4480, 6528, 1024, 3072, 7552, 6656, 3328, 9984, 7168, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3200, 6784, 11520, 4352, 3328, 3584, 12032, 2688, 2304, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[156, 386, 226, 184, 1820, 259, 2778, 283, 62, 375, 649, 425, 879, 709, 141, 2826, 585, 1100, 1384, 1349, 503, 1606, 828, 387, 217, 322, 1035, 4464, 47, 873, 88, 721, 183, 17, 21, 237, 554, 21, 289, 192, 1646, 647, 0, 490, 158, 126, 9, 892, 5962, 280, 1988, 17, 387, 331, 912, 787, 971, 1084, 690, 487, 1115, 226, 389, 377] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[178, 368, 180, 143, 1862, 265, 3092, 276, 39, 387, 748, 468, 854, 531, 136, 2727, 599, 997, 1470, 1208, 435, 1536, 866, 412, 215, 311, 1018, 4555, 40, 925, 69, 722, 167, 9, 10, 250, 538, 25, 301, 190, 1733, 654, 0, 432, 145, 118, 3, 861, 5949, 279, 2090, 19, 292, 349, 1065, 757, 936, 950, 813, 471, 1073, 214, 442, 385] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[143, 389, 235, 156, 1950, 283, 2662, 291, 47, 366, 691, 394, 1015, 641, 157, 2865, 533, 1061, 1424, 1446, 573, 1471, 817, 423, 243, 366, 845, 4445, 35, 1064, 85, 632, 189, 9, 21, 238, 547, 18, 227, 201, 1620, 643, 0, 488, 119, 113, 9, 1027, 5856, 248, 2012, 22, 402, 275, 1042, 675, 1013, 1024, 625, 497, 1111, 225, 455, 453] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[201, 414, 190, 146, 1662, 263, 2766, 337, 64, 426, 698, 436, 969, 659, 171, 3024, 589, 1198, 1379, 1347, 509, 1647, 850, 362, 223, 333, 1174, 4429, 42, 908, 79, 652, 170, 19, 45, 210, 577, 21, 287, 171, 1476, 629, 0, 466, 164, 139, 10, 858, 5777, 259, 1920, 11, 305, 298, 1113, 802, 1052, 1031, 675, 507, 1002, 193, 411, 407] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[190, 417, 168, 147, 1836, 344, 2698, 308, 45, 356, 632, 428, 917, 656, 176, 2835, 508, 1047, 1445, 1311, 504, 1608, 810, 367, 194, 302, 1038, 4456, 32, 929, 87, 628, 136, 16, 18, 259, 569, 20, 209, 187, 1542, 678, 0, 499, 134, 146, 6, 1350, 5910, 242, 2086, 11, 299, 320, 1016, 803, 970, 1029, 715, 532, 976, 205, 451, 399] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[130, 330, 140, 142, 1788, 265, 2875, 314, 30, 351, 643, 353, 891, 689, 105, 2954, 548, 1064, 1447, 1220, 436, 1636, 757, 415, 186, 334, 1178, 4756, 32, 1033, 86, 644, 124, 8, 10, 217, 529, 8, 267, 182, 1604, 644, 0, 470, 141, 101, 9, 1157, 6166, 221, 2150, 8, 352, 263, 931, 671, 1025, 1056, 664, 510, 1030, 160, 359, 343] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[241, 343, 235, 149, 1827, 244, 2785, 320, 65, 414, 681, 448, 868, 712, 170, 2703, 677, 960, 1439, 1342, 538, 1442, 705, 401, 266, 431, 1080, 4182, 39, 950, 103, 591, 181, 16, 13, 230, 623, 12, 285, 216, 1622, 727, 0, 486, 142, 191, 11, 868, 5889, 287, 1969, 11, 292, 357, 1018, 740, 1055, 1118, 848, 562, 958, 271, 414, 389] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[137, 340, 153, 160, 1882, 308, 2892, 248, 38, 306, 597, 399, 921, 644, 147, 2945, 630, 1052, 1643, 1230, 429, 1591, 788, 304, 207, 382, 962, 4619, 18, 893, 50, 698, 121, 19, 16, 245, 448, 17, 269, 154, 1528, 730, 0, 502, 171, 94, 10, 1335, 6046, 219, 2029, 6, 341, 274, 979, 773, 967, 1065, 613, 488, 992, 205, 456, 427] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[178, 368, 180, 143, 1862, 265, 3092, 276, 30, 0, 0, 0, 0, 0, 0, 0, 39, 387, 748, 468, 854, 531, 136, 2727, 55, 0, 0, 0, 0, 0, 0, 0, 599, 997, 1470, 1208, 435, 64, 866, 412, 0, 0, 0, 0, 0, 0, 0, 0, 215, 311, 1018, 2855, 40, 925, 69, 722, 0, 0, 0, 0, 0, 0, 0, 0, 167, 9, 10, 250, 538, 25, 301, 190, 4504, 198, 0, 0, 0, 0, 0, 0, 1733, 654, 0, 432, 145, 118, 3, 861, 1502, 579, 0, 0, 0, 0, 0, 0, 1445, 279, 2090, 19, 292, 349, 1065, 757, 0, 0, 0, 0, 0, 0, 0, 0, 936, 950, 813, 471, 1073, 214, 442, 385, 808, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6394, 5945, 6051, 6155, 6192, 6027, 6296, 6092] avg=6144.0000 max=6394 max_rank=0 min=5945 min_rank=1 imbalance_ratio=1.040690 max_load_reduction=42758 max_load_reduction_pct=0.869914 moved_to_echo=24607 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[156, 386, 226, 184, 1820, 259, 2778, 283, 32, 0, 0, 0, 0, 0, 0, 0, 62, 375, 649, 425, 879, 709, 141, 2826, 58, 0, 0, 0, 0, 0, 0, 0, 585, 1100, 1384, 1349, 503, 66, 828, 387, 0, 0, 0, 0, 0, 0, 0, 0, 217, 322, 1035, 2798, 47, 873, 88, 721, 0, 0, 0, 0, 0, 0, 0, 0, 183, 17, 21, 237, 554, 21, 289, 192, 4514, 194, 0, 0, 0, 0, 0, 0, 1646, 647, 0, 490, 158, 126, 9, 892, 1472, 606, 0, 0, 0, 0, 0, 0, 1448, 280, 1988, 17, 387, 331, 912, 787, 0, 0, 0, 0, 0, 0, 0, 0, 971, 1084, 690, 487, 1115, 226, 389, 377, 844, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6124, 6124, 6202, 6101, 6222, 6046, 6150, 6183] avg=6144.0000 max=6222 max_rank=4 min=6046 min_rank=5 imbalance_ratio=1.012695 max_load_reduction=42930 max_load_reduction_pct=0.873413 moved_to_echo=24601 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[201, 414, 190, 146, 1662, 263, 2766, 337, 33, 0, 0, 0, 0, 0, 0, 0, 64, 426, 698, 436, 969, 659, 171, 3024, 59, 0, 0, 0, 0, 0, 0, 0, 589, 1198, 1379, 1347, 509, 68, 850, 362, 0, 0, 0, 0, 0, 0, 0, 0, 223, 333, 1174, 2775, 42, 908, 79, 652, 0, 0, 0, 0, 0, 0, 0, 0, 170, 19, 45, 210, 577, 21, 287, 171, 4374, 193, 0, 0, 0, 0, 0, 0, 1476, 629, 0, 466, 164, 139, 10, 858, 1461, 621, 0, 0, 0, 0, 0, 0, 1403, 259, 1920, 11, 305, 298, 1113, 802, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1031, 675, 507, 1002, 193, 411, 407, 866, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6012, 6506, 6302, 6186, 6067, 5824, 6111, 6144] avg=6144.0000 max=6506 max_rank=1 min=5824 min_rank=5 imbalance_ratio=1.058919 max_load_reduction=42646 max_load_reduction_pct=0.867635 moved_to_echo=24146 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[190, 417, 168, 147, 1836, 344, 2698, 308, 32, 0, 0, 0, 0, 0, 0, 0, 45, 356, 632, 428, 917, 656, 176, 2835, 58, 0, 0, 0, 0, 0, 0, 0, 508, 1047, 1445, 1311, 504, 65, 810, 367, 0, 0, 0, 0, 0, 0, 0, 0, 194, 302, 1038, 2793, 32, 929, 87, 628, 0, 0, 0, 0, 0, 0, 0, 0, 136, 16, 18, 259, 569, 20, 209, 187, 4474, 194, 0, 0, 0, 0, 0, 0, 1542, 678, 0, 499, 134, 146, 6, 1350, 1469, 607, 0, 0, 0, 0, 0, 0, 1436, 242, 2086, 11, 299, 320, 1016, 803, 0, 0, 0, 0, 0, 0, 0, 0, 970, 1029, 715, 532, 976, 205, 451, 399, 846, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6140, 6103, 6057, 6003, 6082, 6431, 6213, 6123] avg=6144.0000 max=6431 max_rank=5 min=6003 min_rank=3 imbalance_ratio=1.046712 max_load_reduction=42721 max_load_reduction_pct=0.869161 moved_to_echo=24849 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[143, 389, 235, 156, 1950, 283, 2662, 291, 34, 0, 0, 0, 0, 0, 0, 0, 47, 366, 691, 394, 1015, 641, 157, 2865, 56, 0, 0, 0, 0, 0, 0, 0, 533, 1061, 1424, 1446, 573, 44, 817, 423, 0, 0, 0, 0, 0, 0, 0, 0, 243, 366, 845, 2777, 35, 1064, 85, 632, 0, 0, 0, 0, 0, 0, 0, 0, 189, 9, 21, 238, 547, 18, 227, 201, 4438, 197, 0, 0, 0, 0, 0, 0, 1620, 643, 0, 488, 119, 113, 9, 1027, 1471, 560, 0, 0, 0, 0, 0, 0, 1418, 248, 2012, 22, 402, 275, 1042, 675, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1024, 625, 497, 1111, 225, 455, 453, 777, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6232, 6321, 6047, 6085, 6050, 6094, 6180] avg=6144.0000 max=6321 max_rank=2 min=6047 min_rank=3 imbalance_ratio=1.028809 max_load_reduction=42831 max_load_reduction_pct=0.871399 moved_to_echo=24409 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[130, 330, 140, 142, 1788, 265, 2875, 314, 32, 0, 0, 0, 0, 0, 0, 0, 30, 351, 643, 353, 891, 689, 105, 2954, 59, 0, 0, 0, 0, 0, 0, 0, 548, 1064, 1447, 1220, 436, 68, 757, 415, 0, 0, 0, 0, 0, 0, 0, 0, 186, 334, 1178, 2981, 32, 1033, 86, 644, 0, 0, 0, 0, 0, 0, 0, 0, 124, 8, 10, 217, 529, 8, 267, 182, 4668, 207, 0, 0, 0, 0, 0, 0, 1604, 644, 0, 470, 141, 101, 9, 1157, 1568, 617, 0, 0, 0, 0, 0, 0, 1498, 221, 2150, 8, 352, 263, 931, 671, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1056, 664, 510, 1030, 160, 359, 343, 860, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6016, 6075, 5955, 6474, 6220, 6311, 6094, 6007] avg=6144.0000 max=6474 max_rank=3 min=5955 min_rank=2 imbalance_ratio=1.053711 max_load_reduction=42678 max_load_reduction_pct=0.868286 moved_to_echo=24632 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[241, 343, 235, 149, 1827, 244, 2785, 320, 28, 0, 0, 0, 0, 0, 0, 0, 65, 414, 681, 448, 868, 712, 170, 2703, 52, 0, 0, 0, 0, 0, 0, 0, 677, 960, 1439, 1342, 538, 60, 705, 401, 0, 0, 0, 0, 0, 0, 0, 0, 266, 431, 1080, 2621, 39, 950, 103, 591, 0, 0, 0, 0, 0, 0, 0, 0, 181, 16, 13, 230, 623, 12, 285, 216, 4459, 182, 0, 0, 0, 0, 0, 0, 1622, 727, 0, 486, 142, 191, 11, 868, 1379, 544, 0, 0, 0, 0, 0, 0, 1430, 287, 1969, 11, 292, 357, 1018, 740, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1118, 848, 562, 958, 271, 414, 389, 758, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6172, 6113, 6122, 6081, 6217, 5970, 6104, 6373] avg=6144.0000 max=6373 max_rank=7 min=5970 min_rank=5 imbalance_ratio=1.037272 max_load_reduction=42779 max_load_reduction_pct=0.870341 moved_to_echo=24664 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[137, 340, 153, 160, 1882, 308, 2892, 248, 31, 0, 0, 0, 0, 0, 0, 0, 38, 306, 597, 399, 921, 644, 147, 2945, 57, 0, 0, 0, 0, 0, 0, 0, 630, 1052, 1643, 1230, 429, 66, 788, 304, 0, 0, 0, 0, 0, 0, 0, 0, 207, 382, 962, 2895, 18, 893, 50, 698, 0, 0, 0, 0, 0, 0, 0, 0, 121, 19, 16, 245, 448, 17, 269, 154, 4577, 201, 0, 0, 0, 0, 0, 0, 1528, 730, 0, 502, 171, 94, 10, 1335, 1523, 600, 0, 0, 0, 0, 0, 0, 1469, 219, 2029, 6, 341, 274, 979, 773, 0, 0, 0, 0, 0, 0, 0, 0, 967, 1065, 613, 488, 992, 205, 456, 427, 837, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6151, 6054, 6142, 6105, 6067, 6493, 6090, 6050] avg=6144.0000 max=6493 max_rank=5 min=6050 min_rank=7 imbalance_ratio=1.056803 max_load_reduction=42659 max_load_reduction_pct=0.867900 moved_to_echo=24700 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[12800, 5376, 0, 3840, 1280, 1152, 128, 8448, 11904, 4736, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[11648, 2048, 16256, 128, 2688, 2560, 8192, 6016, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4736, 8576, 11648, 10496, 3968, 512, 6528, 3072, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[8064, 8448, 5760, 4096, 8320, 1792, 3456, 3200, 6656, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[512, 3072, 5376, 3456, 7424, 5248, 1280, 22912, 512, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1792, 2816, 8448, 22528, 384, 7680, 768, 5376, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1408, 3072, 1536, 1280, 14720, 2304, 22656, 2432, 256, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1280, 128, 256, 1920, 4480, 256, 2176, 1536, 36096, 1664, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[3, 302, 16, 218, 42, 25, 2091, 4656, 1, 754, 347, 1417, 1686, 196, 182, 120, 2888, 1460, 893, 851, 101, 3, 4567, 314, 386, 1085, 4793, 1132, 652, 245, 168, 35, 213, 479, 1703, 141, 3660, 433, 529, 174, 213, 426, 0, 83, 1292, 2045, 126, 94, 297, 1647, 261, 74, 55, 75, 273, 11, 1262, 80, 57, 64, 4, 687, 30, 1035] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[4, 281, 5, 188, 35, 76, 2030, 4836, 0, 649, 407, 1335, 1936, 131, 169, 117, 3013, 1440, 853, 1092, 57, 4, 4478, 347, 342, 1150, 5177, 995, 634, 185, 173, 27, 158, 389, 1474, 124, 3844, 452, 564, 157, 185, 381, 0, 68, 1184, 2004, 135, 83, 303, 1572, 255, 28, 66, 76, 354, 4, 1286, 60, 41, 66, 9, 605, 30, 1029] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[2, 174, 13, 195, 30, 32, 2124, 5240, 1, 703, 469, 1519, 2085, 134, 133, 74, 2909, 1233, 885, 773, 54, 5, 4679, 338, 250, 1014, 5232, 1055, 702, 219, 131, 20, 110, 432, 1692, 99, 3724, 440, 405, 144, 180, 347, 0, 53, 1312, 1877, 111, 48, 234, 1731, 273, 21, 45, 86, 278, 3, 1276, 54, 22, 39, 2, 738, 12, 937] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[6, 345, 16, 151, 45, 63, 2062, 4748, 2, 749, 420, 1515, 1895, 145, 192, 104, 2604, 1286, 980, 791, 69, 5, 4066, 299, 386, 1136, 5027, 1009, 649, 207, 176, 40, 143, 422, 1623, 103, 4261, 434, 497, 141, 242, 356, 1, 131, 1670, 2042, 131, 104, 298, 1402, 218, 52, 72, 83, 308, 4, 1311, 62, 77, 57, 4, 600, 14, 1101] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[2, 201, 14, 206, 33, 43, 1904, 4724, 2, 671, 385, 1352, 1920, 131, 149, 81, 2703, 1477, 939, 987, 67, 3, 4815, 323, 296, 1109, 5025, 1106, 635, 237, 188, 26, 132, 448, 1598, 105, 3861, 470, 515, 160, 196, 384, 1, 76, 1169, 2017, 141, 77, 294, 1662, 277, 49, 47, 70, 353, 4, 1420, 64, 63, 61, 7, 652, 27, 998] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1, 341, 10, 176, 53, 51, 1852, 4512, 1, 630, 387, 1413, 2058, 198, 239, 144, 2892, 1465, 914, 996, 100, 4, 4343, 298, 303, 1100, 4736, 1095, 678, 232, 194, 44, 237, 414, 1493, 137, 3594, 600, 687, 175, 317, 368, 4, 102, 1239, 2094, 140, 73, 402, 1285, 215, 60, 53, 99, 383, 7, 1481, 112, 35, 67, 7, 662, 46, 1104] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[2, 193, 3, 218, 41, 50, 1923, 4968, 1, 621, 520, 1324, 2081, 125, 156, 99, 2699, 1481, 1033, 702, 66, 3, 4959, 394, 260, 1088, 5354, 1063, 686, 203, 117, 24, 94, 505, 1590, 93, 3667, 487, 605, 141, 184, 302, 0, 75, 1334, 1942, 123, 61, 231, 1556, 234, 29, 44, 57, 247, 4, 1276, 44, 34, 49, 2, 595, 23, 1067] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[4, 230, 13, 223, 27, 43, 1970, 4824, 0, 598, 367, 1265, 1979, 133, 174, 73, 2885, 1557, 924, 1012, 58, 1, 4593, 331, 321, 1183, 5192, 1012, 697, 259, 150, 16, 120, 390, 1468, 88, 3751, 436, 544, 145, 250, 351, 2, 94, 1266, 2114, 146, 49, 335, 1580, 241, 27, 45, 66, 325, 2, 1340, 44, 44, 53, 3, 663, 23, 1033] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[3, 302, 16, 218, 42, 25, 2091, 3477, 0, 0, 0, 0, 0, 0, 0, 0, 1, 754, 347, 1417, 1686, 196, 182, 120, 913, 346, 0, 0, 0, 0, 0, 0, 2542, 1460, 893, 851, 101, 3, 0, 314, 0, 0, 0, 0, 0, 0, 0, 0, 386, 1085, 2440, 1132, 652, 245, 168, 35, 0, 0, 0, 0, 0, 0, 0, 0, 213, 479, 1703, 141, 2605, 433, 529, 174, 0, 0, 0, 0, 0, 0, 0, 0, 213, 426, 0, 83, 1292, 2045, 126, 94, 1179, 543, 142, 0, 0, 0, 0, 0, 297, 1647, 261, 74, 55, 75, 273, 11, 3554, 0, 0, 0, 0, 0, 0, 0, 1262, 80, 57, 64, 4, 687, 30, 1035, 1810, 1013, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6174, 5962, 6164, 6143, 6277, 6143, 6247, 6042] avg=6144.0000 max=6277 max_rank=4 min=5962 min_rank=1 imbalance_ratio=1.021647 max_load_reduction=42875 max_load_reduction_pct=0.872294 moved_to_echo=24709 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4, 281, 5, 188, 35, 76, 2030, 3611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 649, 407, 1335, 1936, 131, 169, 117, 959, 361, 0, 0, 0, 0, 0, 0, 2652, 1440, 853, 1092, 57, 4, 0, 347, 0, 0, 0, 0, 0, 0, 0, 0, 342, 1150, 2635, 995, 634, 185, 173, 27, 0, 0, 0, 0, 0, 0, 0, 0, 158, 389, 1474, 124, 2736, 452, 564, 157, 0, 0, 0, 0, 0, 0, 0, 0, 185, 381, 0, 68, 1184, 2004, 135, 83, 1225, 587, 149, 0, 0, 0, 0, 0, 303, 1572, 255, 28, 66, 76, 354, 4, 3486, 0, 0, 0, 0, 0, 0, 0, 1286, 60, 41, 66, 9, 605, 30, 1029, 1955, 992, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6230, 6064, 6445, 6141, 6054, 6001, 6144, 6073] avg=6144.0000 max=6445 max_rank=2 min=6001 min_rank=5 imbalance_ratio=1.048991 max_load_reduction=42707 max_load_reduction_pct=0.868876 moved_to_echo=24272 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2, 201, 14, 206, 33, 43, 1904, 3528, 0, 0, 0, 0, 0, 0, 0, 0, 2, 671, 385, 1352, 1920, 131, 149, 81, 963, 324, 0, 0, 0, 0, 0, 0, 2379, 1477, 939, 987, 67, 3, 0, 323, 0, 0, 0, 0, 0, 0, 0, 0, 296, 1109, 2558, 1106, 635, 237, 188, 26, 0, 0, 0, 0, 0, 0, 0, 0, 132, 448, 1598, 105, 2748, 470, 515, 160, 0, 0, 0, 0, 0, 0, 0, 0, 196, 384, 1, 76, 1169, 2017, 141, 77, 1196, 569, 150, 0, 0, 0, 0, 0, 294, 1662, 277, 49, 47, 70, 353, 4, 3748, 0, 0, 0, 0, 0, 0, 0, 1420, 64, 63, 61, 7, 652, 27, 998, 1898, 1067, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5931, 5978, 6175, 6155, 6176, 5976, 6504, 6257] avg=6144.0000 max=6504 max_rank=6 min=5931 min_rank=0 imbalance_ratio=1.058594 max_load_reduction=42648 max_load_reduction_pct=0.867676 moved_to_echo=24913 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2, 174, 13, 195, 30, 32, 2124, 3913, 0, 0, 0, 0, 0, 0, 0, 0, 1, 703, 469, 1519, 2085, 134, 133, 74, 929, 348, 0, 0, 0, 0, 0, 0, 2561, 1233, 885, 773, 54, 5, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 250, 1014, 2663, 1055, 702, 219, 131, 20, 0, 0, 0, 0, 0, 0, 0, 0, 110, 432, 1692, 99, 2650, 440, 405, 144, 0, 0, 0, 0, 0, 0, 0, 0, 180, 347, 0, 53, 1312, 1877, 111, 48, 1327, 593, 145, 0, 0, 0, 0, 0, 234, 1731, 273, 21, 45, 86, 278, 3, 3642, 0, 0, 0, 0, 0, 0, 0, 1276, 54, 22, 39, 2, 738, 12, 937, 1976, 1037, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6483, 6395, 5849, 6054, 5972, 5993, 6313, 6093] avg=6144.0000 max=6483 max_rank=0 min=5849 min_rank=2 imbalance_ratio=1.055176 max_load_reduction=42669 max_load_reduction_pct=0.868103 moved_to_echo=24371 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[6, 345, 16, 151, 45, 63, 2062, 3541, 0, 0, 0, 0, 0, 0, 0, 0, 2, 749, 420, 1515, 1895, 145, 192, 104, 1067, 316, 0, 0, 0, 0, 0, 0, 2288, 1286, 980, 791, 69, 5, 0, 299, 0, 0, 0, 0, 0, 0, 0, 0, 386, 1136, 2552, 1009, 649, 207, 176, 40, 0, 0, 0, 0, 0, 0, 0, 0, 143, 422, 1623, 103, 3025, 434, 497, 141, 0, 0, 0, 0, 0, 0, 0, 0, 242, 356, 1, 131, 1670, 2042, 131, 104, 1207, 573, 169, 0, 0, 0, 0, 0, 298, 1402, 218, 52, 72, 83, 308, 4, 3165, 0, 0, 0, 0, 0, 0, 0, 1311, 62, 77, 57, 4, 600, 14, 1101, 1902, 901, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6229, 6405, 5718, 6155, 6388, 6626, 5602, 6029] avg=6144.0000 max=6626 max_rank=5 min=5602 min_rank=6 imbalance_ratio=1.078451 max_load_reduction=42526 max_load_reduction_pct=0.865194 moved_to_echo=24645 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 341, 10, 176, 53, 51, 1852, 3369, 0, 0, 0, 0, 0, 0, 0, 0, 1, 630, 387, 1413, 2058, 198, 239, 144, 897, 346, 0, 0, 0, 0, 0, 0, 2546, 1465, 914, 996, 100, 4, 0, 298, 0, 0, 0, 0, 0, 0, 0, 0, 303, 1100, 2411, 1095, 678, 232, 194, 44, 0, 0, 0, 0, 0, 0, 0, 0, 237, 414, 1493, 137, 2557, 600, 687, 175, 0, 0, 0, 0, 0, 0, 0, 0, 317, 368, 4, 102, 1239, 2094, 140, 73, 1143, 537, 140, 0, 0, 0, 0, 0, 402, 1285, 215, 60, 53, 99, 383, 7, 3381, 0, 0, 0, 0, 0, 0, 0, 1481, 112, 35, 67, 7, 662, 46, 1104, 1788, 962, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5853, 6313, 6323, 6057, 6300, 6157, 5885, 6264] avg=6144.0000 max=6323 max_rank=2 min=5853 min_rank=0 imbalance_ratio=1.029134 max_load_reduction=42829 max_load_reduction_pct=0.871358 moved_to_echo=24606 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4, 230, 13, 223, 27, 43, 1970, 3602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 598, 367, 1265, 1979, 133, 174, 73, 936, 345, 0, 0, 0, 0, 0, 0, 2540, 1557, 924, 1012, 58, 1, 0, 331, 0, 0, 0, 0, 0, 0, 0, 0, 321, 1183, 2643, 1012, 697, 259, 150, 16, 0, 0, 0, 0, 0, 0, 0, 0, 120, 390, 1468, 88, 2669, 436, 544, 145, 0, 0, 0, 0, 0, 0, 0, 0, 250, 351, 2, 94, 1266, 2114, 146, 49, 1222, 588, 146, 0, 0, 0, 0, 0, 335, 1580, 241, 27, 45, 66, 325, 2, 3574, 0, 0, 0, 0, 0, 0, 0, 1340, 44, 44, 53, 3, 663, 23, 1033, 1961, 1019, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6112, 5870, 6423, 6281, 5860, 6228, 6195, 6183] avg=6144.0000 max=6423 max_rank=2 min=5860 min_rank=4 imbalance_ratio=1.045410 max_load_reduction=42729 max_load_reduction_pct=0.869324 moved_to_echo=24466 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2, 193, 3, 218, 41, 50, 1923, 3710, 0, 0, 0, 0, 0, 0, 0, 0, 1, 621, 520, 1324, 2081, 125, 156, 99, 915, 323, 0, 0, 0, 0, 0, 0, 2376, 1481, 1033, 702, 66, 3, 0, 394, 0, 0, 0, 0, 0, 0, 0, 0, 260, 1088, 2725, 1063, 686, 203, 117, 24, 0, 0, 0, 0, 0, 0, 0, 0, 94, 505, 1590, 93, 2610, 487, 605, 141, 0, 0, 0, 0, 0, 0, 0, 0, 184, 302, 0, 75, 1334, 1942, 123, 61, 1258, 607, 142, 0, 0, 0, 0, 0, 231, 1556, 234, 29, 44, 57, 247, 4, 3860, 0, 0, 0, 0, 0, 0, 0, 1276, 44, 34, 49, 2, 595, 23, 1067, 2022, 1099, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6140, 6165, 6055, 6166, 6125, 6028, 6262, 6211] avg=6144.0000 max=6262 max_rank=6 min=6028 min_rank=5 imbalance_ratio=1.019206 max_load_reduction=42890 max_load_reduction_pct=0.872599 moved_to_echo=24626 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1792, 2944, 128, 768, 10496, 16256, 1152, 640, 9856, 4608, 1280, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 5376, 3328, 11264, 15744, 1280, 1408, 896, 7680, 2816, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2560, 8960, 20736, 8576, 5376, 1792, 1408, 256, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[19968, 11520, 7424, 7296, 640, 128, 0, 2688, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[1280, 3584, 12672, 896, 21632, 3840, 4352, 1280, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 2176, 128, 1664, 384, 384, 16000, 28800, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[10752, 640, 384, 512, 128, 5248, 256, 8320, 15360, 8192, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2432, 12544, 2048, 384, 512, 640, 2560, 128, 28416, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[853, 42, 1554, 30, 430, 149, 630, 422, 250, 109, 77, 43, 331, 16, 527, 118, 5653, 402, 312, 1, 526, 141, 1, 3550, 225, 3, 2095, 2441, 962, 1399, 5782, 80, 757, 1016, 27, 43, 900, 325, 968, 268, 386, 1720, 64, 2330, 201, 632, 2, 1192, 1, 161, 1948, 733, 74, 610, 39, 1475, 293, 0, 1037, 85, 287, 1922, 295, 207] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[918, 51, 1673, 35, 552, 189, 480, 506, 248, 124, 75, 41, 331, 18, 709, 231, 5853, 353, 225, 2, 408, 127, 0, 3520, 282, 0, 2241, 2163, 805, 1284, 5712, 81, 554, 872, 7, 45, 770, 257, 1076, 230, 452, 1558, 55, 2260, 179, 591, 7, 1254, 0, 156, 2140, 882, 46, 479, 40, 1445, 274, 0, 1053, 114, 241, 2325, 200, 353] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[918, 49, 1707, 48, 608, 213, 614, 457, 218, 124, 102, 60, 320, 20, 889, 269, 5750, 313, 212, 2, 363, 111, 0, 3564, 283, 0, 2024, 2247, 788, 1188, 5761, 79, 794, 1052, 23, 43, 885, 339, 951, 228, 408, 1659, 61, 2091, 185, 528, 6, 1409, 1, 158, 1978, 928, 78, 650, 26, 1218, 351, 0, 819, 105, 242, 2201, 228, 206] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[852, 34, 1922, 21, 543, 141, 630, 476, 226, 121, 84, 29, 306, 6, 901, 159, 5947, 363, 225, 1, 503, 91, 0, 3742, 311, 0, 2200, 2410, 908, 1116, 5765, 82, 621, 992, 27, 38, 965, 409, 941, 234, 347, 1491, 39, 2315, 178, 540, 0, 1335, 1, 138, 1801, 722, 40, 564, 36, 1549, 235, 0, 865, 97, 126, 1905, 153, 333] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[920, 41, 1547, 20, 555, 130, 657, 351, 158, 120, 97, 50, 346, 31, 715, 210, 5618, 347, 343, 1, 472, 112, 1, 3751, 242, 0, 2178, 2540, 771, 1054, 5880, 57, 737, 959, 17, 60, 1080, 454, 958, 239, 293, 1441, 72, 2314, 189, 524, 3, 1384, 1, 130, 1895, 806, 57, 593, 32, 1250, 238, 0, 1044, 74, 271, 2226, 184, 312] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[901, 83, 1664, 25, 443, 162, 545, 500, 222, 132, 105, 54, 347, 10, 865, 206, 5682, 258, 250, 2, 408, 132, 0, 3356, 280, 1, 2290, 2407, 817, 1195, 5765, 129, 577, 960, 19, 69, 906, 400, 973, 244, 336, 1714, 67, 2353, 238, 495, 5, 1370, 5, 139, 1952, 766, 62, 600, 35, 1288, 263, 0, 929, 137, 293, 2228, 278, 215] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[858, 40, 1744, 27, 515, 146, 681, 453, 167, 98, 77, 43, 337, 12, 622, 220, 6020, 355, 259, 0, 745, 101, 0, 3493, 287, 0, 2049, 2298, 725, 1021, 5883, 129, 863, 1110, 22, 54, 1124, 355, 887, 233, 342, 1482, 55, 2264, 198, 615, 0, 1447, 3, 122, 1863, 782, 40, 541, 42, 1456, 250, 0, 946, 94, 239, 1814, 154, 350] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[890, 53, 1621, 46, 621, 218, 501, 465, 261, 140, 130, 114, 318, 25, 876, 193, 5377, 304, 249, 6, 325, 165, 0, 3383, 359, 4, 2432, 2453, 775, 1169, 5691, 112, 762, 991, 18, 46, 1112, 258, 928, 244, 371, 1709, 87, 2254, 158, 443, 5, 1348, 6, 130, 1958, 1068, 52, 509, 42, 1302, 302, 0, 780, 83, 258, 2115, 252, 285] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[853, 42, 1554, 30, 430, 149, 630, 422, 1722, 0, 0, 0, 0, 0, 0, 0, 250, 109, 77, 43, 331, 16, 527, 118, 4347, 0, 0, 0, 0, 0, 0, 0, 1390, 402, 312, 1, 526, 141, 1, 3550, 0, 0, 0, 0, 0, 0, 0, 0, 225, 3, 2095, 1628, 962, 1399, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 757, 1016, 27, 43, 900, 325, 968, 268, 1827, 0, 0, 0, 0, 0, 0, 0, 386, 1720, 64, 2087, 201, 632, 2, 1192, 0, 0, 0, 0, 0, 0, 0, 0, 1, 161, 1948, 733, 74, 610, 39, 1475, 813, 243, 158, 0, 0, 0, 0, 0, 293, 0, 1037, 85, 287, 1922, 295, 207, 1435, 556, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5832, 5818, 6323, 6392, 6131, 6284, 6255, 6117] avg=6144.0000 max=6392 max_rank=3 min=5818 min_rank=1 imbalance_ratio=1.040365 max_load_reduction=42760 max_load_reduction_pct=0.869954 moved_to_echo=24787 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[918, 51, 1673, 35, 552, 189, 480, 506, 1782, 0, 0, 0, 0, 0, 0, 0, 248, 124, 75, 41, 331, 18, 709, 231, 4295, 0, 0, 0, 0, 0, 0, 0, 1441, 353, 225, 2, 408, 127, 0, 3520, 0, 0, 0, 0, 0, 0, 0, 0, 282, 0, 2241, 1443, 805, 1284, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 554, 872, 7, 45, 770, 257, 1076, 230, 1891, 0, 0, 0, 0, 0, 0, 0, 452, 1558, 55, 2024, 179, 591, 7, 1254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 2140, 882, 46, 479, 40, 1445, 720, 236, 164, 0, 0, 0, 0, 0, 274, 0, 1053, 114, 241, 2325, 200, 353, 1417, 575, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6186, 6072, 6076, 6136, 5702, 6120, 6308, 6552] avg=6144.0000 max=6552 max_rank=7 min=5702 min_rank=4 imbalance_ratio=1.066406 max_load_reduction=42600 max_load_reduction_pct=0.866699 moved_to_echo=24682 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[918, 49, 1707, 48, 608, 213, 614, 457, 1751, 0, 0, 0, 0, 0, 0, 0, 218, 124, 102, 60, 320, 20, 889, 269, 4331, 0, 0, 0, 0, 0, 0, 0, 1415, 313, 212, 2, 363, 111, 0, 3564, 0, 0, 0, 0, 0, 0, 0, 0, 283, 0, 2024, 1499, 788, 1188, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 794, 1052, 23, 43, 885, 339, 951, 228, 1858, 0, 0, 0, 0, 0, 0, 0, 408, 1659, 61, 1873, 185, 528, 6, 1409, 0, 0, 0, 0, 0, 0, 0, 0, 1, 158, 1978, 928, 78, 650, 26, 1218, 748, 218, 161, 0, 0, 0, 0, 0, 351, 0, 819, 105, 242, 2201, 228, 206, 1430, 565, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6365, 6333, 5980, 5861, 6173, 6129, 6164, 6147] avg=6144.0000 max=6365 max_rank=0 min=5861 min_rank=3 imbalance_ratio=1.035970 max_load_reduction=42787 max_load_reduction_pct=0.870504 moved_to_echo=24613 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[852, 34, 1922, 21, 543, 141, 630, 476, 1811, 0, 0, 0, 0, 0, 0, 0, 226, 121, 84, 29, 306, 6, 901, 159, 4335, 0, 0, 0, 0, 0, 0, 0, 1462, 363, 225, 1, 503, 91, 0, 3742, 0, 0, 0, 0, 0, 0, 0, 0, 311, 0, 2200, 1607, 908, 1116, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 621, 992, 27, 38, 965, 409, 941, 234, 1922, 0, 0, 0, 0, 0, 0, 0, 347, 1491, 39, 2074, 178, 540, 0, 1335, 0, 0, 0, 0, 0, 0, 0, 0, 1, 138, 1801, 722, 40, 564, 36, 1549, 803, 241, 167, 0, 0, 0, 0, 0, 235, 0, 865, 97, 126, 1905, 153, 333, 1430, 585, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6430, 6167, 6387, 6224, 6149, 6004, 6062, 5729] avg=6144.0000 max=6430 max_rank=0 min=5729 min_rank=7 imbalance_ratio=1.046549 max_load_reduction=42722 max_load_reduction_pct=0.869181 moved_to_echo=23944 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[920, 41, 1547, 20, 555, 130, 657, 351, 1711, 0, 0, 0, 0, 0, 0, 0, 158, 120, 97, 50, 346, 31, 715, 210, 4422, 0, 0, 0, 0, 0, 0, 0, 1383, 347, 343, 1, 472, 112, 1, 3751, 0, 0, 0, 0, 0, 0, 0, 0, 242, 0, 2178, 1694, 771, 1054, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 737, 959, 17, 60, 1080, 454, 958, 239, 1815, 0, 0, 0, 0, 0, 0, 0, 293, 1441, 72, 2073, 189, 524, 3, 1384, 0, 0, 0, 0, 0, 0, 0, 0, 1, 130, 1895, 806, 57, 593, 32, 1250, 846, 241, 157, 0, 0, 0, 0, 0, 238, 0, 1044, 74, 271, 2226, 184, 312, 1458, 552, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5932, 6149, 6410, 5996, 6319, 5979, 6008, 6359] avg=6144.0000 max=6410 max_rank=2 min=5932 min_rank=0 imbalance_ratio=1.043294 max_load_reduction=42742 max_load_reduction_pct=0.869588 moved_to_echo=24665 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[890, 53, 1621, 46, 621, 218, 501, 465, 1637, 0, 0, 0, 0, 0, 0, 0, 261, 140, 130, 114, 318, 25, 876, 193, 4280, 0, 0, 0, 0, 0, 0, 0, 1323, 304, 249, 6, 325, 165, 0, 3383, 0, 0, 0, 0, 0, 0, 0, 0, 359, 4, 2432, 1636, 775, 1169, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 762, 991, 18, 46, 1112, 258, 928, 244, 1737, 0, 0, 0, 0, 0, 0, 0, 371, 1709, 87, 2019, 158, 443, 5, 1348, 0, 0, 0, 0, 0, 0, 0, 0, 6, 130, 1958, 1068, 52, 509, 42, 1302, 817, 235, 151, 0, 0, 0, 0, 0, 302, 0, 780, 83, 258, 2115, 252, 285, 1411, 529, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6052, 6337, 5755, 6487, 6096, 6140, 6270, 6015] avg=6144.0000 max=6487 max_rank=3 min=5755 min_rank=2 imbalance_ratio=1.055827 max_load_reduction=42665 max_load_reduction_pct=0.868022 moved_to_echo=24521 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[901, 83, 1664, 25, 443, 162, 545, 500, 1735, 0, 0, 0, 0, 0, 0, 0, 222, 132, 105, 54, 347, 10, 865, 206, 4335, 0, 0, 0, 0, 0, 0, 0, 1381, 258, 250, 2, 408, 132, 0, 3356, 0, 0, 0, 0, 0, 0, 0, 0, 280, 1, 2290, 1600, 817, 1195, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 577, 960, 19, 69, 906, 400, 973, 244, 1841, 0, 0, 0, 0, 0, 0, 0, 336, 1714, 67, 2104, 238, 495, 5, 1370, 0, 0, 0, 0, 0, 0, 0, 0, 5, 139, 1952, 766, 62, 600, 35, 1288, 807, 249, 163, 0, 0, 0, 0, 0, 263, 0, 929, 137, 293, 2228, 278, 215, 1430, 562, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6058, 6276, 5787, 6312, 5989, 6329, 6066, 6335] avg=6144.0000 max=6335 max_rank=7 min=5787 min_rank=2 imbalance_ratio=1.031087 max_load_reduction=42817 max_load_reduction_pct=0.871114 moved_to_echo=24719 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[858, 40, 1744, 27, 515, 146, 681, 453, 1833, 0, 0, 0, 0, 0, 0, 0, 167, 98, 77, 43, 337, 12, 622, 220, 4424, 0, 0, 0, 0, 0, 0, 0, 1481, 355, 259, 0, 745, 101, 0, 3493, 0, 0, 0, 0, 0, 0, 0, 0, 287, 0, 2049, 1533, 725, 1021, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 863, 1110, 22, 54, 1124, 355, 887, 233, 1945, 0, 0, 0, 0, 0, 0, 0, 342, 1482, 55, 2028, 198, 615, 0, 1447, 0, 0, 0, 0, 0, 0, 0, 0, 3, 122, 1863, 782, 40, 541, 42, 1456, 765, 236, 169, 0, 0, 0, 0, 0, 250, 0, 946, 94, 239, 1814, 154, 350, 1459, 592, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6297, 6000, 6434, 5744, 6593, 6167, 6019, 5898] avg=6144.0000 max=6593 max_rank=4 min=5744 min_rank=3 imbalance_ratio=1.073079 max_load_reduction=42559 max_load_reduction_pct=0.865865 moved_to_echo=24677 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[2944, 12800, 512, 16384, 1536, 4480, 128, 10752, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[7168, 512, 13440, 256, 4352, 1408, 4864, 3712, 14080, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 1152, 15616, 6784, 512, 4608, 384, 11008, 6400, 1920, 1408, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[2304, 128, 17536, 12672, 6656, 9472, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5760, 8064, 256, 512, 7808, 2816, 7808, 1920, 14848, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2304, 0, 7552, 896, 2048, 16768, 1792, 2304, 11520, 4608, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[11392, 2816, 2176, 128, 3840, 1024, 128, 28416, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1792, 1024, 768, 512, 2688, 256, 6144, 1664, 34816, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[19, 297, 867, 2, 25, 600, 23, 411, 6, 373, 649, 322, 2, 0, 22, 46, 0, 1285, 0, 7078, 466, 268, 320, 953, 59, 3, 616, 4771, 148, 5942, 65, 2, 2, 3, 0, 7614, 2487, 7, 60, 488, 20, 4, 64, 1098, 418, 0, 2468, 95, 0, 46, 1030, 61, 21, 342, 1126, 1298, 10, 553, 2, 36, 4026, 111, 8, 14] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[12, 248, 1190, 0, 20, 560, 32, 365, 1, 423, 517, 282, 0, 0, 8, 62, 0, 1055, 0, 7116, 576, 180, 303, 919, 86, 2, 445, 4532, 164, 5783, 46, 0, 0, 4, 0, 7621, 2596, 3, 49, 667, 11, 7, 18, 1050, 347, 0, 2325, 77, 0, 57, 1348, 33, 7, 338, 1267, 1465, 8, 628, 1, 10, 4206, 69, 5, 8] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[2, 259, 1078, 0, 17, 540, 24, 367, 1, 389, 591, 379, 0, 0, 2, 55, 1, 1120, 0, 7420, 511, 293, 371, 1002, 99, 1, 541, 4605, 102, 5704, 45, 1, 0, 1, 0, 7533, 2644, 5, 60, 609, 13, 8, 39, 1042, 328, 0, 2440, 53, 1, 46, 796, 49, 25, 353, 1391, 1303, 15, 497, 2, 13, 4269, 84, 3, 10] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[6, 258, 1254, 0, 22, 607, 18, 368, 1, 470, 474, 274, 1, 0, 14, 40, 0, 1163, 0, 7146, 706, 189, 226, 782, 69, 5, 582, 4711, 142, 5717, 108, 2, 0, 5, 0, 7549, 2384, 4, 51, 660, 7, 4, 25, 1028, 492, 0, 2507, 79, 0, 58, 1101, 59, 12, 338, 1337, 1409, 6, 617, 0, 19, 3933, 97, 6, 10] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[16, 297, 1124, 3, 24, 588, 20, 327, 6, 574, 622, 429, 0, 0, 24, 44, 0, 1130, 0, 6898, 434, 131, 437, 906, 59, 0, 683, 4872, 155, 5811, 59, 1, 0, 5, 0, 7456, 2143, 9, 48, 623, 28, 3, 67, 1098, 417, 0, 2642, 58, 0, 66, 874, 46, 13, 397, 1450, 1482, 7, 574, 1, 23, 3784, 149, 3, 12] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[3, 199, 1008, 0, 19, 519, 20, 299, 1, 379, 506, 228, 0, 0, 12, 70, 0, 1060, 0, 7199, 638, 184, 339, 837, 84, 0, 501, 4648, 107, 6170, 56, 1, 0, 2, 0, 7607, 2558, 1, 45, 623, 11, 1, 20, 1013, 281, 0, 2613, 56, 0, 42, 1207, 37, 9, 373, 1264, 1521, 6, 580, 0, 14, 4077, 93, 2, 9] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[13, 163, 1118, 0, 14, 512, 20, 341, 1, 376, 521, 201, 0, 0, 9, 55, 0, 1213, 0, 7179, 540, 207, 275, 780, 100, 0, 565, 4952, 83, 6021, 59, 0, 0, 2, 0, 7660, 2707, 8, 54, 601, 5, 2, 30, 1092, 296, 0, 2632, 52, 0, 40, 1235, 31, 16, 301, 1128, 1242, 8, 483, 2, 8, 4076, 99, 2, 22] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[8, 195, 1119, 1, 22, 492, 20, 419, 6, 551, 610, 451, 0, 0, 18, 45, 0, 1107, 0, 7060, 539, 153, 357, 1107, 73, 0, 442, 4850, 129, 5133, 60, 1, 0, 0, 0, 7427, 2632, 10, 80, 848, 13, 11, 43, 1182, 300, 0, 2678, 79, 0, 97, 924, 53, 11, 383, 1346, 1397, 9, 611, 2, 16, 3899, 109, 4, 20] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[12, 248, 1190, 0, 20, 560, 32, 365, 3418, 451, 0, 0, 0, 0, 0, 0, 1, 423, 517, 282, 0, 0, 8, 62, 4749, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 0, 3004, 576, 180, 303, 919, 0, 0, 0, 0, 0, 0, 0, 0, 86, 2, 445, 4532, 164, 583, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2946, 2596, 3, 49, 667, 0, 0, 0, 0, 0, 0, 0, 0, 11, 7, 18, 1050, 347, 0, 2325, 77, 1257, 795, 0, 0, 0, 0, 0, 0, 0, 57, 1348, 33, 7, 338, 1267, 1465, 1925, 0, 0, 0, 0, 0, 0, 0, 8, 628, 1, 10, 4206, 69, 5, 8, 1392, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6296, 6042, 6037, 5858, 6265, 5887, 6440, 6327] avg=6144.0000 max=6440 max_rank=6 min=5858 min_rank=3 imbalance_ratio=1.048177 max_load_reduction=42712 max_load_reduction_pct=0.868978 moved_to_echo=24919 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2, 259, 1078, 0, 17, 540, 24, 367, 3379, 445, 0, 0, 0, 0, 0, 0, 1, 389, 591, 379, 0, 0, 2, 55, 4684, 0, 0, 0, 0, 0, 0, 0, 1, 1120, 0, 3133, 511, 293, 371, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 99, 1, 541, 4605, 102, 575, 45, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2912, 2644, 5, 60, 609, 0, 0, 0, 0, 0, 0, 0, 0, 13, 8, 39, 1042, 328, 0, 2440, 53, 1242, 829, 0, 0, 0, 0, 0, 0, 1, 46, 796, 49, 25, 353, 1391, 1303, 2007, 0, 0, 0, 0, 0, 0, 0, 15, 497, 2, 13, 4269, 84, 3, 10, 1451, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6111, 6101, 6431, 5969, 6231, 5994, 5971, 6344] avg=6144.0000 max=6431 max_rank=2 min=5969 min_rank=3 imbalance_ratio=1.046712 max_load_reduction=42721 max_load_reduction_pct=0.869161 moved_to_echo=24540 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[19, 297, 867, 2, 25, 600, 23, 411, 3415, 463, 0, 0, 0, 0, 0, 0, 6, 373, 649, 322, 2, 0, 22, 46, 4879, 0, 0, 0, 0, 0, 0, 0, 0, 1285, 0, 2988, 466, 268, 320, 953, 0, 0, 0, 0, 0, 0, 0, 0, 59, 3, 616, 4771, 148, 600, 65, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 2943, 2487, 7, 60, 488, 0, 0, 0, 0, 0, 0, 0, 0, 20, 4, 64, 1098, 418, 0, 2468, 95, 1256, 791, 0, 0, 0, 0, 0, 0, 0, 46, 1030, 61, 21, 342, 1126, 1298, 1915, 0, 0, 0, 0, 0, 0, 0, 10, 553, 2, 36, 4026, 111, 8, 14, 1384, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6122, 6299, 6280, 6264, 5990, 6214, 5839, 6144] avg=6144.0000 max=6299 max_rank=1 min=5839 min_rank=6 imbalance_ratio=1.025228 max_load_reduction=42853 max_load_reduction_pct=0.871847 moved_to_echo=24187 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[6, 258, 1254, 0, 22, 607, 18, 368, 3386, 446, 0, 0, 0, 0, 0, 0, 1, 470, 474, 274, 1, 0, 14, 40, 4695, 0, 0, 0, 0, 0, 0, 0, 0, 1163, 0, 3016, 706, 189, 226, 782, 0, 0, 0, 0, 0, 0, 0, 0, 69, 5, 582, 4711, 142, 576, 108, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2918, 2384, 4, 51, 660, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4, 25, 1028, 492, 0, 2507, 79, 1245, 799, 0, 0, 0, 0, 0, 0, 0, 58, 1101, 59, 12, 338, 1337, 1409, 1933, 0, 0, 0, 0, 0, 0, 0, 6, 617, 0, 19, 3933, 97, 6, 10, 1398, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6365, 5969, 6082, 6195, 6022, 6186, 6247, 6086] avg=6144.0000 max=6365 max_rank=0 min=5969 min_rank=1 imbalance_ratio=1.035970 max_load_reduction=42787 max_load_reduction_pct=0.870504 moved_to_echo=24541 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[16, 297, 1124, 3, 24, 588, 20, 327, 3349, 457, 0, 0, 0, 0, 0, 0, 6, 574, 622, 429, 0, 0, 24, 44, 4776, 0, 0, 0, 0, 0, 0, 0, 0, 1130, 0, 2899, 434, 131, 437, 906, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 683, 4872, 155, 578, 59, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2873, 2143, 9, 48, 623, 0, 0, 0, 0, 0, 0, 0, 0, 28, 3, 67, 1098, 417, 0, 2642, 58, 1234, 775, 0, 0, 0, 0, 0, 0, 0, 66, 874, 46, 13, 397, 1450, 1482, 1870, 0, 0, 0, 0, 0, 0, 0, 7, 574, 1, 23, 3784, 149, 3, 12, 1354, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6205, 6475, 5937, 6407, 5701, 6322, 6198, 5907] avg=6144.0000 max=6475 max_rank=1 min=5701 min_rank=4 imbalance_ratio=1.053874 max_load_reduction=42677 max_load_reduction_pct=0.868266 moved_to_echo=24128 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[3, 199, 1008, 0, 19, 519, 20, 299, 3412, 481, 0, 0, 0, 0, 0, 0, 1, 379, 506, 228, 0, 0, 12, 70, 5067, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 0, 3038, 638, 184, 339, 837, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 501, 4648, 107, 622, 56, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2940, 2558, 1, 45, 623, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1, 20, 1013, 281, 0, 2613, 56, 1255, 805, 0, 0, 0, 0, 0, 0, 0, 42, 1207, 37, 9, 373, 1264, 1521, 1948, 0, 0, 0, 0, 0, 0, 0, 6, 580, 0, 14, 4077, 93, 2, 9, 1408, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5960, 6263, 6096, 6019, 6169, 6055, 6401, 6189] avg=6144.0000 max=6401 max_rank=6 min=5960 min_rank=0 imbalance_ratio=1.041829 max_load_reduction=42751 max_load_reduction_pct=0.869771 moved_to_echo=24814 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[8, 195, 1119, 1, 22, 492, 20, 419, 3331, 400, 0, 0, 0, 0, 0, 0, 6, 551, 610, 451, 0, 0, 18, 45, 4215, 0, 0, 0, 0, 0, 0, 0, 0, 1107, 0, 2980, 539, 153, 357, 1107, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 442, 4850, 129, 518, 60, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2871, 2632, 10, 80, 848, 0, 0, 0, 0, 0, 0, 0, 0, 13, 11, 43, 1182, 300, 0, 2678, 79, 1225, 789, 0, 0, 0, 0, 0, 0, 0, 97, 924, 53, 11, 383, 1346, 1397, 1910, 0, 0, 0, 0, 0, 0, 0, 9, 611, 2, 16, 3899, 109, 4, 20, 1381, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6007, 5896, 6243, 6073, 6441, 6320, 6121, 6051] avg=6144.0000 max=6441 max_rank=4 min=5896 min_rank=1 imbalance_ratio=1.048340 max_load_reduction=42711 max_load_reduction_pct=0.868958 moved_to_echo=24933 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[13, 163, 1118, 0, 14, 512, 20, 341, 3436, 469, 0, 0, 0, 0, 0, 0, 1, 376, 521, 201, 0, 0, 9, 55, 4944, 0, 0, 0, 0, 0, 0, 0, 0, 1213, 0, 3031, 540, 207, 275, 780, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 565, 4952, 83, 608, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2961, 2707, 8, 54, 601, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 30, 1092, 296, 0, 2632, 52, 1263, 802, 0, 0, 0, 0, 0, 0, 0, 40, 1235, 31, 16, 301, 1128, 1242, 1942, 0, 0, 0, 0, 0, 0, 0, 8, 483, 2, 8, 4076, 99, 2, 22, 1404, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6086, 6107, 6046, 6367, 6333, 6174, 5935, 6104] avg=6144.0000 max=6367 max_rank=3 min=5935 min_rank=6 imbalance_ratio=1.036296 max_load_reduction=42785 max_load_reduction_pct=0.870463 moved_to_echo=24546 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 1920, 8832, 128, 256, 4480, 256, 2944, 27136, 3712, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 512, 8576, 384, 128, 2944, 10368, 11136, 15488, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 0, 23424, 20224, 128, 512, 5120, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 9216, 0, 24192, 4480, 1664, 2688, 7296, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 384, 8704, 2944, 0, 20352, 640, 9984, 6400, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 4608, 128, 256, 32384, 896, 128, 128, 11264, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[640, 128, 4480, 38016, 1152, 4736, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 3584, 4608, 2688, 128, 0, 128, 512, 38016, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 470, 2046, 0, 339, 37, 78, 220, 117, 0, 11, 1318, 0, 2, 2198, 1299, 5723, 8036, 49, 3, 5025, 1646, 2680, 0, 1, 14, 328, 0, 0, 0, 26, 0, 11, 0, 0, 2, 0, 3, 3067, 181, 6831, 5, 238, 1, 0, 16, 1, 1899, 11, 11, 234, 3469, 6, 34, 1, 668, 5, 40, 1, 31, 0, 612, 0, 108] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 424, 1707, 0, 332, 25, 34, 352, 125, 0, 13, 1467, 1, 7, 2849, 1293, 5592, 7926, 46, 1, 5523, 1311, 2306, 0, 0, 11, 370, 0, 0, 2, 40, 0, 12, 0, 0, 1, 0, 2, 3520, 194, 6656, 5, 320, 1, 0, 11, 0, 1798, 6, 9, 329, 2997, 3, 20, 0, 772, 0, 58, 0, 14, 1, 548, 1, 117] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[5, 493, 1972, 0, 352, 23, 64, 420, 127, 0, 10, 1358, 0, 1, 2414, 1333, 5742, 7952, 42, 2, 5191, 1535, 2196, 0, 0, 16, 375, 0, 0, 1, 26, 0, 15, 0, 0, 1, 0, 2, 3255, 142, 6833, 7, 292, 0, 0, 7, 0, 1528, 12, 17, 255, 3101, 8, 22, 0, 1143, 0, 30, 0, 11, 0, 724, 1, 96] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1, 360, 2072, 0, 304, 21, 28, 232, 82, 0, 3, 1241, 0, 3, 2324, 1313, 5669, 8003, 47, 3, 5790, 1436, 2594, 0, 0, 16, 343, 1, 0, 0, 25, 0, 1, 0, 1, 1, 0, 0, 3440, 137, 7009, 5, 170, 1, 0, 4, 0, 1567, 4, 11, 271, 3366, 4, 17, 0, 550, 0, 26, 0, 7, 0, 542, 0, 107] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1, 500, 1715, 0, 376, 23, 37, 463, 49, 0, 4, 1238, 0, 2, 2160, 1181, 6015, 8012, 26, 2, 5326, 1548, 2285, 0, 0, 6, 342, 0, 0, 1, 20, 0, 7, 0, 2, 0, 0, 2, 3564, 135, 6973, 11, 293, 0, 0, 3, 0, 1743, 11, 6, 244, 3107, 4, 25, 0, 823, 0, 58, 0, 17, 0, 675, 0, 117] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 391, 2067, 0, 272, 20, 39, 233, 65, 0, 1, 1375, 0, 2, 2592, 1160, 5738, 8040, 55, 4, 5426, 1444, 2621, 0, 0, 8, 348, 0, 0, 1, 19, 0, 4, 0, 0, 1, 0, 0, 2933, 155, 7080, 4, 233, 2, 0, 14, 0, 1881, 5, 11, 258, 3198, 3, 11, 0, 696, 0, 38, 0, 17, 0, 597, 0, 90] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[3, 432, 1994, 0, 374, 31, 37, 366, 132, 0, 15, 1348, 0, 1, 2211, 1313, 5508, 7954, 56, 4, 5053, 1540, 2499, 0, 0, 21, 361, 0, 0, 0, 35, 4, 6, 0, 5, 3, 0, 3, 3504, 221, 6854, 8, 308, 2, 1, 5, 4, 2007, 13, 10, 271, 3224, 4, 31, 0, 595, 0, 75, 8, 19, 0, 550, 1, 128] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[4, 469, 1898, 0, 430, 23, 70, 462, 82, 0, 7, 1451, 1, 0, 2077, 1477, 5606, 7839, 20, 3, 4990, 1540, 2348, 0, 0, 31, 337, 0, 0, 1, 33, 0, 5, 0, 0, 2, 0, 3, 3543, 210, 6980, 4, 272, 2, 0, 2, 0, 1568, 14, 16, 248, 3236, 3, 46, 0, 860, 0, 46, 0, 27, 0, 725, 0, 141] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[5, 493, 1972, 0, 352, 23, 64, 420, 2597, 415, 0, 0, 0, 0, 0, 0, 127, 0, 10, 1358, 0, 1, 2414, 1333, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 2, 2119, 1535, 2196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 375, 0, 0, 1, 26, 0, 5735, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 1, 0, 2, 3255, 142, 2557, 0, 0, 0, 0, 0, 0, 0, 4073, 7, 292, 0, 0, 7, 0, 1528, 0, 0, 0, 0, 0, 0, 0, 0, 12, 17, 255, 3101, 8, 22, 0, 1143, 1733, 100, 0, 0, 0, 0, 0, 0, 0, 30, 0, 11, 0, 724, 1, 96, 3145, 2217, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6341, 6270, 5894, 6153, 5972, 5907, 6391, 6224] avg=6144.0000 max=6391 max_rank=6 min=5894 min_rank=2 imbalance_ratio=1.040202 max_load_reduction=42761 max_load_reduction_pct=0.869975 moved_to_echo=24494 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 424, 1707, 0, 332, 25, 34, 352, 2529, 442, 0, 0, 0, 0, 0, 0, 125, 0, 13, 1467, 1, 7, 2849, 1293, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 1, 2253, 1311, 2306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 370, 0, 0, 2, 40, 0, 5716, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 1, 0, 2, 3520, 194, 2721, 0, 0, 0, 0, 0, 0, 0, 3968, 5, 320, 1, 0, 11, 0, 1798, 0, 0, 0, 0, 0, 0, 0, 0, 6, 9, 329, 2997, 3, 20, 0, 772, 1688, 107, 0, 0, 0, 0, 0, 0, 0, 58, 0, 14, 1, 548, 1, 117, 3063, 2210, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5845, 6755, 5917, 6139, 6450, 6103, 5931, 6012] avg=6144.0000 max=6755 max_rank=1 min=5845 min_rank=0 imbalance_ratio=1.099447 max_load_reduction=42397 max_load_reduction_pct=0.862569 moved_to_echo=24496 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 500, 1715, 0, 376, 23, 37, 463, 2720, 426, 0, 0, 0, 0, 0, 0, 49, 0, 4, 1238, 0, 2, 2160, 1181, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 2, 2173, 1548, 2285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 342, 0, 0, 1, 20, 0, 5778, 0, 0, 0, 0, 0, 0, 0, 7, 0, 2, 0, 0, 2, 3564, 135, 2624, 0, 0, 0, 0, 0, 0, 0, 4156, 11, 293, 0, 0, 3, 0, 1743, 0, 0, 0, 0, 0, 0, 0, 0, 11, 6, 244, 3107, 4, 25, 0, 823, 1769, 103, 0, 0, 0, 0, 0, 0, 0, 58, 0, 17, 0, 675, 0, 117, 3295, 2234, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6261, 5682, 6034, 6147, 6334, 6206, 6092, 6396] avg=6144.0000 max=6396 max_rank=7 min=5682 min_rank=1 imbalance_ratio=1.041016 max_load_reduction=42756 max_load_reduction_pct=0.869873 moved_to_echo=25028 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 470, 2046, 0, 339, 37, 78, 220, 2589, 402, 0, 0, 0, 0, 0, 0, 117, 0, 11, 1318, 0, 2, 2198, 1299, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 3, 2050, 1646, 2680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 14, 328, 0, 0, 0, 26, 0, 5797, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 2, 0, 3, 3067, 181, 2476, 0, 0, 0, 0, 0, 0, 0, 4073, 5, 238, 1, 0, 16, 1, 1899, 0, 0, 0, 0, 0, 0, 0, 0, 11, 11, 234, 3469, 6, 34, 1, 668, 1732, 97, 0, 0, 0, 0, 0, 0, 5, 40, 1, 31, 0, 612, 0, 108, 3134, 2239, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6181, 5971, 6428, 6166, 5740, 6233, 6263, 6170] avg=6144.0000 max=6428 max_rank=2 min=5740 min_rank=4 imbalance_ratio=1.046224 max_load_reduction=42724 max_load_reduction_pct=0.869222 moved_to_echo=24406 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3, 432, 1994, 0, 374, 31, 37, 366, 2492, 407, 0, 0, 0, 0, 0, 0, 132, 0, 15, 1348, 0, 1, 2211, 1313, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 4, 2052, 1540, 2499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 361, 0, 0, 0, 35, 4, 5737, 0, 0, 0, 0, 0, 0, 0, 6, 0, 5, 3, 0, 3, 3504, 221, 2494, 0, 0, 0, 0, 0, 0, 0, 4080, 8, 308, 2, 1, 5, 4, 2007, 0, 0, 0, 0, 0, 0, 0, 0, 13, 10, 271, 3224, 4, 31, 0, 595, 1742, 100, 0, 0, 0, 0, 0, 0, 0, 75, 8, 19, 0, 550, 1, 128, 3016, 2217, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6136, 6052, 6151, 6158, 6236, 6415, 5990, 6014] avg=6144.0000 max=6415 max_rank=5 min=5990 min_rank=6 imbalance_ratio=1.044108 max_load_reduction=42737 max_load_reduction_pct=0.869486 moved_to_echo=24655 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 360, 2072, 0, 304, 21, 28, 232, 2565, 463, 0, 0, 0, 0, 0, 0, 82, 0, 3, 1241, 0, 3, 2324, 1313, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 3, 2362, 1436, 2594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 343, 1, 0, 0, 25, 0, 5773, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 3440, 137, 2853, 0, 0, 0, 0, 0, 0, 0, 4178, 5, 170, 1, 0, 4, 0, 1567, 0, 0, 0, 0, 0, 0, 0, 0, 4, 11, 271, 3366, 4, 17, 0, 550, 1778, 112, 0, 0, 0, 0, 0, 0, 0, 26, 0, 7, 0, 542, 0, 107, 3104, 2230, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6046, 6019, 6442, 6158, 6433, 5925, 6113, 6016] avg=6144.0000 max=6442 max_rank=2 min=5925 min_rank=5 imbalance_ratio=1.048503 max_load_reduction=42710 max_load_reduction_pct=0.868937 moved_to_echo=24487 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4, 469, 1898, 0, 430, 23, 70, 462, 2536, 399, 0, 0, 0, 0, 0, 0, 82, 0, 7, 1451, 1, 0, 2077, 1477, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 3, 2037, 1540, 2348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 337, 0, 0, 1, 33, 0, 5654, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 2, 0, 3, 3543, 210, 2458, 0, 0, 0, 0, 0, 0, 0, 4161, 4, 272, 2, 0, 2, 0, 1568, 0, 0, 0, 0, 0, 0, 0, 0, 14, 16, 248, 3236, 3, 46, 0, 860, 1770, 96, 0, 0, 0, 0, 0, 0, 0, 46, 0, 27, 0, 725, 0, 141, 3070, 2185, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6291, 6144, 5948, 6056, 6221, 6009, 6289, 6194] avg=6144.0000 max=6291 max_rank=0 min=5948 min_rank=2 imbalance_ratio=1.023926 max_load_reduction=42861 max_load_reduction_pct=0.872009 moved_to_echo=24713 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 391, 2067, 0, 272, 20, 39, 233, 2595, 434, 0, 0, 0, 0, 0, 0, 65, 0, 1, 1375, 0, 2, 2592, 1160, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 4, 2214, 1444, 2621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 348, 0, 0, 1, 19, 0, 5799, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1, 0, 0, 2933, 155, 2673, 0, 0, 0, 0, 0, 0, 0, 4220, 4, 233, 2, 0, 14, 0, 1881, 0, 0, 0, 0, 0, 0, 0, 0, 5, 11, 258, 3198, 3, 11, 0, 696, 1796, 105, 0, 0, 0, 0, 0, 0, 0, 38, 0, 17, 0, 597, 0, 90, 3143, 2241, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6051, 6259, 6338, 6175, 5766, 6354, 6083, 6126] avg=6144.0000 max=6354 max_rank=5 min=5766 min_rank=4 imbalance_ratio=1.034180 max_load_reduction=42798 max_load_reduction_pct=0.870728 moved_to_echo=24329 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 384, 128, 256, 128, 4992, 128, 1024, 25088, 17792, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 3584, 15488, 0, 2816, 256, 512, 2816, 20736, 3456, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[896, 0, 128, 10880, 128, 128, 18944, 10496, 8320, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 0, 128, 26880, 1408, 20864, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 384, 128, 17280, 12032, 19584, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 2176, 25728, 128, 256, 128, 6144, 14080, 896, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[33024, 128, 2176, 128, 128, 128, 128, 14080, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 2816, 128, 0, 128, 256, 128, 46080, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[2090, 0, 7492, 6, 0, 0, 0, 4302, 0, 0, 1029, 1821, 7031, 6033, 396, 8, 0, 0, 0, 438, 0, 12, 2094, 658, 0, 110, 124, 2082, 0, 1, 2, 0, 19, 0, 4508, 3, 1, 0, 0, 0, 9, 2115, 0, 40, 4, 1, 795, 100, 33, 0, 5235, 485, 14, 12, 0, 0, 31, 9, 0, 0, 8, 0, 1, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1971, 0, 7269, 11, 0, 0, 0, 4691, 0, 0, 909, 1701, 6729, 6236, 463, 12, 0, 0, 0, 509, 0, 5, 1917, 669, 0, 82, 115, 2106, 0, 0, 3, 0, 74, 1, 5596, 2, 0, 0, 0, 0, 4, 2019, 0, 19, 6, 0, 701, 92, 16, 0, 4597, 527, 29, 13, 0, 0, 51, 3, 0, 0, 2, 0, 2, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[2043, 1, 7500, 7, 1, 0, 0, 4780, 0, 0, 1132, 1765, 6434, 6057, 500, 5, 0, 0, 3, 396, 0, 5, 2054, 684, 0, 78, 104, 1722, 0, 0, 5, 0, 65, 2, 5679, 2, 0, 0, 0, 0, 8, 1661, 0, 29, 2, 0, 710, 92, 30, 0, 4819, 679, 18, 12, 1, 0, 56, 1, 0, 0, 0, 0, 8, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[2053, 0, 7423, 8, 0, 0, 0, 4660, 0, 0, 856, 1561, 6549, 5666, 446, 15, 0, 0, 0, 315, 0, 3, 2436, 555, 0, 67, 116, 1930, 0, 0, 1, 0, 48, 0, 5749, 1, 0, 0, 0, 0, 0, 1883, 0, 25, 6, 0, 683, 98, 33, 0, 5282, 604, 19, 2, 0, 0, 53, 2, 0, 0, 4, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1813, 0, 7414, 12, 0, 0, 0, 5167, 0, 0, 830, 1646, 6795, 6135, 458, 11, 0, 0, 0, 275, 0, 6, 2465, 657, 0, 64, 94, 1409, 0, 1, 7, 0, 69, 0, 5566, 3, 0, 0, 0, 0, 9, 2225, 0, 44, 4, 0, 541, 79, 14, 1, 4773, 505, 0, 1, 0, 0, 52, 3, 0, 0, 4, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1996, 0, 7130, 11, 0, 0, 2, 4400, 0, 2, 993, 1657, 6929, 5975, 432, 10, 0, 0, 0, 576, 0, 8, 2270, 739, 0, 87, 111, 1825, 0, 1, 0, 0, 76, 1, 5574, 1, 0, 1, 0, 0, 1, 1858, 0, 48, 10, 0, 773, 82, 19, 0, 4862, 577, 33, 18, 0, 0, 51, 6, 1, 0, 6, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[2114, 0, 7028, 12, 0, 0, 1, 4513, 0, 0, 975, 1956, 6747, 5902, 464, 11, 0, 0, 2, 516, 0, 16, 1854, 897, 0, 103, 137, 2146, 0, 0, 0, 0, 40, 7, 5397, 7, 0, 0, 0, 0, 1, 1996, 0, 52, 8, 0, 689, 109, 26, 0, 4670, 479, 148, 22, 0, 0, 93, 7, 0, 0, 7, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1668, 0, 7349, 12, 0, 0, 0, 4688, 0, 1, 987, 1804, 7035, 5944, 473, 22, 0, 0, 0, 210, 1, 16, 2068, 792, 0, 72, 95, 1788, 0, 0, 3, 0, 84, 8, 5127, 6, 1, 0, 0, 1, 8, 1717, 0, 60, 5, 0, 944, 101, 42, 0, 5322, 547, 73, 6, 0, 0, 60, 1, 1, 0, 8, 0, 2, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2090, 0, 0, 6, 0, 0, 0, 3854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1821, 0, 2997, 396, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 438, 0, 12, 2094, 658, 2269, 646, 0, 0, 0, 0, 0, 0, 0, 110, 124, 2082, 0, 1, 2, 0, 2925, 1276, 0, 0, 0, 0, 0, 0, 19, 0, 4508, 3, 1, 0, 0, 0, 683, 0, 0, 0, 0, 0, 0, 0, 9, 2115, 0, 40, 4, 1, 795, 100, 3460, 0, 0, 0, 0, 0, 0, 0, 33, 0, 5235, 485, 14, 12, 0, 0, 448, 84, 0, 0, 0, 0, 0, 0, 31, 9, 0, 0, 8, 0, 1, 0, 6216, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5950, 6251, 6117, 6520, 5214, 6524, 6311, 6265] avg=6144.0000 max=6524 max_rank=5 min=5214 min_rank=4 imbalance_ratio=1.061849 max_load_reduction=42628 max_load_reduction_pct=0.867269 moved_to_echo=24314 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1971, 0, 0, 11, 0, 0, 0, 4202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 909, 1701, 0, 3097, 463, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 0, 5, 1917, 669, 2346, 618, 0, 0, 0, 0, 0, 0, 0, 82, 115, 2106, 0, 0, 3, 0, 2799, 1238, 0, 0, 0, 0, 0, 0, 74, 1, 5596, 2, 0, 0, 0, 0, 706, 0, 0, 0, 0, 0, 0, 0, 4, 2019, 0, 19, 6, 0, 701, 92, 3312, 0, 0, 0, 0, 0, 0, 0, 16, 0, 4597, 527, 29, 13, 0, 0, 489, 87, 0, 0, 0, 0, 0, 0, 51, 3, 0, 0, 2, 0, 2, 0, 6031, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6184, 6182, 6064, 6343, 6379, 6153, 5758, 6089] avg=6144.0000 max=6379 max_rank=4 min=5758 min_rank=6 imbalance_ratio=1.038249 max_load_reduction=42773 max_load_reduction_pct=0.870219 moved_to_echo=24379 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2043, 1, 0, 7, 1, 0, 0, 4282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1132, 1765, 0, 3007, 500, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 396, 0, 5, 2054, 684, 2279, 591, 0, 0, 0, 0, 0, 0, 0, 78, 104, 1722, 0, 0, 5, 0, 2679, 1279, 0, 0, 0, 0, 0, 0, 65, 2, 5679, 2, 0, 0, 0, 0, 686, 0, 0, 0, 0, 0, 0, 0, 8, 1661, 0, 29, 2, 0, 710, 92, 3164, 0, 0, 0, 0, 0, 0, 0, 30, 0, 4819, 679, 18, 12, 1, 0, 498, 85, 0, 0, 0, 0, 0, 0, 56, 1, 0, 0, 0, 0, 8, 2, 6221, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6334, 6409, 6012, 5867, 6434, 5666, 6142, 6288] avg=6144.0000 max=6434 max_rank=4 min=5666 min_rank=5 imbalance_ratio=1.047201 max_load_reduction=42718 max_load_reduction_pct=0.869100 moved_to_echo=24530 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1996, 0, 0, 11, 0, 0, 2, 3941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 993, 1657, 0, 2967, 432, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 576, 0, 8, 2270, 739, 2248, 637, 0, 0, 0, 0, 0, 0, 0, 87, 111, 1825, 0, 1, 0, 0, 2883, 1215, 0, 0, 0, 0, 0, 0, 76, 1, 5574, 1, 0, 1, 0, 0, 677, 0, 0, 0, 0, 0, 0, 0, 1, 1858, 0, 48, 10, 0, 773, 82, 3409, 0, 0, 0, 0, 0, 0, 0, 19, 0, 4862, 577, 33, 18, 0, 0, 459, 83, 0, 0, 0, 0, 0, 0, 51, 6, 1, 0, 6, 0, 0, 0, 5915, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5950, 6061, 6478, 6122, 6330, 6181, 6051, 5979] avg=6144.0000 max=6478 max_rank=2 min=5950 min_rank=0 imbalance_ratio=1.054362 max_load_reduction=42674 max_load_reduction_pct=0.868205 moved_to_echo=24541 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2053, 0, 0, 8, 0, 0, 0, 4174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 856, 1561, 0, 2814, 446, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 315, 0, 3, 2436, 555, 2131, 603, 0, 0, 0, 0, 0, 0, 0, 67, 116, 1930, 0, 0, 1, 0, 2725, 1266, 0, 0, 0, 0, 0, 0, 48, 0, 5749, 1, 0, 0, 0, 0, 642, 0, 0, 0, 0, 0, 0, 0, 0, 1883, 0, 25, 6, 0, 683, 98, 3221, 0, 0, 0, 0, 0, 0, 0, 33, 0, 5282, 604, 19, 2, 0, 0, 486, 79, 0, 0, 0, 0, 0, 0, 53, 2, 0, 0, 4, 0, 0, 0, 6157, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6235, 5692, 6043, 6105, 6440, 5916, 6505, 6216] avg=6144.0000 max=6505 max_rank=6 min=5692 min_rank=1 imbalance_ratio=1.058757 max_load_reduction=42647 max_load_reduction_pct=0.867655 moved_to_echo=25077 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1668, 0, 0, 12, 0, 0, 0, 4196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 987, 1804, 0, 2941, 473, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 210, 1, 16, 2068, 792, 2240, 648, 0, 0, 0, 0, 0, 0, 0, 72, 95, 1788, 0, 0, 3, 0, 2927, 1253, 0, 0, 0, 0, 0, 0, 84, 8, 5127, 6, 1, 0, 0, 1, 676, 0, 0, 0, 0, 0, 0, 0, 8, 1717, 0, 60, 5, 0, 944, 101, 3460, 0, 0, 0, 0, 0, 0, 0, 42, 0, 5322, 547, 73, 6, 0, 0, 492, 87, 0, 0, 0, 0, 0, 0, 60, 1, 1, 0, 8, 0, 2, 0, 6096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5876, 6228, 5975, 6138, 5903, 6295, 6569, 6168] avg=6144.0000 max=6569 max_rank=6 min=5876 min_rank=0 imbalance_ratio=1.069173 max_load_reduction=42583 max_load_reduction_pct=0.866353 moved_to_echo=24935 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2114, 0, 0, 12, 0, 0, 1, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 975, 1956, 0, 2932, 464, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 516, 0, 16, 1854, 897, 2220, 622, 0, 0, 0, 0, 0, 0, 0, 103, 137, 2146, 0, 0, 0, 0, 2807, 1198, 0, 0, 0, 0, 0, 0, 40, 7, 5397, 7, 0, 0, 0, 0, 668, 0, 0, 0, 0, 0, 0, 0, 1, 1996, 0, 52, 8, 0, 689, 109, 3318, 0, 0, 0, 0, 0, 0, 0, 26, 0, 4670, 479, 148, 22, 0, 0, 470, 82, 0, 0, 0, 0, 0, 0, 93, 7, 0, 0, 7, 0, 0, 0, 5830, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6170, 6338, 6127, 6391, 6119, 6173, 5897, 5937] avg=6144.0000 max=6391 max_rank=3 min=5897 min_rank=6 imbalance_ratio=1.040202 max_load_reduction=42761 max_load_reduction_pct=0.869975 moved_to_echo=24126 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1813, 0, 0, 12, 0, 0, 0, 4628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 830, 1646, 0, 3046, 458, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 275, 0, 6, 2465, 657, 2308, 625, 0, 0, 0, 0, 0, 0, 0, 64, 94, 1409, 0, 1, 7, 0, 2828, 1263, 0, 0, 0, 0, 0, 0, 69, 0, 5566, 3, 0, 0, 0, 0, 695, 0, 0, 0, 0, 0, 0, 0, 9, 2225, 0, 44, 4, 0, 541, 79, 3342, 0, 0, 0, 0, 0, 0, 0, 14, 1, 4773, 505, 0, 1, 0, 0, 539, 86, 0, 0, 0, 0, 0, 0, 52, 3, 0, 0, 4, 0, 0, 0, 6151, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6453, 5991, 6336, 5666, 6333, 6244, 5919, 6210] avg=6144.0000 max=6453 max_rank=0 min=5666 min_rank=3 imbalance_ratio=1.050293 max_load_reduction=42699 max_load_reduction_pct=0.868713 moved_to_echo=24706 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 768, 896, 15104, 0, 128, 128, 0, 22656, 10112, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 0, 128, 3328, 128, 128, 17280, 5760, 18048, 4992, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 128, 7808, 13952, 0, 23808, 3712, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[15872, 128, 0, 128, 128, 0, 128, 33408, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 15488, 0, 384, 128, 128, 5888, 768, 26752, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[256, 128, 39680, 4480, 384, 128, 128, 0, 3968, 768, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[512, 128, 43264, 128, 128, 128, 0, 128, 5504, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[512, 128, 128, 0, 128, 0, 128, 128, 48640, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1468, 499, 0, 112, 0, 0, 0, 0, 2, 73, 7675, 1, 1, 2365, 7264, 94, 0, 0, 0, 0, 1029, 54, 0, 8, 33, 442, 22, 1343, 0, 8189, 1, 49, 2018, 0, 1210, 0, 4682, 0, 56, 0, 410, 2764, 0, 5, 0, 177, 0, 0, 190, 0, 2, 9, 85, 5, 5561, 1, 7, 814, 0, 0, 0, 0, 409, 23] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1007, 439, 0, 64, 0, 0, 0, 3, 0, 67, 7687, 0, 2, 2339, 7324, 32, 0, 0, 2, 0, 1147, 72, 0, 4, 52, 308, 14, 1647, 3, 8181, 1, 40, 2303, 1, 1397, 0, 5297, 1, 33, 0, 396, 1939, 2, 1, 5, 151, 0, 0, 195, 2, 2, 16, 76, 15, 5667, 0, 12, 897, 0, 1, 0, 1, 288, 19] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1174, 447, 0, 50, 0, 0, 0, 0, 1, 67, 7820, 0, 0, 2338, 7234, 45, 0, 0, 0, 0, 1047, 53, 0, 4, 37, 254, 18, 1214, 0, 8192, 2, 41, 2645, 0, 1173, 0, 5192, 0, 55, 0, 414, 2644, 0, 0, 1, 142, 0, 0, 153, 0, 4, 15, 54, 3, 5404, 0, 7, 885, 0, 0, 0, 0, 292, 31] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1202, 456, 0, 62, 1, 0, 0, 0, 5, 52, 7774, 0, 1, 2177, 7421, 96, 0, 0, 0, 0, 1280, 52, 0, 14, 31, 278, 5, 932, 0, 8189, 0, 12, 2453, 1, 940, 0, 5365, 0, 39, 0, 384, 2933, 0, 2, 1, 184, 0, 0, 167, 0, 7, 13, 52, 0, 5520, 0, 12, 668, 0, 0, 0, 0, 337, 34] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1447, 542, 0, 62, 0, 0, 0, 1, 2, 81, 7671, 0, 0, 2158, 7379, 189, 1, 0, 1, 0, 1157, 68, 0, 10, 66, 458, 20, 1373, 0, 8170, 5, 27, 2086, 1, 1042, 0, 5214, 0, 23, 0, 421, 2396, 0, 0, 27, 197, 0, 0, 174, 0, 5, 15, 81, 8, 5347, 0, 9, 807, 0, 0, 0, 0, 353, 58] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1223, 392, 0, 54, 0, 0, 0, 2, 1, 72, 7818, 0, 0, 2436, 7208, 92, 0, 1, 0, 0, 1229, 33, 0, 2, 29, 323, 7, 1436, 1, 8191, 5, 11, 2723, 1, 983, 0, 5567, 0, 20, 0, 387, 2149, 0, 1, 0, 122, 0, 0, 122, 0, 2, 12, 80, 6, 5347, 0, 4, 855, 0, 0, 0, 0, 179, 26] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1193, 535, 0, 37, 0, 0, 0, 0, 2, 123, 7694, 2, 0, 2568, 7193, 64, 0, 0, 0, 0, 985, 57, 0, 2, 91, 308, 14, 1253, 0, 8188, 2, 20, 2301, 3, 1062, 0, 5055, 0, 44, 0, 397, 3051, 0, 1, 0, 202, 0, 0, 132, 0, 3, 12, 75, 11, 5196, 0, 12, 824, 0, 0, 1, 0, 405, 34] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1292, 487, 0, 45, 0, 0, 0, 0, 0, 71, 7872, 0, 0, 2003, 7328, 65, 0, 0, 0, 0, 992, 48, 0, 1, 38, 287, 5, 1643, 0, 8192, 1, 9, 2922, 0, 1033, 0, 5454, 0, 29, 0, 371, 2084, 0, 0, 0, 182, 0, 0, 103, 0, 2, 14, 65, 0, 5382, 0, 0, 861, 0, 0, 0, 0, 239, 32] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1468, 499, 0, 112, 0, 0, 0, 0, 2714, 1631, 0, 0, 0, 0, 0, 0, 2, 73, 0, 1, 1, 2365, 3669, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 54, 0, 8, 4925, 0, 0, 0, 0, 0, 0, 0, 33, 442, 22, 1343, 0, 4370, 1, 49, 0, 0, 0, 0, 0, 0, 0, 0, 2018, 0, 1210, 0, 2302, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 410, 2764, 0, 5, 0, 177, 0, 0, 1962, 881, 0, 0, 0, 0, 0, 0, 190, 0, 2, 9, 85, 5, 5561, 1, 418, 0, 0, 0, 0, 0, 0, 0, 7, 814, 0, 0, 0, 0, 409, 23, 2750, 2188, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6424, 6205, 6016, 6260, 5586, 6199, 6271, 6191] avg=6144.0000 max=6424 max_rank=0 min=5586 min_rank=4 imbalance_ratio=1.045573 max_load_reduction=42728 max_load_reduction_pct=0.869303 moved_to_echo=24247 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1202, 456, 0, 62, 1, 0, 0, 0, 2773, 1631, 0, 0, 0, 0, 0, 0, 5, 52, 0, 0, 1, 2177, 3748, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1280, 52, 0, 14, 4989, 0, 0, 0, 0, 0, 0, 0, 31, 278, 5, 932, 0, 4370, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 2453, 1, 940, 0, 2638, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 384, 2933, 0, 2, 1, 184, 0, 0, 2248, 900, 0, 0, 0, 0, 0, 0, 167, 0, 7, 13, 52, 0, 5520, 0, 479, 0, 0, 0, 0, 0, 0, 0, 12, 668, 0, 0, 0, 0, 337, 34, 2785, 2188, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6125, 6079, 6335, 5628, 6071, 6652, 6238, 6024] avg=6144.0000 max=6652 max_rank=5 min=5628 min_rank=3 imbalance_ratio=1.082682 max_load_reduction=42500 max_load_reduction_pct=0.864665 moved_to_echo=24985 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1174, 447, 0, 50, 0, 0, 0, 0, 2703, 1631, 0, 0, 0, 0, 0, 0, 1, 67, 0, 0, 0, 2338, 3654, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 53, 0, 4, 5018, 0, 0, 0, 0, 0, 0, 0, 37, 254, 18, 1214, 0, 4372, 2, 41, 0, 0, 0, 0, 0, 0, 0, 0, 2645, 0, 1173, 0, 2552, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 414, 2644, 0, 0, 1, 142, 0, 0, 2176, 877, 0, 0, 0, 0, 0, 0, 153, 0, 4, 15, 54, 3, 5404, 0, 464, 0, 0, 0, 0, 0, 0, 0, 7, 885, 0, 0, 0, 0, 292, 31, 2802, 2189, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6005, 6105, 6122, 5938, 6425, 6254, 6097, 6206] avg=6144.0000 max=6425 max_rank=4 min=5938 min_rank=3 imbalance_ratio=1.045736 max_load_reduction=42727 max_load_reduction_pct=0.869283 moved_to_echo=24982 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1007, 439, 0, 64, 0, 0, 0, 3, 2737, 1629, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 2, 2339, 3699, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1147, 72, 0, 4, 4934, 0, 0, 0, 0, 0, 0, 0, 52, 308, 14, 1647, 3, 4366, 1, 40, 0, 0, 0, 0, 0, 0, 0, 0, 2303, 1, 1397, 0, 2604, 1, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 396, 1939, 2, 1, 5, 151, 0, 0, 2220, 888, 0, 0, 0, 0, 0, 0, 195, 2, 2, 16, 76, 15, 5667, 0, 473, 0, 0, 0, 0, 0, 0, 0, 12, 897, 0, 1, 0, 1, 288, 19, 2753, 2186, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5879, 6139, 6159, 6431, 6339, 5602, 6446, 6157] avg=6144.0000 max=6446 max_rank=6 min=5602 min_rank=5 imbalance_ratio=1.049154 max_load_reduction=42706 max_load_reduction_pct=0.868856 moved_to_echo=24544 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1447, 542, 0, 62, 0, 0, 0, 1, 2762, 1632, 0, 0, 0, 0, 0, 0, 2, 81, 0, 0, 0, 2158, 3720, 189, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1157, 68, 0, 10, 4924, 0, 0, 0, 0, 0, 0, 0, 66, 458, 20, 1373, 0, 4351, 5, 27, 0, 0, 0, 0, 0, 0, 0, 0, 2086, 1, 1042, 0, 2555, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 421, 2396, 0, 0, 27, 197, 0, 0, 2189, 897, 0, 0, 0, 0, 0, 0, 174, 0, 5, 15, 81, 8, 5347, 0, 470, 0, 0, 0, 0, 0, 0, 0, 9, 807, 0, 0, 0, 0, 353, 58, 2747, 2187, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6446, 6150, 6161, 6300, 5707, 6127, 6100, 6161] avg=6144.0000 max=6446 max_rank=0 min=5707 min_rank=4 imbalance_ratio=1.049154 max_load_reduction=42706 max_load_reduction_pct=0.868856 moved_to_echo=24095 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1193, 535, 0, 37, 0, 0, 0, 0, 2688, 1631, 0, 0, 0, 0, 0, 0, 2, 123, 0, 2, 0, 2568, 3633, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 985, 57, 0, 2, 4939, 0, 0, 0, 0, 0, 0, 0, 91, 308, 14, 1253, 0, 4369, 2, 20, 0, 0, 0, 0, 0, 0, 0, 0, 2301, 3, 1062, 0, 2485, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 397, 3051, 0, 1, 0, 202, 0, 0, 2118, 872, 0, 0, 0, 0, 0, 0, 132, 0, 3, 12, 75, 11, 5196, 0, 452, 0, 0, 0, 0, 0, 0, 0, 12, 824, 0, 0, 1, 0, 405, 34, 2755, 2188, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6084, 6392, 5983, 6057, 5895, 6641, 5881, 6219] avg=6144.0000 max=6641 max_rank=5 min=5881 min_rank=6 imbalance_ratio=1.080892 max_load_reduction=42511 max_load_reduction_pct=0.864889 moved_to_echo=24636 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1292, 487, 0, 45, 0, 0, 0, 0, 2738, 1631, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 2003, 3702, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 992, 48, 0, 1, 5053, 0, 0, 0, 0, 0, 0, 0, 38, 287, 5, 1643, 0, 4372, 1, 9, 0, 0, 0, 0, 0, 0, 0, 0, 2922, 0, 1033, 0, 2681, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 371, 2084, 0, 0, 0, 182, 0, 0, 2286, 888, 0, 0, 0, 0, 0, 0, 103, 0, 2, 14, 65, 0, 5382, 0, 487, 0, 0, 0, 0, 0, 0, 0, 0, 861, 0, 0, 0, 0, 239, 32, 2819, 2189, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6193, 5841, 6094, 6355, 6665, 5811, 6053, 6140] avg=6144.0000 max=6665 max_rank=4 min=5811 min_rank=5 imbalance_ratio=1.084798 max_load_reduction=42487 max_load_reduction_pct=0.864400 moved_to_echo=24669 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1223, 392, 0, 54, 0, 0, 0, 2, 2694, 1631, 0, 0, 0, 0, 0, 0, 1, 72, 0, 0, 0, 2436, 3640, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1229, 33, 0, 2, 5017, 0, 0, 0, 0, 0, 0, 0, 29, 323, 7, 1436, 1, 4371, 5, 11, 0, 0, 0, 0, 0, 0, 0, 0, 2723, 1, 983, 0, 2737, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 387, 2149, 0, 1, 0, 122, 0, 0, 2333, 874, 0, 0, 0, 0, 0, 0, 122, 0, 2, 12, 80, 6, 5347, 0, 497, 0, 0, 0, 0, 0, 0, 0, 4, 855, 0, 0, 0, 0, 179, 26, 2801, 2189, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5996, 6241, 6282, 6183, 6464, 5866, 6066, 6054] avg=6144.0000 max=6464 max_rank=4 min=5866 min_rank=5 imbalance_ratio=1.052083 max_load_reduction=42688 max_load_reduction_pct=0.868490 moved_to_echo=24450 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[10112, 3840, 0, 512, 128, 0, 0, 128, 21888, 13056, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3200, 19968, 128, 128, 128, 1408, 0, 0, 17536, 7168, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[384, 2688, 128, 10880, 128, 34944, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 640, 0, 128, 128, 18432, 29568, 768, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[19456, 128, 8960, 0, 20608, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 8960, 512, 0, 128, 39808, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 6656, 0, 128, 128, 128, 2560, 384, 22272, 17536, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1280, 128, 128, 128, 640, 128, 43520, 128, 3840, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[3703, 679, 0, 195, 2, 0, 375, 0, 0, 0, 0, 1899, 0, 0, 3, 222, 0, 190, 1, 0, 453, 120, 0, 0, 0, 0, 841, 0, 0, 2364, 2617, 2, 0, 940, 0, 0, 0, 0, 0, 0, 0, 1923, 3, 1, 7858, 0, 0, 4953, 2, 0, 0, 0, 2458, 8, 750, 1, 53, 837, 0, 104, 0, 2, 7969, 7624] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[3650, 803, 0, 183, 8, 0, 368, 0, 0, 0, 0, 2237, 0, 1, 4, 126, 0, 151, 1, 0, 433, 105, 0, 2, 0, 0, 813, 0, 0, 2576, 2654, 2, 1, 1098, 0, 1, 3, 0, 0, 0, 0, 1876, 4, 2, 7760, 0, 0, 4907, 1, 0, 0, 0, 2097, 1, 673, 1, 52, 881, 0, 87, 0, 1, 8051, 7538] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[3256, 741, 0, 206, 14, 1, 477, 0, 0, 0, 0, 2054, 0, 0, 4, 143, 0, 101, 0, 1, 466, 124, 0, 0, 0, 0, 835, 0, 0, 2558, 2646, 2, 0, 961, 0, 0, 0, 0, 0, 0, 0, 2186, 2, 1, 7897, 0, 0, 4506, 4, 0, 1, 0, 2685, 19, 593, 1, 139, 906, 0, 132, 0, 0, 7937, 7553] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[3413, 698, 0, 205, 15, 0, 396, 0, 0, 0, 0, 2035, 0, 1, 3, 155, 1, 187, 1, 0, 427, 134, 1, 0, 0, 0, 878, 0, 2, 2453, 2525, 19, 0, 903, 0, 0, 0, 1, 0, 0, 0, 1818, 11, 0, 7848, 0, 0, 4842, 1, 1, 0, 0, 3048, 8, 506, 0, 39, 947, 0, 119, 1, 10, 7933, 7567] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[3325, 725, 0, 208, 2, 0, 389, 0, 0, 0, 0, 1953, 0, 0, 0, 183, 0, 168, 0, 0, 243, 125, 0, 0, 0, 0, 813, 0, 0, 2624, 2747, 0, 0, 1150, 0, 0, 0, 0, 0, 0, 0, 1885, 0, 0, 7866, 0, 0, 5097, 1, 0, 0, 0, 2240, 3, 564, 0, 56, 923, 0, 108, 0, 6, 8014, 7734] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[3783, 662, 0, 200, 11, 0, 403, 0, 0, 0, 0, 2180, 0, 0, 1, 175, 0, 132, 2, 0, 495, 120, 0, 1, 0, 0, 861, 0, 0, 2657, 2331, 8, 0, 970, 0, 0, 0, 1, 0, 0, 0, 1921, 2, 0, 7696, 0, 0, 4603, 2, 0, 0, 1, 2594, 12, 639, 0, 112, 1131, 0, 100, 0, 0, 7918, 7428] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[3392, 739, 0, 245, 18, 0, 531, 0, 0, 0, 0, 1969, 0, 0, 4, 169, 0, 136, 1, 0, 451, 144, 0, 0, 0, 0, 953, 0, 0, 2520, 2596, 18, 1, 889, 1, 0, 0, 5, 0, 0, 0, 1428, 3, 0, 7821, 0, 0, 5057, 1, 0, 0, 2, 3023, 1, 520, 0, 97, 914, 0, 106, 0, 6, 7893, 7498] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[3257, 775, 0, 223, 8, 1, 378, 0, 0, 0, 0, 1878, 0, 0, 3, 160, 0, 193, 0, 0, 289, 161, 0, 0, 0, 0, 829, 0, 0, 2432, 2465, 2, 0, 1183, 0, 1, 0, 0, 0, 0, 0, 1967, 4, 1, 7919, 0, 0, 5035, 2, 0, 0, 0, 2493, 2, 609, 1, 66, 1072, 0, 100, 0, 2, 7949, 7692] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3650, 803, 0, 183, 8, 0, 368, 0, 884, 424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2237, 0, 1, 4, 126, 3911, 0, 0, 0, 0, 0, 0, 0, 0, 151, 1, 0, 433, 105, 0, 2, 5512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 813, 0, 0, 2576, 2654, 2, 189, 0, 0, 0, 0, 0, 0, 0, 1, 1098, 0, 1, 3, 0, 0, 0, 2596, 2539, 0, 0, 0, 0, 0, 0, 0, 1876, 4, 2, 0, 0, 0, 4294, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2097, 1, 673, 1, 1675, 1253, 0, 0, 0, 0, 0, 0, 52, 881, 0, 87, 0, 1, 0, 4979, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6320, 6279, 6204, 6234, 6238, 6176, 5701, 6000] avg=6144.0000 max=6320 max_rank=0 min=5701 min_rank=6 imbalance_ratio=1.028646 max_load_reduction=42832 max_load_reduction_pct=0.871419 moved_to_echo=24115 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3256, 741, 0, 206, 14, 1, 477, 0, 885, 390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2054, 0, 0, 4, 143, 3980, 0, 0, 0, 0, 0, 0, 0, 0, 101, 0, 1, 466, 124, 0, 0, 5434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 835, 0, 0, 2558, 2646, 2, 174, 0, 0, 0, 0, 0, 0, 0, 0, 961, 0, 0, 0, 0, 0, 0, 2643, 2503, 0, 0, 0, 0, 0, 0, 0, 2186, 2, 1, 0, 0, 0, 3942, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 2685, 19, 593, 1, 1679, 1274, 0, 0, 0, 0, 0, 0, 139, 906, 0, 132, 0, 0, 0, 4989, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5970, 6181, 6126, 6215, 6107, 6131, 6256, 6166] avg=6144.0000 max=6256 max_rank=6 min=5970 min_rank=0 imbalance_ratio=1.018229 max_load_reduction=42896 max_load_reduction_pct=0.872721 moved_to_echo=24660 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3703, 679, 0, 195, 2, 0, 375, 0, 894, 428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1899, 0, 0, 3, 222, 3961, 0, 0, 0, 0, 0, 0, 0, 0, 190, 1, 0, 453, 120, 0, 0, 5456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 841, 0, 0, 2364, 2617, 2, 191, 0, 0, 0, 0, 0, 0, 0, 0, 940, 0, 0, 0, 0, 0, 0, 2630, 2513, 0, 0, 0, 0, 0, 0, 0, 1923, 3, 1, 0, 0, 0, 4334, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2458, 8, 750, 1, 1694, 1267, 0, 0, 0, 0, 0, 0, 53, 837, 0, 104, 0, 2, 0, 5036, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6276, 6085, 6220, 6015, 6083, 6261, 6180, 6032] avg=6144.0000 max=6276 max_rank=0 min=6015 min_rank=3 imbalance_ratio=1.021484 max_load_reduction=42876 max_load_reduction_pct=0.872314 moved_to_echo=24556 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3413, 698, 0, 205, 15, 0, 396, 0, 887, 419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2035, 0, 1, 3, 155, 3956, 0, 0, 0, 0, 0, 0, 0, 1, 187, 1, 0, 427, 134, 1, 0, 5432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 878, 0, 2, 2453, 2525, 19, 187, 0, 0, 0, 0, 0, 0, 0, 0, 903, 0, 0, 0, 1, 0, 0, 2627, 2501, 0, 0, 0, 0, 0, 0, 0, 1818, 11, 0, 0, 0, 0, 4236, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 3048, 8, 506, 0, 1682, 1265, 0, 0, 0, 0, 0, 0, 39, 947, 0, 119, 1, 10, 0, 4998, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6033, 6150, 6183, 6064, 6032, 6065, 6511, 6114] avg=6144.0000 max=6511 max_rank=6 min=6032 min_rank=4 imbalance_ratio=1.059733 max_load_reduction=42641 max_load_reduction_pct=0.867533 moved_to_echo=24722 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3325, 725, 0, 208, 2, 0, 389, 0, 907, 441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1953, 0, 0, 0, 183, 3966, 0, 0, 0, 0, 0, 0, 0, 0, 168, 0, 0, 243, 125, 0, 0, 5488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 813, 0, 0, 2624, 2747, 0, 197, 0, 0, 0, 0, 0, 0, 0, 0, 1150, 0, 0, 0, 0, 0, 0, 2632, 2526, 0, 0, 0, 0, 0, 0, 0, 1885, 0, 0, 0, 0, 0, 4459, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2240, 3, 564, 0, 1719, 1268, 0, 0, 0, 0, 0, 0, 56, 923, 0, 108, 0, 6, 0, 5108, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5997, 6102, 6024, 6381, 6308, 6344, 5795, 6201] avg=6144.0000 max=6381 max_rank=3 min=5795 min_rank=6 imbalance_ratio=1.038574 max_load_reduction=42771 max_load_reduction_pct=0.870178 moved_to_echo=24648 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3392, 739, 0, 245, 18, 0, 531, 0, 882, 442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1969, 0, 0, 4, 169, 3943, 0, 0, 0, 0, 0, 0, 0, 0, 136, 1, 0, 451, 144, 0, 0, 5405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 953, 0, 0, 2520, 2596, 18, 199, 0, 0, 0, 0, 0, 0, 0, 1, 889, 1, 0, 0, 5, 0, 0, 2617, 2488, 0, 0, 0, 0, 0, 0, 0, 1428, 3, 0, 0, 0, 0, 4416, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 3023, 1, 520, 0, 1670, 1261, 0, 0, 0, 0, 0, 0, 97, 914, 0, 106, 0, 6, 0, 4946, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6249, 6085, 6137, 6286, 6001, 5847, 6478, 6069] avg=6144.0000 max=6478 max_rank=6 min=5847 min_rank=5 imbalance_ratio=1.054362 max_load_reduction=42674 max_load_reduction_pct=0.868205 moved_to_echo=24395 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3783, 662, 0, 200, 11, 0, 403, 0, 871, 398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2180, 0, 0, 1, 175, 3881, 0, 0, 0, 0, 0, 0, 0, 0, 132, 2, 0, 495, 120, 0, 1, 5422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 861, 0, 0, 2657, 2331, 8, 178, 0, 0, 0, 0, 0, 0, 0, 0, 970, 0, 0, 0, 1, 0, 0, 2575, 2496, 0, 0, 0, 0, 0, 0, 0, 1921, 2, 0, 0, 0, 0, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 2594, 12, 639, 0, 1651, 1240, 0, 0, 0, 0, 0, 0, 112, 1131, 0, 100, 0, 0, 0, 4906, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6328, 6237, 6172, 6035, 6042, 5950, 6139, 6249] avg=6144.0000 max=6328 max_rank=0 min=5950 min_rank=5 imbalance_ratio=1.029948 max_load_reduction=42824 max_load_reduction_pct=0.871257 moved_to_echo=24380 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3257, 775, 0, 223, 8, 1, 378, 0, 902, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1878, 0, 0, 3, 160, 3992, 0, 0, 0, 0, 0, 0, 0, 0, 193, 0, 0, 289, 161, 0, 0, 5443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 829, 0, 0, 2432, 2465, 2, 194, 0, 0, 0, 0, 0, 0, 0, 0, 1183, 0, 1, 0, 0, 0, 0, 2651, 2506, 0, 0, 0, 0, 0, 0, 0, 1967, 4, 1, 0, 0, 0, 4406, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2493, 2, 609, 1, 1709, 1276, 0, 0, 0, 0, 0, 0, 66, 1072, 0, 100, 0, 2, 0, 5081, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5979, 6033, 6086, 5922, 6341, 6378, 6092, 6321] avg=6144.0000 max=6378 max_rank=5 min=5922 min_rank=3 imbalance_ratio=1.038086 max_load_reduction=42774 max_load_reduction_pct=0.870239 moved_to_echo=25132 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 8192, 128, 128, 128, 128, 0, 0, 20992, 20096, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[27904, 5888, 0, 1792, 128, 128, 3328, 0, 7168, 3456, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 6912, 0, 128, 20224, 20608, 128, 1536, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 20736, 128, 4864, 128, 13568, 10112, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 15104, 128, 128, 0, 0, 0, 34176, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 0, 16256, 0, 128, 128, 1408, 31616, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[640, 7680, 0, 896, 128, 128, 0, 40064, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 1280, 128, 128, 3328, 1152, 128, 128, 43648, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[957, 2499, 5729, 885, 0, 6197, 126, 0, 0, 0, 0, 0, 1, 19, 0, 0, 0, 1, 0, 3746, 0, 0, 0, 342, 3, 198, 2100, 2925, 8191, 0, 193, 1, 0, 0, 1, 0, 3, 0, 105, 0, 1219, 4180, 1, 0, 623, 0, 0, 0, 0, 5316, 0, 947, 0, 0, 0, 0, 0, 29, 1338, 0, 1273, 0, 0, 4] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1112, 2421, 5796, 540, 0, 6023, 229, 0, 0, 0, 0, 0, 1, 7, 0, 0, 0, 0, 0, 4289, 0, 0, 0, 347, 2, 221, 1767, 3317, 8192, 0, 182, 0, 0, 0, 0, 0, 2, 0, 35, 0, 1725, 4272, 0, 0, 555, 0, 0, 0, 0, 4658, 0, 722, 0, 0, 0, 0, 0, 55, 1621, 0, 1061, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[812, 2405, 5624, 779, 0, 6624, 245, 0, 0, 0, 0, 0, 1, 13, 0, 0, 0, 0, 0, 4033, 0, 0, 0, 371, 0, 122, 2009, 2960, 8192, 0, 124, 0, 0, 0, 6, 0, 1, 0, 69, 0, 1198, 4360, 0, 0, 543, 0, 0, 0, 4, 5034, 0, 1061, 0, 0, 0, 0, 0, 19, 1476, 0, 1035, 0, 0, 32] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[1182, 2436, 5770, 958, 0, 6125, 273, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3614, 0, 0, 0, 326, 1, 269, 2175, 3342, 8188, 0, 149, 0, 1, 0, 3, 0, 5, 0, 45, 1, 1607, 3140, 0, 0, 669, 2, 1, 2, 0, 5730, 0, 876, 0, 0, 1, 0, 1, 31, 1271, 0, 946, 0, 0, 9] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1061, 2585, 5817, 821, 0, 6313, 289, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 3698, 0, 0, 0, 376, 3, 253, 1929, 2761, 8192, 0, 161, 0, 0, 0, 0, 0, 1, 0, 45, 0, 1291, 4390, 0, 0, 612, 0, 0, 0, 0, 5159, 0, 862, 0, 0, 0, 0, 0, 62, 1537, 0, 932, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1079, 2641, 5180, 759, 0, 6462, 305, 0, 3, 0, 0, 0, 2, 0, 4, 0, 0, 0, 0, 3830, 0, 0, 0, 362, 9, 254, 1973, 2911, 8192, 0, 138, 0, 0, 0, 0, 0, 3, 2, 87, 0, 1566, 4178, 0, 0, 538, 0, 0, 0, 0, 5342, 0, 930, 0, 0, 0, 0, 0, 20, 1394, 0, 985, 0, 0, 3] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[1085, 2479, 5833, 766, 0, 6006, 355, 0, 1, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 3744, 0, 0, 0, 361, 1, 190, 2052, 3139, 8191, 0, 144, 1, 0, 0, 0, 0, 3, 0, 107, 0, 1222, 4690, 0, 0, 557, 0, 0, 0, 0, 5679, 0, 749, 0, 0, 0, 0, 0, 30, 993, 1, 741, 0, 0, 5] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[798, 2549, 5989, 1117, 0, 6822, 259, 0, 0, 0, 0, 0, 1, 14, 0, 0, 0, 0, 0, 3652, 0, 0, 0, 337, 7, 252, 2084, 3055, 8192, 0, 212, 0, 0, 0, 0, 0, 4, 0, 10, 0, 1426, 3971, 0, 0, 541, 0, 0, 0, 0, 4141, 0, 937, 0, 0, 0, 0, 0, 35, 1555, 0, 1174, 0, 0, 18] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[957, 2499, 1548, 885, 0, 0, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 19, 0, 0, 6131, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3746, 0, 0, 0, 342, 1969, 0, 0, 0, 0, 0, 0, 0, 3, 198, 2100, 2925, 696, 0, 193, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 105, 0, 4619, 1364, 0, 0, 0, 0, 0, 0, 1219, 4180, 1, 0, 623, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 5316, 0, 947, 0, 0, 0, 0, 125, 0, 0, 0, 0, 0, 0, 0, 0, 29, 1338, 0, 1273, 0, 0, 4, 2078, 1578, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6015, 6151, 6058, 6116, 6092, 6032, 6388, 6300] avg=6144.0000 max=6388 max_rank=6 min=6015 min_rank=0 imbalance_ratio=1.039714 max_load_reduction=42764 max_load_reduction_pct=0.870036 moved_to_echo=24812 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[812, 2405, 1520, 779, 0, 0, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 13, 0, 0, 6132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 0, 0, 371, 1933, 0, 0, 0, 0, 0, 0, 0, 0, 122, 2009, 2960, 696, 0, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1, 0, 69, 0, 4937, 1364, 0, 0, 0, 0, 0, 0, 1198, 4360, 0, 0, 543, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 4, 5034, 0, 1061, 0, 0, 0, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0, 19, 1476, 0, 1035, 0, 0, 32, 2040, 1687, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5761, 6146, 6337, 5911, 6377, 6109, 6222, 6289] avg=6144.0000 max=6377 max_rank=4 min=5761 min_rank=0 imbalance_ratio=1.037923 max_load_reduction=42775 max_load_reduction_pct=0.870260 moved_to_echo=24997 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1061, 2585, 1572, 821, 0, 0, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 6132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3698, 0, 0, 0, 376, 1999, 0, 0, 0, 0, 0, 0, 0, 3, 253, 1929, 2761, 696, 0, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 45, 0, 4706, 1364, 0, 0, 0, 0, 0, 0, 1291, 4390, 0, 0, 612, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 5159, 0, 862, 0, 0, 0, 0, 127, 0, 0, 0, 0, 0, 0, 0, 0, 62, 1537, 0, 932, 0, 0, 0, 2110, 1607, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6328, 6134, 6073, 5803, 6116, 6302, 6148, 6248] avg=6144.0000 max=6328 max_rank=0 min=5803 min_rank=3 imbalance_ratio=1.029948 max_load_reduction=42824 max_load_reduction_pct=0.871257 moved_to_echo=24814 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1182, 2436, 1559, 958, 0, 0, 273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 6129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3614, 0, 0, 0, 326, 1983, 0, 0, 0, 0, 0, 0, 0, 1, 269, 2175, 3342, 696, 0, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 5, 0, 45, 1, 4566, 1363, 0, 0, 0, 0, 0, 0, 1607, 3140, 0, 0, 669, 2, 1, 2, 9, 0, 0, 0, 0, 0, 0, 0, 0, 5730, 0, 876, 0, 0, 1, 0, 126, 0, 0, 0, 0, 0, 0, 0, 1, 31, 1271, 0, 946, 0, 0, 9, 2093, 1559, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6408, 6132, 5923, 6632, 5984, 5430, 6733, 5910] avg=6144.0000 max=6733 max_rank=6 min=5430 min_rank=5 imbalance_ratio=1.095866 max_load_reduction=42419 max_load_reduction_pct=0.863017 moved_to_echo=24057 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1112, 2421, 1566, 540, 0, 0, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7, 0, 0, 6132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4289, 0, 0, 0, 347, 1992, 0, 0, 0, 0, 0, 0, 0, 2, 221, 1767, 3317, 696, 0, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 35, 0, 4490, 1364, 0, 0, 0, 0, 0, 0, 1725, 4272, 0, 0, 555, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 4658, 0, 722, 0, 0, 0, 0, 127, 0, 0, 0, 0, 0, 0, 0, 0, 55, 1621, 0, 1061, 0, 0, 0, 2102, 1533, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5868, 6140, 6628, 6185, 5891, 6561, 5507, 6372] avg=6144.0000 max=6628 max_rank=2 min=5507 min_rank=6 imbalance_ratio=1.078776 max_load_reduction=42524 max_load_reduction_pct=0.865153 moved_to_echo=24331 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1079, 2641, 1386, 759, 0, 0, 305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 2, 0, 4, 0, 6134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3830, 0, 0, 0, 362, 1784, 0, 0, 0, 0, 0, 0, 0, 9, 254, 1973, 2911, 691, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 87, 0, 4818, 1367, 0, 0, 0, 0, 0, 0, 1566, 4178, 0, 0, 538, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 5342, 0, 930, 0, 0, 0, 0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 20, 1394, 0, 985, 0, 0, 3, 1882, 1644, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6170, 6143, 5976, 5976, 6277, 6293, 6389, 5928] avg=6144.0000 max=6389 max_rank=6 min=5928 min_rank=7 imbalance_ratio=1.039876 max_load_reduction=42763 max_load_reduction_pct=0.870015 moved_to_echo=24887 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[798, 2549, 1619, 1117, 0, 0, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 14, 0, 0, 6132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3652, 0, 0, 0, 337, 2058, 0, 0, 0, 0, 0, 0, 0, 7, 252, 2084, 3055, 696, 0, 212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 10, 0, 5086, 1364, 0, 0, 0, 0, 0, 0, 1426, 3971, 0, 0, 541, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 4141, 0, 937, 0, 0, 0, 0, 131, 0, 0, 0, 0, 0, 0, 0, 0, 35, 1555, 0, 1174, 0, 0, 18, 2172, 1736, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6342, 6147, 6047, 6306, 6464, 5947, 5209, 6690] avg=6144.0000 max=6690 max_rank=7 min=5209 min_rank=6 imbalance_ratio=1.088867 max_load_reduction=42462 max_load_reduction_pct=0.863892 moved_to_echo=24310 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1085, 2479, 1575, 766, 0, 0, 355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 27, 0, 0, 6131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3744, 0, 0, 0, 361, 2005, 0, 0, 0, 0, 0, 0, 0, 1, 190, 2052, 3139, 696, 0, 144, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 107, 0, 4477, 1364, 0, 0, 0, 0, 0, 0, 1222, 4690, 0, 0, 557, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 5679, 0, 749, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 30, 993, 1, 741, 0, 0, 5, 2116, 1529, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6260, 6159, 6110, 6223, 5951, 6478, 6556, 5415] avg=6144.0000 max=6556 max_rank=6 min=5415 min_rank=7 imbalance_ratio=1.067057 max_load_reduction=42596 max_load_reduction_pct=0.866618 moved_to_echo=24400 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[8192, 20096, 12416, 6656, 0, 0, 2176, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 384, 11264, 128, 8192, 0, 0, 128, 16640, 12928, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 1792, 16128, 24448, 5632, 0, 1408, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 128, 0, 30720, 0, 0, 0, 2944, 15744, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[11264, 33280, 128, 0, 4736, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 41088, 0, 7168, 0, 0, 128, 0, 1024, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 0, 128, 0, 128, 128, 512, 128, 37760, 11008, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 0, 0, 128, 128, 128, 128, 128, 49152, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 7667, 173, 2659, 0, 0, 369, 0, 7109, 1, 0, 0, 0, 124, 0, 1600, 15, 2, 0, 1, 0, 0, 0, 0, 0, 0, 2, 96, 0, 6217, 0, 0, 1, 0, 646, 0, 0, 0, 0, 847, 78, 8178, 2010, 0, 0, 7062, 0, 3004, 0, 128, 0, 0, 0, 379, 0, 0, 250, 359, 130, 45, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 7560, 42, 2471, 0, 0, 371, 0, 7154, 0, 0, 0, 0, 130, 0, 1617, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 102, 0, 6281, 0, 0, 0, 0, 617, 0, 0, 0, 0, 729, 52, 8192, 1871, 0, 0, 7019, 0, 3521, 0, 118, 0, 0, 0, 486, 0, 0, 235, 366, 122, 44, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 7554, 238, 2500, 0, 0, 451, 0, 7265, 0, 0, 0, 0, 157, 0, 1595, 44, 0, 0, 0, 0, 0, 1, 4, 0, 0, 6, 97, 0, 5899, 0, 0, 1, 1, 1107, 0, 0, 0, 0, 952, 102, 8192, 1618, 0, 0, 6562, 0, 3234, 0, 135, 0, 0, 0, 455, 0, 0, 349, 391, 152, 90, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 7498, 21, 2471, 0, 0, 385, 0, 6927, 0, 0, 0, 0, 145, 0, 1670, 13, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 76, 0, 6331, 0, 0, 0, 0, 659, 0, 0, 0, 0, 829, 90, 8192, 1991, 0, 0, 6997, 0, 3498, 0, 122, 0, 0, 0, 474, 0, 0, 381, 241, 70, 69, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 4, 7468, 7, 2632, 0, 2, 420, 0, 7122, 0, 0, 1, 0, 113, 0, 1451, 57, 1, 2, 0, 0, 1, 0, 0, 0, 0, 3, 79, 1, 5725, 1, 0, 1, 0, 864, 3, 1, 0, 0, 720, 140, 8187, 2192, 0, 0, 6725, 0, 3748, 0, 78, 0, 0, 0, 529, 0, 0, 321, 334, 123, 96, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 7569, 215, 2305, 0, 0, 379, 0, 7105, 0, 0, 0, 0, 191, 0, 1757, 74, 4, 0, 0, 0, 0, 0, 0, 0, 0, 5, 101, 0, 5586, 0, 0, 0, 0, 961, 0, 0, 0, 0, 920, 116, 8192, 1959, 0, 0, 6883, 0, 3266, 0, 128, 0, 0, 0, 356, 0, 0, 511, 383, 118, 68, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 7662, 19, 2617, 0, 0, 416, 0, 6922, 0, 0, 0, 0, 155, 0, 1325, 43, 0, 0, 0, 0, 0, 0, 5, 0, 1, 4, 127, 0, 5660, 1, 0, 9, 0, 714, 2, 0, 1, 3, 862, 299, 8187, 2211, 0, 0, 6678, 0, 3664, 0, 127, 0, 0, 0, 417, 0, 0, 478, 345, 131, 67, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 7526, 51, 2639, 0, 0, 389, 0, 7038, 0, 0, 0, 0, 121, 0, 1831, 23, 2, 0, 0, 0, 0, 0, 0, 0, 0, 7, 111, 0, 5830, 0, 0, 0, 0, 1149, 0, 0, 0, 0, 605, 85, 8191, 2160, 0, 0, 6970, 0, 3076, 0, 107, 0, 0, 0, 507, 0, 0, 292, 277, 105, 60, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3510, 42, 2471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 371, 0, 5663, 0, 0, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1617, 44, 0, 0, 0, 0, 0, 2891, 1670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 102, 0, 6281, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 617, 0, 0, 0, 5303, 0, 0, 0, 0, 0, 0, 0, 0, 729, 52, 0, 1871, 0, 0, 3288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3521, 0, 118, 0, 0, 0, 486, 1392, 840, 0, 0, 0, 0, 0, 0, 0, 0, 235, 366, 122, 44, 0, 0, 2889, 2380, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6023, 6164, 6222, 6490, 5920, 5940, 6357, 6036] avg=6144.0000 max=6490 max_rank=3 min=5920 min_rank=4 imbalance_ratio=1.056315 max_load_reduction=42662 max_load_reduction_pct=0.867961 moved_to_echo=24253 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3560, 173, 2659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 369, 0, 5628, 1, 0, 0, 0, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1600, 15, 2, 0, 1, 0, 0, 2909, 1693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 96, 0, 6217, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 646, 0, 0, 0, 5294, 0, 0, 0, 0, 0, 0, 0, 0, 847, 78, 0, 2010, 0, 0, 3308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3004, 0, 128, 0, 0, 0, 379, 1383, 845, 0, 0, 0, 0, 0, 0, 0, 0, 250, 359, 130, 45, 0, 0, 2884, 2414, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6392, 6122, 6220, 6413, 5941, 6243, 5739, 6082] avg=6144.0000 max=6413 max_rank=3 min=5739 min_rank=6 imbalance_ratio=1.043783 max_load_reduction=42739 max_load_reduction_pct=0.869527 moved_to_echo=24005 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3508, 238, 2500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 451, 0, 5752, 0, 0, 0, 0, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1595, 44, 0, 0, 0, 0, 0, 2703, 1668, 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 6, 97, 0, 5899, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1107, 0, 0, 0, 5303, 0, 0, 0, 0, 0, 0, 0, 0, 952, 102, 0, 1618, 0, 0, 3074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3234, 0, 135, 0, 0, 0, 455, 1413, 785, 0, 0, 0, 0, 0, 0, 0, 0, 349, 391, 152, 90, 0, 0, 2889, 2378, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6246, 6360, 6010, 6107, 6412, 5746, 6022, 6249] avg=6144.0000 max=6412 max_rank=4 min=5746 min_rank=5 imbalance_ratio=1.043620 max_load_reduction=42740 max_load_reduction_pct=0.869548 moved_to_echo=24429 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 3468, 7, 2632, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 420, 0, 5639, 0, 0, 1, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1451, 57, 1, 2, 0, 0, 1, 2770, 1649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 79, 1, 5725, 98, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 864, 3, 1, 0, 5301, 0, 0, 0, 0, 0, 0, 0, 0, 720, 140, 0, 2192, 0, 0, 3150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3748, 0, 78, 0, 0, 0, 529, 1385, 805, 0, 0, 0, 0, 0, 0, 0, 0, 321, 334, 123, 96, 0, 0, 2886, 2351, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6113, 6173, 5931, 5906, 6171, 6202, 6545, 6111] avg=6144.0000 max=6545 max_rank=6 min=5906 min_rank=3 imbalance_ratio=1.065267 max_load_reduction=42607 max_load_reduction_pct=0.866842 moved_to_echo=25029 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3482, 21, 2471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 385, 0, 5483, 0, 0, 0, 0, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1670, 13, 0, 0, 0, 0, 0, 2882, 1656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 76, 0, 6331, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 659, 0, 0, 0, 5304, 0, 0, 0, 0, 0, 0, 0, 0, 829, 90, 0, 1991, 0, 0, 3278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3498, 0, 122, 0, 0, 0, 474, 1348, 837, 0, 0, 0, 0, 0, 0, 0, 0, 381, 241, 70, 69, 0, 0, 2888, 2360, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5974, 6013, 6221, 6505, 5963, 6188, 6279, 6009] avg=6144.0000 max=6505 max_rank=3 min=5963 min_rank=4 imbalance_ratio=1.058757 max_load_reduction=42647 max_load_reduction_pct=0.867655 moved_to_echo=24439 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3549, 19, 2617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 416, 0, 5472, 0, 0, 0, 0, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1325, 43, 0, 0, 0, 0, 0, 2752, 1697, 0, 0, 0, 0, 0, 0, 0, 5, 0, 1, 4, 127, 0, 5660, 99, 0, 0, 0, 0, 0, 0, 0, 1, 0, 9, 0, 714, 2, 0, 1, 5301, 0, 0, 0, 0, 0, 0, 0, 3, 862, 299, 0, 2211, 0, 0, 3124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3664, 0, 127, 0, 0, 0, 417, 1351, 802, 0, 0, 0, 0, 0, 0, 0, 0, 478, 345, 131, 67, 0, 0, 2886, 2416, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6185, 6043, 5817, 5896, 6028, 6499, 6361, 6323] avg=6144.0000 max=6499 max_rank=5 min=5817 min_rank=2 imbalance_ratio=1.057780 max_load_reduction=42653 max_load_reduction_pct=0.867778 moved_to_echo=25211 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3514, 215, 2305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 379, 0, 5625, 0, 0, 0, 0, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1757, 74, 4, 0, 0, 0, 0, 2835, 1672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 101, 0, 5586, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 961, 0, 0, 0, 5304, 0, 0, 0, 0, 0, 0, 0, 0, 920, 116, 0, 1959, 0, 0, 3224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3266, 0, 128, 0, 0, 0, 356, 1382, 824, 0, 0, 0, 0, 0, 0, 0, 0, 511, 383, 118, 68, 0, 0, 2888, 2383, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6034, 6195, 6342, 5790, 6265, 6219, 5956, 6351] avg=6144.0000 max=6351 max_rank=7 min=5790 min_rank=3 imbalance_ratio=1.033691 max_load_reduction=42801 max_load_reduction_pct=0.870789 moved_to_echo=24791 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3495, 51, 2639, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 389, 0, 5572, 0, 0, 0, 0, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1831, 23, 2, 0, 0, 0, 0, 2871, 1662, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 111, 0, 5830, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1149, 0, 0, 0, 5303, 0, 0, 0, 0, 0, 0, 0, 0, 605, 85, 0, 2160, 0, 0, 3265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3076, 0, 107, 0, 0, 0, 507, 1369, 834, 0, 0, 0, 0, 0, 0, 0, 0, 292, 277, 105, 60, 0, 0, 2888, 2369, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6185, 6082, 6389, 6045, 6452, 6115, 5893, 5991] avg=6144.0000 max=6452 max_rank=4 min=5893 min_rank=6 imbalance_ratio=1.050130 max_load_reduction=42700 max_load_reduction_pct=0.868734 moved_to_echo=24451 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[0, 12928, 384, 128, 128, 128, 0, 128, 22656, 13440, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 128, 28160, 768, 20352, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 27136, 0, 1024, 0, 0, 0, 3712, 11136, 6656, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 6528, 1024, 0, 16128, 0, 0, 25728, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 2944, 2816, 1024, 640, 0, 0, 23168, 19072, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 0, 128, 128, 6784, 128, 128, 128, 42496, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[3200, 0, 44928, 128, 0, 128, 0, 1152, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 128, 0, 128, 128, 896, 128, 47616, 896, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 7940, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1919, 0, 6878, 0, 0, 0, 14, 469, 0, 701, 0, 1680, 0, 1, 0, 1302, 1877, 0, 6, 6685, 0, 0, 1052, 0, 0, 0, 271, 1, 5813, 0, 0, 429, 0, 0, 0, 0, 0, 0, 69, 0, 1628, 0, 753, 0, 0, 0, 0, 8154, 1496, 0, 0, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 7830, 9, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1980, 0, 6307, 0, 1, 0, 8, 413, 0, 1278, 2, 2171, 0, 0, 0, 1466, 2239, 0, 10, 6637, 0, 0, 672, 0, 0, 0, 312, 0, 5434, 0, 0, 442, 0, 0, 0, 0, 0, 0, 52, 0, 1546, 0, 537, 0, 0, 0, 0, 8158, 1647, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 7987, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1853, 0, 7042, 0, 0, 0, 13, 407, 0, 784, 0, 1712, 0, 2, 0, 1158, 1957, 0, 6, 6447, 0, 0, 1420, 0, 0, 0, 243, 0, 5677, 0, 0, 440, 0, 0, 0, 0, 0, 0, 37, 0, 1571, 0, 561, 0, 0, 0, 0, 8180, 1642, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 7977, 28, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2169, 1, 6953, 2, 0, 0, 11, 419, 0, 1006, 0, 1709, 0, 3, 0, 1352, 1994, 0, 16, 6241, 0, 0, 1354, 0, 0, 0, 237, 0, 5464, 0, 0, 462, 1, 0, 0, 0, 2, 9, 15, 0, 1330, 2, 580, 0, 0, 0, 0, 8182, 1632, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 7991, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2234, 0, 6944, 0, 0, 0, 6, 465, 0, 844, 0, 1041, 0, 3, 0, 1219, 2269, 0, 0, 6221, 0, 0, 1209, 0, 0, 0, 242, 0, 6244, 0, 0, 447, 0, 0, 0, 0, 0, 0, 29, 0, 1639, 0, 418, 0, 0, 0, 0, 8183, 1500, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 8013, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2065, 0, 6954, 0, 0, 0, 1, 608, 0, 933, 0, 1471, 0, 2, 0, 1353, 2368, 0, 4, 5973, 0, 0, 1468, 0, 0, 0, 215, 0, 5822, 0, 0, 481, 0, 0, 0, 0, 0, 0, 17, 0, 1334, 0, 427, 0, 0, 0, 0, 8164, 1465, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 7928, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1511, 0, 6645, 0, 0, 0, 13, 320, 0, 1011, 0, 1703, 0, 2, 0, 1254, 2556, 0, 3, 6711, 0, 2, 1244, 0, 0, 0, 297, 0, 5281, 0, 0, 476, 0, 0, 0, 0, 0, 1, 72, 0, 1665, 0, 685, 0, 0, 0, 1, 8182, 1574, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 7903, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1707, 0, 6644, 0, 0, 0, 9, 377, 0, 1353, 0, 1259, 0, 2, 0, 1363, 2021, 1, 5, 6650, 0, 0, 1113, 1, 0, 0, 271, 0, 5487, 0, 0, 424, 0, 0, 0, 0, 0, 0, 98, 0, 1854, 0, 764, 0, 0, 0, 0, 8172, 1634, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 6123, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1919, 0, 4266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 469, 0, 701, 0, 1680, 1777, 1355, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1302, 1877, 0, 6, 2767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 0, 0, 0, 271, 1, 3918, 910, 0, 0, 0, 0, 0, 0, 5813, 0, 0, 429, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 1628, 0, 753, 0, 2683, 1257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4561, 1496, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6136, 6185, 5996, 5953, 6152, 6282, 6390, 6058] avg=6144.0000 max=6390 max_rank=6 min=5953 min_rank=3 imbalance_ratio=1.040039 max_load_reduction=42762 max_load_reduction_pct=0.869995 moved_to_echo=24882 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 6038, 9, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1980, 0, 3912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8, 413, 0, 1278, 2, 2171, 1752, 1242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1466, 2239, 0, 10, 2747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 672, 0, 0, 0, 312, 0, 3890, 911, 0, 0, 0, 0, 0, 0, 5434, 0, 0, 442, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 1546, 0, 537, 0, 2684, 1153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4563, 1647, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6048, 5892, 6867, 6462, 5785, 5916, 5972, 6210] avg=6144.0000 max=6867 max_rank=2 min=5785 min_rank=4 imbalance_ratio=1.117676 max_load_reduction=42285 max_load_reduction_pct=0.860291 moved_to_echo=23883 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 6159, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1853, 0, 4368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 407, 0, 784, 0, 1712, 1787, 1387, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1158, 1957, 0, 6, 2668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1420, 0, 0, 0, 243, 0, 3779, 913, 0, 0, 0, 0, 0, 0, 5677, 0, 0, 440, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 1571, 0, 561, 0, 2691, 1287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4576, 1642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6172, 6221, 6090, 5791, 6355, 6158, 6147, 6218] avg=6144.0000 max=6355 max_rank=4 min=5791 min_rank=3 imbalance_ratio=1.034342 max_load_reduction=42797 max_load_reduction_pct=0.870707 moved_to_echo=24878 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 6151, 28, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2169, 1, 4312, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 419, 0, 1006, 0, 1709, 1785, 1370, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1352, 1994, 0, 16, 2583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1354, 0, 0, 0, 237, 0, 3658, 913, 0, 0, 0, 0, 0, 0, 5464, 0, 0, 462, 1, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 2, 9, 15, 0, 1330, 2, 580, 0, 2692, 1271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4577, 1632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6180, 6484, 6300, 5948, 6162, 5968, 5901, 6209] avg=6144.0000 max=6484 max_rank=1 min=5901 min_rank=6 imbalance_ratio=1.055339 max_load_reduction=42668 max_load_reduction_pct=0.868083 moved_to_echo=24240 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 6162, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2234, 0, 4307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 465, 0, 844, 0, 1041, 1788, 1368, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1219, 2269, 0, 0, 2575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1209, 0, 0, 0, 242, 0, 3646, 913, 0, 0, 0, 0, 0, 0, 6244, 0, 0, 447, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 1639, 0, 418, 0, 2692, 1269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4578, 1500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6166, 6541, 5512, 6066, 6010, 6732, 6047, 6078] avg=6144.0000 max=6732 max_rank=5 min=5512 min_rank=2 imbalance_ratio=1.095703 max_load_reduction=42420 max_load_reduction_pct=0.863037 moved_to_echo=24867 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 6107, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1511, 0, 4113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 320, 0, 1011, 0, 1703, 1777, 1312, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1254, 2556, 0, 3, 2774, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1244, 0, 0, 0, 297, 0, 3937, 918, 0, 0, 0, 0, 0, 0, 5281, 0, 0, 476, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 1, 72, 0, 1665, 0, 685, 0, 2697, 1220, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4567, 1574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6122, 5624, 6136, 6589, 6398, 5801, 6340, 6142] avg=6144.0000 max=6589 max_rank=3 min=5624 min_rank=1 imbalance_ratio=1.072428 max_load_reduction=42563 max_load_reduction_pct=0.865946 moved_to_echo=24681 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 6095, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1707, 0, 4120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 377, 0, 1353, 0, 1259, 1768, 1309, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1363, 2021, 1, 5, 2752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1113, 1, 0, 0, 271, 0, 3898, 912, 0, 0, 0, 0, 0, 0, 5487, 0, 0, 424, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 1854, 0, 764, 0, 2689, 1215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4571, 1634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6135, 5827, 6075, 6144, 6195, 5951, 6620, 6205] avg=6144.0000 max=6620 max_rank=6 min=5827 min_rank=1 imbalance_ratio=1.077474 max_load_reduction=42532 max_load_reduction_pct=0.865316 moved_to_echo=24971 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 6179, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2065, 0, 4313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 608, 0, 933, 0, 1471, 1793, 1370, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1353, 2368, 0, 4, 2472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1468, 0, 0, 0, 215, 0, 3501, 911, 0, 0, 0, 0, 0, 0, 5822, 0, 0, 481, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 1334, 0, 427, 0, 2686, 1271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4567, 1465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6193, 6378, 6176, 6199, 6095, 6344, 5735, 6032] avg=6144.0000 max=6378 max_rank=1 min=5735 min_rank=6 imbalance_ratio=1.038086 max_load_reduction=42774 max_load_reduction_pct=0.870239 moved_to_echo=24206 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 0, 128, 3584, 0, 7936, 128, 12800, 14336, 10752, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 512, 0, 12672, 128, 4736, 0, 21632, 9984, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 0, 10496, 17408, 128, 128, 21376, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[0, 128, 9600, 128, 0, 0, 2176, 128, 30336, 7424, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 15488, 128, 33792, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 128, 36608, 12672, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[45312, 0, 0, 3712, 128, 0, 0, 0, 384, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 49024, 256, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2195, 0, 0, 1, 0, 2623, 0, 3907, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1775, 1080, 0, 0, 0, 9, 0, 0, 4178, 0, 5924, 8189, 1, 0, 419, 1157, 0, 0, 0, 3, 0, 0, 0, 7797, 0, 0, 0, 248, 0, 288, 0, 0, 1166, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2353, 0, 0, 3, 0, 2395, 0, 4011, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1913, 881, 0, 0, 0, 24, 0, 0, 3916, 4, 6515, 8190, 0, 0, 433, 986, 0, 0, 0, 1, 0, 0, 0, 7760, 0, 0, 0, 190, 0, 376, 0, 0, 1006, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2900, 0, 0, 0, 0, 2470, 0, 3997, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1834, 916, 0, 0, 0, 26, 0, 0, 3349, 3, 5955, 8192, 0, 0, 446, 867, 0, 0, 0, 1, 0, 0, 0, 7760, 0, 0, 0, 285, 0, 524, 0, 0, 1431, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2520, 0, 0, 0, 0, 2434, 0, 4585, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2112, 784, 0, 0, 0, 26, 0, 0, 3558, 0, 6021, 8192, 0, 0, 536, 871, 0, 0, 0, 0, 0, 0, 0, 7812, 0, 0, 0, 182, 0, 246, 0, 0, 1079, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2039, 0, 0, 3, 0, 2658, 0, 3883, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1832, 776, 0, 0, 0, 39, 0, 0, 4430, 1, 6136, 8190, 0, 0, 647, 902, 0, 0, 0, 0, 0, 0, 0, 7780, 0, 0, 0, 288, 0, 283, 0, 0, 1070, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 3053, 0, 0, 1, 0, 2566, 0, 4317, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 2043, 1015, 0, 0, 0, 11, 0, 0, 3348, 3, 5499, 8192, 0, 0, 456, 824, 0, 0, 0, 1, 0, 0, 0, 7719, 0, 0, 0, 312, 0, 474, 0, 0, 1117, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2412, 0, 0, 2, 0, 2513, 0, 3736, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1869, 834, 0, 0, 0, 17, 0, 0, 3967, 1, 6039, 8190, 0, 0, 587, 911, 0, 0, 0, 2, 0, 0, 0, 7787, 0, 0, 0, 324, 0, 380, 0, 0, 1385, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 0, 0, 0, 0, 0, 2346, 0, 0, 2, 0, 2447, 0, 4916, 8190, 1, 2, 0, 0, 0, 0, 0, 1, 6, 0, 0, 1, 2006, 910, 0, 0, 0, 21, 0, 0, 3354, 0, 5631, 8187, 0, 0, 611, 963, 3, 0, 0, 0, 0, 0, 2, 7813, 0, 3, 0, 317, 2, 402, 2, 0, 1010, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6142, 0, 0, 0, 0, 0, 0, 0, 0, 2195, 0, 0, 1, 0, 2623, 0, 1144, 0, 0, 0, 0, 0, 0, 0, 3907, 1976, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1775, 1080, 3317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4178, 0, 1640, 600, 120, 0, 0, 0, 0, 0, 4660, 0, 1, 0, 419, 1157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 6157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 248, 0, 288, 0, 0, 1166, 0, 2298, 2047, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6142, 5963, 5884, 6172, 6547, 6237, 6160, 6047] avg=6144.0000 max=6547 max_rank=4 min=5884 min_rank=2 imbalance_ratio=1.065592 max_load_reduction=42605 max_load_reduction_pct=0.866801 moved_to_echo=24991 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 2353, 0, 0, 3, 0, 2395, 0, 1258, 0, 0, 0, 0, 0, 0, 0, 4011, 1976, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1913, 881, 3317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 3916, 4, 1632, 600, 132, 0, 0, 0, 0, 0, 5125, 0, 0, 0, 433, 986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 6128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 190, 0, 376, 0, 0, 1006, 0, 2299, 2047, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6009, 5988, 6113, 6308, 6544, 6129, 5918] avg=6144.0000 max=6544 max_rank=5 min=5918 min_rank=7 imbalance_ratio=1.065104 max_load_reduction=42608 max_load_reduction_pct=0.866862 moved_to_echo=24899 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 2900, 0, 0, 0, 0, 2470, 0, 1150, 0, 0, 0, 0, 0, 0, 0, 3997, 1976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1834, 916, 3317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 3349, 3, 1632, 600, 121, 0, 0, 0, 0, 0, 4684, 0, 0, 0, 446, 867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 6128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 285, 0, 524, 0, 0, 1431, 0, 2299, 2047, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 6520, 5973, 6071, 5731, 5997, 6129, 6586] avg=6144.0000 max=6586 max_rank=7 min=5731 min_rank=4 imbalance_ratio=1.071940 max_load_reduction=42566 max_load_reduction_pct=0.866007 moved_to_echo=24443 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6146, 0, 0, 0, 0, 0, 0, 0, 0, 2520, 0, 0, 0, 0, 2434, 0, 1162, 0, 0, 0, 0, 0, 0, 0, 4585, 1976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2112, 784, 3317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 3558, 0, 1643, 600, 122, 0, 0, 0, 0, 0, 4737, 0, 0, 0, 536, 871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, 0, 246, 0, 0, 1079, 0, 2299, 2046, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6146, 6116, 6561, 6215, 5949, 6144, 6169, 5852] avg=6144.0000 max=6561 max_rank=2 min=5852 min_rank=7 imbalance_ratio=1.067871 max_load_reduction=42591 max_load_reduction_pct=0.866516 moved_to_echo=24114 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 0, 0, 0, 0, 6141, 0, 0, 0, 0, 0, 0, 0, 0, 2346, 0, 0, 2, 0, 2447, 0, 1087, 0, 0, 0, 0, 0, 0, 0, 4916, 1976, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 0, 1, 2006, 910, 3316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 3354, 0, 1643, 600, 114, 0, 0, 0, 0, 0, 4430, 0, 0, 0, 611, 963, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6170, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 317, 2, 402, 2, 0, 1010, 1, 2298, 2046, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 5882, 6895, 6240, 5732, 6007, 6175, 6078] avg=6144.0000 max=6895 max_rank=2 min=5732 min_rank=4 imbalance_ratio=1.122233 max_load_reduction=42257 max_load_reduction_pct=0.859721 moved_to_echo=23992 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 2412, 0, 0, 2, 0, 2513, 0, 1168, 0, 0, 0, 0, 0, 0, 0, 3736, 1963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1869, 834, 3322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 3967, 1, 1640, 606, 127, 0, 0, 0, 0, 0, 4744, 0, 0, 0, 587, 911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 6147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 324, 0, 380, 0, 0, 1385, 0, 2301, 2046, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6144, 6095, 5699, 6029, 6358, 6242, 6149, 6436] avg=6144.0000 max=6436 max_rank=7 min=5699 min_rank=2 imbalance_ratio=1.047526 max_load_reduction=42716 max_load_reduction_pct=0.869059 moved_to_echo=25185 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6146, 0, 0, 0, 0, 0, 0, 0, 0, 3053, 0, 0, 1, 0, 2566, 0, 1062, 0, 0, 0, 0, 0, 0, 0, 4317, 1976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 2043, 1015, 3317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 3348, 3, 1623, 600, 112, 0, 0, 0, 0, 0, 4325, 0, 0, 0, 456, 824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 6096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 312, 0, 474, 0, 0, 1117, 0, 2299, 2046, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6146, 6682, 6293, 6384, 5697, 5605, 6097, 6248] avg=6144.0000 max=6682 max_rank=1 min=5605 min_rank=5 imbalance_ratio=1.087565 max_load_reduction=42470 max_load_reduction_pct=0.864054 moved_to_echo=23647 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 2039, 0, 0, 3, 0, 2658, 0, 1185, 0, 0, 0, 0, 0, 0, 0, 3883, 1976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1832, 776, 3317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 4430, 1, 1636, 600, 124, 0, 0, 0, 0, 0, 4827, 0, 0, 0, 647, 902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 288, 0, 283, 0, 0, 1070, 0, 2299, 2047, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 5885, 5859, 5928, 6830, 6376, 6144, 5987] avg=6144.0000 max=6830 max_rank=4 min=5859 min_rank=2 imbalance_ratio=1.111654 max_load_reduction=42322 max_load_reduction_pct=0.861043 moved_to_echo=25337 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 0, 128, 15488, 7296, 26624, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 2176, 128, 3072, 128, 0, 9344, 128, 18432, 16384, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 0, 256, 0, 0, 30208, 128, 13184, 4864, 1024, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 19840, 0, 0, 128, 0, 20224, 0, 9216, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[33408, 15872, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[37632, 0, 128, 0, 4224, 7552, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49280 observed_loss=-128 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 49152, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 128, 49152, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4367, 0, 0, 174, 80, 0, 3127, 3486, 1, 0, 1, 206, 0, 0, 4864, 0, 2, 0, 0, 0, 0, 8085, 0, 0, 0, 0, 0, 0, 0, 7841, 0, 0, 1, 0, 0, 0, 0, 2205, 0, 5034, 89, 0, 0, 7743, 0, 0, 0, 0, 1846, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4693, 0, 0, 135, 110, 0, 3328, 3359, 0, 0, 0, 165, 0, 0, 5384, 0, 0, 0, 0, 0, 0, 8049, 0, 0, 0, 0, 0, 0, 0, 7699, 0, 0, 0, 0, 0, 0, 0, 2009, 0, 5177, 76, 0, 0, 7657, 0, 0, 0, 0, 1311, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4866, 0, 0, 151, 64, 1, 3227, 3249, 5, 0, 0, 171, 0, 0, 5103, 0, 2, 1, 0, 0, 0, 8016, 0, 0, 0, 0, 0, 0, 0, 7675, 0, 0, 2, 0, 0, 1, 0, 1883, 0, 5414, 69, 0, 0, 7700, 0, 0, 0, 0, 1552, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4444, 0, 0, 143, 162, 0, 3698, 3415, 5, 0, 0, 174, 0, 0, 4905, 0, 0, 0, 0, 0, 0, 8002, 0, 0, 1, 0, 0, 0, 0, 7550, 0, 0, 0, 0, 0, 0, 0, 2108, 0, 5445, 66, 0, 0, 7672, 0, 0, 0, 0, 1360, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4987, 0, 0, 84, 70, 0, 3527, 2969, 0, 0, 0, 155, 0, 0, 5402, 0, 0, 0, 0, 0, 0, 8064, 0, 0, 0, 0, 0, 0, 0, 7480, 0, 0, 0, 0, 0, 0, 0, 2266, 0, 5025, 75, 0, 0, 7664, 0, 0, 0, 0, 1384, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 2, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4973, 3, 0, 122, 139, 3, 3286, 3554, 0, 0, 1, 165, 0, 1, 4961, 1, 6, 1, 0, 0, 0, 8079, 1, 0, 2, 2, 0, 0, 0, 7548, 0, 0, 0, 0, 0, 0, 0, 2075, 0, 4919, 75, 0, 0, 7656, 1, 0, 0, 1, 1573, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4552, 0, 0, 153, 107, 0, 3509, 3330, 0, 0, 0, 270, 0, 0, 5463, 0, 0, 0, 0, 0, 0, 8004, 0, 0, 0, 0, 0, 0, 0, 7633, 0, 0, 0, 0, 0, 0, 0, 2063, 0, 5268, 92, 0, 0, 7614, 0, 0, 0, 0, 1094, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5129, 0, 0, 79, 54, 0, 3594, 3160, 0, 0, 0, 120, 0, 0, 4908, 0, 0, 0, 0, 0, 0, 8108, 0, 0, 0, 0, 0, 0, 0, 7536, 0, 0, 1, 0, 0, 0, 0, 2006, 0, 5164, 65, 0, 0, 7633, 0, 0, 0, 0, 1594, 1, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 3000, 1899, 1256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4866, 0, 0, 1215, 230, 0, 0, 0, 0, 0, 0, 151, 64, 1, 2458, 3249, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 171, 0, 0, 5103, 0, 2, 1, 0, 769, 27, 0, 0, 0, 0, 0, 0, 0, 0, 6117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6189, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1883, 0, 4172, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4700, 0, 0, 0, 0, 1552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6155, 6311, 5928, 6073, 6117, 6191, 6125, 6252] avg=6144.0000 max=6311 max_rank=1 min=5928 min_rank=2 imbalance_ratio=1.027181 max_load_reduction=42841 max_load_reduction_pct=0.871602 moved_to_echo=24685 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 2983, 1907, 1260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4693, 0, 0, 1162, 231, 0, 0, 0, 0, 0, 0, 135, 110, 0, 2535, 3359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 165, 0, 0, 5384, 0, 0, 0, 0, 793, 26, 0, 0, 0, 0, 0, 0, 0, 0, 6142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2009, 0, 3989, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4674, 0, 0, 0, 0, 1311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6150, 6086, 6139, 6368, 6142, 6208, 6074, 5985] avg=6144.0000 max=6368 max_rank=3 min=5985 min_rank=7 imbalance_ratio=1.036458 max_load_reduction=42784 max_load_reduction_pct=0.870443 moved_to_echo=24409 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 1, 0, 0, 0, 0, 2983, 1914, 1236, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4973, 3, 0, 1104, 226, 0, 0, 0, 0, 0, 0, 122, 139, 3, 2503, 3554, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 165, 0, 1, 4961, 1, 6, 1, 0, 783, 25, 0, 0, 0, 0, 0, 0, 0, 0, 6165, 1, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2075, 0, 3790, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4673, 1, 0, 0, 1, 1573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6136, 6307, 6322, 5943, 6170, 6086, 5940, 6248] avg=6144.0000 max=6322 max_rank=2 min=5940 min_rank=6 imbalance_ratio=1.028971 max_load_reduction=42830 max_load_reduction_pct=0.871379 moved_to_echo=24444 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 2986, 1910, 1225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4987, 0, 0, 1128, 224, 0, 0, 0, 0, 0, 0, 84, 70, 0, 2686, 2969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 0, 0, 5402, 0, 0, 0, 0, 841, 25, 0, 0, 0, 0, 0, 0, 0, 0, 6154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2266, 0, 3872, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4678, 0, 0, 0, 0, 1384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6121, 6339, 5809, 6423, 6154, 6031, 6213, 6062] avg=6144.0000 max=6423 max_rank=3 min=5809 min_rank=2 imbalance_ratio=1.045410 max_load_reduction=42729 max_load_reduction_pct=0.869324 moved_to_echo=24460 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 2994, 1899, 1239, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4444, 0, 0, 1225, 229, 0, 0, 0, 0, 0, 0, 143, 162, 0, 2812, 3415, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 174, 0, 0, 4905, 0, 0, 0, 0, 886, 32, 0, 0, 0, 0, 0, 0, 0, 0, 6103, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2108, 0, 4188, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4678, 0, 0, 0, 0, 1360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6132, 5900, 6537, 5997, 6104, 6082, 6362, 6038] avg=6144.0000 max=6537 max_rank=2 min=5900 min_rank=1 imbalance_ratio=1.063965 max_load_reduction=42615 max_load_reduction_pct=0.867004 moved_to_echo=24586 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 2967, 1896, 1250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4552, 0, 0, 1182, 229, 0, 0, 0, 0, 0, 0, 153, 107, 0, 2673, 3330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 270, 0, 0, 5463, 0, 0, 0, 0, 836, 26, 0, 0, 0, 0, 0, 0, 0, 0, 6108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2063, 0, 4060, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4647, 0, 0, 0, 0, 1094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6113, 5963, 6263, 6595, 6108, 6154, 6215, 5741] avg=6144.0000 max=6595 max_rank=3 min=5741 min_rank=7 imbalance_ratio=1.073405 max_load_reduction=42557 max_load_reduction_pct=0.865824 moved_to_echo=24218 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 2974, 1921, 1234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5129, 0, 0, 1159, 226, 0, 0, 0, 0, 0, 0, 79, 54, 0, 2737, 3160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 0, 0, 4908, 0, 0, 0, 0, 857, 26, 0, 0, 0, 0, 0, 0, 0, 0, 6187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6076, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2006, 0, 3979, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4659, 0, 0, 0, 0, 1594, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6129, 6514, 6030, 5911, 6187, 6077, 6050, 6254] avg=6144.0000 max=6514 max_rank=1 min=5911 min_rank=3 imbalance_ratio=1.060221 max_load_reduction=42638 max_load_reduction_pct=0.867472 moved_to_echo=24568 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 3017, 1915, 1284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4367, 0, 0, 1130, 235, 0, 0, 0, 0, 0, 0, 174, 80, 0, 2382, 3486, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 206, 0, 0, 4864, 0, 2, 0, 0, 745, 25, 0, 0, 0, 0, 0, 0, 0, 0, 6170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6322, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2205, 0, 3879, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4726, 0, 0, 0, 0, 1846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6216, 5732, 6124, 5842, 6170, 6323, 6173, 6572] avg=6144.0000 max=6572 max_rank=7 min=5732 min_rank=1 imbalance_ratio=1.069661 max_load_reduction=42580 max_load_reduction_pct=0.866292 moved_to_echo=25238 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 128, 0, 128, 0, 0, 0, 0, 23936, 15360, 9984, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[128, 0, 16640, 0, 32000, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[37504, 128, 0, 0, 128, 11776, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 0, 38016, 128, 0, 9344, 1920, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1152, 896, 128, 20864, 26624, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1536, 0, 128, 41088, 128, 128, 128, 0, 6528, 256, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49280 observed_loss=-128 tokens_per_expert=[0, 0, 49152, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 49152, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 320, 229, 12, 0, 0, 0, 5278, 0, 0, 0, 8008, 0, 3, 0, 0, 0, 0, 7160, 0, 0, 7213, 0, 0, 0, 0, 0, 0, 0, 544, 113, 0, 0, 3371, 0, 0, 3495, 7213, 216, 0, 0, 187, 0, 0, 0, 0, 0, 4958, 0, 297, 0, 0, 0, 0, 0, 512, 0, 0, 23] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 318, 117, 13, 0, 0, 0, 5063, 0, 0, 0, 8041, 0, 46, 0, 0, 0, 0, 6484, 0, 0, 7354, 0, 0, 0, 0, 0, 0, 0, 809, 134, 0, 0, 2937, 0, 0, 3621, 7234, 369, 0, 0, 176, 0, 0, 0, 0, 0, 5515, 0, 346, 0, 0, 0, 0, 0, 565, 0, 0, 10] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 343, 316, 13, 0, 0, 3, 5201, 0, 0, 2, 8007, 0, 46, 2, 0, 0, 0, 6559, 0, 0, 7216, 0, 0, 0, 0, 0, 0, 0, 836, 113, 1, 0, 3287, 0, 0, 2981, 7235, 308, 1, 0, 138, 0, 0, 0, 0, 0, 5416, 0, 426, 0, 0, 3, 0, 0, 687, 0, 2, 9] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 261, 197, 19, 0, 0, 0, 5019, 0, 0, 0, 8012, 0, 36, 0, 0, 0, 0, 7108, 0, 0, 7033, 0, 0, 0, 0, 0, 0, 0, 610, 136, 0, 0, 3074, 0, 0, 3871, 7089, 197, 0, 0, 214, 0, 0, 0, 0, 0, 5269, 0, 323, 0, 0, 0, 1, 0, 666, 0, 0, 17] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 336, 231, 25, 0, 0, 0, 4660, 0, 0, 0, 8037, 0, 8, 0, 0, 0, 0, 6796, 0, 1, 7076, 0, 0, 0, 0, 0, 0, 0, 785, 141, 0, 0, 3060, 0, 0, 3520, 7174, 264, 0, 0, 146, 0, 0, 0, 0, 0, 5527, 0, 555, 0, 0, 0, 0, 0, 793, 0, 0, 16] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 250, 259, 5, 0, 0, 0, 5515, 0, 0, 0, 8048, 0, 2, 0, 0, 0, 0, 7036, 0, 0, 7017, 0, 0, 0, 0, 1, 0, 0, 524, 110, 0, 0, 3574, 0, 0, 3628, 7096, 201, 0, 0, 144, 0, 0, 0, 0, 0, 4746, 0, 309, 0, 0, 0, 0, 0, 661, 0, 0, 26] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 378, 176, 6, 0, 0, 0, 4713, 0, 0, 0, 7977, 0, 30, 0, 0, 0, 0, 6642, 0, 0, 7100, 0, 0, 0, 0, 0, 0, 0, 823, 91, 0, 0, 3421, 2, 0, 3372, 7252, 344, 0, 0, 227, 0, 0, 0, 0, 0, 5785, 0, 287, 0, 0, 0, 0, 0, 522, 0, 0, 4] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 459, 424, 16, 0, 0, 0, 5541, 0, 0, 0, 8028, 0, 21, 0, 0, 0, 0, 7234, 0, 0, 7281, 0, 0, 0, 0, 0, 0, 0, 522, 112, 0, 0, 2325, 0, 0, 3098, 7259, 443, 0, 0, 168, 0, 0, 0, 0, 0, 5169, 0, 272, 0, 0, 0, 0, 0, 770, 0, 0, 10] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 318, 117, 5581, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 5063, 0, 0, 0, 762, 248, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 5771, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 809, 134, 0, 0, 2937, 0, 1947, 282, 0, 0, 0, 0, 0, 0, 0, 3621, 2242, 369, 0, 0, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5515, 0, 346, 0, 465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 565, 0, 0, 10, 3045, 2460, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6016, 6086, 5817, 6310, 6109, 6408, 6326, 6080] avg=6144.0000 max=6408 max_rank=5 min=5817 min_rank=2 imbalance_ratio=1.042969 max_load_reduction=42744 max_load_reduction_pct=0.869629 moved_to_echo=24923 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 336, 231, 5578, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 4660, 0, 0, 0, 733, 260, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 6048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 785, 141, 0, 0, 3060, 0, 1931, 271, 0, 0, 0, 0, 0, 0, 0, 3520, 2223, 264, 0, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5527, 0, 555, 0, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 793, 0, 0, 16, 3020, 2459, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6146, 5678, 6056, 6073, 6188, 6153, 6570, 6288] avg=6144.0000 max=6570 max_rank=6 min=5678 min_rank=1 imbalance_ratio=1.069336 max_load_reduction=42582 max_load_reduction_pct=0.866333 moved_to_echo=25199 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 343, 316, 5559, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 3, 5201, 0, 0, 2, 748, 251, 0, 0, 0, 0, 0, 0, 0, 0, 46, 2, 0, 0, 0, 5837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 836, 113, 1, 0, 3287, 0, 1948, 277, 0, 0, 0, 0, 0, 0, 0, 2981, 2241, 308, 1, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5416, 0, 426, 0, 471, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 687, 0, 2, 9, 3046, 2448, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6219, 6218, 5885, 6191, 6462, 5669, 6313, 6195] avg=6144.0000 max=6462 max_rank=4 min=5669 min_rank=5 imbalance_ratio=1.051758 max_load_reduction=42690 max_load_reduction_pct=0.868530 moved_to_echo=24639 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 320, 229, 5559, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 5278, 0, 0, 0, 747, 274, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 6372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 113, 0, 0, 3371, 0, 1942, 277, 0, 0, 0, 0, 0, 0, 0, 3495, 2234, 216, 0, 0, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4958, 0, 297, 0, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 0, 0, 23, 3037, 2449, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6108, 6311, 6375, 6189, 6247, 6132, 5769, 6021] avg=6144.0000 max=6375 max_rank=2 min=5769 min_rank=6 imbalance_ratio=1.037598 max_load_reduction=42777 max_load_reduction_pct=0.870300 moved_to_echo=24169 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 261, 197, 5561, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 5019, 0, 0, 0, 729, 272, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 6326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 610, 136, 0, 0, 3074, 0, 1908, 270, 0, 0, 0, 0, 0, 0, 0, 3871, 2197, 197, 0, 0, 214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5269, 0, 323, 0, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 666, 0, 0, 17, 2984, 2451, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6019, 6039, 6362, 6034, 5998, 6479, 6102, 6119] avg=6144.0000 max=6479 max_rank=5 min=5998 min_rank=4 imbalance_ratio=1.054525 max_load_reduction=42673 max_load_reduction_pct=0.868184 moved_to_echo=24698 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 459, 424, 5573, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 5541, 0, 0, 0, 757, 281, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 6431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 112, 0, 0, 2325, 0, 1958, 283, 0, 0, 0, 0, 0, 0, 0, 3098, 2240, 443, 0, 0, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5169, 0, 272, 0, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 770, 0, 0, 10, 3061, 2455, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6456, 6595, 6452, 6241, 5200, 5949, 5963, 6296] avg=6144.0000 max=6595 max_rank=1 min=5200 min_rank=4 imbalance_ratio=1.073405 max_load_reduction=42557 max_load_reduction_pct=0.865824 moved_to_echo=23408 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 378, 176, 5538, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4713, 0, 0, 0, 736, 254, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 5911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 823, 91, 0, 0, 3421, 2, 1952, 272, 0, 0, 0, 0, 0, 0, 0, 3372, 2247, 344, 0, 0, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5785, 0, 287, 0, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 0, 0, 4, 3053, 2439, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6092, 5709, 5941, 6092, 6561, 6190, 6549, 6018] avg=6144.0000 max=6561 max_rank=4 min=5709 min_rank=1 imbalance_ratio=1.067871 max_load_reduction=42591 max_load_reduction_pct=0.866516 moved_to_echo=25318 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 250, 259, 5587, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 5515, 0, 0, 0, 727, 269, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 6262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6021, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 110, 0, 0, 3574, 0, 1910, 269, 0, 0, 0, 0, 0, 0, 0, 3628, 2199, 201, 0, 0, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4746, 0, 309, 0, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 661, 0, 0, 26, 2987, 2461, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6096, 6516, 6264, 6022, 6387, 6172, 5560, 6135] avg=6144.0000 max=6516 max_rank=1 min=5560 min_rank=6 imbalance_ratio=1.060547 max_load_reduction=42636 max_load_reduction_pct=0.867432 moved_to_echo=24254 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 27648, 17920, 2432, 128, 0, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 0, 5504, 1024, 128, 0, 25088, 128, 15616, 2304, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 0, 0, 128, 41088, 0, 0, 128, 6016, 2176, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49280 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 42496, 0, 2816, 0, 3968, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 0, 0, 2688, 2048, 44544, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 5248, 0, 128, 128, 24320, 19712, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 256, 128, 0, 0, 0, 49024, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 128, 49152, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=17 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 6420, 0, 0, 0, 8192, 0, 5285, 0, 517, 0, 0, 0, 0, 0, 0, 0, 0, 246, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 5676, 0, 0, 3423, 0, 0, 7394, 0, 0, 0, 0, 0, 0, 7524, 0, 0, 0, 1, 4471, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=17 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 6494, 0, 0, 0, 8192, 0, 5070, 0, 489, 0, 0, 1, 5, 0, 0, 0, 0, 553, 0, 0, 0, 10, 1, 0, 0, 0, 0, 0, 0, 5177, 0, 0, 3788, 0, 0, 7306, 0, 0, 0, 0, 0, 0, 7520, 0, 0, 0, 3, 4543, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=17 tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 0, 0, 0, 1, 7, 6393, 0, 0, 0, 8188, 0, 6070, 0, 530, 0, 0, 0, 5, 0, 0, 0, 0, 293, 0, 0, 0, 13, 0, 2, 0, 0, 0, 2, 0, 5486, 0, 0, 3755, 0, 0, 7066, 0, 0, 2, 0, 0, 0, 7528, 4, 0, 0, 6, 3798, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=17 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4, 0, 0, 0, 0, 6579, 0, 0, 0, 8191, 0, 5378, 0, 479, 0, 0, 0, 11, 1, 0, 0, 0, 598, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 5347, 0, 3, 3786, 0, 0, 6979, 0, 0, 0, 0, 0, 0, 7485, 0, 0, 0, 0, 4301, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=17 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 5395, 0, 0, 0, 8191, 0, 5855, 0, 589, 0, 0, 0, 1, 0, 0, 0, 0, 478, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 5591, 0, 0, 4022, 0, 0, 7553, 0, 0, 0, 0, 0, 0, 7522, 0, 0, 0, 0, 3945, 0, 0, 0, 5, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=17 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 6822, 0, 0, 0, 8192, 0, 5766, 0, 559, 0, 0, 0, 0, 0, 0, 0, 0, 155, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 5273, 0, 0, 3763, 0, 0, 7085, 0, 0, 0, 0, 0, 0, 7451, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=17 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 6865, 0, 0, 0, 8192, 0, 5659, 0, 506, 0, 0, 0, 0, 0, 0, 0, 0, 517, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 5336, 0, 0, 3409, 0, 0, 7153, 0, 0, 0, 0, 0, 0, 7380, 0, 0, 0, 0, 4126, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=17 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 6586, 0, 0, 0, 8192, 0, 5628, 0, 515, 0, 0, 0, 1, 0, 0, 0, 0, 573, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 5418, 0, 0, 2447, 0, 0, 7480, 0, 0, 0, 0, 0, 0, 7382, 0, 0, 0, 2, 4926, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=17 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 560, 0, 0, 0, 0, 0, 5070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 0, 0, 1, 5, 0, 0, 3414, 2245, 0, 0, 0, 0, 0, 0, 0, 0, 553, 0, 0, 0, 10, 1, 3689, 2049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5177, 0, 734, 0, 0, 0, 0, 0, 0, 0, 0, 3788, 0, 0, 2615, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6181, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4543, 0, 0, 0, 0, 0, 0, 0, 1277, 605, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 5630, 6154, 6302, 5911, 6403, 6184, 6425] avg=6144.0000 max=6425 max_rank=7 min=5630 min_rank=1 imbalance_ratio=1.045736 max_load_reduction=42727 max_load_reduction_pct=0.869283 moved_to_echo=24923 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=17 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 553, 0, 0, 0, 0, 0, 5285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 0, 0, 0, 0, 0, 0, 3455, 2220, 0, 0, 0, 0, 0, 0, 0, 0, 246, 0, 0, 0, 3, 0, 3647, 2049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5676, 0, 735, 0, 0, 0, 0, 0, 0, 0, 0, 3423, 0, 0, 2646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6184, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4471, 0, 0, 0, 0, 0, 0, 0, 1293, 605, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 5838, 6192, 5945, 6411, 6069, 6185, 6369] avg=6144.0000 max=6411 max_rank=4 min=5838 min_rank=1 imbalance_ratio=1.043457 max_load_reduction=42741 max_load_reduction_pct=0.869568 moved_to_echo=25034 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=17 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 0, 0, 0, 1, 6141, 0, 0, 0, 0, 0, 0, 0, 7, 551, 0, 0, 0, 0, 0, 6070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 0, 0, 0, 5, 0, 0, 3302, 2210, 0, 0, 0, 0, 0, 0, 0, 0, 293, 0, 0, 0, 13, 0, 3632, 2047, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 5486, 0, 735, 0, 0, 0, 0, 0, 0, 0, 0, 3755, 0, 0, 2529, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6188, 4, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 3798, 0, 1, 0, 0, 0, 0, 0, 1235, 605, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6144, 6628, 6047, 5985, 6225, 6286, 6198, 5639] avg=6144.0000 max=6628 max_rank=1 min=5639 min_rank=7 imbalance_ratio=1.078776 max_load_reduction=42524 max_load_reduction_pct=0.865153 moved_to_echo=24348 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=17 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 6142, 0, 0, 0, 0, 0, 0, 0, 0, 465, 0, 0, 0, 0, 0, 5855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 589, 0, 0, 0, 1, 0, 0, 3530, 1865, 0, 0, 0, 0, 0, 0, 0, 0, 478, 0, 0, 0, 4, 0, 3065, 2049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5591, 0, 735, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 0, 0, 2702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3945, 0, 0, 0, 5, 0, 0, 0, 1321, 605, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6320, 5985, 5596, 6326, 6724, 6182, 5876] avg=6144.0000 max=6724 max_rank=5 min=5596 min_rank=3 imbalance_ratio=1.094401 max_load_reduction=42428 max_load_reduction_pct=0.863200 moved_to_echo=25108 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=17 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 560, 0, 0, 0, 0, 0, 5378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 0, 0, 0, 11, 1, 0, 3265, 2278, 0, 0, 0, 0, 0, 0, 0, 0, 598, 0, 0, 0, 10, 0, 3741, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5347, 0, 735, 0, 0, 0, 0, 0, 0, 0, 3, 3786, 0, 0, 2491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4301, 0, 0, 0, 0, 0, 0, 0, 1223, 606, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6147, 5938, 6034, 6397, 6082, 6280, 6144, 6130] avg=6144.0000 max=6397 max_rank=3 min=5938 min_rank=1 imbalance_ratio=1.041178 max_load_reduction=42755 max_load_reduction_pct=0.869853 moved_to_echo=24636 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=17 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 587, 0, 0, 0, 0, 0, 5766, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 0, 0, 0, 0, 0, 0, 3311, 2359, 0, 0, 0, 0, 0, 0, 0, 0, 155, 0, 0, 0, 9, 0, 3876, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5273, 0, 728, 0, 0, 0, 0, 0, 0, 0, 0, 3763, 0, 0, 2535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 1239, 599, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6144, 6353, 6229, 6088, 6001, 6298, 6124, 5915] avg=6144.0000 max=6353 max_rank=1 min=5915 min_rank=7 imbalance_ratio=1.034017 max_load_reduction=42799 max_load_reduction_pct=0.870748 moved_to_echo=24338 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=17 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 567, 0, 0, 0, 0, 0, 5628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 0, 0, 0, 1, 0, 0, 3495, 2277, 0, 0, 0, 0, 0, 0, 0, 0, 573, 0, 0, 0, 2, 0, 3742, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5418, 0, 721, 0, 0, 0, 0, 0, 0, 0, 0, 2447, 0, 0, 2677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6067, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4926, 0, 0, 0, 0, 0, 0, 0, 1308, 594, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6144, 6195, 6288, 6365, 6139, 5124, 6069, 6828] avg=6144.0000 max=6828 max_rank=7 min=5124 min_rank=5 imbalance_ratio=1.111328 max_load_reduction=42324 max_load_reduction_pct=0.861084 moved_to_echo=24160 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=17 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 591, 0, 0, 0, 0, 0, 5659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 0, 0, 0, 0, 0, 0, 3343, 2374, 0, 0, 0, 0, 0, 0, 0, 0, 517, 0, 0, 0, 9, 0, 3900, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5336, 0, 721, 0, 0, 0, 0, 0, 0, 0, 0, 3409, 0, 0, 2559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4126, 0, 0, 0, 0, 0, 0, 0, 1251, 593, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6144, 6250, 6223, 6474, 6057, 5968, 6066, 5970] avg=6144.0000 max=6474 max_rank=3 min=5968 min_rank=5 imbalance_ratio=1.053711 max_load_reduction=42678 max_load_reduction_pct=0.868286 moved_to_echo=24061 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 4224, 0, 0, 128, 128, 128, 0, 27136, 17920, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 28416, 0, 0, 20864, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[34304, 0, 128, 0, 128, 0, 0, 0, 10240, 4864, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 3456, 0, 0, 0, 128, 128, 29312, 16512, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[128, 4480, 0, 0, 0, 0, 0, 44800, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 128, 0, 43392, 0, 5888, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 0, 49152, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 128, 0, 0, 128, 49152, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 3091, 0, 0, 0, 1246, 6, 0, 7675, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 951, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 677, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6316, 6288, 7587, 7048] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 2799, 0, 0, 0, 1579, 1, 0, 7711, 0, 0, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 787, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 1, 0, 13, 1, 0, 0, 668, 34, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 6190, 6613, 7573, 6913] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 3595, 0, 0, 0, 1432, 2, 0, 7582, 0, 0, 1, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 783, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 2, 0, 0, 0, 0, 0, 594, 26, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 6768, 5876, 7381, 6857] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 2835, 0, 0, 0, 1466, 11, 0, 7627, 1, 0, 0, 1, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 1, 1, 1026, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 1, 1, 5, 0, 0, 3, 509, 6, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 2, 0, 6547, 5999, 7566, 7289] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 3152, 0, 0, 0, 1270, 0, 0, 7666, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 836, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 544, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6691, 6155, 7449, 7191] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 3031, 0, 0, 0, 1439, 1, 0, 7541, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 759, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 4, 0, 0, 0, 534, 9, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 6784, 6196, 7543, 7081] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 2746, 0, 0, 0, 1576, 9, 0, 7708, 0, 0, 1, 0, 0, 73, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 903, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 1, 0, 0, 0, 0, 536, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6501, 6344, 7402, 7120] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 3293, 0, 0, 0, 1651, 13, 0, 7735, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 895, 0, 0, 0, 0, 0, 0, 8135, 0, 0, 0, 0, 2, 0, 0, 0, 661, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6375, 6171, 7311, 6808] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3091, 0, 0, 0, 1246, 6, 1555, 62, 0, 0, 0, 0, 0, 0, 0, 6090, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 951, 0, 3123, 2017, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 677, 33, 3193, 2231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4817, 1353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5960, 6171, 6234, 6121, 6136, 6134, 6170, 6226] avg=6144.0000 max=6234 max_rank=2 min=5960 min_rank=0 imbalance_ratio=1.014648 max_load_reduction=42918 max_load_reduction_pct=0.873169 moved_to_echo=24666 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3595, 0, 0, 0, 1432, 2, 1536, 58, 0, 0, 0, 0, 0, 0, 0, 6017, 0, 0, 1, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 783, 0, 3347, 2019, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 594, 26, 3421, 2171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4686, 1317, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6623, 6106, 6064, 6179, 6143, 6214, 6004, 5819] avg=6144.0000 max=6623 max_rank=0 min=5819 min_rank=7 imbalance_ratio=1.077962 max_load_reduction=42529 max_load_reduction_pct=0.865255 moved_to_echo=24180 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2799, 0, 0, 0, 1579, 1, 1563, 65, 0, 0, 0, 0, 0, 0, 0, 6118, 0, 0, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 787, 0, 3061, 2022, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 13, 1, 0, 0, 668, 34, 3129, 2188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4725, 1351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6007, 6211, 6222, 5900, 6152, 6034, 6078, 6548] avg=6144.0000 max=6548 max_rank=7 min=5900 min_rank=3 imbalance_ratio=1.065755 max_load_reduction=42604 max_load_reduction_pct=0.866781 moved_to_echo=24812 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2835, 0, 0, 0, 1466, 11, 1546, 59, 0, 0, 0, 0, 0, 0, 0, 6051, 1, 0, 0, 1, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6217, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 1, 1, 1026, 0, 3238, 2020, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 5, 0, 0, 3, 509, 6, 3309, 2308, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 4981, 1349, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 5940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5917, 6131, 6217, 6324, 6146, 6142, 6333, 5942] avg=6144.0000 max=6333 max_rank=6 min=5917 min_rank=0 imbalance_ratio=1.030762 max_load_reduction=42819 max_load_reduction_pct=0.871155 moved_to_echo=24563 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3152, 0, 0, 0, 1270, 0, 1553, 61, 0, 0, 0, 0, 0, 0, 0, 6083, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 836, 0, 3309, 2017, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 27, 3382, 2277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4914, 1328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6036, 6100, 6121, 6192, 6137, 6230, 6242, 6094] avg=6144.0000 max=6242 max_rank=6 min=6036 min_rank=0 imbalance_ratio=1.015951 max_load_reduction=42910 max_load_reduction_pct=0.873006 moved_to_echo=24703 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2746, 0, 0, 0, 1576, 9, 1567, 66, 0, 0, 0, 0, 0, 0, 0, 6107, 0, 0, 1, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 6083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 903, 0, 3215, 2027, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 536, 41, 3286, 2255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4865, 1319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5964, 6181, 6089, 6179, 6158, 6119, 6184, 6278] avg=6144.0000 max=6278 max_rank=7 min=5964 min_rank=0 imbalance_ratio=1.021810 max_load_reduction=42874 max_load_reduction_pct=0.872274 moved_to_echo=24739 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3293, 0, 0, 0, 1651, 13, 1567, 61, 0, 0, 0, 0, 0, 0, 0, 6138, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 895, 0, 3153, 2012, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 661, 20, 3222, 2156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4652, 1303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6585, 6220, 6008, 6090, 6123, 6061, 5955, 6110] avg=6144.0000 max=6585 max_rank=0 min=5955 min_rank=6 imbalance_ratio=1.071777 max_load_reduction=42567 max_load_reduction_pct=0.866028 moved_to_echo=24249 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3031, 0, 0, 0, 1439, 1, 1528, 61, 0, 0, 0, 0, 0, 0, 0, 5984, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 759, 0, 3355, 2024, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 534, 9, 3429, 2242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4839, 1346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6060, 6032, 6197, 6167, 6157, 6218, 6186, 6135] avg=6144.0000 max=6218 max_rank=5 min=6032 min_rank=1 imbalance_ratio=1.012044 max_load_reduction=42934 max_load_reduction_pct=0.873494 moved_to_echo=24696 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[0, 128, 0, 128, 128, 128, 7040, 0, 25856, 16256, 256, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 24576, 0, 0, 0, 11776, 128, 12416, 512, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 0, 128, 128, 0, 38528, 10752, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 0, 128, 4736, 256, 26496, 17920, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 48640, 128, 0, 128, 128, 0, 640, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49280 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 0, 128, 0, 0, 49152, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49152 observed_loss=0 tokens_per_expert=[0, 0, 0, 0, 0, 49152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 128, 128, 0, 0, 49152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49152 slack_slots=49152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8112, 0, 98, 1, 0, 1, 271, 3057, 0, 305, 0, 0, 0, 0, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7256, 8177, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 8192, 4732, 0, 0, 54, 0, 0, 0, 0, 0, 0, 139, 0, 29, 0, 1, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8093, 0, 146, 1, 3, 0, 315, 3026, 0, 635, 0, 0, 0, 0, 419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7047, 8167, 8192, 0, 0, 0, 0, 0, 0, 3, 0, 8192, 4748, 0, 0, 57, 0, 0, 0, 0, 0, 0, 83, 0, 17, 0, 5, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 8182, 0, 314, 1, 2, 3, 443, 2740, 0, 445, 0, 0, 0, 0, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7312, 8180, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 8191, 4432, 0, 0, 51, 0, 0, 0, 0, 0, 0, 140, 0, 10, 0, 6, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8122, 0, 226, 0, 6, 0, 355, 2715, 0, 369, 0, 0, 0, 0, 450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7356, 8172, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 8190, 4825, 0, 0, 40, 0, 0, 0, 0, 0, 0, 97, 0, 27, 0, 7, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 2, 1, 1, 0, 2, 0, 0, 0, 0, 8103, 1, 177, 0, 2, 2, 356, 2645, 2, 401, 0, 2, 0, 0, 646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7222, 8177, 8188, 0, 0, 7, 0, 0, 0, 3, 0, 8183, 4835, 0, 0, 21, 1, 0, 0, 0, 0, 0, 125, 0, 39, 1, 7, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8101, 0, 117, 0, 0, 0, 328, 2619, 0, 338, 0, 0, 0, 0, 351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7345, 8181, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 8192, 5147, 0, 0, 47, 0, 0, 0, 0, 0, 0, 167, 0, 18, 0, 6, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8115, 0, 162, 0, 3, 0, 333, 2523, 0, 308, 0, 0, 0, 0, 437, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7362, 8181, 8192, 0, 0, 0, 0, 0, 0, 6, 0, 8192, 5155, 0, 0, 42, 0, 0, 0, 0, 0, 0, 126, 0, 12, 0, 1, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8116, 0, 98, 0, 3, 1, 254, 3280, 0, 359, 0, 0, 0, 0, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7242, 8185, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 8189, 4338, 0, 0, 36, 0, 0, 0, 0, 0, 0, 229, 0, 67, 0, 5, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5970, 0, 98, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 271, 3057, 0, 305, 0, 0, 0, 1849, 740, 0, 0, 0, 0, 0, 0, 0, 534, 0, 0, 0, 0, 0, 0, 4253, 1402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 4732, 0, 0, 54, 0, 0, 0, 0, 1121, 200, 0, 0, 0, 0, 0, 0, 0, 0, 139, 0, 29, 0, 1, 0, 3924, 2050, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6142, 6069, 6223, 6189, 6135, 6144, 6107, 6143] avg=6144.0000 max=6223 max_rank=2 min=6069 min_rank=1 imbalance_ratio=1.012858 max_load_reduction=42929 max_load_reduction_pct=0.873393 moved_to_echo=24529 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3, 0, 6142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5957, 0, 146, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 315, 3026, 0, 635, 0, 0, 0, 1849, 738, 0, 0, 0, 0, 0, 0, 0, 419, 0, 0, 0, 0, 0, 0, 4248, 1398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 4748, 0, 0, 57, 0, 0, 0, 0, 1089, 200, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 17, 0, 5, 0, 3919, 2050, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 6107, 6563, 6065, 5958, 6146, 6094, 6074] avg=6144.0000 max=6563 max_rank=2 min=5958 min_rank=4 imbalance_ratio=1.068197 max_load_reduction=42589 max_load_reduction_pct=0.866475 moved_to_echo=24272 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 6142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5978, 0, 226, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 355, 2715, 0, 369, 0, 0, 0, 1849, 741, 0, 0, 0, 0, 0, 0, 0, 450, 0, 0, 0, 0, 0, 0, 4250, 1403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 6141, 0, 0, 0, 0, 0, 0, 0, 0, 4825, 0, 0, 40, 0, 0, 0, 0, 1137, 200, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 27, 0, 7, 0, 3922, 2050, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6210, 6029, 6103, 6219, 6143, 6202, 6103] avg=6144.0000 max=6219 max_rank=4 min=6029 min_rank=2 imbalance_ratio=1.012207 max_load_reduction=42933 max_load_reduction_pct=0.873474 moved_to_echo=24667 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 1, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6013, 0, 314, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3, 443, 2740, 0, 445, 0, 0, 0, 1854, 750, 0, 0, 0, 0, 0, 0, 0, 505, 0, 0, 0, 0, 0, 0, 4255, 1419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6135, 0, 0, 0, 0, 0, 0, 0, 0, 4432, 0, 0, 51, 0, 0, 0, 0, 1135, 202, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 10, 0, 6, 0, 3925, 2049, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 6330, 6235, 6179, 6177, 6136, 5820, 6130] avg=6144.0000 max=6330 max_rank=1 min=5820 min_rank=6 imbalance_ratio=1.030273 max_load_reduction=42822 max_load_reduction_pct=0.871216 moved_to_echo=24263 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 1, 1, 0, 2, 0, 6139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5964, 1, 177, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 356, 2645, 2, 401, 0, 2, 0, 1847, 739, 0, 0, 0, 0, 0, 0, 0, 646, 0, 0, 0, 0, 0, 0, 4254, 1400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 3, 0, 6137, 0, 0, 0, 0, 0, 0, 0, 0, 4835, 0, 0, 21, 1, 0, 0, 0, 1116, 199, 0, 0, 0, 0, 0, 0, 0, 0, 125, 0, 39, 1, 7, 0, 3923, 2049, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 6144, 5994, 6300, 6106, 6147, 6172, 6144] avg=6144.0000 max=6300 max_rank=3 min=5994 min_rank=2 imbalance_ratio=1.025391 max_load_reduction=42852 max_load_reduction_pct=0.871826 moved_to_echo=24569 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5962, 0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 328, 2619, 0, 338, 0, 0, 0, 1849, 739, 0, 0, 0, 0, 0, 0, 0, 351, 0, 0, 0, 0, 0, 0, 4256, 1400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 5147, 0, 0, 47, 0, 0, 0, 0, 1135, 200, 0, 0, 0, 0, 0, 0, 0, 0, 167, 0, 18, 0, 6, 0, 3925, 2049, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6144, 6079, 5873, 6007, 6210, 6145, 6529, 6165] avg=6144.0000 max=6529 max_rank=6 min=5873 min_rank=2 imbalance_ratio=1.062663 max_load_reduction=42623 max_load_reduction_pct=0.867167 moved_to_echo=25049 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3, 0, 6142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5974, 0, 98, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 254, 3280, 0, 359, 0, 0, 0, 1848, 740, 0, 0, 0, 0, 0, 0, 0, 554, 0, 0, 0, 0, 0, 0, 4258, 1402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6141, 0, 0, 0, 0, 0, 0, 0, 0, 4338, 0, 0, 36, 0, 0, 0, 0, 1119, 200, 0, 0, 0, 0, 0, 0, 0, 0, 229, 0, 67, 0, 5, 0, 3927, 2050, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 6075, 6482, 6214, 6123, 6142, 5693, 6278] avg=6144.0000 max=6482 max_rank=2 min=5693 min_rank=6 imbalance_ratio=1.055013 max_load_reduction=42670 max_load_reduction_pct=0.868123 moved_to_echo=24236 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 6142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5973, 0, 162, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 333, 2523, 0, 308, 0, 0, 0, 1849, 740, 0, 0, 0, 0, 0, 0, 0, 437, 0, 0, 1, 0, 0, 0, 4255, 1402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 5155, 0, 0, 42, 0, 0, 0, 0, 1138, 200, 0, 0, 0, 0, 0, 0, 0, 0, 126, 0, 12, 0, 1, 0, 3926, 2050, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6138, 5753, 6095, 6224, 6149, 6535, 6115] avg=6144.0000 max=6535 max_rank=6 min=5753 min_rank=2 imbalance_ratio=1.063639 max_load_reduction=42617 max_load_reduction_pct=0.867045 moved_to_echo=25023 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 1152, 0, 256, 128, 128, 0, 31488, 16512, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 3968, 0, 0, 128, 0, 0, 0, 34048, 11264, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 2688, 22656, 128, 3200, 0, 128, 0, 14848, 6016, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[38272, 0, 0, 384, 128, 0, 0, 0, 9088, 1664, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 0, 47872, 128, 1408, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 0, 128, 0, 49152, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 128, 128, 128, 0, 128, 0, 49152, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49152 observed_loss=0 tokens_per_expert=[0, 0, 0, 0, 49152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49152 slack_slots=49152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7555, 387, 1486, 999, 0, 0, 34, 0, 0, 382, 0, 0, 0, 0, 7806, 0, 0, 0, 0, 0, 439, 0, 0, 927, 0, 0, 0, 0, 0, 0, 0, 0, 7416, 6654, 145, 0, 8109, 4, 0, 2322, 130, 0, 0, 0, 0, 4309] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7663, 532, 1141, 1066, 0, 0, 5, 0, 0, 378, 0, 0, 0, 0, 7652, 0, 0, 0, 0, 0, 457, 0, 0, 742, 0, 0, 0, 0, 0, 0, 0, 0, 7563, 6677, 113, 0, 8150, 4, 0, 2292, 75, 0, 0, 0, 0, 4580] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7515, 488, 890, 845, 0, 0, 2, 0, 0, 417, 0, 0, 0, 0, 7690, 0, 0, 0, 0, 0, 490, 0, 0, 673, 0, 0, 0, 0, 0, 0, 0, 0, 7847, 6865, 91, 0, 8128, 15, 0, 2690, 78, 0, 0, 0, 0, 4358] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 1, 71, 0, 0, 0, 1, 0, 1, 0, 3, 1, 7610, 592, 1322, 981, 2, 0, 35, 0, 0, 417, 0, 0, 0, 0, 7599, 0, 1, 0, 0, 0, 478, 1, 1, 794, 0, 3, 0, 1, 0, 2, 1, 0, 7627, 6583, 109, 0, 8114, 10, 1, 2371, 118, 0, 0, 0, 0, 4300] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7594, 495, 1326, 987, 0, 0, 12, 0, 0, 382, 0, 0, 0, 0, 7630, 0, 1, 0, 0, 0, 560, 0, 0, 696, 0, 0, 0, 0, 0, 0, 0, 0, 7544, 6911, 136, 0, 8157, 0, 1, 2120, 95, 0, 0, 0, 0, 4437] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7539, 400, 1548, 1111, 0, 0, 27, 0, 0, 369, 0, 0, 0, 0, 7590, 0, 0, 0, 0, 0, 438, 0, 0, 649, 0, 0, 0, 0, 0, 0, 0, 0, 7581, 6767, 74, 0, 8148, 15, 0, 2320, 81, 0, 0, 0, 0, 4434] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7794, 553, 1341, 994, 0, 0, 22, 0, 0, 403, 0, 0, 0, 0, 7633, 0, 1, 0, 0, 0, 349, 0, 0, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 7543, 6872, 84, 0, 8088, 1, 0, 1775, 57, 0, 0, 0, 0, 4547] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7433, 444, 1033, 860, 0, 0, 1, 0, 0, 385, 0, 0, 0, 0, 7528, 0, 0, 0, 0, 0, 539, 0, 0, 607, 0, 0, 0, 0, 0, 0, 0, 0, 7752, 6605, 111, 0, 8091, 8, 0, 2675, 153, 0, 0, 0, 0, 4891] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6163, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 4081, 1987, 0, 0, 0, 0, 0, 0, 0, 0, 3451, 532, 1141, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 378, 0, 0, 0, 0, 3482, 2254, 0, 0, 0, 0, 0, 0, 5683, 0, 0, 0, 0, 0, 457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 742, 0, 0, 0, 0, 0, 0, 1969, 1958, 773, 706, 0, 0, 0, 0, 0, 0, 0, 5971, 113, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2292, 75, 0, 0, 0, 0, 3807, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6163, 6130, 6190, 6119, 6140, 6148, 6088, 6174] avg=6144.0000 max=6190 max_rank=2 min=6088 min_rank=6 imbalance_ratio=1.007487 max_load_reduction=42962 max_load_reduction_pct=0.874064 moved_to_echo=24550 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6132, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 1, 0, 4002, 1977, 0, 0, 0, 0, 0, 0, 0, 0, 3401, 387, 1486, 999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 382, 0, 0, 0, 0, 3414, 2223, 0, 0, 0, 0, 0, 0, 5798, 0, 0, 0, 0, 0, 439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 927, 0, 0, 0, 0, 0, 0, 2008, 1931, 728, 703, 0, 0, 0, 0, 0, 0, 0, 5951, 145, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2322, 130, 0, 0, 0, 0, 3581, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6132, 6027, 6273, 6053, 6237, 6297, 6100, 6033] avg=6144.0000 max=6297 max_rank=5 min=6027 min_rank=1 imbalance_ratio=1.024902 max_load_reduction=42855 max_load_reduction_pct=0.871887 moved_to_echo=24667 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 0, 0, 0, 6168, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 4071, 1989, 0, 0, 0, 0, 0, 0, 0, 0, 3419, 495, 1326, 987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 382, 0, 0, 0, 0, 3473, 2234, 0, 0, 0, 0, 0, 0, 5667, 0, 1, 0, 0, 0, 560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 696, 0, 0, 0, 0, 0, 0, 1963, 1941, 749, 730, 0, 0, 0, 0, 0, 0, 0, 6181, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2120, 95, 0, 0, 0, 0, 3688, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6170, 6126, 6227, 6101, 6228, 6079, 6317, 5904] avg=6144.0000 max=6317 max_rank=6 min=5904 min_rank=7 imbalance_ratio=1.028158 max_load_reduction=42835 max_load_reduction_pct=0.871480 moved_to_echo=24528 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6148, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 4236, 1980, 0, 0, 0, 0, 0, 0, 0, 1, 3383, 488, 890, 845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 417, 0, 0, 0, 0, 3611, 2211, 0, 0, 0, 0, 0, 0, 5712, 0, 0, 0, 0, 0, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 673, 0, 0, 0, 0, 0, 0, 1978, 1921, 736, 725, 0, 0, 0, 0, 0, 0, 0, 6140, 91, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2690, 78, 0, 0, 0, 0, 3622, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6148, 6285, 5607, 6241, 6202, 6033, 6246, 6390] avg=6144.0000 max=6390 max_rank=7 min=5607 min_rank=2 imbalance_ratio=1.040039 max_load_reduction=42762 max_load_reduction_pct=0.869995 moved_to_echo=24871 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 1, 6137, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 1, 0, 1, 0, 4117, 1977, 0, 0, 0, 0, 0, 0, 3, 1, 3426, 592, 1322, 981, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 417, 0, 0, 0, 0, 3510, 2239, 0, 0, 0, 0, 0, 0, 5644, 0, 1, 0, 0, 0, 478, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 794, 0, 3, 0, 1, 0, 2, 1955, 1945, 726, 696, 0, 0, 0, 0, 1, 0, 0, 5887, 109, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2371, 118, 0, 0, 0, 0, 3574, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6139, 6167, 6327, 6201, 6124, 6123, 6007, 6064] avg=6144.0000 max=6327 max_rank=2 min=6007 min_rank=6 imbalance_ratio=1.029785 max_load_reduction=42825 max_load_reduction_pct=0.871277 moved_to_echo=24318 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6163, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 4092, 1985, 0, 0, 0, 0, 0, 0, 0, 0, 3387, 400, 1548, 1111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 369, 0, 0, 0, 0, 3489, 2222, 0, 0, 0, 0, 0, 0, 5634, 0, 0, 0, 0, 0, 438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 649, 0, 0, 0, 0, 0, 0, 1956, 1930, 752, 719, 0, 0, 0, 0, 0, 0, 0, 6048, 74, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2320, 81, 0, 0, 0, 0, 3682, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6163, 6138, 6446, 6107, 6072, 6006, 6137, 6083] avg=6144.0000 max=6446 max_rank=2 min=6006 min_rank=5 imbalance_ratio=1.049154 max_load_reduction=42706 max_load_reduction_pct=0.868856 moved_to_echo=24298 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6117, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 4072, 1971, 0, 0, 0, 0, 0, 0, 0, 0, 3509, 553, 1341, 994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 403, 0, 0, 0, 0, 3471, 2293, 0, 0, 0, 0, 0, 0, 5669, 0, 1, 0, 0, 0, 349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 0, 0, 0, 0, 0, 0, 1964, 1992, 768, 726, 0, 0, 0, 0, 0, 0, 0, 6146, 84, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1775, 57, 0, 0, 0, 0, 3779, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6117, 6090, 6397, 6189, 6019, 6498, 6231, 5611] avg=6144.0000 max=6498 max_rank=5 min=5611 min_rank=7 imbalance_ratio=1.057617 max_load_reduction=42654 max_load_reduction_pct=0.867798 moved_to_echo=24359 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 6119, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 4184, 1972, 0, 0, 0, 0, 0, 0, 0, 2, 3346, 444, 1033, 860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 385, 0, 0, 0, 0, 3568, 2187, 0, 0, 0, 0, 0, 0, 5591, 0, 0, 0, 0, 0, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 607, 0, 0, 0, 0, 0, 0, 1937, 1900, 826, 698, 0, 0, 0, 0, 0, 0, 0, 5907, 111, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2675, 153, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6120, 6189, 5685, 6141, 6130, 5968, 6026, 6893] avg=6144.0000 max=6893 max_rank=7 min=5685 min_rank=2 imbalance_ratio=1.121908 max_load_reduction=42259 max_load_reduction_pct=0.859762 moved_to_echo=25017 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 6144, 0, 128, 0, 128, 0, 128, 15744, 15616, 6144, 5760, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 27392, 3968, 10112, 7936, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 18688, 896, 0, 0, 0, 0, 29824, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[256, 0, 0, 3200, 0, 0, 0, 0, 28032, 17920, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[45440, 0, 128, 0, 0, 0, 3840, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 128, 49152, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[128, 0, 0, 48256, 896, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[512, 0, 0, 0, 128, 0, 128, 0, 32896, 15872, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=21 tokens_per_expert_current_ep_rank=[0, 4189, 0, 0, 0, 0, 0, 0, 0, 1384, 668, 0, 0, 0, 0, 0, 0, 3017, 0, 0, 0, 0, 0, 789, 106, 7914, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 906, 0, 0, 396, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3700, 0, 4153, 0, 0, 0, 5545, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=21 tokens_per_expert_current_ep_rank=[0, 4101, 0, 1, 0, 0, 0, 0, 0, 1113, 639, 0, 0, 0, 0, 0, 0, 3219, 0, 0, 0, 0, 0, 477, 94, 8000, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1112, 0, 0, 478, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2869, 0, 4884, 0, 0, 0, 5781, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=21 tokens_per_expert_current_ep_rank=[0, 4294, 0, 10, 0, 0, 0, 0, 0, 1303, 579, 0, 0, 0, 0, 0, 0, 3482, 0, 0, 0, 0, 0, 610, 138, 7904, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1097, 0, 0, 585, 0, 0, 0, 0, 8186, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3069, 0, 4286, 0, 0, 0, 5416, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=21 tokens_per_expert_current_ep_rank=[0, 3906, 0, 3, 0, 0, 0, 0, 0, 1303, 611, 0, 0, 0, 0, 0, 0, 2791, 0, 0, 0, 0, 0, 923, 75, 8041, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 1, 974, 0, 0, 308, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3697, 0, 4563, 0, 0, 0, 5572, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=21 tokens_per_expert_current_ep_rank=[0, 3849, 0, 0, 0, 0, 0, 0, 0, 1519, 557, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 0, 0, 514, 134, 8034, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 0, 0, 287, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3135, 0, 4070, 0, 0, 0, 5559, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=21 tokens_per_expert_current_ep_rank=[0, 4029, 0, 0, 0, 0, 0, 0, 0, 1114, 976, 0, 0, 0, 0, 0, 0, 3578, 0, 0, 0, 0, 0, 561, 102, 7821, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 818, 0, 0, 525, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3045, 0, 4618, 0, 0, 0, 5581, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=21 tokens_per_expert_current_ep_rank=[0, 3807, 1, 0, 0, 0, 0, 0, 0, 1357, 528, 1, 1, 2, 1, 0, 1, 3683, 0, 1, 0, 0, 0, 485, 60, 8027, 0, 0, 8189, 0, 1, 0, 0, 0, 0, 0, 0, 0, 833, 0, 0, 516, 0, 0, 1, 0, 8190, 3, 3, 0, 0, 0, 0, 2, 0, 0, 3911, 0, 4030, 0, 0, 0, 5518, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=21 tokens_per_expert_current_ep_rank=[0, 3835, 0, 0, 0, 0, 0, 0, 0, 1562, 502, 0, 0, 0, 0, 0, 0, 3694, 0, 0, 0, 0, 0, 620, 83, 8098, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 907, 0, 0, 378, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3337, 0, 4433, 0, 0, 0, 5318, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 4189, 0, 0, 0, 0, 0, 0, 1421, 707, 0, 0, 0, 0, 0, 0, 0, 1384, 668, 0, 0, 0, 0, 0, 2408, 1774, 0, 0, 0, 0, 0, 0, 0, 3017, 0, 0, 0, 0, 0, 789, 1499, 497, 0, 0, 0, 0, 0, 0, 106, 5996, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 906, 0, 3137, 2048, 0, 0, 0, 0, 0, 0, 0, 396, 0, 0, 0, 0, 5711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 3700, 0, 2654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6317, 6234, 5802, 6103, 6091, 6107, 6144, 6354] avg=6144.0000 max=6354 max_rank=7 min=5802 min_rank=2 imbalance_ratio=1.034180 max_load_reduction=42798 max_load_reduction_pct=0.870728 moved_to_echo=24696 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 4101, 0, 1, 0, 0, 0, 0, 1437, 707, 0, 0, 0, 0, 0, 0, 0, 1113, 639, 0, 0, 0, 0, 0, 2510, 1774, 0, 0, 0, 0, 0, 0, 0, 3219, 0, 0, 0, 0, 0, 477, 1763, 502, 0, 0, 0, 0, 0, 0, 94, 6061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1112, 0, 3271, 2048, 0, 0, 0, 0, 0, 0, 0, 478, 0, 0, 0, 0, 5711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 2869, 0, 3121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6246, 6036, 5961, 6155, 6431, 6189, 6144, 5990] avg=6144.0000 max=6431 max_rank=4 min=5961 min_rank=2 imbalance_ratio=1.046712 max_load_reduction=42721 max_load_reduction_pct=0.869161 moved_to_echo=24754 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 4294, 0, 10, 0, 0, 0, 0, 1420, 706, 0, 0, 0, 0, 0, 0, 0, 1303, 579, 0, 0, 0, 0, 0, 2352, 1773, 0, 0, 0, 0, 0, 0, 0, 3482, 0, 0, 0, 0, 0, 610, 1547, 496, 0, 0, 0, 0, 0, 0, 138, 5988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1097, 0, 3064, 2048, 0, 0, 0, 0, 0, 0, 0, 585, 0, 0, 0, 0, 5707, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 3069, 0, 2739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6430, 6007, 6135, 6126, 6209, 6293, 6144, 5808] avg=6144.0000 max=6430 max_rank=0 min=5808 min_rank=7 imbalance_ratio=1.046549 max_load_reduction=42722 max_load_reduction_pct=0.869181 moved_to_echo=24454 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 3849, 0, 0, 0, 0, 0, 0, 1443, 707, 0, 0, 0, 0, 0, 0, 0, 1519, 557, 0, 0, 0, 0, 0, 2415, 1774, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 0, 0, 514, 1469, 504, 0, 0, 0, 0, 0, 0, 134, 6087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 0, 3144, 2049, 0, 0, 0, 0, 0, 0, 0, 287, 0, 0, 0, 0, 5711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 3135, 0, 2601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5999, 6265, 6534, 6221, 6256, 5998, 6143, 5736] avg=6144.0000 max=6534 max_rank=2 min=5736 min_rank=7 imbalance_ratio=1.063477 max_load_reduction=42618 max_load_reduction_pct=0.867065 moved_to_echo=24133 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 3906, 0, 3, 0, 0, 0, 0, 1448, 709, 0, 0, 0, 0, 0, 0, 0, 1303, 611, 0, 0, 0, 0, 0, 2420, 1780, 0, 0, 0, 0, 0, 0, 0, 2791, 0, 0, 0, 0, 0, 923, 1649, 510, 0, 0, 0, 0, 0, 0, 75, 6083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 974, 0, 3152, 2049, 0, 0, 0, 0, 0, 0, 0, 308, 0, 0, 0, 0, 5703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 3697, 0, 2914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6066, 6114, 5873, 6158, 6176, 6011, 6143, 6611] avg=6144.0000 max=6611 max_rank=7 min=5873 min_rank=2 imbalance_ratio=1.076009 max_load_reduction=42541 max_load_reduction_pct=0.865499 moved_to_echo=24941 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 3807, 1, 0, 0, 0, 0, 0, 1442, 707, 0, 0, 0, 0, 0, 0, 0, 1357, 528, 1, 1, 2, 1, 0, 2397, 1774, 0, 0, 0, 0, 0, 0, 1, 3683, 0, 1, 0, 0, 0, 485, 1454, 504, 0, 0, 0, 0, 0, 0, 60, 6081, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 833, 0, 3121, 2047, 0, 0, 0, 0, 0, 0, 0, 516, 0, 0, 1, 0, 5709, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 2, 0, 0, 6142, 0, 0, 0, 0, 0, 0, 0, 3911, 0, 2576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5957, 6061, 6128, 6142, 6001, 6229, 6147, 6487] avg=6144.0000 max=6487 max_rank=7 min=5957 min_rank=0 imbalance_ratio=1.055827 max_load_reduction=42665 max_load_reduction_pct=0.868022 moved_to_echo=24864 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 4029, 0, 0, 0, 0, 0, 0, 1405, 707, 0, 0, 0, 0, 0, 0, 0, 1114, 976, 0, 0, 0, 0, 0, 2424, 1774, 0, 0, 0, 0, 0, 0, 0, 3578, 0, 0, 0, 0, 0, 561, 1667, 491, 0, 0, 0, 0, 0, 0, 102, 5925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 818, 0, 3157, 2049, 0, 0, 0, 0, 0, 0, 0, 525, 0, 0, 0, 0, 5711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 3045, 0, 2951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6141, 6288, 6297, 6027, 6024, 6236, 6143, 5996] avg=6144.0000 max=6297 max_rank=2 min=5996 min_rank=7 imbalance_ratio=1.024902 max_load_reduction=42855 max_load_reduction_pct=0.871887 moved_to_echo=24399 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 3835, 0, 0, 0, 0, 0, 0, 1454, 707, 0, 0, 0, 0, 0, 0, 0, 1562, 502, 0, 0, 0, 0, 0, 2309, 1774, 0, 0, 0, 0, 0, 0, 0, 3694, 0, 0, 0, 0, 0, 620, 1600, 508, 0, 0, 0, 0, 0, 0, 83, 6136, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 907, 0, 3009, 2048, 0, 0, 0, 0, 0, 0, 0, 378, 0, 0, 0, 0, 5711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 3337, 0, 2833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5996, 6147, 6422, 6220, 5964, 6089, 6144, 6170] avg=6144.0000 max=6422 max_rank=2 min=5964 min_rank=4 imbalance_ratio=1.045247 max_load_reduction=42730 max_load_reduction_pct=0.869344 moved_to_echo=24367 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[0, 10752, 5120, 128, 128, 128, 128, 0, 19328, 14208, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 32128, 128, 128, 0, 0, 0, 0, 11520, 5760, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 27520, 0, 128, 0, 0, 0, 4992, 12672, 4096, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49280 observed_loss=-128 tokens_per_expert=[26880, 0, 22400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 3584, 0, 0, 128, 0, 45696, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 128, 7808, 0, 25088, 16512, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[128, 0, 0, 0, 0, 128, 0, 0, 49152, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[896, 48384, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=22 tokens_per_expert_current_ep_rank=[2291, 0, 0, 8192, 0, 0, 0, 748, 0, 0, 0, 0, 0, 0, 745, 0, 0, 0, 0, 0, 0, 8191, 0, 17, 0, 0, 0, 0, 5157, 0, 0, 0, 0, 1535, 0, 7872, 0, 0, 7483, 0, 0, 0, 0, 130, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 5766, 1021] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=22 tokens_per_expert_current_ep_rank=[1909, 0, 0, 8192, 0, 0, 0, 922, 0, 0, 0, 0, 0, 0, 671, 0, 0, 0, 0, 0, 0, 8192, 0, 6, 0, 0, 0, 0, 5317, 0, 0, 0, 0, 1886, 0, 8042, 0, 0, 7186, 0, 0, 0, 0, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5738, 875] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=22 tokens_per_expert_current_ep_rank=[2222, 0, 0, 8192, 0, 0, 0, 1111, 3, 0, 0, 0, 0, 0, 786, 0, 0, 0, 0, 0, 0, 8192, 0, 20, 0, 0, 0, 0, 4752, 0, 0, 0, 0, 1948, 0, 7972, 0, 0, 7400, 0, 0, 0, 0, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5765, 626] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=22 tokens_per_expert_current_ep_rank=[1858, 0, 0, 8192, 0, 0, 0, 1066, 0, 0, 0, 0, 0, 0, 725, 0, 0, 0, 0, 0, 0, 8192, 0, 5, 0, 0, 0, 0, 4803, 0, 0, 0, 0, 1889, 0, 7871, 0, 0, 7374, 0, 0, 0, 0, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 5956, 980] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=22 tokens_per_expert_current_ep_rank=[2004, 0, 0, 8192, 0, 0, 0, 719, 0, 0, 0, 0, 0, 0, 1122, 0, 0, 0, 0, 0, 0, 8192, 0, 8, 0, 0, 0, 0, 5099, 0, 0, 0, 0, 1532, 0, 8135, 0, 0, 7329, 0, 0, 0, 0, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5656, 1004] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=22 tokens_per_expert_current_ep_rank=[2113, 0, 0, 8190, 0, 2, 0, 842, 0, 0, 0, 1, 1, 0, 948, 0, 0, 0, 0, 0, 0, 8187, 0, 10, 1, 3, 0, 0, 5155, 0, 0, 0, 0, 1664, 0, 8046, 0, 0, 7088, 0, 0, 1, 1, 225, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 5715, 954] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=22 tokens_per_expert_current_ep_rank=[2088, 0, 0, 8192, 0, 0, 0, 691, 0, 0, 0, 0, 0, 0, 609, 0, 0, 0, 0, 0, 0, 8192, 0, 8, 0, 0, 0, 0, 5487, 0, 0, 0, 0, 1474, 0, 8075, 0, 0, 7296, 0, 0, 0, 0, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6048, 884] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=22 tokens_per_expert_current_ep_rank=[2020, 0, 0, 8192, 0, 0, 0, 822, 5, 0, 0, 0, 0, 0, 837, 0, 0, 0, 0, 0, 0, 8192, 0, 41, 0, 0, 0, 0, 4997, 0, 0, 0, 0, 1594, 0, 8080, 0, 0, 7577, 0, 0, 0, 0, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 5562, 1045] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=22 rerouted_tokens_per_expert_current_ep_rank=[1909, 0, 0, 3216, 0, 0, 0, 922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 671, 0, 2826, 1573, 876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6131, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5317, 0, 0, 0, 555, 488, 0, 0, 0, 0, 0, 0, 0, 1886, 0, 0, 0, 0, 4360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 0, 0, 0, 0, 4100, 1875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5183, 875, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6047, 5946, 6137, 6360, 6246, 6191, 6167, 6058] avg=6144.0000 max=6360 max_rank=3 min=5946 min_rank=1 imbalance_ratio=1.035156 max_load_reduction=42792 max_load_reduction_pct=0.870605 moved_to_echo=24662 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=22 rerouted_tokens_per_expert_current_ep_rank=[2291, 0, 0, 3216, 0, 0, 0, 748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 745, 0, 2943, 1573, 876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6130, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5157, 0, 0, 0, 558, 488, 0, 0, 0, 0, 0, 0, 0, 1535, 0, 0, 0, 0, 4540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 0, 0, 0, 0, 4100, 1835, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 6037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 5208, 1021, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6255, 6137, 6147, 6203, 6075, 6065, 6039, 6231] avg=6144.0000 max=6255 max_rank=0 min=6039 min_rank=6 imbalance_ratio=1.018066 max_load_reduction=42897 max_load_reduction_pct=0.872742 moved_to_echo=24410 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=22 rerouted_tokens_per_expert_current_ep_rank=[2222, 0, 0, 3216, 0, 0, 0, 1111, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 786, 0, 2910, 1573, 876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6131, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4752, 0, 0, 0, 558, 488, 0, 0, 0, 0, 0, 0, 0, 1948, 0, 0, 0, 0, 4490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 162, 0, 0, 0, 0, 4100, 1858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6114, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5207, 626, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6549, 6148, 6151, 5798, 6438, 6120, 6114, 5834] avg=6144.0000 max=6549 max_rank=0 min=5798 min_rank=3 imbalance_ratio=1.065918 max_load_reduction=42603 max_load_reduction_pct=0.866760 moved_to_echo=24506 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=22 rerouted_tokens_per_expert_current_ep_rank=[2004, 0, 0, 3216, 0, 0, 0, 719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1122, 0, 2882, 1573, 876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6131, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5099, 0, 0, 0, 547, 488, 0, 0, 0, 0, 0, 0, 0, 1532, 0, 0, 0, 0, 4447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 0, 0, 0, 4100, 1897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5109, 1004, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5939, 6453, 6139, 6134, 5979, 6157, 6238, 6113] avg=6144.0000 max=6453 max_rank=1 min=5939 min_rank=0 imbalance_ratio=1.050293 max_load_reduction=42699 max_load_reduction_pct=0.868713 moved_to_echo=24487 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=22 rerouted_tokens_per_expert_current_ep_rank=[1858, 0, 0, 3216, 0, 0, 0, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 725, 0, 2900, 1573, 876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6131, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4803, 0, 0, 0, 576, 488, 0, 0, 0, 0, 0, 0, 0, 1889, 0, 0, 0, 0, 4474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 0, 0, 0, 0, 4100, 1836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 5380, 980, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6140, 6074, 6136, 5867, 6363, 6152, 6035, 6385] avg=6144.0000 max=6385 max_rank=7 min=5867 min_rank=3 imbalance_ratio=1.039225 max_load_reduction=42767 max_load_reduction_pct=0.870097 moved_to_echo=24935 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=22 rerouted_tokens_per_expert_current_ep_rank=[2113, 0, 0, 3216, 0, 2, 0, 842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 948, 0, 2788, 1572, 875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6128, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 5155, 0, 0, 0, 553, 487, 0, 0, 0, 0, 0, 0, 0, 1664, 0, 0, 0, 0, 4300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 225, 0, 1, 0, 0, 4099, 1877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5162, 954, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6173, 6185, 6138, 6199, 5964, 6204, 6171, 6118] avg=6144.0000 max=6204 max_rank=5 min=5964 min_rank=4 imbalance_ratio=1.009766 max_load_reduction=42948 max_load_reduction_pct=0.873779 moved_to_echo=24457 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=22 rerouted_tokens_per_expert_current_ep_rank=[2020, 0, 0, 3212, 0, 0, 0, 822, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 837, 0, 2985, 1578, 877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6124, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4997, 0, 0, 0, 544, 490, 0, 0, 0, 0, 0, 0, 0, 1594, 0, 0, 0, 0, 4592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, 0, 0, 0, 0, 4103, 1885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 5018, 1045, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6054, 6282, 6165, 6031, 6186, 6173, 6195, 6066] avg=6144.0000 max=6282 max_rank=1 min=6031 min_rank=3 imbalance_ratio=1.022461 max_load_reduction=42870 max_load_reduction_pct=0.872192 moved_to_echo=24620 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=22 rerouted_tokens_per_expert_current_ep_rank=[2088, 0, 0, 3216, 0, 0, 0, 691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 609, 0, 2869, 1573, 876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6131, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5487, 0, 0, 0, 585, 488, 0, 0, 0, 0, 0, 0, 0, 1474, 0, 0, 0, 0, 4427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 0, 0, 0, 0, 4100, 1882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5463, 884, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5995, 5927, 6139, 6560, 5901, 6090, 6193, 6347] avg=6144.0000 max=6560 max_rank=3 min=5901 min_rank=4 imbalance_ratio=1.067708 max_load_reduction=42592 max_load_reduction_pct=0.866536 moved_to_echo=24531 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 0, 0, 128, 128, 0, 6528, 0, 23168, 12672, 7040, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[16512, 0, 0, 25728, 0, 128, 0, 7040, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 128, 1408, 0, 128, 0, 0, 32896, 14976, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49280 observed_loss=-128 tokens_per_expert=[0, 13568, 0, 0, 0, 0, 35712, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 128, 0, 0, 40832, 0, 0, 0, 4480, 3968, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 0, 0, 128, 0, 128, 41856, 7424, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 128, 49152, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49280 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 0, 49152, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=23 tokens_per_expert_current_ep_rank=[9, 0, 62, 0, 7841, 0, 0, 2, 1, 1, 0, 2, 180, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 3410, 0, 0, 2, 1833, 0, 8185, 0, 37, 75, 0, 0, 0, 1, 7, 501, 1, 0, 280, 8189, 0, 4, 8189, 5, 0, 0, 1648, 0, 0, 0, 0, 496, 0, 0, 0, 0, 0, 0, 2, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 46, 0, 7868, 0, 0, 0, 0, 0, 0, 0, 298, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 3998, 0, 0, 0, 1630, 0, 8190, 0, 8, 88, 0, 0, 0, 0, 1, 620, 0, 0, 352, 8192, 0, 0, 8192, 1, 0, 0, 1078, 0, 0, 0, 0, 400, 0, 0, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=23 tokens_per_expert_current_ep_rank=[1, 0, 58, 0, 7920, 0, 0, 0, 0, 0, 0, 0, 182, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 3292, 0, 0, 0, 2067, 0, 8192, 0, 27, 88, 0, 0, 0, 0, 0, 519, 0, 0, 332, 8192, 0, 0, 8192, 2, 0, 0, 1539, 0, 0, 0, 0, 359, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 65, 0, 7888, 0, 0, 0, 0, 0, 0, 0, 136, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 3674, 0, 0, 0, 1797, 0, 8192, 0, 18, 73, 0, 0, 0, 0, 1, 461, 0, 0, 248, 8192, 0, 0, 8192, 4, 0, 0, 1647, 0, 0, 0, 0, 373, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 56, 0, 7896, 0, 0, 0, 0, 0, 0, 0, 349, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 3567, 0, 0, 0, 1674, 0, 8192, 0, 9, 77, 0, 0, 0, 0, 2, 464, 0, 0, 328, 8192, 0, 0, 8192, 6, 0, 0, 1330, 0, 0, 0, 0, 627, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 98, 0, 7885, 0, 0, 0, 0, 0, 0, 0, 132, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 3332, 0, 0, 0, 2179, 0, 8192, 0, 34, 75, 0, 0, 0, 0, 2, 391, 0, 0, 211, 8192, 0, 0, 8192, 20, 0, 0, 1759, 0, 0, 0, 0, 266, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 67, 0, 7943, 0, 0, 0, 0, 0, 0, 0, 188, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 3491, 0, 0, 0, 2038, 0, 8192, 0, 18, 69, 0, 0, 0, 0, 3, 405, 0, 0, 244, 8192, 0, 0, 8192, 1, 0, 0, 1526, 0, 0, 0, 0, 390, 0, 0, 0, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 121, 0, 7825, 0, 0, 0, 0, 0, 0, 0, 795, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 3401, 0, 0, 0, 1760, 0, 8192, 0, 18, 91, 0, 0, 0, 0, 0, 453, 0, 0, 322, 8192, 0, 0, 8192, 42, 0, 0, 1018, 0, 0, 0, 0, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 46, 0, 6060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 298, 0, 0, 0, 3812, 2048, 0, 0, 0, 0, 0, 0, 0, 2623, 0, 0, 0, 0, 0, 3998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1630, 0, 4251, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 0, 1, 620, 0, 3830, 1754, 0, 0, 0, 0, 0, 0, 0, 352, 5845, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1078, 0, 0, 0, 0, 400, 0, 2347, 1808, 109, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6106, 6158, 6621, 5889, 6293, 6198, 5742, 6145] avg=6144.0000 max=6621 max_rank=2 min=5742 min_rank=6 imbalance_ratio=1.077637 max_load_reduction=42531 max_load_reduction_pct=0.865295 moved_to_echo=24378 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=23 rerouted_tokens_per_expert_current_ep_rank=[9, 0, 62, 0, 6039, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 2, 180, 0, 0, 0, 3812, 2047, 0, 0, 0, 0, 0, 0, 0, 2623, 0, 0, 0, 0, 0, 3410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1833, 0, 4248, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 1, 7, 501, 1, 3828, 1754, 0, 0, 0, 0, 0, 0, 0, 280, 5842, 0, 4, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1648, 0, 0, 0, 0, 496, 0, 2347, 1802, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 6142, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6112, 6043, 6033, 6120, 6167, 6131, 6402, 6144] avg=6144.0000 max=6402 max_rank=6 min=6033 min_rank=2 imbalance_ratio=1.041992 max_load_reduction=42750 max_load_reduction_pct=0.869751 moved_to_echo=24844 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 65, 0, 6075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 0, 0, 0, 3812, 2048, 0, 0, 0, 0, 0, 0, 0, 2625, 0, 0, 0, 0, 0, 3674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1797, 0, 4252, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 1, 461, 0, 3831, 1754, 0, 0, 0, 0, 0, 0, 0, 248, 5845, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1647, 0, 0, 0, 0, 373, 0, 2347, 1813, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6140, 5996, 6299, 6067, 6120, 6097, 6289, 6144] avg=6144.0000 max=6299 max_rank=2 min=5996 min_rank=1 imbalance_ratio=1.025228 max_load_reduction=42853 max_load_reduction_pct=0.871847 moved_to_echo=24650 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 56, 0, 6081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 349, 0, 0, 0, 3812, 2048, 0, 0, 0, 0, 0, 0, 0, 2625, 0, 0, 0, 0, 0, 3567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1674, 0, 4252, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 2, 464, 0, 3831, 1754, 0, 0, 0, 0, 0, 0, 0, 328, 5845, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1330, 0, 0, 0, 0, 627, 0, 2347, 1815, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6137, 6209, 6192, 5935, 6128, 6179, 6228, 6144] avg=6144.0000 max=6228 max_rank=6 min=5935 min_rank=3 imbalance_ratio=1.013672 max_load_reduction=42924 max_load_reduction_pct=0.873291 moved_to_echo=24679 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=23 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 58, 0, 6100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, 0, 0, 0, 3812, 2049, 0, 0, 0, 0, 0, 0, 0, 2624, 0, 0, 0, 0, 0, 3292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2067, 0, 4252, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 519, 0, 3831, 1754, 0, 0, 0, 0, 0, 0, 0, 332, 5845, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1539, 0, 0, 0, 0, 359, 0, 2347, 1820, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6159, 6043, 5916, 6346, 6192, 6179, 6174, 6143] avg=6144.0000 max=6346 max_rank=3 min=5916 min_rank=2 imbalance_ratio=1.032878 max_load_reduction=42806 max_load_reduction_pct=0.870890 moved_to_echo=24688 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 121, 0, 6027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 795, 0, 0, 0, 3813, 2048, 0, 0, 0, 0, 0, 0, 0, 2625, 0, 0, 0, 0, 0, 3401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1760, 0, 4252, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 0, 0, 453, 0, 3831, 1754, 0, 0, 0, 0, 0, 0, 0, 322, 5845, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 0, 0, 0, 0, 538, 0, 2347, 1798, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6148, 6656, 6026, 6030, 6129, 6209, 5810, 6144] avg=6144.0000 max=6656 max_rank=1 min=5810 min_rank=6 imbalance_ratio=1.083333 max_load_reduction=42496 max_load_reduction_pct=0.864583 moved_to_echo=24292 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 98, 0, 6068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 0, 0, 3817, 2049, 0, 0, 0, 0, 0, 0, 0, 2617, 0, 0, 0, 0, 0, 3332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2179, 0, 4244, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 0, 2, 391, 0, 3835, 1758, 0, 0, 0, 0, 0, 0, 0, 211, 5838, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1759, 0, 0, 0, 0, 266, 0, 2354, 1817, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6166, 5998, 5949, 6457, 6061, 6069, 6309, 6143] avg=6144.0000 max=6457 max_rank=3 min=5949 min_rank=2 imbalance_ratio=1.050944 max_load_reduction=42695 max_load_reduction_pct=0.868632 moved_to_echo=24582 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 67, 0, 6117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 0, 0, 3813, 2048, 0, 0, 0, 0, 0, 0, 0, 2625, 0, 0, 0, 0, 0, 3491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2038, 0, 4252, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 3, 405, 0, 3831, 1754, 0, 0, 0, 0, 0, 0, 0, 244, 5845, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1526, 0, 0, 0, 0, 390, 0, 2347, 1826, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6184, 6049, 6116, 6308, 6062, 6090, 6198, 6145] avg=6144.0000 max=6308 max_rank=3 min=6049 min_rank=1 imbalance_ratio=1.026693 max_load_reduction=42844 max_load_reduction_pct=0.871663 moved_to_echo=24495 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 11648, 0, 0, 0, 0, 3456, 0, 18816, 14592, 896, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 2304, 0, 0, 0, 30592, 16512, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 128, 15104, 0, 34048, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49280 observed_loss=-128 tokens_per_expert=[0, 20992, 0, 0, 0, 0, 0, 28288, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[640, 0, 0, 0, 128, 128, 3840, 128, 30720, 14080, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 2432, 46848, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 128, 128, 0, 0, 49152, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 0, 640, 0, 48640, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 3212, 0, 0, 0, 0, 2, 0, 404, 8192, 0, 0, 245, 5104, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 1476, 120, 0, 0, 0, 0, 0, 1474, 1794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 4015, 0, 8191, 0, 0, 7911, 1, 0, 0, 0, 0, 192, 6714, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 3495, 0, 0, 0, 0, 0, 0, 258, 8192, 0, 0, 171, 5894, 0, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 1320, 105, 0, 0, 0, 0, 0, 1071, 1286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 4274, 0, 8192, 0, 0, 8060, 0, 0, 0, 0, 0, 77, 6638, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 3975, 0, 0, 0, 0, 2, 0, 225, 8192, 0, 0, 344, 5232, 0, 0, 0, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 1530, 130, 0, 0, 0, 0, 0, 804, 1282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 4345, 0, 8192, 0, 0, 8070, 0, 0, 0, 0, 0, 136, 6601, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 3685, 0, 0, 0, 0, 2, 0, 454, 8185, 0, 0, 223, 5484, 1, 0, 3, 0, 96, 1, 0, 0, 0, 0, 0, 0, 0, 1538, 96, 0, 0, 0, 2, 0, 947, 1591, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8, 0, 3629, 0, 8190, 1, 0, 8104, 1, 0, 2, 0, 0, 142, 6761, 0, 3, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 3423, 0, 0, 0, 0, 2, 0, 288, 8192, 0, 0, 242, 4925, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 1284, 94, 0, 0, 0, 0, 0, 1165, 2218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 4050, 0, 8192, 0, 0, 8068, 0, 0, 0, 0, 0, 235, 6684, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 2994, 0, 0, 0, 0, 3, 0, 226, 8192, 0, 0, 257, 5066, 0, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 1562, 171, 0, 0, 0, 0, 0, 1088, 2183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 3991, 0, 8192, 0, 0, 7856, 0, 0, 0, 0, 0, 196, 7040, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 3411, 0, 0, 0, 0, 2, 0, 366, 8192, 0, 0, 228, 5512, 0, 0, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 1289, 57, 0, 0, 0, 0, 0, 770, 1582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4359, 0, 8192, 0, 0, 8144, 0, 0, 0, 0, 0, 139, 6804, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 3249, 0, 0, 0, 0, 1, 0, 524, 8192, 0, 0, 120, 5505, 0, 0, 0, 0, 137, 0, 0, 0, 0, 0, 0, 0, 0, 1559, 108, 0, 0, 0, 0, 0, 999, 1572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 3987, 0, 8192, 0, 0, 8092, 0, 0, 0, 0, 0, 163, 6723, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3212, 0, 0, 0, 0, 2, 1918, 758, 0, 0, 0, 0, 0, 0, 0, 404, 233, 0, 0, 245, 5104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 3981, 2065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1476, 120, 0, 0, 3978, 601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1474, 1794, 0, 0, 0, 3361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 6126, 0, 0, 0, 0, 0, 0, 0, 4015, 0, 0, 0, 0, 2031, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 5956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5890, 5986, 6120, 6175, 6629, 6157, 6047, 6148] avg=6144.0000 max=6629 max_rank=4 min=5890 min_rank=0 imbalance_ratio=1.078939 max_load_reduction=42523 max_load_reduction_pct=0.865133 moved_to_echo=24981 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3975, 0, 0, 0, 0, 2, 1957, 745, 0, 0, 0, 0, 0, 0, 0, 225, 233, 0, 0, 344, 5232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 0, 3981, 2065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1530, 130, 0, 0, 3978, 613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 804, 1282, 0, 0, 0, 3428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 6127, 0, 0, 0, 0, 0, 0, 0, 4345, 0, 0, 0, 0, 2072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 5856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6679, 6034, 6121, 6251, 5514, 6144, 6417, 5992] avg=6144.0000 max=6679 max_rank=0 min=5514 min_rank=4 imbalance_ratio=1.087077 max_load_reduction=42473 max_load_reduction_pct=0.864115 moved_to_echo=24067 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3495, 0, 0, 0, 0, 0, 1954, 750, 0, 0, 0, 0, 0, 0, 0, 258, 233, 0, 0, 171, 5894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, 0, 3981, 2065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1320, 105, 0, 0, 3978, 612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1071, 1286, 0, 0, 0, 3424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 6127, 0, 0, 0, 0, 0, 0, 0, 4274, 0, 0, 0, 0, 2070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 5888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6199, 6556, 6151, 6015, 5781, 6141, 6344, 5965] avg=6144.0000 max=6556 max_rank=1 min=5781 min_rank=4 imbalance_ratio=1.067057 max_load_reduction=42596 max_load_reduction_pct=0.866618 moved_to_echo=24231 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3685, 0, 0, 0, 0, 2, 1965, 763, 0, 0, 0, 0, 0, 0, 0, 454, 232, 0, 0, 223, 5484, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 96, 1, 0, 0, 0, 3978, 2064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1538, 96, 0, 0, 3975, 616, 0, 0, 0, 0, 0, 0, 0, 2, 0, 947, 1591, 0, 0, 3, 3443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 6126, 0, 0, 0, 0, 0, 0, 0, 3629, 0, 0, 1, 0, 2080, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 142, 5998, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6415, 6394, 6142, 6225, 5986, 6134, 5711, 6145] avg=6144.0000 max=6415 max_rank=0 min=5711 min_rank=6 imbalance_ratio=1.044108 max_load_reduction=42737 max_load_reduction_pct=0.869486 moved_to_echo=23976 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3411, 0, 0, 0, 0, 2, 1975, 768, 0, 0, 0, 0, 0, 0, 0, 366, 233, 0, 0, 228, 5512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 0, 0, 3981, 2066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1289, 57, 0, 0, 3978, 619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 770, 1582, 0, 0, 0, 3460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 6126, 0, 0, 0, 0, 0, 0, 0, 4359, 0, 0, 0, 0, 2090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 6036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6156, 6339, 6142, 5943, 5812, 6136, 6449, 6175] avg=6144.0000 max=6449 max_rank=6 min=5812 min_rank=4 imbalance_ratio=1.049642 max_load_reduction=42703 max_load_reduction_pct=0.868795 moved_to_echo=24572 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3423, 0, 0, 0, 0, 2, 1960, 760, 0, 0, 0, 0, 0, 0, 0, 288, 224, 0, 0, 242, 4925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 3987, 2066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1284, 94, 0, 0, 3981, 616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1165, 2218, 0, 0, 0, 3430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 6126, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 0, 0, 0, 2062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 235, 5924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 5679, 6127, 5975, 6813, 6142, 6112, 6159] avg=6144.0000 max=6813 max_rank=4 min=5679 min_rank=1 imbalance_ratio=1.108887 max_load_reduction=42339 max_load_reduction_pct=0.861389 moved_to_echo=25226 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2994, 0, 0, 0, 0, 3, 1905, 795, 0, 0, 0, 0, 0, 0, 0, 226, 233, 0, 0, 257, 5066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 0, 0, 0, 0, 3981, 2066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1562, 171, 0, 0, 3978, 597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1088, 2183, 0, 0, 0, 3337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 6126, 0, 0, 0, 0, 0, 0, 0, 3991, 0, 0, 0, 0, 2017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 196, 6245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5697, 5782, 6166, 6308, 6608, 6142, 6008, 6441] avg=6144.0000 max=6608 max_rank=4 min=5697 min_rank=0 imbalance_ratio=1.075521 max_load_reduction=42544 max_load_reduction_pct=0.865560 moved_to_echo=25199 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3249, 0, 0, 0, 0, 1, 1962, 759, 0, 0, 0, 0, 0, 0, 0, 524, 233, 0, 0, 120, 5505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 0, 0, 3981, 2065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1559, 108, 0, 0, 3978, 615, 0, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1572, 0, 0, 0, 3438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 6127, 0, 0, 0, 0, 0, 0, 0, 3987, 0, 0, 0, 0, 2077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 163, 5964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5971, 6382, 6183, 6260, 6009, 6156, 6064, 6127] avg=6144.0000 max=6382 max_rank=1 min=5971 min_rank=0 imbalance_ratio=1.038737 max_load_reduction=42770 max_load_reduction_pct=0.870158 moved_to_echo=24356 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 0, 896, 128, 0, 0, 0, 31872, 16640, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 27520, 0, 0, 0, 0, 128, 15616, 6144, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 128, 0, 8320, 13568, 0, 0, 128, 27392, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[32768, 0, 0, 128, 0, 16512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 11648, 896, 0, 0, 31872, 4992, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 2816, 1920, 0, 0, 1920, 42752, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[128, 0, 0, 1280, 47872, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49280 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 0, 0, 256, 0, 49024, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True - [2026-08-26 14:53:45] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 838.8 | throughput per GPU (TFLOP/s/GPU): 152.1 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.476898E+00 | loss scale: 1.0 | grad norm: 7.154 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[2650, 579, 279, 1113, 409, 795, 770, 164, 2693, 277, 417, 1087, 55, 800, 560, 473, 871, 1142, 390, 539, 405, 298, 408, 966, 1087, 916, 331, 416, 1084, 431, 1133, 1060, 1121, 663, 911, 1347, 1222, 848, 914, 580, 268, 660, 446, 315, 959, 1490, 809, 209, 163, 414, 547, 1125, 619, 1569, 514, 356, 382, 811, 1272, 561, 184, 606, 1541, 1128] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[2861, 568, 228, 1132, 529, 871, 685, 98, 2628, 259, 421, 1113, 32, 833, 568, 515, 624, 1144, 420, 499, 350, 251, 397, 868, 1066, 836, 274, 346, 961, 442, 1128, 1031, 1214, 586, 800, 1467, 1393, 903, 835, 601, 231, 600, 433, 393, 912, 1514, 799, 160, 179, 457, 493, 1125, 825, 1601, 563, 261, 460, 653, 1308, 603, 200, 647, 1714, 1244] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[2886, 534, 212, 1293, 376, 798, 653, 163, 2629, 264, 349, 1144, 23, 760, 515, 457, 755, 1235, 281, 583, 363, 234, 404, 895, 1088, 947, 267, 383, 893, 436, 1239, 1076, 1262, 655, 860, 1448, 1331, 764, 803, 647, 317, 673, 456, 383, 944, 1524, 856, 209, 146, 439, 543, 1115, 730, 1601, 504, 304, 432, 665, 1273, 609, 127, 590, 1596, 1211] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[2741, 562, 281, 1254, 440, 886, 723, 122, 2643, 329, 424, 1022, 31, 870, 516, 409, 797, 1240, 459, 547, 359, 303, 377, 955, 1011, 949, 222, 411, 906, 584, 1244, 1066, 1220, 596, 803, 1497, 1396, 736, 808, 589, 257, 631, 444, 304, 923, 1536, 787, 156, 171, 484, 471, 1118, 740, 1582, 488, 292, 418, 651, 1313, 543, 124, 652, 1570, 1169] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[2902, 602, 278, 1154, 392, 750, 603, 129, 2936, 230, 402, 1202, 42, 964, 470, 479, 716, 1115, 364, 536, 329, 277, 412, 1019, 1045, 923, 263, 365, 897, 410, 1121, 972, 1216, 580, 726, 1514, 1309, 795, 756, 585, 267, 690, 448, 320, 910, 1504, 825, 179, 160, 427, 493, 1079, 753, 2006, 514, 264, 397, 701, 1339, 607, 140, 624, 1663, 1062] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[2782, 582, 228, 1171, 377, 791, 720, 134, 2771, 266, 454, 975, 22, 974, 531, 426, 759, 1139, 425, 605, 318, 291, 361, 988, 988, 899, 287, 430, 820, 539, 1139, 928, 1256, 583, 846, 1453, 1392, 812, 778, 536, 282, 596, 418, 334, 925, 1554, 829, 193, 158, 415, 547, 1002, 695, 1898, 535, 334, 420, 676, 1296, 597, 146, 634, 1753, 1139] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[2931, 585, 219, 1065, 312, 786, 657, 148, 2773, 260, 475, 1026, 28, 978, 521, 386, 676, 1116, 383, 544, 342, 309, 446, 990, 906, 797, 288, 383, 886, 521, 1128, 976, 1339, 592, 853, 1397, 1457, 848, 881, 570, 262, 759, 451, 349, 895, 1431, 822, 154, 149, 463, 542, 1078, 737, 1982, 509, 267, 427, 688, 1354, 544, 166, 675, 1630, 1040] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[2704, 547, 254, 977, 310, 812, 783, 168, 2557, 274, 427, 946, 41, 941, 548, 437, 740, 1101, 443, 545, 354, 267, 387, 967, 983, 942, 314, 372, 1036, 517, 1158, 1064, 1121, 617, 820, 1439, 1342, 797, 844, 671, 329, 671, 453, 341, 997, 1529, 746, 164, 158, 438, 516, 1013, 711, 1872, 550, 373, 392, 677, 1385, 569, 207, 666, 1628, 1200] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2179, 568, 228, 1132, 529, 871, 685, 98, 0, 0, 0, 0, 0, 0, 0, 0, 2419, 259, 421, 1113, 32, 833, 568, 515, 0, 0, 0, 0, 0, 0, 0, 0, 624, 1144, 420, 499, 350, 251, 397, 868, 1335, 0, 0, 0, 0, 0, 0, 0, 1066, 836, 274, 346, 961, 442, 1079, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1214, 586, 800, 0, 1287, 903, 835, 601, 0, 0, 0, 0, 0, 0, 0, 0, 231, 600, 433, 393, 912, 1514, 799, 160, 682, 201, 132, 0, 0, 0, 0, 0, 179, 457, 493, 1125, 825, 1601, 563, 261, 262, 209, 106, 49, 0, 0, 0, 0, 460, 653, 1308, 603, 200, 647, 1251, 1244, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6290, 6160, 5888, 6035, 6226, 6057, 6130, 6366] avg=6144.0000 max=6366 max_rank=7 min=5888 min_rank=2 imbalance_ratio=1.036133 max_load_reduction=42786 max_load_reduction_pct=0.870483 moved_to_echo=24779 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2210, 602, 278, 1154, 392, 750, 603, 129, 0, 0, 0, 0, 0, 0, 0, 0, 2702, 230, 402, 1202, 42, 964, 470, 479, 0, 0, 0, 0, 0, 0, 0, 0, 716, 1115, 364, 536, 329, 277, 412, 1019, 1379, 0, 0, 0, 0, 0, 0, 0, 1045, 923, 263, 365, 897, 410, 1072, 972, 0, 0, 0, 0, 0, 0, 0, 0, 1216, 580, 726, 0, 1209, 795, 756, 585, 0, 0, 0, 0, 0, 0, 0, 0, 267, 690, 448, 320, 910, 1504, 825, 179, 692, 195, 135, 0, 0, 0, 0, 0, 160, 427, 493, 1079, 753, 2006, 514, 264, 255, 234, 100, 49, 0, 0, 0, 0, 397, 701, 1339, 607, 140, 624, 1213, 1062, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6118, 6491, 6147, 5947, 5867, 6165, 6334, 6083] avg=6144.0000 max=6491 max_rank=1 min=5867 min_rank=4 imbalance_ratio=1.056478 max_load_reduction=42661 max_load_reduction_pct=0.867940 moved_to_echo=24449 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2018, 579, 279, 1113, 409, 795, 770, 164, 0, 0, 0, 0, 0, 0, 0, 0, 2479, 277, 417, 1087, 55, 800, 560, 473, 0, 0, 0, 0, 0, 0, 0, 0, 871, 1142, 390, 539, 405, 298, 408, 966, 1227, 0, 0, 0, 0, 0, 0, 0, 1087, 916, 331, 416, 1084, 431, 1084, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 1121, 663, 911, 0, 1129, 848, 914, 580, 0, 0, 0, 0, 0, 0, 0, 0, 268, 660, 446, 315, 959, 1490, 809, 209, 632, 180, 120, 0, 0, 0, 0, 0, 163, 414, 547, 1125, 619, 1569, 514, 356, 236, 214, 93, 49, 0, 0, 0, 0, 382, 811, 1272, 561, 184, 606, 1125, 1128, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6127, 6148, 6246, 6409, 6166, 6088, 5899, 6069] avg=6144.0000 max=6409 max_rank=3 min=5899 min_rank=6 imbalance_ratio=1.043132 max_load_reduction=42743 max_load_reduction_pct=0.869609 moved_to_echo=24222 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2198, 534, 212, 1293, 376, 798, 653, 163, 0, 0, 0, 0, 0, 0, 0, 0, 2420, 264, 349, 1144, 23, 760, 515, 457, 0, 0, 0, 0, 0, 0, 0, 0, 755, 1235, 281, 583, 363, 234, 404, 895, 1318, 0, 0, 0, 0, 0, 0, 0, 1088, 947, 267, 383, 893, 436, 1185, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 1262, 655, 860, 0, 1229, 764, 803, 647, 0, 0, 0, 0, 0, 0, 0, 0, 317, 673, 456, 383, 944, 1524, 856, 209, 688, 187, 130, 0, 0, 0, 0, 0, 146, 439, 543, 1115, 730, 1601, 504, 304, 244, 209, 102, 54, 0, 0, 0, 0, 432, 665, 1273, 609, 127, 590, 1165, 1211, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6227, 5932, 6068, 6275, 6220, 6367, 5991, 6072] avg=6144.0000 max=6367 max_rank=5 min=5932 min_rank=1 imbalance_ratio=1.036296 max_load_reduction=42785 max_load_reduction_pct=0.870463 moved_to_echo=24650 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2119, 582, 228, 1171, 377, 791, 720, 134, 0, 0, 0, 0, 0, 0, 0, 0, 2550, 266, 454, 975, 22, 974, 531, 426, 0, 0, 0, 0, 0, 0, 0, 0, 759, 1139, 425, 605, 318, 291, 361, 988, 1323, 0, 0, 0, 0, 0, 0, 0, 988, 899, 287, 430, 820, 539, 1089, 928, 0, 0, 0, 0, 0, 0, 0, 0, 1256, 583, 846, 0, 1286, 812, 778, 536, 0, 0, 0, 0, 0, 0, 0, 0, 282, 596, 418, 334, 925, 1554, 829, 193, 663, 205, 130, 0, 0, 0, 0, 0, 158, 415, 547, 1002, 695, 1898, 535, 334, 268, 221, 106, 50, 0, 0, 0, 0, 420, 676, 1296, 597, 146, 634, 1280, 1139, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6122, 6198, 6209, 5980, 6097, 6129, 6229, 6188] avg=6144.0000 max=6229 max_rank=6 min=5980 min_rank=3 imbalance_ratio=1.013835 max_load_reduction=42923 max_load_reduction_pct=0.873271 moved_to_echo=24643 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2088, 562, 281, 1254, 440, 886, 723, 122, 0, 0, 0, 0, 0, 0, 0, 0, 2433, 329, 424, 1022, 31, 870, 516, 409, 0, 0, 0, 0, 0, 0, 0, 0, 797, 1240, 459, 547, 359, 303, 377, 955, 1364, 0, 0, 0, 0, 0, 0, 0, 1011, 949, 222, 411, 906, 584, 1190, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 1220, 596, 803, 0, 1290, 736, 808, 589, 0, 0, 0, 0, 0, 0, 0, 0, 257, 631, 444, 304, 923, 1536, 787, 156, 653, 184, 133, 0, 0, 0, 0, 0, 171, 484, 471, 1118, 740, 1582, 488, 292, 240, 210, 106, 54, 0, 0, 0, 0, 418, 651, 1313, 543, 124, 652, 1146, 1169, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6356, 6034, 6401, 6339, 6042, 6008, 5956, 6016] avg=6144.0000 max=6401 max_rank=2 min=5956 min_rank=6 imbalance_ratio=1.041829 max_load_reduction=42751 max_load_reduction_pct=0.869771 moved_to_echo=24022 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2229, 585, 219, 1065, 312, 786, 657, 148, 0, 0, 0, 0, 0, 0, 0, 0, 2548, 260, 475, 1026, 28, 978, 521, 386, 0, 0, 0, 0, 0, 0, 0, 0, 676, 1116, 383, 544, 342, 309, 446, 990, 1273, 0, 0, 0, 0, 0, 0, 0, 906, 797, 288, 383, 886, 521, 1074, 976, 0, 0, 0, 0, 0, 0, 0, 0, 1339, 592, 853, 0, 1341, 848, 881, 570, 0, 0, 0, 0, 0, 0, 0, 0, 262, 759, 451, 349, 895, 1431, 822, 154, 702, 195, 124, 0, 0, 0, 0, 0, 149, 463, 542, 1078, 737, 1982, 509, 267, 255, 225, 116, 54, 0, 0, 0, 0, 427, 688, 1354, 544, 166, 675, 1180, 1040, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6001, 6222, 6079, 5831, 6424, 6144, 6377, 6074] avg=6144.0000 max=6424 max_rank=4 min=5831 min_rank=3 imbalance_ratio=1.045573 max_load_reduction=42728 max_load_reduction_pct=0.869303 moved_to_echo=25019 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2060, 547, 254, 977, 310, 812, 783, 168, 0, 0, 0, 0, 0, 0, 0, 0, 2353, 274, 427, 946, 41, 941, 548, 437, 0, 0, 0, 0, 0, 0, 0, 0, 740, 1101, 443, 545, 354, 267, 387, 967, 1310, 0, 0, 0, 0, 0, 0, 0, 983, 942, 314, 372, 1036, 517, 1108, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 1121, 617, 820, 0, 1240, 797, 844, 671, 0, 0, 0, 0, 0, 0, 0, 0, 329, 671, 453, 341, 997, 1529, 746, 164, 644, 191, 129, 0, 0, 0, 0, 0, 158, 438, 516, 1013, 711, 1872, 550, 373, 249, 204, 102, 50, 0, 0, 0, 0, 392, 677, 1385, 569, 207, 666, 1188, 1200, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5911, 5967, 6114, 6336, 6110, 6194, 6236, 6284] avg=6144.0000 max=6336 max_rank=3 min=5911 min_rank=0 imbalance_ratio=1.031250 max_load_reduction=42816 max_load_reduction_pct=0.871094 moved_to_echo=24824 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[19968, 2176, 3456, 8576, 384, 7168, 4352, 3584, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6016, 9344, 3200, 4480, 2944, 2304, 3200, 7680, 10624, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[2304, 5376, 3584, 2816, 7552, 12160, 6528, 1536, 5376, 1664, 1152, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[9856, 4992, 6656, 0, 10112, 6528, 6656, 4864, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[8192, 7296, 2304, 3200, 7552, 3968, 8960, 8192, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[17152, 4608, 2048, 9216, 3200, 6528, 5632, 1152, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3328, 5632, 10624, 4736, 1408, 5120, 9600, 9216, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[1408, 3584, 4224, 8704, 5888, 14208, 4224, 2560, 2048, 1792, 896, 512, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[360, 915, 363, 661, 2422, 90, 4766, 287, 1328, 603, 330, 955, 554, 1058, 51, 1349, 580, 1094, 1253, 1898, 19, 533, 13, 377, 110, 734, 897, 1962, 39, 578, 242, 2566, 45, 17, 676, 214, 502, 23, 265, 149, 446, 527, 320, 825, 225, 158, 263, 1380, 2076, 221, 1460, 550, 477, 161, 818, 116, 1008, 1015, 1446, 1452, 712, 527, 1093, 998] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[320, 864, 353, 551, 2288, 119, 4639, 325, 992, 600, 351, 1173, 555, 1049, 41, 1363, 531, 1003, 1161, 1838, 11, 614, 17, 318, 94, 705, 904, 2454, 31, 640, 210, 2407, 96, 38, 522, 155, 518, 13, 369, 137, 651, 606, 296, 871, 204, 155, 285, 1379, 2722, 224, 1573, 373, 487, 150, 869, 104, 875, 955, 1608, 1210, 576, 533, 1168, 909] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[299, 873, 397, 650, 2240, 92, 4696, 311, 1156, 670, 286, 1011, 519, 1184, 78, 1390, 504, 992, 819, 1741, 29, 561, 18, 363, 134, 650, 839, 2100, 58, 650, 214, 2473, 121, 27, 580, 266, 486, 27, 417, 128, 766, 587, 288, 921, 234, 147, 284, 1332, 2224, 221, 1880, 500, 487, 153, 884, 107, 820, 978, 1537, 1497, 577, 569, 1119, 991] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[327, 823, 344, 660, 2230, 119, 4431, 269, 1238, 670, 256, 1062, 570, 1074, 50, 1417, 546, 944, 867, 2023, 38, 550, 22, 419, 99, 718, 960, 2418, 29, 600, 217, 2516, 95, 31, 568, 225, 537, 28, 360, 136, 799, 537, 324, 851, 172, 190, 292, 1324, 2287, 231, 1618, 432, 508, 185, 985, 127, 930, 936, 1460, 1309, 577, 561, 1021, 1010] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[314, 952, 353, 620, 2315, 77, 4498, 321, 1257, 646, 396, 1038, 554, 1114, 57, 1378, 562, 951, 964, 1708, 43, 565, 14, 392, 121, 797, 1021, 2194, 50, 661, 230, 2240, 139, 33, 603, 234, 561, 27, 400, 153, 709, 575, 309, 909, 257, 150, 326, 1341, 2115, 265, 1770, 363, 439, 187, 950, 139, 799, 991, 1409, 1309, 721, 579, 988, 999] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[374, 935, 344, 643, 2446, 83, 4633, 316, 1109, 637, 331, 942, 511, 1053, 44, 1509, 579, 991, 1245, 1930, 33, 607, 13, 337, 114, 803, 903, 2316, 46, 555, 259, 2318, 111, 38, 510, 152, 532, 14, 294, 130, 744, 599, 289, 814, 208, 151, 315, 1354, 2546, 240, 1786, 376, 465, 172, 835, 148, 867, 1025, 1374, 989, 569, 550, 1112, 884] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[324, 898, 370, 576, 2344, 103, 4316, 323, 1091, 631, 374, 981, 536, 1156, 68, 1412, 596, 1023, 890, 1844, 40, 628, 26, 380, 98, 767, 985, 2285, 69, 675, 282, 2368, 134, 54, 559, 202, 594, 26, 335, 145, 886, 585, 316, 829, 190, 160, 297, 1245, 2433, 248, 1769, 418, 427, 168, 910, 156, 842, 1025, 1326, 1263, 649, 539, 1049, 914] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[348, 886, 296, 639, 2380, 84, 4530, 326, 986, 551, 325, 1119, 542, 1038, 63, 1525, 555, 1000, 1294, 2023, 22, 652, 28, 380, 128, 729, 927, 2184, 43, 642, 248, 2341, 121, 21, 501, 183, 513, 18, 380, 121, 848, 529, 289, 814, 218, 151, 274, 1177, 2611, 241, 1904, 390, 482, 156, 853, 133, 955, 1013, 1306, 969, 577, 496, 1078, 996] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[320, 864, 353, 551, 2288, 119, 1222, 325, 0, 0, 0, 0, 0, 0, 0, 0, 992, 600, 351, 1173, 555, 1049, 41, 1267, 0, 0, 0, 0, 0, 0, 0, 0, 531, 1003, 1161, 1838, 11, 614, 17, 318, 308, 184, 96, 82, 0, 0, 0, 0, 94, 705, 904, 2454, 31, 640, 210, 1220, 0, 0, 0, 0, 0, 0, 0, 0, 96, 38, 522, 155, 518, 13, 369, 137, 3417, 931, 0, 0, 0, 0, 0, 0, 651, 606, 296, 871, 204, 155, 285, 1379, 1105, 677, 0, 0, 0, 0, 0, 0, 2414, 224, 1573, 373, 487, 150, 869, 104, 0, 0, 0, 0, 0, 0, 0, 0, 875, 955, 0, 1026, 576, 533, 1168, 909, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6042, 6028, 6163, 6258, 6196, 6229, 6194, 6042] avg=6144.0000 max=6258 max_rank=3 min=6028 min_rank=1 imbalance_ratio=1.018555 max_load_reduction=42894 max_load_reduction_pct=0.872681 moved_to_echo=24661 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[327, 823, 344, 660, 2230, 119, 1167, 269, 0, 0, 0, 0, 0, 0, 0, 0, 1238, 670, 256, 1062, 570, 1074, 50, 1317, 0, 0, 0, 0, 0, 0, 0, 0, 546, 944, 867, 2023, 38, 550, 22, 419, 258, 199, 100, 86, 0, 0, 0, 0, 99, 718, 960, 2418, 29, 600, 217, 1275, 0, 0, 0, 0, 0, 0, 0, 0, 95, 31, 568, 225, 537, 28, 360, 136, 3264, 847, 0, 0, 0, 0, 0, 0, 799, 537, 324, 851, 172, 190, 292, 1324, 1155, 613, 0, 0, 0, 0, 0, 0, 2029, 231, 1618, 432, 508, 185, 985, 127, 0, 0, 0, 0, 0, 0, 0, 0, 930, 936, 0, 1110, 577, 561, 1021, 1010, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5939, 6237, 6052, 6316, 6091, 6257, 6115, 6145] avg=6144.0000 max=6316 max_rank=3 min=5939 min_rank=0 imbalance_ratio=1.027995 max_load_reduction=42836 max_load_reduction_pct=0.871501 moved_to_echo=24608 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[360, 915, 363, 661, 2422, 90, 1255, 287, 0, 0, 0, 0, 0, 0, 0, 0, 1328, 603, 330, 955, 554, 1058, 51, 1254, 0, 0, 0, 0, 0, 0, 0, 0, 580, 1094, 1253, 1898, 19, 533, 13, 377, 235, 220, 95, 88, 0, 0, 0, 0, 110, 734, 897, 1962, 39, 578, 242, 1300, 0, 0, 0, 0, 0, 0, 0, 0, 45, 17, 676, 214, 502, 23, 265, 149, 3511, 837, 0, 0, 0, 0, 0, 0, 446, 527, 320, 825, 225, 158, 263, 1380, 1178, 609, 0, 0, 0, 0, 0, 0, 1841, 221, 1460, 550, 477, 161, 818, 116, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1015, 0, 1232, 712, 527, 1093, 998, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6353, 6133, 6405, 5862, 6239, 5931, 5644, 6585] avg=6144.0000 max=6585 max_rank=7 min=5644 min_rank=6 imbalance_ratio=1.071777 max_load_reduction=42567 max_load_reduction_pct=0.866028 moved_to_echo=24399 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[299, 873, 397, 650, 2240, 92, 1237, 311, 0, 0, 0, 0, 0, 0, 0, 0, 1156, 670, 286, 1011, 519, 1184, 78, 1292, 0, 0, 0, 0, 0, 0, 0, 0, 504, 992, 819, 1741, 29, 561, 18, 363, 251, 227, 98, 85, 0, 0, 0, 0, 134, 650, 839, 2100, 58, 650, 214, 1253, 0, 0, 0, 0, 0, 0, 0, 0, 121, 27, 580, 266, 486, 27, 417, 128, 3459, 890, 0, 0, 0, 0, 0, 0, 766, 587, 288, 921, 234, 147, 284, 1332, 1135, 647, 0, 0, 0, 0, 0, 0, 1973, 221, 1880, 500, 487, 153, 884, 107, 0, 0, 0, 0, 0, 0, 0, 0, 820, 978, 0, 1270, 577, 569, 1119, 991, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6099, 6196, 5688, 5898, 6401, 6341, 6205, 6324] avg=6144.0000 max=6401 max_rank=4 min=5688 min_rank=2 imbalance_ratio=1.041829 max_load_reduction=42751 max_load_reduction_pct=0.869771 moved_to_echo=25271 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[314, 952, 353, 620, 2315, 77, 1184, 321, 0, 0, 0, 0, 0, 0, 0, 0, 1257, 646, 396, 1038, 554, 1114, 57, 1281, 0, 0, 0, 0, 0, 0, 0, 0, 562, 951, 964, 1708, 43, 565, 14, 392, 239, 199, 97, 77, 0, 0, 0, 0, 121, 797, 1021, 2194, 50, 661, 230, 1135, 0, 0, 0, 0, 0, 0, 0, 0, 139, 33, 603, 234, 561, 27, 400, 153, 3314, 817, 0, 0, 0, 0, 0, 0, 709, 575, 309, 909, 257, 150, 326, 1341, 1028, 592, 0, 0, 0, 0, 0, 0, 1876, 265, 1770, 363, 439, 187, 950, 139, 0, 0, 0, 0, 0, 0, 0, 0, 799, 991, 0, 1110, 721, 579, 988, 999, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6136, 6343, 5811, 6209, 6281, 6196, 5989, 6187] avg=6144.0000 max=6343 max_rank=1 min=5811 min_rank=2 imbalance_ratio=1.032389 max_load_reduction=42809 max_load_reduction_pct=0.870951 moved_to_echo=24653 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[374, 935, 344, 643, 2446, 83, 1215, 316, 0, 0, 0, 0, 0, 0, 0, 0, 1109, 637, 331, 942, 511, 1053, 44, 1398, 0, 0, 0, 0, 0, 0, 0, 0, 579, 991, 1245, 1930, 33, 607, 13, 337, 292, 153, 111, 83, 0, 0, 0, 0, 114, 803, 903, 2316, 46, 555, 259, 1165, 0, 0, 0, 0, 0, 0, 0, 0, 111, 38, 510, 152, 532, 14, 294, 130, 3418, 797, 0, 0, 0, 0, 0, 0, 744, 599, 289, 814, 208, 151, 315, 1354, 1070, 577, 0, 0, 0, 0, 0, 0, 2254, 240, 1786, 376, 465, 172, 835, 148, 0, 0, 0, 0, 0, 0, 0, 0, 867, 1025, 0, 836, 569, 550, 1112, 884, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6356, 6025, 6374, 6161, 5996, 6121, 6276, 5843] avg=6144.0000 max=6374 max_rank=2 min=5843 min_rank=7 imbalance_ratio=1.037435 max_load_reduction=42778 max_load_reduction_pct=0.870321 moved_to_echo=24236 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[348, 886, 296, 639, 2380, 84, 1193, 326, 0, 0, 0, 0, 0, 0, 0, 0, 986, 551, 325, 1119, 542, 1038, 63, 1417, 0, 0, 0, 0, 0, 0, 0, 0, 555, 1000, 1294, 2023, 22, 652, 28, 380, 295, 147, 108, 80, 0, 0, 0, 0, 128, 729, 927, 2184, 43, 642, 248, 1186, 0, 0, 0, 0, 0, 0, 0, 0, 121, 21, 501, 183, 513, 18, 380, 121, 3337, 757, 0, 0, 0, 0, 0, 0, 848, 529, 289, 814, 218, 151, 274, 1177, 1075, 549, 0, 0, 0, 0, 0, 0, 2316, 241, 1904, 390, 482, 156, 853, 133, 0, 0, 0, 0, 0, 0, 0, 0, 955, 1013, 0, 822, 577, 496, 1078, 996, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6152, 6041, 6584, 6087, 5952, 5924, 6475, 5937] avg=6144.0000 max=6584 max_rank=2 min=5924 min_rank=5 imbalance_ratio=1.071615 max_load_reduction=42568 max_load_reduction_pct=0.866048 moved_to_echo=24288 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[324, 898, 370, 576, 2344, 103, 1137, 323, 0, 0, 0, 0, 0, 0, 0, 0, 1091, 631, 374, 981, 536, 1156, 68, 1312, 0, 0, 0, 0, 0, 0, 0, 0, 596, 1023, 890, 1844, 40, 628, 26, 380, 275, 192, 100, 81, 0, 0, 0, 0, 98, 767, 985, 2285, 69, 675, 282, 1200, 0, 0, 0, 0, 0, 0, 0, 0, 134, 54, 559, 202, 594, 26, 335, 145, 3179, 768, 0, 0, 0, 0, 0, 0, 886, 585, 316, 829, 190, 160, 297, 1245, 1087, 558, 0, 0, 0, 0, 0, 0, 2158, 248, 1769, 418, 427, 168, 910, 156, 0, 0, 0, 0, 0, 0, 0, 0, 842, 1025, 0, 1071, 649, 539, 1049, 914, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6075, 6149, 6075, 6361, 5996, 6153, 6254, 6089] avg=6144.0000 max=6361 max_rank=3 min=5996 min_rank=4 imbalance_ratio=1.035319 max_load_reduction=42791 max_load_reduction_pct=0.870585 moved_to_echo=24492 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2688, 7168, 2944, 5120, 18688, 768, 9728, 2560, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[7168, 8064, 0, 8576, 4992, 4480, 8704, 7808, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[4480, 8064, 8576, 15104, 256, 4736, 256, 3072, 2176, 1536, 896, 768, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[9216, 5120, 2688, 8320, 4352, 8832, 512, 10624, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5888, 4608, 2432, 6912, 1792, 1280, 2432, 10624, 8960, 4864, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[16896, 1920, 13824, 3456, 3840, 1408, 7168, 1152, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1024, 6016, 7552, 17920, 384, 5120, 1920, 9856, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[896, 384, 4608, 1664, 4352, 256, 2944, 1152, 27008, 6656, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[699, 259, 438, 1463, 1047, 69, 1029, 611, 1152, 858, 1954, 728, 694, 1595, 122, 254, 3213, 879, 4398, 239, 570, 37, 1392, 1680, 375, 630, 502, 153, 289, 1152, 759, 1149, 412, 280, 882, 1061, 249, 456, 367, 300, 1056, 523, 19, 212, 258, 669, 372, 77, 3042, 1241, 402, 920, 155, 1144, 945, 445, 490, 1134, 297, 27, 237, 296, 14, 781] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[560, 233, 378, 1223, 850, 47, 1229, 832, 950, 1068, 1937, 799, 643, 1449, 105, 274, 3481, 866, 4710, 265, 424, 19, 1534, 1844, 417, 559, 630, 229, 289, 1262, 805, 831, 366, 288, 996, 1120, 372, 348, 235, 172, 1155, 542, 14, 284, 206, 700, 328, 38, 2767, 1189, 416, 930, 69, 937, 1173, 313, 595, 1003, 261, 22, 267, 314, 8, 982] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[558, 265, 541, 1450, 944, 50, 1172, 758, 1075, 1008, 1986, 759, 665, 1514, 132, 303, 3458, 813, 4259, 326, 461, 24, 1510, 1740, 419, 612, 492, 213, 271, 1175, 743, 1045, 364, 284, 973, 1250, 214, 465, 455, 248, 1045, 483, 15, 256, 236, 639, 353, 44, 2648, 1254, 402, 919, 115, 1017, 1024, 400, 568, 1098, 282, 38, 226, 297, 7, 792] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[634, 276, 524, 1457, 900, 61, 1169, 691, 1239, 1025, 1864, 652, 651, 1410, 138, 213, 3521, 874, 4314, 321, 577, 22, 1359, 1741, 404, 616, 487, 137, 216, 1215, 807, 959, 327, 267, 801, 1124, 292, 520, 354, 165, 1230, 507, 7, 284, 242, 661, 310, 52, 3155, 1322, 344, 881, 128, 976, 952, 349, 597, 853, 346, 14, 259, 300, 12, 1047] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[529, 270, 519, 1392, 761, 65, 1197, 808, 1007, 907, 1913, 793, 634, 1308, 124, 289, 3668, 880, 4783, 315, 437, 34, 1617, 1768, 298, 596, 516, 145, 256, 1265, 837, 921, 378, 269, 913, 1012, 386, 397, 362, 144, 1048, 455, 10, 234, 193, 735, 348, 65, 2871, 1113, 508, 947, 89, 934, 994, 439, 618, 1159, 215, 35, 214, 277, 11, 927] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[370, 360, 318, 1254, 767, 67, 1216, 739, 1149, 844, 1826, 777, 624, 1489, 100, 342, 3734, 1029, 4392, 309, 560, 40, 1580, 1782, 468, 630, 640, 172, 270, 1359, 978, 728, 386, 219, 758, 1013, 431, 430, 196, 151, 1193, 451, 14, 245, 207, 747, 276, 71, 3132, 1174, 243, 1022, 93, 876, 1101, 351, 533, 973, 343, 30, 212, 302, 13, 1053] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[499, 287, 514, 1342, 959, 65, 1046, 677, 1190, 779, 1747, 770, 625, 1496, 169, 224, 3230, 1120, 4434, 361, 565, 32, 1359, 1641, 369, 719, 638, 190, 247, 1222, 886, 899, 336, 322, 858, 1301, 263, 438, 405, 251, 1174, 587, 24, 283, 270, 763, 319, 68, 2720, 1137, 450, 906, 131, 1054, 1198, 338, 593, 925, 361, 41, 209, 298, 19, 809] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[508, 278, 540, 1294, 768, 83, 1124, 752, 1049, 1114, 1433, 767, 706, 1459, 155, 264, 3156, 923, 4314, 326, 659, 44, 1387, 1881, 342, 689, 632, 233, 231, 1176, 913, 942, 373, 357, 893, 1127, 323, 349, 512, 204, 1133, 520, 38, 270, 284, 772, 361, 110, 2603, 1257, 462, 961, 108, 1060, 1079, 417, 612, 787, 390, 25, 272, 315, 16, 1020] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[699, 259, 438, 1463, 1047, 69, 1029, 611, 715, 0, 0, 0, 0, 0, 0, 0, 1152, 858, 870, 728, 694, 1595, 122, 254, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 879, 0, 239, 570, 37, 1392, 1680, 0, 0, 0, 0, 0, 0, 0, 0, 375, 630, 502, 153, 289, 1152, 759, 1149, 813, 369, 0, 0, 0, 0, 0, 0, 412, 280, 882, 1061, 249, 456, 367, 300, 1044, 1121, 0, 0, 0, 0, 0, 0, 1056, 523, 19, 212, 258, 669, 372, 77, 2849, 0, 0, 0, 0, 0, 0, 0, 1108, 1241, 402, 920, 155, 1144, 945, 445, 0, 0, 0, 0, 0, 0, 0, 0, 490, 1134, 297, 27, 237, 296, 14, 781, 1549, 1116, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6330, 6273, 5850, 6191, 6172, 6035, 6360, 5941] avg=6144.0000 max=6360 max_rank=6 min=5850 min_rank=2 imbalance_ratio=1.035156 max_load_reduction=42792 max_load_reduction_pct=0.870605 moved_to_echo=24508 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[560, 233, 378, 1223, 850, 47, 1229, 832, 709, 0, 0, 0, 0, 0, 0, 0, 950, 1068, 863, 799, 643, 1449, 105, 274, 0, 0, 0, 0, 0, 0, 0, 0, 1140, 866, 0, 265, 424, 19, 1534, 1844, 0, 0, 0, 0, 0, 0, 0, 0, 417, 559, 630, 229, 289, 1262, 805, 831, 739, 365, 0, 0, 0, 0, 0, 0, 366, 288, 996, 1120, 372, 348, 235, 172, 1132, 1020, 0, 0, 0, 0, 0, 0, 1155, 542, 14, 284, 206, 700, 328, 38, 3051, 0, 0, 0, 0, 0, 0, 0, 1008, 1189, 416, 930, 69, 937, 1173, 313, 0, 0, 0, 0, 0, 0, 0, 0, 595, 1003, 261, 22, 267, 314, 8, 982, 1659, 1209, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6061, 6151, 6092, 6126, 6049, 6318, 6035, 6320] avg=6144.0000 max=6320 max_rank=7 min=6035 min_rank=6 imbalance_ratio=1.028646 max_load_reduction=42832 max_load_reduction_pct=0.871419 moved_to_echo=24722 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[558, 265, 541, 1450, 944, 50, 1172, 758, 727, 0, 0, 0, 0, 0, 0, 0, 1075, 1008, 884, 759, 665, 1514, 132, 303, 0, 0, 0, 0, 0, 0, 0, 0, 1133, 813, 0, 326, 461, 24, 1510, 1740, 0, 0, 0, 0, 0, 0, 0, 0, 419, 612, 492, 213, 271, 1175, 743, 1045, 708, 375, 0, 0, 0, 0, 0, 0, 364, 284, 973, 1250, 214, 465, 455, 248, 1124, 976, 0, 0, 0, 0, 0, 0, 1045, 483, 15, 256, 236, 639, 353, 44, 2760, 0, 0, 0, 0, 0, 0, 0, 964, 1254, 402, 919, 115, 1017, 1024, 400, 0, 0, 0, 0, 0, 0, 0, 0, 568, 1098, 282, 38, 226, 297, 7, 792, 1499, 1201, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6465, 6340, 6007, 6053, 6353, 5831, 6095, 6008] avg=6144.0000 max=6465 max_rank=0 min=5831 min_rank=5 imbalance_ratio=1.052246 max_load_reduction=42687 max_load_reduction_pct=0.868469 moved_to_echo=24287 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[634, 276, 524, 1457, 900, 61, 1169, 691, 682, 0, 0, 0, 0, 0, 0, 0, 1239, 1025, 830, 652, 651, 1410, 138, 213, 0, 0, 0, 0, 0, 0, 0, 0, 1153, 874, 0, 321, 577, 22, 1359, 1741, 0, 0, 0, 0, 0, 0, 0, 0, 404, 616, 487, 137, 216, 1215, 807, 959, 843, 352, 0, 0, 0, 0, 0, 0, 327, 267, 801, 1124, 292, 520, 354, 165, 1145, 1163, 0, 0, 0, 0, 0, 0, 1230, 507, 7, 284, 242, 661, 310, 52, 2794, 0, 0, 0, 0, 0, 0, 0, 1149, 1322, 344, 881, 128, 976, 952, 349, 0, 0, 0, 0, 0, 0, 0, 0, 597, 853, 346, 14, 259, 300, 12, 1047, 1520, 1223, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6394, 6158, 6047, 6036, 6158, 6087, 6101, 6171] avg=6144.0000 max=6394 max_rank=0 min=6036 min_rank=3 imbalance_ratio=1.040690 max_load_reduction=42758 max_load_reduction_pct=0.869914 moved_to_echo=24517 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[529, 270, 519, 1392, 761, 65, 1197, 808, 703, 0, 0, 0, 0, 0, 0, 0, 1007, 907, 846, 793, 634, 1308, 124, 289, 0, 0, 0, 0, 0, 0, 0, 0, 1192, 880, 0, 315, 437, 34, 1617, 1768, 0, 0, 0, 0, 0, 0, 0, 0, 298, 596, 516, 145, 256, 1265, 837, 921, 771, 364, 0, 0, 0, 0, 0, 0, 378, 269, 913, 1012, 386, 397, 362, 144, 1196, 1062, 0, 0, 0, 0, 0, 0, 1048, 455, 10, 234, 193, 735, 348, 65, 3099, 0, 0, 0, 0, 0, 0, 0, 1038, 1113, 508, 947, 89, 934, 994, 439, 0, 0, 0, 0, 0, 0, 0, 0, 618, 1159, 215, 35, 214, 277, 11, 927, 1684, 1280, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6244, 5908, 6243, 5969, 6119, 6187, 6062, 6420] avg=6144.0000 max=6420 max_rank=7 min=5908 min_rank=1 imbalance_ratio=1.044922 max_load_reduction=42732 max_load_reduction_pct=0.869385 moved_to_echo=24788 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[499, 287, 514, 1342, 959, 65, 1046, 677, 639, 0, 0, 0, 0, 0, 0, 0, 1190, 779, 778, 770, 625, 1496, 169, 224, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1120, 0, 361, 565, 32, 1359, 1641, 0, 0, 0, 0, 0, 0, 0, 0, 369, 719, 638, 190, 247, 1222, 886, 899, 727, 330, 0, 0, 0, 0, 0, 0, 336, 322, 858, 1301, 263, 438, 405, 251, 1050, 1002, 0, 0, 0, 0, 0, 0, 1174, 587, 24, 283, 270, 763, 319, 68, 2873, 0, 0, 0, 0, 0, 0, 0, 991, 1137, 450, 906, 131, 1054, 1198, 338, 0, 0, 0, 0, 0, 0, 0, 0, 593, 925, 361, 41, 209, 298, 19, 809, 1561, 1122, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6028, 6031, 6136, 6227, 6226, 6361, 6205, 5938] avg=6144.0000 max=6361 max_rank=5 min=5938 min_rank=7 imbalance_ratio=1.035319 max_load_reduction=42791 max_load_reduction_pct=0.870585 moved_to_echo=24730 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[370, 360, 318, 1254, 767, 67, 1216, 739, 668, 0, 0, 0, 0, 0, 0, 0, 1149, 844, 813, 777, 624, 1489, 100, 342, 0, 0, 0, 0, 0, 0, 0, 0, 1223, 1029, 0, 309, 560, 40, 1580, 1782, 0, 0, 0, 0, 0, 0, 0, 0, 468, 630, 640, 172, 270, 1359, 978, 728, 837, 345, 0, 0, 0, 0, 0, 0, 386, 219, 758, 1013, 431, 430, 196, 151, 1214, 1154, 0, 0, 0, 0, 0, 0, 1193, 451, 14, 245, 207, 747, 276, 71, 2846, 0, 0, 0, 0, 0, 0, 0, 1141, 1174, 243, 1022, 93, 876, 1101, 351, 0, 0, 0, 0, 0, 0, 0, 0, 533, 973, 343, 30, 212, 302, 13, 1053, 1546, 1297, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5759, 6138, 6523, 6427, 5952, 6050, 6001, 6302] avg=6144.0000 max=6523 max_rank=2 min=5759 min_rank=0 imbalance_ratio=1.061686 max_load_reduction=42629 max_load_reduction_pct=0.867289 moved_to_echo=24305 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[508, 278, 540, 1294, 768, 83, 1124, 752, 524, 0, 0, 0, 0, 0, 0, 0, 1049, 1114, 639, 767, 706, 1459, 155, 264, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 923, 0, 326, 659, 44, 1387, 1881, 0, 0, 0, 0, 0, 0, 0, 0, 342, 689, 632, 233, 231, 1176, 913, 942, 695, 270, 0, 0, 0, 0, 0, 0, 373, 357, 893, 1127, 323, 349, 512, 204, 1026, 959, 0, 0, 0, 0, 0, 0, 1133, 520, 38, 270, 284, 772, 361, 110, 2795, 0, 0, 0, 0, 0, 0, 0, 949, 1257, 462, 961, 108, 1060, 1079, 417, 0, 0, 0, 0, 0, 0, 0, 0, 612, 787, 390, 25, 272, 315, 16, 1020, 1519, 1096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5871, 6153, 6254, 6123, 6123, 6283, 6293, 6052] avg=6144.0000 max=6293 max_rank=6 min=5871 min_rank=0 imbalance_ratio=1.024251 max_load_reduction=42859 max_load_reduction_pct=0.871969 moved_to_echo=24751 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[9088, 7424, 0, 2560, 4352, 256, 11776, 14080, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4480, 2304, 3840, 10880, 7040, 512, 9216, 5888, 5376, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[8448, 9728, 3328, 7552, 896, 8064, 8576, 3072, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3200, 5120, 4608, 1536, 2176, 9856, 6784, 7552, 6144, 2816, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[4608, 7936, 2560, 256, 1920, 2432, 128, 7424, 12544, 9600, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[8832, 7680, 6528, 6144, 5248, 11776, 1152, 2176, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2944, 2304, 7168, 9088, 2560, 3456, 2944, 1664, 8960, 8576, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[9088, 4096, 256, 2176, 1920, 5760, 2688, 640, 23168, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[79, 67, 2, 1623, 1306, 2794, 838, 91, 917, 724, 357, 188, 400, 137, 876, 96, 417, 255, 2, 1, 1248, 139, 254, 136, 3177, 1489, 59, 770, 1366, 393, 1229, 1388, 3, 120, 2261, 427, 5054, 161, 988, 242, 455, 452, 644, 754, 359, 281, 0, 564, 749, 359, 1135, 400, 4458, 89, 612, 103, 524, 25, 53, 1194, 16, 1411, 110, 2331] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[80, 53, 5, 1756, 1178, 3087, 424, 129, 1143, 813, 394, 226, 467, 397, 527, 209, 449, 219, 3, 11, 970, 174, 580, 121, 3397, 1384, 194, 734, 843, 348, 1086, 1497, 12, 159, 2529, 458, 4610, 162, 1222, 266, 589, 426, 972, 634, 342, 320, 3, 470, 754, 465, 998, 393, 3802, 61, 949, 156, 713, 43, 34, 1298, 19, 1304, 141, 1950] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[104, 13, 3, 1471, 1399, 2870, 705, 109, 1093, 749, 369, 131, 407, 231, 865, 141, 463, 256, 0, 6, 1305, 177, 240, 126, 3155, 1349, 148, 847, 961, 425, 1183, 1515, 4, 152, 2559, 383, 4902, 204, 1003, 255, 430, 450, 656, 663, 377, 286, 1, 470, 740, 355, 1045, 403, 4271, 88, 560, 188, 553, 30, 71, 1286, 23, 1666, 117, 2145] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1, 26, 1, 2074, 1034, 3122, 279, 116, 852, 756, 286, 181, 451, 475, 580, 164, 196, 253, 1, 4, 1060, 142, 533, 127, 3470, 1749, 13, 806, 736, 331, 867, 1730, 12, 134, 2671, 621, 4988, 109, 899, 244, 550, 373, 824, 535, 310, 247, 3, 634, 997, 431, 985, 412, 4019, 86, 791, 212, 623, 62, 40, 1286, 17, 1282, 146, 2193] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[34, 63, 2, 1826, 1199, 3154, 364, 91, 897, 726, 272, 205, 511, 484, 752, 178, 259, 242, 4, 0, 1232, 129, 459, 129, 3567, 1461, 6, 829, 1024, 370, 995, 1357, 19, 209, 2346, 460, 5165, 166, 1050, 268, 522, 393, 851, 597, 353, 215, 0, 765, 803, 426, 915, 470, 4054, 85, 685, 158, 569, 51, 34, 1110, 38, 1470, 117, 1967] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[69, 63, 7, 1727, 1331, 3406, 612, 51, 927, 755, 298, 177, 391, 215, 667, 119, 382, 211, 2, 11, 1053, 134, 379, 122, 3510, 1452, 21, 869, 1019, 407, 1242, 1375, 3, 179, 2264, 386, 5155, 243, 864, 195, 429, 439, 924, 622, 296, 235, 3, 581, 888, 387, 1000, 430, 4126, 83, 725, 150, 676, 52, 27, 1185, 32, 1227, 130, 2212] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[130, 21, 6, 1612, 1508, 3469, 698, 117, 1042, 753, 386, 142, 415, 109, 940, 215, 538, 237, 5, 2, 1036, 143, 276, 160, 3103, 1368, 89, 871, 1109, 406, 1254, 1286, 4, 168, 2168, 362, 4869, 195, 1212, 232, 513, 421, 916, 625, 213, 307, 4, 493, 650, 378, 982, 461, 4149, 143, 458, 105, 544, 43, 36, 980, 49, 1500, 236, 2290] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[86, 52, 10, 1543, 1273, 2770, 699, 126, 1072, 855, 297, 341, 442, 148, 746, 265, 537, 264, 5, 3, 1105, 174, 493, 137, 3435, 1332, 123, 758, 1024, 377, 1287, 1455, 13, 286, 2008, 412, 4580, 210, 1253, 264, 618, 395, 847, 707, 362, 242, 1, 458, 790, 458, 907, 377, 3787, 80, 617, 163, 679, 40, 41, 1349, 48, 1639, 157, 2130] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[79, 67, 2, 1623, 1306, 2137, 838, 91, 0, 0, 0, 0, 0, 0, 0, 0, 917, 724, 357, 188, 400, 137, 876, 96, 1610, 393, 325, 0, 0, 0, 0, 0, 417, 255, 2, 1, 1248, 139, 254, 136, 3177, 335, 0, 0, 0, 0, 0, 0, 0, 1489, 59, 770, 1366, 393, 1229, 1215, 0, 0, 0, 0, 0, 0, 0, 0, 3, 120, 2261, 427, 1730, 161, 988, 242, 0, 0, 0, 0, 0, 0, 0, 0, 455, 452, 644, 754, 359, 281, 0, 564, 2664, 0, 0, 0, 0, 0, 0, 0, 749, 359, 1135, 400, 2848, 89, 612, 103, 0, 0, 0, 0, 0, 0, 0, 0, 524, 25, 53, 1194, 16, 1411, 110, 2331, 264, 173, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6023, 5964, 6521, 5932, 6173, 6295, 6101] avg=6144.0000 max=6521 max_rank=3 min=5932 min_rank=4 imbalance_ratio=1.061361 max_load_reduction=42631 max_load_reduction_pct=0.867330 moved_to_echo=24501 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 26, 1, 2074, 1034, 2388, 279, 116, 0, 0, 0, 0, 0, 0, 0, 0, 852, 756, 286, 181, 451, 475, 580, 164, 1452, 439, 321, 0, 0, 0, 0, 0, 196, 253, 1, 4, 1060, 142, 533, 127, 3470, 330, 0, 0, 0, 0, 0, 0, 0, 1749, 13, 806, 736, 331, 867, 1514, 0, 0, 0, 0, 0, 0, 0, 0, 12, 134, 2671, 621, 1708, 109, 899, 244, 0, 0, 0, 0, 0, 0, 0, 0, 550, 373, 824, 535, 310, 247, 3, 634, 2629, 0, 0, 0, 0, 0, 0, 0, 997, 431, 985, 412, 2567, 86, 791, 212, 0, 0, 0, 0, 0, 0, 0, 0, 623, 62, 40, 1286, 17, 1282, 146, 2193, 295, 216, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5919, 5957, 6116, 6016, 6398, 6105, 6481, 6160] avg=6144.0000 max=6481 max_rank=6 min=5919 min_rank=0 imbalance_ratio=1.054850 max_load_reduction=42671 max_load_reduction_pct=0.868144 moved_to_echo=25144 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[34, 63, 2, 1826, 1199, 2413, 364, 91, 0, 0, 0, 0, 0, 0, 0, 0, 897, 726, 272, 205, 511, 484, 752, 178, 1464, 443, 332, 0, 0, 0, 0, 0, 259, 242, 4, 0, 1232, 129, 459, 129, 3567, 342, 0, 0, 0, 0, 0, 0, 0, 1461, 6, 829, 1024, 370, 995, 1187, 0, 0, 0, 0, 0, 0, 0, 0, 19, 209, 2346, 460, 1769, 166, 1050, 268, 0, 0, 0, 0, 0, 0, 0, 0, 522, 393, 851, 597, 353, 215, 0, 765, 2722, 0, 0, 0, 0, 0, 0, 0, 803, 426, 915, 470, 2590, 85, 685, 158, 0, 0, 0, 0, 0, 0, 0, 0, 569, 51, 34, 1110, 38, 1470, 117, 1967, 298, 170, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5992, 6264, 6363, 5872, 6287, 6418, 6132, 5824] avg=6144.0000 max=6418 max_rank=5 min=5824 min_rank=7 imbalance_ratio=1.044596 max_load_reduction=42734 max_load_reduction_pct=0.869425 moved_to_echo=24661 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[80, 53, 5, 1756, 1178, 2361, 424, 129, 0, 0, 0, 0, 0, 0, 0, 0, 1143, 813, 394, 226, 467, 397, 527, 209, 1373, 434, 296, 0, 0, 0, 0, 0, 449, 219, 3, 11, 970, 174, 580, 121, 3397, 305, 0, 0, 0, 0, 0, 0, 0, 1384, 194, 734, 843, 348, 1086, 1310, 0, 0, 0, 0, 0, 0, 0, 0, 12, 159, 2529, 458, 1579, 162, 1222, 266, 0, 0, 0, 0, 0, 0, 0, 0, 589, 426, 972, 634, 342, 320, 3, 470, 2430, 0, 0, 0, 0, 0, 0, 0, 754, 465, 998, 393, 2429, 61, 949, 156, 0, 0, 0, 0, 0, 0, 0, 0, 713, 43, 34, 1298, 19, 1304, 141, 1950, 292, 187, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5986, 6279, 6229, 5899, 6387, 6186, 6205, 5981] avg=6144.0000 max=6387 max_rank=4 min=5899 min_rank=3 imbalance_ratio=1.039551 max_load_reduction=42765 max_load_reduction_pct=0.870056 moved_to_echo=24759 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[104, 13, 3, 1471, 1399, 2190, 705, 109, 0, 0, 0, 0, 0, 0, 0, 0, 1093, 749, 369, 131, 407, 231, 865, 141, 1548, 407, 320, 0, 0, 0, 0, 0, 463, 256, 0, 6, 1305, 177, 240, 126, 3155, 329, 0, 0, 0, 0, 0, 0, 0, 1349, 148, 847, 961, 425, 1183, 1322, 0, 0, 0, 0, 0, 0, 0, 0, 4, 152, 2559, 383, 1664, 204, 1003, 255, 0, 0, 0, 0, 0, 0, 0, 0, 430, 450, 656, 663, 377, 286, 1, 470, 2589, 0, 0, 0, 0, 0, 0, 0, 740, 355, 1045, 403, 2723, 88, 560, 188, 0, 0, 0, 0, 0, 0, 0, 0, 553, 30, 71, 1286, 23, 1666, 117, 2145, 273, 193, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5994, 6261, 6057, 6235, 6224, 5922, 6102, 6357] avg=6144.0000 max=6357 max_rank=7 min=5922 min_rank=5 imbalance_ratio=1.034668 max_load_reduction=42795 max_load_reduction_pct=0.870667 moved_to_echo=24605 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[69, 63, 7, 1727, 1331, 2605, 612, 51, 0, 0, 0, 0, 0, 0, 0, 0, 927, 755, 298, 177, 391, 215, 667, 119, 1490, 479, 331, 0, 0, 0, 0, 0, 382, 211, 2, 11, 1053, 134, 379, 122, 3510, 341, 0, 0, 0, 0, 0, 0, 0, 1452, 21, 869, 1019, 407, 1242, 1203, 0, 0, 0, 0, 0, 0, 0, 0, 3, 179, 2264, 386, 1766, 243, 864, 195, 0, 0, 0, 0, 0, 0, 0, 0, 429, 439, 924, 622, 296, 235, 3, 581, 2717, 0, 0, 0, 0, 0, 0, 0, 888, 387, 1000, 430, 2636, 83, 725, 150, 0, 0, 0, 0, 0, 0, 0, 0, 676, 52, 27, 1185, 32, 1227, 130, 2212, 322, 172, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6465, 5849, 6145, 6213, 5900, 6246, 6299, 6035] avg=6144.0000 max=6465 max_rank=0 min=5849 min_rank=1 imbalance_ratio=1.052246 max_load_reduction=42687 max_load_reduction_pct=0.868469 moved_to_echo=24480 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[130, 21, 6, 1612, 1508, 2653, 698, 117, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 753, 386, 142, 415, 109, 940, 215, 1499, 488, 313, 0, 0, 0, 0, 0, 538, 237, 5, 2, 1036, 143, 276, 160, 3103, 322, 0, 0, 0, 0, 0, 0, 0, 1368, 89, 871, 1109, 406, 1254, 1125, 0, 0, 0, 0, 0, 0, 0, 0, 4, 168, 2168, 362, 1668, 195, 1212, 232, 0, 0, 0, 0, 0, 0, 0, 0, 513, 421, 916, 625, 213, 307, 4, 493, 2566, 0, 0, 0, 0, 0, 0, 0, 650, 378, 982, 461, 2650, 143, 458, 105, 0, 0, 0, 0, 0, 0, 0, 0, 544, 43, 36, 980, 49, 1500, 236, 2290, 328, 161, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6745, 6302, 5822, 6222, 6009, 6058, 5827, 6167] avg=6144.0000 max=6745 max_rank=0 min=5822 min_rank=2 imbalance_ratio=1.097819 max_load_reduction=42407 max_load_reduction_pct=0.862773 moved_to_echo=24061 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[86, 52, 10, 1543, 1273, 2119, 699, 126, 0, 0, 0, 0, 0, 0, 0, 0, 1072, 855, 297, 341, 442, 148, 746, 265, 1368, 389, 294, 0, 0, 0, 0, 0, 537, 264, 5, 3, 1105, 174, 493, 137, 3435, 303, 0, 0, 0, 0, 0, 0, 0, 1332, 123, 758, 1024, 377, 1287, 1273, 0, 0, 0, 0, 0, 0, 0, 0, 13, 286, 2008, 412, 1569, 210, 1253, 264, 0, 0, 0, 0, 0, 0, 0, 0, 618, 395, 847, 707, 362, 242, 1, 458, 2414, 0, 0, 0, 0, 0, 0, 0, 790, 458, 907, 377, 2419, 80, 617, 163, 0, 0, 0, 0, 0, 0, 0, 0, 679, 40, 41, 1349, 48, 1639, 157, 2130, 262, 182, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5908, 6217, 6456, 6174, 6015, 6044, 5811, 6527] avg=6144.0000 max=6527 max_rank=7 min=5811 min_rank=6 imbalance_ratio=1.062337 max_load_reduction=42625 max_load_reduction_pct=0.867208 moved_to_echo=24397 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4224, 3456, 6656, 5248, 2688, 2176, 128, 4480, 20736, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[8064, 6144, 2688, 1664, 3584, 2304, 6016, 1408, 11904, 3584, 2560, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[640, 384, 128, 13696, 10240, 18944, 4736, 896, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4992, 384, 384, 9728, 256, 11520, 1280, 17280, 2432, 1536, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6400, 3328, 8064, 3456, 20864, 768, 5504, 1280, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 11648, 768, 6528, 8192, 3072, 9216, 10240, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[3328, 2048, 128, 128, 9088, 1280, 3328, 1152, 26880, 2688, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 1408, 18816, 3584, 13568, 1536, 8576, 2048, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[2832, 1891, 1532, 97, 23, 0, 0, 232, 342, 1824, 0, 689, 149, 2674, 601, 79, 1009, 344, 165, 251, 687, 236, 425, 862, 1057, 583, 84, 853, 1, 1228, 3, 4214, 735, 2044, 1096, 182, 2053, 0, 719, 479, 11, 758, 562, 898, 1, 782, 221, 1825, 706, 3422, 118, 308, 1155, 373, 359, 463, 1275, 6, 294, 758, 640, 1604, 205, 133] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[2135, 2262, 2143, 71, 23, 0, 0, 272, 355, 2132, 0, 783, 161, 1830, 369, 52, 912, 560, 39, 679, 836, 289, 434, 1369, 1122, 652, 91, 1303, 2, 1294, 4, 3577, 429, 1404, 710, 500, 2572, 0, 940, 404, 9, 1001, 678, 1307, 0, 444, 205, 1371, 540, 3105, 210, 312, 1256, 356, 665, 500, 1007, 3, 295, 713, 828, 1123, 377, 137] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[1650, 2185, 1673, 80, 132, 0, 2, 262, 307, 2199, 0, 973, 177, 1459, 470, 65, 922, 568, 15, 822, 664, 349, 440, 1367, 1322, 637, 99, 1045, 2, 1597, 10, 3761, 322, 1504, 454, 558, 2775, 0, 793, 470, 5, 1628, 605, 1629, 0, 439, 251, 1501, 387, 2855, 192, 256, 1141, 232, 743, 522, 1092, 7, 276, 784, 482, 1344, 531, 120] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[2402, 2354, 1706, 80, 32, 0, 0, 268, 332, 1954, 1, 793, 161, 2248, 425, 76, 1110, 386, 104, 448, 672, 271, 417, 1037, 1393, 575, 81, 1306, 3, 1443, 11, 3779, 463, 1747, 869, 347, 2294, 0, 858, 387, 21, 832, 653, 1107, 0, 540, 194, 1651, 483, 3269, 177, 340, 1161, 327, 510, 499, 1157, 18, 248, 687, 600, 1428, 299, 118] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[2345, 2163, 1641, 149, 238, 2, 0, 290, 391, 1995, 0, 665, 112, 2185, 375, 74, 969, 396, 203, 284, 865, 269, 587, 1256, 1321, 675, 76, 978, 0, 1340, 10, 3925, 522, 1748, 599, 279, 1878, 6, 651, 476, 19, 1080, 621, 1206, 0, 923, 203, 1753, 858, 3015, 128, 325, 1215, 495, 486, 476, 1235, 11, 288, 617, 481, 1341, 306, 132] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[2310, 2167, 1654, 47, 81, 3, 0, 227, 281, 1834, 0, 846, 125, 1773, 366, 76, 1023, 472, 21, 533, 696, 242, 572, 1269, 1305, 652, 73, 1051, 0, 1526, 6, 3669, 409, 1749, 728, 328, 2819, 5, 783, 445, 14, 1464, 577, 1389, 1, 589, 234, 1598, 537, 2936, 168, 260, 1007, 305, 662, 485, 1075, 1, 408, 756, 498, 1455, 446, 121] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1612, 2372, 1802, 101, 220, 8, 1, 195, 290, 1940, 6, 873, 148, 1401, 379, 106, 865, 559, 78, 601, 635, 344, 513, 1356, 1336, 626, 96, 1212, 4, 1502, 5, 3556, 488, 1646, 548, 407, 2635, 13, 746, 478, 10, 1369, 679, 1188, 3, 614, 320, 1825, 638, 2681, 312, 324, 1241, 449, 641, 542, 975, 27, 358, 798, 591, 1322, 414, 128] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[2145, 2234, 1638, 60, 30, 0, 0, 216, 340, 1971, 0, 698, 177, 2256, 375, 50, 856, 505, 55, 634, 823, 270, 666, 1337, 906, 640, 84, 1033, 3, 1514, 1, 3805, 417, 1570, 762, 366, 2346, 0, 973, 443, 11, 907, 577, 1584, 0, 553, 222, 1275, 454, 3402, 202, 360, 1272, 278, 506, 439, 1026, 10, 263, 884, 797, 1299, 509, 123] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2832, 1555, 1532, 97, 23, 0, 0, 232, 0, 0, 0, 0, 0, 0, 0, 0, 342, 1824, 0, 689, 149, 2674, 601, 79, 336, 31, 0, 0, 0, 0, 0, 0, 1009, 344, 165, 251, 687, 236, 425, 862, 600, 701, 0, 0, 0, 0, 0, 0, 1057, 583, 84, 853, 1, 1228, 3, 1862, 0, 0, 0, 0, 0, 0, 0, 0, 735, 2044, 1096, 182, 1395, 0, 719, 479, 0, 0, 0, 0, 0, 0, 0, 0, 11, 758, 562, 898, 1, 782, 221, 1825, 646, 58, 0, 0, 0, 0, 0, 0, 706, 2745, 118, 308, 1155, 373, 359, 463, 0, 0, 0, 0, 0, 0, 0, 0, 1275, 6, 294, 758, 640, 1604, 205, 133, 1651, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6271, 6725, 5280, 5671, 6650, 5762, 6227, 6566] avg=6144.0000 max=6725 max_rank=1 min=5280 min_rank=2 imbalance_ratio=1.094564 max_load_reduction=42427 max_load_reduction_pct=0.863180 moved_to_echo=25205 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2402, 1935, 1706, 80, 32, 0, 0, 268, 0, 0, 0, 0, 0, 0, 0, 0, 332, 1954, 1, 793, 161, 2248, 425, 76, 419, 29, 0, 0, 0, 0, 0, 0, 1110, 386, 104, 448, 672, 271, 417, 1037, 671, 629, 0, 0, 0, 0, 0, 0, 1393, 575, 81, 1306, 3, 1443, 11, 1670, 0, 0, 0, 0, 0, 0, 0, 0, 463, 1747, 869, 347, 1558, 0, 858, 387, 0, 0, 0, 0, 0, 0, 0, 0, 21, 832, 653, 1107, 0, 540, 194, 1651, 617, 65, 0, 0, 0, 0, 0, 0, 483, 2623, 177, 340, 1161, 327, 510, 499, 0, 0, 0, 0, 0, 0, 0, 0, 1157, 18, 248, 687, 600, 1428, 299, 118, 1480, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6423, 6438, 5745, 6482, 6229, 5680, 6120, 6035] avg=6144.0000 max=6482 max_rank=3 min=5680 min_rank=5 imbalance_ratio=1.055013 max_load_reduction=42670 max_load_reduction_pct=0.868123 moved_to_echo=24064 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2310, 1782, 1654, 47, 81, 3, 0, 227, 0, 0, 0, 0, 0, 0, 0, 0, 281, 1834, 0, 846, 125, 1773, 366, 76, 385, 26, 0, 0, 0, 0, 0, 0, 1023, 472, 21, 533, 696, 242, 572, 1269, 824, 610, 0, 0, 0, 0, 0, 0, 1305, 652, 73, 1051, 0, 1526, 6, 1622, 0, 0, 0, 0, 0, 0, 0, 0, 409, 1749, 728, 328, 1915, 5, 783, 445, 0, 0, 0, 0, 0, 0, 0, 0, 14, 1464, 577, 1389, 1, 589, 234, 1598, 554, 80, 0, 0, 0, 0, 0, 0, 537, 2356, 168, 260, 1007, 305, 662, 485, 0, 0, 0, 0, 0, 0, 0, 0, 1075, 1, 408, 756, 498, 1455, 446, 121, 1437, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6104, 5712, 6262, 6235, 6362, 6500, 5780, 6197] avg=6144.0000 max=6500 max_rank=5 min=5712 min_rank=1 imbalance_ratio=1.057943 max_load_reduction=42652 max_load_reduction_pct=0.867757 moved_to_echo=24839 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2345, 1778, 1641, 149, 238, 2, 0, 290, 0, 0, 0, 0, 0, 0, 0, 0, 391, 1995, 0, 665, 112, 2185, 375, 74, 385, 27, 0, 0, 0, 0, 0, 0, 969, 396, 203, 284, 865, 269, 587, 1256, 549, 653, 0, 0, 0, 0, 0, 0, 1321, 675, 76, 978, 0, 1340, 10, 1734, 0, 0, 0, 0, 0, 0, 0, 0, 522, 1748, 599, 279, 1276, 6, 651, 476, 0, 0, 0, 0, 0, 0, 0, 0, 19, 1080, 621, 1206, 0, 923, 203, 1753, 569, 53, 0, 0, 0, 0, 0, 0, 858, 2419, 128, 325, 1215, 495, 486, 476, 0, 0, 0, 0, 0, 0, 0, 0, 1235, 11, 288, 617, 481, 1341, 306, 132, 1538, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6443, 6209, 6031, 6134, 5557, 6427, 6402, 5949] avg=6144.0000 max=6443 max_rank=0 min=5557 min_rank=4 imbalance_ratio=1.048665 max_load_reduction=42709 max_load_reduction_pct=0.868917 moved_to_echo=24335 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2135, 1856, 2143, 71, 23, 0, 0, 272, 0, 0, 0, 0, 0, 0, 0, 0, 355, 2132, 0, 783, 161, 1830, 369, 52, 406, 32, 0, 0, 0, 0, 0, 0, 912, 560, 39, 679, 836, 289, 434, 1369, 756, 598, 0, 0, 0, 0, 0, 0, 1122, 652, 91, 1303, 2, 1294, 4, 1574, 0, 0, 0, 0, 0, 0, 0, 0, 429, 1404, 710, 500, 1740, 0, 940, 404, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1001, 678, 1307, 0, 444, 205, 1371, 590, 76, 0, 0, 0, 0, 0, 0, 540, 2483, 210, 312, 1256, 356, 665, 500, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 3, 295, 713, 828, 1123, 377, 137, 1405, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6500, 6120, 6472, 6042, 6127, 5681, 6322, 5888] avg=6144.0000 max=6500 max_rank=0 min=5681 min_rank=5 imbalance_ratio=1.057943 max_load_reduction=42652 max_load_reduction_pct=0.867757 moved_to_echo=24018 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1650, 1797, 1673, 80, 132, 0, 2, 262, 0, 0, 0, 0, 0, 0, 0, 0, 307, 2199, 0, 973, 177, 1459, 470, 65, 388, 26, 0, 0, 0, 0, 0, 0, 922, 568, 15, 822, 664, 349, 440, 1367, 811, 626, 0, 0, 0, 0, 0, 0, 1322, 637, 99, 1045, 2, 1597, 10, 1662, 0, 0, 0, 0, 0, 0, 0, 0, 322, 1504, 454, 558, 1885, 0, 793, 470, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1628, 605, 1629, 0, 439, 251, 1501, 539, 79, 0, 0, 0, 0, 0, 0, 387, 2290, 192, 256, 1141, 232, 743, 522, 0, 0, 0, 0, 0, 0, 0, 0, 1092, 7, 276, 784, 482, 1344, 531, 120, 1473, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5596, 6064, 6584, 6374, 5986, 6676, 5763, 6109] avg=6144.0000 max=6676 max_rank=5 min=5596 min_rank=0 imbalance_ratio=1.086589 max_load_reduction=42476 max_load_reduction_pct=0.864176 moved_to_echo=24534 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2145, 1837, 1638, 60, 30, 0, 0, 216, 0, 0, 0, 0, 0, 0, 0, 0, 340, 1971, 0, 698, 177, 2256, 375, 50, 397, 31, 0, 0, 0, 0, 0, 0, 856, 505, 55, 634, 823, 270, 666, 1337, 686, 633, 0, 0, 0, 0, 0, 0, 906, 640, 84, 1033, 3, 1514, 1, 1681, 0, 0, 0, 0, 0, 0, 0, 0, 417, 1570, 762, 366, 1594, 0, 973, 443, 0, 0, 0, 0, 0, 0, 0, 0, 11, 907, 577, 1584, 0, 553, 222, 1275, 642, 66, 0, 0, 0, 0, 0, 0, 454, 2729, 202, 360, 1272, 278, 506, 439, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 10, 263, 884, 797, 1299, 509, 123, 1491, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5926, 6295, 6465, 5862, 6125, 5837, 6240, 6402] avg=6144.0000 max=6465 max_rank=2 min=5837 min_rank=5 imbalance_ratio=1.052246 max_load_reduction=42687 max_load_reduction_pct=0.868469 moved_to_echo=24604 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1612, 1950, 1802, 101, 220, 8, 1, 195, 0, 0, 0, 0, 0, 0, 0, 0, 290, 1940, 6, 873, 148, 1401, 379, 106, 422, 24, 0, 0, 0, 0, 0, 0, 865, 559, 78, 601, 635, 344, 513, 1356, 770, 592, 0, 0, 0, 0, 0, 0, 1336, 626, 96, 1212, 4, 1502, 5, 1571, 0, 0, 0, 0, 0, 0, 0, 0, 488, 1646, 548, 407, 1790, 13, 746, 478, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1369, 679, 1188, 3, 614, 320, 1825, 506, 75, 0, 0, 0, 0, 0, 0, 638, 2151, 312, 324, 1241, 449, 641, 542, 0, 0, 0, 0, 0, 0, 0, 0, 975, 27, 358, 798, 591, 1322, 414, 128, 1393, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5889, 5589, 6313, 6352, 6116, 6589, 6298, 6006] avg=6144.0000 max=6589 max_rank=5 min=5589 min_rank=1 imbalance_ratio=1.072428 max_load_reduction=42563 max_load_reduction_pct=0.865946 moved_to_echo=25009 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2688, 15872, 128, 6400, 1280, 15872, 3456, 640, 3200, 256, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[4608, 19840, 1536, 2560, 9472, 2816, 4608, 3968, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 9088, 4992, 10368, 128, 4992, 1920, 12800, 4736, 640, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7680, 3840, 768, 4352, 5888, 2304, 4096, 9856, 5760, 5120, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[8960, 128, 2432, 6016, 4992, 11008, 3200, 1024, 11904, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[17536, 14592, 13824, 768, 896, 128, 128, 2048, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3840, 13440, 5888, 3072, 13184, 128, 6528, 3584, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[9856, 5120, 768, 8832, 128, 11520, 128, 13440, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[110, 3142, 1340, 2038, 444, 1953, 37, 71, 506, 672, 89, 1436, 3292, 148, 29, 2523, 484, 0, 179, 677, 11, 26, 461, 851, 36, 1398, 576, 155, 89, 32, 342, 1967, 324, 431, 47, 1567, 112, 2, 760, 14, 1, 294, 7, 144, 2422, 1724, 37, 1206, 1181, 528, 700, 685, 80, 5, 1540, 313, 432, 2762, 503, 1791, 2433, 925, 869, 199] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[109, 1638, 1306, 2029, 175, 1970, 4, 110, 472, 1010, 179, 1071, 2740, 465, 35, 1959, 535, 14, 223, 833, 17, 14, 199, 1383, 89, 836, 523, 143, 162, 25, 286, 2293, 272, 390, 133, 2064, 124, 31, 573, 5, 3, 501, 6, 102, 2406, 1545, 122, 996, 1456, 620, 533, 310, 200, 4, 2588, 436, 486, 3219, 504, 1713, 3189, 603, 1003, 168] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[98, 1855, 1529, 1909, 518, 2288, 55, 50, 416, 639, 107, 1279, 3385, 179, 116, 2250, 631, 61, 171, 753, 27, 12, 650, 746, 29, 1037, 528, 152, 84, 4, 310, 2353, 441, 420, 57, 1736, 107, 1, 927, 18, 7, 360, 20, 99, 2102, 1525, 86, 1161, 1102, 672, 563, 276, 149, 6, 1347, 432, 450, 3551, 491, 1749, 3061, 928, 938, 149] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[126, 2204, 1388, 1805, 457, 1893, 14, 119, 546, 766, 60, 1203, 3346, 198, 23, 2645, 613, 7, 244, 987, 6, 20, 287, 968, 36, 1312, 532, 156, 68, 11, 195, 1484, 364, 434, 28, 1635, 126, 2, 819, 7, 5, 271, 4, 140, 2735, 1929, 38, 1392, 925, 586, 642, 617, 100, 2, 1627, 406, 407, 3193, 564, 1779, 2840, 880, 785, 151] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[110, 2014, 1403, 1624, 437, 2147, 32, 84, 369, 825, 106, 1129, 3005, 287, 23, 2157, 682, 3, 173, 787, 21, 8, 347, 1142, 48, 1215, 522, 158, 82, 4, 233, 2279, 323, 389, 47, 1780, 149, 9, 768, 4, 0, 369, 8, 149, 2298, 1873, 147, 1001, 942, 650, 487, 338, 102, 6, 1749, 436, 500, 3676, 471, 2183, 3092, 772, 856, 122] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[108, 2826, 1525, 1731, 977, 2257, 64, 107, 388, 602, 74, 1353, 3203, 96, 78, 2664, 577, 53, 279, 725, 23, 53, 674, 829, 24, 1437, 551, 137, 63, 10, 225, 1903, 524, 374, 38, 1659, 130, 2, 815, 12, 17, 242, 15, 127, 2410, 1900, 120, 1001, 877, 435, 532, 352, 228, 16, 1135, 348, 499, 3134, 502, 1429, 2846, 881, 792, 144] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[126, 1307, 1438, 2242, 335, 1977, 16, 174, 363, 944, 127, 1047, 3382, 423, 64, 1887, 579, 73, 208, 1020, 12, 18, 253, 1562, 161, 1085, 539, 169, 134, 173, 205, 2090, 400, 437, 139, 1898, 188, 85, 526, 4, 3, 417, 8, 110, 2091, 1933, 163, 1143, 1488, 603, 562, 278, 302, 19, 1916, 403, 485, 3179, 534, 1407, 2592, 622, 853, 231] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[117, 1868, 1408, 1871, 724, 1930, 35, 123, 405, 743, 102, 1238, 2913, 160, 159, 2245, 521, 225, 320, 611, 64, 43, 401, 803, 59, 1327, 535, 173, 46, 197, 255, 1976, 423, 379, 31, 1850, 135, 3, 857, 24, 2, 360, 23, 218, 2480, 1956, 151, 1023, 1015, 562, 579, 430, 135, 13, 1514, 483, 352, 3675, 485, 1637, 2938, 815, 774, 233] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[109, 10, 1306, 2029, 175, 1970, 4, 110, 0, 0, 0, 0, 0, 0, 0, 0, 472, 1010, 179, 1071, 876, 465, 35, 1959, 0, 0, 0, 0, 0, 0, 0, 0, 535, 14, 223, 833, 17, 14, 199, 1383, 2924, 0, 0, 0, 0, 0, 0, 0, 89, 836, 523, 143, 162, 25, 286, 2293, 1350, 0, 0, 0, 0, 0, 0, 0, 272, 390, 133, 2064, 124, 31, 573, 5, 1864, 295, 113, 0, 0, 0, 0, 0, 3, 501, 6, 102, 2406, 1545, 122, 996, 277, 0, 0, 0, 0, 0, 0, 0, 1456, 620, 533, 310, 200, 4, 2588, 436, 1093, 165, 0, 0, 0, 0, 0, 0, 486, 0, 504, 1713, 1819, 603, 1003, 168, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5713, 6067, 6142, 5707, 5864, 5958, 7405, 6296] avg=6144.0000 max=7405 max_rank=6 min=5707 min_rank=3 imbalance_ratio=1.205241 max_load_reduction=41747 max_load_reduction_pct=0.849345 moved_to_echo=25523 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[98, 11, 1529, 1909, 518, 2288, 55, 50, 0, 0, 0, 0, 0, 0, 0, 0, 416, 639, 107, 1279, 1082, 179, 116, 2250, 0, 0, 0, 0, 0, 0, 0, 0, 631, 61, 171, 753, 27, 12, 650, 746, 3225, 0, 0, 0, 0, 0, 0, 0, 29, 1037, 528, 152, 84, 4, 310, 2353, 1529, 0, 0, 0, 0, 0, 0, 0, 441, 420, 57, 1736, 107, 1, 927, 18, 2303, 326, 128, 0, 0, 0, 0, 0, 7, 360, 20, 99, 2102, 1525, 86, 1161, 266, 0, 0, 0, 0, 0, 0, 0, 1102, 672, 563, 276, 149, 6, 1347, 432, 1049, 187, 0, 0, 0, 0, 0, 0, 450, 0, 491, 1749, 1746, 928, 938, 149, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6458, 6068, 6276, 6026, 6464, 5626, 5783, 6451] avg=6144.0000 max=6464 max_rank=4 min=5626 min_rank=5 imbalance_ratio=1.052083 max_load_reduction=42688 max_load_reduction_pct=0.868490 moved_to_echo=24324 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[110, 17, 1340, 2038, 444, 1953, 37, 71, 0, 0, 0, 0, 0, 0, 0, 0, 506, 672, 89, 1436, 1052, 148, 29, 2523, 0, 0, 0, 0, 0, 0, 0, 0, 484, 0, 179, 677, 11, 26, 461, 851, 2509, 0, 0, 0, 0, 0, 0, 0, 36, 1398, 576, 155, 89, 32, 342, 1967, 2590, 0, 0, 0, 0, 0, 0, 0, 324, 431, 47, 1567, 112, 2, 760, 14, 2240, 253, 218, 0, 0, 0, 0, 0, 1, 294, 7, 144, 2422, 1724, 37, 1206, 212, 0, 0, 0, 0, 0, 0, 0, 1181, 528, 700, 685, 80, 5, 1540, 313, 833, 317, 0, 0, 0, 0, 0, 0, 432, 0, 503, 1791, 1388, 925, 869, 199, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6010, 6455, 5198, 7185, 5968, 6047, 6182, 6107] avg=6144.0000 max=7185 max_rank=3 min=5198 min_rank=2 imbalance_ratio=1.169434 max_load_reduction=41967 max_load_reduction_pct=0.853821 moved_to_echo=24304 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[126, 0, 1388, 1805, 457, 1893, 14, 119, 0, 0, 0, 0, 0, 0, 0, 0, 546, 766, 60, 1203, 1066, 198, 23, 2645, 0, 0, 0, 0, 0, 0, 0, 0, 613, 7, 244, 987, 6, 20, 287, 968, 2901, 0, 0, 0, 0, 0, 0, 0, 36, 1312, 532, 156, 68, 11, 195, 1484, 1820, 0, 0, 0, 0, 0, 0, 0, 364, 434, 28, 1635, 126, 2, 819, 7, 2280, 292, 157, 0, 0, 0, 0, 0, 5, 271, 4, 140, 2735, 1929, 38, 1392, 251, 0, 0, 0, 0, 0, 0, 0, 925, 586, 642, 617, 100, 2, 1627, 406, 977, 227, 0, 0, 0, 0, 0, 0, 407, 0, 564, 1779, 1612, 880, 785, 151, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5802, 6507, 6033, 5614, 6144, 6765, 6109, 6178] avg=6144.0000 max=6765 max_rank=5 min=5614 min_rank=3 imbalance_ratio=1.101074 max_load_reduction=42387 max_load_reduction_pct=0.862366 moved_to_echo=25196 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[110, 12, 1403, 1624, 437, 2147, 32, 84, 0, 0, 0, 0, 0, 0, 0, 0, 369, 825, 106, 1129, 961, 287, 23, 2157, 0, 0, 0, 0, 0, 0, 0, 0, 682, 3, 173, 787, 21, 8, 347, 1142, 3340, 0, 0, 0, 0, 0, 0, 0, 48, 1215, 522, 158, 82, 4, 233, 2279, 1660, 0, 0, 0, 0, 0, 0, 0, 323, 389, 47, 1780, 149, 9, 768, 4, 2044, 336, 139, 0, 0, 0, 0, 0, 0, 369, 8, 149, 2298, 1873, 147, 1001, 269, 0, 0, 0, 0, 0, 0, 0, 942, 650, 487, 338, 102, 6, 1749, 436, 1059, 203, 0, 0, 0, 0, 0, 0, 500, 0, 471, 2183, 1764, 772, 856, 122, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5849, 5857, 6503, 6201, 5988, 6114, 5972, 6668] avg=6144.0000 max=6668 max_rank=7 min=5849 min_rank=0 imbalance_ratio=1.085286 max_load_reduction=42484 max_load_reduction_pct=0.864339 moved_to_echo=24742 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[126, 7, 1438, 2242, 335, 1977, 16, 174, 0, 0, 0, 0, 0, 0, 0, 0, 363, 944, 127, 1047, 1081, 423, 64, 1887, 0, 0, 0, 0, 0, 0, 0, 0, 579, 73, 208, 1020, 12, 18, 253, 1562, 2888, 0, 0, 0, 0, 0, 0, 0, 161, 1085, 539, 169, 134, 173, 205, 2090, 1077, 0, 0, 0, 0, 0, 0, 0, 400, 437, 139, 1898, 188, 85, 526, 4, 2301, 291, 90, 0, 0, 0, 0, 0, 3, 417, 8, 110, 2091, 1933, 163, 1143, 225, 0, 0, 0, 0, 0, 0, 0, 1488, 603, 562, 278, 302, 19, 1916, 403, 888, 133, 0, 0, 0, 0, 0, 0, 485, 0, 534, 1407, 1479, 622, 853, 231, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6315, 5936, 6613, 5633, 6359, 6093, 6592, 5611] avg=6144.0000 max=6613 max_rank=2 min=5611 min_rank=7 imbalance_ratio=1.076335 max_load_reduction=42539 max_load_reduction_pct=0.865458 moved_to_echo=24655 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[108, 16, 1525, 1731, 977, 2257, 64, 107, 0, 0, 0, 0, 0, 0, 0, 0, 388, 602, 74, 1353, 1024, 96, 78, 2664, 0, 0, 0, 0, 0, 0, 0, 0, 577, 53, 279, 725, 23, 53, 674, 829, 2848, 0, 0, 0, 0, 0, 0, 0, 24, 1437, 551, 137, 63, 10, 225, 1903, 2329, 0, 0, 0, 0, 0, 0, 0, 524, 374, 38, 1659, 130, 2, 815, 12, 2179, 286, 196, 0, 0, 0, 0, 0, 17, 242, 15, 127, 2410, 1900, 120, 1001, 248, 0, 0, 0, 0, 0, 0, 0, 877, 435, 532, 352, 228, 16, 1135, 348, 975, 285, 0, 0, 0, 0, 0, 0, 499, 0, 502, 1429, 1623, 881, 792, 144, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6785, 6279, 6061, 6679, 6215, 6080, 5183, 5870] avg=6144.0000 max=6785 max_rank=0 min=5183 min_rank=6 imbalance_ratio=1.104329 max_load_reduction=42367 max_load_reduction_pct=0.861959 moved_to_echo=23348 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[117, 12, 1408, 1871, 724, 1930, 35, 123, 0, 0, 0, 0, 0, 0, 0, 0, 405, 743, 102, 1238, 931, 160, 159, 2245, 0, 0, 0, 0, 0, 0, 0, 0, 521, 225, 320, 611, 64, 43, 401, 803, 3338, 0, 0, 0, 0, 0, 0, 0, 59, 1327, 535, 173, 46, 197, 255, 1976, 1539, 0, 0, 0, 0, 0, 0, 0, 423, 379, 31, 1850, 135, 3, 857, 24, 1982, 337, 129, 0, 0, 0, 0, 0, 2, 360, 23, 218, 2480, 1956, 151, 1023, 256, 0, 0, 0, 0, 0, 0, 0, 1015, 562, 579, 430, 135, 13, 1514, 483, 1007, 188, 0, 0, 0, 0, 0, 0, 352, 0, 485, 1637, 1675, 815, 774, 233, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6220, 5983, 6326, 6107, 6150, 6469, 5926, 5971] avg=6144.0000 max=6469 max_rank=5 min=5926 min_rank=6 imbalance_ratio=1.052897 max_load_reduction=42683 max_load_reduction_pct=0.868388 moved_to_echo=24516 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 2816, 128, 1152, 18944, 14464, 896, 8960, 2048, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4736, 512, 1920, 6400, 256, 256, 3328, 8320, 24064, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3584, 6272, 896, 9856, 8192, 2048, 640, 18432, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[9088, 4736, 4608, 3328, 1408, 128, 13440, 3328, 7936, 1792, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[3712, 0, 4096, 13696, 13184, 6528, 6912, 1408, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[512, 9728, 4352, 1280, 768, 512, 2176, 16384, 13952, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3072, 3328, 640, 14208, 1152, 256, 6144, 128, 17280, 2432, 1280, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1024, 128, 11392, 15360, 4096, 16512, 384, 896, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[709, 1168, 91, 7, 138, 164, 687, 4382, 1282, 127, 50, 0, 396, 67, 4648, 95, 1, 9, 24, 739, 161, 9, 69, 13, 36, 1061, 1525, 0, 2171, 141, 38, 3032, 567, 220, 373, 16, 2409, 363, 0, 212, 4, 789, 1924, 291, 31, 213, 43, 188, 1504, 285, 33, 2, 1646, 33, 2, 4713, 79, 150, 32, 3638, 374, 1840, 1838, 2300] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[577, 948, 290, 11, 82, 61, 576, 4425, 910, 56, 1011, 0, 467, 681, 4584, 152, 7, 7, 53, 1880, 81, 24, 181, 69, 35, 873, 2440, 0, 1149, 204, 58, 2370, 359, 60, 1237, 18, 1841, 216, 0, 53, 8, 1094, 974, 433, 19, 196, 95, 452, 1450, 339, 103, 22, 2306, 126, 22, 2420, 312, 894, 29, 3112, 450, 1674, 2541, 2035] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[815, 1399, 270, 15, 132, 73, 502, 4412, 1269, 110, 284, 1, 442, 261, 4304, 107, 7, 9, 58, 1294, 152, 3, 122, 86, 42, 1047, 1918, 0, 2065, 233, 36, 2907, 367, 106, 868, 24, 2030, 265, 0, 133, 8, 1109, 992, 349, 23, 213, 50, 413, 1293, 308, 87, 25, 2047, 81, 7, 3242, 162, 207, 19, 3639, 380, 1737, 2305, 2288] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[609, 1078, 305, 12, 95, 83, 328, 4227, 750, 81, 871, 1, 397, 741, 4622, 176, 6, 4, 108, 1736, 66, 6, 242, 67, 16, 1135, 1876, 0, 1388, 379, 54, 2266, 300, 74, 1171, 56, 1987, 153, 0, 33, 29, 1411, 836, 328, 25, 277, 15, 516, 1708, 277, 225, 39, 2339, 200, 6, 1484, 249, 468, 14, 3801, 447, 1590, 3010, 2359] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[699, 1311, 134, 1, 176, 131, 766, 3431, 1068, 76, 79, 15, 324, 8, 4577, 143, 4, 22, 30, 1316, 205, 5, 74, 18, 16, 1016, 1924, 0, 1522, 243, 60, 2682, 525, 164, 569, 17, 2611, 609, 2, 234, 6, 976, 1992, 157, 29, 266, 12, 290, 1458, 443, 243, 4, 1902, 43, 21, 3954, 139, 701, 38, 3455, 403, 1695, 1884, 2234] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[649, 1053, 422, 4, 126, 60, 479, 4393, 778, 81, 534, 0, 372, 398, 4894, 133, 4, 6, 42, 1661, 71, 12, 287, 46, 16, 925, 1629, 2, 1385, 302, 44, 2437, 347, 83, 927, 23, 1943, 202, 1, 54, 16, 1537, 952, 341, 29, 232, 45, 413, 1710, 271, 268, 21, 2224, 53, 5, 2177, 202, 468, 19, 4009, 463, 1823, 2686, 2363] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[627, 710, 760, 8, 127, 57, 468, 3837, 603, 21, 1159, 12, 437, 1094, 4752, 192, 3, 2, 69, 1985, 87, 11, 175, 84, 29, 1072, 2515, 0, 1234, 246, 50, 2084, 368, 68, 1305, 35, 1793, 126, 1, 25, 43, 1583, 572, 335, 46, 404, 156, 573, 1451, 289, 285, 28, 2551, 64, 14, 995, 349, 1026, 27, 3194, 396, 1351, 3221, 1968] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[693, 871, 359, 10, 151, 67, 679, 3988, 961, 64, 826, 3, 411, 594, 3626, 270, 8, 7, 43, 2284, 145, 4, 151, 74, 37, 1010, 1951, 0, 1284, 485, 53, 2435, 276, 125, 1213, 32, 1820, 239, 7, 105, 41, 1650, 1338, 331, 104, 334, 36, 514, 1485, 405, 413, 107, 2046, 87, 15, 1598, 252, 465, 30, 3977, 488, 1410, 2559, 2106] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[709, 1168, 91, 7, 138, 164, 687, 3487, 0, 0, 0, 0, 0, 0, 0, 0, 1282, 127, 50, 0, 396, 67, 3578, 95, 0, 0, 0, 0, 0, 0, 0, 0, 1, 9, 24, 739, 161, 9, 69, 13, 3638, 375, 0, 0, 0, 0, 0, 0, 36, 1061, 1525, 0, 2171, 141, 38, 1528, 0, 0, 0, 0, 0, 0, 0, 0, 567, 220, 373, 16, 2409, 363, 0, 212, 895, 1339, 667, 0, 0, 0, 0, 0, 4, 789, 1924, 291, 31, 213, 43, 188, 1504, 669, 403, 0, 0, 0, 0, 0, 1504, 285, 33, 2, 1646, 33, 2, 3374, 0, 0, 0, 0, 0, 0, 0, 0, 79, 150, 32, 0, 374, 1840, 794, 2300, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6451, 5595, 5038, 6500, 7061, 6059, 6879, 5569] avg=6144.0000 max=7061 max_rank=4 min=5038 min_rank=2 imbalance_ratio=1.149251 max_load_reduction=42091 max_load_reduction_pct=0.856344 moved_to_echo=25568 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[609, 1078, 305, 12, 95, 83, 328, 3363, 0, 0, 0, 0, 0, 0, 0, 0, 750, 81, 871, 1, 397, 741, 3558, 176, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4, 108, 1736, 66, 6, 242, 67, 3801, 615, 0, 0, 0, 0, 0, 0, 16, 1135, 1876, 0, 1388, 379, 54, 1142, 0, 0, 0, 0, 0, 0, 0, 0, 300, 74, 1171, 56, 1987, 153, 0, 33, 864, 421, 664, 0, 0, 0, 0, 0, 29, 1411, 836, 328, 25, 277, 15, 516, 1124, 1095, 400, 0, 0, 0, 0, 0, 1708, 277, 225, 39, 2339, 200, 6, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 249, 468, 14, 0, 447, 1590, 1300, 2359, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5873, 6575, 6651, 5990, 5723, 6056, 5857, 6427] avg=6144.0000 max=6651 max_rank=2 min=5723 min_rank=4 imbalance_ratio=1.082520 max_load_reduction=42501 max_load_reduction_pct=0.864685 moved_to_echo=24063 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[577, 948, 290, 11, 82, 61, 576, 3521, 0, 0, 0, 0, 0, 0, 0, 0, 910, 56, 1011, 0, 467, 681, 3529, 152, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 53, 1880, 81, 24, 181, 69, 3112, 519, 0, 0, 0, 0, 0, 0, 35, 873, 2440, 0, 1149, 204, 58, 1194, 0, 0, 0, 0, 0, 0, 0, 0, 359, 60, 1237, 18, 1841, 216, 0, 53, 904, 688, 658, 0, 0, 0, 0, 0, 8, 1094, 974, 433, 19, 196, 95, 452, 1176, 925, 397, 0, 0, 0, 0, 0, 1450, 339, 103, 22, 2306, 126, 22, 1732, 0, 0, 0, 0, 0, 0, 0, 0, 312, 894, 29, 0, 450, 1674, 1097, 2035, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6066, 6806, 5933, 5953, 6034, 5769, 6100, 6491] avg=6144.0000 max=6806 max_rank=1 min=5769 min_rank=5 imbalance_ratio=1.107747 max_load_reduction=42346 max_load_reduction_pct=0.861532 moved_to_echo=24394 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[815, 1399, 270, 15, 132, 73, 502, 3511, 0, 0, 0, 0, 0, 0, 0, 0, 1269, 110, 284, 1, 442, 261, 3313, 107, 0, 0, 0, 0, 0, 0, 0, 0, 7, 9, 58, 1294, 152, 3, 122, 86, 3639, 471, 0, 0, 0, 0, 0, 0, 42, 1047, 1918, 0, 2065, 233, 36, 1465, 0, 0, 0, 0, 0, 0, 0, 0, 367, 106, 868, 24, 2030, 265, 0, 133, 901, 921, 618, 0, 0, 0, 0, 0, 8, 1109, 992, 349, 23, 213, 50, 413, 1442, 839, 373, 0, 0, 0, 0, 0, 1293, 308, 87, 25, 2047, 81, 7, 2321, 0, 0, 0, 0, 0, 0, 0, 0, 162, 207, 19, 0, 380, 1737, 995, 2288, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6717, 5787, 5841, 6806, 6233, 5811, 6169, 5788] avg=6144.0000 max=6806 max_rank=3 min=5787 min_rank=1 imbalance_ratio=1.107747 max_load_reduction=42346 max_load_reduction_pct=0.861532 moved_to_echo=24001 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[649, 1053, 422, 4, 126, 60, 479, 3491, 0, 0, 0, 0, 0, 0, 0, 0, 778, 81, 534, 0, 372, 398, 3760, 133, 0, 0, 0, 0, 0, 0, 0, 0, 4, 6, 42, 1661, 71, 12, 287, 46, 4009, 553, 0, 0, 0, 0, 0, 0, 16, 925, 1629, 2, 1385, 302, 44, 1224, 0, 0, 0, 0, 0, 0, 0, 0, 347, 83, 927, 23, 1943, 202, 1, 54, 902, 623, 707, 0, 0, 0, 0, 0, 16, 1537, 952, 341, 29, 232, 45, 413, 1213, 982, 427, 0, 0, 0, 0, 0, 1710, 271, 268, 21, 2224, 53, 5, 1554, 0, 0, 0, 0, 0, 0, 0, 0, 202, 468, 19, 0, 463, 1823, 1151, 2363, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6284, 6056, 6691, 5527, 5812, 6187, 6106, 6489] avg=6144.0000 max=6691 max_rank=2 min=5527 min_rank=3 imbalance_ratio=1.089030 max_load_reduction=42461 max_load_reduction_pct=0.863871 moved_to_echo=24594 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[699, 1311, 134, 1, 176, 131, 766, 2730, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 76, 79, 15, 324, 8, 3524, 143, 0, 0, 0, 0, 0, 0, 0, 0, 4, 22, 30, 1316, 205, 5, 74, 18, 3455, 385, 0, 0, 0, 0, 0, 0, 16, 1016, 1924, 0, 1522, 243, 60, 1352, 0, 0, 0, 0, 0, 0, 0, 0, 525, 164, 569, 17, 2611, 609, 2, 234, 701, 1124, 657, 0, 0, 0, 0, 0, 6, 976, 1992, 157, 29, 266, 12, 290, 1330, 685, 396, 0, 0, 0, 0, 0, 1458, 443, 243, 4, 1902, 43, 21, 2830, 0, 0, 0, 0, 0, 0, 0, 0, 139, 701, 38, 0, 403, 1695, 814, 2234, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5948, 5237, 5514, 6133, 7213, 6139, 6944, 6024] avg=6144.0000 max=7213 max_rank=4 min=5237 min_rank=1 imbalance_ratio=1.173991 max_load_reduction=41939 max_load_reduction_pct=0.853251 moved_to_echo=26320 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[627, 710, 760, 8, 127, 57, 468, 3053, 0, 0, 0, 0, 0, 0, 0, 0, 603, 21, 1159, 12, 437, 1094, 3658, 192, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 69, 1985, 87, 11, 175, 84, 3194, 658, 0, 0, 0, 0, 0, 0, 29, 1072, 2515, 0, 1234, 246, 50, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 368, 68, 1305, 35, 1793, 126, 1, 25, 784, 282, 682, 0, 0, 0, 0, 0, 43, 1583, 572, 335, 46, 404, 156, 573, 1034, 1172, 412, 0, 0, 0, 0, 0, 1451, 289, 285, 28, 2551, 64, 14, 713, 0, 0, 0, 0, 0, 0, 0, 0, 349, 1026, 27, 0, 396, 1351, 1391, 1968, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5810, 7176, 6268, 6196, 5469, 6330, 5395, 6508] avg=6144.0000 max=7176 max_rank=1 min=5395 min_rank=6 imbalance_ratio=1.167969 max_load_reduction=41976 max_load_reduction_pct=0.854004 moved_to_echo=23702 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[693, 871, 359, 10, 151, 67, 679, 3173, 0, 0, 0, 0, 0, 0, 0, 0, 961, 64, 826, 3, 411, 594, 2791, 270, 0, 0, 0, 0, 0, 0, 0, 0, 8, 7, 43, 2284, 145, 4, 151, 74, 3977, 523, 0, 0, 0, 0, 0, 0, 37, 1010, 1951, 0, 1284, 485, 53, 1227, 0, 0, 0, 0, 0, 0, 0, 0, 276, 125, 1213, 32, 1820, 239, 7, 105, 815, 454, 521, 0, 0, 0, 0, 0, 41, 1650, 1338, 331, 104, 334, 36, 514, 1208, 931, 314, 0, 0, 0, 0, 0, 1485, 405, 413, 107, 2046, 87, 15, 1144, 0, 0, 0, 0, 0, 0, 0, 0, 252, 465, 30, 0, 488, 1410, 1105, 2106, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6003, 5920, 7216, 6047, 5607, 6801, 5702, 5856] avg=6144.0000 max=7216 max_rank=2 min=5607 min_rank=4 imbalance_ratio=1.174479 max_load_reduction=41936 max_load_reduction_pct=0.853190 moved_to_echo=23966 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[12160, 2688, 1664, 256, 17152, 768, 128, 14848, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[256, 10240, 9600, 2688, 384, 2176, 512, 3456, 10112, 7424, 3200, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3200, 1024, 7680, 256, 16512, 2176, 128, 896, 6784, 5888, 5248, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[256, 8192, 15872, 128, 12288, 2304, 512, 10240, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[1792, 4480, 256, 0, 3456, 13184, 8704, 17664, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 512, 12928, 1024, 128, 1408, 512, 28928, 4224, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5504, 8576, 2688, 128, 1152, 768, 4608, 26368, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7680, 640, 4864, 128, 3328, 3968, 27776, 1280, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[253, 379, 3144, 36, 318, 4222, 9, 520, 211, 213, 115, 3637, 361, 1, 368, 190, 706, 709, 293, 198, 395, 272, 2, 843, 13, 25, 148, 203, 768, 372, 175, 0, 571, 882, 0, 179, 309, 104, 355, 7, 335, 5, 3377, 73, 20, 171, 374, 702, 54, 499, 9, 15, 1963, 7, 354, 1046, 4748, 1, 5028, 2, 6558, 1516, 580, 209] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[246, 345, 765, 88, 353, 4212, 2, 969, 255, 557, 301, 4523, 130, 150, 388, 978, 1135, 1138, 30, 212, 574, 308, 17, 1634, 14, 183, 63, 335, 906, 408, 253, 0, 436, 273, 2, 144, 1089, 115, 856, 15, 368, 8, 2307, 37, 93, 75, 933, 85, 447, 421, 374, 197, 1984, 22, 1084, 1681, 4073, 8, 2788, 0, 5286, 1340, 931, 208] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[255, 341, 752, 94, 454, 3461, 1, 1162, 250, 648, 386, 4100, 160, 254, 375, 899, 1126, 1429, 24, 231, 1028, 240, 2, 1414, 25, 128, 25, 373, 834, 575, 354, 18, 616, 330, 0, 253, 952, 194, 878, 27, 309, 5, 2374, 70, 171, 116, 1114, 27, 287, 432, 73, 165, 1819, 13, 1032, 1978, 4068, 1, 1959, 0, 5734, 1361, 1169, 207] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[255, 378, 3195, 21, 341, 3469, 11, 568, 186, 237, 223, 2786, 342, 145, 309, 85, 705, 570, 685, 271, 161, 284, 1, 1033, 21, 22, 178, 170, 896, 314, 266, 0, 734, 684, 0, 121, 487, 41, 386, 7, 309, 0, 3647, 91, 34, 155, 512, 402, 26, 542, 51, 143, 2374, 2, 465, 1404, 4566, 1, 4740, 0, 6672, 1687, 525, 216] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[270, 374, 1261, 46, 415, 4124, 5, 1155, 171, 341, 302, 4372, 201, 60, 384, 863, 951, 706, 59, 215, 769, 395, 2, 1313, 13, 96, 28, 358, 1332, 392, 236, 0, 537, 451, 4, 239, 891, 103, 678, 18, 361, 1, 2391, 92, 86, 120, 672, 183, 159, 454, 38, 199, 1984, 14, 913, 1540, 4147, 0, 3386, 0, 5797, 1395, 862, 228] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[239, 398, 995, 68, 404, 3639, 2, 854, 200, 548, 316, 3988, 200, 36, 345, 507, 1009, 1292, 39, 116, 891, 301, 1, 1584, 118, 84, 50, 371, 1317, 409, 377, 0, 368, 237, 0, 215, 1112, 132, 756, 6, 329, 1, 2682, 81, 107, 147, 697, 26, 146, 460, 59, 233, 2028, 6, 986, 1865, 4210, 1, 2700, 0, 6240, 1337, 1054, 233] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[261, 416, 621, 51, 411, 3328, 1, 796, 184, 676, 309, 4531, 232, 30, 323, 1105, 1204, 866, 24, 245, 532, 310, 6, 1853, 109, 143, 198, 296, 1216, 456, 288, 26, 735, 323, 1, 219, 1196, 104, 1481, 50, 315, 4, 2188, 74, 58, 153, 966, 8, 227, 405, 359, 251, 2039, 12, 1040, 1590, 3805, 0, 2798, 1, 5239, 1248, 946, 270] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[251, 331, 584, 35, 506, 3602, 1, 1041, 169, 798, 494, 4541, 169, 215, 375, 837, 1417, 1193, 13, 153, 838, 440, 5, 1381, 11, 149, 14, 431, 1120, 463, 388, 2, 427, 127, 3, 114, 1137, 141, 984, 16, 360, 5, 2092, 70, 160, 159, 1102, 12, 148, 336, 312, 200, 2128, 32, 1138, 1872, 4232, 0, 1776, 0, 5535, 1427, 866, 274] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[246, 345, 765, 88, 353, 3099, 2, 969, 0, 0, 0, 0, 0, 0, 0, 0, 255, 557, 301, 4185, 130, 150, 388, 978, 0, 0, 0, 0, 0, 0, 0, 0, 1135, 1138, 30, 212, 574, 308, 17, 1634, 720, 752, 0, 0, 0, 0, 0, 0, 14, 183, 63, 335, 906, 408, 253, 0, 3436, 0, 0, 0, 0, 0, 0, 0, 436, 273, 2, 144, 1089, 115, 856, 15, 1850, 1063, 0, 0, 0, 0, 0, 0, 368, 8, 2307, 37, 93, 75, 933, 85, 1823, 0, 0, 0, 0, 0, 0, 0, 447, 421, 374, 197, 1984, 22, 1084, 1681, 361, 338, 0, 0, 0, 0, 0, 0, 467, 8, 2788, 0, 0, 1340, 931, 208, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5867, 6944, 6520, 5598, 5843, 5729, 6909, 5742] avg=6144.0000 max=6944 max_rank=1 min=5598 min_rank=3 imbalance_ratio=1.130208 max_load_reduction=42208 max_load_reduction_pct=0.858724 moved_to_echo=24223 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[255, 341, 752, 94, 454, 2546, 1, 1162, 0, 0, 0, 0, 0, 0, 0, 0, 250, 648, 386, 3793, 160, 254, 375, 899, 0, 0, 0, 0, 0, 0, 0, 0, 1126, 1429, 24, 231, 1028, 240, 2, 1414, 719, 618, 0, 0, 0, 0, 0, 0, 25, 128, 25, 373, 834, 575, 354, 18, 3727, 0, 0, 0, 0, 0, 0, 0, 616, 330, 0, 253, 952, 194, 878, 27, 2007, 1061, 0, 0, 0, 0, 0, 0, 309, 5, 2374, 70, 171, 116, 1114, 27, 1820, 0, 0, 0, 0, 0, 0, 0, 287, 432, 73, 165, 1819, 13, 1032, 1978, 297, 307, 0, 0, 0, 0, 0, 0, 468, 1, 1959, 0, 0, 1361, 1169, 207, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5605, 6765, 6831, 6059, 6318, 6006, 6403, 5165] avg=6144.0000 max=6831 max_rank=2 min=5165 min_rank=7 imbalance_ratio=1.111816 max_load_reduction=42321 max_load_reduction_pct=0.861023 moved_to_echo=23892 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[253, 379, 3144, 36, 318, 3106, 9, 520, 0, 0, 0, 0, 0, 0, 0, 0, 211, 213, 115, 3365, 361, 1, 368, 190, 0, 0, 0, 0, 0, 0, 0, 0, 706, 709, 293, 198, 395, 272, 2, 843, 840, 754, 0, 0, 0, 0, 0, 0, 13, 25, 148, 203, 768, 372, 175, 0, 4263, 0, 0, 0, 0, 0, 0, 0, 571, 882, 0, 179, 309, 104, 355, 7, 2295, 1239, 0, 0, 0, 0, 0, 0, 335, 5, 3377, 73, 20, 171, 374, 702, 2125, 0, 0, 0, 0, 0, 0, 0, 54, 499, 9, 15, 1963, 7, 354, 1046, 362, 272, 0, 0, 0, 0, 0, 0, 544, 1, 5028, 2, 0, 1516, 580, 209, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7765, 4824, 5012, 5967, 5941, 7182, 4581, 7880] avg=6144.0000 max=7880 max_rank=7 min=4581 min_rank=6 imbalance_ratio=1.282552 max_load_reduction=41272 max_load_reduction_pct=0.839681 moved_to_echo=25584 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[270, 374, 1261, 46, 415, 3034, 5, 1155, 0, 0, 0, 0, 0, 0, 0, 0, 171, 341, 302, 4045, 201, 60, 384, 863, 0, 0, 0, 0, 0, 0, 0, 0, 951, 706, 59, 215, 769, 395, 2, 1313, 733, 736, 0, 0, 0, 0, 0, 0, 13, 96, 28, 358, 1332, 392, 236, 0, 3769, 0, 0, 0, 0, 0, 0, 0, 537, 451, 4, 239, 891, 103, 678, 18, 2028, 1082, 0, 0, 0, 0, 0, 0, 361, 1, 2391, 92, 86, 120, 672, 183, 1856, 0, 0, 0, 0, 0, 0, 0, 159, 454, 38, 199, 1984, 14, 913, 1540, 354, 327, 0, 0, 0, 0, 0, 0, 476, 0, 3386, 0, 0, 1395, 862, 228, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6560, 6367, 5879, 6224, 6031, 5762, 5982, 6347] avg=6144.0000 max=6560 max_rank=0 min=5762 min_rank=5 imbalance_ratio=1.067708 max_load_reduction=42592 max_load_reduction_pct=0.866536 moved_to_echo=24122 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[239, 398, 995, 68, 404, 2669, 2, 854, 0, 0, 0, 0, 0, 0, 0, 0, 200, 548, 316, 3686, 200, 36, 345, 507, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1292, 39, 116, 891, 301, 1, 1584, 749, 653, 0, 0, 0, 0, 0, 0, 118, 84, 50, 371, 1317, 409, 377, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 368, 237, 0, 215, 1112, 132, 756, 6, 2183, 1102, 0, 0, 0, 0, 0, 0, 329, 1, 2682, 81, 107, 147, 697, 26, 1887, 0, 0, 0, 0, 0, 0, 0, 146, 460, 59, 233, 2028, 6, 986, 1865, 317, 302, 0, 0, 0, 0, 0, 0, 472, 1, 2700, 0, 0, 1337, 1054, 233, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5629, 5838, 6635, 6783, 6111, 5957, 6402, 5797] avg=6144.0000 max=6783 max_rank=3 min=5629 min_rank=0 imbalance_ratio=1.104004 max_load_reduction=42369 max_load_reduction_pct=0.862000 moved_to_echo=24267 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[255, 378, 3195, 21, 341, 2552, 11, 568, 0, 0, 0, 0, 0, 0, 0, 0, 186, 237, 223, 2578, 342, 145, 309, 85, 0, 0, 0, 0, 0, 0, 0, 0, 705, 570, 685, 271, 161, 284, 1, 1033, 808, 619, 0, 0, 0, 0, 0, 0, 21, 22, 178, 170, 896, 314, 266, 0, 4338, 0, 0, 0, 0, 0, 0, 0, 734, 684, 0, 121, 487, 41, 386, 7, 2334, 1191, 0, 0, 0, 0, 0, 0, 309, 0, 3647, 91, 34, 155, 512, 402, 2043, 0, 0, 0, 0, 0, 0, 0, 26, 542, 51, 143, 2374, 2, 465, 1404, 298, 208, 0, 0, 0, 0, 0, 0, 524, 1, 4740, 0, 0, 1687, 525, 216, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7321, 4105, 5137, 6205, 5985, 7193, 5513, 7693] avg=6144.0000 max=7693 max_rank=7 min=4105 min_rank=1 imbalance_ratio=1.252116 max_load_reduction=41459 max_load_reduction_pct=0.843486 moved_to_echo=26384 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[251, 331, 584, 35, 506, 2650, 1, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 169, 798, 494, 4201, 169, 215, 375, 837, 0, 0, 0, 0, 0, 0, 0, 0, 1417, 1193, 13, 153, 838, 440, 5, 1381, 748, 643, 0, 0, 0, 0, 0, 0, 11, 149, 14, 431, 1120, 463, 388, 2, 3599, 0, 0, 0, 0, 0, 0, 0, 427, 127, 3, 114, 1137, 141, 984, 16, 1936, 1104, 0, 0, 0, 0, 0, 0, 360, 5, 2092, 70, 160, 159, 1102, 12, 1894, 0, 0, 0, 0, 0, 0, 0, 148, 336, 312, 200, 2128, 32, 1138, 1872, 309, 340, 0, 0, 0, 0, 0, 0, 486, 0, 1776, 0, 0, 1427, 866, 274, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5399, 7258, 6831, 6177, 5989, 5854, 6815, 4829] avg=6144.0000 max=7258 max_rank=1 min=4829 min_rank=7 imbalance_ratio=1.181315 max_load_reduction=41894 max_load_reduction_pct=0.852336 moved_to_echo=23487 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[261, 416, 621, 51, 411, 2449, 1, 796, 0, 0, 0, 0, 0, 0, 0, 0, 184, 676, 309, 4192, 232, 30, 323, 1105, 0, 0, 0, 0, 0, 0, 0, 0, 1204, 866, 24, 245, 532, 310, 6, 1853, 673, 594, 0, 0, 0, 0, 0, 0, 109, 143, 198, 296, 1216, 456, 288, 26, 3407, 0, 0, 0, 0, 0, 0, 0, 735, 323, 1, 219, 1196, 104, 1481, 50, 1832, 993, 0, 0, 0, 0, 0, 0, 315, 4, 2188, 74, 58, 153, 966, 8, 1703, 0, 0, 0, 0, 0, 0, 0, 227, 405, 359, 251, 2039, 12, 1040, 1590, 285, 339, 0, 0, 0, 0, 0, 0, 436, 0, 2798, 1, 0, 1248, 946, 270, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5006, 7051, 6307, 6139, 6934, 5469, 6547, 5699] avg=6144.0000 max=7051 max_rank=1 min=5006 min_rank=0 imbalance_ratio=1.147624 max_load_reduction=42101 max_load_reduction_pct=0.856547 moved_to_echo=24649 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1536, 3584, 1280, 1408, 16384, 128, 7040, 13056, 2688, 2560, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[8320, 7936, 1280, 1664, 5248, 2560, 128, 11136, 6016, 5376, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2048, 3072, 11392, 512, 3328, 22144, 128, 7168, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3968, 128, 25216, 128, 0, 11392, 7040, 1920, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2688, 128, 21120, 640, 768, 1152, 6400, 1536, 15232, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[4480, 3328, 128, 1536, 7296, 1024, 6400, 256, 16512, 8960, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[384, 896, 768, 2560, 8448, 3456, 2432, 128, 30720, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1664, 4096, 2560, 30080, 1920, 896, 2944, 5504, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[49, 84, 385, 1424, 25, 683, 198, 0, 288, 404, 26, 72, 1205, 3, 160, 218, 1643, 295, 938, 1856, 0, 569, 2288, 981, 1318, 777, 409, 3243, 4, 1, 90, 905, 897, 529, 1, 148, 13, 543, 132, 3557, 1642, 0, 967, 88, 8, 94, 899, 594, 16, 283, 168, 0, 2105, 6038, 71, 161, 4256, 810, 158, 144, 2642, 179, 404, 1064] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[66, 85, 336, 1693, 51, 651, 170, 0, 240, 346, 9, 144, 250, 1, 281, 286, 1794, 479, 1056, 1521, 0, 513, 1907, 933, 1402, 946, 348, 2527, 0, 0, 90, 941, 1433, 346, 0, 88, 5, 729, 114, 2820, 990, 0, 1037, 87, 32, 12, 715, 1185, 23, 142, 110, 0, 2424, 6421, 152, 26, 4493, 674, 206, 168, 2668, 380, 676, 1930] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[69, 87, 371, 1342, 19, 812, 236, 0, 219, 457, 4, 106, 746, 1, 506, 209, 1673, 275, 744, 1763, 2, 425, 1630, 1485, 1716, 1154, 313, 2896, 1, 1, 85, 1154, 624, 518, 0, 132, 10, 693, 142, 3573, 1368, 3, 1003, 161, 9, 45, 698, 533, 29, 188, 127, 0, 2358, 5536, 238, 125, 4158, 671, 296, 171, 3188, 221, 449, 1384] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[46, 32, 478, 183, 28, 943, 193, 0, 290, 683, 55, 0, 3653, 2, 404, 30, 1762, 30, 135, 1062, 0, 390, 2502, 1176, 1225, 303, 84, 4087, 27, 0, 319, 1025, 261, 606, 0, 210, 83, 680, 265, 4771, 2621, 5, 1295, 278, 0, 619, 1364, 25, 3, 581, 116, 0, 1971, 3554, 150, 288, 2259, 904, 121, 159, 4150, 62, 244, 360] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[60, 92, 303, 1647, 44, 820, 199, 0, 79, 427, 10, 267, 163, 3, 221, 304, 1746, 422, 835, 1870, 0, 602, 1879, 1254, 859, 1107, 275, 2968, 1, 0, 87, 820, 714, 514, 0, 102, 6, 970, 118, 2642, 936, 0, 1248, 115, 30, 20, 632, 1305, 26, 263, 163, 1, 2634, 6580, 121, 20, 4120, 679, 229, 198, 2790, 234, 677, 1701] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[41, 35, 361, 192, 36, 881, 188, 0, 300, 578, 35, 2, 3383, 1, 447, 35, 1738, 36, 375, 1736, 0, 368, 2071, 1105, 1425, 771, 107, 3768, 44, 0, 234, 988, 447, 490, 7, 116, 38, 521, 227, 4442, 2337, 3, 1209, 110, 2, 367, 1092, 48, 0, 573, 105, 0, 2031, 4526, 155, 175, 3204, 821, 228, 105, 3782, 35, 265, 410] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[72, 73, 292, 1235, 117, 662, 205, 0, 243, 405, 20, 62, 362, 8, 147, 401, 1800, 528, 935, 2131, 1, 428, 1721, 1005, 767, 767, 486, 2710, 5, 1, 89, 837, 865, 383, 0, 80, 8, 692, 110, 3562, 1329, 1, 949, 77, 13, 52, 714, 970, 31, 232, 120, 0, 2440, 6572, 116, 96, 4281, 712, 229, 151, 3277, 415, 577, 1583] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[67, 77, 368, 1530, 164, 877, 213, 0, 153, 418, 9, 186, 778, 14, 440, 223, 1706, 370, 823, 1540, 8, 478, 2344, 1008, 956, 1119, 309, 2920, 12, 7, 96, 855, 884, 493, 1, 117, 39, 636, 146, 3799, 1417, 5, 1163, 212, 16, 31, 756, 760, 41, 593, 120, 1, 2146, 5848, 145, 118, 4066, 702, 184, 213, 2672, 139, 618, 1003] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[66, 85, 336, 1693, 51, 651, 170, 0, 2883, 496, 446, 0, 0, 0, 0, 0, 240, 346, 9, 144, 250, 1, 281, 286, 4040, 0, 0, 0, 0, 0, 0, 0, 1794, 479, 1056, 1521, 0, 513, 94, 933, 0, 0, 0, 0, 0, 0, 0, 0, 1402, 946, 348, 2163, 0, 0, 90, 941, 0, 0, 0, 0, 0, 0, 0, 0, 1433, 346, 0, 88, 5, 729, 114, 2820, 207, 0, 0, 0, 0, 0, 0, 0, 990, 0, 1037, 87, 32, 12, 715, 1185, 1317, 157, 0, 0, 0, 0, 0, 0, 23, 142, 110, 0, 2424, 3538, 152, 26, 0, 0, 0, 0, 0, 0, 0, 0, 7, 674, 206, 168, 2668, 380, 676, 1930, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6877, 5597, 6390, 5890, 5742, 5532, 6415, 6709] avg=6144.0000 max=6877 max_rank=0 min=5532 min_rank=5 imbalance_ratio=1.119303 max_load_reduction=42275 max_load_reduction_pct=0.860087 moved_to_echo=24398 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[49, 84, 385, 1424, 25, 683, 198, 0, 2711, 595, 423, 0, 0, 0, 0, 0, 288, 404, 26, 72, 1205, 3, 160, 218, 3827, 0, 0, 0, 0, 0, 0, 0, 1643, 295, 938, 1856, 0, 569, 113, 981, 0, 0, 0, 0, 0, 0, 0, 0, 1318, 777, 409, 2776, 4, 1, 90, 905, 0, 0, 0, 0, 0, 0, 0, 0, 897, 529, 1, 148, 13, 543, 132, 3557, 265, 0, 0, 0, 0, 0, 0, 0, 1642, 0, 967, 88, 8, 94, 899, 594, 1580, 202, 0, 0, 0, 0, 0, 0, 16, 283, 168, 0, 2105, 3327, 71, 161, 0, 0, 0, 0, 0, 0, 0, 0, 6, 810, 158, 144, 2642, 179, 404, 1064, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6577, 6203, 6395, 6280, 6085, 6074, 6131, 5407] avg=6144.0000 max=6577 max_rank=0 min=5407 min_rank=7 imbalance_ratio=1.070475 max_load_reduction=42575 max_load_reduction_pct=0.866191 moved_to_echo=23697 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[41, 35, 361, 192, 36, 881, 188, 0, 2032, 538, 318, 0, 0, 0, 0, 0, 300, 578, 35, 2, 3383, 1, 447, 35, 2881, 0, 0, 0, 0, 0, 0, 0, 1738, 36, 375, 1736, 0, 368, 103, 1105, 0, 0, 0, 0, 0, 0, 0, 0, 1425, 771, 107, 3224, 44, 0, 234, 988, 0, 0, 0, 0, 0, 0, 0, 0, 447, 490, 7, 116, 38, 521, 227, 4442, 309, 0, 0, 0, 0, 0, 0, 0, 2337, 3, 1209, 110, 2, 367, 1092, 48, 1430, 235, 0, 0, 0, 0, 0, 0, 0, 573, 105, 0, 2031, 2494, 155, 175, 0, 0, 0, 0, 0, 0, 0, 0, 5, 821, 228, 105, 3782, 35, 265, 410, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4622, 7662, 5461, 6793, 6597, 6833, 5533, 5651] avg=6144.0000 max=7662 max_rank=1 min=4622 min_rank=0 imbalance_ratio=1.247070 max_load_reduction=41490 max_load_reduction_pct=0.844116 moved_to_echo=24614 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[72, 73, 292, 1235, 117, 662, 205, 0, 2951, 447, 425, 0, 0, 0, 0, 0, 243, 405, 20, 62, 362, 8, 147, 401, 3849, 0, 0, 0, 0, 0, 0, 0, 1800, 528, 935, 2131, 1, 428, 86, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 767, 767, 486, 2319, 5, 1, 89, 837, 0, 0, 0, 0, 0, 0, 0, 0, 865, 383, 0, 80, 8, 692, 110, 3562, 222, 0, 0, 0, 0, 0, 0, 0, 1329, 1, 949, 77, 13, 52, 714, 970, 1188, 169, 0, 0, 0, 0, 0, 0, 31, 232, 120, 0, 2440, 3621, 116, 96, 0, 0, 0, 0, 0, 0, 0, 0, 7, 712, 229, 151, 3277, 415, 577, 1583, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6479, 5497, 6914, 5271, 5922, 5462, 6656, 6951] avg=6144.0000 max=6951 max_rank=7 min=5271 min_rank=3 imbalance_ratio=1.131348 max_load_reduction=42201 max_load_reduction_pct=0.858582 moved_to_echo=24991 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[69, 87, 371, 1342, 19, 812, 236, 0, 2490, 428, 418, 0, 0, 0, 0, 0, 219, 457, 4, 106, 746, 1, 506, 209, 3740, 0, 0, 0, 0, 0, 0, 0, 1673, 275, 744, 1763, 2, 425, 73, 1485, 0, 0, 0, 0, 0, 0, 0, 0, 1716, 1154, 313, 2472, 1, 1, 85, 1154, 0, 0, 0, 0, 0, 0, 0, 0, 624, 518, 0, 132, 10, 693, 142, 3573, 240, 0, 0, 0, 0, 0, 0, 0, 1368, 3, 1003, 161, 9, 45, 698, 533, 1129, 184, 0, 0, 0, 0, 0, 0, 29, 188, 127, 0, 2358, 3046, 238, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 671, 296, 171, 3188, 221, 449, 1384, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6272, 5988, 6440, 6896, 5932, 5133, 6111, 6380] avg=6144.0000 max=6896 max_rank=3 min=5133 min_rank=5 imbalance_ratio=1.122396 max_load_reduction=42256 max_load_reduction_pct=0.859701 moved_to_echo=23556 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[46, 32, 478, 183, 28, 943, 193, 0, 1596, 650, 224, 0, 0, 0, 0, 0, 290, 683, 55, 0, 3653, 2, 404, 30, 2033, 0, 0, 0, 0, 0, 0, 0, 1762, 30, 135, 1062, 0, 390, 124, 1176, 0, 0, 0, 0, 0, 0, 0, 0, 1225, 303, 84, 3497, 27, 0, 319, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 261, 606, 0, 210, 83, 680, 265, 4771, 335, 0, 0, 0, 0, 0, 0, 0, 2621, 5, 1295, 278, 0, 619, 1364, 25, 1728, 255, 0, 0, 0, 0, 0, 0, 3, 581, 116, 0, 1971, 1958, 150, 288, 0, 0, 0, 0, 0, 0, 0, 0, 2, 904, 121, 159, 4150, 62, 244, 360, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4373, 7150, 4679, 6480, 7211, 8190, 5067, 6002] avg=6144.0000 max=8190 max_rank=5 min=4373 min_rank=0 imbalance_ratio=1.333008 max_load_reduction=40962 max_load_reduction_pct=0.833374 moved_to_echo=26470 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[60, 92, 303, 1647, 44, 820, 199, 0, 2955, 488, 409, 0, 0, 0, 0, 0, 79, 427, 10, 267, 163, 3, 221, 304, 3704, 0, 0, 0, 0, 0, 0, 0, 1746, 422, 835, 1870, 0, 602, 94, 1254, 0, 0, 0, 0, 0, 0, 0, 0, 859, 1107, 275, 2540, 1, 0, 87, 820, 0, 0, 0, 0, 0, 0, 0, 0, 714, 514, 0, 102, 6, 970, 118, 2642, 243, 0, 0, 0, 0, 0, 0, 0, 936, 0, 1248, 115, 30, 20, 632, 1305, 1297, 185, 0, 0, 0, 0, 0, 0, 26, 263, 163, 1, 2634, 3625, 121, 20, 0, 0, 0, 0, 0, 0, 0, 0, 7, 679, 229, 198, 2790, 234, 677, 1701, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7017, 5178, 6823, 5689, 5309, 5768, 6853, 6515] avg=6144.0000 max=7017 max_rank=0 min=5178 min_rank=1 imbalance_ratio=1.142090 max_load_reduction=42135 max_load_reduction_pct=0.857239 moved_to_echo=24445 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[67, 77, 368, 1530, 164, 877, 213, 0, 2626, 609, 404, 0, 0, 0, 0, 0, 153, 418, 9, 186, 778, 14, 440, 223, 3656, 0, 0, 0, 0, 0, 0, 0, 1706, 370, 823, 1540, 8, 478, 117, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 956, 1119, 309, 2499, 12, 7, 96, 855, 0, 0, 0, 0, 0, 0, 0, 0, 884, 493, 1, 117, 39, 636, 146, 3799, 239, 0, 0, 0, 0, 0, 0, 0, 1417, 5, 1163, 212, 16, 31, 756, 760, 1618, 182, 0, 0, 0, 0, 0, 0, 41, 593, 120, 1, 2146, 3222, 145, 118, 0, 0, 0, 0, 0, 0, 0, 0, 6, 702, 184, 213, 2672, 139, 618, 1003, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6935, 5877, 6050, 5853, 6354, 6160, 6386, 5537] avg=6144.0000 max=6935 max_rank=0 min=5537 min_rank=7 imbalance_ratio=1.128743 max_load_reduction=42217 max_load_reduction_pct=0.858907 moved_to_echo=24437 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[512, 640, 2944, 9344, 512, 6400, 1664, 0, 20352, 4352, 3072, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[9728, 7040, 2432, 21504, 128, 128, 1152, 7552, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[12672, 128, 8960, 1152, 128, 1280, 6912, 5504, 11392, 1664, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[13952, 2560, 5888, 13568, 128, 3840, 896, 8960, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 6016, 1664, 1408, 25216, 1792, 3968, 9472, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[256, 2944, 1152, 128, 18176, 24832, 1152, 1024, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6144, 3968, 128, 1024, 256, 5504, 1280, 29184, 2176, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1920, 3840, 256, 896, 10624, 128, 2688, 1792, 27776, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 305, 229, 2, 1203, 1805, 29, 27, 51, 1813, 20, 85, 21, 8, 2, 2353, 415, 12, 7486, 2148, 136, 874, 1211, 774, 833, 1040, 64, 273, 42, 5485, 107, 3159, 1091, 363, 86, 665, 0, 36, 2844, 6, 242, 0, 742, 252, 10, 351, 662, 3, 4142, 0, 60, 652, 321, 0, 8, 2, 1032, 3, 226, 6, 259, 10, 2839, 227] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 243, 173, 3, 1084, 2156, 5, 9, 12, 1512, 25, 71, 24, 4, 6, 2016, 467, 12, 7487, 2346, 138, 723, 1258, 737, 1060, 818, 28, 227, 0, 5879, 90, 2853, 1420, 447, 87, 516, 0, 2, 2876, 5, 173, 2, 962, 258, 26, 232, 524, 0, 4390, 0, 10, 547, 263, 0, 11, 1, 1310, 0, 151, 3, 263, 13, 3011, 183] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 2110, 56, 0, 5, 31, 3444, 339, 295, 10, 308, 0, 213, 1282, 14, 8041, 8, 344, 2131, 191, 283, 419, 1, 0, 72, 0, 4020, 80, 546, 3165, 232, 109, 2761, 1, 1, 97, 3, 4, 0, 3523, 286, 0, 560, 2914, 76, 2785, 0, 39, 0, 189, 0, 0, 46, 2463, 0, 131, 32, 107, 3, 5307, 74] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[1, 173, 166, 2, 981, 2244, 20, 22, 23, 1116, 18, 78, 38, 26, 10, 1760, 602, 5, 7630, 3009, 145, 813, 1489, 732, 1091, 996, 128, 247, 13, 5717, 111, 2798, 1323, 469, 160, 352, 0, 15, 3163, 6, 247, 4, 750, 244, 7, 315, 447, 0, 4254, 0, 43, 488, 263, 0, 5, 2, 775, 0, 93, 10, 361, 8, 2913, 231] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 228, 155, 0, 2002, 1181, 7, 14, 19, 2008, 55, 161, 12, 177, 6, 1610, 498, 5, 7911, 1018, 264, 1146, 1053, 579, 731, 534, 27, 251, 21, 5973, 86, 2281, 1682, 435, 82, 1152, 0, 5, 1655, 3, 128, 1, 1541, 237, 5, 327, 1146, 5, 3588, 1, 21, 262, 458, 0, 96, 8, 1864, 4, 199, 5, 160, 6, 3969, 94] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 133, 58, 4, 2100, 1230, 14, 13, 47, 2394, 49, 134, 18, 58, 9, 1650, 527, 7, 7825, 889, 261, 1121, 1058, 474, 841, 694, 14, 241, 0, 5920, 90, 2056, 2165, 431, 98, 885, 0, 0, 1513, 8, 237, 1, 1847, 279, 4, 378, 953, 1, 4083, 0, 42, 382, 196, 0, 3, 3, 1730, 0, 212, 6, 149, 10, 3507, 100] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 354, 270, 7, 1674, 1637, 35, 60, 58, 1566, 42, 131, 37, 1, 4, 1349, 526, 13, 7545, 1789, 240, 975, 1333, 497, 887, 1200, 92, 311, 12, 5442, 222, 1833, 1586, 319, 96, 1066, 3, 31, 2851, 35, 531, 11, 1141, 273, 16, 404, 748, 4, 3668, 1, 165, 501, 468, 0, 13, 9, 1150, 15, 179, 2, 306, 29, 3191, 198] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[1, 12, 18, 0, 2072, 219, 8, 34, 39, 2663, 139, 250, 4, 325, 2, 614, 572, 38, 8035, 135, 324, 2032, 378, 362, 476, 106, 12, 47, 0, 5129, 93, 1420, 2603, 271, 86, 2878, 6, 4, 159, 11, 46, 2, 3536, 287, 5, 463, 2023, 29, 2948, 0, 6, 22, 169, 0, 1, 7, 3122, 0, 137, 6, 121, 4, 4561, 80] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 243, 173, 3, 1084, 2156, 5, 9, 2725, 0, 0, 0, 0, 0, 0, 0, 12, 1512, 25, 71, 24, 4, 6, 2016, 1442, 789, 0, 0, 0, 0, 0, 0, 467, 12, 1058, 2346, 138, 723, 1258, 737, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 818, 28, 227, 0, 2361, 90, 2853, 0, 0, 0, 0, 0, 0, 0, 0, 1420, 447, 87, 516, 0, 2, 2876, 5, 638, 0, 0, 0, 0, 0, 0, 0, 173, 2, 962, 258, 26, 232, 524, 0, 2262, 0, 0, 0, 0, 0, 0, 0, 4390, 0, 10, 547, 263, 0, 11, 1, 1823, 0, 0, 0, 0, 0, 0, 0, 1310, 0, 151, 3, 263, 13, 3011, 183, 268, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6398, 5901, 6739, 7437, 5991, 4439, 7045, 5202] avg=6144.0000 max=7437 max_rank=3 min=4439 min_rank=5 imbalance_ratio=1.210449 max_load_reduction=41715 max_load_reduction_pct=0.848694 moved_to_echo=22677 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 305, 229, 2, 1203, 1805, 29, 27, 2725, 0, 0, 0, 0, 0, 0, 0, 51, 1813, 20, 85, 21, 8, 2, 2353, 1442, 736, 0, 0, 0, 0, 0, 0, 415, 12, 1057, 2148, 136, 874, 1211, 774, 0, 0, 0, 0, 0, 0, 0, 0, 833, 1040, 64, 273, 42, 2204, 107, 3159, 0, 0, 0, 0, 0, 0, 0, 0, 1091, 363, 86, 665, 0, 36, 2844, 6, 595, 0, 0, 0, 0, 0, 0, 0, 242, 0, 742, 252, 10, 351, 662, 3, 2262, 0, 0, 0, 0, 0, 0, 0, 4142, 0, 60, 652, 321, 0, 8, 2, 1700, 0, 0, 0, 0, 0, 0, 0, 1032, 3, 226, 6, 259, 10, 2839, 227, 250, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6325, 6531, 6627, 7722, 5686, 4524, 6885, 4852] avg=6144.0000 max=7722 max_rank=3 min=4524 min_rank=5 imbalance_ratio=1.256836 max_load_reduction=41430 max_load_reduction_pct=0.842896 moved_to_echo=21947 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 228, 155, 0, 2002, 1181, 7, 14, 2880, 0, 0, 0, 0, 0, 0, 0, 19, 2008, 55, 161, 12, 177, 6, 1610, 1523, 802, 0, 0, 0, 0, 0, 0, 498, 5, 1117, 1018, 264, 1146, 1053, 579, 0, 0, 0, 0, 0, 0, 0, 0, 731, 534, 27, 251, 21, 2398, 86, 2281, 0, 0, 0, 0, 0, 0, 0, 0, 1682, 435, 82, 1152, 0, 5, 1655, 3, 648, 0, 0, 0, 0, 0, 0, 0, 128, 1, 1541, 237, 5, 327, 1146, 5, 2391, 0, 0, 0, 0, 0, 0, 0, 3588, 1, 21, 262, 458, 0, 96, 8, 1852, 0, 0, 0, 0, 0, 0, 0, 1864, 4, 199, 5, 160, 6, 3969, 94, 273, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6467, 6373, 5680, 6329, 5662, 5781, 6286, 6574] avg=6144.0000 max=6574 max_rank=7 min=5662 min_rank=4 imbalance_ratio=1.069987 max_load_reduction=42578 max_load_reduction_pct=0.866252 moved_to_echo=24303 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 2110, 56, 0, 5, 2927, 0, 0, 0, 0, 0, 0, 0, 31, 3444, 339, 295, 10, 308, 0, 213, 1549, 540, 0, 0, 0, 0, 0, 0, 1282, 14, 1135, 8, 344, 2131, 191, 283, 0, 0, 0, 0, 0, 0, 0, 0, 419, 1, 0, 72, 0, 1615, 80, 546, 0, 0, 0, 0, 0, 0, 0, 0, 3165, 232, 109, 2761, 1, 1, 97, 3, 436, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3523, 286, 0, 560, 2914, 76, 2430, 0, 0, 0, 0, 0, 0, 0, 2785, 0, 39, 0, 189, 0, 0, 46, 1246, 0, 0, 0, 0, 0, 0, 0, 2463, 0, 131, 32, 107, 3, 5307, 74, 183, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5099, 6729, 5388, 2733, 6805, 9793, 4305, 8300] avg=6144.0000 max=9793 max_rank=5 min=2733 min_rank=3 imbalance_ratio=1.593913 max_load_reduction=39359 max_load_reduction_pct=0.800761 moved_to_echo=29203 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 133, 58, 4, 2100, 1230, 14, 13, 2853, 0, 0, 0, 0, 0, 0, 0, 47, 2394, 49, 134, 18, 58, 9, 1650, 1511, 798, 0, 0, 0, 0, 0, 0, 527, 7, 1093, 889, 261, 1121, 1058, 474, 0, 0, 0, 0, 0, 0, 0, 0, 841, 694, 14, 241, 0, 2360, 90, 2056, 0, 0, 0, 0, 0, 0, 0, 0, 2165, 431, 98, 885, 0, 0, 1513, 8, 647, 0, 0, 0, 0, 0, 0, 0, 237, 1, 1847, 279, 4, 378, 953, 1, 2368, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 42, 382, 196, 0, 3, 3, 1840, 0, 0, 0, 0, 0, 0, 0, 1730, 0, 212, 6, 149, 10, 3507, 100, 275, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6405, 6668, 5430, 6296, 5747, 6068, 6549, 5989] avg=6144.0000 max=6668 max_rank=1 min=5430 min_rank=2 imbalance_ratio=1.085286 max_load_reduction=42484 max_load_reduction_pct=0.864339 moved_to_echo=24353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 173, 166, 2, 981, 2244, 20, 22, 2777, 0, 0, 0, 0, 0, 0, 0, 23, 1116, 18, 78, 38, 26, 10, 1760, 1469, 768, 0, 0, 0, 0, 0, 0, 602, 5, 1078, 3009, 145, 813, 1489, 732, 0, 0, 0, 0, 0, 0, 0, 0, 1091, 996, 128, 247, 13, 2296, 111, 2798, 0, 0, 0, 0, 0, 0, 0, 0, 1323, 469, 160, 352, 0, 15, 3163, 6, 620, 0, 0, 0, 0, 0, 0, 0, 247, 4, 750, 244, 7, 315, 447, 0, 2306, 0, 0, 0, 0, 0, 0, 0, 4254, 0, 43, 488, 263, 0, 5, 2, 1772, 0, 0, 0, 0, 0, 0, 0, 775, 0, 93, 10, 361, 8, 2913, 231, 261, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6386, 5306, 7873, 7680, 6108, 4320, 6827, 4652] avg=6144.0000 max=7873 max_rank=2 min=4320 min_rank=5 imbalance_ratio=1.281413 max_load_reduction=41279 max_load_reduction_pct=0.839823 moved_to_echo=21907 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 12, 18, 0, 2072, 219, 8, 34, 2925, 0, 0, 0, 0, 0, 0, 0, 39, 2663, 139, 250, 4, 325, 2, 614, 1547, 689, 0, 0, 0, 0, 0, 0, 572, 38, 1135, 135, 324, 2032, 378, 362, 0, 0, 0, 0, 0, 0, 0, 0, 476, 106, 12, 47, 0, 2059, 93, 1420, 0, 0, 0, 0, 0, 0, 0, 0, 2603, 271, 86, 2878, 6, 4, 159, 11, 557, 0, 0, 0, 0, 0, 0, 0, 46, 2, 3536, 287, 5, 463, 2023, 29, 2428, 0, 0, 0, 0, 0, 0, 0, 2948, 0, 6, 22, 169, 0, 1, 7, 1590, 0, 0, 0, 0, 0, 0, 0, 3122, 0, 137, 6, 121, 4, 4561, 80, 234, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5289, 6272, 4976, 4213, 6575, 8819, 4743, 8265] avg=6144.0000 max=8819 max_rank=5 min=4213 min_rank=3 imbalance_ratio=1.435384 max_load_reduction=40333 max_load_reduction_pct=0.820577 moved_to_echo=28402 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 354, 270, 7, 1674, 1637, 35, 60, 2746, 0, 0, 0, 0, 0, 0, 0, 58, 1566, 42, 131, 37, 1, 4, 1349, 1453, 731, 0, 0, 0, 0, 0, 0, 526, 13, 1066, 1789, 240, 975, 1333, 497, 0, 0, 0, 0, 0, 0, 0, 0, 887, 1200, 92, 311, 12, 2185, 222, 1833, 0, 0, 0, 0, 0, 0, 0, 0, 1586, 319, 96, 1066, 3, 31, 2851, 35, 591, 0, 0, 0, 0, 0, 0, 0, 531, 11, 1141, 273, 16, 404, 748, 4, 2280, 0, 0, 0, 0, 0, 0, 0, 3668, 1, 165, 501, 468, 0, 13, 9, 1687, 0, 0, 0, 0, 0, 0, 0, 1150, 15, 179, 2, 306, 29, 3191, 198, 248, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6783, 5372, 6439, 6742, 6578, 5408, 6512, 5318] avg=6144.0000 max=6783 max_rank=0 min=5318 min_rank=7 imbalance_ratio=1.104004 max_load_reduction=42369 max_load_reduction_pct=0.862000 moved_to_echo=23816 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6400, 5504, 384, 1792, 128, 17536, 896, 17024, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4992, 128, 8832, 11392, 1920, 9856, 8064, 4480, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[384, 16640, 768, 1280, 256, 1024, 128, 11648, 12032, 5888, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1664, 128, 14080, 2176, 128, 3072, 9472, 128, 18816, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 1536, 1152, 128, 13312, 10624, 128, 256, 22656, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[29952, 128, 512, 2944, 2432, 0, 256, 128, 13568, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[13568, 128, 1408, 128, 1792, 128, 29312, 1280, 2048, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[15104, 3072, 896, 10368, 128, 128, 15232, 128, 4736, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[5295, 736, 1864, 14, 275, 24, 1104, 469, 366, 0, 0, 476, 80, 273, 122, 0, 183, 0, 20, 434, 1509, 97, 56, 385, 1223, 4, 1404, 164, 0, 3, 1859, 2, 1851, 2314, 55, 1472, 6, 2184, 674, 3, 53, 0, 2311, 9, 42, 2857, 334, 0, 14, 115, 5554, 1, 168, 194, 332, 52, 489, 7431, 0, 2, 0, 53, 1605, 536] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[3824, 429, 2266, 177, 1, 47, 1120, 1, 749, 0, 22, 65, 342, 243, 0, 0, 1413, 0, 373, 13, 775, 87, 12, 10, 2303, 0, 694, 247, 0, 193, 745, 36, 1283, 3325, 804, 4135, 149, 3985, 54, 0, 117, 23, 5505, 353, 0, 1750, 259, 0, 0, 449, 1269, 0, 43, 1670, 251, 0, 1147, 4505, 0, 57, 0, 801, 820, 211] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[4672, 959, 2449, 230, 168, 33, 1174, 27, 478, 1, 3, 174, 103, 260, 5, 0, 432, 0, 74, 77, 846, 101, 111, 50, 2716, 2, 986, 248, 0, 149, 983, 4, 1161, 2809, 156, 2926, 23, 4061, 68, 3, 108, 9, 4192, 343, 1, 1994, 785, 0, 0, 264, 2769, 0, 101, 1145, 313, 3, 921, 5844, 0, 32, 0, 434, 905, 267] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[2992, 1324, 2167, 205, 106, 149, 1576, 17, 743, 0, 69, 166, 244, 277, 1, 3, 650, 2, 741, 64, 605, 80, 35, 19, 3345, 2, 429, 217, 2, 229, 591, 91, 1215, 4175, 1560, 3852, 233, 2950, 53, 0, 67, 20, 4836, 365, 0, 2192, 158, 2, 0, 517, 1369, 0, 19, 1136, 369, 0, 949, 3558, 0, 119, 0, 1264, 725, 308] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[4536, 692, 2390, 5, 353, 5, 1538, 1517, 187, 0, 0, 700, 59, 574, 350, 0, 21, 0, 0, 385, 1800, 119, 80, 679, 861, 0, 1644, 263, 0, 1, 2429, 0, 819, 967, 3, 1157, 0, 1389, 645, 10, 18, 0, 1415, 1, 138, 2833, 538, 0, 19, 123, 6299, 0, 445, 70, 232, 143, 542, 7766, 0, 0, 0, 6, 1617, 769] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[4418, 1141, 1986, 111, 195, 48, 1467, 75, 626, 0, 18, 174, 83, 297, 23, 0, 541, 0, 136, 43, 937, 95, 14, 38, 2433, 0, 875, 248, 0, 156, 772, 41, 1303, 3321, 369, 3027, 79, 3214, 435, 0, 138, 3, 4711, 345, 0, 1990, 196, 0, 0, 294, 2966, 0, 93, 691, 277, 1, 1289, 5303, 0, 51, 1, 716, 828, 520] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[3043, 1043, 2133, 206, 89, 149, 1336, 0, 842, 0, 14, 52, 85, 271, 0, 0, 1023, 0, 252, 16, 754, 105, 10, 36, 2559, 9, 667, 254, 2, 224, 609, 236, 1426, 3568, 1625, 4138, 72, 3332, 49, 1, 134, 9, 5209, 376, 4, 1735, 526, 1, 0, 802, 1021, 0, 22, 1897, 286, 3, 965, 4406, 0, 27, 3, 845, 380, 271] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[3897, 1438, 2057, 248, 173, 65, 1934, 21, 956, 0, 38, 157, 271, 356, 2, 9, 819, 1, 244, 19, 852, 118, 8, 30, 2802, 3, 892, 226, 2, 262, 633, 188, 1417, 3866, 633, 3614, 117, 2427, 323, 5, 135, 33, 4237, 303, 12, 1869, 308, 7, 0, 375, 1792, 2, 71, 1223, 181, 0, 1201, 4122, 10, 60, 1, 1341, 329, 417] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[893, 429, 2266, 177, 1, 47, 1120, 1, 0, 0, 0, 0, 0, 0, 0, 0, 749, 0, 22, 65, 342, 243, 0, 0, 2931, 1733, 0, 0, 0, 0, 0, 0, 1413, 0, 373, 13, 775, 87, 12, 10, 1848, 1592, 247, 0, 0, 0, 0, 0, 2303, 0, 694, 247, 0, 193, 745, 36, 1147, 719, 0, 0, 0, 0, 0, 0, 1283, 0, 804, 1179, 149, 3985, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 23, 4358, 353, 0, 1750, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 449, 1269, 0, 43, 1670, 251, 0, 1990, 0, 0, 0, 0, 0, 0, 0, 1147, 2657, 0, 57, 0, 801, 820, 211, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4934, 6085, 6370, 6084, 7454, 6860, 5672, 5693] avg=6144.0000 max=7454 max_rank=4 min=4934 min_rank=0 imbalance_ratio=1.213216 max_load_reduction=41698 max_load_reduction_pct=0.848348 moved_to_echo=25679 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1236, 736, 1864, 14, 275, 24, 1104, 469, 0, 0, 0, 0, 0, 0, 0, 0, 366, 0, 0, 476, 80, 273, 122, 0, 4059, 1206, 0, 0, 0, 0, 0, 0, 183, 0, 20, 434, 1509, 97, 56, 385, 3049, 1108, 87, 0, 0, 0, 0, 0, 1223, 4, 1404, 164, 0, 3, 1859, 2, 481, 256, 0, 0, 0, 0, 0, 0, 1851, 0, 55, 421, 6, 2184, 674, 3, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 1830, 9, 42, 2857, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 115, 5554, 1, 168, 194, 332, 52, 708, 0, 0, 0, 0, 0, 0, 0, 489, 4382, 0, 2, 0, 53, 1605, 536, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5722, 6582, 6928, 5396, 5194, 5125, 7138, 7067] avg=6144.0000 max=7138 max_rank=6 min=5125 min_rank=5 imbalance_ratio=1.161784 max_load_reduction=42014 max_load_reduction_pct=0.854777 moved_to_echo=24524 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[699, 1324, 2167, 205, 106, 149, 1576, 17, 0, 0, 0, 0, 0, 0, 0, 0, 743, 0, 69, 166, 244, 277, 1, 3, 2293, 2176, 0, 0, 0, 0, 0, 0, 650, 2, 741, 64, 605, 80, 35, 19, 1459, 1999, 230, 0, 0, 0, 0, 0, 3345, 2, 429, 217, 2, 229, 591, 91, 1008, 670, 0, 0, 0, 0, 0, 0, 1215, 0, 1560, 1098, 233, 2950, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 20, 3828, 365, 0, 2192, 158, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 1369, 0, 19, 1136, 369, 0, 1854, 0, 0, 0, 0, 0, 0, 0, 949, 2099, 0, 119, 0, 1264, 725, 308, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6243, 5972, 5884, 6584, 7109, 6632, 5264, 5464] avg=6144.0000 max=7109 max_rank=4 min=5264 min_rank=6 imbalance_ratio=1.157064 max_load_reduction=42043 max_load_reduction_pct=0.855367 moved_to_echo=24469 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1086, 959, 2449, 230, 168, 33, 1174, 27, 0, 0, 0, 0, 0, 0, 0, 0, 478, 1, 3, 174, 103, 260, 5, 0, 3586, 1465, 0, 0, 0, 0, 0, 0, 432, 0, 74, 77, 846, 101, 111, 50, 2402, 1344, 178, 0, 0, 0, 0, 0, 2716, 2, 986, 248, 0, 149, 983, 4, 878, 512, 0, 0, 0, 0, 0, 0, 1161, 0, 156, 824, 23, 4061, 68, 3, 0, 0, 0, 0, 0, 0, 0, 0, 108, 9, 3314, 343, 1, 1994, 785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 264, 2769, 0, 101, 1145, 313, 3, 1412, 0, 0, 0, 0, 0, 0, 0, 921, 3442, 0, 32, 0, 434, 905, 267, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6126, 6075, 5615, 6478, 6296, 6554, 6007, 6001] avg=6144.0000 max=6554 max_rank=5 min=5615 min_rank=2 imbalance_ratio=1.066732 max_load_reduction=42598 max_load_reduction_pct=0.866659 moved_to_echo=24858 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1032, 1141, 1986, 111, 195, 48, 1467, 75, 0, 0, 0, 0, 0, 0, 0, 0, 626, 0, 18, 174, 83, 297, 23, 0, 3386, 1731, 0, 0, 0, 0, 0, 0, 541, 0, 136, 43, 937, 95, 14, 38, 2175, 1590, 180, 0, 0, 0, 0, 0, 2433, 0, 875, 248, 0, 156, 772, 41, 981, 526, 0, 0, 0, 0, 0, 0, 1303, 0, 369, 864, 79, 3214, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 3, 3730, 345, 0, 1990, 196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 294, 2966, 0, 93, 691, 277, 1, 1457, 0, 0, 0, 0, 0, 0, 0, 1289, 3128, 0, 51, 1, 716, 828, 520, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6055, 6338, 5749, 6032, 6264, 6402, 5779, 6533] avg=6144.0000 max=6533 max_rank=7 min=5749 min_rank=2 imbalance_ratio=1.063314 max_load_reduction=42619 max_load_reduction_pct=0.867086 moved_to_echo=24978 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1059, 692, 2390, 5, 353, 5, 1538, 1517, 0, 0, 0, 0, 0, 0, 0, 0, 187, 0, 0, 700, 59, 574, 350, 0, 3477, 505, 0, 0, 0, 0, 0, 0, 21, 0, 0, 385, 1800, 119, 80, 679, 3186, 462, 69, 0, 0, 0, 0, 0, 861, 0, 1644, 263, 0, 1, 2429, 0, 294, 201, 0, 0, 0, 0, 0, 0, 819, 0, 3, 331, 0, 1389, 645, 10, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 1121, 1, 138, 2833, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 123, 6299, 0, 445, 70, 232, 143, 556, 0, 0, 0, 0, 0, 0, 0, 542, 4580, 0, 0, 0, 6, 1617, 769, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7559, 5852, 6801, 5693, 3197, 4649, 7887, 7514] avg=6144.0000 max=7887 max_rank=6 min=3197 min_rank=4 imbalance_ratio=1.283691 max_load_reduction=41265 max_load_reduction_pct=0.839539 moved_to_echo=23247 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[711, 1043, 2133, 206, 89, 149, 1336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 842, 0, 14, 52, 85, 271, 0, 0, 2332, 1861, 0, 0, 0, 0, 0, 0, 1023, 0, 252, 16, 754, 105, 10, 36, 1807, 1707, 247, 0, 0, 0, 0, 0, 2559, 9, 667, 254, 2, 224, 609, 236, 1085, 720, 0, 0, 0, 0, 0, 0, 1426, 0, 1625, 1180, 72, 3332, 49, 1, 0, 0, 0, 0, 0, 0, 0, 0, 134, 9, 4124, 376, 4, 1735, 526, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 802, 1021, 0, 22, 1897, 286, 3, 1991, 0, 0, 0, 0, 0, 0, 0, 965, 2599, 0, 27, 3, 845, 380, 271, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5667, 5457, 5957, 6365, 7685, 6909, 6022, 5090] avg=6144.0000 max=7685 max_rank=4 min=5090 min_rank=7 imbalance_ratio=1.250814 max_load_reduction=41467 max_load_reduction_pct=0.843648 moved_to_echo=25706 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[910, 1438, 2057, 248, 173, 65, 1934, 21, 0, 0, 0, 0, 0, 0, 0, 0, 956, 0, 38, 157, 271, 356, 2, 9, 2987, 2015, 0, 0, 0, 0, 0, 0, 819, 1, 244, 19, 852, 118, 8, 30, 1691, 1851, 215, 0, 0, 0, 0, 0, 2802, 3, 892, 226, 2, 262, 633, 188, 883, 629, 0, 0, 0, 0, 0, 0, 1417, 0, 633, 1031, 117, 2427, 323, 5, 0, 0, 0, 0, 0, 0, 0, 0, 135, 33, 3354, 303, 12, 1869, 308, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 375, 1792, 2, 71, 1223, 181, 0, 1739, 0, 0, 0, 0, 0, 0, 0, 1201, 2431, 10, 60, 1, 1341, 329, 417, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6846, 6791, 5848, 6520, 5953, 6021, 5383, 5790] avg=6144.0000 max=6846 max_rank=0 min=5383 min_rank=6 imbalance_ratio=1.114258 max_load_reduction=42306 max_load_reduction_pct=0.860718 moved_to_echo=23147 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4992, 128, 256, 2048, 1280, 2560, 512, 128, 25088, 12800, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7552, 25344, 128, 384, 128, 5504, 7296, 3328, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[5120, 128, 1920, 1152, 8192, 896, 384, 1280, 17664, 11776, 1536, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7680, 7808, 17408, 1280, 1408, 640, 11264, 2176, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[18304, 128, 7680, 1920, 128, 1280, 8704, 640, 6784, 4352, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[10496, 0, 5248, 7040, 768, 23552, 2304, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[896, 128, 25728, 2176, 256, 17280, 3200, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 2944, 23040, 128, 1024, 8064, 2304, 256, 11776, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[126, 0, 85, 53, 1061, 354, 41, 1919, 102, 6369, 105, 136, 290, 3160, 6, 1231, 2963, 137, 0, 0, 22, 417, 551, 83, 600, 4, 5580, 475, 20, 277, 6, 17, 413, 0, 0, 639, 139, 1140, 163, 596, 7199, 18, 318, 6, 275, 2942, 663, 0, 982, 54, 1270, 2239, 368, 162, 220, 420, 1940, 51, 3, 22, 235, 379, 18, 88] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[242, 0, 0, 73, 615, 229, 57, 2307, 95, 6561, 260, 97, 153, 2950, 1, 1219, 3448, 166, 0, 0, 17, 251, 610, 39, 518, 47, 5711, 530, 54, 271, 3, 19, 470, 0, 0, 1061, 107, 1221, 183, 534, 7265, 36, 170, 0, 395, 4099, 661, 0, 712, 59, 931, 2435, 531, 152, 237, 298, 543, 74, 5, 3, 232, 73, 27, 95] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[421, 0, 44, 74, 323, 137, 86, 1204, 14, 6853, 187, 18, 179, 3972, 1, 1322, 2985, 105, 0, 0, 74, 129, 704, 70, 448, 27, 5833, 211, 3, 201, 0, 8, 458, 0, 0, 531, 37, 3060, 124, 409, 5477, 81, 517, 0, 199, 4645, 2048, 0, 879, 25, 749, 1355, 343, 84, 61, 1156, 691, 27, 0, 4, 90, 79, 306, 84] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[426, 0, 3, 80, 254, 164, 125, 1781, 91, 7412, 463, 56, 68, 3957, 1, 1595, 2857, 263, 0, 0, 64, 52, 923, 0, 572, 107, 5899, 293, 32, 158, 3, 1, 399, 0, 0, 288, 172, 2715, 103, 559, 5512, 83, 637, 6, 169, 4910, 1464, 0, 890, 30, 428, 1214, 242, 84, 45, 1031, 277, 8, 0, 0, 15, 11, 12, 148] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[739, 0, 2, 80, 37, 60, 123, 1255, 51, 7115, 195, 0, 6, 5292, 1, 1004, 700, 64, 2, 0, 64, 1, 809, 0, 326, 194, 5508, 23, 2, 130, 3, 0, 275, 0, 0, 55, 3, 6557, 6, 375, 1302, 73, 1986, 0, 93, 5309, 5207, 1, 254, 5, 47, 271, 35, 96, 34, 3172, 41, 4, 0, 0, 6, 3, 73, 83] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[345, 9, 53, 70, 454, 485, 118, 1596, 96, 6417, 169, 138, 271, 3985, 3, 1448, 2886, 92, 0, 2, 72, 139, 664, 379, 488, 250, 5510, 620, 34, 401, 40, 34, 459, 0, 2, 517, 77, 1730, 166, 560, 6231, 59, 338, 0, 329, 3277, 1173, 0, 919, 42, 756, 1816, 273, 172, 120, 628, 1549, 133, 18, 28, 91, 267, 36, 118] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[210, 0, 17, 101, 579, 339, 108, 2082, 65, 5988, 138, 338, 253, 3689, 0, 917, 2805, 117, 0, 0, 33, 259, 742, 98, 722, 112, 5726, 502, 32, 370, 35, 6, 807, 0, 0, 772, 148, 653, 207, 522, 7571, 8, 188, 11, 365, 3418, 246, 0, 1333, 38, 930, 2918, 288, 104, 181, 181, 1281, 110, 6, 0, 310, 50, 8, 115] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[531, 0, 0, 59, 103, 53, 127, 991, 54, 6608, 390, 4, 1, 5766, 0, 922, 68, 75, 0, 0, 93, 1, 934, 1, 428, 254, 4681, 0, 1, 130, 0, 0, 266, 0, 0, 0, 0, 7375, 1, 378, 357, 136, 3060, 0, 67, 4810, 6554, 0, 429, 0, 4, 80, 0, 158, 0, 3074, 15, 0, 0, 0, 0, 0, 32, 81] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[242, 0, 0, 73, 615, 229, 57, 2307, 2521, 1469, 0, 0, 0, 0, 0, 0, 95, 275, 260, 97, 153, 2950, 1, 1219, 0, 0, 0, 0, 0, 0, 0, 0, 3448, 166, 0, 0, 17, 251, 610, 39, 3750, 0, 0, 0, 0, 0, 0, 0, 518, 47, 5026, 530, 54, 271, 3, 19, 0, 0, 0, 0, 0, 0, 0, 0, 470, 0, 0, 1061, 107, 1221, 183, 534, 751, 994, 0, 0, 0, 0, 0, 0, 0, 36, 170, 0, 395, 2641, 661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 712, 59, 931, 2435, 531, 152, 237, 298, 707, 685, 0, 0, 0, 0, 0, 0, 543, 74, 5, 3, 232, 73, 27, 95, 4817, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7513, 5050, 8281, 6468, 5321, 3903, 6747, 5869] avg=6144.0000 max=8281 max_rank=2 min=3903 min_rank=5 imbalance_ratio=1.347819 max_load_reduction=40871 max_load_reduction_pct=0.831523 moved_to_echo=21840 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[126, 0, 85, 53, 1061, 354, 41, 1919, 2498, 1426, 0, 0, 0, 0, 0, 0, 102, 267, 105, 136, 290, 3160, 6, 1231, 0, 0, 0, 0, 0, 0, 0, 0, 2963, 137, 0, 0, 22, 417, 551, 83, 3716, 0, 0, 0, 0, 0, 0, 0, 600, 4, 4910, 475, 20, 277, 6, 17, 0, 0, 0, 0, 0, 0, 0, 0, 413, 0, 0, 639, 139, 1140, 163, 596, 539, 985, 0, 0, 0, 0, 0, 0, 0, 18, 318, 6, 275, 1895, 663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 982, 54, 1270, 2239, 368, 162, 220, 420, 508, 670, 0, 0, 0, 0, 0, 0, 1940, 51, 3, 22, 235, 379, 18, 88, 4676, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7563, 5297, 7889, 6309, 4614, 3175, 6893, 7412] avg=6144.0000 max=7889 max_rank=2 min=3175 min_rank=5 imbalance_ratio=1.284017 max_load_reduction=41263 max_load_reduction_pct=0.839498 moved_to_echo=22094 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[421, 0, 44, 74, 323, 137, 86, 1204, 1900, 1534, 0, 0, 0, 0, 0, 0, 14, 287, 187, 18, 179, 3972, 1, 1322, 0, 0, 0, 0, 0, 0, 0, 0, 2985, 105, 0, 0, 74, 129, 704, 70, 2829, 0, 0, 0, 0, 0, 0, 0, 448, 27, 5133, 211, 3, 201, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 458, 0, 0, 531, 37, 3060, 124, 409, 851, 748, 0, 0, 0, 0, 0, 0, 0, 81, 517, 0, 199, 2992, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 879, 25, 749, 1355, 343, 84, 61, 1156, 802, 700, 0, 0, 0, 0, 0, 0, 691, 27, 0, 4, 90, 79, 306, 84, 5032, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5723, 5980, 6896, 6031, 6218, 5837, 6154, 6313] avg=6144.0000 max=6896 max_rank=2 min=5723 min_rank=0 imbalance_ratio=1.122396 max_load_reduction=42256 max_load_reduction_pct=0.859701 moved_to_echo=24522 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[426, 0, 3, 80, 254, 164, 125, 1781, 1914, 1663, 0, 0, 0, 0, 0, 0, 91, 303, 463, 56, 68, 3957, 1, 1595, 0, 0, 0, 0, 0, 0, 0, 0, 2857, 263, 0, 0, 64, 52, 923, 0, 2845, 0, 0, 0, 0, 0, 0, 0, 572, 107, 5187, 293, 32, 158, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 399, 0, 0, 288, 172, 2715, 103, 559, 905, 753, 0, 0, 0, 0, 0, 0, 0, 83, 637, 6, 169, 3153, 1464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 890, 30, 428, 1214, 242, 84, 45, 1031, 852, 712, 0, 0, 0, 0, 0, 0, 277, 8, 0, 0, 15, 11, 12, 148, 5446, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6410, 6534, 7004, 6353, 5894, 5512, 5528, 5917] avg=6144.0000 max=7004 max_rank=2 min=5512 min_rank=5 imbalance_ratio=1.139974 max_load_reduction=42148 max_load_reduction_pct=0.857503 moved_to_echo=22851 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[739, 0, 2, 80, 37, 60, 123, 1255, 451, 1593, 0, 0, 0, 0, 0, 0, 51, 298, 195, 0, 6, 5292, 1, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 700, 64, 2, 0, 64, 1, 809, 0, 672, 0, 0, 0, 0, 0, 0, 0, 326, 194, 4847, 23, 2, 130, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 275, 0, 0, 55, 3, 6557, 6, 375, 973, 179, 0, 0, 0, 0, 0, 0, 0, 73, 1986, 0, 93, 3420, 5207, 1, 0, 0, 0, 0, 0, 0, 0, 0, 254, 5, 47, 271, 35, 96, 34, 3172, 916, 661, 0, 0, 0, 0, 0, 0, 41, 4, 0, 0, 6, 3, 73, 83, 5224, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4340, 6847, 2312, 5525, 8423, 10780, 5491, 5434] avg=6144.0000 max=10780 max_rank=5 min=2312 min_rank=2 imbalance_ratio=1.754557 max_load_reduction=38372 max_load_reduction_pct=0.780680 moved_to_echo=30128 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[531, 0, 0, 59, 103, 53, 127, 991, 125, 1479, 0, 0, 0, 0, 0, 0, 54, 277, 390, 4, 1, 5766, 0, 922, 0, 0, 0, 0, 0, 0, 0, 0, 68, 75, 0, 0, 93, 1, 934, 1, 184, 0, 0, 0, 0, 0, 0, 0, 428, 254, 4119, 0, 1, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 266, 0, 0, 0, 0, 7375, 1, 378, 882, 48, 0, 0, 0, 0, 0, 0, 0, 136, 3060, 0, 67, 3098, 6554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 429, 0, 4, 80, 0, 158, 0, 3074, 830, 562, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 32, 81, 4852, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3468, 7414, 1356, 4932, 8950, 12915, 5137, 4980] avg=6144.0000 max=12915 max_rank=5 min=1356 min_rank=2 imbalance_ratio=2.102051 max_load_reduction=36237 max_load_reduction_pct=0.737244 moved_to_echo=31982 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[210, 0, 17, 101, 579, 339, 108, 2082, 2629, 1341, 0, 0, 0, 0, 0, 0, 65, 251, 138, 338, 253, 3689, 0, 917, 0, 0, 0, 0, 0, 0, 0, 0, 2805, 117, 0, 0, 33, 259, 742, 98, 3908, 0, 0, 0, 0, 0, 0, 0, 722, 112, 5039, 502, 32, 370, 35, 6, 0, 0, 0, 0, 0, 0, 0, 0, 807, 0, 0, 772, 148, 653, 207, 522, 626, 1034, 0, 0, 0, 0, 0, 0, 0, 8, 188, 11, 365, 2202, 246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1333, 38, 930, 2918, 288, 104, 181, 181, 590, 687, 0, 0, 0, 0, 0, 0, 1281, 110, 6, 0, 310, 50, 8, 115, 4396, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7406, 5651, 7962, 6818, 4769, 3020, 7250, 6276] avg=6144.0000 max=7962 max_rank=2 min=3020 min_rank=5 imbalance_ratio=1.295898 max_load_reduction=41190 max_load_reduction_pct=0.838013 moved_to_echo=21315 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[345, 9, 53, 70, 454, 485, 118, 1596, 2162, 1437, 0, 0, 0, 0, 0, 0, 96, 269, 169, 138, 271, 3985, 3, 1448, 0, 0, 0, 0, 0, 0, 0, 0, 2886, 92, 0, 2, 72, 139, 664, 379, 3218, 0, 0, 0, 0, 0, 0, 0, 488, 250, 4849, 620, 34, 401, 40, 34, 0, 0, 0, 0, 0, 0, 0, 0, 459, 0, 2, 517, 77, 1730, 166, 560, 601, 851, 0, 0, 0, 0, 0, 0, 0, 59, 338, 0, 329, 2111, 1173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 919, 42, 756, 1816, 273, 172, 120, 628, 565, 661, 0, 0, 0, 0, 0, 0, 1549, 133, 18, 28, 91, 267, 36, 118, 4711, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6729, 6379, 7452, 6716, 4963, 4010, 5952, 6951] avg=6144.0000 max=7452 max_rank=2 min=4010 min_rank=5 imbalance_ratio=1.212891 max_load_reduction=41700 max_load_reduction_pct=0.848389 moved_to_echo=21876 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3072, 128, 256, 640, 3456, 1920, 896, 13184, 14208, 12032, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 512, 7296, 128, 1920, 21632, 18048, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[18816, 1024, 128, 128, 512, 1280, 6016, 768, 21248, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6400, 256, 5120, 12416, 2176, 1024, 1024, 9984, 5888, 5376, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3584, 0, 128, 3968, 768, 24576, 1024, 3968, 6144, 5632, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[640, 2304, 1920, 896, 1280, 32896, 128, 9728, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6400, 512, 128, 128, 1024, 896, 512, 896, 39168, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4224, 1024, 39168, 2688, 256, 2048, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[248, 0, 41, 0, 1358, 585, 752, 56, 0, 1901, 119, 45, 565, 55, 1035, 13, 6639, 21, 3463, 490, 513, 2148, 45, 3, 0, 3863, 8, 485, 543, 1879, 2138, 128, 2631, 445, 820, 7, 499, 0, 0, 151, 36, 1010, 115, 591, 0, 0, 287, 2825, 2293, 5, 0, 1675, 283, 0, 81, 1324, 526, 0, 432, 3176, 0, 0, 14, 787] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[146, 0, 85, 1, 52, 137, 3875, 176, 0, 101, 2, 1, 37, 0, 74, 0, 1270, 0, 5494, 244, 46, 515, 110, 0, 0, 7569, 0, 95, 3555, 2270, 77, 0, 42, 181, 4419, 0, 805, 1, 0, 18, 1, 30, 21, 693, 0, 3, 259, 7126, 17, 0, 0, 3112, 245, 0, 59, 2784, 274, 0, 74, 35, 53, 4, 0, 2964] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[449, 0, 60, 1, 1451, 699, 1026, 140, 0, 1251, 157, 47, 320, 82, 815, 13, 6192, 49, 3936, 590, 482, 2574, 136, 14, 1, 4614, 1, 579, 630, 2157, 1201, 133, 2284, 440, 1048, 26, 547, 0, 9, 17, 40, 415, 144, 625, 9, 4, 350, 4035, 1279, 0, 0, 1746, 524, 1, 55, 1365, 849, 4, 262, 2129, 11, 3, 57, 1074] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[387, 0, 65, 1, 1483, 552, 867, 43, 0, 1706, 119, 40, 507, 64, 912, 6, 6917, 15, 3915, 531, 583, 2292, 108, 13, 0, 4188, 0, 447, 616, 1698, 1605, 16, 2532, 541, 582, 0, 566, 0, 0, 27, 67, 423, 116, 510, 1, 0, 380, 3777, 1777, 3, 0, 1553, 453, 0, 113, 1410, 560, 0, 581, 2529, 0, 1, 1, 953] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[358, 0, 56, 0, 576, 579, 1456, 65, 0, 918, 43, 20, 177, 22, 761, 7, 5762, 15, 5171, 467, 270, 1527, 158, 16, 0, 5591, 2, 665, 917, 2891, 771, 64, 1502, 387, 1782, 0, 548, 0, 4, 18, 24, 243, 78, 651, 3, 2, 308, 5182, 999, 3, 0, 1849, 388, 0, 10, 2052, 739, 0, 252, 1370, 0, 1, 11, 1421] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[278, 0, 83, 0, 2, 203, 4434, 105, 0, 3, 0, 0, 1, 0, 0, 0, 282, 0, 4910, 163, 83, 407, 155, 4, 0, 7804, 0, 85, 3630, 3013, 4, 0, 8, 0, 4398, 0, 1039, 2, 0, 85, 0, 0, 0, 644, 0, 0, 265, 7295, 3, 0, 0, 3294, 533, 0, 4, 2182, 161, 0, 16, 27, 59, 16, 0, 3472] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[386, 0, 58, 0, 1430, 519, 602, 36, 3, 2341, 155, 39, 207, 54, 902, 5, 6811, 4, 4008, 355, 1082, 2622, 109, 2, 0, 3857, 2, 397, 547, 1884, 1726, 84, 2074, 173, 685, 20, 413, 0, 7, 144, 75, 669, 131, 332, 2, 0, 313, 2901, 1594, 3, 0, 1585, 494, 0, 234, 1089, 719, 0, 489, 4038, 0, 1, 24, 716] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[532, 0, 81, 0, 246, 660, 1839, 43, 1, 529, 122, 12, 88, 68, 529, 2, 5635, 17, 5019, 441, 484, 1468, 144, 5, 0, 6132, 2, 457, 900, 2627, 389, 1, 993, 251, 1591, 7, 912, 1, 0, 64, 31, 62, 41, 584, 1, 4, 293, 5812, 311, 4, 0, 1817, 594, 0, 154, 2668, 1105, 1, 262, 1209, 15, 7, 0, 1885] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[248, 0, 41, 0, 1358, 585, 752, 56, 1101, 1235, 0, 0, 0, 0, 0, 0, 0, 1901, 119, 45, 565, 55, 1035, 13, 5404, 0, 0, 0, 0, 0, 0, 0, 0, 21, 2665, 490, 513, 2148, 45, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 695, 8, 485, 543, 1879, 2138, 128, 0, 0, 0, 0, 0, 0, 0, 0, 2631, 445, 820, 7, 499, 0, 0, 151, 847, 370, 0, 0, 0, 0, 0, 0, 36, 1010, 115, 591, 0, 0, 287, 2776, 0, 0, 0, 0, 0, 0, 0, 0, 2293, 5, 0, 1675, 283, 0, 81, 1324, 428, 49, 0, 0, 0, 0, 0, 0, 526, 0, 432, 3176, 0, 0, 14, 787, 1220, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5376, 9137, 5885, 5876, 5770, 4815, 6138, 6155] avg=6144.0000 max=9137 max_rank=1 min=4815 min_rank=5 imbalance_ratio=1.487142 max_load_reduction=40015 max_load_reduction_pct=0.814107 moved_to_echo=22878 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[387, 0, 65, 1, 1483, 552, 867, 43, 1194, 1287, 0, 0, 0, 0, 0, 0, 0, 1706, 119, 40, 507, 64, 912, 6, 5630, 0, 0, 0, 0, 0, 0, 0, 0, 15, 3013, 531, 583, 2292, 108, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 753, 0, 447, 616, 1698, 1605, 16, 0, 0, 0, 0, 0, 0, 0, 0, 2532, 541, 582, 0, 566, 0, 0, 27, 918, 418, 0, 0, 0, 0, 0, 0, 67, 423, 116, 510, 1, 0, 380, 3711, 0, 0, 0, 0, 0, 0, 0, 0, 1777, 3, 0, 1553, 453, 0, 113, 1410, 484, 66, 0, 0, 0, 0, 0, 0, 560, 0, 581, 2529, 0, 1, 1, 953, 1323, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5879, 8984, 6555, 5135, 5584, 5208, 5859, 5948] avg=6144.0000 max=8984 max_rank=1 min=5135 min_rank=3 imbalance_ratio=1.462240 max_load_reduction=40168 max_load_reduction_pct=0.817220 moved_to_echo=22599 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[146, 0, 85, 1, 52, 137, 3875, 176, 2158, 236, 0, 0, 0, 0, 0, 0, 0, 101, 2, 1, 37, 0, 74, 0, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4228, 244, 46, 515, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1361, 0, 95, 3555, 2270, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 181, 4419, 0, 805, 1, 0, 18, 1659, 587, 0, 0, 0, 0, 0, 0, 1, 30, 21, 693, 0, 3, 259, 7001, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 3112, 245, 0, 59, 2784, 679, 125, 0, 0, 0, 0, 0, 0, 274, 0, 74, 35, 53, 4, 0, 2964, 2391, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6866, 1249, 5143, 7358, 7712, 8008, 7021, 5795] avg=6144.0000 max=8008 max_rank=5 min=1249 min_rank=1 imbalance_ratio=1.303385 max_load_reduction=41144 max_load_reduction_pct=0.837077 moved_to_echo=28536 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[358, 0, 56, 0, 576, 579, 1456, 65, 1594, 1072, 0, 0, 0, 0, 0, 0, 0, 918, 43, 20, 177, 22, 761, 7, 4690, 0, 0, 0, 0, 0, 0, 0, 0, 15, 3980, 467, 270, 1527, 158, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 2, 665, 917, 2891, 771, 64, 0, 0, 0, 0, 0, 0, 0, 0, 1502, 387, 1782, 0, 548, 0, 4, 18, 1226, 552, 0, 0, 0, 0, 0, 0, 24, 243, 78, 651, 3, 2, 308, 5091, 0, 0, 0, 0, 0, 0, 0, 0, 999, 3, 0, 1849, 388, 0, 10, 2052, 639, 91, 0, 0, 0, 0, 0, 0, 739, 0, 252, 1370, 0, 1, 11, 1421, 1766, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5756, 6638, 6433, 6315, 6019, 6400, 6031, 5560] avg=6144.0000 max=6638 max_rank=1 min=5560 min_rank=7 imbalance_ratio=1.080404 max_load_reduction=42514 max_load_reduction_pct=0.864950 moved_to_echo=24010 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[278, 0, 83, 0, 2, 203, 4434, 105, 2225, 53, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3779, 163, 83, 407, 155, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1402, 0, 85, 3630, 3013, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4398, 0, 1039, 2, 0, 85, 1711, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 644, 0, 0, 265, 7167, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3294, 533, 0, 4, 2182, 607, 128, 0, 0, 0, 0, 0, 0, 161, 0, 16, 27, 59, 16, 0, 3472, 2466, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7383, 233, 4591, 8134, 7767, 8076, 6751, 6217] avg=6144.0000 max=8134 max_rank=3 min=233 min_rank=1 imbalance_ratio=1.323893 max_load_reduction=41018 max_load_reduction_pct=0.834513 moved_to_echo=28811 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[532, 0, 81, 0, 246, 660, 1839, 43, 1751, 1049, 0, 0, 0, 0, 0, 0, 1, 529, 122, 12, 88, 68, 529, 2, 4586, 0, 0, 0, 0, 0, 0, 0, 0, 17, 3853, 441, 484, 1468, 144, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1090, 2, 457, 900, 2627, 389, 1, 0, 0, 0, 0, 0, 0, 0, 0, 993, 251, 1591, 7, 912, 1, 0, 64, 1349, 541, 0, 0, 0, 0, 0, 0, 31, 62, 41, 584, 1, 4, 293, 5706, 0, 0, 0, 0, 0, 0, 0, 0, 311, 4, 0, 1817, 594, 0, 154, 2668, 625, 106, 0, 0, 0, 0, 0, 0, 1105, 1, 262, 1209, 15, 7, 0, 1885, 1942, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6201, 5937, 6412, 5466, 5709, 6722, 6279, 6426] avg=6144.0000 max=6722 max_rank=5 min=5466 min_rank=3 imbalance_ratio=1.094076 max_load_reduction=42430 max_load_reduction_pct=0.863241 moved_to_echo=25136 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[386, 0, 58, 0, 1430, 519, 602, 36, 1099, 1268, 0, 0, 0, 0, 0, 0, 3, 2341, 155, 39, 207, 54, 902, 5, 5543, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3085, 355, 1082, 2622, 109, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 695, 2, 397, 547, 1884, 1726, 84, 0, 0, 0, 0, 0, 0, 0, 0, 2074, 173, 685, 20, 413, 0, 7, 144, 845, 428, 0, 0, 0, 0, 0, 0, 75, 669, 131, 332, 2, 0, 313, 2850, 0, 0, 0, 0, 0, 0, 0, 0, 1594, 3, 0, 1585, 494, 0, 234, 1089, 495, 51, 0, 0, 0, 0, 0, 0, 719, 0, 489, 4038, 0, 1, 24, 716, 1218, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5398, 9249, 7259, 5335, 4789, 4372, 5545, 7205] avg=6144.0000 max=9249 max_rank=1 min=4372 min_rank=5 imbalance_ratio=1.505371 max_load_reduction=39903 max_load_reduction_pct=0.811829 moved_to_echo=21911 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[449, 0, 60, 1, 1451, 699, 1026, 140, 1315, 1152, 0, 0, 0, 0, 0, 0, 0, 1251, 157, 47, 320, 82, 815, 13, 5040, 0, 0, 0, 0, 0, 0, 0, 0, 49, 3029, 590, 482, 2574, 136, 14, 0, 0, 0, 0, 0, 0, 0, 0, 1, 831, 1, 579, 630, 2157, 1201, 133, 0, 0, 0, 0, 0, 0, 0, 0, 2284, 440, 1048, 26, 547, 0, 9, 17, 1011, 420, 0, 0, 0, 0, 0, 0, 40, 415, 144, 625, 9, 4, 350, 3964, 0, 0, 0, 0, 0, 0, 0, 0, 1279, 0, 0, 1746, 524, 1, 55, 1365, 487, 71, 0, 0, 0, 0, 0, 0, 849, 4, 262, 2129, 11, 3, 57, 1074, 1457, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6293, 7725, 6874, 5533, 5802, 5551, 5528, 5846] avg=6144.0000 max=7725 max_rank=1 min=5528 min_rank=6 imbalance_ratio=1.257324 max_load_reduction=41427 max_load_reduction_pct=0.842834 moved_to_echo=22727 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[2816, 0, 640, 128, 6656, 3968, 14976, 768, 12544, 7424, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 7936, 128, 3328, 11392, 18432, 7936, 512, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[8320, 128, 0, 16640, 3584, 128, 768, 14976, 4480, 768, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 128, 27648, 3328, 3584, 13568, 1024, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4992, 128, 2432, 14592, 256, 128, 128, 13312, 13824, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 8832, 768, 256, 1920, 384, 5120, 128, 32256, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[384, 2944, 768, 4736, 128, 128, 2560, 38272, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[12160, 2432, 15360, 128, 5376, 128, 128, 640, 9600, 3840, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[1853, 1548, 3699, 6030, 912, 22, 24, 4595, 2213, 1176, 364, 225, 0, 0, 615, 50, 41, 844, 0, 409, 21, 28, 0, 1249, 466, 0, 89, 138, 0, 0, 1692, 963, 181, 3, 0, 54, 1015, 481, 499, 0, 4501, 182, 1, 737, 1484, 2, 0, 1002, 589, 4781, 4, 7, 0, 497, 0, 0, 0, 4, 13, 491, 2702, 594, 0, 62] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[2620, 1186, 3129, 5590, 1592, 45, 45, 4996, 2557, 1345, 289, 215, 0, 0, 556, 34, 122, 950, 0, 382, 39, 28, 0, 807, 967, 3, 98, 149, 0, 0, 1597, 963, 209, 9, 0, 85, 646, 171, 552, 2, 4507, 214, 29, 1108, 1460, 1, 0, 621, 653, 3821, 1, 2, 60, 476, 0, 0, 28, 16, 1, 1042, 2289, 635, 0, 210] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[3077, 5885, 648, 3519, 3032, 4, 1, 427, 170, 294, 16, 2336, 41, 4, 297, 15, 1, 47, 185, 4, 0, 4, 0, 45, 15, 1, 71, 16, 4, 0, 3078, 1178, 27, 5, 0, 1025, 128, 67, 1081, 1, 2058, 372, 0, 30, 1888, 7, 0, 6581, 594, 7475, 3, 216, 0, 986, 5, 0, 0, 4, 145, 1025, 397, 615, 0, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[2674, 3778, 3288, 5389, 1803, 13, 1, 2249, 758, 436, 251, 767, 1, 0, 304, 8, 9, 206, 1, 61, 11, 4, 1, 398, 162, 4, 153, 14, 0, 0, 2469, 1096, 23, 0, 0, 327, 282, 189, 856, 0, 3824, 431, 5, 87, 1885, 32, 0, 2877, 658, 6723, 15, 124, 2, 741, 2, 0, 0, 15, 256, 952, 1899, 613, 0, 25] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[3819, 5822, 390, 1381, 2136, 13, 0, 14, 6, 140, 20, 3575, 137, 2, 257, 12, 0, 5, 359, 0, 0, 0, 2, 19, 0, 1, 5, 0, 0, 0, 3228, 786, 5, 0, 0, 2464, 12, 28, 1203, 0, 1742, 963, 0, 4, 1667, 1, 0, 7442, 547, 7685, 0, 240, 1, 1120, 5, 0, 0, 2, 325, 1054, 18, 493, 0, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[3069, 2069, 2522, 6035, 2030, 24, 7, 2442, 1264, 1186, 237, 575, 0, 0, 425, 34, 30, 378, 0, 211, 13, 12, 0, 400, 428, 1, 119, 80, 0, 0, 2489, 1322, 41, 0, 0, 199, 720, 392, 754, 0, 3772, 284, 17, 503, 1831, 11, 0, 2581, 689, 5948, 0, 81, 1, 501, 0, 0, 23, 2, 18, 887, 1803, 628, 0, 64] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[2249, 1142, 4259, 5505, 906, 36, 5, 4556, 2052, 1284, 292, 157, 0, 0, 716, 23, 38, 740, 0, 416, 10, 29, 0, 1359, 523, 4, 84, 79, 0, 0, 1691, 1223, 204, 19, 0, 80, 689, 224, 665, 0, 4317, 319, 8, 378, 1610, 3, 0, 729, 601, 4879, 1, 1, 33, 667, 0, 0, 19, 24, 152, 890, 2350, 636, 0, 276] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[2631, 1427, 3364, 5179, 1781, 104, 36, 3685, 1720, 1266, 253, 282, 0, 0, 579, 35, 97, 972, 0, 654, 51, 8, 0, 555, 724, 18, 142, 95, 0, 0, 1189, 777, 206, 8, 12, 69, 735, 198, 826, 2, 3212, 371, 55, 743, 1703, 6, 2, 1189, 923, 5605, 1, 33, 17, 514, 0, 0, 120, 21, 51, 1005, 3086, 612, 0, 203] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1120, 0, 3699, 0, 912, 22, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2213, 1176, 364, 225, 0, 0, 615, 50, 733, 873, 0, 0, 0, 0, 0, 0, 41, 844, 0, 409, 21, 28, 0, 1249, 5759, 0, 0, 0, 0, 0, 0, 0, 466, 0, 89, 138, 0, 0, 1692, 963, 2893, 81, 0, 0, 0, 0, 0, 0, 181, 3, 0, 54, 1015, 481, 499, 0, 4595, 459, 271, 0, 0, 0, 0, 0, 919, 182, 1, 737, 1484, 2, 0, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 589, 3827, 4, 7, 0, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 13, 491, 2702, 594, 0, 62, 1089, 689, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5777, 6249, 8351, 6322, 7558, 4327, 4924, 5644] avg=6144.0000 max=8351 max_rank=2 min=4327 min_rank=5 imbalance_ratio=1.359212 max_load_reduction=40801 max_load_reduction_pct=0.830098 moved_to_echo=22453 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1859, 0, 648, 0, 3032, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 170, 294, 16, 2336, 41, 4, 297, 15, 1218, 1366, 0, 0, 0, 0, 0, 0, 1, 47, 185, 4, 0, 4, 0, 45, 3361, 0, 0, 0, 0, 0, 0, 0, 15, 1, 71, 16, 4, 0, 3078, 1178, 1323, 127, 0, 0, 0, 0, 0, 0, 27, 5, 0, 1025, 128, 67, 1081, 1, 427, 1746, 158, 0, 0, 0, 0, 0, 420, 372, 0, 30, 1888, 7, 0, 6581, 0, 0, 0, 0, 0, 0, 0, 0, 594, 5982, 3, 216, 0, 986, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 145, 1025, 397, 615, 0, 2, 4139, 315, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5544, 5757, 3647, 5813, 4665, 9298, 7786, 6642] avg=6144.0000 max=9298 max_rank=5 min=3647 min_rank=2 imbalance_ratio=1.513346 max_load_reduction=39854 max_load_reduction_pct=0.810832 moved_to_echo=28391 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1583, 0, 3129, 0, 1592, 45, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2557, 1345, 289, 215, 0, 0, 556, 34, 1037, 698, 0, 0, 0, 0, 0, 0, 122, 950, 0, 382, 39, 28, 0, 807, 5339, 0, 0, 0, 0, 0, 0, 0, 967, 3, 98, 149, 0, 0, 1597, 963, 2897, 65, 0, 0, 0, 0, 0, 0, 209, 9, 0, 85, 646, 171, 552, 2, 4996, 352, 251, 0, 0, 0, 0, 0, 920, 214, 29, 1108, 1460, 1, 0, 621, 0, 0, 0, 0, 0, 0, 0, 0, 653, 3058, 1, 2, 60, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 16, 1, 1042, 2289, 635, 0, 210, 834, 690, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6394, 6731, 7667, 6739, 7273, 4353, 4250, 5745] avg=6144.0000 max=7667 max_rank=2 min=4250 min_rank=6 imbalance_ratio=1.247884 max_load_reduction=41485 max_load_reduction_pct=0.844014 moved_to_echo=21621 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1610, 0, 3288, 0, 1803, 13, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 758, 436, 251, 767, 1, 0, 304, 8, 1064, 1232, 0, 0, 0, 0, 0, 0, 9, 206, 1, 61, 11, 4, 1, 398, 5148, 0, 0, 0, 0, 0, 0, 0, 162, 4, 153, 14, 0, 0, 2469, 1096, 2462, 118, 0, 0, 0, 0, 0, 0, 23, 0, 0, 327, 282, 189, 856, 0, 2249, 1122, 241, 0, 0, 0, 0, 0, 772, 431, 5, 87, 1885, 32, 0, 2877, 0, 0, 0, 0, 0, 0, 0, 0, 658, 5373, 15, 124, 2, 741, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 256, 952, 1899, 613, 0, 25, 2656, 590, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6715, 4821, 5839, 6478, 5289, 6089, 6915, 7006] avg=6144.0000 max=7006 max_rank=7 min=4821 min_rank=1 imbalance_ratio=1.140299 max_load_reduction=42146 max_load_reduction_pct=0.857463 moved_to_echo=25299 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2307, 0, 390, 0, 2136, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 140, 20, 3575, 137, 2, 257, 12, 1512, 1404, 0, 0, 0, 0, 0, 0, 0, 5, 359, 0, 0, 0, 2, 19, 1320, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 0, 0, 0, 3228, 786, 1119, 131, 0, 0, 0, 0, 0, 0, 5, 0, 0, 2464, 12, 28, 1203, 0, 14, 1728, 61, 0, 0, 0, 0, 0, 357, 963, 0, 4, 1667, 1, 0, 7442, 0, 0, 0, 0, 0, 0, 0, 0, 547, 6150, 0, 240, 1, 1120, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 325, 1054, 18, 493, 0, 2, 4094, 266, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4846, 7065, 1705, 5270, 5515, 10434, 8063, 6254] avg=6144.0000 max=10434 max_rank=5 min=1705 min_rank=2 imbalance_ratio=1.698242 max_load_reduction=38718 max_load_reduction_pct=0.787720 moved_to_echo=30266 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1854, 0, 2522, 0, 2030, 24, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1264, 1186, 237, 575, 0, 0, 425, 34, 1215, 1087, 0, 0, 0, 0, 0, 0, 30, 378, 0, 211, 13, 12, 0, 400, 5765, 0, 0, 0, 0, 0, 0, 0, 428, 1, 119, 80, 0, 0, 2489, 1322, 2424, 101, 0, 0, 0, 0, 0, 0, 41, 0, 0, 199, 720, 392, 754, 0, 2442, 615, 270, 0, 0, 0, 0, 0, 771, 284, 17, 503, 1831, 11, 0, 2581, 0, 0, 0, 0, 0, 0, 0, 0, 689, 4760, 0, 81, 1, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 2, 18, 887, 1803, 628, 0, 64, 1454, 577, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6437, 6023, 6809, 6964, 5433, 5998, 6032, 5456] avg=6144.0000 max=6964 max_rank=3 min=5433 min_rank=4 imbalance_ratio=1.133464 max_load_reduction=42188 max_load_reduction_pct=0.858317 moved_to_echo=22919 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1359, 0, 4259, 0, 906, 36, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2052, 1284, 292, 157, 0, 0, 716, 23, 890, 891, 0, 0, 0, 0, 0, 0, 38, 740, 0, 416, 10, 29, 0, 1359, 5259, 0, 0, 0, 0, 0, 0, 0, 523, 4, 84, 79, 0, 0, 1691, 1223, 2775, 83, 0, 0, 0, 0, 0, 0, 204, 19, 0, 80, 689, 224, 665, 0, 4556, 339, 246, 0, 0, 0, 0, 0, 881, 319, 8, 378, 1610, 3, 0, 729, 0, 0, 0, 0, 0, 0, 0, 0, 601, 3905, 1, 1, 33, 667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 24, 152, 890, 2350, 636, 0, 276, 803, 661, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6565, 6305, 7851, 6462, 7022, 3928, 5208, 5811] avg=6144.0000 max=7851 max_rank=2 min=3928 min_rank=5 imbalance_ratio=1.277832 max_load_reduction=41301 max_load_reduction_pct=0.840271 moved_to_echo=21969 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1589, 0, 3364, 0, 1781, 104, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1720, 1266, 253, 282, 0, 0, 579, 35, 1042, 1024, 0, 0, 0, 0, 0, 0, 97, 972, 0, 654, 51, 8, 0, 555, 4946, 0, 0, 0, 0, 0, 0, 0, 724, 18, 142, 95, 0, 0, 1189, 777, 2064, 95, 0, 0, 0, 0, 0, 0, 206, 8, 12, 69, 735, 198, 826, 2, 3685, 423, 233, 0, 0, 0, 0, 0, 656, 371, 55, 743, 1703, 6, 2, 1189, 0, 0, 0, 0, 0, 0, 0, 0, 923, 4486, 1, 33, 17, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 21, 51, 1005, 3086, 612, 0, 203, 1004, 492, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6874, 6201, 7283, 5104, 6397, 4725, 5974, 6594] avg=6144.0000 max=7283 max_rank=2 min=4725 min_rank=5 imbalance_ratio=1.185384 max_load_reduction=41869 max_load_reduction_pct=0.851827 moved_to_echo=23690 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[10752, 7168, 1792, 8192, 256, 128, 3840, 256, 8832, 8576, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[256, 128, 1024, 7424, 14592, 4864, 0, 896, 16128, 4352, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[13312, 0, 21376, 0, 14208, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5760, 3200, 128, 3712, 13568, 128, 128, 23040, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3328, 128, 768, 640, 128, 0, 17536, 8320, 18048, 896, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[384, 4224, 640, 2176, 256, 128, 128, 4864, 36992, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5376, 37632, 128, 768, 128, 5504, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[896, 128, 128, 4352, 4352, 1792, 6528, 128, 23040, 6784, 1792, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 4136, 28, 309, 196, 0, 528, 12, 3613, 1, 1669, 8, 0, 0, 8148, 0, 388, 20, 4731, 1, 2, 6145, 0, 9, 12, 3, 737, 12, 0, 7, 17, 5145, 328, 0, 18, 8, 726, 1, 0, 79, 0, 29, 5, 249, 3181, 171, 204, 65, 183, 3, 179, 77, 2230, 127, 14, 8, 3726, 38, 3, 196, 16, 0, 1411] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[1, 1, 75, 629, 1366, 698, 0, 294, 23, 1012, 0, 6971, 0, 0, 7, 7744, 109, 2470, 11, 708, 43, 8, 2235, 0, 228, 381, 52, 1285, 450, 1, 196, 625, 814, 532, 7, 222, 4, 166, 384, 0, 529, 5, 441, 480, 157, 1650, 2929, 91, 637, 734, 42, 801, 539, 1586, 2331, 177, 85, 2486, 802, 60, 254, 234, 0, 2350] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[6, 1, 22, 963, 831, 575, 0, 254, 33, 1555, 0, 7030, 0, 0, 6, 7519, 70, 2677, 11, 518, 7, 27, 2236, 0, 182, 186, 190, 993, 464, 2, 136, 1061, 728, 895, 14, 336, 7, 202, 238, 1, 634, 1, 548, 886, 110, 974, 3052, 90, 418, 301, 29, 1334, 981, 825, 2355, 284, 71, 2343, 605, 13, 249, 314, 0, 2759] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[4, 2, 125, 507, 817, 874, 0, 331, 97, 2057, 0, 7058, 0, 2, 10, 7555, 71, 2627, 5, 1086, 101, 20, 2791, 0, 123, 143, 93, 711, 641, 1, 165, 806, 1235, 607, 37, 165, 6, 284, 145, 0, 545, 2, 373, 716, 179, 1549, 2710, 118, 577, 403, 34, 932, 422, 1350, 1690, 321, 56, 2267, 724, 19, 257, 290, 27, 2289] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 1, 5664, 3, 287, 50, 0, 543, 0, 3702, 0, 372, 0, 0, 1, 7816, 0, 13, 1, 6427, 0, 1, 5787, 0, 0, 1, 0, 191, 6, 0, 0, 2, 5954, 60, 0, 0, 0, 1162, 0, 0, 1, 0, 0, 5, 164, 3076, 40, 194, 6, 65, 2, 116, 1, 3185, 3, 0, 0, 3303, 0, 1, 28, 0, 0, 918] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[1, 4, 811, 452, 496, 643, 0, 298, 36, 2147, 0, 6409, 0, 0, 6, 7851, 142, 1949, 5, 1918, 16, 27, 4590, 0, 51, 75, 71, 936, 328, 0, 126, 479, 2112, 579, 5, 163, 5, 441, 82, 0, 205, 2, 235, 359, 212, 1538, 1075, 99, 288, 391, 33, 602, 388, 1495, 923, 143, 52, 3372, 648, 16, 225, 148, 0, 3449] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 7, 1534, 131, 480, 807, 0, 407, 2, 1798, 0, 5261, 0, 0, 2, 8085, 13, 2037, 2, 2337, 7, 2, 3822, 0, 12, 115, 2, 659, 442, 7, 144, 44, 2799, 341, 0, 61, 0, 287, 43, 0, 16, 0, 96, 50, 219, 1637, 1433, 109, 872, 788, 41, 595, 104, 2628, 392, 73, 10, 4222, 323, 2, 250, 114, 0, 3488] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1, 5, 38, 743, 579, 910, 0, 374, 64, 1398, 0, 6797, 1, 0, 22, 7952, 23, 3132, 4, 715, 78, 38, 1745, 0, 77, 216, 27, 804, 731, 0, 234, 497, 1076, 452, 23, 221, 3, 2, 201, 0, 155, 0, 449, 465, 225, 979, 3855, 98, 668, 639, 56, 832, 146, 1528, 2106, 131, 134, 3631, 566, 130, 294, 227, 0, 2655] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4136, 28, 309, 196, 0, 528, 2640, 0, 0, 0, 0, 0, 0, 0, 12, 3613, 1, 1268, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 388, 20, 4731, 1, 2, 3048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 12, 3, 737, 12, 0, 7, 17, 4285, 0, 0, 0, 0, 0, 0, 0, 5145, 328, 0, 18, 8, 726, 1, 0, 2183, 1223, 0, 0, 0, 0, 0, 0, 79, 0, 29, 5, 249, 3181, 171, 204, 204, 914, 0, 0, 0, 0, 0, 0, 65, 183, 3, 179, 77, 2230, 127, 14, 197, 441, 0, 0, 0, 0, 0, 0, 8, 3285, 38, 3, 196, 16, 0, 1411, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7837, 4902, 8190, 5082, 9632, 5036, 3516, 4957] avg=6144.0000 max=9632 max_rank=4 min=3516 min_rank=6 imbalance_ratio=1.567708 max_load_reduction=39520 max_load_reduction_pct=0.804036 moved_to_echo=23141 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 75, 629, 1366, 698, 0, 294, 2509, 0, 0, 0, 0, 0, 0, 0, 23, 1012, 0, 5290, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 2470, 11, 708, 43, 8, 1109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 228, 381, 52, 1285, 450, 1, 196, 625, 4074, 0, 0, 0, 0, 0, 0, 0, 814, 532, 7, 222, 4, 166, 384, 0, 794, 1161, 0, 0, 0, 0, 0, 0, 529, 5, 441, 480, 157, 1650, 2929, 91, 856, 332, 0, 0, 0, 0, 0, 0, 637, 734, 42, 801, 539, 1586, 2331, 177, 825, 294, 0, 0, 0, 0, 0, 0, 85, 2192, 802, 60, 254, 234, 0, 2350, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5573, 6332, 4458, 7292, 4084, 7470, 7966, 5977] avg=6144.0000 max=7966 max_rank=6 min=4084 min_rank=4 imbalance_ratio=1.296549 max_load_reduction=41186 max_load_reduction_pct=0.837931 moved_to_echo=25497 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[6, 1, 22, 963, 831, 575, 0, 254, 2436, 0, 0, 0, 0, 0, 0, 0, 33, 1555, 0, 5335, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 2677, 11, 518, 7, 27, 1110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, 186, 190, 993, 464, 2, 136, 1061, 3954, 0, 0, 0, 0, 0, 0, 0, 728, 895, 14, 336, 7, 202, 238, 1, 794, 1129, 0, 0, 0, 0, 0, 0, 634, 1, 548, 886, 110, 974, 3052, 90, 863, 332, 0, 0, 0, 0, 0, 0, 418, 301, 29, 1334, 981, 825, 2355, 284, 832, 277, 0, 0, 0, 0, 0, 0, 71, 2066, 605, 13, 249, 314, 0, 2759, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5088, 6929, 4420, 7168, 4344, 7490, 7636, 6077] avg=6144.0000 max=7636 max_rank=6 min=4344 min_rank=4 imbalance_ratio=1.242839 max_load_reduction=41516 max_load_reduction_pct=0.844645 moved_to_echo=25547 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 4, 811, 452, 496, 643, 0, 298, 2544, 0, 0, 0, 0, 0, 0, 0, 36, 2147, 0, 4863, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 1949, 5, 1918, 16, 27, 2277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 75, 71, 936, 328, 0, 126, 479, 4130, 0, 0, 0, 0, 0, 0, 0, 2112, 579, 5, 163, 5, 441, 82, 0, 1630, 1177, 0, 0, 0, 0, 0, 0, 205, 2, 235, 359, 212, 1538, 1075, 99, 787, 683, 0, 0, 0, 0, 0, 0, 288, 391, 33, 602, 388, 1495, 923, 143, 759, 399, 0, 0, 0, 0, 0, 0, 52, 2973, 648, 16, 225, 148, 0, 3449, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5249, 7052, 6334, 6196, 6194, 5195, 5421, 7511] avg=6144.0000 max=7511 max_rank=7 min=5195 min_rank=5 imbalance_ratio=1.222493 max_load_reduction=41641 max_load_reduction_pct=0.847188 moved_to_echo=24321 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 5664, 3, 287, 50, 0, 543, 2534, 0, 0, 0, 0, 0, 0, 0, 0, 3702, 0, 283, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1, 6427, 0, 1, 2870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 191, 6, 0, 0, 2, 4111, 0, 0, 0, 0, 0, 0, 0, 5954, 60, 0, 0, 0, 1162, 0, 0, 2056, 1171, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5, 164, 3076, 40, 194, 45, 861, 0, 0, 0, 0, 0, 0, 6, 65, 2, 116, 1, 3185, 3, 0, 44, 391, 0, 0, 0, 0, 0, 0, 0, 2912, 0, 1, 28, 0, 0, 918, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9082, 3986, 9312, 4311, 10403, 4386, 3813, 3859] avg=6144.0000 max=10403 max_rank=4 min=3813 min_rank=6 imbalance_ratio=1.693197 max_load_reduction=38749 max_load_reduction_pct=0.788350 moved_to_echo=22461 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 1534, 131, 480, 807, 0, 407, 2621, 0, 0, 0, 0, 0, 0, 0, 2, 1798, 0, 3984, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 2037, 2, 2337, 7, 2, 1888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 115, 2, 659, 442, 7, 144, 44, 4252, 0, 0, 0, 0, 0, 0, 0, 2799, 341, 0, 61, 0, 287, 43, 0, 1361, 1212, 0, 0, 0, 0, 0, 0, 16, 0, 96, 50, 219, 1637, 1433, 109, 650, 573, 0, 0, 0, 0, 0, 0, 872, 788, 41, 595, 104, 2628, 392, 73, 627, 506, 0, 0, 0, 0, 0, 0, 10, 3716, 323, 2, 250, 114, 0, 3488, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5987, 5786, 6286, 5677, 6104, 4783, 6626, 7903] avg=6144.0000 max=7903 max_rank=7 min=4783 min_rank=5 imbalance_ratio=1.286296 max_load_reduction=41249 max_load_reduction_pct=0.839213 moved_to_echo=25416 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 5, 38, 743, 579, 910, 0, 374, 2578, 0, 0, 0, 0, 0, 0, 0, 64, 1398, 0, 5158, 1, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 3132, 4, 715, 78, 38, 867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 216, 27, 804, 731, 0, 234, 497, 4182, 0, 0, 0, 0, 0, 0, 0, 1076, 452, 23, 221, 3, 2, 201, 0, 619, 1192, 0, 0, 0, 0, 0, 0, 155, 0, 449, 465, 225, 979, 3855, 98, 834, 259, 0, 0, 0, 0, 0, 0, 668, 639, 56, 832, 146, 1528, 2106, 131, 805, 430, 0, 0, 0, 0, 0, 0, 134, 3201, 566, 130, 294, 227, 0, 2655, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5228, 6643, 4857, 6768, 3789, 7319, 7341, 7207] avg=6144.0000 max=7341 max_rank=6 min=3789 min_rank=4 imbalance_ratio=1.194824 max_load_reduction=41811 max_load_reduction_pct=0.850647 moved_to_echo=25656 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4, 2, 125, 507, 817, 874, 0, 331, 2448, 0, 0, 0, 0, 0, 0, 0, 97, 2057, 0, 5356, 0, 2, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 2627, 5, 1086, 101, 20, 1385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 143, 93, 711, 641, 1, 165, 806, 3975, 0, 0, 0, 0, 0, 0, 0, 1235, 607, 37, 165, 6, 284, 145, 0, 991, 1132, 0, 0, 0, 0, 0, 0, 545, 2, 373, 716, 179, 1549, 2710, 118, 866, 415, 0, 0, 0, 0, 0, 0, 577, 403, 34, 932, 422, 1350, 1690, 321, 836, 268, 0, 0, 0, 0, 0, 0, 56, 1999, 724, 19, 257, 290, 27, 2289, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5108, 7522, 5295, 6658, 4602, 7473, 6833, 5661] avg=6144.0000 max=7522 max_rank=1 min=4602 min_rank=4 imbalance_ratio=1.224284 max_load_reduction=41630 max_load_reduction_pct=0.846965 moved_to_echo=24569 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2176, 128, 2176, 3072, 1536, 14592, 15360, 1024, 5120, 4480, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[512, 22400, 3712, 256, 1792, 1408, 128, 19328, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3584, 3584, 256, 5504, 2688, 14848, 9984, 1152, 4992, 3072, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[512, 15360, 128, 18560, 256, 128, 14592, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[19968, 3840, 128, 1280, 128, 3328, 1152, 128, 10496, 9472, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 12416, 3456, 5248, 4864, 0, 3072, 20352, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[384, 17408, 128, 31616, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[768, 1152, 512, 6400, 3200, 128, 1024, 3584, 33024, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=17 tokens_per_expert_current_ep_rank=[2254, 35, 31, 324, 1766, 247, 2, 0, 0, 194, 337, 129, 3768, 16, 1457, 432, 0, 623, 7, 4221, 0, 584, 220, 2, 1302, 0, 762, 2551, 4, 1908, 652, 787, 4156, 129, 789, 380, 247, 458, 123, 996, 0, 414, 364, 148, 1118, 141, 687, 7, 4148, 0, 353, 699, 500, 1289, 1244, 664, 470, 4745, 135, 0, 5, 0, 17, 111] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=17 tokens_per_expert_current_ep_rank=[217, 316, 89, 930, 272, 622, 1, 1, 14, 18, 354, 1, 6913, 0, 36, 23, 0, 13, 0, 544, 0, 76, 214, 1013, 5803, 0, 1, 1846, 3, 4430, 13, 84, 8070, 928, 5224, 24, 3652, 25, 238, 20, 0, 625, 119, 0, 42, 0, 71, 154, 1700, 0, 394, 53, 552, 24, 30, 478, 56, 139, 855, 1755, 3, 0, 3, 71] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=17 tokens_per_expert_current_ep_rank=[1408, 149, 120, 563, 869, 471, 0, 1, 0, 133, 440, 208, 5802, 8, 544, 136, 0, 168, 5, 3688, 0, 499, 137, 75, 3541, 0, 272, 2906, 4, 2744, 359, 742, 6433, 578, 1634, 148, 1284, 229, 121, 659, 0, 483, 355, 55, 420, 90, 487, 65, 3384, 0, 513, 358, 580, 586, 840, 601, 299, 2127, 593, 21, 5, 0, 53, 159] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=17 tokens_per_expert_current_ep_rank=[916, 488, 107, 185, 1697, 624, 0, 1, 1, 50, 460, 40, 6120, 4, 136, 108, 0, 99, 0, 2479, 0, 639, 137, 101, 2968, 0, 86, 2486, 9, 3361, 177, 691, 7653, 1160, 4035, 58, 2174, 337, 208, 204, 0, 834, 347, 15, 144, 44, 191, 96, 2952, 0, 814, 56, 335, 189, 252, 743, 133, 951, 553, 134, 0, 0, 214, 156] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=17 tokens_per_expert_current_ep_rank=[2182, 125, 65, 170, 2042, 211, 0, 0, 0, 119, 336, 137, 4362, 4, 805, 312, 0, 862, 0, 4396, 0, 573, 309, 12, 1661, 0, 323, 2757, 1, 1782, 832, 791, 5067, 341, 1143, 141, 423, 319, 119, 638, 0, 494, 385, 37, 715, 172, 896, 21, 3817, 0, 838, 518, 218, 793, 673, 648, 740, 4498, 185, 0, 15, 0, 7, 122] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=17 tokens_per_expert_current_ep_rank=[3, 221, 86, 1168, 23, 365, 0, 0, 47, 0, 105, 0, 6547, 0, 0, 1, 0, 0, 0, 52, 0, 4, 75, 1728, 6530, 0, 1, 908, 10, 3991, 2, 1, 8185, 1625, 5448, 0, 4381, 0, 126, 1, 0, 696, 31, 1, 1, 3, 2, 185, 945, 0, 473, 6, 372, 5, 5, 481, 2, 14, 684, 3598, 0, 3, 1, 10] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=17 tokens_per_expert_current_ep_rank=[1344, 215, 56, 515, 1902, 339, 0, 0, 0, 145, 467, 100, 4553, 11, 668, 331, 0, 564, 97, 4348, 0, 700, 344, 43, 2019, 0, 505, 2799, 3, 2757, 386, 612, 5201, 522, 1258, 518, 736, 402, 136, 718, 0, 422, 547, 238, 539, 179, 948, 56, 3001, 0, 396, 405, 663, 578, 1291, 590, 626, 2848, 205, 0, 7, 0, 197, 102] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=17 tokens_per_expert_current_ep_rank=[1976, 189, 34, 173, 2170, 306, 0, 0, 0, 260, 432, 81, 4725, 0, 836, 308, 0, 760, 2, 5245, 0, 674, 270, 0, 486, 0, 188, 2134, 10, 2282, 306, 951, 5026, 554, 1510, 246, 380, 966, 129, 537, 0, 538, 534, 132, 422, 145, 734, 48, 3688, 1, 438, 552, 198, 724, 954, 760, 592, 4152, 149, 0, 9, 2, 107, 127] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=17 rerouted_tokens_per_expert_current_ep_rank=[2254, 35, 31, 324, 1766, 247, 2, 0, 1442, 69, 0, 0, 0, 0, 0, 0, 0, 194, 337, 129, 3379, 16, 1457, 432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 623, 7, 4221, 0, 584, 220, 2, 355, 389, 122, 0, 0, 0, 0, 0, 0, 0, 762, 2551, 4, 1740, 652, 787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 781, 380, 247, 458, 123, 996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 414, 364, 148, 1118, 141, 687, 7, 2714, 0, 0, 0, 0, 0, 0, 0, 4148, 0, 353, 699, 500, 1289, 1244, 664, 46, 8, 0, 0, 0, 0, 0, 0, 470, 4745, 135, 0, 5, 0, 17, 111, 878, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6170, 5944, 6523, 6496, 3114, 5593, 8951, 6361] avg=6144.0000 max=8951 max_rank=6 min=3114 min_rank=4 imbalance_ratio=1.456868 max_load_reduction=40201 max_load_reduction_pct=0.817891 moved_to_echo=24019 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=17 rerouted_tokens_per_expert_current_ep_rank=[217, 316, 89, 930, 272, 622, 1, 1, 2800, 307, 0, 0, 0, 0, 0, 0, 14, 18, 354, 1, 6199, 0, 36, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 544, 0, 76, 214, 1013, 1583, 714, 284, 0, 0, 0, 0, 0, 0, 0, 1, 1846, 3, 4039, 13, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 928, 5168, 24, 3652, 25, 238, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 625, 119, 0, 42, 0, 71, 154, 5270, 0, 0, 0, 0, 0, 0, 0, 1700, 0, 394, 53, 552, 24, 30, 478, 107, 56, 0, 0, 0, 0, 0, 0, 56, 139, 855, 1755, 3, 0, 3, 71, 3913, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5555, 6645, 4441, 5986, 10055, 6281, 3394, 6795] avg=6144.0000 max=10055 max_rank=4 min=3394 min_rank=6 imbalance_ratio=1.636556 max_load_reduction=39097 max_load_reduction_pct=0.795431 moved_to_echo=26525 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=17 rerouted_tokens_per_expert_current_ep_rank=[2182, 125, 65, 170, 2042, 211, 0, 0, 1758, 88, 0, 0, 0, 0, 0, 0, 0, 119, 336, 137, 3912, 4, 805, 312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 862, 0, 4396, 0, 573, 309, 12, 453, 450, 114, 0, 0, 0, 0, 0, 0, 0, 323, 2757, 1, 1625, 832, 791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 341, 1131, 141, 423, 319, 119, 638, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 385, 37, 715, 172, 896, 21, 3309, 0, 0, 0, 0, 0, 0, 0, 3817, 0, 838, 518, 218, 793, 673, 648, 43, 12, 0, 0, 0, 0, 0, 0, 740, 4498, 185, 0, 15, 0, 7, 122, 1120, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6641, 5625, 7169, 6329, 3112, 6029, 7560, 6687] avg=6144.0000 max=7560 max_rank=6 min=3112 min_rank=4 imbalance_ratio=1.230469 max_load_reduction=41592 max_load_reduction_pct=0.846191 moved_to_echo=23388 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=17 rerouted_tokens_per_expert_current_ep_rank=[1408, 149, 120, 563, 869, 471, 0, 1, 2232, 187, 0, 0, 0, 0, 0, 0, 0, 133, 440, 208, 5203, 8, 544, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 5, 3688, 0, 499, 137, 75, 968, 599, 176, 0, 0, 0, 0, 0, 0, 0, 272, 2906, 4, 2502, 359, 742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 578, 1617, 148, 1284, 229, 121, 659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 355, 55, 420, 90, 487, 65, 4201, 0, 0, 0, 0, 0, 0, 0, 3384, 0, 513, 358, 580, 586, 840, 601, 66, 17, 0, 0, 0, 0, 0, 0, 299, 2127, 593, 21, 5, 0, 53, 159, 2386, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6000, 6672, 6315, 6785, 4636, 6156, 6945, 5643] avg=6144.0000 max=6945 max_rank=6 min=4636 min_rank=4 imbalance_ratio=1.130371 max_load_reduction=42207 max_load_reduction_pct=0.858704 moved_to_echo=23380 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=17 rerouted_tokens_per_expert_current_ep_rank=[3, 221, 86, 1168, 23, 365, 0, 0, 2840, 348, 0, 0, 0, 0, 0, 0, 47, 0, 105, 0, 5871, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 4, 75, 1728, 1781, 676, 256, 0, 0, 0, 0, 0, 0, 0, 1, 908, 10, 3639, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1625, 5390, 0, 4381, 0, 126, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 696, 31, 1, 1, 3, 2, 185, 5345, 0, 0, 0, 0, 0, 0, 0, 945, 0, 473, 6, 372, 5, 5, 481, 96, 58, 0, 0, 0, 0, 0, 0, 2, 14, 684, 3598, 0, 3, 1, 10, 4401, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5054, 6024, 4572, 4561, 11523, 6264, 2441, 8713] avg=6144.0000 max=11523 max_rank=4 min=2441 min_rank=6 imbalance_ratio=1.875488 max_load_reduction=37629 max_load_reduction_pct=0.765564 moved_to_echo=28941 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=17 rerouted_tokens_per_expert_current_ep_rank=[916, 488, 107, 185, 1697, 624, 0, 1, 2656, 159, 0, 0, 0, 0, 0, 0, 1, 50, 460, 40, 5484, 4, 136, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 0, 2479, 0, 639, 137, 101, 809, 636, 220, 0, 0, 0, 0, 0, 0, 0, 86, 2486, 9, 3057, 177, 691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1160, 3988, 58, 2174, 337, 208, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 834, 347, 15, 144, 44, 191, 96, 4997, 0, 0, 0, 0, 0, 0, 0, 2952, 0, 814, 56, 335, 189, 252, 743, 84, 47, 0, 0, 0, 0, 0, 0, 133, 951, 553, 134, 0, 0, 214, 156, 2000, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6833, 6283, 5120, 6506, 8129, 6668, 5472, 4141] avg=6144.0000 max=8129 max_rank=4 min=4141 min_rank=7 imbalance_ratio=1.323079 max_load_reduction=41023 max_load_reduction_pct=0.834615 moved_to_echo=24410 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=17 rerouted_tokens_per_expert_current_ep_rank=[1976, 189, 34, 173, 2170, 306, 0, 0, 1744, 25, 0, 0, 0, 0, 0, 0, 0, 260, 432, 81, 4237, 0, 836, 308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 760, 2, 5245, 0, 674, 270, 0, 134, 488, 146, 0, 0, 0, 0, 0, 0, 0, 188, 2134, 10, 2081, 306, 951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 1494, 246, 380, 966, 129, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 534, 132, 422, 145, 734, 48, 3282, 0, 0, 0, 0, 0, 0, 0, 3688, 1, 438, 552, 198, 724, 954, 760, 55, 16, 0, 0, 0, 0, 0, 0, 592, 4152, 149, 0, 9, 2, 107, 127, 327, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6617, 6154, 7719, 5670, 4306, 5835, 7386, 5465] avg=6144.0000 max=7719 max_rank=2 min=4306 min_rank=4 imbalance_ratio=1.256348 max_load_reduction=41433 max_load_reduction_pct=0.842957 moved_to_echo=22992 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=17 rerouted_tokens_per_expert_current_ep_rank=[1344, 215, 56, 515, 1902, 339, 0, 0, 1804, 107, 0, 0, 0, 0, 0, 0, 0, 145, 467, 100, 4083, 11, 668, 331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 564, 97, 4348, 0, 700, 344, 43, 550, 470, 177, 0, 0, 0, 0, 0, 0, 0, 505, 2799, 3, 2514, 386, 612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 1245, 518, 736, 402, 136, 718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 422, 547, 238, 539, 179, 948, 56, 3397, 0, 0, 0, 0, 0, 0, 0, 3001, 0, 396, 405, 663, 578, 1291, 590, 66, 13, 0, 0, 0, 0, 0, 0, 626, 2848, 205, 0, 7, 0, 197, 102, 1362, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6282, 5805, 7293, 6819, 4277, 6326, 7003, 5347] avg=6144.0000 max=7293 max_rank=2 min=4277 min_rank=4 imbalance_ratio=1.187012 max_load_reduction=41859 max_load_reduction_pct=0.851624 moved_to_echo=22953 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[10368, 1792, 640, 4096, 10752, 3200, 128, 128, 17280, 1408, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 5888, 20864, 1536, 13312, 2816, 1280, 3840, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[23680, 128, 4224, 2688, 3456, 4224, 5376, 4992, 640, 256, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 3200, 128, 25088, 0, 3840, 1792, 3072, 6656, 4480, 1536, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 2176, 18432, 128, 21248, 2816, 4736, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 4608, 2688, 640, 3456, 896, 4096, 640, 32640, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[2944, 19584, 3456, 5632, 128, 128, 640, 896, 16512, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 1024, 2944, 768, 38400, 128, 4608, 1664, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=18 tokens_per_expert_current_ep_rank=[0, 3, 365, 166, 17, 10, 0, 3, 606, 70, 2439, 0, 2501, 23, 607, 3, 1274, 1073, 963, 68, 27, 989, 4, 2699, 0, 467, 141, 1333, 6653, 6, 0, 1193, 3895, 0, 0, 0, 2687, 216, 1511, 0, 0, 27, 7254, 2288, 0, 756, 0, 1, 12, 494, 2643, 0, 123, 9, 0, 37, 15, 38, 1918, 0, 1295, 157, 0, 73] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=18 tokens_per_expert_current_ep_rank=[380, 8, 28, 42, 20, 108, 0, 1, 175, 3, 482, 0, 27, 191, 5262, 0, 661, 922, 3758, 366, 88, 2, 2, 7124, 0, 103, 37, 10, 7529, 2477, 0, 631, 150, 17, 1, 9, 2632, 5, 4534, 0, 5, 5, 277, 141, 0, 7267, 240, 0, 1, 35, 1109, 0, 466, 1, 281, 194, 886, 0, 430, 1, 5, 15, 0, 8] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=18 tokens_per_expert_current_ep_rank=[10, 20, 530, 234, 13, 24, 0, 0, 1165, 23, 2396, 1, 1272, 30, 931, 1, 1151, 1308, 1586, 60, 33, 434, 6, 3954, 0, 546, 314, 841, 7295, 6, 0, 964, 2901, 0, 0, 0, 2562, 199, 1618, 0, 1, 1, 6721, 2138, 0, 807, 7, 0, 10, 515, 3240, 0, 238, 5, 6, 64, 33, 41, 2277, 0, 470, 92, 2, 56] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=18 tokens_per_expert_current_ep_rank=[385, 3, 0, 0, 45, 239, 0, 0, 96, 0, 311, 0, 4, 410, 5580, 0, 590, 326, 3085, 14, 180, 3, 2, 7397, 0, 0, 2, 0, 7184, 5046, 0, 43, 7, 0, 0, 0, 2492, 0, 3329, 0, 0, 0, 17, 3, 0, 7939, 533, 0, 1, 17, 418, 0, 250, 0, 1230, 271, 1656, 0, 41, 0, 1, 0, 0, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=18 tokens_per_expert_current_ep_rank=[111, 10, 102, 40, 14, 29, 0, 0, 574, 6, 1651, 0, 1017, 102, 1570, 0, 1305, 1724, 2184, 64, 59, 405, 9, 5080, 0, 259, 14, 852, 6911, 329, 0, 1517, 2109, 0, 0, 0, 3643, 176, 3187, 0, 1, 1, 3907, 774, 0, 3131, 85, 0, 0, 439, 2941, 0, 378, 3, 21, 77, 139, 18, 1447, 0, 585, 123, 0, 29] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=18 tokens_per_expert_current_ep_rank=[176, 37, 31, 56, 20, 162, 0, 2, 397, 27, 1227, 0, 121, 107, 2751, 0, 864, 1325, 2763, 87, 210, 21, 20, 6286, 0, 143, 26, 153, 7780, 831, 4, 1302, 641, 1, 0, 0, 2890, 37, 4101, 1, 1, 1, 2094, 343, 0, 5362, 125, 1, 0, 714, 2643, 0, 478, 56, 245, 128, 124, 4, 2064, 0, 126, 0, 2, 41] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=18 tokens_per_expert_current_ep_rank=[0, 15, 330, 239, 5, 46, 6, 0, 487, 51, 1956, 0, 1164, 20, 960, 7, 1344, 1669, 1308, 222, 78, 496, 5, 4463, 0, 534, 257, 383, 7356, 15, 0, 1504, 2394, 2, 0, 1, 2224, 209, 2577, 10, 9, 20, 7004, 2113, 0, 1036, 4, 5, 29, 569, 2799, 0, 259, 89, 11, 21, 20, 54, 2380, 0, 270, 26, 9, 88] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=18 tokens_per_expert_current_ep_rank=[81, 7, 145, 54, 18, 16, 9, 12, 690, 44, 1572, 0, 1420, 80, 1207, 3, 1362, 1565, 929, 120, 57, 799, 6, 4128, 1, 445, 135, 758, 6371, 119, 0, 1385, 2307, 6, 1, 0, 2886, 264, 2538, 2, 2, 17, 6180, 1657, 0, 1448, 44, 8, 2, 447, 3398, 0, 294, 12, 7, 65, 42, 171, 2773, 0, 888, 52, 0, 103] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 365, 166, 17, 10, 0, 3, 3852, 765, 0, 0, 0, 0, 0, 0, 606, 70, 2439, 0, 2501, 23, 607, 3, 344, 0, 0, 0, 0, 0, 0, 0, 1274, 1073, 963, 68, 27, 989, 4, 604, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 141, 1333, 2801, 6, 0, 1193, 0, 0, 0, 0, 0, 0, 0, 0, 3895, 0, 0, 0, 2687, 216, 748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 2344, 2288, 0, 756, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 12, 494, 2643, 0, 123, 9, 0, 37, 3224, 419, 0, 0, 0, 0, 0, 0, 15, 38, 1918, 0, 1295, 157, 0, 73, 1330, 1686, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5181, 6593, 5002, 5941, 7546, 5416, 6961, 6512] avg=6144.0000 max=7546 max_rank=4 min=5002 min_rank=2 imbalance_ratio=1.228190 max_load_reduction=41606 max_load_reduction_pct=0.846476 moved_to_echo=26435 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=18 rerouted_tokens_per_expert_current_ep_rank=[380, 8, 28, 42, 20, 108, 0, 1, 4359, 2021, 0, 0, 0, 0, 0, 0, 175, 3, 482, 0, 27, 191, 5262, 0, 1033, 0, 0, 0, 0, 0, 0, 0, 661, 922, 3758, 366, 88, 2, 2, 1591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 37, 10, 3170, 2477, 0, 631, 0, 0, 0, 0, 0, 0, 0, 0, 150, 17, 1, 9, 2632, 5, 2244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 90, 141, 0, 7267, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 35, 1109, 0, 466, 1, 281, 194, 123, 1257, 0, 0, 0, 0, 0, 0, 886, 0, 430, 1, 5, 15, 0, 8, 3512, 64, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6967, 7173, 7390, 6428, 5058, 7748, 3467, 4921] avg=6144.0000 max=7748 max_rank=5 min=3467 min_rank=6 imbalance_ratio=1.261068 max_load_reduction=41404 max_load_reduction_pct=0.842367 moved_to_echo=21194 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=18 rerouted_tokens_per_expert_current_ep_rank=[10, 20, 530, 234, 13, 24, 0, 0, 4224, 1122, 0, 0, 0, 0, 0, 0, 1165, 23, 2396, 1, 1272, 30, 931, 1, 368, 0, 0, 0, 0, 0, 0, 0, 1151, 1308, 1586, 60, 33, 434, 6, 883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 314, 841, 3071, 6, 0, 964, 0, 0, 0, 0, 0, 0, 0, 0, 2901, 0, 0, 0, 2562, 199, 802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2171, 2138, 0, 807, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 515, 3240, 0, 238, 5, 6, 64, 2987, 448, 0, 0, 0, 0, 0, 0, 33, 41, 2277, 0, 470, 92, 2, 56, 1949, 1563, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6177, 6187, 5461, 5742, 6464, 5125, 7513, 6483] avg=6144.0000 max=7513 max_rank=6 min=5125 min_rank=5 imbalance_ratio=1.222819 max_load_reduction=41639 max_load_reduction_pct=0.847148 moved_to_echo=25585 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=18 rerouted_tokens_per_expert_current_ep_rank=[111, 10, 102, 40, 14, 29, 0, 0, 4001, 1441, 0, 0, 0, 0, 0, 0, 574, 6, 1651, 0, 1017, 102, 1570, 0, 726, 0, 0, 0, 0, 0, 0, 0, 1305, 1724, 2184, 64, 59, 405, 9, 1135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 259, 14, 852, 2910, 329, 0, 1517, 0, 0, 0, 0, 0, 0, 0, 0, 2109, 0, 0, 0, 3643, 176, 1578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1263, 774, 0, 3131, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 439, 2941, 0, 378, 3, 21, 77, 1736, 883, 0, 0, 0, 0, 0, 0, 139, 18, 1447, 0, 585, 123, 0, 29, 2504, 908, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5748, 5646, 6885, 5881, 7506, 5255, 6478, 5753] avg=6144.0000 max=7506 max_rank=4 min=5255 min_rank=5 imbalance_ratio=1.221680 max_load_reduction=41646 max_load_reduction_pct=0.847290 moved_to_echo=24992 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=18 rerouted_tokens_per_expert_current_ep_rank=[385, 3, 0, 0, 45, 239, 0, 0, 4160, 2099, 0, 0, 0, 0, 0, 0, 96, 0, 311, 0, 4, 410, 5580, 0, 758, 0, 0, 0, 0, 0, 0, 0, 590, 326, 3085, 14, 180, 3, 2, 1651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3024, 5046, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 2492, 0, 1648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3, 0, 7939, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 17, 418, 0, 250, 0, 1230, 271, 7, 923, 0, 0, 0, 0, 0, 0, 1656, 0, 41, 0, 1, 0, 0, 2, 3647, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6931, 7159, 5851, 8115, 4147, 8482, 3117, 5350] avg=6144.0000 max=8482 max_rank=5 min=3117 min_rank=6 imbalance_ratio=1.380534 max_load_reduction=40670 max_load_reduction_pct=0.827433 moved_to_echo=21096 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=18 rerouted_tokens_per_expert_current_ep_rank=[176, 37, 31, 56, 20, 162, 0, 2, 4509, 1787, 0, 0, 0, 0, 0, 0, 397, 27, 1227, 0, 121, 107, 2751, 0, 938, 0, 0, 0, 0, 0, 0, 0, 864, 1325, 2763, 87, 210, 21, 20, 1394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 26, 153, 3271, 831, 4, 1302, 0, 0, 0, 0, 0, 0, 0, 0, 641, 1, 0, 0, 2890, 37, 2022, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 670, 343, 0, 5362, 125, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 714, 2643, 0, 478, 56, 245, 128, 933, 1141, 0, 0, 0, 0, 0, 0, 124, 4, 2064, 0, 126, 0, 2, 41, 3105, 491, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6780, 5568, 6684, 5730, 5592, 6503, 6338, 5957] avg=6144.0000 max=6780 max_rank=0 min=5568 min_rank=1 imbalance_ratio=1.103516 max_load_reduction=42372 max_load_reduction_pct=0.862061 moved_to_echo=24390 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 15, 330, 239, 5, 46, 6, 0, 4259, 1266, 0, 0, 0, 0, 0, 0, 487, 51, 1956, 0, 1164, 20, 960, 7, 587, 0, 0, 0, 0, 0, 0, 0, 1344, 1669, 1308, 222, 78, 496, 5, 997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 257, 383, 3097, 15, 0, 1504, 0, 0, 0, 0, 0, 0, 0, 0, 2394, 2, 0, 1, 2224, 209, 1276, 10, 0, 0, 0, 0, 0, 0, 0, 0, 9, 20, 2263, 2113, 0, 1036, 4, 5, 0, 0, 0, 0, 0, 0, 0, 0, 29, 569, 2799, 0, 259, 89, 11, 21, 3113, 714, 0, 0, 0, 0, 0, 0, 20, 54, 2380, 0, 270, 26, 9, 88, 2200, 1628, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6166, 5232, 6119, 5790, 6116, 5450, 7604, 6675] avg=6144.0000 max=7604 max_rank=6 min=5232 min_rank=1 imbalance_ratio=1.237630 max_load_reduction=41548 max_load_reduction_pct=0.845296 moved_to_echo=25845 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=18 rerouted_tokens_per_expert_current_ep_rank=[81, 7, 145, 54, 18, 16, 9, 12, 3689, 1171, 0, 0, 0, 0, 0, 0, 690, 44, 1572, 0, 1420, 80, 1207, 3, 578, 0, 0, 0, 0, 0, 0, 0, 1362, 1565, 929, 120, 57, 799, 6, 922, 0, 0, 0, 0, 0, 0, 0, 0, 1, 445, 135, 758, 2682, 119, 0, 1385, 0, 0, 0, 0, 0, 0, 0, 0, 2307, 6, 1, 0, 2886, 264, 1257, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 17, 1997, 1657, 0, 1448, 44, 8, 0, 0, 0, 0, 0, 0, 0, 0, 2, 447, 3398, 0, 294, 12, 7, 65, 2746, 703, 0, 0, 0, 0, 0, 0, 42, 171, 2773, 0, 888, 52, 0, 103, 2035, 1437, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5202, 5594, 5760, 5525, 6723, 5173, 7674, 7501] avg=6144.0000 max=7674 max_rank=6 min=5173 min_rank=5 imbalance_ratio=1.249023 max_load_reduction=41478 max_load_reduction_pct=0.843872 moved_to_echo=27071 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1152, 128, 1536, 896, 256, 640, 128, 128, 33152, 11776, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[8576, 9984, 16640, 1024, 768, 3200, 128, 9216, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 3328, 19200, 0, 2560, 256, 1920, 896, 14976, 6528, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[2944, 384, 13440, 128, 3712, 512, 128, 512, 20352, 7808, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4224, 256, 12160, 128, 7552, 1024, 18944, 128, 5376, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 2560, 1024, 4352, 24064, 8832, 128, 8576, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[14464, 128, 128, 128, 22016, 1152, 11648, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 10880, 9472, 0, 27776, 1152, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=19 tokens_per_expert_current_ep_rank=[37, 2, 950, 40, 7144, 1, 22, 35, 1608, 1, 1, 5, 13, 29, 3, 2, 52, 286, 7, 71, 0, 0, 933, 29, 14, 41, 5, 0, 35, 177, 34, 6081, 127, 4126, 13, 0, 85, 1, 0, 349, 109, 453, 42, 7435, 83, 34, 507, 5, 4, 5, 453, 2, 67, 4308, 30, 20, 4601, 0, 38, 111, 25, 8024, 29, 408] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=19 tokens_per_expert_current_ep_rank=[1526, 76, 62, 140, 353, 45, 4, 1202, 869, 50, 1, 0, 258, 201, 5, 44, 1409, 551, 715, 777, 1428, 1, 20, 50, 1255, 309, 134, 0, 1941, 56, 15, 6134, 2008, 1058, 84, 0, 2931, 9, 473, 4794, 320, 583, 19, 2717, 712, 337, 2219, 1184, 0, 1096, 0, 101, 1280, 59, 92, 13, 787, 2, 732, 286, 72, 4585, 197, 771] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=19 tokens_per_expert_current_ep_rank=[806, 11, 303, 152, 524, 50, 4, 1340, 1371, 90, 0, 0, 241, 323, 2, 1, 571, 717, 988, 665, 988, 1, 68, 509, 694, 363, 63, 0, 1050, 3, 32, 6545, 1528, 1433, 92, 0, 2588, 0, 147, 4300, 610, 676, 10, 3159, 932, 253, 1967, 810, 0, 785, 7, 66, 2286, 180, 186, 26, 963, 1, 885, 240, 82, 5224, 224, 1017] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=19 tokens_per_expert_current_ep_rank=[522, 17, 349, 148, 1853, 22, 9, 635, 2134, 32, 0, 0, 136, 109, 4, 9, 663, 464, 243, 454, 705, 2, 256, 109, 626, 99, 48, 0, 936, 117, 43, 6454, 1563, 2400, 54, 0, 1419, 0, 285, 3863, 567, 660, 22, 4914, 722, 34, 2058, 443, 0, 289, 141, 51, 860, 801, 34, 19, 2485, 0, 411, 324, 78, 6494, 182, 781] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=19 tokens_per_expert_current_ep_rank=[5, 0, 1408, 1, 8127, 0, 2, 0, 833, 0, 0, 0, 1, 0, 0, 0, 1, 233, 0, 1, 0, 0, 1183, 2, 5, 3, 1, 0, 1, 205, 33, 4326, 2, 3705, 1, 0, 7, 0, 0, 18, 43, 258, 94, 7836, 1, 1, 45, 0, 0, 0, 934, 0, 3, 6042, 0, 26, 5261, 0, 5, 0, 2, 8184, 0, 313] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=19 tokens_per_expert_current_ep_rank=[174, 10, 1378, 206, 4487, 8, 0, 304, 2260, 11, 0, 1, 88, 79, 1, 0, 133, 772, 125, 77, 111, 0, 541, 267, 57, 37, 13, 0, 205, 238, 37, 6244, 450, 2234, 129, 0, 473, 2, 23, 738, 512, 719, 22, 6536, 572, 59, 1835, 50, 0, 110, 386, 16, 949, 2793, 23, 75, 3306, 0, 210, 276, 85, 7611, 209, 885] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=19 tokens_per_expert_current_ep_rank=[748, 66, 288, 205, 873, 69, 0, 1446, 1965, 58, 96, 0, 288, 165, 7, 38, 646, 584, 622, 1173, 1248, 2, 187, 184, 771, 251, 100, 0, 1183, 104, 41, 5866, 1317, 1224, 214, 0, 1881, 27, 410, 4270, 652, 742, 79, 3044, 980, 142, 2031, 848, 0, 554, 10, 135, 1083, 299, 347, 38, 1764, 17, 398, 231, 76, 5682, 191, 1192] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=19 tokens_per_expert_current_ep_rank=[550, 99, 615, 412, 752, 25, 0, 767, 1750, 88, 19, 0, 314, 134, 5, 3, 456, 1042, 412, 357, 207, 4, 145, 662, 1008, 793, 59, 0, 924, 25, 96, 6491, 799, 1371, 128, 0, 1436, 0, 71, 3246, 606, 964, 49, 3920, 1335, 192, 1886, 682, 0, 1071, 33, 182, 2602, 202, 348, 51, 1454, 14, 527, 288, 121, 6190, 282, 888] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=19 rerouted_tokens_per_expert_current_ep_rank=[37, 2, 950, 40, 7144, 1, 22, 35, 125, 0, 0, 0, 0, 0, 0, 0, 1608, 1, 1, 5, 13, 29, 3, 2, 5155, 0, 0, 0, 0, 0, 0, 0, 52, 286, 7, 71, 0, 0, 933, 29, 3924, 488, 0, 0, 0, 0, 0, 0, 14, 41, 5, 0, 35, 177, 34, 4405, 0, 0, 0, 0, 0, 0, 0, 0, 127, 4126, 13, 0, 85, 1, 0, 176, 0, 0, 0, 0, 0, 0, 0, 0, 109, 453, 42, 3267, 83, 34, 507, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5, 453, 2, 67, 4308, 30, 20, 1676, 48, 244, 0, 0, 0, 0, 0, 4601, 0, 38, 111, 25, 2381, 29, 408, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8356, 6817, 5790, 4711, 4528, 4500, 6857, 7593] avg=6144.0000 max=8356 max_rank=0 min=4500 min_rank=5 imbalance_ratio=1.360026 max_load_reduction=40796 max_load_reduction_pct=0.829997 moved_to_echo=23478 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=19 rerouted_tokens_per_expert_current_ep_rank=[1526, 76, 62, 140, 353, 45, 4, 1202, 1720, 0, 0, 0, 0, 0, 0, 0, 869, 50, 1, 0, 258, 201, 5, 44, 2945, 0, 0, 0, 0, 0, 0, 0, 1409, 551, 715, 777, 1428, 1, 20, 50, 1434, 278, 0, 0, 0, 0, 0, 0, 1255, 309, 134, 0, 1941, 56, 15, 4444, 0, 0, 0, 0, 0, 0, 0, 0, 2008, 1058, 84, 0, 2931, 9, 473, 2406, 0, 0, 0, 0, 0, 0, 0, 0, 320, 583, 19, 1194, 712, 337, 2219, 1184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1096, 0, 101, 1280, 59, 92, 13, 1690, 668, 89, 0, 0, 0, 0, 0, 787, 2, 732, 286, 72, 1362, 197, 771, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5128, 4373, 6663, 8154, 8969, 6568, 5088, 4209] avg=6144.0000 max=8969 max_rank=4 min=4209 min_rank=7 imbalance_ratio=1.459798 max_load_reduction=40183 max_load_reduction_pct=0.817525 moved_to_echo=24834 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=19 rerouted_tokens_per_expert_current_ep_rank=[806, 11, 303, 152, 524, 50, 4, 1340, 1543, 0, 0, 0, 0, 0, 0, 0, 1371, 90, 0, 0, 241, 323, 2, 1, 3356, 0, 0, 0, 0, 0, 0, 0, 571, 717, 988, 665, 988, 1, 68, 509, 1667, 317, 0, 0, 0, 0, 0, 0, 694, 363, 63, 0, 1050, 3, 32, 4742, 0, 0, 0, 0, 0, 0, 0, 0, 1528, 1433, 92, 0, 2588, 0, 147, 2157, 0, 0, 0, 0, 0, 0, 0, 0, 610, 676, 10, 1389, 932, 253, 1967, 810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 785, 7, 66, 2286, 180, 186, 26, 1803, 600, 103, 0, 0, 0, 0, 0, 963, 1, 885, 240, 82, 1551, 224, 1017, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4733, 5384, 6491, 6947, 7945, 6647, 6042, 4963] avg=6144.0000 max=7945 max_rank=4 min=4733 min_rank=0 imbalance_ratio=1.293132 max_load_reduction=41207 max_load_reduction_pct=0.838359 moved_to_echo=25597 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=19 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 1408, 1, 8127, 0, 2, 0, 6, 0, 0, 0, 0, 0, 0, 0, 833, 0, 0, 0, 1, 0, 0, 0, 5258, 0, 0, 0, 0, 0, 0, 0, 1, 233, 0, 1, 0, 0, 1183, 2, 4135, 497, 0, 0, 0, 0, 0, 0, 5, 3, 1, 0, 1, 205, 33, 3134, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3705, 1, 0, 7, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 43, 258, 94, 3444, 1, 1, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 934, 0, 3, 6042, 0, 26, 1192, 2, 257, 0, 0, 0, 0, 0, 5261, 0, 5, 0, 2, 2429, 0, 313, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9549, 6092, 6052, 3382, 3725, 3886, 8456, 8010] avg=6144.0000 max=9549 max_rank=0 min=3382 min_rank=3 imbalance_ratio=1.554199 max_load_reduction=39603 max_load_reduction_pct=0.805725 moved_to_echo=24077 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=19 rerouted_tokens_per_expert_current_ep_rank=[522, 17, 349, 148, 1853, 22, 9, 635, 1386, 0, 0, 0, 0, 0, 0, 0, 2134, 32, 0, 0, 136, 109, 4, 9, 4172, 0, 0, 0, 0, 0, 0, 0, 663, 464, 243, 454, 705, 2, 256, 109, 2593, 395, 0, 0, 0, 0, 0, 0, 626, 99, 48, 0, 936, 117, 43, 4676, 0, 0, 0, 0, 0, 0, 0, 0, 1563, 2400, 54, 0, 1419, 0, 285, 1938, 0, 0, 0, 0, 0, 0, 0, 0, 567, 660, 22, 2160, 722, 34, 2058, 443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 289, 141, 51, 860, 801, 34, 19, 1778, 539, 161, 0, 0, 0, 0, 0, 2485, 0, 411, 324, 78, 1927, 182, 781, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4941, 6596, 5884, 6545, 7659, 6666, 4673, 6188] avg=6144.0000 max=7659 max_rank=4 min=4673 min_rank=6 imbalance_ratio=1.246582 max_load_reduction=41493 max_load_reduction_pct=0.844177 moved_to_echo=25186 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=19 rerouted_tokens_per_expert_current_ep_rank=[174, 10, 1378, 206, 4487, 8, 0, 304, 267, 0, 0, 0, 0, 0, 0, 0, 2260, 11, 0, 1, 88, 79, 1, 0, 4893, 0, 0, 0, 0, 0, 0, 0, 133, 772, 125, 77, 111, 0, 541, 267, 3453, 467, 0, 0, 0, 0, 0, 0, 57, 37, 13, 0, 205, 238, 37, 4519, 0, 0, 0, 0, 0, 0, 0, 0, 450, 2234, 129, 0, 473, 2, 23, 364, 0, 0, 0, 0, 0, 0, 0, 0, 512, 719, 22, 2865, 572, 59, 1835, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 386, 16, 949, 2793, 23, 75, 1725, 107, 218, 0, 0, 0, 0, 0, 3306, 0, 210, 276, 85, 2251, 209, 885, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6834, 7333, 5946, 5106, 3675, 6634, 6402, 7222] avg=6144.0000 max=7333 max_rank=1 min=3675 min_rank=4 imbalance_ratio=1.193522 max_load_reduction=41819 max_load_reduction_pct=0.850810 moved_to_echo=23933 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=19 rerouted_tokens_per_expert_current_ep_rank=[550, 99, 615, 412, 752, 25, 0, 767, 1164, 0, 0, 0, 0, 0, 0, 0, 1750, 88, 19, 0, 314, 134, 5, 3, 3977, 0, 0, 0, 0, 0, 0, 0, 456, 1042, 412, 357, 207, 4, 145, 662, 2068, 376, 0, 0, 0, 0, 0, 0, 1008, 793, 59, 0, 924, 25, 96, 4702, 0, 0, 0, 0, 0, 0, 0, 0, 799, 1371, 128, 0, 1436, 0, 71, 1630, 0, 0, 0, 0, 0, 0, 0, 0, 606, 964, 49, 1724, 1335, 192, 1886, 682, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1071, 33, 182, 2602, 202, 348, 51, 1789, 452, 128, 0, 0, 0, 0, 0, 1454, 14, 527, 288, 121, 1837, 282, 888, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4384, 6290, 5729, 7607, 5435, 7438, 6858, 5411] avg=6144.0000 max=7607 max_rank=3 min=4384 min_rank=0 imbalance_ratio=1.238118 max_load_reduction=41545 max_load_reduction_pct=0.845235 moved_to_echo=25142 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=19 rerouted_tokens_per_expert_current_ep_rank=[748, 66, 288, 205, 873, 69, 0, 1446, 1532, 0, 0, 0, 0, 0, 0, 0, 1965, 58, 96, 0, 288, 165, 7, 38, 3650, 0, 0, 0, 0, 0, 0, 0, 646, 584, 622, 1173, 1248, 2, 187, 184, 1606, 345, 0, 0, 0, 0, 0, 0, 771, 251, 100, 0, 1183, 104, 41, 4250, 0, 0, 0, 0, 0, 0, 0, 0, 1317, 1224, 214, 0, 1881, 27, 410, 2143, 0, 0, 0, 0, 0, 0, 0, 0, 652, 742, 79, 1339, 980, 142, 2031, 848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 10, 135, 1083, 299, 347, 38, 1616, 595, 99, 0, 0, 0, 0, 0, 1764, 17, 398, 231, 76, 1687, 191, 1192, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5227, 6267, 6597, 6700, 7216, 6813, 4776, 5556] avg=6144.0000 max=7216 max_rank=4 min=4776 min_rank=6 imbalance_ratio=1.174479 max_load_reduction=41936 max_load_reduction_pct=0.853190 moved_to_echo=24361 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[20736, 128, 3328, 1792, 640, 15488, 1408, 6272, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7808, 17664, 768, 0, 10880, 128, 1536, 10880, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[12800, 384, 128, 128, 1408, 1152, 128, 128, 33408, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3968, 4736, 3200, 3584, 4736, 128, 3456, 1920, 20992, 3200, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4480, 384, 5376, 1408, 24192, 256, 128, 5760, 7808, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3456, 5120, 384, 17408, 5376, 1152, 12672, 4096, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 3968, 2048, 640, 9216, 14720, 1152, 384, 13312, 3072, 1408, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4480, 1920, 512, 0, 6400, 1024, 384, 34944, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 7360, 4, 13, 0, 0, 1, 0, 72, 0, 0, 2084, 0, 0, 1, 1042, 0, 4, 0, 0, 0, 7500, 94, 0, 0, 0, 0, 0, 7, 98, 0, 1, 654, 8, 6876, 118, 0, 0, 3, 7762, 0, 13, 3373, 33, 3, 9, 52, 0, 5038, 3, 0, 0, 437, 0, 0, 0, 0, 1, 1, 6478, 5, 1, 3] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=20 tokens_per_expert_current_ep_rank=[73, 0, 67, 1314, 681, 0, 0, 0, 82, 3954, 105, 474, 1521, 3, 64, 18, 3531, 0, 3464, 0, 0, 41, 2194, 892, 1398, 0, 0, 226, 0, 994, 179, 0, 559, 75, 2955, 1439, 82, 0, 1, 1169, 5792, 588, 172, 358, 930, 1819, 4742, 2211, 2, 1236, 373, 1, 2, 334, 0, 4, 0, 9, 64, 644, 329, 1538, 203, 246] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=20 tokens_per_expert_current_ep_rank=[103, 0, 781, 516, 593, 0, 1, 0, 26, 3837, 56, 431, 2537, 7, 45, 0, 2717, 0, 1905, 0, 4, 2, 4531, 1064, 1189, 0, 0, 213, 0, 577, 354, 0, 358, 167, 1575, 3461, 146, 0, 4, 521, 6296, 223, 184, 1489, 816, 660, 2525, 3177, 3, 2096, 215, 1, 0, 455, 0, 0, 5, 9, 112, 500, 1037, 1393, 96, 139] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=20 tokens_per_expert_current_ep_rank=[41, 0, 1566, 156, 295, 0, 0, 0, 39, 3331, 19, 126, 3114, 4, 12, 6, 4190, 0, 777, 0, 0, 5, 4696, 724, 202, 1, 0, 41, 0, 92, 477, 0, 127, 238, 538, 5882, 334, 0, 0, 206, 7427, 115, 73, 2431, 685, 341, 764, 984, 3, 3469, 152, 2, 7, 781, 0, 4, 0, 8, 14, 396, 3850, 305, 77, 25] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=20 tokens_per_expert_current_ep_rank=[64, 0, 234, 521, 1034, 0, 8, 0, 49, 3030, 110, 551, 2061, 37, 272, 3, 3094, 0, 2871, 0, 41, 15, 2219, 1316, 1937, 0, 0, 568, 0, 229, 201, 0, 544, 106, 2349, 2154, 165, 0, 2, 568, 5492, 283, 172, 466, 1087, 1484, 4031, 3133, 10, 1234, 475, 14, 0, 307, 1, 4, 0, 9, 260, 599, 555, 2615, 261, 307] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=20 tokens_per_expert_current_ep_rank=[19, 0, 20, 1954, 1540, 0, 0, 0, 78, 3374, 89, 339, 1954, 32, 36, 61, 4153, 0, 2775, 0, 0, 1, 1233, 1050, 950, 0, 0, 131, 0, 614, 88, 0, 350, 12, 2193, 2517, 113, 0, 23, 1045, 6334, 277, 160, 290, 1116, 2355, 4764, 1237, 8, 1242, 558, 6, 2, 596, 0, 11, 14, 16, 45, 501, 557, 1632, 468, 219] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=20 tokens_per_expert_current_ep_rank=[96, 0, 33, 954, 895, 0, 0, 0, 63, 2759, 75, 1371, 1019, 2, 228, 1, 2176, 0, 4450, 0, 22, 29, 1486, 1103, 2634, 0, 0, 597, 0, 543, 103, 0, 574, 20, 3669, 815, 38, 0, 2, 1253, 4487, 483, 141, 125, 1001, 1390, 5115, 2572, 8, 840, 386, 8, 0, 158, 0, 1, 1, 1, 368, 773, 133, 3456, 245, 450] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=20 tokens_per_expert_current_ep_rank=[22, 0, 5860, 33, 63, 7, 0, 0, 0, 733, 3, 3, 3780, 0, 16, 7, 1634, 2, 143, 0, 0, 0, 7073, 341, 0, 1, 0, 5, 0, 42, 370, 0, 5, 408, 58, 4831, 225, 0, 0, 86, 7853, 11, 101, 4072, 132, 10, 133, 841, 0, 4400, 80, 1, 8, 338, 0, 13, 1, 8, 3, 70, 5278, 32, 14, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=20 rerouted_tokens_per_expert_current_ep_rank=[73, 0, 67, 1314, 681, 0, 0, 0, 1488, 268, 0, 0, 0, 0, 0, 0, 82, 3954, 105, 474, 1521, 3, 64, 18, 553, 127, 0, 0, 0, 0, 0, 0, 3531, 0, 3464, 0, 0, 41, 247, 892, 0, 0, 0, 0, 0, 0, 0, 0, 1398, 0, 0, 226, 0, 994, 179, 0, 3838, 0, 0, 0, 0, 0, 0, 0, 559, 75, 2955, 1312, 82, 0, 1, 1169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 588, 172, 358, 930, 1819, 1179, 2211, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1236, 373, 1, 2, 334, 0, 4, 1954, 459, 0, 0, 0, 0, 0, 0, 0, 9, 64, 644, 329, 1538, 203, 246, 2742, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3891, 6901, 8175, 6635, 6153, 7257, 4365, 5775] avg=6144.0000 max=8175 max_rank=2 min=3891 min_rank=0 imbalance_ratio=1.330566 max_load_reduction=40977 max_load_reduction_pct=0.833679 moved_to_echo=23550 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=20 rerouted_tokens_per_expert_current_ep_rank=[103, 0, 781, 516, 593, 0, 1, 0, 3073, 143, 0, 0, 0, 0, 0, 0, 26, 3837, 56, 431, 2537, 7, 45, 0, 294, 305, 0, 0, 0, 0, 0, 0, 2717, 0, 1905, 0, 4, 2, 510, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 1189, 0, 0, 213, 0, 577, 354, 0, 4173, 0, 0, 0, 0, 0, 0, 0, 358, 167, 1575, 3156, 146, 0, 4, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 223, 184, 1489, 816, 660, 628, 3177, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2096, 215, 1, 0, 455, 0, 0, 2123, 948, 0, 0, 0, 0, 0, 0, 5, 9, 112, 500, 1037, 1393, 96, 139, 1460, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5210, 7538, 6202, 6506, 5927, 7177, 5841, 4751] avg=6144.0000 max=7538 max_rank=1 min=4751 min_rank=7 imbalance_ratio=1.226888 max_load_reduction=41614 max_load_reduction_pct=0.846639 moved_to_echo=23696 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 7360, 4, 13, 0, 0, 1, 5086, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 2084, 0, 0, 1, 1, 607, 0, 0, 0, 0, 0, 0, 1042, 0, 4, 0, 0, 0, 845, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 98, 0, 5144, 0, 0, 0, 0, 0, 0, 0, 1, 654, 8, 6269, 118, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 3373, 33, 3, 3, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5038, 3, 0, 0, 437, 0, 0, 2618, 1569, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 6478, 5, 1, 3, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12464, 2765, 1985, 5249, 7053, 3477, 9665, 6494] avg=6144.0000 max=12464 max_rank=0 min=1985 min_rank=2 imbalance_ratio=2.028646 max_load_reduction=36688 max_load_reduction_pct=0.746419 moved_to_echo=26689 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=20 rerouted_tokens_per_expert_current_ep_rank=[41, 0, 1566, 156, 295, 0, 0, 0, 3188, 47, 0, 0, 0, 0, 0, 0, 39, 3331, 19, 126, 3114, 4, 12, 6, 92, 522, 0, 0, 0, 0, 0, 0, 4190, 0, 777, 0, 0, 5, 523, 724, 0, 0, 0, 0, 0, 0, 0, 0, 202, 1, 0, 41, 0, 92, 477, 0, 4922, 0, 0, 0, 0, 0, 0, 0, 127, 238, 538, 5360, 334, 0, 0, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 73, 2431, 685, 341, 181, 984, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3469, 152, 2, 7, 781, 0, 4, 2505, 985, 0, 0, 0, 0, 0, 0, 0, 8, 14, 396, 3850, 305, 77, 25, 444, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5293, 7265, 6219, 5735, 6803, 4810, 7908, 5119] avg=6144.0000 max=7908 max_rank=6 min=4810 min_rank=5 imbalance_ratio=1.287109 max_load_reduction=41244 max_load_reduction_pct=0.839111 moved_to_echo=24640 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=20 rerouted_tokens_per_expert_current_ep_rank=[22, 0, 5860, 33, 63, 7, 0, 0, 4797, 7, 0, 0, 0, 0, 0, 0, 0, 733, 3, 3, 3780, 0, 16, 7, 15, 426, 0, 0, 0, 0, 0, 0, 1634, 2, 143, 0, 0, 0, 796, 341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 0, 42, 370, 0, 5204, 0, 0, 0, 0, 0, 0, 0, 5, 408, 58, 4405, 225, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 101, 4072, 132, 10, 35, 841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4400, 80, 1, 8, 338, 0, 13, 2649, 1480, 0, 0, 0, 0, 0, 0, 1, 8, 3, 70, 5278, 32, 14, 2, 76, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10789, 4983, 2916, 5622, 5187, 5202, 8969, 5484] avg=6144.0000 max=10789 max_rank=0 min=2916 min_rank=2 imbalance_ratio=1.756022 max_load_reduction=38363 max_load_reduction_pct=0.780497 moved_to_echo=24842 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=20 rerouted_tokens_per_expert_current_ep_rank=[19, 0, 20, 1954, 1540, 0, 0, 0, 836, 269, 0, 0, 0, 0, 0, 0, 78, 3374, 89, 339, 1954, 32, 36, 61, 556, 222, 0, 0, 0, 0, 0, 0, 4153, 0, 2775, 0, 0, 1, 139, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 950, 0, 0, 131, 0, 614, 88, 0, 4198, 0, 0, 0, 0, 0, 0, 0, 350, 12, 2193, 2295, 113, 0, 23, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 277, 160, 290, 1116, 2355, 1184, 1237, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1242, 558, 6, 2, 596, 0, 11, 2136, 258, 0, 0, 0, 0, 0, 0, 14, 16, 45, 501, 557, 1632, 468, 219, 2755, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4638, 6741, 8118, 5981, 6031, 6619, 4817, 6207] avg=6144.0000 max=8118 max_rank=2 min=4638 min_rank=0 imbalance_ratio=1.321289 max_load_reduction=41034 max_load_reduction_pct=0.834839 moved_to_echo=23674 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=20 rerouted_tokens_per_expert_current_ep_rank=[96, 0, 33, 954, 895, 0, 0, 0, 1007, 289, 0, 0, 0, 0, 0, 0, 63, 2759, 75, 1371, 1019, 2, 228, 1, 597, 71, 0, 0, 0, 0, 0, 0, 2176, 0, 4450, 0, 22, 29, 169, 1103, 0, 0, 0, 0, 0, 0, 0, 0, 2634, 0, 0, 597, 0, 543, 103, 0, 2973, 0, 0, 0, 0, 0, 0, 0, 574, 20, 3669, 744, 38, 0, 2, 1253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 141, 125, 1001, 1390, 1271, 2572, 0, 0, 0, 0, 0, 0, 0, 0, 8, 840, 386, 8, 0, 158, 0, 1, 1514, 310, 0, 0, 0, 0, 0, 0, 1, 1, 368, 773, 133, 3456, 245, 450, 2958, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3274, 6186, 7949, 6850, 6300, 6983, 3225, 8385] avg=6144.0000 max=8385 max_rank=7 min=3225 min_rank=6 imbalance_ratio=1.364746 max_load_reduction=40767 max_load_reduction_pct=0.829407 moved_to_echo=24893 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=20 rerouted_tokens_per_expert_current_ep_rank=[64, 0, 234, 521, 1034, 0, 8, 0, 1504, 228, 0, 0, 0, 0, 0, 0, 49, 3030, 110, 551, 2061, 37, 272, 3, 470, 190, 0, 0, 0, 0, 0, 0, 3094, 0, 2871, 0, 41, 15, 251, 1316, 0, 0, 0, 0, 0, 0, 0, 0, 1937, 0, 0, 568, 0, 229, 201, 0, 3639, 0, 0, 0, 0, 0, 0, 0, 544, 106, 2349, 1964, 165, 0, 2, 568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 283, 172, 466, 1087, 1484, 1002, 3133, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1234, 475, 14, 0, 307, 1, 4, 1853, 464, 0, 0, 0, 0, 0, 0, 0, 9, 260, 599, 555, 2615, 261, 307, 2331, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3593, 6773, 7588, 6574, 5698, 7627, 4362, 6937] avg=6144.0000 max=7627 max_rank=5 min=3593 min_rank=0 imbalance_ratio=1.241374 max_load_reduction=41525 max_load_reduction_pct=0.844828 moved_to_echo=24624 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 19584, 2304, 128, 128, 3456, 128, 128, 17408, 6528, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[512, 0, 16000, 5504, 5120, 128, 128, 128, 20992, 1280, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 896, 3584, 18304, 11008, 1408, 1408, 12800, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 2048, 1024, 12672, 5888, 8064, 5504, 14208, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[384, 21120, 512, 3328, 18176, 128, 768, 128, 2688, 2560, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[22656, 128, 16512, 0, 128, 128, 3584, 6656, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2560, 1792, 13440, 25600, 1280, 0, 128, 4864, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[8320, 128, 0, 1792, 0, 3200, 1920, 0, 34176, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=21 tokens_per_expert_current_ep_rank=[966, 137, 119, 0, 242, 326, 4459, 0, 12, 1074, 1575, 1328, 0, 3962, 492, 2879, 3336, 22, 990, 2514, 121, 10, 1586, 376, 0, 58, 2678, 430, 1008, 166, 814, 0, 0, 125, 7, 2191, 472, 695, 6, 2807, 149, 226, 8, 155, 533, 0, 85, 7, 105, 6401, 0, 758, 0, 339, 56, 130, 168, 0, 0, 28, 0, 249, 1375, 397] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=21 tokens_per_expert_current_ep_rank=[1156, 48, 29, 0, 183, 180, 5331, 0, 22, 745, 1026, 1032, 3, 4586, 1598, 2021, 4678, 0, 524, 1924, 147, 0, 2452, 317, 0, 199, 3092, 141, 633, 83, 2379, 0, 0, 353, 23, 1047, 106, 871, 11, 1468, 265, 108, 2, 51, 833, 0, 1, 1, 74, 4760, 0, 277, 0, 1286, 20, 124, 175, 0, 0, 204, 5, 252, 587, 1719] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=21 tokens_per_expert_current_ep_rank=[1015, 168, 134, 0, 344, 264, 3281, 0, 58, 2184, 1381, 2200, 2, 4167, 230, 2890, 1846, 20, 660, 2451, 33, 4, 997, 617, 0, 232, 3793, 274, 1810, 268, 564, 2, 0, 103, 16, 1878, 153, 1530, 1, 3836, 25, 84, 26, 142, 295, 0, 20, 8, 1, 5889, 0, 799, 2, 106, 35, 148, 350, 0, 0, 10, 0, 170, 1418, 218] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=21 tokens_per_expert_current_ep_rank=[239, 14, 8, 0, 2, 11, 7097, 0, 9, 97, 62, 41, 3, 2220, 5553, 87, 7868, 31, 11, 249, 189, 0, 2506, 4, 3, 3, 334, 4, 5, 10, 6876, 0, 13, 391, 0, 90, 48, 20, 0, 113, 397, 4, 0, 26, 354, 0, 15, 21, 81, 663, 7, 76, 1, 6898, 4, 10, 4, 0, 2, 403, 0, 372, 21, 5582] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=21 tokens_per_expert_current_ep_rank=[12, 0, 1, 0, 0, 2, 7335, 1, 0, 4, 0, 9, 108, 1265, 5160, 17, 8178, 0, 0, 8, 250, 0, 1757, 2, 0, 0, 72, 0, 3, 1, 7401, 0, 40, 315, 1, 10, 2, 2, 0, 13, 139, 0, 0, 0, 396, 0, 0, 0, 146, 235, 0, 2, 1, 8095, 0, 2, 1, 0, 0, 802, 0, 465, 2, 6897] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=21 tokens_per_expert_current_ep_rank=[762, 20, 21, 0, 70, 107, 6890, 0, 0, 287, 307, 229, 5, 4602, 2277, 1114, 7278, 39, 37, 857, 215, 0, 2499, 55, 0, 7, 1012, 110, 191, 28, 3548, 0, 0, 403, 0, 479, 243, 104, 6, 663, 453, 9, 0, 52, 1574, 0, 13, 0, 914, 2997, 0, 276, 0, 4248, 4, 23, 35, 1, 0, 430, 1, 241, 130, 3286] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=21 tokens_per_expert_current_ep_rank=[954, 92, 92, 0, 112, 246, 4964, 0, 15, 779, 1707, 1777, 9, 4675, 578, 3593, 3982, 261, 730, 1167, 163, 11, 1103, 121, 0, 83, 2400, 634, 1208, 189, 1043, 0, 0, 217, 0, 1391, 811, 423, 10, 1562, 336, 90, 15, 140, 952, 0, 20, 126, 268, 5911, 0, 1112, 0, 522, 97, 374, 229, 0, 0, 110, 0, 127, 1018, 603] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=21 tokens_per_expert_current_ep_rank=[943, 90, 84, 0, 318, 219, 4241, 0, 47, 909, 1533, 2135, 4, 4815, 911, 2108, 3418, 12, 603, 2869, 133, 0, 1589, 491, 0, 155, 3540, 542, 1399, 115, 1015, 0, 0, 146, 66, 1465, 318, 1020, 38, 2771, 298, 125, 35, 79, 589, 0, 10, 1, 109, 4537, 0, 506, 0, 367, 44, 63, 277, 0, 0, 71, 11, 184, 808, 976] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=21 rerouted_tokens_per_expert_current_ep_rank=[966, 137, 119, 0, 242, 326, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1074, 1575, 1328, 0, 0, 344, 2879, 0, 0, 0, 0, 0, 0, 0, 0, 1269, 22, 990, 2514, 121, 10, 1586, 376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 2678, 430, 1008, 166, 766, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 125, 7, 2191, 472, 695, 6, 2807, 1550, 148, 427, 48, 0, 0, 0, 0, 149, 226, 8, 155, 533, 0, 85, 7, 3962, 786, 0, 0, 0, 0, 0, 0, 105, 4210, 0, 758, 0, 339, 56, 130, 0, 0, 0, 0, 0, 0, 0, 0, 168, 0, 0, 28, 0, 249, 1375, 397, 1281, 641, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5822, 7212, 6888, 5106, 8476, 5911, 5598, 4139] avg=6144.0000 max=8476 max_rank=4 min=4139 min_rank=7 imbalance_ratio=1.379557 max_load_reduction=40676 max_load_reduction_pct=0.827555 moved_to_echo=24124 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=21 rerouted_tokens_per_expert_current_ep_rank=[1015, 168, 134, 0, 344, 264, 2967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 2184, 1381, 2200, 2, 0, 161, 2890, 0, 0, 0, 0, 0, 0, 0, 0, 702, 20, 660, 2451, 33, 4, 997, 617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 232, 3793, 274, 1810, 268, 531, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 16, 1878, 153, 1530, 1, 3836, 1426, 69, 314, 33, 0, 0, 0, 0, 25, 84, 26, 142, 295, 0, 20, 8, 4167, 435, 0, 0, 0, 0, 0, 0, 1, 3873, 0, 799, 2, 106, 35, 148, 0, 0, 0, 0, 0, 0, 0, 0, 350, 0, 0, 10, 0, 170, 1418, 218, 709, 590, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4892, 8876, 5484, 6910, 9359, 5202, 4964, 3465] avg=6144.0000 max=9359 max_rank=4 min=3465 min_rank=7 imbalance_ratio=1.523275 max_load_reduction=39793 max_load_reduction_pct=0.809591 moved_to_echo=22990 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=21 rerouted_tokens_per_expert_current_ep_rank=[1156, 48, 29, 0, 183, 180, 4821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 745, 1026, 1032, 3, 0, 1116, 2021, 0, 0, 0, 0, 0, 0, 0, 0, 1779, 0, 524, 1924, 147, 0, 2452, 317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 199, 3092, 141, 633, 83, 2238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 353, 23, 1047, 106, 871, 11, 1468, 1153, 482, 510, 141, 0, 0, 0, 0, 265, 108, 2, 51, 833, 0, 1, 1, 4586, 1102, 0, 0, 0, 0, 0, 0, 74, 3130, 0, 277, 0, 1286, 20, 124, 0, 0, 0, 0, 0, 0, 0, 0, 175, 0, 0, 204, 5, 252, 587, 1719, 1797, 477, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6417, 5965, 7143, 6386, 6165, 6949, 4911, 5216] avg=6144.0000 max=7143 max_rank=2 min=4911 min_rank=6 imbalance_ratio=1.162598 max_load_reduction=42009 max_load_reduction_pct=0.854675 moved_to_echo=23241 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=21 rerouted_tokens_per_expert_current_ep_rank=[239, 14, 8, 0, 2, 11, 6418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 97, 62, 41, 3, 0, 3876, 87, 0, 0, 0, 0, 0, 0, 0, 0, 2992, 31, 11, 249, 189, 0, 2506, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 334, 4, 5, 10, 6468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 391, 0, 90, 48, 20, 0, 113, 160, 1677, 679, 408, 0, 0, 0, 0, 397, 4, 0, 26, 354, 0, 15, 21, 2220, 1854, 0, 0, 0, 0, 0, 0, 81, 437, 7, 76, 1, 6898, 4, 10, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2, 403, 0, 372, 21, 5582, 3022, 66, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6692, 4175, 5982, 6827, 3599, 4891, 7514, 9472] avg=6144.0000 max=9472 max_rank=7 min=3599 min_rank=4 imbalance_ratio=1.541667 max_load_reduction=39680 max_load_reduction_pct=0.807292 moved_to_echo=25476 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=21 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 1, 0, 0, 2, 6633, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 9, 108, 0, 3602, 17, 0, 0, 0, 0, 0, 0, 0, 0, 3110, 0, 0, 8, 250, 0, 1757, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 3, 1, 6962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 315, 1, 10, 2, 2, 0, 13, 56, 1558, 702, 439, 0, 0, 0, 0, 139, 0, 0, 0, 396, 0, 0, 0, 1265, 1927, 0, 0, 0, 0, 0, 0, 146, 156, 0, 2, 1, 8095, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 802, 0, 465, 2, 6897, 3141, 23, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6649, 3740, 5127, 7038, 3138, 3727, 8402, 11331] avg=6144.0000 max=11331 max_rank=7 min=3138 min_rank=4 imbalance_ratio=1.844238 max_load_reduction=37821 max_load_reduction_pct=0.769470 moved_to_echo=26598 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=21 rerouted_tokens_per_expert_current_ep_rank=[762, 20, 21, 0, 70, 107, 6226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 287, 307, 229, 5, 0, 1586, 1114, 0, 0, 0, 0, 0, 0, 0, 0, 2758, 39, 37, 857, 215, 0, 2499, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1012, 110, 191, 28, 3335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 403, 0, 479, 243, 104, 6, 663, 730, 691, 664, 213, 0, 0, 0, 0, 453, 9, 0, 52, 1574, 0, 13, 0, 4602, 1720, 0, 0, 0, 0, 0, 0, 914, 1964, 0, 276, 0, 4248, 4, 23, 0, 0, 0, 0, 0, 0, 0, 0, 35, 1, 0, 430, 1, 241, 130, 3286, 2800, 303, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7206, 3528, 6460, 4683, 4196, 8423, 7429, 7227] avg=6144.0000 max=8423 max_rank=5 min=3528 min_rank=1 imbalance_ratio=1.370931 max_load_reduction=40729 max_load_reduction_pct=0.828634 moved_to_echo=27275 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=21 rerouted_tokens_per_expert_current_ep_rank=[954, 92, 92, 0, 112, 246, 4489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 779, 1707, 1777, 9, 0, 404, 3593, 0, 0, 0, 0, 0, 0, 0, 0, 1515, 261, 730, 1167, 163, 11, 1103, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 2400, 634, 1208, 189, 982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 217, 0, 1391, 811, 423, 10, 1562, 1431, 174, 475, 61, 0, 0, 0, 0, 336, 90, 15, 140, 952, 0, 20, 126, 4675, 938, 0, 0, 0, 0, 0, 0, 268, 3888, 0, 1112, 0, 522, 97, 374, 0, 0, 0, 0, 0, 0, 0, 0, 229, 0, 0, 110, 0, 127, 1018, 603, 1529, 592, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5985, 8284, 5071, 5496, 6555, 7292, 6261, 4208] avg=6144.0000 max=8284 max_rank=1 min=4208 min_rank=7 imbalance_ratio=1.348307 max_load_reduction=40868 max_load_reduction_pct=0.831462 moved_to_echo=24316 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=21 rerouted_tokens_per_expert_current_ep_rank=[943, 90, 84, 0, 318, 219, 3835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 909, 1533, 2135, 4, 0, 636, 2108, 0, 0, 0, 0, 0, 0, 0, 0, 1300, 12, 603, 2869, 133, 0, 1589, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 3540, 542, 1399, 115, 955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 66, 1465, 318, 1020, 38, 2771, 1099, 275, 406, 60, 0, 0, 0, 0, 298, 125, 35, 79, 589, 0, 10, 1, 4815, 805, 0, 0, 0, 0, 0, 0, 109, 2984, 0, 506, 0, 367, 44, 63, 0, 0, 0, 0, 0, 0, 0, 0, 277, 0, 0, 71, 11, 184, 808, 976, 1313, 454, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5489, 7372, 6997, 6706, 7664, 6757, 4073, 4094] avg=6144.0000 max=7664 max_rank=4 min=4073 min_rank=6 imbalance_ratio=1.247396 max_load_reduction=41488 max_load_reduction_pct=0.844076 moved_to_echo=22588 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 2176, 128, 8576, 2176, 4736, 128, 13312, 7680, 5120, 4224, 1408, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[256, 6144, 7680, 8832, 256, 0, 11776, 14720, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1280, 128, 128, 2176, 128, 2176, 5376, 19712, 15616, 3200, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[15488, 512, 3584, 12160, 1280, 128, 14592, 2048, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[2176, 768, 128, 768, 5632, 0, 256, 256, 30336, 9600, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 768, 17024, 2176, 6272, 896, 22272, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1792, 20736, 128, 3840, 128, 21888, 384, 896, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[6144, 640, 512, 0, 1280, 1408, 39424, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=22 tokens_per_expert_current_ep_rank=[22, 512, 2219, 2, 0, 174, 321, 0, 0, 920, 3, 1105, 1653, 9, 1, 0, 0, 7098, 75, 0, 0, 0, 2, 0, 86, 27, 167, 16, 19, 250, 4029, 94, 0, 2, 2, 0, 1, 67, 4, 319, 913, 7674, 14, 105, 1669, 6895, 17, 75, 12, 0, 11, 0, 4067, 0, 27, 8069, 4, 95, 116, 0, 7, 95, 85, 3] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=22 tokens_per_expert_current_ep_rank=[16, 328, 1791, 0, 0, 1883, 20, 21, 72, 192, 0, 2815, 6168, 110, 0, 7, 4, 1503, 1896, 0, 0, 146, 0, 0, 261, 25, 835, 52, 10, 160, 1744, 48, 365, 0, 0, 0, 0, 2032, 0, 3657, 3940, 3964, 37, 1849, 0, 2339, 290, 248, 0, 92, 135, 17, 159, 501, 171, 6305, 0, 184, 410, 0, 65, 1128, 1090, 67] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=22 tokens_per_expert_current_ep_rank=[27, 197, 915, 0, 0, 2310, 0, 28, 146, 107, 0, 3383, 6539, 124, 2, 13, 82, 257, 2297, 0, 0, 119, 0, 0, 590, 1, 2621, 29, 70, 78, 542, 16, 271, 0, 0, 0, 0, 3200, 0, 5585, 3271, 1401, 27, 2591, 0, 549, 435, 108, 0, 175, 353, 14, 3, 286, 355, 5291, 6, 200, 549, 0, 55, 2443, 1425, 66] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=22 tokens_per_expert_current_ep_rank=[22, 37, 618, 0, 0, 3608, 0, 70, 481, 71, 0, 1833, 5801, 84, 5, 79, 24, 47, 2924, 0, 0, 192, 0, 0, 242, 1, 3175, 144, 19, 28, 629, 2, 387, 0, 0, 0, 0, 3828, 0, 6289, 3175, 426, 17, 4286, 0, 187, 1218, 141, 0, 222, 175, 33, 2, 705, 250, 4024, 1, 15, 513, 0, 48, 1731, 1173, 170] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=22 tokens_per_expert_current_ep_rank=[21, 2040, 2159, 0, 0, 856, 150, 10, 21, 404, 0, 2889, 4933, 102, 16, 4, 6, 5394, 1249, 0, 0, 22, 1, 0, 204, 60, 632, 17, 14, 279, 1032, 455, 86, 0, 0, 0, 0, 602, 0, 992, 2333, 6710, 19, 421, 91, 4043, 57, 273, 1, 27, 142, 0, 639, 15, 26, 7694, 37, 768, 464, 0, 24, 401, 300, 17] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=22 tokens_per_expert_current_ep_rank=[45, 275, 1025, 3, 0, 2518, 0, 26, 245, 147, 0, 2524, 6383, 161, 38, 30, 31, 355, 3339, 0, 0, 286, 0, 0, 671, 24, 1364, 73, 68, 142, 766, 30, 672, 0, 0, 0, 0, 3146, 0, 5269, 2496, 1913, 62, 2901, 0, 813, 722, 109, 0, 56, 217, 54, 36, 921, 170, 5394, 3, 249, 505, 0, 60, 1483, 1281, 51] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=22 tokens_per_expert_current_ep_rank=[0, 341, 1096, 0, 0, 6, 525, 0, 0, 1117, 0, 377, 373, 0, 0, 1, 0, 7457, 5, 0, 0, 1, 0, 0, 0, 0, 5, 2, 0, 282, 3534, 0, 0, 0, 0, 0, 0, 7, 0, 12, 63, 7912, 0, 9, 4416, 7961, 6, 0, 6, 0, 0, 0, 5140, 0, 6, 8182, 0, 85, 203, 0, 10, 3, 8, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=22 tokens_per_expert_current_ep_rank=[3, 809, 826, 6, 0, 2242, 2, 7, 86, 148, 0, 4027, 6672, 142, 5, 37, 4, 494, 3534, 1, 0, 51, 3, 0, 869, 24, 2310, 146, 108, 142, 386, 21, 105, 0, 0, 0, 0, 2467, 0, 4094, 2172, 2482, 50, 2124, 0, 567, 462, 145, 0, 80, 193, 39, 6, 258, 692, 5964, 8, 294, 492, 0, 71, 2018, 1171, 93] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=22 rerouted_tokens_per_expert_current_ep_rank=[22, 512, 2219, 2, 0, 174, 321, 0, 4448, 626, 0, 0, 0, 0, 0, 0, 0, 920, 3, 1105, 1076, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7098, 75, 0, 0, 0, 2, 0, 333, 735, 0, 0, 0, 0, 0, 0, 86, 27, 167, 16, 19, 250, 4029, 94, 2267, 1222, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 1, 67, 4, 319, 244, 0, 0, 0, 0, 0, 0, 0, 913, 0, 14, 105, 1669, 1712, 17, 75, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 11, 0, 4067, 0, 27, 5176, 0, 0, 0, 0, 0, 0, 0, 0, 4, 95, 116, 0, 7, 95, 85, 3, 6452, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8324, 3114, 8243, 8177, 639, 4505, 9293, 6857] avg=6144.0000 max=9293 max_rank=6 min=639 min_rank=4 imbalance_ratio=1.512533 max_load_reduction=39859 max_load_reduction_pct=0.810933 moved_to_echo=21294 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=22 rerouted_tokens_per_expert_current_ep_rank=[27, 197, 915, 0, 0, 2310, 0, 28, 354, 410, 0, 0, 0, 0, 0, 0, 146, 107, 0, 3383, 4256, 124, 2, 13, 0, 0, 0, 0, 0, 0, 0, 0, 82, 257, 2297, 0, 0, 119, 0, 0, 1317, 58, 0, 0, 0, 0, 0, 0, 590, 1, 2621, 29, 70, 78, 542, 16, 1486, 223, 0, 0, 0, 0, 0, 0, 271, 0, 0, 0, 0, 3200, 0, 5585, 966, 0, 0, 0, 0, 0, 0, 0, 3271, 0, 27, 2591, 0, 137, 435, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 175, 353, 14, 3, 286, 355, 3395, 0, 0, 0, 0, 0, 0, 0, 0, 6, 200, 549, 0, 55, 2443, 1425, 66, 1178, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4241, 8031, 4130, 5656, 10022, 6569, 4581, 5922] avg=6144.0000 max=10022 max_rank=4 min=4130 min_rank=2 imbalance_ratio=1.631185 max_load_reduction=39130 max_load_reduction_pct=0.796102 moved_to_echo=27094 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=22 rerouted_tokens_per_expert_current_ep_rank=[22, 37, 618, 0, 0, 3608, 0, 70, 120, 312, 0, 0, 0, 0, 0, 0, 481, 71, 0, 1833, 3776, 84, 5, 79, 0, 0, 0, 0, 0, 0, 0, 0, 24, 47, 2924, 0, 0, 192, 0, 0, 1168, 19, 0, 0, 0, 0, 0, 0, 242, 1, 3175, 144, 19, 28, 629, 2, 1130, 67, 0, 0, 0, 0, 0, 0, 387, 0, 0, 0, 0, 3828, 0, 6289, 857, 0, 0, 0, 0, 0, 0, 0, 3175, 0, 17, 4286, 0, 48, 1218, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 222, 175, 33, 2, 705, 250, 2582, 0, 0, 0, 0, 0, 0, 0, 0, 1, 15, 513, 0, 48, 1731, 1173, 170, 359, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4787, 6329, 4374, 5437, 11361, 8885, 3969, 4010] avg=6144.0000 max=11361 max_rank=4 min=3969 min_rank=6 imbalance_ratio=1.849121 max_load_reduction=37791 max_load_reduction_pct=0.768860 moved_to_echo=28225 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=22 rerouted_tokens_per_expert_current_ep_rank=[16, 328, 1791, 0, 0, 1883, 20, 21, 1509, 489, 0, 0, 0, 0, 0, 0, 72, 192, 0, 2815, 4015, 110, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1503, 1896, 0, 0, 146, 0, 0, 1242, 249, 0, 0, 0, 0, 0, 0, 261, 25, 835, 52, 10, 160, 1744, 48, 1771, 632, 0, 0, 0, 0, 0, 0, 365, 0, 0, 0, 0, 2032, 0, 3657, 911, 0, 0, 0, 0, 0, 0, 0, 3940, 0, 37, 1849, 0, 581, 290, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 135, 17, 159, 501, 171, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 410, 0, 65, 1128, 1090, 67, 3332, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6057, 7211, 5040, 5538, 6965, 6945, 5120, 6276] avg=6144.0000 max=7211 max_rank=1 min=5040 min_rank=2 imbalance_ratio=1.173665 max_load_reduction=41941 max_load_reduction_pct=0.853292 moved_to_echo=25306 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=22 rerouted_tokens_per_expert_current_ep_rank=[21, 2040, 2159, 0, 0, 856, 150, 10, 2612, 601, 0, 0, 0, 0, 0, 0, 21, 404, 0, 2889, 3205, 102, 16, 4, 0, 0, 0, 0, 0, 0, 0, 0, 6, 5394, 1249, 0, 0, 22, 1, 0, 997, 435, 0, 0, 0, 0, 0, 0, 204, 60, 632, 17, 14, 279, 1032, 455, 2166, 1069, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 602, 0, 992, 731, 0, 0, 0, 0, 0, 0, 0, 2333, 0, 19, 421, 91, 996, 57, 273, 0, 0, 0, 0, 0, 0, 0, 0, 1, 27, 142, 0, 639, 15, 26, 4927, 0, 0, 0, 0, 0, 0, 0, 0, 37, 768, 464, 0, 24, 401, 300, 17, 5641, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8449, 6641, 8104, 5928, 2411, 4190, 5777, 7652] avg=6144.0000 max=8449 max_rank=0 min=2411 min_rank=4 imbalance_ratio=1.375163 max_load_reduction=40703 max_load_reduction_pct=0.828105 moved_to_echo=20030 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=22 rerouted_tokens_per_expert_current_ep_rank=[0, 341, 1096, 0, 0, 6, 525, 0, 5136, 634, 0, 0, 0, 0, 0, 0, 0, 1117, 0, 377, 243, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7457, 5, 0, 0, 1, 0, 0, 75, 849, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 0, 282, 3534, 0, 2298, 1261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 12, 55, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 9, 4416, 1976, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 5140, 0, 6, 5250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 203, 0, 10, 3, 8, 1, 6651, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7738, 1738, 8387, 7382, 74, 6470, 10402, 6961] avg=6144.0000 max=10402 max_rank=6 min=74 min_rank=4 imbalance_ratio=1.693034 max_load_reduction=38750 max_load_reduction_pct=0.788371 moved_to_echo=23907 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=22 rerouted_tokens_per_expert_current_ep_rank=[3, 809, 826, 6, 0, 2242, 2, 7, 365, 462, 0, 0, 0, 0, 0, 0, 86, 148, 0, 4027, 4343, 142, 5, 37, 0, 0, 0, 0, 0, 0, 0, 0, 4, 494, 3534, 1, 0, 51, 3, 0, 1344, 60, 0, 0, 0, 0, 0, 0, 869, 24, 2310, 146, 108, 142, 386, 21, 1675, 396, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, 0, 2467, 0, 4094, 985, 0, 0, 0, 0, 0, 0, 0, 2172, 0, 50, 2124, 0, 142, 462, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 193, 39, 6, 258, 692, 3827, 0, 0, 0, 0, 0, 0, 0, 0, 8, 294, 492, 0, 71, 2018, 1171, 93, 2086, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4722, 8788, 5491, 6077, 7651, 5095, 5095, 6233] avg=6144.0000 max=8788 max_rank=1 min=4722 min_rank=0 imbalance_ratio=1.430339 max_load_reduction=40364 max_load_reduction_pct=0.821208 moved_to_echo=24074 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=22 rerouted_tokens_per_expert_current_ep_rank=[45, 275, 1025, 3, 0, 2518, 0, 26, 524, 418, 0, 0, 0, 0, 0, 0, 245, 147, 0, 2524, 4155, 161, 38, 30, 0, 0, 0, 0, 0, 0, 0, 0, 31, 355, 3339, 0, 0, 286, 0, 0, 1286, 86, 0, 0, 0, 0, 0, 0, 671, 24, 1364, 73, 68, 142, 766, 30, 1515, 304, 0, 0, 0, 0, 0, 0, 672, 0, 0, 0, 0, 3146, 0, 5269, 942, 0, 0, 0, 0, 0, 0, 0, 2496, 0, 62, 2901, 0, 203, 722, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 217, 54, 36, 921, 170, 3461, 0, 0, 0, 0, 0, 0, 0, 0, 3, 249, 505, 0, 60, 1483, 1281, 51, 1609, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4834, 7300, 5383, 4957, 10029, 6493, 4915, 5241] avg=6144.0000 max=10029 max_rank=4 min=4834 min_rank=0 imbalance_ratio=1.632324 max_load_reduction=39123 max_load_reduction_pct=0.795959 moved_to_echo=26678 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1920, 128, 128, 0, 128, 15360, 128, 26240, 5760, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1152, 3200, 128, 19072, 25088, 768, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2944, 256, 11136, 512, 384, 1408, 12672, 768, 14336, 5248, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[256, 4608, 10752, 128, 0, 13696, 1024, 256, 15104, 3968, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 768, 1280, 256, 10112, 2688, 1792, 32768, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[18432, 0, 256, 14336, 6272, 5888, 3328, 1152, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 1920, 3328, 0, 384, 9344, 6656, 512, 27392, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[256, 22656, 15360, 128, 0, 896, 128, 0, 7808, 2560, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=23 tokens_per_expert_current_ep_rank=[5, 19, 0, 11, 1, 5, 7, 1397, 1953, 0, 223, 14, 9, 1803, 5, 69, 223, 65, 32, 805, 1, 3144, 19, 0, 72, 184, 8072, 1, 0, 7, 0, 9, 0, 432, 0, 2, 7989, 0, 1, 0, 17, 99, 1, 41, 4501, 6, 8, 20, 7814, 58, 403, 665, 27, 65, 19, 80, 218, 4251, 3, 37, 2, 1996, 328, 1914] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=23 tokens_per_expert_current_ep_rank=[0, 1, 1, 343, 54, 2407, 23, 99, 11, 0, 130, 59, 1870, 143, 424, 1258, 6440, 210, 311, 1198, 137, 2779, 194, 0, 162, 1477, 4899, 149, 75, 222, 0, 516, 655, 39, 0, 0, 3285, 76, 138, 1, 68, 346, 0, 31, 87, 1287, 0, 139, 2388, 2359, 411, 214, 230, 2614, 32, 104, 2762, 14, 894, 0, 582, 46, 4724, 34] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=23 tokens_per_expert_current_ep_rank=[0, 4, 0, 155, 26, 1790, 23, 538, 50, 0, 178, 308, 1211, 848, 309, 442, 3565, 127, 249, 1351, 211, 3564, 184, 0, 213, 770, 6327, 90, 9, 365, 0, 463, 587, 349, 0, 0, 5108, 58, 231, 2, 215, 49, 0, 74, 409, 1080, 16, 150, 3831, 417, 754, 940, 258, 1387, 35, 151, 2981, 278, 736, 0, 639, 1396, 3272, 379] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 11, 370, 72, 3477, 18, 87, 0, 0, 72, 90, 2747, 65, 589, 1473, 6749, 308, 469, 708, 73, 1829, 429, 0, 182, 836, 3617, 63, 9, 1071, 0, 728, 1250, 14, 0, 0, 1638, 116, 454, 2, 121, 97, 0, 38, 32, 2515, 16, 176, 608, 1067, 480, 156, 244, 3833, 25, 43, 3083, 1, 1289, 0, 1190, 229, 4292, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=23 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 2, 0, 808, 3921, 0, 228, 4, 4, 800, 0, 0, 25, 0, 5, 98, 1, 1542, 0, 0, 1, 2, 8188, 0, 4, 1, 0, 3, 2, 301, 0, 0, 8171, 0, 1, 0, 0, 4, 0, 1, 6868, 3, 0, 0, 8007, 2, 46, 464, 1, 7, 0, 0, 18, 5103, 1, 125, 1, 1347, 19, 3022] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=23 tokens_per_expert_current_ep_rank=[0, 2, 3, 46, 29, 268, 104, 802, 1069, 0, 217, 199, 318, 1045, 17, 333, 1276, 73, 53, 2524, 54, 4306, 110, 0, 127, 320, 7617, 47, 34, 138, 0, 33, 136, 503, 0, 0, 7277, 14, 52, 0, 81, 51, 0, 363, 2786, 171, 3, 35, 7088, 567, 589, 1018, 130, 350, 4, 292, 1029, 1348, 214, 26, 39, 728, 1701, 1393] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=23 tokens_per_expert_current_ep_rank=[0, 50, 1, 457, 242, 1751, 120, 228, 45, 0, 156, 54, 946, 168, 220, 1148, 6450, 202, 747, 1378, 131, 2466, 150, 0, 154, 1211, 5513, 189, 63, 529, 0, 259, 364, 367, 0, 2, 4084, 356, 57, 2, 7, 307, 0, 310, 192, 729, 0, 169, 3742, 1713, 429, 360, 247, 2155, 18, 127, 2307, 15, 820, 0, 509, 255, 4360, 121] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 5, 294, 71, 2801, 23, 147, 14, 5, 155, 127, 2073, 449, 746, 1093, 5446, 143, 341, 1019, 225, 2381, 137, 2, 231, 1168, 5136, 511, 39, 659, 0, 339, 973, 162, 0, 2, 2970, 69, 328, 0, 216, 166, 0, 94, 154, 1798, 19, 188, 2716, 956, 564, 408, 257, 2399, 55, 135, 2707, 26, 1275, 0, 868, 415, 3407, 45] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1, 343, 54, 2407, 23, 99, 4629, 1769, 0, 0, 0, 0, 0, 0, 11, 0, 130, 59, 1870, 143, 424, 1258, 1331, 252, 0, 0, 0, 0, 0, 0, 2704, 210, 311, 1198, 137, 2779, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 162, 1477, 3516, 149, 75, 222, 0, 516, 0, 0, 0, 0, 0, 0, 0, 0, 655, 39, 0, 0, 3285, 76, 138, 1, 52, 0, 0, 0, 0, 0, 0, 0, 68, 346, 0, 31, 87, 1287, 0, 139, 896, 1967, 0, 0, 0, 0, 0, 0, 1240, 2359, 411, 214, 230, 2614, 32, 104, 0, 0, 0, 0, 0, 0, 0, 0, 2762, 14, 894, 0, 582, 46, 95, 34, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9326, 5478, 7533, 6117, 4246, 4821, 7204, 4427] avg=6144.0000 max=9326 max_rank=0 min=4246 min_rank=4 imbalance_ratio=1.517904 max_load_reduction=39826 max_load_reduction_pct=0.810262 moved_to_echo=20698 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=23 rerouted_tokens_per_expert_current_ep_rank=[5, 19, 0, 11, 1, 5, 7, 1397, 321, 61, 0, 0, 0, 0, 0, 0, 1953, 0, 223, 14, 9, 1803, 5, 69, 2194, 825, 0, 0, 0, 0, 0, 0, 94, 65, 32, 805, 1, 3144, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 184, 5792, 1, 0, 7, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 432, 0, 2, 7989, 0, 1, 0, 86, 0, 0, 0, 0, 0, 0, 0, 17, 99, 1, 41, 4501, 6, 8, 20, 2932, 68, 0, 0, 0, 0, 0, 0, 4057, 58, 403, 665, 27, 65, 19, 80, 0, 0, 0, 0, 0, 0, 0, 0, 218, 4251, 3, 37, 2, 1996, 7, 1914, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1827, 7095, 4160, 6065, 8510, 7693, 5374, 8428] avg=6144.0000 max=8510 max_rank=4 min=1827 min_rank=0 imbalance_ratio=1.385091 max_load_reduction=40642 max_load_reduction_pct=0.826864 moved_to_echo=30005 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 11, 370, 72, 3477, 18, 87, 4206, 1854, 0, 0, 0, 0, 0, 0, 0, 0, 72, 90, 2747, 65, 589, 1473, 983, 64, 0, 0, 0, 0, 0, 0, 2834, 308, 469, 708, 73, 1829, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, 836, 2596, 63, 9, 1071, 0, 728, 0, 0, 0, 0, 0, 0, 0, 0, 1250, 14, 0, 0, 1638, 116, 454, 2, 38, 0, 0, 0, 0, 0, 0, 0, 121, 97, 0, 38, 32, 2515, 16, 176, 228, 2061, 0, 0, 0, 0, 0, 0, 316, 1067, 480, 156, 244, 3833, 25, 43, 0, 0, 0, 0, 0, 0, 0, 0, 3083, 1, 1289, 0, 1190, 229, 86, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10095, 6083, 6650, 5485, 3512, 5284, 6164, 5879] avg=6144.0000 max=10095 max_rank=0 min=3512 min_rank=4 imbalance_ratio=1.643066 max_load_reduction=39057 max_load_reduction_pct=0.794617 moved_to_echo=20839 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 155, 26, 1790, 23, 538, 3206, 979, 0, 0, 0, 0, 0, 0, 50, 0, 178, 308, 1211, 848, 309, 442, 1719, 404, 0, 0, 0, 0, 0, 0, 1498, 127, 249, 1351, 211, 3564, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 213, 770, 4540, 90, 9, 365, 0, 463, 0, 0, 0, 0, 0, 0, 0, 0, 587, 349, 0, 0, 5108, 58, 231, 2, 68, 0, 0, 0, 0, 0, 0, 0, 215, 49, 0, 74, 409, 1080, 16, 150, 1437, 1088, 0, 0, 0, 0, 0, 0, 1990, 417, 754, 940, 258, 1387, 35, 151, 0, 0, 0, 0, 0, 0, 0, 0, 2981, 278, 736, 0, 639, 1396, 66, 379, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6721, 5469, 7184, 6450, 6403, 4518, 5932, 6475] avg=6144.0000 max=7184 max_rank=2 min=4518 min_rank=5 imbalance_ratio=1.169271 max_load_reduction=41968 max_load_reduction_pct=0.853841 moved_to_echo=23328 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 2, 0, 808, 18, 6, 0, 0, 0, 0, 0, 0, 3921, 0, 228, 4, 4, 800, 0, 0, 2225, 846, 0, 0, 0, 0, 0, 0, 12, 0, 5, 98, 1, 1542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 5875, 0, 4, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2, 301, 0, 0, 8171, 0, 1, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 6868, 3, 0, 0, 3004, 7, 0, 0, 0, 0, 0, 0, 4157, 2, 46, 464, 1, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 5103, 1, 125, 1, 1347, 1, 3022, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[835, 8028, 1658, 5886, 8563, 9887, 4677, 9618] avg=6144.0000 max=9887 max_rank=5 min=835 min_rank=0 imbalance_ratio=1.609212 max_load_reduction=39265 max_load_reduction_pct=0.798848 moved_to_echo=32745 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 3, 46, 29, 268, 104, 802, 1671, 353, 0, 0, 0, 0, 0, 0, 1069, 0, 217, 199, 318, 1045, 17, 333, 2074, 752, 0, 0, 0, 0, 0, 0, 530, 73, 53, 2524, 54, 4306, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 320, 5457, 47, 34, 138, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 136, 503, 0, 0, 7277, 14, 52, 0, 86, 0, 0, 0, 0, 0, 0, 0, 81, 51, 0, 363, 2786, 171, 3, 35, 2661, 393, 0, 0, 0, 0, 0, 0, 3675, 567, 589, 1018, 130, 350, 4, 292, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1348, 214, 26, 39, 728, 30, 1393, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3278, 6024, 7650, 6156, 8068, 6544, 6625, 4807] avg=6144.0000 max=8068 max_rank=4 min=3278 min_rank=0 imbalance_ratio=1.313151 max_load_reduction=41084 max_load_reduction_pct=0.835856 moved_to_echo=26044 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 50, 1, 457, 242, 1751, 120, 228, 4273, 1772, 0, 0, 0, 0, 0, 0, 45, 0, 156, 54, 946, 168, 220, 1148, 1498, 395, 0, 0, 0, 0, 0, 0, 2708, 202, 747, 1378, 131, 2466, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 1211, 3956, 189, 63, 529, 0, 259, 0, 0, 0, 0, 0, 0, 0, 0, 364, 367, 0, 2, 4084, 356, 57, 2, 59, 0, 0, 0, 0, 0, 0, 0, 7, 307, 0, 310, 192, 729, 0, 169, 1404, 1970, 0, 0, 0, 0, 0, 0, 1943, 1713, 429, 360, 247, 2155, 18, 127, 0, 0, 0, 0, 0, 0, 0, 0, 2307, 15, 820, 0, 509, 255, 87, 121, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8894, 4630, 7782, 6361, 5291, 5088, 6992, 4114] avg=6144.0000 max=8894 max_rank=0 min=4114 min_rank=7 imbalance_ratio=1.447591 max_load_reduction=40258 max_load_reduction_pct=0.819051 moved_to_echo=21485 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 5, 294, 71, 2801, 23, 147, 3339, 1496, 0, 0, 0, 0, 0, 0, 14, 5, 155, 127, 2073, 449, 746, 1093, 1396, 287, 0, 0, 0, 0, 0, 0, 2287, 143, 341, 1019, 225, 2381, 137, 2, 0, 0, 0, 0, 0, 0, 0, 0, 231, 1168, 3685, 511, 39, 659, 0, 339, 0, 0, 0, 0, 0, 0, 0, 0, 973, 162, 0, 2, 2970, 69, 328, 0, 55, 0, 0, 0, 0, 0, 0, 0, 216, 166, 0, 94, 154, 1798, 19, 188, 1019, 1663, 0, 0, 0, 0, 0, 0, 1410, 956, 564, 408, 257, 2399, 55, 135, 0, 0, 0, 0, 0, 0, 0, 0, 2707, 26, 1275, 0, 868, 415, 68, 45, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 6345, 6535, 6632, 4559, 5317, 6184, 5404] avg=6144.0000 max=8176 max_rank=0 min=4559 min_rank=4 imbalance_ratio=1.330729 max_load_reduction=40976 max_load_reduction_pct=0.833659 moved_to_echo=21464 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 128, 1792, 512, 12544, 384, 4224, 21760, 8320, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[12672, 1152, 2304, 9088, 896, 22016, 1280, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[7168, 128, 1408, 896, 9216, 5376, 2432, 5888, 13440, 3840, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[15232, 11136, 5248, 256, 3840, 6528, 512, 6912, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[768, 1152, 128, 1024, 15104, 7680, 128, 896, 13696, 9344, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[18816, 7168, 3712, 4352, 1408, 12928, 256, 1024, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[1152, 6016, 35456, 1152, 256, 3072, 0, 2432, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3968, 2176, 0, 128, 40576, 768, 1280, 128, 640, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=24 tokens_per_expert_current_ep_rank=[34, 2308, 1, 1400, 385, 2032, 1064, 0, 5240, 265, 141, 521, 0, 68, 574, 825, 0, 16, 209, 60, 0, 2481, 1456, 0, 0, 64, 2212, 756, 4, 132, 0, 0, 308, 2142, 215, 317, 188, 2712, 123, 527, 405, 328, 4901, 269, 45, 63, 0, 2409, 65, 250, 0, 1890, 1, 1330, 1, 1827, 0, 1309, 1126, 1742, 0, 2411, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=24 tokens_per_expert_current_ep_rank=[20, 219, 0, 7548, 7688, 7274, 10, 1, 669, 0, 0, 17, 0, 14, 11, 4144, 0, 35, 28, 1, 18, 10, 2622, 0, 3, 83, 150, 318, 0, 0, 18, 0, 3047, 7911, 165, 2, 1, 20, 61, 15, 23, 95, 4497, 390, 28, 14, 0, 147, 13, 692, 0, 694, 0, 12, 6, 83, 179, 73, 71, 4, 1, 7, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=24 tokens_per_expert_current_ep_rank=[71, 2088, 0, 328, 46, 967, 1516, 0, 4115, 163, 486, 851, 0, 28, 814, 746, 0, 0, 92, 28, 0, 3301, 831, 0, 0, 47, 3114, 563, 37, 384, 0, 0, 290, 741, 189, 216, 31, 4249, 86, 460, 550, 643, 3872, 297, 41, 98, 0, 2139, 467, 422, 0, 1499, 3, 2647, 13, 1230, 0, 1355, 1324, 2377, 0, 3297, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=24 tokens_per_expert_current_ep_rank=[3, 749, 0, 5878, 5655, 5022, 177, 0, 4103, 9, 78, 45, 0, 29, 71, 3564, 0, 78, 25, 0, 0, 383, 1281, 0, 0, 85, 610, 1076, 5, 103, 6, 0, 1159, 6962, 601, 67, 8, 495, 55, 40, 45, 36, 5503, 384, 2, 5, 0, 571, 6, 636, 0, 1616, 0, 178, 1, 332, 0, 427, 431, 137, 0, 420, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=24 tokens_per_expert_current_ep_rank=[22, 2009, 0, 2936, 1642, 2994, 701, 0, 4024, 14, 283, 342, 0, 3, 249, 1971, 0, 34, 45, 14, 0, 1432, 2151, 0, 0, 63, 2768, 1110, 12, 168, 0, 0, 1028, 4259, 321, 101, 51, 1549, 39, 213, 128, 410, 5559, 330, 55, 54, 0, 752, 237, 681, 0, 2219, 7, 1146, 3, 747, 1, 1134, 630, 969, 0, 1542, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=24 tokens_per_expert_current_ep_rank=[8, 1647, 0, 1648, 677, 1807, 304, 0, 6520, 26, 35, 291, 0, 82, 529, 2471, 0, 35, 176, 69, 0, 1480, 1852, 0, 0, 83, 1414, 371, 83, 276, 151, 0, 242, 3158, 816, 136, 272, 2100, 432, 371, 432, 966, 5170, 400, 18, 199, 0, 2943, 15, 473, 0, 2217, 2, 976, 50, 1244, 0, 1166, 962, 872, 7, 1477, 0, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=24 tokens_per_expert_current_ep_rank=[0, 2, 0, 7906, 8167, 7663, 1, 0, 27, 0, 0, 6, 0, 0, 3, 2989, 0, 45, 0, 0, 0, 9, 1703, 0, 0, 95, 46, 350, 0, 0, 0, 0, 4896, 8177, 57, 0, 0, 6, 0, 0, 1, 13, 4567, 254, 0, 4, 0, 4, 1, 892, 0, 139, 0, 6, 0, 9, 1106, 2, 2, 1, 0, 3, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=24 tokens_per_expert_current_ep_rank=[20, 2331, 1, 1011, 458, 1797, 953, 0, 5291, 156, 620, 533, 0, 26, 434, 1483, 0, 20, 30, 16, 0, 2229, 1335, 5, 0, 57, 2772, 1035, 18, 166, 1, 0, 325, 2454, 242, 163, 216, 2266, 138, 447, 167, 550, 4616, 255, 14, 124, 0, 1538, 246, 615, 0, 2059, 4, 1667, 37, 1210, 0, 1529, 1154, 1533, 0, 2785, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=24 rerouted_tokens_per_expert_current_ep_rank=[20, 219, 0, 2154, 7688, 0, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 517, 0, 0, 17, 0, 14, 11, 4144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 28, 1, 18, 10, 2622, 0, 5394, 402, 298, 0, 0, 0, 0, 0, 3, 83, 150, 318, 0, 0, 18, 0, 6976, 0, 0, 0, 0, 0, 0, 0, 3047, 3775, 165, 2, 1, 20, 61, 15, 0, 0, 0, 0, 0, 0, 0, 0, 23, 95, 3548, 390, 28, 14, 0, 147, 0, 0, 0, 0, 0, 0, 0, 0, 13, 692, 0, 694, 0, 12, 6, 83, 3609, 0, 0, 0, 0, 0, 0, 0, 179, 73, 71, 4, 1, 7, 0, 0, 949, 152, 125, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10092, 4703, 8808, 7548, 7086, 4245, 5109, 1561] avg=6144.0000 max=10092 max_rank=0 min=1561 min_rank=7 imbalance_ratio=1.642578 max_load_reduction=39060 max_load_reduction_pct=0.794678 moved_to_echo=18001 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=24 rerouted_tokens_per_expert_current_ep_rank=[34, 2308, 1, 400, 385, 0, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 265, 141, 521, 0, 68, 574, 825, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 209, 60, 0, 2481, 1456, 0, 1000, 109, 83, 0, 0, 0, 0, 0, 0, 64, 2212, 756, 4, 132, 0, 0, 1949, 0, 0, 0, 0, 0, 0, 0, 308, 1023, 215, 317, 188, 2712, 123, 527, 0, 0, 0, 0, 0, 0, 0, 0, 405, 328, 3866, 269, 45, 63, 0, 2409, 0, 0, 0, 0, 0, 0, 0, 0, 65, 250, 0, 1890, 1, 1330, 1, 1827, 977, 0, 0, 0, 0, 0, 0, 0, 0, 1309, 1126, 1742, 0, 2411, 0, 0, 1035, 1196, 33, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4192, 6438, 5414, 5117, 5413, 7385, 6341, 8852] avg=6144.0000 max=8852 max_rank=7 min=4192 min_rank=0 imbalance_ratio=1.440755 max_load_reduction=40300 max_load_reduction_pct=0.819906 moved_to_echo=27991 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=24 rerouted_tokens_per_expert_current_ep_rank=[71, 2088, 0, 94, 46, 0, 1516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3176, 163, 486, 851, 0, 28, 814, 746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 28, 0, 3301, 831, 0, 234, 37, 39, 0, 0, 0, 0, 0, 0, 47, 3114, 563, 37, 384, 0, 0, 928, 0, 0, 0, 0, 0, 0, 0, 290, 355, 189, 216, 31, 4249, 86, 460, 0, 0, 0, 0, 0, 0, 0, 0, 550, 643, 3055, 297, 41, 98, 0, 2139, 0, 0, 0, 0, 0, 0, 0, 0, 467, 422, 0, 1499, 3, 2647, 13, 1230, 338, 0, 0, 0, 0, 0, 0, 0, 0, 1355, 1324, 2377, 0, 3297, 0, 0, 817, 939, 11, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3815, 6264, 4562, 5073, 5876, 6823, 6619, 10120] avg=6144.0000 max=10120 max_rank=7 min=3815 min_rank=0 imbalance_ratio=1.647135 max_load_reduction=39032 max_load_reduction_pct=0.794108 moved_to_echo=29438 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=24 rerouted_tokens_per_expert_current_ep_rank=[22, 2009, 0, 838, 1642, 0, 701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3106, 14, 283, 342, 0, 3, 249, 1971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 45, 14, 0, 1432, 2151, 0, 2098, 216, 123, 0, 0, 0, 0, 0, 0, 63, 2768, 1110, 12, 168, 0, 0, 2871, 0, 0, 0, 0, 0, 0, 0, 1028, 2033, 321, 101, 51, 1549, 39, 213, 0, 0, 0, 0, 0, 0, 0, 0, 128, 410, 4385, 330, 55, 54, 0, 752, 0, 0, 0, 0, 0, 0, 0, 0, 237, 681, 0, 2219, 7, 1146, 3, 747, 1943, 0, 0, 0, 0, 0, 0, 0, 1, 1134, 630, 969, 0, 1542, 0, 0, 1174, 918, 67, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5212, 5968, 6113, 6992, 5335, 6114, 6983, 6435] avg=6144.0000 max=6992 max_rank=3 min=5212 min_rank=0 imbalance_ratio=1.138021 max_load_reduction=42160 max_load_reduction_pct=0.857747 moved_to_echo=24867 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=24 rerouted_tokens_per_expert_current_ep_rank=[3, 749, 0, 1672, 5655, 0, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3163, 9, 78, 45, 0, 29, 71, 3564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 25, 0, 0, 383, 1281, 0, 4206, 359, 207, 0, 0, 0, 0, 0, 0, 85, 610, 1076, 5, 103, 6, 0, 4815, 0, 0, 0, 0, 0, 0, 0, 1159, 3309, 601, 67, 8, 495, 55, 40, 0, 0, 0, 0, 0, 0, 0, 0, 45, 36, 4337, 384, 2, 5, 0, 571, 0, 0, 0, 0, 0, 0, 0, 0, 6, 636, 0, 1616, 0, 178, 1, 332, 3180, 0, 0, 0, 0, 0, 0, 0, 0, 427, 431, 137, 0, 420, 0, 0, 1166, 940, 114, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8256, 6959, 6539, 6700, 5734, 5380, 5949, 3635] avg=6144.0000 max=8256 max_rank=0 min=3635 min_rank=7 imbalance_ratio=1.343750 max_load_reduction=40896 max_load_reduction_pct=0.832031 moved_to_echo=20698 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=24 rerouted_tokens_per_expert_current_ep_rank=[8, 1647, 0, 471, 677, 0, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5032, 26, 35, 291, 0, 82, 529, 2471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 176, 69, 0, 1480, 1852, 0, 1177, 160, 75, 0, 0, 0, 0, 0, 0, 83, 1414, 371, 83, 276, 151, 0, 1732, 0, 0, 0, 0, 0, 0, 0, 242, 1508, 816, 136, 272, 2100, 432, 371, 0, 0, 0, 0, 0, 0, 0, 0, 432, 966, 4078, 400, 18, 199, 0, 2943, 0, 0, 0, 0, 0, 0, 0, 0, 15, 473, 0, 2217, 2, 976, 50, 1244, 1440, 0, 0, 0, 0, 0, 0, 0, 0, 1166, 962, 872, 7, 1477, 0, 1, 1092, 1488, 50, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3107, 8466, 5024, 4110, 5877, 9036, 6417, 7115] avg=6144.0000 max=9036 max_rank=5 min=3107 min_rank=0 imbalance_ratio=1.470703 max_load_reduction=40116 max_load_reduction_pct=0.816162 moved_to_echo=28445 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=24 rerouted_tokens_per_expert_current_ep_rank=[20, 2331, 1, 289, 458, 0, 953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 156, 620, 533, 0, 26, 434, 1483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 30, 16, 0, 2229, 1335, 5, 722, 124, 73, 0, 0, 0, 0, 0, 0, 57, 2772, 1035, 18, 166, 1, 0, 1724, 0, 0, 0, 0, 0, 0, 0, 325, 1173, 242, 163, 216, 2266, 138, 447, 0, 0, 0, 0, 0, 0, 0, 0, 167, 550, 3642, 255, 14, 124, 0, 1538, 0, 0, 0, 0, 0, 0, 0, 0, 246, 615, 0, 2059, 4, 1667, 37, 1210, 1119, 0, 0, 0, 0, 0, 0, 0, 0, 1529, 1154, 1533, 0, 2785, 0, 0, 974, 1208, 38, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4052, 7335, 4554, 5773, 4970, 6290, 6957, 9221] avg=6144.0000 max=9221 max_rank=7 min=4052 min_rank=0 imbalance_ratio=1.500814 max_load_reduction=39931 max_load_reduction_pct=0.812398 moved_to_echo=27438 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 2256, 8167, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 6, 0, 0, 3, 2989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 9, 1703, 0, 5650, 416, 315, 0, 0, 0, 0, 0, 0, 95, 46, 350, 0, 0, 0, 0, 7348, 0, 0, 0, 0, 0, 0, 0, 4896, 3902, 57, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 13, 3603, 254, 0, 4, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1, 892, 0, 139, 0, 6, 0, 9, 3730, 0, 0, 0, 0, 0, 0, 0, 1106, 2, 2, 1, 0, 3, 0, 0, 964, 6, 129, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10426, 3019, 8138, 7839, 8861, 3879, 4777, 2213] avg=6144.0000 max=10426 max_rank=0 min=2213 min_rank=7 imbalance_ratio=1.696940 max_load_reduction=38726 max_load_reduction_pct=0.787882 moved_to_echo=19730 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1408, 7040, 5760, 7680, 128, 12032, 0, 128, 8192, 6912, 640, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1792, 3072, 30592, 2688, 256, 640, 0, 10624, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[256, 11392, 128, 8192, 24832, 0, 4736, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[23168, 640, 1664, 2688, 0, 256, 2688, 18304, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[11392, 17152, 2688, 1024, 768, 13440, 1024, 2176, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1152, 4736, 0, 12416, 128, 8064, 128, 6784, 16384, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 640, 13184, 5632, 256, 1280, 256, 0, 28416, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[0, 384, 640, 256, 128, 11392, 13312, 128, 20608, 1920, 1280, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True - [2026-08-26 14:53:45] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 832.4 | throughput per GPU (TFLOP/s/GPU): 153.3 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.114793E+01 | loss scale: 1.0 | grad norm: 400.648 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[2205, 690, 254, 944, 159, 1044, 876, 118, 3068, 558, 735, 412, 3, 2466, 663, 704, 528, 797, 657, 736, 148, 253, 178, 925, 531, 692, 395, 428, 926, 420, 834, 1075, 858, 546, 1230, 1506, 1038, 345, 987, 653, 173, 535, 415, 161, 805, 1327, 386, 74, 204, 434, 469, 1179, 836, 2062, 289, 433, 1425, 527, 1628, 572, 151, 699, 1737, 1046] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[2197, 695, 293, 958, 133, 1232, 807, 123, 3117, 688, 662, 322, 4, 2673, 631, 706, 523, 789, 597, 690, 167, 304, 172, 976, 508, 689, 508, 499, 869, 412, 711, 884, 898, 490, 1067, 1530, 988, 385, 975, 584, 185, 549, 484, 152, 826, 1432, 401, 79, 177, 406, 498, 1026, 736, 2426, 294, 396, 1469, 537, 1720, 577, 146, 604, 1617, 959] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[2165, 634, 227, 1148, 114, 1088, 820, 89, 3232, 497, 736, 375, 5, 2646, 716, 651, 480, 845, 582, 689, 117, 355, 131, 906, 572, 486, 408, 461, 859, 384, 705, 976, 872, 525, 1212, 1498, 1062, 304, 1036, 563, 179, 503, 443, 137, 870, 1431, 386, 86, 162, 364, 534, 1161, 851, 2397, 267, 355, 1348, 548, 1643, 698, 149, 737, 1703, 1029] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[2060, 633, 245, 1100, 122, 1131, 781, 102, 2964, 639, 726, 417, 2, 2479, 634, 718, 599, 909, 608, 710, 144, 260, 180, 988, 604, 520, 468, 402, 879, 547, 821, 1066, 912, 585, 1333, 1483, 1015, 410, 1089, 535, 196, 539, 459, 173, 823, 1404, 421, 91, 137, 378, 481, 1072, 754, 2101, 254, 486, 1363, 565, 1617, 636, 126, 630, 1624, 1002] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[2141, 627, 257, 1006, 153, 1191, 836, 102, 2937, 612, 848, 312, 12, 2574, 686, 570, 583, 834, 663, 727, 174, 252, 174, 925, 664, 745, 449, 519, 897, 468, 724, 1005, 998, 517, 1292, 1401, 1018, 435, 980, 554, 157, 472, 398, 152, 786, 1398, 432, 103, 176, 406, 513, 1045, 729, 2201, 293, 410, 1346, 527, 1637, 652, 159, 698, 1651, 949] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[2245, 645, 258, 1045, 133, 1154, 876, 129, 3171, 479, 690, 413, 11, 2677, 634, 610, 537, 889, 599, 662, 159, 250, 164, 889, 591, 649, 458, 462, 915, 499, 799, 936, 890, 545, 1150, 1529, 1054, 387, 944, 609, 173, 532, 471, 155, 791, 1456, 423, 74, 188, 412, 524, 1092, 802, 1893, 276, 404, 1338, 494, 1710, 636, 103, 704, 1683, 1082] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[2055, 671, 271, 1067, 135, 1149, 797, 112, 3030, 562, 775, 380, 4, 2697, 705, 567, 560, 812, 689, 769, 147, 221, 164, 841, 546, 707, 455, 403, 889, 435, 746, 1046, 921, 485, 1114, 1524, 916, 329, 914, 536, 202, 504, 535, 175, 876, 1413, 423, 63, 177, 439, 484, 1207, 790, 2204, 252, 450, 1428, 445, 1802, 678, 94, 727, 1717, 921] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[2158, 577, 209, 1134, 117, 1063, 776, 87, 3122, 615, 866, 351, 0, 2621, 725, 556, 576, 809, 672, 747, 137, 228, 169, 1026, 573, 616, 460, 580, 858, 464, 854, 962, 903, 562, 1224, 1425, 1079, 306, 915, 553, 155, 475, 443, 158, 838, 1405, 425, 64, 135, 352, 534, 1111, 781, 2358, 300, 387, 1306, 469, 1648, 592, 141, 697, 1706, 997] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2050, 690, 254, 944, 159, 1044, 876, 118, 0, 0, 0, 0, 0, 0, 0, 0, 515, 558, 735, 412, 3, 2466, 663, 704, 0, 0, 0, 0, 0, 0, 0, 0, 528, 797, 657, 736, 148, 253, 178, 925, 1493, 449, 0, 0, 0, 0, 0, 0, 531, 692, 395, 428, 926, 420, 834, 1075, 806, 111, 0, 0, 0, 0, 0, 0, 858, 546, 1230, 574, 1038, 345, 987, 653, 0, 0, 0, 0, 0, 0, 0, 0, 173, 535, 415, 161, 805, 1327, 386, 74, 2104, 0, 0, 0, 0, 0, 0, 0, 204, 434, 469, 1179, 836, 2062, 289, 433, 155, 126, 0, 0, 0, 0, 0, 0, 1425, 527, 1628, 572, 151, 699, 133, 1046, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6135, 6056, 6164, 6218, 6231, 5980, 6187, 6181] avg=6144.0000 max=6231 max_rank=4 min=5980 min_rank=5 imbalance_ratio=1.014160 max_load_reduction=42921 max_load_reduction_pct=0.873230 moved_to_echo=24579 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2043, 695, 293, 958, 133, 1232, 807, 123, 0, 0, 0, 0, 0, 0, 0, 0, 524, 688, 662, 322, 4, 2673, 631, 706, 0, 0, 0, 0, 0, 0, 0, 0, 523, 789, 597, 690, 167, 304, 172, 976, 1390, 456, 0, 0, 0, 0, 0, 0, 508, 689, 508, 499, 869, 412, 711, 884, 819, 104, 0, 0, 0, 0, 0, 0, 898, 490, 1067, 583, 988, 385, 975, 584, 0, 0, 0, 0, 0, 0, 0, 0, 185, 549, 484, 152, 826, 1432, 401, 79, 2137, 0, 0, 0, 0, 0, 0, 0, 177, 406, 498, 1026, 736, 2426, 294, 396, 154, 128, 0, 0, 0, 0, 0, 0, 1469, 537, 1720, 577, 146, 604, 123, 959, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6284, 6210, 6064, 6003, 5970, 6245, 6241, 6135] avg=6144.0000 max=6284 max_rank=0 min=5970 min_rank=4 imbalance_ratio=1.022786 max_load_reduction=42868 max_load_reduction_pct=0.872152 moved_to_echo=24591 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2013, 634, 227, 1148, 114, 1088, 820, 89, 0, 0, 0, 0, 0, 0, 0, 0, 543, 497, 736, 375, 5, 2646, 716, 651, 0, 0, 0, 0, 0, 0, 0, 0, 480, 845, 582, 689, 117, 355, 131, 906, 1463, 473, 0, 0, 0, 0, 0, 0, 572, 486, 408, 461, 859, 384, 705, 976, 802, 109, 0, 0, 0, 0, 0, 0, 872, 525, 1212, 571, 1062, 304, 1036, 563, 0, 0, 0, 0, 0, 0, 0, 0, 179, 503, 443, 137, 870, 1431, 386, 86, 2216, 0, 0, 0, 0, 0, 0, 0, 162, 364, 534, 1161, 851, 2397, 267, 355, 152, 125, 0, 0, 0, 0, 0, 0, 1348, 548, 1643, 698, 149, 737, 131, 1029, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6133, 6169, 6041, 5762, 6145, 6251, 6368, 6283] avg=6144.0000 max=6368 max_rank=6 min=5762 min_rank=3 imbalance_ratio=1.036458 max_load_reduction=42784 max_load_reduction_pct=0.870443 moved_to_echo=25047 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2088, 645, 258, 1045, 133, 1154, 876, 129, 0, 0, 0, 0, 0, 0, 0, 0, 533, 479, 690, 413, 11, 2677, 634, 610, 0, 0, 0, 0, 0, 0, 0, 0, 537, 889, 599, 662, 159, 250, 164, 889, 1446, 464, 0, 0, 0, 0, 0, 0, 591, 649, 458, 462, 915, 499, 799, 936, 819, 108, 0, 0, 0, 0, 0, 0, 890, 545, 1150, 582, 1054, 387, 944, 609, 0, 0, 0, 0, 0, 0, 0, 0, 173, 532, 471, 155, 791, 1456, 423, 74, 2174, 0, 0, 0, 0, 0, 0, 0, 188, 412, 524, 1092, 802, 1893, 276, 404, 157, 128, 0, 0, 0, 0, 0, 0, 1338, 494, 1710, 636, 103, 704, 129, 1082, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6328, 6047, 6059, 6236, 6161, 6249, 5876, 6196] avg=6144.0000 max=6328 max_rank=0 min=5876 min_rank=6 imbalance_ratio=1.029948 max_load_reduction=42824 max_load_reduction_pct=0.871257 moved_to_echo=24482 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1916, 633, 245, 1100, 122, 1131, 781, 102, 0, 0, 0, 0, 0, 0, 0, 0, 498, 639, 726, 417, 2, 2479, 634, 718, 0, 0, 0, 0, 0, 0, 0, 0, 599, 909, 608, 710, 144, 260, 180, 988, 1396, 434, 0, 0, 0, 0, 0, 0, 604, 520, 468, 402, 879, 547, 821, 1066, 794, 104, 0, 0, 0, 0, 0, 0, 912, 585, 1333, 565, 1015, 410, 1089, 535, 0, 0, 0, 0, 0, 0, 0, 0, 196, 539, 459, 173, 823, 1404, 421, 91, 2032, 0, 0, 0, 0, 0, 0, 0, 137, 378, 481, 1072, 754, 2101, 254, 486, 144, 124, 0, 0, 0, 0, 0, 0, 1363, 565, 1617, 636, 126, 630, 124, 1002, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6030, 6113, 6228, 6205, 6444, 6138, 5931, 6063] avg=6144.0000 max=6444 max_rank=4 min=5931 min_rank=6 imbalance_ratio=1.048828 max_load_reduction=42708 max_load_reduction_pct=0.868896 moved_to_echo=24576 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1987, 627, 257, 1006, 153, 1191, 836, 102, 0, 0, 0, 0, 0, 0, 0, 0, 486, 612, 848, 312, 12, 2574, 686, 570, 0, 0, 0, 0, 0, 0, 0, 0, 583, 834, 663, 727, 174, 252, 174, 925, 1422, 433, 0, 0, 0, 0, 0, 0, 664, 745, 449, 519, 897, 468, 724, 1005, 755, 110, 0, 0, 0, 0, 0, 0, 998, 517, 1292, 526, 1018, 435, 980, 554, 0, 0, 0, 0, 0, 0, 0, 0, 157, 472, 398, 152, 786, 1398, 432, 103, 2018, 0, 0, 0, 0, 0, 0, 0, 176, 406, 513, 1045, 729, 2201, 293, 410, 154, 120, 0, 0, 0, 0, 0, 0, 1346, 527, 1637, 652, 159, 698, 119, 949, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6159, 6100, 6187, 6336, 6320, 5916, 6047, 6087] avg=6144.0000 max=6336 max_rank=3 min=5916 min_rank=5 imbalance_ratio=1.031250 max_load_reduction=42816 max_load_reduction_pct=0.871094 moved_to_echo=24370 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2007, 577, 209, 1134, 117, 1063, 776, 87, 0, 0, 0, 0, 0, 0, 0, 0, 524, 615, 866, 351, 0, 2621, 725, 556, 0, 0, 0, 0, 0, 0, 0, 0, 576, 809, 672, 747, 137, 228, 169, 1026, 1466, 457, 0, 0, 0, 0, 0, 0, 573, 616, 460, 580, 858, 464, 854, 962, 763, 109, 0, 0, 0, 0, 0, 0, 903, 562, 1224, 543, 1079, 306, 915, 553, 0, 0, 0, 0, 0, 0, 0, 0, 155, 475, 443, 158, 838, 1405, 425, 64, 2141, 0, 0, 0, 0, 0, 0, 0, 135, 352, 534, 1111, 781, 2358, 300, 387, 151, 119, 0, 0, 0, 0, 0, 0, 1306, 469, 1648, 592, 141, 697, 131, 997, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5970, 6258, 6287, 6239, 6085, 6104, 6228, 5981] avg=6144.0000 max=6287 max_rank=2 min=5970 min_rank=0 imbalance_ratio=1.023275 max_load_reduction=42865 max_load_reduction_pct=0.872091 moved_to_echo=24398 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1911, 671, 271, 1067, 135, 1149, 797, 112, 0, 0, 0, 0, 0, 0, 0, 0, 509, 562, 775, 380, 4, 2697, 705, 567, 0, 0, 0, 0, 0, 0, 0, 0, 560, 812, 689, 769, 147, 221, 164, 841, 1476, 443, 0, 0, 0, 0, 0, 0, 546, 707, 455, 403, 889, 435, 746, 1046, 816, 110, 0, 0, 0, 0, 0, 0, 921, 485, 1114, 581, 916, 329, 914, 536, 0, 0, 0, 0, 0, 0, 0, 0, 202, 504, 535, 175, 876, 1413, 423, 63, 2078, 0, 0, 0, 0, 0, 0, 0, 177, 439, 484, 1207, 790, 2204, 252, 450, 144, 127, 0, 0, 0, 0, 0, 0, 1428, 445, 1802, 678, 94, 727, 131, 921, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6113, 6199, 6122, 6153, 5796, 6269, 6274, 6226] avg=6144.0000 max=6274 max_rank=6 min=5796 min_rank=4 imbalance_ratio=1.021159 max_load_reduction=42878 max_load_reduction_pct=0.872355 moved_to_echo=24565 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[16128, 5248, 2048, 8448, 1152, 9088, 6656, 896, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[4480, 6784, 5120, 5760, 1280, 2176, 1408, 7552, 11648, 3712, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[11136, 4224, 13440, 5120, 1152, 5504, 1024, 8064, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4224, 4736, 6144, 3072, 128, 20864, 5504, 5120, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4608, 5120, 3712, 3840, 7168, 3712, 6272, 8064, 6400, 896, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1536, 4224, 3712, 1280, 6656, 11392, 3328, 640, 17024, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1408, 3200, 4096, 8960, 6400, 17664, 2304, 3328, 1280, 1024, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7296, 4352, 9728, 4608, 8192, 2944, 7936, 4608, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[219, 421, 58, 172, 4703, 51, 2104, 467, 65, 792, 1349, 354, 644, 922, 27, 2973, 490, 1020, 273, 614, 318, 897, 369, 385, 483, 514, 746, 4064, 1, 111, 99, 648, 94, 3, 249, 17, 419, 5, 223, 125, 789, 463, 246, 608, 22, 517, 1, 283, 6723, 351, 3236, 0, 33, 219, 481, 1011, 2666, 401, 937, 629, 339, 139, 1275, 295] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[204, 455, 41, 135, 4298, 85, 1911, 332, 57, 586, 1298, 357, 593, 857, 37, 3024, 321, 1032, 469, 512, 294, 1136, 496, 427, 470, 504, 809, 4205, 3, 138, 122, 593, 51, 3, 246, 8, 432, 1, 271, 106, 797, 494, 244, 423, 14, 588, 0, 285, 6949, 364, 3460, 0, 34, 205, 513, 1047, 2874, 378, 929, 407, 302, 149, 1526, 251] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[238, 459, 38, 160, 4600, 88, 2058, 371, 74, 664, 1282, 352, 638, 776, 49, 2828, 401, 967, 440, 646, 267, 894, 487, 451, 461, 446, 740, 4006, 0, 118, 125, 610, 86, 2, 240, 15, 450, 3, 258, 103, 842, 515, 240, 673, 19, 487, 3, 336, 6883, 345, 3305, 0, 23, 187, 594, 1089, 2631, 331, 926, 651, 351, 154, 1362, 314] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[187, 512, 41, 138, 4588, 80, 1855, 475, 77, 611, 1175, 402, 573, 935, 55, 2862, 397, 1005, 475, 492, 324, 1000, 366, 463, 410, 559, 758, 4072, 6, 136, 96, 538, 114, 0, 260, 9, 381, 1, 262, 111, 795, 527, 257, 624, 28, 584, 2, 307, 6837, 338, 3860, 0, 17, 199, 514, 980, 2737, 110, 840, 620, 316, 160, 1377, 322] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[227, 455, 44, 152, 4652, 53, 2124, 440, 32, 582, 1270, 394, 635, 821, 40, 2998, 398, 1105, 328, 550, 263, 954, 449, 393, 425, 520, 645, 3998, 1, 132, 101, 624, 45, 2, 251, 4, 469, 6, 286, 109, 771, 514, 248, 541, 15, 507, 1, 237, 6897, 305, 3832, 0, 20, 204, 478, 1025, 2940, 215, 971, 529, 270, 132, 1245, 278] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[247, 497, 30, 140, 4462, 79, 1980, 353, 55, 666, 1478, 431, 686, 751, 56, 3006, 363, 1045, 537, 796, 363, 850, 482, 398, 396, 524, 671, 3869, 1, 145, 139, 684, 89, 1, 264, 19, 511, 0, 259, 103, 750, 455, 253, 475, 12, 659, 2, 260, 6623, 393, 3489, 2, 18, 225, 467, 968, 2698, 337, 986, 459, 421, 182, 1311, 281] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[235, 522, 39, 130, 4377, 54, 1826, 425, 72, 686, 1303, 348, 548, 932, 68, 2748, 456, 971, 473, 794, 354, 1063, 446, 461, 454, 503, 758, 3963, 2, 115, 103, 597, 103, 1, 260, 6, 507, 0, 276, 118, 760, 496, 257, 515, 15, 669, 5, 321, 6816, 375, 3236, 0, 31, 209, 505, 909, 2886, 318, 892, 491, 328, 220, 1506, 295] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[172, 564, 25, 150, 4692, 50, 2213, 350, 71, 683, 1298, 389, 657, 970, 40, 2850, 368, 1072, 533, 430, 364, 860, 405, 411, 424, 601, 657, 3886, 4, 140, 106, 741, 118, 1, 253, 16, 488, 4, 260, 99, 782, 524, 252, 636, 14, 538, 5, 234, 6868, 331, 3663, 0, 17, 196, 483, 988, 2541, 202, 845, 608, 321, 202, 1192, 295] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[204, 455, 41, 135, 2626, 85, 1911, 332, 0, 0, 0, 0, 0, 0, 0, 0, 57, 586, 1298, 357, 593, 857, 37, 2285, 0, 0, 0, 0, 0, 0, 0, 0, 321, 1032, 469, 512, 294, 1136, 496, 427, 646, 515, 441, 0, 0, 0, 0, 0, 470, 504, 809, 3764, 3, 138, 122, 593, 0, 0, 0, 0, 0, 0, 0, 0, 51, 3, 246, 8, 432, 1, 271, 106, 5049, 0, 0, 0, 0, 0, 0, 0, 797, 494, 244, 423, 14, 588, 0, 285, 1672, 1341, 93, 0, 0, 0, 0, 0, 559, 364, 3460, 0, 34, 205, 513, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 2359, 378, 929, 407, 302, 149, 1526, 251, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5789, 6070, 6289, 6403, 6167, 5951, 6182, 6301] avg=6144.0000 max=6403 max_rank=3 min=5789 min_rank=0 imbalance_ratio=1.042155 max_load_reduction=42749 max_load_reduction_pct=0.869731 moved_to_echo=24601 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[219, 421, 58, 172, 2874, 51, 2104, 467, 0, 0, 0, 0, 0, 0, 0, 0, 65, 792, 1349, 354, 644, 922, 27, 2246, 0, 0, 0, 0, 0, 0, 0, 0, 490, 1020, 273, 614, 318, 897, 369, 385, 635, 478, 426, 0, 0, 0, 0, 0, 483, 514, 746, 3638, 1, 111, 99, 648, 0, 0, 0, 0, 0, 0, 0, 0, 94, 3, 249, 17, 419, 5, 223, 125, 4885, 0, 0, 0, 0, 0, 0, 0, 789, 463, 246, 608, 22, 517, 1, 283, 1829, 1298, 92, 0, 0, 0, 0, 0, 540, 351, 3236, 0, 33, 219, 481, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 2188, 401, 937, 629, 339, 139, 1275, 295, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6366, 6399, 5905, 6240, 6020, 6148, 5871, 6203] avg=6144.0000 max=6399 max_rank=1 min=5871 min_rank=6 imbalance_ratio=1.041504 max_load_reduction=42753 max_load_reduction_pct=0.869812 moved_to_echo=24242 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[238, 459, 38, 160, 2811, 88, 2058, 371, 0, 0, 0, 0, 0, 0, 0, 0, 74, 664, 1282, 352, 638, 776, 49, 2137, 0, 0, 0, 0, 0, 0, 0, 0, 401, 967, 440, 646, 267, 894, 487, 451, 604, 472, 420, 0, 0, 0, 0, 0, 461, 446, 740, 3586, 0, 118, 125, 610, 0, 0, 0, 0, 0, 0, 0, 0, 86, 2, 240, 15, 450, 3, 258, 103, 5002, 0, 0, 0, 0, 0, 0, 0, 842, 515, 240, 673, 19, 487, 3, 336, 1789, 1329, 87, 0, 0, 0, 0, 0, 552, 345, 3305, 0, 23, 187, 594, 1089, 0, 0, 0, 0, 0, 0, 0, 0, 2159, 331, 926, 651, 351, 154, 1362, 314, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6223, 5972, 6049, 6086, 6159, 6320, 6095, 6248] avg=6144.0000 max=6320 max_rank=5 min=5972 min_rank=1 imbalance_ratio=1.028646 max_load_reduction=42832 max_load_reduction_pct=0.871419 moved_to_echo=24822 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[187, 512, 41, 138, 2803, 80, 1855, 475, 0, 0, 0, 0, 0, 0, 0, 0, 77, 611, 1175, 402, 573, 935, 55, 2163, 0, 0, 0, 0, 0, 0, 0, 0, 397, 1005, 475, 492, 324, 1000, 366, 463, 611, 491, 427, 0, 0, 0, 0, 0, 410, 559, 758, 3645, 6, 136, 96, 538, 0, 0, 0, 0, 0, 0, 0, 0, 114, 0, 260, 9, 381, 1, 262, 111, 4968, 0, 0, 0, 0, 0, 0, 0, 795, 527, 257, 624, 28, 584, 2, 307, 1785, 1320, 88, 0, 0, 0, 0, 0, 549, 338, 3860, 0, 17, 199, 514, 980, 0, 0, 0, 0, 0, 0, 0, 0, 2246, 110, 840, 620, 316, 160, 1377, 322, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6091, 5991, 6051, 6148, 6106, 6317, 6457, 5991] avg=6144.0000 max=6457 max_rank=6 min=5991 min_rank=1 imbalance_ratio=1.050944 max_load_reduction=42695 max_load_reduction_pct=0.868632 moved_to_echo=24871 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[227, 455, 44, 152, 2843, 53, 2124, 440, 0, 0, 0, 0, 0, 0, 0, 0, 32, 582, 1270, 394, 635, 821, 40, 2265, 0, 0, 0, 0, 0, 0, 0, 0, 398, 1105, 328, 550, 263, 954, 449, 393, 640, 527, 419, 0, 0, 0, 0, 0, 425, 520, 645, 3579, 1, 132, 101, 624, 0, 0, 0, 0, 0, 0, 0, 0, 45, 2, 251, 4, 469, 6, 286, 109, 5012, 0, 0, 0, 0, 0, 0, 0, 771, 514, 248, 541, 15, 507, 1, 237, 1809, 1331, 93, 0, 0, 0, 0, 0, 554, 305, 3832, 0, 20, 204, 478, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 2413, 215, 971, 529, 270, 132, 1245, 278, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6338, 6039, 6026, 6027, 6184, 6067, 6418, 6053] avg=6144.0000 max=6418 max_rank=6 min=6026 min_rank=2 imbalance_ratio=1.044596 max_load_reduction=42734 max_load_reduction_pct=0.869425 moved_to_echo=24722 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[235, 522, 39, 130, 2670, 54, 1826, 425, 0, 0, 0, 0, 0, 0, 0, 0, 72, 686, 1303, 348, 548, 932, 68, 2069, 0, 0, 0, 0, 0, 0, 0, 0, 456, 971, 473, 794, 354, 1063, 446, 461, 590, 521, 421, 0, 0, 0, 0, 0, 454, 503, 758, 3542, 2, 115, 103, 597, 0, 0, 0, 0, 0, 0, 0, 0, 103, 1, 260, 6, 507, 0, 276, 118, 4956, 0, 0, 0, 0, 0, 0, 0, 760, 496, 257, 515, 15, 669, 5, 321, 1707, 1319, 89, 0, 0, 0, 0, 0, 541, 375, 3236, 0, 31, 209, 505, 909, 0, 0, 0, 0, 0, 0, 0, 0, 2365, 318, 892, 491, 328, 220, 1506, 295, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5901, 6026, 6550, 6074, 6227, 6153, 5806, 6415] avg=6144.0000 max=6550 max_rank=2 min=5806 min_rank=6 imbalance_ratio=1.066081 max_load_reduction=42602 max_load_reduction_pct=0.866740 moved_to_echo=24601 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[172, 564, 25, 150, 2867, 50, 2213, 350, 0, 0, 0, 0, 0, 0, 0, 0, 71, 683, 1298, 389, 657, 970, 40, 2153, 0, 0, 0, 0, 0, 0, 0, 0, 368, 1072, 533, 430, 364, 860, 405, 411, 609, 456, 408, 0, 0, 0, 0, 0, 424, 601, 657, 3478, 4, 140, 106, 741, 0, 0, 0, 0, 0, 0, 0, 0, 118, 1, 253, 16, 488, 4, 260, 99, 4991, 0, 0, 0, 0, 0, 0, 0, 782, 524, 252, 636, 14, 538, 5, 234, 1825, 1326, 88, 0, 0, 0, 0, 0, 551, 331, 3663, 0, 17, 196, 483, 988, 0, 0, 0, 0, 0, 0, 0, 0, 2085, 202, 845, 608, 321, 202, 1192, 295, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6391, 6261, 5916, 6151, 6230, 6224, 6229, 5750] avg=6144.0000 max=6391 max_rank=0 min=5750 min_rank=7 imbalance_ratio=1.040202 max_load_reduction=42761 max_load_reduction_pct=0.869975 moved_to_echo=24433 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[247, 497, 30, 140, 2727, 79, 1980, 353, 0, 0, 0, 0, 0, 0, 0, 0, 55, 666, 1478, 431, 686, 751, 56, 2271, 0, 0, 0, 0, 0, 0, 0, 0, 363, 1045, 537, 796, 363, 850, 482, 398, 642, 484, 406, 0, 0, 0, 0, 0, 396, 524, 671, 3463, 1, 145, 139, 684, 0, 0, 0, 0, 0, 0, 0, 0, 89, 1, 264, 19, 511, 0, 259, 103, 4813, 0, 0, 0, 0, 0, 0, 0, 750, 455, 253, 475, 12, 659, 2, 260, 1735, 1278, 93, 0, 0, 0, 0, 0, 532, 393, 3489, 2, 18, 225, 467, 968, 0, 0, 0, 0, 0, 0, 0, 0, 2214, 337, 986, 459, 421, 182, 1311, 281, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6053, 6394, 6366, 6023, 6059, 5972, 6094, 6191] avg=6144.0000 max=6394 max_rank=1 min=5972 min_rank=5 imbalance_ratio=1.040690 max_load_reduction=42758 max_load_reduction_pct=0.869914 moved_to_echo=24316 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3200, 8320, 3584, 4864, 2560, 7680, 3584, 3456, 4992, 3968, 3456, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[18048, 2304, 7424, 4480, 2688, 1408, 10880, 2432, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[512, 5376, 10496, 3072, 4992, 7040, 384, 17664, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1792, 3968, 384, 1280, 22272, 640, 16128, 3328, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[6400, 4096, 2048, 4608, 256, 4608, 128, 2304, 14208, 10624, 768, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4480, 2816, 28160, 128, 256, 1664, 4096, 8064, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3584, 4224, 5888, 28800, 128, 1152, 896, 5120, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[768, 128, 2048, 128, 3712, 128, 2176, 896, 39680, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 160, 62, 178, 1, 76, 413, 1733, 4, 1712, 197, 1737, 441, 352, 0, 2, 4534, 832, 5253, 52, 1, 1, 5640, 744, 150, 578, 3079, 3345, 3, 238, 290, 0, 38, 503, 339, 116, 4150, 172, 73, 93, 1478, 0, 1, 30, 428, 2049, 91, 0, 838, 3, 182, 351, 4, 0, 131, 0, 3836, 148, 13, 5, 1, 1315, 0, 956] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 172, 78, 156, 0, 123, 396, 1876, 14, 1632, 241, 1770, 442, 379, 1, 1, 4586, 889, 5359, 63, 1, 0, 5451, 752, 149, 620, 2808, 2781, 2, 349, 278, 0, 48, 462, 498, 286, 3645, 228, 78, 66, 1481, 0, 0, 21, 439, 2165, 116, 1, 1009, 10, 224, 397, 5, 0, 187, 0, 3913, 171, 10, 10, 0, 1388, 0, 925] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 146, 72, 160, 0, 145, 424, 1825, 5, 1725, 238, 1572, 446, 343, 0, 0, 4575, 757, 5438, 64, 1, 0, 5526, 779, 152, 634, 2934, 2839, 2, 335, 259, 0, 29, 526, 596, 156, 3539, 199, 90, 80, 1684, 0, 2, 24, 419, 2083, 95, 0, 943, 4, 171, 372, 7, 0, 213, 0, 4045, 150, 10, 3, 0, 1336, 0, 980] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 131, 60, 136, 1, 105, 411, 1861, 21, 1720, 229, 1746, 456, 348, 0, 0, 4539, 855, 5526, 54, 0, 0, 5118, 666, 147, 600, 2925, 3219, 1, 173, 271, 0, 42, 613, 556, 171, 3704, 198, 98, 77, 1524, 0, 0, 36, 450, 2020, 109, 0, 861, 4, 143, 467, 10, 0, 167, 0, 4015, 156, 10, 17, 0, 1403, 0, 982] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1, 105, 47, 163, 0, 126, 311, 1822, 6, 1644, 213, 1812, 439, 325, 0, 2, 4804, 912, 5422, 63, 0, 0, 5527, 659, 160, 544, 3123, 2943, 3, 102, 251, 1, 21, 536, 498, 153, 3819, 229, 108, 90, 1277, 0, 0, 20, 482, 1974, 97, 0, 811, 8, 166, 445, 2, 0, 176, 1, 4365, 184, 3, 4, 0, 1287, 0, 866] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1, 159, 68, 192, 1, 72, 360, 1943, 24, 1764, 250, 1552, 393, 342, 0, 0, 4583, 961, 5324, 70, 2, 0, 5460, 770, 176, 618, 3057, 2724, 2, 300, 267, 0, 56, 507, 545, 211, 3626, 218, 89, 62, 1436, 0, 3, 32, 541, 2093, 111, 1, 822, 6, 184, 526, 7, 0, 126, 2, 4093, 155, 17, 4, 1, 1285, 1, 957] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 135, 81, 154, 2, 67, 384, 2023, 23, 1684, 297, 1577, 439, 346, 0, 0, 4480, 1064, 5407, 66, 1, 0, 5445, 632, 177, 708, 2979, 2549, 3, 192, 266, 0, 48, 531, 597, 187, 3728, 212, 111, 82, 1374, 0, 0, 17, 443, 2118, 97, 1, 937, 7, 157, 454, 4, 0, 126, 0, 4081, 199, 20, 7, 0, 1472, 1, 960] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 201, 51, 146, 0, 131, 365, 1927, 24, 1599, 221, 1724, 465, 316, 0, 0, 4654, 948, 5216, 61, 2, 0, 5299, 679, 172, 621, 3146, 3070, 2, 407, 277, 0, 45, 482, 393, 172, 3690, 177, 86, 96, 1567, 0, 1, 17, 416, 1938, 106, 1, 895, 7, 160, 472, 3, 0, 202, 0, 3917, 143, 7, 14, 0, 1490, 0, 931] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 105, 47, 163, 0, 126, 311, 1822, 2462, 983, 0, 0, 0, 0, 0, 0, 6, 1644, 213, 1812, 439, 325, 0, 2, 910, 868, 0, 0, 0, 0, 0, 0, 4673, 912, 0, 63, 0, 0, 0, 659, 0, 0, 0, 0, 0, 0, 0, 0, 160, 544, 1975, 2943, 3, 102, 251, 1, 0, 0, 0, 0, 0, 0, 0, 0, 21, 536, 498, 153, 3819, 229, 108, 90, 420, 280, 131, 0, 0, 0, 0, 0, 1277, 0, 0, 20, 482, 1974, 97, 0, 2050, 0, 0, 0, 0, 0, 0, 0, 811, 8, 166, 445, 2, 0, 176, 1, 4544, 0, 0, 0, 0, 0, 0, 0, 3945, 184, 3, 4, 0, 1287, 0, 866, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6020, 6219, 6307, 5979, 6285, 5900, 6153, 6289] avg=6144.0000 max=6307 max_rank=2 min=5900 min_rank=5 imbalance_ratio=1.026530 max_load_reduction=42845 max_load_reduction_pct=0.871684 moved_to_echo=24627 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 160, 62, 178, 1, 76, 413, 1733, 2384, 1002, 0, 0, 0, 0, 0, 0, 4, 1712, 197, 1737, 441, 352, 0, 2, 882, 855, 0, 0, 0, 0, 0, 0, 4411, 832, 0, 52, 1, 1, 0, 744, 0, 0, 0, 0, 0, 0, 0, 0, 150, 578, 1947, 3345, 3, 238, 290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 503, 339, 116, 4150, 172, 73, 93, 369, 277, 123, 0, 0, 0, 0, 0, 1478, 0, 1, 30, 428, 2049, 91, 0, 1987, 0, 0, 0, 0, 0, 0, 0, 838, 3, 182, 351, 4, 0, 131, 0, 4638, 0, 0, 0, 0, 0, 0, 0, 3467, 148, 13, 5, 1, 1315, 0, 956, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6009, 6182, 6041, 6551, 6253, 6064, 6147, 5905] avg=6144.0000 max=6551 max_rank=3 min=5905 min_rank=7 imbalance_ratio=1.066243 max_load_reduction=42601 max_load_reduction_pct=0.866720 moved_to_echo=24369 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 172, 78, 156, 0, 123, 396, 1876, 2432, 968, 0, 0, 0, 0, 0, 0, 14, 1632, 241, 1770, 442, 379, 1, 1, 901, 780, 0, 0, 0, 0, 0, 0, 4461, 889, 0, 63, 1, 0, 0, 752, 0, 0, 0, 0, 0, 0, 0, 0, 149, 620, 1776, 2781, 2, 349, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 462, 498, 286, 3645, 228, 78, 66, 377, 252, 125, 0, 0, 0, 0, 0, 1481, 0, 0, 21, 439, 2165, 116, 1, 2026, 0, 0, 0, 0, 0, 0, 0, 1009, 10, 224, 397, 5, 0, 187, 0, 4483, 0, 0, 0, 0, 0, 0, 0, 3536, 171, 10, 10, 0, 1388, 0, 925, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6201, 6161, 6166, 5955, 6065, 6249, 6315, 6040] avg=6144.0000 max=6315 max_rank=6 min=5955 min_rank=3 imbalance_ratio=1.027832 max_load_reduction=42837 max_load_reduction_pct=0.871521 moved_to_echo=24669 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 146, 72, 160, 0, 145, 424, 1825, 2467, 981, 0, 0, 0, 0, 0, 0, 5, 1725, 238, 1572, 446, 343, 0, 0, 913, 815, 0, 0, 0, 0, 0, 0, 4451, 757, 0, 64, 1, 0, 0, 779, 0, 0, 0, 0, 0, 0, 0, 0, 152, 634, 1856, 2839, 2, 335, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 526, 596, 156, 3539, 199, 90, 80, 389, 263, 124, 0, 0, 0, 0, 0, 1684, 0, 2, 24, 419, 2083, 95, 0, 2058, 0, 0, 0, 0, 0, 0, 0, 943, 4, 171, 372, 7, 0, 213, 0, 4545, 0, 0, 0, 0, 0, 0, 0, 3656, 150, 10, 3, 0, 1336, 0, 980, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6220, 6057, 6052, 6077, 5991, 6365, 6255, 6135] avg=6144.0000 max=6365 max_rank=5 min=5991 min_rank=4 imbalance_ratio=1.035970 max_load_reduction=42787 max_load_reduction_pct=0.870504 moved_to_echo=24746 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 159, 68, 192, 1, 72, 360, 1943, 2416, 970, 0, 0, 0, 0, 0, 0, 24, 1764, 250, 1552, 393, 342, 0, 0, 894, 849, 0, 0, 0, 0, 0, 0, 4458, 961, 0, 70, 2, 0, 0, 770, 0, 0, 0, 0, 0, 0, 0, 0, 176, 618, 1933, 2724, 2, 300, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 507, 545, 211, 3626, 218, 89, 62, 394, 275, 125, 0, 0, 0, 0, 0, 1436, 0, 3, 32, 541, 2093, 111, 1, 2014, 0, 0, 0, 0, 0, 0, 0, 822, 6, 184, 526, 7, 0, 126, 2, 4490, 0, 0, 0, 0, 0, 0, 0, 3699, 155, 17, 4, 1, 1285, 1, 957, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6182, 6068, 6261, 6020, 6108, 6231, 6163, 6119] avg=6144.0000 max=6261 max_rank=2 min=6020 min_rank=3 imbalance_ratio=1.019043 max_load_reduction=42891 max_load_reduction_pct=0.872620 moved_to_echo=24621 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 135, 81, 154, 2, 67, 384, 2023, 2453, 968, 0, 0, 0, 0, 0, 0, 23, 1684, 297, 1577, 439, 346, 0, 0, 909, 828, 0, 0, 0, 0, 0, 0, 4358, 1064, 0, 66, 1, 0, 0, 632, 0, 0, 0, 0, 0, 0, 0, 0, 177, 708, 1883, 2549, 3, 192, 266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 531, 597, 187, 3728, 212, 111, 82, 393, 268, 122, 0, 0, 0, 0, 0, 1374, 0, 0, 17, 443, 2118, 97, 1, 2045, 0, 0, 0, 0, 0, 0, 0, 937, 7, 157, 454, 4, 0, 126, 0, 4477, 0, 0, 0, 0, 0, 0, 0, 3688, 199, 20, 7, 0, 1472, 1, 960, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6267, 6103, 6121, 5778, 6279, 6095, 6162, 6347] avg=6144.0000 max=6347 max_rank=7 min=5778 min_rank=3 imbalance_ratio=1.033040 max_load_reduction=42805 max_load_reduction_pct=0.870870 moved_to_echo=24883 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 131, 60, 136, 1, 105, 411, 1861, 2507, 909, 0, 0, 0, 0, 0, 0, 21, 1720, 229, 1746, 456, 348, 0, 0, 928, 813, 0, 0, 0, 0, 0, 0, 4416, 855, 0, 54, 0, 0, 0, 666, 0, 0, 0, 0, 0, 0, 0, 0, 147, 600, 1849, 3219, 1, 173, 271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 613, 556, 171, 3704, 198, 98, 77, 387, 263, 123, 0, 0, 0, 0, 0, 1524, 0, 0, 36, 450, 2020, 109, 0, 2091, 0, 0, 0, 0, 0, 0, 0, 861, 4, 143, 467, 10, 0, 167, 0, 4209, 0, 0, 0, 0, 0, 0, 0, 3628, 156, 10, 17, 0, 1403, 0, 982, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6121, 6261, 5991, 6260, 6232, 6230, 5861, 6196] avg=6144.0000 max=6261 max_rank=1 min=5861 min_rank=6 imbalance_ratio=1.019043 max_load_reduction=42891 max_load_reduction_pct=0.872620 moved_to_echo=24519 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 201, 51, 146, 0, 131, 365, 1927, 2369, 942, 0, 0, 0, 0, 0, 0, 24, 1599, 221, 1724, 465, 316, 0, 0, 875, 877, 0, 0, 0, 0, 0, 0, 4523, 948, 0, 61, 2, 0, 0, 679, 0, 0, 0, 0, 0, 0, 0, 0, 172, 621, 1983, 3070, 2, 407, 277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 482, 393, 172, 3690, 177, 86, 96, 381, 286, 131, 0, 0, 0, 0, 0, 1567, 0, 1, 17, 416, 1938, 106, 1, 1972, 0, 0, 0, 0, 0, 0, 0, 895, 7, 160, 472, 3, 0, 202, 0, 4357, 0, 0, 0, 0, 0, 0, 0, 3536, 143, 7, 14, 0, 1490, 0, 931, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6132, 6101, 6213, 6532, 5939, 6018, 6096, 6121] avg=6144.0000 max=6532 max_rank=3 min=5939 min_rank=4 imbalance_ratio=1.063151 max_load_reduction=42620 max_load_reduction_pct=0.867106 moved_to_echo=24174 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1408, 4992, 15232, 23552, 128, 2176, 2176, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 1280, 640, 1408, 128, 896, 3072, 15104, 19584, 7808, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[29184, 1408, 128, 128, 128, 11008, 128, 7680, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[384, 4224, 4096, 1536, 29952, 1664, 768, 768, 3200, 2176, 1024, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[11904, 0, 128, 256, 3712, 16512, 896, 128, 16256, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 13568, 1920, 13568, 3584, 2816, 128, 128, 7296, 6784, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[35840, 7296, 0, 512, 128, 128, 0, 5760, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[7168, 128, 1408, 3584, 128, 0, 1408, 128, 35840, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 12, 0, 2107, 1250, 863, 132, 388, 79, 319, 0, 255, 0, 6761, 2, 3541, 1726, 150, 0, 2170, 305, 0, 43, 776, 792, 719, 561, 1165, 745, 465, 0, 0, 0, 82, 26, 1652, 21, 18, 1, 934, 5662, 27, 862, 594, 0, 0, 1725, 0, 0, 14, 498, 30, 1088, 0, 0, 2, 0, 652, 0, 2459, 6585, 3, 891] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 13, 0, 2121, 1157, 928, 116, 426, 86, 370, 0, 246, 0, 6538, 0, 3750, 1878, 103, 0, 1850, 266, 0, 165, 741, 1119, 676, 617, 1054, 601, 480, 0, 0, 8, 79, 20, 1817, 57, 13, 1, 804, 5313, 33, 779, 596, 0, 0, 1691, 0, 0, 7, 504, 40, 1511, 0, 0, 2, 0, 526, 0, 2778, 6502, 2, 768] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 11, 3, 2196, 857, 966, 68, 468, 78, 356, 0, 243, 0, 6845, 1, 3646, 2090, 32, 1, 1913, 290, 0, 35, 647, 819, 679, 624, 1305, 650, 448, 3, 0, 6, 66, 35, 1828, 22, 16, 3, 874, 5596, 14, 759, 525, 3, 2, 1781, 1, 3, 8, 495, 31, 1183, 0, 3, 7, 0, 590, 1, 2554, 6616, 37, 819] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 2, 8, 1, 2154, 1253, 942, 62, 361, 90, 413, 0, 237, 1, 6446, 2, 3640, 1765, 166, 0, 2076, 289, 0, 82, 640, 828, 690, 606, 933, 611, 443, 0, 0, 8, 83, 50, 1874, 35, 19, 0, 1009, 5522, 40, 820, 660, 0, 0, 1797, 1, 1, 9, 477, 31, 1246, 3, 0, 12, 2, 575, 0, 2743, 6600, 2, 792] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4, 2, 2019, 902, 922, 119, 390, 87, 304, 1, 248, 0, 6739, 3, 3725, 1783, 96, 0, 1994, 277, 0, 114, 707, 981, 729, 571, 1013, 662, 451, 2, 0, 4, 89, 33, 2152, 21, 9, 0, 1144, 5706, 14, 814, 527, 0, 1, 1609, 1, 0, 0, 530, 16, 1578, 0, 1, 2, 2, 519, 2, 2735, 6030, 5, 763] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 7, 0, 2424, 1209, 894, 60, 402, 80, 434, 0, 248, 0, 6720, 1, 3460, 1694, 180, 1, 2035, 283, 0, 88, 697, 993, 662, 597, 853, 530, 473, 4, 0, 3, 79, 38, 2010, 68, 8, 0, 997, 5540, 30, 800, 330, 0, 0, 1955, 0, 0, 1, 482, 21, 1382, 0, 0, 3, 0, 490, 2, 2789, 6309, 2, 784] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 10, 0, 2180, 1161, 967, 96, 456, 75, 418, 0, 252, 0, 6441, 0, 3426, 1799, 103, 0, 2174, 318, 0, 94, 765, 919, 656, 552, 1045, 704, 461, 0, 0, 5, 74, 26, 2038, 35, 10, 0, 865, 5444, 34, 713, 557, 0, 0, 1760, 0, 0, 3, 517, 38, 1423, 0, 0, 2, 0, 595, 0, 2722, 6390, 4, 825] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 17, 0, 2166, 1259, 798, 122, 374, 87, 405, 0, 256, 0, 6606, 1, 3469, 1611, 166, 1, 1846, 304, 0, 87, 763, 983, 677, 561, 1228, 577, 451, 0, 0, 7, 79, 33, 1887, 41, 14, 1, 991, 5551, 12, 818, 504, 0, 0, 1845, 0, 0, 21, 477, 32, 1395, 0, 0, 1, 0, 699, 0, 2735, 6389, 7, 798] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 12, 0, 2107, 1250, 863, 132, 1254, 565, 0, 0, 0, 0, 0, 0, 388, 79, 319, 0, 255, 0, 5121, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1819, 1726, 150, 0, 2170, 305, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 776, 792, 719, 561, 1165, 745, 465, 0, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 26, 1652, 21, 18, 1, 3554, 468, 118, 0, 0, 0, 0, 0, 934, 2108, 27, 862, 594, 0, 0, 1725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 498, 30, 1088, 0, 0, 4360, 0, 0, 0, 0, 0, 0, 0, 2, 0, 652, 0, 2459, 2107, 3, 891, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6183, 6164, 6213, 6298, 5940, 6250, 5990, 6114] avg=6144.0000 max=6298 max_rank=3 min=5940 min_rank=4 imbalance_ratio=1.025065 max_load_reduction=42854 max_load_reduction_pct=0.871867 moved_to_echo=24294 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 13, 0, 2121, 1157, 928, 116, 1328, 546, 0, 0, 0, 0, 0, 0, 426, 86, 370, 0, 246, 0, 4952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1926, 1878, 103, 0, 1850, 266, 0, 165, 0, 0, 0, 0, 0, 0, 0, 0, 741, 1119, 676, 617, 1054, 601, 480, 0, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 8, 79, 20, 1817, 57, 13, 1, 3335, 496, 116, 0, 0, 0, 0, 0, 804, 1978, 33, 779, 596, 0, 0, 1691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 504, 40, 1511, 0, 0, 4305, 0, 0, 0, 0, 0, 0, 0, 2, 0, 526, 0, 2778, 2081, 2, 768, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6209, 6080, 6188, 6328, 5942, 5881, 6367, 6157] avg=6144.0000 max=6367 max_rank=6 min=5881 min_rank=5 imbalance_ratio=1.036296 max_load_reduction=42785 max_load_reduction_pct=0.870463 moved_to_echo=24347 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 8, 1, 2154, 1253, 942, 62, 1289, 538, 0, 0, 0, 0, 0, 0, 361, 90, 413, 0, 237, 1, 4883, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1869, 1765, 166, 0, 2076, 289, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 640, 828, 690, 606, 933, 611, 443, 0, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 8, 83, 50, 1874, 35, 19, 0, 3466, 482, 118, 0, 0, 0, 0, 0, 1009, 2056, 40, 820, 660, 0, 0, 1797, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 9, 477, 31, 1246, 3, 0, 4370, 0, 0, 0, 0, 0, 0, 0, 12, 2, 575, 0, 2743, 2112, 2, 792, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6249, 5987, 6247, 5776, 6135, 6382, 6138, 6238] avg=6144.0000 max=6382 max_rank=5 min=5776 min_rank=3 imbalance_ratio=1.038737 max_load_reduction=42770 max_load_reduction_pct=0.870158 moved_to_echo=24893 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 11, 3, 2196, 857, 966, 68, 1291, 572, 0, 0, 0, 0, 0, 0, 468, 78, 356, 0, 243, 0, 5184, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1873, 2090, 32, 1, 1913, 290, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 647, 819, 679, 624, 1305, 650, 448, 3, 1089, 0, 0, 0, 0, 0, 0, 0, 0, 6, 66, 35, 1828, 22, 16, 3, 3512, 482, 118, 0, 0, 0, 0, 0, 874, 2084, 14, 759, 525, 3, 2, 1781, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 8, 495, 31, 1183, 0, 3, 4380, 0, 0, 0, 0, 0, 0, 0, 7, 0, 590, 1, 2554, 2118, 37, 819, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5964, 6330, 6234, 6264, 6088, 6042, 6104, 6126] avg=6144.0000 max=6330 max_rank=1 min=5964 min_rank=0 imbalance_ratio=1.030273 max_load_reduction=42822 max_load_reduction_pct=0.871216 moved_to_echo=24360 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 2, 2019, 902, 922, 119, 1319, 563, 0, 0, 0, 0, 0, 0, 390, 87, 304, 1, 248, 0, 5104, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1913, 1783, 96, 0, 1994, 277, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 707, 981, 729, 571, 1013, 662, 451, 2, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 4, 89, 33, 2152, 21, 9, 0, 3581, 493, 108, 0, 0, 0, 0, 0, 1144, 2125, 14, 814, 527, 0, 1, 1609, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 530, 16, 1578, 0, 1, 3992, 0, 0, 0, 0, 0, 0, 0, 2, 2, 519, 2, 2735, 1930, 5, 763, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5850, 6137, 6177, 6188, 6490, 6234, 6118, 5958] avg=6144.0000 max=6490 max_rank=4 min=5850 min_rank=0 imbalance_ratio=1.056315 max_load_reduction=42662 max_load_reduction_pct=0.867961 moved_to_echo=24800 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 7, 0, 2424, 1209, 894, 60, 1225, 561, 0, 0, 0, 0, 0, 0, 402, 80, 434, 0, 248, 0, 5090, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1777, 1694, 180, 1, 2035, 283, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 697, 993, 662, 597, 853, 530, 473, 4, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 3, 79, 38, 2010, 68, 8, 0, 3477, 458, 113, 0, 0, 0, 0, 0, 997, 2063, 30, 800, 330, 0, 0, 1955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 482, 21, 1382, 0, 0, 4177, 0, 0, 0, 0, 0, 0, 0, 3, 0, 490, 2, 2789, 2019, 2, 784, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6380, 6255, 6058, 5878, 6254, 6175, 6063, 6089] avg=6144.0000 max=6380 max_rank=0 min=5878 min_rank=3 imbalance_ratio=1.038411 max_load_reduction=42772 max_load_reduction_pct=0.870199 moved_to_echo=24581 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 10, 0, 2180, 1161, 967, 96, 1213, 538, 0, 0, 0, 0, 0, 0, 456, 75, 418, 0, 252, 0, 4879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1760, 1799, 103, 0, 2174, 318, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 765, 919, 656, 552, 1045, 704, 461, 0, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 5, 74, 26, 2038, 35, 10, 0, 3417, 453, 114, 0, 0, 0, 0, 0, 865, 2027, 34, 713, 557, 0, 0, 1760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 517, 38, 1423, 0, 0, 4231, 0, 0, 0, 0, 0, 0, 0, 2, 0, 595, 0, 2722, 2045, 4, 825, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6165, 6080, 6248, 6126, 6172, 5956, 6212, 6193] avg=6144.0000 max=6248 max_rank=2 min=5956 min_rank=5 imbalance_ratio=1.016927 max_load_reduction=42904 max_load_reduction_pct=0.872884 moved_to_echo=24533 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 17, 0, 2166, 1259, 798, 122, 1234, 556, 0, 0, 0, 0, 0, 0, 374, 87, 405, 0, 256, 0, 4996, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1772, 1611, 166, 1, 1846, 304, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 763, 983, 677, 561, 1228, 577, 451, 0, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 7, 79, 33, 1887, 41, 14, 1, 3489, 463, 117, 0, 0, 0, 0, 0, 991, 2062, 12, 818, 504, 0, 0, 1845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 477, 32, 1395, 0, 0, 4235, 0, 0, 0, 0, 0, 0, 0, 1, 0, 699, 0, 2735, 2037, 7, 798, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6152, 6119, 5787, 6294, 6131, 6232, 6160, 6277] avg=6144.0000 max=6294 max_rank=3 min=5787 min_rank=2 imbalance_ratio=1.024414 max_load_reduction=42858 max_load_reduction_pct=0.871948 moved_to_echo=24800 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 17408, 9088, 7296, 896, 10240, 4480, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7680, 16512, 256, 6400, 4352, 128, 128, 14208, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[14720, 14464, 1024, 128, 16128, 2432, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5760, 7552, 5504, 4736, 8704, 5120, 3712, 128, 8448, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 4736, 128, 21632, 16512, 128, 6528, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[0, 128, 640, 384, 15360, 384, 128, 128, 27904, 3840, 1024, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 128, 128, 4096, 256, 10880, 128, 128, 34176, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3328, 768, 3072, 128, 2048, 128, 40320, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 474, 3551, 1, 0, 26, 3, 25, 0, 1, 0, 86, 63, 0, 0, 0, 0, 3359, 0, 4032, 0, 191, 5969, 4, 2, 0, 386, 12, 0, 2082, 1, 0, 1, 0, 0, 1643, 1594, 3, 3634, 251, 0, 1560, 1, 7598, 55, 2, 130, 0, 1, 765, 7030, 1, 4, 397, 3547, 473, 0, 0, 0, 5, 187, 0, 0, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 527, 3655, 4, 0, 5, 2, 61, 1, 2, 1, 58, 70, 0, 3, 0, 0, 3484, 0, 3564, 0, 260, 5590, 7, 13, 0, 461, 33, 0, 2158, 0, 3, 1, 0, 3, 1596, 1566, 0, 3824, 235, 3, 1395, 0, 7484, 49, 0, 133, 1, 1, 1213, 6987, 9, 0, 401, 3664, 456, 0, 3, 3, 0, 161, 0, 0, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 517, 3792, 0, 0, 0, 0, 32, 0, 0, 0, 79, 75, 0, 0, 0, 0, 3542, 0, 3780, 1, 257, 5873, 5, 10, 0, 555, 9, 0, 1979, 0, 0, 0, 0, 0, 1673, 1391, 0, 3450, 293, 0, 1416, 0, 7529, 68, 0, 116, 0, 0, 1216, 7100, 0, 0, 423, 3341, 464, 0, 0, 0, 0, 166, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 494, 3417, 0, 0, 1, 0, 36, 0, 0, 0, 54, 73, 0, 0, 0, 0, 3589, 0, 3655, 0, 253, 5803, 1, 9, 0, 429, 33, 0, 2148, 0, 0, 0, 0, 0, 1638, 1708, 0, 3644, 269, 0, 1439, 0, 7581, 52, 0, 140, 0, 0, 1112, 7018, 0, 0, 359, 3582, 455, 0, 0, 0, 0, 160, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 566, 3864, 0, 0, 3, 0, 35, 0, 0, 0, 94, 60, 0, 0, 0, 0, 3566, 0, 3241, 0, 216, 5993, 1, 7, 0, 373, 9, 0, 2113, 0, 0, 0, 0, 0, 1620, 1713, 0, 3345, 322, 0, 1367, 0, 7543, 83, 0, 78, 0, 0, 1163, 6992, 0, 0, 509, 3625, 475, 0, 0, 0, 1, 174, 0, 1, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 350, 3753, 0, 0, 3, 0, 54, 0, 1, 0, 61, 72, 0, 0, 0, 1, 3381, 0, 3594, 3, 263, 5672, 0, 14, 0, 394, 25, 0, 2044, 0, 0, 0, 0, 0, 1738, 1679, 1, 3685, 217, 0, 1475, 0, 7493, 58, 0, 150, 0, 0, 1300, 7141, 0, 0, 475, 3440, 432, 0, 0, 0, 1, 182, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 531, 4123, 1, 0, 0, 0, 36, 0, 1, 0, 92, 125, 0, 0, 0, 0, 3469, 0, 3931, 2, 201, 5709, 3, 7, 0, 359, 8, 0, 2002, 0, 0, 1, 0, 2, 1631, 1473, 0, 3625, 329, 0, 1398, 2, 7477, 79, 0, 136, 0, 0, 930, 7164, 0, 0, 374, 3313, 446, 0, 0, 1, 2, 169, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 458, 3697, 0, 0, 0, 0, 53, 0, 1, 0, 61, 80, 0, 0, 0, 0, 3096, 0, 3871, 0, 250, 5800, 5, 7, 0, 497, 8, 0, 1904, 0, 0, 0, 0, 0, 1794, 1650, 0, 3546, 328, 0, 1615, 0, 7583, 60, 0, 144, 0, 0, 912, 7107, 0, 0, 468, 3533, 461, 0, 0, 0, 1, 162, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 474, 3551, 1, 0, 26, 3, 25, 1006, 956, 0, 0, 0, 0, 0, 0, 0, 1, 0, 86, 63, 0, 0, 0, 5960, 0, 0, 0, 0, 0, 0, 0, 0, 3359, 0, 2684, 0, 191, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 386, 12, 0, 2082, 1, 0, 3067, 392, 235, 0, 0, 0, 0, 0, 1, 0, 0, 1643, 1594, 3, 2628, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1560, 1, 4531, 55, 2, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 765, 674, 1, 4, 397, 3547, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 187, 0, 0, 2, 5734, 396, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6042, 6110, 6238, 6177, 6120, 6279, 5862, 6324] avg=6144.0000 max=6324 max_rank=7 min=5862 min_rank=6 imbalance_ratio=1.029297 max_load_reduction=42828 max_load_reduction_pct=0.871338 moved_to_echo=24585 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 527, 3655, 4, 0, 5, 2, 61, 1059, 845, 0, 0, 0, 0, 0, 0, 1, 2, 1, 58, 70, 0, 3, 0, 5924, 0, 0, 0, 0, 0, 0, 0, 0, 3484, 0, 2373, 0, 260, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 461, 33, 0, 2158, 0, 3, 3021, 346, 220, 0, 0, 0, 0, 0, 1, 0, 3, 1596, 1566, 0, 2765, 235, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1395, 0, 4463, 49, 0, 133, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1213, 669, 9, 0, 401, 3664, 456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 161, 0, 0, 2, 5370, 394, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6158, 6059, 6124, 6255, 6166, 6044, 6413, 5933] avg=6144.0000 max=6413 max_rank=6 min=5933 min_rank=7 imbalance_ratio=1.043783 max_load_reduction=42739 max_load_reduction_pct=0.869527 moved_to_echo=24556 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 566, 3864, 0, 0, 3, 0, 35, 926, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 60, 0, 0, 0, 5928, 0, 0, 0, 0, 0, 0, 0, 0, 3566, 0, 2158, 0, 216, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 373, 9, 0, 2113, 0, 0, 3045, 315, 236, 0, 0, 0, 0, 0, 0, 0, 0, 1620, 1713, 0, 2419, 322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1367, 0, 4498, 83, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1163, 670, 0, 0, 509, 3625, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 174, 0, 1, 0, 5757, 394, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6162, 6082, 5941, 6098, 6074, 6026, 6442, 6327] avg=6144.0000 max=6442 max_rank=6 min=5941 min_rank=2 imbalance_ratio=1.048503 max_load_reduction=42710 max_load_reduction_pct=0.868937 moved_to_echo=24869 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 494, 3417, 0, 0, 1, 0, 36, 1009, 867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 73, 0, 0, 0, 5950, 0, 0, 0, 0, 0, 0, 0, 0, 3589, 0, 2433, 0, 253, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 429, 33, 0, 2148, 0, 0, 3060, 355, 229, 0, 0, 0, 0, 0, 0, 0, 0, 1638, 1708, 0, 2635, 269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1439, 0, 4521, 52, 0, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1112, 673, 0, 0, 359, 3582, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 0, 0, 5574, 395, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5824, 6077, 6276, 6263, 6250, 6152, 6181, 6129] avg=6144.0000 max=6276 max_rank=2 min=5824 min_rank=0 imbalance_ratio=1.021484 max_load_reduction=42876 max_load_reduction_pct=0.872314 moved_to_echo=24712 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 531, 4123, 1, 0, 0, 0, 36, 1004, 932, 0, 0, 0, 0, 0, 0, 0, 1, 0, 92, 125, 0, 0, 0, 6074, 0, 0, 0, 0, 0, 0, 0, 0, 3469, 0, 2617, 2, 201, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 359, 8, 0, 2002, 0, 0, 3018, 382, 225, 0, 0, 0, 0, 0, 1, 0, 2, 1631, 1473, 0, 2621, 329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1398, 2, 4459, 79, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 930, 686, 0, 0, 374, 3313, 446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 169, 0, 0, 0, 5484, 404, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6627, 6292, 6292, 6001, 6057, 6074, 5749, 6060] avg=6144.0000 max=6627 max_rank=0 min=5749 min_rank=6 imbalance_ratio=1.078613 max_load_reduction=42525 max_load_reduction_pct=0.865173 moved_to_echo=23940 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 350, 3753, 0, 0, 3, 0, 54, 1020, 852, 0, 0, 0, 0, 0, 0, 0, 1, 0, 61, 72, 0, 0, 0, 6054, 0, 0, 0, 0, 0, 0, 0, 1, 3381, 0, 2393, 3, 263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 394, 25, 0, 2044, 0, 0, 3025, 349, 224, 0, 0, 0, 0, 0, 0, 0, 0, 1738, 1679, 1, 2665, 217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1475, 0, 4468, 58, 0, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1300, 685, 0, 0, 475, 3440, 432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 182, 0, 0, 0, 5448, 402, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6032, 6188, 6041, 6075, 6300, 6151, 6332, 6033] avg=6144.0000 max=6332 max_rank=6 min=6032 min_rank=0 imbalance_ratio=1.030599 max_load_reduction=42820 max_load_reduction_pct=0.871175 moved_to_echo=24816 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 517, 3792, 0, 0, 0, 0, 32, 955, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 75, 0, 0, 0, 6019, 0, 0, 0, 0, 0, 0, 0, 0, 3542, 0, 2517, 1, 257, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 555, 9, 0, 1979, 0, 0, 3039, 367, 231, 0, 0, 0, 0, 0, 0, 0, 0, 1673, 1391, 0, 2495, 293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1416, 0, 4490, 68, 0, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1216, 681, 0, 0, 423, 3341, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 166, 0, 0, 0, 5642, 400, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6192, 6173, 6322, 6190, 5852, 6090, 6125, 6208] avg=6144.0000 max=6322 max_rank=2 min=5852 min_rank=4 imbalance_ratio=1.028971 max_load_reduction=42830 max_load_reduction_pct=0.871379 moved_to_echo=24275 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 458, 3697, 0, 0, 0, 0, 53, 985, 922, 0, 0, 0, 0, 0, 0, 0, 1, 0, 61, 80, 0, 0, 0, 6029, 0, 0, 0, 0, 0, 0, 0, 0, 3096, 0, 2567, 0, 250, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 497, 8, 0, 1904, 0, 0, 3066, 382, 229, 0, 0, 0, 0, 0, 0, 0, 0, 1794, 1650, 0, 2561, 328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1615, 0, 4517, 60, 0, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 912, 674, 0, 0, 468, 3533, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 162, 0, 0, 0, 5571, 404, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6115, 6171, 5918, 6093, 6333, 6336, 6048, 6138] avg=6144.0000 max=6336 max_rank=5 min=5918 min_rank=2 imbalance_ratio=1.031250 max_load_reduction=42816 max_load_reduction_pct=0.871094 moved_to_echo=24855 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 8704, 5504, 128, 128, 3456, 28160, 3712, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 0, 3456, 256, 0, 16512, 128, 128, 24448, 2944, 1920, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 3968, 29952, 128, 0, 128, 128, 384, 8064, 7040, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 0, 128, 13440, 12800, 128, 20864, 2304, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 11776, 128, 35968, 512, 128, 1152, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 27520, 0, 19840, 128, 1920, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 1408, 0, 128, 128, 44672, 3200, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 128, 640, 640, 0, 128, 0, 48000, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[88, 7270, 87, 3, 5261, 2415, 5179, 2, 0, 2, 0, 924, 122, 0, 0, 2855, 322, 1478, 765, 9, 0, 416, 5575, 0, 2, 1689, 322, 0, 1, 0, 383, 0, 552, 1, 2, 0, 1, 3, 261, 577, 6241, 0, 0, 2, 905, 1, 85, 42, 0, 3, 1682, 86, 0, 0, 1, 799, 0, 0, 86, 0, 0, 2459, 2, 191] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[78, 7232, 76, 4, 5409, 2039, 5303, 0, 0, 0, 0, 863, 194, 0, 0, 2750, 337, 1384, 828, 39, 0, 421, 5773, 0, 0, 1492, 337, 0, 0, 0, 336, 0, 996, 0, 0, 0, 1, 0, 225, 813, 5972, 0, 0, 0, 986, 0, 76, 26, 0, 0, 1860, 106, 0, 0, 0, 778, 0, 0, 76, 0, 3, 2149, 0, 190] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[71, 7325, 70, 0, 5315, 2197, 4991, 45, 0, 0, 0, 695, 246, 0, 2, 2722, 335, 1331, 761, 9, 1, 488, 5652, 0, 0, 1420, 337, 0, 0, 0, 380, 0, 913, 1, 1, 0, 0, 0, 265, 641, 6333, 0, 0, 0, 889, 0, 71, 31, 0, 0, 2241, 64, 0, 0, 0, 673, 0, 0, 71, 0, 4, 2386, 0, 175] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[84, 7299, 81, 0, 5065, 2179, 5349, 0, 0, 0, 0, 824, 182, 0, 0, 2617, 370, 1445, 866, 70, 0, 307, 5673, 0, 0, 1707, 372, 0, 0, 0, 415, 0, 645, 0, 0, 0, 0, 0, 325, 543, 6193, 0, 0, 0, 960, 0, 80, 26, 0, 0, 2001, 52, 0, 0, 0, 725, 0, 0, 81, 0, 4, 2398, 0, 214] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[84, 7337, 87, 1, 5036, 1948, 5247, 6, 0, 0, 3, 826, 149, 0, 0, 2721, 351, 1409, 711, 78, 2, 367, 5603, 0, 0, 1799, 350, 0, 0, 1, 413, 2, 762, 18, 0, 0, 1, 0, 298, 696, 6108, 0, 0, 0, 950, 5, 89, 33, 0, 1, 2168, 56, 0, 0, 1, 699, 0, 0, 91, 0, 8, 2447, 4, 186] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[80, 7239, 78, 0, 5640, 1911, 5610, 31, 0, 0, 0, 723, 121, 0, 0, 2871, 307, 1272, 823, 57, 2, 364, 5501, 0, 0, 1776, 307, 0, 0, 0, 391, 0, 795, 0, 0, 0, 0, 0, 306, 696, 6195, 0, 0, 0, 784, 0, 78, 25, 0, 0, 1795, 63, 0, 0, 0, 731, 0, 0, 78, 0, 4, 2305, 0, 193] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[70, 7356, 77, 3, 5589, 1772, 5145, 0, 0, 0, 1, 825, 117, 0, 0, 2994, 351, 1441, 778, 6, 2, 370, 5789, 0, 2, 1734, 358, 2, 1, 0, 349, 0, 697, 0, 0, 3, 2, 1, 236, 486, 6446, 0, 2, 0, 926, 9, 71, 37, 0, 0, 1771, 57, 0, 0, 1, 700, 0, 0, 75, 0, 2, 2319, 1, 178] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[72, 7248, 72, 1, 5404, 2112, 5286, 0, 0, 0, 0, 706, 154, 0, 0, 2642, 349, 1423, 759, 24, 0, 390, 5656, 0, 0, 1784, 349, 0, 0, 0, 420, 0, 820, 0, 0, 0, 0, 0, 280, 640, 6138, 0, 0, 0, 1068, 0, 72, 32, 0, 0, 1864, 75, 0, 0, 0, 744, 0, 0, 72, 0, 2, 2292, 0, 202] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[78, 0, 76, 4, 0, 2039, 3933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 863, 194, 0, 0, 2750, 1318, 1063, 0, 0, 0, 0, 0, 0, 337, 1384, 828, 39, 0, 421, 3260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1492, 337, 0, 0, 0, 336, 0, 2810, 919, 0, 0, 0, 0, 0, 0, 996, 0, 0, 0, 1, 0, 225, 813, 4422, 0, 0, 0, 0, 0, 0, 0, 4909, 0, 0, 0, 986, 0, 76, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1860, 106, 0, 0, 0, 778, 2513, 919, 52, 0, 0, 0, 0, 0, 0, 0, 76, 0, 3, 2149, 0, 190, 3571, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6130, 6188, 6269, 5894, 6457, 5997, 6228, 5989] avg=6144.0000 max=6457 max_rank=4 min=5894 min_rank=3 imbalance_ratio=1.050944 max_load_reduction=42695 max_load_reduction_pct=0.868632 moved_to_echo=24671 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[70, 0, 77, 3, 0, 1772, 3817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 825, 117, 0, 0, 2994, 1278, 1147, 0, 0, 0, 0, 0, 0, 351, 1441, 778, 6, 2, 370, 3269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1734, 358, 2, 1, 0, 349, 0, 2858, 950, 0, 0, 0, 0, 0, 0, 697, 0, 0, 3, 2, 1, 236, 486, 4498, 0, 0, 0, 0, 0, 0, 0, 5299, 0, 2, 0, 926, 9, 71, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1771, 57, 0, 0, 1, 700, 2520, 948, 50, 0, 0, 0, 0, 0, 0, 0, 75, 0, 2, 2319, 1, 178, 3691, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5739, 6362, 6217, 6254, 5923, 6344, 6047, 6266] avg=6144.0000 max=6362 max_rank=1 min=5739 min_rank=0 imbalance_ratio=1.035482 max_load_reduction=42790 max_load_reduction_pct=0.870565 moved_to_echo=24580 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[84, 0, 87, 1, 0, 1948, 3892, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 826, 149, 0, 0, 2721, 1304, 1087, 0, 0, 0, 0, 0, 0, 351, 1409, 711, 78, 2, 367, 3164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1799, 350, 0, 0, 1, 413, 2, 2851, 856, 0, 0, 0, 0, 0, 0, 762, 18, 0, 0, 1, 0, 298, 696, 4486, 0, 0, 0, 0, 0, 0, 0, 5021, 0, 0, 0, 950, 5, 89, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2168, 56, 0, 0, 1, 699, 2439, 854, 51, 0, 0, 0, 0, 0, 0, 0, 91, 0, 8, 2447, 4, 186, 3326, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6018, 6090, 6082, 6272, 6261, 6098, 6269, 6062] avg=6144.0000 max=6272 max_rank=3 min=6018 min_rank=0 imbalance_ratio=1.020833 max_load_reduction=42880 max_load_reduction_pct=0.872396 moved_to_echo=24690 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[88, 0, 87, 3, 0, 2415, 3841, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 924, 122, 0, 0, 2855, 1287, 1111, 0, 0, 0, 0, 0, 0, 322, 1478, 765, 9, 0, 416, 3148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1689, 322, 0, 1, 0, 383, 0, 2826, 896, 0, 0, 0, 0, 0, 0, 552, 1, 2, 0, 1, 3, 261, 577, 4444, 0, 0, 0, 0, 0, 0, 0, 5130, 0, 0, 2, 905, 1, 85, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1682, 86, 0, 0, 1, 799, 2427, 892, 51, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 2459, 2, 191, 3473, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6436, 6301, 6138, 6119, 5841, 6165, 5941, 6211] avg=6144.0000 max=6436 max_rank=0 min=5841 min_rank=4 imbalance_ratio=1.047526 max_load_reduction=42716 max_load_reduction_pct=0.869059 moved_to_echo=24158 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[84, 0, 81, 0, 0, 2179, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 824, 182, 0, 0, 2617, 1329, 1102, 0, 0, 0, 0, 0, 0, 370, 1445, 866, 70, 0, 307, 3204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1707, 372, 0, 0, 0, 415, 0, 2836, 861, 0, 0, 0, 0, 0, 0, 645, 0, 0, 0, 0, 0, 325, 543, 4463, 0, 0, 0, 0, 0, 0, 0, 5091, 0, 0, 0, 960, 0, 80, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2001, 52, 0, 0, 0, 725, 2469, 860, 52, 0, 0, 0, 0, 0, 0, 0, 81, 0, 4, 2398, 0, 214, 3344, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6312, 6054, 6262, 6191, 5976, 6157, 6159, 6041] avg=6144.0000 max=6312 max_rank=0 min=5976 min_rank=4 imbalance_ratio=1.027344 max_load_reduction=42840 max_load_reduction_pct=0.871582 moved_to_echo=24333 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[71, 0, 70, 0, 0, 2197, 3702, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 695, 246, 0, 2, 2722, 1240, 1127, 0, 0, 0, 0, 0, 0, 335, 1331, 761, 9, 1, 488, 3192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1420, 337, 0, 0, 0, 380, 0, 2847, 904, 0, 0, 0, 0, 0, 0, 913, 1, 1, 0, 0, 0, 265, 641, 4478, 0, 0, 0, 0, 0, 0, 0, 5206, 0, 0, 0, 889, 0, 71, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2241, 64, 0, 0, 0, 673, 2460, 902, 49, 0, 0, 0, 0, 0, 0, 0, 71, 0, 4, 2386, 0, 175, 3509, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6085, 6032, 6117, 5888, 6299, 6197, 6389, 6145] avg=6144.0000 max=6389 max_rank=6 min=5888 min_rank=3 imbalance_ratio=1.039876 max_load_reduction=42763 max_load_reduction_pct=0.870015 moved_to_echo=25030 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[80, 0, 78, 0, 0, 1911, 4154, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 723, 121, 0, 0, 2871, 1397, 1106, 0, 0, 0, 0, 0, 0, 307, 1272, 823, 57, 2, 364, 3103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1776, 307, 0, 0, 0, 391, 0, 2813, 960, 0, 0, 0, 0, 0, 0, 795, 0, 0, 0, 0, 0, 306, 696, 4426, 0, 0, 0, 0, 0, 0, 0, 5089, 0, 0, 0, 784, 0, 78, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1795, 63, 0, 0, 0, 731, 2398, 957, 59, 0, 0, 0, 0, 0, 0, 0, 78, 0, 4, 2305, 0, 193, 3723, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6254, 6218, 5928, 6247, 6223, 5976, 6003, 6303] avg=6144.0000 max=6303 max_rank=7 min=5928 min_rank=2 imbalance_ratio=1.025879 max_load_reduction=42849 max_load_reduction_pct=0.871765 moved_to_echo=24505 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[72, 0, 72, 1, 0, 2112, 3921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 706, 154, 0, 0, 2642, 1313, 1092, 0, 0, 0, 0, 0, 0, 349, 1423, 759, 24, 0, 390, 3194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1784, 349, 0, 0, 0, 420, 0, 2816, 918, 0, 0, 0, 0, 0, 0, 820, 0, 0, 0, 0, 0, 280, 640, 4432, 0, 0, 0, 0, 0, 0, 0, 5046, 0, 0, 0, 1068, 0, 72, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1864, 75, 0, 0, 0, 744, 2462, 919, 52, 0, 0, 0, 0, 0, 0, 0, 72, 0, 2, 2292, 0, 202, 3567, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6178, 5907, 6139, 6287, 6172, 6218, 6116, 6135] avg=6144.0000 max=6287 max_rank=3 min=5907 min_rank=1 imbalance_ratio=1.023275 max_load_reduction=42865 max_load_reduction_pct=0.872091 moved_to_echo=24641 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[0, 128, 128, 6400, 1408, 0, 128, 22272, 10496, 8960, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 128, 15488, 640, 0, 0, 128, 5888, 19712, 7296, 512, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=50176 observed_loss=-1024 tokens_per_expert=[128, 13440, 2816, 128, 128, 128, 3200, 128, 22784, 7296, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2816, 11264, 6400, 384, 128, 3200, 25600, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[640, 0, 640, 128, 0, 16640, 31232, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 640, 0, 128, 18816, 128, 1536, 28288, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[6272, 128, 128, 128, 128, 128, 2304, 5120, 35712, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[40832, 0, 128, 128, 7552, 128, 640, 256, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=50176 slack_slots=48128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[664, 0, 45, 0, 4, 0, 0, 449, 0, 1, 5692, 0, 2490, 7793, 0, 4709, 1, 4, 0, 571, 0, 3, 463, 187, 2, 52, 92, 4, 0, 82, 0, 1, 0, 0, 5412, 0, 1, 0, 3, 0, 5, 7158, 1, 0, 0, 135, 632, 430, 0, 0, 1, 49, 2597, 1302, 6, 3, 0, 0, 6, 0, 1, 3, 8094, 4] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[694, 0, 25, 0, 0, 2, 0, 226, 0, 0, 6256, 1, 2323, 7820, 1, 4455, 0, 0, 1, 481, 0, 1, 446, 148, 1, 63, 102, 0, 1, 71, 1, 0, 3, 0, 4697, 0, 3, 1, 0, 1, 2, 7556, 1, 0, 0, 123, 594, 445, 0, 4, 8, 57, 3164, 1259, 1, 0, 3, 2, 0, 1, 0, 0, 8108, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[714, 0, 22, 0, 0, 0, 0, 338, 0, 0, 6038, 0, 2386, 7836, 0, 4386, 0, 0, 0, 572, 0, 0, 438, 114, 0, 55, 117, 0, 0, 76, 0, 0, 0, 0, 4855, 0, 0, 0, 0, 0, 0, 7336, 0, 0, 0, 118, 663, 461, 0, 0, 0, 92, 3088, 1327, 4, 0, 0, 0, 0, 0, 0, 0, 8116, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[693, 0, 51, 0, 0, 0, 0, 460, 0, 0, 5621, 0, 2581, 7811, 0, 4676, 0, 0, 0, 620, 0, 0, 486, 129, 0, 67, 129, 0, 0, 81, 0, 0, 0, 0, 5345, 0, 0, 0, 0, 0, 0, 7187, 0, 0, 0, 128, 703, 346, 0, 1, 5, 53, 2730, 1136, 2, 0, 0, 0, 0, 0, 0, 0, 8111, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[679, 0, 40, 0, 0, 0, 2, 436, 0, 0, 5397, 0, 2542, 7863, 0, 4114, 0, 3, 1, 676, 0, 0, 444, 113, 0, 54, 193, 1, 1, 86, 0, 0, 0, 0, 5310, 0, 2, 2, 0, 0, 0, 7476, 0, 0, 0, 136, 665, 523, 0, 0, 0, 96, 2956, 1226, 6, 1, 2, 1, 1, 0, 1, 1, 8102, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[694, 0, 23, 0, 0, 0, 0, 416, 0, 0, 5646, 0, 2476, 7814, 0, 4120, 0, 0, 0, 643, 0, 0, 427, 159, 0, 71, 90, 0, 0, 70, 0, 0, 0, 0, 5375, 0, 0, 0, 0, 0, 1, 7528, 0, 0, 0, 130, 609, 346, 0, 0, 0, 61, 2801, 1529, 2, 0, 0, 0, 0, 0, 0, 0, 8121, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[677, 0, 92, 0, 0, 0, 0, 283, 0, 0, 6013, 0, 2570, 7884, 0, 4835, 0, 0, 0, 540, 0, 0, 403, 161, 0, 54, 135, 0, 0, 78, 0, 0, 0, 0, 5053, 0, 0, 0, 0, 0, 1, 7291, 0, 0, 0, 120, 587, 353, 0, 0, 1, 62, 2827, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 8113, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[719, 0, 16, 0, 0, 0, 0, 443, 0, 0, 5823, 0, 2589, 7831, 0, 4636, 0, 0, 0, 677, 0, 0, 432, 160, 0, 73, 23, 0, 0, 72, 0, 0, 0, 0, 4954, 0, 0, 0, 0, 0, 0, 7448, 0, 0, 0, 120, 558, 339, 0, 0, 0, 74, 2690, 1353, 2, 0, 0, 0, 0, 0, 0, 0, 8120, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[714, 0, 22, 0, 0, 0, 0, 338, 3214, 1938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2386, 0, 0, 3564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 572, 0, 0, 438, 114, 2824, 2227, 0, 0, 0, 0, 0, 0, 0, 55, 117, 0, 0, 76, 0, 0, 5898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4855, 0, 0, 0, 0, 0, 822, 174, 0, 0, 0, 0, 0, 0, 0, 4961, 0, 0, 0, 118, 663, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 3088, 1327, 4, 0, 1796, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6146, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6226, 5950, 6175, 6146, 5851, 6203, 6455, 6146] avg=6144.0000 max=6455 max_rank=6 min=5851 min_rank=4 imbalance_ratio=1.050618 max_load_reduction=42697 max_load_reduction_pct=0.868673 moved_to_echo=24655 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[694, 0, 25, 0, 0, 2, 0, 226, 3330, 1934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2323, 0, 1, 3620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 481, 0, 1, 446, 148, 2926, 2294, 0, 0, 0, 0, 0, 0, 1, 63, 102, 0, 1, 71, 1, 0, 5886, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4697, 0, 3, 1, 0, 1, 835, 174, 0, 0, 0, 0, 0, 0, 2, 5110, 1, 0, 0, 123, 594, 445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8, 57, 3164, 1259, 1, 0, 1794, 152, 0, 0, 0, 0, 0, 0, 3, 2, 0, 1, 0, 0, 6140, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6211, 5945, 6297, 6125, 5714, 6275, 6439, 6146] avg=6144.0000 max=6439 max_rank=6 min=5714 min_rank=4 imbalance_ratio=1.048014 max_load_reduction=42713 max_load_reduction_pct=0.868998 moved_to_echo=24574 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[664, 0, 45, 0, 4, 0, 0, 449, 3030, 1927, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2490, 0, 0, 3826, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 571, 0, 3, 463, 187, 2662, 2173, 0, 0, 0, 0, 0, 0, 2, 52, 92, 4, 0, 82, 0, 1, 5866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5412, 0, 1, 0, 3, 0, 883, 174, 0, 0, 0, 0, 0, 0, 5, 4841, 1, 0, 0, 135, 632, 430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 49, 2597, 1302, 6, 3, 1791, 144, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1, 3, 6129, 4, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6119, 6317, 6064, 6099, 6473, 6044, 5893, 6143] avg=6144.0000 max=6473 max_rank=4 min=5893 min_rank=6 imbalance_ratio=1.053548 max_load_reduction=42679 max_load_reduction_pct=0.868306 moved_to_echo=24553 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[693, 0, 51, 0, 0, 0, 0, 460, 2992, 1932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2581, 0, 0, 3799, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 620, 0, 0, 486, 129, 2629, 2182, 0, 0, 0, 0, 0, 0, 0, 67, 129, 0, 0, 81, 0, 0, 5879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5345, 0, 0, 0, 0, 0, 877, 174, 0, 0, 0, 0, 0, 0, 0, 4860, 0, 0, 0, 128, 703, 346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 53, 2730, 1136, 2, 0, 1795, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6142, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6128, 6380, 6046, 6156, 6396, 6037, 5867, 6142] avg=6144.0000 max=6396 max_rank=4 min=5867 min_rank=6 imbalance_ratio=1.041016 max_load_reduction=42756 max_load_reduction_pct=0.869873 moved_to_echo=24442 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[679, 0, 40, 0, 0, 0, 2, 436, 2874, 1946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2542, 0, 0, 3343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 676, 0, 0, 444, 113, 2523, 2270, 0, 0, 0, 0, 0, 0, 0, 54, 193, 1, 1, 86, 0, 0, 5917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5310, 0, 2, 2, 0, 0, 771, 174, 0, 0, 0, 0, 0, 0, 0, 5055, 0, 0, 0, 136, 665, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 2956, 1226, 6, 1, 1793, 151, 0, 0, 0, 0, 0, 0, 2, 1, 1, 0, 1, 1, 6135, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5977, 5885, 6030, 6252, 6259, 6379, 6229, 6141] avg=6144.0000 max=6379 max_rank=5 min=5885 min_rank=1 imbalance_ratio=1.038249 max_load_reduction=42773 max_load_reduction_pct=0.870219 moved_to_echo=25008 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[694, 0, 23, 0, 0, 0, 0, 416, 3006, 1933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2476, 0, 0, 3348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 643, 0, 0, 427, 159, 2640, 2285, 0, 0, 0, 0, 0, 0, 0, 71, 90, 0, 0, 70, 0, 0, 5881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5375, 0, 0, 0, 0, 0, 772, 174, 0, 0, 0, 0, 0, 0, 1, 5091, 0, 0, 0, 130, 609, 346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 2801, 1529, 2, 0, 1797, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6150, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6072, 5824, 6154, 6112, 6321, 6177, 6342, 6150] avg=6144.0000 max=6342 max_rank=6 min=5824 min_rank=1 imbalance_ratio=1.032227 max_load_reduction=42810 max_load_reduction_pct=0.870972 moved_to_echo=24990 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[677, 0, 92, 0, 0, 0, 0, 283, 3202, 1951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2570, 0, 0, 3925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 0, 0, 403, 161, 2811, 2217, 0, 0, 0, 0, 0, 0, 0, 54, 135, 0, 0, 78, 0, 0, 5933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5053, 0, 0, 0, 0, 0, 910, 178, 0, 0, 0, 0, 0, 0, 1, 4922, 0, 0, 0, 120, 587, 353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 62, 2827, 1019, 0, 0, 1800, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6135, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6205, 6495, 6132, 6200, 6141, 5983, 5861, 6135] avg=6144.0000 max=6495 max_rank=1 min=5861 min_rank=6 imbalance_ratio=1.057129 max_load_reduction=42657 max_load_reduction_pct=0.867859 moved_to_echo=24120 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[719, 0, 16, 0, 0, 0, 0, 443, 3099, 1937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2589, 0, 0, 3767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 677, 0, 0, 432, 160, 2724, 2261, 0, 0, 0, 0, 0, 0, 0, 73, 23, 0, 0, 72, 0, 0, 5894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4954, 0, 0, 0, 0, 0, 869, 174, 0, 0, 0, 0, 0, 0, 0, 5037, 0, 0, 0, 120, 558, 339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 2690, 1353, 2, 0, 1797, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6149, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6214, 6356, 6254, 6062, 5997, 6054, 6066, 6149] avg=6144.0000 max=6356 max_rank=1 min=5997 min_rank=4 imbalance_ratio=1.034505 max_load_reduction=42796 max_load_reduction_pct=0.870687 moved_to_echo=24266 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[5632, 0, 384, 0, 128, 128, 128, 3072, 24832, 15616, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[0, 128, 128, 640, 22912, 10240, 128, 128, 14464, 1280, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 0, 128, 19968, 0, 128, 29312, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 39936, 128, 0, 0, 1024, 5120, 3328, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 0, 41088, 0, 128, 128, 128, 128, 6784, 1408, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 128, 4864, 0, 128, 3584, 1280, 21760, 17920, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 512, 896, 128, 128, 640, 128, 128, 47232, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 128, 49152, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[245, 0, 0, 0, 245, 0, 0, 2, 0, 814, 1283, 2005, 0, 0, 288, 4, 1947, 1723, 0, 0, 7729, 0, 0, 1141, 0, 1, 0, 7328, 0, 383, 118, 1, 0, 0, 2, 0, 1319, 577, 282, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3, 444, 7393, 6409, 3, 0, 0, 0, 0, 0, 0, 7461, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[243, 6, 2, 0, 243, 0, 0, 2, 1, 973, 1144, 2315, 2, 5, 349, 2, 1844, 1546, 3, 0, 7736, 2, 0, 1213, 0, 0, 0, 7445, 1, 371, 77, 2, 1, 1, 0, 0, 1173, 624, 271, 0, 2, 3, 0, 0, 0, 0, 0, 2, 1, 3, 0, 2, 458, 7286, 6370, 0, 0, 0, 0, 0, 0, 1, 7427, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[237, 0, 0, 4, 236, 1, 1, 2, 0, 765, 1050, 2356, 0, 0, 321, 8, 2016, 1433, 4, 0, 7732, 0, 0, 1356, 0, 0, 1, 7397, 4, 367, 62, 0, 1, 0, 3, 5, 1206, 493, 356, 0, 3, 0, 0, 1, 2, 0, 0, 0, 2, 6, 0, 64, 495, 7278, 6468, 0, 2, 4, 3, 1, 0, 3, 7402, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[255, 0, 0, 0, 255, 0, 0, 2, 0, 706, 1259, 1905, 0, 0, 339, 7, 2179, 1486, 0, 0, 7669, 0, 0, 1406, 0, 0, 0, 7395, 0, 355, 129, 0, 0, 0, 0, 0, 1234, 659, 378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 500, 7226, 6452, 0, 0, 0, 0, 0, 0, 0, 7352, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[248, 1, 0, 0, 248, 0, 0, 2, 0, 867, 1232, 2028, 0, 0, 143, 3, 2006, 1619, 0, 0, 7792, 0, 0, 1304, 0, 0, 0, 7455, 0, 350, 61, 0, 0, 0, 0, 0, 1168, 565, 260, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4, 461, 7514, 6389, 1, 0, 0, 0, 0, 0, 0, 7429, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[255, 1, 0, 0, 255, 0, 0, 0, 0, 790, 1161, 2161, 0, 0, 298, 0, 1838, 1478, 0, 0, 7719, 0, 0, 1273, 0, 2, 0, 7336, 0, 323, 128, 0, 0, 0, 1, 0, 1265, 599, 290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 451, 7389, 6552, 1, 0, 0, 0, 0, 0, 0, 7433, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[248, 1, 3, 0, 248, 0, 0, 1, 0, 672, 1376, 2157, 1, 1, 109, 7, 2101, 1547, 0, 0, 7612, 0, 0, 1276, 0, 0, 0, 7341, 1, 337, 121, 0, 1, 1, 0, 0, 1296, 568, 284, 0, 0, 1, 1, 0, 0, 0, 0, 2, 1, 2, 0, 1, 519, 7465, 6509, 0, 0, 0, 0, 1, 0, 0, 7338, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[252, 0, 0, 0, 250, 0, 0, 1, 0, 782, 1318, 2224, 0, 0, 314, 5, 2181, 1389, 0, 0, 7668, 0, 0, 1124, 0, 3, 0, 7331, 0, 367, 102, 0, 0, 0, 0, 0, 1146, 589, 322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 400, 7397, 6410, 1, 0, 0, 0, 0, 0, 0, 7470, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[243, 6, 2, 0, 243, 0, 0, 2, 4434, 1213, 0, 0, 0, 0, 0, 0, 1, 973, 1144, 2315, 2, 5, 349, 2, 890, 802, 0, 0, 0, 0, 0, 0, 1844, 1546, 3, 0, 1345, 2, 0, 1213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5737, 1, 371, 77, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1173, 624, 271, 0, 1957, 1708, 384, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 2, 6073, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 2, 458, 0, 5568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6153, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6483, 5953, 6188, 6119, 6080, 6032, 6154] avg=6144.0000 max=6483 max_rank=1 min=5953 min_rank=2 imbalance_ratio=1.055176 max_load_reduction=42669 max_load_reduction_pct=0.868103 moved_to_echo=24385 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[255, 0, 0, 0, 255, 0, 0, 2, 4395, 1203, 0, 0, 0, 0, 0, 0, 0, 706, 1259, 1905, 0, 0, 339, 7, 881, 812, 0, 0, 0, 0, 0, 0, 2179, 1486, 0, 0, 1334, 0, 0, 1406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5699, 0, 355, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1234, 659, 378, 0, 1940, 1696, 380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6023, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 500, 0, 5640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6091, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6110, 5909, 6405, 6183, 6287, 6023, 6144, 6091] avg=6144.0000 max=6405 max_rank=2 min=5909 min_rank=1 imbalance_ratio=1.042480 max_load_reduction=42747 max_load_reduction_pct=0.869690 moved_to_echo=24545 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[245, 0, 0, 0, 245, 0, 0, 2, 4430, 1231, 0, 0, 0, 0, 0, 0, 0, 814, 1283, 2005, 0, 0, 288, 4, 894, 807, 0, 0, 0, 0, 0, 0, 1947, 1723, 0, 0, 1344, 0, 0, 1141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5647, 0, 383, 118, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1319, 577, 282, 0, 1955, 1681, 386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 6162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 444, 0, 5602, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6181, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6153, 6095, 6155, 6150, 6202, 6164, 6052, 6181] avg=6144.0000 max=6202 max_rank=4 min=6052 min_rank=6 imbalance_ratio=1.009440 max_load_reduction=42950 max_load_reduction_pct=0.873820 moved_to_echo=24599 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[237, 0, 0, 4, 236, 1, 1, 2, 4431, 1212, 0, 0, 0, 0, 0, 0, 0, 765, 1050, 2356, 0, 0, 321, 8, 887, 814, 0, 0, 0, 0, 0, 0, 2016, 1433, 4, 0, 1345, 0, 0, 1356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5700, 4, 367, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 5, 1206, 493, 356, 0, 1956, 1697, 383, 0, 0, 0, 0, 0, 3, 0, 0, 1, 2, 0, 0, 0, 6066, 0, 0, 0, 0, 0, 0, 0, 2, 6, 0, 64, 495, 0, 5654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 3, 1, 0, 3, 6132, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6124, 6201, 6154, 6134, 6100, 6072, 6221, 6146] avg=6144.0000 max=6221 max_rank=6 min=6072 min_rank=5 imbalance_ratio=1.012533 max_load_reduction=42931 max_load_reduction_pct=0.873433 moved_to_echo=24539 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[248, 1, 3, 0, 248, 0, 0, 1, 4362, 1244, 0, 0, 0, 0, 0, 0, 0, 672, 1376, 2157, 1, 1, 109, 7, 879, 819, 0, 0, 0, 0, 0, 0, 2101, 1547, 0, 0, 1325, 0, 0, 1276, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5657, 1, 337, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1296, 568, 284, 0, 1925, 1684, 379, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 2, 6221, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 1, 519, 0, 5690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6080, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6107, 6021, 6249, 6116, 6138, 6225, 6213, 6083] avg=6144.0000 max=6249 max_rank=2 min=6021 min_rank=1 imbalance_ratio=1.017090 max_load_reduction=42903 max_load_reduction_pct=0.872864 moved_to_echo=24659 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[248, 1, 0, 0, 248, 0, 0, 2, 4466, 1252, 0, 0, 0, 0, 0, 0, 0, 867, 1232, 2028, 0, 0, 143, 3, 890, 804, 0, 0, 0, 0, 0, 0, 2006, 1619, 0, 0, 1355, 0, 0, 1304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5745, 0, 350, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1168, 565, 260, 0, 1971, 1710, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 6262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 461, 0, 5585, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6155, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6217, 5967, 6284, 6156, 6058, 6264, 6051, 6155] avg=6144.0000 max=6284 max_rank=2 min=5967 min_rank=1 imbalance_ratio=1.022786 max_load_reduction=42868 max_load_reduction_pct=0.872152 moved_to_echo=24528 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[255, 1, 0, 0, 255, 0, 0, 0, 4427, 1231, 0, 0, 0, 0, 0, 0, 0, 790, 1161, 2161, 0, 0, 298, 0, 893, 828, 0, 0, 0, 0, 0, 0, 1838, 1478, 0, 0, 1336, 0, 0, 1273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5650, 0, 323, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1265, 599, 290, 0, 1956, 1686, 387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 451, 0, 5724, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6153, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6169, 6131, 5925, 6103, 6184, 6158, 6329, 6153] avg=6144.0000 max=6329 max_rank=6 min=5925 min_rank=2 imbalance_ratio=1.030111 max_load_reduction=42823 max_load_reduction_pct=0.871236 moved_to_echo=24824 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[252, 0, 0, 0, 250, 0, 0, 1, 4395, 1231, 0, 0, 0, 0, 0, 0, 0, 782, 1318, 2224, 0, 0, 314, 5, 895, 807, 0, 0, 0, 0, 0, 0, 2181, 1389, 0, 0, 1333, 0, 0, 1124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 5650, 0, 367, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1146, 589, 322, 0, 1940, 1681, 386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 400, 0, 5603, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6189, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6129, 6345, 6027, 6122, 6064, 6166, 6110, 6189] avg=6144.0000 max=6345 max_rank=1 min=6027 min_rank=2 imbalance_ratio=1.032715 max_load_reduction=42807 max_load_reduction_pct=0.870911 moved_to_echo=24529 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 6400, 9856, 17152, 128, 128, 2176, 128, 7168, 6528, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[16128, 12288, 128, 0, 10752, 128, 0, 10112, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=50176 observed_loss=-1024 tokens_per_expert=[2048, 128, 128, 128, 2048, 128, 128, 128, 35456, 9856, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 9856, 4736, 2560, 0, 15616, 13568, 3072, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 0, 384, 3840, 0, 45184, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[0, 128, 128, 45568, 128, 2944, 896, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 0, 128, 49152, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 0, 0, 128, 49152, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=50176 slack_slots=48128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[7391, 81, 0, 2699, 71, 1, 201, 0, 1, 0, 4, 298, 8, 0, 3, 8013, 0, 7063, 0, 0, 1, 3, 6, 1, 2, 2618, 0, 4, 0, 89, 1, 0, 3726, 2, 1, 0, 0, 1, 1, 7, 2, 1, 0, 7, 0, 2, 1, 6789, 352, 0, 0, 0, 0, 49, 1, 0, 4, 1, 0, 329, 584, 281, 478, 7974] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[7421, 70, 0, 2758, 49, 0, 175, 0, 0, 0, 0, 307, 0, 0, 0, 8177, 0, 7125, 0, 0, 0, 0, 0, 0, 0, 2763, 0, 0, 0, 68, 0, 0, 3367, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 6961, 321, 0, 0, 0, 0, 58, 0, 0, 4, 0, 0, 286, 474, 232, 460, 8073] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[7445, 86, 3, 2507, 65, 3, 194, 0, 0, 0, 2, 288, 2, 0, 2, 8129, 0, 6995, 0, 0, 1, 0, 2, 0, 0, 3157, 0, 0, 0, 86, 1, 0, 3310, 0, 0, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 1, 0, 6740, 319, 0, 0, 0, 0, 56, 1, 0, 9, 3, 0, 266, 606, 268, 551, 8049] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[7466, 60, 1, 2689, 36, 0, 227, 0, 0, 0, 1, 299, 0, 0, 9, 8042, 0, 7152, 5, 0, 0, 0, 4, 4, 0, 2414, 0, 0, 1, 64, 1, 0, 3915, 0, 0, 1, 0, 1, 11, 0, 0, 3, 0, 0, 7, 3, 5, 6764, 359, 4, 0, 0, 0, 21, 0, 0, 0, 2, 0, 325, 550, 233, 438, 8035] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[7502, 81, 0, 2856, 69, 0, 205, 0, 0, 0, 0, 298, 0, 0, 1, 8181, 0, 7090, 0, 0, 0, 0, 0, 0, 0, 2370, 0, 0, 0, 79, 0, 0, 3616, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 6754, 373, 0, 1, 0, 0, 32, 0, 0, 5, 0, 0, 274, 562, 284, 461, 8056] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[7372, 75, 0, 2473, 61, 0, 219, 0, 0, 0, 0, 305, 0, 0, 0, 8133, 0, 7036, 1, 0, 0, 0, 0, 0, 0, 2756, 0, 0, 0, 72, 0, 0, 3848, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 6629, 338, 0, 0, 0, 0, 37, 0, 0, 1, 0, 0, 359, 647, 287, 482, 8017] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[7403, 75, 0, 2791, 40, 0, 193, 0, 0, 0, 0, 318, 0, 0, 0, 7931, 0, 6913, 1, 0, 0, 0, 0, 0, 0, 2795, 0, 0, 0, 74, 0, 0, 3786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6861, 303, 0, 0, 0, 0, 42, 0, 0, 4, 0, 0, 344, 550, 234, 487, 8007] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[7324, 72, 0, 2738, 43, 0, 208, 0, 0, 0, 0, 298, 0, 0, 0, 8044, 0, 7126, 0, 0, 0, 0, 0, 0, 0, 2660, 0, 0, 0, 71, 0, 0, 3835, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6776, 336, 0, 0, 0, 0, 38, 0, 0, 13, 0, 0, 275, 621, 213, 463, 7997] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3112, 81, 0, 2699, 71, 1, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 298, 8, 0, 3, 5790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6141, 0, 0, 1, 3, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2618, 0, 4, 0, 89, 1, 0, 2017, 1334, 0, 0, 0, 0, 0, 0, 3726, 2, 1, 0, 0, 1, 1, 7, 922, 889, 644, 0, 0, 0, 0, 0, 2, 1, 0, 7, 0, 2, 1, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 352, 0, 0, 0, 0, 49, 1, 0, 4279, 1460, 0, 0, 0, 0, 0, 0, 4, 1, 0, 329, 584, 281, 478, 4497, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6165, 6104, 6152, 6065, 6193, 6158, 6141, 6174] avg=6144.0000 max=6193 max_rank=4 min=6065 min_rank=3 imbalance_ratio=1.007975 max_load_reduction=42959 max_load_reduction_pct=0.874003 moved_to_echo=24666 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3144, 60, 1, 2689, 36, 0, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 299, 0, 0, 9, 5812, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6219, 5, 0, 0, 0, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2414, 0, 0, 1, 64, 1, 0, 2032, 1338, 0, 0, 0, 0, 0, 0, 3915, 0, 0, 1, 0, 1, 11, 0, 933, 892, 642, 0, 0, 0, 0, 0, 0, 3, 0, 0, 7, 3, 5, 6122, 0, 0, 0, 0, 0, 0, 0, 0, 359, 4, 0, 0, 0, 21, 0, 0, 4322, 1472, 0, 0, 0, 0, 0, 0, 0, 2, 0, 325, 550, 233, 438, 4531, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6157, 6121, 6232, 5850, 6395, 6140, 6178, 6079] avg=6144.0000 max=6395 max_rank=4 min=5850 min_rank=3 imbalance_ratio=1.040853 max_load_reduction=42757 max_load_reduction_pct=0.869893 moved_to_echo=24792 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3104, 75, 0, 2473, 61, 0, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 305, 0, 0, 0, 5877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6118, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2756, 0, 0, 0, 72, 0, 0, 2027, 1354, 0, 0, 0, 0, 0, 0, 3848, 0, 0, 0, 0, 0, 4, 0, 918, 902, 629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6000, 0, 0, 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, 37, 0, 0, 4268, 1468, 0, 0, 0, 0, 0, 0, 1, 0, 0, 359, 647, 287, 482, 4522, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5932, 6182, 6119, 6209, 6301, 6000, 6111, 6298] avg=6144.0000 max=6301 max_rank=4 min=5932 min_rank=0 imbalance_ratio=1.025553 max_load_reduction=42851 max_load_reduction_pct=0.871806 moved_to_echo=24710 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3135, 86, 3, 2507, 65, 3, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 288, 2, 0, 2, 5874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6082, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3157, 0, 0, 0, 86, 1, 0, 2036, 1353, 0, 0, 0, 0, 0, 0, 3310, 0, 0, 0, 0, 2, 0, 1, 913, 902, 640, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 6100, 0, 0, 0, 0, 0, 0, 0, 0, 319, 0, 0, 0, 0, 56, 1, 0, 4310, 1474, 0, 0, 0, 0, 0, 0, 9, 3, 0, 266, 606, 268, 551, 4539, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5993, 6168, 6085, 6633, 5768, 6103, 6160, 6242] avg=6144.0000 max=6633 max_rank=3 min=5768 min_rank=4 imbalance_ratio=1.079590 max_load_reduction=42519 max_load_reduction_pct=0.865051 moved_to_echo=24273 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3159, 81, 0, 2856, 69, 0, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 298, 0, 0, 1, 5911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2370, 0, 0, 0, 79, 0, 0, 2037, 1362, 0, 0, 0, 0, 0, 0, 3616, 0, 0, 0, 0, 0, 2, 0, 925, 908, 641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6113, 0, 0, 0, 0, 0, 0, 0, 0, 373, 0, 1, 0, 0, 32, 0, 0, 4343, 1475, 0, 0, 0, 0, 0, 0, 5, 0, 0, 274, 562, 284, 461, 4544, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6370, 6210, 6165, 5848, 6092, 6113, 6224, 6130] avg=6144.0000 max=6370 max_rank=0 min=5848 min_rank=3 imbalance_ratio=1.036784 max_load_reduction=42782 max_load_reduction_pct=0.870402 moved_to_echo=24559 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3125, 70, 0, 2758, 49, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 307, 0, 0, 0, 5909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2763, 0, 0, 0, 68, 0, 0, 2042, 1361, 0, 0, 0, 0, 0, 0, 3367, 0, 0, 0, 0, 0, 3, 0, 930, 907, 661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6300, 0, 0, 0, 0, 0, 0, 0, 0, 321, 0, 0, 0, 0, 58, 0, 0, 4296, 1479, 0, 0, 0, 0, 0, 0, 4, 0, 0, 286, 474, 232, 460, 4552, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6177, 6216, 6195, 6234, 5868, 6300, 6154, 6008] avg=6144.0000 max=6300 max_rank=5 min=5868 min_rank=4 imbalance_ratio=1.025391 max_load_reduction=42852 max_load_reduction_pct=0.871826 moved_to_echo=24330 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3114, 75, 0, 2791, 40, 0, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 318, 0, 0, 0, 5723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6007, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2795, 0, 0, 0, 74, 0, 0, 2029, 1323, 0, 0, 0, 0, 0, 0, 3786, 0, 0, 0, 0, 0, 0, 0, 906, 885, 656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6205, 0, 0, 0, 0, 0, 0, 0, 0, 303, 0, 0, 0, 0, 42, 0, 0, 4289, 1471, 0, 0, 0, 0, 0, 0, 4, 0, 0, 344, 550, 234, 487, 4507, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6213, 6041, 6008, 6221, 6233, 6205, 6105, 6126] avg=6144.0000 max=6233 max_rank=4 min=6008 min_rank=2 imbalance_ratio=1.014486 max_load_reduction=42919 max_load_reduction_pct=0.873189 moved_to_echo=24669 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3084, 72, 0, 2738, 43, 0, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 298, 0, 0, 0, 5812, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2660, 0, 0, 0, 71, 0, 0, 2022, 1339, 0, 0, 0, 0, 0, 0, 3835, 0, 0, 0, 0, 0, 1, 0, 930, 893, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6133, 0, 0, 0, 0, 0, 0, 0, 0, 336, 0, 0, 0, 0, 38, 0, 0, 4240, 1465, 0, 0, 0, 0, 0, 0, 13, 0, 0, 275, 621, 213, 463, 4510, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 6110, 6196, 6092, 6302, 6133, 6079, 6095] avg=6144.0000 max=6302 max_rank=4 min=6079 min_rank=6 imbalance_ratio=1.025716 max_load_reduction=42850 max_load_reduction_pct=0.871785 moved_to_echo=24609 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 21632, 0, 128, 128, 640, 128, 0, 16256, 10880, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2816, 128, 128, 0, 0, 384, 128, 0, 34432, 11776, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=50176 observed_loss=-1024 tokens_per_expert=[29440, 128, 128, 128, 0, 128, 128, 128, 7424, 7296, 5248, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[25088, 640, 128, 21632, 512, 128, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 128, 0, 2560, 4608, 2048, 3840, 36224, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 0, 128, 2432, 128, 0, 128, 46720, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 128, 128, 128, 49152, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 49152, 128, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=50176 slack_slots=48128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 7053, 0, 0, 0, 56, 0, 0, 0, 1, 1, 1, 1, 434, 0, 3, 0, 0, 0, 8185, 0, 0, 324, 8121, 0, 399, 0, 0, 0, 0, 0, 2344, 1, 0, 81, 0, 1, 4575, 7559, 0, 377, 0, 4, 3, 287, 1, 2, 2, 8112, 2, 0, 1108, 0, 2, 0, 2, 1, 2, 0, 96, 2, 1, 1, 7] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 7074, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 462, 0, 0, 0, 0, 0, 8192, 0, 0, 338, 8141, 0, 462, 0, 0, 0, 0, 0, 2486, 0, 0, 71, 0, 0, 4318, 7556, 0, 341, 0, 0, 0, 297, 0, 0, 0, 8106, 0, 0, 1128, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 7063, 0, 0, 0, 45, 0, 2, 0, 0, 0, 0, 0, 472, 0, 0, 0, 0, 0, 8192, 0, 0, 375, 8169, 0, 360, 0, 0, 0, 0, 0, 2159, 0, 0, 81, 0, 0, 4677, 7584, 0, 338, 0, 0, 0, 392, 0, 0, 0, 8073, 0, 0, 979, 0, 0, 0, 0, 0, 1, 0, 190, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 7086, 0, 0, 0, 89, 0, 1, 0, 0, 0, 0, 0, 384, 0, 1, 0, 0, 0, 8192, 0, 0, 320, 8152, 0, 424, 0, 0, 0, 0, 0, 2176, 0, 0, 76, 0, 0, 4876, 7518, 0, 379, 0, 0, 0, 291, 0, 0, 0, 8083, 0, 0, 993, 0, 0, 0, 0, 0, 0, 0, 111, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 7033, 0, 0, 0, 47, 0, 0, 0, 10, 6, 0, 0, 445, 0, 1, 0, 0, 0, 8182, 1, 3, 307, 8132, 0, 437, 1, 2, 0, 0, 0, 2349, 0, 0, 88, 1, 1, 4626, 7544, 3, 346, 1, 7, 8, 318, 3, 0, 1, 8102, 0, 0, 989, 0, 0, 0, 5, 0, 1, 1, 146, 0, 0, 4, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 7001, 0, 0, 1, 79, 0, 0, 0, 0, 2, 3, 1, 416, 0, 0, 0, 0, 0, 8188, 0, 2, 376, 8132, 0, 430, 1, 0, 0, 0, 0, 2347, 0, 0, 87, 2, 0, 4842, 7516, 0, 384, 0, 1, 0, 273, 0, 0, 2, 8061, 0, 0, 839, 0, 3, 0, 2, 0, 2, 0, 158, 1, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 7082, 0, 0, 0, 60, 0, 6, 0, 0, 0, 0, 0, 358, 0, 0, 0, 0, 0, 8192, 0, 0, 413, 8144, 0, 440, 0, 0, 0, 0, 0, 2301, 0, 0, 78, 0, 0, 4470, 7568, 0, 392, 0, 0, 0, 324, 0, 0, 0, 8099, 0, 0, 1053, 0, 0, 0, 0, 0, 0, 0, 172, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 7125, 1, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 483, 0, 0, 0, 0, 0, 8192, 0, 0, 364, 8125, 0, 423, 0, 6, 0, 0, 0, 2257, 0, 0, 72, 0, 0, 4557, 7613, 0, 373, 0, 0, 0, 283, 0, 0, 0, 8068, 0, 0, 983, 0, 0, 0, 0, 0, 0, 0, 154, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 6070, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 434, 0, 3, 3516, 2191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 324, 5778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 399, 0, 0, 0, 0, 0, 2344, 2343, 983, 85, 0, 0, 0, 0, 0, 1, 0, 81, 0, 1, 4575, 1448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 377, 0, 4, 3, 287, 1, 2, 2, 2878, 2595, 0, 0, 0, 0, 0, 0, 5149, 2, 0, 1108, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 96, 2, 1, 1, 7, 5994, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6126, 6148, 6102, 6154, 6106, 6149, 6263, 6104] avg=6144.0000 max=6263 max_rank=6 min=6102 min_rank=2 imbalance_ratio=1.019368 max_load_reduction=42889 max_load_reduction_pct=0.872579 moved_to_echo=24622 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 6098, 0, 0, 0, 89, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 384, 0, 1, 3497, 2193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 320, 5801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 424, 0, 0, 0, 0, 0, 2176, 2351, 988, 84, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 4876, 1440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 379, 0, 0, 0, 291, 0, 0, 0, 2868, 2581, 0, 0, 0, 0, 0, 0, 5131, 0, 0, 993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 0, 0, 0, 0, 5999, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6188, 6075, 6121, 6023, 6392, 6119, 6124, 6110] avg=6144.0000 max=6392 max_rank=4 min=6023 min_rank=3 imbalance_ratio=1.040365 max_load_reduction=42760 max_load_reduction_pct=0.869954 moved_to_echo=24745 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 6079, 0, 0, 0, 45, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 0, 0, 3528, 2193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 375, 5813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 360, 0, 0, 0, 0, 0, 2159, 2356, 984, 84, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 4677, 1452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, 392, 0, 0, 0, 2864, 2604, 0, 0, 0, 0, 0, 0, 5125, 0, 0, 979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 190, 0, 0, 0, 0, 5999, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6126, 6193, 6188, 5943, 6210, 6198, 6104, 6190] avg=6144.0000 max=6210 max_rank=4 min=5943 min_rank=3 imbalance_ratio=1.010742 max_load_reduction=42942 max_load_reduction_pct=0.873657 moved_to_echo=24702 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 6025, 0, 0, 1, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 1, 416, 0, 0, 3496, 2192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 376, 5786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 430, 1, 0, 0, 0, 0, 2347, 2346, 976, 84, 0, 0, 0, 0, 0, 0, 0, 87, 2, 0, 4842, 1439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 384, 0, 1, 0, 273, 0, 0, 2, 2860, 2581, 0, 0, 0, 0, 0, 0, 5117, 0, 0, 839, 0, 3, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 158, 1, 0, 0, 0, 5996, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6105, 6110, 6164, 6184, 6370, 6101, 5961, 6157] avg=6144.0000 max=6370 max_rank=4 min=5961 min_rank=6 imbalance_ratio=1.036784 max_load_reduction=42782 max_load_reduction_pct=0.870402 moved_to_echo=24589 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 6088, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 0, 0, 3515, 2193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 338, 5793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 0, 0, 0, 0, 0, 2486, 2348, 986, 85, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 4318, 1447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 341, 0, 0, 0, 297, 0, 0, 0, 2876, 2594, 0, 0, 0, 0, 0, 0, 5145, 0, 0, 1128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 0, 0, 5999, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6144, 6170, 6131, 6367, 5836, 6108, 6273, 6123] avg=6144.0000 max=6367 max_rank=3 min=5836 min_rank=4 imbalance_ratio=1.036296 max_load_reduction=42785 max_load_reduction_pct=0.870463 moved_to_echo=24340 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 6053, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 6, 0, 0, 445, 0, 1, 3509, 2190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 307, 5786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 437, 1, 2, 0, 0, 0, 2349, 2346, 980, 85, 0, 0, 0, 0, 0, 0, 0, 88, 1, 1, 4626, 1445, 3, 0, 0, 0, 0, 0, 0, 0, 0, 346, 1, 7, 8, 318, 3, 0, 1, 2874, 2590, 0, 0, 0, 0, 0, 0, 5143, 0, 0, 989, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 146, 0, 0, 4, 1, 5992, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6100, 6161, 6097, 6200, 6164, 6148, 6137, 6145] avg=6144.0000 max=6200 max_rank=3 min=6097 min_rank=2 imbalance_ratio=1.009115 max_load_reduction=42952 max_load_reduction_pct=0.873861 moved_to_echo=24594 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 6091, 0, 0, 0, 60, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 358, 0, 0, 3526, 2194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 413, 5791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 440, 0, 0, 0, 0, 0, 2301, 2353, 991, 89, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 4470, 1439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 392, 0, 0, 0, 324, 0, 0, 0, 2878, 2603, 0, 0, 0, 0, 0, 0, 5132, 0, 0, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 0, 0, 0, 0, 5998, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6157, 6078, 6204, 6174, 5987, 6197, 6185, 6170] avg=6144.0000 max=6204 max_rank=2 min=5987 min_rank=4 imbalance_ratio=1.009766 max_load_reduction=42948 max_load_reduction_pct=0.873779 moved_to_echo=24539 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 6132, 1, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 0, 0, 3541, 2193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 364, 5781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 423, 0, 6, 0, 0, 0, 2257, 2344, 993, 84, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 4557, 1458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 373, 0, 0, 0, 283, 0, 0, 0, 2862, 2614, 0, 0, 0, 0, 0, 0, 5122, 0, 0, 983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 0, 0, 0, 0, 5999, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6206, 6217, 6145, 6107, 6087, 6132, 6105, 6153] avg=6144.0000 max=6217 max_rank=1 min=6087 min_rank=4 imbalance_ratio=1.011882 max_load_reduction=42935 max_load_reduction_pct=0.873515 moved_to_echo=24477 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 128, 3456, 0, 128, 28160, 17664, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 3456, 128, 128, 0, 0, 0, 18432, 18816, 7936, 768, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 0, 640, 128, 128, 36992, 11648, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[41088, 128, 0, 8192, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=50176 observed_loss=-1024 tokens_per_expert=[2944, 128, 128, 128, 2560, 128, 128, 128, 23040, 20864, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 128, 2944, 46336, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 48640, 128, 0, 128, 512, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 128, 128, 1152, 128, 128, 128, 128, 48000, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=50176 slack_slots=48128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 733, 1, 401, 0, 3, 6862, 1, 0, 2, 1, 2, 1, 11, 0, 0, 5698, 0, 4, 0, 0, 0, 192, 1, 0, 0, 1, 141, 2, 0, 301, 76, 7935, 8189, 8097, 1, 1, 0, 0, 0, 2270, 57, 1, 0, 0, 56, 1, 0, 3, 0, 0, 8101, 1, 0, 2, 3, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 565, 0, 505, 0, 0, 6799, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5257, 0, 12, 0, 0, 0, 204, 36, 0, 0, 0, 129, 0, 0, 198, 81, 7991, 8192, 8108, 0, 0, 0, 0, 0, 2924, 16, 0, 0, 0, 24, 0, 0, 0, 0, 0, 8111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 602, 0, 468, 0, 0, 6768, 0, 0, 0, 0, 0, 0, 3, 0, 0, 5537, 0, 9, 0, 0, 0, 217, 22, 0, 0, 0, 163, 0, 0, 235, 75, 7966, 8192, 8116, 0, 0, 0, 0, 0, 2631, 9, 0, 0, 0, 23, 0, 0, 0, 0, 0, 8114, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 922, 0, 520, 3, 3, 6830, 0, 0, 0, 0, 3, 2, 6, 0, 0, 5515, 0, 87, 3, 0, 0, 227, 8, 0, 0, 0, 82, 0, 0, 216, 71, 7973, 8187, 8108, 1, 1, 1, 0, 0, 2193, 7, 1, 0, 5, 58, 4, 0, 5, 0, 0, 8097, 0, 2, 6, 3, 1, 0, 0, 0, 1, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 542, 0, 391, 0, 0, 6865, 0, 2, 0, 0, 0, 0, 0, 0, 0, 5806, 0, 17, 0, 0, 0, 175, 7, 0, 0, 0, 143, 0, 0, 208, 70, 7981, 8192, 8117, 0, 0, 0, 0, 0, 2398, 54, 0, 0, 0, 66, 0, 0, 0, 0, 0, 8118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 491, 7, 491, 0, 7, 6725, 1, 0, 7, 1, 1, 4, 10, 0, 0, 5760, 0, 18, 5, 0, 0, 200, 28, 0, 1, 0, 116, 2, 0, 214, 81, 7976, 8181, 8090, 0, 0, 1, 0, 0, 2541, 24, 1, 3, 1, 49, 2, 1, 5, 0, 1, 8084, 0, 4, 9, 8, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 772, 0, 454, 0, 0, 6854, 0, 0, 0, 0, 0, 0, 10, 0, 0, 5612, 0, 13, 0, 0, 0, 193, 46, 0, 0, 0, 136, 0, 0, 194, 77, 7993, 8192, 8113, 0, 0, 0, 0, 0, 2334, 1, 0, 0, 0, 53, 0, 0, 0, 0, 0, 8105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 930, 0, 464, 0, 0, 6975, 0, 0, 0, 0, 0, 0, 2, 0, 0, 5339, 0, 26, 0, 0, 0, 206, 4, 0, 0, 0, 145, 0, 0, 251, 70, 7947, 8192, 8119, 0, 0, 0, 0, 0, 2303, 4, 0, 0, 0, 52, 0, 0, 0, 0, 0, 8120, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 602, 0, 468, 0, 0, 4372, 615, 0, 0, 0, 0, 0, 0, 6077, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5537, 0, 9, 0, 0, 0, 548, 0, 0, 0, 0, 0, 0, 0, 217, 22, 0, 0, 0, 163, 0, 0, 3744, 2051, 0, 0, 0, 0, 0, 0, 235, 75, 5839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2631, 9, 0, 0, 0, 23, 1967, 1512, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 6141, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6057, 6080, 6094, 6197, 6149, 6285, 6147, 6143] avg=6144.0000 max=6285 max_rank=5 min=6057 min_rank=0 imbalance_ratio=1.022949 max_load_reduction=42867 max_load_reduction_pct=0.872131 moved_to_echo=24724 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 922, 0, 520, 3, 3, 4368, 616, 0, 0, 0, 0, 0, 0, 6132, 0, 0, 0, 0, 3, 2, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5515, 0, 87, 3, 0, 0, 553, 0, 0, 0, 0, 0, 0, 0, 227, 8, 0, 0, 0, 82, 0, 0, 3740, 2049, 0, 0, 0, 0, 0, 0, 216, 71, 5844, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2193, 7, 1, 0, 5, 58, 1963, 1513, 145, 0, 0, 0, 0, 0, 4, 0, 5, 0, 0, 6134, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 1, 0, 0, 0, 1, 0, 6138, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6432, 6143, 6158, 6106, 6134, 5885, 6145, 6149] avg=6144.0000 max=6432 max_rank=0 min=5885 min_rank=5 imbalance_ratio=1.046875 max_load_reduction=42720 max_load_reduction_pct=0.869141 moved_to_echo=24313 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 565, 0, 505, 0, 0, 4368, 617, 0, 0, 0, 0, 0, 0, 6104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5257, 0, 12, 0, 0, 0, 551, 0, 0, 0, 0, 0, 0, 0, 204, 36, 0, 0, 0, 129, 0, 0, 3740, 2051, 0, 0, 0, 0, 0, 0, 198, 81, 5858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2924, 16, 0, 0, 0, 24, 1967, 1516, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6141, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6055, 6104, 5820, 6160, 6137, 6591, 6144, 6141] avg=6144.0000 max=6591 max_rank=5 min=5820 min_rank=2 imbalance_ratio=1.072754 max_load_reduction=42561 max_load_reduction_pct=0.865906 moved_to_echo=25013 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 733, 1, 401, 0, 3, 4363, 613, 0, 0, 0, 0, 0, 0, 6161, 1, 0, 2, 1, 2, 1, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5698, 0, 4, 0, 0, 0, 556, 0, 0, 0, 0, 0, 0, 0, 192, 1, 0, 0, 1, 141, 2, 0, 3734, 2051, 0, 0, 0, 0, 0, 0, 301, 76, 5816, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2270, 57, 1, 0, 0, 56, 1964, 1506, 145, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 6137, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 6138, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6114, 6179, 6258, 6122, 6195, 5999, 6142, 6143] avg=6144.0000 max=6258 max_rank=2 min=5999 min_rank=5 imbalance_ratio=1.018555 max_load_reduction=42894 max_load_reduction_pct=0.872681 moved_to_echo=24479 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 491, 7, 491, 0, 7, 4358, 616, 0, 0, 0, 0, 0, 0, 6038, 1, 0, 7, 1, 1, 4, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5760, 0, 18, 5, 0, 0, 545, 0, 0, 0, 0, 0, 0, 0, 200, 28, 0, 1, 0, 116, 2, 0, 3732, 2048, 0, 0, 0, 0, 0, 0, 214, 81, 5846, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2541, 24, 1, 3, 1, 49, 1960, 1514, 142, 0, 0, 0, 0, 0, 2, 1, 5, 0, 1, 6124, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 9, 8, 0, 1, 0, 0, 0, 0, 6133, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5970, 6062, 6328, 6127, 6142, 6235, 6137, 6151] avg=6144.0000 max=6328 max_rank=2 min=5970 min_rank=0 imbalance_ratio=1.029948 max_load_reduction=42824 max_load_reduction_pct=0.871257 moved_to_echo=24665 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 542, 0, 391, 0, 0, 4373, 617, 0, 0, 0, 0, 0, 0, 6164, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5806, 0, 17, 0, 0, 0, 556, 0, 0, 0, 0, 0, 0, 0, 175, 7, 0, 0, 0, 143, 0, 0, 3744, 2051, 0, 0, 0, 0, 0, 0, 208, 70, 5849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2398, 54, 0, 0, 0, 66, 1968, 1515, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6141, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5923, 6166, 6379, 6120, 6127, 6146, 6150, 6141] avg=6144.0000 max=6379 max_rank=2 min=5923 min_rank=0 imbalance_ratio=1.038249 max_load_reduction=42773 max_load_reduction_pct=0.870219 moved_to_echo=24564 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 772, 0, 454, 0, 0, 4371, 622, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5612, 0, 13, 0, 0, 0, 560, 0, 0, 0, 0, 0, 0, 0, 193, 46, 0, 0, 0, 136, 0, 0, 3742, 2052, 0, 0, 0, 0, 0, 0, 194, 77, 5851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2334, 1, 0, 0, 0, 53, 1969, 1520, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6140, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6219, 6154, 6185, 6169, 6122, 6027, 6136, 6140] avg=6144.0000 max=6219 max_rank=0 min=6027 min_rank=5 imbalance_ratio=1.012207 max_load_reduction=42933 max_load_reduction_pct=0.873474 moved_to_echo=24425 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 930, 0, 464, 0, 0, 4374, 614, 0, 0, 0, 0, 0, 0, 6262, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5339, 0, 26, 0, 0, 0, 565, 0, 0, 0, 0, 0, 0, 0, 206, 4, 0, 0, 0, 145, 0, 0, 3745, 2051, 0, 0, 0, 0, 0, 0, 251, 70, 5825, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2303, 4, 0, 0, 0, 52, 1969, 1508, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 6141, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6382, 6264, 5930, 6151, 6146, 5984, 6151, 6144] avg=6144.0000 max=6382 max_rank=0 min=5930 min_rank=2 imbalance_ratio=1.038737 max_load_reduction=42770 max_load_reduction_pct=0.870158 moved_to_echo=24425 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[0, 0, 19712, 256, 128, 128, 128, 384, 15744, 12160, 1280, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 0, 0, 5632, 128, 3712, 128, 128, 35072, 4992, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1664, 256, 0, 128, 128, 1152, 128, 0, 29952, 16512, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 44544, 0, 256, 128, 0, 0, 4480, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1920, 640, 46848, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 128, 49152, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 0, 0, 128, 0, 49152, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[49152, 128, 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 84, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 5982, 0, 0, 45, 0, 0, 0, 1024, 0, 0, 81, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4286, 81, 657, 0, 22, 0, 6298, 0, 5, 0, 1151, 4851, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 15, 1, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 2, 8185, 1, 0, 0, 0, 0, 8189, 5928, 0, 0, 8, 1, 1, 0, 1527, 0, 3, 91, 3, 8187, 3, 2, 0, 0, 1, 2, 0, 1, 0, 3912, 86, 531, 0, 16, 2, 6519, 0, 0, 0, 1281, 4650, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 60, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 5916, 0, 0, 11, 0, 0, 0, 1160, 0, 0, 83, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4275, 75, 552, 0, 24, 0, 6415, 0, 2, 0, 1204, 4796, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 172, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8191, 5968, 0, 0, 27, 0, 0, 0, 966, 0, 0, 76, 1, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3949, 70, 613, 0, 25, 0, 6466, 0, 0, 0, 1274, 4967, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1, 0, 73, 8, 6, 0, 0, 3, 4, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 3, 8178, 5, 0, 0, 0, 1, 8181, 5962, 1, 0, 47, 1, 0, 3, 789, 0, 7, 98, 8, 8163, 2, 13, 0, 0, 2, 5, 0, 1, 0, 4427, 81, 580, 0, 33, 6, 6433, 0, 4, 1, 1172, 4842, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 7, 1, 9, 0, 1, 3, 1, 0, 1, 4, 0, 3, 3, 0, 0, 0, 0, 0, 1, 8185, 5, 0, 1, 0, 0, 8182, 6336, 0, 1, 33, 0, 0, 0, 1096, 0, 4, 93, 3, 8184, 3, 1, 0, 0, 0, 1, 0, 1, 0, 4113, 71, 603, 0, 25, 0, 6150, 0, 7, 0, 1253, 4763, 3, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 92, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 6209, 0, 0, 72, 0, 0, 0, 906, 0, 0, 81, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4290, 72, 613, 0, 14, 0, 6300, 0, 0, 0, 1206, 4720, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 6110, 0, 0, 57, 0, 0, 0, 832, 0, 0, 84, 2, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4391, 78, 667, 0, 25, 0, 6465, 0, 0, 0, 1149, 4698, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 84, 0, 9, 0, 0, 0, 4026, 1994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 5982, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 0, 0, 81, 0, 2129, 2048, 811, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4286, 81, 657, 0, 22, 0, 1236, 0, 0, 0, 0, 0, 0, 0, 143, 0, 5, 0, 1151, 4851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6113, 6142, 6145, 6083, 6093, 6144, 6282, 6150] avg=6144.0000 max=6282 max_rank=6 min=6083 min_rank=3 imbalance_ratio=1.022461 max_load_reduction=42870 max_load_reduction_pct=0.872192 moved_to_echo=24669 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 15, 1, 1, 0, 0, 0, 4167, 1993, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 6140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6141, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 5928, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1527, 0, 3, 91, 3, 2204, 2047, 810, 0, 0, 0, 0, 0, 6140, 3, 2, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3912, 86, 531, 0, 16, 2, 1234, 0, 0, 0, 0, 0, 0, 0, 148, 0, 0, 0, 1281, 4650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6177, 6143, 6144, 5992, 6687, 6148, 5782, 6079] avg=6144.0000 max=6687 max_rank=4 min=5782 min_rank=6 imbalance_ratio=1.088379 max_load_reduction=42465 max_load_reduction_pct=0.863953 moved_to_echo=24696 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 7, 1, 9, 0, 1, 3, 3931, 1991, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4, 0, 3, 3, 0, 6135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6141, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 56, 6336, 0, 1, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1096, 0, 4, 93, 3, 2079, 2046, 810, 0, 0, 0, 0, 0, 6138, 3, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4113, 71, 603, 0, 25, 0, 1234, 0, 0, 0, 0, 0, 0, 0, 140, 0, 7, 0, 1253, 4763, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5943, 6147, 6147, 6427, 6131, 6143, 6047, 6167] avg=6144.0000 max=6427 max_rank=3 min=5943 min_rank=0 imbalance_ratio=1.046061 max_load_reduction=42725 max_load_reduction_pct=0.869242 moved_to_echo=24488 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 73, 8, 6, 0, 0, 3, 4112, 1991, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8, 0, 0, 6134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 6135, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 56, 5962, 1, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 789, 0, 7, 98, 8, 2175, 2041, 810, 0, 0, 0, 0, 0, 6122, 2, 13, 0, 0, 2, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4427, 81, 580, 0, 33, 6, 1233, 0, 0, 0, 0, 0, 0, 0, 146, 0, 4, 1, 1172, 4842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6194, 6146, 6143, 6067, 5932, 6144, 6361, 6165] avg=6144.0000 max=6361 max_rank=6 min=5932 min_rank=4 imbalance_ratio=1.035319 max_load_reduction=42791 max_load_reduction_pct=0.870585 moved_to_echo=24602 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 172, 0, 3, 0, 0, 0, 4133, 1994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 5968, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 966, 0, 0, 76, 1, 2186, 2048, 811, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3949, 70, 613, 0, 25, 0, 1236, 0, 0, 0, 0, 0, 0, 0, 147, 0, 0, 0, 1274, 4967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6302, 6142, 6145, 6050, 6088, 6144, 5893, 6388] avg=6144.0000 max=6388 max_rank=7 min=5893 min_rank=6 imbalance_ratio=1.039714 max_load_reduction=42764 max_load_reduction_pct=0.870036 moved_to_echo=24513 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 60, 0, 3, 0, 0, 0, 4100, 1994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 5916, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1160, 0, 0, 83, 0, 2169, 2048, 811, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4275, 75, 552, 0, 24, 0, 1236, 0, 0, 0, 0, 0, 0, 0, 146, 0, 2, 0, 1204, 4796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6157, 6142, 6145, 5983, 6271, 6144, 6162, 6148] avg=6144.0000 max=6271 max_rank=4 min=5983 min_rank=3 imbalance_ratio=1.020671 max_load_reduction=42881 max_load_reduction_pct=0.872416 moved_to_echo=24725 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 18, 0, 0, 0, 0, 0, 4136, 1998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 6110, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 832, 0, 0, 84, 2, 2190, 2051, 815, 0, 0, 0, 0, 0, 6141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4391, 78, 667, 0, 25, 0, 1239, 0, 0, 0, 0, 0, 0, 0, 139, 0, 0, 0, 1149, 4698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6152, 6148, 6138, 6213, 5974, 6141, 6400, 5986] avg=6144.0000 max=6400 max_rank=6 min=5974 min_rank=4 imbalance_ratio=1.041667 max_load_reduction=42752 max_load_reduction_pct=0.869792 moved_to_echo=24501 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 92, 0, 1, 0, 0, 0, 4027, 1994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 6209, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 906, 0, 0, 81, 0, 2130, 2048, 811, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4290, 72, 613, 0, 14, 0, 1236, 0, 0, 0, 0, 0, 0, 0, 143, 0, 0, 0, 1206, 4720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6114, 6142, 6145, 6337, 5976, 6144, 6225, 6069] avg=6144.0000 max=6337 max_rank=3 min=5976 min_rank=4 imbalance_ratio=1.031413 max_load_reduction=42815 max_load_reduction_pct=0.871073 moved_to_echo=24414 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 0, 640, 128, 128, 0, 128, 128, 32640, 16000, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 128, 8320, 0, 128, 768, 128, 17280, 16384, 6528, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 0, 33664, 640, 4864, 0, 256, 128, 9984, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1152, 0, 128, 128, 9728, 38400, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[49152, 128, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 0, 128, 512, 48512, 128, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 128, 49152, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 0, 128, 128, 128, 128, 128, 128, 49152, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1399, 0, 0, 0, 0, 7893, 0, 0, 8130, 12, 229, 151, 137, 0, 0, 57, 0, 219, 0, 0, 0, 0, 0, 223, 0, 0, 8192, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7544, 0, 7036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 3, 0, 0, 7676, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1373, 0, 0, 0, 0, 7907, 0, 0, 8116, 2, 244, 136, 138, 0, 0, 12, 0, 161, 0, 0, 0, 0, 0, 168, 0, 0, 8192, 0, 1, 96, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7546, 0, 7083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 270, 3, 0, 0, 7703, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 1555, 0, 0, 0, 2, 7847, 0, 0, 8125, 34, 241, 140, 121, 0, 2, 69, 0, 154, 0, 0, 0, 3, 1, 242, 1, 0, 8188, 0, 0, 87, 2, 0, 1, 0, 0, 0, 2, 3, 0, 0, 0, 0, 7585, 3, 6855, 5, 0, 2, 1, 0, 0, 0, 2, 0, 195, 7, 1, 0, 7673, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[3, 0, 0, 0, 1687, 0, 0, 1, 5, 7893, 0, 0, 8114, 10, 260, 126, 129, 0, 4, 9, 0, 154, 3, 1, 0, 3, 2, 102, 2, 1, 8184, 0, 1, 72, 1, 6, 3, 0, 0, 0, 0, 13, 0, 1, 1, 1, 7562, 3, 7050, 0, 0, 2, 0, 0, 0, 0, 4, 0, 145, 1, 1, 0, 7592, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1536, 0, 0, 0, 0, 7879, 0, 0, 8130, 25, 216, 145, 138, 0, 0, 58, 0, 148, 0, 0, 0, 0, 0, 156, 0, 0, 8192, 0, 2, 72, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 7526, 0, 7075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 1, 0, 0, 7702, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1752, 0, 0, 0, 0, 7834, 0, 0, 8114, 4, 236, 138, 129, 0, 0, 15, 0, 39, 0, 0, 0, 0, 0, 205, 0, 0, 8192, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 7578, 0, 7030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 159, 14, 0, 0, 7553, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[6, 1, 3, 6, 1709, 0, 0, 0, 6, 7881, 0, 0, 8114, 19, 250, 133, 111, 0, 3, 73, 2, 46, 1, 0, 0, 4, 0, 217, 4, 0, 8181, 0, 0, 83, 5, 0, 7, 3, 0, 0, 0, 15, 0, 0, 0, 1, 7554, 4, 6961, 1, 0, 1, 0, 0, 0, 7, 2, 0, 126, 5, 0, 4, 7601, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1493, 0, 0, 0, 0, 7871, 0, 0, 8129, 4, 235, 157, 140, 0, 0, 12, 0, 130, 0, 0, 0, 0, 0, 223, 0, 0, 8192, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 7687, 0, 6970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 3, 0, 0, 7620, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1373, 0, 0, 0, 1685, 1447, 875, 590, 0, 0, 0, 0, 0, 5773, 0, 0, 0, 2, 244, 136, 0, 0, 0, 0, 0, 0, 0, 0, 138, 0, 0, 12, 0, 161, 0, 0, 2240, 2062, 1544, 0, 0, 0, 0, 0, 0, 0, 0, 168, 0, 0, 5952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 96, 0, 0, 0, 0, 0, 0, 6054, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 6099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 270, 3, 0, 0, 6018, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5970, 6155, 6157, 6120, 6151, 6100, 6208, 6291] avg=6144.0000 max=6291 max_rank=7 min=5970 min_rank=0 imbalance_ratio=1.023926 max_load_reduction=42861 max_load_reduction_pct=0.872009 moved_to_echo=24750 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1399, 0, 0, 0, 1679, 1446, 869, 589, 0, 0, 0, 0, 0, 5763, 0, 0, 0, 12, 229, 151, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 57, 0, 219, 0, 0, 2240, 2067, 1541, 0, 0, 0, 0, 0, 0, 0, 0, 223, 0, 0, 5952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 6063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 3, 0, 0, 5997, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5982, 6155, 6261, 6175, 6145, 6098, 6167, 6169] avg=6144.0000 max=6261 max_rank=2 min=5982 min_rank=0 imbalance_ratio=1.019043 max_load_reduction=42891 max_load_reduction_pct=0.872620 moved_to_echo=24579 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 0, 1687, 0, 0, 1, 1660, 1450, 871, 589, 0, 0, 0, 0, 5, 5763, 0, 0, 0, 10, 260, 126, 0, 0, 0, 0, 0, 0, 0, 0, 129, 0, 4, 9, 0, 154, 3, 1, 2238, 2062, 1541, 0, 0, 0, 0, 0, 0, 3, 2, 102, 2, 1, 5946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 72, 1, 6, 3, 0, 0, 0, 6052, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 1, 1, 1, 6112, 3, 0, 0, 0, 0, 0, 0, 0, 0, 6179, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 145, 1, 1, 0, 5932, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6261, 6164, 6141, 6056, 6135, 6131, 6181, 6083] avg=6144.0000 max=6261 max_rank=0 min=6056 min_rank=3 imbalance_ratio=1.019043 max_load_reduction=42891 max_load_reduction_pct=0.872620 moved_to_echo=24530 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[6, 1, 3, 6, 1709, 0, 0, 0, 1662, 1448, 860, 589, 0, 0, 0, 0, 6, 5753, 0, 0, 0, 19, 250, 133, 0, 0, 0, 0, 0, 0, 0, 0, 111, 0, 3, 73, 2, 46, 1, 0, 2237, 2062, 1539, 0, 0, 0, 0, 0, 0, 4, 0, 217, 4, 0, 5944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 5, 0, 7, 3, 0, 0, 6052, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 1, 6106, 4, 0, 0, 0, 0, 0, 0, 0, 0, 6101, 1, 0, 1, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 126, 5, 0, 4, 5939, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6284, 6161, 6074, 6169, 6150, 6126, 6110, 6078] avg=6144.0000 max=6284 max_rank=0 min=6074 min_rank=2 imbalance_ratio=1.022786 max_load_reduction=42868 max_load_reduction_pct=0.872152 moved_to_echo=24464 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 1555, 0, 0, 0, 1678, 1454, 847, 586, 0, 0, 0, 0, 2, 5729, 0, 0, 0, 34, 241, 140, 0, 0, 0, 0, 0, 0, 0, 0, 121, 0, 2, 69, 0, 154, 0, 0, 2239, 2066, 1532, 0, 0, 0, 0, 0, 0, 3, 1, 242, 1, 0, 5949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 2, 0, 1, 0, 0, 0, 6059, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 6131, 3, 0, 0, 0, 0, 0, 0, 0, 0, 6008, 5, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 195, 7, 1, 0, 5995, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6122, 6146, 6183, 6196, 6149, 6139, 6016, 6201] avg=6144.0000 max=6201 max_rank=7 min=6016 min_rank=6 imbalance_ratio=1.009277 max_load_reduction=42951 max_load_reduction_pct=0.873840 moved_to_echo=24505 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1536, 0, 0, 0, 1685, 1443, 874, 588, 0, 0, 0, 0, 0, 5753, 0, 0, 0, 25, 216, 145, 0, 0, 0, 0, 0, 0, 0, 0, 138, 0, 0, 58, 0, 148, 0, 0, 2240, 2067, 1538, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 0, 5952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 72, 0, 0, 0, 0, 0, 0, 6063, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 6083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 1, 0, 0, 6017, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6126, 6139, 6189, 6108, 6137, 6102, 6201, 6150] avg=6144.0000 max=6201 max_rank=6 min=6102 min_rank=5 imbalance_ratio=1.009277 max_load_reduction=42951 max_load_reduction_pct=0.873840 moved_to_echo=24590 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1493, 0, 0, 0, 1667, 1474, 861, 588, 0, 0, 0, 0, 0, 5746, 0, 0, 0, 4, 235, 157, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 0, 12, 0, 130, 0, 0, 2240, 2066, 1537, 0, 0, 0, 0, 0, 0, 0, 0, 223, 0, 0, 5952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 0, 0, 0, 6063, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 6213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 3, 0, 0, 5953, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6083, 6142, 6125, 6175, 6154, 6254, 6109, 6110] avg=6144.0000 max=6254 max_rank=5 min=6083 min_rank=0 imbalance_ratio=1.017904 max_load_reduction=42898 max_load_reduction_pct=0.872762 moved_to_echo=24627 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1752, 0, 0, 0, 1656, 1456, 870, 590, 0, 0, 0, 0, 0, 5712, 0, 0, 0, 4, 236, 138, 0, 0, 0, 0, 0, 0, 0, 0, 129, 0, 0, 15, 0, 39, 0, 0, 2244, 2063, 1532, 0, 0, 0, 0, 0, 0, 0, 0, 205, 0, 0, 5948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 6051, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 6122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 159, 14, 0, 0, 5897, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6324, 6090, 6022, 6153, 6131, 6202, 6160, 6070] avg=6144.0000 max=6324 max_rank=0 min=6022 min_rank=2 imbalance_ratio=1.029297 max_load_reduction=42828 max_load_reduction_pct=0.871338 moved_to_echo=24563 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=50176 observed_loss=-1024 tokens_per_expert=[1152, 0, 128, 384, 128, 1152, 128, 128, 17920, 16640, 12416, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 12544, 0, 0, 128, 13440, 11648, 7040, 4736, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=50176 slack_slots=48128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[128, 46080, 0, 0, 0, 128, 1920, 1152, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 0, 1408, 128, 128, 128, 47872, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 128, 1536, 128, 128, 47616, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 256, 0, 128, 128, 128, 49024, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[49152, 128, 0, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 768, 128, 128, 128, 128, 0, 0, 48512, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 347, 0, 0, 0, 8192, 5136, 0, 0, 0, 0, 0, 0, 0, 3913, 5903, 0, 0, 4383, 0, 0, 1290, 1969, 0, 0, 435, 0, 0, 0, 0, 0, 480, 0, 0, 71, 0, 0, 0, 0, 0, 7485, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 700, 0, 0, 0, 84, 0, 0, 0, 0, 576, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[2, 1, 1, 255, 0, 2, 0, 8187, 4870, 0, 1, 3, 0, 3, 0, 1, 3830, 6223, 4, 0, 4454, 0, 0, 1235, 1853, 0, 0, 519, 0, 0, 0, 0, 0, 533, 1, 0, 128, 0, 0, 0, 5, 1, 7563, 8185, 0, 0, 0, 3, 2, 0, 1, 0, 715, 1, 1, 0, 57, 0, 0, 0, 0, 512, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[1, 4, 0, 487, 0, 1, 0, 8184, 4910, 0, 1, 6, 4, 4, 0, 0, 4035, 6158, 4, 0, 4796, 0, 0, 947, 1758, 0, 0, 366, 0, 0, 0, 0, 0, 366, 3, 2, 75, 0, 0, 1, 5, 0, 7459, 8184, 0, 0, 0, 6, 0, 0, 0, 0, 752, 0, 2, 0, 53, 0, 0, 0, 0, 578, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1, 1, 0, 417, 2, 0, 0, 8179, 4952, 0, 4, 1, 8, 9, 0, 1, 3929, 6132, 9, 0, 4375, 0, 5, 1181, 1998, 0, 0, 438, 3, 0, 1, 0, 0, 392, 1, 0, 70, 0, 1, 0, 2, 4, 7488, 8141, 2, 0, 0, 4, 0, 0, 0, 1, 713, 4, 3, 1, 118, 0, 10, 0, 0, 551, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 280, 0, 0, 0, 8192, 4816, 0, 0, 0, 0, 0, 0, 0, 3833, 6178, 0, 0, 4544, 0, 0, 1070, 2111, 0, 0, 392, 0, 0, 0, 0, 0, 597, 0, 0, 103, 0, 0, 0, 0, 0, 7509, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 735, 0, 0, 0, 46, 0, 0, 0, 0, 554, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 225, 0, 0, 0, 8192, 5154, 0, 0, 0, 0, 0, 0, 0, 3967, 6106, 0, 0, 4238, 0, 0, 1241, 1950, 0, 0, 384, 0, 0, 0, 0, 0, 598, 0, 0, 83, 0, 0, 0, 0, 0, 7577, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 675, 0, 0, 0, 96, 0, 0, 0, 0, 495, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 322, 0, 0, 0, 8192, 5282, 0, 0, 0, 0, 0, 1, 0, 3562, 6326, 0, 0, 4392, 0, 0, 1123, 1886, 0, 0, 600, 0, 0, 0, 0, 0, 544, 0, 0, 72, 0, 0, 0, 0, 0, 7564, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 526, 0, 0, 0, 74, 0, 0, 0, 0, 510, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 209, 0, 0, 0, 8192, 5112, 0, 0, 0, 0, 0, 0, 0, 3841, 6273, 0, 0, 4158, 0, 0, 1257, 1924, 0, 0, 525, 0, 0, 0, 0, 0, 668, 0, 0, 110, 0, 0, 0, 0, 0, 7527, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 534, 0, 0, 0, 86, 0, 0, 0, 0, 555, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 347, 0, 0, 0, 5827, 0, 0, 0, 0, 0, 0, 0, 0, 5136, 0, 0, 0, 0, 0, 0, 0, 1103, 0, 0, 0, 0, 0, 0, 0, 3913, 0, 0, 0, 1102, 0, 0, 1290, 0, 0, 0, 0, 0, 0, 0, 0, 1969, 0, 0, 435, 0, 0, 0, 0, 2365, 1109, 271, 0, 0, 0, 0, 0, 0, 480, 0, 0, 71, 0, 0, 0, 5539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 700, 0, 0, 0, 3145, 2172, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 576, 0, 0, 2758, 2649, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6174, 6239, 6305, 6149, 6090, 6111, 6017, 6067] avg=6144.0000 max=6305 max_rank=2 min=6017 min_rank=6 imbalance_ratio=1.026204 max_load_reduction=42847 max_load_reduction_pct=0.871724 moved_to_echo=24285 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 4, 0, 487, 0, 1, 0, 5821, 0, 0, 0, 0, 0, 0, 0, 0, 4910, 0, 1, 6, 4, 4, 0, 0, 1099, 0, 0, 0, 0, 0, 0, 0, 4035, 0, 4, 0, 1206, 0, 0, 947, 0, 0, 0, 0, 0, 0, 0, 0, 1758, 0, 0, 366, 0, 0, 0, 0, 2363, 1213, 270, 0, 0, 0, 0, 0, 0, 366, 3, 2, 75, 0, 0, 1, 5536, 0, 0, 0, 0, 0, 0, 0, 5, 0, 6090, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 752, 0, 2, 0, 3281, 2377, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 578, 0, 0, 2877, 2648, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6314, 6024, 6192, 5970, 5983, 6101, 6412, 6156] avg=6144.0000 max=6412 max_rank=6 min=5970 min_rank=3 imbalance_ratio=1.043620 max_load_reduction=42740 max_load_reduction_pct=0.869548 moved_to_echo=24652 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 280, 0, 0, 0, 5827, 0, 0, 0, 0, 0, 0, 0, 0, 4816, 0, 0, 0, 0, 0, 0, 0, 1107, 0, 0, 0, 0, 0, 0, 0, 3833, 0, 0, 0, 1142, 0, 0, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 2111, 0, 0, 392, 0, 0, 0, 0, 2365, 1150, 271, 0, 0, 0, 0, 0, 0, 597, 0, 0, 103, 0, 0, 0, 5541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 735, 0, 0, 0, 3291, 2252, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 554, 0, 0, 2887, 2651, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6107, 5923, 6045, 6289, 6241, 6131, 6278, 6138] avg=6144.0000 max=6289 max_rank=3 min=5923 min_rank=1 imbalance_ratio=1.023600 max_load_reduction=42863 max_load_reduction_pct=0.872050 moved_to_echo=24788 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 0, 417, 2, 0, 0, 5818, 0, 0, 0, 0, 0, 0, 0, 0, 4952, 0, 4, 1, 8, 9, 0, 1, 1104, 0, 0, 0, 0, 0, 0, 0, 3929, 0, 9, 0, 1100, 0, 5, 1181, 0, 0, 0, 0, 0, 0, 0, 0, 1998, 0, 0, 438, 3, 0, 1, 0, 2361, 1107, 271, 0, 0, 0, 0, 0, 0, 392, 1, 0, 70, 0, 1, 0, 5507, 0, 0, 0, 0, 0, 0, 0, 2, 4, 6113, 0, 2, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 713, 4, 3, 1, 3267, 2168, 0, 0, 0, 0, 0, 0, 118, 0, 10, 0, 0, 551, 0, 0, 2865, 2634, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6239, 6079, 6224, 6179, 5971, 6125, 6157, 6178] avg=6144.0000 max=6239 max_rank=0 min=5971 min_rank=4 imbalance_ratio=1.015462 max_load_reduction=42913 max_load_reduction_pct=0.873067 moved_to_echo=24431 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2, 1, 1, 255, 0, 2, 0, 5824, 0, 0, 0, 0, 0, 0, 0, 0, 4870, 0, 1, 3, 0, 3, 0, 1, 1115, 0, 0, 0, 0, 0, 0, 0, 3830, 0, 4, 0, 1120, 0, 0, 1235, 0, 0, 0, 0, 0, 0, 0, 0, 1853, 0, 0, 519, 0, 0, 0, 0, 2363, 1127, 273, 0, 0, 0, 0, 0, 0, 533, 1, 0, 128, 0, 0, 0, 5537, 0, 0, 0, 0, 0, 0, 0, 5, 1, 6175, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 715, 1, 1, 0, 3316, 2207, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 512, 0, 0, 2907, 2648, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6085, 5993, 6189, 6135, 6199, 6184, 6243, 6124] avg=6144.0000 max=6243 max_rank=6 min=5993 min_rank=1 imbalance_ratio=1.016113 max_load_reduction=42909 max_load_reduction_pct=0.872986 moved_to_echo=24750 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 322, 0, 0, 0, 5827, 0, 0, 0, 0, 0, 0, 0, 0, 5282, 0, 0, 0, 0, 0, 1, 0, 1115, 0, 0, 0, 0, 0, 0, 0, 3562, 0, 0, 0, 1104, 0, 0, 1123, 0, 0, 0, 0, 0, 0, 0, 0, 1886, 0, 0, 600, 0, 0, 0, 0, 2365, 1111, 273, 0, 0, 0, 0, 0, 0, 544, 0, 0, 72, 0, 0, 0, 5531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 0, 0, 0, 3371, 2177, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 510, 0, 0, 2955, 2645, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6149, 6398, 5789, 6235, 6147, 6176, 6074, 6184] avg=6144.0000 max=6398 max_rank=1 min=5789 min_rank=2 imbalance_ratio=1.041341 max_load_reduction=42754 max_load_reduction_pct=0.869832 moved_to_echo=24581 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 225, 0, 0, 0, 5823, 0, 0, 0, 0, 0, 0, 0, 0, 5154, 0, 0, 0, 0, 0, 0, 0, 1121, 0, 0, 0, 0, 0, 0, 0, 3967, 0, 0, 0, 1057, 0, 0, 1241, 0, 0, 0, 0, 0, 0, 0, 0, 1950, 0, 0, 384, 0, 0, 0, 0, 2369, 1076, 278, 0, 0, 0, 0, 0, 0, 598, 0, 0, 83, 0, 0, 0, 5528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 675, 0, 0, 0, 3254, 2105, 0, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 495, 0, 0, 2852, 2643, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6048, 6275, 6265, 6057, 6209, 6178, 6034, 6086] avg=6144.0000 max=6275 max_rank=1 min=6034 min_rank=6 imbalance_ratio=1.021322 max_load_reduction=42877 max_load_reduction_pct=0.872335 moved_to_echo=24507 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 209, 0, 0, 0, 5827, 0, 0, 0, 0, 0, 0, 0, 0, 5112, 0, 0, 0, 0, 0, 0, 0, 1109, 0, 0, 0, 0, 0, 0, 0, 3841, 0, 0, 0, 1045, 0, 0, 1257, 0, 0, 0, 0, 0, 0, 0, 0, 1924, 0, 0, 525, 0, 0, 0, 0, 2365, 1052, 272, 0, 0, 0, 0, 0, 0, 668, 0, 0, 110, 0, 0, 0, 5534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 0, 0, 0, 3342, 2061, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 555, 0, 0, 2931, 2647, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6036, 6221, 6143, 6138, 6312, 6146, 5937, 6219] avg=6144.0000 max=6312 max_rank=4 min=5937 min_rank=6 imbalance_ratio=1.027344 max_load_reduction=42840 max_load_reduction_pct=0.871582 moved_to_echo=24614 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[640, 0, 128, 0, 0, 4352, 0, 0, 23040, 21248, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[15488, 0, 0, 3712, 128, 0, 128, 0, 18944, 8960, 2304, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 0, 128, 128, 5376, 128, 128, 128, 26368, 17536, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[30976, 0, 128, 0, 8960, 0, 128, 9344, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[40320, 0, 128, 128, 128, 128, 128, 128, 8960, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 4224, 128, 128, 768, 0, 128, 128, 44288, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 128, 2560, 128, 128, 0, 46720, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 49152, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 149, 0, 0, 0, 0, 0, 0, 0, 3449, 0, 0, 0, 0, 7868, 193, 3, 0, 0, 0, 0, 0, 0, 0, 7915, 0, 0, 0, 0, 8093, 0, 0, 0, 0, 0, 0, 0, 0, 6765, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5923, 0, 0, 0, 0, 0, 602, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 157, 0, 0, 0, 0, 0, 0, 0, 3798, 0, 0, 0, 0, 7976, 120, 0, 0, 0, 0, 0, 0, 0, 0, 7858, 0, 0, 0, 0, 8108, 0, 0, 0, 1, 0, 0, 0, 0, 6459, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5923, 0, 0, 0, 0, 0, 560, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 3, 7, 147, 4, 0, 1, 0, 3, 1, 0, 3649, 0, 0, 3, 0, 7959, 130, 18, 3, 3, 0, 3, 0, 1, 0, 7976, 0, 0, 2, 0, 8086, 6, 0, 0, 8, 0, 7, 0, 0, 6551, 0, 0, 8180, 0, 1, 0, 0, 1, 2, 0, 0, 0, 0, 5750, 0, 0, 3, 0, 1, 643, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 13, 221, 3, 0, 1, 4, 0, 0, 0, 3586, 1, 11, 0, 1, 7961, 139, 13, 7, 0, 0, 5, 1, 1, 0, 7936, 0, 0, 0, 1, 8080, 8, 0, 0, 0, 0, 2, 0, 0, 6377, 0, 0, 8169, 0, 2, 0, 0, 3, 0, 0, 1, 0, 2, 5965, 0, 0, 11, 0, 0, 621, 0, 2, 4] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 145, 0, 0, 0, 0, 0, 0, 0, 3443, 0, 0, 0, 0, 7916, 192, 0, 0, 0, 0, 0, 0, 0, 0, 7923, 0, 0, 0, 0, 8097, 0, 0, 0, 0, 0, 0, 0, 0, 6607, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6003, 0, 0, 0, 0, 0, 634, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 5, 176, 1, 0, 3, 1, 0, 1, 0, 3824, 1, 0, 0, 0, 7837, 187, 3, 0, 1, 0, 3, 0, 1, 0, 7919, 0, 0, 1, 0, 8073, 4, 0, 0, 3, 0, 0, 0, 0, 6387, 0, 0, 8185, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 5892, 0, 0, 4, 0, 0, 634, 0, 0, 3] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 161, 0, 0, 0, 0, 0, 0, 0, 3606, 0, 0, 0, 0, 7884, 184, 1, 0, 0, 0, 0, 0, 0, 0, 7880, 0, 0, 0, 0, 8107, 0, 0, 0, 17, 0, 0, 0, 0, 6496, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6050, 0, 0, 0, 0, 0, 574, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 129, 0, 0, 0, 0, 0, 0, 0, 3698, 0, 0, 0, 0, 7907, 164, 0, 0, 0, 0, 0, 0, 0, 0, 7968, 0, 0, 0, 0, 8092, 0, 0, 0, 6, 0, 0, 0, 0, 6517, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5818, 0, 0, 0, 0, 0, 661, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 149, 0, 0, 0, 0, 3927, 2051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3449, 0, 0, 0, 0, 1777, 578, 154, 0, 0, 0, 0, 0, 5939, 193, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6141, 0, 0, 0, 0, 0, 0, 0, 6376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5923, 0, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 602, 0, 0, 0, 4166, 1351, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6127, 5958, 6135, 6138, 6141, 6376, 6158, 6119] avg=6144.0000 max=6376 max_rank=5 min=5958 min_rank=1 imbalance_ratio=1.037760 max_load_reduction=42776 max_load_reduction_pct=0.870280 moved_to_echo=24794 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 7, 147, 4, 0, 1, 0, 3924, 2048, 0, 0, 0, 0, 0, 0, 3, 1, 0, 3649, 0, 0, 3, 0, 1790, 584, 149, 0, 0, 0, 0, 0, 6008, 130, 18, 3, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6186, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 8, 0, 7, 0, 0, 6132, 0, 0, 0, 0, 0, 0, 0, 6174, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 5750, 0, 228, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 643, 0, 0, 0, 4162, 1367, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6134, 6179, 6165, 6189, 6153, 6175, 5981, 6176] avg=6144.0000 max=6189 max_rank=3 min=5981 min_rank=6 imbalance_ratio=1.007324 max_load_reduction=42963 max_load_reduction_pct=0.874084 moved_to_echo=24485 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 145, 0, 0, 0, 0, 3930, 2052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3443, 0, 0, 0, 0, 1779, 581, 151, 0, 0, 0, 0, 0, 5976, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6140, 0, 0, 0, 0, 0, 0, 0, 6226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6003, 0, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 634, 0, 0, 0, 4167, 1359, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6127, 5954, 6168, 6144, 6140, 6226, 6233, 6160] avg=6144.0000 max=6233 max_rank=6 min=5954 min_rank=1 imbalance_ratio=1.014486 max_load_reduction=42919 max_load_reduction_pct=0.873189 moved_to_echo=24759 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 13, 221, 3, 0, 1, 4, 3921, 2046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3586, 1, 11, 0, 1, 1781, 585, 145, 0, 0, 0, 0, 0, 6009, 139, 13, 7, 0, 0, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6155, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 2, 0, 0, 6123, 0, 0, 0, 0, 0, 0, 0, 6010, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 2, 5965, 0, 222, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 621, 0, 2, 4, 4159, 1367, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6209, 6110, 6174, 6157, 6133, 6012, 6193, 6164] avg=6144.0000 max=6209 max_rank=0 min=6012 min_rank=5 imbalance_ratio=1.010579 max_load_reduction=42943 max_load_reduction_pct=0.873678 moved_to_echo=24502 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 5, 176, 1, 0, 3, 1, 3918, 2051, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3824, 1, 0, 0, 0, 1778, 576, 146, 0, 0, 0, 0, 0, 5915, 187, 3, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6141, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 3, 0, 0, 0, 0, 6134, 0, 0, 0, 0, 0, 0, 0, 6019, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5892, 0, 222, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 634, 0, 0, 3, 4155, 1346, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6155, 6326, 6109, 6143, 6141, 6021, 6115, 6142] avg=6144.0000 max=6326 max_rank=1 min=6021 min_rank=5 imbalance_ratio=1.029622 max_load_reduction=42826 max_load_reduction_pct=0.871297 moved_to_echo=24419 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 157, 0, 0, 0, 0, 3935, 2052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3798, 0, 0, 0, 0, 1764, 586, 147, 0, 0, 0, 0, 0, 6020, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 6140, 0, 0, 0, 0, 0, 0, 0, 6087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5923, 0, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 560, 0, 0, 0, 4173, 1370, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6144, 6295, 6140, 6094, 6141, 6087, 6148, 6103] avg=6144.0000 max=6295 max_rank=1 min=6087 min_rank=5 imbalance_ratio=1.024577 max_load_reduction=42857 max_load_reduction_pct=0.871928 moved_to_echo=24479 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 161, 0, 0, 0, 0, 3935, 2052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3606, 0, 0, 0, 0, 1772, 582, 153, 0, 0, 0, 0, 0, 5944, 184, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 6140, 0, 0, 0, 0, 0, 0, 0, 6114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6050, 0, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 574, 0, 0, 0, 4172, 1358, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6148, 6113, 6129, 6108, 6157, 6114, 6279, 6104] avg=6144.0000 max=6279 max_rank=6 min=6104 min_rank=7 imbalance_ratio=1.021973 max_load_reduction=42873 max_load_reduction_pct=0.872253 moved_to_echo=24654 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 129, 0, 0, 0, 0, 3927, 2052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3698, 0, 0, 0, 0, 1789, 581, 149, 0, 0, 0, 0, 0, 5968, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 6140, 0, 0, 0, 0, 0, 0, 0, 6141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5818, 0, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 661, 0, 0, 0, 4165, 1358, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6108, 6217, 6132, 6179, 6146, 6141, 6045, 6184] avg=6144.0000 max=6217 max_rank=1 min=6045 min_rank=6 imbalance_ratio=1.011882 max_load_reduction=42935 max_load_reduction_pct=0.873515 moved_to_echo=24516 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[0, 128, 128, 1408, 128, 0, 128, 128, 31488, 16512, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=50176 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 29056, 128, 128, 128, 128, 14336, 4736, 1280, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 128, 0, 128, 4992, 0, 128, 128, 33408, 10880, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[49152, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 0, 128, 47360, 0, 1920, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 0, 49152, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[47872, 1408, 128, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 0, 0, 128, 0, 128, 0, 0, 49152, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=50176 slack_slots=48128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=17 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2091, 0, 0, 0, 0, 0, 2379, 0, 0, 0, 0, 0, 0, 0, 0, 8030, 0, 0, 0, 618, 0, 0, 0, 0, 0, 0, 0, 0, 835, 0, 0, 0, 7987, 8192, 2656, 0, 0, 0, 0, 0, 61, 0, 8192, 0, 0, 0, 8111, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=17 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1627, 0, 0, 0, 0, 0, 2796, 0, 0, 0, 0, 0, 0, 0, 0, 8041, 0, 0, 0, 587, 0, 0, 0, 0, 0, 0, 0, 0, 720, 0, 0, 0, 7972, 8192, 2832, 0, 0, 0, 0, 0, 77, 0, 8192, 0, 0, 0, 8116, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=17 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 2, 1, 0, 0, 1785, 2, 0, 0, 0, 2, 2510, 0, 0, 1, 0, 0, 0, 1, 0, 8033, 0, 0, 1, 512, 1, 6, 5, 2, 0, 4, 0, 0, 648, 1, 6, 0, 7958, 8186, 3112, 0, 0, 1, 2, 0, 68, 0, 8183, 0, 0, 0, 8108, 4, 6, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=17 tokens_per_expert_current_ep_rank=[1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1, 1897, 0, 0, 0, 0, 2, 2719, 0, 0, 0, 1, 0, 3, 3, 0, 8011, 0, 0, 0, 563, 0, 4, 6, 3, 0, 2, 0, 0, 558, 0, 0, 0, 7994, 8185, 2858, 0, 0, 3, 0, 0, 46, 0, 8185, 0, 0, 0, 8098, 1, 4, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=17 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1868, 0, 0, 0, 0, 0, 2717, 0, 0, 0, 0, 0, 0, 0, 0, 8051, 0, 0, 0, 538, 0, 0, 0, 0, 0, 0, 0, 0, 680, 0, 0, 0, 8020, 8192, 2731, 0, 0, 0, 0, 0, 45, 0, 8192, 0, 0, 0, 8118, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=17 tokens_per_expert_current_ep_rank=[2, 2, 0, 0, 2, 2, 1, 1, 1, 0, 1, 1994, 3, 1, 0, 0, 7, 2850, 0, 0, 0, 0, 0, 8, 5, 1, 8006, 1, 0, 1, 553, 0, 13, 8, 3, 0, 8, 1, 0, 575, 0, 7, 1, 7968, 8178, 2574, 0, 1, 2, 0, 1, 100, 0, 8171, 0, 0, 0, 8082, 3, 11, 0, 1, 1, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=17 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2026, 0, 0, 0, 0, 0, 2481, 0, 0, 0, 0, 0, 0, 0, 0, 8036, 0, 0, 0, 533, 0, 0, 0, 0, 0, 0, 0, 0, 654, 0, 0, 0, 7998, 8192, 2868, 0, 0, 0, 0, 0, 58, 0, 8192, 0, 0, 0, 8114, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=17 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1793, 0, 0, 0, 0, 0, 2458, 0, 0, 0, 0, 0, 0, 0, 0, 8048, 0, 0, 0, 499, 0, 0, 0, 0, 0, 0, 0, 0, 692, 0, 0, 0, 7984, 8192, 3095, 0, 0, 0, 0, 0, 79, 0, 8192, 0, 0, 0, 8120, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=17 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1627, 0, 0, 0, 0, 2442, 1262, 552, 0, 0, 0, 0, 0, 0, 2796, 0, 0, 0, 0, 0, 0, 1970, 1559, 0, 0, 0, 0, 0, 0, 0, 0, 5599, 0, 0, 0, 587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 720, 3413, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3297, 0, 2832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 6081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6144, 5883, 6325, 6186, 6181, 6129, 6158, 6146] avg=6144.0000 max=6325 max_rank=2 min=5883 min_rank=1 imbalance_ratio=1.029460 max_load_reduction=42827 max_load_reduction_pct=0.871318 moved_to_echo=24614 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=17 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 2, 6139, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1785, 2, 0, 0, 0, 2440, 1260, 552, 0, 0, 0, 0, 0, 2, 2510, 0, 0, 1, 0, 0, 0, 1968, 1558, 0, 0, 0, 0, 0, 0, 1, 0, 5593, 0, 0, 1, 512, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 5, 2, 0, 4, 0, 0, 648, 3407, 2047, 0, 0, 0, 0, 0, 0, 1, 6, 0, 3291, 0, 3112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 68, 0, 6073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6140, 4, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6142, 6040, 6039, 6108, 6119, 6410, 6144, 6150] avg=6144.0000 max=6410 max_rank=5 min=6039 min_rank=2 imbalance_ratio=1.043294 max_load_reduction=42742 max_load_reduction_pct=0.869588 moved_to_echo=24823 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=17 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2091, 0, 0, 0, 0, 2439, 1265, 552, 0, 0, 0, 0, 0, 0, 2379, 0, 0, 0, 0, 0, 0, 1969, 1559, 0, 0, 0, 0, 0, 0, 0, 0, 5591, 0, 0, 0, 618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 835, 3419, 2047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3303, 0, 2656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 6081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 6347, 5907, 6209, 6301, 5959, 6142, 6142] avg=6144.0000 max=6347 max_rank=1 min=5907 min_rank=2 imbalance_ratio=1.033040 max_load_reduction=42805 max_load_reduction_pct=0.870870 moved_to_echo=24544 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=17 rerouted_tokens_per_expert_current_ep_rank=[2, 2, 0, 0, 2, 2, 1, 1, 6133, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1994, 3, 1, 0, 0, 2432, 1262, 551, 0, 0, 0, 0, 0, 7, 2850, 0, 0, 0, 0, 0, 8, 1961, 1555, 0, 0, 0, 0, 0, 0, 5, 1, 5574, 1, 0, 1, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 8, 3, 0, 8, 1, 0, 575, 3411, 2045, 0, 0, 0, 0, 0, 0, 0, 7, 1, 3295, 0, 2574, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 100, 0, 6065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6121, 3, 11, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6245, 6381, 6135, 6064, 5878, 6168, 6138] avg=6144.0000 max=6381 max_rank=2 min=5878 min_rank=5 imbalance_ratio=1.038574 max_load_reduction=42771 max_load_reduction_pct=0.870178 moved_to_echo=24248 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=17 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 0, 0, 1, 0, 1, 6140, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1897, 0, 0, 0, 0, 2433, 1266, 552, 0, 0, 0, 0, 0, 2, 2719, 0, 0, 0, 1, 0, 3, 1965, 1558, 0, 0, 0, 0, 0, 0, 3, 0, 5578, 0, 0, 0, 563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 6, 3, 0, 2, 0, 0, 558, 3422, 2045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3306, 0, 2858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 46, 0, 6075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6133, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6144, 6150, 6248, 6144, 6040, 6164, 6124, 6138] avg=6144.0000 max=6248 max_rank=2 min=6040 min_rank=4 imbalance_ratio=1.016927 max_load_reduction=42904 max_load_reduction_pct=0.872884 moved_to_echo=24466 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=17 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1868, 0, 0, 0, 0, 2445, 1270, 552, 0, 0, 0, 0, 0, 0, 2717, 0, 0, 0, 0, 0, 0, 1970, 1559, 0, 0, 0, 0, 0, 0, 0, 0, 5606, 0, 0, 0, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 680, 3433, 2047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3317, 0, 2731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 6081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 6135, 6246, 6144, 6160, 6048, 6126, 6148] avg=6144.0000 max=6246 max_rank=2 min=6048 min_rank=5 imbalance_ratio=1.016602 max_load_reduction=42906 max_load_reduction_pct=0.872925 moved_to_echo=24482 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=17 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2026, 0, 0, 0, 0, 2446, 1271, 556, 0, 0, 0, 0, 0, 0, 2481, 0, 0, 0, 0, 0, 0, 1973, 1564, 0, 0, 0, 0, 0, 0, 0, 0, 5590, 0, 0, 0, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 654, 3428, 2047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3299, 0, 2868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 6072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 6299, 6018, 6123, 6129, 6167, 6130, 6141] avg=6144.0000 max=6299 max_rank=1 min=6018 min_rank=2 imbalance_ratio=1.025228 max_load_reduction=42853 max_load_reduction_pct=0.871847 moved_to_echo=24567 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=17 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1793, 0, 0, 0, 0, 2444, 1264, 552, 0, 0, 0, 0, 0, 0, 2458, 0, 0, 0, 0, 0, 0, 1971, 1559, 0, 0, 0, 0, 0, 0, 0, 0, 5604, 0, 0, 0, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 692, 3418, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3302, 0, 3095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 6081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6144, 6053, 5988, 6103, 6158, 6397, 6160, 6149] avg=6144.0000 max=6397 max_rank=5 min=5988 min_rank=2 imbalance_ratio=1.041178 max_load_reduction=42755 max_load_reduction_pct=0.869853 moved_to_echo=24864 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 0, 128, 15104, 128, 128, 0, 0, 19584, 10240, 4480, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 20992, 0, 0, 128, 128, 0, 128, 15872, 12544, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 128, 26496, 0, 22784, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 44800, 128, 0, 128, 4480, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 128, 128, 0, 5376, 27392, 16384, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 49152, 128, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 128, 640, 0, 48640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 128, 128, 0, 128, 128, 128, 128, 49152, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4617, 0, 0, 0, 0, 1849, 0, 0, 0, 0, 8192, 8178, 0, 0, 0, 0, 3752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6743, 8192, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=18 tokens_per_expert_current_ep_rank=[1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 0, 0, 0, 0, 0, 0, 7598, 0, 6, 0, 17, 0, 0, 0, 6, 0, 0, 4290, 3, 2, 1, 0, 1771, 0, 2, 0, 0, 8180, 8153, 0, 9, 5, 0, 4363, 0, 0, 0, 0, 0, 2, 0, 3, 2, 0, 0, 6545, 8184, 1, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 0, 0, 1, 3, 7628, 0, 2, 0, 5, 0, 0, 0, 4, 0, 0, 4773, 0, 0, 0, 0, 1768, 0, 1, 0, 0, 8185, 8185, 0, 5, 1, 0, 3583, 5, 0, 0, 0, 0, 5, 1, 0, 0, 0, 0, 6804, 8188, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4795, 0, 0, 0, 0, 1857, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 3663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6683, 8192, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 5, 11, 0, 0, 0, 1, 0, 9, 7563, 1, 0, 0, 8, 0, 0, 0, 14, 0, 0, 4749, 2, 0, 0, 0, 2059, 1, 0, 0, 0, 8172, 8171, 3, 14, 6, 0, 3605, 29, 0, 0, 0, 0, 10, 1, 2, 0, 0, 0, 6527, 8179, 0, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4971, 0, 0, 0, 0, 1729, 0, 0, 0, 0, 8192, 8191, 0, 0, 0, 0, 3825, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6416, 8191, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4772, 0, 0, 0, 0, 1867, 0, 0, 0, 0, 8192, 8186, 0, 0, 0, 0, 3544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6782, 8192, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4756, 0, 0, 0, 0, 1895, 0, 0, 0, 0, 8192, 8191, 0, 0, 0, 0, 3881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6390, 8192, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=18 rerouted_tokens_per_expert_current_ep_rank=[1, 2, 0, 0, 0, 0, 0, 0, 6139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 0, 4090, 2045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6130, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 6, 0, 0, 4090, 2035, 2, 0, 0, 0, 0, 0, 3906, 3, 2, 1, 0, 1771, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6118, 0, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4363, 0, 0, 0, 0, 0, 2, 0, 1466, 463, 384, 0, 0, 0, 0, 0, 3, 2, 0, 0, 6082, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6142, 6141, 6136, 6150, 5685, 6132, 6678, 6088] avg=6144.0000 max=6678 max_rank=6 min=5685 min_rank=4 imbalance_ratio=1.086914 max_load_reduction=42474 max_load_reduction_pct=0.864136 moved_to_echo=24583 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4097, 2047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2041, 2, 0, 0, 0, 0, 0, 4204, 0, 0, 0, 0, 1849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3752, 0, 0, 0, 0, 0, 0, 0, 1472, 477, 413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 6144, 6155, 6138, 6053, 6137, 6114, 6266] avg=6144.0000 max=6266 max_rank=7 min=6053 min_rank=4 imbalance_ratio=1.019857 max_load_reduction=42886 max_load_reduction_pct=0.872518 moved_to_echo=24570 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8, 0, 0, 6135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 11, 0, 4087, 2044, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 9, 6102, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 14, 0, 0, 4085, 2039, 2, 0, 0, 0, 0, 0, 4324, 2, 0, 0, 0, 2059, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6132, 3, 14, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3605, 29, 0, 0, 0, 0, 10, 1, 1459, 462, 425, 0, 0, 0, 0, 0, 2, 0, 0, 0, 6065, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6147, 6113, 6148, 6386, 6155, 5991, 6069] avg=6144.0000 max=6386 max_rank=4 min=5991 min_rank=6 imbalance_ratio=1.039388 max_load_reduction=42766 max_load_reduction_pct=0.870076 moved_to_echo=24601 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 4094, 2046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 6154, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4, 0, 0, 4091, 2043, 2, 0, 0, 0, 0, 0, 4346, 0, 0, 0, 0, 1768, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6142, 0, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3583, 5, 0, 0, 0, 0, 5, 1, 1472, 482, 427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6142, 6145, 6160, 6145, 6115, 6148, 5975, 6322] avg=6144.0000 max=6322 max_rank=7 min=5975 min_rank=6 imbalance_ratio=1.028971 max_load_reduction=42830 max_load_reduction_pct=0.871379 moved_to_echo=24560 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4097, 2047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2043, 2, 0, 0, 0, 0, 0, 4345, 0, 0, 0, 0, 1867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3544, 0, 0, 0, 0, 0, 0, 0, 1470, 480, 427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 6144, 6145, 6140, 6212, 6143, 5921, 6302] avg=6144.0000 max=6302 max_rank=7 min=5921 min_rank=6 imbalance_ratio=1.025716 max_load_reduction=42850 max_load_reduction_pct=0.871785 moved_to_echo=24578 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4097, 2047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2045, 2, 0, 0, 0, 0, 0, 4366, 0, 0, 0, 0, 1857, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3663, 0, 0, 0, 0, 0, 0, 0, 1462, 473, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 6144, 6114, 6142, 6223, 6147, 6027, 6210] avg=6144.0000 max=6223 max_rank=4 min=6027 min_rank=6 imbalance_ratio=1.012858 max_load_reduction=42929 max_load_reduction_pct=0.873393 moved_to_echo=24607 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4097, 2046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2048, 5, 0, 0, 0, 0, 0, 4523, 0, 0, 0, 0, 1729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3825, 0, 0, 0, 0, 0, 0, 0, 1479, 459, 448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 6143, 6153, 6148, 6252, 6143, 6211, 5957] avg=6144.0000 max=6252 max_rank=4 min=5957 min_rank=7 imbalance_ratio=1.017578 max_load_reduction=42900 max_load_reduction_pct=0.872803 moved_to_echo=24563 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4097, 2047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2044, 2, 0, 0, 0, 0, 0, 4331, 0, 0, 0, 0, 1895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3881, 0, 0, 0, 0, 0, 0, 0, 1477, 452, 425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 6144, 6176, 6141, 6226, 6147, 6235, 5938] avg=6144.0000 max=6235 max_rank=6 min=5938 min_rank=7 imbalance_ratio=1.014811 max_load_reduction=42917 max_load_reduction_pct=0.873149 moved_to_echo=24546 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[30336, 128, 0, 0, 0, 0, 128, 128, 11776, 3840, 3456, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[34432, 128, 128, 128, 0, 14848, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 0, 128, 0, 0, 32768, 16384, 128, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 0, 0, 49152, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 128, 128, 0, 32768, 16384, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 0, 49152, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 128, 0, 0, 0, 128, 0, 0, 49152, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 0, 128, 128, 128, 49152, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=19 tokens_per_expert_current_ep_rank=[8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 7508, 7904, 0, 4524, 0, 8128, 0, 0, 0, 0, 0, 0, 0, 245, 0, 0, 0, 0, 0, 0, 0, 2617, 0, 490, 1, 0, 0, 0, 0, 0, 424, 0, 430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 317, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=19 tokens_per_expert_current_ep_rank=[8175, 0, 11, 0, 15, 0, 0, 8178, 0, 0, 0, 0, 0, 11, 0, 7444, 7810, 4, 4655, 0, 8082, 0, 0, 0, 0, 0, 2, 0, 235, 12, 0, 0, 0, 3, 1, 0, 2688, 0, 371, 1, 2, 0, 0, 8, 23, 398, 0, 469, 0, 0, 0, 1, 0, 13, 0, 0, 9, 119, 393, 0, 2, 13, 0, 4] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=19 tokens_per_expert_current_ep_rank=[8184, 0, 5, 0, 8, 0, 0, 8183, 0, 0, 0, 0, 0, 1, 0, 7579, 7895, 1, 4836, 0, 8079, 1, 3, 0, 0, 0, 1, 0, 243, 4, 0, 0, 0, 1, 7, 0, 2539, 0, 274, 4, 2, 0, 0, 6, 5, 343, 0, 493, 0, 0, 0, 0, 0, 5, 0, 0, 1, 134, 301, 0, 5, 7, 0, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=19 tokens_per_expert_current_ep_rank=[8187, 0, 3, 0, 4, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 1, 7396, 7787, 2, 4863, 0, 8107, 0, 3, 0, 0, 0, 1, 0, 250, 1, 0, 0, 0, 2, 0, 0, 2583, 0, 354, 0, 1, 3, 0, 4, 2, 295, 0, 474, 0, 0, 1, 0, 1, 3, 0, 0, 2, 208, 414, 0, 5, 4, 0, 5] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=19 tokens_per_expert_current_ep_rank=[8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7593, 7839, 0, 4778, 0, 8137, 0, 0, 0, 0, 0, 0, 0, 255, 0, 0, 0, 0, 0, 0, 0, 2520, 0, 365, 1, 0, 0, 0, 0, 0, 341, 0, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 262, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=19 tokens_per_expert_current_ep_rank=[8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7687, 7893, 0, 4107, 0, 8139, 0, 0, 0, 0, 0, 0, 0, 257, 0, 0, 0, 0, 0, 0, 0, 3044, 0, 453, 0, 0, 0, 0, 0, 0, 309, 0, 572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 124, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=19 tokens_per_expert_current_ep_rank=[8192, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 7512, 7915, 0, 4568, 0, 8121, 0, 0, 0, 0, 0, 0, 0, 253, 0, 0, 0, 0, 0, 0, 0, 2637, 0, 528, 0, 0, 0, 0, 0, 0, 362, 0, 421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 353, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=19 tokens_per_expert_current_ep_rank=[8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7482, 7857, 0, 4929, 0, 8124, 0, 0, 0, 0, 0, 1, 0, 251, 0, 0, 0, 0, 0, 0, 0, 2408, 0, 411, 2, 0, 0, 0, 0, 0, 257, 0, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 354, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 5, 0, 8, 0, 0, 6135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6187, 0, 0, 0, 0, 0, 0, 0, 0, 1492, 1, 4836, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 243, 4, 0, 0, 3828, 2046, 0, 0, 0, 0, 0, 0, 0, 1, 7, 0, 2539, 0, 274, 4, 1722, 1392, 0, 0, 0, 0, 0, 0, 2, 0, 0, 6, 5, 343, 0, 493, 5001, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 6138, 0, 0, 0, 0, 0, 0, 0, 1, 134, 301, 0, 5, 7, 0, 2, 4251, 1402, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6148, 6188, 6333, 6122, 5939, 6176, 6143, 6103] avg=6144.0000 max=6333 max_rank=2 min=5939 min_rank=4 imbalance_ratio=1.030762 max_load_reduction=42819 max_load_reduction_pct=0.871155 moved_to_echo=24361 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 11, 0, 15, 0, 0, 6131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 6077, 0, 0, 0, 0, 0, 0, 0, 0, 1476, 4, 4655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 235, 12, 0, 0, 3829, 2043, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 2688, 0, 371, 1, 1721, 1367, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8, 23, 398, 0, 469, 4947, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 13, 0, 0, 6132, 0, 0, 0, 0, 0, 0, 0, 9, 119, 393, 0, 2, 13, 0, 4, 4253, 1387, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6157, 6088, 6135, 6121, 6152, 6173, 6146, 6180] avg=6144.0000 max=6180 max_rank=7 min=6088 min_rank=1 imbalance_ratio=1.005859 max_load_reduction=42972 max_load_reduction_pct=0.874268 moved_to_echo=24651 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 6142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6129, 0, 0, 0, 0, 0, 0, 0, 0, 1494, 0, 4524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, 0, 0, 0, 3852, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2617, 0, 490, 1, 1723, 1379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 424, 0, 430, 5007, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 181, 317, 0, 0, 0, 0, 0, 4276, 1403, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6142, 6129, 6018, 6145, 6210, 6187, 6144, 6177] avg=6144.0000 max=6210 max_rank=4 min=6018 min_rank=2 imbalance_ratio=1.010742 max_load_reduction=42942 max_load_reduction_pct=0.873657 moved_to_echo=24718 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6198, 0, 0, 0, 0, 0, 0, 0, 0, 1481, 0, 4778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 0, 0, 0, 3856, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2520, 0, 365, 1, 1723, 1395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 341, 0, 488, 4966, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 189, 262, 0, 0, 0, 0, 0, 4281, 1392, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6198, 6259, 6159, 6004, 6121, 6144, 6124] avg=6144.0000 max=6259 max_rank=2 min=6004 min_rank=4 imbalance_ratio=1.018717 max_load_reduction=42893 max_load_reduction_pct=0.872660 moved_to_echo=24393 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 4, 0, 0, 6138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6037, 0, 0, 0, 0, 0, 0, 0, 0, 1471, 2, 4863, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 250, 1, 0, 0, 3842, 2047, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2583, 0, 354, 0, 1722, 1359, 0, 0, 0, 0, 0, 0, 1, 3, 0, 4, 2, 295, 0, 474, 4933, 326, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 3, 0, 0, 6140, 0, 0, 0, 0, 0, 0, 0, 2, 208, 414, 0, 5, 4, 0, 5, 4265, 1383, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 6038, 6339, 6141, 6020, 6038, 6145, 6286] avg=6144.0000 max=6339 max_rank=2 min=6020 min_rank=4 imbalance_ratio=1.031738 max_load_reduction=42813 max_load_reduction_pct=0.871033 moved_to_echo=24489 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 6141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6132, 0, 0, 0, 0, 0, 0, 0, 0, 1496, 0, 4568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 253, 0, 0, 0, 3849, 2049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2637, 0, 528, 0, 1723, 1380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 362, 0, 421, 5014, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 100, 353, 0, 0, 0, 0, 0, 4272, 1405, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6141, 6132, 6064, 6151, 6268, 6123, 6143, 6130] avg=6144.0000 max=6268 max_rank=4 min=6064 min_rank=2 imbalance_ratio=1.020182 max_load_reduction=42884 max_load_reduction_pct=0.872477 moved_to_echo=24664 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 6133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6271, 0, 0, 0, 0, 0, 0, 0, 0, 1483, 0, 4107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 257, 0, 0, 0, 3857, 2049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3044, 0, 453, 0, 1728, 1416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 309, 0, 572, 5005, 331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 183, 124, 0, 0, 0, 0, 0, 4282, 1405, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6133, 6271, 5590, 6163, 6641, 6217, 6143, 5994] avg=6144.0000 max=6641 max_rank=4 min=5590 min_rank=2 imbalance_ratio=1.080892 max_load_reduction=42511 max_load_reduction_pct=0.864889 moved_to_echo=24995 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6108, 0, 0, 0, 0, 0, 0, 0, 0, 1485, 0, 4929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 251, 0, 0, 0, 3850, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2408, 0, 411, 2, 1723, 1374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 257, 0, 557, 4977, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 135, 354, 0, 0, 0, 0, 0, 4274, 1395, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6108, 6414, 6150, 5918, 6117, 6144, 6158] avg=6144.0000 max=6414 max_rank=2 min=5918 min_rank=4 imbalance_ratio=1.043945 max_load_reduction=42738 max_load_reduction_pct=0.869507 moved_to_echo=24337 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 2048, 128, 0, 0, 30848, 16384, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 21120, 0, 3328, 128, 13824, 11136, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 1280, 2560, 0, 128, 128, 0, 128, 34176, 11264, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[11904, 128, 37376, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 128, 2816, 0, 3968, 39936, 2688, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 128, 0, 128, 0, 0, 49152, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 128, 128, 128, 128, 0, 0, 49152, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 128, 128, 49152, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4274, 0, 0, 0, 8192, 0, 0, 2, 0, 0, 7984, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 4, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8013, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 4, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4572, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 7923, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3879, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8192, 0, 0, 13, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 0, 4072, 0, 2, 0, 8184, 0, 0, 0, 2, 0, 7942, 3, 0, 0, 0, 8183, 0, 0, 0, 0, 8187, 5, 0, 0, 1, 0, 0, 0, 1, 2, 0, 0, 4363, 0, 0, 3, 0, 0, 0, 0, 0, 2, 0, 0, 8187, 0, 5, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 6, 0, 1, 0, 0, 0, 0, 0, 0, 0, 5, 1, 4236, 3, 2, 0, 8180, 0, 0, 0, 3, 0, 7857, 4, 0, 0, 0, 8177, 0, 0, 0, 0, 8184, 7, 0, 0, 0, 2, 0, 0, 0, 6, 0, 2, 4225, 0, 0, 6, 0, 0, 0, 0, 0, 22, 2, 0, 8184, 0, 7, 30, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 13, 0, 2, 0, 0, 0, 0, 2, 0, 0, 1, 0, 4167, 1, 1, 0, 8170, 0, 0, 0, 9, 0, 7926, 12, 0, 0, 0, 8172, 0, 0, 0, 2, 8177, 15, 0, 0, 2, 0, 2, 0, 0, 15, 1, 0, 4217, 0, 0, 4, 1, 0, 1, 0, 0, 9, 2, 0, 8177, 0, 15, 36, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4396, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8002, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 11, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4245, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 7871, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4238, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 8192, 0, 0, 25, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4274, 0, 1808, 91, 0, 0, 0, 0, 0, 0, 0, 0, 6146, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4120, 0, 1955, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2064, 2046, 2029, 0, 0, 0, 0, 0, 0, 0, 6128, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6146, 6173, 6148, 6176, 6144, 6094, 6139, 6132] avg=6144.0000 max=6176 max_rank=3 min=6094 min_rank=5 imbalance_ratio=1.005208 max_load_reduction=42976 max_load_reduction_pct=0.874349 moved_to_echo=24509 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4572, 0, 1794, 91, 0, 0, 0, 0, 0, 0, 0, 0, 6146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3879, 0, 1955, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2064, 2045, 2029, 0, 0, 0, 0, 0, 0, 0, 6128, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6457, 6146, 6129, 6144, 5853, 6139, 6141] avg=6144.0000 max=6457 max_rank=1 min=5853 min_rank=5 imbalance_ratio=1.050944 max_load_reduction=42695 max_load_reduction_pct=0.868632 moved_to_echo=24277 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6, 0, 1, 0, 0, 0, 6134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 4236, 3, 1779, 91, 0, 0, 0, 0, 0, 0, 2, 0, 6137, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6078, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6138, 7, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2, 4225, 0, 1952, 19, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 22, 2062, 2043, 2027, 0, 0, 0, 0, 0, 2, 0, 6122, 0, 7, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6141, 6115, 6142, 6082, 6147, 6204, 6160, 6161] avg=6144.0000 max=6204 max_rank=5 min=6082 min_rank=3 imbalance_ratio=1.009766 max_load_reduction=42948 max_load_reduction_pct=0.873779 moved_to_echo=24672 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 13, 0, 2, 0, 0, 0, 6131, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 4167, 1, 1795, 91, 0, 0, 0, 0, 0, 0, 1, 0, 6130, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6131, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6132, 15, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 15, 1, 0, 4217, 0, 1949, 19, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 1, 0, 0, 9, 2060, 2041, 2026, 0, 0, 0, 0, 0, 2, 0, 6117, 0, 15, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6146, 6057, 6140, 6143, 6151, 6203, 6142, 6170] avg=6144.0000 max=6203 max_rank=5 min=6057 min_rank=1 imbalance_ratio=1.009603 max_load_reduction=42949 max_load_reduction_pct=0.873800 moved_to_echo=24666 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 0, 0, 0, 6140, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 0, 4072, 0, 1798, 91, 0, 0, 0, 0, 0, 0, 2, 0, 6140, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6140, 5, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 4363, 0, 1953, 19, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 2, 2063, 2043, 2028, 0, 0, 0, 0, 0, 0, 0, 6124, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 5966, 6144, 6147, 6146, 6338, 6139, 6129] avg=6144.0000 max=6338 max_rank=5 min=5966 min_rank=1 imbalance_ratio=1.031576 max_load_reduction=42814 max_load_reduction_pct=0.871053 moved_to_echo=24752 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4396, 0, 1812, 91, 0, 0, 0, 0, 0, 0, 0, 0, 6146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3977, 0, 1955, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2064, 2045, 2029, 0, 0, 0, 0, 0, 0, 0, 6128, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 6299, 6146, 6190, 6144, 5951, 6138, 6139] avg=6144.0000 max=6299 max_rank=1 min=5951 min_rank=5 imbalance_ratio=1.025228 max_load_reduction=42853 max_load_reduction_pct=0.871847 moved_to_echo=24372 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 1814, 91, 0, 0, 0, 0, 0, 0, 0, 0, 6146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4315, 0, 1955, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2064, 2046, 2029, 0, 0, 0, 0, 0, 0, 0, 6128, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6144, 5959, 6146, 6199, 6144, 6289, 6139, 6132] avg=6144.0000 max=6289 max_rank=5 min=5959 min_rank=1 imbalance_ratio=1.023600 max_load_reduction=42863 max_load_reduction_pct=0.872050 moved_to_echo=24704 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4245, 0, 1785, 96, 0, 0, 0, 0, 0, 0, 0, 0, 6140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4238, 0, 1956, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 2069, 2045, 2034, 0, 0, 0, 0, 0, 0, 0, 6123, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6144, 6126, 6140, 6086, 6132, 6220, 6156, 6148] avg=6144.0000 max=6220 max_rank=5 min=6086 min_rank=3 imbalance_ratio=1.012370 max_load_reduction=42932 max_load_reduction_pct=0.873454 moved_to_echo=24656 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 128, 0, 128, 0, 0, 128, 16512, 16384, 16256, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 0, 128, 128, 128, 128, 33408, 0, 15744, 256, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 0, 0, 128, 128, 34048, 128, 14464, 768, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49280 observed_loss=-128 tokens_per_expert=[49152, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 128, 0, 128, 0, 0, 0, 49152, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[128, 0, 49024, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 49152, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 0, 49152, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=21 tokens_per_expert_current_ep_rank=[0, 1, 0, 1, 0, 0, 9, 0, 0, 7823, 8018, 292, 1, 0, 8, 0, 13, 8, 2501, 0, 10, 1772, 0, 7552, 2, 6893, 0, 61, 0, 0, 14, 0, 0, 0, 0, 0, 4897, 8109, 0, 0, 2, 0, 0, 0, 3, 2, 626, 502, 6, 0, 0, 1, 0, 15, 0, 0, 0, 0, 0, 1, 0, 1, 1, 7] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=21 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 7777, 7867, 297, 0, 0, 0, 0, 0, 0, 2577, 0, 0, 1793, 0, 7763, 0, 7126, 0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 4645, 8126, 0, 0, 0, 0, 0, 0, 0, 0, 558, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=21 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 7805, 7933, 298, 0, 0, 0, 0, 0, 0, 2687, 0, 0, 1961, 0, 7825, 0, 6997, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 4361, 8137, 0, 0, 0, 0, 0, 0, 0, 0, 588, 438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=21 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 7847, 8038, 340, 0, 0, 0, 0, 0, 0, 2481, 0, 0, 1938, 0, 7470, 0, 7008, 0, 132, 0, 0, 0, 0, 0, 0, 0, 0, 4733, 8136, 0, 0, 0, 0, 0, 0, 0, 0, 585, 444, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=21 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 1, 0, 0, 7828, 8021, 252, 2, 0, 3, 3, 8, 3, 2474, 0, 6, 1780, 0, 7560, 0, 7133, 0, 90, 0, 0, 6, 0, 0, 1, 0, 0, 4772, 8112, 0, 0, 1, 0, 0, 0, 2, 2, 546, 526, 2, 0, 0, 0, 0, 8, 0, 0, 0, 1, 0, 0, 0, 0, 3, 5] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=21 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 7867, 7924, 351, 3, 0, 3, 0, 4, 2, 3129, 0, 2, 1705, 0, 7499, 1, 6719, 0, 120, 0, 0, 4, 0, 0, 0, 0, 0, 4540, 8130, 0, 0, 1, 0, 0, 0, 2, 0, 702, 431, 2, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 0, 1, 1, 3] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=21 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 7856, 7991, 284, 0, 0, 0, 0, 0, 0, 2537, 0, 0, 2069, 0, 7509, 0, 7004, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 4730, 8121, 0, 0, 0, 0, 0, 0, 0, 0, 526, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=21 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 7820, 7790, 292, 0, 0, 0, 0, 0, 0, 2644, 0, 0, 1767, 0, 7943, 0, 7008, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 4797, 8118, 0, 0, 0, 0, 0, 0, 0, 0, 515, 360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6212, 0, 0, 0, 0, 0, 0, 0, 0, 5855, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2481, 0, 0, 1938, 0, 1624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6060, 0, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4733, 1463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 585, 444, 2121, 1992, 948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3725, 2332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4341, 1826, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6212, 6195, 6043, 6192, 6196, 6090, 6057, 6167] avg=6144.0000 max=6212 max_rank=0 min=6043 min_rank=2 imbalance_ratio=1.011068 max_load_reduction=42940 max_load_reduction_pct=0.873617 moved_to_echo=24510 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 1, 0, 0, 9, 0, 6196, 0, 0, 0, 0, 0, 0, 0, 0, 5837, 0, 292, 1, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 8, 2501, 0, 10, 1772, 0, 1642, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5961, 0, 61, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4897, 1458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 2, 626, 502, 2144, 1986, 932, 0, 0, 0, 0, 0, 6, 0, 0, 1, 0, 15, 0, 0, 3766, 2324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 7, 4327, 1822, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6207, 6138, 5946, 6038, 6355, 6197, 6112, 6159] avg=6144.0000 max=6355 max_rank=4 min=5946 min_rank=2 imbalance_ratio=1.034342 max_load_reduction=42797 max_load_reduction_pct=0.870707 moved_to_echo=24823 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 1, 0, 6198, 0, 0, 0, 0, 0, 0, 0, 0, 5841, 0, 252, 2, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3, 2474, 0, 6, 1780, 0, 1643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6168, 0, 90, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4772, 1459, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 2, 546, 526, 2147, 1987, 965, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8, 0, 0, 3770, 2325, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 5, 4328, 1823, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6200, 6101, 5914, 6264, 6232, 6176, 6105, 6160] avg=6144.0000 max=6264 max_rank=3 min=5914 min_rank=2 imbalance_ratio=1.019531 max_load_reduction=42888 max_load_reduction_pct=0.872559 moved_to_echo=24673 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 6080, 0, 0, 0, 0, 0, 0, 0, 0, 5803, 0, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2577, 0, 0, 1793, 0, 1688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6162, 0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4645, 1461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 558, 505, 2204, 1974, 964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3871, 2329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4336, 1787, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6081, 6100, 6058, 6279, 6106, 6205, 6200, 6123] avg=6144.0000 max=6279 max_rank=3 min=6058 min_rank=2 imbalance_ratio=1.021973 max_load_reduction=42873 max_load_reduction_pct=0.872253 moved_to_echo=24634 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6124, 0, 0, 0, 0, 0, 0, 0, 0, 5870, 0, 351, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 3129, 0, 2, 1705, 0, 1631, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5810, 0, 120, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4540, 1462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 702, 431, 2129, 1997, 909, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 5, 0, 0, 3739, 2330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 3, 4338, 1800, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6124, 6227, 6473, 5935, 6002, 6171, 6076, 6144] avg=6144.0000 max=6473 max_rank=2 min=5935 min_rank=3 imbalance_ratio=1.053548 max_load_reduction=42679 max_load_reduction_pct=0.868306 moved_to_echo=24393 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6176, 0, 0, 0, 0, 0, 0, 0, 0, 5862, 0, 284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2537, 0, 0, 2069, 0, 1633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6057, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4730, 1460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 462, 2132, 1994, 947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3744, 2328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4333, 1815, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6176, 6146, 6239, 6120, 6190, 6061, 6072, 6148] avg=6144.0000 max=6239 max_rank=2 min=6061 min_rank=5 imbalance_ratio=1.015462 max_load_reduction=42913 max_load_reduction_pct=0.873067 moved_to_echo=24471 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6131, 0, 0, 0, 0, 0, 0, 0, 0, 5820, 0, 298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2687, 0, 0, 1961, 0, 1693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6046, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4361, 1454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 588, 438, 2227, 1985, 951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3905, 2337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4346, 1802, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6131, 6118, 6341, 6168, 5815, 6189, 6242, 6148] avg=6144.0000 max=6341 max_rank=2 min=5815 min_rank=4 imbalance_ratio=1.032064 max_load_reduction=42811 max_load_reduction_pct=0.870992 moved_to_echo=24394 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6021, 0, 0, 0, 0, 0, 0, 0, 0, 5835, 0, 292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2644, 0, 0, 1767, 0, 1727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6060, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4797, 1459, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 360, 2255, 1985, 948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3961, 2327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4332, 1769, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6021, 6127, 6138, 6156, 6256, 6063, 6288, 6103] avg=6144.0000 max=6288 max_rank=6 min=6021 min_rank=0 imbalance_ratio=1.023438 max_load_reduction=42864 max_load_reduction_pct=0.872070 moved_to_echo=24710 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 0, 0, 0, 128, 128, 4736, 3712, 17408, 16000, 7680, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 21120, 0, 128, 14848, 0, 13312, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 37504, 11776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 128, 0, 128, 0, 128, 128, 128, 34688, 14464, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 46848, 0, 2432, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 48384, 0, 896, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 0, 128, 128, 0, 128, 0, 49152, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 0, 0, 128, 0, 128, 0, 0, 30592, 18688, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=22 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 801, 344, 0, 0, 0, 0, 0, 0, 1252, 0, 0, 4372, 0, 0, 0, 0, 0, 0, 0, 2781, 0, 0, 0, 0, 0, 7773, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 3556, 0, 0, 0, 0, 0, 0, 0, 0, 419, 0, 0, 3201, 0, 76] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=22 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 679, 338, 0, 0, 0, 0, 0, 0, 1059, 0, 0, 4426, 0, 0, 0, 0, 0, 0, 0, 2607, 0, 0, 0, 0, 0, 7823, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 0, 0, 5, 0, 0, 3721, 0, 13, 0, 0, 0, 0, 0, 0, 369, 0, 0, 3458, 0, 78] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=22 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 1, 857, 412, 6, 0, 0, 0, 0, 0, 1042, 0, 1, 4248, 0, 0, 0, 1, 0, 0, 5, 3175, 0, 0, 0, 0, 2, 7757, 4, 0, 0, 0, 8185, 0, 8186, 0, 4, 1, 8186, 4, 4, 9, 0, 0, 3354, 0, 16, 3, 0, 0, 2, 0, 3, 433, 0, 0, 3164, 0, 86] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=22 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 7, 707, 332, 14, 1, 0, 0, 0, 0, 1173, 0, 5, 4342, 0, 0, 0, 3, 0, 0, 11, 3049, 0, 0, 0, 0, 6, 7755, 12, 2, 0, 0, 8174, 0, 8174, 0, 9, 0, 8172, 15, 12, 12, 0, 1, 3142, 0, 13, 11, 0, 0, 11, 0, 2, 437, 0, 0, 3459, 0, 88] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=22 tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 0, 0, 2, 738, 366, 4, 0, 0, 0, 0, 0, 1093, 0, 8, 4111, 0, 0, 0, 0, 0, 0, 5, 3058, 0, 0, 0, 0, 1, 7791, 4, 1, 0, 0, 8183, 2, 8182, 0, 4, 0, 8182, 9, 7, 32, 0, 2, 3419, 0, 0, 5, 0, 0, 3, 0, 1, 394, 0, 0, 3461, 0, 80] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=22 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 719, 341, 0, 0, 0, 0, 0, 0, 1201, 0, 0, 4446, 0, 0, 0, 0, 0, 0, 0, 3013, 0, 0, 0, 0, 0, 7806, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 3061, 0, 1, 0, 1, 0, 0, 0, 0, 386, 0, 0, 3529, 0, 71] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=22 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 817, 300, 0, 0, 0, 0, 0, 0, 1073, 0, 0, 4303, 0, 0, 0, 0, 0, 0, 0, 2405, 0, 0, 0, 0, 0, 7865, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 0, 0, 7, 0, 0, 3790, 0, 1, 0, 2, 0, 0, 0, 0, 327, 0, 0, 3614, 0, 72] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=22 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 735, 289, 0, 0, 0, 0, 0, 1, 1104, 0, 0, 4226, 0, 0, 0, 0, 0, 0, 0, 2831, 0, 0, 0, 0, 0, 7829, 0, 0, 0, 0, 8191, 0, 8192, 0, 0, 0, 8192, 0, 0, 16, 0, 0, 3404, 0, 0, 0, 0, 0, 0, 0, 0, 363, 0, 0, 3698, 0, 81] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=22 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 801, 5388, 0, 0, 0, 0, 0, 0, 0, 344, 0, 0, 0, 0, 0, 0, 1252, 2804, 1874, 0, 0, 0, 0, 0, 0, 0, 0, 4372, 0, 0, 0, 0, 0, 1653, 173, 0, 0, 0, 0, 0, 0, 0, 0, 2781, 0, 0, 0, 0, 0, 3277, 0, 0, 0, 0, 0, 0, 0, 6120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6129, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3556, 0, 0, 0, 0, 0, 0, 2705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 419, 0, 0, 3201, 0, 76, 1890, 336, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6189, 6274, 6198, 6058, 6120, 6130, 6261, 5922] avg=6144.0000 max=6274 max_rank=1 min=5922 min_rank=7 imbalance_ratio=1.021159 max_load_reduction=42878 max_load_reduction_pct=0.872355 moved_to_echo=24433 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=22 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 0, 0, 2, 738, 5381, 0, 0, 0, 0, 0, 0, 0, 366, 4, 0, 0, 0, 0, 0, 1093, 2801, 1872, 0, 0, 0, 0, 0, 0, 0, 8, 4111, 0, 0, 0, 0, 0, 1657, 173, 0, 0, 0, 0, 0, 0, 0, 5, 3058, 0, 0, 0, 0, 1, 3273, 0, 0, 0, 0, 0, 0, 0, 6134, 4, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 6122, 9, 7, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3419, 0, 0, 5, 0, 0, 3, 2700, 0, 0, 0, 0, 0, 0, 0, 0, 1, 394, 0, 0, 3461, 0, 80, 1887, 338, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6125, 6136, 5949, 6337, 6141, 6174, 6129, 6161] avg=6144.0000 max=6337 max_rank=3 min=5949 min_rank=2 imbalance_ratio=1.031413 max_load_reduction=42815 max_load_reduction_pct=0.871073 moved_to_echo=24605 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=22 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 7, 707, 5376, 0, 0, 0, 0, 0, 0, 0, 332, 14, 1, 0, 0, 0, 0, 1173, 2798, 1870, 0, 0, 0, 0, 0, 0, 0, 5, 4342, 0, 0, 0, 3, 0, 1649, 173, 0, 0, 0, 0, 0, 0, 0, 11, 3049, 0, 0, 0, 0, 6, 3269, 0, 0, 0, 0, 0, 0, 0, 6106, 12, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 6114, 15, 12, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3142, 0, 13, 11, 0, 0, 11, 2699, 0, 0, 0, 0, 0, 0, 0, 0, 2, 437, 0, 0, 3459, 0, 88, 1885, 336, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6091, 6188, 6172, 6335, 6120, 6162, 5877, 6207] avg=6144.0000 max=6335 max_rank=3 min=5877 min_rank=6 imbalance_ratio=1.031087 max_load_reduction=42817 max_load_reduction_pct=0.871114 moved_to_echo=24366 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=22 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 1, 857, 5385, 0, 0, 0, 0, 0, 0, 0, 412, 6, 0, 0, 0, 0, 0, 1042, 2801, 1874, 0, 0, 0, 0, 0, 0, 0, 1, 4248, 0, 0, 0, 1, 0, 1649, 173, 0, 0, 0, 0, 0, 0, 0, 5, 3175, 0, 0, 0, 0, 2, 3274, 0, 0, 0, 0, 0, 0, 0, 6108, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 6125, 4, 4, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3354, 0, 16, 3, 0, 0, 2, 2701, 0, 0, 0, 0, 0, 0, 0, 0, 3, 433, 0, 0, 3164, 0, 86, 1888, 336, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6244, 6135, 6072, 6456, 6112, 6147, 6076, 5910] avg=6144.0000 max=6456 max_rank=3 min=5910 min_rank=7 imbalance_ratio=1.050781 max_load_reduction=42696 max_load_reduction_pct=0.868652 moved_to_echo=24245 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=22 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 735, 5388, 0, 0, 0, 0, 0, 0, 0, 289, 0, 0, 0, 0, 0, 1, 1104, 2804, 1874, 0, 0, 0, 0, 0, 0, 0, 0, 4226, 0, 0, 0, 0, 0, 1665, 173, 0, 0, 0, 0, 0, 0, 0, 0, 2831, 0, 0, 0, 0, 0, 3277, 0, 0, 0, 0, 0, 0, 0, 6164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6129, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3404, 0, 0, 0, 0, 0, 0, 2704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 363, 0, 0, 3698, 0, 81, 1890, 336, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6123, 6072, 6064, 6108, 6164, 6145, 6108, 6368] avg=6144.0000 max=6368 max_rank=7 min=6064 min_rank=2 imbalance_ratio=1.036458 max_load_reduction=42784 max_load_reduction_pct=0.870443 moved_to_echo=24785 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=22 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 719, 5388, 0, 0, 0, 0, 0, 0, 0, 341, 0, 0, 0, 0, 0, 0, 1201, 2804, 1874, 0, 0, 0, 0, 0, 0, 0, 0, 4446, 0, 0, 0, 0, 0, 1660, 173, 0, 0, 0, 0, 0, 0, 0, 0, 3013, 0, 0, 0, 0, 0, 3279, 0, 0, 0, 0, 0, 0, 0, 6146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6129, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3061, 0, 1, 0, 1, 0, 0, 2703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 386, 0, 0, 3529, 0, 71, 1890, 336, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6107, 6220, 6279, 6292, 6146, 6130, 5766, 6212] avg=6144.0000 max=6292 max_rank=3 min=5766 min_rank=6 imbalance_ratio=1.024089 max_load_reduction=42860 max_load_reduction_pct=0.871989 moved_to_echo=24254 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=22 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 679, 5389, 0, 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 1059, 2803, 1876, 0, 0, 0, 0, 0, 0, 0, 0, 4426, 0, 0, 0, 0, 0, 1667, 177, 0, 0, 0, 0, 0, 0, 0, 0, 2607, 0, 0, 0, 0, 0, 3277, 0, 0, 0, 0, 0, 0, 0, 6156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6123, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3721, 0, 13, 0, 0, 0, 0, 2703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 369, 0, 0, 3458, 0, 78, 1892, 336, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6068, 6076, 6270, 5884, 6156, 6128, 6437, 6133] avg=6144.0000 max=6437 max_rank=6 min=5884 min_rank=3 imbalance_ratio=1.047689 max_load_reduction=42715 max_load_reduction_pct=0.869039 moved_to_echo=24854 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=22 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 817, 5388, 0, 0, 0, 0, 0, 0, 0, 300, 0, 0, 0, 0, 0, 0, 1073, 2804, 1874, 0, 0, 0, 0, 0, 0, 0, 0, 4303, 0, 0, 0, 0, 0, 1672, 173, 0, 0, 0, 0, 0, 0, 0, 0, 2405, 0, 0, 0, 0, 0, 3277, 0, 0, 0, 0, 0, 0, 0, 6193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6129, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3790, 0, 1, 0, 2, 0, 0, 2705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 327, 0, 0, 3614, 0, 72, 1890, 336, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6205, 6051, 6148, 5682, 6193, 6136, 6498, 6239] avg=6144.0000 max=6498 max_rank=6 min=5682 min_rank=3 imbalance_ratio=1.057617 max_load_reduction=42654 max_load_reduction_pct=0.867798 moved_to_echo=25066 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 128, 23040, 0, 0, 0, 0, 128, 26240, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 27520, 0, 128, 128, 128, 0, 128, 21632, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 3200, 0, 0, 27648, 0, 640, 15232, 2816, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 128, 34560, 0, 0, 0, 128, 0, 13312, 1408, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[2816, 128, 128, 0, 0, 0, 128, 9088, 22528, 15104, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 0, 0, 128, 6144, 43136, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 128, 49024, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[49152, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=23 tokens_per_expert_current_ep_rank=[2, 0, 1, 0, 0, 0, 1, 0, 9, 0, 1, 0, 8182, 0, 0, 0, 8182, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 4, 0, 0, 0, 0, 186, 0, 0, 10, 8182, 0, 0, 9, 98, 0, 0, 254, 7512, 0, 0, 0, 6, 137, 8, 8182, 0, 0, 5] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=23 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 8185, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 5, 0, 1, 0, 0, 193, 0, 0, 6, 8185, 0, 0, 6, 151, 0, 0, 253, 7538, 0, 0, 0, 3, 53, 5, 8186, 0, 0, 3] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=23 tokens_per_expert_current_ep_rank=[7, 0, 0, 0, 0, 0, 3, 0, 12, 0, 0, 0, 8171, 0, 0, 2, 8175, 3, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 4, 0, 0, 0, 8168, 6, 0, 0, 12, 0, 0, 0, 0, 195, 0, 2, 17, 8174, 0, 0, 19, 125, 0, 1, 242, 7498, 0, 0, 0, 14, 107, 13, 8174, 0, 0, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 0, 0, 8192, 0, 0, 0, 92, 0, 0, 245, 7556, 0, 0, 0, 0, 111, 0, 8192, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 164, 0, 0, 0, 8192, 0, 0, 0, 105, 0, 0, 246, 7589, 0, 0, 0, 0, 88, 0, 8192, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 182, 0, 0, 0, 8192, 0, 0, 0, 78, 0, 0, 255, 7531, 0, 0, 0, 0, 146, 0, 8192, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 196, 0, 0, 0, 8192, 0, 0, 0, 81, 0, 0, 255, 7625, 0, 0, 0, 0, 35, 0, 8192, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 176, 0, 0, 0, 8192, 0, 0, 0, 76, 0, 0, 252, 7597, 0, 0, 0, 0, 91, 0, 8192, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=23 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 1, 0, 0, 0, 1, 0, 2043, 2043, 1756, 284, 0, 0, 0, 0, 9, 0, 1, 0, 6137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6139, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 2231, 2145, 1761, 0, 0, 0, 0, 0, 6135, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 0, 0, 10, 5951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 98, 0, 0, 254, 5756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 137, 8, 6037, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6130, 6147, 6140, 6139, 6139, 6147, 6117, 6193] avg=6144.0000 max=6193 max_rank=7 min=6117 min_rank=6 imbalance_ratio=1.007975 max_load_reduction=42959 max_load_reduction_pct=0.874003 moved_to_echo=24596 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 2047, 2045, 1783, 284, 0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2234, 2148, 1763, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 196, 0, 0, 0, 5958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 255, 5842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 6044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6159, 6145, 6147, 6145, 6145, 6154, 6178, 6079] avg=6144.0000 max=6178 max_rank=6 min=6079 min_rank=7 imbalance_ratio=1.005534 max_load_reduction=42974 max_load_reduction_pct=0.874308 moved_to_echo=24556 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=23 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 0, 0, 0, 0, 3, 0, 2041, 2041, 1753, 284, 0, 0, 0, 0, 12, 0, 0, 0, 6129, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 6134, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 4, 0, 0, 0, 2229, 2143, 1758, 0, 0, 0, 0, 0, 6127, 6, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 195, 0, 2, 17, 5945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 125, 0, 1, 242, 5745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 107, 13, 6031, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6129, 6143, 6137, 6141, 6145, 6159, 6132, 6166] avg=6144.0000 max=6166 max_rank=7 min=6129 min_rank=0 imbalance_ratio=1.003581 max_load_reduction=42986 max_load_reduction_pct=0.874552 moved_to_echo=24602 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 2047, 2045, 1767, 284, 0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2234, 2148, 1763, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 0, 0, 5958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 245, 5789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 0, 6044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6145, 6147, 6145, 6145, 6146, 6126, 6155] avg=6144.0000 max=6155 max_rank=7 min=6126 min_rank=6 imbalance_ratio=1.001790 max_load_reduction=42997 max_load_reduction_pct=0.874776 moved_to_echo=24572 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=23 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 0, 0, 0, 0, 2044, 2044, 1762, 284, 0, 0, 0, 0, 6, 0, 0, 0, 6140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2232, 2146, 1761, 0, 0, 0, 0, 0, 6139, 0, 0, 0, 5, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 193, 0, 0, 6, 5953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 151, 0, 0, 253, 5776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 53, 5, 6040, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6136, 6146, 6143, 6140, 6145, 6152, 6186, 6104] avg=6144.0000 max=6186 max_rank=6 min=6104 min_rank=7 imbalance_ratio=1.006836 max_load_reduction=42966 max_load_reduction_pct=0.874146 moved_to_echo=24587 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 2047, 2045, 1774, 284, 0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2234, 2148, 1763, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 0, 0, 0, 5958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 246, 5815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 6044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6150, 6145, 6147, 6145, 6145, 6122, 6166, 6132] avg=6144.0000 max=6166 max_rank=6 min=6122 min_rank=5 imbalance_ratio=1.003581 max_load_reduction=42986 max_load_reduction_pct=0.874552 moved_to_echo=24565 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 2049, 2048, 1766, 290, 0, 0, 0, 0, 0, 0, 0, 0, 6136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2236, 2150, 1766, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, 0, 0, 0, 5956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 255, 5765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 0, 6042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6153, 6136, 6144, 6152, 6143, 6138, 6098, 6188] avg=6144.0000 max=6188 max_rank=7 min=6098 min_rank=6 imbalance_ratio=1.007161 max_load_reduction=42964 max_load_reduction_pct=0.874105 moved_to_echo=24567 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 2047, 2045, 1776, 284, 0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2234, 2148, 1763, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 0, 0, 0, 5958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 252, 5821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 6044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6152, 6145, 6147, 6145, 6145, 6134, 6149, 6135] avg=6144.0000 max=6152 max_rank=0 min=6134 min_rank=5 imbalance_ratio=1.001302 max_load_reduction=43000 max_load_reduction_pct=0.874837 moved_to_echo=24563 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 0, 128, 0, 0, 0, 128, 0, 16384, 16384, 14208, 2304, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 128, 0, 0, 0, 17920, 17280, 14208, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49280 observed_loss=-128 tokens_per_expert=[49152, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 0, 128, 0, 49152, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[49152, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 128, 768, 128, 48384, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 1536, 0, 128, 128, 47744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 896, 0, 128, 2048, 46336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 9, 0, 345, 375, 0, 0, 0, 5359, 0, 0, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 8041, 0, 0, 0, 0, 0, 0, 0, 0, 8000, 0, 0, 0, 0, 0, 0, 381, 0, 0, 0, 2159, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8098, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 4, 9, 9, 0, 0, 10, 0, 276, 355, 0, 0, 0, 5598, 0, 0, 4, 0, 26, 8, 8180, 3, 0, 1, 4, 0, 0, 8035, 0, 6, 0, 6, 6, 0, 0, 0, 8114, 0, 0, 0, 0, 0, 0, 543, 0, 5, 0, 1836, 1, 0, 1, 0, 8176, 0, 0, 0, 0, 12, 0, 0, 7922, 2, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 9, 9, 18, 0, 0, 24, 0, 417, 394, 0, 0, 0, 4857, 0, 0, 11, 0, 36, 17, 8170, 2, 0, 1, 5, 0, 1, 8035, 0, 8, 0, 12, 15, 0, 0, 0, 8127, 5, 0, 0, 0, 0, 0, 574, 0, 5, 0, 2380, 0, 0, 3, 0, 8161, 0, 0, 0, 0, 3, 0, 0, 7848, 5, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4, 0, 387, 422, 0, 0, 0, 4950, 0, 0, 0, 0, 14, 0, 8192, 0, 0, 0, 0, 0, 0, 8074, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 0, 0, 0, 0, 0, 0, 513, 0, 0, 0, 2333, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7931, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=24 tokens_per_expert_current_ep_rank=[2, 0, 3, 2, 6, 0, 0, 11, 0, 396, 283, 0, 0, 0, 5403, 0, 0, 1, 0, 44, 6, 8184, 3, 0, 1, 3, 0, 0, 8016, 0, 2, 0, 4, 5, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 416, 1, 4, 0, 2161, 1, 0, 0, 0, 8183, 0, 0, 0, 0, 1, 0, 0, 7829, 1, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 8, 0, 347, 534, 0, 0, 0, 4712, 0, 0, 0, 0, 35, 0, 8192, 0, 0, 0, 0, 0, 0, 8058, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 554, 0, 0, 0, 2429, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7914, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 7, 0, 349, 574, 0, 0, 0, 4645, 0, 0, 0, 0, 13, 0, 8192, 0, 0, 0, 0, 0, 0, 8056, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 591, 0, 0, 0, 2428, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7947, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 10, 0, 366, 537, 0, 0, 0, 4839, 0, 0, 0, 0, 12, 0, 8192, 0, 0, 0, 0, 0, 0, 8053, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 475, 0, 0, 0, 2388, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7932, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 9, 4309, 1819, 0, 0, 0, 0, 0, 0, 0, 345, 375, 0, 0, 0, 5359, 0, 310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 381, 0, 0, 0, 1961, 1904, 1514, 253, 0, 0, 0, 0, 2159, 0, 0, 0, 0, 3883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6137, 6389, 6121, 6137, 6039, 6013, 6042, 6274] avg=6144.0000 max=6389 max_rank=1 min=6013 min_rank=5 imbalance_ratio=1.039876 max_load_reduction=42763 max_load_reduction_pct=0.870015 moved_to_echo=24368 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 9, 9, 0, 0, 10, 4301, 1817, 0, 0, 0, 0, 0, 0, 0, 276, 355, 0, 0, 0, 5598, 0, 303, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 26, 8, 6110, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 6132, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 6, 0, 0, 0, 6125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 0, 5, 0, 1989, 1903, 1481, 253, 0, 0, 0, 0, 1836, 1, 0, 1, 0, 3875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 6138, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6150, 6532, 6151, 6143, 6137, 6174, 5713, 6152] avg=6144.0000 max=6532 max_rank=1 min=5713 min_rank=6 imbalance_ratio=1.063151 max_load_reduction=42620 max_load_reduction_pct=0.867106 moved_to_echo=24176 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 9, 9, 18, 0, 0, 24, 4293, 1815, 0, 0, 0, 0, 0, 0, 0, 417, 394, 0, 0, 0, 4857, 0, 300, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 36, 17, 6103, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 0, 1, 6132, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 15, 0, 0, 0, 6134, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 574, 0, 5, 0, 1993, 1903, 1467, 252, 0, 0, 0, 0, 2380, 0, 0, 3, 0, 3868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 6081, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6168, 5968, 6169, 6147, 6166, 6194, 6251, 6089] avg=6144.0000 max=6251 max_rank=6 min=5968 min_rank=1 imbalance_ratio=1.017415 max_load_reduction=42901 max_load_reduction_pct=0.872823 moved_to_echo=24700 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 8, 4309, 1819, 0, 0, 0, 0, 0, 0, 0, 347, 534, 0, 0, 0, 4712, 0, 303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 6120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 0, 0, 0, 2005, 1908, 1480, 253, 0, 0, 0, 0, 2429, 0, 0, 0, 0, 3883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6136, 5896, 6155, 6150, 6172, 6200, 6312, 6131] avg=6144.0000 max=6312 max_rank=6 min=5896 min_rank=1 imbalance_ratio=1.027344 max_load_reduction=42840 max_load_reduction_pct=0.871582 moved_to_echo=24815 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=24 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 3, 2, 6, 0, 0, 11, 4304, 1818, 0, 0, 0, 0, 0, 0, 0, 396, 283, 0, 0, 0, 5403, 0, 300, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 44, 6, 6113, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 6118, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5, 0, 0, 0, 6174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 416, 1, 4, 0, 2006, 1898, 1464, 253, 0, 0, 0, 0, 2161, 1, 0, 0, 0, 3879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6065, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6146, 6382, 6167, 6124, 6183, 6042, 6041, 6067] avg=6144.0000 max=6382 max_rank=1 min=6041 min_rank=6 imbalance_ratio=1.038737 max_load_reduction=42770 max_load_reduction_pct=0.870158 moved_to_echo=24333 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4, 4309, 1819, 0, 0, 0, 0, 0, 0, 0, 387, 422, 0, 0, 0, 4950, 0, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 6120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 0, 0, 0, 1996, 1912, 1483, 253, 0, 0, 0, 0, 2333, 0, 0, 0, 0, 3883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6132, 6063, 6134, 6162, 6144, 6157, 6216, 6144] avg=6144.0000 max=6216 max_rank=6 min=6063 min_rank=1 imbalance_ratio=1.011719 max_load_reduction=42936 max_load_reduction_pct=0.873535 moved_to_echo=24661 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 7, 4314, 1824, 0, 0, 0, 0, 0, 0, 0, 349, 574, 0, 0, 0, 4645, 0, 308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 6110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 591, 0, 0, 0, 2003, 1913, 1489, 258, 0, 0, 0, 0, 2428, 0, 0, 0, 0, 3878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 5876, 6123, 6143, 6155, 6254, 6306, 6150] avg=6144.0000 max=6306 max_rank=6 min=5876 min_rank=1 imbalance_ratio=1.026367 max_load_reduction=42846 max_load_reduction_pct=0.871704 moved_to_echo=24865 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 10, 4309, 1819, 0, 0, 0, 0, 0, 0, 0, 366, 537, 0, 0, 0, 4839, 0, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 6120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 0, 0, 0, 2000, 1907, 1483, 253, 0, 0, 0, 0, 2388, 0, 0, 0, 0, 3883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6138, 6046, 6132, 6146, 6156, 6118, 6271, 6145] avg=6144.0000 max=6271 max_rank=6 min=6046 min_rank=1 imbalance_ratio=1.020671 max_load_reduction=42881 max_load_reduction_pct=0.872416 moved_to_echo=24690 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 4096, 128, 128, 0, 16000, 15360, 11904, 2048, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 128, 0, 0, 128, 34560, 14592, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[18176, 128, 0, 128, 0, 31104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 2944, 3584, 0, 0, 0, 40448, 0, 2432, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 49152, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 128, 0, 0, 49152, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 128, 0, 0, 0, 49152, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 0, 256, 128, 49024, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True - [2026-08-26 14:53:46] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 828.5 | throughput per GPU (TFLOP/s/GPU): 154.0 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.368878E+00 | loss scale: 1.0 | grad norm: 5.139 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1724, 594, 414, 933, 96, 1431, 1043, 146, 2662, 574, 773, 269, 5, 2699, 712, 840, 556, 775, 755, 767, 106, 366, 172, 1076, 346, 646, 634, 385, 862, 274, 695, 971, 802, 610, 1415, 1448, 768, 208, 957, 726, 203, 502, 516, 181, 870, 1159, 333, 119, 304, 399, 552, 817, 723, 2062, 337, 632, 1958, 380, 1868, 720, 196, 728, 1559, 799] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1678, 451, 336, 1008, 122, 1384, 1193, 147, 2529, 564, 1020, 338, 10, 2672, 715, 736, 579, 772, 836, 646, 115, 372, 159, 1025, 341, 558, 684, 484, 894, 365, 773, 1046, 775, 647, 1475, 1316, 743, 231, 901, 784, 247, 503, 477, 198, 938, 1137, 319, 159, 325, 421, 480, 717, 724, 1981, 325, 589, 1881, 446, 1646, 830, 145, 668, 1639, 933] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1743, 569, 425, 947, 81, 1253, 835, 145, 2600, 592, 919, 261, 5, 2767, 694, 857, 570, 820, 799, 683, 153, 374, 154, 1141, 316, 590, 732, 422, 846, 324, 633, 823, 753, 600, 1524, 1338, 765, 193, 940, 824, 215, 478, 486, 182, 828, 1239, 382, 141, 345, 437, 575, 678, 775, 2241, 338, 632, 2126, 385, 1647, 712, 171, 636, 1615, 878] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1818, 556, 358, 963, 128, 1261, 898, 237, 2579, 497, 805, 241, 13, 2643, 750, 848, 556, 777, 730, 769, 136, 419, 201, 1055, 372, 515, 664, 464, 901, 200, 656, 907, 763, 726, 1433, 1445, 822, 228, 888, 738, 229, 579, 520, 186, 960, 1168, 370, 127, 318, 408, 567, 716, 764, 2168, 274, 631, 1976, 411, 1748, 763, 176, 672, 1590, 871] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1738, 449, 446, 814, 118, 1298, 1166, 204, 2409, 628, 1027, 280, 8, 2751, 753, 715, 546, 797, 778, 645, 136, 416, 168, 990, 309, 569, 716, 465, 848, 269, 676, 1032, 856, 672, 1282, 1453, 666, 256, 965, 709, 262, 535, 552, 165, 928, 1132, 337, 108, 400, 410, 525, 627, 748, 2057, 320, 628, 1909, 379, 1685, 803, 264, 750, 1764, 841] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1735, 539, 293, 855, 108, 1388, 990, 120, 2658, 512, 976, 258, 10, 2895, 778, 849, 543, 779, 832, 723, 83, 381, 112, 1109, 305, 573, 622, 431, 830, 248, 722, 920, 829, 636, 1492, 1428, 629, 212, 932, 797, 193, 502, 430, 165, 878, 1187, 378, 104, 310, 378, 506, 681, 783, 2254, 353, 583, 2198, 351, 1761, 720, 155, 665, 1580, 905] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1726, 518, 457, 921, 88, 1264, 1077, 173, 2523, 591, 1086, 227, 11, 2745, 719, 732, 602, 816, 825, 774, 134, 392, 144, 1052, 402, 575, 723, 449, 840, 411, 730, 1003, 825, 649, 1459, 1415, 671, 214, 923, 763, 251, 490, 488, 179, 851, 1234, 406, 133, 338, 386, 492, 732, 724, 1943, 278, 667, 1950, 348, 1700, 708, 174, 650, 1497, 884] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1854, 565, 379, 910, 110, 1378, 1097, 171, 2517, 570, 1035, 255, 10, 2803, 752, 763, 529, 744, 732, 701, 94, 384, 171, 958, 320, 545, 602, 494, 850, 335, 615, 958, 788, 623, 1397, 1418, 560, 197, 968, 755, 196, 589, 525, 130, 921, 1152, 399, 145, 320, 378, 500, 867, 720, 2181, 347, 540, 2036, 441, 1741, 768, 222, 708, 1583, 836] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1655, 569, 425, 947, 81, 1253, 835, 145, 0, 0, 0, 0, 0, 0, 0, 0, 2600, 592, 919, 261, 5, 265, 694, 857, 0, 0, 0, 0, 0, 0, 0, 0, 570, 820, 799, 683, 153, 374, 154, 1141, 1328, 330, 0, 0, 0, 0, 0, 0, 316, 590, 732, 422, 846, 324, 633, 823, 798, 588, 0, 0, 0, 0, 0, 0, 753, 600, 718, 1338, 765, 193, 940, 824, 0, 0, 0, 0, 0, 0, 0, 0, 215, 478, 486, 182, 828, 1239, 382, 141, 2172, 0, 0, 0, 0, 0, 0, 0, 345, 437, 575, 678, 775, 2241, 338, 632, 218, 88, 67, 0, 0, 0, 0, 0, 0, 385, 1580, 712, 171, 636, 1615, 878, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5910, 6193, 6352, 6072, 6131, 6123, 6394, 5977] avg=6144.0000 max=6394 max_rank=6 min=5910 min_rank=0 imbalance_ratio=1.040690 max_load_reduction=42758 max_load_reduction_pct=0.869914 moved_to_echo=24625 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1637, 594, 414, 933, 96, 1431, 1043, 146, 0, 0, 0, 0, 0, 0, 0, 0, 2662, 574, 773, 269, 5, 258, 712, 840, 0, 0, 0, 0, 0, 0, 0, 0, 556, 775, 755, 767, 106, 366, 172, 1076, 1224, 322, 0, 0, 0, 0, 0, 0, 346, 646, 634, 385, 862, 274, 695, 971, 734, 546, 0, 0, 0, 0, 0, 0, 802, 610, 667, 1448, 768, 208, 957, 726, 0, 0, 0, 0, 0, 0, 0, 0, 203, 502, 516, 181, 870, 1159, 333, 119, 2119, 0, 0, 0, 0, 0, 0, 0, 304, 399, 552, 817, 723, 2062, 337, 632, 202, 87, 76, 0, 0, 0, 0, 0, 0, 380, 1792, 720, 196, 728, 1559, 799, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6294, 6093, 6119, 6093, 6186, 6002, 6191, 6174] avg=6144.0000 max=6294 max_rank=0 min=6002 min_rank=5 imbalance_ratio=1.024414 max_load_reduction=42858 max_load_reduction_pct=0.871948 moved_to_echo=24553 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1726, 556, 358, 963, 128, 1261, 898, 237, 0, 0, 0, 0, 0, 0, 0, 0, 2579, 497, 805, 241, 13, 253, 750, 848, 0, 0, 0, 0, 0, 0, 0, 0, 556, 777, 730, 769, 136, 419, 201, 1055, 1234, 315, 0, 0, 0, 0, 0, 0, 372, 515, 664, 464, 901, 200, 656, 907, 742, 552, 0, 0, 0, 0, 0, 0, 763, 726, 676, 1445, 822, 228, 888, 738, 0, 0, 0, 0, 0, 0, 0, 0, 229, 579, 520, 186, 960, 1168, 370, 127, 2075, 0, 0, 0, 0, 0, 0, 0, 318, 408, 567, 716, 764, 2168, 274, 631, 205, 92, 71, 0, 0, 0, 0, 0, 0, 411, 1677, 763, 176, 672, 1590, 871, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6127, 5986, 6192, 5973, 6286, 6214, 6214, 6160] avg=6144.0000 max=6286 max_rank=4 min=5973 min_rank=3 imbalance_ratio=1.023112 max_load_reduction=42866 max_load_reduction_pct=0.872111 moved_to_echo=24874 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1593, 451, 336, 1008, 122, 1384, 1193, 147, 0, 0, 0, 0, 0, 0, 0, 0, 2529, 564, 1020, 338, 10, 255, 715, 736, 0, 0, 0, 0, 0, 0, 0, 0, 579, 772, 836, 646, 115, 372, 159, 1025, 1176, 319, 0, 0, 0, 0, 0, 0, 341, 558, 684, 484, 894, 365, 773, 1046, 705, 569, 0, 0, 0, 0, 0, 0, 775, 647, 695, 1316, 743, 231, 901, 784, 0, 0, 0, 0, 0, 0, 0, 0, 247, 503, 477, 198, 938, 1137, 319, 159, 2098, 0, 0, 0, 0, 0, 0, 0, 325, 421, 480, 717, 724, 1981, 325, 589, 211, 85, 67, 0, 0, 0, 0, 0, 0, 446, 1579, 830, 145, 668, 1639, 933, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6234, 6167, 5999, 6419, 6092, 6076, 5925, 6240] avg=6144.0000 max=6419 max_rank=3 min=5925 min_rank=6 imbalance_ratio=1.044759 max_load_reduction=42733 max_load_reduction_pct=0.869405 moved_to_echo=24333 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1760, 565, 379, 910, 110, 1378, 1097, 171, 0, 0, 0, 0, 0, 0, 0, 0, 2517, 570, 1035, 255, 10, 267, 752, 763, 0, 0, 0, 0, 0, 0, 0, 0, 529, 744, 732, 701, 94, 384, 171, 958, 1272, 335, 0, 0, 0, 0, 0, 0, 320, 545, 602, 494, 850, 335, 615, 958, 764, 539, 0, 0, 0, 0, 0, 0, 788, 623, 658, 1418, 560, 197, 968, 755, 0, 0, 0, 0, 0, 0, 0, 0, 196, 589, 525, 130, 921, 1152, 399, 145, 2201, 0, 0, 0, 0, 0, 0, 0, 320, 378, 500, 867, 720, 2181, 347, 540, 200, 94, 71, 0, 0, 0, 0, 0, 0, 441, 1670, 768, 222, 708, 1583, 836, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6370, 6169, 5920, 6022, 5967, 6258, 6218, 6228] avg=6144.0000 max=6370 max_rank=0 min=5920 min_rank=2 imbalance_ratio=1.036784 max_load_reduction=42782 max_load_reduction_pct=0.870402 moved_to_echo=24671 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1647, 539, 293, 855, 108, 1388, 990, 120, 0, 0, 0, 0, 0, 0, 0, 0, 2658, 512, 976, 258, 10, 276, 778, 849, 0, 0, 0, 0, 0, 0, 0, 0, 543, 779, 832, 723, 83, 381, 112, 1109, 1374, 346, 0, 0, 0, 0, 0, 0, 305, 573, 622, 431, 830, 248, 722, 920, 824, 575, 0, 0, 0, 0, 0, 0, 829, 636, 704, 1428, 629, 212, 932, 797, 0, 0, 0, 0, 0, 0, 0, 0, 193, 502, 430, 165, 878, 1187, 378, 104, 2273, 0, 0, 0, 0, 0, 0, 0, 310, 378, 506, 681, 783, 2254, 353, 583, 213, 88, 71, 0, 0, 0, 0, 0, 0, 351, 1690, 720, 155, 665, 1580, 905, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5940, 6317, 6282, 6050, 6167, 6110, 6220, 6066] avg=6144.0000 max=6317 max_rank=1 min=5940 min_rank=0 imbalance_ratio=1.028158 max_load_reduction=42835 max_load_reduction_pct=0.871480 moved_to_echo=24563 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1645, 449, 446, 814, 118, 1298, 1166, 204, 0, 0, 0, 0, 0, 0, 0, 0, 2409, 628, 1027, 280, 8, 256, 753, 715, 0, 0, 0, 0, 0, 0, 0, 0, 546, 797, 778, 645, 136, 416, 168, 990, 1194, 332, 0, 0, 0, 0, 0, 0, 309, 569, 716, 465, 848, 269, 676, 1032, 715, 497, 0, 0, 0, 0, 0, 0, 856, 672, 598, 1453, 666, 256, 965, 709, 0, 0, 0, 0, 0, 0, 0, 0, 262, 535, 552, 165, 928, 1132, 337, 108, 2163, 0, 0, 0, 0, 0, 0, 0, 400, 410, 525, 627, 748, 2057, 320, 628, 187, 93, 71, 0, 0, 0, 0, 0, 0, 379, 1614, 803, 264, 750, 1764, 841, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6140, 6076, 6002, 6096, 6175, 6182, 6066, 6415] avg=6144.0000 max=6415 max_rank=7 min=6002 min_rank=2 imbalance_ratio=1.044108 max_load_reduction=42737 max_load_reduction_pct=0.869486 moved_to_echo=24838 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1639, 518, 457, 921, 88, 1264, 1077, 173, 0, 0, 0, 0, 0, 0, 0, 0, 2523, 591, 1086, 227, 11, 262, 719, 732, 0, 0, 0, 0, 0, 0, 0, 0, 602, 816, 825, 774, 134, 392, 144, 1052, 1219, 328, 0, 0, 0, 0, 0, 0, 402, 575, 723, 449, 840, 411, 730, 1003, 731, 563, 0, 0, 0, 0, 0, 0, 825, 649, 688, 1415, 671, 214, 923, 763, 0, 0, 0, 0, 0, 0, 0, 0, 251, 490, 488, 179, 851, 1234, 406, 133, 2155, 0, 0, 0, 0, 0, 0, 0, 338, 386, 492, 732, 724, 1943, 278, 667, 208, 87, 69, 0, 0, 0, 0, 0, 0, 348, 1631, 708, 174, 650, 1497, 884, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6137, 6151, 6286, 6427, 6148, 6187, 5924, 5892] avg=6144.0000 max=6427 max_rank=3 min=5892 min_rank=7 imbalance_ratio=1.046061 max_load_reduction=42725 max_load_reduction_pct=0.869242 moved_to_echo=24151 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[20480, 4608, 7680, 2176, 128, 2176, 5888, 6400, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1920, 4224, 4096, 1408, 7296, 9472, 2944, 1152, 17280, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[13312, 4352, 3200, 7424, 896, 10752, 8320, 1408, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[6400, 5248, 5504, 11264, 5632, 1792, 7552, 6144, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2816, 4608, 5504, 3712, 6912, 2432, 5504, 7680, 6016, 4480, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2688, 3328, 4224, 5888, 6016, 16896, 2688, 4992, 1664, 768, 640, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 3200, 13312, 6144, 1536, 5504, 12928, 7040, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[4608, 6400, 6400, 5760, 1024, 3200, 1408, 8448, 10112, 2688, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[232, 485, 45, 230, 3701, 90, 419, 755, 14, 549, 2725, 284, 502, 632, 182, 3294, 387, 975, 528, 312, 893, 404, 1473, 1050, 839, 477, 720, 3036, 0, 60, 52, 154, 86, 7, 249, 6, 720, 6, 380, 148, 1009, 595, 248, 683, 16, 1111, 0, 118, 6337, 610, 4014, 0, 12, 497, 209, 1668, 2074, 376, 843, 226, 577, 94, 491, 243] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[217, 545, 47, 207, 3932, 49, 490, 653, 11, 530, 2701, 226, 552, 650, 171, 3545, 360, 1013, 437, 277, 884, 470, 1534, 890, 893, 502, 697, 3178, 0, 45, 31, 184, 79, 3, 268, 1, 738, 13, 429, 135, 952, 571, 268, 780, 16, 1089, 0, 108, 6505, 561, 3564, 0, 24, 441, 219, 1435, 2157, 333, 848, 277, 547, 88, 586, 196] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[313, 514, 49, 249, 3892, 53, 428, 829, 14, 654, 2692, 301, 604, 563, 215, 3009, 400, 1057, 530, 276, 1033, 427, 1471, 952, 887, 465, 645, 2765, 0, 60, 53, 163, 118, 9, 267, 9, 668, 12, 517, 131, 1056, 537, 257, 690, 19, 1108, 0, 127, 6191, 708, 3752, 0, 15, 540, 261, 1612, 2059, 356, 922, 316, 521, 85, 503, 223] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[252, 478, 47, 222, 3748, 105, 410, 836, 6, 641, 2543, 331, 576, 591, 163, 3220, 352, 1041, 501, 291, 1083, 368, 1476, 1012, 728, 499, 752, 2937, 3, 67, 79, 165, 49, 2, 282, 3, 751, 10, 427, 212, 1123, 671, 279, 812, 17, 840, 1, 107, 6138, 669, 3615, 0, 17, 462, 227, 1770, 2074, 355, 909, 385, 565, 126, 490, 241] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[335, 525, 41, 190, 4101, 50, 337, 796, 17, 504, 2811, 319, 515, 444, 132, 3240, 392, 1127, 553, 269, 949, 400, 1532, 933, 799, 524, 642, 3111, 4, 56, 36, 176, 75, 2, 264, 6, 742, 7, 477, 158, 1036, 554, 265, 702, 9, 833, 1, 141, 6368, 623, 3806, 0, 17, 470, 237, 1507, 1971, 385, 848, 372, 572, 79, 532, 233] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[311, 557, 58, 250, 3757, 70, 347, 800, 14, 620, 2647, 292, 577, 656, 204, 3196, 469, 1057, 553, 311, 931, 450, 1526, 945, 809, 485, 755, 2893, 2, 71, 90, 161, 119, 6, 256, 14, 819, 19, 402, 228, 1229, 610, 259, 688, 19, 1145, 3, 114, 5843, 664, 3096, 1, 16, 662, 275, 1679, 2072, 378, 902, 352, 598, 94, 513, 213] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[215, 546, 32, 206, 4073, 27, 357, 684, 14, 609, 2838, 262, 496, 676, 187, 3370, 451, 1026, 467, 265, 1007, 391, 1319, 1117, 823, 484, 707, 2877, 0, 43, 41, 144, 129, 0, 252, 3, 709, 15, 510, 178, 989, 513, 255, 708, 15, 1114, 0, 98, 6339, 664, 3767, 0, 14, 578, 195, 1437, 2148, 358, 834, 272, 539, 101, 466, 198] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[237, 453, 28, 195, 4158, 62, 337, 717, 6, 530, 2705, 254, 524, 529, 157, 3486, 398, 999, 536, 287, 948, 407, 1387, 950, 870, 487, 667, 3223, 0, 33, 30, 129, 82, 3, 245, 1, 757, 1, 406, 126, 984, 548, 246, 676, 10, 963, 0, 119, 6592, 589, 4034, 0, 6, 488, 201, 1445, 2107, 366, 803, 283, 503, 80, 560, 199] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[217, 545, 47, 207, 3905, 49, 490, 653, 0, 0, 0, 0, 0, 0, 0, 0, 11, 530, 2701, 226, 552, 650, 171, 1345, 0, 0, 0, 0, 0, 0, 0, 0, 360, 1013, 437, 277, 884, 470, 1534, 890, 50, 27, 0, 0, 0, 0, 0, 0, 893, 502, 697, 3178, 0, 45, 31, 184, 456, 409, 0, 0, 0, 0, 0, 0, 79, 3, 268, 1, 738, 13, 429, 135, 4577, 0, 0, 0, 0, 0, 0, 0, 952, 571, 268, 780, 16, 1089, 0, 108, 1928, 563, 0, 0, 0, 0, 0, 0, 0, 561, 3058, 0, 24, 441, 219, 1435, 0, 0, 0, 0, 0, 0, 0, 0, 2157, 333, 848, 277, 547, 88, 586, 196, 1228, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6113, 6186, 5942, 6395, 6243, 6275, 5738, 6260] avg=6144.0000 max=6395 max_rank=3 min=5738 min_rank=6 imbalance_ratio=1.040853 max_load_reduction=42757 max_load_reduction_pct=0.869893 moved_to_echo=24516 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[232, 485, 45, 230, 3675, 90, 419, 755, 0, 0, 0, 0, 0, 0, 0, 0, 14, 549, 2725, 284, 502, 632, 182, 1250, 0, 0, 0, 0, 0, 0, 0, 0, 387, 975, 528, 312, 893, 404, 1473, 1050, 56, 26, 0, 0, 0, 0, 0, 0, 839, 477, 720, 3036, 0, 60, 52, 154, 514, 380, 0, 0, 0, 0, 0, 0, 86, 7, 249, 6, 720, 6, 380, 148, 4460, 0, 0, 0, 0, 0, 0, 0, 1009, 595, 248, 683, 16, 1111, 0, 118, 1877, 523, 0, 0, 0, 0, 0, 0, 0, 610, 3444, 0, 12, 497, 209, 1668, 0, 0, 0, 0, 0, 0, 0, 0, 2074, 376, 843, 226, 577, 94, 491, 243, 1141, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5931, 6138, 6104, 6232, 6062, 6180, 6440, 6065] avg=6144.0000 max=6440 max_rank=6 min=5931 min_rank=0 imbalance_ratio=1.048177 max_load_reduction=42712 max_load_reduction_pct=0.868978 moved_to_echo=24747 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[252, 478, 47, 222, 3722, 105, 410, 836, 0, 0, 0, 0, 0, 0, 0, 0, 6, 641, 2543, 331, 576, 591, 163, 1223, 0, 0, 0, 0, 0, 0, 0, 0, 352, 1041, 501, 291, 1083, 368, 1476, 1012, 51, 26, 0, 0, 0, 0, 0, 0, 728, 499, 752, 2937, 3, 67, 79, 165, 463, 371, 0, 0, 0, 0, 0, 0, 49, 2, 282, 3, 751, 10, 427, 212, 4319, 0, 0, 0, 0, 0, 0, 0, 1123, 671, 279, 812, 17, 840, 1, 107, 1819, 511, 0, 0, 0, 0, 0, 0, 0, 669, 3101, 0, 17, 462, 227, 1770, 0, 0, 0, 0, 0, 0, 0, 0, 2074, 355, 909, 385, 565, 126, 490, 241, 1115, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6072, 6074, 6201, 6064, 6055, 6180, 6246, 6260] avg=6144.0000 max=6260 max_rank=7 min=6055 min_rank=4 imbalance_ratio=1.018880 max_load_reduction=42892 max_load_reduction_pct=0.872640 moved_to_echo=24741 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[335, 525, 41, 190, 4072, 50, 337, 796, 0, 0, 0, 0, 0, 0, 0, 0, 17, 504, 2811, 319, 515, 444, 132, 1229, 0, 0, 0, 0, 0, 0, 0, 0, 392, 1127, 553, 269, 949, 400, 1532, 933, 54, 29, 0, 0, 0, 0, 0, 0, 799, 524, 642, 3111, 4, 56, 36, 176, 487, 374, 0, 0, 0, 0, 0, 0, 75, 2, 264, 6, 742, 7, 477, 158, 4481, 0, 0, 0, 0, 0, 0, 0, 1036, 554, 265, 702, 9, 833, 1, 141, 1887, 515, 0, 0, 0, 0, 0, 0, 0, 623, 3265, 0, 17, 470, 237, 1507, 0, 0, 0, 0, 0, 0, 0, 0, 1971, 385, 848, 372, 572, 79, 532, 233, 1122, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6346, 5971, 6238, 6209, 6212, 5943, 6119, 6114] avg=6144.0000 max=6346 max_rank=0 min=5943 min_rank=5 imbalance_ratio=1.032878 max_load_reduction=42806 max_load_reduction_pct=0.870890 moved_to_echo=24388 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[313, 514, 49, 249, 3865, 53, 428, 829, 0, 0, 0, 0, 0, 0, 0, 0, 14, 654, 2692, 301, 604, 563, 215, 1142, 0, 0, 0, 0, 0, 0, 0, 0, 400, 1057, 530, 276, 1033, 427, 1471, 952, 53, 27, 0, 0, 0, 0, 0, 0, 887, 465, 645, 2765, 0, 60, 53, 163, 480, 347, 0, 0, 0, 0, 0, 0, 118, 9, 267, 9, 668, 12, 517, 131, 4357, 0, 0, 0, 0, 0, 0, 0, 1056, 537, 257, 690, 19, 1108, 0, 127, 1834, 478, 0, 0, 0, 0, 0, 0, 0, 708, 3219, 0, 15, 540, 261, 1612, 0, 0, 0, 0, 0, 0, 0, 0, 2059, 356, 922, 316, 521, 85, 503, 223, 1042, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6300, 6185, 6226, 5865, 6088, 6106, 6355, 6027] avg=6144.0000 max=6355 max_rank=6 min=5865 min_rank=3 imbalance_ratio=1.034342 max_load_reduction=42797 max_load_reduction_pct=0.870707 moved_to_echo=24576 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[237, 453, 28, 195, 4129, 62, 337, 717, 0, 0, 0, 0, 0, 0, 0, 0, 6, 530, 2705, 254, 524, 529, 157, 1322, 0, 0, 0, 0, 0, 0, 0, 0, 398, 999, 536, 287, 948, 407, 1387, 950, 57, 29, 0, 0, 0, 0, 0, 0, 870, 487, 667, 3223, 0, 33, 30, 129, 517, 402, 0, 0, 0, 0, 0, 0, 82, 3, 245, 1, 757, 1, 406, 126, 4639, 0, 0, 0, 0, 0, 0, 0, 984, 548, 246, 676, 10, 963, 0, 119, 1953, 554, 0, 0, 0, 0, 0, 0, 0, 589, 3460, 0, 6, 488, 201, 1445, 0, 0, 0, 0, 0, 0, 0, 0, 2107, 366, 803, 283, 503, 80, 560, 199, 1208, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6158, 6027, 5998, 6358, 6260, 6053, 6189, 6109] avg=6144.0000 max=6358 max_rank=3 min=5998 min_rank=2 imbalance_ratio=1.034831 max_load_reduction=42794 max_load_reduction_pct=0.870646 moved_to_echo=24611 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[311, 557, 58, 250, 3727, 70, 347, 800, 0, 0, 0, 0, 0, 0, 0, 0, 14, 620, 2647, 292, 577, 656, 204, 1200, 0, 0, 0, 0, 0, 0, 0, 0, 469, 1057, 553, 311, 931, 450, 1526, 945, 47, 30, 0, 0, 0, 0, 0, 0, 809, 485, 755, 2893, 2, 71, 90, 161, 401, 372, 0, 0, 0, 0, 0, 0, 119, 6, 256, 14, 819, 19, 402, 228, 4112, 0, 0, 0, 0, 0, 0, 0, 1229, 610, 259, 688, 19, 1145, 3, 114, 1731, 512, 0, 0, 0, 0, 0, 0, 0, 664, 2648, 1, 16, 662, 275, 1679, 0, 0, 0, 0, 0, 0, 0, 0, 2072, 378, 902, 352, 598, 94, 513, 213, 1112, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6120, 6210, 6319, 6039, 5975, 6310, 5945, 6234] avg=6144.0000 max=6319 max_rank=2 min=5945 min_rank=6 imbalance_ratio=1.028483 max_load_reduction=42833 max_load_reduction_pct=0.871440 moved_to_echo=24464 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[215, 546, 32, 206, 4045, 27, 357, 684, 0, 0, 0, 0, 0, 0, 0, 0, 14, 609, 2838, 262, 496, 676, 187, 1279, 0, 0, 0, 0, 0, 0, 0, 0, 451, 1026, 467, 265, 1007, 391, 1319, 1117, 53, 28, 0, 0, 0, 0, 0, 0, 823, 484, 707, 2877, 0, 43, 41, 144, 482, 389, 0, 0, 0, 0, 0, 0, 129, 0, 252, 3, 709, 15, 510, 178, 4461, 0, 0, 0, 0, 0, 0, 0, 989, 513, 255, 708, 15, 1114, 0, 98, 1878, 535, 0, 0, 0, 0, 0, 0, 0, 664, 3232, 0, 14, 578, 195, 1437, 0, 0, 0, 0, 0, 0, 0, 0, 2148, 358, 834, 272, 539, 101, 466, 198, 1167, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6112, 6361, 6124, 5990, 6257, 6105, 6120, 6083] avg=6144.0000 max=6361 max_rank=1 min=5990 min_rank=3 imbalance_ratio=1.035319 max_load_reduction=42791 max_load_reduction_pct=0.870585 moved_to_echo=24565 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[8448, 4608, 2176, 5760, 128, 8320, 128, 1024, 14976, 4224, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[16768, 2944, 6912, 2560, 4480, 768, 4224, 1792, 9216, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6656, 3968, 5632, 24064, 128, 512, 512, 1280, 3840, 3072, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 4736, 21760, 2304, 4352, 4864, 1536, 10112, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 5120, 25472, 128, 128, 4224, 1920, 12672, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2176, 4224, 384, 1792, 31232, 512, 3200, 6144, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3328, 8320, 4224, 2304, 7808, 3328, 11776, 7936, 512, 256, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[768, 128, 2176, 128, 6016, 128, 3584, 1408, 35456, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 567, 207, 138, 0, 164, 121, 380, 42, 665, 731, 2060, 628, 325, 0, 0, 2611, 1534, 1952, 55, 3, 0, 3744, 721, 657, 613, 3667, 4272, 3, 1, 290, 0, 147, 881, 645, 120, 5748, 1149, 598, 114, 529, 0, 4, 13, 1210, 1138, 72, 63, 69, 0, 490, 513, 0, 0, 64, 6, 3528, 319, 0, 3, 0, 2948, 0, 2630] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 693, 181, 161, 0, 124, 163, 248, 55, 581, 898, 1986, 559, 317, 0, 0, 2651, 1652, 2045, 51, 0, 0, 3758, 832, 576, 781, 3689, 4512, 0, 8, 302, 0, 116, 993, 371, 72, 6101, 932, 524, 83, 503, 0, 1, 16, 704, 1506, 63, 63, 94, 1, 359, 458, 0, 0, 52, 3, 3549, 240, 0, 15, 0, 2934, 0, 2576] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 683, 208, 114, 0, 148, 131, 319, 47, 594, 658, 2133, 542, 352, 0, 0, 2304, 1685, 1851, 51, 0, 0, 3987, 670, 569, 723, 3842, 4351, 1, 0, 292, 0, 121, 960, 542, 86, 5968, 820, 508, 74, 628, 0, 2, 17, 846, 1259, 89, 81, 196, 0, 193, 464, 0, 0, 95, 2, 3835, 225, 0, 6, 0, 2815, 0, 3065] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 598, 204, 136, 0, 151, 115, 261, 26, 610, 807, 2229, 499, 392, 0, 0, 2511, 1563, 2131, 55, 2, 0, 3828, 669, 597, 753, 3655, 4546, 0, 1, 264, 0, 135, 932, 344, 54, 6183, 839, 324, 94, 587, 0, 5, 11, 782, 1399, 78, 68, 38, 0, 330, 464, 0, 0, 94, 4, 3619, 251, 1, 6, 0, 2894, 0, 3013] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 683, 206, 146, 0, 162, 130, 336, 62, 558, 838, 2095, 641, 330, 0, 0, 2454, 1609, 1962, 75, 0, 0, 4037, 692, 692, 728, 3465, 4198, 1, 4, 253, 0, 116, 866, 402, 87, 5804, 902, 622, 89, 672, 0, 2, 18, 1062, 1288, 102, 97, 227, 0, 156, 496, 0, 0, 97, 5, 3780, 294, 0, 11, 0, 2947, 0, 2653] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1, 699, 196, 160, 0, 179, 119, 298, 48, 535, 636, 2088, 647, 340, 1, 0, 2370, 1649, 1964, 70, 1, 1, 3869, 673, 710, 772, 3669, 4221, 4, 6, 269, 0, 139, 965, 553, 79, 5633, 996, 599, 91, 651, 2, 9, 23, 1119, 1231, 86, 83, 242, 2, 176, 550, 0, 0, 95, 8, 3875, 303, 5, 8, 0, 2920, 0, 2514] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 616, 202, 144, 0, 163, 130, 254, 58, 545, 859, 2311, 551, 357, 0, 0, 2501, 1599, 2249, 55, 0, 0, 3829, 619, 570, 778, 3708, 4523, 0, 8, 254, 1, 187, 883, 321, 51, 6074, 962, 415, 82, 572, 0, 2, 18, 960, 1105, 85, 87, 162, 0, 156, 502, 0, 0, 91, 1, 3724, 242, 2, 3, 0, 2980, 0, 2601] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 616, 204, 159, 0, 214, 168, 322, 45, 623, 664, 2127, 592, 347, 2, 2, 2235, 1434, 1866, 71, 2, 2, 3903, 623, 547, 727, 3680, 4429, 2, 13, 280, 0, 126, 958, 529, 96, 5804, 906, 608, 95, 644, 4, 4, 27, 1256, 1614, 97, 89, 84, 3, 208, 511, 1, 0, 118, 8, 3545, 284, 2, 12, 5, 3132, 0, 2483] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 693, 181, 161, 0, 124, 163, 248, 2218, 2319, 0, 0, 0, 0, 0, 0, 55, 581, 898, 1986, 559, 317, 0, 0, 1069, 705, 0, 0, 0, 0, 0, 0, 1946, 1652, 2045, 51, 0, 0, 0, 832, 0, 0, 0, 0, 0, 0, 0, 0, 576, 781, 3689, 866, 0, 8, 302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 993, 371, 72, 3076, 932, 524, 83, 0, 0, 0, 0, 0, 0, 0, 0, 503, 0, 1, 16, 704, 1506, 63, 63, 1956, 1112, 0, 0, 0, 0, 0, 0, 94, 1, 359, 458, 0, 0, 52, 3, 3758, 1327, 0, 0, 0, 0, 0, 0, 219, 240, 0, 15, 0, 2934, 0, 2576, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6107, 6170, 6526, 6222, 6167, 5924, 6052, 5984] avg=6144.0000 max=6526 max_rank=2 min=5924 min_rank=5 imbalance_ratio=1.062174 max_load_reduction=42626 max_load_reduction_pct=0.867228 moved_to_echo=24127 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 567, 207, 138, 0, 164, 121, 380, 2205, 2195, 0, 0, 0, 0, 0, 0, 42, 665, 731, 2060, 628, 325, 0, 0, 1007, 694, 0, 0, 0, 0, 0, 0, 1917, 1534, 1952, 55, 3, 0, 0, 721, 0, 0, 0, 0, 0, 0, 0, 0, 657, 613, 3667, 821, 3, 1, 290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 881, 645, 120, 2898, 1149, 598, 114, 0, 0, 0, 0, 0, 0, 0, 0, 529, 0, 4, 13, 1210, 1138, 72, 63, 1843, 1105, 0, 0, 0, 0, 0, 0, 69, 0, 490, 513, 0, 0, 64, 6, 3744, 1256, 0, 0, 0, 0, 0, 0, 218, 319, 0, 3, 0, 2948, 0, 2630, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5977, 6152, 6182, 6052, 6552, 5977, 6142, 6118] avg=6144.0000 max=6552 max_rank=4 min=5977 min_rank=0 imbalance_ratio=1.066406 max_load_reduction=42600 max_load_reduction_pct=0.866699 moved_to_echo=24789 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 683, 208, 114, 0, 148, 131, 319, 2397, 2236, 0, 0, 0, 0, 0, 0, 47, 594, 658, 2133, 542, 352, 0, 0, 1045, 612, 0, 0, 0, 0, 0, 0, 1692, 1685, 1851, 51, 0, 0, 0, 670, 0, 0, 0, 0, 0, 0, 0, 0, 569, 723, 3842, 836, 1, 0, 292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 121, 960, 542, 86, 3009, 820, 508, 74, 0, 0, 0, 0, 0, 0, 0, 0, 628, 0, 2, 17, 846, 1259, 89, 81, 1914, 1201, 0, 0, 0, 0, 0, 0, 196, 0, 193, 464, 0, 0, 95, 2, 3987, 1279, 0, 0, 0, 0, 0, 0, 237, 225, 0, 6, 0, 2815, 0, 3065, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6236, 5983, 5949, 6263, 6120, 6037, 6216, 6348] avg=6144.0000 max=6348 max_rank=7 min=5949 min_rank=2 imbalance_ratio=1.033203 max_load_reduction=42804 max_load_reduction_pct=0.870850 moved_to_echo=24721 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 598, 204, 136, 0, 151, 115, 261, 2262, 2336, 0, 0, 0, 0, 0, 0, 26, 610, 807, 2229, 499, 392, 0, 0, 1083, 667, 0, 0, 0, 0, 0, 0, 1844, 1563, 2131, 55, 2, 0, 0, 669, 0, 0, 0, 0, 0, 0, 0, 0, 597, 753, 3655, 873, 0, 1, 264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 932, 344, 54, 3117, 839, 324, 94, 0, 0, 0, 0, 0, 0, 0, 0, 587, 0, 5, 11, 782, 1399, 78, 68, 1983, 1134, 0, 0, 0, 0, 0, 0, 38, 0, 330, 464, 0, 0, 94, 4, 3828, 1337, 0, 0, 0, 0, 0, 0, 223, 251, 1, 6, 0, 2894, 0, 3013, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6063, 6313, 6264, 6143, 5839, 6047, 6095, 6388] avg=6144.0000 max=6388 max_rank=7 min=5839 min_rank=4 imbalance_ratio=1.039714 max_load_reduction=42764 max_load_reduction_pct=0.870036 moved_to_echo=24369 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 699, 196, 160, 0, 179, 119, 298, 2422, 2169, 0, 0, 0, 0, 0, 0, 48, 535, 636, 2088, 647, 340, 1, 0, 987, 630, 0, 0, 0, 0, 0, 0, 1740, 1649, 1964, 70, 1, 1, 0, 673, 0, 0, 0, 0, 0, 0, 0, 0, 710, 772, 3669, 811, 4, 6, 269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 965, 553, 79, 2840, 996, 599, 91, 0, 0, 0, 0, 0, 0, 0, 0, 651, 2, 9, 23, 1119, 1231, 86, 83, 1806, 1214, 0, 0, 0, 0, 0, 0, 242, 2, 176, 550, 0, 0, 95, 8, 3869, 1241, 0, 0, 0, 0, 0, 0, 239, 303, 5, 8, 0, 2920, 0, 2514, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6243, 5912, 6098, 6241, 6262, 6224, 6183, 5989] avg=6144.0000 max=6262 max_rank=4 min=5912 min_rank=1 imbalance_ratio=1.019206 max_load_reduction=42890 max_load_reduction_pct=0.872599 moved_to_echo=24658 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 683, 206, 146, 0, 162, 130, 336, 2363, 2157, 0, 0, 0, 0, 0, 0, 62, 558, 838, 2095, 641, 330, 0, 0, 1017, 652, 0, 0, 0, 0, 0, 0, 1802, 1609, 1962, 75, 0, 0, 0, 692, 0, 0, 0, 0, 0, 0, 0, 0, 692, 728, 3465, 807, 1, 4, 253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 866, 402, 87, 2926, 902, 622, 89, 0, 0, 0, 0, 0, 0, 0, 0, 672, 0, 2, 18, 1062, 1288, 102, 97, 1861, 1184, 0, 0, 0, 0, 0, 0, 227, 0, 156, 496, 0, 0, 97, 5, 4037, 1234, 0, 0, 0, 0, 0, 0, 233, 294, 0, 11, 0, 2947, 0, 2653, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6183, 6193, 6140, 5950, 6010, 6286, 6252, 6138] avg=6144.0000 max=6286 max_rank=5 min=5950 min_rank=3 imbalance_ratio=1.023112 max_load_reduction=42866 max_load_reduction_pct=0.872111 moved_to_echo=24686 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 616, 202, 144, 0, 163, 130, 254, 2328, 2324, 0, 0, 0, 0, 0, 0, 58, 545, 859, 2311, 551, 357, 0, 0, 1064, 665, 0, 0, 0, 0, 0, 0, 1836, 1599, 2249, 55, 0, 0, 0, 619, 0, 0, 0, 0, 0, 0, 0, 0, 570, 778, 3708, 869, 0, 8, 254, 1, 0, 0, 0, 0, 0, 0, 0, 0, 187, 883, 321, 51, 3062, 962, 415, 82, 0, 0, 0, 0, 0, 0, 0, 0, 572, 0, 2, 18, 960, 1105, 85, 87, 1948, 1167, 0, 0, 0, 0, 0, 0, 162, 0, 156, 502, 0, 0, 91, 1, 3829, 1330, 0, 0, 0, 0, 0, 0, 229, 242, 2, 3, 0, 2980, 0, 2601, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6161, 6410, 6358, 6188, 5963, 5944, 6071, 6057] avg=6144.0000 max=6410 max_rank=1 min=5944 min_rank=5 imbalance_ratio=1.043294 max_load_reduction=42742 max_load_reduction_pct=0.869588 moved_to_echo=24035 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 616, 204, 159, 0, 214, 168, 322, 2219, 2280, 0, 0, 0, 0, 0, 0, 45, 623, 664, 2127, 592, 347, 2, 2, 1019, 598, 0, 0, 0, 0, 0, 0, 1637, 1434, 1866, 71, 2, 2, 0, 623, 0, 0, 0, 0, 0, 0, 0, 0, 547, 727, 3680, 844, 2, 13, 280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 958, 529, 96, 2921, 906, 608, 95, 0, 0, 0, 0, 0, 0, 0, 0, 644, 4, 4, 27, 1256, 1614, 97, 89, 1864, 1114, 0, 0, 0, 0, 0, 0, 84, 3, 208, 511, 1, 0, 118, 8, 3903, 1305, 0, 0, 0, 0, 0, 0, 212, 284, 2, 12, 5, 3132, 0, 2483, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6182, 6019, 5635, 6093, 6239, 6713, 6141, 6130] avg=6144.0000 max=6713 max_rank=5 min=5635 min_rank=2 imbalance_ratio=1.092611 max_load_reduction=42439 max_load_reduction_pct=0.863424 moved_to_echo=25223 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 5248, 1664, 1280, 0, 1408, 1152, 2432, 18432, 18048, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[384, 4736, 6144, 17152, 4736, 2816, 128, 128, 8320, 5248, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[4864, 128, 128, 256, 8064, 10624, 768, 640, 15232, 9344, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1152, 128, 2176, 3968, 128, 0, 768, 128, 30976, 10368, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1152, 7552, 3712, 768, 23936, 7552, 4224, 768, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[14464, 12800, 16128, 512, 128, 128, 0, 5504, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4992, 5888, 29440, 6784, 128, 128, 2304, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1920, 2176, 128, 128, 128, 23680, 0, 21632, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 493, 0, 1085, 489, 532, 310, 4289, 336, 554, 42, 346, 2, 6913, 25, 2994, 25, 633, 364, 508, 204, 0, 54, 804, 1130, 570, 437, 399, 2627, 421, 60, 0, 4, 70, 4, 340, 43, 8, 0, 12, 4546, 2, 463, 147, 1, 0, 4638, 0, 23, 0, 624, 0, 3312, 0, 0, 1, 0, 460, 1, 2625, 4670, 110, 402] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 395, 0, 963, 598, 560, 387, 4524, 329, 403, 9, 304, 1, 6895, 28, 3290, 31, 603, 371, 456, 250, 0, 40, 739, 1170, 500, 525, 392, 2751, 468, 86, 0, 1, 77, 1, 397, 51, 2, 0, 9, 4062, 0, 375, 186, 1, 0, 3995, 0, 41, 0, 612, 1, 3264, 0, 0, 3, 1, 526, 0, 2673, 5203, 126, 477] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 469, 0, 1153, 458, 538, 258, 4623, 233, 378, 0, 296, 1, 7122, 13, 3663, 29, 636, 299, 541, 210, 0, 45, 639, 1127, 539, 406, 368, 2565, 477, 47, 1, 1, 61, 4, 330, 44, 8, 0, 8, 4417, 2, 475, 161, 0, 2, 4638, 0, 10, 1, 506, 2, 3346, 0, 0, 7, 0, 415, 0, 2255, 4913, 67, 345] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 473, 0, 1086, 572, 561, 331, 4534, 273, 458, 0, 291, 0, 7001, 25, 3401, 14, 626, 304, 434, 210, 0, 68, 678, 1192, 522, 478, 384, 2650, 452, 93, 0, 3, 69, 6, 271, 38, 9, 0, 11, 4280, 0, 447, 197, 0, 0, 4404, 0, 10, 0, 555, 0, 3167, 0, 0, 1, 0, 498, 0, 2617, 4991, 74, 393] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 476, 0, 935, 655, 556, 350, 4513, 302, 493, 31, 323, 4, 6812, 29, 3275, 13, 694, 326, 506, 268, 0, 91, 736, 1221, 499, 472, 379, 2683, 512, 93, 0, 8, 81, 6, 371, 58, 14, 0, 16, 4072, 2, 405, 231, 1, 1, 4229, 1, 34, 4, 581, 4, 3039, 0, 0, 3, 2, 542, 1, 2613, 5081, 87, 418] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 351, 0, 1035, 497, 548, 346, 4719, 259, 351, 0, 277, 0, 6947, 10, 3420, 9, 635, 304, 558, 187, 0, 30, 691, 1097, 525, 538, 353, 2891, 424, 89, 0, 2, 64, 1, 266, 30, 5, 0, 18, 4429, 0, 439, 178, 0, 0, 4040, 0, 1, 0, 618, 1, 3267, 0, 0, 2, 0, 497, 0, 2319, 5399, 86, 399] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 506, 2, 997, 579, 562, 366, 4008, 324, 536, 43, 312, 4, 6819, 32, 3315, 22, 605, 329, 588, 279, 0, 64, 754, 1318, 510, 519, 418, 2606, 424, 112, 2, 4, 90, 8, 347, 67, 8, 6, 16, 4294, 2, 417, 223, 4, 0, 4229, 5, 24, 1, 606, 3, 3330, 0, 5, 8, 7, 545, 4, 2395, 5024, 92, 433] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 453, 0, 1139, 738, 510, 373, 4457, 253, 469, 0, 288, 0, 6950, 29, 3046, 8, 586, 317, 516, 234, 0, 63, 751, 1330, 486, 419, 360, 2412, 447, 92, 0, 2, 91, 5, 273, 33, 9, 0, 16, 4454, 0, 415, 249, 0, 0, 4397, 0, 0, 0, 578, 0, 3249, 0, 0, 0, 0, 470, 0, 2542, 5165, 69, 409] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 469, 0, 1153, 458, 538, 258, 2714, 671, 0, 0, 0, 0, 0, 0, 4623, 233, 378, 0, 296, 1, 614, 13, 0, 0, 0, 0, 0, 0, 0, 0, 3663, 29, 636, 299, 541, 210, 0, 45, 711, 327, 0, 0, 0, 0, 0, 0, 639, 1127, 539, 406, 368, 2238, 477, 47, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 61, 4, 330, 44, 8, 0, 5837, 0, 0, 0, 0, 0, 0, 0, 8, 4417, 2, 475, 161, 0, 2, 1271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1, 506, 2, 3346, 0, 0, 1634, 653, 0, 0, 0, 0, 0, 0, 7, 0, 415, 0, 2255, 2568, 67, 345, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6261, 6158, 6461, 5841, 6286, 6336, 6152, 5657] avg=6144.0000 max=6461 max_rank=2 min=5657 min_rank=7 imbalance_ratio=1.051595 max_load_reduction=42691 max_load_reduction_pct=0.868551 moved_to_echo=24431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 395, 0, 963, 598, 560, 387, 2338, 650, 0, 0, 0, 0, 0, 0, 4524, 329, 403, 9, 304, 1, 594, 28, 0, 0, 0, 0, 0, 0, 0, 0, 3290, 31, 603, 371, 456, 250, 0, 40, 753, 351, 0, 0, 0, 0, 0, 0, 739, 1170, 500, 525, 392, 2400, 468, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 77, 1, 397, 51, 2, 0, 5651, 0, 0, 0, 0, 0, 0, 0, 9, 4062, 0, 375, 186, 1, 0, 1095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 612, 1, 3264, 0, 0, 1731, 562, 0, 0, 0, 0, 0, 0, 3, 1, 526, 0, 2673, 2719, 126, 477, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5891, 6192, 6145, 6280, 6180, 5728, 6211, 6525] avg=6144.0000 max=6525 max_rank=7 min=5728 min_rank=5 imbalance_ratio=1.062012 max_load_reduction=42627 max_load_reduction_pct=0.867249 moved_to_echo=24644 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 493, 0, 1085, 489, 532, 310, 2714, 652, 0, 0, 0, 0, 0, 0, 4289, 336, 554, 42, 346, 2, 595, 25, 0, 0, 0, 0, 0, 0, 0, 0, 2994, 25, 633, 364, 508, 204, 0, 54, 676, 335, 0, 0, 0, 0, 0, 0, 804, 1130, 570, 437, 399, 2292, 421, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 70, 4, 340, 43, 8, 0, 5666, 0, 0, 0, 0, 0, 0, 0, 12, 4546, 2, 463, 147, 1, 0, 1271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 624, 0, 3312, 0, 0, 1553, 653, 0, 0, 0, 0, 0, 0, 1, 0, 460, 1, 2625, 2441, 110, 402, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6275, 6189, 5793, 6113, 6135, 6442, 6165, 6040] avg=6144.0000 max=6442 max_rank=5 min=5793 min_rank=2 imbalance_ratio=1.048503 max_load_reduction=42710 max_load_reduction_pct=0.868937 moved_to_echo=24782 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 476, 0, 935, 655, 556, 350, 2475, 642, 0, 0, 0, 0, 0, 0, 4513, 302, 493, 31, 323, 4, 587, 29, 0, 0, 0, 0, 0, 0, 0, 0, 3275, 13, 694, 326, 506, 268, 0, 91, 735, 342, 0, 0, 0, 0, 0, 0, 736, 1221, 499, 472, 379, 2341, 512, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 81, 6, 371, 58, 14, 0, 5583, 0, 0, 0, 0, 0, 0, 0, 16, 4072, 2, 405, 231, 1, 1, 1159, 0, 0, 0, 0, 0, 0, 0, 0, 1, 34, 4, 581, 4, 3039, 0, 0, 1690, 595, 0, 0, 0, 0, 0, 0, 3, 2, 542, 1, 2613, 2656, 87, 418, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6089, 6282, 6250, 6253, 6121, 5887, 5948, 6322] avg=6144.0000 max=6322 max_rank=7 min=5887 min_rank=5 imbalance_ratio=1.028971 max_load_reduction=42830 max_load_reduction_pct=0.871379 moved_to_echo=24278 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 473, 0, 1086, 572, 561, 331, 2577, 660, 0, 0, 0, 0, 0, 0, 4534, 273, 458, 0, 291, 0, 603, 25, 0, 0, 0, 0, 0, 0, 0, 0, 3401, 14, 626, 304, 434, 210, 0, 68, 722, 338, 0, 0, 0, 0, 0, 0, 678, 1192, 522, 478, 384, 2312, 452, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 69, 6, 271, 38, 9, 0, 5738, 0, 0, 0, 0, 0, 0, 0, 11, 4280, 0, 447, 197, 0, 0, 1207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 555, 0, 3167, 0, 0, 1660, 620, 0, 0, 0, 0, 0, 0, 1, 0, 498, 0, 2617, 2609, 74, 393, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6260, 6184, 6117, 6111, 6134, 6142, 6012, 6192] avg=6144.0000 max=6260 max_rank=0 min=6012 min_rank=6 imbalance_ratio=1.018880 max_load_reduction=42892 max_load_reduction_pct=0.872640 moved_to_echo=24480 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 351, 0, 1035, 497, 548, 346, 2364, 655, 0, 0, 0, 0, 0, 0, 4719, 259, 351, 0, 277, 0, 598, 10, 0, 0, 0, 0, 0, 0, 0, 0, 3420, 9, 635, 304, 558, 187, 0, 30, 781, 369, 0, 0, 0, 0, 0, 0, 691, 1097, 525, 538, 353, 2522, 424, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 64, 1, 266, 30, 5, 0, 5694, 0, 0, 0, 0, 0, 0, 0, 18, 4429, 0, 439, 178, 0, 0, 1107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 618, 1, 3267, 0, 0, 1796, 569, 0, 0, 0, 0, 0, 0, 2, 0, 497, 0, 2319, 2822, 86, 399, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5796, 6214, 6293, 6239, 6062, 6171, 6252, 6125] avg=6144.0000 max=6293 max_rank=2 min=5796 min_rank=0 imbalance_ratio=1.024251 max_load_reduction=42859 max_load_reduction_pct=0.871969 moved_to_echo=24610 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 453, 0, 1139, 738, 510, 373, 2573, 655, 0, 0, 0, 0, 0, 0, 4457, 253, 469, 0, 288, 0, 599, 29, 0, 0, 0, 0, 0, 0, 0, 0, 3046, 8, 586, 317, 516, 234, 0, 63, 747, 307, 0, 0, 0, 0, 0, 0, 751, 1330, 486, 419, 360, 2105, 447, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 91, 5, 273, 33, 9, 0, 5696, 0, 0, 0, 0, 0, 0, 0, 16, 4454, 0, 415, 249, 0, 0, 1205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 578, 0, 3249, 0, 0, 1718, 619, 0, 0, 0, 0, 0, 0, 0, 0, 470, 0, 2542, 2700, 69, 409, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6441, 6095, 5824, 5990, 6109, 6339, 6164, 6190] avg=6144.0000 max=6441 max_rank=0 min=5824 min_rank=2 imbalance_ratio=1.048340 max_load_reduction=42711 max_load_reduction_pct=0.868958 moved_to_echo=24802 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 506, 2, 997, 579, 562, 366, 2480, 647, 0, 0, 0, 0, 0, 0, 4008, 324, 536, 43, 312, 4, 579, 32, 0, 0, 0, 0, 0, 0, 0, 0, 3315, 22, 605, 329, 588, 279, 0, 64, 731, 336, 0, 0, 0, 0, 0, 0, 754, 1318, 510, 519, 418, 2270, 424, 112, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 90, 8, 347, 67, 8, 6, 5593, 0, 0, 0, 0, 0, 0, 0, 16, 4294, 2, 417, 223, 4, 0, 1151, 0, 0, 0, 0, 0, 0, 0, 0, 5, 24, 1, 606, 3, 3330, 0, 5, 1676, 598, 0, 0, 0, 0, 0, 0, 8, 7, 545, 4, 2395, 2617, 92, 433, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6139, 5838, 6269, 6325, 6125, 6107, 6248, 6101] avg=6144.0000 max=6325 max_rank=3 min=5838 min_rank=1 imbalance_ratio=1.029460 max_load_reduction=42827 max_load_reduction_pct=0.871318 moved_to_echo=24581 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[26496, 256, 5120, 2688, 4224, 1920, 0, 512, 5888, 2816, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 0, 3712, 128, 8448, 4608, 4480, 2816, 20352, 5248, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 3968, 128, 20096, 21248, 768, 3328, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 34560, 128, 3456, 1664, 128, 128, 9472, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5888, 9600, 4224, 3840, 3072, 18560, 3712, 768, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 256, 128, 4736, 128, 25984, 0, 128, 13568, 4992, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[35712, 2432, 3712, 128, 2560, 128, 4864, 256, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 640, 128, 2688, 384, 128, 128, 45568, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 1557, 1332, 77, 134, 2226, 2, 2880, 0, 656, 438, 42, 65, 0, 0, 0, 0, 455, 0, 162, 2, 478, 5410, 0, 255, 0, 6429, 1, 602, 76, 65, 0, 0, 0, 0, 715, 947, 0, 33, 1188, 0, 1515, 0, 4206, 35, 0, 377, 0, 0, 1087, 2412, 0, 1, 6222, 5511, 470, 1, 37, 0, 435, 615, 0, 1, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 1540, 1431, 44, 82, 2278, 0, 2734, 0, 615, 375, 65, 86, 0, 0, 0, 0, 513, 0, 134, 0, 572, 5382, 1, 226, 0, 6154, 1, 598, 83, 62, 0, 0, 0, 0, 782, 779, 0, 21, 1350, 0, 1659, 0, 4354, 67, 0, 301, 0, 0, 1300, 2253, 0, 0, 6264, 5682, 436, 0, 68, 0, 350, 510, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 1457, 1492, 96, 122, 1784, 0, 2685, 0, 794, 537, 70, 110, 0, 0, 0, 0, 458, 0, 129, 0, 746, 5380, 1, 335, 0, 6174, 2, 475, 74, 60, 0, 1, 0, 0, 720, 931, 0, 21, 1220, 0, 1697, 0, 4139, 70, 0, 354, 0, 0, 1356, 2710, 0, 0, 6137, 5276, 492, 1, 86, 0, 400, 555, 0, 5, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 1367, 1436, 78, 124, 2356, 0, 2955, 0, 605, 446, 92, 126, 0, 0, 0, 0, 485, 0, 110, 0, 620, 5070, 2, 257, 0, 6210, 0, 605, 100, 44, 0, 0, 0, 0, 873, 779, 0, 15, 1211, 0, 1671, 0, 4354, 90, 0, 355, 0, 0, 1214, 2468, 0, 2, 6155, 5376, 431, 0, 96, 0, 397, 577, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 1477, 1373, 60, 138, 2158, 0, 2764, 0, 649, 380, 60, 74, 0, 0, 0, 0, 520, 0, 141, 0, 708, 5289, 7, 259, 0, 6121, 0, 489, 87, 69, 0, 0, 0, 0, 766, 752, 0, 15, 1413, 0, 1739, 0, 4274, 76, 0, 358, 0, 0, 1107, 2660, 0, 0, 6159, 5503, 440, 0, 61, 0, 424, 582, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[2, 1558, 1386, 96, 156, 2154, 3, 2931, 0, 672, 486, 73, 88, 0, 1, 2, 0, 491, 0, 108, 0, 855, 5210, 12, 322, 0, 6120, 2, 594, 101, 68, 0, 3, 0, 0, 856, 996, 0, 19, 1145, 0, 1617, 0, 4311, 84, 1, 281, 1, 0, 1193, 2366, 0, 0, 6138, 5175, 437, 5, 86, 0, 377, 563, 0, 7, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 1509, 1408, 64, 138, 2114, 0, 2885, 0, 761, 366, 59, 69, 0, 0, 0, 0, 451, 0, 113, 0, 668, 5254, 4, 352, 0, 6140, 0, 544, 98, 81, 0, 3, 0, 0, 793, 1033, 0, 31, 1234, 0, 1504, 0, 4277, 71, 0, 405, 0, 0, 1129, 2489, 0, 0, 6166, 5490, 433, 1, 60, 0, 381, 574, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[5, 1553, 1448, 93, 115, 2163, 1, 2758, 0, 515, 639, 70, 92, 0, 5, 2, 0, 478, 7, 144, 1, 687, 5190, 3, 254, 6, 6260, 1, 609, 104, 59, 8, 32, 5, 0, 766, 974, 0, 25, 1224, 2, 1584, 0, 4239, 107, 5, 340, 3, 1, 1185, 2230, 2, 5, 5977, 5471, 439, 0, 83, 0, 372, 810, 0, 1, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1457, 1492, 96, 122, 1784, 0, 828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 794, 537, 70, 110, 0, 0, 0, 3285, 1071, 392, 0, 0, 0, 0, 0, 0, 458, 0, 129, 0, 746, 4960, 1, 0, 0, 0, 0, 0, 0, 0, 0, 335, 0, 5119, 2, 475, 74, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 720, 931, 0, 21, 1220, 1465, 1055, 420, 166, 0, 0, 0, 0, 0, 1697, 0, 3973, 70, 0, 354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1356, 2710, 0, 0, 0, 1991, 492, 0, 0, 0, 0, 0, 0, 0, 0, 1, 86, 0, 400, 555, 0, 5, 0, 5066, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5779, 6259, 6294, 6065, 5999, 6094, 6549, 6113] avg=6144.0000 max=6549 max_rank=6 min=5779 min_rank=0 imbalance_ratio=1.065918 max_load_reduction=42603 max_load_reduction_pct=0.866760 moved_to_echo=24755 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1557, 1332, 77, 134, 2226, 2, 889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 656, 438, 42, 65, 0, 0, 0, 3432, 1085, 420, 0, 0, 0, 0, 0, 0, 455, 0, 162, 2, 478, 4987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 0, 5330, 1, 602, 76, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 715, 947, 0, 33, 1188, 1571, 1099, 423, 168, 0, 0, 0, 0, 0, 1515, 0, 4038, 35, 0, 377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1087, 2412, 0, 1, 0, 2079, 470, 0, 0, 0, 0, 0, 0, 0, 0, 1, 37, 0, 435, 615, 0, 1, 0, 5137, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6217, 6138, 6084, 6329, 6144, 5965, 6049, 6226] avg=6144.0000 max=6329 max_rank=3 min=5965 min_rank=5 imbalance_ratio=1.030111 max_load_reduction=42823 max_load_reduction_pct=0.871236 moved_to_echo=24384 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1477, 1373, 60, 138, 2158, 0, 853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 649, 380, 60, 74, 0, 0, 0, 3427, 1075, 403, 0, 0, 0, 0, 0, 0, 520, 0, 141, 0, 708, 4876, 7, 0, 0, 0, 0, 0, 0, 0, 0, 259, 0, 5075, 0, 489, 87, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 766, 752, 0, 15, 1413, 1508, 1046, 413, 171, 0, 0, 0, 0, 0, 1739, 0, 4103, 76, 0, 358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1107, 2660, 0, 0, 0, 2076, 440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 424, 582, 0, 0, 0, 5084, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6059, 6068, 6252, 5979, 6084, 6276, 6283, 6151] avg=6144.0000 max=6283 max_rank=6 min=5979 min_rank=3 imbalance_ratio=1.022624 max_load_reduction=42869 max_load_reduction_pct=0.872172 moved_to_echo=24794 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1367, 1436, 78, 124, 2356, 0, 912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 605, 446, 92, 126, 0, 0, 0, 3347, 1073, 431, 0, 0, 0, 0, 0, 0, 485, 0, 110, 0, 620, 4674, 2, 0, 0, 0, 0, 0, 0, 0, 0, 257, 0, 5148, 0, 605, 100, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 873, 779, 0, 15, 1211, 1612, 1062, 396, 174, 0, 0, 0, 0, 0, 1671, 0, 4180, 90, 0, 355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1214, 2468, 0, 2, 0, 2029, 431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 397, 577, 0, 0, 0, 5082, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6273, 6120, 5891, 6154, 6122, 6296, 6144, 6152] avg=6144.0000 max=6296 max_rank=5 min=5891 min_rank=2 imbalance_ratio=1.024740 max_load_reduction=42856 max_load_reduction_pct=0.871908 moved_to_echo=24714 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2, 1558, 1386, 96, 156, 2154, 3, 904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 672, 486, 73, 88, 0, 1, 2, 3222, 1071, 428, 0, 0, 0, 0, 0, 0, 491, 0, 108, 0, 855, 4803, 12, 0, 0, 0, 0, 0, 0, 0, 0, 322, 0, 5074, 2, 594, 101, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 856, 996, 0, 19, 1145, 1599, 1046, 407, 173, 0, 0, 0, 0, 0, 1617, 0, 4138, 84, 1, 281, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1193, 2366, 0, 0, 0, 1953, 437, 0, 0, 0, 0, 0, 0, 0, 0, 5, 86, 0, 377, 563, 0, 7, 0, 5067, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6259, 6043, 6269, 6161, 6244, 6122, 5949, 6105] avg=6144.0000 max=6269 max_rank=2 min=5949 min_rank=6 imbalance_ratio=1.020345 max_load_reduction=42883 max_load_reduction_pct=0.872457 moved_to_echo=24420 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1540, 1431, 44, 82, 2278, 0, 844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 615, 375, 65, 86, 0, 0, 0, 3538, 1093, 399, 0, 0, 0, 0, 0, 0, 513, 0, 134, 0, 572, 4962, 1, 0, 0, 0, 0, 0, 0, 0, 0, 226, 0, 5102, 1, 598, 83, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 782, 779, 0, 21, 1350, 1491, 1052, 420, 174, 0, 0, 0, 0, 0, 1659, 0, 4180, 67, 0, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1300, 2253, 0, 0, 0, 2144, 436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 350, 510, 0, 0, 0, 5171, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6219, 6171, 6182, 6072, 6069, 6207, 6133, 6099] avg=6144.0000 max=6219 max_rank=0 min=6069 min_rank=4 imbalance_ratio=1.012207 max_load_reduction=42933 max_load_reduction_pct=0.873474 moved_to_echo=24508 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[5, 1553, 1448, 93, 115, 2163, 1, 845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 639, 70, 92, 0, 5, 2, 3411, 1043, 406, 0, 0, 0, 0, 0, 0, 478, 7, 144, 1, 687, 4780, 3, 0, 0, 0, 0, 0, 0, 0, 0, 254, 6, 5186, 1, 609, 104, 59, 8, 0, 0, 0, 0, 0, 0, 0, 0, 32, 5, 0, 766, 974, 0, 25, 1224, 1507, 1074, 410, 174, 0, 0, 0, 0, 2, 1584, 0, 4065, 107, 5, 340, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1185, 2230, 2, 5, 0, 2060, 439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 372, 810, 0, 1, 0, 4934, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6223, 6183, 6100, 6227, 6191, 6106, 5922, 6200] avg=6144.0000 max=6227 max_rank=3 min=5922 min_rank=6 imbalance_ratio=1.013509 max_load_reduction=42925 max_load_reduction_pct=0.873311 moved_to_echo=24419 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1509, 1408, 64, 138, 2114, 0, 890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 761, 366, 59, 69, 0, 0, 0, 3418, 1076, 421, 0, 0, 0, 0, 0, 0, 451, 0, 113, 0, 668, 4844, 4, 0, 0, 0, 0, 0, 0, 0, 0, 352, 0, 5090, 0, 544, 98, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 793, 1033, 0, 31, 1234, 1574, 1050, 410, 171, 0, 0, 0, 0, 0, 1504, 0, 4106, 71, 0, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1129, 2489, 0, 0, 0, 2072, 433, 0, 0, 0, 0, 0, 0, 0, 0, 1, 60, 0, 381, 574, 0, 0, 0, 5090, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6123, 6170, 6080, 6165, 6299, 6086, 6123, 6106] avg=6144.0000 max=6299 max_rank=4 min=6080 min_rank=2 imbalance_ratio=1.025228 max_load_reduction=42853 max_load_reduction_pct=0.871847 moved_to_echo=24614 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 12032, 11392, 640, 1024, 17280, 128, 7040, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[0, 5376, 3712, 640, 768, 0, 128, 128, 27136, 8704, 3328, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 0, 6272, 7296, 0, 256, 10112, 12416, 8576, 3328, 1408, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 9600, 19712, 128, 128, 0, 16512, 3584, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 13056, 0, 32896, 640, 128, 2816, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 640, 0, 3200, 4864, 0, 128, 0, 40704, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2304, 128, 41216, 128, 4608, 768, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 3968, 128, 1152, 128, 5376, 38912, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[109, 3122, 64, 134, 68, 1901, 1667, 85, 0, 0, 0, 587, 1310, 0, 0, 787, 312, 4643, 7554, 9, 0, 870, 5379, 0, 0, 1694, 2286, 0, 0, 0, 1191, 0, 4191, 41, 0, 0, 0, 0, 289, 181, 349, 0, 0, 66, 1, 0, 139, 1148, 53, 623, 878, 753, 0, 0, 0, 998, 1, 0, 63, 0, 2, 415, 1, 5188] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[172, 2687, 83, 159, 69, 2359, 1672, 165, 0, 0, 1, 527, 1231, 0, 1, 706, 334, 4635, 7293, 2, 2, 908, 5368, 3, 1, 1394, 2489, 1, 0, 0, 985, 0, 4368, 189, 0, 0, 0, 1, 366, 265, 325, 0, 0, 93, 3, 0, 195, 1009, 82, 559, 745, 619, 0, 0, 1, 1167, 23, 0, 86, 0, 48, 506, 0, 5255] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[172, 2789, 80, 143, 69, 2146, 1935, 132, 0, 0, 0, 536, 1162, 0, 0, 797, 346, 4338, 7422, 35, 0, 1018, 5403, 0, 0, 1707, 2514, 0, 0, 0, 914, 0, 4196, 33, 0, 0, 1, 0, 344, 265, 439, 0, 0, 81, 0, 0, 98, 963, 45, 538, 801, 760, 0, 0, 0, 1051, 4, 0, 80, 0, 17, 373, 0, 5405] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[186, 2688, 94, 172, 81, 2321, 1747, 183, 0, 0, 0, 503, 1155, 0, 0, 772, 326, 4705, 7384, 3, 0, 854, 5399, 0, 0, 1466, 2415, 0, 0, 0, 975, 0, 4264, 66, 0, 0, 0, 0, 364, 296, 315, 0, 0, 95, 0, 0, 199, 1021, 87, 560, 778, 636, 0, 0, 0, 1064, 4, 0, 94, 0, 4, 465, 0, 5411] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[131, 2909, 70, 166, 61, 2431, 1428, 121, 0, 0, 0, 574, 1119, 0, 0, 803, 323, 4716, 7513, 11, 0, 931, 5237, 0, 0, 1546, 2294, 0, 0, 0, 1079, 0, 4376, 69, 0, 0, 0, 0, 274, 178, 301, 0, 0, 70, 20, 0, 181, 1032, 62, 674, 919, 710, 0, 0, 0, 1088, 9, 2, 70, 0, 1, 405, 0, 5248] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[171, 2791, 85, 133, 41, 2550, 1694, 171, 0, 0, 0, 534, 1207, 0, 0, 764, 354, 4529, 7269, 9, 0, 1044, 5346, 0, 0, 1489, 2518, 0, 0, 0, 992, 0, 4236, 60, 0, 0, 1, 0, 294, 252, 463, 0, 0, 84, 2, 1, 157, 1055, 75, 409, 801, 701, 0, 0, 0, 1125, 12, 0, 85, 0, 5, 421, 0, 5222] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[166, 2790, 76, 141, 62, 2185, 1510, 109, 0, 0, 0, 547, 1089, 0, 0, 792, 392, 4889, 7476, 13, 0, 990, 5372, 0, 0, 1429, 2458, 0, 0, 0, 1062, 0, 4077, 62, 0, 0, 0, 0, 306, 182, 339, 0, 0, 75, 2, 0, 143, 1146, 62, 575, 899, 767, 0, 0, 0, 1044, 2, 0, 76, 0, 1, 367, 0, 5479] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[194, 2580, 92, 159, 55, 2407, 1746, 171, 0, 5, 2, 542, 998, 0, 2, 800, 341, 4603, 7151, 17, 0, 1033, 5251, 8, 5, 1704, 2530, 8, 0, 1, 991, 6, 4204, 291, 3, 0, 15, 8, 341, 228, 348, 2, 0, 114, 10, 2, 191, 864, 75, 550, 788, 718, 0, 1, 8, 1129, 34, 1, 89, 0, 7, 457, 0, 5272] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[109, 1768, 64, 134, 68, 1901, 1667, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 587, 1310, 0, 0, 787, 2948, 786, 0, 0, 0, 0, 0, 0, 312, 4643, 0, 9, 0, 870, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1694, 2286, 0, 0, 0, 1191, 0, 1013, 220, 0, 0, 0, 0, 0, 0, 4191, 41, 0, 0, 0, 0, 289, 181, 1254, 0, 0, 0, 0, 0, 0, 0, 349, 0, 0, 66, 1, 0, 139, 1148, 4606, 0, 0, 0, 0, 0, 0, 0, 53, 623, 878, 753, 0, 0, 0, 998, 2915, 0, 0, 0, 0, 0, 0, 0, 1, 0, 63, 0, 2, 415, 1, 5188, 341, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5796, 6418, 6038, 6404, 5956, 6309, 6220, 6011] avg=6144.0000 max=6418 max_rank=1 min=5796 min_rank=0 imbalance_ratio=1.044596 max_load_reduction=42734 max_load_reduction_pct=0.869425 moved_to_echo=24496 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[131, 1648, 70, 166, 61, 2431, 1428, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 574, 1119, 0, 0, 803, 2932, 765, 0, 0, 0, 0, 0, 0, 323, 4716, 0, 11, 0, 931, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1546, 2294, 0, 0, 0, 1079, 0, 944, 214, 0, 0, 0, 0, 0, 0, 4376, 69, 0, 0, 0, 0, 274, 178, 1221, 0, 0, 0, 0, 0, 0, 0, 301, 0, 0, 70, 20, 0, 181, 1032, 4581, 0, 0, 0, 0, 0, 0, 0, 62, 674, 919, 710, 0, 0, 0, 1088, 2838, 0, 0, 0, 0, 0, 0, 0, 9, 2, 70, 0, 1, 405, 0, 5248, 317, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6056, 6193, 6180, 6077, 6118, 6185, 6291, 6052] avg=6144.0000 max=6291 max_rank=6 min=6052 min_rank=7 imbalance_ratio=1.023926 max_load_reduction=42861 max_load_reduction_pct=0.872009 moved_to_echo=24646 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[171, 1582, 85, 133, 41, 2550, 1694, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 1207, 0, 0, 764, 2836, 781, 0, 0, 0, 0, 0, 0, 354, 4529, 0, 9, 0, 1044, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1489, 2518, 0, 0, 0, 992, 0, 905, 219, 0, 0, 0, 0, 0, 0, 4236, 60, 0, 0, 1, 0, 294, 252, 1247, 0, 0, 0, 0, 0, 0, 0, 463, 0, 0, 84, 2, 1, 157, 1055, 4433, 0, 0, 0, 0, 0, 0, 0, 75, 409, 801, 701, 0, 0, 0, 1125, 2897, 0, 0, 0, 0, 0, 0, 0, 12, 0, 85, 0, 5, 421, 0, 5222, 304, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6427, 6122, 6138, 6123, 6090, 6195, 6008, 6049] avg=6144.0000 max=6427 max_rank=0 min=6008 min_rank=6 imbalance_ratio=1.046061 max_load_reduction=42725 max_load_reduction_pct=0.869242 moved_to_echo=24342 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[172, 1580, 80, 143, 69, 2146, 1935, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 1162, 0, 0, 797, 2897, 789, 0, 0, 0, 0, 0, 0, 346, 4338, 0, 35, 0, 1018, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1707, 2514, 0, 0, 0, 914, 0, 905, 221, 0, 0, 0, 0, 0, 0, 4196, 33, 0, 0, 1, 0, 344, 265, 1260, 0, 0, 0, 0, 0, 0, 0, 439, 0, 0, 81, 0, 0, 98, 963, 4525, 0, 0, 0, 0, 0, 0, 0, 45, 538, 801, 760, 0, 0, 0, 1051, 2928, 0, 0, 0, 0, 0, 0, 0, 4, 0, 80, 0, 17, 373, 0, 5405, 304, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6257, 6181, 5942, 6261, 6099, 6106, 6123, 6183] avg=6144.0000 max=6261 max_rank=3 min=5942 min_rank=2 imbalance_ratio=1.019043 max_load_reduction=42891 max_load_reduction_pct=0.872620 moved_to_echo=24511 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[172, 1522, 83, 159, 69, 2359, 1672, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 527, 1231, 0, 1, 706, 2847, 784, 0, 0, 0, 0, 0, 0, 334, 4635, 0, 2, 2, 908, 204, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1394, 2489, 1, 0, 0, 985, 0, 872, 219, 0, 0, 0, 0, 0, 0, 4368, 189, 0, 0, 0, 1, 366, 265, 1252, 0, 0, 0, 0, 0, 0, 0, 325, 0, 0, 93, 3, 0, 195, 1009, 4446, 0, 0, 0, 0, 0, 0, 0, 82, 559, 745, 619, 0, 0, 1, 1167, 2909, 0, 0, 0, 0, 0, 0, 0, 23, 0, 86, 0, 48, 506, 0, 5255, 293, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6201, 6097, 6088, 5961, 6441, 6071, 6082, 6211] avg=6144.0000 max=6441 max_rank=4 min=5961 min_rank=3 imbalance_ratio=1.048340 max_load_reduction=42711 max_load_reduction_pct=0.868958 moved_to_echo=24805 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[166, 1581, 76, 141, 62, 2185, 1510, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 1089, 0, 0, 792, 2918, 785, 0, 0, 0, 0, 0, 0, 392, 4889, 0, 13, 0, 990, 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1429, 2458, 0, 0, 0, 1062, 0, 905, 220, 0, 0, 0, 0, 0, 0, 4077, 62, 0, 0, 0, 0, 306, 182, 1253, 0, 0, 0, 0, 0, 0, 0, 339, 0, 0, 75, 2, 0, 143, 1146, 4558, 0, 0, 0, 0, 0, 0, 0, 62, 575, 899, 767, 0, 0, 0, 1044, 2911, 0, 0, 0, 0, 0, 0, 0, 2, 0, 76, 0, 1, 367, 0, 5479, 304, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5830, 6131, 6487, 6074, 5880, 6263, 6258, 6229] avg=6144.0000 max=6487 max_rank=2 min=5830 min_rank=0 imbalance_ratio=1.055827 max_load_reduction=42665 max_load_reduction_pct=0.868022 moved_to_echo=24630 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[194, 1454, 92, 159, 55, 2407, 1746, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 542, 998, 0, 2, 800, 2791, 770, 0, 0, 0, 0, 0, 0, 341, 4603, 0, 17, 0, 1033, 184, 8, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1704, 2530, 8, 0, 1, 991, 6, 840, 218, 0, 0, 0, 0, 0, 0, 4204, 291, 3, 0, 15, 8, 341, 228, 1229, 0, 0, 0, 0, 0, 0, 0, 348, 2, 0, 114, 10, 2, 191, 864, 4360, 0, 0, 0, 0, 0, 0, 0, 75, 550, 788, 718, 0, 1, 8, 1129, 2850, 0, 0, 0, 0, 0, 0, 0, 34, 1, 89, 0, 7, 457, 0, 5272, 286, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6278, 5910, 6186, 6303, 6319, 5891, 6119, 6146] avg=6144.0000 max=6319 max_rank=4 min=5891 min_rank=5 imbalance_ratio=1.028483 max_load_reduction=42833 max_load_reduction_pct=0.871440 moved_to_echo=24475 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[186, 1523, 94, 172, 81, 2321, 1747, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 1155, 0, 0, 772, 2882, 788, 0, 0, 0, 0, 0, 0, 326, 4705, 0, 3, 0, 854, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1466, 2415, 0, 0, 0, 975, 0, 872, 221, 0, 0, 0, 0, 0, 0, 4264, 66, 0, 0, 0, 0, 364, 296, 1259, 0, 0, 0, 0, 0, 0, 0, 315, 0, 0, 95, 0, 0, 199, 1021, 4502, 0, 0, 0, 0, 0, 0, 0, 87, 560, 778, 636, 0, 0, 0, 1064, 2926, 0, 0, 0, 0, 0, 0, 0, 4, 0, 94, 0, 4, 465, 0, 5411, 293, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6307, 6100, 6093, 5949, 6249, 6132, 6051, 6271] avg=6144.0000 max=6307 max_rank=0 min=5949 min_rank=3 imbalance_ratio=1.026530 max_load_reduction=42845 max_load_reduction_pct=0.871684 moved_to_echo=24703 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 128, 128, 4352, 9344, 0, 128, 6272, 23168, 6272, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[640, 4608, 6656, 5760, 0, 128, 128, 8704, 23296, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[1408, 12672, 768, 1280, 512, 18304, 13440, 1152, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 12544, 19584, 128, 0, 128, 8192, 128, 7296, 1792, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2816, 37120, 0, 128, 128, 7680, 1664, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[33920, 896, 128, 0, 128, 128, 2688, 1920, 9984, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[2944, 128, 0, 768, 128, 128, 1408, 8320, 36096, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 768, 0, 128, 3456, 128, 42496, 2560, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[3859, 0, 0, 0, 0, 0, 0, 0, 0, 108, 84, 0, 2207, 6440, 0, 128, 0, 0, 0, 0, 0, 227, 75, 133, 0, 41, 1, 0, 0, 63, 0, 0, 0, 4775, 3335, 0, 0, 0, 0, 0, 738, 997, 0, 0, 0, 63, 2719, 1655, 0, 63, 0, 163, 46, 6571, 5954, 0, 588, 0, 0, 0, 0, 0, 8119, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[3856, 0, 0, 0, 0, 0, 0, 0, 0, 171, 130, 0, 1865, 6615, 0, 107, 0, 0, 0, 0, 0, 116, 72, 153, 0, 59, 0, 0, 0, 70, 0, 0, 0, 4647, 3701, 0, 0, 0, 0, 0, 746, 868, 0, 0, 0, 70, 2320, 1555, 0, 70, 0, 288, 60, 6697, 6024, 0, 827, 0, 0, 0, 0, 0, 8065, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[3780, 0, 0, 0, 0, 0, 0, 0, 0, 165, 177, 0, 2172, 6511, 0, 156, 0, 0, 1, 0, 0, 181, 105, 179, 0, 26, 4, 0, 0, 85, 0, 0, 0, 4482, 3411, 0, 0, 0, 0, 0, 683, 940, 0, 0, 1, 85, 2451, 1682, 0, 85, 0, 289, 86, 6554, 6126, 0, 641, 0, 0, 0, 0, 0, 8094, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[3735, 0, 0, 0, 0, 0, 0, 0, 0, 143, 152, 0, 2346, 6416, 0, 179, 0, 0, 0, 0, 0, 130, 81, 170, 0, 18, 1, 0, 0, 80, 0, 0, 0, 4733, 3313, 0, 0, 0, 0, 0, 704, 908, 0, 0, 0, 80, 2755, 1703, 0, 80, 0, 167, 92, 6336, 5969, 0, 754, 0, 0, 0, 0, 0, 8107, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[3885, 0, 0, 0, 0, 0, 0, 0, 0, 144, 151, 0, 2023, 6388, 0, 221, 0, 0, 0, 0, 0, 245, 103, 194, 0, 81, 15, 0, 0, 94, 0, 0, 0, 4860, 3220, 0, 0, 0, 0, 0, 688, 918, 0, 0, 0, 94, 2508, 1308, 0, 94, 0, 166, 92, 6678, 6199, 0, 691, 0, 0, 0, 0, 0, 8092, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[3948, 0, 0, 0, 0, 0, 0, 0, 0, 111, 155, 0, 2129, 6449, 0, 153, 0, 0, 0, 0, 0, 337, 131, 153, 0, 29, 1, 0, 0, 76, 0, 0, 0, 4748, 3408, 0, 0, 0, 0, 0, 707, 977, 0, 0, 0, 76, 2578, 1306, 0, 76, 0, 225, 89, 6625, 6031, 0, 518, 0, 0, 0, 0, 0, 8116, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[3557, 0, 0, 0, 0, 0, 0, 0, 0, 156, 165, 0, 2168, 6451, 0, 203, 0, 0, 2, 1, 1, 164, 89, 178, 0, 62, 2, 1, 0, 83, 0, 1, 1, 4570, 3468, 0, 0, 1, 0, 0, 677, 872, 1, 0, 0, 83, 2779, 1642, 1, 84, 0, 258, 89, 6662, 6145, 1, 470, 0, 0, 0, 0, 1, 8063, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[3423, 0, 3, 1, 6, 0, 3, 15, 4, 144, 171, 2, 2142, 6312, 2, 188, 0, 0, 10, 0, 5, 150, 137, 190, 1, 43, 27, 2, 0, 90, 1, 0, 1, 4890, 3322, 0, 0, 7, 4, 0, 700, 1050, 10, 0, 0, 88, 2749, 1739, 8, 90, 12, 252, 114, 6252, 6006, 8, 772, 0, 0, 0, 8, 5, 7993, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3859, 0, 0, 0, 0, 0, 0, 0, 1993, 347, 70, 0, 0, 0, 0, 0, 0, 108, 84, 0, 2207, 3550, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 227, 75, 133, 2890, 2261, 536, 0, 0, 0, 0, 0, 0, 41, 1, 0, 0, 63, 0, 0, 6035, 0, 0, 0, 0, 0, 0, 0, 0, 2782, 3335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 738, 997, 0, 0, 0, 63, 2719, 1655, 230, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 163, 46, 0, 5654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 588, 0, 0, 0, 0, 0, 5511, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6269, 6077, 6122, 6140, 6117, 6402, 5926, 6099] avg=6144.0000 max=6402 max_rank=5 min=5926 min_rank=6 imbalance_ratio=1.041992 max_load_reduction=42750 max_load_reduction_pct=0.869751 moved_to_echo=24544 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3856, 0, 0, 0, 0, 0, 0, 0, 1940, 344, 70, 0, 0, 0, 0, 0, 0, 171, 130, 0, 1865, 3646, 0, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 72, 153, 2969, 2246, 547, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 70, 0, 0, 6150, 0, 0, 0, 0, 0, 0, 0, 0, 2707, 3701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 746, 868, 0, 0, 0, 70, 2320, 1555, 233, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 288, 60, 0, 5721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 827, 0, 0, 0, 0, 0, 5475, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6210, 5919, 6103, 6279, 6408, 5792, 6139, 6302] avg=6144.0000 max=6408 max_rank=4 min=5792 min_rank=5 imbalance_ratio=1.042969 max_load_reduction=42744 max_load_reduction_pct=0.869629 moved_to_echo=24641 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3780, 0, 0, 0, 0, 0, 0, 0, 1871, 346, 72, 0, 0, 0, 0, 0, 0, 165, 177, 0, 2172, 3589, 0, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 181, 105, 179, 2922, 2254, 535, 0, 0, 0, 0, 0, 0, 26, 4, 0, 0, 85, 0, 0, 6019, 0, 0, 0, 0, 0, 0, 0, 0, 2611, 3411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 683, 940, 0, 0, 1, 85, 2451, 1682, 237, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 289, 86, 0, 5817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 641, 0, 0, 0, 0, 0, 5494, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6069, 6259, 6177, 6134, 6022, 6079, 6277, 6135] avg=6144.0000 max=6277 max_rank=6 min=6022 min_rank=4 imbalance_ratio=1.021647 max_load_reduction=42875 max_load_reduction_pct=0.872294 moved_to_echo=24513 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3735, 0, 0, 0, 0, 0, 0, 0, 1976, 346, 70, 0, 0, 0, 0, 0, 0, 143, 152, 0, 2346, 3537, 0, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 81, 170, 2879, 2258, 518, 0, 0, 0, 0, 0, 0, 18, 1, 0, 0, 80, 0, 0, 5818, 0, 0, 0, 0, 0, 0, 0, 0, 2757, 3313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 704, 908, 0, 0, 0, 80, 2755, 1703, 231, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 167, 92, 0, 5668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 754, 0, 0, 0, 0, 0, 5503, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6127, 6357, 6036, 5917, 6070, 6381, 6007, 6257] avg=6144.0000 max=6381 max_rank=5 min=5917 min_rank=3 imbalance_ratio=1.038574 max_load_reduction=42771 max_load_reduction_pct=0.870178 moved_to_echo=24715 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3557, 0, 0, 0, 0, 0, 0, 0, 1908, 344, 72, 0, 0, 0, 0, 0, 0, 156, 165, 0, 2168, 3556, 0, 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 164, 89, 178, 2895, 2245, 545, 0, 0, 0, 0, 0, 0, 62, 2, 1, 0, 83, 0, 1, 6117, 0, 0, 0, 0, 0, 0, 0, 1, 2662, 3468, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 677, 872, 1, 0, 0, 83, 2779, 1642, 238, 0, 0, 0, 0, 0, 0, 0, 1, 84, 0, 258, 89, 0, 5835, 1, 0, 0, 0, 0, 0, 0, 0, 0, 470, 0, 0, 0, 0, 1, 5474, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5881, 6248, 6120, 6266, 6132, 6292, 6268, 5945] avg=6144.0000 max=6292 max_rank=5 min=5881 min_rank=0 imbalance_ratio=1.024089 max_load_reduction=42860 max_load_reduction_pct=0.871989 moved_to_echo=24637 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3948, 0, 0, 0, 0, 0, 0, 0, 1982, 347, 71, 0, 0, 0, 0, 0, 0, 111, 155, 0, 2129, 3555, 0, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 337, 131, 153, 2894, 2260, 542, 0, 0, 0, 0, 0, 0, 29, 1, 0, 0, 76, 0, 0, 6083, 0, 0, 0, 0, 0, 0, 0, 0, 2766, 3408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 707, 977, 0, 0, 0, 76, 2578, 1306, 233, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 225, 89, 0, 5727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 0, 0, 0, 0, 0, 5509, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6348, 6103, 6317, 6189, 6174, 5877, 6117, 6027] avg=6144.0000 max=6348 max_rank=0 min=5877 min_rank=5 imbalance_ratio=1.033203 max_load_reduction=42804 max_load_reduction_pct=0.870850 moved_to_echo=24195 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3423, 0, 3, 1, 6, 0, 3, 15, 2046, 345, 73, 0, 0, 0, 0, 0, 4, 144, 171, 2, 2142, 3476, 2, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 5, 150, 137, 190, 2836, 2230, 511, 0, 0, 0, 0, 0, 1, 43, 27, 2, 0, 90, 1, 0, 5741, 0, 0, 0, 0, 0, 0, 0, 1, 2844, 3322, 0, 0, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 700, 1050, 10, 0, 0, 88, 2749, 1739, 237, 0, 0, 0, 0, 0, 0, 0, 8, 90, 12, 252, 114, 0, 5696, 8, 0, 0, 0, 0, 0, 0, 0, 0, 772, 0, 0, 0, 8, 5, 5418, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5915, 6129, 6069, 5905, 6178, 6573, 6180, 6203] avg=6144.0000 max=6573 max_rank=5 min=5905 min_rank=3 imbalance_ratio=1.069824 max_load_reduction=42579 max_load_reduction_pct=0.866272 moved_to_echo=25134 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3885, 0, 0, 0, 0, 0, 0, 0, 2029, 346, 73, 0, 0, 0, 0, 0, 0, 144, 151, 0, 2023, 3521, 0, 221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, 103, 194, 2867, 2253, 546, 0, 0, 0, 0, 0, 0, 81, 15, 0, 0, 94, 0, 0, 6132, 0, 0, 0, 0, 0, 0, 0, 0, 2831, 3220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 688, 918, 0, 0, 0, 94, 2508, 1308, 240, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 166, 92, 0, 5886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 691, 0, 0, 0, 0, 0, 5493, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6333, 6060, 6208, 6322, 6051, 5756, 6238, 6184] avg=6144.0000 max=6333 max_rank=0 min=5756 min_rank=5 imbalance_ratio=1.030762 max_load_reduction=42819 max_load_reduction_pct=0.871155 moved_to_echo=24229 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[0, 0, 128, 128, 128, 1664, 896, 1408, 23168, 18048, 4352, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5760, 7552, 128, 0, 128, 640, 20864, 12672, 1920, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[30080, 0, 128, 128, 128, 0, 128, 128, 15872, 2816, 640, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 1152, 1280, 128, 17152, 28544, 128, 1408, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 22016, 27264, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[5376, 0, 0, 0, 128, 128, 43904, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 384, 128, 128, 0, 768, 128, 128, 48128, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 768, 128, 1920, 768, 0, 46080, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[274, 772, 0, 39, 274, 0, 0, 0, 0, 3777, 116, 5038, 0, 0, 1642, 5225, 110, 910, 0, 0, 574, 0, 0, 399, 559, 1056, 0, 6314, 0, 11, 0, 0, 0, 0, 0, 0, 4378, 936, 287, 0, 19, 0, 0, 0, 0, 60, 0, 0, 3589, 0, 0, 6083, 469, 2410, 1188, 0, 0, 0, 0, 0, 0, 432, 2151, 60] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[264, 601, 0, 19, 264, 0, 0, 0, 0, 4006, 67, 5189, 0, 0, 1664, 5112, 65, 919, 0, 0, 575, 0, 0, 400, 470, 738, 0, 6482, 0, 0, 0, 0, 0, 0, 0, 0, 4147, 1081, 304, 1, 151, 0, 0, 0, 0, 47, 0, 0, 3616, 0, 0, 6309, 468, 2201, 1137, 0, 0, 0, 0, 0, 0, 433, 2376, 46] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[260, 759, 0, 116, 260, 0, 0, 0, 0, 4073, 144, 5006, 0, 0, 1821, 5051, 126, 757, 0, 0, 591, 0, 0, 334, 465, 841, 0, 6456, 0, 0, 0, 0, 0, 0, 0, 0, 4267, 1040, 258, 0, 189, 0, 0, 0, 0, 89, 0, 0, 3115, 0, 0, 6258, 419, 2333, 1177, 0, 0, 0, 0, 0, 0, 537, 2324, 86] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[250, 683, 14, 59, 248, 0, 0, 0, 1, 3816, 115, 5084, 4, 1, 1688, 5232, 91, 900, 13, 3, 599, 0, 0, 362, 476, 1163, 0, 6229, 7, 4, 0, 0, 2, 2, 0, 9, 3916, 1125, 303, 0, 239, 0, 1, 3, 0, 104, 0, 10, 3773, 13, 0, 6190, 454, 2103, 1241, 0, 0, 3, 8, 2, 2, 512, 1991, 104] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[245, 686, 0, 30, 245, 0, 0, 0, 0, 3898, 122, 4922, 0, 0, 1669, 5109, 74, 847, 0, 0, 555, 0, 0, 377, 483, 1011, 0, 6435, 3, 0, 0, 0, 0, 0, 0, 0, 4384, 1100, 274, 0, 167, 0, 0, 0, 0, 92, 0, 0, 3418, 0, 0, 6260, 458, 2304, 1242, 0, 0, 0, 0, 0, 0, 452, 2198, 92] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[252, 747, 1, 11, 252, 0, 0, 0, 0, 4000, 125, 5017, 0, 0, 1856, 5270, 89, 856, 1, 0, 585, 0, 0, 337, 584, 895, 0, 6090, 0, 0, 0, 0, 1, 0, 0, 1, 4252, 1150, 307, 0, 63, 1, 1, 0, 0, 89, 0, 1, 3516, 1, 0, 6201, 420, 2390, 1245, 0, 0, 0, 0, 0, 0, 388, 2067, 90] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[251, 605, 0, 15, 251, 0, 0, 0, 0, 4269, 101, 5104, 0, 0, 1778, 5120, 69, 939, 0, 0, 572, 0, 0, 332, 560, 824, 0, 6318, 0, 0, 0, 0, 0, 0, 0, 0, 4156, 1103, 267, 0, 54, 0, 0, 0, 0, 92, 0, 0, 3377, 0, 0, 6267, 428, 2277, 1185, 0, 0, 0, 0, 0, 0, 525, 2222, 91] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[244, 601, 0, 50, 245, 0, 0, 0, 0, 3915, 117, 5110, 0, 0, 1765, 5039, 86, 878, 0, 0, 553, 0, 0, 372, 590, 841, 0, 6593, 0, 0, 0, 0, 0, 0, 0, 0, 4382, 1103, 297, 0, 87, 0, 0, 0, 0, 90, 0, 0, 3283, 0, 0, 6273, 454, 2212, 1191, 0, 0, 0, 0, 0, 0, 408, 2284, 89] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[274, 772, 0, 39, 274, 0, 0, 0, 4687, 285, 0, 0, 0, 0, 0, 0, 0, 3777, 116, 325, 0, 0, 1642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 910, 0, 0, 574, 0, 0, 399, 3693, 538, 0, 0, 0, 0, 0, 0, 559, 1056, 0, 4660, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4378, 936, 287, 0, 559, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 60, 0, 0, 5798, 0, 0, 0, 0, 0, 0, 0, 2300, 0, 0, 0, 469, 2410, 1188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 432, 2151, 60, 1654, 1020, 730, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6331, 5860, 6224, 6286, 6160, 5877, 6367, 6047] avg=6144.0000 max=6367 max_rank=6 min=5860 min_rank=1 imbalance_ratio=1.036296 max_load_reduction=42785 max_load_reduction_pct=0.870463 moved_to_echo=24451 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[264, 601, 0, 19, 264, 0, 0, 0, 4586, 295, 0, 0, 0, 0, 0, 0, 0, 4006, 67, 334, 0, 0, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 919, 0, 0, 575, 0, 0, 400, 3804, 526, 0, 0, 0, 0, 0, 0, 470, 738, 0, 4783, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4147, 1081, 304, 1, 564, 0, 0, 0, 0, 0, 0, 0, 151, 0, 0, 0, 0, 47, 0, 0, 6014, 0, 0, 0, 0, 0, 0, 0, 2316, 0, 0, 0, 468, 2201, 1137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 433, 2376, 46, 1699, 1051, 736, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6029, 6071, 6289, 5991, 6097, 6212, 6122, 6341] avg=6144.0000 max=6341 max_rank=7 min=5991 min_rank=3 imbalance_ratio=1.032064 max_load_reduction=42811 max_load_reduction_pct=0.870992 moved_to_echo=24772 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[260, 759, 0, 116, 260, 0, 0, 0, 4531, 293, 0, 0, 0, 0, 0, 0, 0, 4073, 144, 323, 0, 0, 1821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 757, 0, 0, 591, 0, 0, 334, 3670, 520, 0, 0, 0, 0, 0, 0, 465, 841, 0, 4764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4267, 1040, 258, 0, 485, 0, 0, 0, 0, 0, 0, 0, 189, 0, 0, 0, 0, 89, 0, 0, 5965, 0, 0, 0, 0, 0, 0, 0, 1996, 0, 0, 0, 419, 2333, 1177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 2324, 86, 1692, 1013, 634, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6219, 6361, 5998, 6070, 6050, 6243, 5925, 6286] avg=6144.0000 max=6361 max_rank=1 min=5925 min_rank=6 imbalance_ratio=1.035319 max_load_reduction=42791 max_load_reduction_pct=0.870585 moved_to_echo=24504 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[245, 686, 0, 30, 245, 0, 0, 0, 4584, 294, 0, 0, 0, 0, 0, 0, 0, 3898, 122, 318, 0, 0, 1669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 847, 0, 0, 555, 0, 0, 377, 3608, 525, 0, 0, 0, 0, 0, 0, 483, 1011, 0, 4749, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4384, 1100, 274, 0, 533, 0, 0, 0, 0, 0, 0, 0, 167, 0, 0, 0, 0, 92, 0, 0, 5966, 0, 0, 0, 0, 0, 0, 0, 2190, 0, 0, 0, 458, 2304, 1242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 452, 2198, 92, 1686, 996, 695, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6084, 6007, 5986, 6246, 6291, 6225, 6194, 6119] avg=6144.0000 max=6291 max_rank=4 min=5986 min_rank=2 imbalance_ratio=1.023926 max_load_reduction=42861 max_load_reduction_pct=0.872009 moved_to_echo=24829 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[252, 747, 1, 11, 252, 0, 0, 0, 4729, 291, 0, 0, 0, 0, 0, 0, 0, 4000, 125, 323, 0, 0, 1856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 856, 1, 0, 585, 0, 0, 337, 3678, 541, 0, 0, 0, 0, 0, 0, 584, 895, 0, 4494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 4252, 1150, 307, 0, 548, 0, 0, 0, 0, 0, 0, 0, 63, 1, 1, 0, 0, 89, 0, 1, 5910, 0, 0, 0, 0, 0, 0, 0, 2253, 1, 0, 0, 420, 2390, 1245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 388, 2067, 90, 1596, 1016, 715, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6283, 6304, 6087, 5973, 6259, 6065, 6309, 5872] avg=6144.0000 max=6309 max_rank=6 min=5872 min_rank=7 imbalance_ratio=1.026855 max_load_reduction=42843 max_load_reduction_pct=0.871643 moved_to_echo=24505 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[244, 601, 0, 50, 245, 0, 0, 0, 4521, 295, 0, 0, 0, 0, 0, 0, 0, 3915, 117, 329, 0, 0, 1765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 878, 0, 0, 553, 0, 0, 372, 3746, 518, 0, 0, 0, 0, 0, 0, 590, 841, 0, 4865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4382, 1103, 297, 0, 512, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 0, 0, 90, 0, 0, 5978, 0, 0, 0, 0, 0, 0, 0, 2103, 0, 0, 0, 454, 2212, 1191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 408, 2284, 89, 1728, 1035, 668, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5956, 6126, 6153, 6296, 6294, 6155, 5960, 6212] avg=6144.0000 max=6296 max_rank=3 min=5956 min_rank=0 imbalance_ratio=1.024740 max_load_reduction=42856 max_load_reduction_pct=0.871908 moved_to_echo=24621 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[250, 683, 14, 59, 248, 0, 0, 0, 4695, 291, 0, 0, 0, 0, 0, 0, 1, 3816, 115, 321, 4, 1, 1688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 900, 13, 3, 599, 0, 0, 362, 3730, 537, 0, 0, 0, 0, 0, 0, 476, 1163, 0, 4594, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 9, 3916, 1125, 303, 0, 592, 0, 0, 0, 0, 0, 0, 0, 239, 0, 1, 3, 0, 104, 0, 10, 5899, 0, 0, 0, 0, 0, 0, 0, 2410, 13, 0, 0, 454, 2103, 1241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8, 2, 2, 512, 1991, 104, 1635, 1033, 771, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6240, 5946, 6235, 6244, 5949, 6256, 6221, 6061] avg=6144.0000 max=6256 max_rank=5 min=5946 min_rank=1 imbalance_ratio=1.018229 max_load_reduction=42896 max_load_reduction_pct=0.872721 moved_to_echo=24487 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[251, 605, 0, 15, 251, 0, 0, 0, 4594, 294, 0, 0, 0, 0, 0, 0, 0, 4269, 101, 329, 0, 0, 1778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 939, 0, 0, 572, 0, 0, 332, 3742, 526, 0, 0, 0, 0, 0, 0, 560, 824, 0, 4662, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4156, 1103, 267, 0, 526, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 92, 0, 0, 5973, 0, 0, 0, 0, 0, 0, 0, 2164, 0, 0, 0, 428, 2277, 1185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 2222, 91, 1656, 1033, 687, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6010, 6477, 6180, 6046, 6052, 6119, 6054, 6214] avg=6144.0000 max=6477 max_rank=1 min=6010 min_rank=0 imbalance_ratio=1.054199 max_load_reduction=42675 max_load_reduction_pct=0.868225 moved_to_echo=24439 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[17792, 128, 0, 0, 3584, 18304, 9728, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=50176 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 128, 128, 3712, 17664, 768, 13440, 8320, 5760, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[768, 7040, 128, 128, 4608, 0, 0, 2944, 29696, 4352, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 31872, 1024, 2688, 128, 128, 13952, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[2048, 5504, 128, 384, 2048, 0, 0, 0, 36992, 2432, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4224, 7424, 0, 37632, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1024, 128, 128, 128, 0, 768, 0, 128, 47616, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 33920, 8704, 2304, 128, 4352, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=50176 slack_slots=48128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[6958, 70, 0, 216, 5343, 0, 370, 149, 0, 0, 0, 887, 0, 0, 0, 7798, 0, 1665, 4304, 0, 12, 0, 340, 0, 0, 948, 0, 0, 0, 140, 0, 0, 1888, 0, 0, 11, 0, 1296, 4652, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1243, 0, 0, 0, 0, 1923, 52, 0, 2753, 0, 0, 217, 487, 3248, 110, 1063] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[7164, 63, 0, 219, 5076, 0, 405, 114, 0, 2, 0, 722, 0, 0, 0, 7817, 0, 1597, 4061, 0, 60, 0, 417, 0, 0, 965, 0, 0, 0, 98, 0, 0, 1930, 0, 0, 34, 0, 1322, 5192, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1329, 0, 0, 0, 0, 1588, 55, 0, 2328, 0, 0, 201, 459, 3663, 65, 1201] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[6938, 85, 0, 259, 5034, 0, 369, 194, 0, 0, 0, 820, 0, 0, 0, 7754, 0, 1730, 4426, 0, 2, 0, 407, 0, 0, 877, 0, 0, 0, 175, 0, 0, 1858, 0, 0, 50, 0, 1297, 5071, 0, 0, 0, 0, 0, 0, 0, 0, 984, 1214, 0, 0, 0, 0, 1783, 33, 0, 2188, 0, 0, 270, 513, 3539, 126, 1156] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[6870, 83, 1, 213, 4956, 0, 363, 186, 0, 4, 1, 811, 1, 0, 0, 7767, 0, 1465, 4371, 0, 61, 0, 609, 0, 1, 815, 2, 0, 0, 152, 0, 1, 1959, 0, 0, 75, 2, 1266, 5219, 1, 0, 0, 0, 0, 0, 2, 0, 1029, 1269, 0, 0, 0, 0, 2071, 62, 1, 2362, 2, 0, 224, 456, 3511, 89, 819] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[6917, 88, 0, 227, 4726, 3, 366, 216, 3, 8, 11, 825, 16, 0, 2, 7735, 0, 1782, 4106, 0, 131, 0, 601, 4, 0, 932, 0, 0, 0, 161, 9, 4, 1795, 0, 2, 85, 0, 1300, 5289, 10, 8, 5, 1, 2, 0, 12, 15, 862, 1279, 3, 0, 0, 1, 1834, 53, 11, 2157, 3, 0, 295, 493, 3746, 102, 916] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[7103, 76, 0, 235, 5087, 0, 396, 176, 0, 0, 0, 891, 0, 1, 0, 7724, 0, 1822, 3926, 0, 0, 0, 581, 0, 0, 939, 0, 0, 0, 140, 0, 0, 1615, 0, 0, 5, 1, 1384, 5204, 0, 0, 0, 0, 0, 0, 0, 0, 929, 1225, 0, 0, 0, 0, 1710, 60, 0, 2200, 0, 0, 238, 439, 3792, 85, 1168] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[7077, 94, 0, 185, 5181, 0, 350, 195, 0, 0, 0, 944, 0, 0, 0, 7772, 0, 1594, 4243, 0, 27, 0, 519, 0, 0, 914, 0, 0, 0, 153, 0, 0, 1793, 0, 0, 8, 0, 1302, 5199, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1208, 0, 0, 0, 0, 1924, 70, 0, 2078, 0, 0, 263, 408, 3591, 69, 953] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[7013, 80, 0, 241, 4843, 0, 370, 186, 0, 0, 0, 860, 0, 0, 0, 7766, 0, 1742, 4623, 0, 3, 0, 528, 0, 0, 934, 0, 0, 0, 140, 0, 0, 1789, 0, 0, 22, 0, 1263, 5243, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1248, 0, 0, 0, 0, 1734, 35, 0, 2042, 0, 0, 227, 459, 3780, 74, 882] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[264, 63, 0, 219, 5076, 0, 405, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 722, 0, 0, 0, 5328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1597, 3751, 0, 60, 0, 417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 965, 0, 0, 0, 98, 0, 0, 2489, 1630, 1025, 0, 0, 0, 0, 0, 1930, 0, 0, 34, 0, 1322, 3008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 5270, 0, 0, 0, 0, 0, 0, 0, 1329, 0, 0, 0, 0, 1588, 55, 0, 1567, 1159, 310, 0, 0, 0, 0, 0, 2328, 0, 0, 201, 459, 2096, 65, 1201, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6141, 6052, 5825, 6207, 6294, 6275, 6008, 6350] avg=6144.0000 max=6350 max_rank=7 min=5825 min_rank=2 imbalance_ratio=1.033529 max_load_reduction=42802 max_load_reduction_pct=0.870809 moved_to_echo=24927 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[257, 70, 0, 216, 5343, 0, 370, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 887, 0, 0, 0, 5315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1665, 3975, 0, 12, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 948, 0, 0, 0, 140, 0, 0, 2483, 1583, 918, 0, 0, 0, 0, 0, 1888, 0, 0, 11, 0, 1296, 2696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 5118, 0, 0, 0, 0, 0, 0, 0, 1243, 0, 0, 0, 0, 1923, 52, 0, 1390, 1038, 329, 0, 0, 0, 0, 0, 2753, 0, 0, 217, 487, 1858, 110, 1063, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6405, 6202, 5992, 6072, 5891, 6127, 5975, 6488] avg=6144.0000 max=6488 max_rank=7 min=5891 min_rank=4 imbalance_ratio=1.055990 max_load_reduction=42664 max_load_reduction_pct=0.868001 moved_to_echo=24481 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[257, 85, 0, 259, 5034, 0, 369, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 820, 0, 0, 0, 5285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1730, 4088, 0, 2, 0, 407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 877, 0, 0, 0, 175, 0, 0, 2469, 1578, 1001, 0, 0, 0, 0, 0, 1858, 0, 0, 50, 0, 1297, 2938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 984, 5103, 0, 0, 0, 0, 0, 0, 0, 1214, 0, 0, 0, 0, 1783, 33, 0, 1514, 1132, 338, 0, 0, 0, 0, 0, 2188, 0, 0, 270, 513, 2025, 126, 1156, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6198, 6105, 6227, 6100, 6143, 6087, 6014, 6278] avg=6144.0000 max=6278 max_rank=7 min=6014 min_rank=6 imbalance_ratio=1.021810 max_load_reduction=42874 max_load_reduction_pct=0.872274 moved_to_echo=24522 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[259, 80, 0, 241, 4843, 0, 370, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 860, 0, 0, 0, 5293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1742, 4270, 0, 3, 0, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 934, 0, 0, 0, 140, 0, 0, 2473, 1595, 1035, 0, 0, 0, 0, 0, 1789, 0, 0, 22, 0, 1263, 3038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 5159, 0, 0, 0, 0, 0, 0, 0, 1248, 0, 0, 0, 0, 1734, 35, 0, 1618, 1170, 353, 0, 0, 0, 0, 0, 2042, 0, 0, 227, 459, 2162, 74, 882, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5979, 6153, 6543, 6177, 6112, 6184, 6158, 5846] avg=6144.0000 max=6543 max_rank=2 min=5846 min_rank=7 imbalance_ratio=1.064941 max_load_reduction=42609 max_load_reduction_pct=0.866882 moved_to_echo=24300 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[254, 83, 1, 213, 4956, 0, 363, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 811, 1, 0, 0, 5294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1465, 4037, 0, 61, 0, 609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 815, 2, 0, 0, 152, 0, 1, 2473, 1563, 1030, 0, 0, 0, 0, 0, 1959, 0, 0, 75, 2, 1266, 3024, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1029, 5053, 0, 0, 0, 0, 0, 0, 0, 1269, 0, 0, 0, 0, 2071, 62, 1, 1502, 1165, 334, 0, 0, 0, 0, 0, 2362, 2, 0, 224, 456, 2009, 89, 819, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6056, 6111, 6172, 6037, 6327, 6084, 6404, 5961] avg=6144.0000 max=6404 max_rank=6 min=5961 min_rank=7 imbalance_ratio=1.042318 max_load_reduction=42748 max_load_reduction_pct=0.869710 moved_to_echo=24776 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[262, 76, 0, 235, 5087, 0, 396, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 891, 0, 1, 0, 5264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1822, 3626, 0, 0, 0, 581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 939, 0, 0, 0, 140, 0, 0, 2460, 1616, 1027, 0, 0, 0, 0, 0, 1615, 0, 0, 5, 1, 1384, 3016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 929, 5225, 0, 0, 0, 0, 0, 0, 0, 1225, 0, 0, 0, 0, 1710, 60, 0, 1623, 1161, 300, 0, 0, 0, 0, 0, 2200, 0, 0, 238, 439, 2169, 85, 1168, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6232, 6156, 6029, 6182, 6021, 6154, 6079, 6299] avg=6144.0000 max=6299 max_rank=7 min=6021 min_rank=4 imbalance_ratio=1.025228 max_load_reduction=42853 max_load_reduction_pct=0.871847 moved_to_echo=24553 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[249, 88, 0, 227, 4726, 3, 366, 216, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8, 11, 825, 16, 0, 2, 5267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1782, 3787, 0, 131, 0, 601, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 932, 0, 0, 0, 161, 9, 4, 2468, 1576, 1049, 0, 0, 0, 0, 0, 1795, 0, 2, 85, 0, 1300, 3056, 10, 0, 0, 0, 0, 0, 0, 0, 0, 8, 5, 1, 2, 0, 12, 15, 862, 5092, 0, 0, 0, 0, 0, 0, 0, 1279, 3, 0, 0, 1, 1834, 53, 11, 1607, 1184, 319, 0, 0, 0, 0, 0, 2157, 3, 0, 295, 493, 2139, 102, 916, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5875, 6132, 6305, 6199, 6248, 5997, 6291, 6105] avg=6144.0000 max=6305 max_rank=2 min=5875 min_rank=0 imbalance_ratio=1.026204 max_load_reduction=42847 max_load_reduction_pct=0.871724 moved_to_echo=24641 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[261, 94, 0, 185, 5181, 0, 350, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 944, 0, 0, 0, 5297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1594, 3919, 0, 27, 0, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 914, 0, 0, 0, 153, 0, 0, 2475, 1610, 1026, 0, 0, 0, 0, 0, 1793, 0, 0, 8, 0, 1302, 3013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 5206, 0, 0, 0, 0, 0, 0, 0, 1208, 0, 0, 0, 0, 1924, 70, 0, 1537, 1160, 324, 0, 0, 0, 0, 0, 2078, 0, 0, 263, 408, 2054, 69, 953, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6266, 6241, 6059, 6178, 6116, 6244, 6223, 5825] avg=6144.0000 max=6266 max_rank=0 min=5825 min_rank=7 imbalance_ratio=1.019857 max_load_reduction=42886 max_load_reduction_pct=0.872518 moved_to_echo=24408 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 7424, 128, 0, 0, 1280, 128, 128, 19840, 12800, 8192, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[10112, 128, 0, 0, 128, 14592, 512, 128, 12416, 9216, 2688, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[18176, 128, 0, 2048, 3840, 16512, 768, 8192, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[14720, 0, 128, 384, 128, 10496, 23808, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 13440, 31488, 0, 384, 0, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[2176, 640, 128, 1920, 40320, 128, 3072, 1536, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 0, 128, 128, 7936, 41344, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 128, 6784, 128, 128, 128, 42368, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 8128, 159, 0, 0, 35, 0, 10, 0, 0, 0, 0, 0, 1048, 0, 98, 0, 0, 0, 8124, 0, 0, 0, 7737, 0, 264, 0, 64, 0, 0, 0, 0, 0, 280, 63, 0, 0, 1, 7519, 0, 2646, 0, 0, 0, 71, 0, 0, 0, 7556, 0, 63, 2, 0, 0, 0, 0, 0, 0, 1358, 3861, 0, 0, 0, 65] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 8121, 166, 0, 0, 70, 0, 41, 0, 0, 0, 0, 0, 1051, 0, 140, 0, 2, 0, 8122, 0, 0, 0, 7677, 0, 274, 0, 61, 0, 0, 0, 0, 0, 289, 70, 0, 0, 0, 7518, 0, 2712, 0, 0, 0, 96, 0, 0, 0, 7468, 0, 69, 0, 0, 0, 0, 0, 0, 0, 1423, 3712, 0, 0, 0, 70] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 8107, 80, 0, 0, 89, 0, 13, 0, 0, 0, 0, 0, 982, 0, 175, 0, 2, 0, 8107, 0, 0, 0, 7681, 0, 260, 0, 40, 0, 0, 0, 0, 0, 266, 85, 0, 0, 0, 7596, 0, 2751, 0, 0, 0, 105, 0, 0, 0, 7420, 0, 85, 0, 0, 0, 0, 0, 0, 0, 1269, 3954, 0, 0, 0, 85] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 8107, 187, 0, 0, 69, 0, 28, 0, 0, 1, 0, 0, 988, 0, 152, 0, 2, 0, 8099, 0, 2, 0, 7699, 0, 251, 1, 68, 0, 0, 0, 0, 0, 277, 83, 1, 1, 2, 7600, 0, 2697, 0, 2, 0, 107, 0, 0, 1, 7506, 1, 83, 0, 0, 0, 0, 1, 0, 0, 1411, 3639, 3, 0, 0, 83] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 8112, 88, 0, 0, 59, 0, 4, 0, 0, 0, 0, 0, 1035, 0, 140, 0, 0, 0, 8112, 0, 0, 0, 7723, 0, 244, 0, 69, 0, 0, 0, 0, 0, 252, 80, 0, 0, 0, 7569, 0, 2588, 0, 0, 0, 89, 0, 0, 0, 7544, 0, 80, 0, 0, 0, 0, 0, 0, 0, 1325, 3959, 0, 0, 0, 80] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 8098, 130, 0, 0, 59, 0, 11, 0, 0, 0, 0, 0, 989, 0, 153, 0, 0, 0, 8098, 0, 0, 0, 7696, 0, 251, 0, 81, 0, 0, 0, 0, 0, 261, 94, 0, 0, 0, 7607, 0, 2668, 0, 0, 0, 110, 0, 0, 0, 7517, 0, 94, 0, 0, 0, 0, 0, 0, 0, 1332, 3809, 0, 0, 0, 94] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 8116, 108, 0, 0, 64, 0, 10, 0, 0, 0, 0, 0, 1066, 0, 140, 0, 0, 0, 8116, 0, 0, 0, 7734, 0, 241, 0, 60, 0, 0, 0, 0, 0, 256, 76, 0, 0, 0, 7570, 0, 2698, 0, 0, 0, 106, 0, 0, 0, 7565, 0, 76, 0, 0, 0, 0, 0, 0, 0, 1392, 3683, 0, 0, 0, 75] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 8082, 223, 0, 0, 66, 1, 30, 0, 5, 11, 1, 0, 1011, 2, 154, 1, 7, 1, 8064, 4, 10, 0, 7657, 0, 248, 2, 53, 0, 0, 1, 0, 8, 270, 88, 9, 1, 1, 7550, 0, 2816, 2, 19, 0, 98, 7, 7, 21, 7472, 5, 88, 2, 0, 2, 0, 17, 0, 2, 1321, 3616, 2, 0, 2, 92] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 5928, 166, 0, 0, 70, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 0, 140, 2282, 2193, 505, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 6122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 274, 0, 61, 0, 0, 0, 0, 5840, 0, 0, 0, 0, 0, 0, 0, 0, 289, 70, 0, 0, 0, 5756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2712, 0, 0, 0, 96, 0, 0, 0, 1555, 1257, 513, 0, 0, 0, 0, 0, 6031, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1423, 3712, 0, 0, 0, 70, 924, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6205, 6171, 6124, 6175, 6115, 6133, 6100, 6129] avg=6144.0000 max=6205 max_rank=0 min=6100 min_rank=6 imbalance_ratio=1.009928 max_load_reduction=42947 max_load_reduction_pct=0.873759 moved_to_echo=24477 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 5933, 159, 0, 0, 35, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 0, 98, 2283, 2195, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 264, 0, 64, 0, 0, 0, 0, 5841, 0, 0, 0, 0, 0, 0, 0, 0, 280, 63, 0, 0, 1, 5757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2646, 0, 0, 0, 71, 0, 0, 0, 1567, 1257, 519, 0, 0, 0, 0, 0, 6102, 0, 63, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1358, 3861, 0, 0, 0, 65, 935, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6137, 6129, 6170, 6169, 6101, 6060, 6167, 6219] avg=6144.0000 max=6219 max_rank=7 min=6060 min_rank=5 imbalance_ratio=1.012207 max_load_reduction=42933 max_load_reduction_pct=0.873474 moved_to_echo=24547 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 5918, 80, 0, 0, 89, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 982, 0, 175, 2278, 2189, 510, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 6125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 260, 0, 40, 0, 0, 0, 0, 5829, 0, 0, 0, 0, 0, 0, 0, 0, 266, 85, 0, 0, 0, 5816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2751, 0, 0, 0, 105, 0, 0, 0, 1556, 1270, 510, 0, 0, 0, 0, 0, 5992, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1269, 3954, 0, 0, 0, 85, 918, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6100, 6134, 6127, 6129, 6167, 6192, 6077, 6226] avg=6144.0000 max=6226 max_rank=7 min=6077 min_rank=6 imbalance_ratio=1.013346 max_load_reduction=42926 max_load_reduction_pct=0.873332 moved_to_echo=24662 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 5922, 88, 0, 0, 59, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 0, 140, 2280, 2190, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 244, 0, 69, 0, 0, 0, 0, 5832, 0, 0, 0, 0, 0, 0, 0, 0, 252, 80, 0, 0, 0, 5796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2588, 0, 0, 0, 89, 0, 0, 0, 1564, 1265, 518, 0, 0, 0, 0, 0, 6092, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1325, 3959, 0, 0, 0, 80, 934, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6073, 6153, 6159, 6145, 6128, 6024, 6172, 6298] avg=6144.0000 max=6298 max_rank=7 min=6024 min_rank=5 imbalance_ratio=1.025065 max_load_reduction=42854 max_load_reduction_pct=0.871867 moved_to_echo=24622 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 5918, 187, 0, 0, 69, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 988, 0, 152, 2277, 2189, 510, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 6139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 1, 68, 0, 0, 0, 0, 5822, 0, 0, 0, 0, 0, 0, 0, 0, 277, 83, 1, 1, 2, 5819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2697, 0, 2, 0, 107, 0, 0, 1, 1560, 1271, 516, 0, 0, 0, 0, 0, 6061, 1, 83, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1411, 3639, 3, 0, 0, 83, 929, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6202, 6117, 6143, 6142, 6183, 6154, 6146, 6065] avg=6144.0000 max=6202 max_rank=0 min=6065 min_rank=7 imbalance_ratio=1.009440 max_load_reduction=42950 max_load_reduction_pct=0.873820 moved_to_echo=24548 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 5925, 108, 0, 0, 64, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1066, 0, 140, 2281, 2191, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 241, 0, 60, 0, 0, 0, 0, 5835, 0, 0, 0, 0, 0, 0, 0, 0, 256, 76, 0, 0, 0, 5796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2698, 0, 0, 0, 106, 0, 0, 0, 1567, 1266, 520, 0, 0, 0, 0, 0, 6109, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1392, 3683, 0, 0, 0, 75, 936, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6107, 6186, 6167, 6136, 6128, 6157, 6185, 6086] avg=6144.0000 max=6186 max_rank=1 min=6086 min_rank=7 imbalance_ratio=1.006836 max_load_reduction=42966 max_load_reduction_pct=0.874146 moved_to_echo=24556 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 5912, 130, 0, 0, 59, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 989, 0, 153, 2275, 2186, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 0, 81, 0, 0, 0, 0, 5823, 0, 0, 0, 0, 0, 0, 0, 0, 261, 94, 0, 0, 0, 5824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2668, 0, 0, 0, 110, 0, 0, 0, 1559, 1272, 516, 0, 0, 0, 0, 0, 6071, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1332, 3809, 0, 0, 0, 94, 930, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6112, 6114, 6137, 6155, 6179, 6125, 6165, 6165] avg=6144.0000 max=6179 max_rank=4 min=6112 min_rank=0 imbalance_ratio=1.005697 max_load_reduction=42973 max_load_reduction_pct=0.874288 moved_to_echo=24634 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 5896, 223, 0, 0, 66, 1, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 11, 1, 0, 1011, 2, 154, 2267, 2186, 511, 0, 0, 0, 0, 0, 1, 7, 1, 0, 4, 10, 0, 6102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 248, 2, 53, 0, 0, 1, 0, 5797, 0, 0, 0, 0, 0, 0, 0, 8, 270, 88, 9, 1, 1, 5774, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2816, 2, 19, 0, 98, 7, 7, 21, 1555, 1265, 517, 0, 0, 0, 0, 0, 6026, 5, 88, 2, 0, 2, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1321, 3616, 2, 0, 2, 92, 929, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6216, 6148, 6125, 6101, 6151, 6307, 6140, 5964] avg=6144.0000 max=6307 max_rank=5 min=5964 min_rank=7 imbalance_ratio=1.026530 max_load_reduction=42845 max_load_reduction_pct=0.871684 moved_to_echo=24562 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 0, 8192, 128, 1152, 18304, 17536, 4096, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=50176 observed_loss=-1024 tokens_per_expert=[21632, 128, 128, 0, 896, 128, 128, 128, 12544, 10240, 4224, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[0, 128, 10880, 30336, 128, 0, 128, 768, 7552, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 2048, 128, 512, 0, 0, 128, 0, 46720, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 47360, 1152, 0, 0, 512, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[48512, 128, 640, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 128, 128, 0, 49152, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 2176, 640, 128, 128, 128, 46464, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=50176 slack_slots=48128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 547, 0, 0, 0, 0, 100, 36, 0, 0, 1093, 1, 0, 0, 0, 0, 1738, 0, 288, 0, 0, 0, 2486, 8031, 0, 3282, 0, 7967, 0, 0, 177, 4309, 0, 7426, 0, 7926, 0, 0, 0, 0, 176, 0, 0, 459, 889, 1, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 2144, 33, 35, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 575, 0, 0, 0, 0, 216, 90, 0, 0, 1112, 2, 0, 0, 0, 0, 1652, 0, 349, 0, 0, 0, 2388, 7932, 0, 2908, 0, 7890, 0, 0, 246, 4376, 1, 7390, 0, 7932, 0, 0, 0, 0, 266, 0, 0, 447, 1045, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 2131, 48, 90, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 521, 0, 0, 0, 0, 183, 70, 0, 0, 1216, 1, 0, 0, 0, 0, 2238, 0, 198, 0, 0, 0, 2586, 7982, 0, 2152, 0, 7921, 0, 0, 225, 4199, 0, 7375, 0, 7918, 0, 0, 0, 0, 230, 0, 0, 439, 816, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 2736, 37, 70, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 515, 0, 0, 0, 0, 134, 60, 0, 0, 1196, 0, 0, 0, 0, 0, 1827, 0, 317, 0, 0, 0, 2375, 7972, 0, 2741, 0, 7903, 0, 0, 231, 4400, 0, 7391, 0, 7947, 0, 0, 0, 0, 193, 0, 0, 433, 1086, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 2251, 37, 60, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[5, 1, 0, 528, 3, 0, 2, 19, 164, 67, 0, 15, 1391, 15, 7, 0, 0, 1, 1694, 0, 356, 12, 0, 4, 2453, 7914, 0, 2393, 11, 7837, 5, 5, 241, 4185, 0, 7203, 1, 7898, 10, 4, 0, 0, 326, 6, 10, 523, 1096, 8, 0, 104, 19, 1, 2, 1, 0, 5, 2, 17, 2414, 106, 66, 0, 0, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 513, 0, 0, 0, 0, 150, 64, 0, 0, 1247, 0, 0, 0, 0, 0, 1814, 0, 364, 0, 0, 0, 2375, 7976, 0, 3215, 0, 7914, 0, 0, 216, 4068, 0, 7341, 0, 7948, 0, 0, 0, 0, 207, 0, 0, 414, 1040, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 2212, 6, 64, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 478, 0, 0, 1, 2, 159, 69, 0, 0, 1214, 9, 1, 1, 0, 0, 1798, 0, 318, 0, 0, 1, 2261, 7955, 1, 2854, 0, 7884, 1, 0, 239, 4469, 0, 7341, 0, 7925, 0, 0, 0, 0, 297, 0, 0, 466, 1096, 5, 0, 89, 1, 2, 0, 0, 0, 0, 0, 2, 2034, 108, 70, 0, 0, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 455, 0, 0, 0, 0, 128, 59, 0, 0, 1128, 0, 0, 0, 0, 0, 1664, 0, 324, 0, 0, 0, 2506, 7945, 0, 2894, 0, 7863, 0, 0, 240, 4310, 0, 7363, 0, 7941, 0, 0, 0, 0, 277, 0, 0, 470, 1041, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 2351, 91, 59, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 547, 0, 0, 0, 0, 3744, 1893, 0, 0, 0, 0, 0, 0, 100, 36, 0, 0, 1093, 1, 0, 0, 4182, 540, 0, 0, 0, 0, 0, 0, 0, 0, 1738, 0, 288, 0, 0, 0, 2048, 2013, 0, 0, 0, 0, 0, 0, 2486, 0, 0, 3282, 0, 917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 177, 4309, 0, 1642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 0, 0, 459, 889, 1, 4462, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 6138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2144, 33, 35, 0, 0, 0, 3771, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6184, 5952, 6087, 6685, 6128, 5987, 6146, 5983] avg=6144.0000 max=6685 max_rank=3 min=5952 min_rank=1 imbalance_ratio=1.088053 max_load_reduction=42467 max_load_reduction_pct=0.863993 moved_to_echo=24244 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 521, 0, 0, 0, 0, 3740, 1882, 0, 0, 0, 0, 0, 0, 183, 70, 0, 0, 1216, 1, 0, 0, 4178, 537, 0, 0, 0, 0, 0, 0, 0, 0, 2238, 0, 198, 0, 0, 0, 2036, 1999, 0, 0, 0, 0, 0, 0, 2586, 0, 0, 2152, 0, 912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 225, 4199, 0, 1631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 230, 0, 0, 439, 816, 0, 4436, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 6100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2736, 37, 70, 0, 0, 0, 3745, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6185, 6471, 5650, 6055, 5921, 6139, 6588] avg=6144.0000 max=6588 max_rank=7 min=5650 min_rank=3 imbalance_ratio=1.072266 max_load_reduction=42564 max_load_reduction_pct=0.865967 moved_to_echo=24703 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 575, 0, 0, 0, 0, 3746, 1870, 0, 0, 0, 0, 0, 0, 216, 90, 0, 0, 1112, 2, 0, 0, 4186, 534, 0, 0, 0, 0, 0, 0, 0, 0, 1652, 0, 349, 0, 0, 0, 2028, 2003, 0, 0, 0, 0, 0, 0, 2388, 0, 0, 2908, 0, 909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 4376, 1, 1635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 266, 0, 0, 447, 1045, 0, 4419, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 6062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2131, 48, 90, 0, 0, 0, 3752, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6191, 6140, 6032, 6205, 6258, 6177, 6128, 6021] avg=6144.0000 max=6258 max_rank=4 min=6021 min_rank=7 imbalance_ratio=1.018555 max_load_reduction=42894 max_load_reduction_pct=0.872681 moved_to_echo=24584 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 515, 0, 0, 0, 0, 3754, 1880, 0, 0, 0, 0, 0, 0, 134, 60, 0, 0, 1196, 0, 0, 0, 4193, 535, 0, 0, 0, 0, 0, 0, 0, 0, 1827, 0, 317, 0, 0, 0, 2032, 2003, 0, 0, 0, 0, 0, 0, 2375, 0, 0, 2741, 0, 910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 231, 4400, 0, 1635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 193, 0, 0, 433, 1086, 0, 4426, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 6092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2251, 37, 60, 0, 0, 0, 3753, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6149, 6118, 6179, 6026, 6266, 6138, 6175, 6101] avg=6144.0000 max=6266 max_rank=4 min=6026 min_rank=3 imbalance_ratio=1.019857 max_load_reduction=42886 max_load_reduction_pct=0.872518 moved_to_echo=24680 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 478, 0, 0, 1, 2, 3744, 1876, 0, 0, 0, 0, 0, 0, 159, 69, 0, 0, 1214, 9, 1, 1, 4181, 534, 0, 0, 0, 0, 0, 0, 0, 0, 1798, 0, 318, 0, 0, 1, 2027, 1990, 0, 0, 0, 0, 0, 0, 2261, 0, 1, 2854, 0, 907, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 239, 4469, 0, 1624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 297, 0, 0, 466, 1096, 5, 4416, 0, 0, 0, 0, 0, 0, 0, 0, 89, 1, 2, 0, 0, 0, 0, 6079, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2034, 108, 70, 0, 0, 1, 3727, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6101, 6168, 6134, 6024, 6332, 6280, 6171, 5942] avg=6144.0000 max=6332 max_rank=4 min=5942 min_rank=7 imbalance_ratio=1.030599 max_load_reduction=42820 max_load_reduction_pct=0.871175 moved_to_echo=24725 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 513, 0, 0, 0, 0, 3755, 1881, 0, 0, 0, 0, 0, 0, 150, 64, 0, 0, 1247, 0, 0, 0, 4193, 536, 0, 0, 0, 0, 0, 0, 0, 0, 1814, 0, 364, 0, 0, 0, 2034, 1990, 0, 0, 0, 0, 0, 0, 2375, 0, 0, 3215, 0, 911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 4068, 0, 1624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 207, 0, 0, 414, 1040, 0, 4433, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 6095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2212, 6, 64, 0, 0, 0, 3727, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6149, 6190, 6202, 6501, 5908, 6094, 6099, 6009] avg=6144.0000 max=6501 max_rank=3 min=5908 min_rank=4 imbalance_ratio=1.058105 max_load_reduction=42651 max_load_reduction_pct=0.867737 moved_to_echo=24110 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[5, 1, 0, 528, 3, 0, 2, 19, 3731, 1867, 0, 0, 0, 0, 0, 0, 164, 67, 0, 15, 1391, 15, 7, 0, 4167, 535, 0, 0, 0, 0, 0, 0, 0, 1, 1694, 0, 356, 12, 0, 4, 2019, 1956, 0, 0, 0, 0, 0, 0, 2453, 0, 0, 2393, 11, 889, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 241, 4185, 0, 1586, 1, 0, 10, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 326, 6, 10, 523, 1096, 8, 4394, 0, 0, 0, 0, 0, 0, 0, 0, 104, 19, 1, 2, 1, 0, 5, 6047, 0, 0, 0, 0, 0, 0, 0, 2, 17, 2414, 106, 66, 0, 0, 2, 3661, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6156, 6361, 6042, 5756, 6027, 6363, 6179, 6268] avg=6144.0000 max=6363 max_rank=5 min=5756 min_rank=3 imbalance_ratio=1.035645 max_load_reduction=42789 max_load_reduction_pct=0.870544 moved_to_echo=24837 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 455, 0, 0, 0, 0, 3751, 1873, 0, 0, 0, 0, 0, 0, 128, 59, 0, 0, 1128, 0, 0, 0, 4190, 533, 0, 0, 0, 0, 0, 0, 0, 0, 1664, 0, 324, 0, 0, 0, 2021, 1996, 0, 0, 0, 0, 0, 0, 2506, 0, 0, 2894, 0, 905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 4310, 0, 1628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 277, 0, 0, 470, 1041, 0, 4404, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 6072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2351, 91, 59, 0, 0, 0, 3739, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6079, 6038, 6005, 6305, 6178, 6192, 6115, 6240] avg=6144.0000 max=6305 max_rank=3 min=6005 min_rank=2 imbalance_ratio=1.026204 max_load_reduction=42847 max_load_reduction_pct=0.871724 moved_to_echo=24725 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 14464, 0, 2560, 128, 0, 128, 16256, 16000, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 128, 0, 4224, 128, 0, 128, 128, 30080, 15104, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[19456, 0, 128, 22528, 128, 7296, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 18304, 512, 640, 0, 0, 128, 29952, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1920, 34432, 128, 13056, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 0, 2048, 128, 128, 3712, 8192, 128, 35456, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1280, 640, 0, 128, 9600, 128, 128, 128, 33536, 4352, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[0, 512, 128, 128, 128, 128, 0, 128, 48768, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 3775, 65, 2737, 2526, 0, 258, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8192, 0, 697, 0, 0, 30, 8192, 0, 0, 0, 1, 342, 0, 0, 564, 0, 0, 118, 476, 230, 0, 0, 5865, 0, 0, 0, 0, 7701, 0, 0, 0, 5998, 0, 4, 0, 2, 0, 237, 176, 857, 43, 0, 63] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4318, 110, 2150, 2454, 0, 322, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 8192, 0, 688, 0, 0, 14, 8192, 0, 0, 0, 0, 277, 0, 0, 591, 0, 0, 174, 460, 336, 0, 0, 5827, 0, 0, 0, 0, 7614, 0, 0, 0, 6098, 0, 3, 0, 24, 0, 153, 160, 856, 57, 0, 70] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 3914, 126, 2486, 2258, 0, 184, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 8192, 0, 1007, 0, 0, 39, 8192, 0, 0, 0, 0, 363, 0, 0, 676, 0, 0, 147, 375, 398, 0, 0, 5730, 0, 0, 0, 0, 7614, 0, 0, 0, 5780, 0, 3, 0, 14, 0, 225, 304, 951, 83, 0, 85] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 3972, 74, 2391, 2323, 0, 160, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 8192, 0, 949, 0, 0, 44, 8192, 0, 0, 0, 0, 382, 0, 0, 636, 0, 0, 135, 447, 297, 0, 0, 5878, 0, 0, 0, 0, 7641, 0, 0, 0, 5901, 0, 1, 0, 6, 0, 195, 299, 863, 91, 0, 79] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4097, 69, 2579, 2439, 0, 292, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 633, 0, 0, 46, 8192, 0, 0, 0, 1, 292, 0, 0, 640, 0, 0, 140, 413, 323, 0, 0, 5861, 0, 0, 0, 0, 7615, 0, 0, 0, 5665, 0, 0, 0, 11, 0, 306, 298, 861, 92, 0, 94] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[1, 3, 4408, 99, 1776, 2462, 6, 340, 0, 0, 0, 4, 6, 10, 0, 5, 7, 0, 0, 0, 20, 8153, 10, 848, 0, 0, 81, 8162, 0, 2, 0, 16, 378, 3, 3, 646, 0, 12, 177, 410, 348, 16, 27, 5617, 0, 0, 20, 1, 7546, 0, 0, 1, 5891, 0, 1, 4, 34, 0, 312, 241, 845, 104, 8, 88] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4329, 88, 2249, 2295, 0, 284, 0, 0, 0, 0, 2, 1, 0, 0, 2, 0, 0, 0, 1, 8185, 0, 808, 0, 0, 49, 8188, 0, 0, 0, 1, 309, 0, 0, 636, 0, 1, 164, 406, 331, 1, 2, 5652, 0, 0, 2, 0, 7598, 0, 5, 1, 5973, 0, 8, 1, 19, 0, 241, 281, 872, 83, 1, 83] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 3790, 86, 2717, 2604, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8192, 0, 1027, 0, 0, 59, 8192, 0, 0, 0, 1, 245, 0, 0, 590, 0, 0, 94, 439, 306, 0, 0, 5718, 0, 0, 0, 0, 7689, 0, 0, 0, 5764, 0, 7, 0, 8, 0, 247, 226, 943, 82, 0, 75] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1072, 110, 2150, 2454, 0, 322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6129, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 5308, 0, 688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 6100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 277, 0, 0, 591, 0, 0, 174, 460, 3246, 1229, 328, 0, 0, 0, 0, 0, 336, 0, 0, 5827, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 6098, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 153, 160, 856, 57, 0, 70, 2556, 2047, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6108, 6129, 6008, 6114, 6305, 6208, 6357, 5923] avg=6144.0000 max=6357 max_rank=6 min=5923 min_rank=7 imbalance_ratio=1.034668 max_load_reduction=42795 max_load_reduction_pct=0.870667 moved_to_echo=24793 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 937, 65, 2737, 2526, 0, 258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6199, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 5308, 0, 697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 6100, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 342, 0, 0, 564, 0, 0, 118, 476, 2838, 1243, 328, 0, 0, 0, 0, 0, 230, 0, 0, 5865, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 259, 0, 0, 0, 5998, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 237, 176, 857, 43, 0, 63, 2556, 2047, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6523, 6199, 6008, 6131, 5909, 6140, 6261, 5981] avg=6144.0000 max=6523 max_rank=0 min=5909 min_rank=4 imbalance_ratio=1.061686 max_load_reduction=42629 max_load_reduction_pct=0.867289 moved_to_echo=24291 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 972, 126, 2486, 2258, 0, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6129, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 5308, 0, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 6100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 363, 0, 0, 676, 0, 0, 147, 375, 2942, 1229, 328, 0, 0, 0, 0, 0, 398, 0, 0, 5730, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 5780, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 225, 304, 951, 83, 0, 85, 2556, 2047, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6026, 6129, 6321, 6139, 6060, 6173, 6039, 6265] avg=6144.0000 max=6321 max_rank=2 min=6026 min_rank=0 imbalance_ratio=1.028809 max_load_reduction=42831 max_load_reduction_pct=0.871399 moved_to_echo=24537 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 986, 74, 2391, 2323, 0, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 6151, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 5308, 0, 949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 6100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 382, 0, 0, 636, 0, 0, 135, 447, 2986, 1233, 328, 0, 0, 0, 0, 0, 297, 0, 0, 5878, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 257, 0, 0, 0, 5901, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 195, 299, 863, 91, 0, 79, 2556, 2047, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5934, 6153, 6259, 6144, 6147, 6220, 6159, 6136] avg=6144.0000 max=6259 max_rank=2 min=5934 min_rank=0 imbalance_ratio=1.018717 max_load_reduction=42893 max_load_reduction_pct=0.872660 moved_to_echo=24662 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1075, 88, 2249, 2295, 0, 284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 6116, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 5303, 0, 808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 6097, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 309, 0, 0, 636, 0, 1, 164, 406, 3254, 1226, 328, 0, 0, 0, 0, 0, 331, 1, 2, 5652, 0, 0, 2, 0, 45, 0, 0, 0, 0, 0, 0, 0, 256, 0, 5, 1, 5973, 0, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 241, 281, 872, 83, 1, 83, 2554, 2046, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5991, 6119, 6114, 6147, 6324, 6033, 6244, 6180] avg=6144.0000 max=6324 max_rank=4 min=5991 min_rank=0 imbalance_ratio=1.029297 max_load_reduction=42828 max_load_reduction_pct=0.871338 moved_to_echo=24781 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 941, 86, 2717, 2604, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6189, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 5308, 0, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 6100, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 245, 0, 0, 590, 0, 0, 94, 439, 2849, 1241, 328, 0, 0, 0, 0, 0, 306, 0, 0, 5718, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 259, 0, 0, 0, 5764, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 247, 226, 943, 82, 0, 75, 2556, 2047, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6397, 6189, 6337, 6160, 5786, 6069, 6030, 6184] avg=6144.0000 max=6397 max_rank=0 min=5786 min_rank=4 imbalance_ratio=1.041178 max_load_reduction=42755 max_load_reduction_pct=0.869853 moved_to_echo=24069 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 3, 1090, 99, 1776, 2462, 6, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 6, 10, 0, 5, 6079, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 20, 5278, 10, 848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 6071, 0, 2, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 378, 3, 3, 646, 0, 12, 177, 410, 3318, 1223, 329, 0, 0, 0, 0, 0, 348, 16, 27, 5617, 0, 0, 20, 1, 51, 0, 0, 0, 0, 0, 0, 0, 244, 0, 0, 1, 5891, 0, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 312, 241, 845, 104, 8, 88, 2546, 2040, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5777, 6104, 6163, 6170, 6499, 6080, 6141, 6218] avg=6144.0000 max=6499 max_rank=4 min=5777 min_rank=0 imbalance_ratio=1.057780 max_load_reduction=42653 max_load_reduction_pct=0.867778 moved_to_echo=24938 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1017, 69, 2579, 2439, 0, 292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6130, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 5308, 0, 633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 6100, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 292, 0, 0, 640, 0, 0, 140, 413, 3080, 1229, 328, 0, 0, 0, 0, 0, 323, 0, 0, 5861, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 5665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 306, 298, 861, 92, 0, 94, 2556, 2047, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6396, 6130, 5942, 6147, 6122, 6229, 5921, 6265] avg=6144.0000 max=6396 max_rank=0 min=5921 min_rank=6 imbalance_ratio=1.041016 max_load_reduction=42756 max_load_reduction_pct=0.869873 moved_to_echo=24537 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 0, 1920, 2048, 7168, 640, 128, 640, 20480, 16384, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 8192, 768, 19200, 19456, 128, 1920, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2688, 128, 128, 4992, 0, 128, 1152, 3456, 24576, 9856, 2688, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2688, 128, 128, 46208, 0, 0, 128, 128, 384, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2048, 0, 128, 128, 47104, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 128, 128, 0, 128, 49152, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 384, 48768, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 128, 42496, 128, 6784, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[833, 0, 0, 0, 1532, 126, 0, 0, 0, 584, 0, 0, 279, 0, 7386, 8107, 6419, 5924, 0, 85, 0, 0, 0, 0, 162, 0, 0, 260, 0, 88, 7370, 1192, 260, 85, 0, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 445, 260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7565, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[718, 0, 0, 0, 1835, 110, 0, 0, 0, 581, 0, 0, 277, 0, 7462, 8122, 6375, 5973, 0, 70, 0, 0, 0, 0, 200, 0, 0, 274, 0, 70, 7474, 1100, 275, 70, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 400, 274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7432, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[804, 0, 0, 0, 1811, 65, 0, 0, 0, 474, 0, 0, 226, 0, 7507, 8129, 6250, 5925, 0, 63, 0, 0, 0, 0, 195, 0, 0, 255, 0, 63, 7606, 1196, 264, 63, 0, 78, 0, 0, 0, 0, 1, 0, 0, 0, 0, 348, 264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7565, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[745, 0, 0, 0, 1881, 87, 0, 0, 0, 570, 0, 0, 284, 0, 7410, 8106, 6251, 5994, 1, 83, 0, 0, 1, 1, 171, 1, 0, 251, 1, 85, 7450, 1184, 253, 83, 1, 105, 2, 0, 0, 0, 2, 0, 0, 1, 0, 409, 251, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 0, 7483, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[967, 0, 0, 0, 1533, 85, 0, 0, 0, 535, 0, 0, 268, 0, 7388, 8116, 6511, 5914, 0, 76, 0, 0, 0, 0, 193, 0, 0, 179, 0, 76, 7469, 1190, 245, 76, 0, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 456, 244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7510, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[833, 0, 0, 0, 1861, 74, 0, 0, 0, 554, 0, 0, 277, 0, 7421, 8112, 6489, 5737, 0, 80, 0, 0, 0, 0, 183, 0, 0, 233, 0, 85, 7497, 1095, 244, 80, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 406, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7541, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[839, 0, 0, 0, 1812, 69, 0, 0, 0, 548, 0, 0, 293, 0, 7426, 8098, 6387, 5872, 0, 94, 0, 0, 0, 0, 170, 0, 0, 251, 0, 94, 7488, 1151, 250, 94, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 385, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7479, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[989, 0, 3, 1, 1751, 91, 1, 0, 8, 635, 0, 0, 306, 0, 7333, 8063, 6275, 5687, 10, 88, 0, 0, 8, 7, 167, 19, 0, 248, 18, 108, 7342, 1280, 246, 89, 4, 124, 12, 2, 0, 0, 3, 0, 4, 10, 4, 420, 255, 24, 0, 0, 0, 3, 2, 0, 9, 0, 12, 0, 7470, 0, 9, 8, 0, 4] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[833, 0, 0, 0, 1532, 126, 0, 0, 2087, 1376, 0, 0, 0, 0, 0, 0, 0, 584, 0, 0, 279, 0, 5286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, 5924, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 162, 0, 0, 260, 0, 88, 4404, 1192, 0, 0, 0, 0, 0, 0, 0, 0, 260, 85, 0, 190, 0, 0, 0, 0, 3760, 1964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 445, 260, 0, 2966, 2474, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5954, 6149, 6194, 6106, 6259, 6158, 6143, 6189] avg=6144.0000 max=6259 max_rank=4 min=5954 min_rank=0 imbalance_ratio=1.018717 max_load_reduction=42893 max_load_reduction_pct=0.872660 moved_to_echo=24749 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[718, 0, 0, 0, 1835, 110, 0, 0, 2108, 1352, 0, 0, 0, 0, 0, 0, 0, 581, 0, 0, 277, 0, 5341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 5973, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 0, 0, 274, 0, 70, 4466, 1100, 0, 0, 0, 0, 0, 0, 0, 0, 275, 70, 0, 60, 0, 0, 0, 0, 3734, 1968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 400, 274, 0, 3008, 2457, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6123, 6199, 6227, 6110, 6107, 6152, 6154, 6080] avg=6144.0000 max=6227 max_rank=2 min=6080 min_rank=7 imbalance_ratio=1.013509 max_load_reduction=42925 max_load_reduction_pct=0.873311 moved_to_echo=24493 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[804, 0, 0, 0, 1811, 65, 0, 0, 2121, 1376, 0, 0, 0, 0, 0, 0, 0, 474, 0, 0, 226, 0, 5372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, 5925, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 195, 0, 0, 255, 0, 63, 4545, 1196, 0, 0, 0, 0, 0, 0, 0, 0, 264, 63, 0, 78, 0, 0, 0, 0, 3661, 1970, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 348, 264, 0, 3061, 2409, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6177, 6072, 6168, 6254, 6036, 6097, 6159, 6189] avg=6144.0000 max=6254 max_rank=3 min=6036 min_rank=4 imbalance_ratio=1.017904 max_load_reduction=42898 max_load_reduction_pct=0.872762 moved_to_echo=24481 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[833, 0, 0, 0, 1861, 74, 0, 0, 2097, 1372, 0, 0, 0, 0, 0, 0, 0, 554, 0, 0, 277, 0, 5311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 187, 5737, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 0, 0, 233, 0, 85, 4479, 1095, 0, 0, 0, 0, 0, 0, 0, 0, 244, 80, 0, 105, 0, 0, 0, 0, 3801, 1966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 406, 245, 0, 3018, 2501, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6237, 6142, 6004, 6075, 6196, 6183, 6146, 6169] avg=6144.0000 max=6237 max_rank=0 min=6004 min_rank=2 imbalance_ratio=1.015137 max_load_reduction=42915 max_load_reduction_pct=0.873108 moved_to_echo=24694 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[745, 0, 0, 0, 1881, 87, 0, 0, 2094, 1361, 0, 0, 0, 0, 0, 0, 0, 570, 0, 0, 284, 0, 5303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 5994, 1, 83, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 171, 1, 0, 251, 1, 85, 4451, 1184, 0, 0, 0, 0, 0, 0, 0, 0, 253, 83, 1, 105, 2, 0, 0, 0, 3662, 1965, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 409, 251, 1, 2999, 2410, 13, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 6141, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6168, 6157, 6259, 6144, 6071, 6086, 6144, 6123] avg=6144.0000 max=6259 max_rank=2 min=6071 min_rank=4 imbalance_ratio=1.018717 max_load_reduction=42893 max_load_reduction_pct=0.872660 moved_to_echo=24424 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[967, 0, 0, 0, 1533, 85, 0, 0, 2088, 1366, 0, 0, 0, 0, 0, 0, 0, 535, 0, 0, 268, 0, 5287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 187, 5914, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 193, 0, 0, 179, 0, 76, 4463, 1190, 0, 0, 0, 0, 0, 0, 0, 0, 245, 76, 0, 121, 0, 0, 0, 0, 3814, 1968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 456, 244, 0, 3006, 2510, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6039, 6090, 6177, 6101, 6224, 6229, 6148, 6144] avg=6144.0000 max=6229 max_rank=5 min=6039 min_rank=0 imbalance_ratio=1.013835 max_load_reduction=42923 max_load_reduction_pct=0.873271 moved_to_echo=24745 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[989, 0, 3, 1, 1751, 91, 1, 0, 2076, 1363, 0, 0, 0, 0, 0, 0, 8, 635, 0, 0, 306, 0, 5238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 173, 5687, 10, 88, 0, 0, 8, 7, 0, 0, 0, 0, 0, 0, 0, 0, 167, 19, 0, 248, 18, 108, 4382, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 246, 89, 4, 124, 12, 2, 0, 0, 3679, 1955, 0, 0, 0, 0, 0, 0, 3, 0, 4, 10, 4, 420, 255, 24, 2960, 2423, 19, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 9, 0, 6108, 0, 0, 0, 0, 0, 0, 0, 12, 0, 6107, 0, 9, 8, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6275, 6187, 5973, 6222, 6111, 6122, 6122, 6140] avg=6144.0000 max=6275 max_rank=0 min=5973 min_rank=2 imbalance_ratio=1.021322 max_load_reduction=42877 max_load_reduction_pct=0.872335 moved_to_echo=24495 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[839, 0, 0, 0, 1812, 69, 0, 0, 2098, 1361, 0, 0, 0, 0, 0, 0, 0, 548, 0, 0, 293, 0, 5315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 5872, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 170, 0, 0, 251, 0, 94, 4474, 1151, 0, 0, 0, 0, 0, 0, 0, 0, 250, 94, 0, 101, 0, 0, 0, 0, 3741, 1962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 385, 251, 0, 3014, 2462, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6179, 6156, 6150, 6140, 6148, 6125, 6136, 6118] avg=6144.0000 max=6179 max_rank=0 min=6118 min_rank=7 imbalance_ratio=1.005697 max_load_reduction=42973 max_load_reduction_pct=0.874288 moved_to_echo=24527 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[6784, 0, 128, 128, 14080, 768, 128, 0, 16896, 11008, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2048, 640, 128, 896, 128, 128, 0, 0, 29952, 15744, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1536, 128, 0, 2048, 128, 768, 35712, 9472, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 4608, 0, 0, 2304, 0, 42496, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1536, 47104, 128, 640, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 0, 128, 128, 128, 3328, 2048, 128, 24064, 19712, 128, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 0, 49152, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 128, 0, 128, 128, 49152, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 840, 0, 0, 0, 72, 0, 0, 868, 0, 0, 0, 811, 0, 6361, 738, 0, 0, 7506, 0, 0, 0, 7074, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 7808, 0, 0, 0, 7849, 0, 132, 144, 0, 0, 0, 0, 0, 0, 0, 0, 7922, 0, 0, 0, 175, 0, 0, 0, 0, 761, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 768, 0, 0, 0, 20, 0, 0, 808, 0, 0, 0, 800, 0, 6354, 759, 0, 0, 7623, 0, 0, 0, 7173, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 7804, 0, 0, 0, 7881, 0, 110, 128, 0, 0, 0, 0, 0, 0, 0, 0, 8033, 0, 0, 0, 98, 0, 0, 0, 0, 722, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 895, 0, 0, 0, 48, 0, 0, 774, 0, 0, 0, 829, 0, 6270, 764, 0, 0, 7546, 0, 0, 0, 7178, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 7794, 0, 0, 0, 7784, 0, 130, 186, 0, 0, 0, 0, 0, 0, 0, 0, 7941, 0, 0, 0, 140, 0, 0, 0, 0, 767, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[6, 3, 0, 842, 0, 0, 0, 87, 0, 0, 777, 11, 7, 26, 808, 6, 6218, 716, 18, 0, 7488, 0, 3, 1, 7065, 0, 0, 178, 2, 2, 0, 4, 0, 0, 5, 0, 7765, 1, 0, 2, 7758, 20, 163, 308, 1, 0, 0, 24, 1, 6, 0, 2, 7884, 16, 8, 0, 153, 0, 4, 0, 0, 762, 1, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 863, 0, 0, 0, 37, 0, 0, 765, 0, 0, 0, 781, 0, 6352, 708, 0, 0, 7550, 0, 0, 0, 7166, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 7801, 0, 0, 0, 7789, 0, 161, 185, 0, 0, 0, 0, 0, 0, 0, 0, 7992, 0, 0, 0, 140, 0, 0, 0, 0, 778, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 788, 0, 0, 0, 63, 0, 0, 789, 0, 0, 0, 759, 0, 6465, 693, 0, 0, 7506, 0, 0, 0, 7156, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 7767, 0, 0, 0, 7882, 0, 178, 168, 0, 0, 0, 0, 0, 0, 0, 0, 7943, 0, 0, 0, 153, 0, 0, 0, 0, 737, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 702, 0, 0, 0, 52, 0, 0, 848, 0, 0, 0, 786, 0, 6299, 727, 0, 0, 7561, 0, 0, 0, 7101, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 7813, 0, 0, 0, 7980, 0, 151, 112, 0, 0, 0, 0, 0, 0, 0, 0, 7977, 0, 0, 0, 139, 0, 0, 0, 0, 828, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 812, 0, 0, 0, 52, 0, 0, 832, 1, 0, 2, 759, 0, 6373, 706, 2, 0, 7527, 0, 0, 2, 7179, 0, 0, 110, 0, 0, 0, 1, 0, 0, 0, 0, 7787, 0, 0, 0, 7838, 2, 168, 133, 0, 0, 0, 1, 2, 1, 0, 1, 7944, 1, 0, 0, 152, 0, 0, 0, 3, 761, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 895, 0, 0, 0, 48, 5280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 774, 0, 0, 0, 829, 0, 1649, 1102, 912, 868, 0, 0, 0, 0, 5358, 764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6076, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5774, 0, 130, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 0, 0, 0, 767, 0, 0, 2266, 2010, 943, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6223, 6134, 6122, 6182, 6145, 6090, 6130, 6126] avg=6144.0000 max=6223 max_rank=0 min=6090 min_rank=5 imbalance_ratio=1.012858 max_load_reduction=42929 max_load_reduction_pct=0.873393 moved_to_echo=24491 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 768, 0, 0, 0, 20, 5334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 808, 0, 0, 0, 800, 0, 1651, 1101, 924, 878, 0, 0, 0, 0, 5430, 759, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6072, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5845, 0, 110, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 0, 0, 722, 0, 0, 2289, 2036, 954, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6122, 6162, 6189, 6143, 6153, 6083, 6201, 6099] avg=6144.0000 max=6201 max_rank=6 min=6083 min_rank=5 imbalance_ratio=1.009277 max_load_reduction=42951 max_load_reduction_pct=0.873840 moved_to_echo=24536 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 840, 0, 0, 0, 72, 5252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 868, 0, 0, 0, 811, 0, 1652, 1086, 925, 866, 0, 0, 0, 0, 5436, 738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5988, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5822, 0, 132, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 175, 0, 0, 0, 0, 761, 0, 0, 2254, 2027, 941, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6164, 6208, 6174, 6079, 6156, 6098, 6115, 6158] avg=6144.0000 max=6208 max_rank=1 min=6079 min_rank=3 imbalance_ratio=1.010417 max_load_reduction=42944 max_load_reduction_pct=0.873698 moved_to_echo=24527 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 863, 0, 0, 0, 37, 5284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 765, 0, 0, 0, 781, 0, 1651, 1100, 924, 874, 0, 0, 0, 0, 5428, 708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6066, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5777, 0, 161, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 0, 0, 0, 778, 0, 0, 2266, 2012, 949, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6184, 6095, 6136, 6150, 6150, 6123, 6169, 6145] avg=6144.0000 max=6184 max_rank=0 min=6095 min_rank=1 imbalance_ratio=1.006510 max_load_reduction=42968 max_load_reduction_pct=0.874186 moved_to_echo=24587 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 812, 0, 0, 0, 52, 5268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 832, 1, 0, 2, 759, 0, 1648, 1102, 927, 869, 0, 0, 0, 0, 5446, 706, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 6077, 0, 0, 110, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5814, 2, 168, 133, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 1, 6132, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 0, 0, 0, 3, 761, 0, 0, 2259, 2024, 943, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6132, 6140, 6156, 6188, 6139, 6118, 6137, 6142] avg=6144.0000 max=6188 max_rank=3 min=6118 min_rank=5 imbalance_ratio=1.007161 max_load_reduction=42964 max_load_reduction_pct=0.874105 moved_to_echo=24536 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 702, 0, 0, 0, 52, 5292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 848, 0, 0, 0, 786, 0, 1653, 1090, 916, 872, 0, 0, 0, 0, 5383, 727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6011, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5919, 0, 151, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 0, 0, 0, 0, 828, 0, 0, 2269, 2061, 947, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6046, 6165, 6110, 6087, 6160, 6182, 6158, 6244] avg=6144.0000 max=6244 max_rank=7 min=6046 min_rank=0 imbalance_ratio=1.016276 max_load_reduction=42908 max_load_reduction_pct=0.872965 moved_to_echo=24744 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[6, 3, 0, 842, 0, 0, 0, 87, 5240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 777, 11, 7, 26, 808, 6, 1647, 1090, 909, 867, 0, 0, 0, 0, 5309, 716, 18, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 5975, 0, 0, 178, 2, 2, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 6118, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 5750, 20, 163, 308, 1, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 2, 6078, 16, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 0, 4, 0, 0, 762, 1, 0, 2248, 2008, 939, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6178, 6148, 6047, 6161, 6126, 6266, 6111, 6115] avg=6144.0000 max=6266 max_rank=5 min=6047 min_rank=2 imbalance_ratio=1.019857 max_load_reduction=42886 max_load_reduction_pct=0.872518 moved_to_echo=24618 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 788, 0, 0, 0, 63, 5252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 789, 0, 0, 0, 759, 0, 1644, 1099, 940, 869, 0, 0, 0, 0, 5525, 693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6057, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5846, 0, 178, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 0, 0, 0, 0, 737, 0, 0, 2254, 2036, 943, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6103, 6100, 6218, 6162, 6123, 6192, 6131, 6123] avg=6144.0000 max=6218 max_rank=2 min=6100 min_rank=1 imbalance_ratio=1.012044 max_load_reduction=42934 max_load_reduction_pct=0.873494 moved_to_echo=24569 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[0, 0, 6528, 128, 128, 128, 6400, 128, 13312, 8832, 7424, 7040, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1152, 0, 128, 0, 128, 6144, 128, 0, 18176, 16256, 7680, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 128, 0, 6528, 0, 0, 0, 512, 42240, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[46592, 128, 1280, 1408, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[43392, 5888, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 49152, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[48384, 0, 0, 896, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 49152, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[59, 0, 0, 63, 0, 0, 0, 0, 7803, 0, 0, 0, 0, 0, 0, 0, 1855, 0, 8191, 0, 0, 0, 0, 0, 369, 0, 0, 0, 0, 63, 0, 0, 0, 0, 0, 1, 0, 0, 1571, 0, 1137, 249, 4037, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6910, 17, 0, 0, 0, 0, 571, 8064, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[85, 0, 0, 85, 0, 0, 0, 0, 7764, 0, 0, 0, 0, 0, 0, 0, 1811, 0, 8192, 0, 0, 0, 0, 0, 541, 0, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 1501, 0, 1261, 355, 3703, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6945, 11, 0, 0, 0, 0, 640, 7981, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[87, 0, 28, 87, 6, 18, 22, 3, 7724, 0, 0, 0, 3, 5, 0, 4, 1524, 1, 8164, 11, 2, 1, 12, 0, 518, 0, 0, 10, 2, 88, 0, 16, 17, 0, 0, 0, 0, 9, 1744, 0, 1120, 339, 4142, 8157, 0, 15, 0, 0, 6, 3, 0, 0, 0, 6, 6643, 11, 6, 22, 0, 10, 608, 7947, 1, 10] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[70, 0, 0, 69, 1, 0, 0, 0, 7778, 0, 0, 0, 0, 0, 0, 0, 1685, 0, 8191, 0, 0, 0, 0, 0, 432, 0, 0, 0, 0, 70, 0, 23, 0, 0, 0, 0, 0, 0, 1473, 0, 1245, 289, 4204, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6786, 8, 0, 0, 0, 0, 624, 8012, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[74, 0, 0, 75, 0, 0, 0, 0, 7808, 0, 0, 0, 0, 0, 0, 0, 1834, 0, 8192, 0, 0, 0, 0, 0, 468, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 1417, 0, 1242, 332, 3873, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6861, 9, 0, 0, 0, 0, 668, 8031, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[80, 0, 0, 80, 0, 0, 0, 0, 7788, 0, 0, 0, 0, 0, 0, 0, 1797, 0, 8192, 0, 0, 0, 0, 0, 487, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 1542, 0, 1092, 323, 4026, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6836, 6, 0, 0, 0, 0, 593, 8038, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[83, 0, 2, 83, 2, 0, 1, 0, 7771, 0, 1, 0, 0, 1, 0, 1, 1841, 0, 8189, 0, 1, 0, 1, 0, 447, 0, 0, 0, 0, 83, 0, 0, 2, 0, 0, 0, 0, 1, 1564, 0, 1294, 339, 3899, 8190, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6750, 7, 0, 2, 0, 0, 577, 8019, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[94, 0, 0, 94, 0, 0, 0, 0, 7760, 0, 0, 0, 0, 0, 0, 0, 1772, 0, 8192, 0, 0, 0, 0, 0, 554, 0, 0, 0, 0, 94, 0, 1, 0, 0, 0, 0, 0, 0, 1437, 0, 1157, 330, 4041, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6833, 10, 0, 0, 0, 0, 562, 8029, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[59, 0, 0, 63, 0, 0, 0, 0, 5978, 0, 0, 0, 0, 0, 0, 0, 6165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1855, 0, 4378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 369, 0, 0, 0, 0, 63, 0, 0, 3813, 1574, 195, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1571, 0, 2302, 1638, 697, 0, 0, 0, 0, 0, 1137, 249, 4037, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6213, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 571, 5567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6100, 6165, 6233, 6014, 6209, 6063, 6230, 6138] avg=6144.0000 max=6233 max_rank=2 min=6014 min_rank=3 imbalance_ratio=1.014486 max_load_reduction=42919 max_load_reduction_pct=0.873189 moved_to_echo=24640 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[70, 0, 0, 69, 1, 0, 0, 0, 5978, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1685, 0, 4378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 432, 0, 0, 0, 0, 70, 0, 23, 3813, 1574, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1473, 0, 2287, 1633, 684, 0, 0, 0, 0, 0, 1245, 289, 4204, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6102, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 624, 5531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6118, 6145, 6063, 6106, 6077, 6378, 6110, 6155] avg=6144.0000 max=6378 max_rank=5 min=6063 min_rank=2 imbalance_ratio=1.038086 max_load_reduction=42774 max_load_reduction_pct=0.870239 moved_to_echo=24720 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[85, 0, 0, 85, 0, 0, 0, 0, 5978, 0, 0, 0, 0, 0, 0, 0, 6134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1811, 0, 4378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 0, 0, 0, 0, 85, 0, 0, 3814, 1574, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1501, 0, 2278, 1630, 700, 0, 0, 0, 0, 0, 1261, 355, 3703, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6245, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 640, 5510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6148, 6134, 6189, 6207, 6109, 5959, 6256, 6150] avg=6144.0000 max=6256 max_rank=6 min=5959 min_rank=5 imbalance_ratio=1.018229 max_load_reduction=42896 max_load_reduction_pct=0.872721 moved_to_echo=24474 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[80, 0, 0, 80, 0, 0, 0, 0, 5978, 0, 0, 0, 0, 0, 0, 0, 6153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1797, 0, 4378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 0, 0, 0, 0, 80, 0, 0, 3814, 1574, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1542, 0, 2295, 1635, 689, 0, 0, 0, 0, 0, 1092, 323, 4026, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6147, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 593, 5549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6138, 6153, 6175, 6149, 6161, 6081, 6153, 6142] avg=6144.0000 max=6175 max_rank=2 min=6081 min_rank=5 imbalance_ratio=1.005046 max_load_reduction=42977 max_load_reduction_pct=0.874369 moved_to_echo=24537 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[74, 0, 0, 75, 0, 0, 0, 0, 5978, 0, 0, 0, 0, 0, 0, 0, 6169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1834, 0, 4378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 0, 0, 0, 0, 76, 0, 0, 3814, 1574, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1417, 0, 2293, 1639, 692, 0, 0, 0, 0, 0, 1242, 332, 3873, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6169, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 668, 5544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6127, 6169, 6212, 6126, 6041, 6087, 6178, 6212] avg=6144.0000 max=6212 max_rank=2 min=6041 min_rank=4 imbalance_ratio=1.011068 max_load_reduction=42940 max_load_reduction_pct=0.873617 moved_to_echo=24518 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[83, 0, 2, 83, 2, 0, 1, 0, 5977, 0, 0, 0, 0, 0, 0, 0, 6140, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1841, 0, 4377, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 447, 0, 0, 0, 0, 83, 0, 0, 3812, 1574, 194, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 1564, 0, 2289, 1631, 681, 0, 0, 0, 0, 0, 1294, 339, 3899, 639, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6069, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 577, 5536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6148, 6143, 6220, 6110, 6168, 6172, 6076, 6115] avg=6144.0000 max=6220 max_rank=2 min=6076 min_rank=6 imbalance_ratio=1.012370 max_load_reduction=42932 max_load_reduction_pct=0.873454 moved_to_echo=24531 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[87, 0, 28, 87, 6, 18, 22, 3, 5956, 0, 0, 0, 0, 0, 0, 0, 6100, 0, 0, 0, 3, 5, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1524, 1, 4359, 11, 2, 1, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 0, 0, 10, 2, 88, 0, 16, 3805, 1573, 197, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 9, 1744, 0, 2272, 1624, 674, 0, 0, 0, 0, 0, 1120, 339, 4142, 628, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 0, 0, 0, 6, 5969, 11, 0, 0, 0, 0, 0, 0, 0, 0, 6, 22, 0, 10, 608, 5478, 1, 10, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6207, 6112, 5910, 6209, 6340, 6244, 5995, 6135] avg=6144.0000 max=6340 max_rank=4 min=5910 min_rank=2 imbalance_ratio=1.031901 max_load_reduction=42812 max_load_reduction_pct=0.871012 moved_to_echo=24714 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[94, 0, 0, 94, 0, 0, 0, 0, 5978, 0, 0, 0, 0, 0, 0, 0, 6131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1772, 0, 4378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 0, 0, 0, 0, 94, 0, 1, 3814, 1574, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1437, 0, 2292, 1629, 689, 0, 0, 0, 0, 0, 1157, 330, 4041, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 562, 5543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6166, 6131, 6150, 6231, 6047, 6168, 6154, 6105] avg=6144.0000 max=6231 max_rank=3 min=6047 min_rank=4 imbalance_ratio=1.014160 max_load_reduction=42921 max_load_reduction_pct=0.873230 moved_to_echo=24474 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 0, 0, 128, 0, 128, 12288, 0, 18432, 13184, 5632, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[14208, 128, 35072, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[9600, 2560, 32000, 5120, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3840, 0, 0, 128, 128, 640, 0, 128, 30592, 12672, 1664, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 4864, 44288, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[640, 0, 128, 640, 128, 128, 128, 128, 47872, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[49152, 0, 128, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 0, 0, 0, 128, 49152, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=17 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 315, 0, 0, 266, 0, 0, 0, 0, 0, 0, 0, 7568, 0, 0, 0, 3832, 0, 0, 0, 0, 8040, 0, 0, 0, 64, 3844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6235, 8192, 2607, 4381, 0, 1, 0, 0, 637, 0, 3109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=17 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 301, 0, 0, 261, 0, 0, 0, 0, 0, 0, 0, 7486, 0, 0, 0, 3367, 0, 0, 0, 0, 8032, 0, 0, 0, 60, 4371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6086, 8192, 2568, 4697, 0, 0, 0, 0, 541, 0, 3121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=17 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 332, 0, 0, 268, 0, 0, 0, 0, 0, 0, 0, 7478, 0, 0, 0, 3098, 0, 0, 0, 0, 8052, 0, 0, 0, 70, 4790, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 6026, 8192, 2412, 4646, 0, 0, 0, 0, 595, 0, 3117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=17 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 343, 0, 0, 247, 0, 0, 0, 0, 0, 0, 0, 7529, 0, 0, 0, 3246, 0, 0, 0, 0, 8022, 0, 0, 0, 90, 4292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6189, 8192, 2854, 4484, 0, 0, 0, 0, 582, 0, 3042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=17 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 1, 313, 0, 0, 247, 0, 1, 0, 0, 0, 0, 0, 7469, 0, 0, 0, 3096, 0, 2, 1, 0, 8024, 0, 0, 0, 69, 4564, 2, 0, 0, 0, 2, 0, 0, 20, 0, 2, 0, 6133, 8190, 2489, 4583, 0, 0, 0, 0, 652, 0, 3219, 0, 0, 0, 0, 0, 2, 0, 0, 0, 70] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=17 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 302, 0, 0, 257, 0, 0, 0, 0, 0, 0, 0, 7521, 0, 0, 0, 3505, 0, 0, 0, 0, 8066, 0, 0, 0, 35, 4414, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 6095, 8192, 2493, 4669, 0, 0, 0, 0, 559, 0, 2979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=17 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 338, 0, 0, 262, 0, 0, 0, 0, 0, 0, 0, 7496, 0, 0, 0, 3232, 0, 0, 0, 0, 8004, 0, 0, 0, 59, 4282, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 6437, 8192, 2461, 4533, 0, 0, 0, 0, 672, 0, 3099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=17 tokens_per_expert_current_ep_rank=[1, 0, 2, 0, 2, 0, 337, 0, 21, 254, 7, 0, 2, 0, 0, 0, 11, 7399, 0, 0, 3, 3436, 1, 5, 3, 2, 7969, 2, 1, 8, 66, 4399, 30, 32, 2, 0, 19, 0, 0, 15, 12, 20, 4, 6222, 8161, 2400, 5087, 7, 4, 0, 4, 684, 0, 2434, 0, 0, 0, 0, 0, 29, 0, 2, 0, 53] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=17 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 332, 0, 3216, 2533, 0, 0, 0, 0, 0, 0, 0, 268, 0, 0, 0, 0, 0, 0, 3493, 2309, 0, 0, 0, 0, 0, 0, 0, 2785, 0, 0, 0, 3098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1715, 0, 0, 0, 70, 4790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 6129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2412, 3612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 595, 0, 3117, 0, 0, 1477, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 4028, 2063, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6081, 6070, 5883, 6575, 6144, 6024, 6223, 6152] avg=6144.0000 max=6575 max_rank=3 min=5883 min_rank=2 imbalance_ratio=1.070150 max_load_reduction=42577 max_load_reduction_pct=0.866231 moved_to_echo=24543 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=17 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 343, 0, 3238, 2602, 0, 0, 0, 0, 0, 0, 0, 247, 0, 0, 0, 0, 0, 0, 3587, 2300, 0, 0, 0, 0, 0, 0, 0, 2804, 0, 0, 0, 3246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1709, 0, 0, 0, 90, 4292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2854, 3486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 582, 0, 3042, 0, 0, 1487, 998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 4013, 2063, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6183, 6134, 6050, 6091, 6129, 6340, 6109, 6116] avg=6144.0000 max=6340 max_rank=5 min=6050 min_rank=2 imbalance_ratio=1.031901 max_load_reduction=42812 max_load_reduction_pct=0.871012 moved_to_echo=24694 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=17 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 301, 0, 3219, 2559, 0, 0, 0, 0, 0, 0, 0, 261, 0, 0, 0, 0, 0, 0, 3527, 2303, 0, 0, 0, 0, 0, 0, 0, 2788, 0, 0, 0, 3367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1711, 0, 0, 0, 60, 4371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2568, 3652, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 0, 3121, 0, 0, 1479, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 4018, 2063, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6079, 6091, 6155, 6142, 6129, 6220, 6186, 6150] avg=6144.0000 max=6220 max_rank=5 min=6079 min_rank=0 imbalance_ratio=1.012370 max_load_reduction=42932 max_load_reduction_pct=0.873454 moved_to_echo=24685 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=17 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 302, 0, 3234, 2562, 0, 0, 0, 0, 0, 0, 0, 257, 0, 0, 0, 0, 0, 0, 3533, 2313, 0, 0, 0, 0, 0, 0, 0, 2802, 0, 0, 0, 3505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1718, 0, 0, 0, 35, 4414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2493, 3630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 0, 2979, 0, 0, 1485, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 4035, 2063, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6098, 6103, 6307, 6167, 6130, 6123, 6062, 6162] avg=6144.0000 max=6307 max_rank=2 min=6062 min_rank=6 imbalance_ratio=1.026530 max_load_reduction=42845 max_load_reduction_pct=0.871684 moved_to_echo=24477 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=17 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 315, 0, 3255, 2622, 0, 0, 0, 0, 0, 0, 0, 266, 0, 0, 0, 0, 0, 0, 3613, 2305, 0, 0, 0, 0, 0, 0, 0, 2818, 0, 0, 0, 3832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1713, 0, 0, 0, 64, 3844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2607, 3406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 637, 0, 3109, 0, 0, 1495, 975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 4022, 2062, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6192, 6184, 6650, 5621, 6130, 6013, 6217, 6145] avg=6144.0000 max=6650 max_rank=2 min=5621 min_rank=3 imbalance_ratio=1.082357 max_load_reduction=42502 max_load_reduction_pct=0.864705 moved_to_echo=24505 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=17 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 1, 313, 0, 3212, 2579, 0, 0, 0, 0, 0, 0, 0, 247, 0, 1, 0, 0, 0, 0, 3554, 2301, 0, 0, 0, 0, 0, 0, 0, 2782, 0, 0, 0, 3096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1709, 0, 0, 0, 69, 4564, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 20, 6128, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2489, 3563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 652, 0, 3219, 0, 0, 1475, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 70, 4014, 2062, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6106, 6103, 5880, 6343, 6152, 6054, 6366, 6148] avg=6144.0000 max=6366 max_rank=6 min=5880 min_rank=2 imbalance_ratio=1.036133 max_load_reduction=42786 max_load_reduction_pct=0.870483 moved_to_echo=24720 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=17 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 338, 0, 3224, 2706, 0, 0, 0, 0, 0, 0, 0, 262, 0, 0, 0, 0, 0, 0, 3731, 2295, 0, 0, 0, 0, 0, 0, 0, 2791, 0, 0, 0, 3232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1705, 0, 0, 0, 59, 4282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 6129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2461, 3524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 672, 0, 3099, 0, 0, 1481, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 4004, 2063, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6268, 6288, 6023, 6046, 6133, 5985, 6261, 6148] avg=6144.0000 max=6288 max_rank=1 min=5985 min_rank=5 imbalance_ratio=1.023438 max_load_reduction=42864 max_load_reduction_pct=0.872070 moved_to_echo=24527 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=17 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 2, 0, 2, 0, 337, 0, 3186, 2617, 0, 0, 0, 0, 0, 0, 21, 254, 7, 0, 2, 0, 0, 0, 3605, 2290, 0, 0, 0, 0, 0, 0, 11, 2748, 0, 0, 3, 3436, 1, 5, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 1686, 2, 1, 8, 66, 4399, 0, 0, 0, 0, 0, 0, 0, 0, 30, 32, 2, 0, 19, 0, 0, 15, 6107, 0, 0, 0, 0, 0, 0, 0, 12, 20, 4, 0, 0, 2400, 3950, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4, 684, 0, 2434, 0, 0, 1465, 1137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 2, 0, 53, 3993, 2054, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 6179, 6204, 6167, 6205, 6393, 5728, 6131] avg=6144.0000 max=6393 max_rank=5 min=5728 min_rank=6 imbalance_ratio=1.040527 max_load_reduction=42759 max_load_reduction_pct=0.869934 moved_to_echo=24457 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 0, 128, 0, 128, 128, 2688, 0, 25856, 20864, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 0, 128, 4992, 0, 24192, 0, 0, 11904, 8320, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 22400, 0, 0, 128, 26880, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 0, 20352, 28928, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 128, 13696, 128, 128, 128, 640, 35072, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 0, 128, 0, 512, 32128, 16512, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 128, 0, 128, 0, 0, 128, 49024, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 2176, 128, 128, 128, 0, 0, 0, 28672, 18432, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 7271, 0, 0, 1262, 0, 0, 0, 0, 0, 6584, 0, 0, 0, 0, 0, 138, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 2386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 0, 0, 6720, 8192, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 7280, 0, 0, 1226, 0, 0, 0, 0, 0, 6489, 0, 0, 0, 0, 0, 231, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 2403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 166, 0, 0, 6696, 8192, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 7309, 0, 0, 1251, 0, 0, 0, 0, 0, 6537, 0, 0, 0, 0, 0, 232, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 2486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 125, 0, 0, 6566, 8192, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 83, 0, 7329, 1, 2, 1261, 0, 0, 1, 0, 1, 6534, 0, 0, 0, 0, 0, 213, 0, 2, 8189, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 2374, 0, 0, 2, 2, 0, 0, 2, 0, 0, 0, 0, 2, 0, 1, 197, 0, 0, 6577, 8186, 1, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 7296, 0, 0, 1136, 0, 0, 0, 0, 0, 6545, 0, 0, 0, 0, 0, 162, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 2677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 0, 0, 6488, 8192, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=18 tokens_per_expert_current_ep_rank=[2, 1, 1, 0, 0, 0, 1, 0, 0, 0, 93, 0, 7216, 10, 33, 1364, 2, 0, 1, 0, 12, 6453, 3, 12, 1, 0, 0, 320, 0, 32, 8156, 0, 0, 0, 1, 0, 0, 8154, 2, 3, 0, 0, 2498, 0, 4, 32, 33, 0, 8, 49, 0, 0, 0, 0, 23, 4, 8, 138, 0, 3, 6322, 8154, 2, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 7254, 0, 0, 1258, 0, 0, 0, 0, 0, 6582, 0, 0, 0, 0, 0, 169, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 2536, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 116, 0, 0, 6584, 8192, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 7325, 0, 0, 1394, 0, 0, 0, 0, 0, 6553, 0, 0, 0, 0, 0, 189, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 2524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 0, 0, 6333, 8192, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 4780, 0, 0, 1262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 0, 0, 5937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2386, 0, 0, 0, 0, 0, 2044, 661, 565, 397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2491, 2045, 1594, 0, 0, 0, 0, 0, 0, 152, 0, 0, 6155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6147, 6105, 6187, 6075, 6148, 6053, 6130, 6307] avg=6144.0000 max=6307 max_rank=7 min=6053 min_rank=5 imbalance_ratio=1.026530 max_load_reduction=42845 max_load_reduction_pct=0.871684 moved_to_echo=24638 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 4805, 0, 0, 1251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 232, 0, 0, 5937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2486, 0, 0, 0, 0, 0, 2044, 661, 552, 394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2504, 2045, 1594, 0, 0, 0, 0, 0, 0, 125, 0, 0, 6014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6147, 6126, 6143, 6169, 6148, 6137, 6143, 6139] avg=6144.0000 max=6169 max_rank=3 min=6126 min_rank=1 imbalance_ratio=1.004069 max_load_reduction=42983 max_load_reduction_pct=0.874491 moved_to_echo=24567 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 4786, 0, 0, 1226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 231, 0, 0, 5937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2403, 0, 0, 0, 0, 0, 2044, 661, 563, 391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2494, 2045, 1594, 0, 0, 0, 0, 0, 0, 166, 0, 0, 6133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6147, 6097, 6098, 6168, 6148, 6062, 6133, 6299] avg=6144.0000 max=6299 max_rank=7 min=6062 min_rank=5 imbalance_ratio=1.025228 max_load_reduction=42853 max_load_reduction_pct=0.871847 moved_to_echo=24642 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 4796, 0, 0, 1136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 162, 0, 0, 5937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2677, 0, 0, 0, 0, 0, 2044, 661, 545, 395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2500, 2044, 1594, 0, 0, 0, 0, 0, 0, 192, 0, 0, 5943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6148, 6012, 6150, 6099, 6148, 6322, 6138, 6135] avg=6144.0000 max=6322 max_rank=5 min=6012 min_rank=1 imbalance_ratio=1.028971 max_load_reduction=42830 max_load_reduction_pct=0.871379 moved_to_echo=24743 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 1, 83, 0, 4818, 1, 2, 1261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 6140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 213, 0, 2, 5935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2374, 0, 0, 2, 2, 0, 2044, 661, 553, 394, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 2511, 2043, 1593, 0, 0, 0, 0, 0, 1, 197, 0, 0, 6024, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6144, 6166, 6142, 6150, 6146, 6030, 6151, 6223] avg=6144.0000 max=6223 max_rank=7 min=6030 min_rank=5 imbalance_ratio=1.012858 max_load_reduction=42929 max_load_reduction_pct=0.873393 moved_to_echo=24550 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 4769, 0, 0, 1258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 0, 0, 5937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2536, 0, 0, 0, 0, 0, 2044, 661, 554, 397, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2485, 2044, 1594, 0, 0, 0, 0, 0, 0, 116, 0, 0, 6030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6148, 6102, 6185, 6106, 6148, 6192, 6125, 6146] avg=6144.0000 max=6192 max_rank=5 min=6102 min_rank=1 imbalance_ratio=1.007812 max_load_reduction=42960 max_load_reduction_pct=0.874023 moved_to_echo=24611 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=18 rerouted_tokens_per_expert_current_ep_rank=[2, 1, 1, 0, 0, 0, 1, 0, 6119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 4740, 10, 33, 1364, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 12, 6059, 3, 12, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 320, 0, 32, 5906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6112, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2498, 0, 4, 32, 33, 0, 2042, 661, 536, 394, 0, 0, 0, 0, 8, 49, 0, 0, 0, 0, 23, 4, 2476, 2035, 1589, 0, 0, 0, 0, 0, 8, 138, 0, 3, 5786, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6124, 6240, 6089, 6259, 6118, 6200, 6184, 5938] avg=6144.0000 max=6259 max_rank=3 min=5938 min_rank=7 imbalance_ratio=1.018717 max_load_reduction=42893 max_load_reduction_pct=0.872660 moved_to_echo=24440 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 4816, 0, 0, 1394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 0, 0, 5937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2524, 0, 0, 0, 0, 0, 2044, 661, 532, 395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2509, 2045, 1594, 0, 0, 0, 0, 0, 0, 164, 0, 0, 5801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6147, 6304, 6158, 6126, 6148, 6156, 6148, 5965] avg=6144.0000 max=6304 max_rank=1 min=5965 min_rank=7 imbalance_ratio=1.026042 max_load_reduction=42848 max_load_reduction_pct=0.871745 moved_to_echo=24417 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 0, 19968, 0, 128, 128, 128, 0, 16384, 5376, 4480, 3200, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 0, 0, 0, 0, 128, 128, 20096, 16384, 12800, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 128, 768, 0, 38400, 128, 128, 10240, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 128, 0, 0, 0, 128, 0, 49152, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 0, 49152, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 1280, 0, 128, 48000, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[128, 0, 0, 1664, 0, 128, 47488, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 128, 49152, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=19 tokens_per_expert_current_ep_rank=[7615, 0, 0, 0, 0, 0, 0, 519, 0, 0, 0, 0, 0, 0, 0, 0, 149, 0, 1314, 137, 8192, 0, 0, 0, 0, 0, 132, 0, 1206, 0, 0, 0, 566, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 7952, 6906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7079, 6921, 0, 71, 0, 0, 323] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=19 tokens_per_expert_current_ep_rank=[7624, 0, 0, 0, 0, 0, 0, 499, 0, 0, 0, 0, 0, 0, 0, 0, 159, 0, 1303, 109, 8192, 0, 0, 0, 0, 0, 115, 0, 894, 0, 0, 0, 541, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 0, 7952, 6918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7079, 7224, 0, 76, 0, 0, 392] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=19 tokens_per_expert_current_ep_rank=[7648, 0, 0, 0, 0, 0, 0, 532, 0, 0, 1, 0, 0, 0, 0, 0, 118, 0, 1270, 164, 8192, 0, 0, 0, 0, 0, 97, 0, 973, 0, 0, 0, 480, 0, 0, 0, 0, 0, 1, 63, 0, 0, 0, 0, 8041, 7011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7018, 7168, 0, 63, 0, 0, 312] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=19 tokens_per_expert_current_ep_rank=[7569, 0, 0, 0, 0, 0, 0, 555, 0, 0, 0, 0, 0, 0, 0, 0, 143, 0, 1250, 148, 8192, 0, 0, 0, 0, 0, 167, 0, 1120, 0, 0, 0, 570, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 7890, 6912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7123, 6989, 0, 85, 0, 0, 354] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=19 tokens_per_expert_current_ep_rank=[7541, 0, 25, 0, 32, 0, 0, 511, 0, 0, 1, 0, 0, 0, 0, 0, 182, 3, 1150, 132, 8134, 0, 15, 0, 0, 0, 129, 0, 1001, 12, 1, 0, 528, 12, 27, 0, 2, 0, 0, 88, 0, 12, 0, 25, 7888, 6891, 0, 0, 0, 0, 3, 0, 0, 27, 0, 0, 10, 7214, 7059, 0, 90, 31, 6, 370] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=19 tokens_per_expert_current_ep_rank=[7658, 0, 0, 0, 0, 0, 0, 457, 0, 0, 0, 0, 0, 0, 0, 0, 159, 0, 1178, 114, 8192, 0, 0, 0, 0, 0, 107, 0, 1134, 0, 0, 0, 541, 0, 0, 0, 0, 0, 0, 94, 0, 0, 0, 0, 7972, 7044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7081, 7001, 0, 94, 0, 0, 326] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=19 tokens_per_expert_current_ep_rank=[7603, 0, 2, 0, 2, 0, 0, 494, 0, 0, 0, 0, 0, 0, 0, 0, 156, 1, 1299, 127, 8189, 1, 2, 0, 0, 0, 108, 0, 1215, 0, 0, 0, 541, 1, 0, 0, 0, 0, 0, 83, 0, 2, 0, 2, 7943, 7006, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 6990, 6962, 0, 83, 2, 2, 334] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=19 tokens_per_expert_current_ep_rank=[7647, 0, 0, 0, 0, 0, 0, 487, 0, 0, 0, 0, 0, 0, 0, 0, 167, 0, 1158, 135, 8192, 0, 0, 0, 0, 0, 94, 0, 1029, 0, 0, 0, 538, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 7966, 6984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7140, 7108, 0, 80, 0, 0, 347] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=19 rerouted_tokens_per_expert_current_ep_rank=[5632, 0, 0, 0, 0, 0, 0, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 149, 0, 1314, 137, 4619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 1206, 0, 0, 0, 1983, 1316, 813, 807, 0, 0, 0, 0, 566, 0, 0, 0, 0, 0, 0, 70, 2766, 2752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4327, 1807, 0, 0, 0, 0, 0, 0, 0, 0, 5605, 0, 71, 0, 0, 323, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6151, 6145, 6219, 6257, 6154, 6093, 6134, 5999] avg=6144.0000 max=6257 max_rank=3 min=5999 min_rank=7 imbalance_ratio=1.018392 max_load_reduction=42895 max_load_reduction_pct=0.872701 moved_to_echo=24380 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=19 rerouted_tokens_per_expert_current_ep_rank=[5656, 0, 0, 0, 0, 0, 0, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6213, 0, 0, 0, 0, 0, 0, 0, 118, 0, 1270, 164, 4619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 973, 0, 0, 0, 1992, 1363, 826, 807, 0, 0, 0, 0, 480, 0, 0, 0, 0, 0, 1, 63, 2766, 2728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4290, 1828, 0, 0, 0, 0, 0, 0, 0, 0, 5805, 0, 63, 0, 0, 312, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6188, 6214, 6171, 6058, 6038, 6185, 6118, 6180] avg=6144.0000 max=6214 max_rank=1 min=6038 min_rank=4 imbalance_ratio=1.011393 max_load_reduction=42938 max_load_reduction_pct=0.873576 moved_to_echo=24521 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=19 rerouted_tokens_per_expert_current_ep_rank=[5597, 0, 0, 0, 0, 0, 0, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6096, 0, 0, 0, 0, 0, 0, 0, 143, 0, 1250, 148, 4619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 167, 0, 1120, 0, 0, 0, 1972, 1329, 814, 807, 0, 0, 0, 0, 570, 0, 0, 0, 0, 0, 0, 85, 2766, 2769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4354, 1794, 0, 0, 0, 0, 0, 0, 0, 0, 5660, 0, 85, 0, 0, 354, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6152, 6096, 6160, 6209, 6190, 6098, 6148, 6099] avg=6144.0000 max=6209 max_rank=3 min=6096 min_rank=1 imbalance_ratio=1.010579 max_load_reduction=42943 max_load_reduction_pct=0.873678 moved_to_echo=24535 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=19 rerouted_tokens_per_expert_current_ep_rank=[5655, 0, 0, 0, 0, 0, 0, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6156, 0, 0, 0, 0, 0, 0, 0, 167, 0, 1158, 135, 4619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 1029, 0, 0, 0, 1992, 1351, 823, 807, 0, 0, 0, 0, 538, 0, 0, 0, 0, 0, 0, 80, 2766, 2777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4363, 1810, 0, 0, 0, 0, 0, 0, 0, 0, 5757, 0, 80, 0, 0, 347, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6142, 6156, 6079, 6096, 6161, 6161, 6173, 6184] avg=6144.0000 max=6184 max_rank=7 min=6079 min_rank=2 imbalance_ratio=1.006510 max_load_reduction=42968 max_load_reduction_pct=0.874186 moved_to_echo=24679 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=19 rerouted_tokens_per_expert_current_ep_rank=[5638, 0, 0, 0, 0, 0, 0, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 159, 0, 1303, 109, 4619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 894, 0, 0, 0, 1986, 1373, 815, 807, 0, 0, 0, 0, 541, 0, 0, 0, 0, 0, 0, 75, 2766, 2753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4326, 1807, 0, 0, 0, 0, 0, 0, 0, 0, 5851, 0, 76, 0, 0, 392, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6137, 6145, 6190, 5990, 6135, 6103, 6133, 6319] avg=6144.0000 max=6319 max_rank=7 min=5990 min_rank=3 imbalance_ratio=1.028483 max_load_reduction=42833 max_load_reduction_pct=0.871440 moved_to_echo=24690 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=19 rerouted_tokens_per_expert_current_ep_rank=[5623, 0, 2, 0, 2, 0, 0, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6138, 0, 0, 0, 0, 0, 0, 0, 156, 1, 1299, 127, 4617, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 0, 1215, 0, 0, 0, 1980, 1324, 825, 807, 0, 0, 0, 0, 541, 1, 0, 0, 0, 0, 0, 83, 2765, 2719, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 6181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4271, 1805, 0, 0, 0, 0, 0, 0, 1, 0, 5638, 0, 83, 2, 2, 334, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6121, 6138, 6203, 6259, 6109, 6185, 6077, 6060] avg=6144.0000 max=6259 max_rank=3 min=6060 min_rank=7 imbalance_ratio=1.018717 max_load_reduction=42893 max_load_reduction_pct=0.872660 moved_to_echo=24431 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=19 rerouted_tokens_per_expert_current_ep_rank=[5573, 0, 25, 0, 32, 0, 0, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6096, 0, 0, 0, 0, 0, 0, 0, 182, 3, 1150, 132, 4578, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 0, 1001, 12, 1, 0, 1968, 1345, 815, 808, 0, 0, 0, 0, 528, 12, 27, 0, 2, 0, 0, 88, 2748, 2806, 0, 0, 0, 0, 0, 0, 0, 12, 0, 25, 0, 6076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 27, 0, 0, 4408, 1792, 0, 0, 0, 0, 0, 0, 10, 0, 5714, 0, 90, 31, 6, 370, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6141, 6097, 6060, 6079, 6211, 6113, 6230, 6221] avg=6144.0000 max=6230 max_rank=6 min=6060 min_rank=2 imbalance_ratio=1.013997 max_load_reduction=42922 max_load_reduction_pct=0.873250 moved_to_echo=24775 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=19 rerouted_tokens_per_expert_current_ep_rank=[5663, 0, 0, 0, 0, 0, 0, 457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6161, 0, 0, 0, 0, 0, 0, 0, 159, 0, 1178, 114, 4619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 0, 1134, 0, 0, 0, 1995, 1331, 830, 807, 0, 0, 0, 0, 541, 0, 0, 0, 0, 0, 0, 94, 2766, 2753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4328, 1811, 0, 0, 0, 0, 0, 0, 0, 0, 5670, 0, 94, 0, 0, 326, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6120, 6161, 6070, 6204, 6154, 6214, 6139, 6090] avg=6144.0000 max=6214 max_rank=5 min=6070 min_rank=2 imbalance_ratio=1.011393 max_load_reduction=42938 max_load_reduction_pct=0.873576 moved_to_echo=24597 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 1024, 0, 8576, 128, 128, 0, 15872, 10752, 6656, 6528, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4352, 128, 128, 0, 128, 0, 128, 640, 22144, 22144, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 128, 0, 0, 128, 0, 0, 34688, 14464, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1280, 128, 9984, 1152, 36992, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 0, 45824, 0, 768, 128, 128, 2816, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49280 observed_loss=-128 tokens_per_expert=[0, 0, 128, 0, 0, 0, 0, 0, 49152, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 128, 0, 128, 0, 49152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[45056, 0, 128, 0, 128, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1252, 0, 0, 0, 51, 0, 8072, 5306, 0, 0, 7352, 0, 0, 0, 0, 826, 0, 0, 0, 0, 8129, 0, 0, 0, 0, 2646, 0, 7689, 0, 0, 0, 0, 19, 0, 0, 0, 0, 63, 0, 0, 0, 144, 1320, 0, 6220, 0, 0, 63, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1788, 0, 0, 0, 64, 0, 8079, 4951, 0, 0, 7323, 0, 0, 0, 0, 863, 0, 0, 0, 0, 8122, 0, 0, 0, 0, 3182, 0, 7573, 0, 0, 0, 0, 7, 0, 0, 0, 0, 70, 0, 0, 0, 200, 1331, 0, 5529, 0, 0, 70, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1256, 0, 0, 0, 93, 0, 8071, 4962, 0, 0, 7296, 0, 0, 0, 0, 898, 0, 0, 0, 0, 8112, 0, 0, 0, 0, 3027, 0, 7621, 0, 0, 0, 0, 31, 0, 0, 0, 0, 80, 0, 0, 0, 232, 1311, 0, 6082, 0, 0, 80, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 1561, 1, 0, 0, 96, 0, 8126, 5010, 1, 0, 7318, 2, 0, 0, 0, 789, 0, 0, 0, 2, 8106, 3, 1, 0, 0, 2966, 0, 7563, 0, 1, 0, 0, 46, 0, 0, 0, 0, 83, 0, 0, 0, 242, 1257, 1, 5887, 0, 2, 83, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1436, 0, 0, 0, 98, 0, 8066, 5014, 0, 0, 7363, 0, 0, 0, 0, 854, 0, 0, 0, 0, 8098, 0, 0, 0, 0, 3081, 0, 7593, 0, 0, 0, 0, 16, 0, 0, 0, 0, 94, 0, 0, 0, 245, 1266, 0, 5834, 0, 0, 94, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1577, 0, 0, 0, 92, 0, 8101, 4724, 0, 0, 7305, 0, 0, 0, 0, 838, 0, 0, 0, 0, 8107, 0, 0, 0, 0, 3270, 0, 7511, 0, 0, 0, 0, 24, 0, 0, 0, 0, 85, 0, 0, 0, 261, 1243, 0, 5929, 0, 0, 85, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 25, 0, 5, 2, 0, 0, 1, 0, 0, 0, 3, 0, 1640, 17, 1, 0, 105, 0, 7996, 4717, 15, 1, 7187, 20, 0, 0, 0, 865, 3, 0, 0, 12, 8055, 30, 0, 0, 0, 3308, 7, 7499, 3, 34, 1, 0, 80, 0, 10, 11, 2, 88, 6, 2, 1, 309, 1304, 5, 5657, 0, 37, 88, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1356, 0, 0, 0, 91, 0, 8097, 5129, 0, 0, 7271, 0, 0, 0, 0, 678, 0, 0, 0, 0, 8117, 0, 0, 0, 0, 2906, 0, 7610, 0, 0, 0, 0, 10, 0, 0, 0, 0, 76, 0, 0, 0, 230, 1344, 0, 6161, 0, 0, 76, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1788, 0, 1993, 1471, 1076, 46, 0, 0, 0, 0, 0, 0, 64, 0, 1080, 4951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5330, 0, 0, 0, 0, 863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3097, 0, 0, 0, 0, 3182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6102, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 200, 4979, 857, 0, 0, 0, 0, 0, 0, 1331, 0, 4453, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6142, 6374, 6095, 6193, 6279, 6109, 6106, 5854] avg=6144.0000 max=6374 max_rank=1 min=5854 min_rank=7 imbalance_ratio=1.037435 max_load_reduction=42778 max_load_reduction_pct=0.870321 moved_to_echo=24348 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1252, 0, 2001, 1493, 1210, 46, 0, 0, 0, 0, 0, 0, 51, 0, 1080, 5306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5351, 0, 0, 0, 0, 826, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3099, 0, 0, 0, 0, 2646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6196, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 144, 4984, 856, 0, 0, 0, 0, 0, 0, 1320, 0, 5010, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6136, 6002, 6437, 6177, 5745, 6215, 6047, 6393] avg=6144.0000 max=6437 max_rank=2 min=5745 min_rank=4 imbalance_ratio=1.047689 max_load_reduction=42715 max_load_reduction_pct=0.869039 moved_to_echo=24400 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1256, 0, 1985, 1480, 1183, 46, 0, 0, 0, 0, 0, 0, 93, 0, 1080, 4962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5311, 0, 0, 0, 0, 898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3093, 0, 0, 0, 0, 3027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6141, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 232, 4973, 856, 0, 0, 0, 0, 0, 0, 1311, 0, 4899, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6135, 5950, 6135, 6209, 6120, 6172, 6141, 6290] avg=6144.0000 max=6290 max_rank=7 min=5950 min_rank=1 imbalance_ratio=1.023763 max_load_reduction=42862 max_load_reduction_pct=0.872030 moved_to_echo=24723 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1577, 0, 1988, 1459, 1154, 46, 0, 0, 0, 0, 0, 0, 92, 0, 1083, 4724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5317, 0, 0, 0, 0, 838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3091, 0, 0, 0, 0, 3270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6052, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 261, 4970, 860, 0, 0, 0, 0, 0, 0, 1243, 0, 4775, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6158, 6224, 5899, 6155, 6361, 6076, 6176, 6103] avg=6144.0000 max=6361 max_rank=4 min=5899 min_rank=2 imbalance_ratio=1.035319 max_load_reduction=42791 max_load_reduction_pct=0.870585 moved_to_echo=24716 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 0, 0, 6177, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 1561, 1, 1991, 1469, 1145, 46, 0, 0, 0, 0, 0, 0, 96, 0, 1087, 5010, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5327, 2, 0, 0, 0, 789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3090, 3, 1, 0, 0, 2966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6094, 0, 1, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 242, 4970, 862, 0, 0, 0, 0, 0, 0, 1257, 1, 4742, 0, 2, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6179, 6216, 6194, 6118, 6062, 6141, 6157, 6085] avg=6144.0000 max=6216 max_rank=1 min=6062 min_rank=4 imbalance_ratio=1.011719 max_load_reduction=42936 max_load_reduction_pct=0.873535 moved_to_echo=24445 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1356, 0, 1978, 1478, 1199, 46, 0, 0, 0, 0, 0, 0, 91, 0, 1083, 5129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5293, 0, 0, 0, 0, 678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3095, 0, 0, 0, 0, 2906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6132, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 230, 4976, 859, 0, 0, 0, 0, 0, 0, 1344, 0, 4962, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6155, 6057, 6303, 5971, 6001, 6142, 6141, 6382] avg=6144.0000 max=6382 max_rank=7 min=5971 min_rank=3 imbalance_ratio=1.038737 max_load_reduction=42770 max_load_reduction_pct=0.870158 moved_to_echo=24666 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1436, 0, 2004, 1475, 1135, 46, 0, 0, 0, 0, 0, 0, 98, 0, 1078, 5014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5359, 0, 0, 0, 0, 854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3087, 0, 0, 0, 0, 3081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6118, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 0, 0, 245, 4965, 856, 0, 0, 0, 0, 0, 0, 1266, 0, 4699, 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6132, 6096, 6190, 6213, 6168, 6134, 6160, 6059] avg=6144.0000 max=6213 max_rank=3 min=6059 min_rank=7 imbalance_ratio=1.011230 max_load_reduction=42939 max_load_reduction_pct=0.873596 moved_to_echo=24521 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 25, 0, 5, 2, 0, 0, 6083, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 1640, 17, 1959, 1461, 1105, 47, 0, 0, 0, 0, 1, 0, 105, 0, 1060, 4717, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 5228, 20, 0, 0, 0, 865, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 3066, 30, 0, 0, 0, 3308, 0, 0, 0, 0, 0, 0, 0, 0, 7, 6038, 3, 34, 1, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 11, 2, 88, 6, 2, 1, 309, 4942, 853, 0, 0, 0, 0, 0, 0, 1304, 5, 4552, 0, 37, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6115, 6233, 5899, 6116, 6416, 6163, 6224, 5986] avg=6144.0000 max=6416 max_rank=4 min=5899 min_rank=2 imbalance_ratio=1.044271 max_load_reduction=42736 max_load_reduction_pct=0.869466 moved_to_echo=24789 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 128, 0, 11904, 128, 16000, 11904, 9216, 384, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 24832, 128, 128, 0, 0, 24448, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[10496, 128, 38144, 0, 128, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[42624, 128, 0, 0, 0, 6656, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 0, 768, 0, 8704, 39936, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 128, 128, 640, 128, 128, 128, 1920, 39808, 6912, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 48896, 128, 128, 128, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 128, 128, 0, 0, 49152, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=21 tokens_per_expert_current_ep_rank=[0, 175, 0, 8066, 720, 83, 0, 0, 0, 0, 7847, 0, 0, 0, 0, 7811, 0, 0, 312, 0, 0, 0, 0, 0, 0, 3708, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8, 0, 0, 0, 212, 0, 0, 0, 3451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 375, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=21 tokens_per_expert_current_ep_rank=[0, 98, 0, 8094, 655, 55, 0, 0, 0, 0, 7865, 0, 0, 0, 0, 8012, 0, 0, 219, 0, 0, 0, 0, 0, 0, 4118, 0, 8192, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 11, 0, 0, 0, 104, 0, 0, 0, 3225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 314, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=21 tokens_per_expert_current_ep_rank=[0, 140, 0, 8072, 654, 79, 0, 0, 0, 0, 7868, 0, 0, 0, 0, 7888, 0, 0, 242, 0, 0, 0, 0, 0, 0, 4110, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 37, 0, 0, 0, 166, 0, 0, 0, 3152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 360, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=21 tokens_per_expert_current_ep_rank=[0, 140, 0, 8051, 736, 70, 0, 0, 0, 0, 7848, 0, 0, 0, 0, 7883, 0, 0, 402, 0, 0, 0, 0, 0, 0, 4013, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 30, 0, 0, 0, 170, 0, 0, 0, 3076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 349, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=21 tokens_per_expert_current_ep_rank=[0, 153, 0, 8049, 654, 94, 0, 0, 0, 0, 7847, 0, 0, 0, 0, 7879, 0, 0, 260, 0, 0, 0, 0, 0, 0, 4017, 0, 8192, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 3, 0, 0, 0, 161, 0, 0, 0, 3303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 351, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=21 tokens_per_expert_current_ep_rank=[0, 152, 1, 8062, 660, 80, 0, 0, 1, 0, 7854, 0, 1, 0, 2, 7865, 2, 0, 347, 0, 0, 0, 0, 0, 0, 4131, 0, 8189, 1, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 24, 0, 0, 0, 176, 0, 2, 0, 3009, 1, 0, 2, 0, 0, 1, 3, 0, 5, 0, 394, 0, 0, 0, 0, 1, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=21 tokens_per_expert_current_ep_rank=[0, 140, 0, 8070, 670, 71, 0, 0, 0, 0, 7872, 0, 0, 0, 0, 7885, 0, 0, 236, 0, 0, 0, 0, 0, 0, 3570, 0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 3, 0, 0, 0, 171, 0, 0, 0, 3703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 378, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=21 tokens_per_expert_current_ep_rank=[0, 154, 1, 8000, 753, 86, 1, 0, 0, 1, 7820, 0, 19, 0, 23, 7822, 32, 8, 267, 0, 21, 0, 0, 0, 1, 3320, 0, 8152, 1, 0, 11, 0, 0, 0, 8129, 0, 0, 0, 86, 0, 6, 0, 195, 0, 8, 1, 3617, 23, 4, 2, 1, 0, 5, 36, 0, 1, 0, 506, 0, 11, 0, 3, 0, 25] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 140, 0, 5231, 736, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 402, 0, 0, 0, 0, 0, 4109, 1744, 0, 0, 0, 0, 0, 0, 0, 4013, 0, 2271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6127, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 170, 0, 0, 0, 3076, 0, 1713, 939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6139, 0, 0, 0, 0, 0, 0, 0, 0, 349, 0, 0, 0, 0, 0, 0, 2820, 1812, 1126, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6177, 6135, 6255, 6284, 6157, 5898, 6139, 6107] avg=6144.0000 max=6284 max_rank=3 min=5898 min_rank=5 imbalance_ratio=1.022786 max_load_reduction=42868 max_load_reduction_pct=0.872152 moved_to_echo=24301 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 98, 0, 5259, 655, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 219, 0, 0, 0, 0, 0, 4109, 1773, 0, 0, 0, 0, 0, 0, 0, 4118, 0, 2271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6126, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 0, 3225, 0, 1717, 938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6239, 0, 0, 0, 0, 0, 0, 0, 0, 314, 0, 0, 0, 0, 0, 0, 2835, 1812, 1126, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6067, 6148, 6101, 6389, 6137, 5984, 6239, 6087] avg=6144.0000 max=6389 max_rank=3 min=5984 min_rank=5 imbalance_ratio=1.039876 max_load_reduction=42763 max_load_reduction_pct=0.870015 moved_to_echo=24447 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 175, 0, 5240, 720, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 312, 0, 0, 0, 0, 0, 4109, 1728, 0, 0, 0, 0, 0, 0, 0, 3708, 0, 2271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6127, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 0, 0, 0, 3451, 0, 1713, 939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6083, 0, 0, 0, 0, 0, 0, 0, 0, 375, 0, 0, 0, 0, 0, 0, 2826, 1812, 1126, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6218, 6134, 6149, 5979, 6135, 6315, 6083, 6139] avg=6144.0000 max=6315 max_rank=5 min=5979 min_rank=3 imbalance_ratio=1.027832 max_load_reduction=42837 max_load_reduction_pct=0.871521 moved_to_echo=24672 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 140, 0, 5244, 654, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 242, 0, 0, 0, 0, 0, 4109, 1746, 0, 0, 0, 0, 0, 0, 0, 4110, 0, 2271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6127, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 166, 0, 0, 0, 3152, 0, 1717, 939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6142, 0, 0, 0, 0, 0, 0, 0, 0, 360, 0, 0, 0, 0, 0, 0, 2828, 1812, 1126, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6117, 6151, 6097, 6381, 6164, 5974, 6142, 6126] avg=6144.0000 max=6381 max_rank=3 min=5974 min_rank=5 imbalance_ratio=1.038574 max_load_reduction=42771 max_load_reduction_pct=0.870178 moved_to_echo=24406 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 152, 1, 5238, 660, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6140, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 347, 0, 0, 0, 0, 0, 4107, 1741, 0, 0, 0, 0, 0, 0, 0, 4131, 0, 2271, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6121, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 0, 2, 0, 3009, 1, 1714, 938, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 3, 0, 5, 6124, 0, 0, 0, 0, 0, 0, 0, 0, 394, 0, 0, 0, 0, 1, 2, 2824, 1811, 1125, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6131, 6144, 6197, 6403, 6145, 5840, 6135, 6157] avg=6144.0000 max=6403 max_rank=3 min=5840 min_rank=5 imbalance_ratio=1.042155 max_load_reduction=42749 max_load_reduction_pct=0.869731 moved_to_echo=24277 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 140, 0, 5243, 670, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 236, 0, 0, 0, 0, 0, 4109, 1746, 0, 0, 0, 0, 0, 0, 0, 3570, 0, 2271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6127, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 171, 0, 0, 0, 3703, 0, 1718, 938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6139, 0, 0, 0, 0, 0, 0, 0, 0, 378, 0, 0, 0, 0, 0, 0, 2827, 1812, 1126, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6124, 6154, 6091, 5841, 6130, 6530, 6139, 6143] avg=6144.0000 max=6530 max_rank=5 min=5841 min_rank=3 imbalance_ratio=1.062826 max_load_reduction=42622 max_load_reduction_pct=0.867147 moved_to_echo=24942 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 154, 1, 5193, 753, 86, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6109, 0, 19, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 8, 267, 0, 21, 0, 0, 0, 4090, 1732, 0, 0, 0, 0, 0, 0, 1, 3320, 0, 2254, 1, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6070, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 195, 0, 8, 1, 3617, 23, 1711, 935, 0, 0, 0, 0, 0, 0, 4, 2, 1, 0, 5, 36, 0, 1, 6090, 0, 0, 0, 0, 0, 0, 0, 0, 506, 0, 11, 0, 3, 0, 25, 2807, 1808, 1124, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6188, 6152, 6150, 5587, 6156, 6496, 6139, 6284] avg=6144.0000 max=6496 max_rank=5 min=5587 min_rank=3 imbalance_ratio=1.057292 max_load_reduction=42656 max_load_reduction_pct=0.867839 moved_to_echo=25075 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 153, 0, 5229, 654, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 260, 0, 0, 0, 0, 0, 4109, 1743, 0, 0, 0, 0, 0, 0, 0, 4017, 0, 2271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6125, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 0, 0, 0, 3303, 0, 1713, 938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6136, 0, 0, 0, 0, 0, 0, 0, 0, 351, 0, 0, 0, 0, 0, 0, 2820, 1812, 1126, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6130, 6134, 6112, 6288, 6128, 6115, 6136, 6109] avg=6144.0000 max=6288 max_rank=3 min=6109 min_rank=7 imbalance_ratio=1.023438 max_load_reduction=42864 max_load_reduction_pct=0.872070 moved_to_echo=24488 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 2304, 0, 128, 0, 0, 0, 32896, 14080, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 0, 1408, 0, 128, 128, 26624, 128, 13824, 7552, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 31104, 0, 18176, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[0, 3072, 0, 128, 0, 128, 128, 128, 22656, 14592, 9088, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 1152, 128, 41984, 5504, 640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 49152, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 128, 128, 0, 128, 49152, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49280 observed_loss=-128 tokens_per_expert=[0, 0, 49024, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=22 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 8192, 478, 0, 1046, 0, 0, 0, 0, 506, 0, 0, 0, 0, 0, 0, 0, 594, 0, 0, 0, 0, 0, 0, 0, 0, 5885, 0, 0, 0, 0, 1334, 0, 8192, 0, 0, 0, 8192, 0, 0, 101, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 310, 0, 6130] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=22 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 8192, 460, 0, 1098, 0, 0, 0, 0, 459, 0, 0, 0, 0, 0, 0, 0, 595, 0, 0, 0, 0, 0, 0, 0, 0, 6043, 0, 0, 0, 0, 1070, 0, 8192, 0, 0, 0, 8192, 0, 0, 53, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 352, 0, 6258] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=22 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 8192, 504, 0, 914, 0, 0, 0, 0, 549, 0, 0, 0, 0, 0, 0, 0, 591, 0, 0, 0, 0, 0, 0, 0, 0, 6059, 0, 0, 0, 0, 1170, 0, 8192, 0, 0, 0, 8192, 0, 0, 76, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 323, 0, 6206] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=22 tokens_per_expert_current_ep_rank=[6, 2, 0, 0, 0, 0, 1, 8185, 426, 2, 1194, 0, 0, 0, 0, 464, 0, 0, 1, 0, 0, 0, 0, 601, 0, 0, 0, 0, 0, 0, 0, 1, 5943, 1, 0, 0, 0, 1091, 0, 8189, 0, 2, 0, 8187, 1, 2, 77, 0, 0, 0, 1, 3, 2, 8170, 0, 2, 0, 1, 0, 0, 0, 335, 0, 6262] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=22 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 8192, 466, 0, 1012, 0, 0, 0, 0, 484, 0, 0, 0, 0, 0, 0, 0, 628, 0, 0, 0, 0, 0, 0, 0, 0, 5696, 0, 0, 0, 0, 1356, 0, 8192, 0, 0, 0, 8192, 0, 0, 85, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 392, 0, 6267] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=22 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 8192, 454, 0, 1108, 0, 0, 0, 0, 474, 0, 0, 0, 0, 0, 0, 0, 606, 0, 0, 0, 0, 0, 0, 0, 0, 6153, 0, 0, 0, 0, 846, 0, 8192, 0, 0, 0, 8192, 0, 0, 100, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 345, 0, 6300] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=22 tokens_per_expert_current_ep_rank=[39, 23, 0, 0, 0, 0, 3, 8108, 439, 37, 1589, 0, 0, 0, 0, 533, 0, 10, 7, 0, 0, 0, 1, 534, 0, 0, 0, 0, 0, 1, 3, 18, 6026, 21, 1, 0, 0, 1144, 0, 8148, 0, 31, 0, 8132, 26, 34, 58, 0, 0, 0, 0, 23, 20, 7997, 0, 20, 0, 9, 4, 1, 1, 347, 0, 5764] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=22 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 8192, 401, 0, 1096, 0, 0, 0, 0, 434, 0, 0, 0, 0, 0, 0, 0, 638, 0, 0, 0, 0, 0, 0, 0, 0, 6033, 0, 0, 0, 0, 1213, 0, 8192, 0, 0, 0, 8192, 0, 0, 100, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 326, 0, 6152] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=22 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 478, 0, 1046, 0, 0, 0, 0, 506, 2034, 975, 681, 364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 594, 2135, 2046, 1367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6146, 0, 0, 0, 0, 0, 0, 0, 4910, 0, 0, 0, 0, 1334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6057, 0, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 310, 0, 5766, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6144, 6084, 6142, 6146, 6244, 6158, 6158, 6076] avg=6144.0000 max=6244 max_rank=4 min=6076 min_rank=7 imbalance_ratio=1.016276 max_load_reduction=42908 max_load_reduction_pct=0.872965 moved_to_echo=24636 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=22 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 504, 0, 914, 0, 0, 0, 0, 549, 2031, 1004, 681, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 591, 2135, 2046, 1367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6146, 0, 0, 0, 0, 0, 0, 0, 5055, 0, 0, 0, 0, 1170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6057, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 323, 0, 5838, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 6051, 6139, 6146, 6225, 6133, 6152, 6161] avg=6144.0000 max=6225 max_rank=4 min=6051 min_rank=1 imbalance_ratio=1.013184 max_load_reduction=42927 max_load_reduction_pct=0.873352 moved_to_echo=24671 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=22 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 460, 0, 1098, 0, 0, 0, 0, 459, 2033, 1002, 681, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 595, 2135, 2046, 1367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6146, 0, 0, 0, 0, 0, 0, 0, 5041, 0, 0, 0, 0, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6057, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 352, 0, 5886, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6144, 6105, 6143, 6146, 6111, 6110, 6155, 6238] avg=6144.0000 max=6238 max_rank=7 min=6105 min_rank=1 imbalance_ratio=1.015299 max_load_reduction=42914 max_load_reduction_pct=0.873088 moved_to_echo=24614 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=22 rerouted_tokens_per_expert_current_ep_rank=[6, 2, 0, 0, 0, 0, 1, 6139, 0, 0, 0, 0, 0, 0, 0, 0, 426, 2, 1194, 0, 0, 0, 0, 464, 2028, 985, 681, 372, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 601, 2133, 2046, 1365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6143, 0, 0, 0, 0, 0, 0, 0, 4958, 1, 0, 0, 0, 1091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 6054, 1, 2, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 2, 6142, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 335, 0, 5890, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6148, 6152, 6146, 6144, 6050, 6136, 6150, 6226] avg=6144.0000 max=6226 max_rank=7 min=6050 min_rank=4 imbalance_ratio=1.013346 max_load_reduction=42926 max_load_reduction_pct=0.873332 moved_to_echo=24562 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=22 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 454, 0, 1108, 0, 0, 0, 0, 474, 2033, 1020, 681, 374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 606, 2135, 2047, 1367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 5133, 0, 0, 0, 0, 846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6057, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 345, 0, 5926, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6144, 6144, 6155, 6145, 5979, 6157, 6157, 6271] avg=6144.0000 max=6271 max_rank=7 min=5979 min_rank=4 imbalance_ratio=1.020671 max_load_reduction=42881 max_load_reduction_pct=0.872416 moved_to_echo=24564 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=22 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 466, 0, 1012, 0, 0, 0, 0, 484, 2033, 944, 681, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 628, 2135, 2047, 1367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 4752, 0, 0, 0, 0, 1356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6057, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 392, 0, 5895, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6144, 5992, 6177, 6145, 6108, 6142, 6157, 6287] avg=6144.0000 max=6287 max_rank=7 min=5992 min_rank=1 imbalance_ratio=1.023275 max_load_reduction=42865 max_load_reduction_pct=0.872091 moved_to_echo=24694 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=22 rerouted_tokens_per_expert_current_ep_rank=[39, 23, 0, 0, 0, 0, 3, 6074, 0, 0, 0, 0, 0, 0, 0, 0, 439, 37, 1589, 0, 0, 0, 0, 533, 1989, 1003, 680, 346, 0, 0, 0, 0, 0, 10, 7, 0, 0, 0, 1, 534, 2122, 2036, 1354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 18, 6112, 0, 0, 0, 0, 0, 0, 0, 5023, 21, 1, 0, 0, 1144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 6010, 26, 34, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 20, 6008, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4, 1, 1, 347, 0, 5418, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6139, 6616, 6064, 6134, 6189, 6159, 6071, 5780] avg=6144.0000 max=6616 max_rank=1 min=5780 min_rank=7 imbalance_ratio=1.076823 max_load_reduction=42536 max_load_reduction_pct=0.865397 moved_to_echo=24199 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=22 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 401, 0, 1096, 0, 0, 0, 0, 434, 2031, 1000, 681, 365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 638, 2135, 2046, 1367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6146, 0, 0, 0, 0, 0, 0, 0, 5033, 0, 0, 0, 0, 1213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6057, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 326, 0, 5787, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6144, 6008, 6186, 6146, 6246, 6157, 6152, 6113] avg=6144.0000 max=6246 max_rank=4 min=6008 min_rank=1 imbalance_ratio=1.016602 max_load_reduction=42906 max_load_reduction_pct=0.872925 moved_to_echo=24668 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 0, 0, 128, 4864, 17152, 16384, 11008, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[3712, 128, 9088, 0, 0, 0, 0, 3968, 16256, 7936, 5504, 2944, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[39936, 128, 128, 0, 0, 9344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 128, 2816, 0, 46464, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 0, 48512, 128, 128, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 128, 128, 128, 49152, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 128, 0, 0, 0, 0, 128, 49152, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 128, 128, 128, 49152, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 2142, 5550, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 337, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 8192, 8190, 0, 0, 5212, 0, 0, 0, 0, 0, 0, 0, 0, 1108, 0, 1928, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 2272, 5787, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 357, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 4972, 0, 0, 0, 0, 0, 0, 0, 0, 996, 0, 1881, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 2349, 5536, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 379, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 4785, 0, 0, 0, 0, 0, 0, 0, 0, 1185, 0, 1989, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 2200, 5537, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 385, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 4973, 0, 0, 0, 0, 0, 0, 0, 0, 1228, 0, 1936, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 2351, 5593, 0, 0, 0, 0, 3, 0, 0, 0, 8189, 0, 0, 0, 2, 405, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 2, 0, 68, 0, 0, 1, 0, 1, 3, 8184, 8187, 0, 3, 4958, 0, 2, 0, 0, 0, 0, 0, 3, 1006, 1, 1912, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 1993, 5589, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 417, 0, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 5120, 0, 0, 0, 0, 0, 0, 0, 0, 1280, 0, 1836, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 2348, 5698, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 355, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 4596, 0, 0, 0, 0, 0, 0, 0, 0, 1159, 0, 2065, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=23 tokens_per_expert_current_ep_rank=[11, 0, 2479, 5460, 0, 0, 0, 1, 35, 0, 0, 0, 8148, 0, 0, 0, 15, 469, 0, 0, 0, 0, 0, 0, 108, 0, 11, 0, 0, 0, 0, 0, 8126, 1, 0, 0, 34, 0, 54, 0, 0, 7, 0, 0, 38, 8137, 8137, 0, 38, 5023, 0, 2, 3, 0, 1, 0, 0, 38, 953, 31, 1788, 0, 0, 4] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2272, 4011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 357, 0, 0, 0, 0, 0, 0, 2125, 2108, 1526, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 6067, 0, 0, 0, 0, 0, 0, 0, 6084, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4972, 0, 0, 0, 0, 0, 0, 1223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 996, 0, 1881, 0, 0, 0, 2048, 553, 522, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6283, 6144, 6116, 6127, 6143, 6144, 6195, 6000] avg=6144.0000 max=6283 max_rank=0 min=6000 min_rank=7 imbalance_ratio=1.022624 max_load_reduction=42869 max_load_reduction_pct=0.872172 moved_to_echo=24482 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2142, 3847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 337, 0, 0, 0, 0, 0, 0, 2124, 2108, 1526, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 6066, 0, 0, 0, 0, 0, 0, 0, 6084, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5212, 0, 0, 0, 0, 0, 0, 1173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1108, 0, 1928, 0, 0, 0, 2048, 530, 522, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5989, 6144, 6095, 6111, 6148, 6144, 6385, 6136] avg=6144.0000 max=6385 max_rank=6 min=5989 min_rank=0 imbalance_ratio=1.039225 max_load_reduction=42767 max_load_reduction_pct=0.870097 moved_to_echo=24813 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2200, 3838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 385, 0, 0, 0, 0, 0, 0, 2124, 2108, 1526, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 6068, 0, 0, 0, 0, 0, 0, 0, 6084, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4973, 0, 0, 0, 0, 0, 0, 1170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1228, 0, 1936, 0, 0, 0, 2048, 529, 522, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6038, 6144, 6143, 6152, 6125, 6144, 6143, 6263] avg=6144.0000 max=6263 max_rank=7 min=6038 min_rank=0 imbalance_ratio=1.019368 max_load_reduction=42889 max_load_reduction_pct=0.872579 moved_to_echo=24675 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2349, 3837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 379, 0, 0, 0, 0, 0, 0, 2125, 2108, 1526, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 6067, 0, 0, 0, 0, 0, 0, 0, 6084, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4785, 0, 0, 0, 0, 0, 0, 1170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1185, 0, 1989, 0, 0, 0, 2048, 529, 522, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6186, 6144, 6138, 6159, 6153, 6144, 5955, 6273] avg=6144.0000 max=6273 max_rank=7 min=5955 min_rank=6 imbalance_ratio=1.020996 max_load_reduction=42879 max_load_reduction_pct=0.872375 moved_to_echo=24525 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2351, 3877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 405, 0, 0, 0, 0, 0, 0, 2124, 2107, 1525, 0, 0, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 6063, 0, 0, 0, 0, 0, 0, 0, 6081, 0, 0, 0, 2, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 3, 6138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4958, 0, 2, 0, 0, 0, 0, 1182, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1006, 1, 1912, 0, 0, 0, 2046, 534, 521, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6228, 6146, 6163, 6153, 6151, 6143, 6145, 6023] avg=6144.0000 max=6228 max_rank=0 min=6023 min_rank=7 imbalance_ratio=1.013672 max_load_reduction=42924 max_load_reduction_pct=0.873291 moved_to_echo=24462 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1993, 3874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 417, 0, 0, 0, 0, 0, 0, 2125, 2108, 1526, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 6067, 0, 0, 0, 0, 0, 0, 0, 6084, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5120, 0, 0, 0, 0, 0, 0, 1181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1280, 0, 1836, 0, 0, 0, 2048, 534, 522, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5867, 6144, 6176, 6156, 6144, 6144, 6301, 6220] avg=6144.0000 max=6301 max_rank=6 min=5867 min_rank=0 imbalance_ratio=1.025553 max_load_reduction=42851 max_load_reduction_pct=0.871806 moved_to_echo=24809 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=23 rerouted_tokens_per_expert_current_ep_rank=[11, 0, 2479, 3773, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 6107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 469, 0, 0, 0, 0, 0, 0, 2111, 2092, 1520, 0, 0, 0, 0, 0, 108, 0, 11, 0, 0, 0, 0, 0, 6026, 0, 0, 0, 0, 0, 0, 0, 6034, 1, 0, 0, 34, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 38, 6100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 5023, 0, 2, 3, 0, 1, 0, 1160, 0, 0, 0, 0, 0, 0, 0, 0, 38, 953, 31, 1788, 0, 0, 4, 2037, 527, 521, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6264, 6142, 6207, 6145, 6123, 6145, 6227, 5899] avg=6144.0000 max=6264 max_rank=0 min=5899 min_rank=7 imbalance_ratio=1.019531 max_load_reduction=42888 max_load_reduction_pct=0.872559 moved_to_echo=24394 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2348, 3949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 355, 0, 0, 0, 0, 0, 0, 2125, 2108, 1526, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 6067, 0, 0, 0, 0, 0, 0, 0, 6084, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4596, 0, 0, 0, 0, 0, 0, 1205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1159, 0, 2065, 0, 0, 0, 2048, 544, 522, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6297, 6144, 6114, 6149, 6165, 6144, 5801, 6338] avg=6144.0000 max=6338 max_rank=7 min=5801 min_rank=6 imbalance_ratio=1.031576 max_load_reduction=42814 max_load_reduction_pct=0.871053 moved_to_echo=24448 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 3200, 0, 0, 0, 0, 0, 0, 17024, 16896, 12288, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 8960, 128, 15360, 0, 0, 128, 16384, 4352, 4224, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 0, 18176, 31104, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 39680, 0, 128, 128, 0, 128, 0, 9472, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[48640, 128, 0, 0, 128, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 128, 0, 128, 128, 49152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[768, 0, 128, 0, 0, 0, 0, 0, 48512, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49280 observed_loss=-128 tokens_per_expert=[128, 0, 0, 0, 49152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 7850, 0, 0, 0, 342, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 344, 0, 0, 7254, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 7848, 0, 0, 0, 310, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 344, 0, 0, 7357, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 7868, 0, 0, 0, 304, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 325, 0, 0, 7367, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 7868, 0, 0, 0, 298, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 325, 0, 0, 7474, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 7873, 0, 0, 0, 308, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 0, 319, 0, 0, 7385, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 3, 2, 0, 0, 2, 8189, 7855, 0, 0, 0, 320, 0, 0, 1, 0, 0, 3, 8189, 0, 0, 0, 1, 0, 0, 8185, 561, 1, 0, 2, 3, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 93, 0, 1, 0, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 333, 0, 0, 7132, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 7847, 0, 0, 0, 350, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 345, 0, 0, 7280, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 24, 30, 40, 16, 0, 2, 1, 8143, 7773, 0, 0, 1, 313, 0, 0, 7, 0, 1, 38, 8136, 4, 0, 7, 20, 0, 0, 8145, 620, 16, 0, 18, 34, 0, 0, 0, 0, 0, 8127, 1, 0, 6, 0, 201, 6, 62, 0, 6, 2, 0, 1, 0, 104, 0, 0, 0, 0, 340, 0, 0, 6907, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5828, 0, 0, 0, 342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5722, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 2047, 2022, 1456, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 2470, 2058, 1537, 0, 0, 0, 0, 0, 0, 0, 344, 0, 0, 5798, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6134, 6170, 6144, 6185, 6145, 6107, 6125, 6142] avg=6144.0000 max=6185 max_rank=3 min=6107 min_rank=5 imbalance_ratio=1.006673 max_load_reduction=42967 max_load_reduction_pct=0.874166 moved_to_echo=24519 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5841, 0, 0, 0, 298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5722, 311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 2047, 2027, 1500, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 2470, 2059, 1537, 0, 0, 0, 0, 0, 0, 0, 325, 0, 0, 5974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6133, 6139, 6144, 6033, 6144, 6148, 6112, 6299] avg=6144.0000 max=6299 max_rank=7 min=6033 min_rank=3 imbalance_ratio=1.025228 max_load_reduction=42853 max_load_reduction_pct=0.871847 moved_to_echo=24703 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5841, 0, 0, 0, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5722, 349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 2047, 2027, 1479, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 2470, 2058, 1537, 0, 0, 0, 0, 0, 0, 0, 325, 0, 0, 5888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6134, 6145, 6144, 6071, 6144, 6144, 6157, 6213] avg=6144.0000 max=6213 max_rank=7 min=6071 min_rank=3 imbalance_ratio=1.011230 max_load_reduction=42939 max_load_reduction_pct=0.873596 moved_to_echo=24658 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5825, 0, 0, 0, 350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5722, 391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 2047, 2022, 1461, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 2470, 2058, 1537, 0, 0, 0, 0, 0, 0, 0, 345, 0, 0, 5819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6134, 6175, 6144, 6113, 6145, 6126, 6151, 6164] avg=6144.0000 max=6175 max_rank=1 min=6113 min_rank=3 imbalance_ratio=1.005046 max_load_reduction=42977 max_load_reduction_pct=0.874369 moved_to_echo=24586 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 3, 2, 0, 0, 6132, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5831, 0, 0, 0, 320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 6142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5717, 561, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 6139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 1, 0, 2046, 2024, 1432, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 0, 2468, 2057, 1536, 0, 0, 0, 0, 0, 0, 0, 333, 0, 0, 5700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6139, 6153, 6146, 6280, 6144, 6107, 6150, 6033] avg=6144.0000 max=6280 max_rank=3 min=6033 min_rank=7 imbalance_ratio=1.022135 max_load_reduction=42872 max_load_reduction_pct=0.872233 moved_to_echo=24434 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5845, 0, 0, 0, 308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5722, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 2047, 2028, 1483, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 2470, 2058, 1537, 0, 0, 0, 0, 0, 0, 0, 319, 0, 0, 5902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6134, 6153, 6144, 6055, 6145, 6145, 6155, 6221] avg=6144.0000 max=6221 max_rank=7 min=6055 min_rank=3 imbalance_ratio=1.012533 max_load_reduction=42931 max_load_reduction_pct=0.873433 moved_to_echo=24666 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5826, 0, 0, 0, 310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5722, 342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 0, 0, 2047, 2022, 1477, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 2470, 2058, 1537, 0, 0, 0, 0, 0, 0, 0, 344, 0, 0, 5880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6134, 6136, 6144, 6064, 6142, 6151, 6157, 6224] avg=6144.0000 max=6224 max_rank=7 min=6064 min_rank=3 imbalance_ratio=1.013021 max_load_reduction=42928 max_load_reduction_pct=0.873372 moved_to_echo=24674 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 24, 30, 40, 16, 0, 2, 6098, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5766, 0, 0, 1, 313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 1, 38, 6092, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 20, 0, 0, 5688, 620, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 34, 0, 0, 0, 0, 0, 6091, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 0, 201, 6, 62, 0, 2036, 2007, 1391, 514, 0, 0, 0, 0, 6, 2, 0, 1, 0, 104, 0, 0, 2457, 2045, 1530, 0, 0, 0, 0, 0, 0, 0, 340, 0, 0, 5516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6210, 6081, 6142, 6351, 6143, 6224, 6145, 5856] avg=6144.0000 max=6351 max_rank=3 min=5856 min_rank=7 imbalance_ratio=1.033691 max_load_reduction=42801 max_load_reduction_pct=0.870789 moved_to_echo=24368 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 768, 128, 128, 0, 16384, 16256, 11776, 4096, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 0, 768, 0, 0, 19840, 16512, 12288, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 0, 0, 45824, 3456, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[128, 128, 0, 0, 0, 0, 0, 49152, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49280 observed_loss=-128 tokens_per_expert=[0, 0, 2688, 0, 0, 46592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 0, 128, 128, 49152, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 0, 128, 128, 128, 128, 0, 128, 49152, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 0, 46720, 0, 0, 128, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True - [2026-08-26 14:53:47] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 818.0 | throughput per GPU (TFLOP/s/GPU): 156.0 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 8.975649E+00 | loss scale: 1.0 | grad norm: 3.681 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1432, 334, 628, 992, 233, 1055, 955, 530, 1610, 291, 872, 321, 98, 1924, 924, 1009, 672, 903, 877, 716, 188, 451, 230, 1172, 339, 707, 720, 437, 799, 114, 726, 1044, 820, 727, 1451, 1250, 803, 244, 583, 915, 475, 623, 603, 588, 911, 928, 458, 340, 626, 481, 757, 704, 952, 1708, 578, 777, 1725, 312, 1570, 819, 179, 687, 1311, 944] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1436, 371, 553, 773, 172, 1122, 1011, 473, 1539, 333, 879, 392, 102, 1901, 884, 962, 674, 862, 849, 744, 184, 452, 209, 1110, 341, 647, 803, 415, 827, 199, 645, 1282, 913, 786, 1487, 1352, 476, 232, 751, 804, 475, 551, 635, 462, 977, 950, 463, 330, 639, 559, 634, 598, 997, 1582, 523, 799, 1655, 419, 1699, 829, 169, 861, 1480, 919] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1484, 280, 523, 822, 163, 1019, 922, 398, 1719, 340, 798, 237, 82, 2054, 881, 1000, 736, 913, 958, 721, 186, 478, 222, 1242, 392, 683, 737, 519, 798, 184, 655, 1093, 793, 717, 1526, 1265, 688, 241, 769, 935, 405, 502, 619, 465, 1047, 1114, 467, 364, 709, 544, 681, 545, 819, 1704, 442, 715, 1736, 350, 1652, 701, 252, 781, 1423, 942] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1470, 390, 680, 1019, 193, 1012, 797, 481, 1603, 391, 746, 287, 97, 1897, 790, 1034, 678, 992, 779, 605, 232, 450, 221, 1346, 399, 659, 727, 310, 793, 134, 771, 1086, 849, 869, 1417, 1250, 750, 198, 650, 863, 490, 570, 665, 548, 944, 996, 503, 369, 590, 553, 684, 716, 949, 1622, 651, 849, 1780, 320, 1536, 774, 195, 657, 1314, 962] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1439, 374, 614, 848, 196, 1074, 829, 473, 1555, 295, 754, 297, 116, 1979, 802, 950, 717, 896, 882, 645, 201, 472, 241, 1288, 398, 617, 743, 338, 751, 166, 767, 1141, 860, 830, 1542, 1305, 704, 217, 720, 874, 372, 507, 614, 523, 968, 1049, 494, 302, 606, 535, 699, 640, 948, 1682, 588, 829, 1941, 375, 1647, 696, 173, 751, 1356, 947] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1426, 353, 675, 937, 168, 1022, 872, 563, 1571, 263, 776, 280, 87, 1992, 900, 917, 770, 958, 827, 680, 188, 429, 247, 1221, 355, 683, 809, 388, 742, 114, 807, 1493, 838, 834, 1519, 1310, 486, 227, 736, 868, 405, 556, 626, 503, 1004, 973, 390, 320, 674, 520, 621, 616, 966, 1780, 485, 665, 1877, 318, 1621, 746, 191, 704, 1408, 852] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1536, 393, 544, 888, 168, 922, 879, 591, 1623, 322, 820, 311, 108, 1937, 839, 911, 677, 897, 791, 642, 182, 475, 220, 1291, 393, 740, 836, 398, 765, 125, 737, 1105, 848, 741, 1435, 1321, 709, 229, 700, 925, 470, 558, 660, 456, 1000, 973, 463, 343, 690, 556, 699, 644, 911, 1721, 519, 730, 1862, 342, 1519, 816, 165, 701, 1414, 966] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1502, 353, 666, 875, 155, 1200, 999, 549, 1536, 267, 846, 270, 120, 1811, 934, 1038, 657, 901, 917, 729, 198, 446, 205, 1377, 426, 714, 709, 346, 905, 142, 745, 1150, 780, 774, 1107, 1229, 706, 199, 625, 907, 426, 577, 593, 494, 982, 1048, 572, 368, 618, 466, 678, 636, 1011, 1607, 541, 742, 1881, 370, 1550, 772, 170, 761, 1394, 880] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1432, 334, 628, 992, 233, 1055, 955, 530, 171, 0, 0, 0, 0, 0, 0, 0, 1610, 291, 872, 321, 98, 1171, 924, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 672, 903, 877, 716, 188, 451, 230, 1172, 673, 103, 86, 0, 0, 0, 0, 0, 339, 707, 720, 437, 799, 114, 726, 1044, 667, 386, 0, 0, 0, 0, 0, 0, 820, 727, 778, 1250, 803, 244, 583, 915, 0, 0, 0, 0, 0, 0, 0, 0, 475, 623, 603, 588, 911, 928, 458, 340, 1160, 0, 0, 0, 0, 0, 0, 0, 626, 481, 757, 704, 952, 1434, 578, 777, 0, 0, 0, 0, 0, 0, 0, 0, 179, 312, 1570, 819, 179, 687, 1311, 944, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6330, 6296, 6071, 5939, 6120, 6086, 6309, 6001] avg=6144.0000 max=6330 max_rank=0 min=5939 min_rank=3 imbalance_ratio=1.030273 max_load_reduction=42822 max_load_reduction_pct=0.871216 moved_to_echo=24516 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1436, 371, 553, 773, 172, 1122, 1011, 473, 158, 0, 0, 0, 0, 0, 0, 0, 1539, 333, 879, 392, 102, 1157, 884, 962, 0, 0, 0, 0, 0, 0, 0, 0, 674, 862, 849, 744, 184, 452, 209, 1110, 689, 95, 85, 0, 0, 0, 0, 0, 341, 647, 803, 415, 827, 199, 645, 1282, 659, 371, 0, 0, 0, 0, 0, 0, 913, 786, 798, 1352, 476, 232, 751, 804, 0, 0, 0, 0, 0, 0, 0, 0, 475, 551, 635, 462, 977, 950, 463, 330, 1113, 0, 0, 0, 0, 0, 0, 0, 639, 559, 634, 598, 997, 1329, 523, 799, 0, 0, 0, 0, 0, 0, 0, 0, 171, 419, 1699, 829, 169, 861, 1480, 919, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6069, 6248, 5953, 6189, 6112, 5956, 6078, 6547] avg=6144.0000 max=6547 max_rank=7 min=5953 min_rank=2 imbalance_ratio=1.065592 max_load_reduction=42605 max_load_reduction_pct=0.866801 moved_to_echo=24693 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1484, 280, 523, 822, 163, 1019, 922, 398, 171, 0, 0, 0, 0, 0, 0, 0, 1719, 340, 798, 237, 82, 1251, 881, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 736, 913, 958, 721, 186, 478, 222, 1242, 707, 103, 91, 0, 0, 0, 0, 0, 392, 683, 737, 519, 798, 184, 655, 1093, 712, 389, 0, 0, 0, 0, 0, 0, 793, 717, 819, 1265, 688, 241, 769, 935, 0, 0, 0, 0, 0, 0, 0, 0, 405, 502, 619, 465, 1047, 1114, 467, 364, 1167, 0, 0, 0, 0, 0, 0, 0, 709, 544, 681, 545, 819, 1430, 442, 715, 0, 0, 0, 0, 0, 0, 0, 0, 180, 350, 1652, 701, 252, 781, 1423, 942, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5782, 6308, 6357, 6162, 6227, 6150, 5885, 6281] avg=6144.0000 max=6357 max_rank=2 min=5782 min_rank=0 imbalance_ratio=1.034668 max_load_reduction=42795 max_load_reduction_pct=0.870667 moved_to_echo=24543 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1470, 390, 680, 1019, 193, 1012, 797, 481, 162, 0, 0, 0, 0, 0, 0, 0, 1603, 391, 746, 287, 97, 1155, 790, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 678, 992, 779, 605, 232, 450, 221, 1346, 657, 98, 84, 0, 0, 0, 0, 0, 399, 659, 727, 310, 793, 134, 771, 1086, 658, 399, 0, 0, 0, 0, 0, 0, 849, 869, 760, 1250, 750, 198, 650, 863, 0, 0, 0, 0, 0, 0, 0, 0, 490, 570, 665, 548, 944, 996, 503, 369, 1197, 0, 0, 0, 0, 0, 0, 0, 590, 553, 684, 716, 949, 1362, 651, 849, 0, 0, 0, 0, 0, 0, 0, 0, 184, 320, 1536, 774, 195, 657, 1314, 962, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6204, 6103, 6142, 5936, 6189, 6282, 6354, 5942] avg=6144.0000 max=6354 max_rank=6 min=5936 min_rank=3 imbalance_ratio=1.034180 max_load_reduction=42798 max_load_reduction_pct=0.870728 moved_to_echo=24767 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1439, 374, 614, 848, 196, 1074, 829, 473, 168, 0, 0, 0, 0, 0, 0, 0, 1555, 295, 754, 297, 116, 1205, 802, 950, 0, 0, 0, 0, 0, 0, 0, 0, 717, 896, 882, 645, 201, 472, 241, 1288, 715, 101, 88, 0, 0, 0, 0, 0, 398, 617, 743, 338, 751, 166, 767, 1141, 686, 435, 0, 0, 0, 0, 0, 0, 860, 830, 827, 1305, 704, 217, 720, 874, 0, 0, 0, 0, 0, 0, 0, 0, 372, 507, 614, 523, 968, 1049, 494, 302, 1305, 0, 0, 0, 0, 0, 0, 0, 606, 535, 699, 640, 948, 1413, 588, 829, 0, 0, 0, 0, 0, 0, 0, 0, 201, 375, 1647, 696, 173, 751, 1356, 947, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6015, 5974, 6246, 6042, 6337, 6134, 6258, 6146] avg=6144.0000 max=6337 max_rank=4 min=5974 min_rank=1 imbalance_ratio=1.031413 max_load_reduction=42815 max_load_reduction_pct=0.871073 moved_to_echo=24875 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1426, 353, 675, 937, 168, 1022, 872, 563, 178, 0, 0, 0, 0, 0, 0, 0, 1571, 263, 776, 280, 87, 1212, 900, 917, 0, 0, 0, 0, 0, 0, 0, 0, 770, 958, 827, 680, 188, 429, 247, 1221, 704, 107, 89, 0, 0, 0, 0, 0, 355, 683, 809, 388, 742, 114, 807, 1493, 691, 420, 0, 0, 0, 0, 0, 0, 838, 834, 815, 1310, 486, 227, 736, 868, 0, 0, 0, 0, 0, 0, 0, 0, 405, 556, 626, 503, 1004, 973, 390, 320, 1262, 0, 0, 0, 0, 0, 0, 0, 674, 520, 621, 616, 966, 1495, 485, 665, 0, 0, 0, 0, 0, 0, 0, 0, 195, 318, 1621, 746, 191, 704, 1408, 852, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6194, 6006, 6220, 6502, 6114, 6039, 6042, 6035] avg=6144.0000 max=6502 max_rank=3 min=6006 min_rank=1 imbalance_ratio=1.058268 max_load_reduction=42650 max_load_reduction_pct=0.867716 moved_to_echo=24230 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1536, 393, 544, 888, 168, 922, 879, 591, 172, 0, 0, 0, 0, 0, 0, 0, 1623, 322, 820, 311, 108, 1179, 839, 911, 0, 0, 0, 0, 0, 0, 0, 0, 677, 897, 791, 642, 182, 475, 220, 1291, 665, 104, 86, 0, 0, 0, 0, 0, 393, 740, 836, 398, 765, 125, 737, 1105, 672, 417, 0, 0, 0, 0, 0, 0, 848, 741, 770, 1321, 709, 229, 700, 925, 0, 0, 0, 0, 0, 0, 0, 0, 470, 558, 660, 456, 1000, 973, 463, 343, 1252, 0, 0, 0, 0, 0, 0, 0, 690, 556, 699, 644, 911, 1445, 519, 730, 0, 0, 0, 0, 0, 0, 0, 0, 193, 342, 1519, 816, 165, 701, 1414, 966, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6093, 6113, 6030, 6188, 6243, 6175, 6194, 6116] avg=6144.0000 max=6243 max_rank=4 min=6030 min_rank=2 imbalance_ratio=1.016113 max_load_reduction=42909 max_load_reduction_pct=0.872986 moved_to_echo=24728 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1502, 353, 666, 875, 155, 1200, 999, 549, 166, 0, 0, 0, 0, 0, 0, 0, 1536, 267, 846, 270, 120, 1093, 934, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 657, 901, 917, 729, 198, 446, 205, 1377, 517, 101, 85, 0, 0, 0, 0, 0, 426, 714, 709, 346, 905, 142, 745, 1150, 633, 424, 0, 0, 0, 0, 0, 0, 780, 774, 590, 1229, 706, 199, 625, 907, 0, 0, 0, 0, 0, 0, 0, 0, 426, 577, 593, 494, 982, 1048, 572, 368, 1270, 0, 0, 0, 0, 0, 0, 0, 618, 466, 678, 636, 1011, 1340, 541, 742, 0, 0, 0, 0, 0, 0, 0, 0, 187, 370, 1550, 772, 170, 761, 1394, 880, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6465, 6104, 6133, 6194, 5810, 6330, 6032, 6084] avg=6144.0000 max=6465 max_rank=0 min=5810 min_rank=4 imbalance_ratio=1.052246 max_load_reduction=42687 max_load_reduction_pct=0.868469 moved_to_echo=24256 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3584, 4480, 5120, 4096, 7936, 8064, 3840, 2816, 9728, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[12800, 2560, 6528, 2432, 896, 9472, 7040, 7936, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[11776, 2944, 4992, 7168, 1536, 8448, 7296, 4096, 1408, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6784, 6400, 6272, 10368, 5376, 1792, 5632, 7168, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[5632, 7424, 6912, 5504, 1664, 3712, 1920, 10112, 5376, 896, 768, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3072, 5504, 6144, 3200, 6400, 1280, 5888, 9472, 5504, 3328, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[5248, 4224, 5504, 5120, 7680, 11264, 4352, 6144, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[1536, 2816, 12800, 6272, 1536, 6016, 11136, 7424, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[564, 697, 299, 435, 2185, 88, 308, 1382, 40, 854, 2177, 365, 614, 874, 485, 2260, 843, 1224, 699, 369, 1151, 228, 1856, 1596, 798, 749, 782, 1453, 27, 149, 103, 68, 257, 40, 276, 44, 1122, 526, 800, 778, 1918, 1203, 424, 979, 128, 1318, 7, 142, 2661, 967, 2212, 0, 38, 997, 182, 1224, 1326, 255, 1089, 453, 755, 561, 346, 402] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[544, 688, 296, 403, 2206, 123, 382, 1124, 39, 969, 2261, 379, 628, 976, 486, 2199, 838, 1226, 830, 425, 1113, 222, 1787, 1442, 776, 669, 800, 1420, 21, 151, 161, 158, 337, 46, 291, 64, 1071, 579, 835, 785, 1824, 1028, 471, 854, 155, 1207, 8, 184, 2327, 1059, 2036, 0, 45, 1101, 234, 1534, 1403, 308, 943, 464, 872, 585, 394, 366] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[482, 809, 355, 409, 1673, 200, 363, 1138, 47, 871, 2054, 306, 781, 1021, 663, 2132, 791, 1344, 855, 369, 1460, 168, 1999, 1208, 784, 632, 709, 1339, 15, 120, 156, 139, 312, 48, 340, 72, 1102, 829, 938, 839, 2040, 1182, 642, 1127, 170, 1062, 18, 184, 2017, 986, 1621, 2, 45, 1274, 243, 1586, 1267, 220, 891, 492, 961, 558, 330, 362] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[483, 843, 470, 381, 2036, 171, 359, 1171, 45, 877, 2245, 332, 652, 952, 640, 2312, 751, 1374, 805, 279, 1515, 193, 1966, 1298, 794, 655, 804, 1177, 13, 130, 104, 138, 310, 44, 320, 75, 1154, 827, 812, 859, 1836, 1202, 553, 988, 168, 1068, 11, 173, 2044, 1058, 1882, 1, 34, 1181, 216, 1445, 1160, 271, 896, 465, 958, 526, 324, 326] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[549, 703, 372, 408, 2367, 127, 345, 1132, 28, 922, 2216, 348, 671, 1036, 521, 2313, 745, 1229, 837, 319, 1246, 225, 1947, 1299, 801, 699, 780, 1299, 17, 133, 137, 138, 300, 57, 262, 46, 1030, 614, 813, 869, 1749, 1178, 475, 980, 165, 1206, 15, 217, 2330, 970, 2091, 0, 31, 1182, 216, 1415, 1278, 290, 904, 488, 828, 500, 426, 318] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[612, 840, 349, 426, 1965, 114, 375, 1216, 34, 908, 2203, 334, 658, 1019, 516, 2193, 750, 1345, 882, 278, 1275, 159, 1912, 1397, 765, 653, 763, 1300, 25, 80, 103, 141, 320, 30, 318, 46, 1165, 701, 890, 816, 1931, 1087, 515, 950, 139, 1203, 7, 136, 2310, 1015, 1893, 0, 20, 1191, 257, 1607, 1369, 210, 944, 483, 865, 499, 302, 343] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[518, 677, 345, 410, 2088, 162, 375, 989, 49, 826, 2193, 305, 685, 952, 515, 2213, 759, 1351, 807, 287, 1342, 166, 1937, 1472, 757, 742, 838, 1351, 16, 103, 138, 122, 290, 38, 290, 45, 1220, 609, 896, 836, 1924, 1149, 526, 1116, 181, 1227, 13, 142, 2074, 1068, 1806, 0, 41, 1185, 202, 1485, 1306, 267, 1000, 489, 932, 576, 363, 366] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[534, 693, 424, 462, 2020, 141, 367, 1141, 69, 849, 2095, 307, 713, 1073, 596, 2086, 906, 1327, 754, 422, 1411, 176, 1843, 1331, 758, 716, 799, 1306, 13, 150, 116, 115, 272, 31, 304, 62, 1129, 788, 852, 825, 1888, 1094, 561, 1038, 154, 1093, 8, 190, 1880, 912, 1905, 2, 46, 1178, 266, 1514, 1348, 295, 957, 491, 1010, 525, 383, 438] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[564, 697, 299, 435, 2185, 88, 308, 1382, 362, 0, 0, 0, 0, 0, 0, 0, 40, 854, 2177, 365, 614, 874, 485, 496, 0, 0, 0, 0, 0, 0, 0, 0, 843, 1224, 699, 369, 1151, 228, 5, 1596, 0, 0, 0, 0, 0, 0, 0, 0, 798, 749, 782, 1453, 27, 149, 103, 68, 1851, 313, 0, 0, 0, 0, 0, 0, 257, 40, 276, 44, 1122, 526, 800, 778, 1978, 324, 0, 0, 0, 0, 0, 0, 1918, 1203, 424, 979, 128, 1318, 7, 142, 79, 0, 0, 0, 0, 0, 0, 0, 370, 967, 2212, 0, 38, 997, 182, 1224, 0, 0, 0, 0, 0, 0, 0, 0, 1326, 255, 1089, 453, 755, 561, 346, 402, 999, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6320, 5905, 6115, 6293, 6145, 6198, 5990, 6186] avg=6144.0000 max=6320 max_rank=0 min=5905 min_rank=1 imbalance_ratio=1.028646 max_load_reduction=42832 max_load_reduction_pct=0.871419 moved_to_echo=24519 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[544, 688, 296, 403, 2206, 123, 382, 1124, 352, 0, 0, 0, 0, 0, 0, 0, 39, 969, 2261, 379, 628, 976, 486, 483, 0, 0, 0, 0, 0, 0, 0, 0, 838, 1226, 830, 425, 1113, 222, 5, 1442, 0, 0, 0, 0, 0, 0, 0, 0, 776, 669, 800, 1420, 21, 151, 161, 158, 1782, 273, 0, 0, 0, 0, 0, 0, 337, 46, 291, 64, 1071, 579, 835, 785, 1729, 315, 0, 0, 0, 0, 0, 0, 1824, 1028, 471, 854, 155, 1207, 8, 184, 77, 0, 0, 0, 0, 0, 0, 0, 325, 1059, 2036, 0, 45, 1101, 234, 1534, 0, 0, 0, 0, 0, 0, 0, 0, 1403, 308, 943, 464, 872, 585, 394, 366, 972, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6118, 6221, 6101, 6211, 6052, 5808, 6334, 6307] avg=6144.0000 max=6334 max_rank=6 min=5808 min_rank=5 imbalance_ratio=1.030924 max_load_reduction=42818 max_load_reduction_pct=0.871134 moved_to_echo=24501 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[483, 843, 470, 381, 2036, 171, 359, 1171, 370, 0, 0, 0, 0, 0, 0, 0, 45, 877, 2245, 332, 652, 952, 640, 508, 0, 0, 0, 0, 0, 0, 0, 0, 751, 1374, 805, 279, 1515, 193, 6, 1298, 0, 0, 0, 0, 0, 0, 0, 0, 794, 655, 804, 1177, 13, 130, 104, 138, 1960, 240, 0, 0, 0, 0, 0, 0, 310, 44, 320, 75, 1154, 827, 812, 859, 1519, 331, 0, 0, 0, 0, 0, 0, 1836, 1202, 553, 988, 168, 1068, 11, 173, 81, 0, 0, 0, 0, 0, 0, 0, 285, 1058, 1882, 1, 34, 1181, 216, 1445, 0, 0, 0, 0, 0, 0, 0, 0, 1160, 271, 896, 465, 958, 526, 324, 326, 1022, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6284, 6251, 6221, 6015, 6251, 6080, 6102, 5948] avg=6144.0000 max=6284 max_rank=0 min=5948 min_rank=7 imbalance_ratio=1.022786 max_load_reduction=42868 max_load_reduction_pct=0.872152 moved_to_echo=24381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[482, 809, 355, 409, 1673, 200, 363, 1138, 341, 0, 0, 0, 0, 0, 0, 0, 47, 871, 2054, 306, 781, 1021, 663, 470, 0, 0, 0, 0, 0, 0, 0, 0, 791, 1344, 855, 369, 1460, 168, 6, 1208, 0, 0, 0, 0, 0, 0, 0, 0, 784, 632, 709, 1339, 15, 120, 156, 139, 1993, 237, 0, 0, 0, 0, 0, 0, 312, 48, 340, 72, 1102, 829, 938, 839, 1499, 305, 0, 0, 0, 0, 0, 0, 2040, 1182, 642, 1127, 170, 1062, 18, 184, 74, 0, 0, 0, 0, 0, 0, 0, 281, 986, 1621, 2, 45, 1274, 243, 1586, 0, 0, 0, 0, 0, 0, 0, 0, 1267, 220, 891, 492, 961, 558, 330, 362, 942, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5770, 6213, 6201, 6124, 6284, 6499, 6038, 6023] avg=6144.0000 max=6499 max_rank=5 min=5770 min_rank=0 imbalance_ratio=1.057780 max_load_reduction=42653 max_load_reduction_pct=0.867778 moved_to_echo=24844 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[549, 703, 372, 408, 2367, 127, 345, 1132, 370, 0, 0, 0, 0, 0, 0, 0, 28, 922, 2216, 348, 671, 1036, 521, 509, 0, 0, 0, 0, 0, 0, 0, 0, 745, 1229, 837, 319, 1246, 225, 5, 1299, 0, 0, 0, 0, 0, 0, 0, 0, 801, 699, 780, 1299, 17, 133, 137, 138, 1942, 274, 0, 0, 0, 0, 0, 0, 300, 57, 262, 46, 1030, 614, 813, 869, 1732, 331, 0, 0, 0, 0, 0, 0, 1749, 1178, 475, 980, 165, 1206, 15, 217, 81, 0, 0, 0, 0, 0, 0, 0, 324, 970, 2091, 0, 31, 1182, 216, 1415, 0, 0, 0, 0, 0, 0, 0, 0, 1278, 290, 904, 488, 828, 500, 426, 318, 1022, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6373, 6251, 5905, 6220, 6054, 6066, 6229, 6054] avg=6144.0000 max=6373 max_rank=0 min=5905 min_rank=2 imbalance_ratio=1.037272 max_load_reduction=42779 max_load_reduction_pct=0.870341 moved_to_echo=24403 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[612, 840, 349, 426, 1965, 114, 375, 1216, 351, 0, 0, 0, 0, 0, 0, 0, 34, 908, 2203, 334, 658, 1019, 516, 483, 0, 0, 0, 0, 0, 0, 0, 0, 750, 1345, 882, 278, 1275, 159, 5, 1397, 0, 0, 0, 0, 0, 0, 0, 0, 765, 653, 763, 1300, 25, 80, 103, 141, 1907, 271, 0, 0, 0, 0, 0, 0, 320, 30, 318, 46, 1165, 701, 890, 816, 1717, 314, 0, 0, 0, 0, 0, 0, 1931, 1087, 515, 950, 139, 1203, 7, 136, 76, 0, 0, 0, 0, 0, 0, 0, 322, 1015, 1893, 0, 20, 1191, 257, 1607, 0, 0, 0, 0, 0, 0, 0, 0, 1369, 210, 944, 483, 865, 499, 302, 343, 969, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6248, 6155, 6091, 6008, 6317, 6044, 6305, 5984] avg=6144.0000 max=6317 max_rank=4 min=5984 min_rank=7 imbalance_ratio=1.028158 max_load_reduction=42835 max_load_reduction_pct=0.871480 moved_to_echo=24650 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[518, 677, 345, 410, 2088, 162, 375, 989, 354, 0, 0, 0, 0, 0, 0, 0, 49, 826, 2193, 305, 685, 952, 515, 487, 0, 0, 0, 0, 0, 0, 0, 0, 759, 1351, 807, 287, 1342, 166, 5, 1472, 0, 0, 0, 0, 0, 0, 0, 0, 757, 742, 838, 1351, 16, 103, 138, 122, 1932, 244, 0, 0, 0, 0, 0, 0, 290, 38, 290, 45, 1220, 609, 896, 836, 1541, 317, 0, 0, 0, 0, 0, 0, 1924, 1149, 526, 1116, 181, 1227, 13, 142, 77, 0, 0, 0, 0, 0, 0, 0, 289, 1068, 1806, 0, 41, 1185, 202, 1485, 0, 0, 0, 0, 0, 0, 0, 0, 1306, 267, 1000, 489, 932, 576, 363, 366, 978, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5918, 6012, 6189, 6243, 6082, 6355, 6076, 6277] avg=6144.0000 max=6355 max_rank=5 min=5918 min_rank=0 imbalance_ratio=1.034342 max_load_reduction=42797 max_load_reduction_pct=0.870707 moved_to_echo=24790 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[534, 693, 424, 462, 2020, 141, 367, 1141, 339, 0, 0, 0, 0, 0, 0, 0, 69, 849, 2095, 307, 713, 1073, 596, 442, 0, 0, 0, 0, 0, 0, 0, 0, 906, 1327, 754, 422, 1411, 176, 2, 1331, 0, 0, 0, 0, 0, 0, 0, 0, 758, 716, 799, 1306, 13, 150, 116, 115, 1841, 224, 0, 0, 0, 0, 0, 0, 272, 31, 304, 62, 1129, 788, 852, 825, 1401, 303, 0, 0, 0, 0, 0, 0, 1888, 1094, 561, 1038, 154, 1093, 8, 190, 76, 0, 0, 0, 0, 0, 0, 0, 255, 912, 1905, 2, 46, 1178, 266, 1514, 0, 0, 0, 0, 0, 0, 0, 0, 1348, 295, 957, 491, 1010, 525, 383, 438, 926, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6121, 6144, 6329, 6038, 5967, 6102, 6078, 6373] avg=6144.0000 max=6373 max_rank=7 min=5967 min_rank=4 imbalance_ratio=1.037272 max_load_reduction=42779 max_load_reduction_pct=0.870341 moved_to_echo=24520 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6272, 5632, 6400, 10752, 256, 1024, 1024, 1024, 15232, 2176, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[15232, 9216, 4224, 8064, 1280, 9472, 128, 1408, 640, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4352, 6016, 2944, 3456, 16640, 1152, 2944, 9344, 2944, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[384, 7168, 17536, 2688, 5504, 7936, 4480, 3968, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2432, 384, 2432, 512, 9088, 5504, 6912, 6656, 13184, 2560, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6400, 10496, 6528, 2816, 10624, 1664, 128, 11136, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[10496, 2176, 7680, 3840, 7296, 4352, 2944, 2944, 7936, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2560, 8064, 15488, 128, 384, 9344, 1920, 11904, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 1286, 563, 236, 2, 1074, 703, 194, 378, 436, 1434, 1514, 1251, 541, 4, 189, 1232, 1636, 498, 278, 24, 42, 1880, 866, 763, 622, 1496, 1754, 30, 35, 994, 40, 289, 1403, 1001, 494, 4718, 1915, 1357, 248, 577, 68, 163, 183, 1731, 1564, 117, 492, 500, 7, 402, 774, 5, 3, 121, 274, 2232, 386, 34, 301, 0, 2528, 13, 3257] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 1197, 503, 260, 16, 1112, 636, 210, 355, 493, 1296, 1734, 1252, 606, 1, 225, 1217, 1555, 432, 247, 35, 56, 1809, 737, 635, 664, 1519, 1804, 26, 264, 754, 51, 312, 1617, 978, 559, 3871, 1649, 1593, 278, 612, 76, 214, 144, 2026, 1647, 129, 608, 542, 27, 400, 864, 3, 10, 166, 308, 2337, 496, 39, 331, 3, 2347, 42, 3223] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 1341, 564, 231, 3, 979, 476, 213, 365, 457, 1420, 1734, 1461, 549, 0, 32, 1402, 1838, 607, 239, 32, 0, 1541, 741, 819, 669, 1655, 1819, 22, 13, 790, 27, 295, 1284, 1032, 487, 4587, 2189, 1344, 220, 689, 80, 183, 200, 1610, 1561, 105, 502, 392, 0, 518, 790, 5, 2, 129, 183, 2421, 527, 9, 176, 0, 2114, 8, 3471] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 1206, 527, 238, 2, 1229, 574, 199, 258, 387, 1334, 1941, 1594, 459, 1, 21, 1347, 1621, 713, 190, 16, 0, 1519, 680, 871, 660, 1776, 2388, 19, 6, 799, 13, 300, 1534, 965, 396, 4662, 1946, 1408, 169, 728, 90, 141, 55, 1507, 1396, 118, 370, 426, 1, 337, 765, 0, 1, 182, 210, 2542, 485, 10, 127, 0, 2223, 2, 3468] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 1362, 502, 248, 11, 1047, 579, 181, 260, 402, 1275, 1908, 1290, 542, 0, 121, 1314, 1638, 451, 204, 30, 5, 1747, 608, 665, 728, 1850, 2113, 22, 12, 849, 16, 355, 1401, 927, 504, 4507, 1981, 1276, 189, 556, 84, 195, 95, 1894, 1557, 126, 497, 471, 1, 376, 837, 5, 8, 104, 228, 2332, 495, 17, 193, 0, 2662, 18, 3281] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[1, 1333, 551, 227, 4, 1149, 567, 199, 315, 444, 1453, 1614, 1281, 571, 0, 90, 1274, 1853, 412, 257, 20, 10, 1612, 671, 822, 626, 1581, 1914, 40, 74, 781, 29, 301, 1379, 946, 563, 4275, 2278, 1570, 232, 538, 77, 187, 105, 1645, 1644, 120, 475, 450, 1, 538, 823, 10, 1, 107, 267, 2394, 454, 18, 239, 0, 2626, 12, 3102] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1, 1119, 550, 259, 1, 1163, 506, 229, 315, 416, 1171, 1992, 1185, 504, 0, 38, 1377, 1714, 711, 256, 30, 0, 1574, 677, 869, 775, 1942, 2203, 29, 29, 750, 32, 303, 1475, 965, 486, 4192, 1998, 1417, 194, 770, 95, 175, 165, 1593, 1753, 162, 334, 436, 2, 319, 693, 7, 1, 225, 186, 2492, 562, 25, 196, 0, 2272, 13, 3229] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 1407, 566, 275, 6, 1217, 565, 206, 332, 413, 1325, 1771, 1598, 636, 3, 156, 1271, 1575, 433, 261, 22, 19, 1700, 726, 823, 573, 1589, 1786, 45, 55, 816, 33, 300, 1480, 1035, 662, 3984, 1921, 1629, 243, 702, 95, 156, 108, 1727, 1369, 128, 505, 461, 7, 572, 840, 5, 3, 182, 272, 2274, 438, 15, 311, 0, 2429, 24, 3072] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1286, 563, 236, 2, 1074, 703, 194, 1861, 329, 0, 0, 0, 0, 0, 0, 378, 436, 1434, 1514, 1251, 541, 4, 189, 242, 0, 0, 0, 0, 0, 0, 0, 1232, 1596, 498, 278, 24, 42, 1880, 866, 0, 0, 0, 0, 0, 0, 0, 0, 763, 622, 1496, 1754, 30, 35, 994, 40, 66, 40, 0, 0, 0, 0, 0, 0, 289, 1403, 1001, 494, 0, 1178, 1357, 248, 0, 0, 0, 0, 0, 0, 0, 0, 577, 68, 163, 183, 1731, 1564, 117, 492, 783, 429, 0, 0, 0, 0, 0, 0, 500, 7, 402, 774, 5, 3, 121, 274, 4389, 0, 0, 0, 0, 0, 0, 0, 2232, 386, 34, 301, 0, 2528, 13, 613, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6248, 5989, 6416, 5840, 5970, 6107, 6475, 6107] avg=6144.0000 max=6475 max_rank=6 min=5840 min_rank=3 imbalance_ratio=1.053874 max_load_reduction=42677 max_load_reduction_pct=0.868266 moved_to_echo=24659 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1197, 503, 260, 16, 1112, 636, 210, 1841, 270, 0, 0, 0, 0, 0, 0, 355, 493, 1296, 1734, 1252, 606, 1, 225, 208, 0, 0, 0, 0, 0, 0, 0, 1217, 1517, 432, 247, 35, 56, 1809, 737, 0, 0, 0, 0, 0, 0, 0, 0, 635, 664, 1519, 1804, 26, 264, 754, 51, 57, 38, 0, 0, 0, 0, 0, 0, 312, 1617, 978, 559, 0, 1015, 1593, 278, 0, 0, 0, 0, 0, 0, 0, 0, 612, 76, 214, 144, 2026, 1647, 129, 608, 775, 369, 0, 0, 0, 0, 0, 0, 542, 27, 400, 864, 3, 10, 166, 308, 3601, 0, 0, 0, 0, 0, 0, 0, 2337, 496, 39, 331, 3, 2347, 42, 607, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6045, 6170, 6050, 5812, 6352, 6600, 5921, 6202] avg=6144.0000 max=6600 max_rank=5 min=5812 min_rank=3 imbalance_ratio=1.074219 max_load_reduction=42552 max_load_reduction_pct=0.865723 moved_to_echo=25075 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 1119, 550, 259, 1, 1163, 506, 229, 1845, 292, 0, 0, 0, 0, 0, 0, 315, 416, 1171, 1992, 1185, 504, 0, 38, 252, 0, 0, 0, 0, 0, 0, 0, 1377, 1672, 711, 256, 30, 0, 1574, 677, 0, 0, 0, 0, 0, 0, 0, 0, 869, 775, 1942, 2203, 29, 29, 750, 32, 69, 42, 0, 0, 0, 0, 0, 0, 303, 1475, 965, 486, 0, 1230, 1417, 194, 0, 0, 0, 0, 0, 0, 0, 0, 770, 95, 175, 165, 1593, 1753, 162, 334, 776, 447, 0, 0, 0, 0, 0, 0, 436, 2, 319, 693, 7, 1, 225, 186, 3900, 0, 0, 0, 0, 0, 0, 0, 2492, 562, 25, 196, 0, 2272, 13, 608, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5965, 5873, 6297, 6740, 6070, 6270, 5769, 6168] avg=6144.0000 max=6740 max_rank=3 min=5769 min_rank=6 imbalance_ratio=1.097005 max_load_reduction=42412 max_load_reduction_pct=0.862874 moved_to_echo=24277 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1206, 527, 238, 2, 1229, 574, 199, 1981, 325, 0, 0, 0, 0, 0, 0, 258, 387, 1334, 1941, 1594, 459, 1, 21, 246, 0, 0, 0, 0, 0, 0, 0, 1347, 1582, 713, 190, 16, 0, 1519, 680, 0, 0, 0, 0, 0, 0, 0, 0, 871, 660, 1776, 2388, 19, 6, 799, 13, 67, 39, 0, 0, 0, 0, 0, 0, 300, 1534, 965, 396, 0, 1198, 1408, 169, 0, 0, 0, 0, 0, 0, 0, 0, 728, 90, 141, 55, 1507, 1396, 118, 370, 833, 435, 0, 0, 0, 0, 0, 0, 426, 1, 337, 765, 0, 1, 182, 210, 4337, 0, 0, 0, 0, 0, 0, 0, 2542, 485, 10, 127, 0, 2223, 2, 654, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6281, 6241, 6047, 6638, 5970, 5673, 6259, 6043] avg=6144.0000 max=6638 max_rank=3 min=5673 min_rank=5 imbalance_ratio=1.080404 max_load_reduction=42514 max_load_reduction_pct=0.864950 moved_to_echo=23945 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1362, 502, 248, 11, 1047, 579, 181, 1875, 315, 0, 0, 0, 0, 0, 0, 260, 402, 1275, 1908, 1290, 542, 0, 121, 250, 0, 0, 0, 0, 0, 0, 0, 1314, 1598, 451, 204, 30, 5, 1747, 608, 0, 0, 0, 0, 0, 0, 0, 0, 665, 728, 1850, 2113, 22, 12, 849, 16, 69, 40, 0, 0, 0, 0, 0, 0, 355, 1401, 927, 504, 0, 1219, 1276, 189, 0, 0, 0, 0, 0, 0, 0, 0, 556, 84, 195, 95, 1894, 1557, 126, 497, 788, 443, 0, 0, 0, 0, 0, 0, 471, 1, 376, 837, 5, 8, 104, 228, 4192, 0, 0, 0, 0, 0, 0, 0, 2332, 495, 17, 193, 0, 2662, 18, 618, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6120, 6048, 5957, 6364, 5871, 6235, 6222, 6335] avg=6144.0000 max=6364 max_rank=3 min=5871 min_rank=4 imbalance_ratio=1.035807 max_load_reduction=42788 max_load_reduction_pct=0.870524 moved_to_echo=24663 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1341, 564, 231, 3, 979, 476, 213, 1983, 321, 0, 0, 0, 0, 0, 0, 365, 457, 1420, 1734, 1461, 549, 0, 32, 276, 0, 0, 0, 0, 0, 0, 0, 1402, 1793, 607, 239, 32, 0, 1541, 741, 0, 0, 0, 0, 0, 0, 0, 0, 819, 669, 1655, 1819, 22, 13, 790, 27, 76, 45, 0, 0, 0, 0, 0, 0, 295, 1284, 1032, 487, 0, 1347, 1344, 220, 0, 0, 0, 0, 0, 0, 0, 0, 689, 80, 183, 200, 1610, 1561, 105, 502, 834, 490, 0, 0, 0, 0, 0, 0, 392, 0, 518, 790, 5, 2, 129, 183, 4266, 0, 0, 0, 0, 0, 0, 0, 2421, 527, 9, 176, 0, 2114, 8, 654, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6111, 6294, 6355, 5935, 6009, 6254, 6285, 5909] avg=6144.0000 max=6355 max_rank=2 min=5909 min_rank=7 imbalance_ratio=1.034342 max_load_reduction=42797 max_load_reduction_pct=0.870707 moved_to_echo=24457 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1407, 566, 275, 6, 1217, 565, 206, 1760, 279, 0, 0, 0, 0, 0, 0, 332, 413, 1325, 1771, 1598, 636, 3, 156, 247, 0, 0, 0, 0, 0, 0, 0, 1271, 1534, 433, 261, 22, 19, 1700, 726, 0, 0, 0, 0, 0, 0, 0, 0, 823, 573, 1589, 1786, 45, 55, 816, 33, 71, 41, 0, 0, 0, 0, 0, 0, 300, 1480, 1035, 662, 0, 1169, 1629, 243, 0, 0, 0, 0, 0, 0, 0, 0, 702, 95, 156, 108, 1727, 1369, 128, 505, 743, 434, 0, 0, 0, 0, 0, 0, 461, 7, 572, 840, 5, 3, 182, 272, 3705, 0, 0, 0, 0, 0, 0, 0, 2274, 438, 15, 311, 0, 2429, 24, 569, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6281, 6481, 5966, 5832, 6518, 5967, 6047, 6060] avg=6144.0000 max=6518 max_rank=4 min=5832 min_rank=3 imbalance_ratio=1.060872 max_load_reduction=42634 max_load_reduction_pct=0.867391 moved_to_echo=24592 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 1333, 551, 227, 4, 1149, 567, 199, 1772, 298, 0, 0, 0, 0, 0, 0, 315, 444, 1453, 1614, 1281, 571, 0, 90, 288, 0, 0, 0, 0, 0, 0, 0, 1274, 1808, 412, 257, 20, 10, 1612, 671, 0, 0, 0, 0, 0, 0, 0, 0, 822, 626, 1581, 1914, 40, 74, 781, 29, 79, 45, 0, 0, 0, 0, 0, 0, 301, 1379, 946, 563, 0, 1401, 1570, 232, 0, 0, 0, 0, 0, 0, 0, 0, 538, 77, 187, 105, 1645, 1644, 120, 475, 745, 510, 0, 0, 0, 0, 0, 0, 450, 1, 538, 823, 10, 1, 107, 267, 3977, 0, 0, 0, 0, 0, 0, 0, 2394, 454, 18, 239, 0, 2626, 12, 585, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6101, 6056, 6064, 5991, 6392, 6046, 6174, 6328] avg=6144.0000 max=6392 max_rank=4 min=5991 min_rank=3 imbalance_ratio=1.040365 max_load_reduction=42760 max_load_reduction_pct=0.869954 moved_to_echo=24940 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[5248, 768, 1536, 1152, 13824, 12544, 1024, 3840, 6400, 3584, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2688, 3456, 10752, 14208, 11008, 4480, 128, 896, 2048, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2560, 11648, 7936, 4224, 0, 9856, 11648, 1792, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 10368, 4352, 2048, 128, 9088, 4608, 1664, 14976, 2432, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[10496, 13184, 4352, 2048, 256, 256, 13440, 5760, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[19072, 3968, 256, 1920, 128, 19328, 256, 4992, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6272, 5376, 13440, 15872, 256, 512, 6656, 256, 640, 384, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3712, 128, 3584, 6400, 128, 128, 1280, 2048, 32384, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[149, 74, 1595, 1, 438, 628, 568, 597, 2734, 2659, 221, 264, 849, 342, 4868, 184, 2817, 23, 2432, 359, 509, 304, 0, 324, 906, 1327, 530, 349, 522, 1819, 1036, 374, 101, 200, 67, 178, 428, 225, 335, 155, 23, 3777, 23, 454, 195, 81, 0, 4796, 0, 95, 63, 945, 116, 2164, 0, 48, 61, 2, 1437, 12, 1930, 1076, 96, 267] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[113, 82, 1391, 5, 505, 935, 438, 683, 3445, 2654, 313, 262, 828, 316, 5169, 144, 2878, 24, 1818, 352, 435, 240, 0, 313, 986, 1379, 507, 328, 458, 1717, 829, 501, 134, 147, 91, 160, 287, 184, 219, 58, 25, 3645, 47, 452, 158, 82, 0, 4617, 0, 114, 89, 973, 65, 1977, 0, 161, 81, 3, 1479, 40, 1970, 1418, 87, 341] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[156, 46, 1491, 3, 591, 876, 537, 736, 3056, 2483, 318, 283, 838, 341, 4894, 134, 3011, 17, 2210, 329, 367, 215, 0, 317, 630, 1505, 456, 351, 387, 1713, 899, 525, 308, 166, 100, 177, 343, 206, 197, 93, 19, 3405, 68, 400, 222, 87, 0, 4373, 0, 105, 94, 1129, 79, 1962, 0, 174, 91, 0, 1574, 308, 2062, 1325, 57, 313] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[143, 83, 1543, 7, 437, 449, 583, 613, 2429, 2604, 206, 290, 991, 361, 4561, 128, 2596, 32, 2614, 379, 471, 296, 0, 447, 731, 1388, 556, 393, 531, 1968, 923, 439, 296, 261, 71, 158, 517, 232, 390, 111, 53, 3211, 68, 476, 248, 120, 0, 4844, 0, 145, 74, 1049, 157, 2016, 78, 78, 48, 9, 1512, 305, 1905, 1066, 149, 313] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[135, 71, 1487, 3, 517, 779, 488, 697, 3258, 2621, 232, 252, 892, 354, 4879, 138, 2659, 28, 2037, 341, 434, 288, 0, 283, 1033, 1408, 590, 355, 412, 1956, 691, 394, 54, 123, 72, 149, 417, 143, 254, 41, 33, 3865, 91, 513, 191, 76, 0, 4828, 0, 95, 71, 1101, 57, 2008, 0, 81, 147, 1, 1308, 36, 2030, 1285, 77, 293] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[118, 86, 1634, 6, 527, 813, 537, 690, 2893, 2259, 247, 283, 943, 358, 4630, 149, 2506, 42, 2415, 345, 401, 272, 0, 385, 851, 1452, 498, 366, 519, 1733, 820, 487, 154, 225, 102, 163, 499, 211, 356, 111, 38, 3599, 69, 431, 193, 79, 1, 4838, 0, 131, 93, 1106, 109, 2031, 16, 140, 77, 1, 1399, 241, 1894, 1179, 71, 330] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[148, 81, 1536, 2, 415, 614, 549, 638, 2627, 2487, 232, 275, 940, 354, 4862, 170, 2705, 43, 2323, 345, 484, 263, 0, 326, 975, 1390, 523, 373, 462, 1784, 879, 398, 191, 180, 81, 177, 421, 218, 235, 90, 22, 3628, 58, 457, 173, 110, 0, 5070, 0, 131, 78, 1070, 87, 2214, 0, 65, 60, 2, 1515, 61, 2021, 1192, 71, 271] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[128, 99, 1424, 1, 505, 705, 540, 679, 2975, 2457, 217, 284, 960, 369, 4791, 101, 2549, 18, 2432, 408, 470, 246, 0, 402, 827, 1299, 494, 380, 447, 2107, 805, 421, 311, 187, 84, 172, 423, 213, 264, 52, 19, 3285, 41, 385, 234, 58, 0, 4798, 0, 143, 81, 1169, 112, 1985, 0, 72, 27, 2, 1537, 48, 2276, 1157, 113, 364] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[149, 74, 1595, 1, 438, 628, 568, 597, 1150, 692, 78, 0, 0, 0, 0, 0, 1584, 2659, 221, 264, 849, 342, 0, 184, 0, 0, 0, 0, 0, 0, 0, 0, 2503, 23, 2432, 359, 509, 304, 0, 324, 0, 0, 0, 0, 0, 0, 0, 0, 906, 1327, 530, 349, 522, 1213, 1036, 374, 0, 0, 0, 0, 0, 0, 0, 0, 101, 200, 67, 178, 428, 225, 335, 155, 4556, 0, 0, 0, 0, 0, 0, 0, 23, 3777, 23, 454, 195, 81, 0, 1781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 63, 945, 116, 2164, 0, 48, 2323, 312, 0, 0, 0, 0, 0, 0, 61, 2, 1437, 12, 1930, 1076, 96, 267, 528, 314, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5970, 6103, 6454, 6257, 6245, 6334, 6066, 5723] avg=6144.0000 max=6454 max_rank=2 min=5723 min_rank=7 imbalance_ratio=1.050456 max_load_reduction=42698 max_load_reduction_pct=0.868693 moved_to_echo=24368 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[143, 83, 1543, 7, 437, 449, 583, 613, 1021, 699, 84, 0, 0, 0, 0, 0, 1408, 2604, 206, 290, 991, 361, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 2306, 32, 2614, 379, 471, 296, 0, 447, 0, 0, 0, 0, 0, 0, 0, 0, 731, 1388, 556, 393, 531, 1312, 923, 439, 0, 0, 0, 0, 0, 0, 0, 0, 296, 261, 71, 158, 517, 232, 390, 111, 4270, 0, 0, 0, 0, 0, 0, 0, 53, 3211, 68, 476, 248, 120, 0, 1799, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 74, 1049, 157, 2016, 78, 78, 2346, 291, 0, 0, 0, 0, 0, 0, 48, 9, 1512, 305, 1905, 1066, 149, 313, 572, 290, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5662, 5988, 6545, 6273, 6306, 5975, 6234, 6169] avg=6144.0000 max=6545 max_rank=2 min=5662 min_rank=0 imbalance_ratio=1.065267 max_load_reduction=42607 max_load_reduction_pct=0.866842 moved_to_echo=24684 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[135, 71, 1487, 3, 517, 779, 488, 697, 1370, 697, 84, 0, 0, 0, 0, 0, 1888, 2621, 232, 252, 892, 354, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 2362, 28, 2037, 341, 434, 288, 0, 283, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1408, 590, 355, 412, 1304, 691, 394, 0, 0, 0, 0, 0, 0, 0, 0, 54, 123, 72, 149, 417, 143, 254, 41, 4568, 0, 0, 0, 0, 0, 0, 0, 33, 3865, 91, 513, 191, 76, 0, 1793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 71, 1101, 57, 2008, 0, 81, 2338, 311, 0, 0, 0, 0, 0, 0, 147, 1, 1308, 36, 2030, 1285, 77, 293, 568, 297, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6328, 6377, 5773, 6187, 5821, 6562, 6062, 6042] avg=6144.0000 max=6562 max_rank=5 min=5773 min_rank=2 imbalance_ratio=1.068034 max_load_reduction=42590 max_load_reduction_pct=0.866496 moved_to_echo=24487 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[156, 46, 1491, 3, 591, 876, 537, 736, 1285, 631, 73, 0, 0, 0, 0, 0, 1771, 2483, 318, 283, 838, 341, 0, 134, 0, 0, 0, 0, 0, 0, 0, 0, 2675, 17, 2210, 329, 367, 215, 0, 317, 0, 0, 0, 0, 0, 0, 0, 0, 630, 1505, 456, 351, 387, 1142, 899, 525, 0, 0, 0, 0, 0, 0, 0, 0, 308, 166, 100, 177, 343, 206, 197, 93, 4581, 0, 0, 0, 0, 0, 0, 0, 19, 3405, 68, 400, 222, 87, 0, 1624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 94, 1129, 79, 1962, 0, 174, 2118, 313, 0, 0, 0, 0, 0, 0, 91, 0, 1574, 308, 2062, 1325, 57, 313, 498, 336, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6425, 6168, 6130, 5895, 6171, 5825, 5974, 6564] avg=6144.0000 max=6564 max_rank=7 min=5825 min_rank=5 imbalance_ratio=1.068359 max_load_reduction=42588 max_load_reduction_pct=0.866455 moved_to_echo=24534 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[128, 99, 1424, 1, 505, 705, 540, 679, 1251, 693, 90, 0, 0, 0, 0, 0, 1724, 2457, 217, 284, 960, 369, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 2265, 18, 2432, 408, 470, 246, 0, 402, 0, 0, 0, 0, 0, 0, 0, 0, 827, 1299, 494, 380, 447, 1405, 805, 421, 0, 0, 0, 0, 0, 0, 0, 0, 311, 187, 84, 172, 423, 213, 264, 52, 4485, 0, 0, 0, 0, 0, 0, 0, 19, 3285, 41, 385, 234, 58, 0, 1781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 81, 1169, 112, 1985, 0, 72, 2324, 306, 0, 0, 0, 0, 0, 0, 27, 2, 1537, 48, 2276, 1157, 113, 364, 612, 284, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6115, 6112, 6241, 6078, 6191, 5803, 6192, 6420] avg=6144.0000 max=6420 max_rank=7 min=5803 min_rank=5 imbalance_ratio=1.044922 max_load_reduction=42732 max_load_reduction_pct=0.869385 moved_to_echo=24606 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[113, 82, 1391, 5, 505, 935, 438, 683, 1449, 667, 74, 0, 0, 0, 0, 0, 1996, 2654, 313, 262, 828, 316, 0, 144, 0, 0, 0, 0, 0, 0, 0, 0, 2557, 24, 1818, 352, 435, 240, 0, 313, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1379, 507, 328, 458, 1144, 829, 501, 0, 0, 0, 0, 0, 0, 0, 0, 134, 147, 91, 160, 287, 184, 219, 58, 4838, 0, 0, 0, 0, 0, 0, 0, 25, 3645, 47, 452, 158, 82, 0, 1714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 89, 973, 65, 1977, 0, 161, 2236, 331, 0, 0, 0, 0, 0, 0, 81, 3, 1479, 40, 1970, 1418, 87, 341, 499, 321, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6342, 6513, 5739, 6132, 6118, 6123, 5946, 6239] avg=6144.0000 max=6513 max_rank=1 min=5739 min_rank=2 imbalance_ratio=1.060059 max_load_reduction=42639 max_load_reduction_pct=0.867493 moved_to_echo=24426 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[118, 86, 1634, 6, 527, 813, 537, 690, 1221, 703, 79, 0, 0, 0, 0, 0, 1672, 2259, 247, 283, 943, 358, 0, 149, 0, 0, 0, 0, 0, 0, 0, 0, 2223, 42, 2415, 345, 401, 272, 0, 385, 0, 0, 0, 0, 0, 0, 0, 0, 851, 1452, 498, 366, 519, 1147, 820, 487, 0, 0, 0, 0, 0, 0, 0, 0, 154, 225, 102, 163, 499, 211, 356, 111, 4335, 0, 0, 0, 0, 0, 0, 0, 38, 3599, 69, 431, 193, 79, 1, 1789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 93, 1106, 109, 2031, 16, 140, 2346, 295, 0, 0, 0, 0, 0, 0, 77, 1, 1399, 241, 1894, 1179, 71, 330, 507, 283, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6414, 5911, 6083, 6140, 6156, 6199, 6267, 5982] avg=6144.0000 max=6414 max_rank=0 min=5911 min_rank=1 imbalance_ratio=1.043945 max_load_reduction=42738 max_load_reduction_pct=0.869507 moved_to_echo=24604 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[148, 81, 1536, 2, 415, 614, 549, 638, 1105, 732, 76, 0, 0, 0, 0, 0, 1522, 2487, 232, 275, 940, 354, 0, 170, 0, 0, 0, 0, 0, 0, 0, 0, 2403, 43, 2323, 345, 484, 263, 0, 326, 0, 0, 0, 0, 0, 0, 0, 0, 975, 1390, 523, 373, 462, 1190, 879, 398, 0, 0, 0, 0, 0, 0, 0, 0, 191, 180, 81, 177, 421, 218, 235, 90, 4551, 0, 0, 0, 0, 0, 0, 0, 22, 3628, 58, 457, 173, 110, 0, 1883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 78, 1070, 87, 2214, 0, 65, 2455, 311, 0, 0, 0, 0, 0, 0, 60, 2, 1515, 61, 2021, 1192, 71, 271, 518, 302, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5896, 5980, 6187, 6190, 6144, 6331, 6411, 6013] avg=6144.0000 max=6411 max_rank=6 min=5896 min_rank=0 imbalance_ratio=1.043457 max_load_reduction=42741 max_load_reduction_pct=0.869568 moved_to_echo=24899 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1152, 640, 12160, 128, 3968, 5888, 4352, 5376, 9856, 5632, 640, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[19328, 256, 18304, 2944, 3584, 2176, 0, 2816, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[640, 128, 11776, 1152, 16128, 9728, 768, 2560, 4352, 2432, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[7040, 11264, 4224, 2944, 3840, 9984, 6912, 3584, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[256, 28416, 512, 3584, 1664, 768, 128, 14208, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[13568, 20224, 2048, 2304, 7296, 2816, 0, 1152, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 1024, 768, 8576, 896, 16384, 128, 896, 18560, 2560, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1664, 1536, 768, 1408, 3456, 1664, 2304, 768, 36224, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 1090, 1279, 1063, 913, 2518, 285, 1766, 17, 1254, 1997, 65, 132, 236, 254, 37, 1200, 0, 0, 35, 55, 1070, 563, 9, 1120, 0, 4251, 91, 3672, 118, 220, 0, 136, 0, 0, 615, 206, 0, 13, 770, 103, 755, 0, 1090, 148, 0, 4195, 0, 0, 977, 113, 343, 403, 4943, 4407, 744, 521, 156, 10, 1670, 887, 306, 34, 297] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 731, 1220, 1358, 869, 2315, 254, 1824, 17, 1263, 1583, 79, 152, 371, 159, 23, 1455, 0, 0, 14, 62, 911, 647, 6, 979, 0, 4565, 79, 4124, 103, 246, 0, 138, 0, 0, 692, 147, 0, 6, 695, 125, 883, 1, 1058, 140, 0, 4312, 0, 0, 1115, 70, 265, 397, 4932, 4706, 729, 392, 178, 10, 1608, 753, 207, 15, 169] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 488, 1179, 1331, 815, 2221, 254, 1802, 7, 1261, 1710, 341, 99, 149, 28, 13, 1073, 0, 0, 3, 32, 1240, 717, 2, 1106, 0, 4049, 103, 4381, 121, 117, 0, 166, 0, 0, 872, 289, 0, 6, 812, 87, 755, 0, 1242, 147, 0, 4232, 0, 0, 942, 104, 375, 113, 5739, 4833, 664, 385, 88, 7, 1496, 671, 256, 3, 226] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 440, 1206, 1042, 697, 2495, 247, 1768, 1, 1565, 1757, 81, 153, 233, 20, 10, 1031, 6, 0, 13, 53, 1158, 828, 1, 867, 0, 3724, 96, 4483, 108, 162, 0, 150, 0, 0, 861, 267, 0, 4, 815, 61, 757, 0, 1197, 161, 0, 4472, 0, 0, 1009, 96, 374, 349, 5947, 4883, 606, 368, 115, 6, 1314, 650, 238, 3, 204] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 477, 1278, 975, 827, 2340, 239, 1772, 1, 1511, 1754, 170, 136, 286, 5, 18, 1293, 0, 0, 2, 23, 1173, 806, 4, 1045, 0, 4296, 83, 4328, 93, 142, 0, 163, 0, 0, 796, 175, 0, 1, 746, 85, 790, 0, 1159, 147, 0, 4341, 0, 0, 1133, 81, 231, 331, 5253, 5008, 657, 320, 135, 6, 1458, 632, 188, 6, 233] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 595, 1294, 1012, 836, 2384, 264, 1713, 5, 1467, 1770, 204, 203, 153, 55, 20, 1255, 2, 0, 3, 24, 1109, 737, 0, 998, 0, 4284, 95, 4545, 108, 196, 0, 126, 0, 0, 747, 190, 0, 7, 631, 53, 842, 0, 1106, 184, 0, 4184, 0, 0, 1007, 82, 342, 290, 5344, 4848, 669, 433, 147, 3, 1555, 677, 193, 6, 155] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 661, 1208, 1246, 882, 2183, 252, 1679, 32, 1550, 1715, 63, 123, 248, 248, 17, 1468, 0, 1, 43, 60, 1202, 694, 5, 1244, 0, 3748, 113, 3803, 117, 185, 1, 164, 0, 2, 820, 307, 0, 13, 791, 99, 866, 4, 1133, 149, 0, 4234, 1, 0, 1050, 117, 417, 407, 5158, 4513, 691, 492, 168, 17, 1329, 732, 383, 10, 294] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 530, 1231, 1030, 925, 2620, 248, 1791, 10, 1556, 1715, 212, 132, 302, 37, 14, 1374, 0, 0, 7, 49, 1237, 797, 3, 1037, 0, 4015, 89, 4186, 129, 205, 0, 144, 0, 0, 643, 187, 0, 3, 695, 75, 784, 0, 1138, 146, 0, 4117, 0, 0, 1005, 91, 233, 265, 5432, 4797, 715, 480, 142, 7, 1390, 716, 193, 6, 267] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 488, 1179, 1331, 815, 265, 254, 1802, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1261, 1710, 341, 99, 149, 28, 13, 1074, 873, 298, 0, 0, 0, 0, 0, 1073, 0, 0, 3, 32, 1240, 717, 2, 1959, 1155, 0, 0, 0, 0, 0, 0, 1106, 0, 4049, 103, 615, 121, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 166, 0, 0, 872, 289, 0, 6, 812, 4584, 0, 0, 0, 0, 0, 0, 0, 87, 755, 0, 1242, 147, 0, 3934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 942, 104, 375, 113, 0, 3759, 664, 0, 0, 0, 0, 0, 0, 0, 0, 385, 88, 7, 1496, 671, 256, 3, 226, 1807, 1083, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6134, 5853, 6181, 6111, 6729, 6165, 5957, 6022] avg=6144.0000 max=6729 max_rank=4 min=5853 min_rank=1 imbalance_ratio=1.095215 max_load_reduction=42423 max_load_reduction_pct=0.863098 moved_to_echo=24873 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1090, 1279, 1063, 913, 300, 285, 1766, 0, 0, 0, 0, 0, 0, 0, 0, 17, 1254, 1997, 65, 132, 236, 254, 37, 979, 990, 296, 0, 0, 0, 0, 0, 1200, 0, 0, 35, 55, 1070, 563, 9, 1642, 996, 0, 0, 0, 0, 0, 0, 1120, 0, 4251, 91, 516, 118, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 0, 0, 615, 206, 0, 13, 770, 3947, 0, 0, 0, 0, 0, 0, 0, 103, 755, 0, 1090, 148, 0, 3899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 977, 113, 343, 403, 0, 3428, 744, 0, 0, 0, 0, 0, 0, 0, 0, 521, 156, 10, 1670, 887, 306, 34, 297, 1514, 1228, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6696, 6257, 5570, 6316, 5687, 5995, 6008, 6623] avg=6144.0000 max=6696 max_rank=0 min=5570 min_rank=2 imbalance_ratio=1.089844 max_load_reduction=42456 max_load_reduction_pct=0.863770 moved_to_echo=24313 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 731, 1220, 1358, 869, 276, 254, 1824, 0, 0, 0, 0, 0, 0, 0, 0, 17, 1263, 1583, 79, 152, 371, 159, 23, 1045, 910, 304, 0, 0, 0, 0, 0, 1455, 0, 0, 14, 62, 911, 647, 6, 1844, 993, 0, 0, 0, 0, 0, 0, 979, 0, 4565, 79, 579, 103, 246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 0, 0, 692, 147, 0, 6, 695, 3939, 0, 0, 0, 0, 0, 0, 0, 125, 883, 1, 1058, 140, 0, 4008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1115, 70, 265, 397, 0, 3661, 729, 0, 0, 0, 0, 0, 0, 0, 0, 392, 178, 10, 1608, 753, 207, 15, 169, 1701, 1129, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6532, 5906, 5932, 6551, 5617, 6215, 6237, 6162] avg=6144.0000 max=6551 max_rank=3 min=5617 min_rank=4 imbalance_ratio=1.066243 max_load_reduction=42601 max_load_reduction_pct=0.866720 moved_to_echo=24231 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 440, 1206, 1042, 697, 297, 247, 1768, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1565, 1757, 81, 153, 233, 20, 10, 1085, 981, 315, 0, 0, 0, 0, 0, 1031, 6, 0, 13, 53, 1158, 828, 1, 2004, 1197, 0, 0, 0, 0, 0, 0, 867, 0, 3724, 96, 630, 108, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 0, 0, 861, 267, 0, 4, 815, 4750, 0, 0, 0, 0, 0, 0, 0, 61, 757, 0, 1197, 161, 0, 4157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 96, 374, 349, 0, 3798, 606, 0, 0, 0, 0, 0, 0, 0, 0, 368, 115, 6, 1314, 650, 238, 3, 204, 1849, 1217, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5697, 6201, 6291, 5587, 6847, 6333, 6232, 5964] avg=6144.0000 max=6847 max_rank=4 min=5587 min_rank=3 imbalance_ratio=1.114421 max_load_reduction=42305 max_load_reduction_pct=0.860697 moved_to_echo=25376 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 477, 1278, 975, 827, 279, 239, 1772, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1511, 1754, 170, 136, 286, 5, 18, 1113, 920, 306, 0, 0, 0, 0, 0, 1293, 0, 0, 2, 23, 1173, 806, 4, 1935, 1058, 0, 0, 0, 0, 0, 0, 1045, 0, 4296, 83, 608, 93, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 163, 0, 0, 796, 175, 0, 1, 746, 4195, 0, 0, 0, 0, 0, 0, 0, 85, 790, 0, 1159, 147, 0, 4035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1133, 81, 231, 331, 0, 3895, 657, 0, 0, 0, 0, 0, 0, 0, 0, 320, 135, 6, 1458, 632, 188, 6, 233, 1785, 1141, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5847, 6220, 6294, 6267, 6076, 6216, 6328, 5904] avg=6144.0000 max=6328 max_rank=6 min=5847 min_rank=0 imbalance_ratio=1.029948 max_load_reduction=42824 max_load_reduction_pct=0.871257 moved_to_echo=24524 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 595, 1294, 1012, 836, 284, 264, 1713, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1467, 1770, 204, 203, 153, 55, 20, 1077, 937, 295, 0, 0, 0, 0, 0, 1255, 2, 0, 3, 24, 1109, 737, 0, 2032, 1076, 0, 0, 0, 0, 0, 0, 998, 0, 4284, 95, 639, 108, 196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 0, 0, 747, 190, 0, 7, 631, 4268, 0, 0, 0, 0, 0, 0, 0, 53, 842, 0, 1106, 184, 0, 3889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 82, 342, 290, 0, 3771, 669, 0, 0, 0, 0, 0, 0, 0, 0, 433, 147, 3, 1555, 677, 193, 6, 155, 1874, 1163, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5998, 6186, 6238, 6320, 5969, 6074, 6161, 6206] avg=6144.0000 max=6320 max_rank=3 min=5969 min_rank=4 imbalance_ratio=1.028646 max_load_reduction=42832 max_load_reduction_pct=0.871419 moved_to_echo=24410 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 661, 1208, 1246, 882, 253, 252, 1679, 0, 0, 0, 0, 0, 0, 0, 0, 32, 1550, 1715, 63, 123, 248, 248, 17, 1006, 861, 302, 0, 0, 0, 0, 0, 1468, 0, 1, 43, 60, 1202, 694, 5, 1703, 1039, 0, 0, 0, 0, 0, 0, 1244, 0, 3748, 113, 528, 117, 185, 1, 0, 0, 0, 0, 0, 0, 0, 0, 164, 0, 2, 820, 307, 0, 13, 791, 4119, 0, 0, 0, 0, 0, 0, 0, 99, 866, 4, 1133, 149, 0, 3932, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 117, 417, 407, 0, 3507, 691, 0, 0, 0, 0, 0, 0, 0, 0, 492, 168, 17, 1329, 732, 383, 10, 294, 1572, 1069, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6181, 6165, 6215, 5936, 6216, 6184, 6189, 6066] avg=6144.0000 max=6216 max_rank=4 min=5936 min_rank=3 imbalance_ratio=1.011719 max_load_reduction=42936 max_load_reduction_pct=0.873535 moved_to_echo=24655 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 530, 1231, 1030, 925, 312, 248, 1791, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1556, 1715, 212, 132, 302, 37, 14, 1066, 1030, 290, 0, 0, 0, 0, 0, 1374, 0, 0, 7, 49, 1237, 797, 3, 1871, 1093, 0, 0, 0, 0, 0, 0, 1037, 0, 4015, 89, 589, 129, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 0, 643, 187, 0, 3, 695, 4339, 0, 0, 0, 0, 0, 0, 0, 75, 784, 0, 1138, 146, 0, 3827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 91, 233, 265, 0, 3731, 715, 0, 0, 0, 0, 0, 0, 0, 0, 480, 142, 7, 1390, 716, 193, 6, 267, 1726, 1278, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6067, 6364, 6431, 6064, 6011, 5970, 6040, 6205] avg=6144.0000 max=6431 max_rank=2 min=5970 min_rank=5 imbalance_ratio=1.046712 max_load_reduction=42721 max_load_reduction_pct=0.869161 moved_to_echo=24226 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 5120, 9984, 9088, 6784, 2304, 2048, 14208, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[10240, 128, 128, 128, 384, 9216, 5888, 128, 15104, 8704, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3456, 1152, 128, 11904, 5760, 2048, 128, 1920, 13952, 9344, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 11520, 14080, 1280, 1152, 2048, 896, 256, 8448, 7552, 2432, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[8448, 0, 33024, 768, 4736, 1024, 1536, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 8320, 768, 2688, 2560, 0, 29568, 5504, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[768, 6528, 128, 9216, 1280, 0, 31744, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1280, 0, 128, 6144, 1792, 0, 128, 6016, 34176, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[133, 1525, 400, 1450, 75, 161, 1406, 407, 0, 0, 5, 538, 3157, 0, 0, 743, 337, 715, 6474, 0, 0, 674, 1236, 0, 0, 4814, 1237, 0, 0, 0, 2483, 0, 4528, 386, 16, 1, 117, 0, 400, 74, 1, 0, 0, 365, 46, 170, 3211, 1280, 470, 1321, 1331, 1229, 0, 0, 0, 911, 650, 237, 73, 20, 76, 139, 163, 3967] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[285, 1474, 161, 1659, 72, 107, 1178, 907, 0, 0, 7, 482, 3263, 0, 0, 818, 337, 678, 6348, 0, 0, 791, 1538, 0, 0, 4262, 1310, 0, 1, 0, 2272, 0, 4451, 329, 13, 0, 76, 0, 347, 121, 0, 0, 0, 417, 42, 117, 3084, 1219, 619, 1254, 1217, 1345, 0, 0, 0, 912, 719, 194, 122, 11, 92, 200, 195, 4106] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[264, 1355, 287, 1807, 60, 115, 1344, 654, 0, 0, 12, 576, 3299, 0, 0, 811, 304, 735, 6406, 0, 1, 698, 1409, 0, 0, 4169, 1374, 0, 1, 0, 2800, 0, 4163, 402, 2, 0, 90, 0, 310, 145, 0, 0, 0, 435, 52, 58, 3001, 1355, 539, 1238, 1094, 1376, 0, 0, 0, 902, 626, 234, 169, 8, 72, 184, 155, 4061] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[192, 1761, 412, 1418, 66, 160, 1644, 417, 0, 0, 29, 605, 2900, 0, 0, 581, 345, 755, 6333, 1, 0, 569, 1114, 0, 0, 4554, 1212, 0, 1, 0, 2640, 0, 4512, 414, 31, 0, 119, 0, 419, 95, 2, 0, 0, 361, 29, 163, 3028, 1111, 610, 1526, 1286, 1350, 0, 7, 0, 957, 679, 229, 305, 41, 87, 166, 184, 3732] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[166, 1611, 368, 1409, 92, 102, 1169, 514, 0, 0, 6, 588, 3417, 0, 0, 765, 349, 758, 6262, 0, 0, 742, 1525, 0, 0, 4378, 1235, 0, 0, 0, 2598, 0, 4434, 431, 29, 0, 101, 0, 383, 113, 0, 0, 0, 351, 49, 117, 2969, 1259, 546, 1391, 1142, 1270, 0, 0, 0, 861, 776, 228, 126, 8, 109, 156, 197, 4052] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[181, 1569, 428, 1610, 44, 248, 1347, 389, 0, 0, 11, 582, 3053, 0, 0, 719, 362, 732, 6424, 1, 0, 776, 1435, 0, 0, 4158, 1383, 0, 0, 0, 2440, 0, 4449, 346, 32, 0, 135, 0, 328, 103, 1, 0, 0, 360, 40, 125, 3217, 1196, 595, 1384, 1111, 1185, 0, 0, 0, 965, 799, 224, 107, 16, 62, 171, 173, 4136] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[268, 1759, 439, 1694, 69, 201, 1439, 510, 1, 1, 4, 545, 2913, 0, 1, 680, 347, 701, 6189, 2, 0, 639, 1327, 0, 1, 4628, 1164, 0, 1, 0, 2558, 0, 4167, 374, 32, 4, 100, 1, 457, 120, 0, 2, 0, 407, 44, 138, 3269, 1069, 610, 1401, 965, 1173, 0, 0, 0, 949, 774, 233, 270, 33, 81, 213, 184, 4001] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[160, 1657, 415, 1349, 69, 125, 1397, 467, 0, 0, 8, 574, 3073, 0, 0, 705, 348, 709, 6380, 0, 0, 637, 1223, 0, 0, 4451, 1122, 0, 0, 0, 2664, 0, 4585, 403, 1, 1, 159, 0, 458, 93, 0, 0, 0, 414, 42, 98, 3128, 1212, 615, 1582, 1034, 1240, 0, 2, 0, 911, 747, 258, 82, 24, 96, 164, 191, 4079] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[285, 1474, 161, 1659, 72, 107, 1178, 907, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 482, 3263, 0, 0, 818, 1708, 0, 0, 0, 0, 0, 0, 0, 337, 678, 3038, 0, 0, 791, 1538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2247, 1310, 0, 1, 0, 2272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4451, 329, 13, 0, 76, 0, 347, 121, 692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 417, 42, 117, 3084, 1219, 1261, 0, 0, 0, 0, 0, 0, 0, 619, 1254, 1217, 1345, 0, 0, 0, 912, 480, 341, 0, 0, 0, 0, 0, 0, 719, 194, 122, 11, 92, 200, 195, 4106, 539, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6147, 6278, 6382, 5830, 6029, 6140, 6168, 6178] avg=6144.0000 max=6382 max_rank=2 min=5830 min_rank=3 imbalance_ratio=1.038737 max_load_reduction=42770 max_load_reduction_pct=0.870158 moved_to_echo=24515 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[133, 1525, 400, 1450, 75, 161, 1406, 407, 344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 538, 3157, 0, 0, 743, 1742, 0, 0, 0, 0, 0, 0, 0, 337, 715, 3098, 0, 0, 674, 1236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2538, 1237, 0, 0, 0, 2483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4528, 386, 16, 1, 117, 0, 400, 74, 781, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 365, 46, 170, 3211, 1280, 1286, 0, 0, 0, 0, 0, 0, 0, 470, 1321, 1331, 1229, 0, 0, 0, 911, 542, 348, 0, 0, 0, 0, 0, 0, 650, 237, 73, 20, 76, 139, 163, 3967, 609, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5901, 6185, 6060, 6258, 6303, 6359, 6152, 5934] avg=6144.0000 max=6359 max_rank=5 min=5901 min_rank=0 imbalance_ratio=1.034993 max_load_reduction=42793 max_load_reduction_pct=0.870626 moved_to_echo=24748 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[264, 1355, 287, 1807, 60, 115, 1344, 654, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 576, 3299, 0, 0, 811, 1724, 0, 0, 0, 0, 0, 0, 0, 304, 735, 3066, 0, 1, 698, 1409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2198, 1374, 0, 1, 0, 2800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4163, 402, 2, 0, 90, 0, 310, 145, 677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 435, 52, 58, 3001, 1355, 1272, 0, 0, 0, 0, 0, 0, 0, 539, 1238, 1094, 1376, 0, 0, 0, 902, 469, 344, 0, 0, 0, 0, 0, 0, 626, 234, 169, 8, 72, 184, 155, 4061, 528, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6183, 6422, 6213, 6373, 5789, 6173, 5962, 6037] avg=6144.0000 max=6422 max_rank=1 min=5789 min_rank=4 imbalance_ratio=1.045247 max_load_reduction=42730 max_load_reduction_pct=0.869344 moved_to_echo=23961 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[192, 1761, 412, 1418, 66, 160, 1644, 417, 325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 605, 2900, 0, 0, 581, 1704, 0, 0, 0, 0, 0, 0, 0, 345, 755, 3031, 1, 0, 569, 1114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2401, 1212, 0, 1, 0, 2640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4512, 414, 31, 0, 119, 0, 419, 95, 739, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 361, 29, 163, 3028, 1111, 1258, 0, 0, 0, 0, 0, 0, 0, 610, 1526, 1286, 1350, 0, 7, 0, 957, 513, 340, 0, 0, 0, 0, 0, 0, 679, 229, 305, 41, 87, 166, 184, 3732, 576, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6395, 5819, 5815, 6254, 6329, 5952, 6589, 5999] avg=6144.0000 max=6589 max_rank=6 min=5815 min_rank=2 imbalance_ratio=1.072428 max_load_reduction=42563 max_load_reduction_pct=0.865946 moved_to_echo=24869 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[166, 1611, 368, 1409, 92, 102, 1169, 514, 312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 588, 3417, 0, 0, 765, 1685, 0, 0, 0, 0, 0, 0, 0, 349, 758, 2996, 0, 0, 742, 1525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2308, 1235, 0, 0, 0, 2598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4434, 431, 29, 0, 101, 0, 383, 113, 711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 351, 49, 117, 2969, 1259, 1244, 0, 0, 0, 0, 0, 0, 0, 546, 1391, 1142, 1270, 0, 0, 0, 861, 493, 337, 0, 0, 0, 0, 0, 0, 776, 228, 126, 8, 109, 156, 197, 4052, 554, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5743, 6461, 6370, 6141, 6202, 5989, 6040, 6206] avg=6144.0000 max=6461 max_rank=1 min=5743 min_rank=0 imbalance_ratio=1.051595 max_load_reduction=42691 max_load_reduction_pct=0.868551 moved_to_echo=24437 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[181, 1569, 428, 1610, 44, 248, 1347, 389, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 582, 3053, 0, 0, 719, 1729, 0, 0, 0, 0, 0, 0, 0, 362, 732, 3074, 1, 0, 776, 1435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2192, 1383, 0, 0, 0, 2440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4449, 346, 32, 0, 135, 0, 328, 103, 675, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 360, 40, 125, 3217, 1196, 1276, 0, 0, 0, 0, 0, 0, 0, 595, 1384, 1111, 1185, 0, 0, 0, 965, 468, 345, 0, 0, 0, 0, 0, 0, 799, 224, 107, 16, 62, 171, 173, 4136, 526, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6113, 6094, 6380, 6015, 6068, 6215, 6053, 6214] avg=6144.0000 max=6380 max_rank=2 min=6015 min_rank=3 imbalance_ratio=1.038411 max_load_reduction=42772 max_load_reduction_pct=0.870199 moved_to_echo=24550 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[268, 1759, 439, 1694, 69, 201, 1439, 510, 334, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4, 545, 2913, 0, 1, 680, 1671, 0, 0, 0, 0, 0, 0, 0, 347, 701, 2946, 2, 0, 639, 1327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2424, 1164, 0, 1, 0, 2558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4167, 374, 32, 4, 100, 1, 457, 120, 755, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 407, 44, 138, 3269, 1069, 1234, 0, 0, 0, 0, 0, 0, 0, 610, 1401, 965, 1173, 0, 0, 0, 949, 524, 338, 0, 0, 0, 0, 0, 0, 774, 233, 270, 33, 81, 213, 184, 4001, 591, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6713, 5816, 5962, 6148, 6010, 6163, 5960, 6380] avg=6144.0000 max=6713 max_rank=0 min=5816 min_rank=1 imbalance_ratio=1.092611 max_load_reduction=42439 max_load_reduction_pct=0.863424 moved_to_echo=24513 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[160, 1657, 415, 1349, 69, 125, 1397, 467, 318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 574, 3073, 0, 0, 705, 1717, 0, 0, 0, 0, 0, 0, 0, 348, 709, 3053, 0, 0, 637, 1223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2347, 1122, 0, 0, 0, 2664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4585, 403, 1, 1, 159, 0, 458, 93, 722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 414, 42, 98, 3128, 1212, 1267, 0, 0, 0, 0, 0, 0, 0, 615, 1582, 1034, 1240, 0, 2, 0, 911, 501, 343, 0, 0, 0, 0, 0, 0, 747, 258, 82, 24, 96, 164, 191, 4079, 563, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5957, 6077, 5970, 6133, 6422, 6161, 6228, 6204] avg=6144.0000 max=6422 max_rank=4 min=5957 min_rank=0 imbalance_ratio=1.045247 max_load_reduction=42730 max_load_reduction_pct=0.869344 moved_to_echo=25015 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4608, 11136, 9216, 10240, 0, 128, 0, 7424, 4096, 2816, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[5888, 1920, 1280, 256, 768, 1408, 1536, 32256, 4608, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 18688, 10112, 0, 128, 0, 20480, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1664, 12800, 2944, 12416, 640, 1280, 11008, 4352, 2560, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 128, 4608, 25088, 0, 128, 5888, 13696, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 0, 3200, 384, 1024, 24960, 9728, 10112, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2816, 5888, 24320, 128, 128, 5632, 10880, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[35328, 3200, 256, 128, 1024, 128, 3200, 896, 5760, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1019, 60, 0, 5035, 0, 0, 20, 0, 0, 220, 108, 4, 805, 369, 371, 439, 0, 0, 239, 0, 0, 536, 385, 403, 0, 75, 196, 0, 48, 65, 0, 0, 92, 5913, 264, 0, 0, 0, 0, 1346, 768, 530, 0, 546, 5061, 65, 356, 4889, 376, 63, 0, 948, 0, 5937, 6761, 0, 3535, 0, 0, 0, 0, 0, 1305, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1063, 45, 0, 5041, 0, 0, 90, 1, 0, 174, 216, 0, 816, 403, 308, 360, 0, 0, 223, 0, 0, 579, 331, 344, 0, 72, 401, 0, 170, 95, 0, 0, 57, 5643, 176, 0, 0, 0, 0, 1022, 725, 478, 0, 519, 4415, 96, 354, 5409, 326, 95, 0, 935, 4, 5826, 6809, 0, 3466, 0, 0, 0, 0, 0, 2064, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1137, 66, 0, 4824, 0, 0, 6, 0, 0, 194, 227, 0, 831, 319, 357, 314, 0, 0, 155, 0, 0, 600, 175, 254, 0, 60, 388, 0, 138, 90, 0, 0, 87, 5828, 163, 0, 0, 0, 0, 1225, 753, 388, 0, 585, 4799, 90, 324, 5147, 392, 90, 0, 1059, 23, 5752, 6714, 0, 3539, 0, 0, 0, 0, 0, 2059, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[999, 74, 0, 4944, 0, 0, 123, 1, 0, 195, 131, 26, 852, 380, 402, 517, 0, 0, 137, 0, 0, 543, 413, 491, 0, 66, 298, 0, 78, 69, 0, 0, 79, 5810, 260, 0, 0, 0, 0, 1180, 814, 529, 0, 610, 4956, 69, 381, 4993, 402, 69, 0, 1194, 0, 5386, 6761, 0, 3685, 0, 0, 0, 0, 0, 1235, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1021, 60, 0, 5078, 0, 0, 6, 0, 0, 202, 141, 1, 881, 391, 406, 350, 0, 0, 179, 0, 0, 555, 221, 453, 0, 92, 309, 0, 77, 71, 0, 0, 76, 5932, 192, 0, 0, 0, 0, 1358, 766, 585, 0, 501, 4872, 74, 372, 4844, 421, 71, 0, 1027, 4, 5586, 6791, 0, 3659, 0, 0, 0, 0, 0, 1526, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1082, 91, 0, 4967, 0, 0, 0, 0, 0, 192, 162, 0, 818, 411, 298, 589, 0, 0, 400, 0, 0, 587, 419, 442, 0, 44, 172, 0, 72, 82, 0, 0, 113, 5665, 214, 0, 0, 0, 0, 959, 753, 468, 0, 517, 5021, 82, 454, 5163, 340, 82, 0, 1030, 0, 5670, 6793, 0, 3604, 0, 0, 0, 0, 0, 1396, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1211, 59, 0, 5166, 0, 1, 169, 7, 1, 195, 194, 13, 823, 408, 358, 433, 0, 0, 138, 1, 0, 531, 419, 549, 0, 69, 418, 0, 142, 92, 0, 0, 69, 5884, 292, 0, 1, 1, 0, 1024, 773, 558, 12, 927, 4639, 93, 342, 4902, 361, 91, 1, 1107, 0, 4965, 6726, 1, 3861, 0, 0, 0, 2, 0, 1123, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1195, 65, 0, 5165, 0, 0, 42, 0, 0, 177, 127, 29, 850, 375, 387, 447, 0, 0, 202, 0, 0, 485, 351, 435, 0, 69, 407, 0, 64, 75, 0, 0, 69, 6024, 222, 0, 0, 0, 0, 1067, 775, 512, 0, 653, 5020, 75, 347, 4947, 387, 75, 0, 1136, 0, 5248, 6815, 0, 3592, 0, 0, 0, 0, 0, 1241, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1137, 66, 0, 4731, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 194, 227, 0, 831, 319, 357, 314, 1647, 1142, 1002, 0, 0, 0, 0, 0, 0, 0, 155, 0, 0, 600, 175, 254, 3500, 1187, 0, 0, 0, 0, 0, 0, 0, 60, 388, 0, 138, 90, 0, 0, 5527, 0, 0, 0, 0, 0, 0, 0, 87, 4686, 163, 0, 0, 0, 0, 1225, 0, 0, 0, 0, 0, 0, 0, 0, 753, 388, 0, 585, 3779, 90, 324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 392, 90, 0, 1059, 23, 4803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3539, 0, 0, 0, 0, 0, 2059, 0, 949, 93, 18, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5940, 6033, 5871, 6203, 6161, 5919, 6367, 6658] avg=6144.0000 max=6658 max_rank=7 min=5871 min_rank=2 imbalance_ratio=1.083659 max_load_reduction=42494 max_load_reduction_pct=0.864543 moved_to_echo=25105 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1063, 45, 0, 4944, 0, 0, 90, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 174, 216, 0, 816, 403, 308, 360, 1731, 1106, 922, 0, 0, 0, 0, 0, 0, 0, 223, 0, 0, 579, 331, 344, 3678, 1204, 0, 0, 0, 0, 0, 0, 0, 72, 401, 0, 170, 95, 0, 0, 5605, 0, 0, 0, 0, 0, 0, 0, 57, 4537, 176, 0, 0, 0, 0, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 725, 478, 0, 519, 3477, 96, 354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 326, 95, 0, 935, 4, 4864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3466, 0, 0, 0, 0, 0, 2064, 1, 962, 97, 16, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6036, 6359, 6343, 5792, 5649, 6224, 6606] avg=6144.0000 max=6606 max_rank=7 min=5649 min_rank=5 imbalance_ratio=1.075195 max_load_reduction=42546 max_load_reduction_pct=0.865601 moved_to_echo=24271 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1019, 60, 0, 4938, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 220, 108, 4, 805, 369, 371, 439, 1565, 1159, 1057, 0, 0, 0, 0, 0, 0, 0, 239, 0, 0, 536, 385, 403, 3324, 1195, 0, 0, 0, 0, 0, 0, 0, 75, 196, 0, 48, 65, 0, 0, 5566, 0, 0, 0, 0, 0, 0, 0, 92, 4754, 264, 0, 0, 0, 0, 1346, 0, 0, 0, 0, 0, 0, 0, 0, 768, 530, 0, 546, 3985, 65, 356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 376, 63, 0, 948, 0, 4957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3535, 0, 0, 0, 0, 0, 1305, 0, 980, 97, 19, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6037, 6097, 6082, 5950, 6456, 6250, 6344, 5936] avg=6144.0000 max=6456 max_rank=4 min=5936 min_rank=7 imbalance_ratio=1.050781 max_load_reduction=42696 max_load_reduction_pct=0.868652 moved_to_echo=24986 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[999, 74, 0, 4848, 0, 0, 123, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 195, 131, 26, 852, 380, 402, 517, 1599, 1139, 1035, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 543, 413, 491, 3394, 1196, 0, 0, 0, 0, 0, 0, 0, 66, 298, 0, 78, 69, 0, 0, 5565, 0, 0, 0, 0, 0, 0, 0, 79, 4671, 260, 0, 0, 0, 0, 1180, 0, 0, 0, 0, 0, 0, 0, 0, 814, 529, 0, 610, 3902, 69, 381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 402, 69, 0, 1194, 0, 4497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3685, 0, 0, 0, 0, 0, 1235, 0, 889, 96, 19, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6045, 6276, 6174, 6076, 6190, 6305, 6162, 5924] avg=6144.0000 max=6305 max_rank=5 min=5924 min_rank=7 imbalance_ratio=1.026204 max_load_reduction=42847 max_load_reduction_pct=0.871724 moved_to_echo=24581 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1021, 60, 0, 4980, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 202, 141, 1, 881, 391, 406, 350, 1551, 1162, 1017, 0, 0, 0, 0, 0, 0, 0, 179, 0, 0, 555, 221, 453, 3293, 1202, 0, 0, 0, 0, 0, 0, 0, 92, 309, 0, 77, 71, 0, 0, 5589, 0, 0, 0, 0, 0, 0, 0, 76, 4770, 192, 0, 0, 0, 0, 1358, 0, 0, 0, 0, 0, 0, 0, 0, 766, 585, 0, 501, 3837, 74, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 421, 71, 0, 1027, 4, 4664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3659, 0, 0, 0, 0, 0, 1526, 1, 922, 98, 18, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6067, 6102, 5903, 6138, 6396, 6135, 6187, 6224] avg=6144.0000 max=6396 max_rank=4 min=5903 min_rank=2 imbalance_ratio=1.041016 max_load_reduction=42756 max_load_reduction_pct=0.869873 moved_to_echo=24942 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1082, 91, 0, 4871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 162, 0, 818, 411, 298, 589, 1653, 1110, 1048, 0, 0, 0, 0, 0, 0, 0, 400, 0, 0, 587, 419, 442, 3510, 1202, 0, 0, 0, 0, 0, 0, 0, 44, 172, 0, 72, 82, 0, 0, 5591, 0, 0, 0, 0, 0, 0, 0, 113, 4555, 214, 0, 0, 0, 0, 959, 0, 0, 0, 0, 0, 0, 0, 0, 753, 468, 0, 517, 3954, 82, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 340, 82, 0, 1030, 0, 4734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3604, 0, 0, 0, 0, 0, 1396, 0, 936, 96, 19, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6044, 6281, 6560, 5961, 5841, 6228, 6186, 6051] avg=6144.0000 max=6560 max_rank=2 min=5841 min_rank=4 imbalance_ratio=1.067708 max_load_reduction=42592 max_load_reduction_pct=0.866536 moved_to_echo=24306 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1211, 59, 0, 5062, 0, 1, 169, 7, 0, 0, 0, 0, 0, 0, 0, 0, 1, 195, 194, 13, 823, 408, 358, 433, 1570, 1157, 972, 0, 0, 0, 0, 0, 0, 0, 138, 1, 0, 531, 419, 549, 3332, 1190, 0, 0, 0, 0, 0, 0, 0, 69, 418, 0, 142, 92, 0, 0, 5536, 0, 0, 0, 0, 0, 0, 0, 69, 4727, 292, 0, 1, 1, 0, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 773, 558, 12, 927, 3646, 93, 342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 361, 91, 1, 1107, 0, 4141, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3861, 0, 0, 0, 2, 0, 1123, 0, 824, 104, 21, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6509, 6124, 6160, 6257, 6114, 6351, 5702, 5935] avg=6144.0000 max=6509 max_rank=0 min=5702 min_rank=6 imbalance_ratio=1.059408 max_load_reduction=42643 max_load_reduction_pct=0.867574 moved_to_echo=24102 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1195, 65, 0, 5065, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 177, 127, 29, 850, 375, 387, 447, 1583, 1180, 1048, 0, 0, 0, 0, 0, 0, 0, 202, 0, 0, 485, 351, 435, 3364, 1206, 0, 0, 0, 0, 0, 0, 0, 69, 407, 0, 64, 75, 0, 0, 5609, 0, 0, 0, 0, 0, 0, 0, 69, 4844, 222, 0, 0, 0, 0, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 775, 512, 0, 653, 3953, 75, 347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 387, 75, 0, 1136, 0, 4382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3592, 0, 0, 0, 0, 0, 1241, 0, 866, 100, 19, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6367, 6203, 6043, 6224, 6202, 6315, 5980, 5818] avg=6144.0000 max=6367 max_rank=0 min=5818 min_rank=7 imbalance_ratio=1.036296 max_load_reduction=42785 max_load_reduction_pct=0.870463 moved_to_echo=24315 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 1664, 1408, 128, 6784, 3072, 2944, 3456, 12928, 9216, 8192, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[29056, 0, 0, 0, 128, 0, 12032, 128, 7424, 896, 256, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3072, 640, 128, 8448, 128, 37120, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[768, 37632, 1792, 0, 128, 128, 0, 9216, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[6144, 4096, 128, 4864, 30592, 768, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 0, 1792, 128, 0, 4480, 2816, 3456, 27520, 9600, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[8832, 640, 0, 39552, 0, 128, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 640, 2688, 0, 896, 640, 0, 0, 44672, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[820, 7576, 0, 0, 371, 0, 42, 0, 0, 300, 42, 3122, 0, 57, 517, 1016, 57, 2721, 0, 0, 176, 42, 0, 392, 6889, 4556, 470, 381, 0, 0, 0, 36, 0, 348, 201, 0, 98, 1250, 484, 28, 512, 0, 0, 72, 0, 3635, 1, 0, 5044, 0, 0, 4266, 420, 56, 1, 0, 0, 0, 0, 0, 0, 2436, 376, 341] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[753, 7490, 0, 0, 324, 0, 60, 0, 0, 353, 60, 3196, 0, 76, 529, 1514, 76, 2476, 0, 0, 213, 60, 0, 511, 6691, 4692, 506, 355, 0, 0, 0, 50, 0, 294, 249, 0, 82, 1437, 433, 7, 431, 0, 0, 92, 0, 3555, 0, 0, 5547, 0, 0, 3705, 491, 17, 0, 0, 0, 0, 0, 0, 0, 2181, 399, 247] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[716, 7517, 0, 0, 350, 0, 53, 0, 0, 300, 53, 2852, 0, 79, 513, 1609, 79, 2484, 0, 0, 285, 53, 2, 514, 6788, 4932, 625, 412, 0, 0, 0, 29, 0, 275, 188, 0, 69, 908, 476, 64, 571, 0, 0, 66, 0, 4120, 0, 0, 5817, 0, 12, 3612, 471, 69, 0, 0, 0, 0, 0, 0, 0, 1604, 372, 213] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[726, 7467, 0, 0, 347, 0, 66, 0, 0, 299, 66, 2942, 0, 87, 519, 1157, 87, 3174, 0, 0, 240, 66, 0, 480, 6858, 4524, 606, 418, 0, 0, 0, 8, 0, 299, 241, 0, 96, 1083, 480, 20, 486, 0, 0, 60, 0, 3998, 0, 0, 5598, 0, 7, 3477, 478, 16, 0, 0, 0, 0, 0, 0, 0, 1916, 449, 311] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[720, 7536, 0, 0, 331, 0, 60, 0, 0, 343, 60, 2705, 0, 92, 539, 1478, 92, 2547, 0, 0, 232, 60, 0, 431, 6832, 4793, 544, 402, 0, 0, 0, 47, 0, 301, 216, 0, 69, 1289, 419, 36, 422, 0, 0, 75, 0, 3820, 0, 0, 5621, 0, 3, 3724, 437, 56, 0, 0, 0, 0, 0, 0, 0, 2223, 409, 188] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[775, 7523, 0, 1, 366, 0, 57, 0, 0, 324, 54, 3095, 1, 70, 530, 1555, 69, 2425, 1, 2, 283, 54, 0, 518, 6794, 4527, 557, 375, 0, 1, 0, 43, 1, 385, 247, 0, 106, 1012, 459, 20, 565, 0, 0, 69, 2, 3791, 0, 1, 5691, 1, 7, 3769, 451, 91, 0, 0, 0, 1, 2, 0, 0, 1865, 316, 300] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[661, 7521, 0, 0, 328, 0, 63, 0, 0, 348, 64, 3099, 0, 69, 516, 1521, 69, 2648, 0, 0, 267, 63, 0, 486, 6809, 4704, 596, 336, 0, 0, 1, 42, 0, 328, 275, 0, 88, 1021, 482, 20, 464, 0, 0, 69, 0, 3853, 0, 0, 5501, 0, 4, 3919, 462, 73, 2, 0, 0, 0, 0, 0, 0, 1887, 285, 208] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[713, 7485, 0, 0, 358, 0, 90, 0, 0, 331, 90, 2929, 0, 113, 588, 1529, 112, 2785, 0, 0, 291, 90, 0, 405, 6803, 4767, 513, 396, 1, 0, 0, 40, 0, 307, 234, 0, 88, 1359, 462, 30, 433, 0, 0, 44, 0, 3825, 0, 0, 5329, 0, 0, 3495, 422, 74, 0, 0, 0, 0, 0, 0, 0, 2049, 382, 190] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[726, 4971, 0, 0, 347, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 299, 66, 2942, 0, 87, 519, 1157, 726, 0, 0, 0, 0, 0, 0, 0, 87, 3174, 0, 0, 240, 66, 0, 480, 2648, 17, 0, 0, 0, 0, 0, 0, 488, 4524, 606, 418, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 299, 241, 0, 96, 1083, 480, 20, 3876, 0, 0, 0, 0, 0, 0, 0, 486, 0, 0, 60, 0, 3998, 0, 0, 1753, 0, 0, 0, 0, 0, 0, 0, 1912, 0, 7, 3477, 478, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1916, 449, 311, 2494, 1038, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6110, 5796, 6712, 6044, 6095, 6297, 5890, 6208] avg=6144.0000 max=6712 max_rank=2 min=5796 min_rank=1 imbalance_ratio=1.092448 max_load_reduction=42440 max_load_reduction_pct=0.863444 moved_to_echo=24490 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[820, 5042, 0, 0, 371, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 42, 3122, 0, 57, 517, 1016, 737, 0, 0, 0, 0, 0, 0, 0, 57, 2721, 0, 0, 176, 42, 0, 392, 2386, 18, 0, 0, 0, 0, 0, 0, 490, 4556, 470, 381, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 348, 201, 0, 98, 1250, 484, 28, 3894, 0, 0, 0, 0, 0, 0, 0, 512, 0, 0, 72, 0, 3635, 1, 0, 1779, 0, 0, 0, 0, 0, 0, 0, 1722, 0, 0, 4266, 420, 56, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2436, 376, 341, 2505, 936, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6275, 5791, 5792, 5933, 6303, 5999, 6465, 6594] avg=6144.0000 max=6594 max_rank=7 min=5791 min_rank=1 imbalance_ratio=1.073242 max_load_reduction=42558 max_load_reduction_pct=0.865845 moved_to_echo=25361 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[720, 5016, 0, 0, 331, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 343, 60, 2705, 0, 92, 539, 1478, 733, 0, 0, 0, 0, 0, 0, 0, 92, 2547, 0, 0, 232, 60, 0, 431, 2659, 17, 0, 0, 0, 0, 0, 0, 486, 4793, 544, 402, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 301, 216, 0, 69, 1289, 419, 36, 3862, 0, 0, 0, 0, 0, 0, 0, 422, 0, 0, 75, 0, 3820, 0, 0, 1770, 0, 0, 0, 0, 0, 0, 0, 1919, 0, 3, 3724, 437, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2223, 409, 188, 2484, 1043, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6127, 5950, 6038, 6272, 6192, 6087, 6139, 6347] avg=6144.0000 max=6347 max_rank=7 min=5950 min_rank=1 imbalance_ratio=1.033040 max_load_reduction=42805 max_load_reduction_pct=0.870870 moved_to_echo=24765 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[716, 5004, 0, 0, 350, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 53, 2852, 0, 79, 513, 1609, 731, 0, 0, 0, 0, 0, 0, 0, 79, 2484, 0, 0, 285, 53, 2, 514, 2752, 17, 0, 0, 0, 0, 0, 0, 483, 4932, 625, 412, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 275, 188, 0, 69, 908, 476, 64, 3837, 0, 0, 0, 0, 0, 0, 0, 571, 0, 0, 66, 0, 4120, 0, 0, 1765, 0, 0, 0, 0, 0, 0, 0, 1986, 0, 12, 3612, 471, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1604, 372, 213, 2468, 1079, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6123, 6137, 6186, 6481, 5817, 6522, 6150, 5736] avg=6144.0000 max=6522 max_rank=5 min=5736 min_rank=7 imbalance_ratio=1.061523 max_load_reduction=42630 max_load_reduction_pct=0.867310 moved_to_echo=24225 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[753, 4985, 0, 0, 324, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 353, 60, 3196, 0, 76, 529, 1514, 729, 0, 0, 0, 0, 0, 0, 0, 76, 2476, 0, 0, 213, 60, 0, 511, 2624, 17, 0, 0, 0, 0, 0, 0, 476, 4692, 506, 355, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 294, 249, 0, 82, 1437, 433, 7, 3782, 0, 0, 0, 0, 0, 0, 0, 431, 0, 0, 92, 0, 3555, 0, 0, 1759, 0, 0, 0, 0, 0, 0, 0, 1894, 0, 0, 3705, 491, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2181, 399, 247, 2433, 1029, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6122, 6457, 5977, 6079, 6284, 5837, 6107, 6289] avg=6144.0000 max=6457 max_rank=1 min=5837 min_rank=5 imbalance_ratio=1.050944 max_load_reduction=42695 max_load_reduction_pct=0.868632 moved_to_echo=24517 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[713, 4982, 0, 0, 358, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 331, 90, 2929, 0, 113, 588, 1529, 728, 0, 0, 0, 0, 0, 0, 0, 112, 2785, 0, 0, 291, 90, 0, 405, 2521, 17, 0, 0, 0, 0, 0, 0, 484, 4767, 513, 396, 1, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 307, 234, 0, 88, 1359, 462, 30, 3845, 0, 0, 0, 0, 0, 0, 0, 433, 0, 0, 44, 0, 3825, 0, 0, 1758, 0, 0, 0, 0, 0, 0, 0, 1820, 0, 0, 3495, 422, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2049, 382, 190, 2474, 988, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6308, 6221, 6201, 6325, 6060, 5811, 6083] avg=6144.0000 max=6325 max_rank=4 min=5811 min_rank=6 imbalance_ratio=1.029460 max_load_reduction=42827 max_load_reduction_pct=0.871318 moved_to_echo=24279 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[775, 4995, 0, 1, 366, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 324, 54, 3095, 1, 70, 530, 1555, 735, 0, 0, 0, 0, 0, 0, 0, 69, 2425, 1, 2, 283, 54, 0, 518, 2698, 23, 0, 0, 0, 0, 0, 0, 476, 4527, 557, 375, 0, 1, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 1, 385, 247, 0, 106, 1012, 459, 20, 3843, 0, 0, 0, 0, 0, 0, 0, 565, 0, 0, 69, 2, 3791, 0, 1, 1770, 0, 0, 0, 0, 0, 0, 0, 1933, 1, 7, 3769, 451, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 1865, 316, 300, 2475, 1060, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6194, 6364, 6073, 5979, 6073, 6198, 6252, 6019] avg=6144.0000 max=6364 max_rank=1 min=5979 min_rank=3 imbalance_ratio=1.035807 max_load_reduction=42788 max_load_reduction_pct=0.870524 moved_to_echo=24542 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[661, 5006, 0, 0, 328, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 348, 64, 3099, 0, 69, 516, 1521, 732, 0, 0, 0, 0, 0, 0, 0, 69, 2648, 0, 0, 267, 63, 0, 486, 2603, 17, 0, 0, 0, 0, 0, 0, 484, 4704, 596, 336, 0, 0, 1, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 328, 275, 0, 88, 1021, 482, 20, 3849, 0, 0, 0, 0, 0, 0, 0, 464, 0, 0, 69, 0, 3853, 0, 0, 1766, 0, 0, 0, 0, 0, 0, 0, 1878, 0, 4, 3919, 462, 73, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1887, 285, 208, 2476, 1020, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6058, 6349, 6153, 6163, 6063, 6152, 6338, 5876] avg=6144.0000 max=6349 max_rank=1 min=5876 min_rank=7 imbalance_ratio=1.033366 max_load_reduction=42803 max_load_reduction_pct=0.870829 moved_to_echo=24429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=50176 observed_loss=-1024 tokens_per_expert=[768, 21376, 128, 128, 2048, 512, 128, 3840, 20992, 256, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3968, 0, 0, 640, 128, 30720, 128, 128, 14208, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 2688, 512, 24064, 128, 768, 4352, 11392, 5888, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 0, 16256, 3072, 2048, 19840, 8320, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[15104, 128, 128, 30080, 3712, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3968, 37504, 4480, 3200, 128, 128, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 2560, 1920, 0, 768, 9472, 3712, 256, 30848, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[5888, 40064, 0, 128, 2816, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=50176 slack_slots=48128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[634, 90, 239, 157, 6769, 0, 7165, 4258, 0, 178, 0, 791, 0, 336, 0, 4226, 248, 0, 2959, 0, 387, 46, 317, 87, 23, 487, 90, 0, 0, 157, 0, 0, 5691, 0, 0, 90, 114, 1163, 2863, 0, 177, 0, 0, 0, 0, 0, 0, 0, 949, 0, 0, 0, 0, 783, 413, 0, 4233, 158, 0, 714, 1999, 161, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[618, 63, 776, 229, 6931, 0, 7355, 4706, 0, 127, 0, 593, 0, 178, 0, 3981, 255, 69, 3234, 0, 403, 42, 238, 92, 8, 436, 65, 0, 0, 125, 0, 0, 5807, 0, 0, 65, 95, 1015, 2909, 0, 155, 0, 0, 0, 0, 0, 0, 0, 1099, 0, 0, 0, 2, 818, 361, 0, 3984, 151, 0, 491, 1611, 65, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[589, 69, 698, 300, 6927, 0, 7308, 4853, 0, 138, 0, 663, 0, 204, 0, 3666, 260, 36, 3161, 0, 822, 42, 6, 79, 0, 412, 69, 0, 0, 123, 0, 0, 5671, 0, 0, 69, 145, 990, 2785, 0, 148, 0, 0, 0, 0, 0, 0, 4, 1123, 0, 0, 0, 1, 840, 402, 0, 4423, 199, 0, 406, 1443, 78, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[611, 71, 402, 181, 6859, 0, 7311, 4458, 0, 145, 0, 729, 0, 212, 0, 4514, 238, 0, 2671, 0, 419, 43, 117, 76, 2, 494, 74, 0, 0, 134, 0, 3, 5815, 0, 0, 74, 93, 1130, 2593, 0, 147, 0, 0, 0, 0, 0, 0, 0, 952, 0, 0, 0, 0, 682, 487, 0, 4355, 313, 0, 567, 2099, 81, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[621, 82, 372, 293, 6937, 0, 7272, 4404, 0, 164, 0, 713, 0, 260, 0, 4525, 264, 0, 2656, 0, 462, 43, 236, 113, 1, 422, 82, 0, 0, 173, 0, 0, 5757, 0, 0, 82, 142, 1123, 2898, 0, 195, 0, 0, 0, 0, 0, 0, 2, 1005, 0, 0, 0, 1, 760, 445, 0, 3962, 282, 0, 545, 1785, 72, 1, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[599, 91, 1016, 234, 6952, 1, 6991, 4665, 1, 182, 2, 715, 3, 287, 0, 4218, 251, 5, 2869, 0, 534, 49, 74, 69, 5, 486, 94, 0, 0, 146, 1, 9, 5796, 2, 0, 91, 127, 1020, 2512, 2, 160, 1, 2, 2, 0, 3, 1, 0, 970, 1, 1, 0, 1, 865, 440, 1, 4101, 394, 0, 477, 1491, 141, 1, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[624, 75, 345, 295, 7093, 0, 7393, 4623, 0, 150, 0, 644, 0, 187, 0, 4108, 245, 5, 3274, 0, 393, 51, 311, 69, 1, 467, 76, 0, 0, 138, 0, 1, 5640, 0, 0, 75, 120, 1228, 2423, 0, 144, 0, 0, 0, 0, 0, 0, 1, 1084, 0, 0, 0, 0, 949, 433, 0, 4094, 136, 0, 549, 1644, 64, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[728, 95, 240, 191, 6790, 0, 7113, 4453, 0, 191, 0, 869, 0, 321, 0, 4083, 253, 0, 2884, 0, 524, 28, 237, 57, 15, 449, 96, 0, 0, 138, 0, 0, 5689, 0, 0, 96, 108, 1065, 2769, 0, 154, 0, 0, 0, 0, 0, 0, 1, 1056, 0, 0, 0, 0, 862, 428, 0, 4276, 135, 0, 590, 1994, 174, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[634, 90, 239, 157, 136, 0, 0, 4258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 178, 0, 791, 0, 336, 0, 4226, 669, 193, 0, 0, 0, 0, 0, 0, 248, 0, 2959, 0, 387, 46, 317, 87, 2090, 0, 0, 0, 0, 0, 0, 0, 23, 487, 90, 0, 0, 157, 0, 0, 4115, 1243, 0, 0, 0, 0, 0, 0, 2121, 0, 0, 90, 114, 1163, 2863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 177, 0, 0, 0, 0, 0, 0, 0, 5922, 0, 0, 0, 0, 0, 0, 0, 949, 0, 0, 0, 0, 783, 413, 0, 2518, 1287, 0, 0, 0, 0, 0, 0, 3564, 158, 0, 714, 1999, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5514, 6393, 6134, 6115, 6351, 6099, 5950, 6596] avg=6144.0000 max=6596 max_rank=7 min=5514 min_rank=0 imbalance_ratio=1.073568 max_load_reduction=42556 max_load_reduction_pct=0.865804 moved_to_echo=24996 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[618, 63, 776, 229, 139, 0, 0, 4706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 0, 593, 0, 178, 0, 3981, 630, 197, 0, 0, 0, 0, 0, 0, 255, 69, 3234, 0, 403, 42, 238, 92, 2133, 0, 0, 0, 0, 0, 0, 0, 8, 436, 65, 0, 0, 125, 0, 0, 4214, 1276, 0, 0, 0, 0, 0, 0, 2163, 0, 0, 65, 95, 1015, 2909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 0, 0, 0, 0, 0, 0, 0, 6079, 0, 0, 0, 0, 0, 0, 0, 1099, 0, 0, 0, 2, 818, 361, 0, 2578, 1314, 0, 0, 0, 0, 0, 0, 3354, 151, 0, 491, 1611, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6531, 5706, 6466, 6124, 6247, 6234, 6172, 5672] avg=6144.0000 max=6531 max_rank=0 min=5672 min_rank=7 imbalance_ratio=1.062988 max_load_reduction=42621 max_load_reduction_pct=0.867126 moved_to_echo=24325 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[589, 69, 698, 300, 139, 0, 0, 4853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 0, 663, 0, 204, 0, 3666, 699, 193, 0, 0, 0, 0, 0, 0, 260, 36, 3161, 0, 822, 42, 6, 79, 2083, 0, 0, 0, 0, 0, 0, 0, 0, 412, 69, 0, 0, 123, 0, 0, 4211, 1269, 0, 0, 0, 0, 0, 0, 2112, 0, 0, 69, 145, 990, 2785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 0, 0, 0, 0, 0, 0, 4, 6039, 0, 0, 0, 0, 0, 0, 0, 1123, 0, 0, 0, 1, 840, 402, 0, 2577, 1283, 0, 0, 0, 0, 0, 0, 3724, 199, 0, 406, 1443, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6648, 5563, 6489, 6084, 6101, 6191, 6226, 5850] avg=6144.0000 max=6648 max_rank=0 min=5563 min_rank=1 imbalance_ratio=1.082031 max_load_reduction=42504 max_load_reduction_pct=0.864746 moved_to_echo=24368 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[611, 71, 402, 181, 138, 0, 0, 4458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 729, 0, 212, 0, 4514, 688, 197, 0, 0, 0, 0, 0, 0, 238, 0, 2671, 0, 419, 43, 117, 76, 2136, 0, 0, 0, 0, 0, 0, 0, 2, 494, 74, 0, 0, 134, 0, 3, 4170, 1270, 0, 0, 0, 0, 0, 0, 2166, 0, 0, 74, 93, 1130, 2593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 0, 0, 0, 0, 0, 0, 0, 6041, 0, 0, 0, 0, 0, 0, 0, 952, 0, 0, 0, 0, 682, 487, 0, 2551, 1316, 0, 0, 0, 0, 0, 0, 3667, 313, 0, 567, 2099, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5861, 6485, 5700, 6147, 6056, 6188, 5988, 6727] avg=6144.0000 max=6727 max_rank=7 min=5700 min_rank=2 imbalance_ratio=1.094889 max_load_reduction=42425 max_load_reduction_pct=0.863139 moved_to_echo=24959 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[621, 82, 372, 293, 140, 0, 0, 4404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 0, 713, 0, 260, 0, 4525, 626, 195, 0, 0, 0, 0, 0, 0, 264, 0, 2656, 0, 462, 43, 236, 113, 2114, 0, 0, 0, 0, 0, 0, 0, 1, 422, 82, 0, 0, 173, 0, 0, 4217, 1263, 0, 0, 0, 0, 0, 0, 2146, 0, 0, 82, 142, 1123, 2898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 195, 0, 0, 0, 0, 0, 0, 2, 6009, 0, 0, 0, 0, 0, 0, 0, 1005, 0, 0, 0, 1, 760, 445, 0, 2580, 1302, 0, 0, 0, 0, 0, 0, 3336, 282, 0, 545, 1785, 72, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5912, 6483, 5888, 6158, 6391, 6206, 6093, 6021] avg=6144.0000 max=6483 max_rank=1 min=5888 min_rank=2 imbalance_ratio=1.055176 max_load_reduction=42669 max_load_reduction_pct=0.868103 moved_to_echo=24711 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[599, 91, 1016, 234, 132, 1, 0, 4665, 0, 0, 0, 0, 0, 0, 0, 0, 1, 182, 2, 715, 3, 287, 0, 4218, 652, 202, 0, 0, 0, 0, 0, 0, 251, 5, 2869, 0, 534, 49, 74, 69, 2133, 0, 0, 0, 0, 0, 0, 0, 5, 486, 94, 0, 0, 146, 1, 9, 4230, 1214, 0, 0, 0, 0, 0, 0, 2146, 2, 0, 91, 127, 1020, 2512, 2, 0, 0, 0, 0, 0, 0, 0, 0, 160, 1, 2, 2, 0, 3, 1, 0, 5777, 0, 0, 0, 0, 0, 0, 0, 970, 1, 1, 0, 1, 865, 440, 1, 2590, 1315, 0, 0, 0, 0, 0, 0, 3449, 394, 0, 477, 1491, 141, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6738, 6262, 5984, 6185, 5900, 5946, 6184, 5953] avg=6144.0000 max=6738 max_rank=0 min=5900 min_rank=4 imbalance_ratio=1.096680 max_load_reduction=42414 max_load_reduction_pct=0.862915 moved_to_echo=23983 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[624, 75, 345, 295, 143, 0, 0, 4623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 0, 644, 0, 187, 0, 4108, 647, 191, 0, 0, 0, 0, 0, 0, 245, 5, 3274, 0, 393, 51, 311, 69, 2071, 0, 0, 0, 0, 0, 0, 0, 1, 467, 76, 0, 0, 138, 0, 1, 4312, 1284, 0, 0, 0, 0, 0, 0, 2102, 0, 0, 75, 120, 1228, 2423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 0, 0, 0, 0, 0, 1, 6109, 0, 0, 0, 0, 0, 0, 0, 1084, 0, 0, 0, 0, 949, 433, 0, 2638, 1276, 0, 0, 0, 0, 0, 0, 3447, 136, 0, 549, 1644, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6105, 5927, 6419, 6279, 5948, 6254, 6380, 5840] avg=6144.0000 max=6419 max_rank=2 min=5840 min_rank=7 imbalance_ratio=1.044759 max_load_reduction=42733 max_load_reduction_pct=0.869405 moved_to_echo=24422 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[728, 95, 240, 191, 136, 0, 0, 4453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 191, 0, 869, 0, 321, 0, 4083, 676, 193, 0, 0, 0, 0, 0, 0, 253, 0, 2884, 0, 524, 28, 237, 57, 2089, 0, 0, 0, 0, 0, 0, 0, 15, 449, 96, 0, 0, 138, 0, 0, 4128, 1234, 0, 0, 0, 0, 0, 0, 2120, 0, 0, 96, 108, 1065, 2769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 0, 0, 0, 0, 0, 0, 1, 5879, 0, 0, 0, 0, 0, 0, 0, 1056, 0, 0, 0, 0, 862, 428, 0, 2526, 1287, 0, 0, 0, 0, 0, 0, 3600, 135, 0, 590, 1994, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5843, 6333, 6072, 6060, 6158, 6034, 6159, 6493] avg=6144.0000 max=6493 max_rank=7 min=5843 min_rank=0 imbalance_ratio=1.056803 max_load_reduction=42659 max_load_reduction_pct=0.867900 moved_to_echo=24844 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[17152, 128, 0, 768, 1024, 8832, 21760, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 3712, 768, 0, 0, 1152, 128, 128, 33664, 10112, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[2048, 128, 23808, 0, 3968, 384, 1536, 768, 16896, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[8320, 128, 128, 0, 128, 6656, 3456, 128, 20608, 10496, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[28160, 1792, 0, 4352, 14080, 896, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[5120, 640, 4096, 1920, 1152, 128, 0, 36480, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 1280, 128, 5760, 128, 2048, 0, 33408, 5376, 1664, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1280, 128, 128, 128, 0, 128, 128, 128, 47872, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 8102, 5563, 0, 0, 66, 0, 466, 0, 0, 0, 0, 0, 687, 3519, 156, 0, 90, 0, 334, 0, 0, 0, 1568, 161, 247, 0, 60, 0, 66, 251, 0, 281, 7870, 90, 0, 811, 0, 6733, 0, 946, 0, 0, 0, 404, 0, 0, 0, 1524, 0, 90, 0, 0, 0, 0, 0, 0, 0, 7449, 891, 89, 97, 0, 541] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 8110, 5416, 0, 0, 90, 0, 467, 0, 0, 0, 0, 0, 757, 3128, 172, 0, 82, 0, 377, 0, 0, 0, 1681, 72, 264, 0, 44, 0, 90, 154, 0, 256, 7843, 82, 0, 954, 0, 6794, 0, 1353, 0, 0, 0, 382, 0, 0, 0, 1411, 0, 83, 0, 0, 0, 0, 0, 0, 0, 7518, 798, 82, 91, 0, 601] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 8123, 5380, 0, 5, 53, 0, 823, 0, 0, 0, 0, 0, 732, 3420, 122, 0, 70, 0, 340, 0, 0, 0, 1922, 78, 261, 0, 70, 0, 53, 147, 0, 211, 7900, 69, 0, 758, 0, 6742, 0, 861, 0, 0, 0, 480, 0, 0, 0, 1187, 0, 69, 0, 0, 0, 0, 0, 0, 0, 7583, 865, 69, 84, 0, 675] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 8097, 5434, 0, 0, 42, 0, 423, 0, 0, 0, 0, 0, 734, 3165, 138, 0, 96, 0, 311, 0, 0, 0, 1597, 174, 254, 0, 72, 0, 42, 269, 0, 210, 7916, 96, 0, 1101, 0, 6619, 0, 961, 0, 0, 0, 455, 0, 0, 0, 1455, 0, 96, 1, 0, 0, 0, 0, 0, 0, 7437, 1150, 95, 102, 0, 610] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 8129, 5517, 0, 0, 60, 0, 545, 0, 0, 0, 0, 0, 740, 3246, 125, 0, 65, 0, 346, 0, 0, 0, 1945, 48, 254, 0, 76, 0, 60, 111, 0, 178, 7891, 65, 0, 1092, 0, 6686, 0, 880, 0, 0, 0, 508, 0, 0, 0, 1202, 0, 66, 0, 0, 0, 0, 0, 0, 0, 7634, 928, 63, 93, 0, 599] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 8121, 5488, 0, 0, 60, 1, 461, 0, 0, 0, 0, 0, 733, 3328, 135, 0, 77, 0, 315, 0, 0, 0, 1689, 81, 239, 0, 92, 0, 60, 152, 0, 257, 7868, 74, 0, 1004, 0, 6708, 0, 1077, 0, 0, 0, 437, 0, 0, 0, 1372, 0, 76, 3, 0, 0, 0, 0, 0, 0, 7591, 875, 71, 86, 0, 621] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 8117, 5393, 0, 0, 63, 0, 457, 0, 0, 0, 0, 0, 730, 3559, 138, 0, 75, 0, 316, 0, 0, 0, 1808, 64, 246, 0, 69, 0, 63, 139, 0, 172, 7912, 75, 0, 741, 0, 6783, 0, 1174, 0, 0, 0, 500, 0, 0, 0, 1357, 0, 75, 0, 0, 0, 0, 0, 0, 0, 7594, 702, 75, 100, 0, 655] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 8094, 5574, 0, 1, 54, 2, 589, 0, 1, 4, 0, 0, 721, 3360, 145, 0, 92, 0, 318, 1, 2, 0, 1635, 140, 249, 0, 74, 0, 54, 231, 1, 235, 7889, 91, 1, 791, 2, 6757, 1, 1201, 1, 4, 0, 571, 2, 0, 2, 1186, 1, 93, 0, 0, 0, 0, 4, 0, 1, 7454, 710, 91, 113, 2, 607] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 83, 5563, 0, 0, 66, 0, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 687, 3519, 156, 1797, 134, 0, 0, 0, 0, 0, 0, 0, 90, 0, 334, 0, 0, 0, 1568, 2874, 1091, 0, 0, 0, 0, 0, 0, 161, 247, 0, 60, 0, 66, 251, 0, 5471, 0, 0, 0, 0, 0, 0, 0, 281, 0, 90, 0, 811, 0, 4936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 946, 0, 0, 0, 404, 0, 0, 0, 4611, 0, 0, 0, 0, 0, 0, 0, 1524, 0, 90, 0, 0, 0, 0, 0, 2548, 2168, 0, 0, 0, 0, 0, 0, 0, 0, 4441, 891, 89, 97, 0, 541, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6178, 6293, 5957, 6256, 6118, 5961, 6330, 6059] avg=6144.0000 max=6330 max_rank=6 min=5957 min_rank=2 imbalance_ratio=1.030273 max_load_reduction=42822 max_load_reduction_pct=0.871216 moved_to_echo=24468 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 83, 5434, 0, 0, 42, 0, 423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 734, 3165, 138, 1766, 134, 0, 0, 0, 0, 0, 0, 0, 96, 0, 311, 0, 0, 0, 1597, 2869, 1097, 0, 0, 0, 0, 0, 0, 174, 254, 0, 72, 0, 42, 269, 0, 5467, 0, 0, 0, 0, 0, 0, 0, 210, 0, 96, 0, 1101, 0, 4853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 961, 0, 0, 0, 455, 0, 0, 0, 4636, 0, 0, 0, 0, 0, 0, 0, 1455, 0, 96, 1, 0, 0, 0, 0, 2547, 2183, 0, 0, 0, 0, 0, 0, 0, 0, 4434, 1150, 95, 102, 0, 610, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5982, 5937, 5970, 6278, 6260, 6052, 6282, 6391] avg=6144.0000 max=6391 max_rank=7 min=5937 min_rank=1 imbalance_ratio=1.040202 max_load_reduction=42761 max_load_reduction_pct=0.869975 moved_to_echo=24985 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 83, 5517, 0, 0, 60, 0, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 740, 3246, 125, 1784, 137, 0, 0, 0, 0, 0, 0, 0, 65, 0, 346, 0, 0, 0, 1945, 2945, 1094, 0, 0, 0, 0, 0, 0, 48, 254, 0, 76, 0, 60, 111, 0, 5489, 0, 0, 0, 0, 0, 0, 0, 178, 0, 65, 0, 1092, 0, 4902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 880, 0, 0, 0, 508, 0, 0, 0, 4621, 0, 0, 0, 0, 0, 0, 0, 1202, 0, 66, 0, 0, 0, 0, 0, 2557, 2176, 0, 0, 0, 0, 0, 0, 0, 0, 4552, 928, 63, 93, 0, 599, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6205, 6032, 6395, 6038, 6237, 6009, 6001, 6235] avg=6144.0000 max=6395 max_rank=2 min=6001 min_rank=6 imbalance_ratio=1.040853 max_load_reduction=42757 max_load_reduction_pct=0.869893 moved_to_echo=24482 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 83, 5380, 0, 5, 53, 0, 823, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 732, 3420, 122, 1799, 137, 0, 0, 0, 0, 0, 0, 0, 70, 0, 340, 0, 0, 0, 1922, 2925, 1095, 0, 0, 0, 0, 0, 0, 78, 261, 0, 70, 0, 53, 147, 0, 5485, 0, 0, 0, 0, 0, 0, 0, 211, 0, 69, 0, 758, 0, 4943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 861, 0, 0, 0, 480, 0, 0, 0, 4628, 0, 0, 0, 0, 0, 0, 0, 1187, 0, 69, 0, 0, 0, 0, 0, 2555, 2177, 0, 0, 0, 0, 0, 0, 0, 0, 4521, 865, 69, 84, 0, 675, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6344, 6210, 6352, 6094, 5981, 5969, 5988, 6214] avg=6144.0000 max=6352 max_rank=2 min=5969 min_rank=5 imbalance_ratio=1.033854 max_load_reduction=42800 max_load_reduction_pct=0.870768 moved_to_echo=24152 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 83, 5416, 0, 0, 90, 0, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 757, 3128, 172, 1813, 135, 0, 0, 0, 0, 0, 0, 0, 82, 0, 377, 0, 0, 0, 1681, 2900, 1088, 0, 0, 0, 0, 0, 0, 72, 264, 0, 44, 0, 90, 154, 0, 5476, 0, 0, 0, 0, 0, 0, 0, 256, 0, 82, 0, 954, 0, 4981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1353, 0, 0, 0, 382, 0, 0, 0, 4594, 0, 0, 0, 0, 0, 0, 0, 1411, 0, 83, 0, 0, 0, 0, 0, 2551, 2161, 0, 0, 0, 0, 0, 0, 0, 0, 4483, 798, 82, 91, 0, 601, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6056, 6005, 6128, 6100, 6273, 6329, 6206, 6055] avg=6144.0000 max=6329 max_rank=5 min=6005 min_rank=1 imbalance_ratio=1.030111 max_load_reduction=42823 max_load_reduction_pct=0.871236 moved_to_echo=24863 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 84, 5488, 0, 0, 60, 1, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 733, 3328, 135, 1790, 137, 0, 0, 0, 0, 0, 0, 0, 77, 0, 315, 0, 0, 0, 1689, 2928, 1092, 0, 0, 0, 0, 0, 0, 81, 239, 0, 92, 0, 60, 152, 0, 5483, 0, 0, 0, 0, 0, 0, 0, 257, 0, 74, 0, 1004, 0, 4918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1077, 0, 0, 0, 437, 0, 0, 0, 4608, 0, 0, 0, 0, 0, 0, 0, 1372, 0, 76, 3, 0, 0, 0, 0, 2554, 2168, 0, 0, 0, 0, 0, 0, 0, 0, 4526, 875, 71, 86, 0, 621, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6094, 6123, 6101, 6107, 6253, 6122, 6173, 6179] avg=6144.0000 max=6253 max_rank=4 min=6094 min_rank=0 imbalance_ratio=1.017741 max_load_reduction=42899 max_load_reduction_pct=0.872782 moved_to_echo=24727 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 83, 5393, 0, 0, 63, 0, 457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 730, 3559, 138, 1810, 137, 0, 0, 0, 0, 0, 0, 0, 75, 0, 316, 0, 0, 0, 1808, 2930, 1097, 0, 0, 0, 0, 0, 0, 64, 246, 0, 69, 0, 63, 139, 0, 5481, 0, 0, 0, 0, 0, 0, 0, 172, 0, 75, 0, 741, 0, 4973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1174, 0, 0, 0, 500, 0, 0, 0, 4635, 0, 0, 0, 0, 0, 0, 0, 1357, 0, 75, 0, 0, 0, 0, 0, 2553, 2180, 0, 0, 0, 0, 0, 0, 0, 0, 4527, 702, 75, 100, 0, 655, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5996, 6374, 6226, 6062, 5961, 6309, 6165, 6059] avg=6144.0000 max=6374 max_rank=1 min=5961 min_rank=4 imbalance_ratio=1.037435 max_load_reduction=42778 max_load_reduction_pct=0.870321 moved_to_echo=24494 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 77, 5574, 0, 1, 54, 2, 589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 721, 3360, 145, 1809, 138, 0, 0, 0, 0, 0, 0, 0, 92, 0, 318, 1, 2, 0, 1635, 2880, 1095, 0, 0, 0, 0, 0, 0, 140, 249, 0, 74, 0, 54, 231, 1, 5468, 0, 0, 0, 0, 0, 0, 0, 235, 0, 91, 1, 791, 2, 4948, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1201, 1, 4, 0, 571, 2, 0, 2, 4620, 0, 0, 0, 0, 0, 0, 0, 1186, 1, 93, 0, 0, 0, 0, 4, 2549, 2174, 0, 0, 0, 0, 0, 0, 0, 1, 4436, 710, 91, 113, 2, 607, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6297, 6178, 6023, 6217, 6069, 6401, 6007, 5960] avg=6144.0000 max=6401 max_rank=5 min=5960 min_rank=7 imbalance_ratio=1.041829 max_load_reduction=42751 max_load_reduction_pct=0.869771 moved_to_echo=24437 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[10752, 128, 768, 128, 0, 0, 0, 128, 20480, 17408, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 768, 0, 2688, 128, 128, 0, 13952, 23296, 8832, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 128, 0, 0, 5888, 26752, 1152, 14464, 1152, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1920, 0, 768, 128, 7296, 128, 39552, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[8576, 128, 128, 0, 3840, 128, 0, 128, 36992, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 35968, 7040, 640, 768, 128, 4992, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[896, 2048, 0, 640, 0, 512, 1536, 128, 43904, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 768, 43776, 0, 128, 512, 128, 4352, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 309, 0, 0, 0, 0, 99, 110, 22, 0, 2029, 95, 0, 2, 0, 0, 0, 0, 0, 57, 0, 0, 425, 7871, 0, 1916, 0, 2535, 0, 0, 191, 1240, 0, 0, 0, 7433, 0, 255, 2405, 260, 182, 0, 326, 2930, 4079, 0, 0, 174, 0, 152, 0, 0, 0, 0, 17, 0, 6746, 6457, 740, 0, 0, 95] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 333, 0, 0, 0, 0, 132, 152, 72, 0, 2454, 69, 0, 0, 2, 0, 0, 0, 0, 79, 0, 0, 450, 7871, 0, 1418, 0, 3031, 0, 0, 138, 1172, 0, 0, 0, 7545, 0, 440, 2527, 127, 286, 0, 402, 2159, 3822, 0, 0, 78, 0, 215, 0, 0, 0, 0, 1, 0, 6686, 6589, 824, 0, 0, 78] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 332, 0, 0, 0, 0, 152, 172, 36, 0, 1997, 63, 0, 0, 1, 0, 0, 0, 0, 92, 0, 0, 554, 7905, 0, 1711, 0, 2930, 0, 0, 128, 1108, 0, 0, 0, 7579, 0, 340, 2869, 82, 233, 0, 370, 2379, 3604, 0, 0, 48, 0, 207, 0, 0, 0, 0, 13, 0, 6799, 6613, 772, 0, 0, 63] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 347, 0, 0, 0, 0, 136, 145, 33, 0, 2082, 71, 0, 3, 0, 0, 0, 0, 0, 76, 0, 0, 536, 7920, 0, 1584, 0, 2946, 0, 0, 145, 1107, 0, 0, 0, 7573, 0, 253, 2631, 187, 192, 0, 420, 2727, 3739, 0, 0, 81, 0, 163, 0, 0, 0, 0, 13, 0, 6846, 6421, 704, 0, 0, 71] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 352, 0, 0, 0, 0, 203, 214, 25, 0, 1998, 82, 0, 3, 0, 0, 0, 0, 0, 113, 0, 0, 464, 7838, 0, 1695, 0, 2860, 0, 0, 164, 1154, 0, 0, 0, 7458, 0, 343, 2351, 194, 289, 0, 339, 2470, 4150, 0, 0, 72, 0, 184, 0, 0, 0, 0, 43, 0, 6737, 6478, 797, 0, 0, 82] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 318, 0, 0, 0, 0, 153, 99, 20, 0, 1807, 90, 0, 2, 0, 0, 0, 0, 0, 87, 0, 0, 527, 7861, 0, 2016, 0, 2591, 0, 0, 180, 1457, 0, 0, 0, 7430, 0, 250, 2616, 277, 254, 0, 392, 2508, 3799, 0, 0, 161, 0, 168, 0, 0, 0, 0, 92, 0, 6819, 6383, 705, 0, 0, 90] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 342, 3, 0, 0, 5, 140, 150, 35, 2, 2241, 91, 3, 11, 2, 0, 0, 0, 7, 70, 0, 0, 467, 7834, 0, 1526, 3, 3071, 2, 2, 182, 1320, 0, 0, 0, 7469, 1, 392, 2535, 263, 271, 0, 362, 2149, 3745, 2, 0, 140, 2, 201, 0, 3, 0, 0, 2, 4, 6720, 6447, 842, 0, 0, 93] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 376, 0, 0, 0, 0, 132, 95, 36, 0, 2007, 75, 0, 10, 1, 0, 0, 0, 0, 69, 0, 0, 616, 7876, 0, 1556, 0, 2954, 0, 0, 150, 1232, 0, 0, 0, 7563, 0, 341, 2680, 194, 276, 0, 377, 2241, 3886, 0, 0, 64, 0, 143, 0, 0, 0, 0, 10, 0, 6835, 6561, 716, 0, 0, 80] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 332, 0, 0, 0, 0, 3321, 1631, 633, 0, 0, 0, 0, 0, 152, 172, 36, 0, 1997, 63, 0, 0, 2468, 1240, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 92, 0, 0, 6087, 0, 0, 0, 0, 0, 0, 0, 554, 1101, 0, 1711, 0, 2930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 1108, 0, 0, 0, 4478, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 2869, 82, 233, 0, 370, 2379, 283, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 207, 0, 0, 0, 0, 5168, 717, 0, 0, 0, 0, 0, 0, 13, 0, 0, 5373, 772, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5917, 6128, 6180, 6296, 6054, 6216, 6140, 6221] avg=6144.0000 max=6296 max_rank=3 min=5917 min_rank=0 imbalance_ratio=1.024740 max_load_reduction=42856 max_load_reduction_pct=0.871908 moved_to_echo=24631 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 309, 0, 0, 0, 0, 3759, 1619, 621, 0, 0, 0, 0, 0, 99, 110, 22, 0, 2029, 95, 0, 2, 2420, 1211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 6061, 0, 0, 0, 0, 0, 0, 0, 425, 1096, 0, 1916, 0, 2535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 191, 1240, 0, 0, 0, 4392, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 2405, 260, 182, 0, 326, 2930, 320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 174, 0, 152, 0, 0, 0, 0, 5127, 714, 0, 0, 0, 0, 0, 0, 17, 0, 0, 5246, 740, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6308, 5988, 6118, 5972, 6078, 6423, 6167, 6098] avg=6144.0000 max=6423 max_rank=5 min=5972 min_rank=3 imbalance_ratio=1.045410 max_load_reduction=42729 max_load_reduction_pct=0.869324 moved_to_echo=24766 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 333, 0, 0, 0, 0, 3522, 1605, 631, 0, 0, 0, 0, 0, 132, 152, 72, 0, 2454, 69, 0, 0, 2457, 1235, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 79, 0, 0, 6061, 0, 0, 0, 0, 0, 0, 0, 450, 1096, 0, 1418, 0, 3031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 1172, 0, 0, 0, 4457, 0, 440, 0, 0, 0, 0, 0, 0, 0, 0, 2527, 127, 286, 0, 402, 2159, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 215, 0, 0, 0, 0, 5081, 714, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5354, 824, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6091, 6571, 6142, 5995, 6207, 5801, 6088, 6257] avg=6144.0000 max=6571 max_rank=1 min=5801 min_rank=5 imbalance_ratio=1.069499 max_load_reduction=42581 max_load_reduction_pct=0.866313 moved_to_echo=24353 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 318, 0, 0, 0, 0, 3500, 1636, 621, 0, 0, 0, 0, 0, 153, 99, 20, 0, 1807, 90, 0, 2, 2419, 1197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 6053, 0, 0, 0, 0, 0, 0, 0, 527, 1095, 0, 2016, 0, 2591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, 1457, 0, 0, 0, 4390, 0, 250, 0, 0, 0, 0, 0, 0, 0, 0, 2616, 277, 254, 0, 392, 2508, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 0, 168, 0, 0, 0, 0, 5183, 713, 0, 0, 0, 0, 0, 0, 92, 0, 0, 5186, 705, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6075, 5787, 6140, 6229, 6277, 6346, 6225, 6073] avg=6144.0000 max=6346 max_rank=5 min=5787 min_rank=1 imbalance_ratio=1.032878 max_load_reduction=42806 max_load_reduction_pct=0.870890 moved_to_echo=24921 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 347, 0, 0, 0, 0, 3445, 1644, 633, 0, 0, 0, 0, 0, 136, 145, 33, 0, 2082, 71, 0, 3, 2466, 1204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 6098, 0, 0, 0, 0, 0, 0, 0, 536, 1104, 0, 1584, 0, 2946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 1107, 0, 0, 0, 4474, 0, 253, 0, 0, 0, 0, 0, 0, 0, 0, 2631, 187, 192, 0, 420, 2727, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 163, 0, 0, 0, 0, 5202, 718, 0, 0, 0, 0, 0, 0, 13, 0, 0, 5217, 704, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6069, 6140, 6174, 6170, 5979, 6451, 6164, 6005] avg=6144.0000 max=6451 max_rank=5 min=5979 min_rank=4 imbalance_ratio=1.049967 max_load_reduction=42701 max_load_reduction_pct=0.868754 moved_to_echo=24599 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 352, 0, 0, 0, 0, 3824, 1617, 623, 0, 0, 0, 0, 0, 203, 214, 25, 0, 1998, 82, 0, 3, 2429, 1215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 0, 0, 6035, 0, 0, 0, 0, 0, 0, 0, 464, 1092, 0, 1695, 0, 2860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 1154, 0, 0, 0, 4406, 0, 343, 0, 0, 0, 0, 0, 0, 0, 0, 2351, 194, 289, 0, 339, 2470, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 184, 0, 0, 0, 0, 5120, 711, 0, 0, 0, 0, 0, 0, 43, 0, 0, 5263, 797, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6416, 6169, 6148, 6111, 6067, 5969, 6087, 6185] avg=6144.0000 max=6416 max_rank=0 min=5969 min_rank=5 imbalance_ratio=1.044271 max_load_reduction=42736 max_load_reduction_pct=0.869466 moved_to_echo=24308 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 376, 0, 0, 0, 0, 3581, 1641, 632, 0, 0, 0, 0, 0, 132, 95, 36, 0, 2007, 75, 0, 10, 2463, 1230, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 69, 0, 0, 6065, 0, 0, 0, 0, 0, 0, 0, 616, 1097, 0, 1556, 0, 2954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 1232, 0, 0, 0, 4468, 0, 341, 0, 0, 0, 0, 0, 0, 0, 0, 2680, 194, 276, 0, 377, 2241, 305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 143, 0, 0, 0, 0, 5194, 714, 0, 0, 0, 0, 0, 0, 10, 0, 0, 5331, 716, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6230, 6048, 6135, 6223, 6191, 6073, 6115, 6137] avg=6144.0000 max=6230 max_rank=0 min=6048 min_rank=1 imbalance_ratio=1.013997 max_load_reduction=42922 max_load_reduction_pct=0.873250 moved_to_echo=24516 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 342, 3, 0, 0, 5, 3454, 1613, 629, 0, 0, 0, 0, 0, 140, 150, 35, 2, 2241, 91, 3, 11, 2436, 1212, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 7, 70, 0, 0, 6036, 0, 0, 0, 0, 0, 0, 0, 467, 1085, 0, 1526, 3, 3071, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 182, 1320, 0, 0, 0, 4404, 1, 392, 0, 0, 0, 0, 0, 0, 0, 0, 2535, 263, 271, 0, 362, 2149, 291, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 2, 201, 0, 3, 0, 0, 5107, 713, 0, 0, 0, 0, 0, 0, 2, 4, 0, 5235, 842, 0, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6046, 6321, 6115, 6156, 6299, 5873, 6166, 6176] avg=6144.0000 max=6321 max_rank=1 min=5873 min_rank=5 imbalance_ratio=1.028809 max_load_reduction=42831 max_load_reduction_pct=0.871399 moved_to_echo=24514 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1152, 1152, 384, 128, 16640, 640, 128, 128, 19584, 9856, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 0, 2816, 128, 0, 0, 128, 28416, 13056, 5120, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4096, 8832, 0, 13440, 128, 23040, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[20736, 1664, 2048, 0, 3072, 19584, 2432, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[1280, 9856, 0, 0, 0, 35584, 128, 2688, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 896, 128, 1536, 0, 128, 0, 0, 41216, 5760, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[256, 128, 0, 42240, 6144, 0, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 128, 768, 0, 0, 48512, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 262, 79, 0, 7597, 0, 414, 0, 0, 1, 0, 69, 0, 0, 0, 646, 0, 0, 0, 0, 5339, 0, 7219, 0, 0, 79, 7562, 0, 0, 0, 53, 1292, 0, 0, 534, 0, 0, 148, 136, 201, 0, 0, 2923, 0, 0, 0, 0, 6424, 0, 751, 0, 6067, 0, 0, 0, 0, 78, 0, 399, 719, 91, 0, 69] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 254, 90, 0, 7641, 0, 432, 0, 0, 0, 0, 96, 0, 0, 0, 466, 0, 0, 0, 0, 5427, 0, 7256, 0, 0, 60, 7614, 0, 0, 0, 41, 973, 0, 1, 629, 0, 0, 152, 168, 194, 0, 0, 3367, 0, 0, 0, 0, 6528, 0, 651, 0, 5617, 0, 0, 0, 0, 174, 0, 486, 562, 177, 0, 96] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 254, 96, 0, 7585, 0, 399, 0, 0, 0, 0, 65, 0, 0, 0, 475, 0, 0, 0, 0, 5374, 0, 7267, 0, 0, 66, 7695, 0, 0, 0, 60, 1228, 0, 0, 515, 0, 0, 155, 140, 215, 0, 0, 3180, 0, 0, 0, 0, 6561, 0, 679, 0, 5988, 0, 0, 0, 0, 48, 0, 382, 595, 65, 0, 65] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 239, 98, 0, 7684, 0, 419, 0, 0, 0, 0, 74, 0, 0, 0, 477, 0, 0, 0, 0, 5303, 0, 7182, 0, 0, 74, 7675, 0, 0, 0, 60, 1091, 0, 0, 523, 0, 0, 147, 166, 207, 0, 0, 3572, 0, 0, 0, 0, 6463, 0, 733, 0, 5761, 0, 0, 0, 0, 81, 0, 388, 565, 96, 0, 74] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 247, 151, 0, 7644, 0, 393, 0, 0, 0, 0, 90, 0, 0, 0, 443, 0, 0, 0, 0, 5303, 0, 7151, 0, 0, 89, 7593, 0, 0, 0, 65, 1060, 0, 1, 646, 0, 0, 177, 149, 243, 0, 0, 3602, 0, 0, 0, 0, 6498, 0, 658, 0, 5555, 0, 0, 0, 0, 161, 0, 442, 526, 175, 0, 90] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[1, 0, 249, 171, 0, 7536, 3, 434, 0, 0, 0, 0, 91, 4, 0, 0, 548, 0, 0, 0, 0, 4976, 3, 7119, 0, 0, 46, 7567, 0, 1, 0, 55, 1154, 9, 1, 597, 0, 4, 160, 161, 214, 5, 8, 3372, 0, 0, 6, 1, 6483, 0, 763, 0, 6048, 0, 0, 1, 1, 140, 0, 432, 541, 153, 1, 93] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 264, 116, 0, 7576, 0, 442, 0, 0, 0, 0, 82, 0, 0, 0, 543, 0, 0, 0, 0, 5397, 0, 7111, 0, 0, 63, 7673, 0, 0, 0, 90, 1081, 0, 0, 574, 0, 0, 195, 126, 285, 0, 0, 3283, 0, 0, 0, 0, 6444, 0, 733, 0, 5770, 0, 0, 0, 0, 72, 0, 440, 630, 80, 0, 82] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 246, 112, 0, 7554, 0, 418, 0, 0, 0, 0, 75, 0, 0, 0, 509, 0, 0, 0, 0, 5127, 0, 7219, 0, 0, 81, 7654, 0, 0, 0, 63, 1271, 0, 0, 497, 0, 0, 144, 144, 207, 0, 0, 3450, 0, 0, 0, 0, 6537, 0, 753, 0, 5915, 0, 0, 0, 0, 64, 0, 398, 560, 79, 0, 75] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 254, 90, 0, 5387, 0, 432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 0, 6118, 0, 0, 0, 0, 0, 0, 0, 466, 0, 0, 0, 0, 5427, 0, 354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 6002, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 973, 0, 1, 629, 0, 0, 152, 168, 2445, 1702, 0, 0, 0, 0, 0, 0, 194, 0, 0, 3367, 0, 0, 0, 0, 1612, 552, 395, 0, 0, 0, 0, 0, 0, 0, 651, 0, 5222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 174, 0, 486, 562, 177, 0, 96, 4083, 784, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6163, 6214, 6247, 6103, 6070, 6120, 5873, 6362] avg=6144.0000 max=6362 max_rank=7 min=5873 min_rank=6 imbalance_ratio=1.035482 max_load_reduction=42790 max_load_reduction_pct=0.870565 moved_to_echo=24425 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 247, 151, 0, 5390, 0, 393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 6029, 0, 0, 0, 0, 0, 0, 0, 443, 0, 0, 0, 0, 5303, 0, 349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 5985, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 0, 1, 646, 0, 0, 177, 149, 2433, 1702, 0, 0, 0, 0, 0, 0, 243, 0, 0, 3602, 0, 0, 0, 0, 1608, 552, 391, 0, 0, 0, 0, 0, 0, 0, 658, 0, 5164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 0, 442, 526, 175, 0, 90, 4065, 773, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6181, 6119, 6095, 6139, 6168, 6396, 5822, 6232] avg=6144.0000 max=6396 max_rank=5 min=5822 min_rank=6 imbalance_ratio=1.041016 max_load_reduction=42756 max_load_reduction_pct=0.869873 moved_to_echo=24618 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 262, 79, 0, 5356, 0, 414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 69, 0, 0, 0, 6087, 0, 0, 0, 0, 0, 0, 0, 646, 0, 0, 0, 0, 5339, 0, 352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 5961, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 1292, 0, 0, 534, 0, 0, 148, 136, 2407, 1692, 0, 0, 0, 0, 0, 0, 201, 0, 0, 2923, 0, 0, 0, 0, 1601, 549, 427, 0, 0, 0, 0, 0, 0, 0, 751, 0, 5640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 399, 719, 91, 0, 69, 4017, 780, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6111, 6157, 6337, 6093, 6209, 5701, 6391, 6153] avg=6144.0000 max=6391 max_rank=6 min=5701 min_rank=5 imbalance_ratio=1.040202 max_load_reduction=42761 max_load_reduction_pct=0.869975 moved_to_echo=24454 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 254, 96, 0, 5348, 0, 399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 6127, 0, 0, 0, 0, 0, 0, 0, 475, 0, 0, 0, 0, 5374, 0, 354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 6066, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 1228, 0, 0, 515, 0, 0, 155, 140, 2457, 1689, 0, 0, 0, 0, 0, 0, 215, 0, 0, 3180, 0, 0, 0, 0, 1629, 548, 421, 0, 0, 0, 0, 0, 0, 0, 679, 0, 5567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 382, 595, 65, 0, 65, 4104, 786, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6097, 6192, 6203, 6192, 6184, 5993, 6246, 6045] avg=6144.0000 max=6246 max_rank=6 min=5993 min_rank=5 imbalance_ratio=1.016602 max_load_reduction=42906 max_load_reduction_pct=0.872925 moved_to_echo=24468 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 239, 98, 0, 5418, 0, 419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 6055, 0, 0, 0, 0, 0, 0, 0, 477, 0, 0, 0, 0, 5303, 0, 351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 6050, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 1091, 0, 0, 523, 0, 0, 147, 166, 2421, 1711, 0, 0, 0, 0, 0, 0, 207, 0, 0, 3572, 0, 0, 0, 0, 1625, 555, 405, 0, 0, 0, 0, 0, 0, 0, 733, 0, 5356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 388, 565, 96, 0, 74, 4042, 776, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6174, 6129, 6131, 6184, 6059, 6364, 6089, 6022] avg=6144.0000 max=6364 max_rank=5 min=6022 min_rank=7 imbalance_ratio=1.035807 max_load_reduction=42788 max_load_reduction_pct=0.870524 moved_to_echo=24534 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 264, 116, 0, 5342, 0, 442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 5996, 0, 0, 0, 0, 0, 0, 0, 543, 0, 0, 0, 0, 5397, 0, 346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 6048, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 1081, 0, 0, 574, 0, 0, 195, 126, 2414, 1687, 0, 0, 0, 0, 0, 0, 285, 0, 0, 3283, 0, 0, 0, 0, 1625, 547, 406, 0, 0, 0, 0, 0, 0, 0, 733, 0, 5364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 440, 630, 80, 0, 82, 4030, 769, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6164, 6078, 6286, 6201, 6077, 6146, 6097, 6103] avg=6144.0000 max=6286 max_rank=2 min=6077 min_rank=4 imbalance_ratio=1.023112 max_load_reduction=42866 max_load_reduction_pct=0.872111 moved_to_echo=24423 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 249, 171, 0, 5302, 3, 434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 4, 0, 0, 6006, 0, 0, 0, 0, 0, 0, 0, 548, 0, 0, 0, 0, 4976, 3, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 5961, 0, 1, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 1154, 9, 1, 597, 0, 4, 160, 161, 2429, 1684, 0, 0, 0, 0, 0, 0, 214, 5, 8, 3372, 0, 0, 6, 1, 1606, 550, 430, 0, 0, 0, 0, 0, 0, 0, 763, 0, 5618, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 140, 0, 432, 541, 153, 1, 93, 4054, 775, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6160, 6101, 5865, 6063, 6199, 6192, 6382, 6190] avg=6144.0000 max=6382 max_rank=6 min=5865 min_rank=2 imbalance_ratio=1.038737 max_load_reduction=42770 max_load_reduction_pct=0.870158 moved_to_echo=24963 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 246, 112, 0, 5326, 0, 418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 6087, 0, 0, 0, 0, 0, 0, 0, 509, 0, 0, 0, 0, 5127, 0, 352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 6033, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 1271, 0, 0, 497, 0, 0, 144, 144, 2448, 1682, 0, 0, 0, 0, 0, 0, 207, 0, 0, 3450, 0, 0, 0, 0, 1621, 546, 416, 0, 0, 0, 0, 0, 0, 0, 753, 0, 5499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 398, 560, 79, 0, 75, 4089, 780, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6102, 6162, 5988, 6177, 6186, 6240, 6252, 6045] avg=6144.0000 max=6252 max_rank=6 min=5988 min_rank=2 imbalance_ratio=1.017578 max_load_reduction=42900 max_load_reduction_pct=0.872803 moved_to_echo=24723 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[9216, 128, 128, 4608, 0, 128, 1280, 1280, 19456, 13568, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1792, 128, 128, 26752, 0, 0, 128, 128, 12928, 4480, 3328, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 896, 0, 3456, 4736, 1024, 128, 768, 32512, 6272, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4224, 0, 0, 0, 0, 42368, 128, 2816, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 5760, 0, 43520, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 0, 2048, 1024, 0, 42880, 128, 3456, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 640, 48128, 0, 128, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 768, 128, 0, 0, 48512, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[6100, 0, 0, 0, 66, 0, 0, 0, 0, 322, 0, 0, 90, 0, 665, 5732, 25, 7236, 0, 90, 0, 0, 60, 0, 7512, 0, 0, 246, 0, 3554, 90, 7651, 2536, 90, 0, 0, 0, 0, 0, 0, 33, 0, 223, 0, 0, 4893, 247, 0, 0, 0, 0, 1589, 0, 0, 0, 0, 0, 102, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[5281, 0, 0, 0, 53, 0, 0, 0, 0, 455, 0, 0, 69, 0, 691, 4843, 66, 7329, 0, 69, 0, 0, 66, 0, 7580, 0, 0, 261, 0, 3818, 69, 7546, 4363, 69, 0, 0, 0, 0, 0, 0, 69, 0, 252, 0, 0, 4210, 261, 0, 0, 0, 0, 1574, 0, 0, 0, 0, 0, 158, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[6227, 0, 0, 0, 42, 0, 0, 0, 0, 374, 0, 0, 95, 0, 585, 5447, 4, 7301, 0, 96, 0, 0, 72, 0, 7473, 0, 1, 254, 0, 3638, 95, 7517, 2873, 95, 0, 0, 0, 0, 0, 0, 29, 0, 291, 0, 0, 4791, 254, 0, 0, 0, 0, 1451, 0, 0, 0, 0, 0, 133, 14, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[6125, 0, 0, 0, 60, 0, 0, 0, 0, 375, 0, 0, 74, 0, 732, 5612, 6, 7320, 0, 74, 0, 0, 92, 0, 7540, 0, 0, 239, 0, 3709, 71, 7596, 2687, 71, 0, 0, 0, 0, 0, 0, 29, 0, 221, 0, 0, 4830, 239, 0, 0, 0, 0, 1325, 0, 0, 0, 0, 0, 125, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[5777, 0, 0, 0, 60, 0, 0, 0, 0, 312, 0, 0, 65, 0, 652, 5237, 43, 7391, 0, 65, 0, 0, 75, 0, 7610, 0, 0, 254, 0, 4097, 63, 7638, 3199, 63, 0, 0, 0, 0, 0, 0, 56, 0, 195, 0, 0, 4389, 254, 0, 0, 0, 0, 1536, 0, 0, 0, 0, 0, 121, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[6036, 0, 0, 0, 90, 0, 0, 0, 0, 352, 0, 0, 80, 0, 592, 5494, 3, 7344, 0, 82, 0, 0, 44, 0, 7542, 0, 0, 264, 0, 4059, 82, 7703, 2989, 82, 0, 0, 0, 0, 0, 0, 52, 0, 196, 0, 0, 4267, 264, 0, 0, 0, 0, 1402, 0, 0, 0, 0, 0, 133, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[5553, 0, 0, 0, 54, 0, 1, 0, 2, 397, 0, 0, 91, 0, 643, 5267, 62, 7290, 2, 91, 0, 0, 76, 0, 7483, 5, 2, 249, 3, 3584, 92, 7607, 3776, 91, 0, 0, 1, 1, 0, 0, 91, 0, 225, 4, 0, 4450, 249, 5, 0, 0, 0, 1537, 2, 0, 0, 0, 5, 155, 1, 0, 1, 4, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[5677, 0, 0, 0, 63, 0, 0, 0, 0, 378, 0, 0, 75, 0, 688, 5414, 42, 7357, 0, 75, 0, 0, 69, 0, 7577, 0, 0, 246, 0, 3947, 75, 7612, 3075, 75, 0, 0, 0, 0, 0, 0, 74, 0, 207, 0, 0, 4565, 246, 0, 0, 0, 0, 1476, 0, 0, 0, 0, 0, 139, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[6100, 0, 0, 0, 66, 0, 0, 0, 251, 0, 0, 0, 0, 0, 0, 0, 0, 322, 0, 0, 90, 0, 665, 5367, 0, 0, 0, 0, 0, 0, 0, 0, 25, 5895, 0, 90, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2004, 0, 0, 246, 0, 3554, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2536, 90, 0, 0, 0, 0, 0, 0, 2463, 399, 0, 0, 0, 0, 0, 0, 33, 0, 223, 0, 0, 4893, 247, 0, 942, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1589, 0, 0, 0, 0, 3045, 1610, 0, 0, 0, 0, 0, 0, 0, 102, 0, 0, 0, 0, 0, 0, 6041, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6417, 6444, 6070, 5894, 5488, 6452, 6244, 6143] avg=6144.0000 max=6452 max_rank=5 min=5488 min_rank=4 imbalance_ratio=1.050130 max_load_reduction=42700 max_load_reduction_pct=0.868734 moved_to_echo=24327 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[6227, 0, 0, 0, 42, 0, 0, 0, 238, 0, 0, 0, 0, 0, 0, 0, 0, 374, 0, 0, 95, 0, 585, 5101, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5947, 0, 96, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1994, 0, 1, 254, 0, 3638, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2873, 95, 0, 0, 0, 0, 0, 0, 2450, 403, 0, 0, 0, 0, 0, 0, 29, 0, 291, 0, 0, 4791, 254, 0, 951, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1451, 0, 0, 0, 0, 3029, 1581, 0, 0, 0, 0, 0, 0, 0, 133, 14, 0, 0, 0, 0, 0, 5936, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6507, 6155, 6119, 5982, 5821, 6424, 6061, 6083] avg=6144.0000 max=6507 max_rank=0 min=5821 min_rank=4 imbalance_ratio=1.059082 max_load_reduction=42645 max_load_reduction_pct=0.867615 moved_to_echo=24389 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[5777, 0, 0, 0, 60, 0, 0, 0, 229, 0, 0, 0, 0, 0, 0, 0, 0, 312, 0, 0, 65, 0, 652, 4904, 0, 0, 0, 0, 0, 0, 0, 0, 43, 6021, 0, 65, 0, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2031, 0, 0, 254, 0, 4097, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3199, 63, 0, 0, 0, 0, 0, 0, 2495, 408, 0, 0, 0, 0, 0, 0, 56, 0, 195, 0, 0, 4389, 254, 0, 962, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1536, 0, 0, 0, 0, 3084, 1607, 0, 0, 0, 0, 0, 0, 0, 121, 0, 0, 0, 0, 0, 0, 6031, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6066, 5933, 6204, 6445, 6165, 5960, 6227, 6152] avg=6144.0000 max=6445 max_rank=3 min=5933 min_rank=1 imbalance_ratio=1.048991 max_load_reduction=42707 max_load_reduction_pct=0.868876 moved_to_echo=24504 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[5281, 0, 0, 0, 53, 0, 0, 0, 212, 0, 0, 0, 0, 0, 0, 0, 0, 455, 0, 0, 69, 0, 691, 4535, 0, 0, 0, 0, 0, 0, 0, 0, 66, 5971, 0, 69, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2023, 0, 0, 261, 0, 3818, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4363, 69, 0, 0, 0, 0, 0, 0, 2485, 404, 0, 0, 0, 0, 0, 0, 69, 0, 252, 0, 0, 4210, 261, 0, 954, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1574, 0, 0, 0, 0, 3072, 1588, 0, 0, 0, 0, 0, 0, 0, 158, 0, 0, 0, 0, 0, 0, 5958, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5546, 5750, 6172, 6171, 7321, 5842, 6234, 6116] avg=6144.0000 max=7321 max_rank=4 min=5546 min_rank=0 imbalance_ratio=1.191569 max_load_reduction=41831 max_load_reduction_pct=0.851054 moved_to_echo=25513 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[6125, 0, 0, 0, 60, 0, 0, 0, 246, 0, 0, 0, 0, 0, 0, 0, 0, 375, 0, 0, 74, 0, 732, 5254, 0, 0, 0, 0, 0, 0, 0, 0, 6, 5963, 0, 74, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2012, 0, 0, 239, 0, 3709, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2687, 71, 0, 0, 0, 0, 0, 0, 2472, 404, 0, 0, 0, 0, 0, 0, 29, 0, 221, 0, 0, 4830, 239, 0, 953, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1325, 0, 0, 0, 0, 3056, 1599, 0, 0, 0, 0, 0, 0, 0, 125, 0, 0, 0, 0, 0, 0, 5997, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6431, 6435, 6135, 6031, 5634, 6384, 5980, 6122] avg=6144.0000 max=6435 max_rank=1 min=5634 min_rank=4 imbalance_ratio=1.047363 max_load_reduction=42717 max_load_reduction_pct=0.869080 moved_to_echo=24120 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[6036, 0, 0, 0, 90, 0, 0, 0, 240, 0, 0, 0, 0, 0, 0, 0, 0, 352, 0, 0, 80, 0, 592, 5145, 0, 0, 0, 0, 0, 0, 0, 0, 3, 5983, 0, 82, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2012, 0, 0, 264, 0, 4059, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2989, 82, 0, 0, 0, 0, 0, 0, 2473, 405, 0, 0, 0, 0, 0, 0, 52, 0, 196, 0, 0, 4267, 264, 0, 956, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1402, 0, 0, 0, 0, 3057, 1621, 0, 0, 0, 0, 0, 0, 0, 133, 0, 0, 0, 0, 0, 0, 6082, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6366, 6169, 6112, 6417, 5949, 5844, 6080, 6215] avg=6144.0000 max=6417 max_rank=3 min=5844 min_rank=5 imbalance_ratio=1.044434 max_load_reduction=42735 max_load_reduction_pct=0.869446 moved_to_echo=24088 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[5553, 0, 0, 0, 54, 0, 1, 0, 234, 0, 0, 0, 0, 0, 0, 0, 2, 397, 0, 0, 91, 0, 643, 4923, 0, 0, 0, 0, 0, 0, 0, 0, 62, 5930, 2, 91, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1987, 5, 2, 249, 3, 3584, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3776, 91, 0, 0, 1, 1, 0, 0, 2459, 406, 0, 0, 0, 0, 0, 0, 91, 0, 225, 4, 0, 4450, 249, 5, 954, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1537, 2, 0, 0, 0, 3037, 1601, 0, 0, 0, 0, 0, 0, 5, 155, 1, 0, 1, 4, 0, 0, 6006, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5842, 6056, 6161, 5922, 6734, 6088, 6177, 6172] avg=6144.0000 max=6734 max_rank=4 min=5842 min_rank=0 imbalance_ratio=1.096029 max_load_reduction=42418 max_load_reduction_pct=0.862996 moved_to_echo=25171 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[5677, 0, 0, 0, 63, 0, 0, 0, 237, 0, 0, 0, 0, 0, 0, 0, 0, 378, 0, 0, 75, 0, 688, 5069, 0, 0, 0, 0, 0, 0, 0, 0, 42, 5993, 0, 75, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2022, 0, 0, 246, 0, 3947, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3075, 75, 0, 0, 0, 0, 0, 0, 2484, 406, 0, 0, 0, 0, 0, 0, 74, 0, 207, 0, 0, 4565, 246, 0, 958, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1476, 0, 0, 0, 0, 3071, 1602, 0, 0, 0, 0, 0, 0, 0, 139, 0, 0, 0, 0, 0, 0, 6010, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5977, 6210, 6179, 6290, 6040, 6158, 6149, 6149] avg=6144.0000 max=6290 max_rank=3 min=5977 min_rank=0 imbalance_ratio=1.023763 max_load_reduction=42862 max_load_reduction_pct=0.872030 moved_to_echo=24496 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[25600, 640, 0, 0, 128, 128, 0, 0, 19840, 3328, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[16128, 128, 128, 2048, 128, 30464, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[512, 0, 1920, 128, 0, 36480, 2048, 128, 7680, 896, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 0, 11904, 128, 0, 0, 0, 24576, 12928, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[128, 3072, 0, 0, 640, 0, 5248, 40320, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[256, 47744, 128, 768, 0, 0, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[46848, 0, 0, 0, 512, 0, 128, 0, 1920, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 1152, 128, 0, 128, 128, 0, 0, 48128, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 530, 0, 0, 911, 0, 0, 3795, 7836, 0, 0, 0, 729, 0, 87, 335, 0, 137, 1973, 0, 0, 0, 1109, 0, 0, 330, 0, 0, 0, 0, 0, 1, 0, 0, 7379, 155, 0, 90, 357, 0, 1608, 0, 0, 0, 0, 0, 2125, 0, 4, 0, 7846, 1, 85, 0, 4749, 0, 0, 0, 6146, 649, 185, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 516, 0, 0, 889, 0, 0, 4082, 7799, 0, 0, 0, 695, 0, 92, 346, 0, 153, 1888, 0, 0, 0, 1154, 0, 0, 286, 0, 0, 0, 0, 0, 0, 0, 0, 7450, 126, 0, 63, 485, 0, 1837, 0, 0, 0, 0, 0, 1869, 0, 0, 0, 7866, 0, 92, 0, 4411, 0, 0, 0, 6335, 685, 33, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 600, 0, 0, 987, 0, 0, 3857, 7822, 0, 0, 0, 734, 0, 76, 318, 0, 135, 1754, 0, 0, 0, 1130, 0, 0, 303, 0, 0, 0, 0, 0, 0, 0, 0, 7408, 135, 0, 71, 449, 0, 1900, 0, 0, 0, 0, 0, 1896, 0, 1, 0, 7859, 0, 68, 0, 4622, 0, 0, 0, 6148, 742, 137, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 544, 0, 0, 900, 0, 0, 4359, 7811, 0, 0, 0, 733, 0, 79, 345, 0, 132, 1318, 0, 0, 0, 1115, 0, 0, 344, 0, 0, 0, 0, 0, 0, 0, 0, 7444, 123, 0, 69, 207, 0, 1942, 0, 0, 0, 0, 0, 2104, 0, 0, 0, 7866, 0, 76, 0, 4365, 0, 0, 0, 6257, 706, 313, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 478, 0, 0, 924, 0, 0, 3887, 7848, 0, 0, 0, 676, 0, 57, 313, 0, 98, 1993, 0, 0, 0, 1043, 0, 0, 333, 0, 0, 0, 0, 0, 0, 0, 0, 7403, 140, 0, 95, 349, 0, 1894, 0, 0, 0, 0, 0, 1938, 0, 3, 0, 7827, 1, 57, 0, 4870, 0, 0, 0, 6086, 683, 156, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 472, 0, 0, 995, 0, 0, 4142, 7770, 0, 0, 0, 686, 0, 113, 380, 0, 183, 1859, 0, 0, 0, 1057, 0, 0, 343, 0, 0, 0, 0, 0, 0, 0, 0, 7430, 172, 0, 82, 336, 0, 1765, 0, 0, 0, 0, 0, 2016, 0, 0, 0, 7801, 0, 109, 0, 4412, 0, 0, 0, 6165, 720, 144, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[1, 0, 0, 523, 1, 1, 850, 0, 0, 4509, 7786, 0, 4, 7, 700, 0, 69, 328, 4, 124, 1419, 0, 0, 0, 1363, 0, 0, 340, 1, 1, 0, 0, 0, 0, 4, 3, 7355, 145, 1, 91, 443, 3, 1889, 0, 1, 0, 0, 7, 2109, 2, 3, 0, 7845, 2, 69, 0, 4230, 0, 2, 0, 6148, 700, 69, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 538, 0, 0, 915, 0, 0, 4300, 7809, 0, 0, 0, 709, 0, 69, 317, 0, 132, 1524, 0, 0, 0, 1245, 0, 0, 325, 0, 0, 0, 0, 0, 0, 0, 0, 7433, 139, 0, 75, 463, 0, 1776, 0, 0, 0, 0, 0, 2139, 0, 0, 0, 7902, 0, 69, 0, 4212, 0, 0, 0, 6253, 747, 61, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 530, 0, 0, 911, 0, 4712, 0, 0, 0, 0, 0, 0, 0, 0, 3795, 1324, 0, 0, 0, 729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 335, 0, 137, 1973, 0, 0, 0, 2382, 1484, 0, 0, 0, 0, 0, 0, 1109, 0, 0, 330, 0, 0, 0, 0, 2850, 1800, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5895, 155, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 357, 0, 1608, 0, 0, 0, 0, 0, 2483, 1427, 0, 0, 0, 0, 0, 0, 2125, 0, 4, 0, 4037, 1, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4749, 0, 0, 0, 813, 649, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6153, 5848, 6398, 6089, 6141, 5875, 6252, 6396] avg=6144.0000 max=6398 max_rank=2 min=5848 min_rank=1 imbalance_ratio=1.041341 max_load_reduction=42754 max_load_reduction_pct=0.869832 moved_to_echo=24664 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 516, 0, 0, 889, 0, 4690, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 1318, 0, 0, 0, 695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 346, 0, 153, 1888, 0, 0, 0, 2388, 1498, 0, 0, 0, 0, 0, 0, 1154, 0, 0, 286, 0, 0, 0, 0, 2937, 1791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5952, 126, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 485, 0, 1837, 0, 0, 0, 0, 0, 2559, 1430, 0, 0, 0, 0, 0, 0, 1869, 0, 0, 0, 4048, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4411, 0, 0, 0, 839, 685, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6095, 6095, 6365, 6168, 6141, 6311, 6009, 5968] avg=6144.0000 max=6365 max_rank=2 min=5968 min_rank=7 imbalance_ratio=1.035970 max_load_reduction=42787 max_load_reduction_pct=0.870504 moved_to_echo=24429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 478, 0, 0, 924, 0, 4719, 0, 0, 0, 0, 0, 0, 0, 0, 3887, 1326, 0, 0, 0, 676, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 313, 0, 98, 1993, 0, 0, 0, 2376, 1489, 0, 0, 0, 0, 0, 0, 1043, 0, 0, 333, 0, 0, 0, 0, 2822, 1803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5914, 140, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 349, 0, 1894, 0, 0, 0, 0, 0, 2458, 1423, 0, 0, 0, 0, 0, 0, 1938, 0, 3, 0, 4028, 1, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4870, 0, 0, 0, 806, 683, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6121, 5889, 6326, 6001, 6149, 6124, 6027, 6515] avg=6144.0000 max=6515 max_rank=7 min=5889 min_rank=1 imbalance_ratio=1.060384 max_load_reduction=42637 max_load_reduction_pct=0.867452 moved_to_echo=24815 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 544, 0, 0, 900, 0, 4697, 0, 0, 0, 0, 0, 0, 0, 0, 4359, 1320, 0, 0, 0, 733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 345, 0, 132, 1318, 0, 0, 0, 2388, 1497, 0, 0, 0, 0, 0, 0, 1115, 0, 0, 344, 0, 0, 0, 0, 2901, 1794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5947, 123, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 207, 0, 1942, 0, 0, 0, 0, 0, 2528, 1430, 0, 0, 0, 0, 0, 0, 2104, 0, 0, 0, 4048, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4365, 0, 0, 0, 828, 706, 313, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6141, 6412, 5759, 6154, 6139, 6107, 6228, 6212] avg=6144.0000 max=6412 max_rank=1 min=5759 min_rank=2 imbalance_ratio=1.043620 max_load_reduction=42740 max_load_reduction_pct=0.869548 moved_to_echo=24686 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 600, 0, 0, 987, 0, 4704, 0, 0, 0, 0, 0, 0, 0, 0, 3857, 1321, 0, 0, 0, 734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 318, 0, 135, 1754, 0, 0, 0, 2386, 1490, 0, 0, 0, 0, 0, 0, 1130, 0, 0, 303, 0, 0, 0, 0, 2851, 1797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5918, 135, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 449, 0, 1900, 0, 0, 0, 0, 0, 2483, 1429, 0, 0, 0, 0, 0, 0, 1896, 0, 1, 0, 4044, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4622, 0, 0, 0, 814, 742, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6291, 5912, 6159, 6081, 6124, 6261, 6009, 6315] avg=6144.0000 max=6315 max_rank=7 min=5912 min_rank=1 imbalance_ratio=1.027832 max_load_reduction=42837 max_load_reduction_pct=0.871521 moved_to_echo=24709 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 472, 0, 0, 995, 0, 4672, 0, 0, 0, 0, 0, 0, 0, 0, 4142, 1313, 0, 0, 0, 686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 380, 0, 183, 1859, 0, 0, 0, 2368, 1494, 0, 0, 0, 0, 0, 0, 1057, 0, 0, 343, 0, 0, 0, 0, 2859, 1785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5936, 172, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 336, 0, 1765, 0, 0, 0, 0, 0, 2490, 1419, 0, 0, 0, 0, 0, 0, 2016, 0, 0, 0, 4014, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4412, 0, 0, 0, 816, 720, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6139, 6141, 6397, 6044, 6190, 6010, 6139, 6092] avg=6144.0000 max=6397 max_rank=2 min=6010 min_rank=5 imbalance_ratio=1.041178 max_load_reduction=42755 max_load_reduction_pct=0.869853 moved_to_echo=24431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 523, 1, 1, 850, 0, 4687, 0, 0, 0, 0, 0, 0, 0, 0, 4509, 1307, 0, 4, 7, 700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 328, 4, 124, 1419, 0, 0, 0, 2385, 1483, 0, 0, 0, 0, 0, 0, 1363, 0, 0, 340, 1, 1, 0, 0, 2855, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 5872, 145, 1, 91, 0, 0, 0, 0, 0, 0, 0, 0, 443, 3, 1889, 0, 1, 0, 0, 7, 2488, 1431, 0, 0, 0, 0, 0, 0, 2109, 2, 3, 0, 4029, 2, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4230, 0, 2, 0, 805, 700, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6063, 6527, 5812, 6352, 6116, 6262, 6214, 5806] avg=6144.0000 max=6527 max_rank=1 min=5806 min_rank=7 imbalance_ratio=1.062337 max_load_reduction=42625 max_load_reduction_pct=0.867208 moved_to_echo=24398 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 538, 0, 0, 915, 0, 4696, 0, 0, 0, 0, 0, 0, 0, 0, 4300, 1319, 0, 0, 0, 709, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 317, 0, 132, 1524, 0, 0, 0, 2399, 1495, 0, 0, 0, 0, 0, 0, 1245, 0, 0, 325, 0, 0, 0, 0, 2899, 1794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5938, 139, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 463, 0, 1776, 0, 0, 0, 0, 0, 2526, 1437, 0, 0, 0, 0, 0, 0, 2139, 0, 0, 0, 4066, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4212, 0, 0, 0, 828, 747, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6149, 6328, 5936, 6263, 6152, 6202, 6274, 5848] avg=6144.0000 max=6328 max_rank=1 min=5848 min_rank=7 imbalance_ratio=1.029948 max_load_reduction=42824 max_load_reduction_pct=0.871257 moved_to_echo=24476 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[768, 2688, 128, 1152, 13824, 0, 0, 0, 19072, 12032, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3200, 128, 14720, 0, 128, 0, 0, 128, 20096, 11520, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[9216, 0, 0, 2688, 128, 128, 0, 0, 23040, 14464, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[16256, 128, 128, 0, 32384, 128, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 33024, 10624, 0, 128, 128, 5760, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[35968, 0, 128, 0, 6656, 5632, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 0, 0, 4224, 128, 128, 7424, 0, 37632, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 47488, 1152, 128, 640, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[573, 0, 0, 96, 350, 6047, 0, 0, 4185, 57, 107, 0, 0, 0, 0, 0, 0, 94, 1249, 0, 0, 88, 0, 0, 167, 0, 0, 0, 2, 96, 0, 0, 0, 0, 174, 0, 0, 0, 2268, 283, 0, 4189, 7703, 8151, 0, 0, 7, 0, 0, 0, 0, 96, 0, 0, 326, 7036, 4718, 0, 0, 0, 545, 390, 4, 151] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[572, 0, 0, 90, 337, 6202, 0, 0, 3420, 85, 119, 0, 0, 0, 0, 0, 0, 72, 1438, 0, 0, 27, 0, 0, 149, 0, 0, 0, 0, 90, 0, 0, 0, 0, 161, 0, 0, 0, 2377, 292, 0, 4226, 7757, 8126, 0, 0, 8, 0, 0, 0, 0, 90, 0, 0, 392, 7087, 5019, 0, 0, 0, 581, 244, 0, 191] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[508, 0, 0, 74, 313, 6172, 0, 0, 3780, 76, 98, 0, 0, 0, 0, 0, 0, 73, 1296, 0, 0, 59, 0, 0, 162, 0, 0, 0, 4, 74, 0, 0, 0, 0, 81, 0, 0, 0, 2233, 272, 0, 4397, 7729, 8132, 0, 0, 9, 0, 0, 0, 0, 74, 0, 0, 421, 7118, 4936, 0, 0, 0, 621, 243, 0, 197] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[502, 0, 0, 65, 319, 6184, 0, 0, 4009, 91, 81, 0, 0, 0, 0, 0, 0, 65, 1043, 0, 0, 63, 0, 0, 143, 0, 0, 0, 30, 65, 0, 0, 0, 0, 58, 0, 0, 0, 2905, 311, 0, 3972, 7742, 8132, 0, 0, 6, 0, 0, 0, 0, 65, 0, 0, 376, 7144, 4744, 0, 0, 0, 614, 247, 0, 176] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[536, 0, 0, 75, 321, 6222, 0, 0, 3732, 68, 93, 0, 0, 0, 0, 0, 0, 75, 1118, 0, 0, 74, 0, 0, 150, 0, 0, 0, 18, 75, 0, 0, 0, 0, 65, 0, 0, 0, 2537, 291, 0, 4442, 7676, 8129, 0, 0, 5, 0, 0, 0, 0, 75, 0, 0, 387, 7103, 4760, 0, 0, 0, 628, 282, 0, 215] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[582, 0, 8, 91, 340, 6043, 1, 0, 3773, 69, 113, 0, 0, 0, 0, 1, 0, 91, 1039, 5, 4, 91, 5, 0, 161, 0, 0, 3, 6, 91, 0, 0, 3, 0, 148, 0, 0, 0, 2850, 285, 15, 4656, 7644, 8128, 0, 6, 6, 0, 1, 2, 0, 91, 0, 0, 361, 7017, 4299, 8, 0, 3, 588, 358, 3, 163] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[576, 0, 0, 82, 346, 6187, 0, 0, 3903, 111, 90, 0, 0, 0, 0, 0, 0, 81, 1262, 0, 0, 77, 0, 0, 132, 0, 0, 0, 4, 82, 0, 0, 0, 0, 72, 0, 0, 0, 2620, 343, 0, 4236, 7710, 8105, 0, 0, 7, 0, 0, 0, 0, 82, 0, 0, 339, 7037, 4565, 0, 0, 0, 659, 260, 0, 184] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[546, 0, 0, 69, 330, 6281, 0, 0, 3896, 79, 85, 0, 0, 0, 0, 0, 0, 69, 924, 0, 0, 69, 0, 0, 136, 0, 0, 0, 59, 69, 0, 0, 0, 0, 92, 0, 0, 0, 3497, 313, 7, 4154, 7695, 8139, 0, 0, 17, 0, 0, 0, 0, 69, 0, 0, 402, 7098, 3963, 0, 0, 0, 606, 315, 4, 169] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[572, 0, 0, 90, 337, 5212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3420, 85, 119, 0, 0, 0, 0, 0, 1393, 551, 189, 0, 0, 0, 0, 0, 0, 72, 1438, 0, 0, 27, 0, 0, 2609, 2231, 0, 0, 0, 0, 0, 0, 149, 0, 0, 0, 0, 90, 0, 0, 5895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 0, 0, 0, 2377, 292, 3097, 0, 0, 0, 0, 0, 0, 0, 0, 4226, 1862, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 392, 5694, 0, 0, 0, 0, 0, 0, 0, 0, 5019, 0, 0, 0, 581, 244, 0, 191, 439, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6211, 5757, 6377, 6134, 5927, 6096, 6176, 6474] avg=6144.0000 max=6474 max_rank=7 min=5757 min_rank=1 imbalance_ratio=1.053711 max_load_reduction=42678 max_load_reduction_pct=0.868286 moved_to_echo=24673 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[573, 0, 0, 96, 350, 5082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4185, 57, 107, 0, 0, 0, 0, 0, 1383, 537, 188, 0, 0, 0, 0, 0, 0, 94, 1249, 0, 0, 88, 0, 0, 2591, 2237, 0, 0, 0, 0, 0, 0, 167, 0, 0, 0, 2, 96, 0, 0, 5914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 174, 0, 0, 0, 2268, 283, 3076, 0, 0, 0, 0, 0, 0, 0, 0, 4189, 1848, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 326, 5653, 0, 0, 0, 0, 0, 0, 0, 0, 4718, 0, 0, 0, 545, 390, 4, 151, 428, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6101, 6457, 6259, 6179, 5801, 6044, 6075, 6236] avg=6144.0000 max=6457 max_rank=1 min=5801 min_rank=4 imbalance_ratio=1.050944 max_load_reduction=42695 max_load_reduction_pct=0.868632 moved_to_echo=24156 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[502, 0, 0, 65, 319, 5196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4009, 91, 81, 0, 0, 0, 0, 0, 1404, 550, 188, 0, 0, 0, 0, 0, 0, 65, 1043, 0, 0, 63, 0, 0, 2604, 2232, 0, 0, 0, 0, 0, 0, 143, 0, 0, 0, 30, 65, 0, 0, 5900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 2905, 311, 3091, 0, 0, 0, 0, 0, 0, 0, 0, 3972, 1859, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 376, 5740, 0, 0, 0, 0, 0, 0, 0, 0, 4744, 0, 0, 0, 614, 247, 0, 176, 438, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6082, 6323, 6007, 6138, 6365, 5837, 6181, 6219] avg=6144.0000 max=6365 max_rank=4 min=5837 min_rank=5 imbalance_ratio=1.035970 max_load_reduction=42787 max_load_reduction_pct=0.870504 moved_to_echo=24602 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[508, 0, 0, 74, 313, 5186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3780, 76, 98, 0, 0, 0, 0, 0, 1399, 549, 188, 0, 0, 0, 0, 0, 0, 73, 1296, 0, 0, 59, 0, 0, 2600, 2233, 0, 0, 0, 0, 0, 0, 162, 0, 0, 0, 4, 74, 0, 0, 5899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 2233, 272, 3086, 0, 0, 0, 0, 0, 0, 0, 0, 4397, 1855, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 421, 5719, 0, 0, 0, 0, 0, 0, 0, 0, 4936, 0, 0, 0, 621, 243, 0, 197, 437, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6081, 6090, 6261, 6139, 5672, 6261, 6214, 6434] avg=6144.0000 max=6434 max_rank=7 min=5672 min_rank=4 imbalance_ratio=1.047201 max_load_reduction=42718 max_load_reduction_pct=0.869100 moved_to_echo=24581 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[576, 0, 0, 82, 346, 5199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3903, 111, 90, 0, 0, 0, 0, 0, 1383, 550, 188, 0, 0, 0, 0, 0, 0, 81, 1262, 0, 0, 77, 0, 0, 2593, 2226, 0, 0, 0, 0, 0, 0, 132, 0, 0, 0, 4, 82, 0, 0, 5879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 2620, 343, 3078, 0, 0, 0, 0, 0, 0, 0, 0, 4236, 1851, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 339, 5654, 0, 0, 0, 0, 0, 0, 0, 0, 4565, 0, 0, 0, 659, 260, 0, 184, 438, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6203, 6225, 6239, 6097, 6113, 6094, 6075, 6106] avg=6144.0000 max=6239 max_rank=2 min=6075 min_rank=6 imbalance_ratio=1.015462 max_load_reduction=42913 max_load_reduction_pct=0.873067 moved_to_echo=24388 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[546, 0, 0, 69, 330, 5278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3896, 79, 85, 0, 0, 0, 0, 0, 1395, 558, 187, 0, 0, 0, 0, 0, 0, 69, 924, 0, 0, 69, 0, 0, 2588, 2235, 0, 0, 0, 0, 0, 0, 136, 0, 0, 0, 59, 69, 0, 0, 5904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 3497, 313, 3072, 0, 0, 0, 0, 0, 0, 0, 7, 4154, 1848, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 402, 5703, 0, 0, 0, 0, 0, 0, 0, 0, 3963, 0, 0, 0, 606, 315, 4, 169, 445, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6223, 6200, 5885, 6168, 6974, 6026, 6174, 5502] avg=6144.0000 max=6974 max_rank=4 min=5502 min_rank=7 imbalance_ratio=1.135091 max_load_reduction=42178 max_load_reduction_pct=0.858114 moved_to_echo=24676 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[582, 0, 8, 91, 340, 5069, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3773, 69, 113, 0, 0, 0, 0, 1, 1384, 541, 190, 0, 0, 0, 0, 0, 0, 91, 1039, 5, 4, 91, 5, 0, 2576, 2232, 0, 0, 0, 0, 0, 0, 161, 0, 0, 3, 6, 91, 0, 0, 5896, 0, 0, 0, 0, 0, 0, 0, 3, 0, 148, 0, 0, 0, 2850, 285, 3056, 0, 0, 0, 0, 0, 0, 0, 15, 4656, 1822, 0, 0, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 91, 0, 0, 361, 5633, 0, 0, 0, 0, 0, 0, 0, 0, 4299, 8, 0, 3, 588, 358, 3, 163, 433, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6091, 6071, 6043, 6157, 6342, 6505, 6088, 5855] avg=6144.0000 max=6505 max_rank=5 min=5855 min_rank=7 imbalance_ratio=1.058757 max_load_reduction=42647 max_load_reduction_pct=0.867655 moved_to_echo=24790 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[536, 0, 0, 75, 321, 5228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3732, 68, 93, 0, 0, 0, 0, 0, 1396, 553, 187, 0, 0, 0, 0, 0, 0, 75, 1118, 0, 0, 74, 0, 0, 2582, 2232, 0, 0, 0, 0, 0, 0, 150, 0, 0, 0, 18, 75, 0, 0, 5897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 2537, 291, 3065, 0, 0, 0, 0, 0, 0, 0, 0, 4442, 1842, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 387, 5707, 0, 0, 0, 0, 0, 0, 0, 0, 4760, 0, 0, 0, 628, 282, 0, 215, 441, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6160, 6029, 6081, 6140, 5958, 6289, 6169, 6326] avg=6144.0000 max=6326 max_rank=7 min=5958 min_rank=4 imbalance_ratio=1.029622 max_load_reduction=42826 max_load_reduction_pct=0.871297 moved_to_echo=24742 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[0, 640, 9472, 128, 128, 640, 128, 0, 20864, 17920, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 0, 896, 0, 0, 0, 21376, 2432, 24704, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[30720, 640, 896, 0, 0, 0, 0, 128, 11264, 4480, 1536, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[37120, 128, 0, 128, 4864, 2432, 128, 1536, 3584, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 34304, 14848, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 0, 768, 0, 0, 3072, 45568, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[1280, 0, 0, 128, 128, 768, 0, 0, 47232, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4480, 0, 128, 768, 2688, 41472, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=17 tokens_per_expert_current_ep_rank=[785, 436, 0, 0, 0, 0, 5191, 0, 0, 428, 0, 0, 0, 0, 4, 0, 0, 180, 0, 0, 0, 936, 0, 0, 0, 0, 7764, 243, 165, 0, 66, 7429, 0, 0, 0, 0, 0, 0, 0, 469, 0, 0, 5719, 153, 7845, 90, 7338, 0, 3298, 0, 0, 397, 0, 90, 0, 0, 0, 0, 0, 0, 66, 0, 0, 60] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=17 tokens_per_expert_current_ep_rank=[805, 317, 0, 0, 0, 0, 4836, 0, 0, 476, 0, 0, 0, 0, 16, 0, 0, 150, 0, 0, 0, 1049, 0, 0, 0, 0, 7796, 230, 214, 0, 63, 7558, 0, 0, 0, 0, 0, 0, 0, 576, 0, 0, 5653, 132, 7807, 75, 7389, 0, 3411, 0, 0, 392, 0, 75, 0, 0, 0, 0, 0, 0, 63, 0, 0, 69] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=17 tokens_per_expert_current_ep_rank=[772, 319, 0, 0, 0, 0, 4931, 0, 0, 476, 0, 0, 0, 0, 14, 0, 0, 128, 0, 0, 0, 1040, 0, 0, 0, 0, 7810, 241, 339, 0, 60, 7590, 0, 0, 0, 0, 0, 0, 0, 558, 0, 0, 5785, 152, 7798, 77, 7432, 0, 3037, 0, 0, 395, 0, 63, 0, 0, 0, 0, 0, 0, 60, 0, 0, 75] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=17 tokens_per_expert_current_ep_rank=[840, 331, 0, 0, 0, 0, 4965, 0, 0, 400, 0, 0, 0, 0, 2, 0, 0, 145, 0, 0, 0, 1042, 0, 0, 0, 0, 7808, 237, 213, 0, 60, 7525, 0, 0, 0, 0, 0, 0, 0, 475, 0, 0, 5422, 136, 7876, 74, 7441, 0, 3533, 0, 0, 404, 0, 71, 0, 0, 0, 0, 0, 0, 60, 0, 0, 92] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=17 tokens_per_expert_current_ep_rank=[794, 430, 0, 0, 0, 0, 4754, 0, 2, 462, 2, 0, 5, 0, 17, 0, 0, 183, 0, 0, 0, 955, 0, 2, 0, 0, 7747, 247, 242, 5, 54, 7438, 7, 4, 6, 0, 7, 0, 0, 497, 0, 3, 5342, 123, 7855, 91, 7410, 3, 3844, 0, 1, 396, 0, 91, 0, 0, 0, 0, 0, 8, 55, 0, 0, 70] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=17 tokens_per_expert_current_ep_rank=[816, 365, 0, 0, 0, 0, 4760, 0, 0, 466, 0, 0, 0, 0, 20, 0, 0, 138, 0, 0, 0, 1038, 0, 0, 0, 0, 7793, 250, 264, 0, 53, 7566, 0, 0, 0, 0, 0, 0, 0, 506, 0, 0, 5253, 132, 7876, 69, 7448, 0, 3762, 0, 0, 389, 0, 69, 0, 0, 0, 0, 0, 0, 53, 0, 0, 66] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=17 tokens_per_expert_current_ep_rank=[759, 441, 0, 0, 0, 0, 5286, 0, 0, 367, 0, 0, 0, 0, 16, 0, 0, 191, 0, 0, 0, 947, 0, 0, 0, 0, 7747, 237, 206, 0, 42, 7416, 0, 0, 0, 0, 0, 0, 0, 459, 0, 0, 5654, 99, 7856, 98, 7432, 0, 3265, 0, 0, 423, 0, 95, 0, 0, 0, 0, 0, 0, 44, 0, 0, 72] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=17 tokens_per_expert_current_ep_rank=[780, 347, 0, 0, 0, 0, 5143, 0, 0, 440, 0, 0, 0, 0, 0, 0, 0, 164, 0, 0, 0, 1027, 0, 0, 0, 0, 7764, 263, 170, 0, 90, 7520, 0, 0, 0, 0, 0, 0, 0, 489, 0, 0, 5542, 203, 7833, 85, 7290, 0, 3395, 0, 0, 391, 0, 82, 0, 0, 0, 0, 0, 0, 90, 0, 0, 44] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=17 rerouted_tokens_per_expert_current_ep_rank=[785, 436, 0, 0, 0, 0, 5185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 428, 0, 0, 0, 0, 4, 0, 3855, 1830, 0, 0, 0, 0, 0, 0, 0, 180, 0, 0, 0, 936, 0, 0, 4939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 165, 0, 66, 5556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 3909, 1707, 0, 0, 0, 0, 0, 0, 0, 0, 5719, 153, 0, 90, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3298, 0, 0, 397, 0, 90, 0, 0, 1873, 320, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 60, 6015, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6406, 6117, 6055, 6030, 6085, 6334, 5984, 6141] avg=6144.0000 max=6406 max_rank=0 min=5984 min_rank=6 imbalance_ratio=1.042643 max_load_reduction=42746 max_load_reduction_pct=0.869670 moved_to_echo=24544 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=17 rerouted_tokens_per_expert_current_ep_rank=[772, 319, 0, 0, 0, 0, 4925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 0, 0, 0, 0, 14, 0, 3877, 1819, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 1040, 0, 0, 5002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 241, 339, 0, 60, 5676, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 558, 3933, 1729, 0, 0, 0, 0, 0, 0, 0, 0, 5785, 152, 0, 77, 376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3037, 0, 0, 395, 0, 63, 0, 0, 1914, 325, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 75, 5979, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6016, 6186, 6170, 6316, 6220, 6390, 5740, 6114] avg=6144.0000 max=6390 max_rank=5 min=5740 min_rank=6 imbalance_ratio=1.040039 max_load_reduction=42762 max_load_reduction_pct=0.869995 moved_to_echo=24464 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=17 rerouted_tokens_per_expert_current_ep_rank=[759, 441, 0, 0, 0, 0, 5280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 367, 0, 0, 0, 0, 16, 0, 3846, 1833, 0, 0, 0, 0, 0, 0, 0, 191, 0, 0, 0, 947, 0, 0, 5002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 237, 206, 0, 42, 5546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 459, 3901, 1729, 0, 0, 0, 0, 0, 0, 0, 0, 5654, 99, 0, 98, 376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3265, 0, 0, 423, 0, 95, 0, 0, 1870, 325, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 72, 6023, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6480, 6062, 6140, 6031, 6089, 6227, 5984, 6139] avg=6144.0000 max=6480 max_rank=0 min=5984 min_rank=6 imbalance_ratio=1.054688 max_load_reduction=42672 max_load_reduction_pct=0.868164 moved_to_echo=24439 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=17 rerouted_tokens_per_expert_current_ep_rank=[816, 365, 0, 0, 0, 0, 4754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 466, 0, 0, 0, 0, 20, 0, 3870, 1838, 0, 0, 0, 0, 0, 0, 0, 138, 0, 0, 0, 1038, 0, 0, 5013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 250, 264, 0, 53, 5658, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 3923, 1732, 0, 0, 0, 0, 0, 0, 0, 0, 5253, 132, 0, 69, 378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3762, 0, 0, 389, 0, 69, 0, 0, 1908, 325, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 66, 6038, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5935, 6194, 6189, 6225, 6161, 5832, 6459, 6157] avg=6144.0000 max=6459 max_rank=6 min=5832 min_rank=5 imbalance_ratio=1.051270 max_load_reduction=42693 max_load_reduction_pct=0.868591 moved_to_echo=24609 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=17 rerouted_tokens_per_expert_current_ep_rank=[840, 331, 0, 0, 0, 0, 4959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 400, 0, 0, 0, 0, 2, 0, 3877, 1838, 0, 0, 0, 0, 0, 0, 0, 145, 0, 0, 0, 1042, 0, 0, 5008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 237, 213, 0, 60, 5628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 3931, 1731, 0, 0, 0, 0, 0, 0, 0, 0, 5422, 136, 0, 74, 377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3533, 0, 0, 404, 0, 71, 0, 0, 1897, 325, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 92, 6038, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6130, 6117, 6195, 6138, 6137, 6009, 6236, 6190] avg=6144.0000 max=6236 max_rank=6 min=6009 min_rank=5 imbalance_ratio=1.014974 max_load_reduction=42916 max_load_reduction_pct=0.873128 moved_to_echo=24572 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=17 rerouted_tokens_per_expert_current_ep_rank=[780, 347, 0, 0, 0, 0, 5137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 440, 0, 0, 0, 0, 0, 0, 3855, 1828, 0, 0, 0, 0, 0, 0, 0, 164, 0, 0, 0, 1027, 0, 0, 4907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 263, 170, 0, 90, 5624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 3909, 1695, 0, 0, 0, 0, 0, 0, 0, 0, 5542, 203, 0, 85, 370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3395, 0, 0, 391, 0, 82, 0, 0, 1896, 318, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 44, 6005, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6264, 6123, 6098, 6147, 6093, 6200, 6088, 6139] avg=6144.0000 max=6264 max_rank=0 min=6088 min_rank=6 imbalance_ratio=1.019531 max_load_reduction=42888 max_load_reduction_pct=0.872559 moved_to_echo=24520 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=17 rerouted_tokens_per_expert_current_ep_rank=[805, 317, 0, 0, 0, 0, 4830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 0, 0, 0, 0, 16, 0, 3871, 1822, 0, 0, 0, 0, 0, 0, 0, 150, 0, 0, 0, 1049, 0, 0, 4973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 230, 214, 0, 63, 5652, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 576, 3925, 1719, 0, 0, 0, 0, 0, 0, 0, 0, 5653, 132, 0, 75, 374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3411, 0, 0, 392, 0, 75, 0, 0, 1906, 323, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 69, 5985, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5952, 6185, 6172, 6159, 6220, 6234, 6113, 6117] avg=6144.0000 max=6234 max_rank=5 min=5952 min_rank=0 imbalance_ratio=1.014648 max_load_reduction=42918 max_load_reduction_pct=0.873169 moved_to_echo=24684 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=17 rerouted_tokens_per_expert_current_ep_rank=[794, 430, 0, 0, 0, 0, 4745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 462, 2, 0, 5, 0, 17, 0, 3847, 1833, 0, 0, 0, 0, 0, 0, 0, 183, 0, 0, 0, 955, 0, 2, 4993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 242, 5, 54, 5558, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4, 6, 0, 7, 0, 0, 497, 3900, 1726, 0, 0, 0, 0, 0, 0, 0, 3, 5342, 123, 0, 91, 364, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3844, 0, 1, 396, 0, 91, 0, 0, 1880, 327, 9, 0, 0, 0, 0, 0, 0, 0, 0, 8, 55, 0, 0, 70, 6022, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5969, 6168, 6133, 6106, 6147, 5926, 6548, 6155] avg=6144.0000 max=6548 max_rank=6 min=5926 min_rank=5 imbalance_ratio=1.065755 max_load_reduction=42604 max_load_reduction_pct=0.866781 moved_to_echo=24776 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 3584, 128, 0, 128, 0, 128, 0, 30976, 14720, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 128, 0, 0, 4096, 31360, 13824, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[27648, 0, 128, 3200, 0, 640, 0, 0, 15232, 2688, 128, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[6400, 3072, 0, 0, 0, 0, 39936, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 1280, 0, 0, 0, 8064, 0, 128, 39936, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 44416, 1152, 0, 768, 3072, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 0, 2048, 1920, 128, 512, 44928, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 512, 0, 0, 640, 48128, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 315, 0, 0, 0, 1165, 0, 0, 0, 40, 0, 7421, 0, 0, 7850, 74, 0, 0, 0, 0, 735, 0, 0, 0, 0, 0, 8121, 0, 0, 8191, 0, 0, 0, 71, 0, 0, 8132, 0, 0, 0, 0, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 320, 0, 0, 543, 5966, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 383, 0, 0, 0, 1024, 0, 0, 0, 63, 0, 7481, 0, 0, 7693, 65, 0, 0, 0, 0, 974, 0, 0, 0, 0, 0, 8129, 0, 0, 8151, 0, 0, 0, 63, 0, 0, 8132, 0, 0, 0, 0, 215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 302, 0, 0, 506, 5893, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 308, 0, 0, 0, 1058, 0, 0, 0, 69, 0, 7483, 0, 0, 7780, 69, 0, 0, 0, 0, 2297, 0, 0, 0, 0, 0, 8123, 0, 0, 8112, 0, 0, 0, 69, 0, 0, 8139, 0, 0, 0, 0, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 339, 0, 0, 545, 4532, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 408, 0, 0, 0, 1121, 0, 0, 0, 48, 0, 7424, 0, 0, 7624, 90, 0, 0, 0, 0, 803, 0, 0, 0, 0, 0, 8102, 0, 0, 8180, 0, 0, 0, 90, 0, 0, 8126, 0, 0, 0, 0, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 408, 0, 0, 494, 5991, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 235, 0, 0, 0, 1127, 0, 0, 0, 77, 0, 7449, 0, 0, 7771, 82, 0, 0, 0, 0, 1036, 0, 0, 0, 0, 0, 8110, 0, 0, 8140, 0, 0, 0, 82, 0, 0, 8102, 0, 0, 0, 0, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 336, 0, 0, 544, 5772, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 316, 0, 0, 0, 1196, 0, 0, 0, 30, 0, 7397, 0, 0, 7799, 95, 0, 0, 0, 0, 930, 0, 0, 0, 0, 0, 8097, 0, 0, 8186, 0, 0, 0, 94, 0, 0, 8150, 0, 0, 0, 0, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 428, 0, 0, 543, 5691, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=18 tokens_per_expert_current_ep_rank=[1, 0, 322, 0, 0, 0, 1012, 0, 0, 0, 91, 0, 7408, 0, 8, 7701, 94, 0, 0, 0, 2, 1409, 0, 6, 0, 0, 0, 8086, 0, 8, 8112, 0, 0, 0, 91, 0, 0, 8129, 6, 3, 1, 0, 214, 0, 2, 9, 7, 0, 0, 1, 0, 0, 0, 4, 4, 0, 1, 389, 0, 1, 739, 5290, 0, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 247, 0, 0, 0, 1084, 0, 0, 0, 63, 0, 7402, 0, 0, 7901, 75, 0, 0, 0, 0, 1025, 0, 0, 0, 0, 0, 8117, 0, 0, 8151, 0, 0, 0, 75, 0, 0, 8129, 0, 0, 0, 0, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 310, 0, 0, 657, 5707, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 408, 0, 0, 0, 1121, 0, 1965, 1348, 1023, 413, 0, 0, 0, 0, 0, 0, 48, 0, 6076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 0, 803, 0, 0, 3798, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 6061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 0, 0, 0, 0, 0, 3826, 2031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6149, 0, 0, 0, 0, 0, 0, 0, 0, 408, 0, 0, 494, 5578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6278, 6124, 5733, 6137, 6151, 6100, 6149, 6480] avg=6144.0000 max=6480 max_rank=7 min=5733 min_rank=2 imbalance_ratio=1.054688 max_load_reduction=42672 max_load_reduction_pct=0.868164 moved_to_echo=24880 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 383, 0, 0, 0, 1024, 0, 1972, 1358, 1024, 406, 0, 0, 0, 0, 0, 0, 63, 0, 6123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 974, 0, 0, 3833, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 6066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 215, 0, 0, 0, 0, 0, 3860, 2024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 6127, 0, 0, 0, 0, 0, 0, 0, 0, 302, 0, 0, 506, 5487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6167, 6186, 5914, 6157, 6129, 6099, 6205, 6295] avg=6144.0000 max=6295 max_rank=7 min=5914 min_rank=2 imbalance_ratio=1.024577 max_load_reduction=42857 max_load_reduction_pct=0.871928 moved_to_echo=24728 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 316, 0, 0, 0, 1196, 0, 1964, 1343, 1026, 392, 0, 0, 0, 0, 0, 0, 30, 0, 6054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 0, 0, 930, 0, 0, 3885, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 0, 6079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 194, 0, 0, 0, 0, 0, 3914, 2033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 6153, 0, 0, 0, 0, 0, 0, 0, 0, 428, 0, 0, 543, 5299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6237, 6084, 5955, 6133, 6173, 6141, 6159, 6270] avg=6144.0000 max=6270 max_rank=7 min=5955 min_rank=2 imbalance_ratio=1.020508 max_load_reduction=42882 max_load_reduction_pct=0.872437 moved_to_echo=24743 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 315, 0, 0, 0, 1165, 0, 1970, 1348, 1024, 411, 0, 0, 0, 0, 0, 0, 40, 0, 6073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 735, 0, 0, 3912, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 6066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 207, 0, 0, 0, 0, 0, 3938, 2035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6156, 0, 0, 0, 0, 0, 0, 0, 0, 320, 0, 0, 543, 5555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6233, 6113, 5763, 6151, 6137, 6180, 6157, 6418] avg=6144.0000 max=6418 max_rank=7 min=5763 min_rank=2 imbalance_ratio=1.044596 max_load_reduction=42734 max_load_reduction_pct=0.869425 moved_to_echo=24892 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 308, 0, 0, 0, 1058, 0, 1970, 1359, 1025, 312, 0, 0, 0, 0, 0, 0, 69, 0, 6124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 2297, 0, 0, 3877, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 6071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 201, 0, 0, 0, 0, 0, 3903, 2015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 6097, 0, 0, 0, 0, 0, 0, 0, 0, 339, 0, 0, 545, 4220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6032, 6193, 7286, 6153, 6140, 6119, 6125, 5104] avg=6144.0000 max=7286 max_rank=2 min=5104 min_rank=7 imbalance_ratio=1.185872 max_load_reduction=41866 max_load_reduction_pct=0.851766 moved_to_echo=23488 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 235, 0, 0, 0, 1127, 0, 1967, 1353, 1020, 398, 0, 0, 0, 0, 0, 0, 77, 0, 6096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 1036, 0, 0, 3872, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 6044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 285, 0, 0, 0, 0, 0, 3899, 2022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 6118, 0, 0, 0, 0, 0, 0, 0, 0, 336, 0, 0, 544, 5374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6100, 6173, 6028, 6143, 6126, 6206, 6122, 6254] avg=6144.0000 max=6254 max_rank=7 min=6028 min_rank=2 imbalance_ratio=1.017904 max_load_reduction=42898 max_load_reduction_pct=0.872762 moved_to_echo=24708 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=18 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 322, 0, 0, 0, 1012, 0, 1964, 1349, 1026, 370, 0, 0, 0, 0, 0, 0, 91, 0, 6059, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 0, 0, 2, 1409, 0, 6, 3838, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6122, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 6057, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 214, 0, 2, 9, 7, 0, 3863, 2015, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 4, 0, 6097, 0, 0, 0, 0, 0, 0, 0, 1, 389, 0, 1, 739, 4920, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6044, 6158, 6395, 6130, 6157, 6111, 6106, 6051] avg=6144.0000 max=6395 max_rank=2 min=6044 min_rank=0 imbalance_ratio=1.040853 max_load_reduction=42757 max_load_reduction_pct=0.869893 moved_to_echo=24425 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 247, 0, 0, 0, 1084, 0, 1969, 1344, 1023, 394, 0, 0, 0, 0, 0, 0, 63, 0, 6058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 0, 1025, 0, 0, 3936, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 6064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 207, 0, 0, 0, 0, 0, 3965, 2024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 6127, 0, 0, 0, 0, 0, 0, 0, 0, 310, 0, 0, 657, 5313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6061, 6121, 6078, 6148, 6139, 6196, 6129, 6280] avg=6144.0000 max=6280 max_rank=7 min=6061 min_rank=0 imbalance_ratio=1.022135 max_load_reduction=42872 max_load_reduction_pct=0.872233 moved_to_echo=24744 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 0, 2560, 0, 0, 0, 8832, 0, 15744, 10880, 8192, 3200, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[768, 0, 0, 0, 128, 9216, 0, 128, 30976, 8448, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 512, 0, 48768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49280 observed_loss=-128 tokens_per_expert=[0, 0, 0, 49152, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 2944, 0, 128, 4608, 41856, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 0, 1792, 0, 128, 128, 128, 0, 31232, 16256, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 640, 0, 0, 48512, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 0, 128, 128, 0, 49024, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 407, 0, 0, 0, 450, 0, 1477, 0, 191, 7837, 8192, 0, 0, 0, 0, 0, 940, 0, 7717, 0, 42, 0, 7921, 0, 0, 0, 0, 497, 7154, 96, 0, 0, 0, 0, 102, 306, 0, 0, 0, 4187, 0, 53, 0, 0, 0, 0, 0, 3, 42, 0, 95, 0, 1106, 337] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 438, 0, 0, 0, 229, 0, 1864, 0, 128, 8016, 8192, 0, 0, 0, 0, 0, 847, 0, 7731, 0, 60, 0, 7894, 0, 0, 0, 0, 697, 7150, 65, 0, 0, 0, 0, 57, 337, 0, 0, 0, 3900, 0, 71, 0, 0, 0, 0, 0, 0, 60, 17, 63, 0, 999, 337] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 0, 0, 0, 484, 0, 1745, 0, 145, 7973, 8192, 0, 0, 0, 0, 0, 1032, 0, 7710, 0, 60, 0, 7877, 0, 0, 0, 0, 407, 7059, 74, 0, 0, 0, 0, 93, 526, 0, 0, 0, 3600, 0, 75, 0, 0, 0, 0, 0, 10, 60, 1, 72, 0, 1122, 349] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 0, 0, 0, 398, 0, 1737, 0, 180, 7855, 8192, 0, 0, 0, 0, 0, 913, 0, 7733, 0, 66, 0, 7865, 0, 0, 0, 0, 556, 7088, 90, 0, 0, 0, 0, 111, 424, 0, 0, 0, 3623, 0, 90, 0, 0, 0, 0, 0, 49, 66, 2, 98, 0, 1245, 303] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 433, 0, 0, 0, 233, 0, 1824, 0, 138, 7976, 8192, 0, 0, 0, 0, 0, 787, 0, 7725, 0, 53, 0, 7930, 0, 0, 0, 0, 899, 7156, 69, 0, 0, 0, 0, 105, 300, 0, 0, 0, 3790, 0, 64, 0, 0, 0, 0, 0, 0, 54, 0, 69, 0, 1010, 345] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 400, 0, 0, 0, 276, 0, 1819, 0, 164, 7961, 8192, 0, 0, 0, 0, 0, 909, 0, 7659, 0, 90, 0, 7862, 0, 0, 0, 0, 638, 7052, 82, 0, 0, 0, 0, 141, 454, 0, 0, 0, 3659, 0, 110, 0, 0, 0, 0, 0, 2, 91, 3, 83, 0, 1144, 361] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 421, 0, 0, 0, 207, 0, 1732, 0, 150, 7986, 8192, 0, 0, 0, 0, 0, 782, 0, 7710, 0, 63, 0, 7887, 0, 0, 0, 0, 730, 7160, 75, 0, 0, 0, 0, 109, 468, 0, 0, 0, 3817, 0, 65, 0, 0, 0, 0, 0, 0, 63, 1, 75, 0, 1115, 344] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 8, 0, 8, 0, 1, 4, 0, 0, 443, 3, 0, 0, 317, 0, 2052, 1, 193, 7856, 8176, 0, 5, 0, 0, 0, 735, 0, 7686, 1, 55, 0, 7872, 2, 4, 0, 1, 761, 7152, 91, 0, 3, 0, 6, 127, 438, 0, 0, 0, 3393, 0, 70, 0, 7, 0, 0, 5, 0, 70, 2, 91, 8, 1159, 346] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 407, 0, 0, 0, 450, 0, 4324, 1062, 0, 0, 0, 0, 0, 0, 1477, 0, 191, 4152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 940, 0, 5221, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 5440, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 306, 0, 0, 3597, 2050, 0, 0, 0, 0, 0, 0, 0, 4187, 0, 53, 0, 0, 0, 0, 1714, 616, 0, 0, 0, 0, 0, 0, 0, 3, 42, 0, 95, 0, 1106, 337, 2623, 1880, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6142, 6243, 5820, 6203, 6033, 6055, 6570, 6086] avg=6144.0000 max=6570 max_rank=6 min=5820 min_rank=2 imbalance_ratio=1.069336 max_load_reduction=42582 max_load_reduction_pct=0.866333 moved_to_echo=24744 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 438, 0, 0, 0, 229, 0, 4309, 1087, 0, 0, 0, 0, 0, 0, 1864, 0, 128, 4246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 847, 0, 5230, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 697, 5437, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 337, 0, 0, 3585, 2049, 0, 0, 0, 0, 0, 0, 0, 3900, 0, 71, 0, 0, 0, 0, 1713, 617, 0, 0, 0, 0, 0, 0, 0, 0, 60, 17, 63, 0, 999, 337, 2683, 1884, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6063, 6238, 6137, 6199, 6028, 6301, 6043] avg=6144.0000 max=6301 max_rank=6 min=6028 min_rank=5 imbalance_ratio=1.025553 max_load_reduction=42851 max_load_reduction_pct=0.871806 moved_to_echo=24571 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 0, 0, 0, 398, 0, 4293, 1065, 0, 0, 0, 0, 0, 0, 1737, 0, 180, 4161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 913, 0, 5232, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 5390, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 424, 0, 0, 3572, 2049, 0, 0, 0, 0, 0, 0, 0, 3623, 0, 90, 0, 0, 0, 0, 1698, 617, 0, 0, 0, 0, 0, 0, 0, 49, 66, 2, 98, 0, 1245, 303, 2629, 1884, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6224, 6078, 6211, 6036, 6156, 6028, 6276] avg=6144.0000 max=6276 max_rank=7 min=6028 min_rank=6 imbalance_ratio=1.021484 max_load_reduction=42876 max_load_reduction_pct=0.872314 moved_to_echo=24496 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 0, 0, 0, 484, 0, 4301, 1081, 0, 0, 0, 0, 0, 0, 1745, 0, 145, 4223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 0, 5217, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 407, 5368, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 526, 0, 0, 3576, 2049, 0, 0, 0, 0, 0, 0, 0, 3600, 0, 75, 0, 0, 0, 0, 1691, 615, 0, 0, 0, 0, 0, 0, 0, 10, 60, 1, 72, 0, 1122, 349, 2669, 1878, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6352, 6113, 6309, 5849, 6244, 5981, 6161] avg=6144.0000 max=6352 max_rank=1 min=5849 min_rank=4 imbalance_ratio=1.033854 max_load_reduction=42800 max_load_reduction_pct=0.870768 moved_to_echo=24235 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 433, 0, 0, 0, 233, 0, 4329, 1081, 0, 0, 0, 0, 0, 0, 1824, 0, 138, 4225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 787, 0, 5227, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 899, 5442, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 300, 0, 0, 3601, 2049, 0, 0, 0, 0, 0, 0, 0, 3790, 0, 64, 0, 0, 0, 0, 1714, 616, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 69, 0, 1010, 345, 2670, 1882, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6076, 6187, 6067, 6410, 6055, 6184, 6030] avg=6144.0000 max=6410 max_rank=4 min=6030 min_rank=7 imbalance_ratio=1.043294 max_load_reduction=42742 max_load_reduction_pct=0.869588 moved_to_echo=24679 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 400, 0, 0, 0, 276, 0, 4293, 1079, 0, 0, 0, 0, 0, 0, 1819, 0, 164, 4217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 909, 0, 5182, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 638, 5363, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 454, 0, 0, 3569, 2049, 0, 0, 0, 0, 0, 0, 0, 3659, 0, 110, 0, 0, 0, 0, 1689, 611, 0, 0, 0, 0, 0, 0, 0, 2, 91, 3, 83, 0, 1144, 361, 2665, 1866, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6048, 6200, 6181, 6083, 6213, 6069, 6215] avg=6144.0000 max=6215 max_rank=7 min=6048 min_rank=1 imbalance_ratio=1.011556 max_load_reduction=42937 max_load_reduction_pct=0.873556 moved_to_echo=24580 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8, 0, 8, 0, 1, 4, 6131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 443, 3, 0, 0, 317, 0, 4298, 1069, 0, 0, 0, 0, 0, 0, 2052, 1, 193, 4153, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 735, 0, 5191, 1, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 0, 1, 761, 5433, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 6, 127, 438, 0, 0, 3574, 2045, 0, 0, 0, 0, 0, 0, 0, 3393, 0, 70, 0, 7, 0, 0, 1719, 618, 0, 0, 0, 0, 0, 0, 5, 0, 70, 2, 91, 8, 1159, 346, 2634, 1877, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6152, 6130, 6404, 5982, 6292, 6193, 5807, 6192] avg=6144.0000 max=6404 max_rank=2 min=5807 min_rank=6 imbalance_ratio=1.042318 max_load_reduction=42748 max_load_reduction_pct=0.869710 moved_to_echo=24484 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 421, 0, 0, 0, 207, 0, 4305, 1083, 0, 0, 0, 0, 0, 0, 1732, 0, 150, 4230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 782, 0, 5217, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 730, 5445, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 468, 0, 0, 3582, 2049, 0, 0, 0, 0, 0, 0, 0, 3817, 0, 65, 0, 0, 0, 0, 1715, 615, 0, 0, 0, 0, 0, 0, 0, 0, 63, 1, 75, 0, 1115, 344, 2673, 1878, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6016, 6112, 6062, 6250, 6208, 6212, 6149] avg=6144.0000 max=6250 max_rank=4 min=6016 min_rank=1 imbalance_ratio=1.017253 max_load_reduction=42902 max_load_reduction_pct=0.872843 moved_to_echo=24819 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 0, 128, 896, 3328, 0, 0, 28672, 16512, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 512, 128, 768, 128, 8960, 2816, 21248, 15104, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 3584, 128, 0, 0, 2688, 0, 34560, 8704, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 30080, 0, 640, 0, 128, 0, 0, 13696, 4992, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[14336, 128, 1408, 33664, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 7040, 0, 41728, 128, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 128, 5248, 43392, 768, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 128, 0, 128, 128, 49152, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 412, 0, 0, 0, 78, 434, 8192, 1053, 0, 0, 8001, 0, 0, 7295, 78, 53, 0, 0, 0, 0, 148, 0, 0, 0, 0, 142, 0, 7794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1302, 6749, 0, 165, 55, 7063, 69, 0, 0, 0, 69, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 400, 0, 0, 0, 84, 467, 8186, 1040, 0, 0, 7986, 0, 0, 7325, 81, 60, 0, 0, 0, 0, 162, 0, 0, 0, 0, 149, 0, 7813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 923, 6742, 0, 121, 33, 7438, 71, 0, 0, 0, 71, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 379, 0, 0, 0, 179, 505, 8179, 958, 0, 0, 7959, 0, 0, 7227, 174, 42, 0, 0, 0, 0, 271, 0, 0, 0, 0, 106, 0, 7725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 874, 6766, 0, 73, 107, 7428, 95, 0, 0, 0, 96, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 428, 0, 0, 0, 166, 452, 8145, 970, 0, 0, 7946, 0, 0, 7363, 161, 66, 0, 0, 0, 0, 284, 0, 0, 0, 0, 157, 0, 7736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 934, 6621, 0, 80, 27, 7436, 90, 0, 0, 0, 90, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 398, 0, 0, 0, 69, 460, 8185, 1020, 0, 0, 7979, 0, 0, 7349, 64, 63, 0, 0, 0, 0, 139, 0, 0, 0, 0, 137, 0, 7816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 991, 6816, 0, 152, 23, 7341, 75, 0, 0, 0, 75, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 0, 0, 0, 75, 462, 8187, 1022, 0, 0, 7938, 0, 0, 7278, 72, 90, 0, 0, 0, 0, 165, 0, 0, 0, 0, 213, 0, 7773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 6755, 0, 136, 55, 7263, 82, 0, 0, 0, 82, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 425, 0, 0, 0, 52, 456, 8192, 1007, 0, 0, 8004, 0, 0, 7344, 48, 60, 0, 0, 0, 0, 111, 0, 0, 0, 0, 158, 0, 7828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1114, 6795, 0, 158, 18, 7256, 63, 0, 0, 0, 63, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 7, 7, 3, 5, 0, 0, 0, 0, 0, 0, 0, 0, 397, 9, 0, 0, 142, 496, 8134, 995, 1, 0, 7930, 6, 0, 7293, 140, 54, 0, 0, 0, 7, 248, 9, 0, 0, 0, 126, 5, 7737, 1, 8, 0, 0, 0, 0, 2, 1, 1, 1061, 6792, 1, 176, 77, 7087, 94, 0, 0, 9, 91, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 428, 0, 1626, 1427, 1339, 1331, 0, 0, 0, 0, 0, 0, 166, 452, 4547, 970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6024, 161, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 284, 0, 0, 0, 0, 157, 3598, 1823, 356, 0, 0, 0, 0, 0, 0, 6110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 934, 4838, 0, 80, 27, 0, 0, 0, 0, 0, 0, 0, 0, 6105, 90, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6123, 6151, 6135, 6251, 6218, 6110, 5879, 6285] avg=6144.0000 max=6285 max_rank=7 min=5879 min_rank=6 imbalance_ratio=1.022949 max_load_reduction=42867 max_load_reduction_pct=0.872131 moved_to_echo=24492 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 9, 0, 0, 0, 0, 6133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 379, 0, 1624, 1458, 1314, 1330, 0, 0, 0, 0, 0, 0, 179, 505, 4566, 958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5913, 174, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 271, 0, 0, 0, 0, 106, 3613, 1826, 363, 0, 0, 0, 0, 0, 0, 6101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 874, 4945, 0, 73, 107, 0, 0, 0, 0, 0, 0, 0, 0, 6098, 95, 0, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6142, 6105, 6208, 6129, 6179, 6101, 5999, 6289] avg=6144.0000 max=6289 max_rank=7 min=5999 min_rank=6 imbalance_ratio=1.023600 max_load_reduction=42863 max_load_reduction_pct=0.872050 moved_to_echo=24568 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 412, 0, 1638, 1455, 1326, 1265, 0, 0, 0, 0, 0, 0, 78, 434, 4574, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5969, 78, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 0, 0, 0, 0, 142, 3618, 1835, 363, 0, 0, 0, 0, 0, 0, 6156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1302, 4931, 0, 165, 55, 0, 0, 0, 0, 0, 0, 0, 0, 5798, 69, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6166, 6096, 6139, 6100, 6106, 6156, 6453, 5936] avg=6144.0000 max=6453 max_rank=6 min=5936 min_rank=7 imbalance_ratio=1.050293 max_load_reduction=42699 max_load_reduction_pct=0.868713 moved_to_echo=24651 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 400, 0, 1642, 1453, 1332, 1332, 0, 0, 0, 0, 0, 0, 84, 467, 4570, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5993, 81, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 162, 0, 0, 0, 0, 149, 3616, 1831, 362, 0, 0, 0, 0, 0, 0, 6171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 923, 4927, 0, 121, 33, 0, 0, 0, 0, 0, 0, 0, 0, 6106, 71, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6155, 6159, 6161, 6134, 6120, 6171, 6004, 6248] avg=6144.0000 max=6248 max_rank=7 min=6004 min_rank=6 imbalance_ratio=1.016927 max_load_reduction=42904 max_load_reduction_pct=0.872884 moved_to_echo=24543 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 425, 0, 1646, 1465, 1335, 1299, 0, 0, 0, 0, 0, 0, 52, 456, 4574, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6009, 48, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 0, 0, 0, 0, 158, 3618, 1837, 365, 0, 0, 0, 0, 0, 0, 6182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1114, 4965, 0, 158, 18, 0, 0, 0, 0, 0, 0, 0, 0, 5957, 63, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6167, 6170, 6089, 6117, 6089, 6182, 6255, 6083] avg=6144.0000 max=6255 max_rank=6 min=6083 min_rank=7 imbalance_ratio=1.018066 max_load_reduction=42897 max_load_reduction_pct=0.872742 moved_to_echo=24609 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 0, 1634, 1456, 1323, 1300, 0, 0, 0, 0, 0, 0, 75, 462, 4571, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5955, 72, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 165, 0, 0, 0, 0, 213, 3616, 1820, 363, 0, 0, 0, 0, 0, 0, 6139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 4936, 0, 136, 55, 0, 0, 0, 0, 0, 0, 0, 0, 5963, 82, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6118, 6203, 6130, 6117, 6177, 6139, 6141, 6127] avg=6144.0000 max=6203 max_rank=1 min=6117 min_rank=3 imbalance_ratio=1.009603 max_load_reduction=42949 max_load_reduction_pct=0.873800 moved_to_echo=24584 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 398, 0, 1643, 1469, 1336, 1314, 0, 0, 0, 0, 0, 0, 69, 460, 4570, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6013, 64, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 0, 0, 0, 0, 137, 3615, 1831, 366, 0, 0, 0, 0, 0, 0, 6173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 991, 4981, 0, 152, 23, 0, 0, 0, 0, 0, 0, 0, 0, 6027, 75, 0, 0, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6148, 6160, 6119, 6140, 6088, 6173, 6147, 6177] avg=6144.0000 max=6177 max_rank=7 min=6088 min_rank=4 imbalance_ratio=1.005371 max_load_reduction=42975 max_load_reduction_pct=0.874329 moved_to_echo=24585 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 7, 7, 3, 5, 0, 0, 6111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 397, 9, 1631, 1468, 1329, 1274, 0, 0, 0, 0, 0, 0, 142, 496, 4537, 995, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 5964, 140, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 248, 9, 0, 0, 0, 126, 3597, 1819, 369, 0, 0, 0, 0, 0, 5, 6106, 1, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 1061, 4955, 1, 176, 77, 0, 0, 0, 0, 0, 0, 0, 0, 5813, 94, 0, 0, 9, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6133, 6108, 6171, 6164, 6175, 6120, 6274, 6007] avg=6144.0000 max=6274 max_rank=6 min=6007 min_rank=7 imbalance_ratio=1.021159 max_load_reduction=42878 max_load_reduction_pct=0.872355 moved_to_echo=24576 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 0, 0, 3456, 128, 13184, 11776, 10752, 10496, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 896, 3840, 36608, 8192, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 1536, 128, 0, 0, 0, 1280, 28928, 14720, 2944, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 128, 128, 8320, 39552, 128, 1152, 512, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49280 observed_loss=-128 tokens_per_expert=[47872, 640, 0, 0, 128, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 128, 0, 47872, 896, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 49152, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 128, 128, 128, 128, 0, 0, 49152, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=21 tokens_per_expert_current_ep_rank=[0, 414, 332, 1416, 7976, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5571, 0, 0, 5520, 0, 0, 313, 0, 0, 0, 0, 1, 7801, 200, 0, 0, 0, 0, 0, 7848, 0, 0, 0, 2745, 0, 0, 0, 248, 0, 0, 0, 161, 0, 0, 278, 0, 0, 0, 0, 0, 7854, 90, 204, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=21 tokens_per_expert_current_ep_rank=[0, 485, 339, 1456, 8004, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5525, 0, 0, 5914, 0, 0, 319, 0, 0, 0, 0, 0, 7796, 144, 0, 0, 0, 0, 0, 7976, 0, 0, 0, 2428, 0, 0, 0, 157, 0, 0, 0, 78, 0, 0, 234, 0, 0, 0, 0, 0, 7966, 69, 127, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=21 tokens_per_expert_current_ep_rank=[0, 450, 345, 1366, 7994, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5645, 0, 0, 5324, 0, 0, 314, 0, 0, 0, 0, 0, 7799, 83, 0, 0, 0, 0, 0, 8016, 0, 0, 0, 3055, 0, 0, 0, 140, 0, 0, 0, 48, 0, 0, 247, 0, 0, 0, 0, 0, 7989, 63, 157, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=21 tokens_per_expert_current_ep_rank=[0, 422, 307, 1407, 7969, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5490, 0, 0, 5541, 0, 0, 336, 0, 0, 0, 0, 0, 7797, 83, 0, 0, 0, 0, 0, 7966, 0, 0, 0, 2946, 0, 0, 0, 157, 0, 0, 0, 81, 0, 0, 265, 0, 0, 0, 0, 0, 7964, 71, 206, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=21 tokens_per_expert_current_ep_rank=[0, 366, 299, 1540, 7983, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5707, 0, 0, 5362, 0, 0, 336, 0, 0, 0, 0, 0, 7783, 148, 0, 0, 0, 0, 0, 7826, 0, 0, 0, 2742, 0, 0, 0, 231, 0, 0, 0, 174, 0, 0, 246, 0, 0, 0, 0, 0, 7866, 95, 263, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=21 tokens_per_expert_current_ep_rank=[0, 464, 367, 1439, 7976, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5425, 0, 0, 5629, 0, 0, 310, 0, 0, 0, 0, 0, 7813, 131, 0, 0, 0, 0, 0, 7950, 0, 0, 0, 2790, 0, 0, 0, 185, 0, 0, 0, 72, 0, 0, 231, 0, 0, 0, 0, 0, 7925, 82, 199, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=21 tokens_per_expert_current_ep_rank=[0, 433, 315, 1428, 7985, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5555, 0, 0, 5763, 0, 0, 352, 0, 0, 0, 0, 1, 7802, 111, 0, 0, 0, 0, 0, 7977, 0, 0, 0, 2586, 0, 0, 0, 133, 0, 0, 0, 64, 0, 0, 272, 0, 0, 0, 0, 0, 7984, 75, 168, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=21 tokens_per_expert_current_ep_rank=[0, 502, 318, 1358, 7958, 176, 1, 0, 2, 0, 5, 0, 4, 0, 7, 5598, 10, 0, 5915, 0, 6, 344, 0, 0, 0, 0, 5, 7762, 201, 0, 8, 0, 0, 0, 7852, 0, 0, 0, 2336, 0, 0, 0, 209, 0, 5, 0, 141, 7, 1, 245, 3, 0, 0, 9, 0, 7877, 91, 188, 1, 7, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 414, 332, 1416, 3791, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5571, 577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5520, 0, 0, 313, 0, 0, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6011, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3407, 0, 0, 0, 2745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 248, 0, 0, 0, 161, 0, 2798, 2018, 1020, 0, 0, 0, 0, 0, 0, 278, 0, 0, 0, 0, 0, 5836, 0, 0, 0, 0, 0, 0, 0, 0, 90, 204, 0, 0, 0, 0, 0, 0, 4441, 1387, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6133, 6148, 6026, 6212, 6152, 6245, 6114, 6122] avg=6144.0000 max=6245 max_rank=5 min=6026 min_rank=2 imbalance_ratio=1.016439 max_load_reduction=42907 max_load_reduction_pct=0.872945 moved_to_echo=24633 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 366, 299, 1540, 3793, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5707, 576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5362, 0, 0, 336, 0, 0, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5998, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3397, 0, 0, 0, 2742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 231, 0, 0, 0, 174, 0, 2801, 2022, 1017, 0, 0, 0, 0, 0, 0, 246, 0, 0, 0, 0, 0, 5844, 0, 0, 0, 0, 0, 0, 0, 0, 95, 263, 0, 0, 0, 0, 0, 0, 4429, 1389, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6183, 6283, 5890, 6146, 6139, 6245, 6090, 6176] avg=6144.0000 max=6283 max_rank=1 min=5890 min_rank=2 imbalance_ratio=1.022624 max_load_reduction=42869 max_load_reduction_pct=0.872172 moved_to_echo=24650 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 485, 339, 1456, 3804, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5525, 577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5914, 0, 0, 319, 0, 0, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6007, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3463, 0, 0, 0, 2428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 0, 0, 0, 78, 0, 2808, 2047, 1019, 0, 0, 0, 0, 0, 0, 234, 0, 0, 0, 0, 0, 5919, 0, 0, 0, 0, 0, 0, 0, 0, 69, 127, 0, 0, 0, 0, 0, 0, 4513, 1392, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6219, 6102, 6426, 6151, 5891, 6109, 6153, 6101] avg=6144.0000 max=6426 max_rank=2 min=5891 min_rank=4 imbalance_ratio=1.045898 max_load_reduction=42726 max_load_reduction_pct=0.869263 moved_to_echo=24254 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 450, 345, 1366, 3800, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5645, 577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5324, 0, 0, 314, 0, 0, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6009, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3480, 0, 0, 0, 3055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 0, 0, 48, 0, 2804, 2053, 1020, 0, 0, 0, 0, 0, 0, 247, 0, 0, 0, 0, 0, 5936, 0, 0, 0, 0, 0, 0, 0, 0, 63, 157, 0, 0, 0, 0, 0, 0, 4536, 1390, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6078, 6222, 5831, 6092, 6535, 6065, 6183, 6146] avg=6144.0000 max=6535 max_rank=4 min=5831 min_rank=2 imbalance_ratio=1.063639 max_load_reduction=42617 max_load_reduction_pct=0.867045 moved_to_echo=24929 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 422, 307, 1407, 3787, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5490, 577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5541, 0, 0, 336, 0, 0, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6008, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3458, 0, 0, 0, 2946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 0, 0, 0, 81, 0, 2796, 2047, 1019, 0, 0, 0, 0, 0, 0, 265, 0, 0, 0, 0, 0, 5917, 0, 0, 0, 0, 0, 0, 0, 0, 71, 206, 0, 0, 0, 0, 0, 0, 4508, 1386, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6067, 6067, 6070, 6091, 6404, 6100, 6182, 6171] avg=6144.0000 max=6404 max_rank=4 min=6067 min_rank=0 imbalance_ratio=1.042318 max_load_reduction=42748 max_load_reduction_pct=0.869710 moved_to_echo=24857 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 464, 367, 1439, 3791, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5425, 578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5629, 0, 0, 310, 0, 0, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6021, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3451, 0, 0, 0, 2790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, 0, 0, 0, 72, 0, 2798, 2037, 1021, 0, 0, 0, 0, 0, 0, 231, 0, 0, 0, 0, 0, 5888, 0, 0, 0, 0, 0, 0, 0, 0, 82, 199, 0, 0, 0, 0, 0, 0, 4499, 1387, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6225, 6003, 6132, 6152, 6241, 6113, 6119, 6167] avg=6144.0000 max=6241 max_rank=4 min=6003 min_rank=1 imbalance_ratio=1.015788 max_load_reduction=42911 max_load_reduction_pct=0.873027 moved_to_echo=24640 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 433, 315, 1428, 3795, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5555, 577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5763, 0, 0, 352, 0, 0, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6012, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3463, 0, 0, 0, 2586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 133, 0, 0, 0, 64, 0, 2801, 2052, 1020, 0, 0, 0, 0, 0, 0, 272, 0, 0, 0, 0, 0, 5932, 0, 0, 0, 0, 0, 0, 0, 0, 75, 168, 0, 0, 0, 0, 0, 0, 4514, 1389, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6119, 6132, 6308, 6124, 6049, 6070, 6204, 6146] avg=6144.0000 max=6308 max_rank=2 min=6049 min_rank=4 imbalance_ratio=1.026693 max_load_reduction=42844 max_load_reduction_pct=0.871663 moved_to_echo=24469 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 502, 318, 1358, 3773, 176, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5, 0, 4, 0, 7, 5598, 579, 0, 0, 0, 0, 0, 0, 0, 10, 0, 5915, 0, 6, 344, 0, 0, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5970, 201, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3405, 0, 0, 0, 2336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 209, 0, 5, 0, 141, 7, 2796, 2028, 1019, 0, 0, 0, 0, 0, 1, 245, 3, 0, 0, 9, 0, 5849, 0, 0, 0, 0, 0, 0, 0, 0, 91, 188, 1, 7, 0, 0, 0, 0, 4447, 1389, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6128, 6195, 6469, 6184, 5741, 6205, 6107, 6123] avg=6144.0000 max=6469 max_rank=2 min=5741 min_rank=4 imbalance_ratio=1.052897 max_load_reduction=42683 max_load_reduction_pct=0.868388 moved_to_echo=24176 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 0, 1536, 0, 128, 0, 896, 128, 22528, 16384, 8192, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[640, 1536, 128, 128, 0, 0, 0, 0, 35968, 11136, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 3584, 2688, 11520, 30336, 1280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49280 observed_loss=-128 tokens_per_expert=[0, 0, 27648, 0, 0, 0, 21632, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 128, 0, 128, 44544, 4736, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 0, 45056, 0, 128, 2688, 0, 0, 1664, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 2048, 128, 0, 0, 128, 0, 47232, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 128, 48128, 1152, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=22 tokens_per_expert_current_ep_rank=[7980, 0, 0, 0, 0, 0, 0, 1990, 421, 0, 6255, 0, 0, 0, 0, 0, 0, 0, 0, 150, 63, 81, 0, 7706, 0, 0, 0, 0, 0, 0, 0, 0, 483, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 7418, 0, 0, 218, 0, 0, 0, 0, 239, 0, 7616, 0, 0, 0, 0, 0, 218, 71, 0, 0, 74] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=22 tokens_per_expert_current_ep_rank=[7991, 0, 0, 0, 0, 0, 0, 2026, 402, 0, 6174, 0, 0, 0, 0, 0, 0, 0, 0, 155, 19, 78, 0, 7719, 0, 0, 0, 0, 0, 0, 0, 0, 474, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 7458, 0, 0, 177, 0, 0, 0, 0, 261, 0, 7708, 0, 0, 0, 0, 0, 209, 70, 0, 0, 69] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=22 tokens_per_expert_current_ep_rank=[7875, 0, 0, 0, 0, 0, 0, 1929, 392, 0, 6338, 0, 0, 0, 0, 0, 0, 0, 0, 176, 78, 161, 0, 7634, 0, 0, 0, 0, 0, 0, 0, 0, 560, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 7463, 0, 0, 164, 0, 0, 0, 0, 247, 0, 7502, 0, 0, 0, 0, 0, 292, 90, 0, 0, 89] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=22 tokens_per_expert_current_ep_rank=[7880, 0, 0, 0, 0, 0, 0, 1912, 327, 0, 6392, 0, 0, 0, 0, 0, 0, 0, 0, 148, 94, 174, 0, 7596, 0, 0, 0, 0, 0, 0, 0, 0, 595, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 7458, 0, 0, 200, 0, 0, 0, 0, 254, 0, 7431, 0, 0, 0, 0, 0, 330, 96, 0, 0, 95] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=22 tokens_per_expert_current_ep_rank=[7948, 0, 0, 0, 0, 0, 0, 1961, 340, 0, 6373, 0, 0, 0, 0, 0, 0, 0, 0, 184, 79, 72, 0, 7730, 0, 0, 0, 0, 0, 0, 0, 0, 462, 0, 0, 0, 0, 0, 0, 8120, 0, 0, 0, 7499, 0, 0, 175, 0, 0, 0, 0, 264, 0, 7582, 0, 0, 0, 0, 0, 199, 82, 0, 0, 82] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=22 tokens_per_expert_current_ep_rank=[8018, 0, 0, 0, 0, 0, 0, 1828, 377, 0, 6466, 0, 0, 0, 0, 0, 0, 0, 0, 133, 54, 48, 0, 7750, 0, 0, 0, 0, 0, 0, 0, 0, 440, 0, 0, 0, 0, 0, 0, 8139, 0, 0, 0, 7477, 0, 0, 184, 0, 0, 0, 0, 254, 0, 7656, 0, 0, 0, 0, 0, 197, 66, 0, 0, 65] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=22 tokens_per_expert_current_ep_rank=[7891, 5, 0, 0, 0, 0, 7, 1994, 360, 10, 6190, 0, 1, 0, 0, 0, 0, 1, 1, 153, 87, 142, 0, 7633, 0, 0, 0, 0, 0, 2, 0, 2, 578, 4, 0, 0, 0, 0, 0, 8169, 0, 10, 0, 7434, 6, 8, 197, 0, 0, 0, 0, 250, 4, 7556, 0, 4, 0, 0, 0, 260, 99, 3, 0, 91] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=22 tokens_per_expert_current_ep_rank=[7989, 0, 0, 0, 0, 0, 0, 1855, 387, 0, 6433, 0, 0, 0, 0, 0, 0, 0, 0, 158, 69, 64, 0, 7738, 0, 0, 0, 0, 0, 0, 0, 0, 459, 0, 0, 0, 0, 0, 0, 8140, 0, 0, 0, 7453, 0, 0, 193, 0, 0, 0, 0, 246, 0, 7607, 0, 0, 0, 0, 0, 210, 76, 0, 0, 75] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=22 rerouted_tokens_per_expert_current_ep_rank=[4171, 0, 0, 0, 0, 0, 0, 1912, 0, 0, 0, 0, 0, 0, 0, 0, 327, 0, 5826, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 94, 174, 0, 5747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3709, 2407, 0, 0, 0, 0, 0, 0, 595, 0, 0, 0, 0, 0, 0, 5649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5953, 0, 0, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 254, 0, 5774, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 330, 96, 0, 0, 95, 1849, 1657, 1505, 566, 114, 0, 0, 0] rerouted_tokens_per_ep_rank=[6083, 6153, 6163, 6116, 6244, 6153, 6028, 6212] avg=6144.0000 max=6244 max_rank=4 min=6028 min_rank=6 imbalance_ratio=1.016276 max_load_reduction=42908 max_load_reduction_pct=0.872965 moved_to_echo=24637 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=22 rerouted_tokens_per_expert_current_ep_rank=[4168, 0, 0, 0, 0, 0, 0, 1929, 0, 0, 0, 0, 0, 0, 0, 0, 392, 0, 5777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 78, 161, 0, 5776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3707, 2405, 0, 0, 0, 0, 0, 0, 560, 0, 0, 0, 0, 0, 0, 5643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5957, 0, 0, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 0, 5829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 292, 90, 0, 0, 89, 1858, 1673, 1506, 561, 114, 0, 0, 0] rerouted_tokens_per_ep_rank=[6097, 6169, 6191, 6112, 6203, 6121, 6076, 6183] avg=6144.0000 max=6203 max_rank=4 min=6076 min_rank=6 imbalance_ratio=1.009603 max_load_reduction=42949 max_load_reduction_pct=0.873800 moved_to_echo=24583 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=22 rerouted_tokens_per_expert_current_ep_rank=[4230, 0, 0, 0, 0, 0, 0, 2026, 0, 0, 0, 0, 0, 0, 0, 0, 402, 0, 5627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 19, 78, 0, 5840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3761, 2405, 0, 0, 0, 0, 0, 0, 474, 0, 0, 0, 0, 0, 0, 5643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5953, 0, 0, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 261, 0, 5989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 209, 70, 0, 0, 69, 1879, 1719, 1505, 547, 114, 0, 0, 0] rerouted_tokens_per_ep_rank=[6256, 6029, 6092, 6166, 6117, 6130, 6250, 6112] avg=6144.0000 max=6256 max_rank=0 min=6029 min_rank=1 imbalance_ratio=1.018229 max_load_reduction=42896 max_load_reduction_pct=0.872721 moved_to_echo=24609 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=22 rerouted_tokens_per_expert_current_ep_rank=[4224, 0, 0, 0, 0, 0, 0, 1990, 0, 0, 0, 0, 0, 0, 0, 0, 421, 0, 5701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 63, 81, 0, 5830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3756, 2407, 0, 0, 0, 0, 0, 0, 483, 0, 0, 0, 0, 0, 0, 5648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5921, 0, 0, 218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 239, 0, 5918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 218, 71, 0, 0, 74, 1876, 1698, 1497, 554, 114, 0, 0, 0] rerouted_tokens_per_ep_rank=[6214, 6122, 6124, 6163, 6131, 6139, 6157, 6102] avg=6144.0000 max=6214 max_rank=0 min=6102 min_rank=7 imbalance_ratio=1.011393 max_load_reduction=42938 max_load_reduction_pct=0.873576 moved_to_echo=24529 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=22 rerouted_tokens_per_expert_current_ep_rank=[4244, 0, 0, 0, 0, 0, 0, 1828, 0, 0, 0, 0, 0, 0, 0, 0, 377, 0, 5893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 133, 54, 48, 0, 5863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3774, 2398, 0, 0, 0, 0, 0, 0, 440, 0, 0, 0, 0, 0, 0, 5628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5969, 0, 0, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 254, 0, 5949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 197, 66, 0, 0, 65, 1887, 1707, 1508, 573, 113, 0, 0, 0] rerouted_tokens_per_ep_rank=[6072, 6270, 6098, 6172, 6068, 6153, 6203, 6116] avg=6144.0000 max=6270 max_rank=1 min=6068 min_rank=4 imbalance_ratio=1.020508 max_load_reduction=42882 max_load_reduction_pct=0.872437 moved_to_echo=24540 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=22 rerouted_tokens_per_expert_current_ep_rank=[4207, 0, 0, 0, 0, 0, 0, 1961, 0, 0, 0, 0, 0, 0, 0, 0, 340, 0, 5809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 79, 72, 0, 5848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3741, 2392, 0, 0, 0, 0, 0, 0, 462, 0, 0, 0, 0, 0, 0, 5615, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5986, 0, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 264, 0, 5891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 199, 82, 0, 0, 82, 1882, 1691, 1513, 564, 113, 0, 0, 0] rerouted_tokens_per_ep_rank=[6168, 6149, 6183, 6133, 6077, 6161, 6155, 6126] avg=6144.0000 max=6183 max_rank=2 min=6077 min_rank=4 imbalance_ratio=1.006348 max_load_reduction=42969 max_load_reduction_pct=0.874207 moved_to_echo=24519 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=22 rerouted_tokens_per_expert_current_ep_rank=[4229, 0, 0, 0, 0, 0, 0, 1855, 0, 0, 0, 0, 0, 0, 0, 0, 387, 0, 5863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 69, 64, 0, 5854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3760, 2398, 0, 0, 0, 0, 0, 0, 459, 0, 0, 0, 0, 0, 0, 5629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5949, 0, 0, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 0, 5911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 210, 76, 0, 0, 75, 1884, 1696, 1504, 570, 113, 0, 0, 0] rerouted_tokens_per_ep_rank=[6084, 6250, 6145, 6158, 6088, 6142, 6157, 6128] avg=6144.0000 max=6250 max_rank=1 min=6084 min_rank=0 imbalance_ratio=1.017253 max_load_reduction=42902 max_load_reduction_pct=0.872843 moved_to_echo=24515 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=22 rerouted_tokens_per_expert_current_ep_rank=[4172, 5, 0, 0, 0, 0, 7, 1994, 0, 0, 0, 0, 0, 0, 0, 0, 360, 10, 5639, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 153, 87, 142, 0, 5772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 3719, 2409, 0, 0, 0, 0, 0, 0, 578, 4, 0, 0, 0, 0, 0, 5642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 5932, 6, 8, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 250, 4, 5868, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 260, 99, 3, 0, 91, 1861, 1688, 1502, 551, 118, 0, 0, 0] rerouted_tokens_per_ep_rank=[6178, 6010, 6156, 6132, 6224, 6153, 6126, 6173] avg=6144.0000 max=6224 max_rank=4 min=6010 min_rank=1 imbalance_ratio=1.013021 max_load_reduction=42928 max_load_reduction_pct=0.873372 moved_to_echo=24676 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 0, 0, 1920, 768, 128, 0, 640, 14976, 13568, 12160, 4608, 1024, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 128, 0, 128, 29952, 19328, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[33664, 128, 0, 0, 0, 0, 128, 15616, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[4096, 128, 0, 0, 0, 0, 0, 45184, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 128, 1280, 640, 896, 0, 46592, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[3072, 128, 46208, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 0, 47744, 128, 128, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 0, 2048, 128, 47232, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 8040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8127, 0, 88, 0, 0, 338, 29, 0, 0, 0, 60, 0, 168, 0, 0, 0, 0, 0, 0, 5868, 7796, 0, 0, 0, 0, 0, 11, 0, 68, 123, 0, 0, 0, 4635, 8192, 0, 0, 65, 0, 0, 154, 0, 65, 0, 0, 0, 5325, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 8093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8097, 0, 17, 0, 0, 337, 5, 0, 0, 0, 42, 0, 254, 0, 0, 0, 0, 0, 0, 4953, 7810, 0, 0, 0, 0, 0, 0, 0, 110, 137, 0, 0, 0, 4757, 8192, 0, 0, 95, 0, 0, 65, 0, 96, 0, 0, 0, 6092, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 8051, 0, 2, 0, 0, 1, 9, 1, 0, 0, 8086, 0, 155, 0, 0, 345, 48, 0, 0, 0, 54, 0, 221, 2, 1, 0, 0, 0, 0, 5565, 7767, 0, 0, 0, 9, 0, 8, 0, 101, 145, 0, 2, 10, 4717, 8182, 0, 11, 93, 0, 0, 94, 0, 91, 0, 0, 11, 5361, 9, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 8060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8123, 0, 59, 0, 0, 345, 48, 0, 0, 0, 53, 0, 153, 0, 0, 0, 0, 0, 0, 6033, 7793, 0, 3, 0, 0, 0, 7, 0, 72, 119, 0, 0, 0, 4717, 8192, 0, 0, 69, 0, 0, 201, 0, 69, 0, 0, 0, 5036, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 8056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8118, 0, 76, 0, 0, 348, 13, 0, 0, 0, 60, 0, 177, 0, 0, 0, 0, 0, 0, 4943, 7779, 0, 0, 0, 0, 0, 0, 0, 101, 131, 0, 0, 0, 5021, 8192, 0, 0, 74, 0, 0, 20, 0, 74, 0, 0, 0, 5969, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 8039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8102, 0, 63, 0, 0, 303, 1, 0, 0, 0, 65, 0, 268, 0, 0, 0, 0, 0, 0, 4742, 7820, 0, 1, 0, 0, 0, 0, 0, 136, 155, 0, 0, 0, 4896, 8192, 0, 0, 90, 0, 0, 33, 0, 90, 0, 0, 0, 6156, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 7989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8110, 0, 50, 0, 0, 361, 12, 0, 0, 0, 90, 0, 193, 0, 0, 0, 0, 0, 0, 5109, 7740, 0, 1, 0, 0, 0, 0, 0, 152, 171, 0, 0, 0, 4643, 8192, 0, 0, 82, 0, 0, 198, 0, 82, 0, 0, 0, 5977, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 8060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 0, 174, 0, 0, 344, 18, 0, 0, 0, 63, 0, 134, 0, 0, 0, 0, 0, 0, 5289, 7782, 0, 5, 0, 0, 0, 1, 0, 116, 133, 0, 0, 0, 4745, 8192, 0, 0, 75, 0, 0, 56, 0, 75, 0, 0, 0, 5773, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6048, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 337, 5, 0, 0, 0, 42, 0, 2049, 1915, 1181, 584, 0, 0, 0, 0, 254, 0, 0, 0, 0, 0, 0, 4953, 636, 0, 0, 0, 0, 0, 0, 0, 6159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 137, 0, 0, 0, 4757, 1131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 65, 0, 96, 0, 5880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6092, 0, 0, 0, 0, 0, 431, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6178, 6065, 6113, 5843, 6159, 6135, 6136, 6523] avg=6144.0000 max=6523 max_rank=7 min=5843 min_rank=3 imbalance_ratio=1.061686 max_load_reduction=42629 max_load_reduction_pct=0.867289 moved_to_echo=24953 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6071, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 338, 29, 0, 0, 0, 60, 0, 2056, 1902, 1181, 583, 0, 0, 0, 0, 168, 0, 0, 0, 0, 0, 0, 5868, 635, 0, 0, 0, 0, 0, 0, 0, 6148, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 123, 0, 0, 0, 4635, 1131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 154, 0, 65, 0, 5880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5325, 0, 0, 0, 0, 0, 430, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6138, 6159, 6149, 6671, 6159, 5957, 6164, 5755] avg=6144.0000 max=6671 max_rank=3 min=5755 min_rank=7 imbalance_ratio=1.085775 max_load_reduction=42481 max_load_reduction_pct=0.864278 moved_to_echo=24035 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6052, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 303, 1, 0, 0, 0, 65, 0, 2050, 1902, 1181, 584, 0, 0, 0, 0, 268, 0, 0, 0, 0, 0, 0, 4742, 637, 0, 0, 0, 0, 0, 0, 0, 6167, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 155, 0, 0, 0, 4896, 1131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 33, 0, 90, 0, 5880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6156, 0, 0, 0, 0, 0, 432, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6137, 6115, 6086, 5647, 6168, 6318, 6093, 6588] avg=6144.0000 max=6588 max_rank=7 min=5647 min_rank=3 imbalance_ratio=1.072266 max_load_reduction=42564 max_load_reduction_pct=0.865967 moved_to_echo=25167 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6068, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 345, 48, 0, 0, 0, 53, 0, 2055, 1907, 1181, 582, 0, 0, 0, 0, 153, 0, 0, 0, 0, 0, 0, 6033, 635, 0, 0, 0, 0, 0, 0, 0, 6146, 0, 3, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 119, 0, 0, 0, 4717, 1131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 201, 0, 69, 0, 5880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5036, 0, 0, 0, 0, 0, 430, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6153, 6127, 6171, 6821, 6156, 6039, 6219, 5466] avg=6144.0000 max=6821 max_rank=3 min=5466 min_rank=7 imbalance_ratio=1.110189 max_load_reduction=42331 max_load_reduction_pct=0.861226 moved_to_echo=23880 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6064, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 348, 13, 0, 0, 0, 60, 0, 2054, 1906, 1181, 581, 0, 0, 0, 0, 177, 0, 0, 0, 0, 0, 0, 4943, 634, 0, 0, 0, 0, 0, 0, 0, 6135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 131, 0, 0, 0, 5021, 1131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 20, 0, 74, 0, 5880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5969, 0, 0, 0, 0, 0, 429, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6150, 6140, 6143, 5754, 6135, 6384, 6048, 6398] avg=6144.0000 max=6398 max_rank=7 min=5754 min_rank=3 imbalance_ratio=1.041341 max_load_reduction=42754 max_load_reduction_pct=0.869832 moved_to_echo=24965 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6058, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 361, 12, 0, 0, 0, 90, 0, 2052, 1890, 1181, 578, 0, 0, 0, 0, 193, 0, 0, 0, 0, 0, 0, 5109, 631, 0, 0, 0, 0, 0, 0, 0, 6104, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 171, 0, 0, 0, 4643, 1131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 198, 0, 82, 0, 5880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5977, 0, 0, 0, 0, 0, 427, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6099, 6108, 6164, 5933, 6105, 6097, 6242, 6404] avg=6144.0000 max=6404 max_rank=7 min=5933 min_rank=3 imbalance_ratio=1.042318 max_load_reduction=42748 max_load_reduction_pct=0.869710 moved_to_echo=24848 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6141, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 0, 0, 6036, 0, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 345, 48, 0, 0, 0, 54, 0, 2050, 1910, 1185, 585, 0, 0, 0, 0, 221, 2, 1, 0, 0, 0, 0, 5565, 637, 0, 0, 0, 0, 0, 0, 0, 6111, 0, 0, 0, 9, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 145, 0, 2, 10, 4717, 1122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 93, 0, 0, 94, 0, 91, 0, 5875, 0, 0, 0, 0, 0, 0, 0, 0, 11, 5361, 9, 0, 0, 0, 0, 434, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6144, 6201, 6177, 6426, 6128, 6097, 6164, 5815] avg=6144.0000 max=6426 max_rank=3 min=5815 min_rank=7 imbalance_ratio=1.045898 max_load_reduction=42726 max_load_reduction_pct=0.869263 moved_to_echo=24204 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6063, 0, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 344, 18, 0, 0, 0, 63, 0, 2054, 1907, 1181, 582, 0, 0, 0, 0, 134, 0, 0, 0, 0, 0, 0, 5289, 634, 0, 0, 0, 0, 0, 0, 0, 6136, 0, 5, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 133, 0, 0, 0, 4745, 1131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 56, 0, 75, 0, 5880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5773, 0, 0, 0, 0, 0, 430, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6153, 6237, 6149, 6057, 6142, 6125, 6086, 6203] avg=6144.0000 max=6237 max_rank=1 min=6057 min_rank=3 imbalance_ratio=1.015137 max_load_reduction=42915 max_load_reduction_pct=0.873108 moved_to_echo=24556 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 2816, 256, 0, 0, 0, 512, 0, 16512, 15360, 9472, 4736, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1664, 128, 128, 0, 0, 0, 0, 42624, 5120, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[896, 1152, 0, 128, 128, 38144, 9088, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 128, 45696, 128, 0, 0, 0, 0, 3456, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 768, 0, 0, 896, 0, 768, 0, 47104, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[49152, 0, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 49152, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 128, 0, 0, 48512, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 299, 0, 0, 7856, 7069, 0, 0, 69, 28, 0, 0, 0, 0, 0, 0, 6807, 0, 0, 0, 0, 0, 0, 4092, 701, 0, 0, 0, 0, 0, 0, 466, 7371, 0, 8123, 0, 0, 0, 3, 69, 0, 24, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0, 330, 5701, 0, 0, 0, 66] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 295, 0, 0, 7846, 7138, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 6601, 0, 0, 0, 0, 0, 0, 4596, 682, 0, 0, 0, 0, 0, 0, 361, 7540, 0, 8129, 0, 0, 0, 16, 63, 0, 6, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 319, 5374, 0, 0, 0, 75] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 295, 0, 0, 7860, 7110, 0, 0, 90, 76, 0, 0, 0, 0, 0, 0, 6469, 0, 0, 0, 0, 0, 0, 5030, 753, 0, 0, 0, 0, 0, 0, 490, 7164, 0, 8102, 0, 0, 0, 2, 90, 0, 66, 0, 0, 0, 0, 0, 0, 161, 0, 0, 0, 0, 337, 4997, 0, 0, 0, 60] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 279, 0, 0, 7888, 7084, 0, 0, 91, 77, 0, 0, 0, 0, 0, 0, 6801, 0, 0, 0, 0, 0, 0, 5083, 687, 0, 0, 0, 0, 0, 0, 407, 7177, 0, 8097, 0, 0, 0, 21, 95, 0, 84, 0, 0, 0, 0, 0, 0, 174, 0, 0, 0, 0, 349, 4686, 0, 0, 0, 72] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 321, 0, 0, 7893, 7076, 0, 0, 70, 98, 0, 0, 0, 0, 0, 0, 6929, 0, 0, 0, 0, 0, 0, 4944, 726, 0, 0, 0, 0, 0, 0, 578, 7042, 0, 8121, 0, 0, 0, 1, 71, 0, 56, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 313, 4740, 0, 0, 0, 92] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 319, 0, 0, 7867, 7070, 0, 0, 80, 12, 0, 0, 0, 0, 0, 0, 6644, 0, 0, 0, 0, 0, 0, 4879, 706, 0, 0, 0, 0, 0, 0, 460, 7261, 0, 8110, 0, 0, 0, 2, 82, 0, 78, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 346, 5121, 0, 0, 0, 43] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 347, 0, 0, 7878, 7052, 0, 0, 73, 3, 0, 0, 0, 0, 0, 0, 6841, 0, 0, 0, 0, 0, 0, 4182, 650, 0, 0, 0, 0, 0, 0, 433, 7361, 0, 8117, 0, 0, 0, 18, 75, 0, 45, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 321, 5623, 0, 0, 0, 69] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 7, 7, 11, 5, 306, 1, 0, 7860, 7010, 0, 0, 91, 17, 0, 0, 2, 0, 0, 10, 6570, 7, 0, 0, 2, 0, 0, 4530, 761, 7, 0, 1, 10, 0, 0, 510, 7250, 0, 8090, 0, 0, 0, 12, 96, 0, 100, 0, 1, 0, 0, 0, 0, 137, 0, 0, 0, 0, 341, 5329, 0, 1, 1, 69] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 295, 0, 3936, 1567, 302, 0, 0, 0, 0, 0, 0, 0, 6056, 0, 0, 90, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5030, 753, 0, 0, 752, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 5597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 90, 0, 66, 0, 3924, 2066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 0, 0, 6036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 337, 4997, 0, 0, 0, 60, 527, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6100, 6222, 5924, 6553, 6087, 6148, 6197, 5921] avg=6144.0000 max=6553 max_rank=3 min=5921 min_rank=7 imbalance_ratio=1.066569 max_load_reduction=42599 max_load_reduction_pct=0.866679 moved_to_echo=24353 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 295, 0, 3929, 1650, 303, 0, 0, 0, 0, 0, 0, 0, 6080, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4596, 682, 0, 0, 755, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 361, 5890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 63, 0, 6, 0, 3917, 2073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 6056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 319, 5374, 0, 0, 0, 75, 538, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6177, 6143, 6045, 6051, 6251, 6075, 6104, 6306] avg=6144.0000 max=6306 max_rank=7 min=6045 min_rank=2 imbalance_ratio=1.026367 max_load_reduction=42846 max_load_reduction_pct=0.871704 moved_to_echo=24736 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 279, 0, 3950, 1570, 301, 0, 0, 0, 0, 0, 0, 0, 6034, 0, 0, 91, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5083, 687, 0, 0, 749, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 407, 5607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 95, 0, 84, 0, 3938, 2065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 174, 0, 0, 6032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 349, 4686, 0, 0, 0, 72, 554, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6100, 6202, 6228, 6538, 6014, 6203, 6206, 5661] avg=6144.0000 max=6538 max_rank=3 min=5661 min_rank=7 imbalance_ratio=1.064128 max_load_reduction=42614 max_load_reduction_pct=0.866984 moved_to_echo=24084 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 299, 0, 3934, 1613, 300, 0, 0, 0, 0, 0, 0, 0, 6022, 0, 0, 69, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 701, 0, 0, 747, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 466, 5758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 69, 0, 24, 0, 3922, 2073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 6050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 330, 5701, 0, 0, 0, 66, 555, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6146, 6119, 6233, 5559, 6224, 6091, 6128, 6652] avg=6144.0000 max=6652 max_rank=7 min=5559 min_rank=3 imbalance_ratio=1.082682 max_load_reduction=42500 max_load_reduction_pct=0.864665 moved_to_echo=25095 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 321, 0, 3953, 1541, 300, 0, 0, 0, 0, 0, 0, 0, 6028, 0, 0, 70, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4944, 726, 0, 0, 748, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 578, 5501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 71, 0, 56, 0, 3940, 2072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 6049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 313, 4740, 0, 0, 0, 92, 565, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6115, 6196, 6345, 6437, 6079, 6140, 6130, 5710] avg=6144.0000 max=6437 max_rank=3 min=5710 min_rank=7 imbalance_ratio=1.047689 max_load_reduction=42715 max_load_reduction_pct=0.869039 moved_to_echo=24059 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 319, 0, 3940, 1589, 300, 0, 0, 0, 0, 0, 0, 0, 6023, 0, 0, 80, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4879, 706, 0, 0, 747, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 460, 5672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 82, 0, 78, 0, 3927, 2069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 6041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 346, 5121, 0, 0, 0, 43, 541, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6148, 6115, 6085, 6350, 6132, 6158, 6113, 6051] avg=6144.0000 max=6350 max_rank=3 min=6051 min_rank=7 imbalance_ratio=1.033529 max_load_reduction=42802 max_load_reduction_pct=0.870809 moved_to_echo=24454 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 7, 7, 11, 5, 306, 1, 3937, 1589, 301, 0, 0, 0, 0, 0, 0, 0, 5964, 0, 0, 91, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 10, 6008, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4530, 761, 7, 0, 745, 22, 0, 0, 0, 0, 0, 0, 1, 10, 0, 0, 510, 5661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 96, 0, 100, 0, 3923, 2064, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 137, 0, 0, 6026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 341, 5329, 0, 1, 1, 69, 540, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6164, 6072, 6027, 6067, 6182, 6195, 6164, 6281] avg=6144.0000 max=6281 max_rank=7 min=6027 min_rank=2 imbalance_ratio=1.022298 max_load_reduction=42871 max_load_reduction_pct=0.872213 moved_to_echo=24822 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 347, 0, 3945, 1611, 299, 0, 0, 0, 0, 0, 0, 0, 6007, 0, 0, 73, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4182, 650, 0, 0, 746, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 433, 5750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 75, 0, 45, 0, 3933, 2071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 6046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 321, 5623, 0, 0, 0, 69, 557, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6202, 6083, 6265, 5597, 6183, 6142, 6110, 6570] avg=6144.0000 max=6570 max_rank=7 min=5597 min_rank=3 imbalance_ratio=1.069336 max_load_reduction=42582 max_load_reduction_pct=0.866333 moved_to_echo=25005 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 128, 128, 2560, 128, 31616, 12800, 2432, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 37376, 5760, 128, 0, 6016, 256, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 2688, 41600, 0, 128, 128, 640, 4480, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 768, 0, 512, 0, 31488, 16640, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[128, 0, 0, 0, 0, 896, 0, 0, 48384, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49280 observed_loss=-128 tokens_per_expert=[0, 0, 48256, 0, 0, 640, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[128, 128, 0, 0, 3712, 45440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 128, 49152, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True - [2026-08-26 14:53:48] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 843.2 | throughput per GPU (TFLOP/s/GPU): 151.3 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.785368E+00 | loss scale: 1.0 | grad norm: 5.697 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1320, 293, 790, 888, 241, 957, 865, 870, 1417, 190, 659, 305, 262, 1565, 893, 1080, 691, 1002, 821, 677, 227, 510, 227, 1413, 472, 696, 737, 349, 711, 124, 804, 1145, 845, 784, 1243, 1149, 641, 193, 622, 1007, 538, 586, 651, 756, 879, 915, 508, 502, 732, 571, 762, 489, 1019, 1499, 812, 757, 1612, 317, 1562, 725, 184, 761, 1354, 976] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1434, 296, 789, 881, 246, 858, 880, 829, 1403, 217, 641, 280, 262, 1703, 1008, 1213, 663, 995, 819, 590, 248, 569, 251, 1372, 418, 719, 828, 375, 737, 109, 774, 1181, 752, 840, 1211, 1081, 616, 221, 604, 915, 568, 574, 543, 776, 929, 941, 453, 478, 729, 611, 766, 511, 1016, 1441, 850, 744, 1680, 247, 1484, 776, 199, 760, 1351, 897] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1379, 347, 778, 928, 257, 938, 839, 921, 1312, 283, 733, 289, 284, 1647, 847, 1085, 683, 1042, 808, 598, 256, 456, 244, 1340, 431, 735, 825, 359, 727, 80, 886, 1179, 816, 844, 1335, 1121, 638, 200, 571, 839, 624, 650, 593, 731, 987, 950, 473, 481, 656, 587, 743, 512, 989, 1471, 754, 816, 1681, 263, 1472, 686, 155, 726, 1318, 954] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1390, 278, 725, 981, 243, 967, 738, 720, 1410, 235, 622, 276, 233, 1713, 838, 1120, 719, 1030, 857, 603, 247, 471, 294, 1392, 455, 618, 764, 381, 785, 114, 855, 1141, 899, 920, 1140, 1118, 733, 215, 561, 1004, 679, 576, 592, 746, 904, 993, 543, 577, 630, 564, 777, 513, 1013, 1445, 696, 918, 1520, 358, 1457, 665, 164, 710, 1282, 1025] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1407, 310, 859, 913, 255, 895, 802, 896, 1408, 226, 704, 312, 301, 1535, 879, 1050, 670, 955, 880, 587, 281, 530, 251, 1357, 453, 720, 806, 369, 717, 96, 822, 1408, 848, 913, 1133, 1094, 484, 227, 571, 936, 574, 586, 529, 739, 877, 974, 490, 601, 688, 564, 842, 586, 1071, 1359, 779, 805, 1524, 290, 1485, 766, 187, 722, 1311, 943] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1328, 290, 766, 932, 265, 1052, 864, 796, 1376, 209, 641, 330, 241, 1491, 966, 1088, 722, 950, 853, 593, 235, 520, 234, 1361, 514, 647, 785, 430, 764, 146, 843, 1235, 832, 778, 1206, 1129, 463, 203, 648, 948, 509, 517, 529, 712, 921, 1002, 521, 575, 754, 569, 769, 508, 1024, 1447, 766, 825, 1596, 407, 1586, 771, 132, 796, 1295, 947] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1374, 304, 702, 1002, 264, 1013, 836, 849, 1469, 248, 705, 352, 305, 1588, 963, 1199, 659, 920, 867, 664, 295, 543, 265, 1240, 470, 582, 742, 390, 722, 104, 819, 1296, 752, 885, 917, 1108, 664, 195, 634, 924, 537, 611, 531, 695, 952, 940, 486, 545, 713, 512, 749, 570, 1016, 1458, 716, 827, 1570, 369, 1495, 741, 158, 803, 1292, 1036] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1370, 317, 742, 959, 303, 836, 834, 858, 1419, 238, 555, 323, 323, 1587, 894, 1133, 656, 990, 781, 548, 288, 506, 272, 1217, 437, 709, 803, 344, 739, 104, 868, 1122, 814, 782, 1257, 1126, 788, 191, 657, 941, 533, 587, 645, 818, 896, 991, 503, 600, 684, 586, 758, 517, 1017, 1375, 758, 799, 1644, 327, 1584, 679, 163, 734, 1254, 1069] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1212, 293, 790, 888, 241, 957, 865, 870, 0, 0, 0, 0, 0, 0, 0, 0, 1417, 190, 659, 305, 262, 1209, 893, 1080, 0, 0, 0, 0, 0, 0, 0, 0, 691, 1002, 821, 677, 227, 510, 227, 1413, 349, 207, 108, 0, 0, 0, 0, 0, 472, 696, 737, 349, 711, 124, 804, 1145, 959, 0, 0, 0, 0, 0, 0, 0, 845, 784, 1036, 1149, 641, 193, 622, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 538, 586, 651, 756, 879, 915, 508, 502, 476, 278, 7, 0, 0, 0, 0, 0, 732, 571, 762, 489, 1019, 1023, 812, 757, 0, 0, 0, 0, 0, 0, 0, 0, 375, 317, 1562, 725, 184, 761, 1354, 976, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6116, 6015, 6232, 5997, 6277, 6096, 6165, 6254] avg=6144.0000 max=6277 max_rank=4 min=5997 min_rank=3 imbalance_ratio=1.021647 max_load_reduction=42875 max_load_reduction_pct=0.872294 moved_to_echo=24792 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1292, 310, 859, 913, 255, 895, 802, 896, 0, 0, 0, 0, 0, 0, 0, 0, 1408, 226, 704, 312, 301, 1186, 879, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 670, 955, 880, 587, 281, 530, 251, 1357, 342, 188, 115, 0, 0, 0, 0, 0, 453, 720, 806, 369, 717, 96, 822, 1408, 906, 0, 0, 0, 0, 0, 0, 0, 848, 913, 945, 1094, 484, 227, 571, 936, 0, 0, 0, 0, 0, 0, 0, 0, 574, 586, 529, 739, 877, 974, 490, 601, 431, 262, 7, 0, 0, 0, 0, 0, 688, 564, 842, 586, 1071, 928, 779, 805, 0, 0, 0, 0, 0, 0, 0, 0, 356, 290, 1485, 766, 187, 722, 1311, 943, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6222, 6066, 6156, 6297, 6018, 6070, 6263, 6060] avg=6144.0000 max=6297 max_rank=3 min=6018 min_rank=4 imbalance_ratio=1.024902 max_load_reduction=42855 max_load_reduction_pct=0.871887 moved_to_echo=24411 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1316, 296, 789, 881, 246, 858, 880, 829, 0, 0, 0, 0, 0, 0, 0, 0, 1403, 217, 641, 280, 262, 1315, 1008, 1213, 0, 0, 0, 0, 0, 0, 0, 0, 663, 995, 819, 590, 248, 569, 251, 1372, 380, 201, 118, 0, 0, 0, 0, 0, 418, 719, 828, 375, 737, 109, 774, 1181, 999, 0, 0, 0, 0, 0, 0, 0, 752, 840, 1010, 1081, 616, 221, 604, 915, 0, 0, 0, 0, 0, 0, 0, 0, 568, 574, 543, 776, 929, 941, 453, 478, 457, 289, 8, 0, 0, 0, 0, 0, 729, 611, 766, 511, 1016, 984, 850, 744, 0, 0, 0, 0, 0, 0, 0, 0, 392, 247, 1484, 776, 199, 760, 1351, 897, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6095, 6339, 6206, 6140, 6039, 6016, 6211, 6106] avg=6144.0000 max=6339 max_rank=1 min=6016 min_rank=5 imbalance_ratio=1.031738 max_load_reduction=42813 max_load_reduction_pct=0.871033 moved_to_echo=24372 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1219, 290, 766, 932, 265, 1052, 864, 796, 0, 0, 0, 0, 0, 0, 0, 0, 1376, 209, 641, 330, 241, 1151, 966, 1088, 0, 0, 0, 0, 0, 0, 0, 0, 722, 950, 853, 593, 235, 520, 234, 1361, 333, 201, 109, 0, 0, 0, 0, 0, 514, 647, 785, 430, 764, 146, 843, 1235, 949, 0, 0, 0, 0, 0, 0, 0, 832, 778, 1005, 1129, 463, 203, 648, 948, 0, 0, 0, 0, 0, 0, 0, 0, 509, 517, 529, 712, 921, 1002, 521, 575, 459, 275, 7, 0, 0, 0, 0, 0, 754, 569, 769, 508, 1024, 988, 766, 825, 0, 0, 0, 0, 0, 0, 0, 0, 372, 407, 1586, 771, 132, 796, 1295, 947, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6184, 6002, 6111, 6313, 6006, 6027, 6203, 6306] avg=6144.0000 max=6313 max_rank=3 min=6002 min_rank=1 imbalance_ratio=1.027507 max_load_reduction=42839 max_load_reduction_pct=0.871562 moved_to_echo=24542 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1276, 278, 725, 981, 243, 967, 738, 720, 0, 0, 0, 0, 0, 0, 0, 0, 1410, 235, 622, 276, 233, 1323, 838, 1120, 0, 0, 0, 0, 0, 0, 0, 0, 719, 1030, 857, 603, 247, 471, 294, 1392, 382, 190, 114, 0, 0, 0, 0, 0, 455, 618, 764, 381, 785, 114, 855, 1141, 904, 0, 0, 0, 0, 0, 0, 0, 899, 920, 950, 1118, 733, 215, 561, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 679, 576, 592, 746, 904, 993, 543, 577, 459, 262, 8, 0, 0, 0, 0, 0, 630, 564, 777, 513, 1013, 986, 696, 918, 0, 0, 0, 0, 0, 0, 0, 0, 354, 358, 1457, 665, 164, 710, 1282, 1025, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5928, 6057, 6299, 6017, 6400, 6339, 6097, 6015] avg=6144.0000 max=6400 max_rank=4 min=5928 min_rank=0 imbalance_ratio=1.041667 max_load_reduction=42752 max_load_reduction_pct=0.869792 moved_to_echo=24851 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1266, 347, 778, 928, 257, 938, 839, 921, 0, 0, 0, 0, 0, 0, 0, 0, 1312, 283, 733, 289, 284, 1272, 847, 1085, 0, 0, 0, 0, 0, 0, 0, 0, 683, 1042, 808, 598, 256, 456, 244, 1340, 367, 222, 113, 0, 0, 0, 0, 0, 431, 735, 825, 359, 727, 80, 886, 1179, 1000, 0, 0, 0, 0, 0, 0, 0, 816, 844, 1113, 1121, 638, 200, 571, 839, 0, 0, 0, 0, 0, 0, 0, 0, 624, 650, 593, 731, 987, 950, 473, 481, 467, 290, 8, 0, 0, 0, 0, 0, 656, 587, 743, 512, 989, 1004, 754, 816, 0, 0, 0, 0, 0, 0, 0, 0, 391, 263, 1472, 686, 155, 726, 1318, 954, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6274, 6105, 6129, 6222, 6142, 6254, 6061, 5965] avg=6144.0000 max=6274 max_rank=0 min=5965 min_rank=7 imbalance_ratio=1.021159 max_load_reduction=42878 max_load_reduction_pct=0.872355 moved_to_echo=24422 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1253, 317, 742, 959, 303, 836, 834, 858, 0, 0, 0, 0, 0, 0, 0, 0, 1419, 238, 555, 323, 323, 1216, 894, 1133, 0, 0, 0, 0, 0, 0, 0, 0, 656, 990, 781, 548, 288, 506, 272, 1217, 359, 213, 117, 0, 0, 0, 0, 0, 437, 709, 803, 344, 739, 104, 868, 1122, 981, 0, 0, 0, 0, 0, 0, 0, 814, 782, 1044, 1126, 788, 191, 657, 941, 0, 0, 0, 0, 0, 0, 0, 0, 533, 587, 645, 818, 896, 991, 503, 600, 440, 287, 12, 0, 0, 0, 0, 0, 684, 586, 758, 517, 1017, 935, 758, 799, 0, 0, 0, 0, 0, 0, 0, 0, 376, 327, 1584, 679, 163, 734, 1254, 1069, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6102, 6101, 5947, 6107, 6343, 6312, 6054, 6186] avg=6144.0000 max=6343 max_rank=4 min=5947 min_rank=2 imbalance_ratio=1.032389 max_load_reduction=42809 max_load_reduction_pct=0.870951 moved_to_echo=24895 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1261, 304, 702, 1002, 264, 1013, 836, 849, 0, 0, 0, 0, 0, 0, 0, 0, 1469, 248, 705, 352, 305, 1226, 963, 1199, 0, 0, 0, 0, 0, 0, 0, 0, 659, 920, 867, 664, 295, 543, 265, 1240, 354, 152, 113, 0, 0, 0, 0, 0, 470, 582, 742, 390, 722, 104, 819, 1296, 934, 0, 0, 0, 0, 0, 0, 0, 752, 885, 765, 1108, 664, 195, 634, 924, 0, 0, 0, 0, 0, 0, 0, 0, 537, 611, 531, 695, 952, 940, 486, 545, 463, 270, 8, 0, 0, 0, 0, 0, 713, 512, 749, 570, 1016, 995, 716, 827, 0, 0, 0, 0, 0, 0, 0, 0, 366, 369, 1495, 741, 158, 803, 1292, 1036, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6231, 6467, 6072, 6059, 5927, 6038, 6098, 6260] avg=6144.0000 max=6467 max_rank=1 min=5927 min_rank=4 imbalance_ratio=1.052572 max_load_reduction=42685 max_load_reduction_pct=0.868429 moved_to_echo=24323 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[10112, 2560, 6272, 7552, 2176, 7552, 6784, 6784, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[11264, 1920, 5376, 2560, 2304, 9984, 7296, 9088, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[5632, 4608, 6272, 4224, 8192, 7936, 6144, 6528, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6656, 6784, 7936, 8960, 5120, 1664, 4992, 7552, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[4608, 4736, 4736, 6016, 7424, 7808, 4096, 4480, 3712, 2304, 128, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3072, 2688, 12160, 5888, 1408, 6016, 10496, 7936, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[5504, 7936, 6784, 4864, 2176, 4224, 2048, 10752, 2944, 1664, 1024, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3712, 5504, 6400, 3072, 6016, 896, 6784, 9728, 7680, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[623, 893, 673, 450, 1068, 291, 320, 1191, 61, 904, 1598, 320, 601, 1088, 722, 1708, 1083, 1319, 820, 484, 1512, 129, 1837, 1042, 714, 719, 740, 892, 81, 306, 176, 123, 425, 162, 462, 205, 1103, 1536, 1018, 1051, 2107, 1386, 1057, 1438, 313, 913, 16, 186, 882, 1139, 1001, 20, 121, 1583, 199, 1440, 881, 321, 863, 535, 1042, 565, 317, 377] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[623, 850, 584, 453, 1404, 276, 383, 1274, 112, 962, 1644, 361, 709, 1035, 700, 1767, 974, 1173, 825, 349, 1378, 148, 1693, 1079, 750, 633, 788, 966, 71, 201, 190, 132, 424, 169, 431, 101, 1163, 1291, 926, 930, 1982, 1406, 930, 1355, 309, 1092, 35, 218, 1124, 1317, 1304, 9, 84, 1375, 236, 1319, 996, 329, 926, 606, 940, 576, 352, 410] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[523, 998, 570, 420, 1290, 260, 338, 1190, 79, 1011, 1814, 367, 603, 1027, 779, 1778, 1068, 1188, 817, 416, 1289, 195, 1639, 1234, 752, 805, 750, 884, 63, 205, 180, 121, 353, 170, 458, 120, 1203, 1213, 975, 998, 1920, 1384, 971, 1275, 267, 1024, 17, 214, 1093, 1315, 1447, 4, 67, 1388, 217, 1459, 895, 307, 893, 530, 943, 575, 373, 431] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[644, 733, 810, 585, 1269, 163, 260, 1186, 83, 987, 1590, 334, 618, 1062, 801, 1656, 1159, 1155, 860, 455, 1290, 160, 1610, 1298, 691, 701, 765, 916, 64, 371, 224, 128, 362, 181, 389, 109, 1115, 1386, 1014, 920, 2187, 1382, 975, 1223, 272, 1094, 10, 201, 1070, 1240, 1276, 5, 107, 1463, 191, 1320, 934, 347, 950, 561, 945, 541, 320, 434] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[582, 783, 884, 428, 1297, 233, 307, 1152, 70, 933, 1644, 348, 640, 943, 688, 1781, 1001, 1228, 936, 451, 1385, 183, 1833, 1185, 735, 793, 846, 899, 85, 337, 200, 125, 343, 232, 390, 114, 1137, 1258, 848, 917, 2185, 1332, 939, 1245, 276, 947, 20, 191, 1184, 1238, 1259, 8, 115, 1268, 219, 1428, 977, 318, 921, 587, 930, 586, 355, 450] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[609, 818, 901, 389, 1059, 233, 323, 1148, 67, 960, 1488, 335, 613, 980, 707, 1618, 1070, 1213, 889, 533, 1391, 205, 1829, 1120, 763, 746, 751, 910, 82, 323, 165, 133, 340, 219, 434, 148, 1196, 1560, 1007, 935, 2202, 1483, 997, 1232, 357, 1096, 16, 191, 867, 1198, 1104, 9, 134, 1498, 239, 1414, 997, 338, 781, 514, 949, 572, 375, 379] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[650, 837, 797, 401, 1326, 236, 351, 1134, 92, 891, 1544, 347, 622, 1113, 804, 1761, 1001, 1154, 886, 432, 1526, 168, 1783, 1152, 681, 712, 717, 932, 80, 251, 205, 144, 369, 154, 421, 114, 1099, 1515, 844, 1013, 2177, 1358, 972, 1311, 280, 998, 37, 218, 1035, 1170, 998, 7, 120, 1398, 244, 1413, 1002, 358, 920, 628, 908, 602, 322, 417] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[639, 856, 703, 449, 1089, 207, 322, 1283, 107, 932, 1539, 338, 619, 1018, 769, 1630, 1204, 1277, 963, 454, 1322, 138, 1684, 1085, 621, 711, 660, 915, 65, 292, 168, 105, 413, 208, 463, 161, 1256, 1546, 1009, 938, 2115, 1476, 997, 1334, 305, 1080, 15, 154, 917, 1238, 1126, 7, 109, 1551, 241, 1332, 966, 308, 908, 561, 946, 573, 318, 417] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[623, 893, 673, 450, 1068, 291, 320, 1191, 389, 177, 0, 0, 0, 0, 0, 0, 61, 904, 1598, 320, 601, 1088, 722, 753, 0, 0, 0, 0, 0, 0, 0, 0, 1083, 1319, 820, 484, 1335, 129, 0, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 714, 719, 740, 892, 81, 306, 176, 123, 1837, 623, 0, 0, 0, 0, 0, 0, 425, 162, 462, 205, 1103, 1536, 1018, 1051, 407, 126, 0, 0, 0, 0, 0, 0, 916, 1386, 1057, 1438, 313, 913, 16, 186, 0, 0, 0, 0, 0, 0, 0, 0, 882, 1139, 1001, 20, 121, 1068, 199, 1440, 0, 0, 0, 0, 0, 0, 0, 0, 881, 321, 863, 535, 1042, 565, 317, 377, 568, 548, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6075, 6047, 6212, 6211, 6495, 6225, 5870, 6017] avg=6144.0000 max=6495 max_rank=4 min=5870 min_rank=6 imbalance_ratio=1.057129 max_load_reduction=42657 max_load_reduction_pct=0.867859 moved_to_echo=24607 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[582, 783, 884, 428, 1297, 233, 307, 1152, 311, 162, 0, 0, 0, 0, 0, 0, 70, 933, 1644, 348, 640, 943, 688, 784, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1228, 936, 451, 1223, 183, 0, 1185, 0, 0, 0, 0, 0, 0, 0, 0, 735, 793, 846, 899, 85, 337, 200, 125, 1833, 646, 0, 0, 0, 0, 0, 0, 343, 232, 390, 114, 1137, 1258, 848, 917, 425, 101, 0, 0, 0, 0, 0, 0, 950, 1332, 939, 1245, 276, 947, 20, 191, 0, 0, 0, 0, 0, 0, 0, 0, 1184, 1238, 1259, 8, 115, 856, 219, 1428, 0, 0, 0, 0, 0, 0, 0, 0, 977, 318, 921, 587, 930, 586, 355, 450, 589, 572, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6139, 6050, 6207, 6499, 5765, 5900, 6307, 6285] avg=6144.0000 max=6499 max_rank=3 min=5765 min_rank=4 imbalance_ratio=1.057780 max_load_reduction=42653 max_load_reduction_pct=0.867778 moved_to_echo=24257 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[623, 850, 584, 453, 1404, 276, 383, 1274, 337, 161, 0, 0, 0, 0, 0, 0, 112, 962, 1644, 361, 709, 1035, 700, 779, 0, 0, 0, 0, 0, 0, 0, 0, 974, 1173, 825, 349, 1217, 148, 0, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 750, 633, 788, 966, 71, 201, 190, 132, 1693, 586, 0, 0, 0, 0, 0, 0, 424, 169, 431, 101, 1163, 1291, 926, 930, 421, 109, 0, 0, 0, 0, 0, 0, 862, 1406, 930, 1355, 309, 1092, 35, 218, 0, 0, 0, 0, 0, 0, 0, 0, 1124, 1317, 1304, 9, 84, 929, 236, 1319, 0, 0, 0, 0, 0, 0, 0, 0, 996, 329, 926, 606, 940, 576, 352, 410, 534, 567, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6345, 6302, 5765, 6010, 5965, 6207, 6322, 6236] avg=6144.0000 max=6345 max_rank=0 min=5765 min_rank=2 imbalance_ratio=1.032715 max_load_reduction=42807 max_load_reduction_pct=0.870911 moved_to_echo=24730 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[644, 733, 810, 585, 1269, 163, 260, 1186, 359, 151, 0, 0, 0, 0, 0, 0, 83, 987, 1590, 334, 618, 1062, 801, 729, 0, 0, 0, 0, 0, 0, 0, 0, 1159, 1155, 860, 455, 1139, 160, 0, 1298, 0, 0, 0, 0, 0, 0, 0, 0, 691, 701, 765, 916, 64, 371, 224, 128, 1610, 646, 0, 0, 0, 0, 0, 0, 362, 181, 389, 109, 1115, 1386, 1014, 920, 395, 116, 0, 0, 0, 0, 0, 0, 951, 1382, 975, 1223, 272, 1094, 10, 201, 0, 0, 0, 0, 0, 0, 0, 0, 1070, 1240, 1276, 5, 107, 988, 191, 1320, 0, 0, 0, 0, 0, 0, 0, 0, 934, 347, 950, 561, 945, 541, 320, 434, 590, 532, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6160, 6204, 6226, 6116, 5987, 6108, 6197, 6154] avg=6144.0000 max=6226 max_rank=2 min=5987 min_rank=4 imbalance_ratio=1.013346 max_load_reduction=42926 max_load_reduction_pct=0.873332 moved_to_echo=24446 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[523, 998, 570, 420, 1290, 260, 338, 1190, 341, 150, 0, 0, 0, 0, 0, 0, 79, 1011, 1814, 367, 603, 1027, 779, 783, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 1188, 817, 416, 1139, 195, 0, 1234, 0, 0, 0, 0, 0, 0, 0, 0, 752, 805, 750, 884, 63, 205, 180, 121, 1639, 567, 0, 0, 0, 0, 0, 0, 353, 170, 458, 120, 1203, 1213, 975, 998, 424, 110, 0, 0, 0, 0, 0, 0, 835, 1384, 971, 1275, 267, 1024, 17, 214, 0, 0, 0, 0, 0, 0, 0, 0, 1093, 1315, 1447, 4, 67, 937, 217, 1459, 0, 0, 0, 0, 0, 0, 0, 0, 895, 307, 893, 530, 943, 575, 373, 431, 518, 571, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6080, 6463, 6057, 5966, 6024, 5987, 6539, 6036] avg=6144.0000 max=6539 max_rank=6 min=5966 min_rank=3 imbalance_ratio=1.064290 max_load_reduction=42613 max_load_reduction_pct=0.866964 moved_to_echo=24586 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[609, 818, 901, 389, 1059, 233, 323, 1148, 368, 162, 0, 0, 0, 0, 0, 0, 67, 960, 1488, 335, 613, 980, 707, 713, 0, 0, 0, 0, 0, 0, 0, 0, 1070, 1213, 889, 533, 1229, 205, 0, 1120, 0, 0, 0, 0, 0, 0, 0, 0, 763, 746, 751, 910, 82, 323, 165, 133, 1829, 651, 0, 0, 0, 0, 0, 0, 340, 219, 434, 148, 1196, 1560, 1007, 935, 386, 119, 0, 0, 0, 0, 0, 0, 957, 1483, 997, 1232, 357, 1096, 16, 191, 0, 0, 0, 0, 0, 0, 0, 0, 867, 1198, 1104, 9, 134, 1011, 239, 1414, 0, 0, 0, 0, 0, 0, 0, 0, 997, 338, 781, 514, 949, 572, 375, 379, 594, 519, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6010, 5863, 6259, 6353, 6344, 6329, 5976, 6018] avg=6144.0000 max=6353 max_rank=3 min=5863 min_rank=1 imbalance_ratio=1.034017 max_load_reduction=42799 max_load_reduction_pct=0.870748 moved_to_echo=24667 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[650, 837, 797, 401, 1326, 236, 351, 1134, 346, 182, 0, 0, 0, 0, 0, 0, 92, 891, 1544, 347, 622, 1113, 804, 769, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1154, 886, 432, 1344, 168, 0, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 681, 712, 717, 932, 80, 251, 205, 144, 1783, 646, 0, 0, 0, 0, 0, 0, 369, 154, 421, 114, 1099, 1515, 844, 1013, 423, 115, 0, 0, 0, 0, 0, 0, 941, 1358, 972, 1311, 280, 998, 37, 218, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1170, 998, 7, 120, 937, 244, 1413, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 358, 920, 628, 908, 602, 322, 417, 590, 569, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6260, 6182, 6137, 6151, 6067, 6115, 5924, 6316] avg=6144.0000 max=6316 max_rank=7 min=5924 min_rank=6 imbalance_ratio=1.027995 max_load_reduction=42836 max_load_reduction_pct=0.871501 moved_to_echo=24422 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[639, 856, 703, 449, 1089, 207, 322, 1283, 381, 154, 0, 0, 0, 0, 0, 0, 107, 932, 1539, 338, 619, 1018, 769, 719, 0, 0, 0, 0, 0, 0, 0, 0, 1204, 1277, 963, 454, 1168, 138, 0, 1085, 0, 0, 0, 0, 0, 0, 0, 0, 621, 711, 660, 915, 65, 292, 168, 105, 1684, 625, 0, 0, 0, 0, 0, 0, 413, 208, 463, 161, 1256, 1546, 1009, 938, 388, 123, 0, 0, 0, 0, 0, 0, 920, 1476, 997, 1334, 305, 1080, 15, 154, 0, 0, 0, 0, 0, 0, 0, 0, 917, 1238, 1126, 7, 109, 1047, 241, 1332, 0, 0, 0, 0, 0, 0, 0, 0, 966, 308, 908, 561, 946, 573, 318, 417, 570, 523, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6083, 6041, 6289, 5846, 6505, 6281, 6017, 6090] avg=6144.0000 max=6505 max_rank=4 min=5846 min_rank=3 imbalance_ratio=1.058757 max_load_reduction=42647 max_load_reduction_pct=0.867655 moved_to_echo=24893 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4992, 6784, 6016, 3584, 9856, 1920, 2688, 9600, 2944, 1408, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[7424, 11264, 7936, 10496, 2432, 8320, 256, 1664, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5760, 5888, 6144, 7424, 640, 2304, 1536, 1024, 13952, 4992, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[768, 7680, 12928, 2816, 5120, 8320, 6016, 6144, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7680, 2688, 7168, 4608, 7680, 4608, 2816, 3328, 4608, 4480, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3072, 1536, 3456, 1152, 9344, 11392, 7680, 7808, 3328, 1024, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[8576, 9728, 7040, 3584, 9856, 1408, 0, 9216, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[8192, 9856, 9600, 128, 896, 7808, 1792, 11136, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[92, 991, 564, 263, 82, 1274, 856, 300, 621, 341, 1085, 1279, 1624, 1169, 176, 1325, 968, 1551, 348, 368, 73, 219, 1239, 911, 449, 684, 1071, 827, 294, 495, 935, 338, 405, 1103, 1201, 718, 2255, 1817, 1817, 533, 350, 244, 337, 192, 1717, 1201, 138, 1193, 758, 134, 1112, 790, 106, 165, 268, 745, 1553, 408, 102, 505, 120, 1540, 420, 2393] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[12, 1129, 664, 292, 15, 1207, 765, 236, 618, 443, 1138, 1373, 1600, 832, 20, 741, 1095, 1750, 366, 329, 86, 122, 1176, 995, 639, 731, 1161, 1076, 230, 432, 831, 156, 394, 1140, 1196, 697, 2892, 2346, 1660, 477, 503, 186, 316, 237, 1630, 1244, 137, 1105, 640, 80, 945, 976, 49, 58, 165, 552, 1817, 505, 43, 406, 47, 1927, 133, 2389] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[16, 983, 510, 330, 25, 1385, 738, 234, 598, 426, 1216, 1561, 1566, 882, 38, 861, 1137, 1527, 422, 278, 81, 167, 1341, 1041, 532, 691, 1215, 1115, 230, 413, 982, 168, 401, 1361, 1234, 599, 3057, 1974, 1555, 516, 526, 157, 227, 266, 1555, 1209, 139, 1270, 767, 99, 847, 770, 33, 97, 267, 595, 1580, 455, 73, 442, 118, 1772, 128, 2354] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[35, 1272, 607, 344, 30, 1328, 834, 251, 656, 372, 1231, 1359, 1851, 1095, 65, 851, 1018, 1561, 331, 306, 66, 225, 1267, 931, 571, 709, 1053, 910, 269, 484, 1046, 208, 353, 1175, 1068, 653, 2635, 1907, 1802, 482, 555, 234, 322, 201, 1672, 1219, 193, 1208, 833, 148, 732, 752, 56, 102, 317, 558, 1651, 393, 66, 431, 159, 1564, 187, 2388] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[14, 1042, 566, 337, 19, 1195, 745, 279, 697, 413, 1277, 1285, 1780, 952, 17, 805, 1080, 1768, 371, 326, 218, 175, 1237, 890, 684, 686, 1222, 879, 125, 439, 840, 196, 396, 1067, 1169, 620, 2855, 2376, 1712, 408, 467, 209, 387, 278, 1650, 1295, 132, 865, 748, 98, 673, 809, 44, 112, 227, 515, 1788, 512, 73, 377, 79, 1842, 180, 2630] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[23, 879, 555, 311, 47, 1422, 859, 256, 572, 418, 1251, 1386, 1619, 1002, 43, 1219, 1073, 1611, 421, 313, 92, 165, 1253, 1150, 517, 640, 1115, 1100, 291, 488, 1068, 206, 390, 1165, 1156, 596, 2403, 2020, 1652, 483, 517, 157, 318, 331, 1539, 1116, 145, 1153, 832, 128, 938, 849, 43, 104, 248, 635, 1562, 372, 83, 463, 90, 1653, 198, 2448] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[46, 1015, 549, 370, 69, 1379, 819, 269, 565, 254, 1107, 1416, 1641, 1103, 202, 1391, 1064, 1320, 270, 362, 83, 261, 1332, 872, 499, 680, 1118, 973, 254, 494, 882, 247, 398, 1161, 1153, 736, 2164, 1811, 1937, 520, 371, 207, 269, 143, 1855, 1271, 157, 1130, 908, 118, 795, 771, 105, 148, 284, 769, 1595, 413, 67, 432, 151, 1600, 392, 2415] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[31, 1052, 521, 318, 34, 1375, 795, 269, 646, 462, 1096, 1421, 1547, 996, 48, 956, 1050, 1614, 366, 339, 80, 162, 1270, 955, 581, 730, 1065, 981, 226, 498, 982, 176, 383, 1301, 1217, 557, 2678, 1859, 1766, 466, 438, 198, 316, 221, 1668, 1284, 159, 1165, 716, 87, 937, 858, 47, 126, 232, 655, 1652, 526, 56, 503, 97, 1679, 181, 2482] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[92, 991, 564, 263, 82, 1274, 856, 300, 1342, 154, 92, 0, 0, 0, 0, 0, 621, 341, 1085, 1279, 465, 1169, 176, 1325, 0, 0, 0, 0, 0, 0, 0, 0, 968, 1551, 348, 368, 73, 219, 1239, 911, 283, 0, 0, 0, 0, 0, 0, 0, 449, 684, 1071, 827, 294, 495, 935, 338, 899, 0, 0, 0, 0, 0, 0, 0, 405, 1103, 1201, 718, 0, 475, 1817, 533, 0, 0, 0, 0, 0, 0, 0, 0, 350, 244, 337, 192, 1717, 1201, 138, 1193, 637, 106, 0, 0, 0, 0, 0, 0, 758, 134, 1112, 790, 106, 165, 268, 745, 2163, 0, 0, 0, 0, 0, 0, 0, 1553, 408, 102, 505, 120, 1540, 420, 1473, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6010, 6461, 5960, 5992, 6252, 6115, 6241, 6121] avg=6144.0000 max=6461 max_rank=1 min=5960 min_rank=2 imbalance_ratio=1.051595 max_load_reduction=42691 max_load_reduction_pct=0.868551 moved_to_echo=24729 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[16, 983, 510, 330, 25, 1385, 738, 234, 1458, 149, 125, 0, 0, 0, 0, 0, 598, 426, 1216, 1561, 448, 882, 38, 861, 0, 0, 0, 0, 0, 0, 0, 0, 1137, 1527, 422, 278, 81, 167, 1341, 1041, 279, 0, 0, 0, 0, 0, 0, 0, 532, 691, 1215, 1115, 230, 413, 982, 168, 867, 0, 0, 0, 0, 0, 0, 0, 401, 1361, 1234, 599, 0, 516, 1555, 516, 0, 0, 0, 0, 0, 0, 0, 0, 526, 157, 227, 266, 1555, 1209, 139, 1270, 627, 102, 0, 0, 0, 0, 0, 0, 767, 99, 847, 770, 33, 97, 267, 595, 2932, 0, 0, 0, 0, 0, 0, 0, 1580, 455, 73, 442, 118, 1772, 128, 1448, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5953, 6030, 6273, 6213, 6182, 6078, 6407, 6016] avg=6144.0000 max=6407 max_rank=6 min=5953 min_rank=0 imbalance_ratio=1.042806 max_load_reduction=42745 max_load_reduction_pct=0.869649 moved_to_echo=24683 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[12, 1129, 664, 292, 15, 1207, 765, 236, 1733, 152, 120, 0, 0, 0, 0, 0, 618, 443, 1138, 1373, 459, 832, 20, 741, 0, 0, 0, 0, 0, 0, 0, 0, 1095, 1750, 366, 329, 86, 122, 1176, 995, 283, 0, 0, 0, 0, 0, 0, 0, 639, 731, 1161, 1076, 230, 432, 831, 156, 885, 0, 0, 0, 0, 0, 0, 0, 394, 1140, 1196, 697, 0, 613, 1660, 477, 0, 0, 0, 0, 0, 0, 0, 0, 503, 186, 316, 237, 1630, 1244, 137, 1105, 636, 104, 0, 0, 0, 0, 0, 0, 640, 80, 945, 976, 49, 58, 165, 552, 2772, 0, 0, 0, 0, 0, 0, 0, 1817, 505, 43, 406, 47, 1927, 133, 1470, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6325, 5624, 6202, 6141, 6177, 6098, 6237, 6348] avg=6144.0000 max=6348 max_rank=7 min=5624 min_rank=1 imbalance_ratio=1.033203 max_load_reduction=42804 max_load_reduction_pct=0.870850 moved_to_echo=24860 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[35, 1272, 607, 344, 30, 1328, 834, 251, 1409, 176, 108, 0, 0, 0, 0, 0, 656, 372, 1231, 1359, 531, 1095, 65, 851, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1561, 331, 306, 66, 225, 1267, 931, 283, 0, 0, 0, 0, 0, 0, 0, 571, 709, 1053, 910, 269, 484, 1046, 208, 1024, 0, 0, 0, 0, 0, 0, 0, 353, 1175, 1068, 653, 0, 498, 1802, 482, 0, 0, 0, 0, 0, 0, 0, 0, 555, 234, 322, 201, 1672, 1219, 193, 1208, 636, 120, 0, 0, 0, 0, 0, 0, 833, 148, 732, 752, 56, 102, 317, 558, 2527, 0, 0, 0, 0, 0, 0, 0, 1651, 393, 66, 431, 159, 1564, 187, 1469, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6394, 6160, 5988, 6274, 6031, 6360, 6025, 5920] avg=6144.0000 max=6394 max_rank=0 min=5920 min_rank=7 imbalance_ratio=1.040690 max_load_reduction=42758 max_load_reduction_pct=0.869914 moved_to_echo=24336 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[23, 879, 555, 311, 47, 1422, 859, 256, 1492, 154, 99, 0, 0, 0, 0, 0, 572, 418, 1251, 1386, 464, 1002, 43, 1219, 0, 0, 0, 0, 0, 0, 0, 0, 1073, 1611, 421, 313, 92, 165, 1253, 1150, 290, 0, 0, 0, 0, 0, 0, 0, 517, 640, 1115, 1100, 291, 488, 1068, 206, 896, 0, 0, 0, 0, 0, 0, 0, 390, 1165, 1156, 596, 0, 528, 1652, 483, 0, 0, 0, 0, 0, 0, 0, 0, 517, 157, 318, 331, 1539, 1116, 145, 1153, 652, 105, 0, 0, 0, 0, 0, 0, 832, 128, 938, 849, 43, 104, 248, 635, 2304, 0, 0, 0, 0, 0, 0, 0, 1562, 372, 83, 463, 90, 1653, 198, 1506, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6097, 6355, 6368, 6321, 5970, 6033, 6081, 5927] avg=6144.0000 max=6368 max_rank=2 min=5927 min_rank=7 imbalance_ratio=1.036458 max_load_reduction=42784 max_load_reduction_pct=0.870443 moved_to_echo=24011 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[14, 1042, 566, 337, 19, 1195, 745, 279, 1756, 169, 118, 0, 0, 0, 0, 0, 697, 413, 1277, 1285, 510, 952, 17, 805, 0, 0, 0, 0, 0, 0, 0, 0, 1080, 1768, 371, 326, 218, 175, 1237, 890, 312, 0, 0, 0, 0, 0, 0, 0, 684, 686, 1222, 879, 125, 439, 840, 196, 985, 0, 0, 0, 0, 0, 0, 0, 396, 1067, 1169, 620, 0, 620, 1712, 408, 0, 0, 0, 0, 0, 0, 0, 0, 467, 209, 387, 278, 1650, 1295, 132, 865, 700, 116, 0, 0, 0, 0, 0, 0, 748, 98, 673, 809, 44, 112, 227, 515, 2737, 0, 0, 0, 0, 0, 0, 0, 1788, 512, 73, 377, 79, 1842, 180, 1618, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6240, 5956, 6377, 6056, 5992, 6099, 5963, 6469] avg=6144.0000 max=6469 max_rank=7 min=5956 min_rank=1 imbalance_ratio=1.052897 max_load_reduction=42683 max_load_reduction_pct=0.868388 moved_to_echo=24523 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[31, 1052, 521, 318, 34, 1375, 795, 269, 1379, 149, 111, 0, 0, 0, 0, 0, 646, 462, 1096, 1421, 433, 996, 48, 956, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1614, 366, 339, 80, 162, 1270, 955, 298, 0, 0, 0, 0, 0, 0, 0, 581, 730, 1065, 981, 226, 498, 982, 176, 861, 0, 0, 0, 0, 0, 0, 0, 383, 1301, 1217, 557, 0, 480, 1766, 466, 0, 0, 0, 0, 0, 0, 0, 0, 438, 198, 316, 221, 1668, 1284, 159, 1165, 663, 104, 0, 0, 0, 0, 0, 0, 716, 87, 937, 858, 47, 126, 232, 655, 2567, 0, 0, 0, 0, 0, 0, 0, 1652, 526, 56, 503, 97, 1679, 181, 1521, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6034, 6058, 6134, 6100, 6170, 6216, 6225, 6215] avg=6144.0000 max=6225 max_rank=6 min=6034 min_rank=0 imbalance_ratio=1.013184 max_load_reduction=42927 max_load_reduction_pct=0.873352 moved_to_echo=24826 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[46, 1015, 549, 370, 69, 1379, 819, 269, 1338, 156, 89, 0, 0, 0, 0, 0, 565, 254, 1107, 1416, 470, 1103, 202, 1391, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 1320, 270, 362, 83, 261, 1332, 872, 286, 0, 0, 0, 0, 0, 0, 0, 499, 680, 1118, 973, 254, 494, 882, 247, 908, 0, 0, 0, 0, 0, 0, 0, 398, 1161, 1153, 736, 0, 473, 1937, 520, 0, 0, 0, 0, 0, 0, 0, 0, 371, 207, 269, 143, 1855, 1271, 157, 1130, 643, 107, 0, 0, 0, 0, 0, 0, 908, 118, 795, 771, 105, 148, 284, 769, 2075, 0, 0, 0, 0, 0, 0, 0, 1595, 413, 67, 432, 151, 1600, 392, 1486, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6099, 6508, 5850, 6055, 6378, 6153, 5973, 6136] avg=6144.0000 max=6508 max_rank=1 min=5850 min_rank=2 imbalance_ratio=1.059245 max_load_reduction=42644 max_load_reduction_pct=0.867594 moved_to_echo=24640 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3840, 1664, 2560, 1920, 13312, 9856, 1280, 9216, 5248, 896, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4992, 3200, 9472, 11136, 3840, 8064, 640, 8192, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6272, 896, 7040, 6656, 512, 1024, 2048, 5120, 20096, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[384, 8448, 4608, 2688, 384, 10624, 6528, 2176, 12032, 1280, 896, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3200, 9600, 9472, 5248, 0, 4224, 13952, 3968, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4480, 5632, 9088, 7936, 1920, 3840, 7680, 1792, 7424, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[8576, 12800, 2944, 2688, 896, 1536, 10240, 7808, 2432, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[13312, 3584, 640, 3584, 896, 13696, 1920, 12032, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[226, 259, 1695, 106, 607, 744, 590, 914, 2373, 3022, 341, 369, 1694, 384, 1936, 141, 1969, 51, 2469, 421, 708, 329, 0, 633, 971, 1389, 641, 415, 441, 1244, 1056, 469, 319, 524, 117, 320, 852, 384, 862, 388, 108, 3110, 91, 469, 244, 294, 272, 4050, 4, 187, 90, 1079, 303, 1193, 66, 65, 156, 62, 1376, 580, 1316, 913, 360, 391] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[230, 491, 1395, 220, 515, 626, 672, 820, 2358, 2909, 274, 326, 1267, 419, 1943, 193, 2006, 55, 2265, 418, 455, 310, 6, 552, 1219, 1149, 616, 460, 535, 1003, 1304, 855, 308, 578, 214, 325, 927, 494, 1051, 450, 204, 2722, 98, 480, 268, 379, 370, 3831, 3, 224, 164, 979, 328, 1303, 202, 182, 247, 76, 1502, 616, 1017, 1047, 340, 357] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[200, 286, 1587, 110, 588, 608, 699, 850, 2512, 3042, 364, 366, 1343, 423, 2254, 200, 2196, 51, 2248, 430, 654, 404, 1, 645, 934, 1337, 549, 445, 620, 1012, 1218, 597, 382, 522, 119, 249, 796, 466, 996, 456, 155, 2981, 91, 434, 289, 296, 360, 3472, 2, 169, 123, 984, 410, 1084, 138, 134, 197, 71, 1441, 563, 1295, 959, 318, 427] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[322, 660, 1479, 162, 445, 534, 700, 807, 1711, 2694, 269, 409, 1468, 376, 1456, 398, 1720, 59, 2406, 445, 471, 243, 10, 746, 1261, 1060, 651, 429, 531, 974, 1240, 630, 250, 601, 183, 287, 763, 650, 1316, 477, 311, 2658, 78, 498, 312, 513, 372, 4012, 16, 208, 172, 1011, 410, 1369, 187, 213, 161, 148, 1609, 647, 1276, 891, 408, 379] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[292, 452, 1426, 188, 539, 606, 718, 807, 2075, 2834, 340, 396, 1280, 430, 1850, 306, 2203, 79, 2186, 458, 632, 378, 4, 683, 934, 1211, 632, 456, 551, 1075, 1239, 616, 237, 521, 159, 290, 838, 506, 1141, 412, 206, 2960, 100, 507, 299, 315, 369, 3585, 6, 228, 126, 1074, 289, 1036, 140, 131, 232, 115, 1570, 643, 1268, 927, 449, 597] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[209, 390, 1509, 206, 497, 581, 734, 801, 2062, 2862, 258, 412, 1438, 350, 1838, 322, 2096, 63, 2616, 452, 578, 343, 3, 644, 1228, 1223, 646, 404, 652, 1068, 1240, 632, 252, 625, 142, 274, 987, 597, 1074, 449, 154, 2573, 94, 474, 266, 325, 303, 3893, 15, 229, 99, 1088, 370, 1190, 117, 146, 242, 114, 1387, 618, 1110, 851, 412, 325] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[230, 329, 1786, 105, 587, 707, 693, 906, 2020, 2865, 248, 436, 1560, 341, 1914, 420, 2066, 30, 2545, 420, 681, 323, 1, 683, 1051, 1277, 661, 377, 409, 1078, 1341, 511, 504, 727, 166, 331, 874, 442, 806, 521, 203, 2898, 101, 506, 248, 454, 276, 3739, 4, 192, 116, 1025, 231, 1264, 119, 94, 238, 57, 1547, 547, 1130, 725, 159, 307] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[394, 759, 1332, 222, 425, 577, 720, 800, 1693, 2938, 260, 466, 1325, 378, 1574, 407, 1787, 76, 2586, 491, 459, 303, 8, 703, 1222, 886, 597, 476, 545, 918, 1348, 725, 162, 764, 177, 301, 793, 522, 1184, 470, 310, 2465, 111, 467, 313, 542, 348, 3657, 36, 237, 188, 1025, 415, 1307, 219, 280, 277, 143, 1574, 627, 1144, 948, 411, 335] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[322, 660, 1479, 162, 445, 534, 700, 807, 662, 332, 65, 0, 0, 0, 0, 0, 1222, 0, 269, 409, 1468, 376, 1456, 398, 0, 0, 0, 0, 0, 0, 0, 0, 1720, 59, 2057, 445, 471, 243, 10, 746, 0, 0, 0, 0, 0, 0, 0, 0, 1261, 1060, 651, 429, 531, 974, 578, 630, 0, 0, 0, 0, 0, 0, 0, 0, 250, 601, 183, 287, 763, 650, 1316, 477, 2020, 6, 0, 0, 0, 0, 0, 0, 311, 2658, 78, 498, 312, 513, 372, 1660, 0, 0, 0, 0, 0, 0, 0, 0, 16, 208, 172, 1011, 410, 1369, 187, 213, 2688, 0, 0, 0, 0, 0, 0, 0, 161, 148, 1609, 647, 1276, 891, 408, 379, 424, 349, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6168, 5598, 5751, 6114, 6553, 6402, 6274, 6292] avg=6144.0000 max=6553 max_rank=4 min=5598 min_rank=1 imbalance_ratio=1.066569 max_load_reduction=42599 max_load_reduction_pct=0.866679 moved_to_echo=25521 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[226, 259, 1695, 106, 607, 744, 590, 914, 563, 335, 90, 0, 0, 0, 0, 0, 1694, 0, 341, 369, 1694, 384, 1936, 141, 0, 0, 0, 0, 0, 0, 0, 0, 1969, 51, 2111, 421, 708, 329, 0, 633, 0, 0, 0, 0, 0, 0, 0, 0, 971, 1389, 641, 415, 441, 1244, 493, 469, 0, 0, 0, 0, 0, 0, 0, 0, 319, 524, 117, 320, 852, 384, 862, 388, 2039, 7, 0, 0, 0, 0, 0, 0, 108, 3110, 91, 469, 244, 294, 272, 1676, 0, 0, 0, 0, 0, 0, 0, 0, 4, 187, 90, 1079, 303, 1193, 66, 65, 3015, 0, 0, 0, 0, 0, 0, 0, 156, 62, 1376, 580, 1316, 913, 360, 391, 589, 358, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6129, 6559, 6222, 6063, 5812, 6264, 6002, 6101] avg=6144.0000 max=6559 max_rank=1 min=5812 min_rank=4 imbalance_ratio=1.067546 max_load_reduction=42593 max_load_reduction_pct=0.866557 moved_to_echo=24179 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[230, 491, 1395, 220, 515, 626, 672, 820, 696, 317, 89, 0, 0, 0, 0, 0, 1684, 0, 274, 326, 1267, 419, 1943, 193, 0, 0, 0, 0, 0, 0, 0, 0, 2006, 55, 1937, 418, 455, 310, 6, 552, 0, 0, 0, 0, 0, 0, 0, 0, 1219, 1149, 616, 460, 535, 1003, 608, 855, 0, 0, 0, 0, 0, 0, 0, 0, 308, 578, 214, 325, 927, 494, 1051, 450, 1928, 6, 0, 0, 0, 0, 0, 0, 204, 2722, 98, 480, 268, 379, 370, 1586, 0, 0, 0, 0, 0, 0, 0, 0, 3, 224, 164, 979, 328, 1303, 202, 182, 2903, 0, 0, 0, 0, 0, 0, 0, 247, 76, 1502, 616, 1017, 1047, 340, 357, 585, 328, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6071, 6106, 5739, 6445, 6281, 6107, 6288, 6115] avg=6144.0000 max=6445 max_rank=3 min=5739 min_rank=2 imbalance_ratio=1.048991 max_load_reduction=42707 max_load_reduction_pct=0.868876 moved_to_echo=24791 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[200, 286, 1587, 110, 588, 608, 699, 850, 650, 287, 95, 0, 0, 0, 0, 0, 1794, 0, 364, 366, 1343, 423, 2254, 200, 0, 0, 0, 0, 0, 0, 0, 0, 2196, 51, 1922, 430, 654, 404, 1, 645, 0, 0, 0, 0, 0, 0, 0, 0, 934, 1337, 549, 445, 620, 1012, 568, 597, 0, 0, 0, 0, 0, 0, 0, 0, 382, 522, 119, 249, 796, 466, 996, 456, 1748, 6, 0, 0, 0, 0, 0, 0, 155, 2981, 91, 434, 289, 296, 360, 1437, 0, 0, 0, 0, 0, 0, 0, 0, 2, 169, 123, 984, 410, 1084, 138, 134, 3036, 0, 0, 0, 0, 0, 0, 0, 197, 71, 1441, 563, 1295, 959, 318, 427, 623, 326, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5960, 6744, 6303, 6062, 5740, 6043, 6080, 6220] avg=6144.0000 max=6744 max_rank=1 min=5740 min_rank=4 imbalance_ratio=1.097656 max_load_reduction=42408 max_load_reduction_pct=0.862793 moved_to_echo=24083 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[209, 390, 1509, 206, 497, 581, 734, 801, 662, 322, 78, 0, 0, 0, 0, 0, 1472, 0, 258, 412, 1438, 350, 1838, 322, 0, 0, 0, 0, 0, 0, 0, 0, 2096, 63, 2237, 452, 578, 343, 3, 644, 0, 0, 0, 0, 0, 0, 0, 0, 1228, 1223, 646, 404, 652, 1068, 578, 632, 0, 0, 0, 0, 0, 0, 0, 0, 252, 625, 142, 274, 987, 597, 1074, 449, 1960, 7, 0, 0, 0, 0, 0, 0, 154, 2573, 94, 474, 266, 325, 303, 1611, 0, 0, 0, 0, 0, 0, 0, 0, 15, 229, 99, 1088, 370, 1190, 117, 146, 2855, 0, 0, 0, 0, 0, 0, 0, 242, 114, 1387, 618, 1110, 851, 412, 325, 512, 379, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5989, 6090, 6416, 6431, 6367, 5800, 6109, 5950] avg=6144.0000 max=6431 max_rank=3 min=5800 min_rank=5 imbalance_ratio=1.046712 max_load_reduction=42721 max_load_reduction_pct=0.869161 moved_to_echo=24226 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[230, 329, 1786, 105, 587, 707, 693, 906, 716, 309, 77, 0, 0, 0, 0, 0, 1442, 0, 248, 436, 1560, 341, 1914, 420, 0, 0, 0, 0, 0, 0, 0, 0, 2066, 30, 2176, 420, 681, 323, 1, 683, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1277, 661, 377, 409, 1078, 625, 511, 0, 0, 0, 0, 0, 0, 0, 0, 504, 727, 166, 331, 874, 442, 806, 521, 1882, 7, 0, 0, 0, 0, 0, 0, 203, 2898, 101, 506, 248, 454, 276, 1548, 0, 0, 0, 0, 0, 0, 0, 0, 4, 192, 116, 1025, 231, 1264, 119, 94, 2858, 0, 0, 0, 0, 0, 0, 0, 238, 57, 1547, 547, 1130, 725, 159, 307, 501, 369, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6445, 6361, 6380, 5989, 6260, 6234, 5903, 5580] avg=6144.0000 max=6445 max_rank=0 min=5580 min_rank=7 imbalance_ratio=1.048991 max_load_reduction=42707 max_load_reduction_pct=0.868876 moved_to_echo=23977 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[292, 452, 1426, 188, 539, 606, 718, 807, 664, 301, 83, 0, 0, 0, 0, 0, 1473, 0, 340, 396, 1280, 430, 1850, 306, 0, 0, 0, 0, 0, 0, 0, 0, 2203, 79, 1866, 458, 632, 378, 4, 683, 0, 0, 0, 0, 0, 0, 0, 0, 934, 1211, 632, 456, 551, 1075, 575, 616, 0, 0, 0, 0, 0, 0, 0, 0, 237, 521, 159, 290, 838, 506, 1141, 412, 1808, 7, 0, 0, 0, 0, 0, 0, 206, 2960, 100, 507, 299, 315, 369, 1476, 0, 0, 0, 0, 0, 0, 0, 0, 6, 228, 126, 1074, 289, 1036, 140, 131, 2827, 0, 0, 0, 0, 0, 0, 0, 232, 115, 1570, 643, 1268, 927, 449, 597, 519, 320, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6076, 6075, 6303, 6050, 5919, 6232, 5857, 6640] avg=6144.0000 max=6640 max_rank=7 min=5857 min_rank=6 imbalance_ratio=1.080729 max_load_reduction=42512 max_load_reduction_pct=0.864909 moved_to_echo=24648 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[394, 759, 1332, 222, 425, 577, 720, 800, 719, 302, 64, 0, 0, 0, 0, 0, 1209, 0, 260, 466, 1325, 378, 1574, 407, 0, 0, 0, 0, 0, 0, 0, 0, 1787, 76, 2211, 491, 459, 303, 8, 703, 0, 0, 0, 0, 0, 0, 0, 0, 1222, 886, 597, 476, 545, 918, 629, 725, 0, 0, 0, 0, 0, 0, 0, 0, 162, 764, 177, 301, 793, 522, 1184, 470, 1841, 6, 0, 0, 0, 0, 0, 0, 310, 2465, 111, 467, 313, 542, 348, 1514, 0, 0, 0, 0, 0, 0, 0, 0, 36, 237, 188, 1025, 415, 1307, 219, 280, 2932, 0, 0, 0, 0, 0, 0, 0, 277, 143, 1574, 627, 1144, 948, 411, 335, 420, 375, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6314, 5619, 6038, 5998, 6220, 6070, 6639, 6254] avg=6144.0000 max=6639 max_rank=6 min=5619 min_rank=1 imbalance_ratio=1.080566 max_load_reduction=42513 max_load_reduction_pct=0.864929 moved_to_echo=25183 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[1664, 22400, 768, 3840, 2304, 3200, 2688, 12544, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[16128, 512, 16640, 3584, 4736, 2688, 128, 5376, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[8832, 9600, 5120, 3584, 4352, 8448, 4736, 5120, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[2432, 4864, 1280, 2432, 6912, 4096, 8448, 3712, 15232, 128, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[2176, 3712, 12288, 1408, 4224, 4992, 5632, 6784, 5376, 2560, 768, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1792, 896, 12032, 4864, 9600, 7296, 2944, 3200, 4224, 2816, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[12032, 0, 2432, 3200, 11392, 3200, 14848, 2432, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 1792, 1152, 8320, 2816, 9856, 1280, 1280, 23168, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 1798, 1547, 1000, 1083, 1184, 685, 1380, 225, 1268, 2490, 73, 151, 467, 336, 678, 1737, 78, 100, 64, 93, 834, 289, 176, 945, 323, 2302, 402, 2297, 727, 624, 52, 222, 0, 11, 930, 834, 77, 95, 1028, 315, 446, 46, 915, 636, 0, 2649, 2, 102, 976, 250, 1001, 391, 1633, 1380, 1135, 1231, 446, 170, 1932, 1153, 1755, 1001, 982] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 1588, 1491, 1048, 1127, 1574, 695, 1709, 236, 1077, 2425, 65, 147, 597, 262, 436, 1725, 6, 86, 57, 102, 968, 367, 104, 828, 17, 2739, 257, 2861, 535, 418, 60, 187, 0, 2, 1254, 603, 31, 60, 1040, 265, 704, 20, 935, 420, 0, 3008, 0, 20, 1077, 251, 715, 433, 1879, 1698, 1018, 855, 508, 128, 1674, 1316, 1617, 754, 1073] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 1671, 1343, 1074, 1156, 1251, 683, 1575, 185, 1395, 2805, 76, 156, 497, 284, 510, 1788, 6, 69, 99, 90, 866, 337, 103, 875, 5, 2813, 265, 2672, 716, 418, 44, 161, 0, 0, 1337, 621, 27, 66, 1082, 292, 672, 18, 942, 377, 0, 2861, 0, 17, 915, 236, 688, 362, 1723, 1634, 1047, 893, 484, 128, 2193, 1208, 1813, 714, 814] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 1593, 1407, 1055, 1048, 1310, 672, 1700, 256, 1325, 2436, 75, 92, 508, 266, 385, 1963, 4, 156, 70, 98, 979, 293, 130, 988, 192, 2425, 294, 2622, 660, 329, 67, 221, 2, 2, 1356, 690, 17, 50, 1180, 270, 747, 59, 969, 432, 1, 2994, 1, 32, 964, 287, 761, 435, 2018, 1765, 968, 1031, 332, 163, 1601, 1101, 1634, 677, 994] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 1350, 1326, 1215, 1034, 1342, 618, 1705, 147, 1362, 2430, 71, 117, 468, 256, 304, 1746, 8, 16, 15, 81, 1113, 311, 95, 797, 0, 3207, 180, 3334, 457, 308, 22, 178, 0, 0, 1357, 510, 4, 39, 978, 288, 674, 4, 991, 340, 0, 3322, 0, 3, 1062, 192, 680, 432, 2379, 2085, 1038, 727, 416, 50, 1929, 1048, 1600, 490, 901] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 1245, 1407, 1442, 1067, 1259, 641, 2050, 169, 1777, 2310, 59, 138, 404, 245, 278, 2003, 10, 8, 22, 100, 1425, 272, 94, 1104, 0, 2274, 190, 2864, 437, 404, 51, 203, 0, 0, 1501, 559, 11, 60, 934, 195, 676, 3, 922, 380, 0, 3162, 0, 5, 1068, 177, 660, 386, 2365, 2099, 1060, 714, 396, 46, 1858, 944, 1581, 513, 925] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 1327, 1332, 1342, 1103, 1142, 621, 1629, 137, 1567, 2836, 61, 130, 452, 244, 200, 2028, 2, 0, 17, 70, 1430, 347, 134, 1026, 0, 2933, 134, 3494, 346, 336, 0, 159, 0, 0, 1587, 468, 2, 36, 863, 209, 677, 7, 910, 297, 0, 3225, 0, 4, 1204, 151, 532, 407, 2512, 2135, 876, 744, 362, 32, 1734, 1012, 1155, 506, 926] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 1815, 1484, 1161, 1133, 1142, 681, 1411, 277, 1461, 2618, 116, 171, 479, 324, 564, 1664, 131, 132, 111, 120, 729, 326, 119, 944, 199, 1904, 310, 2251, 772, 513, 54, 242, 0, 8, 1013, 828, 50, 71, 1091, 278, 531, 96, 895, 475, 0, 2600, 0, 106, 934, 263, 1236, 389, 1743, 1234, 1144, 1253, 418, 178, 1800, 1171, 2067, 833, 1089] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 462, 1326, 1215, 1034, 1342, 618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 1362, 2430, 71, 117, 468, 256, 304, 522, 0, 0, 0, 0, 0, 0, 0, 1746, 8, 16, 15, 81, 1113, 311, 95, 1855, 817, 338, 0, 0, 0, 0, 0, 797, 0, 3207, 180, 1479, 457, 308, 22, 0, 0, 0, 0, 0, 0, 0, 0, 178, 0, 0, 1357, 510, 4, 39, 978, 1705, 1317, 0, 0, 0, 0, 0, 0, 288, 674, 4, 991, 340, 0, 3322, 0, 532, 366, 0, 0, 0, 0, 0, 0, 3, 1062, 192, 680, 432, 1030, 2085, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 727, 416, 50, 274, 1048, 1600, 490, 901, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5997, 5677, 6395, 6450, 6088, 6517, 6522, 5506] avg=6144.0000 max=6522 max_rank=6 min=5506 min_rank=7 imbalance_ratio=1.061523 max_load_reduction=42630 max_load_reduction_pct=0.867310 moved_to_echo=24633 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 614, 1547, 1000, 1083, 1184, 685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 225, 1268, 2490, 73, 151, 467, 336, 678, 696, 0, 0, 0, 0, 0, 0, 0, 1737, 78, 100, 64, 93, 834, 289, 176, 1278, 561, 339, 0, 0, 0, 0, 0, 945, 323, 2302, 402, 1019, 727, 624, 52, 0, 0, 0, 0, 0, 0, 0, 0, 222, 0, 11, 930, 834, 77, 95, 1028, 1380, 1319, 0, 0, 0, 0, 0, 0, 315, 446, 46, 915, 636, 0, 2649, 2, 365, 488, 0, 0, 0, 0, 0, 0, 102, 976, 250, 1001, 391, 707, 1380, 1135, 0, 0, 0, 0, 0, 0, 0, 0, 1231, 446, 170, 274, 1153, 1755, 1001, 982, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6113, 6384, 5549, 6394, 5896, 5862, 5942, 7012] avg=6144.0000 max=7012 max_rank=7 min=5549 min_rank=2 imbalance_ratio=1.141276 max_load_reduction=42140 max_load_reduction_pct=0.857340 moved_to_echo=24712 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 545, 1407, 1055, 1048, 1310, 672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 1325, 2436, 75, 92, 508, 266, 385, 616, 0, 0, 0, 0, 0, 0, 0, 1963, 4, 156, 70, 98, 979, 293, 130, 1459, 693, 281, 0, 0, 0, 0, 0, 988, 192, 2425, 294, 1163, 660, 329, 67, 0, 0, 0, 0, 0, 0, 0, 0, 221, 2, 2, 1356, 690, 17, 50, 1180, 1700, 1093, 0, 0, 0, 0, 0, 0, 270, 747, 59, 969, 432, 1, 2994, 1, 451, 432, 0, 0, 0, 0, 0, 0, 32, 964, 287, 761, 435, 874, 1765, 968, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 332, 163, 227, 1101, 1634, 677, 994, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6037, 5959, 6126, 6118, 6311, 6356, 6086, 6159] avg=6144.0000 max=6356 max_rank=5 min=5959 min_rank=1 imbalance_ratio=1.034505 max_load_reduction=42796 max_load_reduction_pct=0.870687 moved_to_echo=24912 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 572, 1343, 1074, 1156, 1251, 683, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, 1395, 2805, 76, 156, 497, 284, 510, 646, 0, 0, 0, 0, 0, 0, 0, 1788, 6, 69, 99, 90, 866, 337, 103, 1487, 592, 385, 0, 0, 0, 0, 0, 875, 5, 2813, 265, 1185, 716, 418, 44, 0, 0, 0, 0, 0, 0, 0, 0, 161, 0, 0, 1337, 621, 27, 66, 1082, 1575, 1497, 0, 0, 0, 0, 0, 0, 292, 672, 18, 942, 377, 0, 2861, 0, 385, 453, 0, 0, 0, 0, 0, 0, 17, 915, 236, 688, 362, 746, 1634, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 893, 484, 128, 311, 1208, 1813, 714, 814, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6079, 6554, 5822, 6321, 6366, 6000, 5645, 6365] avg=6144.0000 max=6554 max_rank=1 min=5645 min_rank=6 imbalance_ratio=1.066732 max_load_reduction=42598 max_load_reduction_pct=0.866659 moved_to_echo=24376 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 454, 1332, 1342, 1103, 1142, 621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 1567, 2836, 61, 130, 452, 244, 200, 513, 0, 0, 0, 0, 0, 0, 0, 2028, 2, 0, 17, 70, 1430, 347, 134, 1944, 863, 304, 0, 0, 0, 0, 0, 1026, 0, 2933, 134, 1550, 346, 336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 159, 0, 0, 1587, 468, 2, 36, 863, 1629, 1184, 0, 0, 0, 0, 0, 0, 209, 677, 7, 910, 297, 0, 3225, 0, 561, 360, 0, 0, 0, 0, 0, 0, 4, 1204, 151, 532, 407, 1088, 2135, 876, 0, 0, 0, 0, 0, 0, 0, 0, 744, 362, 32, 246, 1012, 1155, 506, 926, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5994, 6140, 7139, 6325, 5928, 6246, 6397, 4983] avg=6144.0000 max=7139 max_rank=2 min=4983 min_rank=7 imbalance_ratio=1.161947 max_load_reduction=42013 max_load_reduction_pct=0.854757 moved_to_echo=23554 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 534, 1491, 1048, 1127, 1574, 695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 236, 1077, 2425, 65, 147, 597, 262, 436, 619, 0, 0, 0, 0, 0, 0, 0, 1725, 6, 86, 57, 102, 968, 367, 104, 1596, 649, 297, 0, 0, 0, 0, 0, 828, 17, 2739, 257, 1265, 535, 418, 60, 0, 0, 0, 0, 0, 0, 0, 0, 187, 0, 2, 1254, 603, 31, 60, 1040, 1709, 1147, 0, 0, 0, 0, 0, 0, 265, 704, 20, 935, 420, 0, 3008, 0, 424, 435, 0, 0, 0, 0, 0, 0, 20, 1077, 251, 715, 433, 806, 1698, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 855, 508, 128, 230, 1316, 1617, 754, 1073, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6469, 5864, 5957, 6119, 6033, 6211, 6018, 6481] avg=6144.0000 max=6481 max_rank=7 min=5864 min_rank=1 imbalance_ratio=1.054850 max_load_reduction=42671 max_load_reduction_pct=0.868144 moved_to_echo=24743 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 425, 1407, 1442, 1067, 1259, 641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 1777, 2310, 59, 138, 404, 245, 278, 482, 0, 0, 0, 0, 0, 0, 0, 2003, 10, 8, 22, 100, 1425, 272, 94, 1594, 812, 326, 0, 0, 0, 0, 0, 1104, 0, 2274, 190, 1270, 437, 404, 51, 0, 0, 0, 0, 0, 0, 0, 0, 203, 0, 0, 1501, 559, 11, 60, 934, 2050, 1269, 0, 0, 0, 0, 0, 0, 195, 676, 3, 922, 380, 0, 3162, 0, 528, 338, 0, 0, 0, 0, 0, 0, 5, 1068, 177, 660, 386, 1025, 2099, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 714, 396, 46, 263, 944, 1581, 513, 925, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6241, 5862, 6666, 5730, 6587, 6204, 6480, 5382] avg=6144.0000 max=6666 max_rank=2 min=5382 min_rank=7 imbalance_ratio=1.084961 max_load_reduction=42486 max_load_reduction_pct=0.864380 moved_to_echo=24653 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 621, 1484, 1161, 1133, 1142, 681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 277, 1461, 2618, 116, 171, 479, 324, 564, 702, 0, 0, 0, 0, 0, 0, 0, 1664, 131, 132, 111, 120, 729, 326, 119, 1252, 598, 316, 0, 0, 0, 0, 0, 944, 199, 1904, 310, 999, 772, 513, 54, 0, 0, 0, 0, 0, 0, 0, 0, 242, 0, 8, 1013, 828, 50, 71, 1091, 1411, 1229, 0, 0, 0, 0, 0, 0, 278, 531, 96, 895, 475, 0, 2600, 0, 389, 492, 0, 0, 0, 0, 0, 0, 106, 934, 263, 1236, 389, 756, 1234, 1144, 0, 0, 0, 0, 0, 0, 0, 0, 1253, 418, 178, 255, 1171, 2067, 833, 1089, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6222, 6712, 5498, 5695, 5943, 5756, 6062, 7264] avg=6144.0000 max=7264 max_rank=7 min=5498 min_rank=2 imbalance_ratio=1.182292 max_load_reduction=41888 max_load_reduction_pct=0.852214 moved_to_echo=25025 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 4352, 11392, 9344, 8832, 10240, 5376, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7552, 3456, 896, 2176, 8960, 13312, 5504, 7808, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[7552, 768, 20608, 2048, 9984, 4736, 3456, 384, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[384, 8320, 1920, 6400, 3328, 7040, 14080, 8320, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1664, 11264, 20352, 640, 1152, 3968, 2304, 3456, 4864, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[14720, 256, 640, 512, 768, 8448, 2560, 1024, 12544, 5632, 2688, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[2176, 5248, 256, 7552, 3456, 128, 23936, 128, 3712, 3456, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1664, 128, 128, 10368, 5120, 256, 512, 8320, 13184, 10112, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[499, 1949, 841, 602, 77, 232, 2490, 571, 0, 0, 173, 587, 2348, 0, 1, 668, 525, 847, 4658, 19, 30, 777, 533, 0, 88, 3252, 763, 94, 156, 191, 3438, 128, 3254, 464, 77, 293, 400, 12, 1407, 521, 36, 1, 8, 1411, 69, 416, 2026, 799, 862, 1274, 1354, 1045, 20, 516, 1, 1151, 1153, 369, 500, 506, 134, 288, 608, 1640] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[240, 1969, 522, 669, 71, 398, 2134, 628, 0, 0, 157, 607, 3042, 0, 0, 349, 391, 794, 5049, 15, 30, 1056, 555, 0, 0, 3711, 759, 6, 46, 84, 3846, 42, 3264, 399, 80, 115, 403, 0, 854, 473, 7, 0, 0, 1103, 55, 457, 2735, 827, 828, 1492, 1492, 1565, 7, 390, 0, 1004, 1245, 264, 218, 326, 108, 211, 536, 1524] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[447, 1810, 753, 474, 78, 361, 2630, 397, 1, 0, 260, 670, 2272, 0, 0, 578, 447, 1015, 4470, 23, 57, 554, 402, 0, 182, 2970, 773, 93, 149, 204, 3787, 161, 2878, 487, 73, 271, 488, 180, 1297, 723, 36, 0, 14, 1724, 61, 415, 2292, 595, 869, 1624, 1094, 895, 32, 541, 0, 1022, 1171, 416, 494, 667, 168, 370, 647, 1590] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[359, 2055, 609, 578, 59, 414, 2234, 718, 0, 0, 148, 566, 2594, 0, 0, 512, 417, 787, 4948, 11, 23, 980, 580, 0, 1, 3443, 829, 41, 62, 134, 3760, 92, 3056, 385, 70, 198, 386, 0, 1084, 522, 19, 0, 1, 1295, 57, 452, 2270, 955, 804, 1334, 1679, 1333, 8, 479, 0, 1018, 1251, 271, 199, 418, 99, 221, 535, 1799] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[374, 2124, 583, 480, 62, 400, 2428, 496, 0, 0, 168, 664, 2499, 0, 0, 472, 378, 859, 4834, 24, 32, 854, 453, 0, 62, 3691, 756, 118, 66, 204, 3889, 116, 2916, 422, 77, 215, 375, 0, 1122, 442, 20, 0, 4, 1441, 80, 471, 2334, 805, 745, 1675, 1702, 935, 15, 458, 0, 1051, 1174, 275, 474, 457, 108, 340, 614, 1319] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[317, 2261, 536, 326, 88, 412, 2055, 677, 0, 0, 165, 640, 2900, 0, 0, 314, 399, 940, 5311, 9, 34, 1184, 463, 0, 56, 2858, 692, 31, 75, 47, 3899, 85, 3610, 421, 113, 218, 315, 0, 760, 502, 23, 0, 0, 1201, 52, 435, 2488, 900, 777, 1772, 1442, 1207, 21, 444, 0, 1044, 1249, 297, 443, 385, 132, 247, 511, 1369] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[484, 1747, 853, 561, 83, 198, 2622, 579, 1, 0, 247, 674, 2075, 0, 0, 672, 480, 909, 4781, 18, 69, 592, 400, 0, 222, 2744, 765, 118, 130, 154, 3944, 125, 2980, 542, 79, 333, 500, 109, 1366, 558, 29, 0, 8, 1490, 85, 315, 2197, 622, 839, 1522, 1087, 923, 78, 500, 0, 1138, 1069, 440, 460, 665, 186, 431, 716, 1638] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[367, 1697, 619, 488, 75, 425, 2556, 681, 0, 0, 279, 658, 2337, 0, 0, 552, 468, 915, 4550, 19, 32, 860, 496, 0, 45, 3519, 897, 121, 71, 196, 3696, 129, 2806, 495, 90, 218, 485, 0, 1069, 509, 41, 0, 7, 1569, 55, 426, 2315, 808, 817, 1348, 1702, 1068, 9, 494, 0, 1071, 1195, 271, 482, 430, 132, 283, 597, 1612] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[499, 1949, 841, 602, 77, 232, 1662, 571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 173, 587, 2348, 0, 1, 668, 2005, 78, 0, 0, 0, 0, 0, 0, 525, 847, 3307, 19, 30, 777, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 3252, 763, 94, 156, 191, 1433, 128, 0, 0, 0, 0, 0, 0, 0, 0, 3254, 464, 77, 293, 400, 12, 1407, 521, 147, 0, 0, 0, 0, 0, 0, 0, 36, 1, 8, 1411, 69, 416, 2026, 799, 828, 163, 113, 0, 0, 0, 0, 0, 862, 1014, 1354, 1045, 20, 516, 1, 1151, 0, 0, 0, 0, 0, 0, 0, 0, 1153, 369, 500, 506, 134, 288, 608, 1640, 1110, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6433, 5860, 6038, 6105, 6575, 5870, 5963, 6308] avg=6144.0000 max=6575 max_rank=4 min=5860 min_rank=1 imbalance_ratio=1.070150 max_load_reduction=42577 max_load_reduction_pct=0.866231 moved_to_echo=24716 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[359, 2055, 609, 578, 59, 414, 1491, 718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 566, 2594, 0, 0, 512, 2193, 83, 0, 0, 0, 0, 0, 0, 417, 787, 3513, 11, 23, 980, 580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3443, 829, 41, 62, 134, 1567, 92, 0, 0, 0, 0, 0, 0, 0, 0, 3056, 385, 70, 198, 386, 0, 1084, 522, 154, 0, 0, 0, 0, 0, 0, 0, 19, 0, 1, 1295, 57, 452, 2270, 955, 743, 173, 118, 0, 0, 0, 0, 0, 804, 1062, 1679, 1333, 8, 479, 0, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1251, 271, 199, 418, 99, 221, 535, 1799, 1179, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6283, 6096, 6311, 6169, 5855, 6083, 6383, 5972] avg=6144.0000 max=6383 max_rank=6 min=5855 min_rank=4 imbalance_ratio=1.038900 max_load_reduction=42769 max_load_reduction_pct=0.870138 moved_to_echo=24293 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[447, 1810, 753, 474, 78, 361, 1756, 397, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 260, 670, 2272, 0, 0, 578, 2209, 75, 0, 0, 0, 0, 0, 0, 447, 1015, 3174, 23, 57, 554, 402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, 2970, 773, 93, 149, 204, 1578, 161, 0, 0, 0, 0, 0, 0, 0, 0, 2878, 487, 73, 271, 488, 180, 1297, 723, 187, 0, 0, 0, 0, 0, 0, 0, 36, 0, 14, 1724, 61, 415, 2292, 595, 874, 156, 144, 0, 0, 0, 0, 0, 869, 1293, 1094, 895, 32, 541, 0, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1171, 416, 494, 667, 168, 370, 647, 1590, 1065, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6076, 6065, 5672, 6110, 6584, 6311, 5746, 6588] avg=6144.0000 max=6588 max_rank=7 min=5672 min_rank=2 imbalance_ratio=1.072266 max_load_reduction=42564 max_load_reduction_pct=0.865967 moved_to_echo=25229 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[240, 1969, 522, 669, 71, 398, 1425, 628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 607, 3042, 0, 0, 349, 2243, 84, 0, 0, 0, 0, 0, 0, 391, 794, 3585, 15, 30, 1056, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3711, 759, 6, 46, 84, 1603, 42, 0, 0, 0, 0, 0, 0, 0, 0, 3264, 399, 80, 115, 403, 0, 854, 473, 172, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 1103, 55, 457, 2735, 827, 709, 176, 133, 0, 0, 0, 0, 0, 828, 1187, 1492, 1565, 7, 390, 0, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1245, 264, 218, 326, 108, 211, 536, 1524, 1204, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5922, 6482, 6426, 6251, 5760, 6202, 6473, 5636] avg=6144.0000 max=6482 max_rank=1 min=5636 min_rank=7 imbalance_ratio=1.055013 max_load_reduction=42670 max_load_reduction_pct=0.868123 moved_to_echo=24071 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[317, 2261, 536, 326, 88, 412, 1372, 677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 165, 640, 2900, 0, 0, 314, 2274, 89, 0, 0, 0, 0, 0, 0, 399, 940, 3771, 9, 34, 1184, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 2858, 692, 31, 75, 47, 1625, 85, 0, 0, 0, 0, 0, 0, 0, 0, 3610, 421, 113, 218, 315, 0, 760, 502, 204, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 1201, 52, 435, 2488, 900, 683, 185, 158, 0, 0, 0, 0, 0, 777, 1410, 1442, 1207, 21, 444, 0, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1249, 297, 443, 385, 132, 247, 511, 1369, 1266, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5989, 6382, 6800, 5469, 6143, 6125, 6345, 5899] avg=6144.0000 max=6800 max_rank=2 min=5469 min_rank=3 imbalance_ratio=1.106771 max_load_reduction=42352 max_load_reduction_pct=0.861654 moved_to_echo=24512 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[367, 1697, 619, 488, 75, 425, 1702, 681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 279, 658, 2337, 0, 0, 552, 2161, 78, 0, 0, 0, 0, 0, 0, 468, 915, 3221, 19, 32, 860, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 3519, 897, 121, 71, 196, 1535, 129, 0, 0, 0, 0, 0, 0, 0, 0, 2806, 495, 90, 218, 485, 0, 1069, 509, 159, 0, 0, 0, 0, 0, 0, 0, 41, 0, 7, 1569, 55, 426, 2315, 808, 854, 162, 124, 0, 0, 0, 0, 0, 817, 1065, 1702, 1068, 9, 494, 0, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 1195, 271, 482, 430, 132, 283, 597, 1612, 1089, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6054, 6065, 6011, 6513, 5831, 6361, 6226, 6091] avg=6144.0000 max=6513 max_rank=3 min=5831 min_rank=4 imbalance_ratio=1.060059 max_load_reduction=42639 max_load_reduction_pct=0.867493 moved_to_echo=24509 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[374, 2124, 583, 480, 62, 400, 1621, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 664, 2499, 0, 0, 472, 2268, 81, 0, 0, 0, 0, 0, 0, 378, 859, 3432, 24, 32, 854, 453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 3691, 756, 118, 66, 204, 1621, 116, 0, 0, 0, 0, 0, 0, 0, 0, 2916, 422, 77, 215, 375, 0, 1122, 442, 193, 0, 0, 0, 0, 0, 0, 0, 20, 0, 4, 1441, 80, 471, 2334, 805, 807, 169, 149, 0, 0, 0, 0, 0, 745, 1333, 1702, 935, 15, 458, 0, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1174, 275, 474, 457, 108, 340, 614, 1319, 1152, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6140, 6152, 6032, 6634, 5762, 6280, 6239, 5913] avg=6144.0000 max=6634 max_rank=3 min=5762 min_rank=4 imbalance_ratio=1.079753 max_load_reduction=42518 max_load_reduction_pct=0.865031 moved_to_echo=24194 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[484, 1747, 853, 561, 83, 198, 1750, 579, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 247, 674, 2075, 0, 0, 672, 2301, 80, 0, 0, 0, 0, 0, 0, 480, 909, 3394, 18, 69, 592, 400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 222, 2744, 765, 118, 130, 154, 1643, 125, 0, 0, 0, 0, 0, 0, 0, 0, 2980, 542, 79, 333, 500, 109, 1366, 558, 175, 0, 0, 0, 0, 0, 0, 0, 29, 0, 8, 1490, 85, 315, 2197, 622, 872, 167, 135, 0, 0, 0, 0, 0, 839, 1212, 1087, 923, 78, 500, 0, 1138, 0, 0, 0, 0, 0, 0, 0, 0, 1069, 440, 460, 665, 186, 431, 716, 1638, 1140, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6255, 6050, 5862, 5901, 6642, 5920, 5777, 6745] avg=6144.0000 max=6745 max_rank=7 min=5777 min_rank=6 imbalance_ratio=1.097819 max_load_reduction=42407 max_load_reduction_pct=0.862773 moved_to_echo=25084 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3200, 15616, 5376, 4224, 640, 2944, 12800, 4864, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[256, 128, 128, 11264, 640, 3456, 18688, 6400, 6400, 1408, 1152, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[24832, 3712, 768, 1920, 3456, 384, 8960, 4352, 1408, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6656, 9600, 11648, 9088, 256, 3840, 128, 8576, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 0, 1664, 5120, 20096, 0, 128, 4224, 17664, 768, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[9600, 2688, 3328, 3968, 1152, 2432, 4864, 12544, 9216, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[768, 26240, 6272, 640, 768, 1280, 12672, 896, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3584, 7168, 27520, 256, 384, 6912, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[718, 77, 0, 5326, 3, 0, 188, 79, 0, 329, 128, 226, 823, 722, 396, 814, 0, 0, 227, 0, 0, 464, 275, 1255, 120, 71, 420, 0, 54, 64, 0, 0, 79, 2448, 248, 0, 0, 0, 0, 2736, 872, 275, 184, 1399, 5731, 65, 572, 4831, 391, 65, 0, 2028, 52, 2908, 6705, 0, 4336, 0, 0, 0, 0, 0, 386, 62] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[704, 80, 0, 5097, 25, 0, 201, 63, 0, 304, 233, 330, 847, 644, 367, 693, 0, 0, 344, 0, 0, 477, 359, 993, 90, 59, 436, 0, 128, 77, 0, 0, 66, 2602, 173, 0, 0, 0, 0, 2940, 926, 276, 192, 1375, 5673, 78, 653, 4818, 365, 77, 0, 1772, 112, 2954, 6581, 0, 4391, 0, 0, 0, 0, 0, 500, 77] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[733, 66, 0, 4684, 107, 0, 191, 50, 0, 397, 310, 383, 946, 571, 414, 905, 0, 0, 369, 0, 2, 456, 469, 1309, 93, 66, 532, 0, 159, 166, 0, 1, 49, 2546, 234, 0, 0, 1, 0, 2257, 959, 262, 192, 1739, 5447, 124, 871, 4486, 406, 133, 0, 1741, 359, 2223, 6462, 1, 4635, 0, 0, 1, 1, 0, 537, 107] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[706, 102, 0, 4881, 84, 0, 204, 88, 0, 338, 249, 160, 898, 756, 443, 1064, 0, 0, 368, 0, 1, 382, 412, 2003, 104, 50, 564, 0, 314, 471, 0, 0, 88, 2644, 347, 0, 0, 0, 0, 1832, 896, 256, 201, 1695, 5475, 89, 889, 4080, 422, 88, 0, 1718, 259, 1828, 6398, 0, 4763, 0, 0, 1, 0, 0, 420, 121] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[690, 91, 0, 4952, 63, 0, 202, 82, 0, 388, 239, 203, 915, 681, 443, 994, 0, 0, 443, 0, 0, 420, 399, 1353, 120, 66, 497, 0, 195, 399, 0, 0, 82, 2578, 204, 0, 0, 0, 0, 2852, 936, 273, 199, 1417, 5589, 77, 822, 4339, 431, 80, 0, 1531, 107, 2571, 6486, 0, 4205, 0, 0, 0, 0, 0, 461, 77] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[707, 85, 0, 5241, 23, 0, 180, 82, 0, 406, 174, 273, 832, 731, 412, 722, 0, 0, 337, 1, 0, 423, 282, 1411, 112, 88, 411, 0, 84, 73, 0, 0, 86, 2559, 268, 0, 0, 0, 0, 2304, 916, 273, 175, 1455, 5665, 74, 827, 4567, 416, 73, 0, 2025, 82, 2874, 6531, 0, 4327, 0, 0, 0, 0, 0, 495, 70] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[717, 97, 0, 5098, 62, 0, 207, 95, 0, 383, 251, 194, 847, 772, 427, 900, 0, 0, 534, 0, 0, 462, 375, 1406, 121, 53, 443, 0, 213, 153, 0, 0, 95, 2534, 235, 0, 0, 0, 0, 2308, 940, 268, 208, 1587, 5624, 75, 757, 4781, 410, 72, 0, 2206, 86, 2068, 6458, 0, 4133, 0, 0, 1, 0, 0, 420, 76] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[688, 103, 0, 4848, 88, 0, 158, 94, 0, 330, 349, 191, 878, 686, 409, 923, 0, 0, 331, 0, 1, 393, 396, 1772, 122, 54, 562, 0, 334, 358, 0, 0, 94, 2567, 341, 0, 0, 0, 0, 1976, 895, 215, 148, 1839, 5420, 92, 1068, 4350, 397, 105, 0, 1930, 349, 1573, 6421, 0, 4733, 0, 0, 0, 0, 0, 458, 113] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[704, 80, 0, 5097, 25, 0, 201, 63, 5, 0, 0, 0, 0, 0, 0, 0, 0, 304, 233, 330, 847, 644, 367, 693, 2263, 213, 0, 0, 0, 0, 0, 0, 0, 0, 344, 0, 0, 477, 359, 993, 3070, 504, 0, 0, 0, 0, 0, 0, 90, 59, 436, 0, 128, 77, 0, 0, 5169, 0, 0, 0, 0, 0, 0, 0, 66, 2602, 173, 0, 0, 0, 0, 2940, 900, 0, 0, 0, 0, 0, 0, 0, 926, 276, 192, 1375, 0, 78, 653, 2472, 0, 0, 0, 0, 0, 0, 0, 0, 365, 77, 0, 1772, 112, 2954, 1248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4391, 0, 0, 0, 0, 0, 500, 77, 1228, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6175, 5894, 5747, 5959, 6681, 5972, 6528, 6196] avg=6144.0000 max=6681 max_rank=4 min=5747 min_rank=2 imbalance_ratio=1.087402 max_load_reduction=42471 max_load_reduction_pct=0.864075 moved_to_echo=25377 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[718, 77, 0, 5326, 3, 0, 188, 79, 5, 0, 0, 0, 0, 0, 0, 0, 0, 329, 128, 226, 823, 722, 396, 814, 2306, 214, 0, 0, 0, 0, 0, 0, 0, 0, 227, 0, 0, 464, 275, 1255, 3128, 510, 0, 0, 0, 0, 0, 0, 120, 71, 420, 0, 54, 64, 0, 0, 5221, 0, 0, 0, 0, 0, 0, 0, 79, 2448, 248, 0, 0, 0, 0, 2736, 903, 0, 0, 0, 0, 0, 0, 0, 872, 275, 184, 1399, 0, 65, 572, 2477, 0, 0, 0, 0, 0, 0, 0, 0, 391, 65, 0, 2028, 52, 2908, 1271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4336, 0, 0, 0, 0, 0, 386, 62, 1232, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6396, 5958, 5859, 5950, 6414, 5844, 6715, 6016] avg=6144.0000 max=6715 max_rank=6 min=5844 min_rank=5 imbalance_ratio=1.092936 max_load_reduction=42437 max_load_reduction_pct=0.863383 moved_to_echo=24989 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[706, 102, 0, 4881, 84, 0, 204, 88, 4, 0, 0, 0, 0, 0, 0, 0, 0, 338, 249, 160, 898, 756, 443, 1064, 2200, 181, 0, 0, 0, 0, 0, 0, 0, 0, 368, 0, 1, 382, 412, 2003, 2984, 486, 0, 0, 0, 0, 0, 0, 104, 50, 564, 0, 314, 471, 0, 0, 4989, 0, 0, 0, 0, 0, 0, 0, 88, 2644, 347, 0, 0, 0, 0, 1832, 762, 0, 0, 0, 0, 0, 0, 0, 896, 256, 201, 1695, 0, 89, 889, 2093, 0, 0, 0, 0, 0, 0, 0, 0, 422, 88, 0, 1718, 259, 1828, 1214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4763, 0, 0, 1, 0, 0, 420, 121, 1040, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6069, 6289, 6636, 6492, 5673, 6119, 5529, 6345] avg=6144.0000 max=6636 max_rank=2 min=5529 min_rank=6 imbalance_ratio=1.080078 max_load_reduction=42516 max_load_reduction_pct=0.864990 moved_to_echo=23666 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[733, 66, 0, 4684, 107, 0, 191, 50, 5, 0, 0, 0, 0, 0, 0, 0, 0, 397, 310, 383, 946, 571, 414, 905, 2222, 199, 0, 0, 0, 0, 0, 0, 0, 0, 369, 0, 2, 456, 469, 1309, 3014, 484, 0, 0, 0, 0, 0, 0, 93, 66, 532, 0, 159, 166, 0, 1, 4963, 0, 0, 0, 0, 0, 0, 0, 49, 2546, 234, 0, 0, 1, 0, 2257, 838, 0, 0, 0, 0, 0, 0, 0, 959, 262, 192, 1739, 0, 124, 871, 2300, 0, 0, 0, 0, 0, 0, 0, 0, 406, 133, 0, 1741, 359, 2223, 1226, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4635, 0, 0, 1, 1, 0, 537, 107, 1144, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5836, 6347, 6103, 5980, 5925, 6447, 6089, 6425] avg=6144.0000 max=6447 max_rank=5 min=5836 min_rank=0 imbalance_ratio=1.049316 max_load_reduction=42705 max_load_reduction_pct=0.868835 moved_to_echo=24886 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[717, 97, 0, 5098, 62, 0, 207, 95, 5, 0, 0, 0, 0, 0, 0, 0, 0, 383, 251, 194, 847, 772, 427, 900, 2221, 212, 0, 0, 0, 0, 0, 0, 0, 0, 534, 0, 0, 462, 375, 1406, 3012, 501, 0, 0, 0, 0, 0, 0, 121, 53, 443, 0, 213, 153, 0, 0, 5123, 0, 0, 0, 0, 0, 0, 0, 95, 2534, 235, 0, 0, 0, 0, 2308, 894, 0, 0, 0, 0, 0, 0, 0, 940, 268, 208, 1587, 0, 75, 757, 2451, 0, 0, 0, 0, 0, 0, 0, 0, 410, 72, 0, 2206, 86, 2068, 1225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4133, 0, 0, 1, 0, 0, 420, 76, 1219, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6281, 6207, 6290, 6106, 6066, 6286, 6067, 5849] avg=6144.0000 max=6290 max_rank=2 min=5849 min_rank=7 imbalance_ratio=1.023763 max_load_reduction=42862 max_load_reduction_pct=0.872030 moved_to_echo=24268 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[690, 91, 0, 4952, 63, 0, 202, 82, 8, 0, 0, 0, 0, 0, 0, 0, 0, 388, 239, 203, 915, 681, 443, 994, 2233, 196, 0, 0, 0, 0, 0, 0, 0, 0, 443, 0, 0, 420, 399, 1353, 3030, 498, 0, 0, 0, 0, 0, 0, 120, 66, 497, 0, 195, 399, 0, 0, 5091, 0, 0, 0, 0, 0, 0, 0, 82, 2578, 204, 0, 0, 0, 0, 2852, 815, 0, 0, 0, 0, 0, 0, 0, 936, 273, 199, 1417, 0, 77, 822, 2211, 0, 0, 0, 0, 0, 0, 0, 0, 431, 80, 0, 1531, 107, 2571, 1223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4205, 0, 0, 0, 0, 0, 461, 77, 1109, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6088, 6292, 6143, 6368, 6531, 5935, 5943, 5852] avg=6144.0000 max=6531 max_rank=4 min=5852 min_rank=7 imbalance_ratio=1.062988 max_load_reduction=42621 max_load_reduction_pct=0.867126 moved_to_echo=24261 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[707, 85, 0, 5241, 23, 0, 180, 82, 5, 0, 0, 0, 0, 0, 0, 0, 0, 406, 174, 273, 832, 731, 412, 722, 2246, 202, 0, 0, 0, 0, 0, 0, 0, 0, 337, 1, 0, 423, 282, 1411, 3046, 504, 0, 0, 0, 0, 0, 0, 112, 88, 411, 0, 84, 73, 0, 0, 5161, 0, 0, 0, 0, 0, 0, 0, 86, 2559, 268, 0, 0, 0, 0, 2304, 854, 0, 0, 0, 0, 0, 0, 0, 916, 273, 175, 1455, 0, 74, 827, 2342, 0, 0, 0, 0, 0, 0, 0, 0, 416, 73, 0, 2025, 82, 2874, 1239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4327, 0, 0, 0, 0, 0, 495, 70, 1164, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6323, 5998, 6004, 5929, 6071, 6062, 6709, 6056] avg=6144.0000 max=6709 max_rank=6 min=5929 min_rank=3 imbalance_ratio=1.091960 max_load_reduction=42443 max_load_reduction_pct=0.863505 moved_to_echo=24898 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[688, 103, 0, 4848, 88, 0, 158, 94, 5, 0, 0, 0, 0, 0, 0, 0, 0, 330, 349, 191, 878, 686, 409, 923, 2208, 193, 0, 0, 0, 0, 0, 0, 0, 0, 331, 0, 1, 393, 396, 1772, 2995, 482, 0, 0, 0, 0, 0, 0, 122, 54, 562, 0, 334, 358, 0, 0, 4938, 0, 0, 0, 0, 0, 0, 0, 94, 2567, 341, 0, 0, 0, 0, 1976, 813, 0, 0, 0, 0, 0, 0, 0, 895, 215, 148, 1839, 0, 92, 1068, 2230, 0, 0, 0, 0, 0, 0, 0, 0, 397, 105, 0, 1930, 349, 1573, 1218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4733, 0, 0, 0, 0, 0, 458, 113, 1109, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5984, 6167, 6370, 6368, 5791, 6487, 5572, 6413] avg=6144.0000 max=6487 max_rank=5 min=5572 min_rank=6 imbalance_ratio=1.055827 max_load_reduction=42665 max_load_reduction_pct=0.868022 moved_to_echo=24263 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7424, 2176, 1536, 12544, 0, 768, 6528, 18688, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 2944, 2048, 2048, 7040, 5632, 3328, 7040, 17920, 1664, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3328, 768, 0, 14976, 1408, 19072, 9984, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[640, 20480, 2176, 0, 0, 128, 0, 19328, 6784, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[0, 0, 3072, 128, 128, 3584, 3072, 11520, 24320, 4096, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[35584, 0, 0, 128, 128, 0, 3712, 768, 9344, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[5760, 768, 0, 40192, 512, 0, 1536, 640, 128, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[896, 512, 3968, 0, 1536, 1792, 0, 128, 40704, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[2337, 7499, 0, 6, 366, 0, 50, 0, 0, 357, 50, 500, 98, 304, 490, 821, 69, 1972, 0, 0, 164, 54, 139, 610, 5906, 4252, 2015, 294, 0, 0, 77, 209, 0, 376, 266, 0, 79, 2570, 579, 77, 711, 0, 0, 59, 126, 6232, 13, 0, 5403, 0, 84, 931, 441, 74, 0, 0, 0, 11, 0, 0, 0, 1975, 235, 271] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[2581, 7518, 0, 2, 340, 0, 50, 0, 0, 318, 50, 644, 16, 216, 429, 898, 90, 1930, 0, 0, 141, 52, 189, 652, 5684, 4251, 2222, 204, 0, 0, 97, 174, 0, 373, 256, 0, 65, 2154, 576, 65, 574, 1, 0, 71, 69, 6325, 2, 0, 5519, 0, 77, 957, 453, 63, 0, 0, 0, 6, 0, 0, 0, 2243, 346, 209] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1842, 7203, 0, 40, 607, 1, 57, 0, 0, 316, 118, 495, 100, 307, 425, 999, 192, 1331, 0, 0, 264, 96, 973, 1287, 4373, 4730, 2125, 421, 0, 0, 89, 223, 0, 246, 580, 0, 86, 1415, 565, 99, 631, 1, 0, 50, 393, 6386, 21, 0, 5920, 0, 380, 951, 504, 85, 163, 0, 0, 302, 0, 0, 0, 1249, 241, 270] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[2351, 7284, 0, 19, 418, 0, 59, 0, 0, 295, 61, 648, 65, 349, 377, 794, 144, 1709, 0, 0, 209, 93, 372, 873, 5160, 4716, 2047, 381, 0, 0, 87, 236, 0, 278, 398, 0, 72, 2071, 481, 75, 570, 0, 0, 53, 303, 6255, 23, 0, 5898, 0, 186, 1005, 481, 72, 0, 0, 0, 10, 0, 0, 0, 1740, 210, 224] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[2161, 7293, 1, 16, 596, 0, 37, 0, 0, 291, 78, 626, 152, 291, 509, 861, 160, 1836, 0, 0, 236, 58, 549, 841, 5143, 4206, 2144, 315, 0, 1, 136, 205, 0, 344, 395, 0, 107, 1916, 586, 108, 758, 0, 0, 66, 262, 6004, 26, 1, 5654, 1, 359, 989, 512, 107, 2, 0, 0, 69, 1, 0, 1, 1576, 234, 332] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1977, 7343, 0, 12, 354, 0, 59, 0, 0, 316, 59, 404, 86, 320, 493, 933, 159, 1920, 0, 0, 231, 80, 365, 859, 5253, 4296, 2204, 295, 0, 0, 55, 202, 0, 358, 349, 0, 77, 2070, 623, 79, 683, 0, 0, 66, 183, 6304, 13, 0, 5724, 0, 283, 1018, 507, 74, 0, 0, 0, 46, 0, 0, 0, 1984, 197, 239] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[2304, 7472, 0, 14, 349, 0, 51, 0, 0, 320, 55, 609, 31, 322, 372, 679, 114, 1695, 0, 0, 146, 78, 381, 716, 5396, 4672, 2280, 310, 1, 0, 118, 179, 0, 356, 301, 0, 75, 2142, 459, 76, 542, 0, 0, 88, 75, 6321, 4, 0, 5683, 0, 60, 1092, 482, 73, 0, 0, 0, 10, 0, 0, 0, 2176, 229, 244] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1607, 7212, 1, 45, 585, 3, 66, 0, 0, 169, 103, 656, 132, 221, 410, 922, 303, 1210, 0, 0, 179, 84, 1063, 1220, 4461, 4343, 2201, 412, 0, 1, 225, 198, 0, 259, 551, 0, 91, 1546, 604, 121, 647, 0, 0, 54, 374, 6539, 34, 0, 5896, 0, 433, 1017, 483, 91, 266, 0, 1, 367, 0, 0, 0, 1262, 183, 301] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2337, 3544, 0, 6, 366, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 357, 50, 500, 98, 304, 490, 821, 2163, 1601, 0, 0, 0, 0, 0, 0, 69, 1972, 0, 0, 164, 54, 139, 610, 1084, 1042, 426, 0, 0, 0, 0, 0, 0, 3210, 2015, 294, 0, 0, 77, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 376, 266, 0, 79, 2570, 579, 77, 1792, 909, 0, 0, 0, 0, 0, 0, 711, 0, 0, 59, 126, 5148, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 84, 931, 441, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 1975, 235, 271, 4305, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6303, 6384, 5560, 5805, 6648, 6057, 5598, 6797] avg=6144.0000 max=6797 max_rank=7 min=5560 min_rank=2 imbalance_ratio=1.106283 max_load_reduction=42355 max_load_reduction_pct=0.861715 moved_to_echo=25100 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1842, 3404, 0, 40, 607, 1, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 316, 118, 495, 100, 307, 425, 999, 2078, 1185, 0, 0, 0, 0, 0, 0, 192, 1331, 0, 0, 264, 96, 973, 1287, 1110, 1159, 467, 0, 0, 0, 0, 0, 0, 3571, 2125, 421, 0, 0, 89, 223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 580, 0, 86, 1415, 565, 99, 1721, 996, 0, 0, 0, 0, 0, 0, 631, 1, 0, 50, 393, 5276, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4457, 0, 380, 951, 504, 85, 163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 302, 0, 0, 0, 1249, 241, 270, 3188, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5951, 6023, 6879, 6429, 5708, 6372, 6540, 5250] avg=6144.0000 max=6879 max_rank=2 min=5250 min_rank=7 imbalance_ratio=1.119629 max_load_reduction=42273 max_load_reduction_pct=0.860046 moved_to_echo=23870 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2161, 3446, 1, 16, 596, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 291, 78, 626, 152, 291, 509, 861, 2104, 1394, 0, 0, 0, 0, 0, 0, 160, 1836, 0, 0, 236, 58, 549, 841, 1044, 1031, 446, 0, 0, 0, 0, 0, 0, 3175, 2144, 315, 0, 1, 136, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 344, 395, 0, 107, 1916, 586, 108, 1743, 952, 0, 0, 0, 0, 0, 0, 758, 0, 0, 66, 262, 4960, 26, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4256, 1, 359, 989, 512, 107, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 1, 0, 1, 1576, 234, 332, 3749, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6257, 6306, 6201, 5976, 6151, 6073, 6226, 5962] avg=6144.0000 max=6306 max_rank=1 min=5962 min_rank=7 imbalance_ratio=1.026367 max_load_reduction=42846 max_load_reduction_pct=0.871704 moved_to_echo=24412 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2581, 3552, 0, 2, 340, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 318, 50, 644, 16, 216, 429, 898, 2169, 1542, 0, 0, 0, 0, 0, 0, 90, 1930, 0, 0, 141, 52, 189, 652, 1100, 1042, 435, 0, 0, 0, 0, 0, 0, 3209, 2222, 204, 0, 0, 97, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 373, 256, 0, 65, 2154, 576, 65, 1797, 929, 0, 0, 0, 0, 0, 0, 574, 1, 0, 71, 69, 5225, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4155, 0, 77, 957, 453, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 2243, 346, 209, 4142, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6525, 6282, 5631, 5906, 6215, 5942, 5705, 6946] avg=6144.0000 max=6946 max_rank=7 min=5631 min_rank=2 imbalance_ratio=1.130534 max_load_reduction=42206 max_load_reduction_pct=0.858683 moved_to_echo=24808 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2351, 3442, 0, 19, 418, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 295, 61, 648, 65, 349, 377, 794, 2101, 1399, 0, 0, 0, 0, 0, 0, 144, 1709, 0, 0, 209, 93, 372, 873, 1088, 1156, 465, 0, 0, 0, 0, 0, 0, 3560, 2047, 381, 0, 0, 87, 236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 278, 398, 0, 72, 2071, 481, 75, 1741, 993, 0, 0, 0, 0, 0, 0, 570, 0, 0, 53, 303, 5167, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4440, 0, 186, 1005, 481, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 1740, 210, 224, 3761, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6289, 6089, 6109, 6311, 6109, 6116, 6184, 5945] avg=6144.0000 max=6311 max_rank=3 min=5945 min_rank=7 imbalance_ratio=1.027181 max_load_reduction=42841 max_load_reduction_pct=0.871602 moved_to_echo=24354 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1977, 3463, 0, 12, 354, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 316, 59, 404, 86, 320, 493, 933, 2122, 1425, 0, 0, 0, 0, 0, 0, 159, 1920, 0, 0, 231, 80, 365, 859, 1099, 1058, 454, 0, 0, 0, 0, 0, 0, 3238, 2204, 295, 0, 0, 55, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 358, 349, 0, 77, 2070, 623, 79, 1758, 968, 0, 0, 0, 0, 0, 0, 683, 0, 0, 66, 183, 5205, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4302, 0, 283, 1018, 507, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 1984, 197, 239, 3828, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5865, 6158, 6225, 5994, 6282, 6150, 6184, 6294] avg=6144.0000 max=6294 max_rank=7 min=5865 min_rank=0 imbalance_ratio=1.024414 max_load_reduction=42858 max_load_reduction_pct=0.871948 moved_to_echo=24910 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2304, 3530, 0, 14, 349, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 320, 55, 609, 31, 322, 372, 679, 2156, 1463, 0, 0, 0, 0, 0, 0, 114, 1695, 0, 0, 146, 78, 381, 716, 1099, 1145, 448, 0, 0, 0, 0, 0, 0, 3527, 2280, 310, 1, 0, 118, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 356, 301, 0, 75, 2142, 459, 76, 1786, 956, 0, 0, 0, 0, 0, 0, 542, 0, 0, 88, 75, 5222, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4279, 0, 60, 1092, 482, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 2176, 229, 244, 3933, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6248, 6007, 5822, 6415, 6151, 5931, 5986, 6592] avg=6144.0000 max=6592 max_rank=7 min=5822 min_rank=2 imbalance_ratio=1.072917 max_load_reduction=42560 max_load_reduction_pct=0.865885 moved_to_echo=24660 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1607, 3407, 1, 45, 585, 3, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 103, 656, 132, 221, 410, 922, 2081, 1209, 0, 0, 0, 0, 0, 0, 303, 1210, 0, 0, 179, 84, 1063, 1220, 1137, 1064, 465, 0, 0, 0, 0, 0, 0, 3279, 2201, 412, 0, 1, 225, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 259, 551, 0, 91, 1546, 604, 121, 1724, 992, 0, 0, 0, 0, 0, 0, 647, 0, 0, 54, 374, 5402, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4439, 0, 433, 1017, 483, 91, 266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 367, 0, 0, 0, 1262, 183, 301, 3252, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5714, 5903, 6725, 6316, 5888, 6511, 6729, 5366] avg=6144.0000 max=6729 max_rank=6 min=5366 min_rank=7 imbalance_ratio=1.095215 max_load_reduction=42423 max_load_reduction_pct=0.863098 moved_to_echo=24494 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1280, 13696, 0, 0, 1664, 640, 4096, 7168, 8832, 8704, 3712, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 2432, 640, 4608, 768, 2432, 3584, 6912, 17024, 11264, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[17280, 27904, 128, 256, 3712, 128, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 2688, 3200, 0, 768, 16000, 4480, 768, 14080, 7808, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[34432, 128, 1920, 8064, 3968, 640, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 26880, 17280, 2688, 128, 128, 896, 1664, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 896, 128, 0, 128, 14208, 1920, 2176, 30208, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5120, 128, 0, 512, 1792, 41728, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[625, 64, 4795, 1359, 3766, 0, 6587, 5092, 0, 127, 0, 812, 0, 507, 0, 783, 243, 0, 2338, 0, 912, 53, 152, 82, 50, 64, 476, 0, 0, 115, 0, 176, 6045, 611, 0, 65, 195, 1264, 763, 0, 143, 0, 0, 0, 0, 0, 0, 3, 1733, 0, 0, 71, 1, 1618, 659, 0, 2849, 455, 1, 1583, 1915, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[582, 77, 5041, 1360, 3337, 0, 6651, 5285, 0, 155, 0, 798, 0, 435, 0, 738, 251, 0, 2618, 0, 934, 57, 277, 85, 50, 77, 461, 0, 0, 128, 0, 183, 5995, 614, 0, 78, 199, 1110, 760, 0, 190, 0, 0, 0, 0, 0, 0, 0, 1938, 0, 0, 59, 6, 1802, 581, 0, 2349, 441, 1, 1559, 1881, 9, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[562, 86, 5520, 1591, 3748, 0, 6546, 5549, 0, 167, 0, 707, 0, 432, 0, 460, 241, 0, 2890, 0, 972, 61, 352, 139, 57, 84, 549, 0, 0, 142, 0, 80, 5790, 599, 0, 85, 192, 621, 730, 0, 188, 0, 0, 0, 0, 0, 0, 0, 2111, 0, 1, 50, 8, 1848, 415, 0, 2829, 504, 1, 1058, 1176, 11, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[596, 110, 5429, 1473, 2995, 1, 6449, 5535, 0, 214, 0, 809, 1, 424, 1, 442, 246, 7, 2513, 0, 992, 47, 390, 116, 36, 107, 559, 0, 0, 144, 1, 181, 5869, 652, 0, 107, 207, 862, 756, 0, 258, 1, 0, 0, 0, 1, 3, 0, 1969, 0, 61, 66, 1, 1580, 475, 1, 2803, 512, 14, 1410, 1725, 0, 1, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[600, 73, 4935, 1141, 3772, 0, 6539, 5057, 0, 147, 0, 766, 0, 516, 0, 537, 241, 0, 3025, 0, 916, 50, 212, 121, 51, 73, 491, 0, 0, 125, 0, 140, 5781, 482, 0, 74, 194, 1131, 741, 0, 239, 0, 0, 0, 0, 0, 0, 22, 2010, 0, 0, 88, 4, 2178, 489, 0, 2485, 489, 0, 1293, 1923, 1, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[535, 72, 5271, 1695, 3218, 0, 6465, 5605, 0, 144, 0, 744, 0, 370, 0, 618, 259, 0, 2697, 0, 979, 58, 394, 151, 59, 72, 503, 0, 0, 131, 0, 168, 5990, 572, 0, 72, 208, 802, 703, 0, 193, 0, 0, 0, 0, 0, 0, 0, 2053, 0, 0, 53, 2, 1703, 407, 0, 2619, 482, 2, 1354, 1722, 7, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[586, 104, 5712, 1401, 3796, 0, 6389, 5667, 0, 182, 0, 753, 0, 516, 0, 444, 252, 0, 3258, 0, 949, 50, 320, 148, 64, 91, 521, 0, 0, 157, 0, 129, 5660, 517, 0, 91, 214, 585, 713, 0, 212, 0, 0, 0, 0, 0, 0, 0, 2044, 0, 18, 57, 0, 1768, 400, 0, 2740, 483, 5, 872, 1235, 49, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[566, 77, 5064, 1393, 3150, 0, 6446, 5569, 0, 153, 0, 708, 0, 546, 0, 582, 246, 0, 2806, 0, 990, 50, 271, 138, 59, 77, 549, 0, 0, 136, 0, 180, 5984, 665, 0, 77, 226, 993, 812, 0, 221, 0, 0, 0, 0, 0, 0, 0, 2040, 0, 0, 66, 3, 1651, 535, 0, 2655, 505, 4, 1365, 1592, 2, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[582, 77, 561, 1360, 3337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 0, 798, 0, 435, 0, 738, 3272, 767, 0, 0, 0, 0, 0, 0, 251, 0, 2618, 0, 934, 57, 277, 85, 1730, 0, 0, 0, 0, 0, 0, 0, 50, 77, 461, 0, 0, 128, 0, 183, 4518, 586, 0, 0, 0, 0, 0, 0, 3667, 614, 0, 78, 199, 1110, 760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 190, 0, 0, 0, 0, 0, 0, 0, 6065, 0, 0, 0, 0, 0, 0, 0, 1938, 0, 0, 59, 6, 1802, 581, 0, 1208, 575, 0, 0, 0, 0, 0, 0, 2349, 441, 1, 1559, 1881, 9, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5917, 6165, 5952, 6003, 6428, 6255, 6169, 6263] avg=6144.0000 max=6428 max_rank=4 min=5917 min_rank=0 imbalance_ratio=1.046224 max_load_reduction=42724 max_load_reduction_pct=0.869222 moved_to_echo=25115 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[562, 86, 615, 1591, 3748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 167, 0, 707, 0, 432, 0, 460, 3583, 805, 0, 0, 0, 0, 0, 0, 241, 0, 2890, 0, 972, 61, 352, 139, 1671, 0, 0, 0, 0, 0, 0, 0, 57, 84, 549, 0, 0, 142, 0, 80, 4744, 577, 0, 0, 0, 0, 0, 0, 3542, 599, 0, 85, 192, 621, 730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 0, 0, 0, 0, 0, 0, 5969, 0, 0, 0, 0, 0, 0, 0, 2111, 0, 1, 50, 8, 1848, 415, 0, 1322, 555, 0, 0, 0, 0, 0, 0, 2829, 504, 1, 1058, 1176, 11, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6602, 6154, 6326, 6233, 5769, 6157, 6310, 5601] avg=6144.0000 max=6602 max_rank=0 min=5601 min_rank=7 imbalance_ratio=1.074544 max_load_reduction=42550 max_load_reduction_pct=0.865682 moved_to_echo=23837 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[625, 64, 533, 1359, 3766, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 0, 812, 0, 507, 0, 783, 3113, 740, 0, 0, 0, 0, 0, 0, 243, 0, 2338, 0, 912, 53, 152, 82, 1745, 0, 0, 0, 0, 0, 0, 0, 50, 64, 476, 0, 0, 115, 0, 176, 4352, 582, 0, 0, 0, 0, 0, 0, 3698, 611, 0, 65, 195, 1264, 763, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 0, 0, 0, 0, 0, 0, 3, 6005, 0, 0, 0, 0, 0, 0, 0, 1733, 0, 0, 71, 1, 1618, 659, 0, 1149, 579, 0, 0, 0, 0, 0, 0, 2849, 455, 1, 1583, 1915, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6347, 6082, 5525, 5815, 6596, 6151, 5810, 6826] avg=6144.0000 max=6826 max_rank=7 min=5525 min_rank=2 imbalance_ratio=1.111003 max_load_reduction=42326 max_load_reduction_pct=0.861125 moved_to_echo=25383 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[596, 110, 605, 1473, 2995, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 214, 0, 809, 1, 424, 1, 442, 3524, 803, 0, 0, 0, 0, 0, 0, 246, 7, 2513, 0, 992, 47, 390, 116, 1694, 0, 0, 0, 0, 0, 0, 0, 36, 107, 559, 0, 0, 144, 1, 181, 4732, 568, 0, 0, 0, 0, 0, 0, 3590, 652, 0, 107, 207, 862, 756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 258, 1, 0, 0, 0, 1, 3, 0, 5881, 0, 0, 0, 0, 0, 0, 0, 1969, 0, 61, 66, 1, 1580, 475, 1, 1300, 563, 0, 0, 0, 0, 0, 0, 2803, 512, 14, 1410, 1725, 0, 1, 0, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5780, 6218, 6005, 6328, 6174, 6144, 6016, 6487] avg=6144.0000 max=6487 max_rank=7 min=5780 min_rank=0 imbalance_ratio=1.055827 max_load_reduction=42665 max_load_reduction_pct=0.868022 moved_to_echo=24821 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[535, 72, 586, 1695, 3218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 744, 0, 370, 0, 618, 3422, 815, 0, 0, 0, 0, 0, 0, 259, 0, 2697, 0, 979, 58, 394, 151, 1729, 0, 0, 0, 0, 0, 0, 0, 59, 72, 503, 0, 0, 131, 0, 168, 4790, 571, 0, 0, 0, 0, 0, 0, 3664, 572, 0, 72, 208, 802, 703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 193, 0, 0, 0, 0, 0, 0, 0, 5894, 0, 0, 0, 0, 0, 0, 0, 2053, 0, 0, 53, 2, 1703, 407, 0, 1263, 574, 0, 0, 0, 0, 0, 0, 2619, 482, 2, 1354, 1722, 7, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6106, 6113, 6267, 6294, 6021, 6087, 6055, 6209] avg=6144.0000 max=6294 max_rank=3 min=6021 min_rank=4 imbalance_ratio=1.024414 max_load_reduction=42858 max_load_reduction_pct=0.871948 moved_to_echo=24372 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[566, 77, 556, 1393, 3150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 0, 708, 0, 546, 0, 582, 3291, 809, 0, 0, 0, 0, 0, 0, 246, 0, 2806, 0, 990, 50, 271, 138, 1731, 0, 0, 0, 0, 0, 0, 0, 59, 77, 549, 0, 0, 136, 0, 180, 4760, 569, 0, 0, 0, 0, 0, 0, 3651, 665, 0, 77, 226, 993, 812, 0, 0, 0, 0, 0, 0, 0, 0, 0, 221, 0, 0, 0, 0, 0, 0, 0, 5877, 0, 0, 0, 0, 0, 0, 0, 2040, 0, 0, 66, 3, 1651, 535, 0, 1217, 577, 0, 0, 0, 0, 0, 0, 2655, 505, 4, 1365, 1592, 2, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5742, 6089, 6232, 6330, 6424, 6098, 6089, 6148] avg=6144.0000 max=6424 max_rank=4 min=5742 min_rank=0 imbalance_ratio=1.045573 max_load_reduction=42728 max_load_reduction_pct=0.869303 moved_to_echo=24759 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[600, 73, 549, 1141, 3772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 0, 766, 0, 516, 0, 537, 3204, 735, 0, 0, 0, 0, 0, 0, 241, 0, 3025, 0, 916, 50, 212, 121, 1669, 0, 0, 0, 0, 0, 0, 0, 51, 73, 491, 0, 0, 125, 0, 140, 4322, 577, 0, 0, 0, 0, 0, 0, 3536, 482, 0, 74, 194, 1131, 741, 0, 0, 0, 0, 0, 0, 0, 0, 0, 239, 0, 0, 0, 0, 0, 0, 22, 5962, 0, 0, 0, 0, 0, 0, 0, 2010, 0, 0, 88, 4, 2178, 489, 0, 1182, 554, 0, 0, 0, 0, 0, 0, 2485, 489, 0, 1293, 1923, 1, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6135, 5905, 6234, 5779, 6158, 6223, 6505, 6213] avg=6144.0000 max=6505 max_rank=6 min=5779 min_rank=3 imbalance_ratio=1.058757 max_load_reduction=42647 max_load_reduction_pct=0.867655 moved_to_echo=25099 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[586, 104, 636, 1401, 3796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, 0, 753, 0, 516, 0, 444, 3708, 823, 0, 0, 0, 0, 0, 0, 252, 0, 3258, 0, 949, 50, 320, 148, 1634, 0, 0, 0, 0, 0, 0, 0, 64, 91, 521, 0, 0, 157, 0, 129, 4844, 564, 0, 0, 0, 0, 0, 0, 3462, 517, 0, 91, 214, 585, 713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 0, 0, 0, 0, 0, 0, 0, 5825, 0, 0, 0, 0, 0, 0, 0, 2044, 0, 18, 57, 0, 1768, 400, 0, 1368, 543, 0, 0, 0, 0, 0, 0, 2740, 483, 5, 872, 1235, 49, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6523, 6426, 6611, 6370, 5582, 6037, 6198, 5405] avg=6144.0000 max=6611 max_rank=2 min=5405 min_rank=7 imbalance_ratio=1.076009 max_load_reduction=42541 max_load_reduction_pct=0.865499 moved_to_echo=23222 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[16000, 0, 128, 512, 128, 14208, 3968, 128, 10112, 4608, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2048, 128, 22272, 0, 7680, 512, 2432, 1024, 13696, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 1408, 0, 6144, 128, 3840, 128, 4608, 27136, 6400, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4736, 768, 4736, 11520, 27904, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[21376, 3968, 128, 10496, 13184, 128, 128, 0, 256, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[28928, 4736, 0, 768, 1664, 7424, 6016, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[512, 768, 4224, 0, 0, 1152, 128, 1280, 37120, 4608, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1664, 128, 0, 0, 0, 128, 128, 128, 47488, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 7474, 3807, 0, 0, 37, 368, 3561, 0, 1, 0, 0, 0, 674, 2863, 145, 0, 255, 152, 291, 3, 1, 1, 1133, 315, 247, 1, 127, 4, 37, 259, 0, 251, 7847, 107, 1, 2445, 0, 2439, 0, 1284, 0, 1, 0, 2029, 241, 0, 1, 107, 1, 107, 466, 0, 43, 107, 1, 0, 405, 6912, 191, 108, 651, 1, 1650] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 7610, 3611, 0, 0, 51, 301, 3051, 0, 0, 0, 0, 0, 630, 3244, 125, 0, 204, 30, 329, 0, 0, 0, 1029, 209, 244, 0, 123, 0, 51, 155, 0, 261, 7878, 74, 0, 3171, 0, 2844, 0, 1454, 0, 0, 0, 2057, 212, 0, 0, 76, 0, 74, 328, 0, 51, 59, 0, 0, 389, 7060, 276, 73, 510, 0, 1308] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 7507, 3812, 0, 1, 59, 382, 3865, 0, 0, 0, 0, 0, 679, 2639, 137, 0, 241, 81, 261, 45, 0, 0, 1157, 213, 255, 0, 95, 0, 59, 163, 0, 274, 7815, 77, 0, 2807, 0, 2267, 0, 1473, 0, 0, 0, 2075, 244, 0, 0, 77, 0, 80, 409, 0, 59, 75, 0, 0, 425, 6997, 243, 77, 658, 0, 1369] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 7507, 3909, 0, 0, 50, 376, 3309, 0, 0, 0, 0, 0, 685, 2819, 128, 0, 225, 37, 287, 0, 0, 0, 1105, 259, 252, 0, 79, 0, 50, 189, 0, 272, 7858, 78, 0, 3407, 0, 2443, 0, 1349, 0, 0, 0, 2019, 241, 0, 0, 78, 0, 78, 403, 0, 50, 60, 0, 0, 335, 7069, 298, 77, 500, 0, 1271] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 7680, 3439, 0, 0, 50, 387, 3072, 0, 0, 0, 0, 0, 657, 3298, 115, 0, 249, 0, 321, 0, 0, 0, 1077, 157, 246, 0, 79, 0, 50, 116, 0, 264, 7911, 65, 0, 3538, 0, 2425, 0, 1499, 0, 0, 0, 2141, 113, 0, 0, 65, 0, 66, 353, 0, 50, 49, 0, 0, 329, 7145, 285, 64, 535, 0, 1262] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 7537, 3040, 0, 11, 57, 350, 4404, 0, 0, 0, 0, 0, 622, 2192, 145, 0, 216, 94, 249, 75, 0, 0, 1085, 220, 249, 0, 115, 62, 57, 184, 0, 106, 7853, 85, 0, 1468, 0, 3401, 0, 1697, 0, 0, 0, 2186, 176, 0, 0, 85, 0, 85, 395, 0, 59, 83, 0, 0, 414, 6945, 146, 84, 1292, 0, 1628] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 7568, 3489, 0, 1, 59, 305, 3788, 0, 0, 0, 0, 0, 614, 2605, 131, 0, 245, 72, 299, 54, 0, 0, 1075, 213, 258, 0, 88, 0, 59, 152, 0, 261, 7879, 72, 0, 2575, 0, 2890, 0, 1597, 0, 0, 0, 2205, 249, 0, 0, 73, 0, 72, 344, 0, 61, 71, 0, 0, 408, 7011, 277, 72, 643, 0, 1317] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 7540, 3326, 0, 4, 66, 324, 4311, 0, 0, 0, 0, 0, 623, 2376, 157, 0, 229, 126, 301, 32, 0, 0, 1047, 268, 253, 0, 117, 90, 66, 225, 0, 67, 7800, 91, 0, 1436, 0, 3759, 0, 1579, 0, 0, 0, 1881, 153, 0, 0, 91, 0, 92, 410, 0, 107, 90, 0, 0, 373, 6911, 148, 91, 1095, 0, 1497] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3909, 0, 0, 50, 376, 1971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 685, 2819, 128, 2629, 0, 0, 0, 0, 0, 0, 0, 0, 225, 37, 287, 0, 0, 0, 1105, 4217, 215, 0, 0, 0, 0, 0, 0, 259, 252, 0, 79, 0, 50, 189, 0, 3211, 2082, 0, 0, 0, 0, 0, 0, 272, 430, 78, 0, 3407, 0, 2443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1349, 0, 0, 0, 2019, 241, 0, 0, 1338, 897, 0, 0, 0, 0, 0, 0, 78, 0, 78, 403, 0, 50, 60, 0, 5425, 0, 0, 0, 0, 0, 0, 0, 0, 335, 3328, 298, 77, 500, 0, 1271, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6306, 6261, 6086, 6122, 6630, 5844, 6094, 5809] avg=6144.0000 max=6630 max_rank=4 min=5809 min_rank=7 imbalance_ratio=1.079102 max_load_reduction=42522 max_load_reduction_pct=0.865112 moved_to_echo=24377 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3807, 0, 0, 37, 368, 2121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 674, 2863, 145, 2570, 0, 0, 0, 0, 0, 0, 0, 0, 255, 152, 291, 3, 1, 1, 1133, 4211, 210, 0, 0, 0, 0, 0, 0, 315, 247, 1, 127, 4, 37, 259, 0, 3206, 2073, 0, 0, 0, 0, 0, 0, 251, 430, 107, 1, 2445, 0, 2439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1284, 0, 1, 0, 2029, 241, 0, 1, 1440, 877, 0, 0, 0, 0, 0, 0, 107, 1, 107, 466, 0, 43, 107, 1, 5401, 0, 0, 0, 0, 0, 0, 0, 0, 405, 3255, 191, 108, 651, 1, 1650, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6333, 6253, 6257, 6269, 5673, 5873, 6233, 6261] avg=6144.0000 max=6333 max_rank=0 min=5673 min_rank=4 imbalance_ratio=1.030762 max_load_reduction=42819 max_load_reduction_pct=0.871155 moved_to_echo=24040 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3040, 0, 11, 57, 350, 2622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 622, 2192, 145, 2583, 0, 0, 0, 0, 0, 0, 0, 0, 216, 94, 249, 75, 0, 0, 1085, 4214, 211, 0, 0, 0, 0, 0, 0, 220, 249, 0, 115, 62, 57, 184, 0, 3209, 2090, 0, 0, 0, 0, 0, 0, 106, 430, 85, 0, 1468, 0, 3401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1697, 0, 0, 0, 2186, 176, 0, 0, 1782, 881, 0, 0, 0, 0, 0, 0, 85, 0, 85, 395, 0, 59, 83, 0, 5447, 0, 0, 0, 0, 0, 0, 0, 0, 414, 3270, 146, 84, 1292, 0, 1628, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6080, 5542, 6144, 6186, 5490, 6722, 6154, 6834] avg=6144.0000 max=6834 max_rank=7 min=5490 min_rank=4 imbalance_ratio=1.112305 max_load_reduction=42318 max_load_reduction_pct=0.860962 moved_to_echo=25200 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3439, 0, 0, 50, 387, 1829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 657, 3298, 115, 2657, 0, 0, 0, 0, 0, 0, 0, 0, 249, 0, 321, 0, 0, 0, 1077, 4246, 217, 0, 0, 0, 0, 0, 0, 157, 246, 0, 79, 0, 50, 116, 0, 3232, 2131, 0, 0, 0, 0, 0, 0, 264, 433, 65, 0, 3538, 0, 2425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1499, 0, 0, 0, 2141, 113, 0, 0, 1243, 906, 0, 0, 0, 0, 0, 0, 65, 0, 66, 353, 0, 50, 49, 0, 5549, 0, 0, 0, 0, 0, 0, 0, 0, 329, 3365, 285, 64, 535, 0, 1262, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5705, 6727, 6110, 6011, 6725, 5902, 6132, 5840] avg=6144.0000 max=6727 max_rank=1 min=5705 min_rank=0 imbalance_ratio=1.094889 max_load_reduction=42425 max_load_reduction_pct=0.863139 moved_to_echo=24599 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3489, 0, 1, 59, 305, 2256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 614, 2605, 131, 2607, 0, 0, 0, 0, 0, 0, 0, 0, 245, 72, 299, 54, 0, 0, 1075, 4228, 213, 0, 0, 0, 0, 0, 0, 213, 258, 0, 88, 0, 59, 152, 0, 3219, 2100, 0, 0, 0, 0, 0, 0, 261, 432, 72, 0, 2575, 0, 2890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1597, 0, 0, 0, 2205, 249, 0, 0, 1532, 889, 0, 0, 0, 0, 0, 0, 73, 0, 72, 344, 0, 61, 71, 0, 5468, 0, 0, 0, 0, 0, 0, 0, 0, 408, 3302, 277, 72, 643, 0, 1317, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6110, 5957, 6186, 6089, 6230, 6472, 6089, 6019] avg=6144.0000 max=6472 max_rank=5 min=5957 min_rank=1 imbalance_ratio=1.053385 max_load_reduction=42680 max_load_reduction_pct=0.868327 moved_to_echo=24810 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3611, 0, 0, 51, 301, 1817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 630, 3244, 125, 2625, 0, 0, 0, 0, 0, 0, 0, 0, 204, 30, 329, 0, 0, 0, 1029, 4228, 215, 0, 0, 0, 0, 0, 0, 209, 244, 0, 123, 0, 51, 155, 0, 3219, 2111, 0, 0, 0, 0, 0, 0, 261, 431, 74, 0, 3171, 0, 2844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1454, 0, 0, 0, 2057, 212, 0, 0, 1234, 896, 0, 0, 0, 0, 0, 0, 76, 0, 74, 328, 0, 51, 59, 0, 5499, 0, 0, 0, 0, 0, 0, 0, 0, 389, 3324, 276, 73, 510, 0, 1308, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5780, 6624, 6035, 6112, 6781, 5853, 6087, 5880] avg=6144.0000 max=6781 max_rank=4 min=5780 min_rank=0 imbalance_ratio=1.103678 max_load_reduction=42371 max_load_reduction_pct=0.862040 moved_to_echo=24601 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3812, 0, 1, 59, 382, 2297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 679, 2639, 137, 2607, 0, 0, 0, 0, 0, 0, 0, 0, 241, 81, 261, 45, 0, 0, 1157, 4199, 219, 0, 0, 0, 0, 0, 0, 213, 255, 0, 95, 0, 59, 163, 0, 3198, 2083, 0, 0, 0, 0, 0, 0, 274, 418, 77, 0, 2807, 0, 2267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1473, 0, 0, 0, 2075, 244, 0, 0, 1568, 892, 0, 0, 0, 0, 0, 0, 77, 0, 80, 409, 0, 59, 75, 0, 5424, 0, 0, 0, 0, 0, 0, 0, 0, 425, 3279, 243, 77, 658, 0, 1369, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6551, 6062, 6203, 6066, 5843, 6252, 6124, 6051] avg=6144.0000 max=6551 max_rank=0 min=5843 min_rank=4 imbalance_ratio=1.066243 max_load_reduction=42601 max_load_reduction_pct=0.866720 moved_to_echo=24270 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3326, 0, 4, 66, 324, 2567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 623, 2376, 157, 2570, 0, 0, 0, 0, 0, 0, 0, 0, 229, 126, 301, 32, 0, 0, 1047, 4186, 210, 0, 0, 0, 0, 0, 0, 268, 253, 0, 117, 90, 66, 225, 0, 3187, 2091, 0, 0, 0, 0, 0, 0, 67, 427, 91, 0, 1436, 0, 3759, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1579, 0, 0, 0, 1881, 153, 0, 0, 1744, 877, 0, 0, 0, 0, 0, 0, 91, 0, 92, 410, 0, 107, 90, 0, 5449, 0, 0, 0, 0, 0, 0, 0, 0, 373, 3254, 148, 91, 1095, 0, 1497, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6287, 5726, 6131, 6297, 5780, 6234, 6239, 6458] avg=6144.0000 max=6458 max_rank=7 min=5726 min_rank=1 imbalance_ratio=1.051107 max_load_reduction=42694 max_load_reduction_pct=0.868612 moved_to_echo=24711 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[12032, 0, 128, 0, 16640, 1664, 0, 128, 11904, 7168, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 3200, 26496, 1920, 768, 5888, 128, 11392, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[1792, 3456, 768, 128, 20864, 0, 22528, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 28544, 0, 128, 512, 2816, 17536, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 0, 5248, 22144, 1152, 20864, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[0, 1920, 640, 2432, 256, 128, 128, 8832, 33792, 1792, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1920, 2048, 128, 896, 256, 512, 1536, 0, 25728, 16768, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[768, 128, 768, 3200, 0, 512, 640, 128, 43776, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 359, 0, 0, 0, 0, 386, 256, 1385, 0, 1021, 312, 0, 84, 50, 0, 1, 0, 0, 88, 0, 0, 173, 5481, 0, 300, 0, 1992, 0, 0, 399, 1612, 0, 0, 0, 3927, 0, 759, 2508, 545, 4096, 0, 588, 919, 2179, 0, 0, 101, 0, 830, 0, 0, 48, 0, 456, 0, 7068, 6517, 4286, 257, 0, 169] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 355, 0, 0, 0, 0, 367, 277, 908, 0, 953, 81, 0, 77, 59, 0, 0, 0, 0, 66, 0, 0, 209, 6838, 0, 297, 0, 1581, 0, 0, 229, 2019, 0, 0, 0, 4020, 0, 746, 4229, 343, 2455, 0, 498, 1581, 2705, 0, 0, 112, 0, 533, 0, 0, 8, 0, 578, 0, 7031, 6561, 3184, 97, 0, 155] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 333, 0, 0, 0, 0, 373, 429, 677, 0, 997, 64, 0, 64, 71, 0, 0, 0, 0, 79, 0, 0, 207, 6821, 0, 274, 0, 1797, 0, 0, 191, 1923, 0, 0, 0, 4226, 0, 731, 4164, 201, 2719, 0, 425, 1755, 2489, 0, 0, 52, 0, 488, 0, 0, 0, 0, 521, 0, 7320, 6556, 2999, 77, 0, 129] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 345, 0, 1, 0, 1, 331, 189, 1058, 1, 829, 312, 1, 107, 66, 1, 0, 0, 1, 49, 0, 0, 193, 6409, 0, 277, 0, 1748, 0, 2, 299, 2293, 0, 0, 0, 3885, 1, 734, 3443, 476, 3299, 1, 595, 1197, 2293, 0, 0, 152, 1, 631, 0, 0, 66, 1, 536, 1, 6724, 6427, 3754, 208, 0, 214] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 345, 0, 0, 0, 0, 386, 275, 950, 0, 987, 171, 0, 77, 66, 0, 0, 0, 0, 82, 0, 0, 208, 6565, 0, 268, 0, 1947, 0, 0, 236, 1954, 0, 0, 0, 3776, 0, 752, 3569, 373, 3123, 0, 617, 1341, 2318, 0, 0, 86, 0, 624, 0, 0, 34, 0, 574, 0, 6978, 6469, 3696, 151, 0, 154] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 345, 0, 0, 0, 0, 380, 369, 1059, 0, 889, 140, 0, 73, 88, 0, 0, 0, 0, 86, 0, 0, 207, 6224, 0, 300, 0, 1905, 0, 0, 226, 2104, 0, 0, 0, 4031, 0, 655, 3790, 265, 2788, 0, 502, 1642, 2466, 0, 0, 82, 0, 470, 0, 0, 6, 0, 560, 0, 7226, 6520, 3464, 143, 0, 147] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 369, 0, 0, 0, 0, 412, 335, 1246, 0, 755, 195, 0, 72, 53, 0, 0, 0, 0, 95, 0, 0, 219, 6548, 0, 298, 0, 1846, 0, 0, 273, 1796, 0, 0, 0, 3868, 0, 808, 3454, 364, 3232, 0, 542, 1261, 2297, 0, 0, 80, 0, 611, 0, 0, 29, 0, 573, 0, 6956, 6514, 3777, 130, 0, 144] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 301, 0, 0, 0, 0, 412, 291, 1339, 0, 977, 341, 0, 90, 54, 0, 0, 0, 0, 94, 0, 0, 161, 5465, 0, 295, 0, 1836, 0, 0, 325, 2039, 0, 0, 0, 3755, 0, 727, 2478, 609, 3921, 0, 527, 1010, 2447, 0, 0, 134, 0, 788, 0, 0, 52, 0, 500, 0, 6952, 6469, 4376, 205, 0, 182] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 359, 0, 0, 0, 0, 4777, 1036, 0, 0, 0, 0, 0, 0, 386, 256, 1385, 0, 1021, 312, 0, 84, 2213, 769, 584, 0, 0, 0, 0, 0, 50, 0, 1, 0, 0, 88, 0, 0, 6032, 0, 0, 0, 0, 0, 0, 0, 173, 3334, 0, 300, 0, 1992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 399, 1612, 0, 0, 0, 3158, 0, 759, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 1883, 0, 588, 919, 2179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 0, 830, 0, 0, 48, 0, 2508, 1563, 141, 0, 0, 0, 0, 0, 456, 0, 0, 1599, 4286, 257, 0, 169, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6172, 7010, 6171, 5799, 5928, 6114, 5191, 6767] avg=6144.0000 max=7010 max_rank=1 min=5191 min_rank=6 imbalance_ratio=1.140951 max_load_reduction=42142 max_load_reduction_pct=0.857381 moved_to_echo=24000 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 355, 0, 0, 0, 0, 4809, 1031, 0, 0, 0, 0, 0, 0, 367, 277, 908, 0, 953, 81, 0, 77, 1326, 787, 729, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 66, 0, 0, 6000, 0, 0, 0, 0, 0, 0, 0, 209, 4159, 0, 297, 0, 1581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 229, 2019, 0, 0, 0, 3233, 0, 746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 343, 1129, 0, 498, 1581, 2705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 533, 0, 0, 8, 0, 4229, 1950, 142, 0, 0, 0, 0, 0, 578, 0, 0, 1610, 3184, 97, 0, 155, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6195, 5505, 6125, 6246, 6227, 6256, 6974, 5624] avg=6144.0000 max=6974 max_rank=6 min=5505 min_rank=1 imbalance_ratio=1.135091 max_load_reduction=42178 max_load_reduction_pct=0.858114 moved_to_echo=25081 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 345, 0, 1, 0, 1, 4711, 986, 0, 0, 0, 0, 0, 0, 331, 189, 1058, 1, 829, 312, 1, 107, 1782, 760, 683, 0, 0, 0, 0, 0, 66, 1, 0, 0, 1, 49, 0, 0, 5738, 0, 0, 0, 0, 0, 0, 0, 193, 3898, 0, 277, 0, 1748, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 299, 2293, 0, 0, 0, 3125, 1, 734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 1517, 1, 595, 1197, 2293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 1, 631, 0, 0, 66, 1, 3443, 1828, 139, 0, 0, 0, 0, 0, 536, 1, 0, 1577, 3754, 208, 0, 214, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6044, 6053, 5855, 6118, 6452, 6079, 6261, 6290] avg=6144.0000 max=6452 max_rank=4 min=5855 min_rank=2 imbalance_ratio=1.050130 max_load_reduction=42700 max_load_reduction_pct=0.868734 moved_to_echo=25082 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 333, 0, 0, 0, 0, 4806, 1074, 0, 0, 0, 0, 0, 0, 373, 429, 677, 0, 997, 64, 0, 64, 1469, 827, 727, 0, 0, 0, 0, 0, 71, 0, 0, 0, 0, 79, 0, 0, 6246, 0, 0, 0, 0, 0, 0, 0, 207, 4148, 0, 274, 0, 1797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 191, 1923, 0, 0, 0, 3399, 0, 731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 201, 1250, 0, 425, 1755, 2489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 488, 0, 0, 0, 0, 4164, 1946, 142, 0, 0, 0, 0, 0, 521, 0, 0, 1608, 2999, 77, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6213, 5627, 6396, 6426, 6244, 6120, 6792, 5334] avg=6144.0000 max=6792 max_rank=6 min=5334 min_rank=7 imbalance_ratio=1.105469 max_load_reduction=42360 max_load_reduction_pct=0.861816 moved_to_echo=24490 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 369, 0, 0, 0, 0, 4775, 1021, 0, 0, 0, 0, 0, 0, 412, 335, 1246, 0, 755, 195, 0, 72, 1746, 757, 698, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 95, 0, 0, 5935, 0, 0, 0, 0, 0, 0, 0, 219, 3982, 0, 298, 0, 1846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 273, 1796, 0, 0, 0, 3111, 0, 808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 364, 1486, 0, 542, 1261, 2297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 611, 0, 0, 29, 0, 3454, 1868, 141, 0, 0, 0, 0, 0, 573, 0, 0, 1598, 3777, 130, 0, 144, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6165, 6216, 6083, 6345, 5988, 5950, 6183, 6222] avg=6144.0000 max=6345 max_rank=3 min=5950 min_rank=5 imbalance_ratio=1.032715 max_load_reduction=42807 max_load_reduction_pct=0.870911 moved_to_echo=24343 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 345, 0, 0, 0, 0, 4747, 1024, 0, 0, 0, 0, 0, 0, 386, 275, 950, 0, 987, 171, 0, 77, 1692, 743, 705, 0, 0, 0, 0, 0, 66, 0, 0, 0, 0, 82, 0, 0, 5954, 0, 0, 0, 0, 0, 0, 0, 208, 3983, 0, 268, 0, 1947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 236, 1954, 0, 0, 0, 3033, 0, 752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 373, 1431, 0, 617, 1341, 2318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 624, 0, 0, 34, 0, 3569, 1877, 146, 0, 0, 0, 0, 0, 574, 0, 0, 1576, 3696, 151, 0, 154, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6116, 5986, 6102, 6406, 5975, 6080, 6336, 6151] avg=6144.0000 max=6406 max_rank=3 min=5975 min_rank=4 imbalance_ratio=1.042643 max_load_reduction=42746 max_load_reduction_pct=0.869670 moved_to_echo=24542 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 345, 0, 0, 0, 0, 4779, 1060, 0, 0, 0, 0, 0, 0, 380, 369, 1059, 0, 889, 140, 0, 73, 1506, 789, 664, 0, 0, 0, 0, 0, 88, 0, 0, 0, 0, 86, 0, 0, 6166, 0, 0, 0, 0, 0, 0, 0, 207, 3785, 0, 300, 0, 1905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 226, 2104, 0, 0, 0, 3242, 0, 655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 265, 1282, 0, 502, 1642, 2466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 470, 0, 0, 6, 0, 3790, 1775, 141, 0, 0, 0, 0, 0, 560, 0, 0, 1600, 3464, 143, 0, 147, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6184, 5869, 6340, 6197, 6227, 6157, 6264, 5914] avg=6144.0000 max=6340 max_rank=2 min=5869 min_rank=1 imbalance_ratio=1.031901 max_load_reduction=42812 max_load_reduction_pct=0.871012 moved_to_echo=24562 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 301, 0, 0, 0, 0, 4742, 1020, 0, 0, 0, 0, 0, 0, 412, 291, 1339, 0, 977, 341, 0, 90, 2118, 735, 583, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 94, 0, 0, 5932, 0, 0, 0, 0, 0, 0, 0, 161, 3323, 0, 295, 0, 1836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 325, 2039, 0, 0, 0, 3020, 0, 727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 609, 1803, 0, 527, 1010, 2447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, 0, 788, 0, 0, 52, 0, 2478, 1559, 140, 0, 0, 0, 0, 0, 500, 0, 0, 1587, 4376, 205, 0, 182, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6063, 6886, 6080, 5615, 6111, 6396, 5151, 6850] avg=6144.0000 max=6886 max_rank=1 min=5151 min_rank=6 imbalance_ratio=1.120768 max_load_reduction=42266 max_load_reduction_pct=0.859904 moved_to_echo=24508 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3072, 2432, 8704, 128, 7424, 1664, 128, 768, 13952, 6272, 5376, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1664, 30720, 0, 2432, 0, 14720, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 896, 128, 4992, 0, 0, 256, 128, 27648, 14464, 1152, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[2304, 15744, 0, 0, 0, 25344, 128, 6016, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 3200, 11904, 128, 4352, 10752, 19200, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 0, 2816, 0, 128, 0, 128, 38272, 8320, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4352, 128, 0, 12800, 29568, 1280, 0, 1408, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[512, 128, 128, 0, 128, 640, 0, 0, 48128, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 312, 351, 0, 6934, 0, 375, 0, 0, 13, 0, 72, 0, 0, 0, 2814, 0, 0, 0, 0, 3207, 0, 3795, 0, 0, 62, 4032, 0, 0, 0, 59, 5673, 2, 72, 3551, 0, 0, 167, 540, 226, 0, 0, 2862, 95, 0, 0, 0, 690, 0, 3467, 72, 6225, 0, 415, 0, 0, 95, 939, 624, 863, 397, 0, 151] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 267, 323, 0, 6860, 0, 399, 0, 0, 13, 0, 65, 0, 0, 0, 2794, 0, 0, 0, 0, 3374, 0, 3777, 0, 0, 80, 5440, 0, 0, 0, 50, 5472, 0, 65, 2826, 0, 0, 143, 527, 193, 0, 0, 3545, 73, 0, 0, 0, 740, 0, 3175, 62, 6233, 0, 224, 0, 0, 54, 626, 606, 816, 217, 0, 113] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 292, 285, 2, 7277, 0, 415, 0, 0, 12, 0, 85, 0, 0, 0, 2987, 0, 0, 0, 0, 2866, 0, 3508, 0, 0, 27, 2845, 0, 0, 0, 57, 6072, 0, 85, 3703, 0, 0, 172, 695, 229, 0, 0, 2582, 88, 0, 0, 0, 610, 0, 4044, 83, 6116, 0, 585, 0, 0, 147, 1148, 670, 868, 419, 0, 178] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 8, 311, 290, 0, 6905, 0, 470, 0, 0, 4, 0, 107, 0, 0, 0, 2438, 0, 0, 0, 1, 2714, 0, 4275, 0, 0, 68, 3631, 1, 0, 0, 37, 5651, 2, 107, 2634, 0, 1, 156, 585, 193, 1, 0, 3378, 49, 0, 1, 0, 704, 1, 4314, 107, 5659, 3, 552, 1, 0, 182, 1269, 720, 877, 490, 1, 254] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 288, 317, 0, 6726, 0, 444, 0, 0, 5, 0, 78, 0, 0, 0, 2547, 0, 0, 0, 0, 3278, 0, 4402, 0, 0, 43, 4673, 0, 0, 0, 50, 5368, 1, 78, 3182, 0, 0, 143, 502, 193, 0, 0, 3392, 65, 0, 0, 0, 789, 0, 3205, 77, 6145, 0, 249, 0, 0, 118, 843, 642, 845, 278, 0, 186] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 322, 330, 0, 6566, 0, 371, 0, 0, 6, 0, 76, 0, 0, 0, 2913, 0, 0, 0, 0, 3366, 0, 3926, 0, 0, 43, 4748, 0, 0, 0, 51, 5429, 0, 74, 3688, 0, 0, 159, 576, 210, 0, 0, 3289, 84, 0, 0, 0, 822, 0, 2809, 71, 6181, 0, 320, 0, 0, 89, 721, 629, 815, 334, 0, 134] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 3, 307, 328, 0, 7072, 0, 441, 0, 0, 22, 0, 77, 0, 0, 0, 2473, 0, 0, 0, 0, 3101, 0, 4298, 0, 0, 45, 4040, 0, 0, 0, 59, 5749, 0, 77, 3133, 0, 0, 159, 576, 218, 0, 0, 3116, 82, 0, 0, 0, 658, 0, 3568, 77, 6184, 0, 374, 0, 0, 117, 929, 632, 816, 258, 0, 163] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 304, 331, 3, 7215, 0, 408, 0, 0, 7, 0, 92, 0, 0, 0, 2941, 0, 0, 0, 0, 3138, 0, 3685, 0, 0, 30, 2889, 0, 0, 0, 66, 5735, 0, 91, 3297, 0, 0, 185, 579, 251, 0, 0, 2728, 93, 0, 0, 0, 572, 0, 3885, 90, 6045, 4, 581, 0, 1, 220, 1340, 717, 824, 583, 0, 222] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 288, 317, 0, 4947, 0, 444, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 78, 0, 0, 0, 4740, 1498, 0, 0, 0, 0, 0, 0, 2547, 0, 0, 0, 0, 3278, 0, 307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 4673, 0, 0, 0, 50, 1779, 157, 0, 0, 0, 0, 0, 0, 1978, 1, 78, 3182, 0, 0, 143, 502, 0, 0, 0, 0, 0, 0, 0, 0, 193, 0, 0, 3392, 65, 0, 0, 0, 2605, 0, 0, 0, 0, 0, 0, 0, 789, 0, 3205, 77, 1405, 0, 249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 843, 642, 845, 278, 0, 186, 2597, 628, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5996, 6321, 6132, 6702, 5884, 6255, 5725, 6137] avg=6144.0000 max=6702 max_rank=3 min=5725 min_rank=6 imbalance_ratio=1.090820 max_load_reduction=42450 max_load_reduction_pct=0.863647 moved_to_echo=24001 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 292, 285, 2, 5352, 0, 415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 85, 0, 0, 0, 4718, 1194, 0, 0, 0, 0, 0, 0, 2987, 0, 0, 0, 0, 2866, 0, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 2845, 0, 0, 0, 57, 1925, 178, 0, 0, 0, 0, 0, 0, 2236, 0, 85, 3703, 0, 0, 172, 695, 0, 0, 0, 0, 0, 0, 0, 0, 229, 0, 0, 2582, 88, 0, 0, 0, 2947, 0, 0, 0, 0, 0, 0, 0, 610, 0, 4044, 83, 1398, 0, 585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 1148, 670, 868, 419, 0, 178, 2069, 711, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6346, 6009, 6098, 5032, 6891, 5846, 6720, 6210] avg=6144.0000 max=6891 max_rank=4 min=5032 min_rank=3 imbalance_ratio=1.121582 max_load_reduction=42261 max_load_reduction_pct=0.859802 moved_to_echo=25667 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 311, 290, 0, 5078, 0, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 107, 0, 0, 0, 4365, 1455, 0, 0, 0, 0, 0, 0, 2438, 0, 0, 0, 1, 2714, 0, 298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 3631, 1, 0, 0, 37, 1827, 165, 0, 0, 0, 0, 0, 0, 2082, 2, 107, 2634, 0, 1, 156, 585, 0, 0, 0, 0, 0, 0, 0, 0, 193, 1, 0, 3378, 49, 0, 1, 0, 2743, 0, 0, 0, 0, 0, 0, 0, 704, 1, 4314, 107, 1294, 3, 552, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, 1269, 720, 877, 490, 1, 254, 2522, 661, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6157, 5931, 5451, 5729, 5567, 6365, 6976, 6976] avg=6144.0000 max=6976 max_rank=6 min=5451 min_rank=2 imbalance_ratio=1.135417 max_load_reduction=42176 max_load_reduction_pct=0.858073 moved_to_echo=25884 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 267, 323, 0, 5045, 0, 399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 65, 0, 0, 0, 4808, 1286, 0, 0, 0, 0, 0, 0, 2794, 0, 0, 0, 0, 3374, 0, 263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 5440, 0, 0, 0, 50, 1815, 160, 0, 0, 0, 0, 0, 0, 2016, 0, 65, 2826, 0, 0, 143, 527, 0, 0, 0, 0, 0, 0, 0, 0, 193, 0, 0, 3545, 73, 0, 0, 0, 2656, 0, 0, 0, 0, 0, 0, 0, 740, 0, 3175, 62, 1425, 0, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 626, 606, 816, 217, 0, 113, 2228, 640, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6034, 6172, 6431, 7545, 5577, 6467, 5626, 5300] avg=6144.0000 max=7545 max_rank=3 min=5300 min_rank=7 imbalance_ratio=1.228027 max_load_reduction=41607 max_load_reduction_pct=0.846497 moved_to_echo=22970 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 312, 351, 0, 5100, 0, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 72, 0, 0, 0, 4802, 1292, 0, 0, 0, 0, 0, 0, 2814, 0, 0, 0, 0, 3207, 0, 264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 4032, 0, 0, 0, 59, 1834, 166, 0, 0, 0, 0, 0, 0, 2090, 2, 72, 3551, 0, 0, 167, 540, 0, 0, 0, 0, 0, 0, 0, 0, 226, 0, 0, 2862, 95, 0, 0, 0, 2753, 0, 0, 0, 0, 0, 0, 0, 690, 0, 3467, 72, 1423, 0, 415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 939, 624, 863, 397, 0, 151, 2239, 664, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6138, 6179, 6285, 6153, 6422, 5936, 6067, 5972] avg=6144.0000 max=6422 max_rank=4 min=5936 min_rank=5 imbalance_ratio=1.045247 max_load_reduction=42730 max_load_reduction_pct=0.869344 moved_to_echo=24397 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 322, 330, 0, 4829, 0, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 76, 0, 0, 0, 4768, 1336, 0, 0, 0, 0, 0, 0, 2913, 0, 0, 0, 0, 3366, 0, 274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 4748, 0, 0, 0, 51, 1737, 159, 0, 0, 0, 0, 0, 0, 2000, 0, 74, 3688, 0, 0, 159, 576, 0, 0, 0, 0, 0, 0, 0, 0, 210, 0, 0, 3289, 84, 0, 0, 0, 2635, 0, 0, 0, 0, 0, 0, 0, 822, 0, 2809, 71, 1413, 0, 320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 721, 629, 815, 334, 0, 134, 2316, 635, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5852, 6186, 6553, 6738, 6497, 6218, 5435, 5673] avg=6144.0000 max=6738 max_rank=3 min=5435 min_rank=6 imbalance_ratio=1.096680 max_load_reduction=42414 max_load_reduction_pct=0.862915 moved_to_echo=23823 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 307, 328, 0, 5198, 0, 441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 77, 0, 0, 0, 4772, 1467, 0, 0, 0, 0, 0, 0, 2473, 0, 0, 0, 0, 3101, 0, 292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 4040, 0, 0, 0, 59, 1874, 173, 0, 0, 0, 0, 0, 0, 2104, 0, 77, 3133, 0, 0, 159, 576, 0, 0, 0, 0, 0, 0, 0, 0, 218, 0, 0, 3116, 82, 0, 0, 0, 2794, 0, 0, 0, 0, 0, 0, 0, 658, 0, 3568, 77, 1412, 0, 374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 929, 632, 816, 258, 0, 163, 2539, 678, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6277, 6338, 5866, 6191, 6049, 6210, 6089, 6132] avg=6144.0000 max=6338 max_rank=1 min=5866 min_rank=2 imbalance_ratio=1.031576 max_load_reduction=42814 max_load_reduction_pct=0.871053 moved_to_echo=24480 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 304, 331, 3, 5306, 0, 408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 92, 0, 0, 0, 4663, 1254, 0, 0, 0, 0, 0, 0, 2941, 0, 0, 0, 0, 3138, 0, 257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 2889, 0, 0, 0, 66, 1909, 168, 0, 0, 0, 0, 0, 0, 2113, 0, 91, 3297, 0, 0, 185, 579, 0, 0, 0, 0, 0, 0, 0, 0, 251, 0, 0, 2728, 93, 0, 0, 0, 2783, 0, 0, 0, 0, 0, 0, 0, 572, 0, 3885, 90, 1382, 4, 581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 220, 1340, 717, 824, 583, 0, 222, 2174, 671, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6352, 6016, 6336, 5062, 6265, 5855, 6514, 6752] avg=6144.0000 max=6752 max_rank=7 min=5062 min_rank=3 imbalance_ratio=1.098958 max_load_reduction=42400 max_load_reduction_pct=0.862630 moved_to_echo=25386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 1024, 7936, 5248, 6784, 3072, 128, 1408, 18688, 5376, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5632, 128, 28544, 640, 11264, 128, 3328, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[22016, 0, 0, 0, 128, 25088, 0, 2304, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1792, 128, 0, 24960, 640, 0, 128, 0, 22016, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[16640, 128, 768, 26112, 0, 128, 1408, 4608, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 512, 32384, 128, 0, 0, 512, 14720, 1408, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 128, 2432, 2560, 128, 40960, 0, 3328, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 768, 0, 0, 0, 37760, 10880, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[369, 0, 56, 0, 57, 0, 0, 0, 0, 3032, 0, 100, 84, 27, 474, 1689, 0, 7290, 0, 85, 0, 0, 50, 0, 7481, 0, 89, 241, 0, 1388, 84, 7571, 7094, 784, 0, 0, 0, 0, 0, 0, 85, 0, 1305, 0, 0, 2074, 241, 0, 317, 0, 0, 4542, 0, 0, 0, 0, 0, 2493, 0, 0, 0, 0, 0, 50] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[1196, 0, 0, 0, 50, 0, 0, 0, 0, 2488, 0, 52, 65, 2, 462, 1757, 0, 7391, 0, 65, 0, 0, 71, 0, 7541, 0, 88, 244, 0, 1455, 64, 7743, 6877, 672, 0, 0, 0, 0, 0, 0, 65, 0, 914, 0, 0, 2677, 244, 0, 4, 0, 0, 5158, 0, 0, 0, 0, 0, 1736, 0, 0, 0, 0, 0, 71] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[1101, 0, 7, 0, 50, 0, 0, 0, 0, 2390, 0, 112, 78, 1, 423, 1843, 1, 7337, 0, 79, 0, 0, 59, 0, 7461, 0, 51, 251, 0, 1429, 77, 7698, 6840, 585, 0, 0, 0, 0, 0, 0, 78, 0, 877, 0, 0, 3080, 251, 0, 8, 0, 0, 5168, 0, 0, 0, 0, 0, 1758, 0, 0, 0, 0, 0, 59] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[559, 0, 37, 0, 37, 0, 0, 0, 1, 2778, 0, 152, 107, 14, 472, 1782, 0, 7244, 1, 107, 0, 0, 67, 1, 7417, 1, 96, 245, 1, 1388, 107, 7642, 6849, 664, 1, 0, 1, 0, 0, 1, 107, 0, 1228, 1, 0, 2643, 245, 0, 209, 0, 0, 4931, 0, 0, 0, 0, 0, 1950, 0, 0, 0, 0, 0, 66] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[785, 0, 39, 0, 59, 0, 0, 0, 0, 2694, 0, 80, 72, 1, 464, 1766, 0, 7313, 0, 72, 0, 0, 53, 0, 7530, 0, 118, 258, 0, 1480, 72, 7696, 6919, 593, 0, 0, 0, 0, 0, 0, 72, 0, 1025, 0, 0, 2367, 258, 0, 25, 0, 0, 5114, 0, 0, 0, 0, 0, 2174, 0, 0, 0, 0, 0, 53] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[739, 0, 43, 0, 59, 0, 0, 0, 0, 2651, 0, 86, 77, 8, 497, 1714, 0, 7293, 0, 77, 0, 0, 66, 0, 7458, 0, 73, 246, 0, 1615, 77, 7670, 6840, 701, 0, 0, 0, 0, 0, 0, 77, 0, 1033, 0, 0, 2603, 246, 0, 13, 0, 0, 4981, 0, 0, 0, 0, 0, 2143, 0, 0, 0, 0, 0, 66] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[816, 0, 7, 0, 51, 0, 0, 0, 0, 2680, 0, 82, 74, 3, 504, 1635, 0, 7326, 0, 74, 0, 0, 88, 0, 7514, 0, 23, 244, 0, 1779, 73, 7717, 6920, 633, 0, 0, 0, 0, 0, 0, 74, 0, 1015, 0, 0, 2270, 244, 0, 29, 0, 0, 5375, 0, 0, 0, 0, 0, 1814, 0, 0, 0, 0, 0, 88] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[489, 0, 57, 0, 66, 0, 0, 0, 0, 3135, 0, 134, 91, 17, 460, 1708, 0, 7257, 0, 91, 0, 0, 54, 0, 7437, 0, 108, 252, 0, 1363, 91, 7560, 7182, 793, 0, 0, 0, 0, 0, 0, 91, 0, 1411, 0, 0, 1829, 251, 0, 191, 0, 0, 4705, 0, 0, 0, 0, 0, 2276, 0, 0, 0, 0, 0, 53] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[369, 0, 56, 0, 57, 0, 0, 0, 5239, 0, 0, 0, 0, 0, 0, 0, 0, 3032, 0, 100, 84, 27, 474, 1689, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 5986, 0, 85, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4250, 0, 89, 241, 0, 1388, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5588, 784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 1305, 0, 0, 2074, 241, 0, 1553, 733, 0, 0, 0, 0, 0, 0, 317, 0, 0, 4542, 0, 0, 0, 0, 773, 290, 0, 0, 0, 0, 0, 0, 0, 2493, 0, 0, 0, 0, 0, 50, 2332, 1678, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5721, 6420, 6121, 6052, 6372, 5991, 5922, 6553] avg=6144.0000 max=6553 max_rank=7 min=5721 min_rank=0 imbalance_ratio=1.066569 max_load_reduction=42599 max_load_reduction_pct=0.866679 moved_to_echo=24838 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1101, 0, 7, 0, 50, 0, 0, 0, 5327, 0, 0, 0, 0, 0, 0, 0, 0, 2390, 0, 112, 78, 1, 423, 1843, 1021, 0, 0, 0, 0, 0, 0, 0, 1, 6025, 0, 79, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4238, 0, 51, 251, 0, 1429, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5387, 585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 877, 0, 0, 3080, 251, 0, 1549, 707, 0, 0, 0, 0, 0, 0, 8, 0, 0, 5168, 0, 0, 0, 0, 746, 291, 0, 0, 0, 0, 0, 0, 0, 1758, 0, 0, 0, 0, 0, 59, 2371, 1674, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6485, 5868, 6164, 6046, 5972, 6542, 6213, 5862] avg=6144.0000 max=6542 max_rank=5 min=5862 min_rank=7 imbalance_ratio=1.064779 max_load_reduction=42610 max_load_reduction_pct=0.866903 moved_to_echo=24589 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1196, 0, 0, 0, 50, 0, 0, 0, 5359, 0, 0, 0, 0, 0, 0, 0, 0, 2488, 0, 52, 65, 2, 462, 1757, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 6069, 0, 65, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4283, 0, 88, 244, 0, 1455, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5416, 672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 914, 0, 0, 2677, 244, 0, 1566, 711, 0, 0, 0, 0, 0, 0, 4, 0, 0, 5158, 0, 0, 0, 0, 750, 294, 0, 0, 0, 0, 0, 0, 0, 1736, 0, 0, 0, 0, 0, 71, 2384, 1692, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6605, 5854, 6205, 6134, 6088, 6177, 6206, 5883] avg=6144.0000 max=6605 max_rank=0 min=5854 min_rank=1 imbalance_ratio=1.075033 max_load_reduction=42547 max_load_reduction_pct=0.865621 moved_to_echo=24354 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[559, 0, 37, 0, 37, 0, 0, 0, 5288, 0, 0, 0, 0, 0, 0, 0, 1, 2778, 0, 152, 107, 14, 472, 1782, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 5948, 1, 107, 0, 0, 67, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4213, 1, 96, 245, 1, 1388, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5394, 664, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 107, 0, 1228, 1, 0, 2643, 245, 0, 1540, 708, 0, 0, 0, 0, 0, 0, 209, 0, 0, 4931, 0, 0, 0, 0, 747, 288, 0, 0, 0, 0, 0, 0, 0, 1950, 0, 0, 0, 0, 0, 66, 2354, 1664, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5921, 6314, 6124, 6051, 6061, 6472, 6175, 6034] avg=6144.0000 max=6472 max_rank=5 min=5921 min_rank=0 imbalance_ratio=1.053385 max_load_reduction=42680 max_load_reduction_pct=0.868327 moved_to_echo=24742 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[785, 0, 39, 0, 59, 0, 0, 0, 5327, 0, 0, 0, 0, 0, 0, 0, 0, 2694, 0, 80, 72, 1, 464, 1766, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 6005, 0, 72, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4277, 0, 118, 258, 0, 1480, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5450, 593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 1025, 0, 0, 2367, 258, 0, 1564, 715, 0, 0, 0, 0, 0, 0, 25, 0, 0, 5114, 0, 0, 0, 0, 754, 291, 0, 0, 0, 0, 0, 0, 0, 2174, 0, 0, 0, 0, 0, 53, 2369, 1689, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6210, 6094, 6130, 6205, 6043, 6001, 6184, 6285] avg=6144.0000 max=6285 max_rank=7 min=6001 min_rank=5 imbalance_ratio=1.022949 max_load_reduction=42867 max_load_reduction_pct=0.872131 moved_to_echo=24513 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[739, 0, 43, 0, 59, 0, 0, 0, 5309, 0, 0, 0, 0, 0, 0, 0, 0, 2651, 0, 86, 77, 8, 497, 1714, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 5982, 0, 77, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4227, 0, 73, 246, 0, 1615, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5381, 701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 1033, 0, 0, 2603, 246, 0, 1554, 710, 0, 0, 0, 0, 0, 0, 13, 0, 0, 4981, 0, 0, 0, 0, 749, 293, 0, 0, 0, 0, 0, 0, 0, 2143, 0, 0, 0, 0, 0, 66, 2361, 1677, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6150, 6051, 6125, 6238, 6082, 6223, 6036, 6247] avg=6144.0000 max=6247 max_rank=7 min=6036 min_rank=6 imbalance_ratio=1.016764 max_load_reduction=42905 max_load_reduction_pct=0.872904 moved_to_echo=24588 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[816, 0, 7, 0, 51, 0, 0, 0, 5341, 0, 0, 0, 0, 0, 0, 0, 0, 2680, 0, 82, 74, 3, 504, 1635, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 6016, 0, 74, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4268, 0, 23, 244, 0, 1779, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5451, 633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 1015, 0, 0, 2270, 244, 0, 1560, 715, 0, 0, 0, 0, 0, 0, 29, 0, 0, 5375, 0, 0, 0, 0, 754, 291, 0, 0, 0, 0, 0, 0, 0, 1814, 0, 0, 0, 0, 0, 88, 2376, 1686, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6215, 5997, 6178, 6387, 6084, 5878, 6449, 5964] avg=6144.0000 max=6449 max_rank=6 min=5878 min_rank=5 imbalance_ratio=1.049642 max_load_reduction=42703 max_load_reduction_pct=0.868795 moved_to_echo=24375 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[489, 0, 57, 0, 66, 0, 0, 0, 5233, 0, 0, 0, 0, 0, 0, 0, 0, 3135, 0, 134, 91, 17, 460, 1708, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 5960, 0, 91, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4225, 0, 108, 252, 0, 1363, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5657, 793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 1411, 0, 0, 1829, 251, 0, 1544, 742, 0, 0, 0, 0, 0, 0, 191, 0, 0, 4705, 0, 0, 0, 0, 783, 288, 0, 0, 0, 0, 0, 0, 0, 2276, 0, 0, 0, 0, 0, 53, 2327, 1668, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5845, 6554, 6105, 6039, 6450, 5868, 5967, 6324] avg=6144.0000 max=6554 max_rank=1 min=5845 min_rank=0 imbalance_ratio=1.066732 max_load_reduction=42598 max_load_reduction_pct=0.866659 moved_to_echo=24609 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[768, 0, 8832, 128, 0, 19584, 2048, 0, 12544, 5760, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 21888, 0, 896, 768, 128, 3840, 13952, 8192, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49280 observed_loss=-128 tokens_per_expert=[0, 16384, 0, 0, 0, 0, 0, 512, 18944, 13440, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[34048, 128, 768, 2048, 128, 11904, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[896, 0, 0, 40064, 0, 0, 0, 0, 6144, 2432, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[6144, 0, 256, 0, 512, 0, 0, 0, 42496, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[43776, 5504, 128, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 48000, 128, 768, 0, 0, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 56, 636, 0, 0, 1474, 0, 0, 4083, 4741, 0, 0, 0, 646, 0, 88, 329, 0, 355, 1, 47, 0, 0, 2886, 0, 0, 84, 0, 0, 0, 0, 50, 326, 0, 0, 4579, 161, 0, 84, 4, 0, 4929, 0, 0, 0, 0, 0, 721, 0, 83, 0, 7531, 0, 88, 72, 7052, 0, 0, 0, 6825, 645, 576, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 34, 647, 0, 0, 1730, 0, 0, 3722, 5472, 0, 1, 2, 548, 0, 49, 295, 0, 248, 2, 61, 0, 0, 2945, 0, 0, 107, 1, 0, 0, 1, 68, 352, 1, 1, 3977, 154, 0, 107, 14, 0, 4936, 0, 0, 0, 0, 1, 704, 1, 107, 0, 7545, 1, 49, 104, 7093, 0, 0, 0, 6844, 627, 601, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 542, 0, 0, 2191, 0, 0, 2836, 6269, 0, 0, 0, 693, 0, 66, 317, 0, 259, 6, 52, 0, 0, 1935, 0, 0, 77, 0, 0, 0, 0, 59, 329, 0, 0, 4659, 146, 0, 77, 35, 0, 4912, 0, 0, 0, 0, 0, 726, 0, 71, 0, 7611, 0, 66, 2, 7224, 0, 0, 0, 6843, 595, 554, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 36, 610, 0, 0, 2031, 0, 0, 2996, 5479, 0, 0, 0, 703, 0, 82, 328, 0, 325, 1, 60, 0, 0, 2392, 0, 0, 77, 0, 0, 0, 0, 66, 323, 0, 0, 4824, 145, 0, 77, 10, 0, 4962, 0, 0, 0, 0, 0, 678, 0, 76, 0, 7549, 0, 82, 45, 7163, 0, 0, 0, 6802, 617, 613, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 601, 0, 0, 2059, 0, 0, 2778, 6223, 0, 0, 0, 728, 0, 86, 330, 0, 277, 1, 77, 0, 0, 2003, 0, 0, 78, 0, 0, 0, 0, 88, 312, 0, 0, 4361, 158, 0, 73, 21, 0, 5409, 0, 0, 0, 0, 0, 673, 0, 65, 0, 7575, 0, 86, 6, 7148, 0, 0, 0, 6836, 527, 573, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 556, 0, 0, 2196, 0, 0, 2617, 6154, 0, 0, 0, 690, 0, 79, 323, 0, 257, 0, 62, 0, 0, 1832, 0, 0, 64, 0, 0, 0, 0, 71, 308, 0, 0, 4807, 133, 0, 64, 28, 0, 5128, 0, 0, 0, 0, 0, 691, 0, 57, 0, 7619, 0, 79, 3, 7250, 0, 0, 0, 6943, 552, 589, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 600, 0, 0, 1892, 0, 0, 3599, 5413, 0, 0, 0, 715, 0, 95, 353, 0, 328, 0, 52, 0, 0, 2516, 0, 0, 72, 0, 0, 0, 0, 53, 330, 0, 0, 4613, 143, 0, 72, 1, 0, 4744, 0, 0, 0, 0, 0, 676, 0, 71, 0, 7583, 0, 95, 24, 7138, 0, 0, 0, 6855, 531, 588, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 46, 592, 0, 0, 1479, 0, 0, 4305, 4643, 0, 0, 0, 627, 0, 94, 346, 0, 353, 0, 54, 0, 0, 2891, 0, 0, 91, 0, 0, 0, 0, 54, 342, 0, 0, 4471, 173, 0, 91, 1, 0, 4949, 0, 0, 0, 0, 0, 652, 0, 90, 0, 7497, 0, 94, 79, 7096, 0, 0, 0, 6917, 571, 554, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 56, 636, 0, 0, 1474, 0, 1601, 1745, 0, 0, 0, 0, 0, 0, 0, 4083, 1802, 0, 0, 0, 646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 329, 0, 355, 1, 47, 0, 0, 5307, 0, 0, 0, 0, 0, 0, 0, 2886, 0, 0, 84, 0, 0, 0, 0, 2061, 1338, 0, 0, 0, 0, 0, 0, 50, 326, 0, 0, 4579, 161, 0, 84, 978, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4929, 0, 0, 0, 0, 0, 890, 237, 0, 0, 0, 0, 0, 0, 721, 0, 83, 0, 5233, 0, 88, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4957, 645, 576, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5512, 6531, 6127, 6369, 6178, 6060, 6197, 6178] avg=6144.0000 max=6531 max_rank=1 min=5512 min_rank=0 imbalance_ratio=1.062988 max_load_reduction=42621 max_load_reduction_pct=0.867126 moved_to_echo=24613 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 542, 0, 0, 2191, 0, 2117, 1788, 0, 0, 0, 0, 0, 0, 0, 2836, 2382, 0, 0, 0, 693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 317, 0, 259, 6, 52, 0, 0, 5436, 0, 0, 0, 0, 0, 0, 0, 1935, 0, 0, 77, 0, 0, 0, 0, 2083, 1770, 0, 0, 0, 0, 0, 0, 59, 329, 0, 0, 4659, 146, 0, 77, 981, 0, 0, 0, 0, 0, 0, 0, 35, 0, 4912, 0, 0, 0, 0, 0, 892, 240, 0, 0, 0, 0, 0, 0, 726, 0, 71, 0, 5288, 0, 66, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4970, 595, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6638, 5911, 6136, 5865, 6251, 6079, 6153, 6119] avg=6144.0000 max=6638 max_rank=0 min=5865 min_rank=3 imbalance_ratio=1.080404 max_load_reduction=42514 max_load_reduction_pct=0.864950 moved_to_echo=24602 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 34, 647, 0, 0, 1730, 0, 1848, 1755, 0, 0, 0, 0, 0, 0, 0, 3722, 2079, 0, 1, 2, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 295, 0, 248, 2, 61, 0, 0, 5338, 0, 0, 0, 0, 0, 0, 0, 2945, 0, 0, 107, 1, 0, 0, 1, 2065, 1545, 0, 0, 0, 0, 0, 0, 68, 352, 1, 1, 3977, 154, 0, 107, 981, 0, 0, 0, 0, 0, 0, 0, 14, 0, 4936, 0, 0, 0, 0, 1, 892, 238, 0, 0, 0, 0, 0, 0, 704, 1, 107, 0, 5242, 1, 49, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4971, 627, 601, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6014, 6352, 5993, 6664, 5641, 6081, 6208, 6199] avg=6144.0000 max=6664 max_rank=3 min=5641 min_rank=4 imbalance_ratio=1.084635 max_load_reduction=42488 max_load_reduction_pct=0.864421 moved_to_echo=24129 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 36, 610, 0, 0, 2031, 0, 1853, 1774, 0, 0, 0, 0, 0, 0, 0, 2996, 2075, 0, 0, 0, 703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 328, 0, 325, 1, 60, 0, 0, 5389, 0, 0, 0, 0, 0, 0, 0, 2392, 0, 0, 77, 0, 0, 0, 0, 2069, 1551, 0, 0, 0, 0, 0, 0, 66, 323, 0, 0, 4824, 145, 0, 77, 980, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4962, 0, 0, 0, 0, 0, 890, 243, 0, 0, 0, 0, 0, 0, 678, 0, 76, 0, 5237, 0, 82, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4932, 617, 613, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6304, 5774, 6185, 6089, 6415, 6105, 6118, 6162] avg=6144.0000 max=6415 max_rank=4 min=5774 min_rank=1 imbalance_ratio=1.044108 max_load_reduction=42737 max_load_reduction_pct=0.869486 moved_to_echo=24800 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 556, 0, 0, 2196, 0, 2078, 1795, 0, 0, 0, 0, 0, 0, 0, 2617, 2339, 0, 0, 0, 690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 323, 0, 257, 0, 62, 0, 0, 5455, 0, 0, 0, 0, 0, 0, 0, 1832, 0, 0, 64, 0, 0, 0, 0, 2085, 1737, 0, 0, 0, 0, 0, 0, 71, 308, 0, 0, 4807, 133, 0, 64, 995, 0, 0, 0, 0, 0, 0, 0, 28, 0, 5128, 0, 0, 0, 0, 0, 905, 240, 0, 0, 0, 0, 0, 0, 691, 0, 57, 0, 5294, 0, 79, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5043, 552, 589, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6625, 5646, 6176, 5718, 6378, 6301, 6124, 6184] avg=6144.0000 max=6625 max_rank=0 min=5646 min_rank=1 imbalance_ratio=1.078288 max_load_reduction=42527 max_load_reduction_pct=0.865214 moved_to_echo=24987 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 600, 0, 0, 1892, 0, 1828, 1767, 0, 0, 0, 0, 0, 0, 0, 3599, 2057, 0, 0, 0, 715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 353, 0, 328, 0, 52, 0, 0, 5371, 0, 0, 0, 0, 0, 0, 0, 2516, 0, 0, 72, 0, 0, 0, 0, 2075, 1528, 0, 0, 0, 0, 0, 0, 53, 330, 0, 0, 4613, 143, 0, 72, 983, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4744, 0, 0, 0, 0, 0, 894, 239, 0, 0, 0, 0, 0, 0, 676, 0, 71, 0, 5269, 0, 95, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4978, 531, 588, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6087, 6371, 6199, 6191, 6194, 5878, 6135, 6097] avg=6144.0000 max=6371 max_rank=1 min=5878 min_rank=5 imbalance_ratio=1.036947 max_load_reduction=42781 max_load_reduction_pct=0.870382 moved_to_echo=24304 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 601, 0, 0, 2059, 0, 2101, 1770, 0, 0, 0, 0, 0, 0, 0, 2778, 2365, 0, 0, 0, 728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 330, 0, 277, 1, 77, 0, 0, 5378, 0, 0, 0, 0, 0, 0, 0, 2003, 0, 0, 78, 0, 0, 0, 0, 2073, 1757, 0, 0, 0, 0, 0, 0, 88, 312, 0, 0, 4361, 158, 0, 73, 980, 0, 0, 0, 0, 0, 0, 0, 21, 0, 5409, 0, 0, 0, 0, 0, 891, 239, 0, 0, 0, 0, 0, 0, 673, 0, 65, 0, 5263, 0, 86, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4965, 527, 573, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6531, 5871, 6149, 5911, 5972, 6560, 6093, 6065] avg=6144.0000 max=6560 max_rank=5 min=5871 min_rank=1 imbalance_ratio=1.067708 max_load_reduction=42592 max_load_reduction_pct=0.866536 moved_to_echo=24690 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 46, 592, 0, 0, 1479, 0, 1568, 1756, 0, 0, 0, 0, 0, 0, 0, 4305, 1764, 0, 0, 0, 627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 346, 0, 353, 0, 54, 0, 0, 5340, 0, 0, 0, 0, 0, 0, 0, 2891, 0, 0, 91, 0, 0, 0, 0, 2052, 1311, 0, 0, 0, 0, 0, 0, 54, 342, 0, 0, 4471, 173, 0, 91, 992, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4949, 0, 0, 0, 0, 0, 902, 236, 0, 0, 0, 0, 0, 0, 652, 0, 90, 0, 5209, 0, 94, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5023, 571, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5441, 6696, 6187, 6345, 6123, 6088, 6124, 6148] avg=6144.0000 max=6696 max_rank=1 min=5441 min_rank=0 imbalance_ratio=1.089844 max_load_reduction=42456 max_load_reduction_pct=0.863770 moved_to_echo=24483 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 256, 4864, 0, 0, 15104, 0, 15104, 14208, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 27008, 16896, 0, 128, 128, 5376, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[19456, 0, 0, 768, 128, 0, 0, 128, 16640, 12544, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[128, 0, 40064, 0, 0, 0, 0, 128, 7168, 1920, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[512, 2688, 128, 128, 36352, 1280, 0, 768, 7936, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[640, 2688, 0, 2432, 128, 512, 0, 0, 43136, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5632, 128, 640, 0, 42112, 128, 640, 384, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 39936, 4736, 4736, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[3360, 0, 0, 85, 746, 5058, 0, 0, 1141, 88, 345, 0, 0, 0, 0, 0, 0, 85, 145, 0, 0, 83, 0, 0, 137, 0, 0, 0, 0, 85, 0, 0, 0, 0, 1097, 0, 0, 0, 2411, 388, 0, 6904, 5642, 8047, 0, 0, 4444, 0, 0, 0, 0, 85, 0, 0, 0, 6867, 894, 0, 0, 0, 623, 150, 241, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[3196, 0, 0, 78, 693, 5635, 0, 0, 1278, 63, 339, 0, 0, 0, 0, 0, 0, 78, 117, 0, 0, 77, 0, 0, 151, 0, 0, 0, 0, 78, 0, 0, 0, 0, 845, 0, 0, 0, 2162, 390, 0, 7116, 6102, 8081, 0, 0, 3807, 0, 0, 0, 0, 78, 0, 0, 0, 6812, 918, 0, 0, 0, 636, 171, 251, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[3488, 0, 2, 107, 756, 5253, 0, 0, 1212, 49, 358, 0, 0, 0, 0, 0, 0, 107, 86, 1, 0, 107, 1, 0, 182, 0, 0, 1, 1, 107, 0, 1, 1, 0, 1085, 0, 0, 1, 2031, 399, 0, 7014, 5621, 8104, 0, 1, 4100, 0, 0, 0, 0, 107, 0, 0, 4, 6781, 1007, 2, 0, 5, 604, 218, 245, 3] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[3369, 0, 0, 65, 700, 5862, 0, 0, 1330, 61, 349, 0, 0, 0, 0, 0, 0, 65, 130, 0, 0, 64, 0, 0, 136, 0, 0, 0, 0, 65, 0, 0, 0, 0, 699, 0, 0, 0, 1885, 315, 0, 7198, 5937, 8085, 0, 0, 3882, 0, 0, 0, 0, 65, 0, 0, 0, 7021, 884, 0, 0, 0, 603, 123, 244, 15] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[3814, 0, 0, 74, 730, 5681, 0, 0, 1103, 83, 353, 0, 0, 0, 0, 0, 0, 74, 137, 0, 0, 73, 0, 0, 171, 0, 0, 0, 0, 74, 0, 0, 0, 0, 842, 0, 0, 0, 1508, 310, 0, 7130, 5984, 8058, 0, 0, 3892, 0, 0, 0, 0, 74, 0, 0, 0, 7009, 978, 0, 0, 1, 566, 170, 244, 19] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[3163, 0, 0, 77, 754, 5560, 0, 0, 1030, 82, 354, 0, 0, 0, 0, 0, 0, 77, 141, 0, 0, 77, 0, 0, 151, 0, 0, 0, 0, 77, 0, 0, 0, 0, 1048, 0, 0, 0, 2039, 385, 0, 7043, 6012, 8051, 0, 0, 4047, 0, 0, 0, 0, 77, 0, 0, 0, 6871, 989, 0, 0, 0, 649, 152, 246, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[3418, 0, 0, 91, 722, 5093, 0, 0, 1053, 94, 372, 0, 0, 0, 0, 0, 0, 90, 160, 0, 0, 90, 0, 0, 147, 0, 0, 0, 0, 91, 0, 0, 0, 0, 1185, 0, 0, 0, 2143, 446, 0, 6811, 5829, 8032, 0, 0, 4401, 0, 0, 0, 0, 91, 0, 0, 3, 6886, 892, 0, 0, 1, 569, 188, 252, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[3634, 0, 0, 72, 736, 5423, 0, 0, 1196, 95, 305, 0, 0, 0, 0, 0, 0, 72, 154, 0, 0, 72, 0, 0, 127, 0, 0, 0, 0, 72, 0, 0, 0, 0, 984, 0, 0, 0, 2352, 365, 0, 7090, 5639, 8039, 0, 0, 3898, 0, 0, 0, 0, 72, 0, 0, 0, 6921, 835, 0, 0, 1, 607, 133, 258, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3196, 0, 0, 78, 693, 1970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1278, 63, 339, 0, 0, 0, 0, 0, 3376, 1266, 0, 0, 0, 0, 0, 0, 0, 78, 117, 0, 0, 77, 0, 0, 3740, 2155, 0, 0, 0, 0, 0, 0, 151, 0, 0, 0, 0, 78, 0, 0, 5926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 845, 0, 0, 0, 2162, 390, 1961, 823, 0, 0, 0, 0, 0, 0, 0, 0, 2177, 0, 0, 0, 3807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 5989, 0, 0, 0, 0, 0, 0, 0, 0, 918, 0, 0, 0, 636, 171, 251, 0, 2659, 1704, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5937, 6322, 6167, 6155, 6181, 5984, 6067, 6339] avg=6144.0000 max=6339 max_rank=7 min=5937 min_rank=0 imbalance_ratio=1.031738 max_load_reduction=42813 max_load_reduction_pct=0.871033 moved_to_echo=24571 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3360, 0, 0, 85, 746, 1768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1141, 88, 345, 0, 0, 0, 0, 0, 3275, 1170, 0, 0, 0, 0, 0, 0, 0, 85, 145, 0, 0, 83, 0, 0, 3629, 2145, 0, 0, 0, 0, 0, 0, 137, 0, 0, 0, 0, 85, 0, 0, 5902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1097, 0, 0, 0, 2411, 388, 1760, 830, 0, 0, 0, 0, 0, 0, 0, 0, 2013, 0, 0, 0, 4444, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 6037, 0, 0, 0, 0, 0, 0, 0, 0, 894, 0, 0, 0, 623, 150, 241, 1, 2459, 1530, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5959, 6019, 6087, 6124, 6486, 6457, 6122, 5898] avg=6144.0000 max=6486 max_rank=4 min=5898 min_rank=7 imbalance_ratio=1.055664 max_load_reduction=42666 max_load_reduction_pct=0.868042 moved_to_echo=24963 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3369, 0, 0, 65, 700, 2049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1330, 61, 349, 0, 0, 0, 0, 0, 3416, 1231, 0, 0, 0, 0, 0, 0, 0, 65, 130, 0, 0, 64, 0, 0, 3782, 2156, 0, 0, 0, 0, 0, 0, 136, 0, 0, 0, 0, 65, 0, 0, 5929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 699, 0, 0, 0, 1885, 315, 2040, 849, 0, 0, 0, 0, 0, 0, 0, 0, 2119, 0, 0, 0, 3882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 6172, 0, 0, 0, 0, 0, 0, 0, 0, 884, 0, 0, 0, 603, 123, 244, 15, 2587, 1773, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6183, 6387, 6197, 6130, 5788, 6001, 6237, 6229] avg=6144.0000 max=6387 max_rank=1 min=5788 min_rank=4 imbalance_ratio=1.039551 max_load_reduction=42765 max_load_reduction_pct=0.870056 moved_to_echo=24255 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3488, 0, 2, 107, 756, 1836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1212, 49, 358, 0, 0, 0, 0, 0, 3327, 1166, 0, 0, 0, 0, 0, 0, 0, 107, 86, 1, 0, 107, 1, 0, 3687, 2161, 0, 0, 0, 0, 0, 0, 182, 0, 0, 1, 1, 107, 0, 1, 5943, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1085, 0, 0, 1, 2031, 399, 1828, 820, 0, 0, 0, 0, 0, 0, 0, 0, 2006, 0, 0, 1, 4100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 0, 0, 4, 5961, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 2, 0, 5, 604, 218, 245, 3, 2449, 1589, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6189, 6112, 6150, 6235, 6165, 6107, 6072, 6122] avg=6144.0000 max=6235 max_rank=3 min=6072 min_rank=6 imbalance_ratio=1.014811 max_load_reduction=42917 max_load_reduction_pct=0.873149 moved_to_echo=24466 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3634, 0, 0, 72, 736, 1896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1196, 95, 305, 0, 0, 0, 0, 0, 3365, 1169, 0, 0, 0, 0, 0, 0, 0, 72, 154, 0, 0, 72, 0, 0, 3725, 2144, 0, 0, 0, 0, 0, 0, 127, 0, 0, 0, 0, 72, 0, 0, 5895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 984, 0, 0, 0, 2352, 365, 1887, 837, 0, 0, 0, 0, 0, 0, 0, 0, 2013, 0, 0, 0, 3898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 6084, 0, 0, 0, 0, 0, 0, 0, 0, 835, 0, 0, 1, 607, 133, 258, 0, 2457, 1640, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6338, 6130, 6167, 6094, 6425, 5911, 6156, 5931] avg=6144.0000 max=6425 max_rank=4 min=5911 min_rank=5 imbalance_ratio=1.045736 max_load_reduction=42727 max_load_reduction_pct=0.869283 moved_to_echo=24423 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3814, 0, 0, 74, 730, 1986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1103, 83, 353, 0, 0, 0, 0, 0, 3384, 1241, 0, 0, 0, 0, 0, 0, 0, 74, 137, 0, 0, 73, 0, 0, 3746, 2149, 0, 0, 0, 0, 0, 0, 171, 0, 0, 0, 0, 74, 0, 0, 5909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 842, 0, 0, 0, 1508, 310, 1977, 847, 0, 0, 0, 0, 0, 0, 0, 0, 2135, 0, 0, 0, 3892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 6162, 0, 0, 0, 0, 0, 0, 0, 0, 978, 0, 0, 1, 566, 170, 244, 19, 2608, 1718, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6604, 6164, 6179, 6154, 5484, 6027, 6236, 6304] avg=6144.0000 max=6604 max_rank=0 min=5484 min_rank=4 imbalance_ratio=1.074870 max_load_reduction=42548 max_load_reduction_pct=0.865641 moved_to_echo=24051 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3163, 0, 0, 77, 754, 1936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 82, 354, 0, 0, 0, 0, 0, 3342, 1251, 0, 0, 0, 0, 0, 0, 0, 77, 141, 0, 0, 77, 0, 0, 3701, 2147, 0, 0, 0, 0, 0, 0, 151, 0, 0, 0, 0, 77, 0, 0, 5904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 0, 0, 0, 2039, 385, 1939, 834, 0, 0, 0, 0, 0, 0, 0, 0, 2137, 0, 0, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 6037, 0, 0, 0, 0, 0, 0, 0, 0, 989, 0, 0, 0, 649, 152, 246, 0, 2624, 1685, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5930, 6059, 6143, 6132, 6245, 6184, 6114, 6345] avg=6144.0000 max=6345 max_rank=7 min=5930 min_rank=0 imbalance_ratio=1.032715 max_load_reduction=42807 max_load_reduction_pct=0.870911 moved_to_echo=24888 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3418, 0, 0, 91, 722, 1781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 94, 372, 0, 0, 0, 0, 0, 3231, 1209, 0, 0, 0, 0, 0, 0, 0, 90, 160, 0, 0, 90, 0, 0, 3580, 2142, 0, 0, 0, 0, 0, 0, 147, 0, 0, 0, 0, 91, 0, 0, 5890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1185, 0, 0, 0, 2143, 446, 1772, 832, 0, 0, 0, 0, 0, 0, 0, 0, 2080, 0, 0, 0, 4401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 3, 6054, 0, 0, 0, 0, 0, 0, 0, 0, 892, 0, 0, 1, 569, 188, 252, 2, 2540, 1540, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6012, 5959, 6062, 6128, 6378, 6481, 6148, 5984] avg=6144.0000 max=6481 max_rank=5 min=5959 min_rank=1 imbalance_ratio=1.054850 max_load_reduction=42671 max_load_reduction_pct=0.868144 moved_to_echo=24991 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[7424, 128, 0, 128, 4864, 1408, 2048, 128, 20480, 13184, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49280 observed_loss=-128 tokens_per_expert=[9344, 640, 2816, 0, 0, 0, 0, 0, 26752, 9728, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 0, 7808, 0, 0, 128, 16640, 3072, 15232, 6784, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 16768, 0, 0, 128, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[27520, 0, 128, 768, 5888, 15232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1280, 0, 0, 128, 128, 768, 0, 128, 47360, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 0, 768, 0, 0, 128, 48512, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[0, 768, 1152, 128, 0, 768, 128, 0, 29696, 17280, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=17 tokens_per_expert_current_ep_rank=[1239, 864, 0, 0, 0, 0, 3160, 0, 0, 2978, 0, 12, 0, 0, 381, 0, 0, 155, 0, 0, 0, 608, 0, 0, 0, 0, 7090, 251, 1516, 0, 196, 4347, 0, 0, 0, 0, 0, 0, 0, 2283, 0, 0, 1936, 115, 7206, 78, 6681, 0, 6898, 0, 0, 161, 0, 77, 0, 0, 11, 0, 0, 0, 643, 76, 0, 190] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=17 tokens_per_expert_current_ep_rank=[1323, 789, 0, 0, 0, 0, 2702, 0, 0, 3250, 0, 34, 0, 0, 378, 0, 0, 144, 0, 0, 0, 618, 0, 0, 0, 0, 7051, 258, 1639, 0, 194, 3789, 0, 0, 0, 0, 0, 0, 0, 2780, 0, 0, 1720, 154, 7211, 72, 6852, 0, 7067, 0, 0, 154, 0, 72, 0, 0, 57, 0, 0, 0, 617, 71, 0, 156] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=17 tokens_per_expert_current_ep_rank=[1703, 845, 0, 0, 0, 0, 2179, 0, 0, 3036, 0, 78, 0, 0, 324, 0, 0, 169, 0, 0, 0, 576, 0, 0, 0, 0, 6976, 240, 1725, 0, 230, 3157, 0, 0, 0, 0, 0, 0, 0, 3502, 0, 0, 1461, 145, 7208, 90, 7052, 0, 7067, 0, 0, 169, 0, 84, 0, 0, 88, 0, 0, 0, 741, 84, 0, 223] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=17 tokens_per_expert_current_ep_rank=[1492, 1053, 0, 0, 0, 0, 2265, 0, 0, 3086, 0, 95, 1, 0, 348, 0, 1, 214, 2, 0, 1, 554, 0, 1, 0, 0, 6902, 245, 1511, 1, 161, 3569, 2, 0, 0, 0, 1, 0, 0, 3114, 1, 2, 1605, 86, 7314, 107, 6903, 1, 6943, 1, 0, 198, 0, 107, 0, 0, 49, 0, 0, 1, 719, 107, 0, 389] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=17 tokens_per_expert_current_ep_rank=[1384, 813, 0, 0, 0, 0, 2956, 0, 0, 3175, 0, 51, 0, 0, 412, 0, 0, 154, 0, 0, 0, 578, 0, 0, 0, 0, 6968, 248, 1539, 0, 214, 3456, 0, 0, 0, 0, 0, 0, 0, 2875, 0, 0, 1728, 141, 7195, 77, 6830, 0, 6996, 0, 0, 178, 0, 77, 0, 0, 65, 0, 0, 0, 797, 76, 0, 169] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=17 tokens_per_expert_current_ep_rank=[1155, 834, 0, 0, 0, 0, 3421, 0, 0, 2587, 0, 85, 0, 0, 188, 0, 0, 129, 0, 0, 0, 593, 0, 0, 0, 0, 7064, 244, 1164, 0, 149, 4457, 0, 0, 0, 0, 0, 0, 0, 2384, 0, 0, 2003, 129, 7311, 65, 6890, 0, 7058, 0, 0, 169, 0, 64, 0, 0, 0, 0, 0, 0, 776, 57, 0, 176] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=17 tokens_per_expert_current_ep_rank=[1808, 921, 0, 0, 0, 0, 2162, 0, 0, 3183, 0, 62, 0, 0, 334, 0, 0, 182, 1, 0, 0, 528, 0, 0, 0, 0, 7020, 252, 1661, 0, 219, 3060, 0, 0, 0, 0, 0, 0, 0, 3045, 0, 0, 1609, 160, 7225, 92, 7019, 0, 7097, 0, 0, 188, 0, 91, 0, 0, 91, 0, 0, 0, 777, 90, 0, 275] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=17 tokens_per_expert_current_ep_rank=[1165, 1065, 0, 0, 0, 0, 2903, 0, 0, 2975, 0, 15, 0, 0, 474, 0, 0, 147, 0, 0, 0, 553, 0, 0, 0, 0, 7117, 244, 1204, 0, 163, 4341, 0, 0, 0, 0, 0, 0, 0, 2711, 0, 0, 2149, 137, 7204, 75, 6498, 0, 6879, 0, 0, 187, 0, 73, 0, 0, 9, 0, 0, 0, 587, 71, 0, 206] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=17 rerouted_tokens_per_expert_current_ep_rank=[1239, 864, 0, 0, 0, 0, 3160, 0, 1096, 5, 0, 0, 0, 0, 0, 0, 0, 2978, 0, 12, 0, 0, 381, 0, 2620, 18, 0, 0, 0, 0, 0, 0, 0, 155, 0, 0, 0, 608, 0, 0, 5384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 444, 251, 1516, 0, 196, 4347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2283, 3338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1936, 115, 0, 78, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5802, 0, 0, 161, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 643, 76, 0, 190, 3290, 1822, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6364, 6009, 6147, 6754, 5621, 6185, 6040, 6032] avg=6144.0000 max=6754 max_rank=3 min=5621 min_rank=4 imbalance_ratio=1.099284 max_load_reduction=42398 max_load_reduction_pct=0.862590 moved_to_echo=23878 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=17 rerouted_tokens_per_expert_current_ep_rank=[1703, 845, 0, 0, 0, 0, 2179, 0, 1123, 6, 0, 0, 0, 0, 0, 0, 0, 3036, 0, 78, 0, 0, 324, 0, 2765, 18, 0, 0, 0, 0, 0, 0, 0, 169, 0, 0, 0, 576, 0, 0, 5385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 437, 240, 1725, 0, 230, 3157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3502, 3284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1461, 145, 0, 90, 4281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5944, 0, 0, 169, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 741, 84, 0, 223, 3237, 1823, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5856, 6221, 6130, 5789, 6786, 5977, 6197, 6196] avg=6144.0000 max=6786 max_rank=4 min=5789 min_rank=3 imbalance_ratio=1.104492 max_load_reduction=42366 max_load_reduction_pct=0.861938 moved_to_echo=25156 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=17 rerouted_tokens_per_expert_current_ep_rank=[1492, 1053, 0, 0, 0, 0, 2265, 0, 1103, 6, 0, 0, 0, 0, 0, 0, 0, 3086, 0, 95, 1, 0, 348, 0, 2707, 18, 0, 0, 0, 0, 0, 0, 1, 214, 2, 0, 1, 554, 0, 1, 5465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 432, 245, 1511, 1, 161, 3569, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 3114, 3249, 0, 0, 0, 0, 0, 0, 0, 1, 2, 1605, 86, 0, 107, 4190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 5840, 1, 0, 198, 0, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 1, 719, 107, 0, 389, 3203, 1849, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5919, 6255, 6238, 5919, 6366, 5992, 6146, 6317] avg=6144.0000 max=6366 max_rank=4 min=5919 min_rank=0 imbalance_ratio=1.036133 max_load_reduction=42786 max_load_reduction_pct=0.870483 moved_to_echo=24821 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=17 rerouted_tokens_per_expert_current_ep_rank=[1323, 789, 0, 0, 0, 0, 2702, 0, 1123, 6, 0, 0, 0, 0, 0, 0, 0, 3250, 0, 34, 0, 0, 378, 0, 2687, 18, 0, 0, 0, 0, 0, 0, 0, 144, 0, 0, 0, 618, 0, 0, 5389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 442, 258, 1639, 0, 194, 3789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2780, 3319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1720, 154, 0, 72, 4159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5944, 0, 0, 154, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 617, 71, 0, 156, 3272, 1822, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5943, 6367, 6151, 6322, 6099, 6105, 6170, 5995] avg=6144.0000 max=6367 max_rank=1 min=5943 min_rank=0 imbalance_ratio=1.036296 max_load_reduction=42785 max_load_reduction_pct=0.870463 moved_to_echo=24369 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=17 rerouted_tokens_per_expert_current_ep_rank=[1155, 834, 0, 0, 0, 0, 3421, 0, 1121, 6, 0, 0, 0, 0, 0, 0, 0, 2587, 0, 85, 0, 0, 188, 0, 2702, 18, 0, 0, 0, 0, 0, 0, 0, 129, 0, 0, 0, 593, 0, 0, 5463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 443, 244, 1164, 0, 149, 4457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2384, 3325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2003, 129, 0, 65, 4182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5937, 0, 0, 169, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 776, 57, 0, 176, 3278, 1848, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6537, 5580, 6185, 6457, 5709, 6379, 6170, 6135] avg=6144.0000 max=6537 max_rank=0 min=5580 min_rank=1 imbalance_ratio=1.063965 max_load_reduction=42615 max_load_reduction_pct=0.867004 moved_to_echo=24393 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=17 rerouted_tokens_per_expert_current_ep_rank=[1384, 813, 0, 0, 0, 0, 2956, 0, 1116, 8, 0, 0, 0, 0, 0, 0, 0, 3175, 0, 51, 0, 0, 412, 0, 2682, 21, 0, 0, 0, 0, 0, 0, 0, 154, 0, 0, 0, 578, 0, 0, 5377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 425, 248, 1539, 0, 214, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2875, 3285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1728, 141, 0, 77, 4140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5880, 0, 0, 178, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 797, 76, 0, 169, 3237, 1818, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6277, 6341, 6109, 5882, 6160, 6086, 6135, 6162] avg=6144.0000 max=6341 max_rank=1 min=5882 min_rank=3 imbalance_ratio=1.032064 max_load_reduction=42811 max_load_reduction_pct=0.870992 moved_to_echo=24543 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=17 rerouted_tokens_per_expert_current_ep_rank=[1165, 1065, 0, 0, 0, 0, 2903, 0, 1093, 5, 0, 0, 0, 0, 0, 0, 0, 2975, 0, 15, 0, 0, 474, 0, 2548, 18, 0, 0, 0, 0, 0, 0, 0, 147, 0, 0, 0, 553, 0, 0, 5383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 446, 244, 1204, 0, 163, 4341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2711, 3350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2149, 137, 0, 75, 3945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5786, 0, 0, 187, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 587, 71, 0, 206, 3303, 1821, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6231, 6030, 6083, 6398, 6061, 6306, 6046, 5997] avg=6144.0000 max=6398 max_rank=3 min=5997 min_rank=7 imbalance_ratio=1.041341 max_load_reduction=42754 max_load_reduction_pct=0.869832 moved_to_echo=24410 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=17 rerouted_tokens_per_expert_current_ep_rank=[1808, 921, 0, 0, 0, 0, 2162, 0, 1128, 6, 0, 0, 0, 0, 0, 0, 0, 3183, 0, 62, 0, 0, 334, 0, 2752, 18, 0, 0, 0, 0, 0, 0, 0, 182, 1, 0, 0, 528, 0, 0, 5398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 439, 252, 1661, 0, 219, 3060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3045, 3305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1609, 160, 0, 92, 4261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5969, 0, 0, 188, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 777, 90, 0, 275, 3258, 1827, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6025, 6349, 6109, 5631, 6350, 6122, 6248, 6318] avg=6144.0000 max=6350 max_rank=4 min=5631 min_rank=3 imbalance_ratio=1.033529 max_load_reduction=42802 max_load_reduction_pct=0.870809 moved_to_echo=25038 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[384, 0, 0, 128, 5760, 640, 0, 1792, 26112, 14720, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[11392, 7296, 0, 0, 0, 0, 21760, 0, 8960, 128, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 128, 0, 0, 22784, 26496, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 24320, 0, 512, 128, 0, 2944, 0, 21504, 256, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 14336, 1152, 0, 768, 33280, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 3584, 2048, 12032, 128, 1536, 30208, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[47104, 128, 0, 1408, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 1408, 128, 0, 128, 4608, 0, 128, 43264, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 1281, 0, 0, 0, 1674, 0, 0, 0, 114, 79, 7354, 0, 0, 7633, 65, 0, 0, 0, 0, 1335, 0, 0, 0, 0, 0, 8083, 0, 0, 7656, 0, 0, 0, 64, 0, 0, 7890, 0, 0, 0, 0, 317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 244, 0, 65, 0, 1832, 0, 0, 448, 3018, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 833, 0, 0, 0, 1443, 0, 0, 0, 141, 88, 7347, 0, 0, 7595, 85, 0, 0, 0, 0, 3313, 0, 0, 0, 0, 0, 8108, 0, 0, 7492, 0, 0, 0, 84, 0, 0, 7735, 0, 0, 0, 0, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 0, 85, 0, 2503, 0, 0, 502, 1153, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 894, 0, 0, 0, 1583, 0, 0, 0, 144, 49, 7293, 0, 1, 7534, 107, 0, 0, 1, 0, 2720, 0, 1, 1, 0, 0, 8082, 0, 2, 7550, 0, 0, 0, 107, 0, 0, 7733, 2, 0, 1, 0, 420, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 245, 0, 107, 0, 2566, 0, 0, 534, 1472, 1, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 1088, 0, 0, 0, 1629, 0, 0, 0, 127, 66, 7310, 0, 0, 7580, 78, 0, 0, 0, 0, 1765, 0, 0, 0, 0, 0, 8080, 0, 0, 7637, 0, 0, 0, 77, 0, 0, 7821, 0, 0, 0, 0, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 0, 78, 0, 2180, 0, 0, 506, 2516, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 825, 0, 0, 0, 1317, 0, 0, 0, 157, 94, 7400, 0, 0, 7532, 91, 0, 0, 0, 0, 3384, 0, 0, 0, 0, 0, 8101, 0, 0, 7459, 0, 0, 0, 91, 0, 0, 7671, 0, 0, 0, 0, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 0, 91, 0, 2532, 0, 0, 459, 1234, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 1123, 0, 0, 0, 1546, 0, 0, 0, 124, 87, 7371, 0, 0, 7570, 74, 0, 0, 0, 0, 1599, 0, 0, 0, 0, 0, 8067, 0, 0, 7617, 0, 0, 0, 73, 0, 0, 7830, 0, 0, 0, 0, 385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 244, 0, 74, 0, 2121, 0, 0, 463, 2784, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 928, 0, 0, 0, 1438, 0, 0, 0, 131, 95, 7399, 0, 0, 7575, 72, 0, 0, 0, 0, 2607, 0, 0, 0, 0, 0, 8114, 0, 0, 7515, 0, 0, 0, 72, 0, 0, 7814, 0, 0, 0, 0, 362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 258, 0, 72, 0, 2249, 0, 0, 484, 1967, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 962, 0, 0, 0, 1355, 0, 0, 0, 136, 82, 7378, 0, 0, 7581, 77, 0, 0, 0, 0, 2468, 0, 0, 0, 0, 0, 8104, 0, 0, 7558, 0, 0, 0, 77, 0, 0, 7811, 0, 0, 0, 0, 369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 0, 77, 0, 2451, 0, 0, 470, 1950, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 833, 0, 0, 0, 1443, 0, 1713, 1425, 431, 71, 0, 0, 0, 0, 0, 0, 141, 88, 5922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 3313, 0, 0, 3673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 6022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 405, 0, 0, 0, 0, 0, 3491, 2281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 0, 85, 5827, 0, 0, 0, 0, 0, 0, 0, 0, 2503, 0, 0, 502, 1153, 0, 0, 1332, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5916, 6151, 7071, 6089, 6106, 6177, 6152, 5490] avg=6144.0000 max=7071 max_rank=2 min=5490 min_rank=7 imbalance_ratio=1.150879 max_load_reduction=42081 max_load_reduction_pct=0.856140 moved_to_echo=23925 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1088, 0, 0, 0, 1629, 0, 1732, 1417, 430, 72, 0, 0, 0, 0, 0, 0, 127, 66, 5893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0, 1765, 0, 0, 3666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 6089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 363, 0, 0, 0, 0, 0, 3484, 2273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 0, 78, 5807, 0, 0, 0, 0, 0, 0, 0, 0, 2180, 0, 0, 506, 2516, 0, 0, 1358, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6368, 6086, 5509, 6207, 6166, 6120, 6136, 6560] avg=6144.0000 max=6560 max_rank=7 min=5509 min_rank=2 imbalance_ratio=1.067708 max_load_reduction=42592 max_load_reduction_pct=0.866536 moved_to_echo=24982 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 894, 0, 0, 0, 1583, 0, 1712, 1414, 427, 72, 0, 0, 0, 0, 0, 0, 144, 49, 5879, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 0, 0, 1, 0, 2720, 0, 1, 3643, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 6136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 0, 0, 6021, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 420, 0, 0, 2, 0, 0, 3464, 2273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, 0, 107, 5809, 0, 0, 0, 0, 0, 0, 0, 0, 2566, 0, 0, 534, 1472, 1, 0, 1342, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6102, 6073, 6472, 6139, 6130, 6160, 6161, 5915] avg=6144.0000 max=6472 max_rank=2 min=5915 min_rank=7 imbalance_ratio=1.053385 max_load_reduction=42680 max_load_reduction_pct=0.868327 moved_to_echo=24366 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1281, 0, 0, 0, 1674, 0, 1747, 1426, 433, 73, 0, 0, 0, 0, 0, 0, 114, 79, 5928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 1335, 0, 0, 3693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 6143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 317, 0, 0, 0, 0, 0, 3507, 2275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 244, 0, 65, 5808, 0, 0, 0, 0, 0, 0, 0, 0, 1832, 0, 0, 448, 3018, 0, 0, 1361, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6634, 6121, 5093, 6222, 6207, 6099, 6117, 6659] avg=6144.0000 max=6659 max_rank=7 min=5093 min_rank=2 imbalance_ratio=1.083822 max_load_reduction=42493 max_load_reduction_pct=0.864522 moved_to_echo=25082 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 928, 0, 0, 0, 1438, 0, 1730, 1435, 431, 71, 0, 0, 0, 0, 0, 0, 131, 95, 5964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 2607, 0, 0, 3663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 6084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 362, 0, 0, 0, 0, 0, 3481, 2283, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 258, 0, 72, 5831, 0, 0, 0, 0, 0, 0, 0, 0, 2249, 0, 0, 484, 1967, 0, 0, 1336, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6033, 6190, 6342, 6108, 6156, 6126, 6161, 6036] avg=6144.0000 max=6342 max_rank=2 min=6033 min_rank=0 imbalance_ratio=1.032227 max_load_reduction=42810 max_load_reduction_pct=0.870972 moved_to_echo=24479 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 962, 0, 0, 0, 1355, 0, 1733, 1434, 431, 76, 0, 0, 0, 0, 0, 0, 136, 82, 5944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 2468, 0, 0, 3666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 6078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 369, 0, 0, 0, 0, 0, 3484, 2281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 0, 77, 5823, 0, 0, 0, 0, 0, 0, 0, 0, 2451, 0, 0, 470, 1950, 0, 0, 1348, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5991, 6162, 6211, 6134, 6155, 6134, 6146, 6219] avg=6144.0000 max=6219 max_rank=7 min=5991 min_rank=0 imbalance_ratio=1.012207 max_load_reduction=42933 max_load_reduction_pct=0.873474 moved_to_echo=24654 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1123, 0, 0, 0, 1546, 0, 1734, 1429, 430, 72, 0, 0, 0, 0, 0, 0, 124, 87, 5942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 1599, 0, 0, 3662, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 6096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 385, 0, 0, 0, 0, 0, 3478, 2270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 244, 0, 74, 5797, 0, 0, 0, 0, 0, 0, 0, 0, 2121, 0, 0, 463, 2784, 0, 0, 1354, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6334, 6153, 5335, 6191, 6169, 6133, 6115, 6722] avg=6144.0000 max=6722 max_rank=7 min=5335 min_rank=2 imbalance_ratio=1.094076 max_load_reduction=42430 max_load_reduction_pct=0.863241 moved_to_echo=25139 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 825, 0, 0, 0, 1317, 0, 1699, 1435, 427, 71, 0, 0, 0, 0, 0, 0, 157, 94, 5965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 0, 3384, 0, 0, 3644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 5972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 463, 0, 0, 0, 0, 0, 3461, 2279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 0, 91, 5822, 0, 0, 0, 0, 0, 0, 0, 0, 2532, 0, 0, 459, 1234, 0, 0, 1326, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5774, 6216, 7119, 6062, 6063, 6203, 6164, 5551] avg=6144.0000 max=7119 max_rank=2 min=5551 min_rank=7 imbalance_ratio=1.158691 max_load_reduction=42033 max_load_reduction_pct=0.855164 moved_to_echo=23981 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 7936, 0, 0, 0, 12032, 0, 13824, 11520, 3456, 640, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[768, 0, 0, 128, 0, 19200, 0, 128, 29312, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 18560, 0, 0, 3968, 16128, 128, 0, 10880, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 2048, 0, 768, 46592, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 1152, 640, 47488, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 0, 3200, 0, 0, 128, 0, 0, 27904, 18304, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[128, 0, 0, 0, 0, 128, 49152, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 768, 0, 0, 48512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 0, 0, 50, 576, 0, 2804, 0, 155, 7924, 7866, 0, 0, 0, 0, 0, 1952, 0, 7697, 0, 50, 0, 7694, 0, 1, 0, 0, 2367, 7573, 78, 0, 0, 0, 0, 0, 48, 0, 59, 0, 552, 0, 178, 0, 0, 0, 0, 0, 0, 2, 38, 91, 0, 510, 367] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 610, 0, 0, 37, 408, 0, 2551, 0, 214, 7823, 7758, 0, 1, 0, 0, 0, 1157, 0, 7683, 0, 38, 0, 7641, 1, 0, 0, 1, 3558, 7523, 107, 0, 0, 0, 1, 1, 37, 0, 66, 0, 574, 1, 201, 0, 1, 0, 0, 2, 0, 0, 8, 328, 2, 452, 364] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 589, 0, 0, 57, 305, 0, 2252, 0, 169, 7912, 7873, 0, 0, 0, 0, 0, 884, 0, 7719, 0, 57, 0, 7531, 0, 1, 0, 0, 4404, 7491, 85, 0, 0, 0, 0, 0, 57, 0, 50, 0, 614, 0, 188, 0, 0, 0, 0, 0, 0, 0, 1, 227, 0, 355, 331] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 584, 0, 0, 59, 590, 0, 2774, 0, 154, 7951, 7886, 0, 0, 0, 0, 0, 1413, 0, 7692, 0, 59, 0, 7600, 0, 1, 0, 0, 3023, 7476, 77, 0, 0, 0, 0, 0, 56, 0, 66, 0, 493, 0, 168, 0, 0, 0, 0, 0, 0, 3, 44, 122, 0, 497, 364] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 0, 0, 59, 614, 0, 2516, 0, 144, 7964, 7915, 0, 0, 0, 0, 0, 1163, 0, 7758, 0, 59, 0, 7700, 0, 0, 0, 0, 3465, 7502, 72, 0, 0, 0, 0, 0, 59, 0, 53, 0, 561, 0, 175, 0, 0, 0, 0, 0, 0, 0, 13, 77, 0, 426, 303] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 0, 0, 50, 486, 0, 2174, 0, 129, 8009, 7940, 0, 0, 0, 0, 0, 2306, 0, 7742, 0, 50, 0, 7754, 0, 2, 0, 0, 2438, 7529, 65, 0, 0, 0, 0, 0, 49, 0, 71, 0, 671, 0, 131, 0, 0, 0, 0, 0, 0, 1, 60, 146, 0, 494, 335] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 0, 0, 51, 626, 0, 2273, 0, 147, 7962, 7875, 0, 0, 0, 0, 0, 1971, 0, 7769, 0, 51, 0, 7768, 0, 1, 0, 0, 2705, 7475, 74, 0, 0, 0, 0, 0, 50, 0, 88, 0, 652, 0, 168, 0, 0, 0, 0, 0, 0, 1, 75, 82, 0, 431, 298] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 438, 0, 0, 66, 249, 0, 2175, 0, 182, 7874, 7822, 0, 0, 0, 0, 0, 838, 0, 7718, 0, 66, 0, 7569, 0, 0, 0, 0, 4577, 7498, 91, 0, 0, 0, 0, 0, 64, 0, 54, 0, 645, 0, 225, 0, 0, 0, 0, 0, 0, 0, 3, 362, 0, 319, 317] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 0, 0, 50, 576, 0, 3082, 1969, 0, 0, 0, 0, 0, 0, 2804, 0, 155, 0, 3543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1952, 0, 4615, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2367, 2769, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 59, 4264, 1783, 0, 0, 0, 0, 0, 0, 0, 552, 0, 178, 0, 0, 0, 0, 3430, 1973, 0, 0, 0, 0, 0, 0, 0, 0, 2, 38, 91, 0, 510, 367, 2831, 2354, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6141, 6197, 6502, 6617, 5215, 6154, 6133, 6193] avg=6144.0000 max=6617 max_rank=3 min=5215 min_rank=4 imbalance_ratio=1.076986 max_load_reduction=42535 max_load_reduction_pct=0.865377 moved_to_echo=23695 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 589, 0, 0, 57, 305, 0, 3091, 1970, 0, 0, 0, 0, 0, 0, 2252, 0, 169, 0, 3546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 884, 0, 4628, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4404, 2740, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 50, 4174, 1781, 0, 0, 0, 0, 0, 0, 0, 614, 0, 188, 0, 0, 0, 0, 3357, 1951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 227, 0, 355, 331, 2800, 2357, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6131, 6012, 5967, 5569, 7230, 6062, 6110, 6071] avg=6144.0000 max=7230 max_rank=4 min=5569 min_rank=3 imbalance_ratio=1.176758 max_load_reduction=41922 max_load_reduction_pct=0.852905 moved_to_echo=25473 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 2, 0, 0, 0, 6062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 610, 0, 0, 37, 408, 0, 3076, 1942, 0, 0, 0, 0, 0, 0, 2551, 0, 214, 0, 3494, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1157, 0, 4607, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 3558, 2751, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 37, 0, 66, 4235, 1761, 0, 0, 0, 0, 0, 0, 0, 574, 1, 201, 0, 1, 0, 0, 3406, 1960, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8, 328, 2, 452, 364, 2812, 2322, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6065, 6073, 6260, 5802, 6418, 6101, 6143, 6290] avg=6144.0000 max=6418 max_rank=4 min=5802 min_rank=3 imbalance_ratio=1.044596 max_load_reduction=42734 max_load_reduction_pct=0.869425 moved_to_echo=24952 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 0, 0, 50, 486, 0, 3100, 1987, 0, 0, 0, 0, 0, 0, 2174, 0, 129, 0, 3576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2306, 0, 4642, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2438, 2753, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 71, 4298, 1802, 0, 0, 0, 0, 0, 0, 0, 671, 0, 131, 0, 0, 0, 0, 3456, 1961, 0, 0, 0, 0, 0, 0, 0, 0, 1, 60, 146, 0, 494, 335, 2815, 2377, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6207, 6143, 5879, 6998, 5258, 6220, 6219, 6228] avg=6144.0000 max=6998 max_rank=3 min=5258 min_rank=4 imbalance_ratio=1.138997 max_load_reduction=42154 max_load_reduction_pct=0.857625 moved_to_echo=23925 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 0, 0, 59, 614, 0, 3106, 1981, 0, 0, 0, 0, 0, 0, 2516, 0, 144, 0, 3565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1163, 0, 4652, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3465, 2744, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 53, 4268, 1791, 0, 0, 0, 0, 0, 0, 0, 561, 0, 175, 0, 0, 0, 0, 3432, 1954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 77, 0, 426, 303, 2804, 2369, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6173, 6314, 6225, 5874, 6281, 6171, 6122, 5992] avg=6144.0000 max=6314 max_rank=1 min=5874 min_rank=3 imbalance_ratio=1.027669 max_load_reduction=42838 max_load_reduction_pct=0.871541 moved_to_echo=24566 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 584, 0, 0, 59, 590, 0, 3084, 1977, 0, 0, 0, 0, 0, 0, 2774, 0, 154, 0, 3544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1413, 0, 4608, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3023, 2725, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 66, 4213, 1789, 0, 0, 0, 0, 0, 0, 0, 493, 0, 168, 0, 0, 0, 0, 3387, 1951, 0, 0, 0, 0, 0, 0, 0, 0, 3, 44, 122, 0, 497, 364, 2800, 2365, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6162, 6294, 6472, 6080, 5826, 6124, 5999, 6195] avg=6144.0000 max=6472 max_rank=2 min=5826 min_rank=4 imbalance_ratio=1.053385 max_load_reduction=42680 max_load_reduction_pct=0.868327 moved_to_echo=24144 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 0, 0, 51, 626, 0, 3111, 1971, 0, 0, 0, 0, 0, 0, 2273, 0, 147, 0, 3547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1971, 0, 4658, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2705, 2734, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 88, 4306, 1791, 0, 0, 0, 0, 0, 0, 0, 652, 0, 168, 0, 0, 0, 0, 3462, 1947, 0, 0, 0, 0, 0, 0, 0, 0, 1, 75, 82, 0, 431, 298, 2794, 2357, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6171, 6318, 5967, 6680, 5514, 6235, 6229, 6038] avg=6144.0000 max=6680 max_rank=3 min=5514 min_rank=4 imbalance_ratio=1.087240 max_load_reduction=42472 max_load_reduction_pct=0.864095 moved_to_echo=24016 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 6102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 438, 0, 0, 66, 249, 0, 3090, 1958, 0, 0, 0, 0, 0, 0, 2175, 0, 182, 0, 3523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 838, 0, 4628, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4577, 2742, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 54, 4195, 1772, 0, 0, 0, 0, 0, 0, 0, 645, 0, 225, 0, 0, 0, 0, 3374, 1953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 362, 0, 319, 317, 2803, 2341, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6102, 5801, 5880, 5532, 7410, 6085, 6197, 6145] avg=6144.0000 max=7410 max_rank=4 min=5532 min_rank=3 imbalance_ratio=1.206055 max_load_reduction=41742 max_load_reduction_pct=0.849243 moved_to_echo=25837 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 0, 128, 256, 1536, 128, 3584, 2688, 22528, 18944, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 4480, 0, 0, 512, 3968, 0, 24832, 15872, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 128, 26624, 22016, 768, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[19584, 0, 1408, 0, 28416, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 11776, 0, 37120, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 4864, 128, 1536, 0, 128, 0, 0, 27392, 15744, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 128, 512, 0, 512, 34048, 14336, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 128, 0, 128, 0, 0, 0, 49152, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 1129, 0, 0, 0, 0, 0, 0, 0, 78, 0, 78, 1195, 0, 0, 0, 416, 1818, 8113, 941, 0, 54, 7624, 0, 0, 7551, 153, 50, 0, 0, 0, 0, 80, 0, 0, 0, 0, 1034, 0, 5760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1944, 6752, 0, 1034, 152, 2812, 77, 0, 0, 0, 222, 85, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 1127, 0, 0, 0, 0, 0, 0, 0, 84, 0, 84, 1396, 0, 0, 0, 663, 2291, 8103, 1022, 0, 84, 7764, 0, 0, 7582, 157, 57, 0, 0, 0, 0, 84, 0, 0, 0, 0, 1436, 0, 4295, 0, 0, 0, 0, 79, 0, 0, 0, 0, 2743, 6643, 0, 471, 157, 2466, 84, 0, 0, 0, 278, 2, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 1251, 0, 0, 0, 0, 0, 0, 0, 91, 0, 90, 1161, 0, 0, 0, 637, 2351, 8100, 972, 0, 90, 7880, 0, 0, 7573, 201, 66, 0, 0, 0, 0, 91, 0, 0, 0, 0, 1392, 0, 4245, 0, 0, 0, 0, 86, 0, 0, 0, 0, 3046, 6616, 0, 336, 200, 2309, 91, 0, 0, 0, 277, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 1, 1339, 0, 0, 0, 0, 0, 0, 0, 107, 0, 107, 933, 0, 0, 0, 739, 2198, 8079, 920, 1, 107, 7852, 1, 0, 7505, 191, 37, 0, 0, 0, 2, 109, 2, 1, 0, 0, 976, 1, 4848, 0, 2, 0, 0, 29, 0, 0, 0, 0, 2384, 6677, 0, 693, 188, 2718, 108, 0, 0, 2, 246, 49, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 779, 0, 0, 0, 0, 0, 0, 0, 62, 0, 65, 1329, 0, 0, 0, 238, 1928, 8075, 911, 0, 27, 7686, 0, 0, 7587, 84, 50, 0, 0, 0, 0, 70, 0, 0, 0, 0, 1132, 0, 5944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1894, 6723, 0, 1075, 87, 3088, 64, 0, 0, 0, 218, 36, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 1156, 0, 0, 0, 0, 0, 0, 0, 77, 0, 77, 1400, 0, 0, 0, 419, 1934, 8114, 1029, 0, 64, 7635, 0, 0, 7545, 145, 59, 0, 0, 0, 0, 77, 0, 0, 0, 0, 951, 0, 5271, 0, 0, 0, 0, 19, 0, 0, 0, 0, 2596, 6604, 0, 707, 141, 2753, 77, 0, 0, 0, 256, 46, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 1158, 0, 0, 0, 0, 0, 0, 0, 72, 0, 71, 1276, 0, 0, 0, 495, 2193, 8120, 962, 0, 68, 7706, 0, 0, 7633, 137, 59, 0, 0, 0, 0, 72, 0, 0, 0, 0, 954, 0, 5064, 0, 0, 0, 0, 14, 0, 0, 0, 0, 2608, 6766, 0, 697, 139, 2527, 72, 0, 0, 0, 258, 31, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 1066, 0, 0, 0, 0, 0, 0, 0, 74, 0, 74, 1461, 0, 0, 0, 330, 2082, 8117, 856, 0, 48, 7712, 0, 0, 7607, 131, 51, 0, 0, 0, 0, 73, 0, 0, 0, 0, 1352, 0, 5261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1897, 6552, 0, 958, 129, 2980, 73, 0, 0, 0, 256, 12, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1129, 0, 0, 0, 0, 4947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 78, 1195, 0, 3380, 1355, 0, 0, 0, 0, 0, 0, 0, 0, 416, 1818, 2537, 941, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5924, 153, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 1034, 2677, 2196, 0, 0, 0, 0, 0, 0, 0, 5760, 0, 0, 0, 0, 0, 0, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1944, 2897, 0, 1034, 152, 0, 0, 0, 0, 0, 0, 0, 0, 2812, 77, 0, 0, 0, 222, 85, 0, 2500, 601, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6076, 6086, 5766, 6127, 5987, 6786, 6027, 6297] avg=6144.0000 max=6786 max_rank=5 min=5766 min_rank=2 imbalance_ratio=1.104492 max_load_reduction=42366 max_load_reduction_pct=0.861938 moved_to_echo=25097 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1127, 0, 0, 0, 0, 5038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 84, 1396, 0, 3375, 1333, 0, 0, 0, 0, 0, 0, 0, 0, 663, 2291, 2535, 1022, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5949, 157, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 1436, 2726, 2193, 0, 0, 0, 0, 0, 0, 0, 4295, 0, 0, 0, 0, 79, 0, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2743, 2850, 0, 471, 157, 0, 0, 0, 0, 0, 0, 0, 0, 2466, 84, 0, 0, 0, 278, 2, 0, 2460, 603, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6165, 6272, 6595, 6163, 6439, 5404, 6221, 5893] avg=6144.0000 max=6595 max_rank=2 min=5404 min_rank=5 imbalance_ratio=1.073405 max_load_reduction=42557 max_load_reduction_pct=0.865824 moved_to_echo=23957 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 779, 0, 0, 0, 0, 4988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 65, 1329, 0, 3364, 1349, 0, 0, 0, 0, 0, 0, 0, 0, 238, 1928, 2525, 911, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5952, 84, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 1132, 2698, 2186, 0, 0, 0, 0, 0, 0, 0, 5944, 0, 0, 0, 0, 0, 0, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1894, 2884, 0, 1075, 87, 0, 0, 0, 0, 0, 0, 0, 0, 3088, 64, 0, 0, 0, 218, 36, 0, 2490, 604, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5767, 6169, 5629, 6086, 6086, 6975, 5940, 6500] avg=6144.0000 max=6975 max_rank=5 min=5629 min_rank=2 imbalance_ratio=1.135254 max_load_reduction=42177 max_load_reduction_pct=0.858093 moved_to_echo=25501 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 1339, 0, 0, 0, 0, 5095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 0, 107, 933, 0, 3365, 1340, 0, 0, 0, 0, 0, 0, 0, 0, 739, 2198, 2527, 920, 1, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5888, 191, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 109, 2, 1, 0, 0, 976, 2757, 2187, 0, 0, 0, 0, 0, 0, 1, 4848, 0, 2, 0, 0, 29, 0, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2384, 2865, 0, 693, 188, 0, 0, 0, 0, 0, 0, 0, 0, 2718, 108, 0, 0, 2, 246, 49, 0, 2472, 597, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6435, 5852, 6492, 6117, 6034, 5900, 6130, 6192] avg=6144.0000 max=6492 max_rank=2 min=5852 min_rank=1 imbalance_ratio=1.056641 max_load_reduction=42660 max_load_reduction_pct=0.867920 moved_to_echo=24256 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1158, 0, 0, 0, 0, 5002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 71, 1276, 0, 3382, 1357, 0, 0, 0, 0, 0, 0, 0, 0, 495, 2193, 2540, 962, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5988, 137, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 954, 2704, 2198, 0, 0, 0, 0, 0, 0, 0, 5064, 0, 0, 0, 0, 14, 0, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2608, 2904, 0, 697, 139, 0, 0, 0, 0, 0, 0, 0, 0, 2527, 72, 0, 0, 0, 258, 31, 0, 2505, 608, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6160, 6158, 6258, 6184, 5928, 6115, 6348, 6001] avg=6144.0000 max=6348 max_rank=6 min=5928 min_rank=4 imbalance_ratio=1.033203 max_load_reduction=42804 max_load_reduction_pct=0.870850 moved_to_echo=24392 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1066, 0, 0, 0, 0, 5006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 74, 1461, 0, 3381, 1314, 0, 0, 0, 0, 0, 0, 0, 0, 330, 2082, 2539, 856, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5969, 131, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 1352, 2706, 2197, 0, 0, 0, 0, 0, 0, 0, 5261, 0, 0, 0, 0, 0, 0, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1897, 2812, 0, 958, 129, 0, 0, 0, 0, 0, 0, 0, 0, 2980, 73, 0, 0, 0, 256, 12, 0, 2426, 605, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6072, 6304, 5855, 6151, 6328, 6294, 5796, 6352] avg=6144.0000 max=6352 max_rank=7 min=5796 min_rank=6 imbalance_ratio=1.033854 max_load_reduction=42800 max_load_reduction_pct=0.870768 moved_to_echo=24770 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1156, 0, 0, 0, 0, 4956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 77, 1400, 0, 3385, 1329, 0, 0, 0, 0, 0, 0, 0, 0, 419, 1934, 2528, 1029, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5912, 145, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 951, 2679, 2201, 0, 0, 0, 0, 0, 0, 0, 5271, 0, 0, 0, 0, 19, 0, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2596, 2825, 0, 707, 141, 0, 0, 0, 0, 0, 0, 0, 0, 2753, 77, 0, 0, 0, 256, 46, 0, 2450, 605, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6112, 6268, 5974, 6116, 5908, 6318, 6269, 6187] avg=6144.0000 max=6318 max_rank=5 min=5908 min_rank=4 imbalance_ratio=1.028320 max_load_reduction=42834 max_load_reduction_pct=0.871460 moved_to_echo=24682 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1251, 0, 0, 0, 0, 5114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 90, 1161, 0, 3374, 1327, 0, 0, 0, 0, 0, 0, 0, 0, 637, 2351, 2533, 972, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5941, 201, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 0, 1392, 2766, 2193, 0, 0, 0, 0, 0, 0, 0, 4245, 0, 0, 0, 0, 86, 0, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3046, 2839, 0, 336, 200, 0, 0, 0, 0, 0, 0, 0, 0, 2309, 91, 0, 0, 0, 277, 0, 0, 2450, 603, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6365, 6043, 6583, 6208, 6442, 5360, 6421, 5730] avg=6144.0000 max=6583 max_rank=2 min=5360 min_rank=5 imbalance_ratio=1.071452 max_load_reduction=42569 max_load_reduction_pct=0.866069 moved_to_echo=23953 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 0, 768, 0, 768, 10240, 0, 27008, 10752, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 3968, 16896, 20352, 7680, 128, 640, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 0, 19200, 22912, 0, 6016, 1280, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[0, 128, 768, 128, 128, 0, 0, 9344, 21760, 17664, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[21760, 768, 0, 0, 128, 2048, 384, 0, 19840, 4864, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 128, 9088, 0, 0, 0, 0, 40192, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 40704, 0, 128, 0, 0, 256, 0, 8320, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 128, 0, 47616, 1280, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=21 tokens_per_expert_current_ep_rank=[0, 4914, 411, 1397, 8001, 609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 802, 0, 0, 578, 0, 0, 0, 0, 283, 6811, 86, 0, 0, 0, 0, 0, 2223, 0, 0, 5552, 7140, 57, 0, 0, 247, 0, 0, 0, 0, 0, 0, 819, 0, 0, 0, 0, 88, 7840, 84, 1195, 0, 0, 15, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=21 tokens_per_expert_current_ep_rank=[0, 4130, 394, 1460, 8006, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 380, 0, 0, 649, 0, 0, 0, 0, 308, 7019, 77, 0, 0, 0, 0, 0, 3624, 0, 0, 3507, 7493, 50, 0, 0, 462, 0, 0, 0, 0, 0, 0, 784, 0, 0, 0, 0, 66, 7870, 77, 2256, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=21 tokens_per_expert_current_ep_rank=[0, 4362, 405, 1611, 7990, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 0, 0, 623, 0, 0, 0, 0, 253, 6873, 77, 0, 0, 0, 0, 0, 2872, 0, 0, 4328, 7438, 59, 0, 0, 402, 0, 0, 0, 0, 0, 0, 879, 0, 0, 0, 0, 82, 7858, 77, 1821, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=21 tokens_per_expert_current_ep_rank=[0, 4041, 403, 1408, 7980, 574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 823, 0, 0, 716, 0, 0, 0, 0, 269, 6885, 73, 0, 0, 0, 0, 0, 3526, 0, 0, 4387, 7525, 51, 0, 0, 226, 0, 0, 0, 0, 0, 0, 671, 0, 0, 0, 0, 86, 7886, 73, 1549, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=21 tokens_per_expert_current_ep_rank=[0, 4727, 435, 1405, 7981, 590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 667, 0, 0, 581, 0, 0, 0, 0, 310, 6686, 103, 0, 0, 0, 0, 0, 2231, 0, 0, 5545, 7215, 66, 0, 0, 431, 0, 0, 0, 0, 0, 0, 817, 0, 0, 0, 0, 94, 7784, 91, 1328, 0, 0, 65, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=21 tokens_per_expert_current_ep_rank=[0, 4339, 425, 1310, 8008, 570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 635, 0, 0, 622, 0, 0, 0, 0, 269, 6931, 72, 0, 0, 0, 0, 0, 2763, 0, 0, 5010, 7422, 59, 0, 0, 327, 0, 0, 0, 0, 0, 0, 744, 0, 0, 0, 0, 95, 7852, 72, 1627, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=21 tokens_per_expert_current_ep_rank=[0, 4673, 406, 1500, 7978, 636, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 2, 0, 597, 0, 1, 572, 0, 0, 0, 0, 309, 6892, 104, 0, 1, 0, 0, 0, 2404, 0, 0, 4957, 7234, 39, 0, 0, 313, 0, 2, 1, 0, 1, 0, 1005, 1, 0, 0, 2, 49, 7795, 107, 1562, 0, 2, 3, 0, 0, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=21 tokens_per_expert_current_ep_rank=[0, 3979, 387, 1354, 8007, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 798, 0, 0, 747, 0, 0, 0, 0, 223, 6973, 64, 0, 0, 0, 0, 0, 3729, 0, 0, 3586, 7635, 50, 0, 0, 145, 0, 0, 0, 0, 0, 0, 691, 0, 0, 0, 0, 79, 7933, 64, 2181, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 4157, 411, 1397, 0, 609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 802, 0, 0, 578, 0, 0, 3299, 1442, 0, 0, 0, 0, 0, 0, 0, 0, 283, 5723, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2223, 0, 0, 3817, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 0, 0, 0, 0, 0, 3841, 1852, 0, 0, 0, 0, 0, 0, 0, 819, 0, 0, 0, 0, 88, 5255, 0, 0, 0, 0, 0, 0, 0, 0, 84, 1195, 0, 0, 15, 0, 0, 0, 1735, 1143, 1088, 757, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6574, 6149, 6121, 6092, 6097, 5940, 6162, 6017] avg=6144.0000 max=6574 max_rank=0 min=5940 min_rank=5 imbalance_ratio=1.069987 max_load_reduction=42578 max_load_reduction_pct=0.866252 moved_to_echo=24216 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 3494, 394, 1460, 0, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 380, 0, 0, 649, 0, 0, 3462, 1447, 0, 0, 0, 0, 0, 0, 0, 0, 308, 5898, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3624, 0, 0, 2412, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 0, 0, 0, 0, 0, 4031, 1854, 0, 0, 0, 0, 0, 0, 0, 784, 0, 0, 0, 0, 66, 5276, 0, 0, 0, 0, 0, 0, 0, 0, 77, 2256, 0, 0, 0, 0, 0, 0, 1095, 1147, 1121, 636, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5888, 6152, 5938, 6283, 6086, 6347, 6126, 6332] avg=6144.0000 max=6347 max_rank=5 min=5888 min_rank=0 imbalance_ratio=1.033040 max_load_reduction=42805 max_load_reduction_pct=0.870870 moved_to_echo=24891 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 3954, 406, 1500, 0, 636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 6131, 0, 0, 0, 0, 0, 0, 0, 2, 0, 597, 0, 1, 572, 0, 0, 3342, 1433, 0, 0, 0, 0, 0, 0, 0, 0, 309, 5791, 104, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2404, 0, 0, 3408, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 313, 0, 2, 1, 0, 1, 3892, 1847, 0, 0, 0, 0, 0, 0, 0, 1005, 1, 0, 0, 2, 49, 5226, 0, 0, 0, 0, 0, 0, 0, 0, 107, 1562, 0, 2, 3, 0, 0, 1, 1549, 1136, 1101, 719, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6496, 6134, 5947, 6205, 5851, 6056, 6283, 6180] avg=6144.0000 max=6496 max_rank=0 min=5851 min_rank=4 imbalance_ratio=1.057292 max_load_reduction=42656 max_load_reduction_pct=0.867839 moved_to_echo=24370 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 3687, 405, 1611, 0, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 0, 0, 623, 0, 0, 3437, 1450, 0, 0, 0, 0, 0, 0, 0, 0, 253, 5773, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2872, 0, 0, 2971, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 402, 0, 0, 0, 0, 0, 4001, 1849, 0, 0, 0, 0, 0, 0, 0, 879, 0, 0, 0, 0, 82, 5259, 0, 0, 0, 0, 0, 0, 0, 0, 77, 1821, 0, 0, 0, 0, 0, 0, 1357, 1149, 1100, 675, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6351, 6141, 6004, 6103, 5902, 6252, 6220, 6179] avg=6144.0000 max=6351 max_rank=0 min=5902 min_rank=4 imbalance_ratio=1.033691 max_load_reduction=42801 max_load_reduction_pct=0.870789 moved_to_echo=24553 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 3366, 387, 1354, 0, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 798, 0, 0, 747, 0, 0, 3528, 1459, 0, 0, 0, 0, 0, 0, 0, 0, 223, 5859, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3729, 0, 0, 2466, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 0, 0, 0, 0, 4107, 1853, 0, 0, 0, 0, 0, 0, 0, 691, 0, 0, 0, 0, 79, 5317, 0, 0, 0, 0, 0, 0, 0, 0, 64, 2181, 0, 0, 0, 0, 0, 0, 1120, 1157, 1114, 613, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5634, 6154, 6532, 6146, 6245, 6105, 6087, 6249] avg=6144.0000 max=6532 max_rank=2 min=5634 min_rank=0 imbalance_ratio=1.063151 max_load_reduction=42620 max_load_reduction_pct=0.867106 moved_to_echo=24686 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 3419, 403, 1408, 0, 574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 823, 0, 0, 716, 0, 0, 3478, 1450, 0, 0, 0, 0, 0, 0, 0, 0, 269, 5785, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3526, 0, 0, 3017, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 226, 0, 0, 0, 0, 0, 4047, 1847, 0, 0, 0, 0, 0, 0, 0, 671, 0, 0, 0, 0, 86, 5286, 0, 0, 0, 0, 0, 0, 0, 0, 73, 1549, 0, 0, 0, 0, 0, 0, 1370, 1150, 1100, 622, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5804, 6133, 6467, 6127, 6594, 6120, 6043, 5864] avg=6144.0000 max=6594 max_rank=4 min=5804 min_rank=0 imbalance_ratio=1.073242 max_load_reduction=42558 max_load_reduction_pct=0.865845 moved_to_echo=24621 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 3671, 425, 1310, 0, 570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 635, 0, 0, 622, 0, 0, 3430, 1444, 0, 0, 0, 0, 0, 0, 0, 0, 269, 5824, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2763, 0, 0, 3445, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 327, 0, 0, 0, 0, 0, 3992, 1853, 0, 0, 0, 0, 0, 0, 0, 744, 0, 0, 0, 0, 95, 5263, 0, 0, 0, 0, 0, 0, 0, 0, 72, 1627, 0, 0, 0, 0, 0, 0, 1565, 1145, 1107, 668, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5976, 6155, 6131, 6165, 6267, 6172, 6102, 6184] avg=6144.0000 max=6267 max_rank=4 min=5976 min_rank=0 imbalance_ratio=1.020020 max_load_reduction=42885 max_load_reduction_pct=0.872498 moved_to_echo=24725 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 3999, 435, 1405, 0, 590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 667, 0, 0, 581, 0, 0, 3333, 1431, 0, 0, 0, 0, 0, 0, 0, 0, 310, 5618, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2231, 0, 0, 3813, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 431, 0, 0, 0, 0, 0, 3882, 1847, 0, 0, 0, 0, 0, 0, 0, 817, 0, 0, 0, 0, 94, 5218, 0, 0, 0, 0, 0, 0, 0, 0, 91, 1328, 0, 0, 65, 0, 0, 0, 1732, 1135, 1068, 728, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6429, 6134, 6012, 6031, 6110, 6160, 6129, 6147] avg=6144.0000 max=6429 max_rank=0 min=6012 min_rank=2 imbalance_ratio=1.046387 max_load_reduction=42723 max_load_reduction_pct=0.869202 moved_to_echo=24546 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[768, 13568, 0, 128, 128, 0, 0, 128, 11648, 9216, 8832, 5504, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 0, 5248, 0, 128, 5120, 0, 0, 27392, 11648, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 2560, 0, 128, 128, 0, 128, 31872, 14848, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 23424, 0, 0, 25472, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 29824, 3328, 11520, 0, 4736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 6528, 128, 0, 0, 128, 640, 42112, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 2304, 46336, 768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 128, 49152, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=22 tokens_per_expert_current_ep_rank=[7701, 0, 0, 0, 0, 0, 0, 1380, 503, 0, 360, 0, 0, 0, 0, 906, 0, 0, 0, 5380, 282, 396, 0, 5097, 0, 0, 0, 0, 0, 0, 0, 0, 619, 0, 0, 0, 0, 0, 0, 7349, 0, 0, 0, 7324, 0, 0, 51, 0, 0, 0, 0, 2389, 0, 7849, 0, 0, 0, 0, 0, 1309, 172, 0, 0, 85] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=22 tokens_per_expert_current_ep_rank=[7811, 0, 0, 0, 0, 0, 0, 1223, 508, 0, 491, 0, 0, 0, 0, 880, 0, 0, 0, 4167, 244, 384, 0, 5492, 0, 0, 0, 0, 0, 0, 0, 0, 640, 0, 0, 0, 0, 0, 0, 7545, 0, 0, 0, 7305, 0, 0, 69, 0, 0, 0, 0, 2675, 0, 7812, 0, 0, 0, 0, 0, 1670, 159, 0, 0, 77] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=22 tokens_per_expert_current_ep_rank=[7882, 0, 0, 0, 0, 0, 0, 879, 452, 0, 603, 0, 0, 0, 0, 805, 0, 0, 0, 3238, 209, 270, 0, 6251, 0, 0, 0, 0, 0, 0, 0, 0, 460, 0, 0, 0, 0, 0, 0, 7814, 0, 0, 0, 7325, 0, 0, 83, 0, 0, 0, 0, 2804, 0, 7568, 0, 0, 0, 0, 0, 2301, 143, 0, 0, 65] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=22 tokens_per_expert_current_ep_rank=[7722, 1, 0, 0, 0, 0, 0, 944, 512, 2, 522, 0, 0, 0, 0, 933, 0, 2, 0, 4703, 283, 434, 0, 5508, 0, 0, 0, 0, 0, 0, 0, 0, 656, 1, 0, 0, 0, 0, 0, 7713, 0, 2, 0, 7311, 0, 2, 68, 1, 0, 0, 0, 2146, 1, 7783, 0, 1, 0, 0, 0, 1637, 157, 0, 0, 107] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=22 tokens_per_expert_current_ep_rank=[7826, 0, 0, 0, 0, 0, 0, 832, 490, 0, 510, 0, 0, 0, 0, 870, 0, 0, 0, 3224, 240, 355, 0, 6564, 0, 0, 0, 0, 0, 0, 0, 0, 508, 0, 0, 0, 0, 0, 0, 7522, 0, 0, 0, 7309, 0, 0, 91, 0, 0, 0, 0, 2887, 0, 7622, 0, 0, 0, 0, 0, 2069, 159, 0, 0, 74] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=22 tokens_per_expert_current_ep_rank=[7814, 0, 0, 0, 0, 0, 0, 976, 481, 0, 538, 0, 0, 0, 0, 861, 0, 0, 0, 3856, 278, 393, 0, 6020, 0, 0, 0, 0, 0, 0, 0, 0, 566, 0, 0, 0, 0, 0, 0, 7525, 0, 0, 0, 7316, 0, 0, 54, 0, 0, 0, 0, 2778, 0, 7841, 0, 0, 0, 0, 0, 1616, 167, 0, 0, 72] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=22 tokens_per_expert_current_ep_rank=[7628, 0, 0, 0, 0, 0, 0, 1199, 479, 0, 416, 0, 0, 0, 0, 878, 0, 0, 0, 5428, 244, 449, 0, 5465, 0, 0, 0, 0, 0, 0, 0, 0, 673, 0, 0, 0, 0, 0, 0, 7212, 0, 0, 0, 7395, 0, 0, 54, 0, 0, 0, 0, 2340, 0, 7800, 0, 0, 0, 0, 0, 1216, 185, 0, 0, 91] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=22 tokens_per_expert_current_ep_rank=[7809, 0, 0, 0, 0, 0, 0, 1033, 443, 0, 582, 0, 0, 0, 0, 829, 0, 0, 0, 3429, 191, 380, 0, 5962, 0, 0, 0, 0, 0, 0, 0, 0, 562, 0, 0, 0, 0, 0, 0, 7829, 0, 0, 0, 7310, 0, 0, 71, 0, 0, 0, 0, 2816, 0, 7544, 0, 0, 0, 0, 0, 2141, 143, 0, 0, 78] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=22 rerouted_tokens_per_expert_current_ep_rank=[5038, 0, 0, 0, 0, 0, 0, 1380, 0, 0, 0, 0, 0, 0, 0, 0, 503, 0, 360, 0, 0, 0, 0, 906, 2541, 1768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5380, 282, 396, 0, 1176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3921, 1712, 0, 0, 0, 0, 0, 0, 619, 0, 0, 0, 0, 0, 0, 5401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6076, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2389, 0, 3596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1309, 172, 0, 0, 85, 1948, 1248, 895, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6418, 6078, 7234, 5633, 6020, 6127, 5985, 5657] avg=6144.0000 max=7234 max_rank=2 min=5633 min_rank=3 imbalance_ratio=1.177409 max_load_reduction=41918 max_load_reduction_pct=0.852824 moved_to_echo=23789 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=22 rerouted_tokens_per_expert_current_ep_rank=[5109, 0, 0, 0, 0, 0, 0, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 443, 0, 582, 0, 0, 0, 0, 829, 2442, 1793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3429, 191, 380, 0, 1376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4586, 1645, 0, 0, 0, 0, 0, 0, 562, 0, 0, 0, 0, 0, 0, 5754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6064, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2816, 0, 3457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2141, 143, 0, 0, 78, 2075, 1246, 907, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6142, 6089, 5376, 6231, 6316, 6135, 6273, 6590] avg=6144.0000 max=6590 max_rank=7 min=5376 min_rank=2 imbalance_ratio=1.072591 max_load_reduction=42562 max_load_reduction_pct=0.865926 moved_to_echo=25314 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=22 rerouted_tokens_per_expert_current_ep_rank=[5157, 0, 0, 0, 0, 0, 0, 879, 0, 0, 0, 0, 0, 0, 0, 0, 452, 0, 603, 0, 0, 0, 0, 805, 2450, 1809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3238, 209, 270, 0, 1442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4809, 1650, 0, 0, 0, 0, 0, 0, 460, 0, 0, 0, 0, 0, 0, 5743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6077, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2804, 0, 3468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2301, 143, 0, 0, 65, 2071, 1248, 916, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6036, 6119, 5159, 6459, 6203, 6160, 6272, 6744] avg=6144.0000 max=6744 max_rank=7 min=5159 min_rank=2 imbalance_ratio=1.097656 max_load_reduction=42408 max_load_reduction_pct=0.862793 moved_to_echo=25379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=22 rerouted_tokens_per_expert_current_ep_rank=[5052, 1, 0, 0, 0, 0, 0, 944, 0, 0, 0, 0, 0, 0, 0, 0, 512, 2, 522, 0, 0, 0, 0, 933, 2520, 1773, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4703, 283, 434, 0, 1271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4237, 1697, 0, 0, 0, 0, 0, 0, 656, 1, 0, 0, 0, 0, 0, 5669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 6065, 0, 2, 68, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2146, 1, 3566, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1637, 157, 0, 0, 107, 2044, 1246, 897, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5997, 6262, 6693, 5934, 6326, 6138, 5714, 6088] avg=6144.0000 max=6693 max_rank=2 min=5714 min_rank=6 imbalance_ratio=1.089355 max_load_reduction=42459 max_load_reduction_pct=0.863831 moved_to_echo=24266 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=22 rerouted_tokens_per_expert_current_ep_rank=[5105, 0, 0, 0, 0, 0, 0, 1223, 0, 0, 0, 0, 0, 0, 0, 0, 508, 0, 491, 0, 0, 0, 0, 880, 2533, 1796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4167, 244, 384, 0, 1263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4229, 1709, 0, 0, 0, 0, 0, 0, 640, 0, 0, 0, 0, 0, 0, 5542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6057, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2675, 0, 3570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1670, 159, 0, 0, 77, 2003, 1248, 910, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6328, 6208, 6058, 5938, 6182, 6126, 6245, 6067] avg=6144.0000 max=6328 max_rank=0 min=5938 min_rank=3 imbalance_ratio=1.029948 max_load_reduction=42824 max_load_reduction_pct=0.871257 moved_to_echo=24620 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=22 rerouted_tokens_per_expert_current_ep_rank=[5112, 0, 0, 0, 0, 0, 0, 976, 0, 0, 0, 0, 0, 0, 0, 0, 481, 0, 538, 0, 0, 0, 0, 861, 2539, 1794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3856, 278, 393, 0, 1389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4631, 1710, 0, 0, 0, 0, 0, 0, 566, 0, 0, 0, 0, 0, 0, 5530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6069, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2778, 0, 3592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1616, 167, 0, 0, 72, 1995, 1247, 908, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6088, 6213, 5916, 6341, 6096, 6123, 6370, 6005] avg=6144.0000 max=6370 max_rank=6 min=5916 min_rank=2 imbalance_ratio=1.036784 max_load_reduction=42782 max_load_reduction_pct=0.870402 moved_to_echo=24594 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=22 rerouted_tokens_per_expert_current_ep_rank=[5121, 0, 0, 0, 0, 0, 0, 832, 0, 0, 0, 0, 0, 0, 0, 0, 490, 0, 510, 0, 0, 0, 0, 870, 2468, 1796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3224, 240, 355, 0, 1515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5049, 1662, 0, 0, 0, 0, 0, 0, 508, 0, 0, 0, 0, 0, 0, 5528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6063, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2887, 0, 3492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2069, 159, 0, 0, 74, 1994, 1246, 909, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5953, 6134, 5334, 6711, 6036, 6154, 6379, 6451] avg=6144.0000 max=6711 max_rank=3 min=5334 min_rank=2 imbalance_ratio=1.092285 max_load_reduction=42441 max_load_reduction_pct=0.863464 moved_to_echo=25020 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=22 rerouted_tokens_per_expert_current_ep_rank=[4991, 0, 0, 0, 0, 0, 0, 1199, 0, 0, 0, 0, 0, 0, 0, 0, 479, 0, 416, 0, 0, 0, 0, 878, 2525, 1751, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5428, 244, 449, 0, 1261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4204, 1701, 0, 0, 0, 0, 0, 0, 673, 0, 0, 0, 0, 0, 0, 5300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6135, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2340, 0, 3574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1216, 185, 0, 0, 91, 1912, 1260, 886, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6190, 6049, 7382, 5905, 5973, 6189, 5914, 5550] avg=6144.0000 max=7382 max_rank=2 min=5550 min_rank=7 imbalance_ratio=1.201497 max_load_reduction=41770 max_load_reduction_pct=0.849813 moved_to_echo=23626 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3968, 128, 4096, 0, 0, 0, 0, 7040, 20096, 14336, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 0, 0, 14080, 1408, 0, 0, 768, 16128, 10112, 7296, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49280 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 35712, 13568, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 0, 0, 20864, 128, 28416, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 128, 0, 33536, 2048, 3072, 0, 10752, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[40704, 128, 0, 0, 0, 0, 0, 8576, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[4736, 128, 0, 0, 0, 0, 0, 44544, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 0, 48640, 0, 128, 640, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 7761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8107, 0, 429, 0, 0, 331, 57, 0, 0, 0, 57, 0, 189, 0, 0, 0, 0, 0, 0, 7474, 6990, 0, 84, 0, 0, 0, 90, 0, 203, 379, 0, 0, 0, 6868, 8051, 0, 0, 85, 0, 0, 241, 0, 85, 0, 0, 0, 1671, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 7761, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8127, 0, 375, 0, 0, 335, 50, 0, 0, 0, 50, 0, 193, 0, 0, 0, 0, 0, 0, 7033, 6287, 0, 64, 0, 0, 0, 129, 0, 192, 369, 0, 0, 0, 7059, 8078, 0, 0, 65, 0, 0, 244, 0, 65, 0, 0, 0, 2674, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 7745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8120, 0, 433, 0, 1, 303, 59, 0, 0, 0, 59, 0, 187, 0, 0, 0, 0, 0, 0, 7156, 6732, 0, 72, 0, 0, 0, 86, 0, 185, 376, 0, 0, 0, 6938, 8061, 0, 0, 71, 0, 0, 258, 0, 72, 0, 0, 0, 2238, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 7799, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8114, 0, 429, 0, 0, 367, 50, 0, 0, 0, 50, 0, 237, 0, 0, 0, 0, 0, 0, 6992, 6267, 0, 77, 0, 0, 0, 85, 0, 195, 389, 0, 0, 0, 7076, 8065, 0, 0, 78, 0, 0, 251, 0, 78, 0, 0, 0, 2552, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 7746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8118, 0, 412, 0, 0, 298, 51, 0, 0, 0, 51, 0, 180, 0, 0, 0, 0, 0, 0, 7012, 6578, 0, 73, 0, 0, 0, 147, 0, 173, 340, 0, 0, 0, 6990, 8068, 0, 0, 74, 0, 0, 244, 0, 74, 0, 0, 0, 2523, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 7772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8115, 0, 414, 0, 1, 364, 59, 0, 0, 0, 59, 0, 187, 0, 0, 0, 0, 0, 0, 7050, 6683, 0, 77, 0, 0, 0, 99, 0, 188, 393, 0, 0, 0, 7061, 8056, 0, 0, 76, 0, 0, 246, 0, 77, 0, 0, 0, 2175, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 7817, 0, 0, 0, 0, 0, 2, 13, 0, 0, 8081, 0, 446, 0, 1, 363, 37, 0, 0, 0, 37, 0, 228, 0, 1, 0, 0, 0, 0, 7000, 6560, 0, 107, 0, 2, 0, 108, 0, 181, 444, 0, 0, 2, 6969, 8046, 0, 2, 106, 0, 0, 245, 0, 107, 0, 0, 2, 2243, 1, 1, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 7751, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8101, 0, 480, 0, 2, 317, 66, 0, 0, 0, 66, 0, 229, 0, 0, 0, 0, 0, 0, 7489, 6861, 0, 92, 0, 0, 0, 74, 0, 222, 410, 0, 0, 0, 6842, 8035, 0, 0, 88, 0, 0, 252, 0, 91, 0, 0, 0, 1684, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5713, 0, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 331, 57, 0, 0, 0, 57, 0, 2394, 2321, 981, 0, 0, 0, 0, 0, 189, 0, 0, 0, 0, 0, 0, 6209, 0, 0, 0, 0, 0, 0, 0, 0, 6295, 0, 84, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 203, 379, 0, 0, 0, 5479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 241, 0, 85, 0, 5730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1671, 0, 0, 0, 0, 0, 1389, 1265, 695, 641, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6139, 6142, 6141, 6398, 6469, 6061, 6141, 5661] avg=6144.0000 max=6469 max_rank=4 min=5661 min_rank=7 imbalance_ratio=1.052897 max_load_reduction=42683 max_load_reduction_pct=0.868388 moved_to_echo=24332 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5718, 0, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 367, 50, 0, 0, 0, 50, 0, 2396, 2325, 986, 0, 0, 0, 0, 0, 237, 0, 0, 0, 0, 0, 0, 5809, 0, 0, 0, 0, 0, 0, 0, 0, 5644, 0, 77, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 195, 389, 0, 0, 0, 5645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 251, 0, 78, 0, 5740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2552, 0, 0, 0, 0, 0, 1431, 1183, 623, 644, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6169, 6148, 6174, 6046, 5806, 6229, 6147, 6433] avg=6144.0000 max=6433 max_rank=7 min=5806 min_rank=4 imbalance_ratio=1.047038 max_load_reduction=42719 max_load_reduction_pct=0.869120 moved_to_echo=24615 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 5727, 0, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 335, 50, 0, 0, 0, 50, 0, 2400, 2329, 981, 0, 0, 0, 0, 0, 193, 0, 0, 0, 0, 0, 0, 5843, 0, 0, 0, 0, 0, 0, 0, 0, 5662, 0, 64, 0, 0, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 369, 0, 0, 0, 5631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 244, 0, 65, 0, 5749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2674, 0, 0, 0, 0, 0, 1428, 1190, 625, 641, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6139, 6104, 6145, 6036, 5855, 6192, 6123, 6558] avg=6144.0000 max=6558 max_rank=7 min=5855 min_rank=4 imbalance_ratio=1.067383 max_load_reduction=42594 max_load_reduction_pct=0.866577 moved_to_echo=24728 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5722, 0, 433, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 303, 59, 0, 0, 0, 59, 0, 2398, 2324, 979, 0, 0, 0, 0, 0, 187, 0, 0, 0, 0, 0, 0, 5945, 0, 0, 0, 0, 0, 0, 0, 0, 6062, 0, 72, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, 376, 0, 0, 0, 5535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 258, 0, 72, 0, 5737, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2238, 0, 0, 0, 0, 0, 1403, 1211, 670, 640, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6126, 6155, 6123, 6132, 6220, 6096, 6138, 6162] avg=6144.0000 max=6220 max_rank=4 min=6096 min_rank=5 imbalance_ratio=1.012370 max_load_reduction=42932 max_load_reduction_pct=0.873454 moved_to_echo=24616 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 13, 0, 0, 5694, 0, 446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 363, 37, 0, 0, 0, 37, 0, 2387, 2319, 988, 0, 0, 0, 0, 0, 228, 0, 1, 0, 0, 0, 0, 5815, 0, 0, 0, 0, 0, 0, 0, 0, 5907, 0, 107, 0, 2, 0, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 444, 0, 0, 2, 5559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 106, 0, 0, 245, 0, 107, 0, 5727, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2243, 1, 1, 0, 0, 0, 1410, 1185, 653, 646, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6183, 6155, 6132, 6044, 6124, 6186, 6187, 6141] avg=6144.0000 max=6187 max_rank=6 min=6044 min_rank=3 imbalance_ratio=1.006999 max_load_reduction=42965 max_load_reduction_pct=0.874125 moved_to_echo=24638 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5714, 0, 414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 364, 59, 0, 0, 0, 59, 0, 2401, 2323, 987, 0, 0, 0, 0, 0, 187, 0, 0, 0, 0, 0, 0, 5853, 0, 0, 0, 0, 0, 0, 0, 0, 6014, 0, 77, 0, 0, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 393, 0, 0, 0, 5628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 246, 0, 77, 0, 5733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2175, 0, 0, 0, 0, 0, 1433, 1197, 669, 647, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6138, 6128, 6194, 6040, 6190, 6209, 6132, 6121] avg=6144.0000 max=6209 max_rank=5 min=6040 min_rank=3 imbalance_ratio=1.010579 max_load_reduction=42943 max_load_reduction_pct=0.873678 moved_to_echo=24652 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5720, 0, 412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 298, 51, 0, 0, 0, 51, 0, 2398, 2326, 979, 0, 0, 0, 0, 0, 180, 0, 0, 0, 0, 0, 0, 5825, 0, 0, 0, 0, 0, 0, 0, 0, 5924, 0, 73, 0, 0, 0, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 173, 340, 0, 0, 0, 5576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 244, 0, 74, 0, 5742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2523, 0, 0, 0, 0, 0, 1414, 1187, 654, 640, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6127, 6132, 6103, 6005, 6144, 6089, 6134, 6418] avg=6144.0000 max=6418 max_rank=7 min=6005 min_rank=3 imbalance_ratio=1.044596 max_load_reduction=42734 max_load_reduction_pct=0.869425 moved_to_echo=24785 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5708, 0, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 317, 66, 0, 0, 0, 66, 0, 2393, 2316, 980, 0, 0, 0, 0, 0, 229, 0, 0, 0, 0, 0, 0, 6222, 0, 0, 0, 0, 0, 0, 0, 0, 6178, 0, 92, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 222, 410, 0, 0, 0, 5458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 252, 0, 91, 0, 5719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1684, 0, 0, 0, 0, 0, 1384, 1267, 683, 640, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6131, 6188, 6140, 6451, 6344, 6090, 6150, 5658] avg=6144.0000 max=6451 max_rank=3 min=5658 min_rank=7 imbalance_ratio=1.049967 max_load_reduction=42701 max_load_reduction_pct=0.868754 moved_to_echo=24242 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 2688, 512, 0, 0, 0, 512, 0, 19200, 18688, 7936, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[0, 128, 17792, 128, 128, 0, 0, 0, 11392, 9728, 5376, 5248, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[1664, 3200, 0, 0, 128, 44544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 128, 0, 0, 45824, 0, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[1664, 0, 128, 0, 0, 0, 0, 47616, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 768, 0, 0, 2048, 0, 768, 0, 45952, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[47744, 0, 768, 0, 128, 0, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49152 observed_loss=0 tokens_per_expert=[0, 0, 49152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49152 slack_slots=49152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 319, 0, 0, 7481, 6206, 0, 0, 64, 0, 0, 0, 0, 0, 50, 0, 540, 0, 0, 0, 0, 0, 0, 1345, 7468, 0, 0, 0, 0, 0, 0, 793, 7329, 0, 8128, 65, 0, 0, 825, 64, 0, 158, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 309, 7874, 0, 0, 0, 79] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=24 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 342, 0, 0, 7443, 5776, 0, 0, 84, 0, 0, 0, 0, 0, 57, 0, 593, 0, 0, 0, 0, 0, 0, 1309, 7742, 0, 0, 0, 0, 0, 0, 826, 7257, 0, 8108, 85, 0, 0, 906, 84, 0, 85, 0, 0, 0, 0, 0, 0, 174, 0, 0, 0, 0, 326, 7819, 0, 0, 0, 134] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 350, 0, 0, 7518, 6185, 0, 0, 77, 0, 0, 0, 0, 0, 50, 0, 530, 0, 0, 0, 0, 0, 0, 1357, 7463, 0, 0, 0, 0, 0, 0, 757, 7292, 0, 8115, 78, 0, 0, 805, 77, 0, 149, 0, 0, 0, 0, 0, 0, 122, 0, 0, 0, 0, 329, 7834, 0, 0, 0, 64] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=24 tokens_per_expert_current_ep_rank=[0, 4, 1, 2, 2, 1, 317, 0, 0, 7490, 5916, 0, 0, 107, 0, 1, 0, 0, 0, 36, 2, 538, 2, 0, 0, 0, 0, 0, 1360, 7675, 0, 0, 0, 2, 0, 0, 802, 7155, 0, 8083, 107, 0, 0, 853, 111, 0, 112, 0, 0, 0, 0, 0, 0, 248, 0, 0, 0, 0, 352, 7781, 0, 0, 0, 92] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 342, 0, 0, 7434, 6082, 0, 0, 72, 0, 0, 0, 0, 0, 59, 0, 529, 0, 0, 0, 0, 0, 0, 1271, 7663, 0, 0, 0, 0, 0, 0, 826, 7294, 0, 8120, 72, 0, 0, 875, 72, 0, 78, 0, 0, 0, 0, 0, 0, 124, 0, 0, 0, 0, 330, 7839, 0, 0, 0, 70] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=24 tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 0, 0, 290, 0, 0, 7456, 5892, 0, 0, 91, 0, 0, 0, 0, 0, 66, 0, 467, 0, 0, 0, 0, 0, 0, 1299, 7761, 0, 0, 0, 0, 0, 0, 835, 7225, 0, 8101, 91, 0, 0, 875, 91, 0, 97, 0, 0, 0, 0, 0, 0, 214, 0, 0, 0, 0, 343, 7852, 0, 0, 0, 100] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=24 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 324, 0, 0, 7454, 6054, 0, 0, 73, 0, 0, 0, 0, 0, 51, 0, 498, 0, 0, 0, 0, 0, 0, 1389, 7602, 0, 0, 0, 0, 0, 0, 746, 7302, 0, 8119, 74, 0, 0, 890, 73, 0, 112, 0, 0, 0, 0, 0, 0, 110, 0, 0, 0, 0, 318, 7863, 0, 0, 0, 100] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=24 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 321, 0, 0, 7433, 6085, 0, 0, 77, 0, 0, 0, 0, 0, 59, 0, 499, 0, 0, 0, 0, 0, 0, 1438, 7573, 0, 0, 0, 0, 0, 0, 839, 7268, 0, 8115, 77, 0, 0, 877, 77, 0, 85, 0, 0, 0, 0, 0, 0, 104, 0, 0, 0, 0, 323, 7835, 0, 0, 0, 66] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 342, 0, 3694, 2110, 0, 0, 0, 0, 0, 0, 0, 40, 5776, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 593, 0, 0, 3709, 1875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1309, 4877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 826, 5335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 906, 84, 0, 85, 0, 2105, 1922, 990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 174, 0, 0, 5998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 326, 5714, 0, 0, 0, 134, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6148, 5900, 6234, 6186, 6161, 6177, 6172, 6174] avg=6144.0000 max=6234 max_rank=2 min=5900 min_rank=1 imbalance_ratio=1.014648 max_load_reduction=42918 max_load_reduction_pct=0.873169 moved_to_echo=24684 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 350, 0, 3731, 2112, 0, 0, 0, 0, 0, 0, 0, 41, 6185, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 530, 0, 0, 3746, 1808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1357, 4701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 757, 5361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 805, 77, 0, 149, 0, 2109, 1931, 954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 0, 0, 6003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 329, 5725, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6193, 6303, 6134, 6058, 6118, 6103, 6125, 6118] avg=6144.0000 max=6303 max_rank=1 min=6058 min_rank=3 imbalance_ratio=1.025879 max_load_reduction=42849 max_load_reduction_pct=0.871765 moved_to_echo=24464 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 319, 0, 3713, 2115, 0, 0, 0, 0, 0, 0, 0, 40, 6206, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 540, 0, 0, 3728, 1809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1345, 4704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 793, 5388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 825, 64, 0, 158, 0, 2119, 1941, 955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 6013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 309, 5755, 0, 0, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6147, 6310, 6127, 6049, 6181, 6127, 6068, 6143] avg=6144.0000 max=6310 max_rank=1 min=6049 min_rank=3 imbalance_ratio=1.027018 max_load_reduction=42842 max_load_reduction_pct=0.871623 moved_to_echo=24519 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 1, 2, 2, 1, 317, 0, 3718, 2103, 0, 0, 0, 0, 0, 0, 0, 39, 5916, 0, 0, 107, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 2, 538, 2, 0, 3733, 1859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1360, 4835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 802, 5260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 0, 0, 853, 111, 0, 112, 0, 2094, 1895, 981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 248, 0, 0, 5980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 352, 5687, 0, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6148, 6063, 6170, 6195, 6064, 6153, 6228, 6131] avg=6144.0000 max=6228 max_rank=6 min=6063 min_rank=1 imbalance_ratio=1.013672 max_load_reduction=42924 max_load_reduction_pct=0.873291 moved_to_echo=24576 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 342, 0, 3690, 2114, 0, 0, 0, 0, 0, 0, 0, 39, 6082, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 529, 0, 0, 3705, 1856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1271, 4827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 826, 5362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 875, 72, 0, 78, 0, 2110, 1932, 980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 6006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 330, 5729, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6146, 6193, 6149, 6098, 6188, 6119, 6130, 6129] avg=6144.0000 max=6193 max_rank=1 min=6098 min_rank=3 imbalance_ratio=1.007975 max_load_reduction=42959 max_load_reduction_pct=0.874003 moved_to_echo=24566 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 324, 0, 3700, 2113, 0, 0, 0, 0, 0, 0, 0, 39, 6054, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 498, 0, 0, 3715, 1841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1389, 4789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 746, 5368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 890, 73, 0, 112, 0, 2116, 1934, 972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0, 0, 6006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 318, 5747, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6137, 6166, 6105, 6178, 6114, 6171, 6116, 6165] avg=6144.0000 max=6178 max_rank=3 min=6105 min_rank=2 imbalance_ratio=1.005534 max_load_reduction=42974 max_load_reduction_pct=0.874308 moved_to_echo=24566 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 321, 0, 3693, 2113, 0, 0, 0, 0, 0, 0, 0, 33, 6085, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 499, 0, 0, 3707, 1839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1438, 4762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 839, 5340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 877, 77, 0, 85, 0, 2113, 1928, 972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 6002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 323, 5722, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6128, 6195, 6104, 6200, 6179, 6129, 6106, 6111] avg=6144.0000 max=6200 max_rank=3 min=6104 min_rank=2 imbalance_ratio=1.009115 max_load_reduction=42952 max_load_reduction_pct=0.873861 moved_to_echo=24525 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 0, 0, 290, 0, 3701, 2108, 0, 0, 0, 0, 0, 0, 0, 39, 5892, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 467, 0, 0, 3716, 1880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1299, 4889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 835, 5312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 875, 91, 0, 97, 0, 2114, 1913, 992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 214, 0, 0, 5993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 343, 5738, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6105, 6022, 6129, 6188, 6147, 6173, 6207, 6181] avg=6144.0000 max=6207 max_rank=6 min=6022 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=42945 max_load_reduction_pct=0.873718 moved_to_echo=24708 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[768, 0, 0, 6912, 768, 0, 896, 0, 16896, 15488, 7808, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 128, 128, 2688, 0, 29696, 16896, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49280 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 10880, 38400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 0, 0, 512, 128, 4224, 128, 0, 29824, 14848, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49280 observed_loss=-128 tokens_per_expert=[0, 0, 2688, 45824, 0, 0, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 6528, 42752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49280 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 0, 1152, 0, 0, 48128, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 384, 48256, 0, 0, 768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49280 slack_slots=49024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True - [2026-08-26 14:53:49] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 824.6 | throughput per GPU (TFLOP/s/GPU): 154.7 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.613759E+00 | loss scale: 1.0 | grad norm: 3.002 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1331, 307, 753, 989, 292, 902, 715, 1115, 1370, 239, 613, 373, 388, 1296, 807, 1140, 634, 966, 879, 518, 371, 532, 311, 1424, 490, 679, 781, 355, 722, 80, 823, 1456, 882, 803, 1018, 1081, 526, 191, 529, 907, 666, 629, 498, 891, 914, 921, 550, 590, 766, 538, 798, 561, 1042, 1233, 866, 894, 1530, 470, 1477, 750, 193, 670, 1156, 961] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1283, 288, 816, 883, 280, 932, 861, 1057, 1405, 233, 676, 322, 362, 1464, 1054, 984, 728, 1007, 796, 662, 335, 521, 268, 1431, 483, 576, 750, 365, 778, 82, 855, 1339, 793, 789, 986, 1115, 508, 232, 527, 911, 653, 611, 506, 853, 848, 937, 422, 633, 867, 623, 786, 562, 1034, 1263, 824, 832, 1442, 510, 1566, 662, 167, 758, 1162, 894] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1350, 333, 739, 968, 302, 812, 788, 1134, 1337, 198, 642, 336, 417, 1296, 839, 984, 694, 1002, 734, 593, 307, 514, 283, 1393, 474, 685, 783, 390, 713, 81, 915, 1339, 858, 878, 1082, 1102, 526, 212, 505, 893, 718, 657, 591, 923, 819, 933, 622, 611, 720, 551, 835, 546, 1081, 1225, 828, 928, 1538, 444, 1421, 730, 161, 639, 1120, 1080] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1312, 232, 788, 954, 286, 746, 791, 1137, 1314, 188, 572, 329, 375, 1308, 970, 1184, 670, 985, 843, 648, 278, 529, 254, 1508, 573, 692, 733, 384, 775, 80, 773, 1402, 781, 841, 951, 1004, 421, 198, 553, 1060, 599, 568, 524, 916, 910, 905, 565, 687, 799, 555, 857, 472, 1160, 1226, 946, 919, 1490, 412, 1423, 686, 159, 803, 1181, 968] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1325, 295, 774, 906, 337, 1041, 838, 1036, 1355, 242, 663, 315, 347, 1334, 963, 1159, 681, 988, 824, 611, 383, 521, 255, 1401, 501, 645, 771, 402, 734, 115, 777, 1353, 809, 783, 1008, 1087, 513, 217, 485, 1005, 644, 673, 511, 822, 852, 917, 454, 572, 861, 545, 812, 556, 1073, 1280, 893, 762, 1530, 265, 1415, 808, 243, 762, 1180, 923] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1280, 260, 718, 924, 316, 996, 807, 1063, 1290, 190, 669, 372, 366, 1293, 955, 1123, 681, 967, 838, 618, 340, 526, 285, 1362, 534, 673, 773, 390, 738, 162, 914, 1153, 882, 755, 1110, 1116, 693, 209, 526, 967, 638, 607, 556, 858, 915, 938, 535, 664, 710, 549, 772, 518, 1058, 1288, 838, 890, 1508, 267, 1446, 736, 191, 684, 1251, 901] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1375, 259, 729, 849, 290, 1081, 856, 912, 1342, 258, 685, 337, 291, 1483, 854, 1132, 769, 966, 754, 602, 347, 548, 288, 1415, 342, 638, 843, 384, 704, 105, 836, 1410, 823, 785, 1062, 1165, 432, 253, 487, 762, 624, 625, 582, 826, 990, 895, 429, 571, 886, 553, 787, 624, 1012, 1346, 809, 884, 1521, 310, 1544, 756, 195, 815, 1270, 845] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1401, 272, 734, 891, 283, 907, 760, 976, 1427, 182, 611, 358, 336, 1473, 855, 1128, 618, 945, 786, 618, 294, 512, 254, 1356, 474, 720, 765, 358, 679, 74, 822, 1072, 851, 827, 1028, 1146, 774, 223, 589, 945, 605, 589, 582, 817, 923, 970, 411, 636, 686, 575, 834, 583, 1024, 1394, 869, 806, 1600, 481, 1602, 768, 187, 737, 1209, 940] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1105, 307, 753, 989, 292, 902, 715, 1115, 0, 0, 0, 0, 0, 0, 0, 0, 1370, 239, 613, 373, 388, 1149, 807, 1140, 0, 0, 0, 0, 0, 0, 0, 0, 634, 966, 879, 518, 371, 532, 311, 1424, 274, 226, 9, 0, 0, 0, 0, 0, 490, 679, 781, 355, 722, 80, 823, 1456, 866, 0, 0, 0, 0, 0, 0, 0, 882, 803, 1018, 1081, 526, 191, 529, 907, 138, 0, 0, 0, 0, 0, 0, 0, 666, 629, 498, 891, 914, 921, 550, 590, 333, 185, 0, 0, 0, 0, 0, 0, 766, 538, 798, 561, 1042, 626, 866, 894, 0, 0, 0, 0, 0, 0, 0, 0, 479, 470, 1477, 750, 193, 670, 1156, 961, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6178, 6079, 6144, 6252, 6075, 6177, 6091, 6156] avg=6144.0000 max=6252 max_rank=3 min=6075 min_rank=4 imbalance_ratio=1.017578 max_load_reduction=42900 max_load_reduction_pct=0.872803 moved_to_echo=24499 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1065, 288, 816, 883, 280, 932, 861, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1405, 233, 676, 322, 362, 1298, 1054, 984, 0, 0, 0, 0, 0, 0, 0, 0, 728, 1007, 796, 662, 335, 521, 268, 1431, 281, 218, 10, 0, 0, 0, 0, 0, 483, 576, 750, 365, 778, 82, 855, 1339, 816, 0, 0, 0, 0, 0, 0, 0, 793, 789, 986, 1115, 508, 232, 527, 911, 156, 0, 0, 0, 0, 0, 0, 0, 653, 611, 506, 853, 848, 937, 422, 633, 341, 175, 0, 0, 0, 0, 0, 0, 867, 623, 786, 562, 1034, 641, 824, 832, 0, 0, 0, 0, 0, 0, 0, 0, 451, 510, 1566, 662, 167, 758, 1162, 894, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6182, 6334, 6257, 6044, 6017, 5979, 6169, 6170] avg=6144.0000 max=6334 max_rank=1 min=5979 min_rank=5 imbalance_ratio=1.030924 max_load_reduction=42818 max_load_reduction_pct=0.871134 moved_to_echo=24335 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1116, 333, 739, 968, 302, 812, 788, 1134, 0, 0, 0, 0, 0, 0, 0, 0, 1337, 198, 642, 336, 417, 1140, 839, 984, 0, 0, 0, 0, 0, 0, 0, 0, 694, 1002, 734, 593, 307, 514, 283, 1393, 277, 234, 13, 0, 0, 0, 0, 0, 474, 685, 783, 390, 713, 81, 915, 1339, 875, 0, 0, 0, 0, 0, 0, 0, 858, 878, 1082, 1102, 526, 212, 505, 893, 143, 0, 0, 0, 0, 0, 0, 0, 718, 657, 591, 923, 819, 933, 622, 611, 335, 190, 0, 0, 0, 0, 0, 0, 720, 551, 835, 546, 1081, 613, 828, 928, 0, 0, 0, 0, 0, 0, 0, 0, 473, 444, 1421, 730, 161, 639, 1120, 1080, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6192, 5893, 6044, 6255, 6199, 6399, 6102, 6068] avg=6144.0000 max=6399 max_rank=5 min=5893 min_rank=1 imbalance_ratio=1.041504 max_load_reduction=42753 max_load_reduction_pct=0.869812 moved_to_echo=24768 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1089, 232, 788, 954, 286, 746, 791, 1137, 0, 0, 0, 0, 0, 0, 0, 0, 1314, 188, 572, 329, 375, 1159, 970, 1184, 0, 0, 0, 0, 0, 0, 0, 0, 670, 985, 843, 648, 278, 529, 254, 1508, 273, 223, 9, 0, 0, 0, 0, 0, 573, 692, 733, 384, 775, 80, 773, 1402, 843, 0, 0, 0, 0, 0, 0, 0, 781, 841, 951, 1004, 421, 198, 553, 1060, 140, 0, 0, 0, 0, 0, 0, 0, 599, 568, 524, 916, 910, 905, 565, 687, 331, 180, 0, 0, 0, 0, 0, 0, 799, 555, 857, 472, 1160, 622, 946, 919, 0, 0, 0, 0, 0, 0, 0, 0, 467, 412, 1423, 686, 159, 803, 1181, 968, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6023, 6091, 6220, 6255, 5949, 6185, 6330, 6099] avg=6144.0000 max=6330 max_rank=6 min=5949 min_rank=4 imbalance_ratio=1.030273 max_load_reduction=42822 max_load_reduction_pct=0.871216 moved_to_echo=24563 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1141, 259, 729, 849, 290, 1081, 856, 912, 0, 0, 0, 0, 0, 0, 0, 0, 1342, 258, 685, 337, 291, 1315, 854, 1132, 0, 0, 0, 0, 0, 0, 0, 0, 769, 966, 754, 602, 347, 548, 288, 1415, 300, 234, 10, 0, 0, 0, 0, 0, 342, 638, 843, 384, 704, 105, 836, 1410, 861, 0, 0, 0, 0, 0, 0, 0, 823, 785, 1062, 1165, 432, 253, 487, 762, 158, 0, 0, 0, 0, 0, 0, 0, 624, 625, 582, 826, 990, 895, 429, 571, 364, 184, 0, 0, 0, 0, 0, 0, 886, 553, 787, 624, 1012, 682, 809, 884, 0, 0, 0, 0, 0, 0, 0, 0, 476, 310, 1544, 756, 195, 815, 1270, 845, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6117, 6214, 6233, 6123, 5927, 6090, 6237, 6211] avg=6144.0000 max=6237 max_rank=6 min=5927 min_rank=4 imbalance_ratio=1.015137 max_load_reduction=42915 max_load_reduction_pct=0.873108 moved_to_echo=24465 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1063, 260, 718, 924, 316, 996, 807, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 1290, 190, 669, 372, 366, 1146, 955, 1123, 0, 0, 0, 0, 0, 0, 0, 0, 681, 967, 838, 618, 340, 526, 285, 1362, 287, 217, 9, 0, 0, 0, 0, 0, 534, 673, 773, 390, 738, 162, 914, 1153, 853, 0, 0, 0, 0, 0, 0, 0, 882, 755, 1110, 1116, 693, 209, 526, 967, 138, 0, 0, 0, 0, 0, 0, 0, 638, 607, 556, 858, 915, 938, 535, 664, 348, 183, 0, 0, 0, 0, 0, 0, 710, 549, 772, 518, 1058, 653, 838, 890, 0, 0, 0, 0, 0, 0, 0, 0, 472, 267, 1446, 736, 191, 684, 1251, 901, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6147, 6111, 6130, 6190, 6396, 6242, 5988, 5948] avg=6144.0000 max=6396 max_rank=4 min=5948 min_rank=7 imbalance_ratio=1.041016 max_load_reduction=42756 max_load_reduction_pct=0.869873 moved_to_echo=24574 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1100, 295, 774, 906, 337, 1041, 838, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1355, 242, 663, 315, 347, 1183, 963, 1159, 0, 0, 0, 0, 0, 0, 0, 0, 681, 988, 824, 611, 383, 521, 255, 1401, 285, 225, 9, 0, 0, 0, 0, 0, 501, 645, 771, 402, 734, 115, 777, 1353, 866, 0, 0, 0, 0, 0, 0, 0, 809, 783, 1008, 1087, 513, 217, 485, 1005, 142, 0, 0, 0, 0, 0, 0, 0, 644, 673, 511, 822, 852, 917, 454, 572, 346, 185, 0, 0, 0, 0, 0, 0, 861, 545, 812, 556, 1073, 649, 893, 762, 0, 0, 0, 0, 0, 0, 0, 0, 479, 265, 1415, 808, 243, 762, 1180, 923, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6327, 6227, 6183, 6164, 6049, 5976, 6151, 6075] avg=6144.0000 max=6327 max_rank=0 min=5976 min_rank=5 imbalance_ratio=1.029785 max_load_reduction=42825 max_load_reduction_pct=0.871277 moved_to_echo=24251 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1163, 272, 734, 891, 283, 907, 760, 976, 0, 0, 0, 0, 0, 0, 0, 0, 1427, 182, 611, 358, 336, 1306, 855, 1128, 0, 0, 0, 0, 0, 0, 0, 0, 618, 945, 786, 618, 294, 512, 254, 1356, 310, 238, 10, 0, 0, 0, 0, 0, 474, 720, 765, 358, 679, 74, 822, 1072, 905, 0, 0, 0, 0, 0, 0, 0, 851, 827, 1028, 1146, 774, 223, 589, 945, 157, 0, 0, 0, 0, 0, 0, 0, 605, 589, 582, 817, 923, 970, 411, 636, 377, 194, 0, 0, 0, 0, 0, 0, 686, 575, 834, 583, 1024, 707, 869, 806, 0, 0, 0, 0, 0, 0, 0, 0, 501, 481, 1602, 768, 187, 737, 1209, 940, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5986, 6203, 5941, 5869, 6540, 6104, 6084, 6425] avg=6144.0000 max=6540 max_rank=4 min=5869 min_rank=3 imbalance_ratio=1.064453 max_load_reduction=42612 max_load_reduction_pct=0.866943 moved_to_echo=25153 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[8960, 2304, 6144, 7424, 2432, 7424, 6528, 8448, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[5504, 7936, 6528, 4992, 2688, 4224, 2304, 11392, 2304, 1920, 128, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5248, 4992, 4352, 6912, 7296, 7424, 4096, 4992, 2816, 1536, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[3840, 3200, 11904, 6016, 1536, 5888, 9600, 7552, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[10880, 1792, 5248, 2816, 2944, 9728, 7424, 8960, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6784, 6528, 8320, 8832, 4480, 1792, 4224, 7552, 1280, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3968, 5376, 6272, 3072, 5888, 896, 6784, 10624, 6912, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6400, 4608, 6528, 4480, 8576, 5248, 6912, 7040, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[780, 742, 961, 410, 888, 295, 402, 1093, 108, 1059, 1403, 346, 600, 1055, 878, 1481, 1197, 1250, 835, 412, 1417, 135, 1630, 1004, 592, 645, 654, 801, 95, 299, 262, 133, 488, 295, 533, 245, 1146, 1815, 1074, 1132, 2194, 1516, 1292, 1372, 438, 770, 59, 207, 679, 1273, 689, 32, 195, 1628, 219, 1399, 766, 301, 722, 585, 1042, 556, 292, 336] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[648, 899, 923, 463, 874, 217, 256, 1088, 80, 1004, 1314, 335, 550, 945, 840, 1391, 1197, 1144, 964, 524, 1313, 161, 1695, 1098, 634, 725, 632, 819, 117, 494, 200, 115, 480, 364, 518, 137, 1012, 1830, 1191, 1072, 2178, 1537, 1221, 1491, 445, 893, 52, 219, 753, 1127, 891, 14, 199, 1625, 219, 1328, 768, 340, 804, 570, 976, 545, 288, 406] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[697, 840, 786, 346, 963, 262, 322, 1057, 86, 926, 1414, 399, 614, 969, 839, 1496, 1029, 1234, 992, 410, 1426, 174, 2014, 1096, 690, 644, 689, 824, 83, 300, 172, 102, 519, 311, 447, 135, 1115, 1750, 1194, 1026, 2049, 1548, 1143, 1290, 386, 1041, 70, 202, 805, 1250, 860, 20, 173, 1631, 261, 1283, 947, 370, 666, 575, 937, 590, 256, 407] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[717, 806, 893, 481, 992, 260, 296, 1149, 82, 965, 1411, 378, 577, 1012, 886, 1655, 1154, 1150, 968, 400, 1352, 147, 1752, 1111, 631, 742, 779, 885, 103, 261, 231, 123, 344, 209, 446, 123, 1154, 1657, 1016, 944, 2181, 1487, 1066, 1309, 405, 1021, 41, 212, 901, 1198, 1030, 14, 125, 1475, 186, 1261, 924, 334, 890, 606, 859, 635, 348, 402] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[758, 812, 974, 432, 893, 242, 271, 1115, 91, 962, 1089, 308, 540, 940, 1055, 1387, 1375, 1185, 891, 437, 1544, 121, 1678, 1050, 558, 686, 659, 787, 94, 411, 167, 68, 453, 232, 561, 203, 1156, 2011, 1059, 1044, 2251, 1524, 1395, 1584, 472, 688, 29, 183, 624, 1365, 682, 26, 169, 1716, 203, 1224, 674, 322, 864, 584, 1017, 573, 282, 402] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[685, 945, 923, 432, 963, 258, 334, 1194, 112, 960, 1366, 367, 547, 1047, 864, 1429, 1163, 1196, 853, 444, 1401, 189, 1544, 1121, 625, 709, 707, 840, 119, 412, 269, 142, 393, 275, 482, 124, 1096, 1859, 1100, 973, 2058, 1560, 1124, 1378, 509, 950, 36, 174, 671, 1260, 860, 21, 179, 1618, 237, 1249, 930, 316, 765, 606, 877, 634, 298, 380] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[644, 785, 1014, 473, 1024, 235, 315, 1093, 150, 1157, 1192, 362, 669, 1064, 987, 1460, 1260, 1260, 864, 338, 1367, 123, 1724, 1002, 624, 564, 588, 756, 104, 303, 222, 103, 460, 238, 588, 271, 1053, 1902, 1018, 1086, 2283, 1516, 1255, 1444, 429, 756, 28, 217, 732, 1296, 704, 45, 188, 1564, 197, 1268, 784, 335, 807, 592, 1015, 576, 329, 350] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[739, 860, 902, 420, 1037, 209, 301, 1241, 125, 1004, 1340, 378, 593, 1014, 937, 1528, 1172, 1166, 809, 439, 1268, 140, 1568, 1195, 598, 765, 632, 801, 135, 372, 204, 130, 441, 236, 530, 134, 1041, 1766, 1068, 941, 2171, 1372, 1222, 1357, 463, 880, 38, 203, 864, 1274, 989, 20, 152, 1561, 220, 1158, 825, 359, 962, 643, 869, 633, 298, 440] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[648, 899, 923, 463, 874, 217, 256, 1088, 296, 170, 119, 0, 0, 0, 0, 0, 80, 1004, 1314, 335, 550, 945, 840, 808, 0, 0, 0, 0, 0, 0, 0, 0, 1197, 1144, 964, 524, 1135, 161, 0, 1098, 0, 0, 0, 0, 0, 0, 0, 0, 634, 725, 632, 819, 117, 494, 200, 115, 1722, 845, 0, 0, 0, 0, 0, 0, 480, 364, 518, 137, 1012, 1534, 1191, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 456, 1537, 1221, 1491, 445, 893, 52, 219, 0, 0, 0, 0, 0, 0, 0, 0, 753, 1127, 891, 14, 199, 1625, 219, 1328, 8, 0, 0, 0, 0, 0, 0, 0, 768, 340, 804, 570, 976, 545, 288, 406, 850, 464, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5953, 5876, 6223, 6303, 6308, 6314, 6164, 6011] avg=6144.0000 max=6314 max_rank=5 min=5876 min_rank=1 imbalance_ratio=1.027669 max_load_reduction=42838 max_load_reduction_pct=0.871541 moved_to_echo=24797 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[780, 742, 961, 410, 888, 295, 402, 1093, 294, 184, 127, 0, 0, 0, 0, 0, 108, 1059, 1403, 346, 600, 1055, 878, 859, 0, 0, 0, 0, 0, 0, 0, 0, 1197, 1250, 835, 412, 1224, 135, 0, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 592, 645, 654, 801, 95, 299, 262, 133, 1734, 813, 0, 0, 0, 0, 0, 0, 488, 295, 533, 245, 1146, 1521, 1074, 1132, 0, 0, 0, 0, 0, 0, 0, 0, 460, 1516, 1292, 1372, 438, 770, 59, 207, 0, 0, 0, 0, 0, 0, 0, 0, 679, 1273, 689, 32, 195, 1628, 219, 1399, 9, 0, 0, 0, 0, 0, 0, 0, 766, 301, 722, 585, 1042, 556, 292, 336, 817, 495, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6176, 6308, 6057, 6028, 6434, 6114, 6123, 5912] avg=6144.0000 max=6434 max_rank=4 min=5912 min_rank=7 imbalance_ratio=1.047201 max_load_reduction=42718 max_load_reduction_pct=0.869100 moved_to_echo=24583 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[697, 840, 786, 346, 963, 262, 322, 1057, 283, 185, 128, 0, 0, 0, 0, 0, 86, 926, 1414, 399, 614, 969, 839, 868, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1234, 992, 410, 1232, 174, 0, 1096, 0, 0, 0, 0, 0, 0, 0, 0, 690, 644, 689, 824, 83, 300, 172, 102, 1620, 1005, 0, 0, 0, 0, 0, 0, 519, 311, 447, 135, 1115, 1467, 1194, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 429, 1548, 1143, 1290, 386, 1041, 70, 202, 0, 0, 0, 0, 0, 0, 0, 0, 805, 1250, 860, 20, 173, 1631, 261, 1283, 9, 0, 0, 0, 0, 0, 0, 0, 947, 370, 666, 575, 937, 590, 256, 407, 1009, 500, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5869, 6115, 6167, 6129, 6214, 6109, 6292, 6257] avg=6144.0000 max=6292 max_rank=6 min=5869 min_rank=0 imbalance_ratio=1.024089 max_load_reduction=42860 max_load_reduction_pct=0.871989 moved_to_echo=24872 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[685, 945, 923, 432, 963, 258, 334, 1194, 301, 182, 122, 0, 0, 0, 0, 0, 112, 960, 1366, 367, 547, 1047, 864, 830, 0, 0, 0, 0, 0, 0, 0, 0, 1163, 1196, 853, 444, 1210, 189, 0, 1121, 0, 0, 0, 0, 0, 0, 0, 0, 625, 709, 707, 840, 119, 412, 269, 142, 1627, 771, 0, 0, 0, 0, 0, 0, 393, 275, 482, 124, 1096, 1558, 1100, 973, 0, 0, 0, 0, 0, 0, 0, 0, 431, 1560, 1124, 1378, 509, 950, 36, 174, 0, 0, 0, 0, 0, 0, 0, 0, 671, 1260, 860, 21, 179, 1618, 237, 1249, 9, 0, 0, 0, 0, 0, 0, 0, 930, 316, 765, 606, 877, 634, 298, 380, 773, 477, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6339, 6093, 6176, 6221, 6001, 6162, 6104, 6056] avg=6144.0000 max=6339 max_rank=0 min=6001 min_rank=4 imbalance_ratio=1.031738 max_load_reduction=42813 max_load_reduction_pct=0.871033 moved_to_echo=24323 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[758, 812, 974, 432, 893, 242, 271, 1115, 325, 200, 119, 0, 0, 0, 0, 0, 91, 962, 1089, 308, 540, 940, 1055, 805, 0, 0, 0, 0, 0, 0, 0, 0, 1375, 1185, 891, 437, 1334, 121, 0, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 558, 686, 659, 787, 94, 411, 167, 68, 1780, 838, 0, 0, 0, 0, 0, 0, 453, 232, 561, 203, 1156, 1686, 1059, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 471, 1524, 1395, 1584, 472, 688, 29, 183, 0, 0, 0, 0, 0, 0, 0, 0, 624, 1365, 682, 26, 169, 1716, 203, 1224, 10, 0, 0, 0, 0, 0, 0, 0, 674, 322, 864, 584, 1017, 573, 282, 402, 840, 463, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6141, 5790, 6393, 6048, 6394, 6346, 6019, 6021] avg=6144.0000 max=6394 max_rank=4 min=5790 min_rank=1 imbalance_ratio=1.040690 max_load_reduction=42758 max_load_reduction_pct=0.869914 moved_to_echo=24780 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[644, 785, 1014, 473, 1024, 235, 315, 1093, 311, 182, 129, 0, 0, 0, 0, 0, 150, 1157, 1192, 362, 669, 1064, 987, 840, 0, 0, 0, 0, 0, 0, 0, 0, 1260, 1260, 864, 338, 1173, 123, 0, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 624, 564, 588, 756, 104, 303, 222, 103, 1809, 861, 0, 0, 0, 0, 0, 0, 460, 238, 588, 271, 1053, 1591, 1018, 1086, 0, 0, 0, 0, 0, 0, 0, 0, 474, 1516, 1255, 1444, 429, 756, 28, 217, 0, 0, 0, 0, 0, 0, 0, 0, 732, 1296, 704, 45, 188, 1564, 197, 1268, 12, 0, 0, 0, 0, 0, 0, 0, 784, 335, 807, 592, 1015, 576, 329, 350, 863, 491, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6205, 6421, 6020, 5934, 6305, 6119, 6006, 6142] avg=6144.0000 max=6421 max_rank=1 min=5934 min_rank=3 imbalance_ratio=1.045085 max_load_reduction=42731 max_load_reduction_pct=0.869364 moved_to_echo=24572 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[739, 860, 902, 420, 1037, 209, 301, 1241, 286, 164, 131, 0, 0, 0, 0, 0, 125, 1004, 1340, 378, 593, 1014, 937, 887, 0, 0, 0, 0, 0, 0, 0, 0, 1172, 1166, 809, 439, 1096, 140, 0, 1195, 0, 0, 0, 0, 0, 0, 0, 0, 598, 765, 632, 801, 135, 372, 204, 130, 1716, 783, 0, 0, 0, 0, 0, 0, 441, 236, 530, 134, 1041, 1480, 1068, 941, 0, 0, 0, 0, 0, 0, 0, 0, 455, 1372, 1222, 1357, 463, 880, 38, 203, 0, 0, 0, 0, 0, 0, 0, 0, 864, 1274, 989, 20, 152, 1561, 220, 1158, 8, 0, 0, 0, 0, 0, 0, 0, 825, 359, 962, 643, 869, 633, 298, 440, 785, 510, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6290, 6278, 6017, 6136, 5871, 5990, 6246, 6324] avg=6144.0000 max=6324 max_rank=7 min=5871 min_rank=4 imbalance_ratio=1.029297 max_load_reduction=42828 max_load_reduction_pct=0.871338 moved_to_echo=24431 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[717, 806, 893, 481, 992, 260, 296, 1149, 268, 175, 142, 0, 0, 0, 0, 0, 82, 965, 1411, 378, 577, 1012, 886, 960, 0, 0, 0, 0, 0, 0, 0, 0, 1154, 1150, 968, 400, 1169, 147, 0, 1111, 0, 0, 0, 0, 0, 0, 0, 0, 631, 742, 779, 885, 103, 261, 231, 123, 1724, 874, 0, 0, 0, 0, 0, 0, 344, 209, 446, 123, 1154, 1389, 1016, 944, 0, 0, 0, 0, 0, 0, 0, 0, 457, 1487, 1066, 1309, 405, 1021, 41, 212, 0, 0, 0, 0, 0, 0, 0, 0, 901, 1198, 1030, 14, 125, 1475, 186, 1261, 8, 0, 0, 0, 0, 0, 0, 0, 924, 334, 890, 606, 859, 635, 348, 402, 878, 553, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6179, 6271, 6099, 6353, 5625, 5998, 6198, 6429] avg=6144.0000 max=6429 max_rank=7 min=5625 min_rank=4 imbalance_ratio=1.046387 max_load_reduction=42723 max_load_reduction_pct=0.869202 moved_to_echo=24250 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[3712, 12160, 9728, 11264, 3584, 7040, 384, 1664, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[896, 8064, 10624, 2944, 4736, 8064, 7296, 6912, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[5760, 6784, 7424, 3584, 7680, 2048, 2560, 9088, 2432, 1536, 1024, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6144, 10112, 6784, 256, 1408, 12928, 1792, 10240, 128, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4992, 5504, 5376, 6528, 896, 2944, 1792, 1024, 13824, 6912, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6656, 2688, 6528, 4864, 7680, 4864, 2432, 3200, 6912, 3968, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3584, 2176, 4224, 1408, 8832, 12288, 8832, 8320, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[9600, 9600, 7296, 3456, 9600, 1280, 0, 8704, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[208, 952, 557, 364, 86, 1365, 1051, 252, 865, 327, 1022, 1207, 1632, 1110, 680, 1681, 1000, 1343, 318, 366, 93, 336, 1169, 1037, 365, 680, 1075, 678, 372, 484, 803, 311, 452, 964, 1128, 564, 1702, 1586, 1884, 516, 462, 270, 328, 251, 1530, 1164, 193, 1415, 1017, 250, 1016, 891, 128, 324, 304, 751, 1285, 360, 69, 656, 202, 1404, 482, 1815] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[182, 927, 481, 386, 75, 1159, 979, 275, 628, 365, 1043, 1159, 1589, 1256, 491, 1560, 980, 1272, 320, 440, 96, 330, 1225, 1023, 272, 789, 1135, 826, 442, 546, 881, 267, 381, 1074, 985, 515, 1674, 1453, 1821, 547, 561, 285, 392, 288, 1741, 1231, 174, 1520, 1177, 287, 651, 836, 105, 183, 307, 718, 1366, 305, 104, 893, 140, 1540, 447, 2052] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[226, 836, 553, 366, 89, 1197, 1057, 229, 896, 317, 991, 1117, 1489, 1244, 819, 1723, 905, 1310, 304, 375, 106, 438, 1303, 1042, 328, 698, 1024, 674, 426, 510, 803, 330, 387, 888, 1029, 628, 1759, 1304, 1896, 492, 478, 322, 385, 262, 1637, 1107, 225, 1516, 1030, 327, 916, 878, 101, 270, 390, 813, 1248, 298, 75, 678, 292, 1379, 541, 1876] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[161, 916, 686, 341, 95, 1298, 891, 207, 902, 346, 1016, 1146, 1677, 972, 329, 1329, 1030, 1299, 295, 410, 87, 298, 1026, 915, 494, 742, 1168, 783, 364, 452, 859, 304, 420, 1084, 1253, 614, 1929, 1767, 1881, 479, 412, 298, 371, 206, 1551, 1378, 173, 1479, 1001, 219, 936, 824, 190, 240, 267, 727, 1489, 510, 89, 536, 179, 1298, 407, 2107] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[362, 852, 513, 308, 148, 1242, 1169, 286, 789, 357, 946, 977, 1525, 1396, 930, 1850, 887, 1134, 312, 423, 84, 479, 1106, 951, 225, 711, 916, 552, 394, 563, 832, 556, 470, 849, 1017, 682, 1573, 1349, 1873, 543, 392, 315, 308, 239, 1609, 1068, 190, 1711, 1042, 367, 974, 694, 178, 279, 273, 852, 1341, 327, 109, 858, 149, 1307, 716, 1723] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[77, 861, 628, 310, 50, 1232, 954, 235, 691, 383, 984, 1323, 1634, 961, 141, 1433, 1027, 1562, 383, 354, 106, 213, 1179, 840, 497, 725, 1033, 897, 276, 497, 878, 220, 448, 1068, 1198, 571, 2281, 1880, 1840, 512, 457, 185, 272, 331, 1544, 1242, 166, 1523, 1030, 146, 891, 1019, 65, 155, 209, 757, 1539, 487, 67, 555, 99, 1413, 236, 2382] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[364, 783, 611, 331, 165, 1279, 1040, 273, 832, 312, 838, 1070, 1422, 1218, 1038, 1930, 834, 1100, 238, 363, 105, 595, 1213, 797, 182, 690, 784, 504, 370, 516, 785, 412, 416, 1020, 1126, 678, 1571, 1414, 2110, 590, 327, 316, 341, 156, 1700, 1171, 205, 1559, 1182, 328, 909, 736, 231, 444, 331, 984, 1258, 364, 74, 707, 231, 1200, 665, 1814] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[264, 907, 628, 330, 137, 1266, 931, 256, 854, 301, 922, 1073, 1679, 1322, 873, 1732, 929, 1213, 249, 465, 78, 426, 1068, 908, 237, 715, 910, 495, 347, 534, 774, 491, 360, 986, 1102, 732, 1663, 1564, 2094, 684, 327, 317, 377, 195, 1702, 1101, 175, 1521, 880, 310, 894, 935, 152, 312, 236, 913, 1286, 391, 77, 715, 146, 1182, 585, 1924] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[226, 836, 553, 366, 89, 1197, 1057, 229, 1156, 372, 0, 0, 0, 0, 0, 0, 896, 317, 991, 1117, 986, 1244, 819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 905, 1310, 304, 375, 106, 438, 1303, 1042, 503, 46, 46, 0, 0, 0, 0, 0, 328, 698, 1024, 674, 426, 510, 803, 330, 794, 567, 0, 0, 0, 0, 0, 0, 387, 888, 1029, 628, 919, 1304, 0, 492, 0, 0, 0, 0, 0, 0, 0, 0, 478, 322, 385, 262, 1637, 1107, 225, 1516, 299, 0, 0, 0, 0, 0, 0, 0, 1030, 327, 916, 878, 101, 270, 390, 813, 1524, 0, 0, 0, 0, 0, 0, 0, 1248, 298, 75, 678, 292, 1379, 541, 1531, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6081, 6370, 6378, 6154, 5647, 6231, 6249, 6042] avg=6144.0000 max=6378 max_rank=2 min=5647 min_rank=4 imbalance_ratio=1.038086 max_load_reduction=42774 max_load_reduction_pct=0.870239 moved_to_echo=24169 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[182, 927, 481, 386, 75, 1159, 979, 275, 1046, 358, 0, 0, 0, 0, 0, 0, 628, 365, 1043, 1159, 1053, 1256, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 980, 1272, 320, 440, 96, 330, 1225, 1023, 536, 51, 44, 0, 0, 0, 0, 0, 272, 789, 1135, 826, 442, 546, 881, 267, 756, 514, 0, 0, 0, 0, 0, 0, 381, 1074, 985, 515, 874, 1453, 0, 547, 0, 0, 0, 0, 0, 0, 0, 0, 561, 285, 392, 288, 1741, 1231, 174, 1520, 327, 0, 0, 0, 0, 0, 0, 0, 1177, 287, 651, 836, 105, 183, 307, 718, 1463, 0, 0, 0, 0, 0, 0, 0, 1366, 305, 104, 893, 140, 1540, 447, 1674, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5868, 5995, 6317, 6428, 5829, 6519, 5727, 6469] avg=6144.0000 max=6519 max_rank=5 min=5727 min_rank=6 imbalance_ratio=1.061035 max_load_reduction=42633 max_load_reduction_pct=0.867371 moved_to_echo=24544 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[208, 952, 557, 364, 86, 1365, 1051, 252, 1129, 371, 0, 0, 0, 0, 0, 0, 865, 327, 1022, 1207, 1081, 1110, 680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1343, 318, 366, 93, 336, 1169, 1037, 551, 45, 44, 0, 0, 0, 0, 0, 365, 680, 1075, 678, 372, 484, 803, 311, 769, 552, 0, 0, 0, 0, 0, 0, 452, 964, 1128, 564, 889, 1586, 0, 516, 0, 0, 0, 0, 0, 0, 0, 0, 462, 270, 328, 251, 1530, 1164, 193, 1415, 289, 0, 0, 0, 0, 0, 0, 0, 1017, 250, 1016, 891, 128, 324, 304, 751, 1513, 0, 0, 0, 0, 0, 0, 0, 1285, 360, 69, 656, 202, 1404, 482, 1481, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6335, 6292, 6302, 6089, 6099, 5902, 6194, 5939] avg=6144.0000 max=6335 max_rank=0 min=5902 min_rank=5 imbalance_ratio=1.031087 max_load_reduction=42817 max_load_reduction_pct=0.871114 moved_to_echo=24134 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[362, 852, 513, 308, 148, 1242, 1169, 286, 1241, 368, 0, 0, 0, 0, 0, 0, 789, 357, 946, 977, 1010, 1396, 930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 887, 1134, 312, 423, 84, 479, 1106, 951, 515, 43, 41, 0, 0, 0, 0, 0, 225, 711, 916, 552, 394, 563, 832, 556, 710, 609, 0, 0, 0, 0, 0, 0, 470, 849, 1017, 682, 822, 1349, 0, 543, 0, 0, 0, 0, 0, 0, 0, 0, 392, 315, 308, 239, 1609, 1068, 190, 1711, 275, 0, 0, 0, 0, 0, 0, 0, 1042, 367, 974, 694, 178, 279, 273, 852, 1505, 0, 0, 0, 0, 0, 0, 0, 1341, 327, 109, 858, 149, 1307, 716, 1405, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6489, 6405, 5975, 6068, 5732, 6107, 6164, 6212] avg=6144.0000 max=6489 max_rank=0 min=5732 min_rank=4 imbalance_ratio=1.056152 max_load_reduction=42663 max_load_reduction_pct=0.867981 moved_to_echo=24215 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[161, 916, 686, 341, 95, 1298, 891, 207, 892, 370, 0, 0, 0, 0, 0, 0, 902, 346, 1016, 1146, 1111, 972, 329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1299, 295, 410, 87, 298, 1026, 915, 566, 52, 50, 0, 0, 0, 0, 0, 494, 742, 1168, 783, 364, 452, 859, 304, 871, 437, 0, 0, 0, 0, 0, 0, 420, 1084, 1253, 614, 1008, 1767, 0, 479, 0, 0, 0, 0, 0, 0, 0, 0, 412, 298, 371, 206, 1551, 1378, 173, 1479, 336, 0, 0, 0, 0, 0, 0, 0, 1001, 219, 936, 824, 190, 240, 267, 727, 1511, 0, 0, 0, 0, 0, 0, 0, 1489, 510, 89, 536, 179, 1298, 407, 1719, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5857, 5822, 6028, 6474, 6625, 6204, 5915, 6227] avg=6144.0000 max=6625 max_rank=4 min=5822 min_rank=1 imbalance_ratio=1.078288 max_load_reduction=42527 max_load_reduction_pct=0.865214 moved_to_echo=24971 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[77, 861, 628, 310, 50, 1232, 954, 235, 961, 362, 0, 0, 0, 0, 0, 0, 691, 383, 984, 1323, 1082, 961, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1562, 383, 354, 106, 213, 1179, 840, 552, 59, 60, 0, 0, 0, 0, 0, 497, 725, 1033, 897, 276, 497, 878, 220, 1030, 472, 0, 0, 0, 0, 0, 0, 448, 1068, 1198, 571, 1191, 1880, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 457, 185, 272, 331, 1544, 1242, 166, 1523, 380, 0, 0, 0, 0, 0, 0, 0, 1030, 146, 891, 1019, 65, 155, 209, 757, 1478, 0, 0, 0, 0, 0, 0, 0, 1539, 487, 67, 555, 99, 1413, 236, 1943, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5670, 5565, 6335, 6525, 6868, 6100, 5750, 6339] avg=6144.0000 max=6868 max_rank=4 min=5565 min_rank=1 imbalance_ratio=1.117839 max_load_reduction=42284 max_load_reduction_pct=0.860270 moved_to_echo=25057 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[364, 783, 611, 331, 165, 1279, 1040, 273, 1296, 416, 0, 0, 0, 0, 0, 0, 832, 312, 838, 1070, 942, 1218, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 834, 1100, 238, 363, 105, 595, 1213, 797, 480, 45, 41, 0, 0, 0, 0, 0, 182, 690, 784, 504, 370, 516, 785, 412, 709, 634, 0, 0, 0, 0, 0, 0, 416, 1020, 1126, 678, 821, 1414, 0, 590, 0, 0, 0, 0, 0, 0, 0, 0, 327, 316, 341, 156, 1700, 1171, 205, 1559, 289, 0, 0, 0, 0, 0, 0, 0, 1182, 328, 909, 736, 231, 444, 331, 984, 1694, 0, 0, 0, 0, 0, 0, 0, 1258, 364, 74, 707, 231, 1200, 665, 1480, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6558, 6250, 5811, 5586, 6065, 6064, 6839, 5979] avg=6144.0000 max=6839 max_rank=6 min=5586 min_rank=3 imbalance_ratio=1.113118 max_load_reduction=42313 max_load_reduction_pct=0.860860 moved_to_echo=24947 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[264, 907, 628, 330, 137, 1266, 931, 256, 1163, 412, 0, 0, 0, 0, 0, 0, 854, 301, 922, 1073, 1108, 1322, 873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 929, 1213, 249, 465, 78, 426, 1068, 908, 571, 51, 48, 0, 0, 0, 0, 0, 237, 715, 910, 495, 347, 534, 774, 491, 756, 569, 0, 0, 0, 0, 0, 0, 360, 986, 1102, 732, 859, 1564, 0, 684, 0, 0, 0, 0, 0, 0, 0, 0, 327, 317, 377, 195, 1702, 1101, 175, 1521, 310, 0, 0, 0, 0, 0, 0, 0, 880, 310, 894, 935, 152, 312, 236, 913, 1682, 0, 0, 0, 0, 0, 0, 0, 1286, 391, 77, 715, 146, 1182, 585, 1563, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6294, 6453, 6006, 5828, 6287, 6025, 6314, 5945] avg=6144.0000 max=6453 max_rank=1 min=5828 min_rank=3 imbalance_ratio=1.050293 max_load_reduction=42699 max_load_reduction_pct=0.868713 moved_to_echo=24571 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3456, 2432, 2816, 2048, 13056, 9472, 1536, 12288, 2560, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1920, 7040, 4736, 2816, 896, 10112, 8192, 2048, 8960, 3072, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[8448, 2304, 7296, 6912, 1152, 2304, 2432, 6528, 12416, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6528, 2816, 7808, 9088, 8448, 9600, 5376, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2688, 5760, 8064, 5504, 3072, 4224, 6656, 2944, 6400, 4480, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[10880, 3072, 768, 5632, 1536, 10752, 4096, 12800, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7680, 10240, 2432, 3200, 768, 3200, 9344, 7552, 4352, 512, 384, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3456, 7936, 8960, 4992, 7424, 12416, 0, 4480, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[498, 819, 1478, 317, 621, 529, 717, 820, 1445, 2688, 323, 385, 1030, 596, 1095, 286, 1640, 134, 2225, 606, 645, 312, 18, 797, 906, 762, 657, 487, 664, 776, 1188, 884, 503, 707, 228, 348, 677, 558, 1452, 692, 552, 2393, 128, 479, 313, 690, 418, 3243, 128, 273, 230, 1102, 437, 1064, 325, 488, 282, 355, 1396, 596, 964, 842, 569, 372] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[477, 950, 1358, 516, 485, 539, 736, 712, 1505, 2361, 275, 373, 858, 633, 1023, 375, 1486, 70, 2198, 590, 482, 353, 15, 715, 798, 727, 628, 510, 698, 689, 1295, 1063, 491, 747, 297, 341, 653, 610, 1779, 700, 442, 2129, 121, 511, 393, 695, 624, 3147, 303, 316, 322, 985, 479, 991, 529, 676, 285, 473, 1204, 596, 940, 809, 656, 415] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[708, 1451, 1197, 560, 365, 386, 723, 574, 1138, 2186, 226, 370, 825, 538, 838, 412, 1404, 57, 1882, 548, 458, 247, 36, 761, 669, 461, 652, 589, 593, 597, 1268, 1072, 276, 827, 385, 414, 521, 645, 1858, 734, 861, 1933, 158, 559, 368, 1105, 978, 2909, 628, 424, 575, 789, 535, 1132, 657, 963, 228, 468, 1232, 651, 940, 674, 619, 315] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[582, 1150, 1181, 614, 398, 448, 787, 655, 1183, 2155, 276, 363, 829, 491, 899, 351, 1567, 85, 2029, 604, 537, 317, 60, 844, 576, 613, 699, 673, 781, 756, 1352, 946, 555, 689, 295, 370, 508, 708, 1729, 765, 580, 1941, 149, 563, 440, 879, 677, 3088, 353, 464, 426, 928, 648, 1091, 385, 731, 189, 407, 1414, 620, 1073, 700, 660, 326] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[425, 1063, 1398, 564, 507, 497, 725, 567, 1173, 2292, 214, 469, 914, 556, 1019, 406, 1969, 83, 2207, 474, 470, 400, 27, 766, 1087, 679, 616, 580, 640, 848, 1540, 972, 223, 712, 235, 361, 695, 579, 2081, 745, 391, 1825, 162, 519, 322, 612, 548, 3237, 220, 315, 247, 1021, 657, 946, 281, 515, 391, 401, 1215, 590, 1026, 817, 692, 424] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[446, 736, 1742, 286, 553, 720, 796, 860, 1807, 2796, 408, 346, 1299, 515, 1324, 199, 1645, 68, 2282, 541, 593, 336, 19, 749, 809, 965, 620, 446, 487, 851, 1230, 741, 556, 631, 225, 269, 649, 526, 1201, 673, 474, 2964, 126, 476, 264, 573, 431, 3161, 69, 243, 254, 1090, 369, 943, 274, 341, 205, 332, 1375, 576, 884, 911, 482, 390] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[281, 577, 1582, 314, 648, 582, 734, 731, 2020, 2974, 352, 301, 1349, 449, 1356, 371, 2046, 45, 2679, 464, 602, 395, 8, 683, 910, 881, 653, 491, 656, 872, 1245, 717, 323, 620, 219, 243, 726, 467, 1334, 670, 310, 2753, 94, 497, 229, 531, 415, 3151, 34, 221, 187, 1131, 355, 874, 173, 247, 236, 311, 1370, 627, 918, 869, 523, 526] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[719, 1257, 1265, 556, 402, 447, 765, 588, 1075, 2261, 240, 440, 1038, 481, 811, 419, 1534, 61, 2149, 609, 528, 373, 46, 901, 929, 736, 633, 674, 581, 618, 1339, 907, 220, 740, 238, 363, 668, 556, 1787, 686, 715, 2055, 150, 513, 304, 683, 692, 3549, 320, 294, 296, 916, 530, 1180, 378, 587, 210, 316, 1402, 623, 1078, 746, 645, 330] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[477, 950, 1358, 516, 485, 539, 736, 712, 226, 40, 37, 0, 0, 0, 0, 0, 1505, 1063, 275, 373, 858, 633, 1023, 375, 0, 0, 0, 0, 0, 0, 0, 0, 1486, 70, 2158, 590, 482, 353, 15, 715, 0, 0, 0, 0, 0, 0, 0, 0, 798, 727, 628, 510, 698, 689, 1069, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 491, 747, 297, 341, 653, 610, 1779, 700, 490, 342, 0, 0, 0, 0, 0, 0, 442, 2129, 121, 511, 393, 695, 624, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 303, 316, 322, 985, 479, 991, 529, 676, 1761, 0, 0, 0, 0, 0, 0, 0, 285, 473, 1204, 596, 940, 809, 656, 415, 771, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6076, 6105, 5869, 6182, 6450, 5959, 6362, 6149] avg=6144.0000 max=6450 max_rank=4 min=5869 min_rank=2 imbalance_ratio=1.049805 max_load_reduction=42702 max_load_reduction_pct=0.868774 moved_to_echo=24920 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[582, 1150, 1181, 614, 398, 448, 787, 655, 236, 37, 34, 0, 0, 0, 0, 0, 1183, 970, 276, 363, 829, 491, 899, 351, 0, 0, 0, 0, 0, 0, 0, 0, 1567, 85, 1992, 604, 537, 317, 60, 844, 0, 0, 0, 0, 0, 0, 0, 0, 576, 613, 699, 673, 781, 756, 1116, 946, 0, 0, 0, 0, 0, 0, 0, 0, 555, 689, 295, 370, 508, 708, 1729, 765, 447, 336, 0, 0, 0, 0, 0, 0, 580, 1941, 149, 563, 440, 879, 677, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 353, 464, 426, 928, 648, 1091, 385, 731, 1728, 0, 0, 0, 0, 0, 0, 0, 189, 407, 1414, 620, 1073, 700, 660, 326, 704, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6122, 5362, 6006, 6160, 6402, 6253, 6754, 6093] avg=6144.0000 max=6754 max_rank=6 min=5362 min_rank=1 imbalance_ratio=1.099284 max_load_reduction=42398 max_load_reduction_pct=0.862590 moved_to_echo=25502 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[498, 819, 1478, 317, 621, 529, 717, 820, 207, 40, 43, 0, 0, 0, 0, 0, 1445, 1209, 323, 385, 1030, 596, 1095, 286, 0, 0, 0, 0, 0, 0, 0, 0, 1640, 134, 2185, 606, 645, 312, 18, 797, 0, 0, 0, 0, 0, 0, 0, 0, 906, 762, 657, 487, 664, 776, 981, 884, 0, 0, 0, 0, 0, 0, 0, 0, 503, 707, 228, 348, 677, 558, 1452, 692, 558, 352, 0, 0, 0, 0, 0, 0, 552, 2393, 128, 479, 313, 690, 418, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 128, 273, 230, 1102, 437, 1064, 325, 488, 1815, 0, 0, 0, 0, 0, 0, 0, 282, 355, 1396, 596, 964, 842, 569, 372, 878, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6089, 6369, 6337, 6117, 6075, 6049, 5862, 6254] avg=6144.0000 max=6369 max_rank=1 min=5862 min_rank=6 imbalance_ratio=1.036621 max_load_reduction=42783 max_load_reduction_pct=0.870422 moved_to_echo=24240 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[425, 1063, 1398, 564, 507, 497, 725, 567, 269, 40, 36, 0, 0, 0, 0, 0, 1173, 1032, 214, 469, 914, 556, 1019, 406, 0, 0, 0, 0, 0, 0, 0, 0, 1969, 83, 2167, 474, 470, 400, 27, 766, 0, 0, 0, 0, 0, 0, 0, 0, 1087, 679, 616, 580, 640, 848, 1271, 972, 0, 0, 0, 0, 0, 0, 0, 0, 223, 712, 235, 361, 695, 579, 2081, 745, 476, 352, 0, 0, 0, 0, 0, 0, 391, 1825, 162, 519, 322, 612, 548, 1073, 0, 0, 0, 0, 0, 0, 0, 0, 220, 315, 247, 1021, 657, 946, 281, 515, 1812, 0, 0, 0, 0, 0, 0, 0, 391, 401, 1215, 590, 1026, 817, 692, 424, 748, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6091, 5783, 6356, 6693, 6459, 5452, 6014, 6304] avg=6144.0000 max=6693 max_rank=3 min=5452 min_rank=5 imbalance_ratio=1.089355 max_load_reduction=42459 max_load_reduction_pct=0.863831 moved_to_echo=24229 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[708, 1451, 1197, 560, 365, 386, 723, 574, 221, 34, 35, 0, 0, 0, 0, 0, 1138, 983, 226, 370, 825, 538, 838, 412, 0, 0, 0, 0, 0, 0, 0, 0, 1404, 57, 1848, 548, 458, 247, 36, 761, 0, 0, 0, 0, 0, 0, 0, 0, 669, 461, 652, 589, 593, 597, 1047, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 276, 827, 385, 414, 521, 645, 1858, 734, 454, 316, 0, 0, 0, 0, 0, 0, 861, 1933, 158, 559, 368, 1105, 978, 965, 0, 0, 0, 0, 0, 0, 0, 0, 628, 424, 575, 789, 535, 1132, 657, 963, 1628, 0, 0, 0, 0, 0, 0, 0, 228, 468, 1232, 651, 940, 674, 619, 315, 714, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6254, 5330, 5359, 5680, 6430, 6927, 7331, 5841] avg=6144.0000 max=7331 max_rank=6 min=5330 min_rank=1 imbalance_ratio=1.193197 max_load_reduction=41821 max_load_reduction_pct=0.850850 moved_to_echo=26529 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[446, 736, 1742, 286, 553, 720, 796, 860, 215, 41, 44, 0, 0, 0, 0, 0, 1807, 1259, 408, 346, 1299, 515, 1324, 199, 0, 0, 0, 0, 0, 0, 0, 0, 1645, 68, 2241, 541, 593, 336, 19, 749, 0, 0, 0, 0, 0, 0, 0, 0, 809, 965, 620, 446, 487, 851, 1015, 741, 0, 0, 0, 0, 0, 0, 0, 0, 556, 631, 225, 269, 649, 526, 1201, 673, 580, 343, 0, 0, 0, 0, 0, 0, 474, 2964, 126, 476, 264, 573, 431, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 69, 243, 254, 1090, 369, 943, 274, 341, 1769, 0, 0, 0, 0, 0, 0, 0, 205, 332, 1375, 576, 884, 911, 482, 390, 913, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6439, 7157, 6192, 5934, 5653, 6357, 5352, 6068] avg=6144.0000 max=7157 max_rank=1 min=5352 min_rank=6 imbalance_ratio=1.164876 max_load_reduction=41995 max_load_reduction_pct=0.854390 moved_to_echo=23430 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[281, 577, 1582, 314, 648, 582, 734, 731, 217, 48, 47, 0, 0, 0, 0, 0, 2020, 1339, 352, 301, 1349, 449, 1356, 371, 0, 0, 0, 0, 0, 0, 0, 0, 2046, 45, 2631, 464, 602, 395, 8, 683, 0, 0, 0, 0, 0, 0, 0, 0, 910, 881, 653, 491, 656, 872, 1028, 717, 0, 0, 0, 0, 0, 0, 0, 0, 323, 620, 219, 243, 726, 467, 1334, 670, 617, 342, 0, 0, 0, 0, 0, 0, 310, 2753, 94, 497, 229, 531, 415, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 34, 221, 187, 1131, 355, 874, 173, 247, 1763, 0, 0, 0, 0, 0, 0, 0, 236, 311, 1370, 627, 918, 869, 523, 526, 971, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5761, 7537, 6874, 6208, 5561, 5875, 4985, 6351] avg=6144.0000 max=7537 max_rank=1 min=4985 min_rank=6 imbalance_ratio=1.226725 max_load_reduction=41615 max_load_reduction_pct=0.846659 moved_to_echo=22772 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[719, 1257, 1265, 556, 402, 447, 765, 588, 239, 42, 40, 0, 0, 0, 0, 0, 1075, 1005, 240, 440, 1038, 481, 811, 419, 0, 0, 0, 0, 0, 0, 0, 0, 1534, 61, 2107, 609, 528, 373, 46, 901, 0, 0, 0, 0, 0, 0, 0, 0, 929, 736, 633, 674, 581, 618, 1100, 907, 0, 0, 0, 0, 0, 0, 0, 0, 220, 740, 238, 363, 668, 556, 1787, 686, 474, 390, 0, 0, 0, 0, 0, 0, 715, 2055, 150, 513, 304, 683, 692, 1168, 0, 0, 0, 0, 0, 0, 0, 0, 320, 294, 296, 916, 530, 1180, 378, 587, 1991, 0, 0, 0, 0, 0, 0, 0, 210, 316, 1402, 623, 1078, 746, 645, 330, 742, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6320, 5509, 6159, 6178, 6122, 6280, 6492, 6092] avg=6144.0000 max=6492 max_rank=6 min=5509 min_rank=1 imbalance_ratio=1.056641 max_load_reduction=42660 max_load_reduction_pct=0.867920 moved_to_echo=24986 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4352, 18048, 1152, 4224, 2688, 5888, 4864, 8448, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[13312, 640, 17408, 4480, 4352, 2816, 256, 6272, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6784, 5888, 5248, 4480, 5120, 6016, 8704, 7424, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2048, 3072, 10624, 4992, 7936, 6400, 4864, 3200, 6528, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3200, 5760, 2176, 2816, 5120, 4736, 13312, 5760, 4096, 2816, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2176, 2560, 2560, 8064, 4096, 8320, 3072, 4608, 14336, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[11392, 8960, 2432, 3072, 8192, 4352, 8448, 2944, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[4224, 8064, 11264, 3840, 4096, 4224, 6016, 5632, 1920, 384, 384, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[34, 1959, 1536, 398, 1110, 1364, 1027, 823, 641, 448, 1449, 326, 306, 424, 471, 1135, 1868, 260, 177, 220, 196, 300, 255, 206, 415, 376, 1822, 608, 1681, 1131, 870, 126, 386, 94, 162, 756, 1083, 844, 307, 1037, 421, 604, 1002, 912, 595, 20, 1802, 19, 340, 620, 249, 1762, 544, 987, 878, 1198, 1062, 484, 282, 1259, 1015, 2071, 1377, 1018] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[33, 1529, 1515, 580, 1124, 1242, 1192, 763, 475, 480, 1480, 249, 512, 394, 430, 1466, 1392, 569, 164, 192, 303, 265, 184, 225, 395, 404, 1507, 620, 1477, 1116, 949, 196, 548, 217, 423, 523, 965, 996, 341, 989, 433, 309, 1124, 874, 974, 82, 1781, 46, 439, 350, 277, 2275, 568, 815, 792, 1211, 1084, 444, 387, 1067, 1208, 1656, 1399, 1133] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[12, 1348, 1466, 677, 1244, 1143, 793, 1265, 343, 614, 2138, 155, 207, 442, 379, 972, 1313, 172, 181, 90, 191, 416, 337, 278, 574, 367, 2054, 458, 2138, 997, 714, 84, 357, 20, 74, 975, 1113, 464, 171, 1221, 325, 439, 338, 897, 817, 9, 2408, 8, 210, 672, 271, 1844, 430, 1304, 965, 1289, 878, 501, 260, 1441, 1065, 2012, 1668, 1144] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[105, 1752, 1537, 416, 975, 976, 1352, 650, 523, 395, 1177, 162, 429, 351, 784, 1212, 1573, 499, 291, 200, 282, 281, 210, 190, 333, 447, 817, 616, 947, 1020, 878, 745, 778, 605, 518, 414, 1135, 1219, 378, 1189, 412, 302, 1656, 873, 982, 181, 1733, 299, 678, 193, 197, 2567, 664, 812, 640, 858, 945, 317, 435, 815, 887, 1902, 1018, 1425] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[35, 1429, 1590, 601, 1139, 1076, 952, 902, 422, 532, 1614, 178, 301, 389, 430, 1495, 1441, 228, 274, 165, 238, 363, 218, 249, 561, 372, 1989, 620, 1419, 951, 693, 130, 457, 67, 179, 679, 1414, 796, 265, 1277, 385, 302, 706, 921, 936, 41, 1917, 27, 391, 359, 307, 2147, 586, 900, 868, 1251, 961, 430, 409, 1242, 962, 2294, 1469, 1211] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 713, 1538, 1185, 1120, 980, 708, 1559, 287, 768, 2549, 139, 164, 468, 361, 646, 1510, 121, 190, 48, 131, 635, 402, 224, 594, 304, 2120, 212, 2378, 790, 524, 77, 378, 4, 56, 1315, 938, 309, 132, 1246, 298, 383, 114, 901, 669, 0, 2571, 1, 112, 971, 218, 1779, 449, 1635, 1258, 1190, 833, 389, 212, 1562, 1034, 1887, 1403, 1460] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 1020, 1513, 1084, 1077, 1113, 706, 1714, 321, 826, 2500, 97, 171, 474, 349, 770, 1832, 81, 107, 62, 124, 580, 317, 156, 510, 359, 2420, 284, 2431, 717, 576, 60, 302, 1, 14, 1267, 841, 172, 101, 1059, 327, 439, 64, 916, 593, 0, 2704, 0, 56, 914, 219, 1742, 464, 1517, 1406, 1076, 655, 515, 159, 1589, 1180, 1875, 1322, 1312] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[30, 1716, 1707, 723, 1109, 1014, 1000, 776, 643, 579, 1541, 179, 343, 374, 427, 1323, 1836, 422, 145, 148, 211, 361, 172, 192, 696, 393, 1438, 627, 1274, 897, 892, 200, 449, 110, 305, 546, 1415, 901, 295, 984, 375, 594, 814, 849, 962, 42, 1961, 56, 396, 640, 265, 1999, 567, 923, 825, 1071, 1133, 386, 325, 1048, 1025, 1965, 1488, 1050] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[33, 1234, 0, 580, 1124, 1242, 1192, 763, 0, 0, 0, 0, 0, 0, 0, 0, 475, 480, 1480, 249, 512, 394, 430, 1466, 557, 147, 29, 0, 0, 0, 0, 0, 1392, 569, 164, 192, 303, 265, 184, 225, 1656, 858, 0, 0, 0, 0, 0, 0, 395, 404, 950, 620, 1477, 1116, 949, 196, 0, 0, 0, 0, 0, 0, 0, 0, 548, 217, 423, 523, 965, 996, 341, 989, 657, 734, 0, 0, 0, 0, 0, 0, 433, 309, 1124, 874, 974, 82, 1781, 46, 425, 295, 0, 0, 0, 0, 0, 0, 439, 350, 277, 1394, 568, 815, 792, 1211, 0, 0, 0, 0, 0, 0, 0, 0, 1084, 444, 387, 1067, 1208, 0, 945, 1133, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6168, 6219, 5808, 6107, 6393, 6343, 5846, 6268] avg=6144.0000 max=6393 max_rank=4 min=5808 min_rank=2 imbalance_ratio=1.040527 max_load_reduction=42759 max_load_reduction_pct=0.869934 moved_to_echo=24850 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[35, 1153, 0, 601, 1139, 1076, 952, 902, 0, 0, 0, 0, 0, 0, 0, 0, 422, 532, 1614, 178, 301, 389, 430, 1495, 735, 139, 31, 0, 0, 0, 0, 0, 1441, 228, 274, 165, 238, 363, 218, 249, 2294, 900, 0, 0, 0, 0, 0, 0, 561, 372, 1254, 620, 1419, 951, 693, 130, 0, 0, 0, 0, 0, 0, 0, 0, 457, 67, 179, 679, 1414, 796, 265, 1277, 690, 692, 0, 0, 0, 0, 0, 0, 385, 302, 706, 921, 936, 41, 1917, 27, 447, 276, 0, 0, 0, 0, 0, 0, 391, 359, 307, 1316, 586, 900, 868, 1251, 0, 0, 0, 0, 0, 0, 0, 0, 961, 430, 409, 1242, 962, 0, 991, 1211, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5858, 6266, 6370, 6000, 6516, 5958, 5978, 6206] avg=6144.0000 max=6516 max_rank=4 min=5858 min_rank=0 imbalance_ratio=1.060547 max_load_reduction=42636 max_load_reduction_pct=0.867432 moved_to_echo=24658 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[34, 1581, 0, 398, 1110, 1364, 1027, 823, 0, 0, 0, 0, 0, 0, 0, 0, 641, 448, 1449, 326, 306, 424, 471, 1135, 673, 114, 29, 0, 0, 0, 0, 0, 1868, 260, 177, 220, 196, 300, 255, 206, 2071, 870, 0, 0, 0, 0, 0, 0, 415, 376, 1149, 608, 1681, 1131, 870, 126, 0, 0, 0, 0, 0, 0, 0, 0, 386, 94, 162, 756, 1083, 844, 307, 1037, 666, 568, 0, 0, 0, 0, 0, 0, 421, 604, 1002, 912, 595, 20, 1802, 19, 419, 378, 0, 0, 0, 0, 0, 0, 340, 620, 249, 1080, 544, 987, 878, 1198, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 484, 282, 1259, 1015, 0, 929, 1018, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6337, 6016, 6423, 6356, 5903, 6172, 5896, 6049] avg=6144.0000 max=6423 max_rank=2 min=5896 min_rank=6 imbalance_ratio=1.045410 max_load_reduction=42729 max_load_reduction_pct=0.869324 moved_to_echo=24020 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[12, 1088, 0, 677, 1244, 1143, 793, 1265, 0, 0, 0, 0, 0, 0, 0, 0, 343, 614, 2138, 155, 207, 442, 379, 972, 759, 119, 35, 0, 0, 0, 0, 0, 1313, 172, 181, 90, 191, 416, 337, 278, 2012, 831, 0, 0, 0, 0, 0, 0, 574, 367, 1295, 458, 2138, 997, 714, 84, 0, 0, 0, 0, 0, 0, 0, 0, 357, 20, 74, 975, 1113, 464, 171, 1221, 635, 595, 0, 0, 0, 0, 0, 0, 325, 439, 338, 897, 817, 9, 2408, 8, 507, 260, 0, 0, 0, 0, 0, 0, 210, 672, 271, 1130, 430, 1304, 965, 1289, 0, 0, 0, 0, 0, 0, 0, 0, 878, 501, 260, 1441, 1065, 0, 1126, 1144, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6222, 6163, 5821, 6627, 5625, 6008, 6271, 6415] avg=6144.0000 max=6627 max_rank=3 min=5625 min_rank=4 imbalance_ratio=1.078613 max_load_reduction=42525 max_load_reduction_pct=0.865173 moved_to_echo=24319 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[105, 1414, 0, 416, 975, 976, 1352, 650, 0, 0, 0, 0, 0, 0, 0, 0, 523, 395, 1177, 162, 429, 351, 784, 1212, 302, 166, 21, 0, 0, 0, 0, 0, 1573, 499, 291, 200, 282, 281, 210, 190, 1902, 871, 0, 0, 0, 0, 0, 0, 333, 447, 515, 616, 947, 1020, 878, 745, 0, 0, 0, 0, 0, 0, 0, 0, 778, 605, 518, 414, 1135, 1219, 378, 1189, 666, 828, 0, 0, 0, 0, 0, 0, 412, 302, 1656, 873, 982, 181, 1733, 299, 309, 338, 0, 0, 0, 0, 0, 0, 678, 193, 197, 1573, 664, 812, 640, 858, 0, 0, 0, 0, 0, 0, 0, 0, 945, 317, 435, 815, 887, 0, 688, 1425, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5888, 5522, 6299, 5501, 7730, 7085, 5615, 5512] avg=6144.0000 max=7730 max_rank=4 min=5501 min_rank=3 imbalance_ratio=1.258138 max_load_reduction=41422 max_load_reduction_pct=0.842733 moved_to_echo=25942 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 576, 0, 1185, 1120, 980, 708, 1559, 0, 0, 0, 0, 0, 0, 0, 0, 287, 768, 2549, 139, 164, 468, 361, 646, 783, 115, 29, 0, 0, 0, 0, 0, 1510, 121, 190, 48, 131, 635, 402, 224, 1887, 872, 0, 0, 0, 0, 0, 0, 594, 304, 1337, 212, 2378, 790, 524, 77, 0, 0, 0, 0, 0, 0, 0, 0, 378, 4, 56, 1315, 938, 309, 132, 1246, 666, 574, 0, 0, 0, 0, 0, 0, 298, 383, 114, 901, 669, 0, 2571, 1, 427, 137, 0, 0, 0, 0, 0, 0, 112, 971, 218, 1090, 449, 1635, 1258, 1190, 0, 0, 0, 0, 0, 0, 0, 0, 833, 389, 212, 1562, 1034, 0, 947, 1460, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6128, 6309, 6020, 6216, 5618, 5501, 6923, 6437] avg=6144.0000 max=6923 max_rank=6 min=5501 min_rank=5 imbalance_ratio=1.126790 max_load_reduction=42229 max_load_reduction_pct=0.859151 moved_to_echo=24479 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 823, 0, 1084, 1077, 1113, 706, 1714, 0, 0, 0, 0, 0, 0, 0, 0, 321, 826, 2500, 97, 171, 474, 349, 770, 894, 113, 28, 0, 0, 0, 0, 0, 1832, 81, 107, 62, 124, 580, 317, 156, 1875, 857, 0, 0, 0, 0, 0, 0, 510, 359, 1526, 284, 2431, 717, 576, 60, 0, 0, 0, 0, 0, 0, 0, 0, 302, 1, 14, 1267, 841, 172, 101, 1059, 656, 562, 0, 0, 0, 0, 0, 0, 327, 439, 64, 916, 593, 0, 2704, 0, 402, 197, 0, 0, 0, 0, 0, 0, 56, 914, 219, 1067, 464, 1517, 1406, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 655, 515, 159, 1589, 1180, 0, 892, 1312, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6517, 6543, 5991, 6463, 4975, 5642, 6719, 6302] avg=6144.0000 max=6719 max_rank=6 min=4975 min_rank=4 imbalance_ratio=1.093587 max_load_reduction=42433 max_load_reduction_pct=0.863302 moved_to_echo=23638 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[30, 1382, 0, 723, 1109, 1014, 1000, 776, 0, 0, 0, 0, 0, 0, 0, 0, 643, 579, 1541, 179, 343, 374, 427, 1323, 535, 134, 36, 0, 0, 0, 0, 0, 1836, 422, 145, 148, 211, 361, 172, 192, 1965, 968, 0, 0, 0, 0, 0, 0, 696, 393, 903, 627, 1274, 897, 892, 200, 0, 0, 0, 0, 0, 0, 0, 0, 449, 110, 305, 546, 1415, 901, 295, 984, 739, 648, 0, 0, 0, 0, 0, 0, 375, 594, 814, 849, 962, 42, 1961, 56, 456, 334, 0, 0, 0, 0, 0, 0, 396, 640, 265, 1217, 567, 923, 825, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 1133, 386, 325, 1048, 1025, 0, 996, 1050, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6034, 6114, 6420, 5882, 6392, 6443, 5904, 5963] avg=6144.0000 max=6443 max_rank=5 min=5882 min_rank=3 imbalance_ratio=1.048665 max_load_reduction=42709 max_load_reduction_pct=0.868917 moved_to_echo=24702 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[256, 9344, 0, 5760, 8960, 8960, 7808, 8576, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7552, 3584, 2560, 10112, 8448, 0, 7552, 9856, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2688, 4736, 2048, 9984, 4352, 8960, 7680, 9216, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[12800, 2432, 1536, 1152, 1792, 3328, 2176, 1792, 15744, 7040, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3712, 4736, 14464, 1536, 2560, 3328, 3712, 9088, 5248, 1152, 256, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3712, 1152, 1792, 6528, 8960, 5760, 2048, 9088, 5376, 5248, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4096, 3072, 8960, 4096, 13824, 7680, 6144, 1664, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3072, 3456, 5888, 7168, 6528, 384, 16896, 512, 3456, 2304, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[718, 1196, 1246, 326, 583, 230, 2579, 264, 66, 75, 1180, 970, 1225, 0, 66, 1229, 688, 952, 2854, 88, 296, 286, 334, 38, 187, 1662, 793, 234, 529, 283, 2939, 438, 1448, 741, 316, 1214, 1041, 365, 1647, 881, 301, 13, 170, 2052, 157, 421, 1021, 440, 1032, 1288, 893, 670, 152, 801, 0, 1292, 766, 507, 466, 868, 307, 508, 1560, 1260] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[922, 942, 1582, 252, 611, 180, 1523, 258, 167, 265, 1353, 967, 867, 4, 112, 1937, 722, 723, 1513, 67, 493, 121, 252, 813, 486, 1042, 641, 198, 741, 471, 2810, 391, 1139, 968, 692, 2511, 906, 704, 1609, 793, 302, 186, 378, 1916, 338, 359, 541, 607, 1138, 1077, 659, 353, 312, 713, 57, 1526, 449, 544, 514, 1259, 218, 351, 1879, 728] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[773, 1614, 1206, 321, 399, 141, 3227, 445, 38, 3, 1012, 887, 1323, 0, 31, 1154, 714, 959, 2985, 63, 257, 335, 261, 8, 283, 2057, 849, 145, 483, 270, 3292, 449, 1682, 627, 217, 1095, 741, 295, 1455, 1098, 201, 1, 156, 2030, 140, 361, 1153, 473, 944, 1176, 977, 723, 82, 704, 0, 1274, 829, 462, 477, 843, 217, 452, 1247, 1036] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[654, 1450, 910, 316, 217, 169, 2952, 475, 8, 5, 884, 807, 1668, 0, 16, 795, 575, 1008, 3565, 31, 225, 505, 288, 2, 188, 2295, 807, 147, 347, 249, 3542, 440, 1850, 575, 185, 762, 811, 298, 1357, 990, 105, 4, 117, 2080, 92, 516, 1490, 456, 970, 1239, 1124, 906, 85, 635, 5, 1259, 1085, 401, 450, 837, 209, 483, 1120, 1116] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[788, 1871, 1236, 412, 502, 254, 2385, 316, 46, 11, 995, 905, 1703, 0, 33, 1554, 653, 805, 2687, 61, 279, 364, 359, 26, 174, 1536, 852, 216, 485, 298, 2836, 427, 1753, 789, 266, 1362, 680, 334, 1966, 871, 327, 6, 62, 1779, 234, 344, 1215, 452, 1023, 1482, 996, 509, 140, 631, 0, 1422, 865, 489, 447, 829, 309, 402, 1278, 821] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[620, 1650, 1221, 386, 432, 217, 2386, 251, 55, 60, 1053, 891, 1428, 1, 50, 1059, 765, 814, 2400, 55, 372, 271, 308, 42, 167, 1466, 792, 126, 614, 252, 3368, 282, 1419, 674, 299, 1448, 816, 309, 2098, 1223, 172, 18, 207, 2355, 160, 338, 1410, 421, 1020, 1397, 840, 743, 133, 739, 6, 1241, 754, 504, 462, 1121, 288, 440, 1236, 1007] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[481, 1608, 822, 285, 154, 111, 2776, 541, 3, 0, 786, 671, 1947, 0, 4, 703, 563, 917, 3417, 26, 133, 886, 285, 0, 191, 2608, 788, 101, 256, 196, 4023, 303, 2137, 462, 161, 554, 881, 223, 1214, 1194, 53, 0, 51, 1755, 55, 639, 1594, 565, 857, 1151, 1297, 1341, 92, 612, 0, 1157, 1187, 379, 392, 726, 147, 350, 981, 1360] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[426, 1718, 861, 404, 85, 221, 2581, 549, 1, 0, 757, 690, 2457, 0, 1, 630, 488, 964, 3702, 16, 112, 1162, 366, 0, 112, 2771, 781, 101, 249, 284, 3976, 278, 1905, 471, 129, 592, 738, 230, 1465, 784, 36, 0, 20, 1663, 60, 514, 1722, 577, 872, 1289, 1454, 1018, 44, 554, 0, 1155, 1216, 424, 476, 648, 128, 293, 852, 1080] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[773, 1614, 1206, 321, 399, 141, 1951, 445, 0, 0, 0, 0, 0, 0, 0, 0, 38, 3, 1012, 887, 1323, 0, 31, 1154, 1385, 88, 0, 0, 0, 0, 0, 0, 714, 959, 2985, 63, 257, 335, 261, 8, 606, 0, 0, 0, 0, 0, 0, 0, 283, 2057, 849, 145, 483, 270, 1886, 449, 0, 0, 0, 0, 0, 0, 0, 0, 256, 627, 217, 1095, 741, 295, 1455, 1098, 0, 0, 0, 0, 0, 0, 0, 0, 201, 1, 156, 2030, 140, 361, 1153, 473, 1406, 41, 0, 0, 0, 0, 0, 0, 944, 1176, 977, 723, 82, 704, 0, 1259, 0, 0, 0, 0, 0, 0, 0, 0, 829, 462, 477, 843, 217, 452, 1247, 1036, 582, 15, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6850, 5921, 6188, 6422, 5784, 5962, 5865, 6160] avg=6144.0000 max=6850 max_rank=0 min=5784 min_rank=4 imbalance_ratio=1.114909 max_load_reduction=42302 max_load_reduction_pct=0.860636 moved_to_echo=23771 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[718, 1196, 1246, 326, 583, 230, 1559, 264, 0, 0, 0, 0, 0, 0, 0, 0, 66, 75, 1180, 970, 1225, 0, 66, 1229, 1192, 71, 0, 0, 0, 0, 0, 0, 688, 952, 2854, 88, 296, 286, 334, 38, 484, 0, 0, 0, 0, 0, 0, 0, 187, 1662, 793, 234, 529, 283, 1684, 438, 0, 0, 0, 0, 0, 0, 0, 0, 221, 741, 316, 1214, 1041, 365, 1647, 881, 0, 0, 0, 0, 0, 0, 0, 0, 301, 13, 170, 2052, 157, 421, 1021, 440, 1255, 35, 0, 0, 0, 0, 0, 0, 1032, 1288, 893, 670, 152, 801, 0, 1276, 0, 0, 0, 0, 0, 0, 0, 0, 766, 507, 466, 868, 307, 508, 1560, 1260, 465, 16, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6122, 6074, 6020, 5810, 6426, 5865, 6112, 6723] avg=6144.0000 max=6723 max_rank=7 min=5810 min_rank=3 imbalance_ratio=1.094238 max_load_reduction=42429 max_load_reduction_pct=0.863220 moved_to_echo=25126 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[654, 1450, 910, 316, 217, 169, 1785, 475, 0, 0, 0, 0, 0, 0, 0, 0, 8, 5, 884, 807, 1668, 0, 16, 795, 1523, 81, 0, 0, 0, 0, 0, 0, 575, 1008, 3565, 31, 225, 505, 288, 2, 554, 0, 0, 0, 0, 0, 0, 0, 188, 2295, 807, 147, 347, 249, 2029, 440, 0, 0, 0, 0, 0, 0, 0, 0, 282, 575, 185, 762, 811, 298, 1357, 990, 0, 0, 0, 0, 0, 0, 0, 0, 105, 4, 117, 2080, 92, 516, 1490, 456, 1513, 45, 0, 0, 0, 0, 0, 0, 970, 1239, 1124, 906, 85, 635, 5, 1244, 0, 0, 0, 0, 0, 0, 0, 0, 1085, 401, 450, 837, 209, 483, 1120, 1116, 532, 15, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5976, 5787, 6753, 6502, 5260, 6418, 6208, 6248] avg=6144.0000 max=6753 max_rank=2 min=5260 min_rank=4 imbalance_ratio=1.099121 max_load_reduction=42399 max_load_reduction_pct=0.862610 moved_to_echo=24134 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[788, 1871, 1236, 412, 502, 254, 1443, 316, 0, 0, 0, 0, 0, 0, 0, 0, 46, 11, 995, 905, 1703, 0, 33, 1554, 1443, 65, 0, 0, 0, 0, 0, 0, 653, 805, 2687, 61, 279, 364, 359, 26, 447, 0, 0, 0, 0, 0, 0, 0, 174, 1536, 852, 216, 485, 298, 1625, 427, 0, 0, 0, 0, 0, 0, 0, 0, 268, 789, 266, 1362, 680, 334, 1966, 871, 0, 0, 0, 0, 0, 0, 0, 0, 327, 6, 62, 1779, 234, 344, 1215, 452, 1211, 42, 0, 0, 0, 0, 0, 0, 1023, 1482, 996, 509, 140, 631, 0, 1405, 0, 0, 0, 0, 0, 0, 0, 0, 865, 489, 447, 829, 309, 402, 1278, 821, 430, 17, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6822, 6755, 5681, 5613, 6536, 5672, 6186, 5887] avg=6144.0000 max=6822 max_rank=0 min=5613 min_rank=3 imbalance_ratio=1.110352 max_load_reduction=42330 max_load_reduction_pct=0.861206 moved_to_echo=24281 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[922, 942, 1582, 252, 611, 180, 922, 258, 0, 0, 0, 0, 0, 0, 0, 0, 167, 265, 1353, 967, 867, 4, 112, 1937, 938, 41, 0, 0, 0, 0, 0, 0, 722, 723, 1513, 67, 493, 121, 252, 813, 286, 0, 0, 0, 0, 0, 0, 0, 486, 1042, 641, 198, 741, 471, 1610, 391, 0, 0, 0, 0, 0, 0, 0, 0, 174, 968, 692, 2511, 906, 704, 1609, 793, 0, 0, 0, 0, 0, 0, 0, 0, 302, 186, 378, 1916, 338, 359, 541, 607, 1200, 27, 0, 0, 0, 0, 0, 0, 1138, 1077, 659, 353, 312, 713, 57, 1507, 0, 0, 0, 0, 0, 0, 0, 0, 449, 544, 514, 1259, 218, 351, 1879, 728, 274, 19, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5669, 6651, 4990, 5580, 8357, 5854, 5816, 6235] avg=6144.0000 max=8357 max_rank=4 min=4990 min_rank=2 imbalance_ratio=1.360189 max_load_reduction=40795 max_load_reduction_pct=0.829976 moved_to_echo=26262 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[481, 1608, 822, 285, 154, 111, 1678, 541, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 786, 671, 1947, 0, 4, 703, 1760, 76, 0, 0, 0, 0, 0, 0, 563, 917, 3417, 26, 133, 886, 285, 0, 521, 0, 0, 0, 0, 0, 0, 0, 191, 2608, 788, 101, 256, 196, 2305, 303, 0, 0, 0, 0, 0, 0, 0, 0, 325, 462, 161, 554, 881, 223, 1214, 1194, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 51, 1755, 55, 639, 1594, 565, 1718, 52, 0, 0, 0, 0, 0, 0, 857, 1151, 1297, 1341, 92, 612, 0, 1143, 0, 0, 0, 0, 0, 0, 0, 0, 1187, 379, 392, 726, 147, 350, 981, 1360, 501, 14, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5680, 5950, 6748, 6748, 5014, 6482, 6493, 6037] avg=6144.0000 max=6748 max_rank=2 min=5014 min_rank=4 imbalance_ratio=1.098307 max_load_reduction=42404 max_load_reduction_pct=0.862712 moved_to_echo=24026 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[426, 1718, 861, 404, 85, 221, 1561, 549, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 757, 690, 2457, 0, 1, 630, 1569, 71, 0, 0, 0, 0, 0, 0, 488, 964, 3702, 16, 112, 1162, 366, 0, 484, 0, 0, 0, 0, 0, 0, 0, 112, 2771, 781, 101, 249, 284, 2278, 278, 0, 0, 0, 0, 0, 0, 0, 0, 290, 471, 129, 592, 738, 230, 1465, 784, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 20, 1663, 60, 514, 1722, 577, 1698, 46, 0, 0, 0, 0, 0, 0, 872, 1289, 1454, 1018, 44, 554, 0, 1141, 0, 0, 0, 0, 0, 0, 0, 0, 1216, 424, 476, 648, 128, 293, 852, 1080, 465, 14, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5825, 6176, 7294, 6854, 4699, 6336, 6372, 5596] avg=6144.0000 max=7294 max_rank=2 min=4699 min_rank=4 imbalance_ratio=1.187174 max_load_reduction=41858 max_load_reduction_pct=0.851603 moved_to_echo=23003 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[620, 1650, 1221, 386, 432, 217, 1431, 251, 0, 0, 0, 0, 0, 0, 0, 0, 55, 60, 1053, 891, 1428, 1, 50, 1059, 1172, 69, 0, 0, 0, 0, 0, 0, 765, 814, 2400, 55, 372, 271, 308, 42, 451, 0, 0, 0, 0, 0, 0, 0, 167, 1466, 792, 126, 614, 252, 1924, 282, 0, 0, 0, 0, 0, 0, 0, 0, 209, 674, 299, 1448, 816, 309, 2098, 1223, 0, 0, 0, 0, 0, 0, 0, 0, 172, 18, 207, 2355, 160, 338, 1410, 421, 1444, 38, 0, 0, 0, 0, 0, 0, 1020, 1397, 840, 743, 133, 739, 6, 1222, 0, 0, 0, 0, 0, 0, 0, 0, 754, 504, 462, 1121, 288, 440, 1236, 1007, 435, 19, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6208, 5838, 5478, 5623, 7076, 6563, 6100, 6266] avg=6144.0000 max=7076 max_rank=4 min=5478 min_rank=2 imbalance_ratio=1.151693 max_load_reduction=42076 max_load_reduction_pct=0.856038 moved_to_echo=26005 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5504, 12160, 9088, 2816, 3072, 1536, 12416, 3200, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1536, 256, 1280, 15744, 1280, 3584, 10240, 4096, 11520, 384, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7168, 3712, 3712, 7168, 1920, 3328, 10240, 8448, 3712, 256, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7936, 10112, 8320, 6272, 1152, 5504, 128, 10240, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5248, 7168, 23168, 512, 2176, 3968, 2560, 1024, 3840, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[384, 512, 8064, 6912, 12672, 128, 384, 9088, 11008, 640, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2048, 5376, 2304, 9600, 6656, 2816, 12928, 7936, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[1792, 15488, 6400, 1280, 3712, 2304, 15360, 3072, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[886, 260, 2, 2910, 76, 55, 261, 118, 2, 691, 1411, 917, 1107, 429, 450, 1541, 0, 35, 853, 30, 288, 561, 515, 2492, 140, 244, 715, 2, 683, 285, 0, 26, 104, 1683, 493, 37, 2, 0, 0, 1678, 1281, 314, 241, 1978, 4805, 357, 843, 3087, 364, 1171, 4, 737, 380, 1155, 5796, 0, 3699, 16, 1, 45, 363, 0, 373, 160] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[702, 150, 0, 4230, 80, 3, 217, 89, 0, 512, 1127, 746, 1023, 693, 421, 1216, 1, 6, 756, 3, 73, 510, 393, 2126, 136, 66, 663, 0, 366, 171, 0, 5, 91, 1864, 205, 5, 5, 4, 6, 2433, 1157, 254, 185, 1667, 5443, 128, 1056, 3863, 375, 143, 1, 1568, 329, 1025, 5984, 2, 4047, 0, 0, 26, 281, 1, 382, 138] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[976, 515, 100, 832, 172, 693, 390, 140, 166, 942, 1015, 766, 1028, 185, 447, 1304, 17, 259, 913, 92, 2183, 683, 890, 1295, 188, 1095, 566, 316, 1494, 535, 35, 185, 231, 1655, 557, 273, 21, 340, 69, 2573, 1159, 275, 840, 1538, 2403, 1331, 364, 1352, 433, 2798, 243, 294, 786, 607, 3690, 305, 1842, 747, 74, 281, 1042, 44, 315, 253] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[711, 151, 0, 3366, 100, 29, 251, 67, 0, 519, 1684, 950, 1081, 528, 421, 1396, 4, 8, 769, 7, 349, 509, 474, 2426, 113, 63, 942, 0, 547, 295, 0, 7, 69, 1845, 303, 18, 0, 0, 0, 1752, 1259, 233, 221, 1828, 5122, 332, 802, 3725, 380, 566, 0, 982, 383, 976, 5883, 0, 3735, 4, 0, 7, 346, 0, 460, 154] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[808, 294, 2, 3444, 76, 67, 279, 126, 0, 583, 1524, 935, 1087, 550, 431, 1610, 3, 31, 827, 21, 852, 549, 459, 2407, 101, 328, 629, 5, 673, 478, 0, 29, 109, 1891, 400, 41, 0, 0, 0, 2031, 1283, 259, 239, 1592, 4851, 374, 951, 3010, 403, 642, 4, 1066, 307, 850, 5242, 0, 3502, 5, 1, 42, 343, 0, 294, 212] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[653, 120, 0, 4574, 78, 0, 209, 97, 0, 456, 582, 532, 1043, 526, 440, 1018, 0, 0, 601, 4, 3, 447, 404, 1434, 102, 59, 528, 0, 177, 441, 0, 0, 97, 2033, 170, 1, 0, 0, 0, 3079, 1078, 268, 209, 1606, 5531, 87, 963, 4555, 422, 96, 0, 1647, 325, 1619, 6345, 0, 3981, 0, 0, 0, 4, 0, 364, 144] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[880, 311, 1, 3013, 76, 33, 303, 137, 2, 546, 1437, 751, 1064, 583, 413, 1548, 1, 17, 777, 37, 344, 476, 479, 2582, 114, 199, 989, 1, 712, 230, 0, 19, 71, 1926, 548, 11, 0, 5, 4, 1944, 1152, 250, 216, 1726, 5082, 396, 784, 3144, 388, 807, 1, 839, 327, 910, 6004, 5, 3693, 7, 3, 32, 378, 0, 302, 122] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[623, 91, 0, 4569, 83, 1, 183, 74, 0, 453, 731, 557, 971, 777, 389, 946, 0, 1, 553, 0, 26, 430, 370, 1780, 96, 74, 572, 0, 276, 227, 0, 0, 74, 1956, 205, 1, 0, 0, 0, 2827, 1098, 254, 178, 1664, 5559, 132, 876, 4278, 379, 100, 0, 1630, 297, 1505, 6332, 0, 4285, 0, 0, 5, 146, 0, 383, 135] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[711, 151, 0, 3366, 100, 29, 251, 67, 1298, 0, 0, 0, 0, 0, 0, 0, 0, 519, 1684, 950, 1081, 528, 421, 1396, 268, 0, 0, 0, 0, 0, 0, 0, 4, 8, 769, 7, 349, 509, 474, 2426, 926, 869, 0, 0, 0, 0, 0, 0, 113, 63, 942, 0, 547, 295, 0, 7, 4253, 0, 0, 0, 0, 0, 0, 0, 69, 1845, 303, 18, 0, 0, 0, 1752, 992, 520, 0, 0, 0, 0, 0, 0, 1259, 233, 221, 1828, 0, 332, 802, 1639, 0, 0, 0, 0, 0, 0, 0, 0, 380, 566, 0, 982, 383, 976, 2415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3735, 4, 0, 7, 346, 0, 460, 154, 1550, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5973, 6847, 6341, 6220, 5499, 6314, 5702, 6256] avg=6144.0000 max=6847 max_rank=1 min=5499 min_rank=4 imbalance_ratio=1.114421 max_load_reduction=42305 max_load_reduction_pct=0.860697 moved_to_echo=23771 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[886, 260, 2, 2910, 76, 55, 261, 118, 1075, 0, 0, 0, 0, 0, 0, 0, 2, 691, 1411, 917, 1107, 429, 450, 1541, 222, 0, 0, 0, 0, 0, 0, 0, 0, 35, 853, 30, 288, 561, 515, 2492, 912, 815, 0, 0, 0, 0, 0, 0, 140, 244, 715, 2, 683, 285, 0, 26, 3990, 0, 0, 0, 0, 0, 0, 0, 104, 1683, 493, 37, 2, 0, 0, 1678, 977, 431, 0, 0, 0, 0, 0, 0, 1281, 314, 241, 1978, 0, 357, 843, 1359, 0, 0, 0, 0, 0, 0, 0, 0, 364, 1171, 4, 737, 380, 1155, 2380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3699, 16, 1, 45, 363, 0, 373, 160, 1527, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5643, 6770, 6501, 6085, 5405, 6373, 6191, 6184] avg=6144.0000 max=6770 max_rank=1 min=5405 min_rank=4 imbalance_ratio=1.101888 max_load_reduction=42382 max_load_reduction_pct=0.862264 moved_to_echo=24153 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[808, 294, 2, 3444, 76, 67, 279, 126, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 583, 1524, 935, 1087, 550, 431, 1610, 217, 0, 0, 0, 0, 0, 0, 0, 3, 31, 827, 21, 852, 549, 459, 2407, 825, 823, 0, 0, 0, 0, 0, 0, 101, 328, 629, 5, 673, 478, 0, 29, 4028, 0, 0, 0, 0, 0, 0, 0, 109, 1891, 400, 41, 0, 0, 0, 2031, 884, 420, 0, 0, 0, 0, 0, 0, 1283, 259, 239, 1592, 0, 374, 951, 1324, 0, 0, 0, 0, 0, 0, 0, 0, 403, 642, 4, 1066, 307, 850, 2152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3502, 5, 1, 42, 343, 0, 294, 212, 1381, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6145, 6937, 6797, 6271, 5776, 6022, 5424, 5780] avg=6144.0000 max=6937 max_rank=1 min=5424 min_rank=6 imbalance_ratio=1.129069 max_load_reduction=42215 max_load_reduction_pct=0.858866 moved_to_echo=23002 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[702, 150, 0, 4230, 80, 3, 217, 89, 1346, 0, 0, 0, 0, 0, 0, 0, 0, 512, 1127, 746, 1023, 693, 421, 1216, 278, 0, 0, 0, 0, 0, 0, 0, 1, 6, 756, 3, 73, 510, 393, 2126, 942, 925, 0, 0, 0, 0, 0, 0, 136, 66, 663, 0, 366, 171, 0, 5, 4518, 0, 0, 0, 0, 0, 0, 0, 91, 1864, 205, 5, 5, 4, 6, 2433, 1009, 539, 0, 0, 0, 0, 0, 0, 1157, 254, 185, 1667, 0, 128, 1056, 1700, 0, 0, 0, 0, 0, 0, 0, 0, 375, 143, 1, 1568, 329, 1025, 2456, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 26, 281, 1, 382, 138, 1577, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6817, 6016, 5735, 5925, 6161, 6147, 5899, 6452] avg=6144.0000 max=6817 max_rank=0 min=5735 min_rank=2 imbalance_ratio=1.109538 max_load_reduction=42335 max_load_reduction_pct=0.861308 moved_to_echo=24659 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[976, 515, 100, 832, 172, 693, 390, 140, 471, 0, 0, 0, 0, 0, 0, 0, 166, 942, 1015, 766, 1028, 185, 447, 1304, 97, 0, 0, 0, 0, 0, 0, 0, 17, 259, 913, 92, 2183, 683, 890, 1295, 581, 408, 0, 0, 0, 0, 0, 0, 188, 1095, 566, 316, 1494, 535, 35, 185, 1995, 0, 0, 0, 0, 0, 0, 0, 231, 1655, 557, 273, 21, 340, 69, 2573, 622, 188, 0, 0, 0, 0, 0, 0, 1159, 275, 840, 1538, 0, 1331, 364, 596, 0, 0, 0, 0, 0, 0, 0, 0, 433, 2798, 243, 294, 786, 607, 1515, 305, 0, 0, 0, 0, 0, 0, 0, 0, 1842, 747, 74, 281, 1042, 44, 315, 253, 972, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4289, 5950, 7321, 6409, 6529, 6103, 6981, 5570] avg=6144.0000 max=7321 max_rank=2 min=4289 min_rank=0 imbalance_ratio=1.191569 max_load_reduction=41831 max_load_reduction_pct=0.851054 moved_to_echo=25183 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[623, 91, 0, 4569, 83, 1, 183, 74, 1490, 0, 0, 0, 0, 0, 0, 0, 0, 453, 731, 557, 971, 777, 389, 946, 308, 0, 0, 0, 0, 0, 0, 0, 0, 1, 553, 0, 26, 430, 370, 1780, 997, 944, 0, 0, 0, 0, 0, 0, 96, 74, 572, 0, 276, 227, 0, 0, 4615, 0, 0, 0, 0, 0, 0, 0, 74, 1956, 205, 1, 0, 0, 0, 2827, 1067, 597, 0, 0, 0, 0, 0, 0, 1098, 254, 178, 1664, 0, 132, 876, 1883, 0, 0, 0, 0, 0, 0, 0, 0, 379, 100, 0, 1630, 297, 1505, 2600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4285, 0, 0, 5, 146, 0, 383, 135, 1668, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7114, 5132, 5101, 5860, 6727, 6085, 6511, 6622] avg=6144.0000 max=7114 max_rank=0 min=5101 min_rank=2 imbalance_ratio=1.157878 max_load_reduction=42038 max_load_reduction_pct=0.855265 moved_to_echo=25945 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[653, 120, 0, 4574, 78, 0, 209, 97, 1587, 0, 0, 0, 0, 0, 0, 0, 0, 456, 582, 532, 1043, 526, 440, 1018, 328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 601, 4, 3, 447, 404, 1434, 999, 939, 0, 0, 0, 0, 0, 0, 102, 59, 528, 0, 177, 441, 0, 0, 4592, 0, 0, 0, 0, 0, 0, 0, 97, 2033, 170, 1, 0, 0, 0, 3079, 1070, 636, 0, 0, 0, 0, 0, 0, 1078, 268, 209, 1606, 0, 87, 963, 2004, 0, 0, 0, 0, 0, 0, 0, 0, 422, 96, 0, 1647, 325, 1619, 2604, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3981, 0, 0, 0, 4, 0, 364, 144, 1672, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7318, 4925, 4831, 5899, 7086, 6215, 6713, 6165] avg=6144.0000 max=7318 max_rank=0 min=4831 min_rank=2 imbalance_ratio=1.191081 max_load_reduction=41834 max_load_reduction_pct=0.851115 moved_to_echo=26179 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[880, 311, 1, 3013, 76, 33, 303, 137, 1099, 0, 0, 0, 0, 0, 0, 0, 2, 546, 1437, 751, 1064, 583, 413, 1548, 231, 0, 0, 0, 0, 0, 0, 0, 1, 17, 777, 37, 344, 476, 479, 2582, 949, 863, 0, 0, 0, 0, 0, 0, 114, 199, 989, 1, 712, 230, 0, 19, 4219, 0, 0, 0, 0, 0, 0, 0, 71, 1926, 548, 11, 0, 5, 4, 1944, 1015, 445, 0, 0, 0, 0, 0, 0, 1152, 250, 216, 1726, 0, 396, 784, 1369, 0, 0, 0, 0, 0, 0, 0, 0, 388, 807, 1, 839, 327, 910, 2454, 5, 0, 0, 0, 0, 0, 0, 0, 0, 3693, 7, 3, 32, 378, 0, 302, 122, 1586, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5853, 6575, 6525, 6483, 5969, 5893, 5731, 6123] avg=6144.0000 max=6575 max_rank=1 min=5731 min_rank=6 imbalance_ratio=1.070150 max_load_reduction=42577 max_load_reduction_pct=0.866231 moved_to_echo=23716 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 384, 6144, 256, 4224, 4224, 4096, 16640, 7168, 6656, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[9472, 2176, 2432, 13696, 0, 3200, 6656, 11904, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[256, 4736, 9600, 6272, 8448, 4352, 3456, 10624, 2048, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[28800, 896, 128, 512, 2944, 128, 2944, 1408, 12032, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[896, 14976, 2944, 512, 128, 384, 128, 18432, 7680, 3840, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[6272, 1920, 128, 27008, 768, 896, 2176, 896, 9472, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3200, 6400, 256, 8832, 3200, 8704, 18688, 384, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1024, 2176, 5632, 384, 4992, 2688, 128, 384, 32256, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1545, 4404, 0, 211, 1099, 132, 57, 35, 0, 290, 145, 236, 160, 313, 556, 714, 361, 922, 0, 6, 600, 109, 2873, 1172, 2844, 3576, 1130, 361, 0, 304, 144, 392, 0, 241, 1815, 0, 100, 1357, 757, 205, 746, 60, 0, 54, 1997, 5843, 650, 0, 5469, 0, 1973, 739, 476, 124, 350, 5, 10, 374, 0, 0, 0, 545, 144, 427] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1611, 3495, 1, 372, 1236, 185, 77, 107, 1, 401, 190, 265, 52, 325, 528, 641, 372, 921, 0, 11, 852, 112, 3083, 1304, 2617, 3714, 1198, 366, 0, 305, 122, 464, 0, 272, 2498, 0, 74, 1398, 810, 251, 665, 115, 0, 71, 1958, 5401, 628, 0, 5439, 0, 1843, 572, 461, 105, 356, 9, 13, 389, 0, 0, 0, 455, 132, 309] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1705, 3807, 0, 243, 1283, 178, 70, 79, 2, 466, 215, 533, 82, 334, 584, 743, 407, 824, 0, 41, 799, 98, 2393, 1478, 2625, 3495, 1186, 258, 6, 347, 184, 434, 0, 246, 1710, 0, 76, 1830, 936, 296, 722, 81, 0, 68, 1842, 5261, 809, 0, 5567, 0, 1715, 602, 437, 131, 368, 3, 13, 383, 0, 0, 0, 508, 250, 399] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[988, 1028, 249, 607, 3213, 766, 798, 482, 11, 534, 1401, 732, 169, 617, 681, 787, 432, 1543, 248, 642, 754, 133, 2191, 1872, 1878, 855, 768, 155, 241, 414, 336, 787, 434, 560, 1919, 141, 94, 1113, 1288, 191, 807, 407, 68, 609, 1159, 1930, 211, 48, 3101, 738, 2893, 481, 558, 288, 362, 546, 243, 816, 153, 63, 26, 94, 860, 639] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1864, 6042, 11, 143, 635, 39, 61, 9, 0, 306, 120, 225, 117, 331, 439, 681, 265, 1243, 6, 3, 509, 109, 2336, 1264, 3358, 4070, 1697, 321, 1, 62, 133, 340, 0, 315, 1078, 0, 76, 1674, 671, 129, 665, 8, 0, 63, 1271, 6110, 206, 7, 5509, 11, 1247, 799, 453, 81, 286, 1, 4, 374, 5, 1, 2, 835, 196, 335] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1916, 6896, 0, 102, 581, 18, 53, 0, 0, 296, 85, 216, 130, 339, 474, 729, 159, 1471, 0, 0, 225, 79, 1887, 1219, 4052, 3921, 2145, 306, 0, 0, 157, 262, 0, 358, 606, 0, 90, 1567, 597, 116, 696, 9, 0, 72, 797, 6227, 82, 0, 5278, 0, 1320, 859, 466, 83, 248, 0, 0, 359, 0, 0, 0, 1029, 227, 348] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[2258, 6890, 0, 43, 553, 0, 68, 0, 0, 335, 98, 206, 79, 343, 581, 676, 172, 1829, 0, 0, 194, 107, 1242, 1203, 4435, 3885, 2232, 336, 1, 0, 82, 261, 0, 339, 581, 0, 80, 1648, 633, 93, 700, 0, 0, 59, 801, 6185, 75, 0, 5425, 0, 842, 874, 500, 65, 216, 0, 0, 424, 0, 0, 0, 972, 253, 278] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1696, 3678, 8, 246, 1209, 190, 52, 82, 0, 275, 171, 392, 65, 322, 556, 814, 394, 969, 7, 15, 866, 93, 3203, 1698, 2485, 3422, 1325, 277, 1, 369, 93, 391, 8, 259, 1689, 0, 73, 1425, 813, 205, 696, 77, 0, 78, 1536, 5395, 516, 5, 5642, 8, 2478, 581, 457, 109, 374, 10, 15, 403, 5, 4, 3, 390, 166, 368] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1545, 2429, 0, 211, 1099, 132, 57, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 290, 145, 236, 160, 313, 556, 714, 1975, 1141, 295, 0, 0, 0, 0, 0, 361, 922, 0, 6, 600, 109, 2848, 1172, 0, 0, 0, 0, 0, 0, 0, 0, 2844, 553, 1130, 361, 0, 304, 144, 392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 241, 1815, 0, 100, 1357, 757, 205, 1827, 25, 0, 0, 0, 0, 0, 0, 746, 60, 0, 54, 1997, 3721, 650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2726, 0, 1973, 739, 476, 124, 350, 5, 0, 0, 0, 0, 0, 0, 0, 0, 10, 374, 0, 0, 0, 545, 144, 427, 3023, 1602, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5508, 5825, 6018, 5728, 6327, 7228, 6393, 6125] avg=6144.0000 max=7228 max_rank=5 min=5508 min_rank=0 imbalance_ratio=1.176432 max_load_reduction=41924 max_load_reduction_pct=0.852946 moved_to_echo=26073 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1611, 1928, 1, 372, 1236, 185, 77, 107, 0, 0, 0, 0, 0, 0, 0, 0, 1, 401, 190, 265, 52, 325, 528, 641, 1567, 1134, 273, 0, 0, 0, 0, 0, 372, 921, 0, 11, 852, 112, 3057, 1304, 0, 0, 0, 0, 0, 0, 0, 0, 2617, 575, 1198, 366, 0, 305, 122, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 272, 2498, 0, 74, 1398, 810, 251, 1689, 26, 0, 0, 0, 0, 0, 0, 665, 115, 0, 71, 1958, 3439, 628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2712, 0, 1843, 572, 461, 105, 356, 9, 0, 0, 0, 0, 0, 0, 0, 0, 13, 389, 0, 0, 0, 455, 132, 309, 3139, 1593, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5517, 5377, 6629, 5647, 7018, 6876, 6058, 6030] avg=6144.0000 max=7018 max_rank=4 min=5377 min_rank=1 imbalance_ratio=1.142253 max_load_reduction=42134 max_load_reduction_pct=0.857218 moved_to_echo=25982 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1705, 2100, 0, 243, 1283, 178, 70, 79, 0, 0, 0, 0, 0, 0, 0, 0, 2, 466, 215, 533, 82, 334, 584, 743, 1707, 1161, 266, 0, 0, 0, 0, 0, 407, 824, 0, 41, 799, 98, 2373, 1478, 0, 0, 0, 0, 0, 0, 0, 0, 2625, 541, 1186, 258, 6, 347, 184, 434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 1710, 0, 76, 1830, 936, 296, 1645, 20, 0, 0, 0, 0, 0, 0, 722, 81, 0, 68, 1842, 3350, 809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2776, 0, 1715, 602, 437, 131, 368, 3, 0, 0, 0, 0, 0, 0, 0, 0, 13, 383, 0, 0, 0, 508, 250, 399, 2954, 1630, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5658, 6093, 6020, 5581, 6759, 6872, 6032, 6137] avg=6144.0000 max=6872 max_rank=5 min=5581 min_rank=3 imbalance_ratio=1.118490 max_load_reduction=42280 max_load_reduction_pct=0.860189 moved_to_echo=25800 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1864, 3333, 11, 143, 635, 39, 61, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 306, 120, 225, 117, 331, 439, 681, 2709, 1149, 309, 0, 0, 0, 0, 0, 265, 1243, 6, 3, 509, 109, 2316, 1264, 0, 0, 0, 0, 0, 0, 0, 0, 3358, 630, 1697, 321, 1, 62, 133, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 315, 1078, 0, 76, 1674, 671, 129, 1911, 20, 0, 0, 0, 0, 0, 0, 665, 8, 0, 63, 1271, 3890, 206, 7, 0, 0, 0, 0, 0, 0, 0, 0, 2747, 11, 1247, 799, 453, 81, 286, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4, 374, 5, 1, 2, 835, 196, 335, 3440, 1613, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6095, 6386, 5715, 6542, 5874, 6110, 5625, 6805] avg=6144.0000 max=6805 max_rank=7 min=5625 min_rank=6 imbalance_ratio=1.107585 max_load_reduction=42347 max_load_reduction_pct=0.861552 moved_to_echo=24414 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[988, 567, 249, 607, 3213, 766, 798, 482, 0, 0, 0, 0, 0, 0, 0, 0, 11, 534, 1401, 732, 169, 617, 681, 787, 461, 646, 97, 0, 0, 0, 0, 0, 432, 1543, 248, 642, 754, 133, 2172, 1872, 0, 0, 0, 0, 0, 0, 0, 0, 1878, 133, 768, 155, 241, 414, 336, 787, 0, 0, 0, 0, 0, 0, 0, 0, 434, 560, 1919, 141, 94, 1113, 1288, 191, 603, 19, 0, 0, 0, 0, 0, 0, 807, 407, 68, 609, 1159, 1230, 211, 48, 0, 0, 0, 0, 0, 0, 0, 0, 1547, 738, 2893, 481, 558, 288, 362, 546, 0, 0, 0, 0, 0, 0, 0, 0, 243, 816, 153, 63, 26, 94, 860, 639, 722, 908, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7670, 6136, 7796, 4712, 6362, 4539, 7413, 4524] avg=6144.0000 max=7796 max_rank=2 min=4524 min_rank=7 imbalance_ratio=1.268880 max_load_reduction=41356 max_load_reduction_pct=0.841390 moved_to_echo=22838 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1916, 3804, 0, 102, 581, 18, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 296, 85, 216, 130, 339, 474, 729, 3092, 1101, 315, 0, 0, 0, 0, 0, 159, 1471, 0, 0, 225, 79, 1871, 1219, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 607, 2145, 306, 0, 0, 157, 262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 358, 606, 0, 90, 1567, 597, 116, 1947, 16, 0, 0, 0, 0, 0, 0, 696, 9, 0, 72, 797, 3965, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2631, 0, 1320, 859, 466, 83, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 359, 0, 0, 0, 1029, 227, 348, 3314, 1546, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6474, 6777, 5024, 7529, 5297, 5621, 5607, 6823] avg=6144.0000 max=7529 max_rank=3 min=5024 min_rank=2 imbalance_ratio=1.225423 max_load_reduction=41623 max_load_reduction_pct=0.846822 moved_to_echo=23348 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2258, 3800, 0, 43, 553, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 335, 98, 206, 79, 343, 581, 676, 3090, 1131, 313, 0, 0, 0, 0, 0, 172, 1829, 0, 0, 194, 107, 1232, 1203, 0, 0, 0, 0, 0, 0, 0, 0, 4435, 601, 2232, 336, 1, 0, 82, 261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 339, 581, 0, 80, 1648, 633, 93, 1934, 10, 0, 0, 0, 0, 0, 0, 700, 0, 0, 59, 801, 3938, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2705, 0, 842, 874, 500, 65, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 424, 0, 0, 0, 972, 253, 278, 3284, 1589, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6722, 6852, 4737, 7948, 5318, 5573, 5202, 6800] avg=6144.0000 max=7948 max_rank=3 min=4737 min_rank=2 imbalance_ratio=1.293620 max_load_reduction=41204 max_load_reduction_pct=0.838298 moved_to_echo=22893 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1696, 2025, 8, 246, 1209, 190, 52, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 275, 171, 392, 65, 322, 556, 814, 1653, 1181, 277, 0, 0, 0, 0, 0, 394, 969, 7, 15, 866, 93, 3171, 1698, 0, 0, 0, 0, 0, 0, 0, 0, 2485, 524, 1325, 277, 1, 369, 93, 391, 0, 0, 0, 0, 0, 0, 0, 0, 8, 259, 1689, 0, 73, 1425, 813, 205, 1693, 32, 0, 0, 0, 0, 0, 0, 696, 77, 0, 78, 1536, 3425, 516, 5, 0, 0, 0, 0, 0, 0, 0, 0, 2805, 8, 2478, 581, 457, 109, 374, 10, 0, 0, 0, 0, 0, 0, 0, 0, 15, 403, 5, 4, 3, 390, 166, 368, 2898, 1656, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5508, 5706, 7213, 5465, 6197, 6333, 6822, 5908] avg=6144.0000 max=7213 max_rank=2 min=5465 min_rank=3 imbalance_ratio=1.173991 max_load_reduction=41939 max_load_reduction_pct=0.853251 moved_to_echo=25260 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 2944, 2432, 2816, 896, 2944, 4480, 5888, 16256, 8704, 2176, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2688, 9728, 384, 768, 4864, 896, 19072, 11264, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[5760, 768, 128, 1152, 11392, 27008, 3200, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[20736, 768, 14336, 5632, 3840, 1024, 2560, 640, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[384, 3584, 256, 128, 128, 4864, 2304, 3200, 22784, 12160, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[13696, 20096, 384, 2048, 9856, 1536, 1280, 896, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[24320, 4224, 11776, 2432, 256, 1920, 1280, 3456, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[512, 2688, 11904, 256, 768, 12032, 6528, 1536, 13312, 256, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[587, 530, 4055, 3631, 1269, 0, 6341, 5915, 0, 148, 0, 655, 0, 1095, 0, 277, 256, 40, 1729, 28, 846, 71, 401, 161, 70, 102, 1134, 0, 4, 145, 0, 174, 5458, 789, 0, 92, 260, 763, 721, 0, 386, 0, 125, 0, 27, 0, 363, 93, 2251, 0, 217, 194, 129, 1942, 69, 0, 2560, 446, 75, 855, 1408, 1, 264, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[543, 449, 5080, 2586, 2212, 2, 6099, 5779, 7, 153, 4, 787, 9, 751, 0, 270, 287, 0, 2439, 45, 935, 55, 363, 135, 64, 76, 942, 8, 0, 137, 6, 171, 5643, 668, 0, 76, 232, 867, 720, 1, 373, 1, 10, 4, 0, 5, 325, 21, 2210, 5, 8, 133, 15, 1563, 207, 5, 2220, 454, 22, 1275, 1656, 0, 39, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[584, 527, 5017, 2718, 1925, 0, 6147, 6048, 0, 199, 0, 808, 0, 961, 0, 222, 279, 0, 1997, 45, 922, 50, 346, 123, 55, 112, 1155, 0, 0, 155, 0, 165, 5460, 765, 0, 138, 244, 748, 749, 0, 359, 0, 22, 0, 0, 0, 345, 144, 2473, 0, 112, 201, 5, 1281, 93, 0, 2733, 478, 87, 843, 1162, 3, 146, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[669, 2263, 2130, 1143, 422, 332, 3669, 1939, 135, 204, 222, 611, 277, 1276, 140, 1071, 502, 359, 1522, 75, 804, 125, 1356, 1493, 63, 585, 2011, 144, 1014, 200, 421, 1208, 3141, 697, 169, 167, 466, 1709, 768, 226, 511, 917, 1363, 1187, 307, 257, 620, 323, 1086, 58, 273, 407, 973, 553, 308, 221, 877, 738, 360, 204, 956, 13, 784, 128] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[603, 630, 3877, 3372, 1973, 0, 6245, 6351, 0, 146, 0, 616, 0, 1126, 0, 196, 244, 17, 2234, 51, 869, 77, 359, 171, 64, 128, 1326, 0, 10, 138, 0, 167, 5479, 733, 0, 96, 214, 648, 589, 0, 370, 0, 76, 0, 125, 0, 330, 31, 2412, 0, 44, 215, 8, 1228, 106, 0, 2777, 468, 80, 803, 1047, 3, 278, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[583, 444, 5051, 2191, 2400, 0, 5900, 5125, 0, 178, 0, 899, 0, 994, 0, 240, 278, 0, 2654, 23, 891, 46, 131, 136, 53, 89, 781, 0, 0, 143, 0, 218, 5770, 654, 0, 90, 206, 1063, 710, 0, 342, 5, 5, 0, 0, 0, 192, 22, 2308, 0, 1, 142, 9, 1779, 482, 0, 2469, 499, 30, 1382, 1535, 0, 9, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[628, 590, 4148, 3267, 1550, 7, 6140, 5996, 4, 144, 9, 684, 7, 1086, 0, 322, 260, 1, 1654, 60, 840, 54, 433, 159, 47, 85, 1417, 0, 9, 122, 12, 194, 5488, 737, 1, 78, 235, 740, 628, 4, 387, 3, 75, 3, 44, 6, 370, 21, 2245, 0, 162, 234, 147, 2034, 95, 3, 3239, 547, 111, 590, 675, 0, 317, 4] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[541, 336, 4705, 2490, 1931, 0, 6251, 5189, 0, 160, 0, 824, 0, 1016, 0, 370, 277, 0, 2393, 28, 967, 46, 85, 148, 68, 80, 756, 0, 0, 148, 0, 190, 5635, 709, 0, 80, 203, 1415, 758, 0, 422, 0, 7, 0, 0, 0, 72, 3, 2641, 0, 34, 102, 3, 1566, 443, 0, 1988, 509, 18, 1494, 2020, 6, 25, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[584, 527, 476, 2718, 1925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 199, 0, 808, 0, 961, 0, 222, 2627, 1687, 0, 0, 0, 0, 0, 0, 279, 0, 1997, 45, 922, 50, 346, 123, 2323, 0, 0, 0, 0, 0, 0, 0, 55, 112, 1155, 0, 0, 155, 0, 165, 3421, 1134, 0, 0, 0, 0, 0, 0, 3444, 765, 0, 138, 244, 748, 749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 359, 0, 22, 0, 0, 0, 345, 144, 5013, 0, 0, 0, 0, 0, 0, 0, 2473, 0, 112, 201, 5, 1281, 93, 0, 1311, 531, 0, 0, 0, 0, 0, 0, 2733, 478, 87, 843, 1162, 3, 146, 1, 705, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6230, 6504, 6085, 6197, 6088, 5883, 6007, 6158] avg=6144.0000 max=6504 max_rank=1 min=5883 min_rank=5 imbalance_ratio=1.058594 max_load_reduction=42648 max_load_reduction_pct=0.867676 moved_to_echo=24136 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[587, 530, 385, 3631, 1269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 0, 655, 0, 1095, 0, 277, 2569, 1363, 0, 0, 0, 0, 0, 0, 256, 40, 1729, 28, 846, 71, 401, 161, 1878, 0, 0, 0, 0, 0, 0, 0, 70, 102, 1134, 0, 4, 145, 0, 174, 3346, 1170, 0, 0, 0, 0, 0, 0, 3442, 789, 0, 92, 260, 763, 721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 386, 0, 125, 0, 27, 0, 363, 93, 5171, 0, 0, 0, 0, 0, 0, 0, 2251, 0, 217, 194, 129, 1942, 69, 0, 1311, 429, 0, 0, 0, 0, 0, 0, 2560, 446, 75, 855, 1408, 1, 264, 0, 705, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6402, 6107, 5410, 6145, 6067, 6165, 6542, 6314] avg=6144.0000 max=6542 max_rank=6 min=5410 min_rank=2 imbalance_ratio=1.064779 max_load_reduction=42610 max_load_reduction_pct=0.866903 moved_to_echo=25088 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[603, 630, 369, 3372, 1973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 0, 616, 0, 1126, 0, 196, 2759, 1303, 0, 0, 0, 0, 0, 0, 244, 17, 2234, 51, 869, 77, 359, 171, 1795, 0, 0, 0, 0, 0, 0, 0, 64, 128, 1326, 0, 10, 138, 0, 167, 3592, 1152, 0, 0, 0, 0, 0, 0, 3455, 733, 0, 96, 214, 648, 589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 370, 0, 76, 0, 125, 0, 330, 31, 5093, 0, 0, 0, 0, 0, 0, 0, 2412, 0, 44, 215, 8, 1228, 106, 0, 1316, 410, 0, 0, 0, 0, 0, 0, 2777, 468, 80, 803, 1047, 3, 278, 2, 708, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6947, 6146, 5817, 6577, 5735, 6025, 5739, 6166] avg=6144.0000 max=6947 max_rank=0 min=5735 min_rank=4 imbalance_ratio=1.130697 max_load_reduction=42205 max_load_reduction_pct=0.858663 moved_to_echo=23665 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[543, 449, 482, 2586, 2212, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 153, 4, 787, 9, 751, 0, 270, 2511, 1708, 0, 0, 0, 0, 0, 0, 287, 0, 2439, 45, 935, 55, 363, 135, 2352, 0, 0, 0, 0, 0, 0, 0, 64, 76, 942, 8, 0, 137, 6, 171, 3268, 1127, 0, 0, 0, 0, 0, 0, 3559, 668, 0, 76, 232, 867, 720, 1, 0, 0, 0, 0, 0, 0, 0, 0, 373, 1, 10, 4, 0, 5, 325, 21, 4972, 0, 0, 0, 0, 0, 0, 0, 2210, 5, 8, 133, 15, 1563, 207, 5, 1355, 538, 0, 0, 0, 0, 0, 0, 2220, 454, 22, 1275, 1656, 0, 39, 0, 729, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6274, 6200, 6611, 5799, 6123, 5711, 6039, 6395] avg=6144.0000 max=6611 max_rank=2 min=5711 min_rank=5 imbalance_ratio=1.076009 max_load_reduction=42541 max_load_reduction_pct=0.865499 moved_to_echo=24268 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[669, 2263, 203, 1143, 422, 332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 204, 222, 611, 277, 1276, 140, 1071, 843, 716, 0, 0, 0, 0, 0, 0, 502, 359, 1522, 75, 804, 125, 1356, 1493, 986, 0, 0, 0, 0, 0, 0, 0, 63, 585, 2011, 144, 1014, 200, 421, 1208, 1096, 678, 0, 0, 0, 0, 0, 0, 1981, 697, 169, 167, 466, 1709, 768, 226, 0, 0, 0, 0, 0, 0, 0, 0, 511, 917, 1363, 1187, 307, 257, 620, 323, 2991, 0, 0, 0, 0, 0, 0, 0, 1086, 58, 273, 407, 973, 553, 308, 221, 754, 225, 0, 0, 0, 0, 0, 0, 877, 738, 360, 204, 956, 13, 784, 128, 406, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5032, 5495, 7222, 7420, 6183, 8476, 4858, 4466] avg=6144.0000 max=8476 max_rank=5 min=4466 min_rank=7 imbalance_ratio=1.379557 max_load_reduction=40676 max_load_reduction_pct=0.827555 moved_to_echo=23983 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[583, 444, 479, 2191, 2400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 178, 0, 899, 0, 994, 0, 240, 2227, 1698, 0, 0, 0, 0, 0, 0, 278, 0, 2654, 23, 891, 46, 131, 136, 2339, 0, 0, 0, 0, 0, 0, 0, 53, 89, 781, 0, 0, 143, 0, 218, 2898, 1090, 0, 0, 0, 0, 0, 0, 3639, 654, 0, 90, 206, 1063, 710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 342, 5, 5, 0, 0, 0, 192, 22, 4810, 0, 0, 0, 0, 0, 0, 0, 2308, 0, 1, 142, 9, 1779, 482, 0, 1386, 535, 0, 0, 0, 0, 0, 0, 2469, 499, 30, 1382, 1535, 0, 9, 0, 745, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6097, 6236, 6498, 5272, 6362, 5376, 6642, 6669] avg=6144.0000 max=6669 max_rank=7 min=5272 min_rank=3 imbalance_ratio=1.085449 max_load_reduction=42483 max_load_reduction_pct=0.864319 moved_to_echo=25049 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[541, 336, 446, 2490, 1931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 824, 0, 1016, 0, 370, 2254, 1582, 0, 0, 0, 0, 0, 0, 277, 0, 2393, 28, 967, 46, 85, 148, 2179, 0, 0, 0, 0, 0, 0, 0, 68, 80, 756, 0, 0, 148, 0, 190, 2935, 1154, 0, 0, 0, 0, 0, 0, 3554, 709, 0, 80, 203, 1415, 758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 422, 0, 7, 0, 0, 0, 72, 3, 5097, 0, 0, 0, 0, 0, 0, 0, 2641, 0, 34, 102, 3, 1566, 443, 0, 1353, 498, 0, 0, 0, 0, 0, 0, 1988, 509, 18, 1494, 2020, 6, 25, 0, 728, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5744, 6206, 6123, 5331, 6719, 5601, 6640, 6788] avg=6144.0000 max=6788 max_rank=7 min=5331 min_rank=3 imbalance_ratio=1.104818 max_load_reduction=42364 max_load_reduction_pct=0.861898 moved_to_echo=25748 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[628, 590, 384, 3267, 1550, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 144, 9, 684, 7, 1086, 0, 322, 2605, 1397, 0, 0, 0, 0, 0, 0, 260, 1, 1654, 60, 840, 54, 433, 159, 1925, 0, 0, 0, 0, 0, 0, 0, 47, 85, 1417, 0, 9, 122, 12, 194, 3391, 1134, 0, 0, 0, 0, 0, 0, 3452, 737, 1, 78, 235, 740, 628, 4, 0, 0, 0, 0, 0, 0, 0, 0, 387, 3, 75, 3, 44, 6, 370, 21, 5006, 0, 0, 0, 0, 0, 0, 0, 2245, 0, 162, 234, 147, 2034, 95, 3, 1323, 442, 0, 0, 0, 0, 0, 0, 3239, 547, 111, 590, 675, 0, 317, 4, 713, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6426, 6258, 5386, 6411, 5875, 5915, 6685, 6196] avg=6144.0000 max=6685 max_rank=6 min=5386 min_rank=2 imbalance_ratio=1.088053 max_load_reduction=42467 max_load_reduction_pct=0.863993 moved_to_echo=24671 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[256, 1408, 256, 5888, 384, 8320, 256, 3072, 18432, 11520, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[26624, 5760, 256, 896, 2176, 8064, 5760, 256, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[18944, 4224, 896, 7552, 10496, 128, 1920, 256, 5504, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2432, 512, 16640, 384, 7168, 640, 3584, 2560, 15872, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[17664, 128, 896, 1664, 1408, 12032, 1920, 256, 10112, 3712, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4864, 5888, 3328, 21504, 13696, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[512, 1280, 9600, 256, 1152, 1280, 512, 2560, 24064, 8704, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3200, 1024, 1792, 1280, 512, 384, 2688, 768, 38272, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 6715, 1873, 7, 7, 64, 494, 5195, 0, 2, 0, 0, 0, 512, 2204, 161, 324, 353, 52, 240, 85, 0, 211, 991, 431, 313, 27, 166, 237, 82, 128, 2, 53, 7361, 105, 0, 737, 0, 1608, 0, 1047, 0, 0, 0, 2327, 149, 0, 0, 53, 0, 77, 822, 71, 534, 73, 0, 0, 377, 6580, 131, 98, 2030, 21, 4022] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 7070, 2217, 0, 2, 55, 580, 4998, 0, 0, 0, 0, 0, 582, 2244, 162, 252, 333, 160, 247, 67, 0, 137, 964, 593, 346, 0, 135, 232, 69, 262, 0, 91, 7399, 109, 0, 809, 0, 1704, 0, 1047, 0, 0, 0, 2347, 112, 0, 0, 95, 0, 111, 846, 54, 495, 99, 0, 0, 361, 6405, 81, 113, 1869, 5, 3293] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 6875, 2428, 20, 9, 72, 584, 4135, 0, 0, 0, 0, 0, 473, 2229, 164, 361, 417, 86, 252, 84, 0, 249, 962, 491, 325, 2, 175, 250, 91, 157, 4, 56, 7353, 85, 0, 771, 0, 1781, 0, 863, 0, 0, 0, 2187, 211, 0, 0, 62, 0, 75, 1010, 32, 550, 74, 0, 0, 359, 6622, 91, 85, 1931, 15, 4044] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 7477, 2701, 0, 3, 68, 580, 4156, 0, 0, 0, 0, 0, 686, 2747, 148, 0, 464, 75, 247, 91, 0, 8, 943, 312, 327, 0, 119, 181, 75, 164, 1, 141, 7579, 80, 0, 2317, 0, 1206, 0, 1751, 0, 0, 0, 2210, 68, 0, 0, 80, 0, 90, 591, 0, 421, 80, 0, 0, 420, 6829, 239, 109, 1559, 0, 1809] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[63, 3364, 4198, 607, 164, 278, 346, 485, 213, 1230, 357, 260, 151, 780, 2264, 257, 433, 218, 1528, 15, 291, 412, 253, 985, 225, 865, 550, 452, 260, 70, 915, 24, 799, 2336, 474, 216, 319, 1426, 1857, 31, 299, 146, 822, 411, 411, 388, 64, 292, 145, 422, 70, 674, 808, 953, 876, 711, 1063, 579, 2223, 129, 461, 2627, 2534, 2073] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 7439, 2204, 0, 2, 53, 554, 4514, 0, 3, 0, 0, 0, 685, 2770, 145, 0, 392, 116, 265, 70, 0, 4, 884, 389, 352, 0, 156, 218, 53, 227, 1, 97, 7640, 90, 0, 2128, 0, 1504, 0, 1624, 0, 0, 0, 2409, 74, 0, 0, 90, 0, 106, 672, 0, 399, 85, 0, 0, 350, 6765, 167, 116, 1555, 0, 1785] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 7327, 2161, 0, 7, 64, 567, 4689, 0, 18, 1, 0, 0, 629, 2273, 140, 44, 413, 116, 239, 88, 8, 53, 889, 460, 315, 0, 137, 236, 61, 192, 0, 131, 7581, 81, 6, 1696, 0, 1745, 1, 1383, 0, 11, 0, 2446, 99, 0, 11, 77, 9, 82, 648, 10, 510, 76, 9, 2, 374, 6703, 165, 94, 1630, 0, 2445] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 6986, 2390, 1, 8, 50, 580, 4786, 0, 7, 3, 2, 0, 513, 2553, 127, 348, 357, 64, 255, 64, 7, 144, 984, 272, 331, 16, 182, 256, 65, 140, 0, 58, 7334, 86, 10, 759, 7, 1593, 0, 901, 2, 10, 5, 1799, 194, 3, 11, 58, 11, 84, 855, 35, 532, 72, 10, 1, 385, 6472, 75, 102, 2519, 23, 3655] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1873, 7, 7, 64, 494, 3640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 512, 2204, 161, 2838, 55, 0, 0, 0, 0, 0, 0, 324, 353, 52, 240, 85, 0, 211, 991, 4137, 0, 0, 0, 0, 0, 0, 0, 431, 313, 27, 166, 237, 82, 128, 2, 4543, 0, 0, 0, 0, 0, 0, 0, 53, 3020, 105, 0, 737, 0, 1608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 0, 0, 0, 2327, 149, 0, 0, 1503, 1555, 0, 0, 0, 0, 0, 0, 53, 0, 77, 822, 71, 534, 73, 0, 2172, 2227, 0, 0, 0, 0, 0, 0, 0, 377, 161, 131, 98, 2030, 21, 4022, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6085, 5772, 6393, 5929, 5523, 6581, 6029, 6840] avg=6144.0000 max=6840 max_rank=7 min=5523 min_rank=4 imbalance_ratio=1.113281 max_load_reduction=42312 max_load_reduction_pct=0.860840 moved_to_echo=24973 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2217, 0, 2, 55, 580, 3502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 582, 2244, 162, 2853, 54, 0, 0, 0, 0, 0, 0, 252, 333, 160, 247, 67, 0, 137, 964, 4027, 0, 0, 0, 0, 0, 0, 0, 593, 346, 0, 135, 232, 69, 262, 0, 4784, 0, 0, 0, 0, 0, 0, 0, 91, 3035, 109, 0, 809, 0, 1704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 0, 0, 0, 2347, 112, 0, 0, 1511, 1496, 0, 0, 0, 0, 0, 0, 95, 0, 111, 846, 54, 495, 99, 0, 2286, 2167, 0, 0, 0, 0, 0, 0, 0, 361, 157, 81, 113, 1869, 5, 3293, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6356, 5895, 6187, 6421, 5748, 6513, 6153, 5879] avg=6144.0000 max=6513 max_rank=5 min=5748 min_rank=4 imbalance_ratio=1.060059 max_load_reduction=42639 max_load_reduction_pct=0.867493 moved_to_echo=24293 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2428, 20, 9, 72, 584, 2898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 2229, 164, 2835, 55, 0, 0, 0, 0, 0, 0, 361, 417, 86, 252, 84, 0, 249, 962, 4163, 0, 0, 0, 0, 0, 0, 0, 491, 325, 2, 175, 250, 91, 157, 4, 4652, 0, 0, 0, 0, 0, 0, 0, 56, 3016, 85, 0, 771, 0, 1781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 863, 0, 0, 0, 2187, 211, 0, 0, 1502, 1237, 0, 0, 0, 0, 0, 0, 62, 0, 75, 1010, 32, 550, 74, 0, 2223, 2241, 0, 0, 0, 0, 0, 0, 0, 359, 163, 91, 85, 1931, 15, 4044, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6011, 5756, 6574, 6147, 5709, 6000, 6267, 6688] avg=6144.0000 max=6688 max_rank=7 min=5709 min_rank=4 imbalance_ratio=1.088542 max_load_reduction=42464 max_load_reduction_pct=0.863932 moved_to_echo=24664 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[63, 0, 4198, 607, 164, 278, 346, 340, 0, 0, 0, 0, 0, 0, 0, 0, 213, 1230, 357, 260, 151, 780, 2264, 257, 900, 18, 0, 0, 0, 0, 0, 0, 433, 218, 1528, 15, 291, 412, 253, 985, 1397, 0, 0, 0, 0, 0, 0, 0, 225, 865, 550, 452, 260, 70, 915, 24, 2277, 0, 0, 0, 0, 0, 0, 0, 799, 959, 474, 216, 319, 1426, 1857, 31, 0, 0, 0, 0, 0, 0, 0, 0, 299, 146, 822, 411, 411, 388, 64, 292, 477, 145, 0, 0, 0, 0, 0, 0, 145, 422, 70, 674, 808, 953, 876, 711, 1087, 752, 0, 0, 0, 0, 0, 0, 1063, 579, 56, 129, 461, 2627, 2534, 2073, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5996, 6430, 5532, 5638, 6081, 3455, 6498, 9522] avg=6144.0000 max=9522 max_rank=7 min=3455 min_rank=5 imbalance_ratio=1.549805 max_load_reduction=39630 max_load_reduction_pct=0.806274 moved_to_echo=25556 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2204, 0, 2, 53, 554, 3163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 685, 2770, 145, 2945, 57, 0, 0, 0, 0, 0, 0, 0, 392, 116, 265, 70, 0, 4, 884, 4253, 0, 0, 0, 0, 0, 0, 0, 389, 352, 0, 156, 218, 53, 227, 1, 5034, 0, 0, 0, 0, 0, 0, 0, 97, 3135, 90, 0, 2128, 0, 1504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1624, 0, 0, 0, 2409, 74, 0, 0, 1560, 1351, 0, 0, 0, 0, 0, 0, 90, 0, 106, 672, 0, 399, 85, 0, 2405, 2289, 0, 0, 0, 0, 0, 0, 0, 350, 166, 167, 116, 1555, 0, 1785, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5976, 6605, 5984, 6430, 6954, 7018, 6046, 4139] avg=6144.0000 max=7018 max_rank=5 min=4139 min_rank=7 imbalance_ratio=1.142253 max_load_reduction=42134 max_load_reduction_pct=0.857218 moved_to_echo=24157 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2701, 0, 3, 68, 580, 2912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 686, 2747, 148, 2922, 57, 0, 0, 0, 0, 0, 0, 0, 464, 75, 247, 91, 0, 8, 943, 4293, 0, 0, 0, 0, 0, 0, 0, 312, 327, 0, 119, 181, 75, 164, 1, 5059, 0, 0, 0, 0, 0, 0, 0, 141, 3109, 80, 0, 2317, 0, 1206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1751, 0, 0, 0, 2210, 68, 0, 0, 1548, 1244, 0, 0, 0, 0, 0, 0, 80, 0, 90, 591, 0, 421, 80, 0, 2418, 2311, 0, 0, 0, 0, 0, 0, 0, 420, 168, 239, 109, 1559, 0, 1809, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6264, 6560, 6121, 6238, 6853, 6821, 5991, 4304] avg=6144.0000 max=6853 max_rank=4 min=4304 min_rank=7 imbalance_ratio=1.115397 max_load_reduction=42299 max_load_reduction_pct=0.860575 moved_to_echo=23969 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2161, 0, 7, 64, 567, 3286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 1, 0, 0, 629, 2273, 140, 2923, 56, 0, 0, 0, 0, 0, 0, 44, 413, 116, 239, 88, 8, 53, 889, 4214, 0, 0, 0, 0, 0, 0, 0, 460, 315, 0, 137, 236, 61, 192, 0, 4958, 0, 0, 0, 0, 0, 0, 0, 131, 3110, 81, 6, 1696, 0, 1745, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1383, 0, 11, 0, 2446, 99, 0, 11, 1548, 1403, 0, 0, 0, 0, 0, 0, 77, 9, 82, 648, 10, 510, 76, 9, 2369, 2268, 0, 0, 0, 0, 0, 0, 2, 374, 165, 165, 94, 1630, 0, 2445, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6085, 6040, 6064, 6359, 6770, 6901, 6058, 4875] avg=6144.0000 max=6901 max_rank=5 min=4875 min_rank=7 imbalance_ratio=1.123210 max_load_reduction=42251 max_load_reduction_pct=0.859599 moved_to_echo=24604 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2390, 1, 8, 50, 580, 3350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3, 2, 0, 513, 2553, 127, 2831, 58, 0, 0, 0, 0, 0, 0, 348, 357, 64, 255, 64, 7, 144, 984, 4074, 0, 0, 0, 0, 0, 0, 0, 272, 331, 16, 182, 256, 65, 140, 0, 4728, 0, 0, 0, 0, 0, 0, 0, 58, 3001, 86, 10, 759, 7, 1593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 901, 2, 10, 5, 1799, 194, 3, 11, 1502, 1436, 0, 0, 0, 0, 0, 0, 58, 11, 84, 855, 35, 532, 72, 10, 2258, 2195, 0, 0, 0, 0, 0, 0, 1, 385, 145, 75, 102, 2519, 23, 3655, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6379, 6094, 6297, 5990, 5514, 5863, 6110, 6905] avg=6144.0000 max=6905 max_rank=7 min=5514 min_rank=4 imbalance_ratio=1.123861 max_load_reduction=42247 max_load_reduction_pct=0.859517 moved_to_echo=24392 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[768, 512, 768, 6144, 1024, 4480, 1536, 768, 17280, 16512, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1152, 3328, 1280, 1152, 1280, 15744, 2688, 23168, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1536, 22400, 1152, 256, 9600, 1536, 13056, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[256, 1280, 384, 384, 256, 4864, 19328, 1408, 21120, 512, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 0, 20224, 640, 256, 768, 4352, 23168, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[8960, 256, 896, 512, 16256, 1408, 128, 384, 11264, 9984, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3200, 3200, 640, 1536, 1920, 640, 2304, 128, 36096, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1792, 3072, 2304, 1792, 896, 512, 1152, 7680, 30592, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 414, 0, 0, 0, 0, 389, 153, 986, 0, 1122, 313, 0, 73, 72, 0, 17, 0, 0, 85, 0, 0, 24, 4325, 0, 298, 170, 976, 0, 0, 1429, 823, 0, 0, 0, 2231, 0, 862, 1299, 1502, 5996, 0, 565, 545, 521, 0, 0, 160, 0, 4504, 0, 0, 71, 0, 694, 0, 5429, 6077, 5125, 1754, 0, 148] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 387, 0, 0, 0, 0, 369, 247, 1610, 0, 922, 346, 0, 99, 54, 0, 3, 0, 0, 67, 0, 0, 5, 4883, 0, 282, 178, 1322, 0, 0, 1224, 1182, 0, 0, 0, 2266, 0, 860, 1643, 1318, 5004, 0, 553, 652, 754, 0, 0, 264, 0, 3631, 0, 0, 54, 0, 651, 0, 5634, 6174, 4819, 1496, 0, 199] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 401, 0, 0, 0, 0, 405, 130, 1374, 0, 1140, 325, 0, 83, 71, 0, 13, 0, 0, 84, 0, 0, 24, 4477, 0, 283, 237, 975, 0, 1, 951, 1358, 0, 0, 0, 2753, 0, 954, 1569, 1979, 5602, 0, 550, 524, 717, 0, 0, 160, 0, 4246, 0, 0, 68, 0, 741, 0, 4623, 5949, 4285, 1951, 0, 149] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[817, 497, 10, 974, 1038, 207, 79, 734, 581, 176, 395, 690, 2038, 879, 485, 196, 979, 644, 76, 15, 855, 233, 132, 1118, 289, 2824, 165, 647, 862, 1811, 594, 936, 222, 473, 11, 525, 30, 1515, 149, 1193, 2044, 3725, 2531, 318, 825, 450, 815, 254, 9, 208, 649, 1272, 332, 1006, 657, 238, 889, 555, 2543, 1274, 1058, 1193, 28, 185] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 257, 0, 0, 0, 0, 412, 403, 2124, 0, 867, 322, 0, 80, 59, 0, 1, 0, 0, 97, 0, 0, 3, 5125, 0, 309, 206, 1683, 0, 0, 422, 1931, 0, 0, 0, 2451, 0, 871, 2878, 595, 4479, 0, 629, 1180, 1519, 0, 0, 81, 0, 2095, 0, 0, 59, 0, 692, 0, 6384, 6318, 3796, 664, 0, 160] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 1, 361, 1, 0, 1, 9, 388, 422, 1570, 8, 909, 317, 3, 76, 63, 0, 4, 0, 8, 90, 0, 3, 20, 4952, 2, 313, 188, 1238, 9, 7, 805, 1254, 0, 1, 0, 2260, 5, 866, 2281, 1044, 5432, 2, 556, 1030, 958, 1, 0, 134, 13, 2807, 1, 2, 63, 0, 531, 10, 6002, 6292, 4748, 940, 0, 151] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[1, 2, 3, 362, 2, 0, 2, 10, 367, 113, 1122, 12, 1057, 330, 4, 75, 77, 3, 5, 0, 7, 64, 0, 6, 33, 4212, 2, 238, 193, 1063, 2, 10, 949, 971, 0, 5, 1, 2810, 2, 886, 1427, 1512, 5626, 2, 558, 389, 785, 1, 0, 171, 12, 4003, 0, 9, 76, 3, 675, 10, 5689, 6185, 4779, 2089, 1, 149] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[2, 0, 0, 242, 0, 0, 0, 0, 392, 397, 1914, 0, 857, 354, 0, 89, 72, 0, 2, 0, 0, 74, 0, 0, 14, 4974, 0, 281, 165, 1670, 0, 0, 559, 1511, 0, 0, 0, 2552, 0, 899, 2743, 623, 4569, 0, 555, 1206, 1194, 1, 0, 139, 0, 2204, 0, 0, 72, 0, 560, 0, 6505, 6414, 4354, 817, 0, 176] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 387, 0, 0, 0, 0, 5481, 94, 0, 0, 0, 0, 0, 0, 369, 247, 1610, 0, 922, 346, 0, 99, 2340, 39, 0, 0, 0, 0, 0, 0, 54, 0, 3, 0, 0, 67, 0, 0, 6080, 0, 0, 0, 0, 0, 0, 0, 5, 4290, 0, 282, 178, 1322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1224, 1182, 0, 0, 0, 2266, 0, 860, 554, 302, 0, 0, 0, 0, 0, 0, 1643, 1318, 302, 0, 553, 652, 754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 264, 0, 3631, 0, 0, 54, 0, 2362, 153, 0, 0, 0, 0, 0, 0, 651, 0, 0, 0, 4517, 1496, 0, 199, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5962, 5972, 6204, 6077, 6388, 5222, 6464, 6863] avg=6144.0000 max=6863 max_rank=7 min=5222 min_rank=5 imbalance_ratio=1.117025 max_load_reduction=42289 max_load_reduction_pct=0.860372 moved_to_echo=24937 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 414, 0, 0, 0, 0, 5282, 93, 0, 0, 0, 0, 0, 0, 389, 153, 986, 0, 1122, 313, 0, 73, 2804, 34, 0, 0, 0, 0, 0, 0, 72, 0, 17, 0, 0, 85, 0, 0, 5984, 0, 0, 0, 0, 0, 0, 0, 24, 3801, 0, 298, 170, 976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1429, 823, 0, 0, 0, 2231, 0, 862, 490, 321, 0, 0, 0, 0, 0, 0, 1299, 1502, 362, 0, 565, 545, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 4504, 0, 0, 71, 0, 2830, 147, 0, 0, 0, 0, 0, 0, 694, 0, 0, 0, 4804, 1754, 0, 148, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5789, 5874, 6158, 5269, 6156, 4794, 7712, 7400] avg=6144.0000 max=7712 max_rank=6 min=4794 min_rank=5 imbalance_ratio=1.255208 max_load_reduction=41440 max_load_reduction_pct=0.843099 moved_to_echo=26062 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 401, 0, 0, 0, 0, 4498, 91, 0, 0, 0, 0, 0, 0, 405, 130, 1374, 0, 1140, 325, 0, 83, 2620, 36, 0, 0, 0, 0, 0, 0, 71, 0, 13, 0, 0, 84, 0, 0, 5858, 0, 0, 0, 0, 0, 0, 0, 24, 3933, 0, 283, 237, 975, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 951, 1358, 0, 0, 0, 2753, 0, 954, 508, 269, 0, 0, 0, 0, 0, 0, 1569, 1979, 338, 0, 550, 524, 717, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 4246, 0, 0, 68, 0, 2644, 125, 0, 0, 0, 0, 0, 0, 741, 0, 0, 0, 4016, 1951, 0, 149, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4990, 6113, 6026, 5453, 6793, 5677, 7243, 6857] avg=6144.0000 max=7243 max_rank=6 min=4990 min_rank=0 imbalance_ratio=1.178874 max_load_reduction=41909 max_load_reduction_pct=0.852641 moved_to_echo=26570 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[817, 497, 10, 974, 1038, 207, 79, 734, 2475, 20, 0, 0, 0, 0, 0, 0, 581, 176, 395, 690, 2038, 879, 485, 196, 1183, 22, 0, 0, 0, 0, 0, 0, 979, 644, 76, 15, 855, 233, 132, 1118, 1254, 0, 0, 0, 0, 0, 0, 0, 289, 2482, 165, 647, 862, 1811, 594, 936, 0, 0, 0, 0, 0, 0, 0, 0, 222, 473, 11, 525, 30, 1515, 149, 1193, 320, 66, 0, 0, 0, 0, 0, 0, 2044, 3725, 154, 318, 825, 450, 815, 254, 0, 0, 0, 0, 0, 0, 0, 0, 9, 208, 649, 1272, 332, 1006, 657, 238, 1194, 68, 0, 0, 0, 0, 0, 0, 889, 555, 0, 0, 992, 1193, 28, 185, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6851, 6645, 5306, 7786, 4504, 8585, 5633, 3842] avg=6144.0000 max=8585 max_rank=5 min=3842 min_rank=7 imbalance_ratio=1.397298 max_load_reduction=40567 max_load_reduction_pct=0.825338 moved_to_echo=22564 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 361, 1, 0, 1, 9, 5841, 96, 0, 0, 0, 0, 0, 0, 388, 422, 1570, 8, 909, 317, 3, 76, 2540, 39, 0, 0, 0, 0, 0, 0, 63, 0, 4, 0, 8, 90, 0, 3, 6196, 0, 0, 0, 0, 0, 0, 0, 20, 4351, 2, 313, 188, 1238, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 805, 1254, 0, 1, 0, 2260, 5, 866, 562, 298, 0, 0, 0, 0, 0, 0, 2281, 1044, 328, 2, 556, 1030, 958, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, 13, 2807, 1, 2, 63, 0, 2564, 161, 0, 0, 0, 0, 0, 0, 531, 10, 0, 0, 4450, 940, 0, 151, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6310, 6272, 6364, 6128, 6051, 6200, 5745, 6082] avg=6144.0000 max=6364 max_rank=2 min=5745 min_rank=6 imbalance_ratio=1.035807 max_load_reduction=42788 max_load_reduction_pct=0.870524 moved_to_echo=24078 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 242, 0, 0, 0, 0, 6330, 99, 0, 0, 0, 0, 0, 0, 392, 397, 1914, 0, 857, 354, 0, 89, 2137, 40, 0, 0, 0, 0, 0, 0, 72, 0, 2, 0, 0, 74, 0, 0, 6315, 0, 0, 0, 0, 0, 0, 0, 14, 4370, 0, 281, 165, 1670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 1511, 0, 0, 0, 2552, 0, 899, 564, 273, 0, 0, 0, 0, 0, 0, 2743, 623, 276, 0, 555, 1206, 1194, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 0, 2204, 0, 0, 72, 0, 2156, 175, 0, 0, 0, 0, 0, 0, 560, 0, 0, 0, 4081, 817, 0, 176, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6673, 6180, 6463, 6500, 6358, 6598, 4746, 5634] avg=6144.0000 max=6673 max_rank=0 min=4746 min_rank=6 imbalance_ratio=1.086100 max_load_reduction=42479 max_load_reduction_pct=0.864237 moved_to_echo=23336 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 257, 0, 0, 0, 0, 6211, 96, 0, 0, 0, 0, 0, 0, 412, 403, 2124, 0, 867, 322, 0, 80, 2095, 41, 0, 0, 0, 0, 0, 0, 59, 0, 1, 0, 0, 97, 0, 0, 6222, 0, 0, 0, 0, 0, 0, 0, 3, 4503, 0, 309, 206, 1683, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 422, 1931, 0, 0, 0, 2451, 0, 871, 581, 238, 0, 0, 0, 0, 0, 0, 2878, 595, 270, 0, 629, 1180, 1519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 2095, 0, 0, 59, 0, 2114, 173, 0, 0, 0, 0, 0, 0, 692, 0, 0, 0, 3558, 664, 0, 160, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6564, 6344, 6379, 6704, 6494, 7071, 4522, 5074] avg=6144.0000 max=7071 max_rank=5 min=4522 min_rank=6 imbalance_ratio=1.150879 max_load_reduction=42081 max_load_reduction_pct=0.856140 moved_to_echo=23161 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 2, 3, 362, 2, 0, 2, 10, 5536, 95, 0, 0, 0, 0, 0, 0, 367, 113, 1122, 12, 1057, 330, 4, 75, 2635, 37, 0, 0, 0, 0, 0, 0, 77, 3, 5, 0, 7, 64, 0, 6, 6090, 0, 0, 0, 0, 0, 0, 0, 33, 3694, 2, 238, 193, 1063, 2, 10, 0, 0, 0, 0, 0, 0, 0, 0, 949, 971, 0, 5, 1, 2810, 2, 886, 481, 303, 0, 0, 0, 0, 0, 0, 1427, 1512, 331, 2, 558, 389, 785, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 171, 12, 4003, 0, 9, 76, 3, 2660, 153, 0, 0, 0, 0, 0, 0, 675, 10, 0, 0, 4476, 2089, 1, 149, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6013, 5752, 6252, 5235, 6408, 5005, 7087, 7400] avg=6144.0000 max=7400 max_rank=7 min=5005 min_rank=5 imbalance_ratio=1.204427 max_load_reduction=41752 max_load_reduction_pct=0.849447 moved_to_echo=25900 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3328, 2048, 11136, 768, 8960, 3200, 512, 896, 18432, 384, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[16000, 12416, 2432, 384, 4864, 6016, 7296, 384, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[6656, 9600, 128, 640, 128, 18944, 256, 7424, 4096, 2176, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 1408, 768, 24832, 384, 1024, 1152, 256, 18560, 1280, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5504, 640, 0, 0, 30976, 11008, 128, 1408, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[512, 31488, 256, 2688, 2304, 10752, 640, 1024, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[896, 512, 128, 3456, 1152, 256, 128, 768, 41728, 768, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1536, 768, 128, 128, 896, 896, 256, 1152, 44032, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 64, 311, 300, 264, 6915, 0, 418, 0, 0, 23, 0, 74, 0, 0, 0, 1555, 0, 0, 0, 0, 2733, 0, 1889, 0, 52, 15, 609, 0, 0, 0, 64, 5972, 44, 74, 4047, 0, 0, 158, 1522, 299, 0, 0, 1365, 85, 0, 0, 207, 391, 50, 3830, 73, 4734, 64, 3941, 0, 27, 1453, 2884, 601, 1437, 534, 0, 74] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 61, 305, 334, 72, 6277, 4, 394, 0, 2, 40, 4, 78, 8, 0, 0, 1677, 3, 0, 0, 2, 2808, 3, 2073, 0, 27, 48, 1128, 0, 0, 0, 66, 5985, 24, 78, 4971, 0, 10, 164, 829, 239, 10, 15, 1654, 88, 2, 13, 0, 388, 0, 3584, 76, 5827, 61, 3609, 4, 22, 876, 2888, 628, 1074, 447, 6, 166] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 55, 301, 314, 132, 6661, 0, 461, 0, 0, 14, 0, 100, 0, 0, 0, 1257, 0, 0, 0, 0, 2559, 0, 2111, 0, 140, 14, 594, 0, 0, 0, 55, 5790, 0, 101, 4735, 0, 0, 166, 1081, 306, 0, 0, 1427, 67, 0, 0, 3, 350, 0, 3523, 99, 5140, 54, 4263, 0, 33, 1163, 3431, 702, 1300, 530, 0, 120] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 67, 306, 344, 24, 6289, 0, 460, 0, 0, 15, 0, 80, 0, 0, 0, 1996, 0, 0, 0, 0, 3131, 0, 2695, 0, 0, 66, 1800, 0, 0, 0, 68, 5854, 0, 80, 4315, 0, 0, 177, 699, 245, 0, 0, 2382, 96, 0, 0, 0, 425, 0, 3723, 80, 5756, 37, 2802, 0, 0, 548, 2354, 652, 996, 429, 0, 161] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 70, 319, 335, 110, 6791, 0, 438, 0, 0, 17, 0, 75, 0, 0, 0, 1323, 0, 0, 0, 0, 2593, 0, 2973, 0, 82, 13, 463, 0, 0, 0, 70, 5953, 0, 75, 3494, 0, 0, 158, 1363, 302, 0, 0, 1886, 84, 0, 0, 119, 386, 3, 4473, 74, 4332, 70, 3983, 0, 39, 1186, 2639, 644, 1507, 635, 0, 75] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[505, 412, 872, 481, 815, 1700, 1067, 879, 88, 266, 52, 357, 727, 1556, 194, 431, 81, 0, 99, 27, 86, 636, 368, 1826, 272, 637, 792, 573, 708, 237, 69, 456, 3696, 805, 160, 1183, 68, 953, 189, 1244, 333, 1035, 2165, 1035, 155, 183, 1417, 489, 418, 682, 1414, 233, 856, 55, 2667, 76, 469, 2625, 2294, 685, 873, 2338, 363, 725] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 53, 336, 339, 39, 6471, 0, 432, 0, 0, 19, 0, 90, 0, 0, 0, 1653, 0, 0, 0, 0, 2953, 0, 2320, 0, 0, 35, 1544, 0, 0, 0, 53, 5818, 5, 90, 4780, 0, 0, 163, 695, 223, 0, 0, 2213, 74, 0, 0, 0, 410, 0, 3411, 87, 5839, 41, 2894, 0, 12, 653, 2948, 689, 1090, 454, 0, 226] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 47, 334, 320, 194, 7252, 4, 440, 0, 3, 24, 0, 74, 13, 3, 4, 1563, 1, 0, 0, 4, 1928, 4, 2072, 1, 63, 12, 489, 3, 0, 5, 50, 6276, 20, 74, 3575, 0, 5, 136, 1257, 259, 14, 17, 1506, 64, 2, 14, 13, 359, 2, 4193, 73, 4722, 47, 4057, 2, 51, 1818, 2654, 704, 1526, 726, 7, 72] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 64, 311, 300, 264, 5052, 0, 418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 74, 0, 0, 0, 5262, 605, 0, 0, 0, 0, 0, 0, 1555, 0, 0, 0, 0, 2733, 0, 1889, 19, 0, 0, 0, 0, 0, 0, 0, 0, 52, 15, 609, 0, 0, 0, 64, 4129, 762, 0, 0, 0, 0, 0, 0, 710, 44, 74, 4047, 0, 0, 158, 1522, 0, 0, 0, 0, 0, 0, 0, 0, 299, 0, 0, 1365, 85, 0, 0, 207, 1729, 1101, 908, 0, 0, 0, 0, 0, 391, 50, 3830, 73, 0, 64, 2212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 1453, 1957, 601, 1437, 534, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6409, 5964, 6196, 5631, 6555, 5694, 6620, 6083] avg=6144.0000 max=6620 max_rank=6 min=5631 min_rank=3 imbalance_ratio=1.077474 max_load_reduction=42532 max_load_reduction_pct=0.865316 moved_to_echo=24952 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 55, 301, 314, 132, 4867, 0, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 100, 0, 0, 0, 5102, 657, 0, 0, 0, 0, 0, 0, 1257, 0, 0, 0, 0, 2559, 0, 2111, 23, 0, 0, 0, 0, 0, 0, 0, 0, 140, 14, 594, 0, 0, 0, 55, 4483, 734, 0, 0, 0, 0, 0, 0, 688, 0, 101, 4735, 0, 0, 166, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 306, 0, 0, 1427, 67, 0, 0, 3, 1871, 1060, 1081, 0, 0, 0, 0, 0, 350, 0, 3523, 99, 0, 54, 2392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 1163, 2327, 702, 1300, 530, 0, 120, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6130, 5873, 5950, 6020, 6771, 5815, 6418, 6175] avg=6144.0000 max=6771 max_rank=4 min=5815 min_rank=5 imbalance_ratio=1.102051 max_load_reduction=42381 max_load_reduction_pct=0.862244 moved_to_echo=25179 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 70, 319, 335, 110, 4961, 0, 438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 75, 0, 0, 0, 5246, 554, 0, 0, 0, 0, 0, 0, 1323, 0, 0, 0, 0, 2593, 0, 2973, 17, 0, 0, 0, 0, 0, 0, 0, 0, 82, 13, 463, 0, 0, 0, 70, 3778, 749, 0, 0, 0, 0, 0, 0, 707, 0, 75, 3494, 0, 0, 158, 1363, 0, 0, 0, 0, 0, 0, 0, 0, 302, 0, 0, 1886, 84, 0, 0, 119, 1748, 1081, 831, 0, 0, 0, 0, 0, 386, 3, 4473, 74, 0, 70, 2235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 1186, 1791, 644, 1507, 635, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6233, 5892, 6906, 5155, 5797, 6051, 7241, 5877] avg=6144.0000 max=7241 max_rank=6 min=5155 min_rank=3 imbalance_ratio=1.178548 max_load_reduction=41911 max_load_reduction_pct=0.852681 moved_to_echo=24966 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 61, 305, 334, 72, 4586, 4, 394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 40, 4, 78, 8, 0, 0, 5274, 746, 0, 0, 0, 0, 0, 0, 1677, 3, 0, 0, 2, 2808, 3, 2073, 19, 0, 0, 0, 0, 0, 0, 0, 0, 27, 48, 1128, 0, 0, 0, 66, 5081, 692, 0, 0, 0, 0, 0, 0, 711, 24, 78, 4971, 0, 10, 164, 829, 0, 0, 0, 0, 0, 0, 0, 0, 239, 10, 15, 1654, 88, 2, 13, 0, 1584, 999, 909, 0, 0, 0, 0, 0, 388, 0, 3584, 76, 0, 61, 2025, 4, 0, 0, 0, 0, 0, 0, 0, 0, 22, 876, 1960, 628, 1074, 447, 6, 166, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5756, 6152, 6585, 7042, 6787, 5513, 6138, 5179] avg=6144.0000 max=7042 max_rank=3 min=5179 min_rank=7 imbalance_ratio=1.146159 max_load_reduction=42110 max_load_reduction_pct=0.856730 moved_to_echo=23617 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[505, 412, 872, 481, 815, 1243, 1067, 879, 0, 0, 0, 0, 0, 0, 0, 0, 88, 266, 52, 357, 727, 1556, 194, 431, 3257, 110, 0, 0, 0, 0, 0, 0, 81, 0, 99, 27, 86, 636, 368, 1826, 15, 0, 0, 0, 0, 0, 0, 0, 272, 637, 792, 573, 708, 237, 69, 456, 746, 187, 0, 0, 0, 0, 0, 0, 439, 805, 160, 1183, 68, 953, 189, 1244, 0, 0, 0, 0, 0, 0, 0, 0, 333, 1035, 2165, 1035, 155, 183, 1417, 489, 1170, 270, 722, 0, 0, 0, 0, 0, 418, 682, 1414, 233, 0, 55, 1497, 76, 0, 0, 0, 0, 0, 0, 0, 0, 469, 2625, 1557, 685, 873, 2338, 363, 725, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6274, 7038, 3138, 4677, 5041, 8974, 4375, 9635] avg=6144.0000 max=9635 max_rank=7 min=3138 min_rank=2 imbalance_ratio=1.568197 max_load_reduction=39517 max_load_reduction_pct=0.803975 moved_to_echo=28025 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 53, 336, 339, 39, 4728, 0, 432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 90, 0, 0, 0, 5127, 748, 0, 0, 0, 0, 0, 0, 1653, 0, 0, 0, 0, 2953, 0, 2320, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 1544, 0, 0, 0, 53, 5091, 713, 0, 0, 0, 0, 0, 0, 691, 5, 90, 4780, 0, 0, 163, 695, 0, 0, 0, 0, 0, 0, 0, 0, 223, 0, 0, 2213, 74, 0, 0, 0, 1270, 1030, 928, 0, 0, 0, 0, 0, 410, 0, 3411, 87, 0, 41, 1624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 653, 2000, 689, 1090, 454, 0, 226, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5927, 5984, 6946, 7436, 6424, 5738, 5573, 5124] avg=6144.0000 max=7436 max_rank=3 min=5124 min_rank=7 imbalance_ratio=1.210286 max_load_reduction=41716 max_load_reduction_pct=0.848714 moved_to_echo=22859 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 67, 306, 344, 24, 4595, 0, 460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 80, 0, 0, 0, 5158, 737, 0, 0, 0, 0, 0, 0, 1996, 0, 0, 0, 0, 3131, 0, 2695, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 1800, 0, 0, 0, 68, 5019, 693, 0, 0, 0, 0, 0, 0, 696, 0, 80, 4315, 0, 0, 177, 699, 0, 0, 0, 0, 0, 0, 0, 0, 245, 0, 0, 2382, 96, 0, 0, 0, 1229, 1001, 741, 0, 0, 0, 0, 0, 425, 0, 3723, 80, 0, 37, 1573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 1598, 652, 996, 429, 0, 161, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5796, 5990, 7837, 7646, 5967, 5694, 5838, 4384] avg=6144.0000 max=7837 max_rank=2 min=4384 min_rank=7 imbalance_ratio=1.275553 max_load_reduction=41315 max_load_reduction_pct=0.840556 moved_to_echo=21883 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 47, 334, 320, 194, 5288, 4, 440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 24, 0, 74, 13, 3, 4, 5533, 605, 0, 0, 0, 0, 0, 0, 1563, 1, 0, 0, 4, 1928, 4, 2072, 22, 0, 0, 0, 0, 0, 0, 0, 1, 63, 12, 489, 3, 0, 5, 50, 4117, 805, 0, 0, 0, 0, 0, 0, 743, 20, 74, 3575, 0, 5, 136, 1257, 0, 0, 0, 0, 0, 0, 0, 0, 259, 14, 17, 1506, 64, 2, 14, 13, 1784, 1159, 841, 0, 0, 0, 0, 0, 359, 2, 4193, 73, 0, 47, 2273, 2, 0, 0, 0, 0, 0, 0, 0, 0, 51, 1818, 1791, 704, 1526, 726, 7, 72, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6627, 6259, 5594, 5545, 5810, 5673, 6949, 6695] avg=6144.0000 max=6949 max_rank=6 min=5545 min_rank=3 imbalance_ratio=1.131022 max_load_reduction=42203 max_load_reduction_pct=0.858622 moved_to_echo=25127 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[2304, 1152, 2304, 13568, 768, 256, 1536, 896, 12416, 7808, 7040, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[768, 10368, 15104, 5376, 9856, 6144, 384, 1664, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[3200, 768, 28160, 896, 0, 512, 15872, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5504, 1024, 768, 31104, 128, 1024, 1408, 8704, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[512, 896, 3200, 2816, 1664, 35328, 1152, 3968, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[11136, 128, 128, 128, 128, 19456, 384, 18048, 256, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 384, 256, 384, 1408, 1664, 256, 512, 40064, 4864, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[384, 1024, 1024, 7296, 768, 256, 128, 896, 32512, 5376, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[114, 0, 52, 0, 60, 0, 0, 0, 0, 3042, 0, 208, 88, 54, 387, 1548, 0, 7268, 0, 130, 0, 0, 72, 0, 7394, 0, 54, 265, 0, 866, 89, 7158, 7366, 2587, 0, 1, 0, 0, 0, 0, 90, 62, 1032, 0, 0, 2006, 265, 0, 340, 0, 0, 3377, 0, 0, 0, 0, 0, 3105, 0, 0, 0, 0, 0, 72] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 70, 0, 75, 0, 0, 0, 0, 1606, 0, 167, 75, 525, 366, 1609, 0, 7319, 0, 437, 0, 0, 68, 0, 7343, 0, 124, 251, 0, 676, 74, 6691, 7530, 2669, 0, 9, 0, 0, 0, 0, 75, 68, 3228, 0, 0, 2157, 251, 0, 424, 0, 0, 2344, 0, 0, 0, 0, 0, 2853, 0, 0, 0, 0, 0, 68] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[17, 0, 64, 0, 92, 0, 0, 0, 0, 1765, 0, 163, 74, 419, 392, 1562, 0, 7323, 0, 433, 0, 0, 71, 0, 7372, 0, 4, 240, 0, 651, 73, 6359, 7555, 3464, 0, 0, 0, 0, 0, 0, 74, 66, 2783, 0, 0, 2039, 240, 0, 381, 0, 0, 1891, 0, 0, 0, 0, 0, 3514, 0, 0, 0, 0, 0, 71] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[158, 187, 536, 154, 91, 253, 146, 10, 428, 1379, 51, 196, 267, 2005, 471, 702, 35, 2953, 630, 480, 205, 22, 1644, 238, 2572, 953, 254, 306, 228, 2426, 233, 3476, 3887, 783, 1507, 280, 312, 484, 7, 747, 686, 483, 1819, 1506, 108, 1694, 1036, 2279, 288, 541, 115, 1200, 278, 148, 135, 220, 541, 1295, 20, 190, 535, 1610, 98, 631] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[12, 0, 55, 0, 91, 0, 0, 0, 0, 2125, 0, 266, 100, 311, 378, 1744, 0, 7211, 0, 442, 0, 0, 54, 0, 7393, 0, 69, 247, 0, 736, 99, 6841, 7517, 2593, 0, 0, 0, 0, 0, 0, 100, 53, 1960, 0, 0, 2086, 247, 0, 417, 0, 0, 2750, 0, 0, 0, 0, 0, 3201, 0, 0, 0, 0, 0, 54] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[124, 0, 61, 0, 83, 0, 0, 0, 2, 2525, 0, 202, 76, 99, 381, 1564, 1, 7300, 6, 449, 0, 0, 63, 1, 7410, 12, 142, 239, 5, 837, 76, 6896, 7391, 2516, 2, 9, 12, 10, 0, 0, 77, 57, 1344, 7, 0, 2150, 245, 10, 365, 1, 0, 3232, 0, 0, 1, 1, 4, 3075, 9, 0, 3, 12, 0, 65] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[201, 0, 68, 0, 72, 0, 0, 0, 0, 2526, 0, 130, 80, 93, 443, 1545, 0, 7295, 0, 99, 0, 0, 59, 0, 7414, 0, 107, 247, 0, 1274, 80, 7292, 7044, 2381, 0, 0, 0, 0, 0, 0, 80, 37, 1243, 0, 0, 2005, 247, 0, 365, 0, 0, 3756, 0, 0, 0, 0, 0, 2910, 0, 0, 0, 0, 0, 59] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[8, 0, 47, 1, 70, 1, 0, 0, 5, 1979, 0, 147, 75, 423, 393, 1476, 0, 7303, 9, 455, 0, 0, 83, 6, 7471, 15, 94, 256, 8, 749, 72, 6552, 7520, 2510, 12, 3, 5, 0, 0, 1, 78, 79, 2978, 9, 0, 1906, 261, 14, 413, 0, 0, 2166, 1, 0, 0, 1, 8, 3393, 0, 0, 1, 15, 0, 80] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 55, 0, 91, 0, 0, 0, 6279, 0, 0, 0, 0, 0, 0, 0, 0, 2125, 0, 266, 100, 311, 378, 1744, 978, 424, 0, 0, 0, 0, 0, 0, 0, 5741, 0, 442, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 247, 0, 736, 99, 4808, 0, 0, 0, 0, 0, 0, 0, 0, 3536, 2593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 53, 1960, 0, 0, 2086, 247, 0, 1046, 0, 0, 0, 0, 0, 0, 0, 417, 0, 0, 2750, 0, 0, 0, 0, 2129, 1114, 0, 0, 0, 0, 0, 0, 0, 3201, 0, 0, 0, 0, 0, 54, 1852, 1055, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6437, 6326, 6237, 5959, 6129, 5492, 6410, 6162] avg=6144.0000 max=6437 max_rank=0 min=5492 min_rank=5 imbalance_ratio=1.047689 max_load_reduction=42715 max_load_reduction_pct=0.869039 moved_to_echo=24193 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[17, 0, 64, 0, 92, 0, 0, 0, 6261, 0, 0, 0, 0, 0, 0, 0, 0, 1765, 0, 163, 74, 419, 392, 1562, 909, 431, 0, 0, 0, 0, 0, 0, 0, 5830, 0, 433, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 240, 0, 651, 73, 4469, 0, 0, 0, 0, 0, 0, 0, 0, 3554, 3464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 66, 2783, 0, 0, 2039, 240, 0, 1062, 0, 0, 0, 0, 0, 0, 0, 381, 0, 0, 1891, 0, 0, 0, 0, 2140, 1111, 0, 0, 0, 0, 0, 0, 0, 3514, 0, 0, 0, 0, 0, 71, 1861, 981, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6434, 5715, 6334, 5437, 7018, 6264, 5523, 6427] avg=6144.0000 max=7018 max_rank=4 min=5437 min_rank=3 imbalance_ratio=1.142253 max_load_reduction=42134 max_load_reduction_pct=0.857218 moved_to_echo=25232 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 70, 0, 75, 0, 0, 0, 6236, 0, 0, 0, 0, 0, 0, 0, 0, 1606, 0, 167, 75, 525, 366, 1609, 957, 431, 0, 0, 0, 0, 0, 0, 0, 5826, 0, 437, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 251, 0, 676, 74, 4702, 0, 0, 0, 0, 0, 0, 0, 0, 3542, 2669, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 68, 3228, 0, 0, 2157, 251, 0, 1062, 0, 0, 0, 0, 0, 0, 0, 424, 0, 0, 2344, 0, 0, 0, 0, 2133, 1107, 0, 0, 0, 0, 0, 0, 0, 2853, 0, 0, 0, 0, 0, 68, 1855, 1032, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6381, 5736, 6331, 5827, 6220, 6841, 6008, 5808] avg=6144.0000 max=6841 max_rank=5 min=5736 min_rank=1 imbalance_ratio=1.113444 max_load_reduction=42311 max_load_reduction_pct=0.860819 moved_to_echo=24877 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[114, 0, 52, 0, 60, 0, 0, 0, 6281, 0, 0, 0, 0, 0, 0, 0, 0, 3042, 0, 208, 88, 54, 387, 1548, 1024, 428, 0, 0, 0, 0, 0, 0, 0, 5786, 0, 130, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 265, 0, 866, 89, 5030, 0, 0, 0, 0, 0, 0, 0, 0, 3466, 2587, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 62, 1032, 0, 0, 2006, 265, 0, 1054, 0, 0, 0, 0, 0, 0, 0, 340, 0, 0, 3377, 0, 0, 0, 0, 2086, 1113, 0, 0, 0, 0, 0, 0, 0, 3105, 0, 0, 0, 0, 0, 72, 1814, 1104, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6507, 6779, 5988, 6304, 6054, 4509, 6916, 6095] avg=6144.0000 max=6916 max_rank=6 min=4509 min_rank=5 imbalance_ratio=1.125651 max_load_reduction=42236 max_load_reduction_pct=0.859294 moved_to_echo=23574 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[158, 187, 536, 154, 91, 253, 146, 10, 2185, 0, 0, 0, 0, 0, 0, 0, 428, 1379, 51, 196, 267, 2005, 471, 702, 497, 173, 0, 0, 0, 0, 0, 0, 35, 2352, 630, 480, 205, 22, 1644, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 953, 254, 306, 228, 2426, 233, 2443, 0, 0, 0, 0, 0, 0, 0, 0, 1829, 783, 1507, 280, 312, 484, 7, 747, 0, 0, 0, 0, 0, 0, 0, 0, 686, 483, 1819, 1506, 108, 1694, 1036, 2279, 428, 0, 0, 0, 0, 0, 0, 0, 288, 541, 115, 1200, 278, 148, 135, 220, 1101, 387, 0, 0, 0, 0, 0, 0, 541, 1295, 20, 190, 535, 1610, 98, 631, 957, 536, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3720, 6169, 5606, 6843, 5949, 10039, 4413, 6413] avg=6144.0000 max=10039 max_rank=5 min=3720 min_rank=0 imbalance_ratio=1.633952 max_load_reduction=39113 max_load_reduction_pct=0.795756 moved_to_echo=26814 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[124, 0, 61, 0, 83, 0, 0, 0, 6294, 0, 0, 0, 0, 0, 0, 0, 2, 2525, 0, 202, 76, 99, 381, 1564, 986, 430, 0, 0, 0, 0, 0, 0, 1, 5811, 6, 449, 0, 0, 63, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 142, 239, 5, 837, 76, 4847, 0, 0, 0, 0, 0, 0, 0, 0, 3477, 2516, 2, 9, 12, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 57, 1344, 7, 0, 2150, 245, 10, 1059, 0, 0, 0, 0, 0, 0, 0, 365, 1, 0, 3232, 0, 0, 1, 1, 2093, 1116, 0, 0, 0, 0, 0, 0, 4, 3075, 9, 0, 3, 12, 0, 65, 1821, 1063, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6562, 6265, 6331, 6158, 6026, 4949, 6809, 6052] avg=6144.0000 max=6809 max_rank=6 min=4949 min_rank=5 imbalance_ratio=1.108236 max_load_reduction=42343 max_load_reduction_pct=0.861471 moved_to_echo=23836 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[201, 0, 68, 0, 72, 0, 0, 0, 6297, 0, 0, 0, 0, 0, 0, 0, 0, 2526, 0, 130, 80, 93, 443, 1545, 1043, 429, 0, 0, 0, 0, 0, 0, 0, 5808, 0, 99, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 247, 0, 1274, 80, 5125, 0, 0, 0, 0, 0, 0, 0, 0, 3314, 2381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 37, 1243, 0, 0, 2005, 247, 0, 1058, 0, 0, 0, 0, 0, 0, 0, 365, 0, 0, 3756, 0, 0, 0, 0, 1995, 1117, 0, 0, 0, 0, 0, 0, 0, 2910, 0, 0, 0, 0, 0, 59, 1735, 1124, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6638, 6289, 5966, 6833, 5695, 4670, 7233, 5828] avg=6144.0000 max=7233 max_rank=6 min=4670 min_rank=5 imbalance_ratio=1.177246 max_load_reduction=41919 max_load_reduction_pct=0.852844 moved_to_echo=23426 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 47, 1, 70, 1, 0, 0, 6346, 0, 0, 0, 0, 0, 0, 0, 5, 1979, 0, 147, 75, 423, 393, 1476, 941, 434, 0, 0, 0, 0, 0, 0, 0, 5806, 9, 455, 0, 0, 83, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 94, 256, 8, 749, 72, 4597, 0, 0, 0, 0, 0, 0, 0, 0, 3530, 2510, 12, 3, 5, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 78, 79, 2978, 9, 0, 1906, 261, 14, 1063, 0, 0, 0, 0, 0, 0, 0, 413, 0, 0, 2166, 1, 0, 0, 1, 2134, 1125, 0, 0, 0, 0, 0, 0, 8, 3393, 0, 0, 1, 15, 0, 80, 1856, 1014, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6473, 5873, 6359, 5791, 6061, 6388, 5840, 6367] avg=6144.0000 max=6473 max_rank=0 min=5791 min_rank=3 imbalance_ratio=1.053548 max_load_reduction=42679 max_load_reduction_pct=0.868306 moved_to_echo=24656 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[512, 17024, 128, 1536, 896, 3968, 3328, 11776, 7424, 3200, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[640, 23424, 128, 256, 640, 1664, 128, 1152, 13824, 7936, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[26368, 19584, 1536, 384, 384, 512, 128, 768, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1280, 1024, 16512, 1536, 128, 16128, 2816, 2304, 7936, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3072, 640, 128, 20736, 384, 256, 256, 256, 15872, 8192, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 1024, 896, 2176, 256, 8320, 896, 36096, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 43008, 768, 2944, 256, 128, 2176, 256, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[640, 256, 1024, 256, 640, 256, 256, 128, 46208, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 160, 1397, 0, 0, 933, 0, 0, 4958, 1997, 0, 0, 0, 625, 0, 67, 314, 0, 1024, 0, 54, 0, 0, 5137, 59, 0, 99, 0, 0, 0, 0, 54, 347, 0, 0, 3742, 164, 0, 99, 0, 0, 4916, 0, 0, 0, 1, 0, 636, 0, 98, 0, 6948, 0, 8, 99, 7140, 0, 0, 0, 6715, 699, 662, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 118, 1054, 0, 1, 1402, 0, 0, 3806, 2419, 1, 7, 16, 615, 0, 88, 327, 8, 823, 15, 60, 2, 0, 4825, 0, 0, 85, 2, 0, 0, 5, 65, 316, 6, 4, 3976, 150, 1, 76, 15, 9, 5341, 0, 1, 1, 1, 13, 671, 15, 76, 0, 7297, 13, 90, 77, 7138, 0, 0, 0, 6823, 675, 623, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 52, 1783, 0, 0, 1029, 0, 0, 4930, 1723, 0, 0, 0, 628, 0, 85, 325, 0, 1072, 0, 68, 0, 0, 5461, 78, 0, 73, 0, 0, 0, 0, 71, 314, 0, 0, 3841, 180, 0, 73, 0, 0, 4463, 0, 0, 0, 0, 0, 682, 0, 73, 0, 6760, 0, 7, 77, 7094, 0, 0, 0, 6872, 694, 644, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 82, 1922, 0, 0, 929, 0, 0, 5276, 2091, 0, 0, 0, 614, 0, 84, 335, 0, 1109, 0, 67, 0, 0, 5850, 35, 0, 74, 0, 0, 0, 0, 68, 326, 0, 0, 2835, 213, 0, 74, 0, 0, 4250, 0, 0, 0, 0, 0, 685, 0, 74, 0, 6926, 0, 48, 75, 7021, 0, 0, 0, 6686, 718, 685, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[106, 217, 828, 600, 660, 527, 1008, 2, 0, 2044, 2184, 76, 1710, 2326, 396, 23, 82, 451, 1052, 1412, 365, 72, 3, 26, 1610, 767, 200, 270, 307, 129, 251, 610, 251, 234, 2461, 1170, 552, 464, 410, 115, 834, 675, 2610, 70, 517, 268, 768, 2492, 576, 664, 138, 80, 4245, 837, 390, 340, 1662, 103, 857, 397, 2516, 743, 1118, 281] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 136, 943, 0, 0, 1566, 0, 0, 3334, 2953, 0, 0, 0, 649, 0, 75, 339, 0, 693, 0, 72, 0, 0, 4314, 0, 0, 89, 0, 0, 0, 0, 72, 355, 0, 0, 3777, 186, 0, 89, 4, 0, 5728, 0, 0, 0, 0, 0, 624, 0, 87, 0, 7278, 5, 74, 85, 7281, 0, 0, 0, 7020, 685, 639, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 81, 909, 0, 0, 1805, 0, 0, 2880, 3581, 0, 0, 0, 669, 0, 97, 344, 0, 655, 0, 59, 0, 0, 4066, 0, 0, 80, 0, 0, 0, 0, 59, 327, 0, 0, 3695, 174, 0, 80, 0, 0, 5474, 0, 0, 0, 0, 0, 671, 0, 80, 0, 7469, 0, 96, 79, 7249, 0, 0, 0, 6981, 770, 722, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[1, 1, 63, 1484, 3, 3, 868, 0, 0, 5029, 1975, 0, 15, 19, 640, 1, 64, 328, 5, 936, 15, 74, 0, 1, 5726, 41, 0, 75, 0, 1, 2, 4, 77, 342, 12, 5, 3149, 252, 12, 72, 3, 9, 4883, 0, 4, 3, 10, 18, 656, 15, 73, 0, 6849, 4, 28, 74, 7065, 0, 2, 0, 6816, 707, 638, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 160, 1397, 0, 0, 933, 0, 2162, 1442, 0, 0, 0, 0, 0, 0, 0, 3259, 1997, 0, 0, 0, 625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 314, 0, 1024, 0, 54, 0, 0, 4978, 0, 0, 0, 0, 0, 0, 0, 5137, 59, 0, 99, 0, 0, 0, 0, 812, 457, 0, 0, 0, 0, 0, 0, 54, 347, 0, 0, 3742, 164, 0, 99, 1325, 466, 0, 0, 0, 0, 0, 0, 0, 0, 4916, 0, 0, 0, 1, 0, 887, 0, 0, 0, 0, 0, 0, 0, 636, 0, 98, 0, 5166, 0, 8, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4807, 699, 662, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6094, 5881, 6437, 6564, 6197, 5804, 6007, 6168] avg=6144.0000 max=6564 max_rank=3 min=5804 min_rank=5 imbalance_ratio=1.068359 max_load_reduction=42588 max_load_reduction_pct=0.866455 moved_to_echo=24176 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 52, 1783, 0, 0, 1029, 0, 2148, 1476, 0, 0, 0, 0, 0, 0, 0, 3240, 1723, 0, 0, 0, 628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 325, 0, 1072, 0, 68, 0, 0, 4946, 0, 0, 0, 0, 0, 0, 0, 5461, 78, 0, 73, 0, 0, 0, 0, 808, 445, 0, 0, 0, 0, 0, 0, 71, 314, 0, 0, 3841, 180, 0, 73, 1289, 477, 0, 0, 0, 0, 0, 0, 0, 0, 4463, 0, 0, 0, 0, 0, 882, 0, 0, 0, 0, 0, 0, 0, 682, 0, 73, 0, 5026, 0, 7, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4919, 694, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6488, 5591, 6496, 6865, 6245, 5345, 5865, 6257] avg=6144.0000 max=6865 max_rank=3 min=5345 min_rank=5 imbalance_ratio=1.117350 max_load_reduction=42287 max_load_reduction_pct=0.860331 moved_to_echo=23712 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 82, 1922, 0, 0, 929, 0, 2126, 1436, 0, 0, 0, 0, 0, 0, 0, 3469, 2091, 0, 0, 0, 614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 335, 0, 1109, 0, 67, 0, 0, 4895, 0, 0, 0, 0, 0, 0, 0, 5850, 35, 0, 74, 0, 0, 0, 0, 864, 455, 0, 0, 0, 0, 0, 0, 68, 326, 0, 0, 2835, 213, 0, 74, 1321, 464, 0, 0, 0, 0, 0, 0, 0, 0, 4250, 0, 0, 0, 0, 0, 943, 0, 0, 0, 0, 0, 0, 0, 685, 0, 74, 0, 5150, 0, 48, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4786, 718, 685, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6495, 6174, 6490, 7278, 5301, 5193, 6032, 6189] avg=6144.0000 max=7278 max_rank=3 min=5193 min_rank=5 imbalance_ratio=1.184570 max_load_reduction=41874 max_load_reduction_pct=0.851929 moved_to_echo=22715 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 118, 1054, 0, 1, 1402, 0, 2163, 1465, 0, 0, 0, 0, 0, 0, 0, 2503, 2419, 1, 7, 16, 615, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 327, 8, 823, 15, 60, 2, 0, 4975, 0, 0, 0, 0, 0, 0, 0, 4825, 0, 0, 85, 2, 0, 0, 5, 623, 480, 0, 0, 0, 0, 0, 0, 65, 316, 6, 4, 3976, 150, 1, 76, 1392, 474, 0, 0, 0, 0, 0, 0, 15, 9, 5341, 0, 1, 1, 1, 13, 680, 0, 0, 0, 0, 0, 0, 0, 671, 15, 76, 0, 5425, 13, 90, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4884, 675, 623, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6203, 5561, 6298, 6020, 6460, 6061, 6367, 6182] avg=6144.0000 max=6460 max_rank=4 min=5561 min_rank=1 imbalance_ratio=1.051432 max_load_reduction=42692 max_load_reduction_pct=0.868571 moved_to_echo=25070 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[106, 217, 828, 600, 660, 527, 1008, 2, 504, 540, 0, 0, 0, 0, 0, 0, 0, 1344, 2184, 76, 1710, 2326, 396, 23, 0, 0, 0, 0, 0, 0, 0, 0, 82, 451, 1052, 1412, 365, 72, 3, 26, 1158, 0, 0, 0, 0, 0, 0, 0, 1610, 767, 200, 270, 307, 129, 251, 610, 335, 279, 0, 0, 0, 0, 0, 0, 251, 234, 2461, 1170, 552, 464, 410, 115, 809, 174, 0, 0, 0, 0, 0, 0, 834, 675, 2610, 70, 517, 268, 768, 2492, 365, 0, 0, 0, 0, 0, 0, 0, 576, 664, 138, 80, 3157, 837, 390, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 857, 397, 1802, 743, 1118, 281, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4992, 8059, 4621, 4758, 6640, 8599, 6182, 5301] avg=6144.0000 max=8599 max_rank=5 min=4621 min_rank=2 imbalance_ratio=1.399577 max_load_reduction=40553 max_load_reduction_pct=0.825053 moved_to_echo=26722 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 136, 943, 0, 0, 1566, 0, 2206, 1508, 0, 0, 0, 0, 0, 0, 0, 2192, 2953, 0, 0, 0, 649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 339, 0, 693, 0, 72, 0, 0, 5075, 0, 0, 0, 0, 0, 0, 0, 4314, 0, 0, 89, 0, 0, 0, 0, 546, 479, 0, 0, 0, 0, 0, 0, 72, 355, 0, 0, 3777, 186, 0, 89, 1388, 487, 0, 0, 0, 0, 0, 0, 4, 0, 5728, 0, 0, 0, 0, 0, 596, 0, 0, 0, 0, 0, 0, 0, 624, 0, 87, 0, 5411, 5, 74, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5025, 685, 639, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6359, 5794, 6254, 5428, 6354, 6328, 6286, 6349] avg=6144.0000 max=6359 max_rank=0 min=5428 min_rank=3 imbalance_ratio=1.034993 max_load_reduction=42793 max_load_reduction_pct=0.870626 moved_to_echo=25317 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 81, 909, 0, 0, 1805, 0, 2195, 1499, 0, 0, 0, 0, 0, 0, 0, 1893, 3581, 0, 0, 0, 669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 344, 0, 655, 0, 59, 0, 0, 5054, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 80, 0, 0, 0, 0, 472, 491, 0, 0, 0, 0, 0, 0, 59, 327, 0, 0, 3695, 174, 0, 80, 1425, 485, 0, 0, 0, 0, 0, 0, 0, 0, 5474, 0, 0, 0, 0, 0, 515, 0, 0, 0, 0, 0, 0, 0, 671, 0, 80, 0, 5553, 0, 96, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4997, 770, 722, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6489, 6143, 6209, 5109, 6245, 5989, 6479, 6489] avg=6144.0000 max=6489 max_rank=0 min=5109 min_rank=3 imbalance_ratio=1.056152 max_load_reduction=42663 max_load_reduction_pct=0.867981 moved_to_echo=25202 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 63, 1484, 3, 3, 868, 0, 2141, 1468, 0, 0, 0, 0, 0, 0, 0, 3299, 1975, 0, 15, 19, 640, 1, 0, 0, 0, 0, 0, 0, 0, 0, 64, 328, 5, 936, 15, 74, 0, 1, 4924, 0, 0, 0, 0, 0, 0, 0, 5726, 41, 0, 75, 0, 1, 2, 4, 827, 454, 0, 0, 0, 0, 0, 0, 77, 342, 12, 5, 3149, 252, 12, 72, 1311, 478, 0, 0, 0, 0, 0, 0, 3, 9, 4883, 0, 4, 3, 10, 18, 903, 0, 0, 0, 0, 0, 0, 0, 656, 15, 73, 0, 5084, 4, 28, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4870, 707, 638, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6032, 5949, 6347, 7130, 5710, 5833, 5934, 6217] avg=6144.0000 max=7130 max_rank=3 min=5710 min_rank=4 imbalance_ratio=1.160482 max_load_reduction=42022 max_load_reduction_pct=0.854940 moved_to_echo=23694 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 256, 1536, 10112, 768, 640, 9600, 128, 15744, 10880, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[768, 2688, 2560, 1280, 25600, 1792, 512, 768, 10368, 3584, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 21248, 18944, 128, 1792, 2432, 4864, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[768, 2816, 1152, 7808, 512, 640, 128, 128, 36096, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[896, 768, 37760, 128, 640, 384, 896, 2560, 5888, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[36992, 1024, 256, 896, 384, 256, 256, 640, 5376, 3584, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 128, 896, 512, 36096, 5760, 5760, 384, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5248, 768, 768, 128, 40064, 896, 768, 1024, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[2977, 0, 0, 147, 704, 1949, 0, 0, 1488, 85, 771, 0, 0, 0, 0, 0, 0, 74, 132, 0, 0, 73, 0, 0, 10, 0, 0, 0, 23, 74, 0, 0, 0, 0, 5470, 0, 0, 0, 4364, 727, 0, 7146, 1701, 8025, 0, 0, 5142, 71, 0, 0, 0, 74, 0, 0, 118, 6195, 299, 0, 0, 329, 611, 124, 240, 9] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[2646, 0, 0, 112, 724, 2864, 0, 0, 1207, 67, 585, 0, 0, 0, 0, 0, 0, 100, 122, 0, 0, 99, 0, 0, 19, 0, 0, 0, 5, 100, 0, 0, 0, 0, 4987, 0, 0, 0, 4126, 569, 0, 7140, 2529, 7981, 0, 0, 4989, 96, 0, 0, 0, 100, 0, 0, 45, 6240, 268, 0, 0, 356, 609, 215, 247, 5] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[433, 72, 3175, 247, 668, 3361, 1101, 611, 1360, 270, 209, 0, 358, 85, 275, 238, 14, 43, 594, 2152, 285, 105, 1279, 603, 163, 141, 724, 323, 126, 192, 279, 375, 1749, 243, 1148, 62, 391, 660, 2156, 675, 143, 2868, 1486, 1089, 176, 1241, 2863, 621, 301, 1611, 337, 134, 84, 306, 548, 1275, 1257, 2924, 97, 808, 591, 129, 1023, 295] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[3287, 0, 0, 101, 692, 2442, 0, 0, 1641, 84, 694, 0, 0, 0, 0, 0, 0, 75, 150, 0, 0, 74, 0, 0, 16, 0, 0, 0, 60, 75, 0, 0, 0, 0, 4911, 0, 0, 0, 4165, 494, 0, 7006, 1823, 8027, 0, 0, 5026, 74, 0, 0, 0, 75, 0, 0, 23, 6446, 285, 0, 0, 362, 642, 150, 251, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[2579, 0, 18, 76, 689, 4021, 1, 0, 1363, 92, 467, 0, 0, 4, 1, 1, 0, 76, 161, 15, 3, 76, 9, 0, 53, 0, 0, 9, 0, 76, 0, 0, 3, 0, 4395, 0, 0, 5, 3627, 506, 0, 7189, 2775, 7950, 0, 11, 4839, 66, 5, 2, 6, 76, 0, 1, 10, 6187, 321, 14, 0, 308, 587, 179, 240, 60] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[2589, 0, 0, 80, 747, 4999, 0, 0, 916, 96, 353, 0, 0, 0, 0, 0, 0, 80, 164, 0, 0, 80, 0, 0, 113, 0, 0, 0, 0, 80, 0, 0, 0, 0, 3965, 0, 0, 0, 2979, 445, 0, 7171, 3623, 8009, 0, 0, 4273, 46, 0, 0, 0, 80, 0, 0, 0, 6355, 440, 0, 0, 379, 686, 140, 247, 17] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[2701, 0, 24, 131, 712, 3234, 6, 0, 1327, 72, 597, 0, 1, 1, 0, 5, 0, 72, 112, 17, 6, 72, 8, 5, 7, 0, 0, 5, 1, 74, 0, 2, 9, 0, 5045, 0, 0, 2, 3723, 496, 2, 7164, 2075, 7822, 1, 15, 5270, 72, 2, 5, 1, 72, 0, 5, 19, 6363, 392, 19, 0, 368, 605, 137, 256, 20] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[2720, 0, 0, 90, 732, 4530, 0, 0, 913, 74, 367, 0, 0, 0, 0, 0, 0, 90, 128, 0, 0, 88, 0, 0, 123, 0, 0, 0, 0, 90, 0, 0, 0, 0, 3879, 0, 0, 0, 2958, 501, 0, 7185, 3558, 8002, 0, 0, 4793, 65, 0, 0, 0, 90, 0, 0, 0, 6392, 466, 0, 0, 223, 595, 203, 265, 32] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2646, 0, 0, 112, 724, 1836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1207, 67, 585, 0, 0, 0, 0, 0, 2977, 1629, 0, 0, 0, 0, 0, 0, 0, 100, 122, 0, 0, 99, 0, 0, 4163, 1555, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 5, 100, 0, 0, 6426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1995, 0, 0, 0, 4126, 569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2529, 0, 0, 0, 3606, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 45, 6181, 0, 0, 0, 0, 0, 0, 0, 0, 268, 0, 0, 356, 609, 215, 247, 5, 1383, 1028, 1363, 59, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5318, 6465, 6039, 6550, 6690, 6231, 6326, 5533] avg=6144.0000 max=6690 max_rank=4 min=5318 min_rank=0 imbalance_ratio=1.088867 max_load_reduction=42462 max_load_reduction_pct=0.863892 moved_to_echo=24780 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2977, 0, 0, 147, 704, 1250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1488, 85, 771, 0, 0, 0, 0, 0, 2979, 1786, 0, 0, 0, 0, 0, 0, 0, 74, 132, 0, 0, 73, 0, 0, 4167, 1564, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 23, 74, 0, 0, 6461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2189, 0, 0, 0, 4364, 727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1701, 0, 0, 0, 3717, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 118, 6137, 0, 0, 0, 0, 0, 0, 0, 0, 299, 0, 0, 329, 611, 124, 240, 9, 1425, 699, 1495, 58, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5078, 7109, 6010, 6568, 7280, 5489, 6329, 5289] avg=6144.0000 max=7280 max_rank=4 min=5078 min_rank=0 imbalance_ratio=1.184896 max_load_reduction=41872 max_load_reduction_pct=0.851888 moved_to_echo=24387 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3287, 0, 0, 101, 692, 1566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1641, 84, 694, 0, 0, 0, 0, 0, 2921, 1604, 0, 0, 0, 0, 0, 0, 0, 75, 150, 0, 0, 74, 0, 0, 4085, 1564, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 60, 75, 0, 0, 6463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1965, 0, 0, 0, 4165, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1823, 0, 0, 0, 3633, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 23, 6385, 0, 0, 0, 0, 0, 0, 0, 0, 285, 0, 0, 362, 642, 150, 251, 1, 1393, 876, 1342, 61, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5646, 6944, 5948, 6614, 6624, 5530, 6483, 5363] avg=6144.0000 max=6944 max_rank=1 min=5363 min_rank=7 imbalance_ratio=1.130208 max_load_reduction=42208 max_load_reduction_pct=0.858724 moved_to_echo=24000 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[433, 72, 3175, 247, 668, 2154, 1101, 611, 0, 0, 0, 0, 0, 0, 0, 0, 1360, 270, 209, 0, 358, 85, 275, 238, 1196, 375, 0, 0, 0, 0, 0, 0, 14, 43, 594, 2152, 285, 105, 1279, 603, 1672, 213, 0, 0, 0, 0, 0, 0, 163, 141, 724, 323, 126, 192, 279, 375, 876, 0, 0, 0, 0, 0, 0, 0, 1749, 243, 460, 62, 391, 660, 2156, 675, 0, 0, 0, 0, 0, 0, 0, 0, 143, 0, 1486, 0, 176, 1241, 2070, 621, 0, 0, 0, 0, 0, 0, 0, 0, 301, 1611, 337, 134, 84, 306, 548, 1263, 0, 0, 0, 0, 0, 0, 0, 0, 1257, 2924, 97, 808, 591, 129, 1023, 295, 793, 1207, 313, 12, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8461, 4366, 6960, 3199, 6396, 5737, 4584, 9449] avg=6144.0000 max=9449 max_rank=7 min=3199 min_rank=3 imbalance_ratio=1.537923 max_load_reduction=39703 max_load_reduction_pct=0.807760 moved_to_echo=26166 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2579, 0, 18, 76, 689, 2577, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1363, 92, 467, 0, 0, 4, 1, 1, 2998, 1435, 0, 0, 0, 0, 0, 0, 0, 76, 161, 15, 3, 76, 9, 0, 4191, 1550, 0, 0, 0, 0, 0, 0, 53, 0, 0, 9, 0, 76, 0, 0, 6400, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1759, 0, 0, 5, 3627, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2775, 0, 0, 11, 3498, 66, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 6, 76, 0, 1, 10, 6129, 0, 0, 0, 0, 0, 0, 0, 0, 321, 14, 0, 308, 587, 179, 240, 60, 1341, 1444, 1201, 58, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5940, 6361, 6081, 6538, 5900, 6350, 6229, 5753] avg=6144.0000 max=6538 max_rank=3 min=5753 min_rank=7 imbalance_ratio=1.064128 max_load_reduction=42614 max_load_reduction_pct=0.866984 moved_to_echo=24232 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2720, 0, 0, 90, 732, 2904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 913, 74, 367, 0, 0, 0, 0, 0, 2996, 1267, 0, 0, 0, 0, 0, 0, 0, 90, 128, 0, 0, 88, 0, 0, 4189, 1560, 0, 0, 0, 0, 0, 0, 123, 0, 0, 0, 0, 90, 0, 0, 6442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1552, 0, 0, 0, 2958, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3558, 0, 0, 0, 3465, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 6332, 0, 0, 0, 0, 0, 0, 0, 0, 466, 0, 0, 223, 595, 203, 265, 32, 1328, 1626, 1060, 60, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6446, 5617, 6055, 6655, 5011, 7088, 6422, 5858] avg=6144.0000 max=7088 max_rank=5 min=5011 min_rank=4 imbalance_ratio=1.153646 max_load_reduction=42064 max_load_reduction_pct=0.855794 moved_to_echo=24379 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2589, 0, 0, 80, 747, 3204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 916, 96, 353, 0, 0, 0, 0, 0, 2989, 1295, 0, 0, 0, 0, 0, 0, 0, 80, 164, 0, 0, 80, 0, 0, 4182, 1560, 0, 0, 0, 0, 0, 0, 113, 0, 0, 0, 0, 80, 0, 0, 6449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1586, 0, 0, 0, 2979, 445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3623, 0, 0, 0, 3089, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 6295, 0, 0, 0, 0, 0, 0, 0, 0, 440, 0, 0, 379, 686, 140, 247, 17, 1184, 1795, 1084, 60, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6620, 5649, 6066, 6642, 5010, 6758, 6375, 6032] avg=6144.0000 max=6758 max_rank=5 min=5010 min_rank=4 imbalance_ratio=1.099935 max_load_reduction=42394 max_load_reduction_pct=0.862508 moved_to_echo=24175 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2701, 0, 24, 131, 712, 2069, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1327, 72, 597, 0, 1, 1, 0, 5, 2988, 1650, 0, 0, 0, 0, 0, 0, 0, 72, 112, 17, 6, 72, 8, 5, 4176, 1525, 0, 0, 0, 0, 0, 0, 7, 0, 0, 5, 1, 74, 0, 2, 6297, 0, 0, 0, 0, 0, 0, 0, 9, 0, 2011, 0, 0, 2, 3723, 496, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2075, 0, 1, 15, 3804, 72, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5, 1, 72, 0, 5, 19, 6300, 0, 0, 0, 0, 0, 0, 0, 0, 392, 19, 0, 368, 605, 137, 256, 20, 1466, 1165, 1384, 63, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5643, 6641, 5993, 6386, 6241, 5969, 6404, 5875] avg=6144.0000 max=6641 max_rank=1 min=5643 min_rank=0 imbalance_ratio=1.080892 max_load_reduction=42511 max_load_reduction_pct=0.864889 moved_to_echo=24489 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[10240, 896, 4096, 0, 384, 128, 384, 256, 22144, 11136, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3840, 3072, 128, 3200, 4992, 1280, 2816, 512, 10368, 9856, 9344, 512, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1792, 256, 13568, 128, 512, 768, 28160, 4480, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[19968, 128, 3328, 1024, 5760, 17664, 1152, 640, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[256, 0, 19584, 0, 256, 1280, 27008, 1152, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 640, 1664, 2304, 384, 768, 1408, 640, 30848, 11136, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[384, 1664, 384, 768, 128, 384, 768, 45056, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[512, 256, 768, 384, 256, 768, 384, 384, 45824, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=17 tokens_per_expert_current_ep_rank=[2922, 653, 0, 0, 0, 1, 2069, 10, 0, 4005, 1, 429, 57, 0, 598, 0, 3, 152, 550, 0, 7, 573, 0, 4, 1, 0, 5862, 610, 2473, 8, 326, 1222, 17, 16, 0, 0, 15, 1, 0, 3559, 0, 15, 717, 149, 6309, 138, 6518, 11, 6529, 0, 1, 175, 0, 76, 0, 2, 88, 0, 0, 19, 1534, 76, 2, 649] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=17 tokens_per_expert_current_ep_rank=[3158, 439, 0, 0, 0, 0, 1826, 3, 0, 5762, 0, 452, 154, 0, 748, 0, 0, 147, 788, 0, 0, 525, 6, 0, 0, 0, 4704, 624, 1488, 0, 233, 639, 0, 0, 0, 0, 0, 0, 0, 5274, 0, 0, 233, 131, 5413, 219, 6875, 0, 6771, 12, 0, 181, 0, 73, 0, 0, 85, 0, 0, 0, 1500, 74, 0, 615] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=17 tokens_per_expert_current_ep_rank=[2833, 1038, 0, 0, 0, 0, 2258, 4, 0, 3560, 0, 402, 7, 0, 407, 0, 0, 179, 391, 0, 0, 534, 0, 0, 0, 0, 6584, 548, 2273, 0, 338, 1223, 0, 0, 0, 0, 0, 0, 0, 3222, 0, 0, 1063, 119, 6440, 133, 6282, 0, 6596, 0, 0, 179, 0, 89, 0, 0, 68, 0, 0, 0, 1566, 88, 0, 728] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=17 tokens_per_expert_current_ep_rank=[3029, 604, 0, 0, 0, 0, 1953, 1, 0, 4667, 0, 547, 129, 0, 754, 0, 0, 199, 625, 0, 0, 523, 0, 0, 0, 0, 5210, 581, 2003, 0, 375, 820, 0, 0, 0, 0, 0, 0, 0, 4312, 0, 0, 350, 122, 6113, 261, 6677, 0, 6605, 0, 0, 166, 0, 99, 0, 0, 67, 0, 0, 0, 1486, 100, 0, 774] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=17 tokens_per_expert_current_ep_rank=[2413, 537, 0, 0, 0, 0, 1734, 0, 0, 5444, 0, 589, 189, 0, 797, 0, 0, 149, 711, 1, 0, 487, 0, 0, 0, 0, 4739, 607, 1467, 0, 262, 669, 0, 0, 0, 0, 0, 0, 0, 5728, 0, 0, 203, 141, 5797, 216, 6906, 0, 6817, 14, 0, 202, 0, 74, 0, 0, 84, 0, 0, 0, 1437, 75, 0, 663] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=17 tokens_per_expert_current_ep_rank=[1529, 944, 275, 16, 226, 1026, 554, 198, 514, 1223, 1258, 522, 724, 284, 91, 64, 622, 223, 408, 811, 497, 1193, 169, 446, 60, 520, 750, 782, 1204, 985, 327, 737, 3251, 883, 525, 44, 2954, 80, 287, 1596, 346, 1703, 1801, 59, 1936, 213, 1630, 1623, 2472, 345, 189, 298, 0, 71, 47, 315, 59, 241, 80, 3407, 1943, 264, 844, 464] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=17 tokens_per_expert_current_ep_rank=[2744, 914, 0, 0, 0, 0, 2413, 0, 0, 3430, 0, 392, 0, 0, 334, 0, 0, 160, 432, 0, 0, 621, 0, 0, 0, 0, 6399, 652, 2066, 0, 302, 1573, 0, 0, 0, 0, 0, 0, 0, 3346, 0, 0, 1090, 163, 6436, 100, 6266, 0, 6489, 0, 0, 143, 0, 80, 0, 0, 87, 0, 0, 0, 1762, 80, 0, 678] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=17 tokens_per_expert_current_ep_rank=[3019, 534, 0, 0, 3, 2, 1667, 6, 2, 4317, 12, 700, 172, 2, 908, 0, 7, 144, 660, 10, 1, 520, 1, 2, 5, 1, 5133, 639, 1585, 6, 232, 653, 24, 12, 2, 0, 22, 4, 0, 5417, 1, 15, 376, 109, 5745, 239, 6735, 14, 6858, 2, 0, 176, 0, 72, 0, 12, 64, 1, 0, 22, 1560, 72, 4, 651] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=17 rerouted_tokens_per_expert_current_ep_rank=[3158, 439, 0, 0, 0, 0, 1826, 3, 685, 0, 0, 0, 0, 0, 0, 0, 0, 5762, 0, 452, 154, 0, 748, 0, 424, 174, 0, 0, 0, 0, 0, 0, 0, 147, 788, 0, 0, 525, 6, 0, 5120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2155, 624, 1488, 0, 233, 639, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5274, 1012, 10, 0, 0, 0, 0, 0, 0, 0, 0, 233, 131, 4544, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6347, 12, 0, 181, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 1500, 74, 0, 615, 1537, 1755, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6111, 7714, 6586, 5139, 6296, 5127, 6613, 5566] avg=6144.0000 max=7714 max_rank=1 min=5127 min_rank=5 imbalance_ratio=1.255534 max_load_reduction=41438 max_load_reduction_pct=0.843058 moved_to_echo=23602 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=17 rerouted_tokens_per_expert_current_ep_rank=[3029, 604, 0, 0, 0, 0, 1953, 1, 774, 0, 0, 0, 0, 0, 0, 0, 0, 4667, 0, 547, 129, 0, 754, 0, 413, 197, 0, 0, 0, 0, 0, 0, 0, 199, 625, 0, 0, 523, 0, 0, 4973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2386, 581, 2003, 0, 375, 820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4312, 1121, 11, 0, 0, 0, 0, 0, 0, 0, 0, 350, 122, 5131, 261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6192, 0, 0, 166, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 1486, 100, 0, 774, 1703, 1704, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6361, 6707, 6320, 6165, 5444, 5864, 6457, 5834] avg=6144.0000 max=6707 max_rank=1 min=5444 min_rank=4 imbalance_ratio=1.091634 max_load_reduction=42445 max_load_reduction_pct=0.863546 moved_to_echo=23599 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=17 rerouted_tokens_per_expert_current_ep_rank=[2413, 537, 0, 0, 0, 0, 1734, 0, 734, 0, 0, 0, 0, 0, 0, 0, 0, 5444, 0, 589, 189, 0, 797, 0, 427, 187, 0, 0, 0, 0, 0, 0, 0, 149, 711, 1, 0, 487, 0, 0, 5144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2171, 607, 1467, 0, 262, 669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5728, 1019, 11, 0, 0, 0, 0, 0, 0, 0, 0, 203, 141, 4865, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6390, 14, 0, 202, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 1437, 75, 0, 663, 1549, 1762, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5418, 7633, 6492, 5176, 6758, 5425, 6680, 5570] avg=6144.0000 max=7633 max_rank=1 min=5176 min_rank=3 imbalance_ratio=1.242350 max_load_reduction=41519 max_load_reduction_pct=0.844706 moved_to_echo=24433 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=17 rerouted_tokens_per_expert_current_ep_rank=[2922, 653, 0, 0, 0, 1, 2069, 10, 799, 0, 0, 0, 0, 0, 0, 0, 0, 4005, 1, 429, 57, 0, 598, 0, 408, 203, 0, 0, 0, 0, 0, 0, 3, 152, 550, 0, 7, 573, 0, 4, 4855, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2685, 610, 2473, 8, 326, 1222, 0, 0, 0, 0, 0, 0, 0, 0, 17, 16, 0, 0, 15, 1, 0, 3559, 1261, 12, 0, 0, 0, 0, 0, 0, 0, 15, 717, 149, 5295, 138, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 6121, 0, 1, 175, 0, 76, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 19, 1534, 76, 2, 649, 1916, 1663, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6454, 5701, 6144, 7325, 4881, 6325, 6375, 5947] avg=6144.0000 max=7325 max_rank=3 min=4881 min_rank=4 imbalance_ratio=1.192220 max_load_reduction=41827 max_load_reduction_pct=0.850972 moved_to_echo=23528 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=17 rerouted_tokens_per_expert_current_ep_rank=[2833, 1038, 0, 0, 0, 0, 2258, 4, 815, 0, 0, 0, 0, 0, 0, 0, 0, 3560, 0, 402, 7, 0, 407, 0, 413, 207, 0, 0, 0, 0, 0, 0, 0, 179, 391, 0, 0, 534, 0, 0, 4680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3016, 548, 2273, 0, 338, 1223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3222, 1416, 12, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 119, 5406, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6183, 0, 0, 179, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 1566, 88, 0, 728, 2152, 1602, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6948, 4996, 5784, 7398, 4650, 6721, 6451, 6204] avg=6144.0000 max=7398 max_rank=3 min=4650 min_rank=4 imbalance_ratio=1.204102 max_load_reduction=41754 max_load_reduction_pct=0.849487 moved_to_echo=24026 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=17 rerouted_tokens_per_expert_current_ep_rank=[1529, 944, 275, 16, 226, 1026, 554, 198, 245, 0, 0, 0, 0, 0, 0, 0, 514, 1223, 1258, 522, 724, 284, 91, 64, 154, 62, 0, 0, 0, 0, 0, 0, 622, 223, 408, 811, 497, 1193, 169, 446, 1215, 0, 0, 0, 0, 0, 0, 0, 60, 520, 344, 782, 1204, 985, 327, 737, 0, 0, 0, 0, 0, 0, 0, 0, 3251, 883, 525, 44, 2954, 80, 287, 1596, 161, 3, 0, 0, 0, 0, 0, 0, 346, 1703, 1801, 59, 1626, 213, 0, 1623, 0, 0, 0, 0, 0, 0, 0, 0, 2318, 345, 189, 298, 0, 71, 47, 315, 0, 0, 0, 0, 0, 0, 0, 0, 59, 241, 80, 3407, 1943, 264, 844, 464, 245, 415, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5013, 4896, 5584, 4959, 9784, 7371, 3583, 7962] avg=6144.0000 max=9784 max_rank=4 min=3583 min_rank=6 imbalance_ratio=1.592448 max_load_reduction=39368 max_load_reduction_pct=0.800944 moved_to_echo=28700 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=17 rerouted_tokens_per_expert_current_ep_rank=[2744, 914, 0, 0, 0, 0, 2413, 0, 815, 0, 0, 0, 0, 0, 0, 0, 0, 3430, 0, 392, 0, 0, 334, 0, 406, 207, 0, 0, 0, 0, 0, 0, 0, 160, 432, 0, 0, 621, 0, 0, 4667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2931, 652, 2066, 0, 302, 1573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3346, 1377, 12, 0, 0, 0, 0, 0, 0, 0, 0, 1090, 163, 5402, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6083, 0, 0, 143, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 0, 1762, 80, 0, 678, 2091, 1599, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6886, 4769, 5880, 7524, 4735, 6755, 6306, 6297] avg=6144.0000 max=7524 max_rank=3 min=4735 min_rank=4 imbalance_ratio=1.224609 max_load_reduction=41628 max_load_reduction_pct=0.846924 moved_to_echo=24093 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=17 rerouted_tokens_per_expert_current_ep_rank=[3019, 534, 0, 0, 3, 2, 1667, 6, 730, 0, 0, 0, 0, 0, 0, 0, 2, 4317, 12, 700, 172, 2, 908, 0, 433, 190, 0, 0, 0, 0, 0, 0, 7, 144, 660, 10, 1, 520, 1, 2, 5017, 0, 0, 0, 0, 0, 0, 0, 5, 1, 2345, 639, 1585, 6, 232, 653, 0, 0, 0, 0, 0, 0, 0, 0, 24, 12, 2, 0, 22, 4, 0, 5417, 1108, 15, 0, 0, 0, 0, 0, 0, 1, 15, 376, 109, 4810, 239, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 6425, 2, 0, 176, 0, 72, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 64, 1, 0, 22, 1560, 72, 4, 651, 1680, 1718, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5961, 6736, 6362, 5466, 6604, 5564, 6687, 5772] avg=6144.0000 max=6736 max_rank=1 min=5466 min_rank=3 imbalance_ratio=1.096354 max_load_reduction=42416 max_load_reduction_pct=0.862956 moved_to_echo=24627 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[21760, 5760, 384, 128, 256, 1152, 14592, 256, 5632, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[640, 256, 128, 3456, 12800, 896, 896, 5248, 12928, 12288, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 640, 18048, 5120, 14592, 1024, 2432, 7552, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3328, 1024, 640, 128, 3072, 128, 384, 32512, 8576, 128, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[640, 32512, 1280, 4096, 1536, 384, 4736, 128, 3200, 1536, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[384, 1792, 5888, 1024, 37120, 1536, 0, 1664, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[640, 1408, 4608, 896, 512, 4992, 256, 512, 35712, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[46080, 384, 256, 1536, 0, 640, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=18 tokens_per_expert_current_ep_rank=[0, 58, 644, 0, 0, 0, 2577, 0, 0, 0, 145, 104, 7491, 0, 0, 7593, 2, 0, 0, 0, 0, 4890, 0, 0, 0, 0, 0, 8116, 0, 0, 3002, 0, 0, 0, 73, 0, 0, 6802, 0, 0, 0, 0, 475, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 257, 0, 74, 0, 5790, 0, 0, 829, 225, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=18 tokens_per_expert_current_ep_rank=[0, 30, 778, 0, 0, 0, 2584, 0, 0, 0, 158, 81, 7422, 0, 0, 7504, 61, 0, 0, 0, 0, 4241, 0, 0, 0, 0, 0, 8091, 0, 0, 3559, 0, 0, 0, 99, 0, 0, 7204, 0, 0, 0, 0, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 100, 0, 5398, 0, 0, 818, 225, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=18 tokens_per_expert_current_ep_rank=[3, 4, 1273, 1, 0, 0, 2073, 0, 0, 0, 137, 103, 7352, 0, 18, 7506, 64, 1, 0, 0, 0, 3429, 7, 12, 4, 0, 0, 8065, 0, 19, 4135, 0, 0, 1, 76, 0, 1, 7584, 5, 7, 3, 0, 531, 0, 2, 19, 13, 0, 0, 12, 0, 0, 0, 251, 14, 76, 1, 4785, 0, 0, 747, 811, 7, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=18 tokens_per_expert_current_ep_rank=[0, 62, 555, 0, 0, 0, 2791, 0, 0, 0, 148, 97, 7423, 0, 0, 7556, 13, 0, 0, 0, 0, 4689, 0, 0, 0, 0, 0, 8118, 0, 0, 3398, 0, 7, 0, 74, 0, 0, 6979, 0, 0, 0, 0, 468, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 251, 0, 75, 0, 5517, 0, 0, 833, 94, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 1791, 0, 0, 0, 2119, 0, 0, 0, 148, 131, 7289, 0, 0, 7548, 78, 0, 0, 0, 0, 2380, 0, 0, 0, 0, 0, 8062, 0, 0, 5314, 0, 0, 0, 80, 0, 0, 7684, 0, 0, 0, 0, 422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 249, 0, 80, 0, 4238, 0, 0, 509, 1030, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=18 tokens_per_expert_current_ep_rank=[0, 0, 1747, 0, 0, 0, 1898, 0, 0, 0, 142, 92, 7339, 0, 0, 7491, 25, 0, 0, 0, 0, 2666, 0, 0, 0, 0, 0, 8087, 0, 0, 5144, 0, 0, 0, 89, 0, 0, 7639, 0, 2, 0, 0, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 330, 0, 90, 0, 4536, 0, 0, 513, 854, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=18 tokens_per_expert_current_ep_rank=[83, 473, 1786, 451, 21, 97, 975, 4, 2, 215, 95, 54, 3014, 4, 2275, 2301, 247, 86, 14, 664, 764, 378, 230, 2208, 151, 706, 94, 2611, 79, 3574, 1193, 551, 213, 4, 106, 63, 138, 1495, 1437, 326, 659, 0, 86, 0, 1424, 3950, 1111, 129, 0, 450, 91, 471, 698, 1, 2342, 183, 152, 832, 0, 343, 690, 2553, 3345, 460] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=18 tokens_per_expert_current_ep_rank=[5, 71, 893, 1, 0, 0, 2266, 0, 0, 0, 125, 81, 7457, 2, 25, 7577, 5, 0, 0, 0, 3, 4278, 2, 16, 2, 2, 0, 8072, 2, 26, 3609, 2, 8, 0, 73, 0, 0, 7120, 3, 4, 0, 0, 422, 0, 3, 28, 15, 0, 0, 6, 17, 1, 1, 255, 18, 72, 3, 5542, 0, 5, 785, 237, 10, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 30, 778, 0, 0, 0, 2584, 0, 2583, 288, 0, 0, 0, 0, 0, 0, 0, 0, 158, 81, 6066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 4241, 0, 0, 1068, 856, 525, 0, 0, 0, 0, 0, 0, 0, 0, 1994, 0, 0, 3559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 0, 0, 6348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 0, 0, 0, 0, 0, 3514, 1724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 100, 5780, 0, 0, 0, 0, 0, 0, 0, 0, 4873, 0, 0, 818, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6263, 6305, 6751, 5553, 6447, 5781, 6136, 5916] avg=6144.0000 max=6751 max_rank=2 min=5553 min_rank=3 imbalance_ratio=1.098796 max_load_reduction=42401 max_load_reduction_pct=0.862651 moved_to_echo=24280 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 58, 644, 0, 0, 0, 2577, 0, 2591, 291, 0, 0, 0, 0, 0, 0, 0, 0, 145, 104, 6122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4890, 0, 0, 1078, 808, 563, 0, 0, 0, 0, 0, 0, 0, 0, 2000, 0, 0, 3002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 5994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 0, 0, 0, 0, 0, 3525, 1744, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 257, 0, 74, 5849, 0, 0, 0, 0, 0, 0, 0, 0, 5227, 0, 0, 829, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6161, 6371, 7341, 5002, 6067, 5744, 6185, 6281] avg=6144.0000 max=7341 max_rank=2 min=5002 min_rank=3 imbalance_ratio=1.194824 max_load_reduction=41811 max_load_reduction_pct=0.850647 moved_to_echo=24277 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 62, 555, 0, 0, 0, 2791, 0, 2591, 288, 0, 0, 0, 0, 0, 0, 0, 0, 148, 97, 6067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 4689, 0, 0, 1068, 829, 537, 0, 0, 0, 0, 0, 0, 0, 0, 2001, 0, 0, 3398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 74, 0, 0, 6150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 0, 0, 0, 0, 0, 3526, 1736, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 251, 0, 75, 5820, 0, 0, 0, 0, 0, 0, 0, 0, 4980, 0, 0, 833, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6287, 6312, 7136, 5399, 6231, 5730, 6150, 5907] avg=6144.0000 max=7136 max_rank=2 min=5399 min_rank=3 imbalance_ratio=1.161458 max_load_reduction=42016 max_load_reduction_pct=0.854818 moved_to_echo=24018 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=18 rerouted_tokens_per_expert_current_ep_rank=[3, 4, 1273, 1, 0, 0, 2073, 0, 2575, 285, 0, 0, 0, 0, 0, 0, 0, 0, 137, 103, 6009, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 1, 0, 0, 0, 3429, 7, 12, 1058, 901, 465, 0, 0, 0, 0, 0, 4, 0, 0, 1987, 0, 19, 4135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 76, 0, 1, 6683, 5, 7, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 531, 0, 2, 19, 13, 0, 3503, 1724, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 251, 14, 76, 5782, 0, 0, 0, 0, 0, 0, 0, 1, 4320, 0, 0, 747, 811, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6214, 6267, 5937, 6145, 6773, 5795, 6135, 5886] avg=6144.0000 max=6773 max_rank=4 min=5795 min_rank=5 imbalance_ratio=1.102376 max_load_reduction=42379 max_load_reduction_pct=0.862203 moved_to_echo=24589 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=18 rerouted_tokens_per_expert_current_ep_rank=[83, 473, 1786, 451, 21, 97, 975, 4, 833, 117, 0, 0, 0, 0, 0, 0, 2, 215, 95, 54, 2464, 4, 2275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 86, 14, 664, 764, 378, 230, 2208, 433, 177, 81, 0, 0, 0, 0, 0, 151, 706, 94, 644, 79, 3574, 1193, 551, 0, 0, 0, 0, 0, 0, 0, 0, 213, 4, 106, 63, 138, 1318, 1437, 326, 0, 0, 0, 0, 0, 0, 0, 0, 659, 0, 86, 0, 1424, 3950, 1111, 129, 1134, 529, 0, 0, 0, 0, 0, 0, 0, 450, 91, 471, 698, 1, 2342, 183, 1772, 0, 0, 0, 0, 0, 0, 0, 152, 751, 0, 343, 690, 2553, 3345, 460, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4840, 5109, 5282, 6992, 3605, 9022, 6008, 8294] avg=6144.0000 max=9022 max_rank=5 min=3605 min_rank=4 imbalance_ratio=1.468424 max_load_reduction=40130 max_load_reduction_pct=0.816447 moved_to_echo=26929 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1747, 0, 0, 0, 1898, 0, 2582, 285, 0, 0, 0, 0, 0, 0, 0, 0, 142, 92, 5998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 2666, 0, 0, 1056, 908, 441, 0, 0, 0, 0, 0, 0, 0, 0, 1992, 0, 0, 5144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 0, 6731, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 0, 0, 0, 0, 0, 3513, 1722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 330, 0, 90, 5769, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 513, 854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6512, 6232, 5096, 7136, 6822, 5703, 6189, 5462] avg=6144.0000 max=7136 max_rank=3 min=5096 min_rank=2 imbalance_ratio=1.161458 max_load_reduction=42016 max_load_reduction_pct=0.854818 moved_to_echo=24176 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1791, 0, 0, 0, 2119, 0, 2574, 283, 0, 0, 0, 0, 0, 0, 0, 0, 148, 131, 5957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0, 2380, 0, 0, 1049, 913, 412, 0, 0, 0, 0, 0, 0, 0, 0, 1986, 0, 0, 5314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 6771, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 422, 0, 0, 0, 0, 0, 3502, 1734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 249, 0, 80, 5814, 0, 0, 0, 0, 0, 0, 0, 0, 3826, 0, 0, 509, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6767, 6236, 4832, 7300, 6851, 5658, 6143, 5365] avg=6144.0000 max=7300 max_rank=3 min=4832 min_rank=2 imbalance_ratio=1.188151 max_load_reduction=41852 max_load_reduction_pct=0.851481 moved_to_echo=24017 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=18 rerouted_tokens_per_expert_current_ep_rank=[5, 71, 893, 1, 0, 0, 2266, 0, 2580, 292, 0, 0, 0, 0, 0, 0, 0, 0, 125, 81, 6087, 2, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 3, 4278, 2, 16, 1078, 852, 543, 0, 0, 0, 0, 0, 2, 2, 0, 1982, 2, 26, 3609, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 73, 0, 0, 6268, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 422, 0, 3, 28, 15, 0, 3510, 1741, 0, 0, 0, 0, 0, 0, 0, 6, 17, 1, 1, 255, 18, 72, 5836, 0, 0, 0, 0, 0, 0, 0, 3, 4999, 0, 5, 785, 237, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6108, 6320, 6777, 5625, 6356, 5719, 6206, 6041] avg=6144.0000 max=6777 max_rank=2 min=5625 min_rank=3 imbalance_ratio=1.103027 max_load_reduction=42375 max_load_reduction_pct=0.862122 moved_to_echo=24322 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 768, 9472, 512, 128, 128, 17408, 128, 18944, 2176, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[512, 128, 128, 768, 768, 27008, 256, 2304, 7936, 6272, 3584, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[256, 768, 128, 14592, 128, 3712, 29440, 640, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[768, 0, 3456, 0, 1536, 4096, 1152, 256, 25728, 12672, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[256, 33152, 0, 384, 5760, 6144, 3456, 512, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 256, 1152, 768, 44800, 128, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 512, 128, 512, 768, 1920, 2432, 768, 42496, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[256, 128, 768, 128, 256, 46336, 1536, 384, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=19 tokens_per_expert_current_ep_rank=[29, 0, 13, 0, 19, 47, 1, 1, 0, 0, 158, 6, 0, 63, 896, 0, 3285, 15, 153, 7586, 7793, 0, 10, 5, 0, 0, 925, 0, 7696, 1, 68, 0, 5658, 8, 11, 1, 2, 4610, 7468, 76, 0, 5, 0, 16, 3, 61, 0, 63, 0, 983, 0, 210, 0, 17, 0, 0, 2, 0, 21, 25, 448, 18, 344, 332] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=19 tokens_per_expert_current_ep_rank=[52, 0, 0, 0, 0, 22, 0, 0, 0, 0, 206, 0, 0, 53, 961, 0, 3106, 0, 179, 7539, 7797, 0, 0, 0, 0, 0, 1225, 0, 7693, 0, 53, 0, 6475, 0, 6, 0, 0, 3781, 7525, 90, 0, 0, 0, 0, 0, 48, 0, 72, 0, 852, 0, 212, 0, 0, 0, 0, 0, 0, 5, 68, 468, 0, 321, 343] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=19 tokens_per_expert_current_ep_rank=[8, 0, 0, 0, 0, 92, 0, 0, 0, 0, 221, 0, 0, 55, 728, 0, 3754, 0, 198, 7525, 7617, 0, 0, 0, 0, 0, 630, 0, 7675, 0, 55, 0, 4569, 0, 0, 0, 148, 5399, 7539, 100, 0, 0, 0, 0, 0, 55, 0, 136, 0, 1260, 0, 227, 0, 0, 0, 0, 0, 0, 0, 9, 454, 0, 336, 362] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=19 tokens_per_expert_current_ep_rank=[10, 0, 0, 0, 0, 69, 0, 0, 0, 0, 175, 0, 0, 64, 323, 0, 3847, 0, 78, 7740, 7727, 0, 0, 0, 0, 55, 594, 0, 7708, 0, 64, 0, 4112, 0, 0, 0, 176, 6061, 7510, 74, 0, 0, 0, 0, 0, 26, 0, 125, 0, 1377, 0, 137, 0, 0, 0, 0, 0, 0, 0, 7, 430, 0, 318, 345] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=19 tokens_per_expert_current_ep_rank=[8, 0, 0, 0, 0, 73, 0, 0, 0, 0, 190, 0, 0, 70, 629, 0, 3946, 0, 115, 7687, 7622, 0, 0, 0, 0, 1, 581, 0, 7683, 0, 70, 0, 4429, 0, 0, 0, 240, 5056, 7532, 75, 0, 0, 0, 0, 0, 64, 0, 123, 0, 1657, 0, 163, 0, 0, 0, 0, 0, 0, 0, 2, 433, 0, 339, 364] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=19 tokens_per_expert_current_ep_rank=[15, 30, 3762, 1, 4175, 131, 364, 865, 104, 9, 875, 402, 21, 333, 154, 300, 1068, 897, 208, 1797, 1333, 28, 1305, 882, 0, 737, 711, 0, 1878, 2132, 889, 2, 878, 194, 691, 480, 196, 198, 1588, 69, 14, 698, 6, 2964, 47, 162, 0, 36, 241, 42, 1620, 1589, 0, 3107, 0, 41, 1113, 91, 1166, 262, 409, 3674, 1753, 415] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=19 tokens_per_expert_current_ep_rank=[0, 0, 21, 0, 27, 72, 12, 5, 0, 0, 172, 1, 0, 61, 520, 0, 3787, 13, 145, 7645, 7578, 0, 14, 4, 0, 5, 625, 0, 7668, 10, 49, 0, 4244, 7, 5, 1, 254, 5742, 7494, 72, 0, 1, 0, 19, 3, 47, 0, 119, 0, 1315, 2, 143, 0, 24, 0, 0, 16, 0, 14, 43, 458, 27, 301, 367] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=19 tokens_per_expert_current_ep_rank=[80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 196, 0, 0, 68, 969, 0, 3397, 0, 160, 7626, 7869, 0, 0, 0, 0, 0, 1397, 0, 7650, 0, 68, 0, 6722, 0, 5, 0, 0, 2986, 7464, 80, 0, 0, 0, 0, 0, 68, 0, 59, 0, 745, 0, 177, 0, 0, 0, 0, 0, 0, 0, 160, 468, 0, 358, 380] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=19 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 0, 0, 0, 92, 0, 0, 3506, 1906, 0, 0, 0, 0, 0, 0, 0, 0, 221, 0, 0, 55, 728, 0, 4033, 1544, 0, 0, 0, 0, 0, 0, 3754, 0, 198, 2533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 630, 0, 5269, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1511, 0, 0, 0, 148, 5399, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 136, 6073, 0, 0, 0, 0, 0, 0, 0, 0, 1260, 0, 227, 0, 0, 0, 0, 3086, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 454, 0, 336, 362, 2406, 1778, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5512, 6581, 6485, 5954, 7158, 6264, 5853, 5345] avg=6144.0000 max=7158 max_rank=4 min=5345 min_rank=7 imbalance_ratio=1.165039 max_load_reduction=41994 max_load_reduction_pct=0.854370 moved_to_echo=24620 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=19 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 0, 0, 0, 69, 0, 0, 3493, 1961, 0, 0, 0, 0, 0, 0, 0, 0, 175, 0, 0, 64, 323, 0, 4017, 1567, 0, 0, 0, 0, 0, 0, 3847, 0, 78, 2605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 594, 0, 5292, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1360, 0, 0, 0, 176, 6061, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 125, 6160, 0, 0, 0, 0, 0, 0, 0, 0, 1377, 0, 137, 0, 0, 0, 0, 3174, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 430, 0, 318, 345, 2416, 1600, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5533, 6146, 6530, 6005, 7671, 6311, 5840, 5116] avg=6144.0000 max=7671 max_rank=4 min=5116 min_rank=7 imbalance_ratio=1.248535 max_load_reduction=41481 max_load_reduction_pct=0.843933 moved_to_echo=24938 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=19 rerouted_tokens_per_expert_current_ep_rank=[29, 0, 13, 0, 19, 47, 1, 1, 3474, 1922, 0, 0, 0, 0, 0, 0, 0, 0, 158, 6, 0, 63, 896, 0, 3994, 1581, 0, 0, 0, 0, 0, 0, 3285, 15, 153, 2553, 0, 0, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 925, 0, 5284, 1, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1871, 8, 11, 1, 2, 4610, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 16, 3, 61, 0, 63, 6212, 0, 0, 0, 0, 0, 0, 0, 0, 983, 0, 210, 0, 17, 0, 0, 3111, 1585, 0, 0, 0, 0, 0, 0, 2, 0, 21, 25, 448, 18, 344, 332, 2412, 2202, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5506, 6698, 6021, 6278, 6579, 6360, 5906, 5804] avg=6144.0000 max=6698 max_rank=1 min=5506 min_rank=0 imbalance_ratio=1.090169 max_load_reduction=42454 max_load_reduction_pct=0.863729 moved_to_echo=24649 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=19 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 0, 0, 0, 73, 0, 0, 3503, 1947, 0, 0, 0, 0, 0, 0, 0, 0, 190, 0, 0, 70, 629, 0, 4029, 1545, 0, 0, 0, 0, 0, 0, 3946, 0, 115, 2587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 581, 0, 5275, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1464, 0, 0, 0, 240, 5056, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 123, 6077, 0, 0, 0, 0, 0, 0, 0, 0, 1657, 0, 163, 0, 0, 0, 0, 3153, 1241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 433, 0, 339, 364, 2408, 1724, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5531, 6463, 6648, 5927, 6835, 6264, 6214, 5270] avg=6144.0000 max=6835 max_rank=4 min=5270 min_rank=7 imbalance_ratio=1.112467 max_load_reduction=42317 max_load_reduction_pct=0.860942 moved_to_echo=24583 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=19 rerouted_tokens_per_expert_current_ep_rank=[52, 0, 0, 0, 0, 22, 0, 0, 3501, 1910, 0, 0, 0, 0, 0, 0, 0, 0, 206, 0, 0, 53, 961, 0, 4024, 1582, 0, 0, 0, 0, 0, 0, 3106, 0, 179, 2537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1225, 0, 5282, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2141, 0, 6, 0, 0, 3781, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 72, 6215, 0, 0, 0, 0, 0, 0, 0, 0, 852, 0, 212, 0, 0, 0, 0, 3092, 1814, 0, 0, 0, 0, 0, 0, 0, 0, 5, 68, 468, 0, 321, 343, 2411, 2520, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5485, 6826, 5822, 6560, 6018, 6335, 5970, 6136] avg=6144.0000 max=6826 max_rank=1 min=5485 min_rank=0 imbalance_ratio=1.111003 max_load_reduction=42326 max_load_reduction_pct=0.861125 moved_to_echo=24459 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=19 rerouted_tokens_per_expert_current_ep_rank=[15, 30, 3762, 1, 4175, 131, 364, 865, 739, 455, 0, 0, 0, 0, 0, 0, 104, 9, 875, 402, 21, 333, 154, 300, 849, 271, 0, 0, 0, 0, 0, 0, 1068, 897, 208, 605, 0, 28, 1305, 882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 737, 711, 0, 1290, 2132, 889, 2, 0, 0, 0, 0, 0, 0, 0, 0, 291, 194, 691, 480, 196, 198, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 14, 698, 6, 2964, 47, 162, 0, 36, 1062, 0, 0, 0, 0, 0, 0, 0, 241, 42, 1620, 1589, 0, 3107, 0, 41, 737, 246, 0, 0, 0, 0, 0, 0, 1113, 91, 1166, 262, 409, 3674, 1753, 415, 588, 341, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10537, 3318, 4993, 5761, 2119, 4989, 7623, 9812] avg=6144.0000 max=10537 max_rank=0 min=2119 min_rank=4 imbalance_ratio=1.715007 max_load_reduction=38615 max_load_reduction_pct=0.785624 moved_to_echo=24543 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=19 rerouted_tokens_per_expert_current_ep_rank=[80, 0, 0, 0, 0, 0, 0, 0, 3473, 1932, 0, 0, 0, 0, 0, 0, 0, 0, 196, 0, 0, 68, 969, 0, 3991, 1596, 0, 0, 0, 0, 0, 0, 3397, 0, 160, 2567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1397, 0, 5252, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2223, 0, 5, 0, 0, 2986, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 59, 6273, 0, 0, 0, 0, 0, 0, 0, 0, 745, 0, 177, 0, 0, 0, 0, 3127, 1883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 468, 0, 358, 380, 2398, 2616, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5485, 6820, 6124, 6717, 5294, 6400, 5932, 6380] avg=6144.0000 max=6820 max_rank=1 min=5294 min_rank=4 imbalance_ratio=1.110026 max_load_reduction=42332 max_load_reduction_pct=0.861247 moved_to_echo=24006 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=19 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 21, 0, 27, 72, 12, 5, 3486, 1940, 0, 0, 0, 0, 0, 0, 0, 0, 172, 1, 0, 61, 520, 0, 4008, 1538, 0, 0, 0, 0, 0, 0, 3787, 13, 145, 2566, 0, 0, 14, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 625, 0, 5261, 10, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1397, 7, 5, 1, 254, 5742, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 19, 3, 47, 0, 119, 6040, 0, 0, 0, 0, 0, 0, 0, 0, 1315, 2, 143, 0, 24, 0, 0, 3139, 1191, 0, 0, 0, 0, 0, 0, 16, 0, 14, 43, 458, 27, 301, 367, 2407, 1656, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5563, 6300, 6529, 5950, 7478, 6229, 5814, 5289] avg=6144.0000 max=7478 max_rank=4 min=5289 min_rank=7 imbalance_ratio=1.217122 max_load_reduction=41674 max_load_reduction_pct=0.847860 moved_to_echo=24810 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[256, 128, 3840, 128, 4224, 512, 384, 896, 25216, 14080, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[26240, 1024, 1280, 18560, 0, 128, 1408, 896, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1152, 128, 1280, 640, 3584, 3840, 4096, 2944, 17536, 14464, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[12288, 256, 768, 512, 1024, 33920, 0, 640, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[256, 8320, 1664, 2944, 0, 3200, 0, 128, 22656, 10496, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 896, 6784, 0, 38272, 2176, 1408, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 2304, 512, 128, 768, 5248, 384, 29056, 11264, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 768, 128, 3072, 128, 640, 0, 768, 44160, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 2163, 0, 0, 0, 0, 0, 0, 0, 100, 0, 100, 1642, 0, 0, 0, 2796, 1738, 8092, 903, 0, 100, 7576, 0, 0, 7477, 550, 55, 0, 0, 0, 0, 99, 0, 6, 0, 0, 2290, 0, 1543, 0, 0, 0, 0, 96, 0, 0, 0, 0, 2544, 5069, 0, 794, 211, 1475, 99, 0, 0, 0, 1490, 141, 3] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 2305, 0, 0, 0, 0, 0, 0, 5, 75, 0, 75, 1587, 0, 0, 0, 3054, 1931, 8117, 906, 0, 75, 7613, 0, 0, 7518, 526, 70, 0, 0, 0, 0, 72, 0, 39, 0, 0, 1696, 0, 1144, 0, 0, 0, 0, 74, 0, 0, 0, 0, 2527, 5018, 0, 698, 113, 1296, 74, 0, 0, 0, 2364, 180, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 15, 2309, 3, 17, 0, 0, 0, 0, 0, 76, 7, 75, 1959, 10, 0, 0, 2208, 1559, 8053, 738, 4, 76, 7604, 8, 0, 7396, 429, 66, 0, 0, 0, 16, 115, 17, 0, 0, 0, 2694, 10, 1925, 0, 18, 1, 0, 71, 0, 4, 0, 3, 2079, 5465, 2, 1276, 185, 1426, 78, 10, 0, 18, 1069, 53, 5] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 2219, 0, 0, 0, 0, 0, 0, 6, 74, 0, 74, 1774, 0, 0, 0, 3070, 1717, 8115, 825, 0, 74, 7636, 0, 0, 7518, 466, 64, 0, 0, 0, 0, 73, 0, 32, 0, 0, 1941, 0, 1267, 0, 0, 0, 0, 71, 0, 0, 0, 0, 2843, 4784, 0, 662, 85, 1401, 73, 0, 0, 0, 2136, 152, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=20 tokens_per_expert_current_ep_rank=[152, 8, 2599, 2526, 179, 774, 21, 0, 39, 89, 366, 30, 1816, 0, 607, 1079, 90, 1137, 74, 279, 512, 379, 43, 92, 1936, 673, 23, 557, 268, 88, 88, 71, 0, 2875, 795, 4325, 58, 1, 1, 942, 2391, 158, 911, 3556, 45, 0, 110, 0, 465, 753, 526, 1022, 3548, 695, 2215, 373, 1050, 403, 409, 36, 4184, 137, 298, 275] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 2241, 0, 0, 0, 0, 0, 0, 0, 80, 0, 80, 1814, 0, 0, 0, 1500, 1536, 8085, 709, 0, 80, 7584, 0, 0, 7434, 257, 68, 0, 0, 0, 0, 94, 0, 0, 0, 0, 2496, 0, 2551, 0, 0, 0, 0, 77, 0, 0, 0, 0, 1775, 5774, 0, 1562, 151, 2111, 80, 0, 0, 0, 912, 98, 3] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 18, 2302, 7, 20, 0, 0, 1, 0, 7, 72, 10, 72, 1692, 5, 0, 0, 2486, 1783, 8019, 953, 4, 72, 7594, 13, 0, 7455, 484, 47, 0, 0, 0, 22, 81, 30, 26, 0, 0, 2292, 13, 1286, 3, 25, 1, 0, 73, 0, 0, 4, 11, 2641, 5237, 2, 620, 88, 1322, 74, 0, 0, 27, 2018, 137, 3] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 2240, 0, 0, 0, 0, 0, 0, 0, 89, 0, 88, 1875, 0, 0, 0, 1764, 1746, 8065, 732, 0, 87, 7617, 0, 0, 7496, 392, 53, 0, 0, 0, 0, 109, 0, 0, 0, 0, 2787, 0, 2240, 0, 0, 0, 0, 75, 0, 0, 0, 0, 1850, 5462, 0, 1327, 190, 1816, 89, 0, 0, 0, 878, 76, 9] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2163, 0, 0, 0, 0, 3833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 100, 1642, 0, 4340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2796, 1738, 1680, 903, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6281, 550, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 0, 6, 0, 0, 2290, 1749, 1355, 0, 0, 0, 0, 0, 0, 0, 1543, 0, 0, 0, 0, 96, 0, 3236, 830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2544, 2370, 0, 794, 211, 0, 0, 0, 0, 0, 0, 0, 0, 1475, 99, 0, 0, 0, 1490, 141, 3, 1344, 1196, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5996, 6182, 7217, 6886, 5499, 5705, 5919, 5748] avg=6144.0000 max=7217 max_rank=2 min=5499 min_rank=4 imbalance_ratio=1.174642 max_load_reduction=41935 max_load_reduction_pct=0.853170 moved_to_echo=22871 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2219, 0, 0, 0, 0, 3844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 74, 0, 74, 1774, 0, 4374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3070, 1717, 1684, 825, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6315, 466, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 32, 0, 0, 1941, 1754, 1279, 0, 0, 0, 0, 0, 0, 0, 1267, 0, 0, 0, 0, 71, 0, 3262, 833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2843, 2236, 0, 662, 85, 0, 0, 0, 0, 0, 0, 0, 0, 1401, 73, 0, 0, 0, 2136, 152, 0, 1269, 1203, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6063, 6302, 7370, 6845, 5079, 5433, 5826, 6234] avg=6144.0000 max=7370 max_rank=2 min=5079 min_rank=4 imbalance_ratio=1.199544 max_load_reduction=41782 max_load_reduction_pct=0.850057 moved_to_echo=22572 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2305, 0, 0, 0, 0, 3845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 75, 0, 75, 1587, 0, 4361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3054, 1931, 1685, 906, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6315, 526, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 39, 0, 0, 1696, 1754, 1342, 0, 0, 0, 0, 0, 0, 0, 1144, 0, 0, 0, 0, 74, 0, 3252, 833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2527, 2345, 0, 698, 113, 0, 0, 0, 0, 0, 0, 0, 0, 1296, 74, 0, 0, 0, 2364, 180, 0, 1331, 1203, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6150, 6103, 7651, 6911, 4903, 5303, 5683, 6448] avg=6144.0000 max=7651 max_rank=2 min=4903 min_rank=4 imbalance_ratio=1.245280 max_load_reduction=41501 max_load_reduction_pct=0.844340 moved_to_echo=22337 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 15, 2309, 3, 17, 0, 0, 3815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 7, 75, 1959, 10, 4357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2208, 1559, 1672, 738, 4, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 6213, 429, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 115, 17, 0, 0, 0, 2694, 1740, 1461, 0, 0, 0, 0, 0, 0, 10, 1925, 0, 18, 1, 0, 71, 0, 3247, 826, 0, 0, 0, 0, 0, 0, 4, 0, 3, 2079, 2555, 2, 1276, 185, 0, 0, 0, 0, 0, 0, 0, 0, 1426, 78, 10, 0, 18, 1069, 53, 5, 1449, 1183, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6159, 6484, 6257, 6716, 6043, 6098, 6104, 5291] avg=6144.0000 max=6716 max_rank=3 min=5291 min_rank=7 imbalance_ratio=1.093099 max_load_reduction=42436 max_load_reduction_pct=0.863363 moved_to_echo=23536 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=20 rerouted_tokens_per_expert_current_ep_rank=[152, 8, 2599, 2526, 179, 774, 21, 0, 242, 0, 0, 0, 0, 0, 0, 0, 39, 89, 366, 30, 1816, 0, 607, 1079, 1110, 0, 0, 0, 0, 0, 0, 0, 90, 1137, 74, 279, 108, 379, 43, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 673, 23, 468, 268, 88, 88, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2875, 795, 4325, 58, 1, 1, 942, 110, 949, 0, 0, 0, 0, 0, 0, 2391, 158, 911, 3556, 45, 0, 110, 0, 826, 52, 0, 0, 0, 0, 0, 0, 465, 753, 526, 1022, 1658, 695, 2215, 373, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 403, 409, 36, 4184, 137, 298, 275, 941, 89, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6501, 5136, 2202, 1679, 10056, 8049, 7707, 7822] avg=6144.0000 max=10056 max_rank=4 min=1679 min_rank=3 imbalance_ratio=1.636719 max_load_reduction=39096 max_load_reduction_pct=0.795410 moved_to_echo=33634 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2240, 0, 0, 0, 0, 3821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 88, 1875, 0, 4364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1764, 1746, 1673, 732, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6297, 392, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 0, 0, 0, 0, 2787, 1743, 1461, 0, 0, 0, 0, 0, 0, 0, 2240, 0, 0, 0, 0, 75, 0, 3253, 828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1850, 2553, 0, 1327, 190, 0, 0, 0, 0, 0, 0, 0, 0, 1816, 89, 0, 0, 0, 878, 76, 9, 1448, 1199, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6061, 6416, 6002, 6742, 6100, 6396, 5920, 5515] avg=6144.0000 max=6742 max_rank=3 min=5515 min_rank=7 imbalance_ratio=1.097331 max_load_reduction=42410 max_load_reduction_pct=0.862834 moved_to_echo=23931 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2241, 0, 0, 0, 0, 3830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 80, 1814, 0, 4345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1500, 1536, 1678, 709, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6245, 257, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 0, 0, 0, 2496, 1747, 1544, 0, 0, 0, 0, 0, 0, 0, 2551, 0, 0, 0, 0, 77, 0, 3239, 830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1775, 2699, 0, 1562, 151, 0, 0, 0, 0, 0, 0, 0, 0, 2111, 80, 0, 0, 0, 912, 98, 3, 1531, 1189, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6071, 6319, 5503, 6570, 5881, 6697, 6187, 5924] avg=6144.0000 max=6697 max_rank=5 min=5503 min_rank=2 imbalance_ratio=1.090007 max_load_reduction=42455 max_load_reduction_pct=0.863749 moved_to_echo=24689 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 18, 2302, 7, 20, 0, 0, 3804, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7, 72, 10, 72, 1692, 5, 4351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2486, 1783, 1652, 953, 4, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 6259, 484, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 81, 30, 26, 0, 0, 2292, 1736, 1404, 0, 0, 0, 0, 0, 0, 13, 1286, 3, 25, 1, 0, 73, 0, 3243, 827, 0, 0, 0, 0, 0, 0, 0, 4, 11, 2641, 2440, 2, 620, 88, 0, 0, 0, 0, 0, 0, 0, 0, 1322, 74, 0, 0, 27, 2018, 137, 3, 1393, 1196, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6151, 6210, 6950, 6803, 5591, 5471, 5806, 6170] avg=6144.0000 max=6950 max_rank=2 min=5471 min_rank=5 imbalance_ratio=1.131185 max_load_reduction=42202 max_load_reduction_pct=0.858602 moved_to_echo=23038 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2432, 12160, 1024, 3712, 128, 0, 768, 0, 23680, 5888, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 1152, 17024, 12416, 11904, 6272, 128, 768, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[512, 768, 640, 17408, 18944, 768, 9216, 1408, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[0, 2944, 1536, 4480, 256, 128, 128, 17152, 12416, 10880, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[256, 128, 2688, 18432, 256, 896, 128, 0, 27136, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[11904, 1024, 512, 128, 4352, 11008, 1152, 384, 10752, 8576, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 384, 640, 1920, 640, 13056, 1152, 31616, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 768, 128, 44416, 3456, 512, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=21 tokens_per_expert_current_ep_rank=[0, 6014, 529, 1333, 7555, 4937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 0, 443, 0, 0, 0, 0, 260, 2647, 73, 0, 0, 0, 0, 0, 410, 0, 0, 6514, 4443, 149, 0, 0, 722, 0, 0, 0, 0, 0, 0, 1212, 1, 0, 0, 0, 85, 7740, 135, 3732, 0, 0, 73, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=21 tokens_per_expert_current_ep_rank=[1, 381, 212, 466, 773, 1868, 195, 0, 137, 211, 350, 141, 330, 1512, 1178, 1194, 4005, 873, 1, 27, 3677, 827, 25, 3, 59, 0, 443, 1396, 83, 20, 1478, 162, 251, 135, 325, 64, 26, 1277, 518, 18, 1068, 79, 279, 471, 2446, 345, 153, 2077, 1923, 557, 896, 0, 64, 4688, 245, 924, 141, 1557, 258, 3948, 43, 236, 258, 1854] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=21 tokens_per_expert_current_ep_rank=[0, 4867, 655, 1288, 7752, 2881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 0, 0, 596, 0, 0, 0, 0, 307, 4513, 88, 0, 0, 0, 0, 0, 599, 0, 0, 5627, 6135, 127, 0, 0, 658, 0, 0, 0, 0, 0, 0, 1068, 0, 0, 0, 0, 74, 7171, 89, 4369, 0, 0, 88, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=21 tokens_per_expert_current_ep_rank=[0, 5642, 528, 1206, 7435, 5207, 4, 0, 1, 0, 0, 2, 3, 4, 11, 10, 26, 8, 156, 0, 12, 495, 0, 0, 0, 0, 246, 3036, 72, 0, 17, 0, 0, 1, 597, 0, 0, 6485, 4073, 111, 2, 0, 674, 0, 16, 2, 2, 16, 14, 1290, 7, 0, 0, 30, 65, 7723, 148, 3662, 0, 27, 72, 1, 0, 13] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=21 tokens_per_expert_current_ep_rank=[0, 5930, 568, 1068, 7459, 5570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 0, 0, 503, 0, 0, 0, 0, 333, 3616, 74, 0, 0, 0, 0, 0, 499, 0, 0, 6419, 3720, 154, 0, 0, 683, 0, 0, 0, 0, 0, 0, 1317, 0, 0, 0, 0, 84, 7646, 140, 3148, 0, 0, 74, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=21 tokens_per_expert_current_ep_rank=[0, 5435, 685, 1325, 7507, 4256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 476, 0, 0, 0, 0, 330, 3262, 99, 0, 0, 0, 0, 0, 571, 0, 0, 6189, 4798, 122, 0, 0, 738, 0, 0, 0, 0, 0, 0, 1221, 0, 0, 0, 0, 67, 7500, 145, 4203, 0, 0, 99, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=21 tokens_per_expert_current_ep_rank=[0, 4835, 592, 1211, 7624, 2809, 5, 0, 1, 0, 6, 1, 3, 0, 12, 9, 19, 1, 199, 0, 9, 622, 0, 0, 0, 0, 328, 4171, 76, 0, 16, 0, 0, 0, 788, 0, 0, 5898, 5787, 149, 1, 0, 690, 0, 14, 0, 2, 24, 0, 1033, 22, 0, 0, 21, 88, 7313, 87, 4584, 5, 15, 76, 1, 1, 4] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=21 tokens_per_expert_current_ep_rank=[0, 4671, 580, 1219, 7807, 2591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 207, 0, 0, 596, 0, 0, 0, 0, 281, 4905, 80, 0, 0, 0, 0, 0, 783, 0, 0, 5509, 6402, 164, 0, 0, 651, 0, 0, 0, 0, 0, 0, 990, 0, 0, 0, 0, 97, 7082, 80, 4377, 0, 0, 80, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 854, 529, 1333, 0, 4937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 0, 443, 0, 0, 4061, 1367, 0, 0, 0, 0, 0, 0, 0, 0, 260, 2647, 73, 0, 0, 0, 1425, 1099, 0, 0, 0, 0, 0, 0, 0, 0, 410, 0, 0, 1074, 4443, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 722, 0, 0, 0, 0, 0, 4073, 1384, 0, 0, 0, 0, 0, 0, 0, 1212, 1, 0, 0, 0, 85, 4664, 0, 0, 0, 0, 0, 0, 0, 0, 135, 3732, 0, 0, 73, 0, 0, 0, 1651, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7653, 6171, 6016, 5504, 6076, 6179, 5962, 5591] avg=6144.0000 max=7653 max_rank=0 min=5504 min_rank=3 imbalance_ratio=1.245605 max_load_reduction=41499 max_load_reduction_pct=0.844299 moved_to_echo=23808 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 772, 685, 1325, 0, 4256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 476, 0, 0, 3670, 1299, 0, 0, 0, 0, 0, 0, 0, 0, 330, 3262, 99, 0, 0, 0, 1380, 993, 0, 0, 0, 0, 0, 0, 0, 0, 571, 0, 0, 1020, 4798, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 738, 0, 0, 0, 0, 0, 3870, 1376, 0, 0, 0, 0, 0, 0, 0, 1221, 0, 0, 0, 0, 67, 4520, 0, 0, 0, 0, 0, 0, 0, 0, 145, 4203, 0, 0, 99, 0, 0, 0, 1600, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7038, 6131, 5569, 6064, 6511, 5984, 5808, 6047] avg=6144.0000 max=7038 max_rank=0 min=5569 min_rank=2 imbalance_ratio=1.145508 max_load_reduction=42114 max_load_reduction_pct=0.856812 moved_to_echo=24350 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 843, 568, 1068, 0, 5570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 0, 0, 503, 0, 0, 4004, 1347, 0, 0, 0, 0, 0, 0, 0, 0, 333, 3616, 74, 0, 0, 0, 1407, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 499, 0, 0, 1059, 3720, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 683, 0, 0, 0, 0, 0, 4013, 1367, 0, 0, 0, 0, 0, 0, 0, 1317, 0, 0, 0, 0, 84, 4608, 0, 0, 0, 0, 0, 0, 0, 0, 140, 3148, 0, 0, 74, 0, 0, 0, 1631, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8049, 6092, 6001, 6513, 5432, 6063, 6009, 4993] avg=6144.0000 max=8049 max_rank=0 min=4993 min_rank=7 imbalance_ratio=1.310059 max_load_reduction=41103 max_load_reduction_pct=0.836243 moved_to_echo=22497 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=21 rerouted_tokens_per_expert_current_ep_rank=[1, 55, 212, 466, 0, 1868, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 211, 350, 141, 330, 1512, 1178, 1194, 632, 0, 0, 0, 0, 0, 0, 0, 4005, 873, 1, 27, 3677, 827, 25, 3, 257, 268, 0, 0, 0, 0, 0, 0, 59, 0, 443, 1396, 83, 20, 1478, 162, 170, 69, 0, 0, 0, 0, 0, 0, 251, 135, 325, 64, 26, 211, 518, 18, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 79, 279, 471, 2446, 345, 153, 2077, 798, 141, 0, 0, 0, 0, 0, 0, 1923, 557, 896, 0, 64, 4688, 245, 557, 0, 0, 0, 0, 0, 0, 0, 0, 141, 1557, 258, 3948, 43, 236, 258, 1854, 197, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2797, 5685, 9963, 3880, 1548, 7857, 8930, 8492] avg=6144.0000 max=9963 max_rank=2 min=1548 min_rank=4 imbalance_ratio=1.621582 max_load_reduction=39189 max_load_reduction_pct=0.797302 moved_to_echo=26827 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 688, 592, 1211, 0, 2809, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 1, 3, 0, 12, 9, 6228, 0, 0, 0, 0, 0, 0, 0, 19, 1, 199, 0, 9, 622, 0, 0, 3264, 1238, 0, 0, 0, 0, 0, 0, 0, 0, 328, 4171, 76, 0, 16, 0, 1346, 883, 0, 0, 0, 0, 0, 0, 0, 0, 788, 0, 0, 972, 5787, 149, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 690, 0, 14, 0, 2, 24, 3688, 1396, 0, 0, 0, 0, 0, 0, 0, 1033, 22, 0, 0, 21, 88, 4407, 0, 0, 0, 0, 0, 0, 0, 0, 87, 4584, 5, 15, 76, 1, 1, 4, 1560, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5305, 6260, 5352, 6820, 7696, 5815, 5571, 6333] avg=6144.0000 max=7696 max_rank=4 min=5305 min_rank=0 imbalance_ratio=1.252604 max_load_reduction=41456 max_load_reduction_pct=0.843424 moved_to_echo=25415 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 692, 655, 1288, 0, 2881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 0, 0, 596, 0, 0, 3286, 1181, 0, 0, 0, 0, 0, 0, 0, 0, 307, 4513, 88, 0, 0, 0, 1320, 889, 0, 0, 0, 0, 0, 0, 0, 0, 599, 0, 0, 928, 6135, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 658, 0, 0, 0, 0, 0, 3518, 1420, 0, 0, 0, 0, 0, 0, 0, 1068, 0, 0, 0, 0, 74, 4322, 0, 0, 0, 0, 0, 0, 0, 0, 89, 4369, 0, 0, 88, 0, 0, 0, 1529, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5516, 6332, 5263, 7117, 7789, 5596, 5464, 6075] avg=6144.0000 max=7789 max_rank=4 min=5263 min_rank=2 imbalance_ratio=1.267741 max_load_reduction=41363 max_load_reduction_pct=0.841532 moved_to_echo=24924 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 664, 580, 1219, 0, 2591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 207, 0, 0, 596, 0, 0, 3154, 1156, 0, 0, 0, 0, 0, 0, 0, 0, 281, 4905, 80, 0, 0, 0, 1303, 853, 0, 0, 0, 0, 0, 0, 0, 0, 783, 0, 0, 909, 6402, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 651, 0, 0, 0, 0, 0, 3444, 1431, 0, 0, 0, 0, 0, 0, 0, 990, 0, 0, 0, 0, 97, 4269, 0, 0, 0, 0, 0, 0, 0, 0, 80, 4377, 0, 0, 80, 0, 0, 0, 1510, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5054, 6376, 5113, 7422, 8258, 5526, 5356, 6047] avg=6144.0000 max=8258 max_rank=4 min=5054 min_rank=0 imbalance_ratio=1.344076 max_load_reduction=40894 max_load_reduction_pct=0.831991 moved_to_echo=25187 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 795, 528, 1206, 0, 5207, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 3, 4, 11, 10, 6074, 0, 0, 0, 0, 0, 0, 0, 26, 8, 156, 0, 12, 495, 0, 0, 3812, 1366, 0, 0, 0, 0, 0, 0, 0, 0, 246, 3036, 72, 0, 17, 0, 1426, 1035, 0, 0, 0, 0, 0, 0, 0, 1, 597, 0, 0, 1060, 4073, 111, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 674, 0, 16, 2, 2, 16, 4059, 1361, 0, 0, 0, 0, 0, 0, 14, 1290, 7, 0, 0, 30, 65, 4646, 0, 0, 0, 0, 0, 0, 0, 0, 148, 3662, 0, 27, 72, 1, 0, 13, 1651, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7740, 6105, 5875, 5832, 5842, 6132, 6052, 5574] avg=6144.0000 max=7740 max_rank=0 min=5574 min_rank=7 imbalance_ratio=1.259766 max_load_reduction=41412 max_load_reduction_pct=0.842529 moved_to_echo=23600 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[4096, 896, 1280, 128, 3712, 4608, 128, 128, 25600, 9344, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 5376, 4352, 9216, 0, 30208, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 0, 2560, 27648, 768, 128, 1536, 256, 9856, 6912, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1024, 29696, 384, 4096, 640, 256, 384, 1920, 11392, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[256, 256, 4608, 128, 128, 7296, 35968, 1024, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2048, 8704, 1024, 0, 128, 4864, 896, 32000, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1152, 128, 5120, 512, 2560, 384, 256, 2176, 27520, 9984, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[256, 256, 384, 256, 384, 1536, 1280, 1280, 44160, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=22 tokens_per_expert_current_ep_rank=[2080, 3727, 100, 119, 82, 3, 1285, 140, 140, 4858, 2223, 265, 1160, 1, 117, 393, 23, 1277, 75, 297, 209, 52, 234, 251, 124, 101, 0, 327, 199, 530, 4, 167, 1961, 5691, 233, 0, 86, 0, 10, 351, 0, 4826, 24, 40, 2408, 3975, 59, 59, 95, 137, 29, 597, 441, 1033, 0, 2110, 226, 852, 283, 2197, 425, 346, 65, 30] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=22 tokens_per_expert_current_ep_rank=[6859, 0, 0, 0, 0, 0, 0, 1909, 544, 0, 182, 0, 0, 0, 6, 2464, 0, 0, 0, 5799, 244, 611, 0, 853, 0, 0, 0, 0, 0, 0, 0, 0, 1667, 0, 0, 0, 0, 0, 0, 6666, 0, 0, 0, 7502, 0, 0, 129, 0, 0, 0, 0, 2562, 0, 7916, 0, 0, 0, 0, 0, 2973, 166, 0, 0, 100] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=22 tokens_per_expert_current_ep_rank=[6442, 0, 0, 0, 0, 0, 0, 2398, 522, 0, 116, 0, 0, 0, 0, 2438, 0, 0, 0, 6353, 256, 633, 0, 818, 0, 0, 0, 0, 0, 0, 0, 0, 1827, 0, 0, 0, 0, 0, 0, 6678, 0, 0, 0, 7552, 0, 0, 144, 0, 0, 0, 0, 1368, 0, 8034, 0, 0, 0, 0, 0, 3340, 158, 0, 0, 75] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=22 tokens_per_expert_current_ep_rank=[7533, 0, 0, 0, 0, 0, 0, 1831, 531, 0, 311, 0, 0, 0, 38, 1686, 0, 0, 0, 4759, 205, 448, 0, 975, 0, 0, 0, 0, 0, 0, 0, 0, 942, 0, 0, 0, 0, 0, 0, 6893, 0, 0, 0, 7354, 0, 0, 184, 0, 0, 0, 0, 4017, 0, 7753, 0, 0, 0, 0, 0, 3439, 163, 0, 0, 90] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=22 tokens_per_expert_current_ep_rank=[6390, 0, 0, 0, 0, 0, 0, 2230, 547, 0, 148, 0, 0, 0, 5, 2316, 0, 0, 0, 6282, 241, 688, 0, 766, 0, 0, 0, 0, 0, 0, 0, 0, 2333, 0, 0, 0, 0, 0, 0, 6633, 0, 0, 0, 7642, 0, 0, 144, 0, 0, 0, 0, 1712, 0, 8029, 0, 0, 0, 0, 0, 2814, 158, 0, 0, 74] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=22 tokens_per_expert_current_ep_rank=[7415, 18, 0, 1, 0, 0, 1, 1866, 538, 21, 349, 1, 0, 0, 41, 2103, 1, 7, 1, 4611, 155, 449, 1, 971, 0, 0, 0, 0, 0, 2, 0, 7, 1449, 13, 1, 0, 0, 0, 0, 6733, 0, 19, 0, 7288, 7, 18, 195, 0, 0, 0, 0, 3351, 9, 7746, 0, 15, 0, 6, 0, 3501, 164, 2, 0, 76] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=22 tokens_per_expert_current_ep_rank=[6559, 19, 0, 0, 1, 0, 3, 2287, 519, 31, 216, 2, 4, 0, 2, 2444, 0, 12, 7, 6282, 242, 566, 1, 827, 0, 0, 0, 0, 0, 5, 0, 3, 1809, 26, 2, 0, 1, 0, 0, 6517, 0, 30, 0, 7433, 11, 27, 118, 0, 0, 0, 0, 1908, 8, 7920, 0, 19, 0, 2, 1, 3073, 136, 4, 3, 72] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=22 tokens_per_expert_current_ep_rank=[7658, 0, 0, 0, 0, 0, 0, 1836, 584, 0, 285, 0, 0, 0, 4, 1746, 0, 0, 0, 4339, 164, 414, 0, 1136, 0, 0, 0, 0, 0, 0, 0, 0, 862, 0, 0, 0, 0, 0, 0, 6957, 0, 0, 0, 7310, 0, 0, 184, 0, 0, 0, 0, 3816, 0, 7784, 0, 0, 0, 0, 0, 3817, 176, 0, 0, 80] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=22 rerouted_tokens_per_expert_current_ep_rank=[3946, 0, 0, 0, 0, 0, 0, 1909, 0, 0, 0, 0, 0, 0, 0, 0, 544, 0, 182, 0, 0, 0, 6, 2464, 2235, 506, 47, 0, 0, 0, 0, 0, 0, 0, 0, 5293, 244, 611, 0, 853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4119, 2481, 0, 0, 0, 0, 0, 0, 1667, 0, 0, 0, 0, 0, 0, 4251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5267, 0, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2562, 0, 3797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2973, 166, 0, 0, 100, 2368, 432, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5855, 5984, 7001, 6600, 5918, 5396, 6359, 6039] avg=6144.0000 max=7001 max_rank=2 min=5396 min_rank=5 imbalance_ratio=1.139486 max_load_reduction=42151 max_load_reduction_pct=0.857564 moved_to_echo=23712 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=22 rerouted_tokens_per_expert_current_ep_rank=[3676, 0, 0, 0, 0, 0, 0, 2230, 0, 0, 0, 0, 0, 0, 0, 0, 547, 0, 148, 0, 0, 0, 5, 2316, 2277, 548, 47, 0, 0, 0, 0, 0, 0, 0, 0, 5734, 241, 688, 0, 766, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4178, 2311, 0, 0, 0, 0, 0, 0, 2333, 0, 0, 0, 0, 0, 0, 4230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5365, 0, 0, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1712, 0, 3851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2814, 158, 0, 0, 74, 2356, 403, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5906, 5888, 7429, 6489, 6563, 5509, 5563, 5805] avg=6144.0000 max=7429 max_rank=2 min=5509 min_rank=5 imbalance_ratio=1.209147 max_load_reduction=41723 max_load_reduction_pct=0.848857 moved_to_echo=23440 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=22 rerouted_tokens_per_expert_current_ep_rank=[3706, 0, 0, 0, 0, 0, 0, 2398, 0, 0, 0, 0, 0, 0, 0, 0, 522, 0, 116, 0, 0, 0, 0, 2438, 2250, 555, 47, 0, 0, 0, 0, 0, 0, 0, 0, 5798, 256, 633, 0, 818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4181, 2330, 0, 0, 0, 0, 0, 0, 1827, 0, 0, 0, 0, 0, 0, 4259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5302, 0, 0, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1368, 0, 3853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3340, 158, 0, 0, 75, 2372, 406, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6104, 5928, 7505, 6511, 6086, 5446, 5221, 6351] avg=6144.0000 max=7505 max_rank=2 min=5221 min_rank=6 imbalance_ratio=1.221517 max_load_reduction=41647 max_load_reduction_pct=0.847310 moved_to_echo=23104 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=22 rerouted_tokens_per_expert_current_ep_rank=[1197, 3727, 100, 119, 82, 3, 1285, 140, 0, 0, 0, 0, 0, 0, 0, 0, 140, 4858, 2223, 265, 1160, 1, 117, 393, 11, 25, 2, 0, 0, 0, 0, 0, 23, 1277, 75, 272, 209, 52, 234, 251, 0, 0, 0, 0, 0, 0, 0, 0, 124, 101, 0, 327, 199, 530, 4, 167, 537, 752, 0, 0, 0, 0, 0, 0, 1961, 5691, 233, 0, 86, 0, 10, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4826, 24, 29, 2408, 3975, 59, 59, 0, 0, 0, 0, 0, 0, 0, 0, 95, 137, 29, 597, 441, 496, 0, 2110, 0, 0, 0, 0, 0, 0, 0, 0, 226, 852, 283, 2197, 425, 346, 65, 30, 124, 131, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6653, 9195, 2393, 2741, 8206, 11380, 3905, 4679] avg=6144.0000 max=11380 max_rank=5 min=2393 min_rank=2 imbalance_ratio=1.852214 max_load_reduction=37772 max_load_reduction_pct=0.768473 moved_to_echo=28170 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=22 rerouted_tokens_per_expert_current_ep_rank=[4266, 18, 0, 1, 0, 0, 1, 1866, 0, 0, 0, 0, 0, 0, 0, 0, 538, 21, 349, 1, 0, 0, 41, 2103, 2171, 402, 47, 0, 0, 0, 0, 0, 1, 7, 1, 4209, 155, 449, 1, 971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 7, 4031, 2682, 0, 0, 0, 0, 0, 0, 1449, 13, 1, 0, 0, 0, 0, 4294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 5117, 7, 18, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3351, 9, 3715, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 3501, 164, 2, 0, 76, 2392, 467, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6152, 5673, 5794, 6722, 5757, 5356, 7090, 6608] avg=6144.0000 max=7090 max_rank=6 min=5356 min_rank=5 imbalance_ratio=1.153971 max_load_reduction=42062 max_load_reduction_pct=0.855754 moved_to_echo=24811 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=22 rerouted_tokens_per_expert_current_ep_rank=[4333, 0, 0, 0, 0, 0, 0, 1831, 0, 0, 0, 0, 0, 0, 0, 0, 531, 0, 311, 0, 0, 0, 38, 1686, 2191, 415, 49, 0, 0, 0, 0, 0, 0, 0, 0, 4344, 205, 448, 0, 975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 2725, 0, 0, 0, 0, 0, 0, 942, 0, 0, 0, 0, 0, 0, 4395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5163, 0, 0, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4017, 0, 3718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3439, 163, 0, 0, 90, 2449, 475, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6164, 5221, 5972, 6760, 5337, 5347, 7735, 6616] avg=6144.0000 max=7735 max_rank=6 min=5221 min_rank=1 imbalance_ratio=1.258952 max_load_reduction=41417 max_load_reduction_pct=0.842631 moved_to_echo=25035 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=22 rerouted_tokens_per_expert_current_ep_rank=[4405, 0, 0, 0, 0, 0, 0, 1836, 0, 0, 0, 0, 0, 0, 0, 0, 584, 0, 285, 0, 0, 0, 4, 1746, 2178, 379, 49, 0, 0, 0, 0, 0, 0, 0, 0, 3960, 164, 414, 0, 1136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 2770, 0, 0, 0, 0, 0, 0, 862, 0, 0, 0, 0, 0, 0, 4437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5132, 0, 0, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3816, 0, 3733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3817, 176, 0, 0, 80, 2471, 483, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6241, 5225, 5674, 6821, 5299, 5316, 7549, 7027] avg=6144.0000 max=7549 max_rank=6 min=5225 min_rank=1 imbalance_ratio=1.228678 max_load_reduction=41603 max_load_reduction_pct=0.846415 moved_to_echo=25191 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=22 rerouted_tokens_per_expert_current_ep_rank=[3767, 19, 0, 0, 1, 0, 3, 2287, 0, 0, 0, 0, 0, 0, 0, 0, 519, 31, 216, 2, 4, 0, 2, 2444, 2217, 553, 50, 0, 0, 0, 0, 0, 0, 12, 7, 5729, 242, 566, 1, 827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3, 4125, 2375, 0, 0, 0, 0, 0, 0, 1809, 26, 2, 0, 1, 0, 0, 4148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 5216, 11, 27, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1908, 8, 3795, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 3073, 136, 4, 3, 72, 2319, 417, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6077, 6038, 7384, 6508, 5986, 5402, 5730, 6027] avg=6144.0000 max=7384 max_rank=2 min=5402 min_rank=5 imbalance_ratio=1.201823 max_load_reduction=41768 max_load_reduction_pct=0.849772 moved_to_echo=23145 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 0, 384, 256, 640, 128, 256, 29312, 18432, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3968, 4992, 3840, 384, 1280, 128, 256, 15616, 15616, 3456, 384, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[256, 896, 384, 25216, 1664, 384, 128, 640, 16896, 3328, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 256, 128, 19456, 512, 27008, 0, 2176, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[12928, 5760, 256, 0, 128, 0, 128, 30336, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[29312, 3840, 128, 128, 128, 128, 1408, 14592, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 4992, 128, 36608, 2432, 4096, 1280, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 1408, 128, 35456, 1792, 3968, 256, 6656, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 7122, 0, 0, 0, 0, 0, 0, 218, 0, 0, 7878, 0, 570, 0, 1, 345, 64, 0, 0, 0, 86, 0, 176, 0, 0, 0, 0, 0, 0, 7330, 7158, 0, 288, 0, 0, 0, 144, 0, 692, 750, 0, 0, 0, 6582, 8055, 0, 0, 0, 0, 0, 240, 0, 87, 0, 0, 0, 1366, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 7414, 0, 0, 0, 0, 0, 0, 79, 0, 0, 7859, 0, 517, 0, 34, 380, 68, 0, 0, 0, 245, 0, 354, 0, 0, 0, 0, 0, 0, 6857, 6043, 0, 171, 0, 0, 0, 139, 0, 1155, 769, 0, 0, 0, 6711, 8044, 0, 0, 5, 0, 0, 247, 0, 80, 0, 0, 0, 1981, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=23 tokens_per_expert_current_ep_rank=[0, 2, 7162, 6, 0, 0, 2, 0, 17, 207, 0, 0, 7833, 0, 557, 0, 22, 333, 61, 0, 0, 0, 184, 1, 403, 1, 9, 0, 0, 0, 0, 6869, 6513, 0, 191, 0, 19, 0, 139, 0, 1170, 725, 1, 5, 20, 6722, 8028, 1, 23, 6, 0, 1, 242, 0, 77, 0, 0, 22, 1561, 17, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 7019, 0, 0, 0, 0, 0, 0, 246, 0, 0, 7867, 0, 687, 0, 0, 365, 70, 0, 0, 0, 71, 0, 170, 0, 0, 0, 0, 0, 0, 7316, 7020, 0, 229, 0, 0, 0, 142, 0, 626, 687, 0, 0, 0, 6685, 8048, 0, 0, 0, 0, 0, 251, 0, 76, 0, 0, 0, 1577, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 7006, 0, 0, 0, 0, 0, 0, 204, 0, 0, 7845, 0, 654, 0, 6, 362, 55, 0, 0, 0, 75, 0, 263, 0, 0, 0, 0, 0, 0, 7267, 6893, 0, 248, 0, 0, 0, 153, 0, 1033, 680, 0, 0, 0, 6503, 8038, 0, 0, 0, 0, 0, 247, 0, 100, 0, 0, 0, 1520, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 7359, 0, 0, 0, 0, 0, 0, 154, 0, 0, 7840, 0, 602, 0, 19, 343, 53, 0, 0, 0, 171, 0, 377, 0, 0, 0, 0, 0, 0, 6907, 6111, 0, 225, 0, 0, 0, 161, 0, 1191, 762, 0, 0, 0, 6563, 8050, 0, 0, 6, 0, 0, 265, 0, 90, 0, 0, 0, 1903, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=23 tokens_per_expert_current_ep_rank=[491, 762, 868, 493, 47, 1, 117, 89, 3448, 60, 331, 24, 291, 13, 379, 232, 294, 972, 70, 63, 0, 2, 1, 203, 262, 578, 192, 42, 1475, 6, 0, 166, 39, 86, 58, 0, 3777, 86, 858, 1, 1934, 1630, 286, 2897, 5041, 691, 246, 28, 5286, 212, 198, 1436, 1595, 0, 123, 184, 3, 5153, 396, 4440, 290, 146, 16, 44] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=23 tokens_per_expert_current_ep_rank=[1, 1, 6808, 6, 3, 1, 3, 0, 13, 257, 0, 0, 7801, 0, 632, 1, 10, 379, 49, 0, 0, 0, 70, 0, 162, 4, 6, 1, 0, 1, 1, 7231, 6911, 0, 222, 0, 27, 0, 149, 0, 913, 852, 2, 20, 32, 6632, 8028, 0, 33, 3, 0, 1, 271, 0, 76, 0, 0, 33, 1474, 29, 1, 2, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 218, 0, 0, 5573, 0, 570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 345, 64, 0, 0, 0, 86, 0, 6215, 0, 0, 0, 0, 0, 0, 0, 176, 0, 0, 0, 0, 0, 0, 6556, 0, 0, 0, 0, 0, 0, 0, 0, 6384, 0, 288, 0, 0, 0, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 692, 750, 0, 0, 0, 3530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 0, 87, 0, 3052, 1840, 419, 0, 0, 0, 0, 0, 0, 0, 1366, 0, 0, 0, 0, 0, 1886, 774, 774, 509, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6613, 6361, 6711, 6732, 6816, 4972, 5638, 5309] avg=6144.0000 max=6816 max_rank=4 min=4972 min_rank=5 imbalance_ratio=1.109375 max_load_reduction=42336 max_load_reduction_pct=0.861328 moved_to_echo=22735 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 204, 0, 0, 5550, 0, 654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 362, 55, 0, 0, 0, 75, 0, 6202, 0, 0, 0, 0, 0, 0, 0, 263, 0, 0, 0, 0, 0, 0, 6500, 0, 0, 0, 0, 0, 0, 0, 0, 6148, 0, 248, 0, 0, 0, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 680, 0, 0, 0, 3488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 0, 100, 0, 3015, 1836, 417, 0, 0, 0, 0, 0, 0, 0, 1520, 0, 0, 0, 0, 0, 1878, 767, 745, 501, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6505, 6408, 6700, 6763, 6549, 5201, 5615, 5411] avg=6144.0000 max=6763 max_rank=3 min=5201 min_rank=5 imbalance_ratio=1.100749 max_load_reduction=42389 max_load_reduction_pct=0.862406 moved_to_echo=22776 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 6650, 6, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 207, 0, 0, 5540, 0, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 333, 61, 0, 0, 0, 184, 1, 6195, 0, 0, 0, 0, 0, 0, 0, 403, 1, 9, 0, 0, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 5809, 0, 191, 0, 19, 0, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1170, 725, 1, 5, 20, 3605, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 23, 6, 0, 1, 242, 0, 77, 0, 3117, 1833, 417, 0, 0, 0, 0, 0, 0, 22, 1561, 17, 0, 0, 0, 0, 1876, 725, 704, 512, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6660, 6321, 6796, 6557, 6158, 5527, 5716, 5417] avg=6144.0000 max=6796 max_rank=2 min=5417 min_rank=7 imbalance_ratio=1.106120 max_load_reduction=42356 max_load_reduction_pct=0.861735 moved_to_echo=22818 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=23 rerouted_tokens_per_expert_current_ep_rank=[491, 762, 806, 493, 47, 1, 117, 89, 0, 0, 0, 0, 0, 0, 0, 0, 3448, 60, 331, 24, 207, 13, 379, 232, 0, 0, 0, 0, 0, 0, 0, 0, 294, 972, 70, 63, 0, 2, 1, 203, 190, 0, 0, 0, 0, 0, 0, 0, 262, 578, 192, 42, 1475, 6, 0, 149, 0, 0, 0, 0, 0, 0, 0, 0, 35, 86, 58, 0, 3777, 86, 858, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1934, 1630, 286, 2897, 5041, 371, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 5286, 212, 198, 1436, 1595, 0, 123, 184, 320, 56, 15, 0, 0, 0, 0, 0, 3, 5153, 396, 4440, 290, 146, 16, 44, 69, 17, 4, 62, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2806, 4694, 1795, 2704, 4901, 12187, 9425, 10640] avg=6144.0000 max=12187 max_rank=5 min=1795 min_rank=2 imbalance_ratio=1.983561 max_load_reduction=36965 max_load_reduction_pct=0.752055 moved_to_echo=37153 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 0, 0, 5546, 0, 602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 343, 53, 0, 0, 0, 171, 0, 6212, 0, 0, 0, 0, 0, 0, 0, 377, 0, 0, 0, 0, 0, 0, 6178, 0, 0, 0, 0, 0, 0, 0, 0, 5451, 0, 225, 0, 0, 0, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1191, 762, 0, 0, 0, 3520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 265, 0, 90, 0, 3043, 1838, 417, 0, 0, 0, 0, 0, 0, 0, 1903, 0, 0, 0, 0, 0, 1877, 729, 660, 526, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6833, 6302, 6798, 6555, 5837, 5473, 5659, 5695] avg=6144.0000 max=6833 max_rank=0 min=5473 min_rank=5 imbalance_ratio=1.112142 max_load_reduction=42319 max_load_reduction_pct=0.860982 moved_to_echo=22664 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 0, 0, 5565, 0, 687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 365, 70, 0, 0, 0, 71, 0, 6209, 0, 0, 0, 0, 0, 0, 0, 170, 0, 0, 0, 0, 0, 0, 6544, 0, 0, 0, 0, 0, 0, 0, 0, 6261, 0, 229, 0, 0, 0, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 626, 687, 0, 0, 0, 3585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 0, 76, 0, 3100, 1839, 418, 0, 0, 0, 0, 0, 0, 0, 1577, 0, 0, 0, 0, 0, 1884, 772, 759, 502, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6517, 6498, 6715, 6714, 6632, 4898, 5684, 5494] avg=6144.0000 max=6715 max_rank=2 min=4898 min_rank=5 imbalance_ratio=1.092936 max_load_reduction=42437 max_load_reduction_pct=0.863383 moved_to_echo=22708 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 5559, 0, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 380, 68, 0, 0, 0, 245, 0, 6207, 0, 0, 0, 0, 0, 0, 0, 354, 0, 0, 0, 0, 0, 0, 6133, 0, 0, 0, 0, 0, 0, 0, 0, 5390, 0, 171, 0, 0, 0, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1155, 769, 0, 0, 0, 3599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 247, 0, 80, 0, 3112, 1837, 418, 0, 0, 0, 0, 0, 0, 0, 1981, 0, 0, 0, 0, 0, 1882, 724, 653, 530, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6884, 6155, 6934, 6487, 5700, 5523, 5699, 5770] avg=6144.0000 max=6934 max_rank=2 min=5523 min_rank=5 imbalance_ratio=1.128581 max_load_reduction=42218 max_load_reduction_pct=0.858927 moved_to_echo=22692 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=23 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 6319, 6, 3, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 257, 0, 0, 5510, 0, 632, 1, 0, 0, 0, 0, 0, 0, 0, 0, 10, 379, 49, 0, 0, 0, 70, 0, 6195, 0, 0, 0, 0, 0, 0, 0, 162, 4, 6, 1, 0, 1, 1, 6465, 0, 0, 0, 0, 0, 0, 0, 0, 6161, 0, 222, 0, 27, 0, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 913, 852, 2, 20, 32, 3552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 3, 0, 1, 271, 0, 76, 0, 3080, 1833, 419, 0, 0, 0, 0, 0, 0, 33, 1474, 29, 1, 2, 0, 0, 1872, 766, 750, 489, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6334, 6413, 6703, 6640, 6559, 5371, 5716, 5416] avg=6144.0000 max=6703 max_rank=2 min=5371 min_rank=5 imbalance_ratio=1.090983 max_load_reduction=42449 max_load_reduction_pct=0.863627 moved_to_echo=23062 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=50304 observed_loss=-1152 tokens_per_expert=[128, 5248, 11904, 4608, 384, 256, 128, 128, 13312, 5376, 5120, 3712, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[8832, 6912, 384, 2944, 5120, 25344, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5376, 256, 256, 1536, 3456, 0, 768, 256, 21888, 12928, 2944, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3584, 1536, 384, 128, 39168, 128, 4608, 256, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[512, 3584, 512, 128, 0, 128, 1024, 256, 43648, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2176, 640, 256, 128, 1536, 128, 128, 44672, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[41728, 128, 1664, 0, 3840, 128, 1920, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[512, 768, 47232, 512, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=50304 slack_slots=48000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=24 tokens_per_expert_current_ep_rank=[0, 55, 0, 0, 0, 0, 450, 0, 95, 7501, 2354, 0, 0, 99, 0, 0, 0, 0, 0, 55, 0, 360, 0, 0, 0, 0, 0, 0, 945, 7739, 0, 0, 0, 0, 0, 0, 497, 7130, 0, 8093, 100, 0, 0, 2951, 133, 0, 73, 0, 0, 0, 0, 0, 0, 2215, 0, 0, 0, 0, 347, 7812, 0, 0, 0, 148] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=24 tokens_per_expert_current_ep_rank=[0, 64, 0, 0, 0, 0, 495, 0, 67, 7479, 2097, 0, 0, 73, 0, 0, 0, 0, 0, 64, 0, 396, 0, 0, 0, 0, 0, 0, 830, 7728, 0, 0, 0, 0, 0, 0, 536, 7270, 0, 8119, 74, 0, 0, 2798, 113, 0, 29, 0, 0, 0, 0, 0, 0, 2630, 0, 0, 0, 0, 314, 7864, 0, 0, 0, 112] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=24 tokens_per_expert_current_ep_rank=[0, 61, 8, 21, 23, 14, 378, 9, 73, 7464, 3298, 0, 0, 77, 1, 0, 1, 0, 0, 66, 21, 397, 17, 0, 0, 0, 0, 0, 1006, 7692, 15, 0, 0, 20, 0, 0, 680, 7129, 1, 8094, 76, 0, 1, 2651, 129, 2, 152, 0, 6, 1, 0, 1, 0, 1277, 0, 0, 0, 0, 317, 7752, 0, 0, 0, 221] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=24 tokens_per_expert_current_ep_rank=[25, 41, 3909, 4997, 4764, 3087, 53, 15, 84, 821, 64, 0, 0, 2287, 27, 140, 22, 833, 0, 9, 4799, 56, 1884, 0, 25, 1184, 2, 114, 274, 422, 2939, 0, 136, 5364, 3, 67, 1715, 55, 1172, 2319, 25, 94, 5, 183, 1910, 448, 1446, 106, 205, 42, 168, 14, 31, 1, 0, 0, 0, 5, 321, 34, 92, 67, 146, 101] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=24 tokens_per_expert_current_ep_rank=[0, 70, 0, 0, 0, 0, 543, 0, 74, 7494, 2229, 0, 0, 74, 0, 0, 0, 0, 0, 70, 0, 488, 0, 0, 0, 0, 0, 0, 839, 7688, 0, 0, 0, 0, 0, 0, 487, 7276, 0, 8118, 75, 0, 0, 2689, 88, 0, 41, 0, 0, 0, 0, 0, 0, 2523, 0, 0, 0, 0, 326, 7849, 0, 0, 0, 111] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=24 tokens_per_expert_current_ep_rank=[0, 49, 0, 0, 0, 0, 312, 0, 64, 7476, 3596, 0, 0, 89, 0, 0, 0, 0, 0, 53, 0, 358, 0, 0, 0, 0, 0, 0, 1051, 7712, 0, 0, 0, 0, 0, 0, 562, 7141, 0, 8103, 90, 0, 0, 2146, 119, 0, 110, 0, 0, 0, 0, 0, 0, 1669, 0, 0, 0, 0, 355, 7822, 0, 0, 0, 275] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=24 tokens_per_expert_current_ep_rank=[0, 64, 0, 0, 0, 0, 351, 0, 27, 7428, 3787, 0, 0, 80, 0, 0, 0, 0, 0, 68, 0, 388, 0, 0, 0, 0, 0, 0, 1048, 7635, 0, 0, 0, 0, 0, 0, 646, 7198, 0, 8112, 80, 0, 0, 2351, 84, 0, 100, 0, 0, 0, 0, 0, 0, 1239, 0, 0, 0, 0, 327, 7800, 0, 0, 0, 339] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=24 tokens_per_expert_current_ep_rank=[0, 47, 21, 32, 32, 18, 462, 2, 72, 7445, 1852, 0, 0, 78, 5, 2, 1, 6, 0, 48, 32, 430, 20, 0, 0, 3, 0, 0, 895, 7662, 20, 0, 1, 32, 0, 1, 631, 7157, 2, 8084, 72, 1, 0, 3317, 186, 3, 112, 0, 3, 0, 0, 0, 0, 2147, 0, 0, 0, 0, 341, 7722, 0, 3, 2, 150] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 64, 0, 0, 0, 0, 495, 0, 2279, 1784, 0, 0, 0, 0, 0, 0, 67, 3703, 2097, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 396, 0, 0, 5254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 830, 5406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 5284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 2798, 113, 0, 29, 0, 1986, 1427, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2630, 0, 0, 2865, 1992, 0, 0, 0, 0, 0, 0, 0, 0, 314, 6437, 0, 0, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4622, 5940, 5714, 6236, 5820, 6470, 7487, 6863] avg=6144.0000 max=7487 max_rank=6 min=4622 min_rank=0 imbalance_ratio=1.218587 max_load_reduction=41665 max_load_reduction_pct=0.847677 moved_to_echo=26640 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 55, 0, 0, 0, 0, 450, 0, 2282, 1789, 0, 0, 0, 0, 0, 0, 95, 3714, 2354, 0, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 360, 0, 0, 5237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 945, 5413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 5183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 2951, 133, 0, 73, 0, 1947, 1418, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2215, 0, 0, 2856, 1998, 0, 0, 0, 0, 0, 0, 0, 0, 347, 6394, 0, 0, 0, 148, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4576, 6262, 5652, 6358, 5680, 6666, 7069, 6889] avg=6144.0000 max=7069 max_rank=6 min=4576 min_rank=0 imbalance_ratio=1.150553 max_load_reduction=42083 max_load_reduction_pct=0.856181 moved_to_echo=26304 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 70, 0, 0, 0, 0, 543, 0, 2267, 1788, 0, 0, 0, 0, 0, 0, 74, 3710, 2229, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 488, 0, 0, 5253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 839, 5378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 5289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 2689, 88, 0, 41, 0, 1987, 1425, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2523, 0, 0, 2865, 1996, 0, 0, 0, 0, 0, 0, 0, 0, 326, 6424, 0, 0, 0, 111, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4668, 6087, 5811, 6217, 5776, 6348, 7384, 6861] avg=6144.0000 max=7384 max_rank=6 min=4668 min_rank=0 imbalance_ratio=1.201823 max_load_reduction=41768 max_load_reduction_pct=0.849772 moved_to_echo=26369 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 61, 8, 21, 23, 14, 378, 9, 2268, 1781, 0, 0, 0, 0, 0, 0, 73, 3695, 3298, 0, 0, 77, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 66, 21, 397, 17, 0, 5239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 5381, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 680, 5182, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 1, 2651, 129, 2, 152, 0, 1947, 1407, 43, 0, 0, 0, 0, 0, 6, 1, 0, 1, 0, 1277, 0, 0, 2855, 1988, 0, 0, 0, 0, 0, 0, 0, 0, 317, 6345, 0, 0, 0, 221, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4563, 7144, 5741, 6402, 5883, 6408, 6128, 6883] avg=6144.0000 max=7144 max_rank=1 min=4563 min_rank=0 imbalance_ratio=1.162760 max_load_reduction=42008 max_load_reduction_pct=0.854655 moved_to_echo=25302 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=24 rerouted_tokens_per_expert_current_ep_rank=[25, 41, 3909, 4997, 4764, 3087, 53, 15, 124, 195, 0, 0, 0, 0, 0, 0, 84, 408, 64, 0, 0, 2287, 27, 140, 0, 0, 0, 0, 0, 0, 0, 0, 22, 833, 0, 9, 4799, 56, 1884, 0, 1501, 0, 0, 0, 0, 0, 0, 0, 25, 1184, 2, 114, 274, 296, 2939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 5364, 3, 67, 1715, 40, 1172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 94, 5, 183, 1910, 448, 1446, 106, 15, 6, 2, 0, 0, 0, 0, 0, 205, 42, 168, 14, 31, 1, 0, 0, 818, 218, 0, 0, 0, 0, 0, 0, 0, 5, 321, 28, 92, 67, 146, 101, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17210, 3010, 9104, 4834, 8497, 4240, 1497, 760] avg=6144.0000 max=17210 max_rank=0 min=760 min_rank=7 imbalance_ratio=2.801107 max_load_reduction=31942 max_load_reduction_pct=0.649862 moved_to_echo=14994 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 49, 0, 0, 0, 0, 312, 0, 2274, 1783, 0, 0, 0, 0, 0, 0, 64, 3702, 3596, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 358, 0, 0, 5244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 5395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 562, 5191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 2146, 119, 0, 110, 0, 1950, 1420, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1669, 0, 0, 2859, 1991, 0, 0, 0, 0, 0, 0, 0, 0, 355, 6402, 0, 0, 0, 275, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4418, 7451, 5655, 6446, 5753, 5878, 6519, 7032] avg=6144.0000 max=7451 max_rank=1 min=4418 min_rank=0 imbalance_ratio=1.212728 max_load_reduction=41701 max_load_reduction_pct=0.848409 moved_to_echo=25182 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 64, 0, 0, 0, 0, 351, 0, 2251, 1772, 0, 0, 0, 0, 0, 0, 27, 3678, 3787, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 388, 0, 0, 5250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 5341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 646, 5232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 2351, 84, 0, 100, 0, 1966, 1416, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1239, 0, 0, 2862, 1978, 0, 0, 0, 0, 0, 0, 0, 0, 327, 6384, 0, 0, 0, 339, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4438, 7572, 5706, 6389, 5878, 6040, 6079, 7050] avg=6144.0000 max=7572 max_rank=1 min=4438 min_rank=0 imbalance_ratio=1.232422 max_load_reduction=41580 max_load_reduction_pct=0.845947 moved_to_echo=25047 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 47, 21, 32, 32, 18, 462, 2, 2262, 1781, 0, 0, 0, 0, 0, 0, 72, 3677, 1852, 0, 0, 78, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 48, 32, 430, 20, 0, 5232, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 895, 5352, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 32, 0, 1, 631, 5198, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 1, 0, 3317, 186, 3, 112, 0, 1959, 1404, 48, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 2147, 0, 0, 2852, 1987, 0, 0, 0, 0, 0, 0, 0, 0, 341, 6318, 0, 3, 2, 150, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4657, 5686, 5769, 6270, 5865, 7102, 6989, 6814] avg=6144.0000 max=7102 max_rank=5 min=4657 min_rank=0 imbalance_ratio=1.155924 max_load_reduction=42050 max_load_reduction_pct=0.855509 moved_to_echo=26770 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 512, 3968, 5120, 4864, 3200, 3072, 128, 16128, 12800, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[640, 128, 128, 19200, 2816, 512, 2176, 128, 13824, 9984, 384, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[256, 128, 256, 128, 128, 13824, 0, 0, 20864, 14208, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[640, 26368, 19328, 0, 0, 2944, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 1280, 128, 128, 6912, 38016, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 896, 0, 512, 4864, 2944, 2048, 0, 38272, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 128, 2688, 44800, 128, 128, 256, 1536, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[256, 5504, 128, 128, 5760, 36608, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True - [2026-08-26 14:53:50] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 829.2 | throughput per GPU (TFLOP/s/GPU): 153.9 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.899922E+00 | loss scale: 1.0 | grad norm: 1155.511 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1337, 263, 856, 910, 310, 927, 839, 1023, 1350, 225, 589, 330, 369, 1444, 968, 1076, 723, 1029, 846, 675, 293, 536, 265, 1351, 557, 671, 724, 377, 727, 95, 838, 1290, 790, 831, 961, 1010, 504, 206, 574, 1022, 655, 566, 556, 754, 884, 998, 534, 678, 795, 522, 910, 444, 1003, 1356, 880, 863, 1585, 308, 1500, 650, 159, 771, 1168, 902] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1309, 301, 768, 889, 249, 987, 843, 998, 1264, 231, 658, 318, 382, 1461, 946, 1022, 713, 1016, 769, 600, 308, 528, 269, 1374, 447, 653, 863, 361, 730, 124, 850, 1453, 894, 833, 1060, 1135, 370, 195, 650, 878, 613, 600, 569, 899, 967, 928, 498, 634, 760, 612, 756, 538, 1020, 1246, 877, 912, 1539, 265, 1450, 738, 187, 737, 1223, 885] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1225, 257, 788, 939, 315, 1027, 844, 924, 1242, 220, 550, 357, 385, 1306, 926, 1191, 744, 1060, 788, 527, 342, 513, 242, 1489, 428, 642, 858, 311, 680, 136, 956, 1449, 815, 810, 1059, 1145, 362, 212, 567, 939, 578, 651, 590, 894, 905, 868, 569, 639, 692, 521, 756, 532, 1119, 1228, 964, 897, 1529, 378, 1492, 793, 177, 725, 1165, 920] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1426, 336, 755, 935, 309, 1151, 881, 797, 1237, 321, 758, 313, 288, 1669, 822, 912, 714, 964, 829, 647, 324, 630, 278, 1300, 376, 663, 837, 445, 699, 165, 740, 1265, 834, 797, 1100, 1187, 534, 191, 578, 970, 543, 594, 622, 733, 922, 1004, 465, 456, 749, 608, 755, 560, 923, 1435, 867, 778, 1432, 317, 1533, 700, 196, 768, 1315, 900] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1347, 270, 813, 863, 299, 826, 790, 1071, 1234, 200, 536, 307, 416, 1311, 907, 1084, 743, 940, 790, 605, 312, 499, 307, 1391, 561, 654, 771, 357, 681, 77, 917, 1366, 844, 867, 978, 1135, 437, 233, 501, 924, 691, 601, 538, 891, 921, 943, 585, 779, 791, 582, 839, 483, 1003, 1232, 828, 910, 1548, 497, 1466, 701, 174, 754, 1202, 1029] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1348, 327, 818, 924, 322, 836, 830, 1074, 1242, 215, 611, 343, 330, 1383, 933, 1152, 694, 937, 848, 647, 244, 523, 293, 1401, 532, 711, 772, 393, 695, 80, 900, 1413, 791, 841, 1057, 1065, 456, 223, 515, 957, 639, 630, 582, 840, 904, 933, 542, 642, 773, 549, 885, 488, 990, 1301, 738, 879, 1561, 363, 1410, 720, 135, 727, 1222, 1023] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1340, 281, 722, 829, 273, 885, 741, 1106, 1377, 196, 599, 331, 366, 1344, 915, 1176, 661, 928, 766, 587, 278, 508, 264, 1501, 542, 683, 741, 388, 713, 101, 879, 1138, 868, 795, 1119, 1017, 730, 223, 478, 1118, 642, 596, 555, 980, 882, 984, 608, 659, 687, 509, 820, 576, 1028, 1363, 835, 880, 1504, 379, 1403, 713, 120, 721, 1152, 1049] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1341, 263, 746, 867, 265, 792, 905, 916, 1315, 257, 582, 338, 295, 1451, 903, 1085, 701, 1019, 805, 671, 267, 536, 284, 1334, 455, 589, 777, 457, 798, 95, 801, 1106, 840, 795, 1306, 1139, 658, 210, 636, 908, 664, 527, 552, 892, 962, 981, 492, 595, 768, 603, 736, 561, 983, 1409, 814, 943, 1491, 375, 1567, 666, 183, 789, 1266, 825] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1138, 263, 856, 910, 310, 927, 839, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1350, 225, 589, 330, 369, 1354, 968, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 723, 1029, 846, 675, 293, 536, 265, 1351, 358, 180, 0, 0, 0, 0, 0, 0, 557, 671, 724, 377, 727, 95, 838, 1290, 858, 0, 0, 0, 0, 0, 0, 0, 790, 831, 961, 1010, 504, 206, 574, 1022, 60, 0, 0, 0, 0, 0, 0, 0, 655, 566, 556, 754, 884, 998, 534, 678, 220, 199, 30, 0, 0, 0, 0, 0, 795, 522, 910, 444, 1003, 778, 880, 863, 0, 0, 0, 0, 0, 0, 0, 0, 547, 308, 1500, 650, 159, 771, 1168, 902, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6266, 6261, 6256, 6137, 5958, 6074, 6195, 6005] avg=6144.0000 max=6266 max_rank=0 min=5958 min_rank=4 imbalance_ratio=1.019857 max_load_reduction=42886 max_load_reduction_pct=0.872518 moved_to_echo=24232 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1042, 257, 788, 939, 315, 1027, 844, 924, 0, 0, 0, 0, 0, 0, 0, 0, 1242, 220, 550, 357, 385, 1225, 926, 1191, 0, 0, 0, 0, 0, 0, 0, 0, 744, 1060, 788, 527, 342, 513, 242, 1489, 324, 174, 0, 0, 0, 0, 0, 0, 428, 642, 858, 311, 680, 136, 956, 1449, 827, 0, 0, 0, 0, 0, 0, 0, 815, 810, 1059, 1145, 362, 212, 567, 939, 54, 0, 0, 0, 0, 0, 0, 0, 578, 651, 590, 894, 905, 868, 569, 639, 199, 183, 27, 0, 0, 0, 0, 0, 692, 521, 756, 532, 1119, 705, 964, 897, 0, 0, 0, 0, 0, 0, 0, 0, 528, 378, 1492, 793, 177, 725, 1165, 920, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6136, 6096, 6203, 6287, 5963, 6103, 6186, 6178] avg=6144.0000 max=6287 max_rank=3 min=5963 min_rank=4 imbalance_ratio=1.023275 max_load_reduction=42865 max_load_reduction_pct=0.872091 moved_to_echo=24430 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1114, 301, 768, 889, 249, 987, 843, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1264, 231, 658, 318, 382, 1370, 946, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 713, 1016, 769, 600, 308, 528, 269, 1374, 329, 175, 0, 0, 0, 0, 0, 0, 447, 653, 863, 361, 730, 124, 850, 1453, 833, 0, 0, 0, 0, 0, 0, 0, 894, 833, 1060, 1135, 370, 195, 650, 878, 61, 0, 0, 0, 0, 0, 0, 0, 613, 600, 569, 899, 967, 928, 498, 634, 202, 195, 30, 0, 0, 0, 0, 0, 760, 612, 756, 538, 1020, 715, 877, 912, 0, 0, 0, 0, 0, 0, 0, 0, 531, 265, 1450, 738, 187, 737, 1223, 885, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6149, 6191, 6081, 6314, 6076, 6135, 6190, 6016] avg=6144.0000 max=6314 max_rank=3 min=6016 min_rank=7 imbalance_ratio=1.027669 max_load_reduction=42838 max_load_reduction_pct=0.871541 moved_to_echo=24417 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1213, 336, 755, 935, 309, 1151, 881, 797, 0, 0, 0, 0, 0, 0, 0, 0, 1237, 321, 758, 313, 288, 1566, 822, 912, 0, 0, 0, 0, 0, 0, 0, 0, 714, 964, 829, 647, 324, 630, 278, 1300, 378, 163, 0, 0, 0, 0, 0, 0, 376, 663, 837, 445, 699, 165, 740, 1265, 775, 0, 0, 0, 0, 0, 0, 0, 834, 797, 1100, 1187, 534, 191, 578, 970, 69, 0, 0, 0, 0, 0, 0, 0, 543, 594, 622, 733, 922, 1004, 465, 456, 233, 213, 34, 0, 0, 0, 0, 0, 749, 608, 755, 560, 923, 824, 867, 778, 0, 0, 0, 0, 0, 0, 0, 0, 494, 317, 1533, 700, 196, 768, 1315, 900, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6377, 6217, 6227, 5965, 6260, 5819, 6064, 6223] avg=6144.0000 max=6377 max_rank=0 min=5819 min_rank=5 imbalance_ratio=1.037923 max_load_reduction=42775 max_load_reduction_pct=0.870260 moved_to_echo=24366 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1146, 270, 813, 863, 299, 826, 790, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 1234, 200, 536, 307, 416, 1230, 907, 1084, 0, 0, 0, 0, 0, 0, 0, 0, 743, 940, 790, 605, 312, 499, 307, 1391, 325, 176, 0, 0, 0, 0, 0, 0, 561, 654, 771, 357, 681, 77, 917, 1366, 838, 0, 0, 0, 0, 0, 0, 0, 844, 867, 978, 1135, 437, 233, 501, 924, 54, 0, 0, 0, 0, 0, 0, 0, 691, 601, 538, 891, 921, 943, 585, 779, 200, 201, 27, 0, 0, 0, 0, 0, 791, 582, 839, 483, 1003, 707, 828, 910, 0, 0, 0, 0, 0, 0, 0, 0, 534, 497, 1466, 701, 174, 754, 1202, 1029, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6078, 5914, 6088, 6222, 5973, 6377, 6143, 6357] avg=6144.0000 max=6377 max_rank=5 min=5914 min_rank=1 imbalance_ratio=1.037923 max_load_reduction=42775 max_load_reduction_pct=0.870260 moved_to_echo=24850 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1140, 281, 722, 829, 273, 885, 741, 1106, 0, 0, 0, 0, 0, 0, 0, 0, 1377, 196, 599, 331, 366, 1260, 915, 1176, 0, 0, 0, 0, 0, 0, 0, 0, 661, 928, 766, 587, 278, 508, 264, 1501, 359, 171, 0, 0, 0, 0, 0, 0, 542, 683, 741, 388, 713, 101, 879, 1138, 814, 0, 0, 0, 0, 0, 0, 0, 868, 795, 1119, 1017, 730, 223, 478, 1118, 56, 0, 0, 0, 0, 0, 0, 0, 642, 596, 555, 980, 882, 984, 608, 659, 221, 200, 28, 0, 0, 0, 0, 0, 687, 509, 820, 576, 1028, 783, 835, 880, 0, 0, 0, 0, 0, 0, 0, 0, 519, 379, 1403, 713, 120, 721, 1152, 1049, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5977, 6220, 6023, 5999, 6404, 6355, 6118, 6056] avg=6144.0000 max=6404 max_rank=4 min=5977 min_rank=0 imbalance_ratio=1.042318 max_load_reduction=42748 max_load_reduction_pct=0.869710 moved_to_echo=24933 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1147, 327, 818, 924, 322, 836, 830, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 1242, 215, 611, 343, 330, 1298, 933, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 694, 937, 848, 647, 244, 523, 293, 1401, 343, 178, 0, 0, 0, 0, 0, 0, 532, 711, 772, 393, 695, 80, 900, 1413, 845, 0, 0, 0, 0, 0, 0, 0, 791, 841, 1057, 1065, 456, 223, 515, 957, 57, 0, 0, 0, 0, 0, 0, 0, 639, 630, 582, 840, 904, 933, 542, 642, 211, 201, 28, 0, 0, 0, 0, 0, 773, 549, 885, 488, 990, 747, 738, 879, 0, 0, 0, 0, 0, 0, 0, 0, 538, 363, 1410, 720, 135, 727, 1222, 1023, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6278, 6124, 6108, 6341, 5962, 6152, 6049, 6138] avg=6144.0000 max=6341 max_rank=3 min=5962 min_rank=4 imbalance_ratio=1.032064 max_load_reduction=42811 max_load_reduction_pct=0.870992 moved_to_echo=24301 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1137, 263, 746, 867, 265, 792, 905, 916, 0, 0, 0, 0, 0, 0, 0, 0, 1315, 257, 582, 338, 295, 1354, 903, 1085, 0, 0, 0, 0, 0, 0, 0, 0, 701, 1019, 805, 671, 267, 536, 284, 1334, 375, 174, 0, 0, 0, 0, 0, 0, 455, 589, 777, 457, 798, 95, 801, 1106, 809, 0, 0, 0, 0, 0, 0, 0, 840, 795, 1306, 1139, 658, 210, 636, 908, 64, 0, 0, 0, 0, 0, 0, 0, 664, 527, 552, 892, 962, 981, 492, 595, 235, 204, 33, 0, 0, 0, 0, 0, 768, 603, 736, 561, 983, 799, 814, 943, 0, 0, 0, 0, 0, 0, 0, 0, 508, 375, 1567, 666, 183, 789, 1266, 825, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5891, 6129, 6166, 5887, 6556, 6137, 6207, 6179] avg=6144.0000 max=6556 max_rank=4 min=5887 min_rank=3 imbalance_ratio=1.067057 max_load_reduction=42596 max_load_reduction_pct=0.866618 moved_to_echo=25079 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[6784, 6656, 8704, 8960, 4096, 1792, 4608, 7808, 512, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[9088, 2304, 6272, 7168, 2432, 7552, 6784, 7936, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3968, 5376, 6400, 3200, 5760, 896, 6912, 10496, 6656, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5760, 7936, 6528, 4992, 2432, 4352, 2304, 11264, 2816, 1408, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6016, 4608, 6528, 4224, 8192, 6144, 6912, 7168, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[10368, 1920, 4992, 2688, 2944, 10752, 7424, 8704, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5120, 4864, 4608, 6912, 7424, 7680, 4352, 5120, 1792, 1664, 256, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4224, 2944, 11904, 5760, 1408, 6016, 9728, 7552, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[728, 825, 1014, 496, 905, 231, 322, 1193, 97, 983, 1271, 302, 553, 1014, 998, 1476, 1189, 1306, 887, 424, 1496, 188, 1852, 1069, 587, 669, 609, 750, 103, 373, 215, 120, 488, 310, 574, 186, 1025, 1847, 980, 1009, 2279, 1432, 1274, 1192, 437, 945, 55, 190, 743, 1289, 750, 24, 130, 1629, 228, 1357, 797, 315, 695, 534, 979, 555, 266, 393] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[750, 909, 800, 384, 1025, 224, 272, 1212, 108, 922, 1534, 397, 517, 807, 778, 1589, 998, 1381, 884, 475, 1393, 254, 1847, 1181, 755, 796, 796, 911, 129, 330, 209, 114, 438, 329, 415, 98, 1175, 1329, 1091, 889, 2172, 1498, 894, 1188, 447, 878, 78, 210, 1042, 1180, 1069, 18, 152, 1524, 229, 1289, 938, 317, 775, 537, 1005, 560, 341, 366] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[695, 746, 1037, 557, 875, 173, 337, 1287, 108, 1077, 1218, 342, 565, 998, 969, 1467, 1202, 1188, 832, 425, 1315, 163, 1537, 1298, 594, 798, 656, 879, 157, 383, 228, 106, 399, 254, 561, 144, 1251, 1661, 1029, 924, 2231, 1512, 1155, 1473, 415, 710, 30, 183, 786, 1203, 1064, 15, 124, 1526, 240, 1083, 816, 361, 861, 593, 920, 590, 351, 475] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[663, 836, 1022, 425, 1036, 268, 320, 1055, 148, 959, 1346, 341, 713, 1088, 879, 1473, 1176, 1245, 931, 412, 1460, 114, 1661, 976, 616, 643, 687, 837, 111, 292, 233, 126, 379, 203, 527, 227, 1205, 1749, 1081, 1123, 2132, 1553, 1327, 1413, 485, 733, 26, 180, 661, 1234, 898, 12, 151, 1453, 246, 1227, 746, 305, 802, 591, 1017, 568, 334, 472] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[674, 826, 1091, 440, 990, 236, 307, 1111, 90, 941, 1367, 410, 606, 1049, 953, 1475, 1120, 1082, 1030, 416, 1421, 143, 1670, 1064, 698, 641, 667, 826, 108, 354, 173, 108, 429, 271, 490, 194, 1089, 1809, 1090, 1048, 2163, 1532, 1150, 1320, 431, 968, 22, 192, 744, 1248, 829, 14, 160, 1431, 238, 1237, 972, 358, 844, 567, 959, 575, 288, 403] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[744, 849, 893, 406, 1009, 257, 234, 1225, 85, 911, 1584, 375, 634, 994, 804, 1560, 1134, 1161, 895, 466, 1330, 162, 1807, 1055, 687, 651, 696, 899, 81, 300, 239, 125, 426, 287, 448, 114, 1231, 1539, 1055, 1051, 2211, 1542, 1014, 1255, 410, 959, 56, 216, 902, 1238, 946, 9, 150, 1711, 207, 1251, 871, 323, 755, 520, 972, 517, 305, 409] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[690, 798, 856, 535, 1083, 186, 312, 1428, 97, 1012, 1413, 414, 597, 1094, 820, 1574, 1122, 1110, 863, 383, 1186, 202, 1609, 1379, 737, 636, 754, 909, 106, 391, 223, 125, 403, 254, 451, 99, 1157, 1330, 889, 896, 2280, 1469, 940, 1229, 374, 973, 45, 251, 1002, 1309, 1140, 11, 124, 1503, 229, 1095, 955, 342, 904, 572, 830, 635, 358, 459] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[641, 789, 1064, 477, 917, 255, 304, 1082, 112, 922, 1209, 372, 694, 1114, 858, 1480, 1219, 1211, 920, 436, 1565, 170, 1631, 1080, 647, 665, 763, 821, 96, 355, 229, 77, 435, 240, 509, 189, 1214, 1750, 1158, 1060, 2097, 1507, 1098, 1452, 427, 707, 33, 159, 739, 1190, 883, 23, 176, 1601, 207, 1223, 860, 378, 939, 591, 876, 593, 264, 399] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[728, 825, 1014, 496, 905, 231, 322, 1193, 272, 151, 50, 25, 0, 0, 0, 0, 97, 983, 1271, 302, 553, 1014, 998, 805, 0, 0, 0, 0, 0, 0, 0, 0, 1189, 1306, 887, 424, 1224, 188, 0, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 587, 669, 609, 750, 103, 373, 215, 120, 1852, 770, 0, 0, 0, 0, 0, 0, 488, 310, 574, 186, 1025, 1797, 980, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 716, 1432, 1274, 1192, 437, 945, 55, 190, 0, 0, 0, 0, 0, 0, 0, 0, 743, 1289, 750, 24, 130, 1604, 228, 1357, 0, 0, 0, 0, 0, 0, 0, 0, 797, 315, 695, 534, 979, 555, 266, 393, 793, 520, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6212, 6023, 6287, 6048, 6369, 6241, 6125, 5847] avg=6144.0000 max=6369 max_rank=4 min=5847 min_rank=7 imbalance_ratio=1.036621 max_load_reduction=42783 max_load_reduction_pct=0.870422 moved_to_echo=24582 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[695, 746, 1037, 557, 875, 173, 337, 1287, 239, 150, 45, 23, 0, 0, 0, 0, 108, 1077, 1218, 342, 565, 998, 969, 800, 0, 0, 0, 0, 0, 0, 0, 0, 1202, 1188, 832, 425, 1076, 163, 0, 1298, 0, 0, 0, 0, 0, 0, 0, 0, 594, 798, 656, 879, 157, 383, 228, 106, 1537, 754, 0, 0, 0, 0, 0, 0, 399, 254, 561, 144, 1251, 1616, 1029, 924, 0, 0, 0, 0, 0, 0, 0, 0, 700, 1512, 1155, 1473, 415, 710, 30, 183, 0, 0, 0, 0, 0, 0, 0, 0, 786, 1203, 1064, 15, 124, 1503, 240, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 816, 361, 861, 593, 920, 590, 351, 475, 777, 517, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6164, 6077, 6184, 6092, 6178, 6178, 6018, 6261] avg=6144.0000 max=6261 max_rank=7 min=6018 min_rank=6 imbalance_ratio=1.019043 max_load_reduction=42891 max_load_reduction_pct=0.872620 moved_to_echo=24635 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[663, 836, 1022, 425, 1036, 268, 320, 1055, 266, 150, 47, 22, 0, 0, 0, 0, 148, 959, 1346, 341, 713, 1088, 879, 804, 0, 0, 0, 0, 0, 0, 0, 0, 1176, 1245, 931, 412, 1194, 114, 0, 976, 0, 0, 0, 0, 0, 0, 0, 0, 616, 643, 687, 837, 111, 292, 233, 126, 1661, 720, 0, 0, 0, 0, 0, 0, 379, 203, 527, 227, 1205, 1702, 1081, 1123, 0, 0, 0, 0, 0, 0, 0, 0, 670, 1553, 1327, 1413, 485, 733, 26, 180, 0, 0, 0, 0, 0, 0, 0, 0, 661, 1234, 898, 12, 151, 1431, 246, 1227, 0, 0, 0, 0, 0, 0, 0, 0, 746, 305, 802, 591, 1017, 568, 334, 472, 742, 519, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6110, 6278, 6048, 5926, 6447, 6387, 5860, 6096] avg=6144.0000 max=6447 max_rank=4 min=5860 min_rank=6 imbalance_ratio=1.049316 max_load_reduction=42705 max_load_reduction_pct=0.868835 moved_to_echo=24790 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[674, 826, 1091, 440, 990, 236, 307, 1111, 258, 151, 49, 22, 0, 0, 0, 0, 90, 941, 1367, 410, 606, 1049, 953, 804, 0, 0, 0, 0, 0, 0, 0, 0, 1120, 1082, 1030, 416, 1163, 143, 0, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 698, 641, 667, 826, 108, 354, 173, 108, 1670, 731, 0, 0, 0, 0, 0, 0, 429, 271, 490, 194, 1089, 1760, 1090, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 679, 1532, 1150, 1320, 431, 968, 22, 192, 0, 0, 0, 0, 0, 0, 0, 0, 744, 1248, 829, 14, 160, 1409, 238, 1237, 0, 0, 0, 0, 0, 0, 0, 0, 972, 358, 844, 567, 959, 575, 288, 403, 753, 520, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6155, 6220, 6018, 5976, 6371, 6294, 5879, 6239] avg=6144.0000 max=6371 max_rank=4 min=5879 min_rank=6 imbalance_ratio=1.036947 max_load_reduction=42781 max_load_reduction_pct=0.870382 moved_to_echo=24783 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[750, 909, 800, 384, 1025, 224, 272, 1212, 253, 162, 36, 23, 0, 0, 0, 0, 108, 922, 1534, 397, 517, 807, 778, 867, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1381, 884, 475, 1140, 254, 0, 1181, 0, 0, 0, 0, 0, 0, 0, 0, 755, 796, 796, 911, 129, 330, 209, 114, 1847, 734, 0, 0, 0, 0, 0, 0, 438, 329, 415, 98, 1175, 1293, 1091, 889, 0, 0, 0, 0, 0, 0, 0, 0, 682, 1498, 894, 1188, 447, 878, 78, 210, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1180, 1069, 18, 152, 1501, 229, 1289, 0, 0, 0, 0, 0, 0, 0, 0, 938, 317, 775, 537, 1005, 560, 341, 366, 756, 560, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6050, 5930, 6313, 6621, 5728, 5875, 6480, 6155] avg=6144.0000 max=6621 max_rank=3 min=5728 min_rank=4 imbalance_ratio=1.077637 max_load_reduction=42531 max_load_reduction_pct=0.865295 moved_to_echo=24238 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[744, 849, 893, 406, 1009, 257, 234, 1225, 242, 159, 41, 26, 0, 0, 0, 0, 85, 911, 1584, 375, 634, 994, 804, 851, 0, 0, 0, 0, 0, 0, 0, 0, 1134, 1161, 895, 466, 1088, 162, 0, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 687, 651, 696, 899, 81, 300, 239, 125, 1807, 747, 0, 0, 0, 0, 0, 0, 426, 287, 448, 114, 1231, 1498, 1055, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 694, 1542, 1014, 1255, 410, 959, 56, 216, 0, 0, 0, 0, 0, 0, 0, 0, 902, 1238, 946, 9, 150, 1685, 207, 1251, 0, 0, 0, 0, 0, 0, 0, 0, 871, 323, 755, 520, 972, 517, 305, 409, 770, 550, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6085, 6238, 5961, 6232, 6110, 6146, 6388, 5992] avg=6144.0000 max=6388 max_rank=6 min=5961 min_rank=2 imbalance_ratio=1.039714 max_load_reduction=42764 max_load_reduction_pct=0.870036 moved_to_echo=24636 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[690, 798, 856, 535, 1083, 186, 312, 1428, 220, 165, 38, 28, 0, 0, 0, 0, 97, 1012, 1413, 414, 597, 1094, 820, 851, 0, 0, 0, 0, 0, 0, 0, 0, 1122, 1110, 863, 383, 966, 202, 0, 1379, 0, 0, 0, 0, 0, 0, 0, 0, 737, 636, 754, 909, 106, 391, 223, 125, 1609, 774, 0, 0, 0, 0, 0, 0, 403, 254, 451, 99, 1157, 1292, 889, 896, 0, 0, 0, 0, 0, 0, 0, 0, 709, 1469, 940, 1229, 374, 973, 45, 251, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1309, 1140, 11, 124, 1475, 229, 1095, 0, 0, 0, 0, 0, 0, 0, 0, 955, 342, 904, 572, 830, 635, 358, 459, 797, 558, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6339, 6298, 6025, 6264, 5441, 5990, 6385, 6410] avg=6144.0000 max=6410 max_rank=7 min=5441 min_rank=4 imbalance_ratio=1.043294 max_load_reduction=42742 max_load_reduction_pct=0.869588 moved_to_echo=24226 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[641, 789, 1064, 477, 917, 255, 304, 1082, 285, 151, 47, 25, 0, 0, 0, 0, 112, 922, 1209, 372, 694, 1114, 858, 807, 0, 0, 0, 0, 0, 0, 0, 0, 1219, 1211, 920, 436, 1280, 170, 0, 1080, 0, 0, 0, 0, 0, 0, 0, 0, 647, 665, 763, 821, 96, 355, 229, 77, 1631, 709, 0, 0, 0, 0, 0, 0, 435, 240, 509, 189, 1214, 1703, 1158, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 658, 1507, 1098, 1452, 427, 707, 33, 159, 0, 0, 0, 0, 0, 0, 0, 0, 739, 1190, 883, 23, 176, 1576, 207, 1223, 0, 0, 0, 0, 0, 0, 0, 0, 860, 378, 939, 591, 876, 593, 264, 399, 730, 522, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6037, 6088, 6316, 5993, 6508, 6041, 6017, 6152] avg=6144.0000 max=6508 max_rank=4 min=5993 min_rank=3 imbalance_ratio=1.059245 max_load_reduction=42644 max_load_reduction_pct=0.867594 moved_to_echo=24718 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5376, 5504, 5632, 6912, 896, 2816, 1792, 1024, 13696, 6016, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5632, 12160, 8960, 10624, 3456, 6912, 384, 1664, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5632, 6656, 7808, 3840, 7936, 1920, 2432, 9600, 2048, 1280, 384, 256, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[896, 7808, 11008, 3072, 4992, 8192, 7168, 6656, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[7040, 2816, 6656, 4608, 7680, 4608, 2560, 3456, 6144, 4352, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[9216, 9728, 7296, 3456, 9216, 1408, 0, 9216, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3456, 2176, 4096, 1280, 9472, 12672, 8448, 8064, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6656, 9984, 7680, 128, 1280, 12288, 1920, 9856, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[220, 878, 433, 421, 100, 1069, 1108, 231, 682, 247, 1026, 1022, 1539, 1567, 867, 1770, 969, 1308, 236, 411, 225, 396, 1121, 965, 272, 801, 1054, 605, 384, 503, 793, 336, 396, 986, 982, 659, 1791, 1423, 1884, 545, 511, 342, 469, 260, 1570, 1128, 206, 1728, 1154, 321, 606, 826, 135, 301, 319, 795, 1192, 328, 91, 825, 193, 1202, 618, 1807] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[182, 816, 547, 338, 88, 1101, 888, 233, 717, 352, 1084, 1198, 1647, 1219, 594, 1603, 951, 1329, 287, 394, 96, 382, 1252, 1041, 329, 681, 944, 600, 355, 539, 962, 309, 334, 951, 1124, 707, 1934, 1712, 1923, 565, 403, 312, 468, 270, 1607, 1129, 174, 1389, 999, 239, 890, 892, 95, 274, 308, 783, 1404, 463, 57, 601, 170, 1385, 427, 2105] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[101, 1002, 503, 344, 81, 1175, 1011, 199, 736, 332, 1207, 1230, 1505, 1121, 254, 1364, 954, 1608, 306, 458, 223, 278, 1296, 1016, 344, 733, 1187, 784, 416, 476, 955, 196, 372, 971, 1005, 538, 2219, 1561, 1677, 501, 575, 244, 363, 266, 1525, 1423, 208, 1534, 1045, 209, 712, 820, 93, 144, 280, 706, 1248, 372, 80, 658, 156, 1692, 311, 2249] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[202, 851, 584, 461, 61, 1419, 983, 253, 860, 306, 1053, 1074, 1658, 1161, 380, 1462, 1049, 1295, 302, 403, 98, 332, 1049, 1019, 450, 629, 911, 684, 367, 516, 965, 317, 383, 1125, 1177, 680, 1725, 1728, 1969, 527, 473, 231, 309, 259, 1589, 1203, 173, 1378, 1193, 253, 927, 847, 209, 393, 287, 818, 1376, 402, 78, 422, 180, 1135, 423, 2126] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[245, 949, 465, 478, 135, 1193, 976, 237, 849, 357, 1063, 1146, 1720, 1343, 715, 1558, 910, 1229, 267, 472, 72, 385, 1213, 854, 389, 693, 807, 547, 386, 600, 942, 377, 371, 1044, 1099, 742, 1851, 1667, 1921, 527, 489, 259, 321, 295, 1609, 1176, 151, 1341, 925, 289, 890, 823, 109, 339, 288, 724, 1397, 419, 55, 650, 135, 1308, 440, 1926] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[298, 1004, 436, 436, 96, 1138, 862, 261, 822, 362, 1081, 1241, 1705, 1490, 785, 1461, 951, 1286, 281, 466, 82, 357, 1334, 972, 306, 666, 849, 434, 499, 525, 863, 350, 344, 977, 1059, 710, 1694, 1587, 1811, 558, 528, 273, 338, 252, 1617, 1150, 188, 1360, 1006, 321, 916, 737, 143, 338, 335, 864, 1364, 360, 72, 604, 191, 1344, 505, 1907] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[80, 1122, 437, 422, 33, 1260, 1033, 228, 641, 311, 1314, 1373, 1328, 1031, 243, 1372, 1052, 1361, 586, 341, 156, 239, 1234, 1115, 483, 686, 1189, 855, 398, 499, 1011, 195, 403, 1063, 1100, 425, 2425, 1573, 1607, 523, 687, 234, 360, 406, 1435, 1297, 165, 1468, 1097, 167, 492, 822, 87, 165, 264, 650, 1334, 306, 124, 614, 120, 1642, 286, 2183] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[92, 1065, 611, 387, 35, 1408, 847, 249, 659, 392, 1147, 1227, 1785, 835, 126, 992, 1109, 1475, 472, 381, 124, 203, 1088, 1050, 631, 782, 1173, 1052, 337, 479, 884, 203, 392, 1159, 1301, 497, 2461, 1991, 1608, 466, 647, 222, 319, 333, 1449, 1280, 202, 1218, 974, 125, 779, 733, 103, 186, 271, 692, 1499, 423, 79, 396, 144, 1528, 161, 2214] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[220, 878, 433, 421, 100, 1069, 1108, 231, 1347, 164, 0, 0, 0, 0, 0, 0, 682, 247, 1026, 1022, 0, 1567, 867, 1271, 0, 0, 0, 0, 0, 0, 0, 0, 969, 1308, 236, 411, 225, 396, 1121, 965, 456, 0, 0, 0, 0, 0, 0, 0, 272, 801, 1054, 605, 384, 503, 793, 336, 1017, 192, 42, 0, 0, 0, 0, 0, 396, 986, 982, 659, 0, 1423, 867, 545, 0, 0, 0, 0, 0, 0, 0, 0, 511, 342, 469, 260, 1570, 1128, 206, 1728, 257, 1, 0, 0, 0, 0, 0, 0, 1154, 321, 606, 826, 135, 301, 319, 795, 1627, 0, 0, 0, 0, 0, 0, 0, 1192, 328, 91, 825, 193, 1202, 618, 1550, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5971, 6682, 6087, 5999, 5858, 6472, 6084, 5999] avg=6144.0000 max=6682 max_rank=1 min=5858 min_rank=4 imbalance_ratio=1.087565 max_load_reduction=42470 max_load_reduction_pct=0.864054 moved_to_echo=24413 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[202, 851, 584, 461, 61, 1419, 983, 253, 1450, 158, 0, 0, 0, 0, 0, 0, 860, 306, 1053, 1074, 0, 1161, 380, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1295, 302, 403, 98, 332, 1049, 1019, 376, 0, 0, 0, 0, 0, 0, 0, 450, 629, 911, 684, 367, 516, 965, 317, 1063, 208, 34, 0, 0, 0, 0, 0, 383, 1125, 1177, 680, 0, 1728, 906, 527, 0, 0, 0, 0, 0, 0, 0, 0, 473, 231, 309, 259, 1589, 1203, 173, 1378, 302, 1, 0, 0, 0, 0, 0, 0, 1193, 253, 927, 847, 209, 393, 287, 818, 1567, 0, 0, 0, 0, 0, 0, 0, 1376, 402, 78, 422, 180, 1135, 423, 1824, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6422, 5885, 5923, 6144, 6526, 5918, 6494, 5840] avg=6144.0000 max=6526 max_rank=4 min=5840 min_rank=7 imbalance_ratio=1.062174 max_load_reduction=42626 max_load_reduction_pct=0.867228 moved_to_echo=24778 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[182, 816, 547, 338, 88, 1101, 888, 233, 1441, 177, 0, 0, 0, 0, 0, 0, 717, 352, 1084, 1198, 0, 1219, 594, 1151, 0, 0, 0, 0, 0, 0, 0, 0, 951, 1329, 287, 394, 96, 382, 1252, 1041, 413, 0, 0, 0, 0, 0, 0, 0, 329, 681, 944, 600, 355, 539, 962, 309, 1038, 206, 38, 0, 0, 0, 0, 0, 334, 951, 1124, 707, 0, 1712, 885, 565, 0, 0, 0, 0, 0, 0, 0, 0, 403, 312, 468, 270, 1607, 1129, 174, 1389, 300, 1, 0, 0, 0, 0, 0, 0, 999, 239, 890, 892, 95, 274, 308, 783, 1757, 0, 0, 0, 0, 0, 0, 0, 1404, 463, 57, 601, 170, 1385, 427, 1805, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5811, 6315, 6145, 6001, 6278, 6053, 6237, 6312] avg=6144.0000 max=6315 max_rank=1 min=5811 min_rank=0 imbalance_ratio=1.027832 max_load_reduction=42837 max_load_reduction_pct=0.871521 moved_to_echo=24880 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[245, 949, 465, 478, 135, 1193, 976, 237, 1505, 169, 0, 0, 0, 0, 0, 0, 849, 357, 1063, 1146, 0, 1343, 715, 1119, 0, 0, 0, 0, 0, 0, 0, 0, 910, 1229, 267, 472, 72, 385, 1213, 854, 401, 0, 0, 0, 0, 0, 0, 0, 389, 693, 807, 547, 386, 600, 942, 377, 1037, 215, 37, 0, 0, 0, 0, 0, 371, 1044, 1099, 742, 0, 1667, 884, 527, 0, 0, 0, 0, 0, 0, 0, 0, 489, 259, 321, 295, 1609, 1176, 151, 1341, 274, 1, 0, 0, 0, 0, 0, 0, 925, 289, 890, 823, 109, 339, 288, 724, 1682, 0, 0, 0, 0, 0, 0, 0, 1397, 419, 55, 650, 135, 1308, 440, 1652, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6352, 6592, 5803, 6030, 6334, 5916, 6069, 6056] avg=6144.0000 max=6592 max_rank=1 min=5803 min_rank=2 imbalance_ratio=1.072917 max_load_reduction=42560 max_load_reduction_pct=0.865885 moved_to_echo=24375 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[101, 1002, 503, 344, 81, 1175, 1011, 199, 1317, 204, 0, 0, 0, 0, 0, 0, 736, 332, 1207, 1230, 0, 1121, 254, 980, 0, 0, 0, 0, 0, 0, 0, 0, 954, 1608, 306, 458, 223, 278, 1296, 1016, 351, 0, 0, 0, 0, 0, 0, 0, 344, 733, 1187, 784, 416, 476, 955, 196, 905, 188, 32, 0, 0, 0, 0, 0, 372, 971, 1005, 538, 0, 1561, 772, 501, 0, 0, 0, 0, 0, 0, 0, 0, 575, 244, 363, 266, 1525, 1423, 208, 1534, 320, 1, 0, 0, 0, 0, 0, 0, 1045, 209, 712, 820, 93, 144, 280, 706, 2015, 0, 0, 0, 0, 0, 0, 0, 1248, 372, 80, 658, 156, 1692, 311, 1929, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5937, 5860, 6490, 6216, 5720, 6459, 6024, 6446] avg=6144.0000 max=6490 max_rank=2 min=5720 min_rank=4 imbalance_ratio=1.056315 max_load_reduction=42662 max_load_reduction_pct=0.867961 moved_to_echo=24649 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[80, 1122, 437, 422, 33, 1260, 1033, 228, 1162, 222, 0, 0, 0, 0, 0, 0, 641, 311, 1314, 1373, 0, 1031, 243, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1361, 586, 341, 156, 239, 1234, 1115, 353, 0, 0, 0, 0, 0, 0, 0, 483, 686, 1189, 855, 398, 499, 1011, 195, 868, 166, 32, 0, 0, 0, 0, 0, 403, 1063, 1100, 425, 0, 1573, 739, 523, 0, 0, 0, 0, 0, 0, 0, 0, 687, 234, 360, 406, 1435, 1297, 165, 1468, 311, 1, 0, 0, 0, 0, 0, 0, 1097, 167, 492, 822, 87, 165, 264, 650, 2203, 0, 0, 0, 0, 0, 0, 0, 1334, 306, 124, 614, 120, 1642, 286, 1872, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5999, 5899, 6437, 6382, 5826, 6364, 5947, 6298] avg=6144.0000 max=6437 max_rank=2 min=5826 min_rank=4 imbalance_ratio=1.047689 max_load_reduction=42715 max_load_reduction_pct=0.869039 moved_to_echo=24435 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[298, 1004, 436, 436, 96, 1138, 862, 261, 1492, 155, 0, 0, 0, 0, 0, 0, 822, 362, 1081, 1241, 0, 1490, 785, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 951, 1286, 281, 466, 82, 357, 1334, 972, 376, 0, 0, 0, 0, 0, 0, 0, 306, 666, 849, 434, 499, 525, 863, 350, 978, 213, 34, 0, 0, 0, 0, 0, 344, 977, 1059, 710, 0, 1587, 833, 558, 0, 0, 0, 0, 0, 0, 0, 0, 528, 273, 338, 252, 1617, 1150, 188, 1360, 271, 1, 0, 0, 0, 0, 0, 0, 1006, 321, 916, 737, 143, 338, 335, 864, 1539, 0, 0, 0, 0, 0, 0, 0, 1364, 360, 72, 604, 191, 1344, 505, 1636, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6178, 6831, 6105, 5717, 6068, 5978, 6199, 6076] avg=6144.0000 max=6831 max_rank=1 min=5717 min_rank=3 imbalance_ratio=1.111816 max_load_reduction=42321 max_load_reduction_pct=0.861023 moved_to_echo=24321 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[92, 1065, 611, 387, 35, 1408, 847, 249, 1562, 226, 0, 0, 0, 0, 0, 0, 659, 392, 1147, 1227, 0, 835, 126, 702, 0, 0, 0, 0, 0, 0, 0, 0, 1109, 1475, 472, 381, 124, 203, 1088, 1050, 260, 0, 0, 0, 0, 0, 0, 0, 631, 782, 1173, 1052, 337, 479, 884, 203, 872, 223, 27, 0, 0, 0, 0, 0, 392, 1159, 1301, 497, 0, 1991, 736, 466, 0, 0, 0, 0, 0, 0, 0, 0, 647, 222, 319, 333, 1449, 1280, 202, 1218, 319, 3, 0, 0, 0, 0, 0, 0, 974, 125, 779, 733, 103, 186, 271, 692, 2235, 0, 0, 0, 0, 0, 0, 0, 1499, 423, 79, 396, 144, 1528, 161, 1895, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6482, 5088, 6162, 6663, 6542, 5992, 6098, 6125] avg=6144.0000 max=6663 max_rank=3 min=5088 min_rank=1 imbalance_ratio=1.084473 max_load_reduction=42489 max_load_reduction_pct=0.864441 moved_to_echo=24757 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[8448, 2048, 6272, 6528, 1024, 2176, 2432, 6144, 14720, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[3072, 8320, 8960, 4992, 0, 13312, 6656, 4224, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[4352, 2176, 3072, 2432, 12416, 9856, 1536, 11520, 2432, 128, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[6016, 2688, 9088, 9600, 0, 9856, 3968, 8320, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3328, 5760, 8192, 5632, 3200, 4224, 7424, 2304, 7808, 1664, 384, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1536, 7808, 4096, 3328, 640, 9856, 7808, 1920, 11392, 1536, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[8064, 11008, 2816, 3328, 1152, 2688, 9600, 8064, 3072, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[10880, 3200, 640, 4864, 1408, 11264, 3200, 14208, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[456, 858, 1414, 437, 673, 547, 739, 682, 1583, 2739, 328, 416, 1122, 643, 1020, 363, 1589, 69, 1925, 546, 639, 269, 20, 656, 1202, 836, 623, 451, 605, 827, 1388, 951, 262, 664, 328, 339, 734, 702, 1440, 668, 554, 2642, 163, 473, 233, 595, 497, 2950, 93, 295, 339, 943, 437, 902, 427, 525, 342, 397, 1317, 600, 791, 918, 558, 408] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[527, 800, 1477, 369, 600, 563, 818, 745, 1701, 2581, 280, 363, 1206, 640, 1099, 318, 1327, 72, 2236, 497, 570, 409, 11, 762, 1179, 1069, 676, 489, 540, 821, 1273, 845, 252, 625, 241, 327, 1038, 554, 1628, 752, 376, 2473, 162, 568, 355, 496, 498, 3272, 90, 224, 209, 1044, 426, 959, 300, 425, 272, 276, 1117, 656, 1025, 753, 574, 322] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[481, 812, 1368, 405, 516, 557, 769, 649, 1422, 2582, 248, 428, 1246, 555, 978, 392, 1785, 71, 2425, 508, 547, 347, 15, 727, 964, 903, 707, 522, 588, 814, 1233, 909, 221, 728, 251, 305, 844, 605, 1700, 613, 437, 2328, 139, 583, 348, 561, 473, 3161, 110, 310, 216, 909, 545, 1034, 329, 537, 342, 414, 1321, 675, 887, 667, 583, 503] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[257, 689, 1625, 461, 640, 560, 764, 671, 1700, 2935, 256, 440, 1063, 501, 1298, 415, 1965, 100, 2005, 459, 674, 318, 17, 697, 1102, 822, 560, 515, 672, 833, 1604, 917, 272, 760, 271, 329, 901, 470, 1736, 791, 272, 2101, 154, 401, 249, 596, 374, 2748, 41, 271, 206, 956, 663, 936, 299, 327, 466, 404, 1324, 509, 969, 898, 496, 427] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[318, 605, 1663, 352, 575, 584, 749, 924, 1626, 2901, 263, 413, 1143, 443, 1119, 300, 1664, 87, 2257, 451, 721, 429, 4, 677, 1140, 894, 632, 486, 639, 941, 1286, 770, 296, 697, 197, 326, 861, 652, 1829, 791, 245, 2491, 143, 473, 337, 499, 366, 3134, 26, 215, 204, 1148, 526, 921, 192, 288, 261, 251, 1299, 579, 1031, 856, 519, 443] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[466, 937, 1286, 399, 562, 474, 782, 710, 1359, 2376, 201, 384, 1159, 600, 1082, 391, 1484, 102, 2149, 529, 536, 328, 28, 786, 1160, 713, 682, 509, 539, 860, 1325, 1011, 216, 768, 294, 355, 728, 594, 1857, 642, 421, 2332, 165, 550, 355, 560, 514, 3168, 240, 355, 259, 958, 671, 1080, 430, 614, 275, 310, 1350, 629, 918, 719, 596, 320] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[279, 466, 1649, 358, 739, 790, 727, 909, 2250, 3074, 398, 390, 1115, 601, 1348, 143, 2043, 62, 1918, 493, 679, 459, 6, 617, 980, 990, 648, 447, 610, 939, 1406, 716, 319, 613, 240, 297, 751, 556, 1252, 637, 224, 2807, 170, 480, 238, 560, 365, 2866, 10, 243, 193, 1029, 364, 972, 213, 259, 322, 220, 1408, 570, 844, 915, 461, 505] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[523, 1185, 1291, 652, 542, 572, 734, 685, 1159, 2127, 202, 416, 1083, 653, 759, 390, 1456, 75, 2175, 470, 544, 287, 18, 786, 1031, 655, 543, 504, 674, 623, 1520, 1049, 253, 724, 324, 401, 730, 568, 2074, 683, 500, 1948, 150, 460, 388, 840, 681, 3024, 220, 292, 389, 960, 572, 1040, 350, 578, 311, 542, 1386, 529, 953, 821, 675, 403] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[456, 858, 1414, 437, 673, 547, 739, 682, 344, 0, 0, 0, 0, 0, 0, 0, 1583, 738, 328, 416, 1122, 643, 1020, 363, 0, 0, 0, 0, 0, 0, 0, 0, 1589, 69, 1925, 546, 639, 269, 20, 656, 72, 0, 0, 0, 0, 0, 0, 0, 1202, 836, 623, 451, 605, 827, 734, 951, 0, 0, 0, 0, 0, 0, 0, 0, 262, 664, 328, 339, 734, 702, 1440, 668, 530, 238, 0, 0, 0, 0, 0, 0, 554, 2642, 163, 473, 233, 595, 497, 1310, 0, 0, 0, 0, 0, 0, 0, 0, 93, 295, 339, 943, 437, 902, 427, 525, 2001, 295, 0, 0, 0, 0, 0, 0, 342, 397, 1317, 600, 791, 918, 558, 408, 815, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6150, 6213, 5785, 6229, 5905, 6467, 6257, 6146] avg=6144.0000 max=6467 max_rank=5 min=5785 min_rank=2 imbalance_ratio=1.052572 max_load_reduction=42685 max_load_reduction_pct=0.868429 moved_to_echo=24775 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[481, 812, 1368, 405, 516, 557, 769, 649, 305, 0, 0, 0, 0, 0, 0, 0, 1422, 696, 248, 428, 1246, 555, 978, 392, 0, 0, 0, 0, 0, 0, 0, 0, 1785, 71, 2425, 508, 547, 347, 15, 727, 64, 0, 0, 0, 0, 0, 0, 0, 964, 903, 707, 522, 588, 814, 652, 909, 0, 0, 0, 0, 0, 0, 0, 0, 221, 728, 251, 305, 844, 605, 1700, 613, 568, 212, 0, 0, 0, 0, 0, 0, 437, 2328, 139, 583, 348, 561, 473, 1404, 0, 0, 0, 0, 0, 0, 0, 0, 110, 310, 216, 909, 545, 1034, 329, 537, 1886, 316, 0, 0, 0, 0, 0, 0, 342, 414, 1321, 675, 887, 667, 583, 503, 873, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5862, 5965, 6489, 6059, 6047, 6273, 6192, 6265] avg=6144.0000 max=6489 max_rank=2 min=5862 min_rank=0 imbalance_ratio=1.056152 max_load_reduction=42663 max_load_reduction_pct=0.867981 moved_to_echo=24777 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[523, 1185, 1291, 652, 542, 572, 734, 685, 376, 0, 0, 0, 0, 0, 0, 0, 1159, 573, 202, 416, 1083, 653, 759, 390, 0, 0, 0, 0, 0, 0, 0, 0, 1456, 75, 2175, 470, 544, 287, 18, 786, 79, 0, 0, 0, 0, 0, 0, 0, 1031, 655, 543, 504, 674, 623, 804, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 253, 724, 324, 401, 730, 568, 2074, 683, 544, 261, 0, 0, 0, 0, 0, 0, 500, 1948, 150, 460, 388, 840, 681, 1343, 0, 0, 0, 0, 0, 0, 0, 0, 220, 292, 389, 960, 572, 1040, 350, 578, 1554, 302, 0, 0, 0, 0, 0, 0, 311, 542, 1386, 529, 953, 821, 675, 403, 835, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6560, 5235, 5890, 5883, 6562, 6310, 6257, 6455] avg=6144.0000 max=6562 max_rank=4 min=5235 min_rank=1 imbalance_ratio=1.068034 max_load_reduction=42590 max_load_reduction_pct=0.866496 moved_to_echo=25584 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[318, 605, 1663, 352, 575, 584, 749, 924, 318, 0, 0, 0, 0, 0, 0, 0, 1626, 781, 263, 413, 1143, 443, 1119, 300, 0, 0, 0, 0, 0, 0, 0, 0, 1664, 87, 2257, 451, 721, 429, 4, 677, 67, 0, 0, 0, 0, 0, 0, 0, 1140, 894, 632, 486, 639, 941, 680, 770, 0, 0, 0, 0, 0, 0, 0, 0, 296, 697, 197, 326, 861, 652, 1829, 791, 563, 221, 0, 0, 0, 0, 0, 0, 245, 2491, 143, 473, 337, 499, 366, 1392, 0, 0, 0, 0, 0, 0, 0, 0, 26, 215, 204, 1148, 526, 921, 192, 288, 2120, 314, 0, 0, 0, 0, 0, 0, 261, 251, 1299, 579, 1031, 856, 519, 443, 865, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6088, 6088, 6357, 6182, 6433, 5946, 5954, 6104] avg=6144.0000 max=6433 max_rank=4 min=5946 min_rank=5 imbalance_ratio=1.047038 max_load_reduction=42719 max_load_reduction_pct=0.869120 moved_to_echo=24437 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[257, 689, 1625, 461, 640, 560, 764, 671, 397, 0, 0, 0, 0, 0, 0, 0, 1700, 791, 256, 440, 1063, 501, 1298, 415, 0, 0, 0, 0, 0, 0, 0, 0, 1965, 100, 2005, 459, 674, 318, 17, 697, 83, 0, 0, 0, 0, 0, 0, 0, 1102, 822, 560, 515, 672, 833, 848, 917, 0, 0, 0, 0, 0, 0, 0, 0, 272, 760, 271, 329, 901, 470, 1736, 791, 494, 276, 0, 0, 0, 0, 0, 0, 272, 2101, 154, 401, 249, 596, 374, 1220, 0, 0, 0, 0, 0, 0, 0, 0, 41, 271, 206, 956, 663, 936, 299, 327, 2144, 275, 0, 0, 0, 0, 0, 0, 466, 404, 1324, 509, 969, 898, 496, 427, 759, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6064, 6464, 6318, 6269, 6300, 5367, 6118, 6252] avg=6144.0000 max=6464 max_rank=1 min=5367 min_rank=5 imbalance_ratio=1.052083 max_load_reduction=42688 max_load_reduction_pct=0.868490 moved_to_echo=24037 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[527, 800, 1477, 369, 600, 563, 818, 745, 315, 0, 0, 0, 0, 0, 0, 0, 1701, 695, 280, 363, 1206, 640, 1099, 318, 0, 0, 0, 0, 0, 0, 0, 0, 1327, 72, 2236, 497, 570, 409, 11, 762, 66, 0, 0, 0, 0, 0, 0, 0, 1179, 1069, 676, 489, 540, 821, 673, 845, 0, 0, 0, 0, 0, 0, 0, 0, 252, 625, 241, 327, 1038, 554, 1628, 752, 588, 219, 0, 0, 0, 0, 0, 0, 376, 2473, 162, 568, 355, 496, 498, 1454, 0, 0, 0, 0, 0, 0, 0, 0, 90, 224, 209, 1044, 426, 959, 300, 425, 1886, 327, 0, 0, 0, 0, 0, 0, 272, 276, 1117, 656, 1025, 753, 574, 322, 903, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6214, 6302, 5950, 6292, 6224, 6382, 5890, 5898] avg=6144.0000 max=6382 max_rank=5 min=5890 min_rank=6 imbalance_ratio=1.038737 max_load_reduction=42770 max_load_reduction_pct=0.870158 moved_to_echo=24394 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[466, 937, 1286, 399, 562, 474, 782, 710, 328, 0, 0, 0, 0, 0, 0, 0, 1359, 640, 201, 384, 1159, 600, 1082, 391, 0, 0, 0, 0, 0, 0, 0, 0, 1484, 102, 2149, 529, 536, 328, 28, 786, 69, 0, 0, 0, 0, 0, 0, 0, 1160, 713, 682, 509, 539, 860, 700, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 216, 768, 294, 355, 728, 594, 1857, 642, 570, 228, 0, 0, 0, 0, 0, 0, 421, 2332, 165, 550, 355, 560, 514, 1406, 0, 0, 0, 0, 0, 0, 0, 0, 240, 355, 259, 958, 671, 1080, 430, 614, 1736, 317, 0, 0, 0, 0, 0, 0, 275, 310, 1350, 629, 918, 719, 596, 320, 875, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5944, 5816, 6011, 6174, 6252, 6303, 6660, 5992] avg=6144.0000 max=6660 max_rank=6 min=5816 min_rank=1 imbalance_ratio=1.083984 max_load_reduction=42492 max_load_reduction_pct=0.864502 moved_to_echo=25207 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[279, 466, 1649, 358, 739, 790, 727, 909, 353, 0, 0, 0, 0, 0, 0, 0, 2250, 824, 398, 390, 1115, 601, 1348, 143, 0, 0, 0, 0, 0, 0, 0, 0, 2043, 62, 1918, 493, 679, 459, 6, 617, 75, 0, 0, 0, 0, 0, 0, 0, 980, 990, 648, 447, 610, 939, 734, 716, 0, 0, 0, 0, 0, 0, 0, 0, 319, 613, 240, 297, 751, 556, 1252, 637, 520, 244, 0, 0, 0, 0, 0, 0, 224, 2807, 170, 480, 238, 560, 365, 1260, 0, 0, 0, 0, 0, 0, 0, 0, 10, 243, 193, 1029, 364, 972, 213, 259, 2250, 291, 0, 0, 0, 0, 0, 0, 322, 220, 1408, 570, 844, 915, 461, 505, 795, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6270, 7069, 6352, 6064, 5429, 6104, 5824, 6040] avg=6144.0000 max=7069 max_rank=1 min=5429 min_rank=4 imbalance_ratio=1.150553 max_load_reduction=42083 max_load_reduction_pct=0.856181 moved_to_echo=23397 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[13440, 640, 17152, 3968, 4992, 2944, 128, 5760, 640, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3072, 19200, 1280, 4096, 2560, 4736, 3840, 10880, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[896, 2304, 2048, 8064, 4224, 7936, 2560, 3584, 15616, 2560, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2176, 5632, 2176, 2688, 6656, 4736, 13568, 5632, 4480, 1920, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[8832, 6912, 5120, 3968, 4992, 6784, 5888, 7168, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[3328, 6400, 11776, 3456, 4864, 4736, 6144, 6016, 2816, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2688, 2816, 10624, 4864, 7424, 6656, 4480, 3456, 6784, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[12800, 5760, 2176, 3328, 9216, 4736, 8704, 2816, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 1384, 1490, 965, 1176, 1200, 667, 1424, 447, 648, 2326, 124, 156, 396, 342, 926, 1565, 39, 190, 88, 169, 531, 278, 249, 600, 374, 2501, 316, 2316, 763, 585, 65, 290, 3, 16, 1382, 877, 278, 90, 1258, 286, 317, 139, 959, 671, 0, 2244, 0, 107, 959, 256, 1539, 414, 1235, 1232, 1238, 706, 583, 179, 1560, 1063, 2155, 1621, 1195] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[2, 1764, 1648, 736, 1173, 1117, 780, 1004, 477, 787, 1979, 110, 185, 417, 370, 1105, 1499, 148, 147, 89, 189, 389, 223, 217, 564, 436, 1890, 504, 1593, 1115, 826, 106, 371, 13, 46, 930, 1093, 537, 197, 1315, 336, 619, 335, 904, 744, 2, 2011, 0, 329, 654, 257, 1456, 512, 1176, 901, 1315, 756, 588, 198, 1210, 1269, 2572, 1533, 1384] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[3, 1536, 1707, 794, 1168, 1016, 733, 1307, 335, 847, 2161, 73, 143, 519, 382, 869, 1408, 169, 215, 83, 191, 451, 289, 307, 699, 404, 1656, 523, 1676, 995, 627, 80, 497, 21, 41, 1131, 1094, 669, 163, 1423, 312, 620, 400, 964, 620, 4, 2002, 15, 197, 687, 251, 1225, 456, 1268, 890, 1127, 919, 543, 233, 1078, 1077, 2554, 1535, 1770] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[3, 1819, 1640, 724, 1208, 1084, 744, 1186, 279, 790, 2240, 152, 168, 421, 352, 936, 1323, 120, 176, 76, 138, 416, 235, 362, 766, 418, 1967, 490, 1942, 775, 741, 98, 315, 11, 40, 1185, 1111, 335, 118, 1406, 389, 419, 250, 936, 708, 1, 2292, 4, 214, 761, 279, 1262, 509, 1299, 874, 1280, 946, 458, 206, 1230, 1203, 2509, 1563, 1250] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[20, 1888, 1488, 544, 1193, 1648, 689, 1136, 526, 718, 2105, 130, 162, 536, 397, 850, 1545, 47, 185, 139, 259, 428, 285, 224, 447, 335, 1935, 487, 2008, 1054, 707, 91, 382, 40, 32, 1367, 785, 557, 163, 1332, 374, 749, 455, 1032, 606, 26, 1954, 43, 230, 630, 261, 1242, 440, 1297, 1052, 1061, 694, 542, 224, 1210, 1140, 2301, 1365, 1360] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[31, 2025, 1578, 577, 1148, 1267, 923, 863, 516, 514, 1683, 245, 298, 388, 775, 1348, 1332, 92, 205, 219, 212, 317, 218, 290, 613, 330, 1383, 707, 1293, 1255, 864, 172, 463, 97, 143, 847, 1408, 928, 307, 1141, 323, 316, 784, 864, 900, 26, 1666, 47, 573, 353, 305, 1665, 512, 888, 697, 1131, 978, 463, 306, 889, 1105, 2490, 1426, 1430] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 648, 1435, 1153, 1046, 1166, 647, 1664, 487, 798, 2839, 74, 136, 540, 323, 723, 1429, 36, 156, 39, 149, 545, 393, 201, 388, 360, 2300, 244, 2712, 672, 428, 92, 333, 2, 2, 1957, 758, 227, 51, 1294, 261, 394, 18, 833, 517, 0, 2824, 0, 51, 926, 201, 1603, 465, 1674, 1440, 1123, 525, 514, 148, 1557, 1008, 1977, 1184, 1462] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[7, 1675, 1601, 659, 1210, 1103, 880, 694, 398, 679, 1837, 148, 188, 406, 369, 1171, 1367, 99, 196, 115, 191, 369, 228, 371, 684, 412, 1540, 678, 1488, 1228, 706, 92, 487, 43, 58, 716, 1298, 587, 188, 1397, 435, 699, 497, 1053, 790, 15, 1794, 13, 451, 531, 297, 1797, 513, 1059, 843, 1276, 956, 570, 292, 1110, 1171, 2608, 1457, 1362] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 1177, 794, 1168, 1016, 733, 1307, 0, 0, 0, 0, 0, 0, 0, 0, 335, 847, 2161, 73, 143, 519, 382, 869, 373, 242, 0, 0, 0, 0, 0, 0, 1408, 169, 215, 83, 191, 451, 289, 307, 2554, 739, 0, 0, 0, 0, 0, 0, 699, 404, 897, 523, 1676, 995, 627, 80, 0, 0, 0, 0, 0, 0, 0, 0, 497, 21, 41, 1131, 1094, 669, 163, 1423, 797, 535, 0, 0, 0, 0, 0, 0, 312, 620, 400, 964, 620, 4, 2002, 15, 676, 288, 224, 0, 0, 0, 0, 0, 197, 687, 251, 852, 456, 1268, 890, 1127, 0, 0, 0, 0, 0, 0, 0, 0, 919, 543, 233, 1078, 1077, 0, 859, 1770, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6198, 5944, 6406, 5901, 6371, 6125, 5728, 6479] avg=6144.0000 max=6479 max_rank=7 min=5728 min_rank=6 imbalance_ratio=1.054525 max_load_reduction=42673 max_load_reduction_pct=0.868184 moved_to_echo=24703 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[31, 0, 1088, 577, 1148, 1267, 923, 863, 0, 0, 0, 0, 0, 0, 0, 0, 516, 514, 1683, 245, 298, 388, 775, 1348, 507, 224, 0, 0, 0, 0, 0, 0, 1332, 92, 205, 219, 212, 317, 218, 290, 2490, 975, 0, 0, 0, 0, 0, 0, 613, 330, 750, 707, 1293, 1255, 864, 172, 0, 0, 0, 0, 0, 0, 0, 0, 463, 97, 143, 847, 1408, 928, 307, 1141, 1050, 446, 0, 0, 0, 0, 0, 0, 323, 316, 784, 864, 900, 26, 1666, 47, 628, 266, 187, 0, 0, 0, 0, 0, 573, 353, 305, 1158, 512, 888, 697, 1131, 0, 0, 0, 0, 0, 0, 0, 0, 978, 463, 306, 889, 1105, 0, 798, 1430, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5897, 6498, 6350, 5984, 6830, 6007, 5617, 5969] avg=6144.0000 max=6830 max_rank=4 min=5617 min_rank=6 imbalance_ratio=1.111654 max_load_reduction=42322 max_load_reduction_pct=0.861043 moved_to_echo=24423 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 1130, 724, 1208, 1084, 744, 1186, 0, 0, 0, 0, 0, 0, 0, 0, 279, 790, 2240, 152, 168, 421, 352, 936, 384, 233, 0, 0, 0, 0, 0, 0, 1323, 120, 176, 76, 138, 416, 235, 362, 2509, 876, 0, 0, 0, 0, 0, 0, 766, 418, 1065, 490, 1942, 775, 741, 98, 0, 0, 0, 0, 0, 0, 0, 0, 315, 11, 40, 1185, 1111, 335, 118, 1406, 943, 635, 0, 0, 0, 0, 0, 0, 389, 419, 250, 936, 708, 1, 2292, 4, 689, 277, 267, 0, 0, 0, 0, 0, 214, 761, 279, 878, 509, 1299, 874, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 946, 458, 206, 1230, 1203, 0, 874, 1250, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6079, 5955, 6231, 6295, 6099, 6232, 6094, 6167] avg=6144.0000 max=6295 max_rank=3 min=5955 min_rank=1 imbalance_ratio=1.024577 max_load_reduction=42857 max_load_reduction_pct=0.871928 moved_to_echo=24592 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 1136, 736, 1173, 1117, 780, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 477, 787, 1979, 110, 185, 417, 370, 1105, 443, 234, 0, 0, 0, 0, 0, 0, 1499, 148, 147, 89, 189, 389, 223, 217, 2572, 850, 0, 0, 0, 0, 0, 0, 564, 436, 1024, 504, 1593, 1115, 826, 106, 0, 0, 0, 0, 0, 0, 0, 0, 371, 13, 46, 930, 1093, 537, 197, 1315, 914, 610, 0, 0, 0, 0, 0, 0, 336, 619, 335, 904, 744, 2, 2011, 0, 675, 278, 256, 0, 0, 0, 0, 0, 329, 654, 257, 1013, 512, 1176, 901, 1315, 0, 0, 0, 0, 0, 0, 0, 0, 756, 588, 198, 1210, 1269, 0, 858, 1384, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5948, 6107, 6323, 6168, 6026, 6160, 6157, 6263] avg=6144.0000 max=6323 max_rank=2 min=5948 min_rank=0 imbalance_ratio=1.029134 max_load_reduction=42829 max_load_reduction_pct=0.871358 moved_to_echo=24606 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1028, 965, 1176, 1200, 667, 1424, 0, 0, 0, 0, 0, 0, 0, 0, 447, 648, 2326, 124, 156, 396, 342, 926, 469, 211, 0, 0, 0, 0, 0, 0, 1565, 39, 190, 88, 169, 531, 278, 249, 2155, 667, 0, 0, 0, 0, 0, 0, 600, 374, 1354, 316, 2316, 763, 585, 65, 0, 0, 0, 0, 0, 0, 0, 0, 290, 3, 16, 1382, 877, 278, 90, 1258, 717, 808, 0, 0, 0, 0, 0, 0, 286, 317, 139, 959, 671, 0, 2244, 0, 714, 251, 339, 0, 0, 0, 0, 0, 107, 959, 256, 1070, 414, 1235, 1232, 1238, 0, 0, 0, 0, 0, 0, 0, 0, 706, 583, 179, 1560, 1063, 0, 907, 1195, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6460, 6045, 5931, 6373, 5719, 5920, 6511, 6193] avg=6144.0000 max=6511 max_rank=6 min=5719 min_rank=4 imbalance_ratio=1.059733 max_load_reduction=42641 max_load_reduction_pct=0.867533 moved_to_echo=24343 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[20, 0, 1026, 544, 1193, 1648, 689, 1136, 0, 0, 0, 0, 0, 0, 0, 0, 526, 718, 2105, 130, 162, 536, 397, 850, 378, 211, 0, 0, 0, 0, 0, 0, 1545, 47, 185, 139, 259, 428, 285, 224, 2301, 910, 0, 0, 0, 0, 0, 0, 447, 335, 1048, 487, 2008, 1054, 707, 91, 0, 0, 0, 0, 0, 0, 0, 0, 382, 40, 32, 1367, 785, 557, 163, 1332, 978, 625, 0, 0, 0, 0, 0, 0, 374, 749, 455, 1032, 606, 26, 1954, 43, 601, 251, 262, 0, 0, 0, 0, 0, 230, 630, 261, 864, 440, 1297, 1052, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 694, 542, 224, 1210, 1140, 0, 764, 1360, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6256, 6013, 6323, 6177, 6261, 6353, 5835, 5934] avg=6144.0000 max=6353 max_rank=5 min=5835 min_rank=6 imbalance_ratio=1.034017 max_load_reduction=42799 max_load_reduction_pct=0.870748 moved_to_echo=24383 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 1104, 659, 1210, 1103, 880, 694, 0, 0, 0, 0, 0, 0, 0, 0, 398, 679, 1837, 148, 188, 406, 369, 1171, 547, 227, 0, 0, 0, 0, 0, 0, 1367, 99, 196, 115, 191, 369, 228, 371, 2608, 806, 0, 0, 0, 0, 0, 0, 684, 412, 834, 678, 1488, 1228, 706, 92, 0, 0, 0, 0, 0, 0, 0, 0, 487, 43, 58, 716, 1298, 587, 188, 1397, 869, 497, 0, 0, 0, 0, 0, 0, 435, 699, 497, 1053, 790, 15, 1794, 13, 642, 270, 209, 0, 0, 0, 0, 0, 451, 531, 297, 1250, 513, 1059, 843, 1276, 0, 0, 0, 0, 0, 0, 0, 0, 956, 570, 292, 1110, 1171, 0, 815, 1362, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5657, 5970, 6350, 6122, 6140, 6417, 6220, 6276] avg=6144.0000 max=6417 max_rank=5 min=5657 min_rank=0 imbalance_ratio=1.044434 max_load_reduction=42735 max_load_reduction_pct=0.869446 moved_to_echo=25053 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 981, 1153, 1046, 1166, 647, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 487, 798, 2839, 74, 136, 540, 323, 723, 493, 207, 0, 0, 0, 0, 0, 0, 1429, 36, 156, 39, 149, 545, 393, 201, 1977, 313, 0, 0, 0, 0, 0, 0, 388, 360, 1236, 244, 2712, 672, 428, 92, 0, 0, 0, 0, 0, 0, 0, 0, 333, 2, 2, 1957, 758, 227, 51, 1294, 335, 747, 0, 0, 0, 0, 0, 0, 261, 394, 18, 833, 517, 0, 2824, 0, 527, 247, 317, 0, 0, 0, 0, 0, 51, 926, 201, 1110, 465, 1674, 1440, 1123, 0, 0, 0, 0, 0, 0, 0, 0, 525, 514, 148, 1557, 1008, 0, 657, 1462, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6657, 6620, 5238, 6132, 5706, 5938, 6990, 5871] avg=6144.0000 max=6990 max_rank=6 min=5238 min_rank=2 imbalance_ratio=1.137695 max_load_reduction=42162 max_load_reduction_pct=0.857788 moved_to_echo=24505 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 0, 8704, 6272, 9344, 9728, 6144, 9344, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[6528, 4352, 1792, 9856, 9088, 0, 6656, 11264, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3584, 5888, 17280, 1152, 1536, 3712, 3328, 7936, 3712, 1792, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[2176, 5504, 2176, 8320, 3840, 9984, 7936, 9600, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[11520, 768, 1536, 896, 1536, 3456, 2176, 2304, 19200, 6144, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[2816, 4224, 2944, 7552, 5632, 128, 16896, 128, 5248, 2176, 2176, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3200, 256, 384, 9600, 8448, 4224, 1280, 10624, 6656, 4992, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4864, 3072, 8320, 3968, 15104, 7936, 5504, 896, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[714, 1604, 1016, 274, 174, 171, 2651, 397, 26, 2, 844, 757, 1516, 1, 30, 909, 725, 881, 3186, 58, 223, 571, 273, 14, 219, 1982, 676, 133, 388, 209, 3775, 471, 1754, 637, 207, 1641, 695, 315, 1522, 1183, 79, 4, 131, 1856, 135, 518, 1567, 547, 832, 1077, 1287, 937, 137, 602, 5, 1507, 1192, 484, 489, 746, 156, 348, 864, 828] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[647, 2023, 908, 270, 83, 174, 2643, 528, 9, 0, 735, 775, 1977, 0, 1, 788, 573, 939, 3614, 37, 141, 1193, 311, 0, 177, 2195, 833, 106, 163, 175, 3531, 358, 1932, 454, 154, 1009, 704, 310, 1407, 1004, 84, 0, 24, 1828, 87, 633, 1424, 479, 977, 1248, 1428, 1115, 101, 543, 0, 1299, 1333, 370, 457, 592, 197, 388, 800, 864] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[699, 2022, 1039, 255, 145, 228, 2961, 440, 40, 0, 896, 892, 1293, 0, 7, 861, 731, 896, 2924, 34, 247, 683, 282, 0, 176, 2187, 849, 98, 304, 201, 3692, 332, 1435, 479, 204, 1082, 727, 309, 1564, 1134, 205, 0, 145, 1997, 94, 536, 1396, 465, 855, 1376, 1441, 867, 118, 643, 0, 1252, 1277, 412, 493, 677, 209, 438, 901, 1007] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[747, 1635, 1426, 318, 536, 239, 2571, 336, 55, 3, 915, 884, 1439, 0, 36, 1292, 969, 918, 2909, 72, 338, 393, 267, 5, 213, 1513, 825, 144, 428, 243, 3139, 615, 1472, 472, 240, 1628, 819, 303, 1960, 1039, 338, 1, 102, 2069, 121, 400, 1390, 441, 833, 1299, 849, 638, 104, 690, 0, 1301, 908, 475, 413, 658, 360, 478, 1120, 808] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[530, 1747, 812, 276, 107, 139, 2485, 662, 4, 0, 809, 673, 2368, 0, 0, 703, 524, 832, 3378, 16, 88, 1434, 281, 0, 133, 2366, 944, 132, 176, 182, 3910, 366, 1893, 394, 183, 1096, 618, 233, 1215, 1217, 23, 0, 22, 1253, 77, 846, 1383, 616, 875, 1252, 1616, 1736, 100, 512, 0, 1329, 1231, 451, 422, 516, 132, 251, 755, 828] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[827, 1707, 1183, 260, 244, 255, 2529, 369, 44, 1, 894, 944, 1370, 0, 27, 1174, 787, 992, 3024, 96, 344, 456, 215, 8, 197, 1514, 728, 135, 383, 216, 3058, 393, 1367, 490, 267, 1527, 567, 342, 1791, 1064, 336, 0, 127, 2154, 158, 508, 1536, 449, 994, 1373, 1265, 824, 152, 670, 0, 1507, 1060, 463, 487, 850, 207, 525, 947, 771] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[589, 1799, 1031, 278, 212, 243, 2662, 341, 22, 0, 803, 924, 1487, 0, 6, 925, 811, 881, 3268, 48, 223, 603, 245, 2, 150, 1851, 733, 124, 350, 233, 3633, 409, 1520, 549, 256, 1392, 699, 303, 1758, 1029, 165, 0, 122, 2129, 88, 518, 1602, 465, 950, 1473, 1215, 830, 69, 647, 0, 1289, 1125, 466, 509, 698, 219, 409, 945, 827] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[781, 2043, 958, 336, 189, 243, 2605, 530, 28, 14, 725, 792, 1944, 0, 5, 1283, 717, 859, 3178, 26, 264, 853, 327, 25, 182, 1912, 808, 150, 309, 248, 2941, 441, 1771, 507, 249, 1937, 642, 379, 1750, 943, 122, 43, 51, 1249, 146, 457, 1386, 582, 988, 1171, 1471, 1010, 224, 521, 23, 1575, 1059, 455, 414, 437, 177, 229, 831, 637] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[714, 1604, 1016, 274, 174, 171, 1434, 397, 0, 0, 0, 0, 0, 0, 0, 0, 26, 2, 844, 757, 1516, 1, 30, 909, 1413, 492, 0, 0, 0, 0, 0, 0, 725, 881, 3179, 58, 223, 571, 273, 14, 0, 0, 0, 0, 0, 0, 0, 0, 219, 1982, 676, 133, 388, 209, 2343, 471, 0, 0, 0, 0, 0, 0, 0, 0, 341, 637, 207, 1641, 695, 315, 1522, 1183, 0, 0, 0, 0, 0, 0, 0, 0, 79, 4, 131, 1856, 135, 518, 1567, 547, 940, 571, 0, 0, 0, 0, 0, 0, 832, 1077, 1287, 937, 137, 602, 5, 980, 0, 0, 0, 0, 0, 0, 0, 0, 1192, 484, 489, 746, 156, 348, 864, 828, 646, 527, 7, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5784, 5990, 5924, 6421, 6541, 6348, 5857, 6287] avg=6144.0000 max=6541 max_rank=4 min=5784 min_rank=0 imbalance_ratio=1.064616 max_load_reduction=42611 max_load_reduction_pct=0.866923 moved_to_echo=25033 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[699, 2022, 1039, 255, 145, 228, 1602, 440, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 896, 892, 1293, 0, 7, 861, 1156, 481, 0, 0, 0, 0, 0, 0, 731, 896, 2918, 34, 247, 683, 282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 2187, 849, 98, 304, 201, 2292, 332, 0, 0, 0, 0, 0, 0, 0, 0, 279, 479, 204, 1082, 727, 309, 1564, 1134, 0, 0, 0, 0, 0, 0, 0, 0, 205, 0, 145, 1997, 94, 536, 1396, 465, 919, 637, 0, 0, 0, 0, 0, 0, 855, 1376, 1441, 867, 118, 643, 0, 814, 0, 0, 0, 0, 0, 0, 0, 0, 1277, 412, 493, 677, 209, 438, 901, 1007, 722, 438, 6, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6430, 5626, 5791, 6439, 5778, 6394, 6114, 6580] avg=6144.0000 max=6580 max_rank=7 min=5626 min_rank=1 imbalance_ratio=1.070964 max_load_reduction=42572 max_load_reduction_pct=0.866130 moved_to_echo=24866 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[647, 2023, 908, 270, 83, 174, 1430, 528, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 735, 775, 1977, 0, 1, 788, 1556, 460, 0, 0, 0, 0, 0, 0, 573, 939, 3606, 37, 141, 1193, 311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 177, 2195, 833, 106, 163, 175, 2192, 358, 0, 0, 0, 0, 0, 0, 0, 0, 376, 454, 154, 1009, 704, 310, 1407, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 24, 1828, 87, 633, 1424, 479, 879, 569, 0, 0, 0, 0, 0, 0, 977, 1248, 1428, 1115, 101, 543, 0, 845, 0, 0, 0, 0, 0, 0, 0, 0, 1333, 370, 457, 592, 197, 388, 800, 864, 644, 454, 8, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6063, 6301, 6800, 6199, 5418, 6007, 6257, 6107] avg=6144.0000 max=6800 max_rank=2 min=5418 min_rank=4 imbalance_ratio=1.106771 max_load_reduction=42352 max_load_reduction_pct=0.861654 moved_to_echo=23789 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[589, 1799, 1031, 278, 212, 243, 1440, 341, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 803, 924, 1487, 0, 6, 925, 1224, 473, 0, 0, 0, 0, 0, 0, 811, 881, 3261, 48, 223, 603, 245, 2, 0, 0, 0, 0, 0, 0, 0, 0, 150, 1851, 733, 124, 350, 233, 2255, 409, 0, 0, 0, 0, 0, 0, 0, 0, 296, 549, 256, 1392, 699, 303, 1758, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 165, 0, 122, 2129, 88, 518, 1602, 465, 905, 573, 0, 0, 0, 0, 0, 0, 950, 1473, 1215, 830, 69, 647, 0, 838, 0, 0, 0, 0, 0, 0, 0, 0, 1125, 466, 509, 698, 219, 409, 945, 827, 649, 451, 7, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5933, 5864, 6074, 6105, 6282, 6567, 6022, 6305] avg=6144.0000 max=6567 max_rank=5 min=5864 min_rank=1 imbalance_ratio=1.068848 max_load_reduction=42585 max_load_reduction_pct=0.866394 moved_to_echo=25176 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[781, 2043, 958, 336, 189, 243, 1409, 530, 0, 0, 0, 0, 0, 0, 0, 0, 28, 14, 725, 792, 1944, 0, 5, 1283, 1427, 383, 0, 0, 0, 0, 0, 0, 717, 859, 3171, 26, 264, 853, 327, 25, 0, 0, 0, 0, 0, 0, 0, 0, 182, 1912, 808, 150, 309, 248, 1826, 441, 0, 0, 0, 0, 0, 0, 0, 0, 344, 507, 249, 1937, 642, 379, 1750, 943, 0, 0, 0, 0, 0, 0, 0, 0, 122, 43, 51, 1249, 146, 457, 1386, 582, 732, 561, 0, 0, 0, 0, 0, 0, 988, 1171, 1471, 1010, 224, 521, 23, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 455, 414, 437, 177, 229, 831, 637, 635, 551, 7, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6489, 6601, 6242, 5876, 6751, 5329, 6432, 5432] avg=6144.0000 max=6751 max_rank=4 min=5329 min_rank=5 imbalance_ratio=1.098796 max_load_reduction=42401 max_load_reduction_pct=0.862651 moved_to_echo=23944 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[747, 1635, 1426, 318, 536, 239, 1392, 336, 0, 0, 0, 0, 0, 0, 0, 0, 55, 3, 915, 884, 1439, 0, 36, 1292, 1186, 409, 0, 0, 0, 0, 0, 0, 969, 918, 2903, 72, 338, 393, 267, 5, 0, 0, 0, 0, 0, 0, 0, 0, 213, 1513, 825, 144, 428, 243, 1948, 615, 0, 0, 0, 0, 0, 0, 0, 0, 286, 472, 240, 1628, 819, 303, 1960, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 338, 1, 102, 2069, 121, 400, 1390, 441, 782, 553, 0, 0, 0, 0, 0, 0, 833, 1299, 849, 638, 104, 690, 0, 846, 0, 0, 0, 0, 0, 0, 0, 0, 908, 475, 413, 658, 360, 478, 1120, 808, 626, 455, 6, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6629, 6219, 5865, 5929, 6747, 6197, 5259, 6307] avg=6144.0000 max=6747 max_rank=4 min=5259 min_rank=6 imbalance_ratio=1.098145 max_load_reduction=42405 max_load_reduction_pct=0.862732 moved_to_echo=24510 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[530, 1747, 812, 276, 107, 139, 1337, 662, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 809, 673, 2368, 0, 0, 703, 1528, 513, 0, 0, 0, 0, 0, 0, 524, 832, 3366, 16, 88, 1434, 281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 133, 2366, 944, 132, 176, 182, 2419, 366, 0, 0, 0, 0, 0, 0, 0, 0, 365, 394, 183, 1096, 618, 233, 1215, 1217, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 22, 1253, 77, 846, 1383, 616, 978, 539, 0, 0, 0, 0, 0, 0, 875, 1252, 1616, 1736, 100, 512, 0, 862, 0, 0, 0, 0, 0, 0, 0, 0, 1231, 451, 422, 516, 132, 251, 755, 828, 609, 467, 12, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5610, 6598, 6541, 6718, 5321, 5737, 6953, 5674] avg=6144.0000 max=6953 max_rank=6 min=5321 min_rank=4 imbalance_ratio=1.131673 max_load_reduction=42199 max_load_reduction_pct=0.858541 moved_to_echo=23685 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[827, 1707, 1183, 260, 244, 255, 1369, 369, 0, 0, 0, 0, 0, 0, 0, 0, 44, 1, 894, 944, 1370, 0, 27, 1174, 1101, 398, 0, 0, 0, 0, 0, 0, 787, 992, 3017, 96, 344, 456, 215, 8, 0, 0, 0, 0, 0, 0, 0, 0, 197, 1514, 728, 135, 383, 216, 1899, 393, 0, 0, 0, 0, 0, 0, 0, 0, 266, 490, 267, 1527, 567, 342, 1791, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 336, 0, 127, 2154, 158, 508, 1536, 449, 761, 544, 0, 0, 0, 0, 0, 0, 994, 1373, 1265, 824, 152, 670, 0, 980, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 463, 487, 850, 207, 525, 947, 771, 616, 527, 7, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6214, 5953, 5915, 5465, 6314, 6573, 6258, 6460] avg=6144.0000 max=6573 max_rank=5 min=5465 min_rank=3 imbalance_ratio=1.069824 max_load_reduction=42579 max_load_reduction_pct=0.866272 moved_to_echo=25605 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1408, 128, 768, 14592, 1024, 4480, 11776, 4096, 6912, 4608, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[7424, 10368, 10624, 8064, 1024, 4864, 128, 7296, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5632, 14592, 8448, 2304, 1792, 1792, 11520, 3712, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[9216, 3584, 3712, 5248, 1664, 3072, 7168, 6656, 5248, 3968, 128, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1536, 15616, 6400, 1024, 2560, 1792, 17280, 3456, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[256, 128, 6656, 6656, 13440, 128, 128, 7936, 10624, 3712, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2560, 4096, 1792, 11392, 5504, 2560, 13056, 8704, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5888, 7296, 25472, 512, 1920, 6272, 2304, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[724, 122, 1, 2651, 88, 6, 241, 108, 0, 700, 1733, 977, 1025, 609, 390, 1341, 0, 25, 824, 8, 290, 499, 494, 2739, 116, 66, 1129, 1, 523, 262, 0, 14, 75, 1472, 389, 19, 0, 0, 0, 1865, 1375, 219, 210, 2026, 5116, 361, 1065, 3145, 328, 818, 0, 1349, 360, 741, 5701, 0, 3983, 0, 0, 39, 317, 0, 323, 150] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[718, 116, 0, 3879, 63, 1, 236, 62, 0, 869, 1082, 897, 1033, 669, 495, 926, 0, 0, 788, 1, 42, 482, 414, 2177, 121, 91, 1030, 0, 373, 178, 0, 0, 62, 1699, 251, 8, 0, 0, 0, 2396, 1339, 276, 186, 1747, 5400, 104, 1052, 3558, 449, 183, 0, 1431, 331, 808, 5944, 0, 4429, 0, 0, 25, 297, 0, 319, 115] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[763, 127, 15, 3389, 107, 25, 199, 93, 21, 997, 844, 911, 942, 370, 442, 752, 3, 30, 852, 30, 108, 627, 400, 1406, 123, 91, 757, 0, 347, 208, 3, 13, 103, 1609, 140, 31, 12, 60, 35, 2758, 1520, 235, 252, 2066, 5324, 228, 940, 4020, 326, 822, 24, 1379, 456, 1125, 5730, 90, 4084, 52, 36, 52, 127, 1, 394, 126] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[645, 96, 0, 3947, 98, 0, 177, 82, 0, 628, 988, 721, 1042, 479, 400, 1312, 0, 0, 728, 1, 8, 435, 427, 2026, 105, 61, 656, 0, 205, 157, 0, 0, 82, 1680, 177, 1, 0, 0, 0, 3156, 1139, 239, 176, 1774, 5524, 112, 973, 4257, 382, 459, 0, 1317, 333, 1130, 6110, 0, 4189, 0, 0, 5, 0, 0, 354, 159] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[667, 88, 0, 3906, 78, 0, 223, 70, 0, 768, 921, 802, 1067, 637, 447, 880, 0, 0, 713, 6, 35, 423, 443, 2277, 94, 86, 827, 0, 371, 186, 0, 0, 70, 1700, 216, 8, 0, 0, 0, 2743, 1241, 288, 203, 1771, 5449, 115, 1066, 3709, 418, 320, 0, 1295, 339, 916, 6121, 0, 4523, 0, 0, 8, 181, 0, 308, 129] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[681, 92, 3, 3722, 91, 3, 224, 64, 2, 894, 1219, 786, 1037, 695, 434, 833, 0, 7, 804, 3, 71, 530, 436, 1675, 127, 56, 1206, 3, 401, 119, 6, 10, 71, 1657, 165, 18, 0, 3, 12, 2591, 1250, 246, 193, 1780, 5266, 148, 945, 3712, 421, 238, 7, 1828, 399, 1065, 5848, 6, 4115, 3, 2, 38, 396, 4, 351, 140] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[772, 127, 0, 3397, 80, 14, 234, 62, 0, 745, 1319, 956, 1092, 621, 464, 780, 1, 7, 820, 16, 117, 489, 440, 2627, 116, 83, 1181, 4, 486, 192, 0, 2, 62, 1669, 273, 10, 0, 0, 1, 2091, 1369, 274, 206, 1935, 5230, 260, 933, 3321, 408, 560, 2, 1262, 353, 866, 5778, 0, 4182, 4, 0, 21, 394, 3, 307, 134] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[606, 82, 0, 4370, 89, 0, 180, 69, 0, 1027, 574, 662, 952, 511, 403, 548, 0, 0, 662, 0, 5, 498, 384, 1225, 127, 95, 507, 0, 262, 397, 0, 0, 69, 1632, 136, 1, 0, 0, 0, 3685, 1338, 281, 180, 1645, 5550, 92, 813, 4506, 376, 138, 0, 1673, 375, 1410, 6225, 0, 4169, 0, 0, 3, 2, 0, 469, 149] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[724, 122, 1, 2651, 88, 6, 241, 108, 649, 497, 0, 0, 0, 0, 0, 0, 0, 700, 1733, 977, 1025, 609, 390, 1341, 362, 0, 0, 0, 0, 0, 0, 0, 0, 25, 824, 8, 290, 499, 494, 2739, 1356, 869, 0, 0, 0, 0, 0, 0, 116, 66, 1129, 1, 523, 262, 0, 14, 4247, 0, 0, 0, 0, 0, 0, 0, 75, 1472, 389, 19, 0, 0, 0, 1865, 1464, 0, 0, 0, 0, 0, 0, 0, 1375, 219, 210, 2026, 0, 361, 1065, 1142, 0, 0, 0, 0, 0, 0, 0, 0, 328, 818, 0, 1349, 360, 741, 2384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3983, 0, 0, 39, 317, 0, 323, 150, 992, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5087, 7137, 7104, 6358, 5284, 6398, 5980, 5804] avg=6144.0000 max=7137 max_rank=1 min=5087 min_rank=0 imbalance_ratio=1.161621 max_load_reduction=42015 max_load_reduction_pct=0.854797 moved_to_echo=23466 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[681, 92, 3, 3722, 91, 3, 224, 64, 766, 510, 0, 0, 0, 0, 0, 0, 2, 894, 1219, 786, 1037, 695, 434, 833, 427, 0, 0, 0, 0, 0, 0, 0, 0, 7, 804, 3, 71, 530, 436, 1675, 1391, 894, 0, 0, 0, 0, 0, 0, 127, 56, 1206, 3, 401, 119, 6, 10, 4372, 0, 0, 0, 0, 0, 0, 0, 71, 1657, 165, 18, 0, 3, 12, 2591, 1502, 0, 0, 0, 0, 0, 0, 0, 1250, 246, 193, 1780, 0, 148, 945, 1348, 0, 0, 0, 0, 0, 0, 0, 0, 421, 238, 7, 1828, 399, 1065, 2445, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4115, 3, 2, 38, 396, 4, 351, 140, 1171, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6156, 6327, 5811, 6300, 6019, 5910, 6409, 6220] avg=6144.0000 max=6409 max_rank=6 min=5811 min_rank=2 imbalance_ratio=1.043132 max_load_reduction=42743 max_load_reduction_pct=0.869609 moved_to_echo=24558 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[718, 116, 0, 3879, 63, 1, 236, 62, 734, 518, 0, 0, 0, 0, 0, 0, 0, 869, 1082, 897, 1033, 669, 495, 926, 409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 788, 1, 42, 482, 414, 2177, 1414, 918, 0, 0, 0, 0, 0, 0, 121, 91, 1030, 0, 373, 178, 0, 0, 4482, 0, 0, 0, 0, 0, 0, 0, 62, 1699, 251, 8, 0, 0, 0, 2396, 1526, 0, 0, 0, 0, 0, 0, 0, 1339, 276, 186, 1747, 0, 104, 1052, 1292, 0, 0, 0, 0, 0, 0, 0, 0, 449, 183, 0, 1431, 331, 808, 2486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4429, 0, 0, 25, 297, 0, 319, 115, 1123, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6327, 6380, 6236, 6275, 5942, 5996, 5688, 6308] avg=6144.0000 max=6380 max_rank=1 min=5688 min_rank=6 imbalance_ratio=1.038411 max_load_reduction=42772 max_load_reduction_pct=0.870199 moved_to_echo=23934 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[667, 88, 0, 3906, 78, 0, 223, 70, 765, 533, 0, 0, 0, 0, 0, 0, 0, 768, 921, 802, 1067, 637, 447, 880, 427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 713, 6, 35, 423, 443, 2277, 1456, 925, 0, 0, 0, 0, 0, 0, 94, 86, 827, 0, 371, 186, 0, 0, 4524, 0, 0, 0, 0, 0, 0, 0, 70, 1700, 216, 8, 0, 0, 0, 2743, 1572, 0, 0, 0, 0, 0, 0, 0, 1241, 288, 203, 1771, 0, 115, 1066, 1347, 0, 0, 0, 0, 0, 0, 0, 0, 418, 320, 0, 1295, 339, 916, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4523, 0, 0, 8, 181, 0, 308, 129, 1170, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6330, 5949, 6278, 6088, 6309, 6031, 5848, 6319] avg=6144.0000 max=6330 max_rank=0 min=5848 min_rank=6 imbalance_ratio=1.030273 max_load_reduction=42822 max_load_reduction_pct=0.871216 moved_to_echo=24507 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[645, 96, 0, 3947, 98, 0, 177, 82, 878, 532, 0, 0, 0, 0, 0, 0, 0, 628, 988, 721, 1042, 479, 400, 1312, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 728, 1, 8, 435, 427, 2026, 1453, 939, 0, 0, 0, 0, 0, 0, 105, 61, 656, 0, 205, 157, 0, 0, 4585, 0, 0, 0, 0, 0, 0, 0, 82, 1680, 177, 1, 0, 0, 0, 3156, 1569, 0, 0, 0, 0, 0, 0, 0, 1139, 239, 176, 1774, 0, 112, 973, 1546, 0, 0, 0, 0, 0, 0, 0, 0, 382, 459, 0, 1317, 333, 1130, 2556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4189, 0, 0, 5, 0, 0, 354, 159, 1343, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6455, 6060, 6017, 5769, 6665, 5959, 6177, 6050] avg=6144.0000 max=6665 max_rank=4 min=5769 min_rank=3 imbalance_ratio=1.084798 max_load_reduction=42487 max_load_reduction_pct=0.864400 moved_to_echo=24851 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[763, 127, 15, 3389, 107, 25, 199, 93, 829, 499, 0, 0, 0, 0, 0, 0, 21, 997, 844, 911, 942, 370, 442, 752, 462, 0, 0, 0, 0, 0, 0, 0, 3, 30, 852, 30, 108, 627, 400, 1406, 1363, 905, 0, 0, 0, 0, 0, 0, 123, 91, 757, 0, 347, 208, 3, 13, 4419, 0, 0, 0, 0, 0, 0, 0, 103, 1609, 140, 31, 12, 60, 35, 2758, 1471, 0, 0, 0, 0, 0, 0, 0, 1520, 235, 252, 2066, 0, 228, 940, 1461, 0, 0, 0, 0, 0, 0, 0, 0, 326, 822, 24, 1379, 456, 1125, 2397, 90, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 52, 36, 52, 127, 1, 394, 126, 1268, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6046, 5741, 5724, 5961, 6219, 6702, 6619, 6140] avg=6144.0000 max=6702 max_rank=5 min=5724 min_rank=2 imbalance_ratio=1.090820 max_load_reduction=42450 max_load_reduction_pct=0.863647 moved_to_echo=25680 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[772, 127, 0, 3397, 80, 14, 234, 62, 685, 503, 0, 0, 0, 0, 0, 0, 0, 745, 1319, 956, 1092, 621, 464, 780, 382, 0, 0, 0, 0, 0, 0, 0, 1, 7, 820, 16, 117, 489, 440, 2627, 1374, 888, 0, 0, 0, 0, 0, 0, 116, 83, 1181, 4, 486, 192, 0, 2, 4342, 0, 0, 0, 0, 0, 0, 0, 62, 1669, 273, 10, 0, 0, 1, 2091, 1484, 0, 0, 0, 0, 0, 0, 0, 1369, 274, 206, 1935, 0, 260, 933, 1206, 0, 0, 0, 0, 0, 0, 0, 0, 408, 560, 2, 1262, 353, 866, 2417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4182, 4, 0, 21, 394, 3, 307, 134, 1048, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5874, 6359, 6779, 6406, 5590, 6183, 5868, 6093] avg=6144.0000 max=6779 max_rank=2 min=5590 min_rank=4 imbalance_ratio=1.103353 max_load_reduction=42373 max_load_reduction_pct=0.862081 moved_to_echo=23734 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[606, 82, 0, 4370, 89, 0, 180, 69, 934, 547, 0, 0, 0, 0, 0, 0, 0, 1027, 574, 662, 952, 511, 403, 548, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 662, 0, 5, 498, 384, 1225, 1486, 943, 0, 0, 0, 0, 0, 0, 127, 95, 507, 0, 262, 397, 0, 0, 4607, 0, 0, 0, 0, 0, 0, 0, 69, 1632, 136, 1, 0, 0, 0, 3685, 1601, 0, 0, 0, 0, 0, 0, 0, 1338, 281, 180, 1645, 0, 92, 813, 1624, 0, 0, 0, 0, 0, 0, 0, 0, 376, 138, 0, 1673, 375, 1410, 2591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4169, 0, 0, 3, 2, 0, 469, 149, 1426, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6877, 5199, 5203, 5995, 7124, 5973, 6563, 6218] avg=6144.0000 max=7124 max_rank=4 min=5199 min_rank=1 imbalance_ratio=1.159505 max_load_reduction=42028 max_load_reduction_pct=0.855062 moved_to_echo=25878 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 6656, 8704, 6784, 8192, 4608, 3584, 7424, 3584, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 6272, 128, 768, 4096, 3456, 16256, 11392, 7296, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[640, 13184, 1792, 128, 128, 128, 128, 21376, 12288, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[10624, 2176, 1664, 14848, 0, 1536, 7808, 11008, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3200, 3584, 128, 11648, 3072, 8064, 19840, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5632, 896, 128, 29312, 768, 128, 1792, 640, 6272, 4224, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[33792, 128, 128, 256, 1792, 128, 2944, 1152, 9600, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1024, 640, 7296, 128, 3072, 1792, 128, 128, 35584, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1981, 4002, 0, 299, 952, 190, 67, 121, 0, 347, 205, 269, 104, 326, 583, 644, 411, 705, 0, 8, 677, 88, 2843, 1564, 2071, 3566, 1093, 262, 1, 345, 234, 414, 0, 221, 1926, 0, 91, 1651, 779, 198, 723, 71, 0, 57, 2390, 5574, 922, 0, 5392, 0, 2040, 493, 413, 115, 374, 1, 9, 346, 0, 0, 0, 484, 114, 396] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[2454, 5344, 105, 166, 928, 37, 96, 65, 1, 487, 253, 297, 99, 371, 576, 531, 378, 1711, 95, 76, 277, 88, 1515, 1264, 3130, 3107, 1912, 219, 59, 230, 208, 386, 40, 450, 667, 9, 79, 2154, 1064, 159, 625, 14, 31, 74, 1983, 5521, 511, 63, 4497, 117, 1463, 516, 429, 93, 258, 12, 75, 355, 80, 26, 13, 609, 347, 353] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[2760, 6590, 0, 89, 538, 19, 64, 6, 0, 289, 104, 202, 110, 313, 507, 629, 178, 1415, 0, 0, 184, 87, 1388, 1110, 4107, 3282, 2108, 290, 0, 0, 133, 386, 0, 290, 611, 0, 99, 2454, 633, 138, 695, 4, 0, 60, 1696, 6100, 262, 0, 4598, 0, 961, 754, 477, 98, 190, 0, 4, 391, 0, 0, 0, 1166, 268, 315] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[2313, 5459, 0, 143, 673, 47, 54, 33, 0, 288, 131, 197, 89, 302, 431, 693, 318, 1103, 0, 0, 295, 99, 2335, 1383, 2864, 3704, 1753, 208, 2, 212, 156, 354, 0, 269, 1167, 0, 63, 1981, 610, 119, 704, 15, 0, 88, 2007, 6075, 434, 0, 5196, 0, 1497, 652, 507, 65, 339, 0, 12, 463, 0, 0, 0, 763, 152, 335] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[2748, 6014, 0, 139, 638, 35, 51, 13, 0, 325, 122, 181, 98, 313, 459, 704, 237, 994, 0, 1, 167, 81, 2248, 1059, 3307, 3622, 1378, 300, 1, 39, 89, 362, 0, 258, 1071, 0, 78, 2341, 651, 133, 754, 18, 0, 84, 1986, 5922, 433, 0, 4787, 0, 1299, 693, 495, 87, 333, 0, 3, 437, 0, 0, 0, 1000, 222, 342] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[2410, 5737, 20, 160, 737, 52, 53, 29, 0, 357, 166, 184, 133, 316, 501, 735, 402, 1124, 15, 6, 227, 80, 2232, 1470, 2482, 3559, 1793, 222, 2, 217, 261, 314, 8, 404, 854, 1, 88, 1624, 872, 138, 701, 13, 0, 68, 1692, 6148, 392, 4, 5368, 11, 1718, 539, 492, 91, 262, 1, 21, 422, 5, 4, 1, 659, 198, 357] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[3047, 6905, 1, 47, 541, 0, 61, 0, 0, 310, 65, 166, 124, 321, 574, 536, 155, 1740, 0, 0, 139, 75, 1100, 1027, 4378, 3326, 2083, 276, 0, 13, 87, 332, 0, 424, 458, 1, 90, 2512, 828, 99, 757, 5, 0, 95, 999, 6241, 136, 0, 4547, 0, 844, 713, 465, 73, 123, 0, 7, 374, 0, 0, 0, 1195, 371, 366] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[2193, 4356, 0, 181, 849, 59, 52, 62, 0, 452, 177, 278, 108, 304, 558, 690, 376, 1053, 0, 5, 484, 85, 2575, 1304, 2647, 3756, 1303, 271, 0, 354, 176, 355, 0, 233, 1605, 0, 81, 1811, 684, 181, 722, 21, 0, 72, 2402, 5758, 802, 0, 5004, 0, 1692, 484, 489, 96, 349, 0, 37, 432, 0, 0, 0, 666, 132, 336] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1981, 1873, 0, 299, 952, 190, 67, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 347, 205, 269, 104, 326, 583, 644, 1467, 1675, 0, 0, 0, 0, 0, 0, 411, 705, 0, 8, 677, 88, 2843, 1564, 971, 0, 0, 0, 0, 0, 0, 0, 2071, 380, 1093, 262, 1, 345, 234, 414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 221, 1926, 0, 91, 1651, 779, 198, 662, 869, 0, 0, 0, 0, 0, 0, 723, 71, 0, 57, 2390, 2776, 922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3552, 0, 2040, 493, 413, 115, 374, 1, 0, 0, 0, 0, 0, 0, 0, 0, 9, 346, 0, 0, 0, 484, 114, 396, 3186, 1123, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5483, 5620, 7267, 4800, 6397, 6939, 6988, 5658] avg=6144.0000 max=7267 max_rank=2 min=4800 min_rank=3 imbalance_ratio=1.182780 max_load_reduction=41885 max_load_reduction_pct=0.852153 moved_to_echo=25982 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2748, 2814, 0, 139, 638, 35, 51, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 325, 122, 181, 98, 313, 459, 704, 2205, 1780, 0, 0, 0, 0, 0, 0, 237, 994, 0, 1, 167, 81, 2248, 1059, 862, 0, 0, 0, 0, 0, 0, 0, 3307, 386, 1378, 300, 1, 39, 89, 362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 258, 1071, 0, 78, 2341, 651, 133, 995, 772, 0, 0, 0, 0, 0, 0, 754, 18, 0, 84, 1986, 2949, 433, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3153, 0, 1299, 693, 495, 87, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 437, 0, 0, 0, 1000, 222, 342, 3236, 1193, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6438, 6187, 5649, 5862, 6299, 6224, 6060, 6433] avg=6144.0000 max=6438 max_rank=0 min=5649 min_rank=2 imbalance_ratio=1.047852 max_load_reduction=42714 max_load_reduction_pct=0.869019 moved_to_echo=25016 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2313, 2555, 0, 143, 673, 47, 54, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 288, 131, 197, 89, 302, 431, 693, 2001, 1826, 0, 0, 0, 0, 0, 0, 318, 1103, 0, 0, 295, 99, 2335, 1383, 936, 0, 0, 0, 0, 0, 0, 0, 2864, 395, 1753, 208, 2, 212, 156, 354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 269, 1167, 0, 63, 1981, 610, 119, 903, 838, 0, 0, 0, 0, 0, 0, 704, 15, 0, 88, 2007, 3025, 434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3422, 0, 1497, 652, 507, 65, 339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 463, 0, 0, 0, 763, 152, 335, 3309, 1224, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5818, 5958, 6469, 5944, 5950, 6273, 6482, 6258] avg=6144.0000 max=6482 max_rank=6 min=5818 min_rank=0 imbalance_ratio=1.055013 max_load_reduction=42670 max_load_reduction_pct=0.868123 moved_to_echo=24963 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2410, 2685, 20, 160, 737, 52, 53, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 357, 166, 184, 133, 316, 501, 735, 2103, 1848, 0, 0, 0, 0, 0, 0, 402, 1124, 15, 6, 227, 80, 2232, 1470, 967, 0, 0, 0, 0, 0, 0, 0, 2482, 379, 1793, 222, 2, 217, 261, 314, 0, 0, 0, 0, 0, 0, 0, 0, 8, 404, 854, 1, 88, 1624, 872, 138, 949, 865, 0, 0, 0, 0, 0, 0, 701, 13, 0, 68, 1692, 3062, 392, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3536, 11, 1718, 539, 492, 91, 262, 1, 0, 0, 0, 0, 0, 0, 0, 0, 21, 422, 5, 4, 1, 659, 198, 357, 3180, 1238, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6146, 6343, 6523, 5670, 5803, 5932, 6650, 6085] avg=6144.0000 max=6650 max_rank=6 min=5670 min_rank=3 imbalance_ratio=1.082357 max_load_reduction=42502 max_load_reduction_pct=0.864705 moved_to_echo=24470 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2760, 3084, 0, 89, 538, 19, 64, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 289, 104, 202, 110, 313, 507, 629, 2416, 1833, 0, 0, 0, 0, 0, 0, 178, 1415, 0, 0, 184, 87, 1388, 1110, 828, 0, 0, 0, 0, 0, 0, 0, 4107, 350, 2108, 290, 0, 0, 133, 386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 290, 611, 0, 99, 2454, 633, 138, 1090, 741, 0, 0, 0, 0, 0, 0, 695, 4, 0, 60, 1696, 3038, 262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3029, 0, 961, 754, 477, 98, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 391, 0, 0, 0, 1166, 268, 315, 2932, 1229, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6560, 6403, 5190, 7374, 6056, 5755, 5509, 6305] avg=6144.0000 max=7374 max_rank=3 min=5190 min_rank=2 imbalance_ratio=1.200195 max_load_reduction=41778 max_load_reduction_pct=0.849976 moved_to_echo=23625 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2454, 2501, 105, 166, 928, 37, 96, 65, 0, 0, 0, 0, 0, 0, 0, 0, 1, 487, 253, 297, 99, 371, 576, 531, 1959, 1659, 0, 0, 0, 0, 0, 0, 378, 1711, 95, 76, 277, 88, 1515, 1264, 810, 0, 0, 0, 0, 0, 0, 0, 3130, 331, 1912, 219, 59, 230, 208, 386, 0, 0, 0, 0, 0, 0, 0, 0, 40, 450, 667, 9, 79, 2154, 1064, 159, 884, 725, 0, 0, 0, 0, 0, 0, 625, 14, 31, 74, 1983, 2750, 511, 63, 0, 0, 0, 0, 0, 0, 0, 0, 2962, 117, 1463, 516, 429, 93, 258, 12, 0, 0, 0, 0, 0, 0, 0, 0, 75, 355, 80, 26, 13, 609, 347, 353, 2776, 1112, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6352, 6233, 6214, 6475, 6231, 6051, 5850, 5746] avg=6144.0000 max=6475 max_rank=3 min=5746 min_rank=7 imbalance_ratio=1.053874 max_load_reduction=42677 max_load_reduction_pct=0.868266 moved_to_echo=23878 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2193, 2039, 0, 181, 849, 59, 52, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 452, 177, 278, 108, 304, 558, 690, 1597, 1731, 0, 0, 0, 0, 0, 0, 376, 1053, 0, 5, 484, 85, 2575, 1304, 901, 0, 0, 0, 0, 0, 0, 0, 2647, 400, 1303, 271, 0, 354, 176, 355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 233, 1605, 0, 81, 1811, 684, 181, 720, 807, 0, 0, 0, 0, 0, 0, 722, 21, 0, 72, 2402, 2867, 802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3296, 0, 1692, 484, 489, 96, 349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 432, 0, 0, 0, 666, 132, 336, 3356, 1160, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5435, 5895, 6783, 5506, 6122, 6886, 6406, 6119] avg=6144.0000 max=6886 max_rank=5 min=5435 min_rank=0 imbalance_ratio=1.120768 max_load_reduction=42266 max_load_reduction_pct=0.859904 moved_to_echo=25533 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3047, 3223, 1, 47, 541, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 310, 65, 166, 124, 321, 574, 536, 2537, 1880, 0, 0, 0, 0, 0, 0, 155, 1740, 0, 0, 139, 75, 1100, 1027, 821, 0, 0, 0, 0, 0, 0, 0, 4378, 352, 2083, 276, 0, 13, 87, 332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 424, 458, 1, 90, 2512, 828, 99, 1145, 737, 0, 0, 0, 0, 0, 0, 757, 5, 0, 95, 999, 3100, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2989, 0, 844, 713, 465, 73, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 374, 0, 0, 0, 1195, 371, 366, 2974, 1261, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6920, 6513, 5057, 7521, 6294, 5092, 5207, 6548] avg=6144.0000 max=7521 max_rank=3 min=5057 min_rank=2 imbalance_ratio=1.224121 max_load_reduction=41631 max_load_reduction_pct=0.846985 moved_to_echo=23141 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 2944, 1280, 1792, 896, 2688, 4224, 5248, 16384, 14336, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5760, 256, 128, 640, 15232, 23680, 3968, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[19968, 20864, 128, 1280, 5888, 512, 512, 384, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[256, 3328, 128, 128, 128, 6656, 1920, 2816, 24960, 9600, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2560, 9856, 128, 128, 2560, 768, 16256, 10240, 7168, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 2560, 8448, 128, 768, 16640, 6144, 1280, 7424, 6400, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[25088, 3072, 13440, 2048, 128, 1536, 1408, 2944, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[25984, 128, 11520, 4864, 3840, 768, 2304, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[567, 468, 4642, 3829, 1405, 0, 5530, 5164, 0, 124, 0, 747, 0, 1549, 0, 147, 264, 8, 1893, 47, 886, 57, 144, 148, 54, 61, 1308, 0, 0, 117, 0, 175, 5615, 893, 0, 63, 250, 1043, 623, 0, 364, 0, 23, 0, 0, 0, 354, 45, 1982, 0, 180, 322, 10, 1611, 468, 0, 2167, 521, 86, 1688, 1323, 0, 187, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[600, 639, 4203, 4252, 904, 0, 5807, 5833, 0, 176, 0, 736, 0, 1445, 0, 169, 260, 5, 1786, 19, 752, 75, 678, 140, 62, 88, 1723, 0, 0, 150, 0, 175, 5487, 863, 0, 135, 259, 780, 590, 0, 255, 0, 95, 0, 144, 0, 366, 10, 1785, 0, 250, 316, 18, 1679, 85, 0, 1859, 452, 113, 1374, 1179, 0, 380, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[575, 456, 4500, 3601, 1318, 0, 5775, 4732, 0, 195, 0, 825, 0, 1470, 0, 159, 261, 0, 2059, 22, 710, 89, 349, 130, 63, 97, 1154, 0, 0, 162, 0, 197, 5698, 867, 0, 98, 256, 1496, 653, 0, 410, 0, 13, 0, 0, 0, 330, 11, 1786, 0, 74, 195, 91, 1665, 542, 0, 1371, 479, 37, 2089, 2001, 0, 91, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[571, 508, 4951, 3862, 1274, 0, 5594, 4907, 0, 156, 0, 836, 0, 1529, 0, 155, 321, 1, 1770, 28, 894, 43, 333, 139, 51, 78, 1284, 0, 0, 129, 0, 204, 5687, 946, 0, 104, 208, 1213, 623, 0, 392, 0, 14, 0, 0, 0, 354, 107, 1745, 0, 164, 322, 9, 1519, 496, 0, 2096, 552, 72, 1440, 1449, 0, 22, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[567, 564, 4717, 3370, 1505, 7, 5318, 4673, 1, 177, 10, 880, 18, 1542, 4, 196, 254, 1, 1857, 24, 790, 69, 443, 126, 51, 94, 1341, 0, 0, 140, 11, 187, 5615, 929, 0, 89, 253, 1001, 659, 6, 394, 7, 71, 17, 2, 12, 358, 120, 1865, 2, 253, 312, 4, 1715, 399, 8, 2099, 500, 81, 1645, 1584, 0, 214, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[547, 554, 4643, 3626, 857, 72, 5511, 4087, 36, 161, 65, 673, 137, 1723, 15, 301, 258, 6, 1639, 41, 490, 99, 584, 163, 55, 122, 1197, 27, 28, 140, 116, 189, 5495, 898, 39, 117, 284, 1407, 611, 28, 420, 58, 312, 82, 11, 116, 378, 104, 1779, 15, 184, 142, 329, 1627, 307, 115, 1356, 416, 137, 1682, 2183, 0, 338, 20] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[590, 467, 4726, 3154, 1277, 0, 5418, 4024, 0, 179, 0, 1035, 0, 1657, 0, 167, 267, 0, 1516, 16, 644, 65, 264, 151, 61, 89, 838, 0, 0, 151, 0, 210, 5817, 851, 0, 90, 309, 1496, 718, 0, 410, 0, 6, 0, 0, 0, 341, 59, 2012, 0, 235, 205, 34, 1788, 734, 0, 1644, 465, 57, 2366, 2463, 0, 86, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[557, 562, 4485, 3791, 1487, 0, 5796, 5501, 0, 160, 0, 748, 0, 1465, 0, 190, 244, 152, 1754, 51, 914, 52, 561, 133, 52, 80, 1443, 0, 1, 132, 0, 188, 5515, 886, 0, 81, 202, 856, 586, 0, 384, 3, 81, 0, 19, 0, 355, 106, 1402, 0, 212, 293, 27, 1787, 90, 0, 2215, 490, 84, 1445, 1296, 0, 238, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[600, 639, 89, 4252, 904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 0, 736, 0, 1445, 0, 169, 2284, 1072, 0, 0, 0, 0, 0, 0, 260, 5, 1786, 19, 752, 75, 678, 140, 1830, 629, 0, 0, 0, 0, 0, 0, 62, 88, 1723, 0, 0, 150, 0, 175, 4761, 403, 0, 0, 0, 0, 0, 0, 3029, 863, 0, 135, 259, 780, 590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 0, 95, 0, 144, 0, 366, 10, 5404, 0, 0, 0, 0, 0, 0, 0, 1785, 0, 250, 316, 18, 1679, 85, 0, 1705, 0, 0, 0, 0, 0, 0, 0, 1859, 452, 113, 1374, 1179, 0, 380, 1, 124, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6484, 5882, 6174, 7362, 5656, 6274, 5838, 5482] avg=6144.0000 max=7362 max_rank=3 min=5482 min_rank=7 imbalance_ratio=1.198242 max_load_reduction=41790 max_load_reduction_pct=0.850220 moved_to_echo=23250 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[567, 468, 98, 3829, 1405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 747, 0, 1549, 0, 147, 2523, 950, 0, 0, 0, 0, 0, 0, 264, 8, 1893, 47, 886, 57, 144, 148, 2021, 644, 0, 0, 0, 0, 0, 0, 54, 61, 1308, 0, 0, 117, 0, 175, 4214, 385, 0, 0, 0, 0, 0, 0, 3099, 893, 0, 63, 250, 1043, 623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 364, 0, 23, 0, 0, 0, 354, 45, 5145, 0, 0, 0, 0, 0, 0, 0, 1982, 0, 180, 322, 10, 1611, 468, 0, 1745, 0, 0, 0, 0, 0, 0, 0, 2167, 521, 86, 1688, 1323, 0, 187, 0, 127, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6367, 6040, 6112, 6314, 5971, 5931, 6318, 6099] avg=6144.0000 max=6367 max_rank=0 min=5931 min_rank=5 imbalance_ratio=1.036296 max_load_reduction=42785 max_load_reduction_pct=0.870463 moved_to_echo=24319 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[567, 564, 99, 3370, 1505, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 177, 10, 880, 18, 1542, 4, 196, 2564, 858, 0, 0, 0, 0, 0, 0, 254, 1, 1857, 24, 790, 69, 443, 126, 2054, 644, 0, 0, 0, 0, 0, 0, 51, 94, 1341, 0, 0, 140, 11, 187, 3815, 369, 0, 0, 0, 0, 0, 0, 3099, 929, 0, 89, 253, 1001, 659, 6, 0, 0, 0, 0, 0, 0, 0, 0, 394, 7, 71, 17, 2, 12, 358, 120, 4949, 0, 0, 0, 0, 0, 0, 0, 1865, 2, 253, 312, 4, 1715, 399, 8, 1745, 0, 0, 0, 0, 0, 0, 0, 2099, 500, 81, 1645, 1584, 0, 214, 1, 127, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6112, 6250, 6262, 6008, 6036, 5930, 6303, 6251] avg=6144.0000 max=6303 max_rank=6 min=5930 min_rank=5 imbalance_ratio=1.025879 max_load_reduction=42849 max_load_reduction_pct=0.871765 moved_to_echo=24520 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[571, 508, 104, 3862, 1274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 836, 0, 1529, 0, 155, 2691, 901, 0, 0, 0, 0, 0, 0, 321, 1, 1770, 28, 894, 43, 333, 139, 2156, 652, 0, 0, 0, 0, 0, 0, 51, 78, 1284, 0, 0, 129, 0, 204, 4006, 388, 0, 0, 0, 0, 0, 0, 3140, 946, 0, 104, 208, 1213, 623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 392, 0, 14, 0, 0, 0, 354, 107, 5206, 0, 0, 0, 0, 0, 0, 0, 1745, 0, 164, 322, 9, 1519, 496, 0, 1767, 0, 0, 0, 0, 0, 0, 0, 2096, 552, 72, 1440, 1449, 0, 22, 0, 128, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6319, 6268, 6337, 6140, 6234, 6073, 6022, 5759] avg=6144.0000 max=6337 max_rank=2 min=5759 min_rank=7 imbalance_ratio=1.031413 max_load_reduction=42815 max_load_reduction_pct=0.871073 moved_to_echo=24088 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[575, 456, 94, 3601, 1318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 195, 0, 825, 0, 1470, 0, 159, 2446, 870, 0, 0, 0, 0, 0, 0, 261, 0, 2059, 22, 710, 89, 349, 130, 1960, 653, 0, 0, 0, 0, 0, 0, 63, 97, 1154, 0, 0, 162, 0, 197, 3862, 402, 0, 0, 0, 0, 0, 0, 3146, 867, 0, 98, 256, 1496, 653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 410, 0, 13, 0, 0, 0, 330, 11, 5373, 0, 0, 0, 0, 0, 0, 0, 1786, 0, 74, 195, 91, 1665, 542, 0, 1770, 0, 0, 0, 0, 0, 0, 0, 1371, 479, 37, 2089, 2001, 0, 91, 0, 129, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6044, 5965, 6233, 5937, 6516, 6137, 6123, 6197] avg=6144.0000 max=6516 max_rank=4 min=5937 min_rank=3 imbalance_ratio=1.060547 max_load_reduction=42636 max_load_reduction_pct=0.867432 moved_to_echo=24973 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[547, 554, 98, 3626, 857, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 161, 65, 673, 137, 1723, 15, 301, 2523, 752, 0, 0, 0, 0, 0, 0, 258, 6, 1639, 41, 490, 99, 584, 163, 2022, 630, 0, 0, 0, 0, 0, 0, 55, 122, 1197, 27, 28, 140, 116, 189, 3335, 384, 0, 0, 0, 0, 0, 0, 3034, 898, 39, 117, 284, 1407, 611, 28, 0, 0, 0, 0, 0, 0, 0, 0, 420, 58, 312, 82, 11, 116, 378, 104, 5127, 0, 0, 0, 0, 0, 0, 0, 1779, 15, 184, 142, 329, 1627, 307, 115, 1707, 0, 0, 0, 0, 0, 0, 0, 1356, 416, 137, 1682, 2183, 0, 338, 20, 124, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5754, 6386, 5932, 5593, 6418, 6608, 6205, 6256] avg=6144.0000 max=6608 max_rank=5 min=5593 min_rank=3 imbalance_ratio=1.075521 max_load_reduction=42544 max_load_reduction_pct=0.865560 moved_to_echo=25487 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[557, 562, 95, 3791, 1487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 748, 0, 1465, 0, 190, 2437, 1011, 0, 0, 0, 0, 0, 0, 244, 152, 1754, 51, 914, 52, 561, 133, 1953, 632, 0, 0, 0, 0, 0, 0, 52, 80, 1443, 0, 1, 132, 0, 188, 4490, 402, 0, 0, 0, 0, 0, 0, 3046, 886, 0, 81, 202, 856, 586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 384, 3, 81, 0, 19, 0, 355, 106, 5394, 0, 0, 0, 0, 0, 0, 0, 1402, 0, 212, 293, 27, 1787, 90, 0, 1713, 0, 0, 0, 0, 0, 0, 0, 2215, 490, 84, 1445, 1296, 0, 238, 0, 124, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6492, 6011, 6446, 6788, 5657, 6342, 5524, 5892] avg=6144.0000 max=6788 max_rank=3 min=5524 min_rank=6 imbalance_ratio=1.104818 max_load_reduction=42364 max_load_reduction_pct=0.861898 moved_to_echo=23415 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[590, 467, 92, 3154, 1277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 0, 1035, 0, 1657, 0, 167, 2572, 740, 0, 0, 0, 0, 0, 0, 267, 0, 1516, 16, 644, 65, 264, 151, 2062, 671, 0, 0, 0, 0, 0, 0, 61, 89, 838, 0, 0, 151, 0, 210, 3284, 377, 0, 0, 0, 0, 0, 0, 3200, 851, 0, 90, 309, 1496, 718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 410, 0, 6, 0, 0, 0, 341, 59, 5041, 0, 0, 0, 0, 0, 0, 0, 2012, 0, 235, 205, 34, 1788, 734, 0, 1811, 0, 0, 0, 0, 0, 0, 0, 1644, 465, 57, 2366, 2463, 0, 86, 0, 135, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5580, 6350, 5656, 5010, 6664, 5857, 6819, 7216] avg=6144.0000 max=7216 max_rank=7 min=5010 min_rank=3 imbalance_ratio=1.174479 max_load_reduction=41936 max_load_reduction_pct=0.853190 moved_to_echo=26556 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 1408, 128, 6528, 256, 12416, 128, 1536, 20096, 7168, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[24832, 7168, 128, 896, 2048, 9344, 5120, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[14464, 128, 1664, 2176, 640, 13440, 3200, 128, 14080, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[14848, 3968, 768, 13824, 13568, 0, 1664, 128, 1024, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2176, 256, 14336, 256, 6144, 640, 3456, 1152, 16128, 5248, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4608, 4224, 896, 29568, 10112, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[512, 768, 10368, 128, 128, 1152, 128, 1536, 31872, 3200, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3072, 128, 640, 128, 256, 128, 2944, 640, 41728, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 7170, 1924, 1, 2, 53, 720, 4389, 0, 5, 0, 0, 0, 700, 2464, 131, 1, 484, 98, 244, 70, 0, 180, 781, 698, 322, 0, 155, 244, 53, 179, 4, 115, 7459, 83, 0, 1799, 0, 747, 0, 1409, 0, 0, 0, 2429, 243, 0, 0, 78, 0, 84, 817, 25, 491, 78, 0, 0, 417, 6627, 147, 277, 1988, 0, 2767] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 6632, 1853, 1, 1, 65, 684, 4820, 0, 0, 0, 0, 0, 515, 2113, 160, 362, 433, 104, 257, 72, 0, 202, 864, 673, 345, 64, 140, 256, 71, 199, 0, 58, 7381, 123, 0, 894, 0, 1064, 0, 957, 0, 0, 0, 2308, 170, 0, 0, 15, 0, 90, 1175, 79, 506, 89, 0, 0, 325, 6427, 82, 124, 2398, 73, 3928] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 7188, 1909, 12, 1, 54, 710, 4508, 0, 1, 0, 0, 0, 686, 2389, 124, 93, 471, 89, 242, 62, 0, 153, 806, 558, 303, 13, 177, 239, 57, 150, 0, 108, 7485, 67, 0, 1647, 0, 850, 0, 1458, 0, 0, 0, 2328, 164, 0, 0, 63, 0, 64, 793, 22, 514, 61, 0, 0, 446, 6617, 140, 222, 2165, 0, 2943] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[1, 7144, 2187, 11, 1, 56, 662, 4216, 1, 37, 12, 3, 1, 649, 2628, 143, 67, 606, 139, 246, 67, 3, 154, 769, 421, 343, 33, 157, 251, 67, 230, 1, 106, 7333, 92, 6, 1560, 16, 1063, 1, 1250, 5, 29, 7, 2263, 252, 3, 18, 81, 20, 165, 1105, 34, 519, 105, 19, 18, 401, 6375, 100, 249, 2183, 24, 2444] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 7322, 1803, 4, 0, 64, 705, 3854, 0, 4, 0, 0, 0, 640, 2532, 162, 0, 693, 110, 231, 82, 0, 169, 710, 758, 328, 0, 141, 223, 80, 209, 0, 142, 7537, 101, 0, 2583, 0, 725, 0, 1560, 0, 0, 0, 2246, 202, 0, 0, 98, 0, 100, 692, 8, 464, 97, 0, 0, 381, 6544, 199, 287, 1938, 0, 2424] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 6960, 1997, 17, 1, 60, 639, 4485, 0, 4, 0, 0, 0, 573, 2276, 140, 297, 475, 108, 241, 62, 0, 183, 852, 644, 322, 5, 144, 242, 53, 189, 1, 104, 7353, 102, 0, 1329, 0, 944, 0, 1129, 0, 0, 0, 2384, 136, 0, 0, 67, 0, 80, 1081, 54, 529, 80, 0, 0, 409, 6419, 101, 179, 2228, 13, 3461] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[21, 6894, 3353, 70, 5, 55, 817, 2685, 7, 239, 120, 30, 9, 662, 2584, 149, 160, 639, 126, 199, 89, 101, 209, 649, 655, 346, 117, 146, 262, 97, 204, 6, 375, 6907, 89, 49, 1650, 125, 527, 2, 861, 2, 198, 18, 1930, 292, 18, 155, 90, 135, 80, 893, 31, 554, 151, 207, 19, 328, 6198, 244, 284, 2041, 136, 2858] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 7395, 1998, 0, 0, 61, 684, 3672, 0, 5, 0, 0, 0, 703, 2721, 151, 0, 744, 124, 252, 69, 0, 129, 682, 607, 340, 0, 165, 216, 67, 216, 0, 139, 7526, 90, 0, 3264, 0, 662, 0, 1320, 0, 0, 0, 2310, 252, 0, 0, 90, 0, 92, 771, 3, 417, 86, 0, 0, 376, 6669, 211, 282, 1689, 0, 1902] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1853, 1, 1, 65, 684, 3825, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 2113, 160, 2643, 192, 0, 0, 0, 0, 0, 0, 362, 433, 104, 257, 72, 0, 202, 864, 4044, 0, 0, 0, 0, 0, 0, 0, 673, 345, 64, 140, 256, 71, 199, 0, 4206, 0, 0, 0, 0, 0, 0, 0, 58, 3223, 123, 0, 894, 0, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 957, 0, 0, 0, 2308, 170, 0, 0, 1515, 995, 0, 0, 0, 0, 0, 0, 15, 0, 90, 1175, 79, 506, 89, 0, 2426, 1780, 0, 0, 0, 0, 0, 0, 0, 325, 411, 82, 124, 2398, 73, 3928, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6429, 5623, 6338, 5954, 5362, 5945, 6160, 7341] avg=6144.0000 max=7341 max_rank=7 min=5362 min_rank=4 imbalance_ratio=1.194824 max_load_reduction=41811 max_load_reduction_pct=0.850647 moved_to_echo=24808 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 2187, 11, 1, 56, 662, 3346, 0, 0, 0, 0, 0, 0, 0, 0, 1, 37, 12, 3, 1, 649, 2628, 143, 2626, 191, 0, 0, 0, 0, 0, 0, 67, 606, 139, 246, 67, 3, 154, 769, 4012, 0, 0, 0, 0, 0, 0, 0, 421, 343, 33, 157, 251, 67, 230, 1, 4531, 0, 0, 0, 0, 0, 0, 0, 106, 3202, 92, 6, 1560, 16, 1063, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1250, 5, 29, 7, 2263, 252, 3, 18, 1505, 870, 0, 0, 0, 0, 0, 0, 81, 20, 165, 1105, 34, 519, 105, 19, 2613, 1766, 0, 0, 0, 0, 0, 0, 18, 401, 406, 100, 249, 2183, 24, 2444, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6264, 6291, 6063, 6034, 6046, 6202, 6427, 5825] avg=6144.0000 max=6427 max_rank=6 min=5825 min_rank=7 imbalance_ratio=1.046061 max_load_reduction=42725 max_load_reduction_pct=0.869242 moved_to_echo=24500 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1924, 1, 2, 53, 720, 3483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 700, 2464, 131, 2671, 198, 0, 0, 0, 0, 0, 0, 1, 484, 98, 244, 70, 0, 180, 781, 4170, 0, 0, 0, 0, 0, 0, 0, 698, 322, 0, 155, 244, 53, 179, 4, 4547, 0, 0, 0, 0, 0, 0, 0, 115, 3257, 83, 0, 1799, 0, 747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1409, 0, 0, 0, 2429, 243, 0, 0, 1531, 906, 0, 0, 0, 0, 0, 0, 78, 0, 84, 817, 25, 491, 78, 0, 2623, 1835, 0, 0, 0, 0, 0, 0, 0, 417, 424, 147, 277, 1988, 0, 2767, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6183, 6169, 6028, 6202, 6001, 6518, 6031, 6020] avg=6144.0000 max=6518 max_rank=5 min=6001 min_rank=4 imbalance_ratio=1.060872 max_load_reduction=42634 max_load_reduction_pct=0.867391 moved_to_echo=24570 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1909, 12, 1, 54, 710, 3577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 686, 2389, 124, 2680, 198, 0, 0, 0, 0, 0, 0, 93, 471, 89, 242, 62, 0, 153, 806, 4164, 0, 0, 0, 0, 0, 0, 0, 558, 303, 13, 177, 239, 57, 150, 0, 4559, 0, 0, 0, 0, 0, 0, 0, 108, 3269, 67, 0, 1647, 0, 850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1458, 0, 0, 0, 2328, 164, 0, 0, 1536, 931, 0, 0, 0, 0, 0, 0, 63, 0, 64, 793, 22, 514, 61, 0, 2629, 1833, 0, 0, 0, 0, 0, 0, 0, 446, 422, 140, 222, 2165, 0, 2943, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6263, 6078, 6080, 6056, 5941, 6417, 5979, 6338] avg=6144.0000 max=6417 max_rank=5 min=5941 min_rank=4 imbalance_ratio=1.044434 max_load_reduction=42735 max_load_reduction_pct=0.869446 moved_to_echo=24675 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1803, 4, 0, 64, 705, 3059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 640, 2532, 162, 2699, 196, 0, 0, 0, 0, 0, 0, 0, 693, 110, 231, 82, 0, 169, 710, 4118, 0, 0, 0, 0, 0, 0, 0, 758, 328, 0, 141, 223, 80, 209, 0, 4645, 0, 0, 0, 0, 0, 0, 0, 142, 3291, 101, 0, 2583, 0, 725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1560, 0, 0, 0, 2246, 202, 0, 0, 1547, 795, 0, 0, 0, 0, 0, 0, 98, 0, 100, 692, 8, 464, 97, 0, 2677, 1812, 0, 0, 0, 0, 0, 0, 0, 381, 418, 199, 287, 1938, 0, 2424, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5635, 6233, 6113, 6384, 6842, 6350, 5948, 5647] avg=6144.0000 max=6842 max_rank=4 min=5635 min_rank=0 imbalance_ratio=1.113607 max_load_reduction=42310 max_load_reduction_pct=0.860799 moved_to_echo=24787 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[21, 0, 3353, 70, 5, 55, 817, 2131, 0, 0, 0, 0, 0, 0, 0, 0, 7, 239, 120, 30, 9, 662, 2584, 149, 2473, 185, 0, 0, 0, 0, 0, 0, 160, 639, 126, 199, 89, 101, 209, 649, 3900, 0, 0, 0, 0, 0, 0, 0, 655, 346, 117, 146, 262, 97, 204, 6, 4373, 0, 0, 0, 0, 0, 0, 0, 375, 3017, 89, 49, 1650, 125, 527, 2, 0, 0, 0, 0, 0, 0, 0, 0, 861, 2, 198, 18, 1930, 292, 18, 155, 1417, 554, 0, 0, 0, 0, 0, 0, 90, 135, 80, 893, 31, 554, 151, 207, 2521, 1717, 0, 0, 0, 0, 0, 0, 19, 328, 396, 244, 284, 2041, 136, 2858, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6452, 6458, 6072, 6206, 5834, 5445, 6379, 6306] avg=6144.0000 max=6458 max_rank=1 min=5445 min_rank=5 imbalance_ratio=1.051107 max_load_reduction=42694 max_load_reduction_pct=0.868612 moved_to_echo=23964 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1997, 17, 1, 60, 639, 3559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 573, 2276, 140, 2633, 192, 0, 0, 0, 0, 0, 0, 297, 475, 108, 241, 62, 0, 183, 852, 4039, 0, 0, 0, 0, 0, 0, 0, 644, 322, 5, 144, 242, 53, 189, 1, 4414, 0, 0, 0, 0, 0, 0, 0, 104, 3211, 102, 0, 1329, 0, 944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1129, 0, 0, 0, 2384, 136, 0, 0, 1509, 926, 0, 0, 0, 0, 0, 0, 67, 0, 80, 1081, 54, 529, 80, 0, 2546, 1778, 0, 0, 0, 0, 0, 0, 0, 409, 410, 101, 179, 2228, 13, 3461, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6273, 5818, 6257, 6014, 5690, 6084, 6215, 6801] avg=6144.0000 max=6801 max_rank=7 min=5690 min_rank=4 imbalance_ratio=1.106934 max_load_reduction=42351 max_load_reduction_pct=0.861633 moved_to_echo=24790 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1998, 0, 0, 61, 684, 2910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 703, 2721, 151, 2698, 204, 0, 0, 0, 0, 0, 0, 0, 744, 124, 252, 69, 0, 129, 682, 4201, 0, 0, 0, 0, 0, 0, 0, 607, 340, 0, 165, 216, 67, 216, 0, 4691, 0, 0, 0, 0, 0, 0, 0, 139, 3281, 90, 0, 3264, 0, 662, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1320, 0, 0, 0, 2310, 252, 0, 0, 1547, 762, 0, 0, 0, 0, 0, 0, 90, 0, 92, 771, 3, 417, 86, 0, 2704, 1850, 0, 0, 0, 0, 0, 0, 0, 376, 414, 211, 282, 1689, 0, 1902, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5653, 6482, 6201, 6302, 7436, 6191, 6013, 4874] avg=6144.0000 max=7436 max_rank=4 min=4874 min_rank=7 imbalance_ratio=1.210286 max_load_reduction=41716 max_load_reduction_pct=0.848714 moved_to_echo=24514 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[640, 256, 768, 7424, 256, 4096, 768, 256, 20864, 14464, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 3200, 3328, 1280, 1920, 16640, 256, 22784, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1152, 25856, 768, 128, 14848, 256, 6656, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[9984, 128, 256, 128, 18304, 1792, 128, 256, 12160, 6784, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5120, 2688, 256, 1280, 2048, 640, 1664, 128, 35968, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 0, 17024, 128, 128, 512, 5632, 25984, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1024, 4608, 1024, 1920, 640, 128, 1408, 6144, 32768, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=50176 observed_loss=-1024 tokens_per_expert=[128, 384, 256, 128, 128, 5248, 19712, 1280, 21248, 1664, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=50176 slack_slots=48128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 351, 0, 0, 0, 0, 365, 459, 1805, 0, 827, 322, 0, 78, 84, 0, 6, 0, 0, 70, 0, 0, 3, 4329, 0, 261, 238, 1383, 1, 0, 918, 1474, 0, 0, 0, 1612, 0, 1073, 2410, 983, 5936, 0, 618, 772, 737, 4, 0, 200, 0, 3345, 0, 0, 84, 0, 862, 0, 5353, 5987, 4226, 1821, 0, 155] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 301, 0, 0, 0, 0, 358, 345, 1619, 0, 958, 302, 0, 61, 88, 0, 3, 0, 0, 62, 0, 0, 1, 4189, 0, 250, 231, 1308, 1, 0, 821, 1770, 0, 0, 0, 1783, 0, 1015, 1983, 848, 6349, 0, 629, 647, 681, 3, 0, 163, 0, 3613, 0, 0, 88, 0, 891, 0, 5026, 6061, 4664, 1917, 0, 123] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[6, 8, 0, 325, 3, 1, 4, 17, 367, 224, 1362, 12, 913, 340, 7, 88, 57, 5, 1, 0, 24, 63, 0, 54, 5, 4117, 2, 281, 249, 1372, 26, 51, 898, 2163, 0, 6, 1, 1629, 8, 1050, 2296, 1272, 6038, 14, 589, 687, 842, 10, 1, 214, 21, 3186, 11, 19, 63, 4, 898, 19, 5132, 5677, 4361, 1892, 0, 167] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 303, 0, 0, 0, 0, 376, 556, 1379, 0, 841, 328, 0, 97, 60, 0, 0, 0, 0, 82, 0, 0, 2, 4675, 0, 321, 240, 1440, 0, 0, 680, 2315, 0, 0, 0, 1771, 0, 1030, 3154, 765, 5991, 0, 555, 1160, 1108, 0, 0, 186, 0, 2658, 0, 0, 60, 0, 816, 0, 5296, 6043, 3253, 1416, 0, 195] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 319, 0, 0, 0, 0, 391, 203, 1113, 0, 1059, 345, 0, 104, 58, 0, 2, 0, 0, 72, 0, 0, 0, 4111, 0, 280, 219, 1092, 0, 0, 1382, 1366, 0, 0, 0, 1811, 0, 974, 1350, 1638, 6380, 0, 484, 603, 736, 0, 0, 224, 0, 4555, 0, 0, 57, 0, 716, 0, 4070, 5875, 4729, 2658, 0, 176] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[33, 25, 2, 353, 141, 13, 41, 212, 412, 324, 2014, 113, 1064, 385, 77, 85, 42, 43, 0, 9, 121, 110, 0, 116, 70, 4288, 10, 372, 294, 1389, 84, 160, 594, 1898, 1, 63, 0, 1941, 60, 1116, 3480, 1492, 5137, 178, 521, 862, 1461, 56, 2, 190, 194, 2201, 32, 150, 63, 33, 1101, 91, 4180, 5033, 3052, 1415, 7, 146] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 363, 0, 0, 1, 0, 356, 253, 1347, 0, 961, 322, 0, 82, 65, 0, 9, 0, 0, 62, 0, 2, 1, 4340, 0, 272, 256, 1120, 3, 0, 1234, 1491, 0, 0, 0, 1760, 0, 973, 1876, 1352, 6440, 0, 614, 598, 701, 3, 0, 232, 0, 3690, 2, 0, 62, 0, 863, 0, 4734, 5857, 4445, 2252, 0, 158] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[2, 0, 0, 201, 0, 0, 0, 0, 382, 475, 1942, 0, 880, 341, 0, 89, 95, 0, 0, 0, 0, 69, 0, 0, 4, 4464, 0, 286, 241, 1492, 6, 0, 434, 2614, 0, 0, 0, 1928, 0, 1004, 3474, 594, 5023, 0, 554, 1507, 1153, 7, 0, 132, 0, 2392, 0, 0, 95, 0, 940, 0, 5761, 5962, 3216, 1222, 0, 171] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 351, 0, 0, 0, 0, 5658, 141, 0, 0, 0, 0, 0, 0, 365, 459, 1805, 0, 827, 322, 0, 78, 2378, 273, 0, 0, 0, 0, 0, 0, 84, 0, 6, 0, 0, 70, 0, 0, 5987, 140, 0, 0, 0, 0, 0, 0, 3, 4244, 0, 261, 238, 1383, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 918, 1474, 0, 0, 0, 1612, 0, 1073, 504, 85, 0, 0, 0, 0, 0, 0, 2410, 983, 138, 0, 618, 772, 737, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 0, 3345, 0, 0, 84, 0, 2834, 0, 0, 0, 0, 0, 0, 0, 862, 0, 0, 0, 3449, 1821, 0, 155, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6150, 6507, 6287, 6130, 5666, 5662, 6463, 6287] avg=6144.0000 max=6507 max_rank=1 min=5662 min_rank=5 imbalance_ratio=1.059082 max_load_reduction=42645 max_load_reduction_pct=0.867615 moved_to_echo=24078 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 319, 0, 0, 0, 0, 6081, 107, 0, 0, 0, 0, 0, 0, 391, 203, 1113, 0, 1059, 345, 0, 104, 1809, 306, 0, 0, 0, 0, 0, 0, 58, 0, 2, 0, 0, 72, 0, 0, 5875, 151, 0, 0, 0, 0, 0, 0, 0, 4031, 0, 280, 219, 1092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1382, 1366, 0, 0, 0, 1811, 0, 974, 564, 80, 0, 0, 0, 0, 0, 0, 1350, 1638, 148, 0, 484, 603, 736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 224, 0, 4555, 0, 0, 57, 0, 2154, 0, 0, 0, 0, 0, 0, 0, 716, 0, 0, 0, 3859, 2658, 0, 176, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6507, 5330, 6158, 5622, 6177, 4959, 6990, 7409] avg=6144.0000 max=7409 max_rank=7 min=4959 min_rank=5 imbalance_ratio=1.205892 max_load_reduction=41743 max_load_reduction_pct=0.849264 moved_to_echo=25535 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[6, 8, 0, 325, 3, 1, 4, 17, 5755, 135, 0, 0, 0, 0, 0, 0, 367, 224, 1362, 12, 913, 340, 7, 88, 2280, 282, 0, 0, 0, 0, 0, 0, 57, 5, 1, 0, 24, 63, 0, 54, 5677, 143, 0, 0, 0, 0, 0, 0, 5, 4037, 2, 281, 249, 1372, 26, 51, 0, 0, 0, 0, 0, 0, 0, 0, 898, 2163, 0, 6, 1, 1629, 8, 1050, 520, 80, 0, 0, 0, 0, 0, 0, 2296, 1272, 140, 14, 589, 687, 842, 10, 0, 0, 0, 0, 0, 0, 0, 0, 1, 214, 21, 3186, 11, 19, 63, 4, 2717, 0, 0, 0, 0, 0, 0, 0, 898, 19, 0, 0, 3559, 1892, 0, 167, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6254, 5875, 6024, 6023, 6355, 5850, 6236, 6535] avg=6144.0000 max=6535 max_rank=7 min=5850 min_rank=5 imbalance_ratio=1.063639 max_load_reduction=42617 max_load_reduction_pct=0.867045 moved_to_echo=24976 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 301, 0, 0, 0, 0, 6051, 133, 0, 0, 0, 0, 0, 0, 358, 345, 1619, 0, 958, 302, 0, 61, 2234, 302, 0, 0, 0, 0, 0, 0, 88, 0, 3, 0, 0, 62, 0, 0, 6061, 150, 0, 0, 0, 0, 0, 0, 1, 4107, 0, 250, 231, 1308, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 821, 1770, 0, 0, 0, 1783, 0, 1015, 556, 82, 0, 0, 0, 0, 0, 0, 1983, 848, 148, 0, 629, 647, 681, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 163, 0, 3613, 0, 0, 88, 0, 2659, 0, 0, 0, 0, 0, 0, 0, 891, 0, 0, 0, 3806, 1917, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6485, 6179, 6364, 5898, 6027, 4939, 6523, 6737] avg=6144.0000 max=6737 max_rank=7 min=4939 min_rank=5 imbalance_ratio=1.096517 max_load_reduction=42415 max_load_reduction_pct=0.862935 moved_to_echo=24226 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 303, 0, 0, 0, 0, 5710, 141, 0, 0, 0, 0, 0, 0, 376, 556, 1379, 0, 841, 328, 0, 97, 2353, 210, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 82, 0, 0, 6043, 142, 0, 0, 0, 0, 0, 0, 2, 4584, 0, 321, 240, 1440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 680, 2315, 0, 0, 0, 1771, 0, 1030, 388, 91, 0, 0, 0, 0, 0, 0, 3154, 765, 139, 0, 555, 1160, 1108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 0, 2658, 0, 0, 60, 0, 2802, 0, 0, 0, 0, 0, 0, 0, 816, 0, 0, 0, 2655, 1416, 0, 195, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6154, 6140, 6327, 6587, 6275, 6881, 5706, 5082] avg=6144.0000 max=6881 max_rank=5 min=5082 min_rank=7 imbalance_ratio=1.119954 max_load_reduction=42271 max_load_reduction_pct=0.860006 moved_to_echo=23944 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[33, 25, 2, 353, 141, 13, 41, 212, 4896, 111, 0, 0, 0, 0, 0, 0, 412, 324, 2014, 113, 1064, 385, 77, 85, 1858, 197, 0, 0, 0, 0, 0, 0, 42, 43, 0, 9, 121, 110, 0, 116, 5033, 121, 0, 0, 0, 0, 0, 0, 70, 4204, 10, 372, 294, 1389, 84, 160, 0, 0, 0, 0, 0, 0, 0, 0, 594, 1898, 1, 63, 0, 1941, 60, 1116, 364, 84, 0, 0, 0, 0, 0, 0, 3480, 1492, 120, 178, 521, 862, 1461, 56, 0, 0, 0, 0, 0, 0, 0, 0, 2, 190, 194, 2201, 32, 150, 63, 33, 2211, 0, 0, 0, 0, 0, 0, 0, 1101, 91, 0, 0, 2491, 1415, 7, 146, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5827, 6529, 5595, 6583, 6121, 8170, 5076, 5251] avg=6144.0000 max=8170 max_rank=5 min=5076 min_rank=6 imbalance_ratio=1.329753 max_load_reduction=40982 max_load_reduction_pct=0.833781 moved_to_echo=24618 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 363, 0, 0, 1, 0, 6138, 125, 0, 0, 0, 0, 0, 0, 356, 253, 1347, 0, 961, 322, 0, 82, 2104, 287, 0, 0, 0, 0, 0, 0, 65, 0, 9, 0, 0, 62, 0, 2, 5857, 152, 0, 0, 0, 0, 0, 0, 1, 4255, 0, 272, 256, 1120, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1234, 1491, 0, 0, 0, 1760, 0, 973, 530, 85, 0, 0, 0, 0, 0, 0, 1876, 1352, 150, 0, 614, 598, 701, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 232, 0, 3690, 2, 0, 62, 0, 2505, 0, 0, 0, 0, 0, 0, 0, 863, 0, 0, 0, 3628, 2252, 0, 158, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6627, 5712, 6147, 5907, 6073, 5294, 6491, 6901] avg=6144.0000 max=6901 max_rank=7 min=5294 min_rank=5 imbalance_ratio=1.123210 max_load_reduction=42251 max_load_reduction_pct=0.859599 moved_to_echo=24759 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 201, 0, 0, 0, 0, 4791, 154, 0, 0, 0, 0, 0, 0, 382, 475, 1942, 0, 880, 341, 0, 89, 2559, 212, 0, 0, 0, 0, 0, 0, 95, 0, 0, 0, 0, 69, 0, 0, 5962, 124, 0, 0, 0, 0, 0, 0, 4, 4373, 0, 286, 241, 1492, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 434, 2614, 0, 0, 0, 1928, 0, 1004, 387, 91, 0, 0, 0, 0, 0, 0, 3474, 594, 108, 0, 554, 1507, 1153, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 2392, 0, 0, 95, 0, 3048, 0, 0, 0, 0, 0, 0, 0, 940, 0, 0, 0, 2617, 1222, 0, 171, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5148, 6880, 6250, 6402, 6458, 7397, 5667, 4950] avg=6144.0000 max=7397 max_rank=5 min=4950 min_rank=7 imbalance_ratio=1.203939 max_load_reduction=41755 max_load_reduction_pct=0.849508 moved_to_echo=24472 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3072, 2944, 12672, 128, 7552, 2688, 128, 768, 17664, 2176, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[7040, 15104, 128, 128, 128, 14336, 128, 8320, 3840, 768, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7168, 128, 0, 0, 26112, 14720, 128, 1408, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[20096, 8960, 1152, 256, 4608, 6912, 7424, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 1664, 256, 25728, 128, 256, 640, 128, 20992, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 33920, 128, 2432, 2048, 10624, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[640, 128, 128, 128, 256, 640, 0, 256, 46592, 1152, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 128, 128, 2560, 256, 128, 128, 256, 45184, 1152, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 51, 325, 314, 147, 4584, 0, 450, 0, 0, 10, 0, 78, 0, 0, 0, 1392, 0, 0, 0, 0, 2411, 0, 1513, 0, 0, 22, 575, 0, 0, 0, 52, 6113, 2, 78, 5562, 0, 0, 148, 1055, 214, 0, 0, 1498, 70, 0, 0, 0, 399, 0, 2997, 78, 5329, 51, 5930, 0, 37, 1374, 3443, 696, 1425, 553, 0, 176] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 62, 314, 329, 218, 5272, 0, 457, 0, 0, 6, 0, 150, 0, 0, 0, 1133, 0, 0, 0, 0, 2003, 0, 1565, 0, 100, 2, 401, 0, 0, 0, 62, 6141, 0, 89, 4837, 0, 0, 160, 1233, 309, 0, 0, 1224, 72, 0, 0, 58, 294, 0, 3869, 89, 4173, 62, 6002, 0, 26, 1955, 3448, 710, 1833, 397, 0, 97] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 63, 291, 313, 49, 4364, 0, 431, 0, 0, 9, 0, 98, 0, 0, 0, 1512, 0, 0, 0, 0, 2506, 0, 1603, 0, 0, 47, 945, 0, 0, 0, 63, 6158, 0, 98, 5469, 0, 0, 179, 835, 248, 0, 0, 1940, 82, 0, 0, 0, 377, 0, 3335, 97, 5332, 61, 5880, 0, 16, 1427, 2897, 708, 1209, 302, 0, 208] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[39, 70, 330, 381, 199, 3374, 173, 437, 8, 40, 27, 53, 179, 233, 2, 39, 1102, 0, 4, 0, 29, 2018, 88, 1992, 40, 101, 363, 823, 33, 18, 13, 96, 5823, 99, 78, 4273, 1, 158, 159, 648, 229, 159, 326, 1411, 87, 22, 249, 86, 376, 4, 3534, 69, 4757, 49, 5721, 49, 32, 1587, 3534, 843, 1279, 982, 56, 168] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 54, 330, 304, 151, 4716, 0, 417, 0, 0, 8, 0, 63, 0, 0, 0, 1393, 0, 0, 0, 0, 2423, 0, 1512, 0, 20, 21, 590, 0, 0, 0, 54, 6224, 1, 63, 5224, 0, 0, 123, 1114, 191, 0, 0, 1463, 62, 0, 0, 0, 398, 0, 3323, 61, 5221, 54, 5920, 0, 53, 1513, 3284, 631, 1452, 588, 0, 133] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[2, 63, 305, 319, 135, 4576, 39, 442, 1, 12, 16, 2, 98, 26, 0, 12, 975, 0, 1, 0, 1, 1989, 5, 1581, 10, 103, 78, 634, 1, 5, 0, 60, 6219, 36, 91, 4996, 0, 20, 151, 958, 219, 24, 42, 1399, 63, 9, 28, 15, 360, 9, 3788, 81, 4591, 51, 5964, 7, 34, 1513, 3897, 735, 1431, 752, 5, 173] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 52, 304, 304, 154, 4758, 0, 435, 0, 0, 17, 0, 85, 0, 0, 0, 1358, 0, 0, 0, 0, 2206, 0, 1347, 0, 105, 11, 499, 0, 0, 0, 54, 6259, 4, 82, 5289, 0, 0, 142, 1271, 253, 0, 0, 1410, 62, 0, 0, 7, 390, 0, 3553, 80, 4630, 52, 5989, 0, 44, 1585, 3382, 682, 1681, 429, 0, 187] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 61, 347, 321, 47, 4370, 0, 433, 0, 0, 7, 0, 90, 0, 0, 0, 1390, 0, 0, 0, 0, 2738, 0, 1791, 0, 1, 57, 1106, 0, 0, 0, 61, 5978, 1, 90, 5227, 0, 0, 158, 678, 222, 0, 0, 2254, 69, 0, 0, 0, 402, 0, 3150, 88, 5215, 53, 5854, 0, 1, 972, 3266, 701, 1198, 541, 0, 214] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 62, 314, 329, 218, 5272, 0, 457, 204, 120, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 150, 0, 0, 0, 5998, 0, 0, 0, 0, 0, 0, 0, 1133, 0, 0, 0, 0, 2003, 0, 1565, 961, 0, 0, 0, 0, 0, 0, 0, 0, 100, 2, 401, 0, 0, 0, 62, 5180, 143, 0, 0, 0, 0, 0, 0, 0, 0, 89, 4633, 0, 0, 160, 1233, 0, 0, 0, 0, 0, 0, 0, 0, 309, 0, 0, 1224, 72, 0, 0, 58, 2316, 822, 612, 0, 0, 0, 0, 0, 294, 0, 3869, 89, 1857, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 1955, 1755, 710, 1833, 397, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6976, 6154, 5662, 5888, 6115, 5413, 6171, 6773] avg=6144.0000 max=6976 max_rank=0 min=5413 min_rank=5 imbalance_ratio=1.135417 max_load_reduction=42176 max_load_reduction_pct=0.858073 moved_to_echo=24472 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 51, 325, 314, 147, 4584, 0, 450, 235, 120, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 78, 0, 0, 0, 5971, 0, 0, 0, 0, 0, 0, 0, 1392, 0, 0, 0, 0, 2411, 0, 1513, 960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 575, 0, 0, 0, 52, 5118, 142, 0, 0, 0, 0, 0, 0, 0, 2, 78, 5327, 0, 0, 148, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 214, 0, 0, 1498, 70, 0, 0, 0, 2958, 812, 611, 0, 0, 0, 0, 0, 399, 0, 2997, 78, 2371, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 1374, 1752, 696, 1425, 553, 0, 176, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6226, 6059, 6276, 5909, 6610, 6163, 5896, 6013] avg=6144.0000 max=6610 max_rank=4 min=5896 min_rank=6 imbalance_ratio=1.075846 max_load_reduction=42542 max_load_reduction_pct=0.865519 moved_to_echo=24682 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 54, 330, 304, 151, 4716, 0, 417, 220, 114, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 63, 0, 0, 0, 6080, 0, 0, 0, 0, 0, 0, 0, 1393, 0, 0, 0, 0, 2423, 0, 1512, 915, 0, 0, 0, 0, 0, 0, 0, 0, 20, 21, 590, 0, 0, 0, 54, 5110, 144, 0, 0, 0, 0, 0, 0, 0, 1, 63, 5004, 0, 0, 123, 1114, 0, 0, 0, 0, 0, 0, 0, 0, 191, 0, 0, 1463, 62, 0, 0, 0, 2898, 810, 583, 0, 0, 0, 0, 0, 398, 0, 3323, 61, 2323, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 1513, 1672, 631, 1452, 588, 0, 133, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6306, 6151, 6243, 5939, 6305, 6007, 6159, 6042] avg=6144.0000 max=6306 max_rank=0 min=5939 min_rank=3 imbalance_ratio=1.026367 max_load_reduction=42846 max_load_reduction_pct=0.871704 moved_to_echo=24513 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2, 63, 305, 319, 135, 4576, 39, 442, 211, 136, 0, 0, 0, 0, 0, 0, 1, 12, 16, 2, 98, 26, 0, 12, 6075, 0, 0, 0, 0, 0, 0, 0, 975, 0, 1, 0, 1, 1989, 5, 1581, 1086, 0, 0, 0, 0, 0, 0, 0, 10, 103, 78, 634, 1, 5, 0, 60, 5148, 144, 0, 0, 0, 0, 0, 0, 0, 36, 91, 4785, 0, 20, 151, 958, 0, 0, 0, 0, 0, 0, 0, 0, 219, 24, 42, 1399, 63, 9, 28, 15, 2548, 816, 692, 0, 0, 0, 0, 0, 360, 9, 3788, 81, 2043, 51, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 34, 1513, 1983, 735, 1431, 752, 5, 173, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6228, 6242, 5638, 6183, 6041, 5855, 6339, 6626] avg=6144.0000 max=6626 max_rank=7 min=5638 min_rank=2 imbalance_ratio=1.078451 max_load_reduction=42526 max_load_reduction_pct=0.865194 moved_to_echo=24861 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 63, 291, 313, 49, 4364, 0, 431, 231, 101, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 98, 0, 0, 0, 6016, 0, 0, 0, 0, 0, 0, 0, 1512, 0, 0, 0, 0, 2506, 0, 1603, 807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 945, 0, 0, 0, 63, 5076, 142, 0, 0, 0, 0, 0, 0, 0, 0, 98, 5238, 0, 0, 179, 835, 0, 0, 0, 0, 0, 0, 0, 0, 248, 0, 0, 1940, 82, 0, 0, 0, 2959, 804, 514, 0, 0, 0, 0, 0, 377, 0, 3335, 97, 2373, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 1427, 1475, 708, 1209, 302, 0, 208, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5843, 6123, 6428, 6273, 6350, 6547, 6243, 5345] avg=6144.0000 max=6547 max_rank=5 min=5345 min_rank=7 imbalance_ratio=1.065592 max_load_reduction=42605 max_load_reduction_pct=0.866801 moved_to_echo=24485 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[39, 70, 330, 381, 199, 3374, 173, 437, 180, 123, 0, 0, 0, 0, 0, 0, 8, 40, 27, 53, 179, 233, 2, 39, 5689, 0, 0, 0, 0, 0, 0, 0, 1102, 0, 4, 0, 29, 2018, 88, 1992, 985, 0, 0, 0, 0, 0, 0, 0, 40, 101, 363, 823, 33, 18, 13, 96, 4939, 134, 0, 0, 0, 0, 0, 0, 0, 99, 78, 4093, 1, 158, 159, 648, 0, 0, 0, 0, 0, 0, 0, 0, 229, 159, 326, 1411, 87, 22, 249, 86, 2640, 782, 627, 0, 0, 0, 0, 0, 376, 4, 3534, 69, 2117, 49, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 32, 1587, 1799, 843, 1279, 982, 56, 168, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5306, 6270, 6218, 6560, 5236, 6618, 6198, 6746] avg=6144.0000 max=6746 max_rank=7 min=5236 min_rank=4 imbalance_ratio=1.097982 max_load_reduction=42406 max_load_reduction_pct=0.862752 moved_to_echo=24798 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 52, 304, 304, 154, 4758, 0, 435, 223, 118, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 85, 0, 0, 0, 6114, 0, 0, 0, 0, 0, 0, 0, 1358, 0, 0, 0, 0, 2206, 0, 1347, 943, 0, 0, 0, 0, 0, 0, 0, 0, 105, 11, 499, 0, 0, 0, 54, 5169, 145, 0, 0, 0, 0, 0, 0, 0, 4, 82, 5066, 0, 0, 142, 1271, 0, 0, 0, 0, 0, 0, 0, 0, 253, 0, 0, 1410, 62, 0, 0, 7, 2570, 820, 600, 0, 0, 0, 0, 0, 390, 0, 3553, 80, 2060, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 1585, 1721, 682, 1681, 429, 0, 187, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6348, 6216, 5854, 5983, 6565, 5722, 6135, 6329] avg=6144.0000 max=6565 max_rank=4 min=5722 min_rank=5 imbalance_ratio=1.068522 max_load_reduction=42587 max_load_reduction_pct=0.866435 moved_to_echo=24751 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 61, 347, 321, 47, 4370, 0, 433, 224, 116, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 90, 0, 0, 0, 5840, 0, 0, 0, 0, 0, 0, 0, 1390, 0, 0, 0, 0, 2738, 0, 1791, 914, 0, 0, 0, 0, 0, 0, 0, 0, 1, 57, 1106, 0, 0, 0, 61, 5054, 138, 0, 0, 0, 0, 0, 0, 0, 1, 90, 5003, 0, 0, 158, 678, 0, 0, 0, 0, 0, 0, 0, 0, 222, 0, 0, 2254, 69, 0, 0, 0, 2897, 800, 585, 0, 0, 0, 0, 0, 402, 0, 3150, 88, 2318, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 972, 1651, 701, 1198, 541, 0, 214, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5919, 5937, 6833, 6417, 5930, 6827, 6011, 5278] avg=6144.0000 max=6833 max_rank=2 min=5278 min_rank=7 imbalance_ratio=1.112142 max_load_reduction=42319 max_load_reduction_pct=0.860982 moved_to_echo=24046 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[256, 12032, 13824, 5760, 11520, 4608, 128, 1408, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[1920, 256, 384, 12672, 640, 128, 384, 256, 21888, 6528, 4864, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3072, 128, 27648, 768, 17536, 512, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[10368, 0, 128, 0, 128, 18304, 128, 12928, 7680, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 512, 2560, 2688, 1152, 36096, 256, 3584, 1792, 1024, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 256, 768, 39168, 128, 256, 1280, 7808, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 512, 640, 5632, 128, 128, 128, 512, 40832, 1152, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 896, 384, 128, 128, 47872, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[6, 0, 51, 0, 75, 0, 0, 0, 0, 2474, 0, 208, 78, 203, 419, 1631, 0, 7285, 0, 495, 0, 0, 84, 0, 7151, 0, 100, 244, 0, 553, 78, 6547, 7087, 4417, 0, 4, 0, 0, 0, 0, 78, 82, 1329, 0, 0, 1264, 244, 0, 416, 0, 0, 3015, 0, 0, 0, 0, 0, 3450, 0, 0, 0, 0, 0, 84] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[6, 0, 54, 0, 97, 0, 0, 0, 0, 2279, 0, 161, 63, 313, 445, 1586, 0, 7250, 0, 508, 0, 0, 88, 0, 7206, 0, 126, 242, 0, 593, 61, 6453, 7025, 4975, 0, 1, 0, 0, 0, 0, 63, 88, 1601, 0, 0, 1199, 242, 0, 394, 0, 0, 2747, 0, 0, 0, 0, 0, 3198, 0, 0, 0, 0, 0, 88] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[45, 0, 51, 16, 53, 5, 1, 0, 24, 2476, 0, 225, 96, 286, 403, 1501, 0, 7147, 19, 497, 1, 0, 74, 3, 7006, 18, 142, 250, 7, 588, 93, 6301, 6994, 4561, 23, 79, 14, 7, 0, 3, 137, 54, 1660, 31, 3, 1466, 257, 48, 399, 3, 1, 2704, 4, 0, 8, 12, 14, 3236, 0, 3, 7, 32, 0, 64] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[45, 0, 63, 0, 67, 0, 0, 0, 0, 1955, 0, 213, 98, 107, 380, 1545, 0, 7275, 0, 473, 0, 0, 60, 0, 7167, 0, 124, 231, 0, 704, 97, 6524, 7108, 4923, 0, 5, 0, 0, 0, 0, 98, 59, 1028, 0, 0, 1517, 231, 0, 396, 0, 0, 3418, 0, 0, 0, 0, 0, 3181, 0, 0, 0, 0, 0, 60] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[3, 0, 62, 0, 77, 0, 0, 0, 0, 1980, 0, 248, 88, 855, 325, 1532, 0, 7266, 0, 390, 0, 0, 57, 0, 7114, 0, 116, 257, 0, 583, 88, 5973, 7508, 4446, 0, 55, 0, 0, 0, 0, 88, 57, 2493, 0, 0, 1388, 257, 0, 415, 0, 0, 2125, 0, 0, 0, 0, 0, 3249, 0, 0, 0, 0, 0, 57] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[130, 12, 60, 34, 70, 49, 8, 0, 43, 2200, 0, 159, 154, 677, 319, 1002, 9, 6800, 133, 424, 5, 0, 160, 90, 6232, 191, 317, 269, 66, 1407, 86, 6357, 6561, 3418, 119, 378, 88, 81, 0, 61, 170, 59, 1320, 203, 16, 1830, 314, 279, 414, 26, 4, 3115, 20, 5, 29, 33, 124, 2497, 0, 25, 152, 231, 22, 95] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[2, 0, 52, 0, 82, 0, 0, 0, 0, 1880, 0, 217, 80, 472, 409, 1871, 0, 7218, 0, 492, 0, 0, 62, 0, 7112, 0, 127, 242, 0, 592, 80, 6263, 7144, 4802, 0, 21, 0, 0, 0, 0, 83, 62, 2115, 0, 0, 1381, 242, 0, 402, 0, 0, 2271, 0, 0, 0, 0, 0, 3314, 0, 0, 0, 0, 0, 62] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[112, 0, 61, 1, 62, 0, 0, 0, 0, 1932, 0, 210, 88, 78, 377, 1600, 0, 7266, 0, 464, 0, 0, 95, 0, 7092, 0, 183, 252, 0, 557, 89, 6685, 6958, 5265, 0, 14, 0, 0, 0, 0, 90, 94, 976, 0, 0, 1673, 252, 0, 390, 0, 0, 3293, 0, 0, 0, 0, 0, 2848, 0, 0, 0, 0, 0, 95] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 51, 0, 75, 0, 0, 0, 6074, 0, 0, 0, 0, 0, 0, 0, 0, 2474, 0, 208, 78, 203, 419, 1631, 1433, 0, 0, 0, 0, 0, 0, 0, 0, 5634, 0, 495, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 244, 0, 553, 78, 5048, 0, 0, 0, 0, 0, 0, 0, 0, 1433, 4417, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 82, 1329, 0, 0, 1264, 244, 0, 1651, 927, 66, 0, 0, 0, 0, 0, 416, 0, 0, 3015, 0, 0, 0, 0, 1841, 1077, 0, 0, 0, 0, 0, 0, 0, 3450, 0, 0, 0, 0, 0, 84, 2886, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6206, 6446, 6213, 6023, 5854, 5641, 6349, 6420] avg=6144.0000 max=6446 max_rank=1 min=5641 min_rank=5 imbalance_ratio=1.049154 max_load_reduction=42706 max_load_reduction_pct=0.868856 moved_to_echo=24264 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[45, 0, 51, 16, 53, 5, 1, 0, 5951, 0, 0, 0, 0, 0, 0, 0, 24, 2476, 0, 225, 96, 286, 403, 1501, 1379, 0, 0, 0, 0, 0, 0, 0, 0, 5527, 19, 497, 1, 0, 74, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 142, 250, 7, 588, 93, 4859, 0, 0, 0, 0, 0, 0, 0, 0, 1415, 4561, 23, 79, 14, 7, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 137, 54, 1660, 31, 3, 1466, 257, 48, 1620, 914, 63, 0, 0, 0, 0, 0, 399, 3, 1, 2704, 4, 0, 8, 12, 1817, 1055, 0, 0, 0, 0, 0, 0, 14, 3236, 0, 3, 7, 32, 0, 64, 2848, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6122, 6390, 6121, 5957, 6102, 6253, 6003, 6204] avg=6144.0000 max=6390 max_rank=1 min=5957 min_rank=3 imbalance_ratio=1.040039 max_load_reduction=42762 max_load_reduction_pct=0.869995 moved_to_echo=24562 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 62, 0, 77, 0, 0, 0, 6043, 0, 0, 0, 0, 0, 0, 0, 0, 1980, 0, 248, 88, 855, 325, 1532, 1307, 0, 0, 0, 0, 0, 0, 0, 0, 5619, 0, 390, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 257, 0, 583, 88, 4606, 0, 0, 0, 0, 0, 0, 0, 0, 1518, 4446, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 57, 2493, 0, 0, 1388, 257, 0, 1647, 982, 60, 0, 0, 0, 0, 0, 415, 0, 0, 2125, 0, 0, 0, 0, 1951, 1071, 0, 0, 0, 0, 0, 0, 0, 3249, 0, 0, 0, 0, 0, 57, 3057, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6185, 6335, 6066, 5650, 6019, 6972, 5562, 6363] avg=6144.0000 max=6972 max_rank=5 min=5562 min_rank=6 imbalance_ratio=1.134766 max_load_reduction=42180 max_load_reduction_pct=0.858154 moved_to_echo=24916 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 54, 0, 97, 0, 0, 0, 6122, 0, 0, 0, 0, 0, 0, 0, 0, 2279, 0, 161, 63, 313, 445, 1586, 1412, 0, 0, 0, 0, 0, 0, 0, 0, 5607, 0, 508, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 242, 0, 593, 61, 4976, 0, 0, 0, 0, 0, 0, 0, 0, 1421, 4975, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 88, 1601, 0, 0, 1199, 242, 0, 1643, 918, 65, 0, 0, 0, 0, 0, 394, 0, 0, 2747, 0, 0, 0, 0, 1825, 1084, 0, 0, 0, 0, 0, 0, 0, 3198, 0, 0, 0, 0, 0, 88, 2861, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6279, 6259, 6203, 5998, 6397, 5819, 6050, 6147] avg=6144.0000 max=6397 max_rank=4 min=5819 min_rank=5 imbalance_ratio=1.041178 max_load_reduction=42755 max_load_reduction_pct=0.869853 moved_to_echo=24413 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[45, 0, 63, 0, 67, 0, 0, 0, 6089, 0, 0, 0, 0, 0, 0, 0, 0, 1955, 0, 213, 98, 107, 380, 1545, 1428, 0, 0, 0, 0, 0, 0, 0, 0, 5626, 0, 473, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 231, 0, 704, 97, 5030, 0, 0, 0, 0, 0, 0, 0, 0, 1438, 4923, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 59, 1028, 0, 0, 1517, 231, 0, 1649, 929, 66, 0, 0, 0, 0, 0, 396, 0, 0, 3418, 0, 0, 0, 0, 1847, 1078, 0, 0, 0, 0, 0, 0, 0, 3181, 0, 0, 0, 0, 0, 60, 2894, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6264, 5726, 6159, 6186, 6366, 5577, 6739, 6135] avg=6144.0000 max=6739 max_rank=6 min=5577 min_rank=5 imbalance_ratio=1.096842 max_load_reduction=42413 max_load_reduction_pct=0.862895 moved_to_echo=24817 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[130, 12, 60, 34, 70, 49, 8, 0, 5295, 0, 0, 0, 0, 0, 0, 0, 43, 2200, 0, 159, 154, 677, 319, 1002, 1391, 0, 0, 0, 0, 0, 0, 0, 9, 5259, 133, 424, 5, 0, 160, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 191, 317, 269, 66, 1407, 86, 4902, 0, 0, 0, 0, 0, 0, 0, 0, 1326, 3418, 119, 378, 88, 81, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 170, 59, 1320, 203, 16, 1830, 314, 279, 1541, 858, 64, 0, 0, 0, 0, 0, 414, 26, 4, 3115, 20, 5, 29, 33, 1705, 937, 0, 0, 0, 0, 0, 0, 124, 2497, 0, 25, 152, 231, 22, 95, 2672, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5658, 5945, 6080, 7238, 5471, 6654, 6288, 5818] avg=6144.0000 max=7238 max_rank=3 min=5471 min_rank=4 imbalance_ratio=1.178060 max_load_reduction=41914 max_load_reduction_pct=0.852743 moved_to_echo=24231 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 52, 0, 82, 0, 0, 0, 6041, 0, 0, 0, 0, 0, 0, 0, 0, 1880, 0, 217, 80, 472, 409, 1871, 1371, 0, 0, 0, 0, 0, 0, 0, 0, 5582, 0, 492, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 242, 0, 592, 80, 4829, 0, 0, 0, 0, 0, 0, 0, 0, 1445, 4802, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 62, 2115, 0, 0, 1381, 242, 0, 1636, 934, 63, 0, 0, 0, 0, 0, 402, 0, 0, 2271, 0, 0, 0, 0, 1856, 1071, 0, 0, 0, 0, 0, 0, 0, 3314, 0, 0, 0, 0, 0, 62, 2909, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6177, 6300, 6136, 5870, 6268, 6516, 5600, 6285] avg=6144.0000 max=6516 max_rank=5 min=5600 min_rank=6 imbalance_ratio=1.060547 max_load_reduction=42636 max_load_reduction_pct=0.867432 moved_to_echo=24669 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[112, 0, 61, 1, 62, 0, 0, 0, 6025, 0, 0, 0, 0, 0, 0, 0, 0, 1932, 0, 210, 88, 78, 377, 1600, 1466, 0, 0, 0, 0, 0, 0, 0, 0, 5615, 0, 464, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 252, 0, 557, 89, 5149, 0, 0, 0, 0, 0, 0, 0, 0, 1396, 5265, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 94, 976, 0, 0, 1673, 252, 0, 1651, 914, 70, 0, 0, 0, 0, 0, 390, 0, 0, 3293, 0, 0, 0, 0, 1811, 1067, 0, 0, 0, 0, 0, 0, 0, 2848, 0, 0, 0, 0, 0, 95, 2837, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6261, 5751, 6174, 6230, 6675, 5720, 6561, 5780] avg=6144.0000 max=6675 max_rank=4 min=5720 min_rank=5 imbalance_ratio=1.086426 max_load_reduction=42477 max_load_reduction_pct=0.864197 moved_to_echo=24736 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[896, 640, 12544, 256, 128, 11776, 2048, 384, 13056, 7424, 640, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 17280, 0, 1664, 768, 3072, 3200, 12288, 11264, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[256, 25088, 0, 128, 256, 384, 128, 640, 23040, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[3328, 128, 128, 22784, 128, 128, 128, 128, 14720, 8448, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[11392, 36864, 256, 640, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 256, 1280, 2048, 128, 5632, 768, 39424, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 44544, 256, 3840, 128, 0, 768, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[384, 128, 512, 128, 640, 128, 128, 0, 47744, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 89, 2170, 0, 0, 1192, 0, 0, 3211, 2025, 0, 0, 0, 715, 0, 62, 304, 0, 1223, 0, 88, 0, 0, 6130, 19, 0, 69, 0, 0, 0, 0, 96, 305, 0, 0, 2834, 189, 0, 61, 0, 0, 5584, 0, 0, 0, 105, 0, 529, 0, 61, 0, 6749, 0, 43, 62, 7144, 0, 0, 0, 6637, 713, 743, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 104, 2515, 0, 0, 864, 0, 0, 4294, 1764, 0, 0, 0, 585, 0, 72, 329, 0, 1622, 0, 57, 0, 0, 6040, 71, 0, 88, 0, 0, 0, 0, 64, 345, 0, 0, 2656, 211, 0, 88, 0, 0, 5257, 0, 0, 0, 116, 0, 594, 0, 88, 0, 6132, 0, 0, 88, 7031, 0, 0, 0, 6691, 702, 684, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 110, 1636, 0, 0, 1791, 0, 0, 2167, 2741, 0, 0, 0, 672, 0, 82, 313, 0, 1062, 0, 60, 0, 0, 5789, 0, 0, 97, 0, 0, 0, 0, 80, 330, 0, 0, 3057, 228, 0, 97, 0, 0, 5655, 0, 0, 0, 15, 0, 517, 0, 97, 0, 6998, 0, 82, 98, 7199, 0, 0, 0, 6640, 702, 837, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[2, 3, 134, 1673, 0, 9, 1218, 0, 0, 2773, 2533, 2, 28, 41, 666, 1, 63, 320, 25, 1418, 9, 52, 0, 6, 5857, 43, 0, 95, 7, 1, 0, 8, 156, 364, 27, 24, 2635, 233, 56, 88, 21, 21, 5613, 3, 5, 2, 215, 39, 523, 14, 88, 0, 6754, 31, 41, 93, 7085, 1, 9, 2, 6565, 702, 755, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 98, 2067, 0, 0, 1508, 0, 0, 2662, 1993, 0, 0, 0, 699, 0, 70, 316, 0, 1107, 1, 83, 0, 0, 5907, 9, 0, 92, 0, 0, 0, 0, 86, 322, 0, 0, 3139, 179, 0, 78, 2, 0, 5675, 0, 0, 1, 41, 0, 510, 0, 78, 0, 6862, 2, 61, 78, 7165, 0, 0, 0, 6660, 774, 827, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 108, 2443, 0, 0, 1117, 0, 0, 3464, 1827, 0, 0, 0, 701, 0, 63, 304, 0, 1338, 4, 55, 0, 0, 6203, 39, 0, 83, 0, 0, 0, 0, 71, 324, 0, 0, 2800, 187, 0, 80, 1, 0, 5282, 0, 0, 0, 117, 0, 564, 0, 80, 0, 6713, 0, 23, 87, 6925, 0, 0, 0, 6654, 739, 756, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 124, 1508, 0, 0, 2061, 0, 0, 1774, 3150, 0, 0, 0, 691, 0, 71, 321, 0, 882, 0, 95, 0, 0, 5586, 0, 0, 92, 0, 0, 0, 0, 148, 342, 0, 0, 2877, 255, 0, 89, 9, 0, 5801, 0, 0, 0, 36, 0, 467, 0, 88, 0, 6945, 21, 69, 87, 7320, 0, 0, 0, 6700, 720, 823, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[42, 19, 126, 1279, 44, 73, 1777, 0, 0, 1630, 3601, 12, 242, 335, 599, 19, 82, 375, 180, 1015, 61, 43, 4, 11, 5461, 39, 0, 128, 80, 12, 12, 35, 185, 338, 223, 103, 2038, 181, 282, 80, 55, 139, 5430, 0, 46, 57, 350, 323, 631, 183, 78, 1, 6893, 248, 103, 97, 6299, 3, 84, 15, 6013, 638, 700, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 104, 2515, 0, 0, 864, 0, 2586, 74, 0, 0, 0, 0, 0, 0, 0, 4294, 1764, 0, 0, 0, 585, 0, 174, 0, 0, 0, 0, 0, 0, 0, 72, 329, 0, 1622, 0, 57, 0, 0, 4445, 0, 0, 0, 0, 0, 0, 0, 6040, 71, 0, 88, 0, 0, 0, 0, 120, 0, 0, 0, 0, 0, 0, 0, 64, 345, 0, 0, 2656, 211, 0, 88, 1878, 654, 0, 0, 0, 0, 0, 0, 0, 0, 5257, 0, 0, 0, 116, 0, 355, 0, 0, 0, 0, 0, 0, 0, 594, 0, 88, 0, 4829, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4739, 702, 684, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6817, 6525, 6319, 5896, 5728, 5599, 6125] avg=6144.0000 max=6817 max_rank=1 min=5599 min_rank=6 imbalance_ratio=1.109538 max_load_reduction=42335 max_load_reduction_pct=0.861308 moved_to_echo=23348 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2, 3, 134, 1673, 0, 9, 1218, 0, 2606, 72, 0, 0, 0, 0, 0, 0, 0, 2773, 2533, 2, 28, 41, 666, 1, 191, 0, 0, 0, 0, 0, 0, 0, 63, 320, 25, 1418, 9, 52, 0, 6, 4479, 0, 0, 0, 0, 0, 0, 0, 5857, 43, 0, 95, 7, 1, 0, 8, 132, 0, 0, 0, 0, 0, 0, 0, 156, 364, 27, 24, 2635, 233, 56, 88, 1842, 720, 0, 0, 0, 0, 0, 0, 21, 21, 5613, 3, 5, 2, 215, 39, 391, 0, 0, 0, 0, 0, 0, 0, 523, 14, 88, 0, 5320, 31, 41, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 9, 2, 4651, 702, 755, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5717, 6235, 6372, 6143, 6145, 6310, 6110, 6120] avg=6144.0000 max=6372 max_rank=2 min=5717 min_rank=0 imbalance_ratio=1.037109 max_load_reduction=42780 max_load_reduction_pct=0.870361 moved_to_echo=24685 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 89, 2170, 0, 0, 1192, 0, 2629, 73, 0, 0, 0, 0, 0, 0, 0, 3211, 2025, 0, 0, 0, 715, 0, 191, 0, 0, 0, 0, 0, 0, 0, 62, 304, 0, 1223, 0, 88, 0, 0, 4515, 0, 0, 0, 0, 0, 0, 0, 6130, 19, 0, 69, 0, 0, 0, 0, 132, 0, 0, 0, 0, 0, 0, 0, 96, 305, 0, 0, 2834, 189, 0, 61, 1862, 720, 0, 0, 0, 0, 0, 0, 0, 0, 5584, 0, 0, 0, 105, 0, 391, 0, 0, 0, 0, 0, 0, 0, 529, 0, 61, 0, 5315, 0, 43, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4702, 713, 743, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6153, 6142, 6192, 6350, 6067, 6080, 6010, 6158] avg=6144.0000 max=6350 max_rank=3 min=6010 min_rank=6 imbalance_ratio=1.033529 max_load_reduction=42802 max_load_reduction_pct=0.870809 moved_to_echo=24315 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 98, 2067, 0, 0, 1508, 0, 2635, 74, 0, 0, 0, 0, 0, 0, 0, 2662, 1993, 0, 0, 0, 699, 0, 194, 0, 0, 0, 0, 0, 0, 0, 70, 316, 0, 1107, 1, 83, 0, 0, 4530, 0, 0, 0, 0, 0, 0, 0, 5907, 9, 0, 92, 0, 0, 0, 0, 134, 0, 0, 0, 0, 0, 0, 0, 86, 322, 0, 0, 3139, 179, 0, 78, 1869, 732, 0, 0, 0, 0, 0, 0, 2, 0, 5675, 0, 0, 1, 41, 0, 398, 0, 0, 0, 0, 0, 0, 0, 510, 0, 78, 0, 5404, 2, 61, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4717, 774, 827, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6382, 5548, 6107, 6142, 6405, 6117, 6133, 6318] avg=6144.0000 max=6405 max_rank=4 min=5548 min_rank=1 imbalance_ratio=1.042480 max_load_reduction=42747 max_load_reduction_pct=0.869690 moved_to_echo=24973 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 110, 1636, 0, 0, 1791, 0, 2649, 73, 0, 0, 0, 0, 0, 0, 0, 2167, 2741, 0, 0, 0, 672, 0, 198, 0, 0, 0, 0, 0, 0, 0, 82, 313, 0, 1062, 0, 60, 0, 0, 4550, 0, 0, 0, 0, 0, 0, 0, 5789, 0, 0, 97, 0, 0, 0, 0, 137, 0, 0, 0, 0, 0, 0, 0, 80, 330, 0, 0, 3057, 228, 0, 97, 1863, 746, 0, 0, 0, 0, 0, 0, 0, 0, 5655, 0, 0, 0, 15, 0, 406, 0, 0, 0, 0, 0, 0, 0, 517, 0, 97, 0, 5511, 0, 82, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4704, 702, 837, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6259, 5778, 6067, 6023, 6401, 6076, 6305, 6243] avg=6144.0000 max=6401 max_rank=4 min=5778 min_rank=1 imbalance_ratio=1.041829 max_load_reduction=42751 max_load_reduction_pct=0.869771 moved_to_echo=25025 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[42, 19, 126, 1279, 44, 73, 1777, 0, 2318, 66, 0, 0, 0, 0, 0, 0, 0, 1630, 3601, 12, 242, 335, 599, 19, 195, 0, 0, 0, 0, 0, 0, 0, 82, 375, 180, 1015, 61, 43, 4, 11, 3981, 0, 0, 0, 0, 0, 0, 0, 5461, 39, 0, 128, 80, 12, 12, 35, 135, 0, 0, 0, 0, 0, 0, 0, 185, 338, 223, 103, 2038, 181, 282, 80, 1687, 735, 0, 0, 0, 0, 0, 0, 55, 139, 5430, 0, 46, 57, 350, 323, 399, 0, 0, 0, 0, 0, 0, 0, 631, 183, 78, 1, 5429, 248, 103, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 84, 15, 4260, 638, 700, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5744, 6633, 5752, 5902, 5852, 6799, 6770, 5700] avg=6144.0000 max=6799 max_rank=5 min=5700 min_rank=7 imbalance_ratio=1.106608 max_load_reduction=42353 max_load_reduction_pct=0.861674 moved_to_echo=25121 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 108, 2443, 0, 0, 1117, 0, 2547, 73, 0, 0, 0, 0, 0, 0, 0, 3464, 1827, 0, 0, 0, 701, 0, 190, 0, 0, 0, 0, 0, 0, 0, 63, 304, 0, 1338, 4, 55, 0, 0, 4378, 0, 0, 0, 0, 0, 0, 0, 6203, 39, 0, 83, 0, 0, 0, 0, 131, 0, 0, 0, 0, 0, 0, 0, 71, 324, 0, 0, 2800, 187, 0, 80, 1867, 716, 0, 0, 0, 0, 0, 0, 1, 0, 5282, 0, 0, 0, 117, 0, 389, 0, 0, 0, 0, 0, 0, 0, 564, 0, 80, 0, 5287, 0, 23, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4714, 739, 756, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6288, 6182, 6142, 6456, 6045, 5789, 6041, 6209] avg=6144.0000 max=6456 max_rank=3 min=5789 min_rank=5 imbalance_ratio=1.050781 max_load_reduction=42696 max_load_reduction_pct=0.868652 moved_to_echo=24084 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 124, 1508, 0, 0, 2061, 0, 2694, 79, 0, 0, 0, 0, 0, 0, 0, 1774, 3150, 0, 0, 0, 691, 0, 202, 0, 0, 0, 0, 0, 0, 0, 71, 321, 0, 882, 0, 95, 0, 0, 4626, 0, 0, 0, 0, 0, 0, 0, 5586, 0, 0, 92, 0, 0, 0, 0, 139, 0, 0, 0, 0, 0, 0, 0, 148, 342, 0, 0, 2877, 255, 0, 89, 1885, 745, 0, 0, 0, 0, 0, 0, 9, 0, 5801, 0, 0, 0, 36, 0, 407, 0, 0, 0, 0, 0, 0, 0, 467, 0, 88, 0, 5452, 21, 69, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4736, 720, 823, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6466, 5817, 5995, 5817, 6341, 6253, 6184, 6279] avg=6144.0000 max=6466 max_rank=0 min=5817 min_rank=1 imbalance_ratio=1.052409 max_load_reduction=42686 max_load_reduction_pct=0.868449 moved_to_echo=25057 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[896, 2688, 256, 128, 22144, 1664, 384, 768, 14848, 5888, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 896, 15360, 128, 128, 11648, 0, 20736, 640, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 22016, 19712, 128, 384, 384, 5376, 128, 1536, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[640, 2688, 256, 9728, 128, 640, 128, 128, 35584, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 128, 128, 128, 37248, 5760, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 256, 44416, 128, 128, 128, 1024, 384, 3200, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4352, 256, 768, 128, 42624, 384, 512, 768, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[46976, 256, 0, 768, 128, 128, 128, 128, 1152, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[3133, 0, 0, 122, 701, 3267, 0, 0, 1229, 72, 873, 0, 0, 0, 0, 0, 0, 88, 98, 0, 0, 88, 0, 0, 58, 0, 0, 0, 34, 88, 0, 0, 0, 0, 6297, 0, 0, 0, 3701, 825, 0, 7313, 1092, 7592, 62, 0, 5195, 88, 0, 0, 0, 88, 45, 0, 0, 5140, 463, 0, 0, 368, 599, 161, 257, 15] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[2672, 1, 47, 84, 759, 4519, 13, 7, 1168, 64, 463, 0, 4, 5, 3, 0, 0, 87, 137, 39, 1, 88, 9, 5, 61, 2, 4, 12, 13, 89, 2, 2, 19, 0, 5883, 2, 0, 13, 3319, 904, 6, 7193, 1735, 7464, 56, 32, 4831, 88, 3, 23, 3, 87, 23, 7, 37, 4853, 671, 41, 7, 362, 598, 181, 270, 81] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[2933, 0, 0, 78, 747, 4468, 0, 0, 1025, 69, 428, 0, 0, 0, 0, 0, 0, 78, 121, 0, 0, 78, 0, 0, 91, 0, 0, 0, 1, 78, 0, 0, 0, 0, 6309, 1, 0, 0, 3350, 865, 0, 7375, 1449, 7908, 46, 0, 4726, 77, 0, 0, 0, 78, 0, 0, 0, 4881, 399, 0, 0, 372, 644, 182, 244, 51] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[2914, 0, 0, 65, 792, 4265, 0, 0, 1166, 62, 480, 0, 0, 0, 0, 0, 0, 63, 114, 0, 0, 61, 0, 0, 95, 0, 0, 0, 0, 63, 0, 0, 0, 0, 6404, 0, 0, 0, 3255, 920, 0, 7293, 1341, 7680, 46, 0, 4945, 61, 0, 0, 0, 63, 0, 0, 0, 5013, 555, 0, 0, 356, 604, 177, 242, 57] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1819, 5, 394, 69, 643, 5626, 135, 72, 1263, 164, 393, 0, 22, 34, 34, 7, 1, 69, 321, 234, 78, 77, 133, 23, 68, 13, 18, 126, 82, 77, 25, 13, 135, 1, 4787, 3, 11, 88, 3711, 754, 1, 6662, 1742, 7009, 51, 229, 4530, 92, 53, 116, 35, 76, 0, 42, 188, 3787, 868, 370, 17, 512, 576, 107, 311, 250] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[2669, 0, 0, 98, 718, 4922, 0, 0, 1050, 80, 425, 0, 0, 0, 0, 0, 0, 98, 147, 0, 0, 97, 0, 0, 72, 0, 0, 0, 0, 98, 0, 2, 0, 0, 6187, 0, 0, 0, 3186, 834, 0, 7390, 1620, 7922, 52, 0, 4396, 87, 0, 0, 0, 98, 0, 0, 0, 5130, 360, 0, 0, 334, 593, 170, 231, 86] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[2165, 0, 0, 90, 718, 5142, 0, 0, 1023, 69, 414, 0, 0, 0, 0, 0, 0, 90, 130, 0, 0, 89, 0, 0, 116, 0, 0, 0, 0, 90, 0, 0, 0, 0, 6145, 0, 0, 0, 3004, 841, 0, 7350, 2081, 7802, 49, 0, 4491, 72, 0, 0, 0, 90, 0, 0, 0, 5183, 418, 0, 0, 344, 593, 211, 252, 90] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[2857, 0, 0, 80, 786, 3545, 0, 0, 1184, 61, 527, 0, 0, 0, 0, 0, 0, 80, 113, 0, 0, 80, 0, 0, 63, 0, 0, 0, 2, 80, 0, 1, 0, 0, 6401, 0, 0, 0, 4031, 890, 0, 7319, 1253, 7632, 47, 0, 4880, 80, 0, 0, 0, 80, 0, 0, 1, 5075, 602, 0, 0, 355, 622, 170, 244, 11] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3133, 0, 0, 122, 701, 1899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1229, 72, 873, 0, 0, 0, 0, 0, 3189, 1305, 0, 0, 0, 0, 0, 0, 0, 88, 98, 0, 0, 88, 0, 0, 4124, 1679, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 34, 88, 0, 0, 5913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1886, 0, 0, 0, 3701, 825, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1092, 0, 62, 0, 4857, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 45, 0, 0, 5140, 582, 338, 0, 0, 0, 0, 0, 0, 463, 0, 0, 368, 599, 161, 257, 15, 3106, 786, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5855, 6668, 6077, 6093, 6412, 6099, 6193, 5755] avg=6144.0000 max=6668 max_rank=1 min=5755 min_rank=7 imbalance_ratio=1.085286 max_load_reduction=42484 max_load_reduction_pct=0.864339 moved_to_echo=24459 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2672, 1, 47, 84, 759, 2626, 13, 7, 0, 0, 0, 0, 0, 0, 0, 0, 1168, 64, 463, 0, 4, 5, 3, 0, 3137, 1219, 0, 0, 0, 0, 0, 0, 0, 87, 137, 39, 1, 88, 9, 5, 4056, 1651, 0, 0, 0, 0, 0, 0, 61, 2, 4, 12, 13, 89, 2, 2, 5813, 0, 0, 0, 0, 0, 0, 0, 19, 0, 1762, 2, 0, 13, 3319, 904, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1735, 0, 56, 32, 4517, 88, 0, 0, 0, 0, 0, 0, 0, 0, 3, 23, 3, 87, 23, 7, 37, 4853, 805, 314, 0, 0, 0, 0, 0, 0, 671, 41, 7, 362, 598, 181, 270, 81, 2902, 1088, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6209, 6063, 6073, 5998, 6019, 6434, 6155, 6201] avg=6144.0000 max=6434 max_rank=5 min=5998 min_rank=3 imbalance_ratio=1.047201 max_load_reduction=42718 max_load_reduction_pct=0.869100 moved_to_echo=24809 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2914, 0, 0, 65, 792, 2478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1166, 62, 480, 0, 0, 0, 0, 0, 3182, 1327, 0, 0, 0, 0, 0, 0, 0, 63, 114, 0, 0, 61, 0, 0, 4111, 1699, 0, 0, 0, 0, 0, 0, 95, 0, 0, 0, 0, 63, 0, 0, 5981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1918, 0, 0, 0, 3255, 920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1341, 0, 46, 0, 4623, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 5013, 760, 322, 0, 0, 0, 0, 0, 0, 555, 0, 0, 356, 604, 177, 242, 57, 3159, 1027, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6249, 6217, 6048, 6139, 6093, 6071, 6158, 6177] avg=6144.0000 max=6249 max_rank=0 min=6048 min_rank=2 imbalance_ratio=1.017090 max_load_reduction=42903 max_load_reduction_pct=0.872864 moved_to_echo=24499 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2933, 0, 0, 78, 747, 2596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 69, 428, 0, 0, 0, 0, 0, 3217, 1307, 0, 0, 0, 0, 0, 0, 0, 78, 121, 0, 0, 78, 0, 0, 4158, 1749, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 1, 78, 0, 0, 6159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1890, 1, 0, 0, 3350, 865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1449, 0, 46, 0, 4418, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 4881, 796, 308, 0, 0, 0, 0, 0, 0, 399, 0, 0, 372, 644, 182, 244, 51, 3112, 1076, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6354, 6046, 6184, 6329, 6106, 5990, 6063, 6080] avg=6144.0000 max=6354 max_rank=0 min=5990 min_rank=5 imbalance_ratio=1.034180 max_load_reduction=42798 max_load_reduction_pct=0.870728 moved_to_echo=24239 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2669, 0, 0, 98, 718, 2860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 80, 425, 0, 0, 0, 0, 0, 3224, 1282, 0, 0, 0, 0, 0, 0, 0, 98, 147, 0, 0, 97, 0, 0, 4166, 1753, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 98, 0, 2, 6169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1853, 0, 0, 0, 3186, 834, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1620, 0, 52, 0, 4110, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 0, 5130, 877, 286, 0, 0, 0, 0, 0, 0, 360, 0, 0, 334, 593, 170, 231, 86, 3052, 1185, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6345, 6061, 6261, 6341, 5873, 5869, 6391, 6011] avg=6144.0000 max=6391 max_rank=6 min=5869 min_rank=5 imbalance_ratio=1.040202 max_load_reduction=42761 max_load_reduction_pct=0.869975 moved_to_echo=24144 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1819, 5, 394, 69, 643, 3269, 135, 72, 0, 0, 0, 0, 0, 0, 0, 0, 1263, 164, 393, 0, 22, 34, 34, 7, 2906, 992, 0, 0, 0, 0, 0, 0, 1, 69, 321, 234, 78, 77, 133, 23, 3756, 1551, 0, 0, 0, 0, 0, 0, 68, 13, 18, 126, 82, 77, 25, 13, 5458, 0, 0, 0, 0, 0, 0, 0, 135, 1, 1434, 3, 11, 88, 3711, 754, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1742, 0, 51, 229, 4235, 92, 0, 0, 0, 0, 0, 0, 0, 0, 53, 116, 35, 76, 0, 42, 188, 3787, 1003, 295, 0, 0, 0, 0, 0, 0, 868, 370, 17, 512, 576, 107, 311, 250, 2361, 1354, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6406, 5815, 6243, 5880, 6137, 6350, 5595, 6726] avg=6144.0000 max=6726 max_rank=7 min=5595 min_rank=6 imbalance_ratio=1.094727 max_load_reduction=42426 max_load_reduction_pct=0.863159 moved_to_echo=24808 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2857, 0, 0, 80, 786, 2060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1184, 61, 527, 0, 0, 0, 0, 0, 3192, 1327, 0, 0, 0, 0, 0, 0, 0, 80, 113, 0, 0, 80, 0, 0, 4127, 1688, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 2, 80, 0, 1, 5944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1916, 0, 0, 0, 4031, 890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1253, 0, 47, 0, 4562, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 1, 5075, 632, 318, 0, 0, 0, 0, 0, 0, 602, 0, 0, 355, 622, 170, 244, 11, 3158, 853, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5783, 6291, 6088, 6090, 6837, 5942, 6106, 6015] avg=6144.0000 max=6837 max_rank=4 min=5783 min_rank=0 imbalance_ratio=1.112793 max_load_reduction=42315 max_load_reduction_pct=0.860901 moved_to_echo=24900 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2165, 0, 0, 90, 718, 2978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 69, 414, 0, 0, 0, 0, 0, 3207, 1278, 0, 0, 0, 0, 0, 0, 0, 90, 130, 0, 0, 89, 0, 0, 4143, 1726, 0, 0, 0, 0, 0, 0, 116, 0, 0, 0, 0, 90, 0, 0, 6076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1830, 0, 0, 0, 3004, 841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2081, 0, 49, 0, 4195, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 5183, 922, 296, 0, 0, 0, 0, 0, 0, 418, 0, 0, 344, 593, 211, 252, 90, 3037, 1242, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5951, 5991, 6178, 6282, 5675, 6397, 6491, 6187] avg=6144.0000 max=6491 max_rank=6 min=5675 min_rank=4 imbalance_ratio=1.056478 max_load_reduction=42661 max_load_reduction_pct=0.867940 moved_to_echo=24750 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[9216, 768, 4096, 0, 128, 128, 128, 128, 25344, 10112, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[4352, 512, 128, 3072, 4864, 1408, 2176, 768, 23936, 8704, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[256, 128, 14592, 128, 128, 128, 27648, 6912, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[21248, 128, 512, 768, 5888, 20864, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 256, 128, 768, 128, 128, 256, 39168, 6400, 2560, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 0, 12416, 0, 512, 384, 35584, 768, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=50176 observed_loss=-1024 tokens_per_expert=[128, 768, 1280, 384, 128, 768, 256, 128, 32768, 13568, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[640, 128, 128, 256, 256, 768, 128, 128, 47616, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=50176 slack_slots=48128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=17 tokens_per_expert_current_ep_rank=[4521, 1183, 5, 1, 7, 35, 1593, 36, 30, 3918, 25, 980, 186, 6, 782, 0, 15, 175, 1754, 14, 11, 424, 10, 13, 0, 3, 4470, 650, 1636, 19, 593, 486, 47, 12, 9, 0, 38, 0, 1, 3912, 6, 38, 258, 98, 4867, 257, 5757, 14, 6456, 19, 2, 185, 0, 87, 8, 18, 63, 8, 0, 50, 2486, 91, 1, 783] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=17 tokens_per_expert_current_ep_rank=[4383, 705, 0, 0, 0, 0, 1422, 4, 0, 4168, 0, 1133, 132, 0, 937, 0, 0, 176, 2086, 0, 0, 421, 62, 0, 0, 0, 3331, 582, 1383, 0, 469, 360, 0, 0, 0, 0, 0, 0, 0, 5272, 0, 0, 251, 82, 4715, 361, 6491, 0, 6952, 52, 0, 171, 0, 88, 0, 0, 72, 0, 0, 0, 1985, 88, 0, 818] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=17 tokens_per_expert_current_ep_rank=[4034, 1012, 0, 0, 0, 0, 1885, 70, 0, 3363, 0, 790, 0, 0, 683, 0, 0, 195, 1592, 0, 0, 489, 0, 0, 0, 0, 4584, 613, 2177, 0, 546, 618, 0, 0, 0, 0, 0, 0, 0, 3973, 0, 0, 512, 146, 5517, 178, 5812, 0, 6303, 0, 0, 167, 0, 97, 0, 1, 81, 0, 0, 0, 2866, 98, 0, 750] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=17 tokens_per_expert_current_ep_rank=[4146, 722, 0, 0, 0, 0, 1896, 38, 0, 3793, 0, 884, 11, 0, 807, 0, 0, 156, 1764, 0, 0, 490, 0, 0, 0, 0, 4185, 661, 2151, 0, 593, 519, 0, 0, 0, 0, 0, 0, 0, 4319, 0, 0, 342, 120, 5571, 218, 6154, 0, 6468, 1, 0, 187, 0, 78, 0, 0, 70, 0, 0, 0, 1975, 78, 0, 755] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=17 tokens_per_expert_current_ep_rank=[4499, 659, 0, 0, 0, 0, 1823, 16, 0, 4339, 0, 983, 50, 0, 803, 0, 0, 124, 1575, 0, 0, 442, 0, 0, 0, 0, 4297, 688, 1878, 0, 515, 488, 0, 0, 0, 0, 0, 0, 0, 4466, 0, 0, 307, 116, 5210, 214, 6072, 0, 6490, 0, 0, 172, 0, 61, 0, 0, 62, 0, 0, 0, 2046, 63, 0, 694] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=17 tokens_per_expert_current_ep_rank=[3721, 1608, 20, 1, 56, 80, 1514, 95, 104, 3220, 169, 809, 149, 56, 567, 1, 73, 154, 1395, 59, 71, 531, 75, 89, 33, 12, 3234, 589, 2365, 121, 484, 812, 405, 189, 63, 0, 317, 12, 11, 3469, 21, 296, 933, 86, 5151, 225, 5208, 170, 5925, 84, 16, 178, 0, 76, 1, 84, 82, 22, 1, 394, 2697, 92, 19, 658] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=17 tokens_per_expert_current_ep_rank=[3809, 1483, 0, 0, 0, 0, 2149, 82, 0, 3590, 0, 691, 0, 0, 656, 0, 0, 179, 1442, 0, 0, 459, 0, 0, 0, 0, 5134, 628, 1965, 0, 472, 646, 0, 0, 0, 0, 0, 0, 0, 3243, 0, 0, 669, 94, 5474, 123, 5449, 0, 6298, 0, 0, 229, 0, 89, 0, 1, 69, 0, 0, 0, 3107, 89, 0, 833] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=17 tokens_per_expert_current_ep_rank=[4021, 620, 0, 0, 0, 0, 1611, 10, 0, 4861, 0, 901, 129, 0, 891, 0, 0, 160, 2198, 0, 0, 464, 32, 0, 0, 0, 3545, 651, 1759, 0, 535, 471, 0, 0, 0, 0, 0, 0, 0, 4671, 0, 0, 183, 104, 5137, 396, 6227, 0, 6513, 3, 0, 163, 0, 81, 1, 0, 62, 0, 0, 0, 1927, 80, 0, 745] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=17 rerouted_tokens_per_expert_current_ep_rank=[3645, 1183, 5, 1, 7, 35, 1593, 36, 0, 0, 0, 0, 0, 0, 0, 0, 30, 3918, 25, 980, 186, 6, 782, 0, 446, 0, 0, 0, 0, 0, 0, 0, 15, 175, 1754, 14, 11, 424, 10, 13, 3634, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2725, 650, 1636, 19, 593, 486, 0, 0, 0, 0, 0, 0, 0, 0, 47, 12, 9, 0, 38, 0, 1, 3912, 982, 876, 137, 0, 0, 0, 0, 0, 6, 38, 258, 98, 4867, 257, 86, 14, 0, 0, 0, 0, 0, 0, 0, 0, 5873, 19, 2, 185, 0, 87, 8, 18, 0, 0, 0, 0, 0, 0, 0, 0, 63, 8, 0, 50, 2486, 91, 1, 783, 2037, 763, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6505, 6373, 6050, 6112, 6014, 5624, 6192, 6282] avg=6144.0000 max=6505 max_rank=0 min=5624 min_rank=5 imbalance_ratio=1.058757 max_load_reduction=42647 max_load_reduction_pct=0.867655 moved_to_echo=24112 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=17 rerouted_tokens_per_expert_current_ep_rank=[3534, 705, 0, 0, 0, 0, 1422, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4168, 0, 1133, 132, 0, 937, 0, 480, 0, 0, 0, 0, 0, 0, 0, 0, 176, 2086, 0, 0, 421, 62, 0, 4098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2032, 582, 1383, 0, 469, 360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5272, 731, 849, 148, 0, 0, 0, 0, 0, 0, 0, 251, 82, 4715, 361, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6324, 52, 0, 171, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 1985, 88, 0, 818, 2297, 568, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5665, 6850, 6843, 4826, 7000, 5505, 6635, 5828] avg=6144.0000 max=7000 max_rank=4 min=4826 min_rank=3 imbalance_ratio=1.139323 max_load_reduction=42152 max_load_reduction_pct=0.857585 moved_to_echo=24968 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=17 rerouted_tokens_per_expert_current_ep_rank=[3343, 722, 0, 0, 0, 0, 1896, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3793, 0, 884, 11, 0, 807, 0, 447, 0, 0, 0, 0, 0, 0, 0, 0, 156, 1764, 0, 0, 490, 0, 0, 3885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2552, 661, 2151, 0, 593, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4319, 919, 803, 138, 0, 0, 0, 0, 0, 0, 0, 342, 120, 5571, 218, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5883, 1, 0, 187, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 1975, 78, 0, 755, 2178, 714, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5999, 5942, 6295, 6476, 6179, 6342, 6149, 5770] avg=6144.0000 max=6476 max_rank=3 min=5770 min_rank=7 imbalance_ratio=1.054036 max_load_reduction=42676 max_load_reduction_pct=0.868245 moved_to_echo=24440 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=17 rerouted_tokens_per_expert_current_ep_rank=[3627, 659, 0, 0, 0, 0, 1823, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4339, 0, 983, 50, 0, 803, 0, 448, 0, 0, 0, 0, 0, 0, 0, 0, 124, 1575, 0, 0, 442, 0, 0, 3833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2620, 688, 1878, 0, 515, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4466, 944, 872, 138, 0, 0, 0, 0, 0, 0, 0, 307, 116, 5210, 214, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5904, 0, 0, 172, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 2046, 63, 0, 694, 2149, 733, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6125, 6623, 5974, 6189, 6420, 5937, 6137, 5747] avg=6144.0000 max=6623 max_rank=1 min=5747 min_rank=7 imbalance_ratio=1.077962 max_load_reduction=42529 max_load_reduction_pct=0.865255 moved_to_echo=24241 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=17 rerouted_tokens_per_expert_current_ep_rank=[3253, 1012, 0, 0, 0, 0, 1885, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3363, 0, 790, 0, 0, 683, 0, 435, 0, 0, 0, 0, 0, 0, 0, 0, 195, 1592, 0, 0, 489, 0, 0, 3669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2795, 613, 2177, 0, 546, 618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3973, 1007, 781, 134, 0, 0, 0, 0, 0, 0, 0, 512, 146, 5517, 178, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5734, 0, 0, 167, 0, 97, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 2866, 98, 0, 750, 2057, 782, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6220, 5271, 5945, 6749, 5895, 6439, 5999, 6634] avg=6144.0000 max=6749 max_rank=3 min=5271 min_rank=1 imbalance_ratio=1.098470 max_load_reduction=42403 max_load_reduction_pct=0.862691 moved_to_echo=24967 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=17 rerouted_tokens_per_expert_current_ep_rank=[3000, 1608, 20, 1, 56, 80, 1514, 95, 0, 0, 0, 0, 0, 0, 0, 0, 104, 3220, 169, 809, 149, 56, 567, 1, 409, 0, 0, 0, 0, 0, 0, 0, 73, 154, 1395, 59, 71, 531, 75, 89, 3288, 0, 0, 0, 0, 0, 0, 0, 33, 12, 1972, 589, 2365, 121, 484, 812, 0, 0, 0, 0, 0, 0, 0, 0, 405, 189, 63, 0, 317, 12, 11, 3469, 710, 721, 126, 0, 0, 0, 0, 0, 21, 296, 933, 86, 5151, 225, 77, 170, 0, 0, 0, 0, 0, 0, 0, 0, 5390, 84, 16, 178, 0, 76, 1, 84, 0, 0, 0, 0, 0, 0, 0, 0, 82, 22, 1, 394, 2697, 92, 19, 658, 1843, 552, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6374, 5484, 5735, 6388, 6023, 6959, 5829, 6360] avg=6144.0000 max=6959 max_rank=5 min=5484 min_rank=1 imbalance_ratio=1.132650 max_load_reduction=42193 max_load_reduction_pct=0.858419 moved_to_echo=25171 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=17 rerouted_tokens_per_expert_current_ep_rank=[3242, 620, 0, 0, 0, 0, 1611, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4861, 0, 901, 129, 0, 891, 0, 450, 0, 0, 0, 0, 0, 0, 0, 0, 160, 2198, 0, 0, 464, 32, 0, 3931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2162, 651, 1759, 0, 535, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4671, 778, 779, 139, 0, 0, 0, 0, 0, 0, 0, 183, 104, 5137, 396, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5924, 3, 0, 163, 0, 81, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 1927, 80, 0, 745, 2203, 605, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5483, 7232, 6785, 5578, 6367, 5913, 6172, 5622] avg=6144.0000 max=7232 max_rank=1 min=5483 min_rank=0 imbalance_ratio=1.177083 max_load_reduction=41920 max_load_reduction_pct=0.852865 moved_to_echo=24074 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=17 rerouted_tokens_per_expert_current_ep_rank=[3067, 1483, 0, 0, 0, 0, 2149, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3590, 0, 691, 0, 0, 656, 0, 440, 0, 0, 0, 0, 0, 0, 0, 0, 179, 1442, 0, 0, 459, 0, 0, 3445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3123, 628, 1965, 0, 472, 646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3243, 1131, 742, 138, 0, 0, 0, 0, 0, 0, 0, 669, 94, 5474, 123, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5720, 0, 0, 229, 0, 89, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 3107, 89, 0, 833, 1931, 880, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6781, 5377, 5525, 6834, 5254, 6433, 6039, 6909] avg=6144.0000 max=6909 max_rank=7 min=5254 min_rank=4 imbalance_ratio=1.124512 max_load_reduction=42243 max_load_reduction_pct=0.859436 moved_to_echo=24635 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[26752, 8064, 128, 128, 128, 128, 13952, 384, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[640, 128, 128, 512, 19200, 768, 128, 6144, 16768, 5632, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[512, 256, 128, 0, 384, 128, 128, 33408, 7296, 6528, 1152, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 20096, 5120, 15360, 256, 4224, 4480, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 1408, 13824, 128, 128, 3840, 256, 128, 29824, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[256, 31360, 256, 7296, 768, 128, 6144, 128, 3584, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 384, 3456, 896, 41728, 2048, 768, 256, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[46848, 256, 128, 1536, 0, 768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=18 tokens_per_expert_current_ep_rank=[0, 92, 2612, 116, 0, 0, 2849, 0, 0, 16, 138, 510, 7245, 0, 52, 7465, 10, 0, 1, 3, 8, 2318, 5, 32, 23, 0, 9, 7825, 6, 55, 2385, 2, 11, 0, 125, 1, 16, 7138, 24, 7, 6, 0, 459, 0, 49, 57, 30, 20, 1, 37, 34, 11, 3, 354, 41, 87, 6, 5695, 0, 18, 453, 645, 38, 9] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=18 tokens_per_expert_current_ep_rank=[0, 76, 2100, 38, 0, 0, 3466, 0, 0, 0, 132, 373, 7291, 0, 0, 7552, 0, 0, 0, 0, 0, 2439, 0, 0, 0, 0, 0, 8025, 0, 0, 2523, 0, 0, 0, 124, 0, 0, 7260, 0, 0, 0, 0, 469, 0, 0, 0, 0, 21, 0, 0, 19, 0, 0, 393, 0, 78, 0, 6035, 0, 0, 432, 306, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=18 tokens_per_expert_current_ep_rank=[0, 74, 2808, 70, 0, 0, 3494, 0, 0, 0, 160, 308, 7171, 0, 0, 7541, 0, 0, 0, 0, 0, 1502, 6, 0, 0, 0, 0, 7956, 0, 0, 2733, 0, 0, 0, 132, 0, 0, 7517, 0, 0, 0, 0, 457, 0, 0, 0, 0, 16, 0, 0, 3, 0, 0, 326, 0, 98, 0, 5831, 0, 0, 417, 532, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=18 tokens_per_expert_current_ep_rank=[0, 88, 3203, 5, 0, 0, 3513, 0, 0, 0, 150, 298, 7150, 0, 0, 7497, 1, 0, 0, 0, 0, 888, 34, 0, 0, 0, 0, 8003, 0, 0, 3154, 0, 0, 0, 93, 0, 0, 7606, 0, 0, 0, 0, 465, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 290, 0, 90, 0, 5531, 0, 0, 318, 744, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=18 tokens_per_expert_current_ep_rank=[0, 60, 1961, 10, 0, 0, 3610, 0, 0, 0, 118, 383, 7402, 0, 0, 7591, 0, 0, 0, 0, 0, 2515, 1, 0, 0, 0, 0, 8063, 0, 0, 2408, 0, 0, 0, 105, 0, 0, 7178, 0, 0, 0, 0, 452, 0, 0, 0, 0, 13, 0, 0, 34, 0, 0, 396, 0, 63, 0, 6147, 0, 0, 424, 218, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=18 tokens_per_expert_current_ep_rank=[0, 85, 1213, 68, 0, 0, 3611, 0, 0, 0, 156, 439, 7376, 0, 0, 7546, 2, 0, 0, 0, 0, 4018, 0, 0, 0, 0, 0, 8060, 0, 0, 2063, 0, 0, 0, 105, 0, 0, 6069, 0, 0, 0, 0, 542, 0, 0, 0, 0, 33, 0, 0, 49, 0, 0, 447, 0, 88, 0, 6381, 0, 0, 704, 97, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=18 tokens_per_expert_current_ep_rank=[17, 113, 3488, 293, 16, 3, 2908, 0, 0, 6, 121, 236, 6606, 17, 375, 6905, 34, 4, 1, 46, 35, 1018, 42, 284, 25, 9, 18, 7059, 26, 456, 2203, 21, 60, 5, 81, 8, 40, 7012, 183, 102, 18, 0, 346, 0, 153, 490, 198, 101, 0, 91, 20, 90, 21, 285, 399, 82, 49, 4301, 0, 50, 383, 1938, 201, 60] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=18 tokens_per_expert_current_ep_rank=[0, 80, 1477, 175, 0, 0, 4020, 0, 1, 0, 139, 322, 7369, 0, 0, 7578, 0, 0, 0, 0, 0, 3183, 0, 0, 0, 0, 0, 7885, 0, 0, 2178, 0, 1, 0, 132, 0, 0, 6753, 0, 0, 0, 0, 475, 0, 0, 0, 0, 20, 0, 0, 30, 0, 0, 384, 0, 80, 0, 6164, 0, 0, 635, 71, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 92, 2612, 116, 0, 0, 2849, 0, 165, 0, 0, 0, 0, 0, 0, 0, 0, 16, 138, 510, 5622, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 1, 3, 8, 2318, 5, 32, 1623, 1103, 589, 536, 0, 0, 0, 0, 23, 0, 9, 3592, 6, 55, 2385, 2, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 125, 1, 16, 6035, 24, 7, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 459, 0, 49, 57, 30, 20, 3644, 1869, 0, 0, 0, 0, 0, 0, 1, 37, 34, 11, 3, 354, 41, 87, 5596, 0, 0, 0, 0, 0, 0, 0, 6, 4994, 0, 18, 453, 645, 38, 9, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5834, 6338, 6228, 6072, 6219, 6134, 6164, 6163] avg=6144.0000 max=6338 max_rank=1 min=5834 min_rank=0 imbalance_ratio=1.031576 max_load_reduction=42814 max_load_reduction_pct=0.871053 moved_to_echo=24680 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 85, 1213, 68, 0, 0, 3611, 0, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 439, 5724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4018, 0, 0, 1652, 937, 607, 601, 0, 0, 0, 0, 0, 0, 0, 3700, 0, 0, 2063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 5132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 0, 0, 0, 0, 33, 3753, 1890, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 447, 0, 88, 5656, 0, 0, 0, 0, 0, 0, 0, 0, 5595, 0, 0, 704, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5162, 6319, 7817, 5763, 5237, 6218, 6240, 6396] avg=6144.0000 max=7817 max_rank=2 min=5162 min_rank=0 imbalance_ratio=1.272298 max_load_reduction=41335 max_load_reduction_pct=0.840963 moved_to_echo=24091 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 76, 2100, 38, 0, 0, 3466, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 373, 5658, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2439, 0, 0, 1633, 1121, 604, 568, 0, 0, 0, 0, 0, 0, 0, 3684, 0, 0, 2523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 6139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 0, 0, 0, 0, 21, 3737, 1891, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 393, 0, 78, 5661, 0, 0, 0, 0, 0, 0, 0, 0, 5292, 0, 0, 432, 306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5855, 6163, 6365, 6207, 6263, 6118, 6151, 6030] avg=6144.0000 max=6365 max_rank=2 min=5855 min_rank=0 imbalance_ratio=1.035970 max_load_reduction=42787 max_load_reduction_pct=0.870504 moved_to_echo=24562 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 60, 1961, 10, 0, 0, 3610, 0, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 383, 5744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2515, 1, 0, 1658, 1109, 607, 579, 0, 0, 0, 0, 0, 0, 0, 3701, 0, 0, 2408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 6069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 452, 0, 0, 0, 0, 13, 3755, 1902, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 396, 0, 63, 5689, 0, 0, 0, 0, 0, 0, 0, 0, 5390, 0, 0, 424, 218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5819, 6245, 6469, 6109, 6174, 6122, 6182, 6032] avg=6144.0000 max=6469 max_rank=2 min=5819 min_rank=0 imbalance_ratio=1.052897 max_load_reduction=42683 max_load_reduction_pct=0.868388 moved_to_echo=24510 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 74, 2808, 70, 0, 0, 3494, 0, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 308, 5565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1502, 6, 0, 1606, 1161, 599, 549, 0, 0, 0, 0, 0, 0, 0, 3652, 0, 0, 2733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 0, 6356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 457, 0, 0, 0, 0, 16, 3705, 1889, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 326, 0, 98, 5652, 0, 0, 0, 0, 0, 0, 0, 0, 5113, 0, 0, 417, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6615, 6033, 5423, 6385, 6488, 6067, 6079, 6062] avg=6144.0000 max=6615 max_rank=0 min=5423 min_rank=2 imbalance_ratio=1.076660 max_load_reduction=42537 max_load_reduction_pct=0.865417 moved_to_echo=24696 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=18 rerouted_tokens_per_expert_current_ep_rank=[17, 113, 3488, 293, 16, 3, 2908, 0, 125, 0, 0, 0, 0, 0, 0, 0, 0, 6, 121, 236, 5126, 17, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 4, 1, 46, 35, 1018, 42, 284, 1480, 1083, 531, 405, 0, 0, 0, 0, 25, 9, 18, 3241, 26, 456, 2203, 21, 0, 0, 0, 0, 0, 0, 0, 0, 60, 5, 81, 8, 40, 5929, 183, 102, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 346, 0, 153, 490, 198, 101, 3287, 1730, 0, 0, 0, 0, 0, 0, 0, 91, 20, 90, 21, 285, 399, 82, 5175, 0, 0, 0, 0, 0, 0, 0, 49, 3771, 0, 50, 383, 1938, 201, 60, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6963, 5881, 4963, 5999, 6408, 6323, 6163, 6452] avg=6144.0000 max=6963 max_rank=0 min=4963 min_rank=2 imbalance_ratio=1.133301 max_load_reduction=42189 max_load_reduction_pct=0.858337 moved_to_echo=25346 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 80, 1477, 175, 0, 0, 4020, 0, 179, 0, 0, 0, 0, 0, 0, 0, 1, 0, 139, 322, 5718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3183, 0, 0, 1651, 1043, 594, 580, 0, 0, 0, 0, 0, 0, 0, 3619, 0, 0, 2178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 132, 0, 0, 5710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 0, 0, 0, 0, 20, 3672, 1898, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 384, 0, 80, 5680, 0, 0, 0, 0, 0, 0, 0, 0, 5405, 0, 0, 635, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5931, 6180, 7051, 5797, 5843, 6065, 6174, 6111] avg=6144.0000 max=7051 max_rank=2 min=5797 min_rank=3 imbalance_ratio=1.147624 max_load_reduction=42101 max_load_reduction_pct=0.856547 moved_to_echo=24193 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 88, 3203, 5, 0, 0, 3513, 0, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 298, 5545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 888, 34, 0, 1605, 1179, 606, 523, 0, 0, 0, 0, 0, 0, 0, 3666, 0, 0, 3154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 0, 6427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 0, 0, 0, 0, 31, 3731, 1878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 290, 0, 90, 5619, 0, 0, 0, 0, 0, 0, 0, 0, 4844, 0, 0, 318, 744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6973, 5993, 4836, 6820, 6520, 6105, 5999, 5906] avg=6144.0000 max=6973 max_rank=0 min=4836 min_rank=2 imbalance_ratio=1.134928 max_load_reduction=42179 max_load_reduction_pct=0.858134 moved_to_echo=24530 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 17920, 128, 384, 12928, 8832, 4864, 4352, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 0, 3712, 0, 256, 640, 256, 256, 29312, 14976, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 128, 28928, 128, 512, 19712, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 768, 18944, 896, 128, 128, 27520, 0, 1408, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 1152, 2944, 44800, 128, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 256, 128, 128, 2944, 512, 768, 44800, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 40448, 0, 128, 3840, 4608, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 1024, 128, 128, 47872, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=19 tokens_per_expert_current_ep_rank=[71, 0, 0, 0, 0, 74, 0, 0, 0, 0, 156, 0, 0, 51, 1214, 0, 4382, 104, 89, 7616, 7523, 0, 0, 0, 0, 144, 1052, 0, 7601, 0, 51, 0, 5486, 0, 0, 0, 0, 3196, 7492, 78, 0, 0, 0, 0, 0, 51, 0, 84, 0, 992, 0, 199, 0, 0, 0, 0, 0, 0, 0, 249, 495, 0, 330, 372] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=19 tokens_per_expert_current_ep_rank=[35, 2, 333, 0, 477, 70, 244, 46, 25, 7, 205, 109, 4, 100, 964, 4, 4158, 373, 141, 6901, 6266, 2, 255, 113, 0, 178, 2902, 0, 6798, 175, 227, 13, 4265, 105, 173, 74, 148, 1595, 6380, 88, 2, 72, 13, 332, 11, 40, 2, 37, 25, 536, 51, 966, 0, 361, 0, 1, 146, 25, 297, 204, 450, 440, 788, 398] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=19 tokens_per_expert_current_ep_rank=[68, 0, 0, 0, 0, 88, 0, 0, 0, 0, 176, 0, 0, 62, 1089, 0, 4376, 14, 88, 7567, 7523, 0, 0, 0, 0, 349, 702, 0, 7669, 0, 62, 0, 4123, 0, 0, 0, 0, 4503, 7578, 88, 0, 0, 0, 0, 0, 62, 0, 57, 0, 1458, 0, 226, 0, 0, 0, 0, 0, 0, 0, 112, 434, 0, 309, 369] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=19 tokens_per_expert_current_ep_rank=[99, 0, 0, 0, 0, 80, 0, 0, 0, 0, 196, 0, 0, 64, 1269, 0, 4222, 222, 115, 7559, 7553, 0, 0, 0, 0, 0, 1421, 0, 7605, 0, 64, 0, 6030, 0, 10, 0, 0, 2440, 7396, 98, 0, 0, 0, 0, 0, 61, 0, 60, 0, 986, 0, 199, 0, 0, 0, 0, 0, 0, 1, 250, 479, 0, 339, 334] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=19 tokens_per_expert_current_ep_rank=[63, 0, 0, 0, 0, 61, 0, 0, 0, 0, 126, 0, 0, 54, 1163, 0, 4192, 64, 63, 7676, 7591, 0, 0, 0, 0, 182, 1026, 0, 7654, 0, 54, 0, 5258, 0, 3, 0, 0, 3528, 7470, 63, 0, 0, 0, 0, 0, 53, 0, 88, 0, 1139, 0, 169, 0, 0, 0, 0, 0, 0, 0, 246, 507, 0, 303, 356] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=19 tokens_per_expert_current_ep_rank=[79, 0, 0, 0, 0, 80, 3, 0, 0, 1, 161, 0, 0, 52, 1144, 0, 4087, 8, 81, 7600, 7567, 0, 0, 0, 0, 247, 818, 0, 7680, 0, 52, 0, 4931, 0, 0, 0, 1, 4126, 7526, 80, 0, 0, 0, 0, 0, 52, 0, 63, 0, 1116, 0, 197, 0, 0, 0, 0, 0, 0, 0, 221, 488, 0, 336, 355] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=19 tokens_per_expert_current_ep_rank=[44, 0, 50, 0, 62, 87, 42, 11, 19, 1, 166, 13, 0, 67, 1221, 7, 3917, 261, 133, 7460, 7302, 2, 24, 15, 0, 42, 1478, 0, 7503, 20, 61, 0, 4982, 11, 29, 4, 24, 3518, 7319, 87, 1, 13, 0, 37, 0, 53, 0, 51, 24, 929, 22, 419, 0, 45, 0, 0, 22, 25, 32, 224, 493, 52, 379, 349] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=19 tokens_per_expert_current_ep_rank=[88, 0, 0, 0, 0, 44, 0, 0, 0, 0, 179, 0, 0, 61, 1347, 0, 3717, 203, 135, 7548, 7470, 0, 0, 0, 0, 0, 1967, 0, 7613, 0, 61, 0, 6295, 0, 49, 0, 0, 2133, 7409, 90, 0, 0, 0, 0, 0, 61, 0, 95, 0, 914, 0, 243, 0, 0, 0, 0, 0, 0, 0, 288, 467, 0, 331, 344] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=19 rerouted_tokens_per_expert_current_ep_rank=[68, 0, 0, 0, 0, 88, 0, 0, 4466, 1543, 0, 0, 0, 0, 0, 0, 0, 0, 176, 0, 0, 62, 1089, 0, 4006, 812, 0, 0, 0, 0, 0, 0, 4376, 14, 88, 0, 1740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 349, 702, 0, 4568, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2274, 0, 0, 0, 0, 4503, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 57, 6024, 0, 0, 0, 0, 0, 0, 0, 0, 1458, 0, 226, 0, 0, 0, 0, 3112, 1777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 434, 0, 309, 369, 1849, 2289, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6165, 6145, 6218, 5681, 6865, 6143, 6573, 5362] avg=6144.0000 max=6865 max_rank=4 min=5362 min_rank=7 imbalance_ratio=1.117350 max_load_reduction=42287 max_load_reduction_pct=0.860331 moved_to_echo=24943 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=19 rerouted_tokens_per_expert_current_ep_rank=[71, 0, 0, 0, 0, 74, 0, 0, 4416, 1553, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 0, 51, 1214, 0, 4006, 804, 0, 0, 0, 0, 0, 0, 4382, 104, 89, 0, 1740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 1052, 0, 4528, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3025, 0, 0, 0, 0, 3196, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 84, 6063, 0, 0, 0, 0, 0, 0, 0, 0, 992, 0, 199, 0, 0, 0, 0, 3076, 1777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 249, 495, 0, 330, 372, 2461, 2269, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6114, 6231, 6315, 5775, 6299, 6198, 6044, 6176] avg=6144.0000 max=6315 max_rank=2 min=5775 min_rank=3 imbalance_ratio=1.027832 max_load_reduction=42837 max_load_reduction_pct=0.871521 moved_to_echo=24717 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=19 rerouted_tokens_per_expert_current_ep_rank=[44, 0, 50, 0, 62, 87, 42, 11, 4314, 1521, 0, 0, 0, 0, 0, 0, 19, 1, 166, 13, 0, 67, 1221, 7, 3888, 794, 0, 0, 0, 0, 0, 0, 3917, 261, 133, 0, 1689, 2, 24, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 1478, 0, 4469, 20, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2747, 11, 29, 4, 24, 3518, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 1, 13, 0, 37, 0, 53, 0, 51, 5939, 0, 0, 0, 0, 0, 0, 0, 24, 929, 22, 419, 0, 45, 0, 0, 3005, 1725, 0, 0, 0, 0, 0, 0, 22, 25, 32, 224, 493, 52, 379, 349, 2235, 2240, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6131, 6176, 6041, 6070, 6420, 6094, 6169, 6051] avg=6144.0000 max=6420 max_rank=4 min=6041 min_rank=2 imbalance_ratio=1.044922 max_load_reduction=42732 max_load_reduction_pct=0.869385 moved_to_echo=24734 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=19 rerouted_tokens_per_expert_current_ep_rank=[63, 0, 0, 0, 0, 61, 0, 0, 4404, 1567, 0, 0, 0, 0, 0, 0, 0, 0, 126, 0, 0, 54, 1163, 0, 4042, 810, 0, 0, 0, 0, 0, 0, 4192, 64, 63, 0, 1756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, 1026, 0, 4559, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2900, 0, 3, 0, 0, 3528, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 88, 6109, 0, 0, 0, 0, 0, 0, 0, 0, 1139, 0, 169, 0, 0, 0, 0, 3066, 1793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 507, 0, 303, 356, 2358, 2285, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6095, 6195, 6075, 5821, 6494, 6250, 6167, 6055] avg=6144.0000 max=6494 max_rank=4 min=5821 min_rank=3 imbalance_ratio=1.056966 max_load_reduction=42658 max_load_reduction_pct=0.867879 moved_to_echo=24966 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=19 rerouted_tokens_per_expert_current_ep_rank=[35, 2, 333, 0, 477, 70, 244, 46, 3761, 1408, 0, 0, 0, 0, 0, 0, 25, 7, 205, 109, 4, 100, 964, 4, 3336, 719, 0, 0, 0, 0, 0, 0, 4158, 373, 141, 0, 1450, 2, 255, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 178, 2902, 0, 4050, 175, 227, 13, 0, 0, 0, 0, 0, 0, 0, 0, 2352, 105, 173, 74, 148, 1595, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 2, 72, 13, 332, 11, 40, 2, 37, 5493, 0, 0, 0, 0, 0, 0, 0, 25, 536, 51, 966, 0, 361, 0, 1, 2619, 1480, 0, 0, 0, 0, 0, 0, 146, 25, 297, 204, 450, 440, 788, 398, 1913, 2029, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6376, 5473, 6492, 7545, 4535, 6002, 6039, 6690] avg=6144.0000 max=7545 max_rank=3 min=4535 min_rank=4 imbalance_ratio=1.228027 max_load_reduction=41607 max_load_reduction_pct=0.846497 moved_to_echo=23266 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=19 rerouted_tokens_per_expert_current_ep_rank=[99, 0, 0, 0, 0, 80, 0, 0, 4360, 1543, 0, 0, 0, 0, 0, 0, 0, 0, 196, 0, 0, 64, 1269, 0, 4022, 805, 0, 0, 0, 0, 0, 0, 4222, 222, 115, 0, 1747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1421, 0, 4530, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3325, 0, 10, 0, 0, 2440, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 60, 6016, 0, 0, 0, 0, 0, 0, 0, 0, 986, 0, 199, 0, 0, 0, 0, 3036, 1784, 0, 0, 0, 0, 0, 0, 0, 0, 1, 250, 479, 0, 339, 334, 2705, 2270, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6082, 6356, 6306, 6015, 5873, 6137, 6005, 6378] avg=6144.0000 max=6378 max_rank=7 min=5873 min_rank=4 imbalance_ratio=1.038086 max_load_reduction=42774 max_load_reduction_pct=0.870239 moved_to_echo=24393 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=19 rerouted_tokens_per_expert_current_ep_rank=[79, 0, 0, 0, 0, 80, 3, 0, 4436, 1551, 0, 0, 0, 0, 0, 0, 0, 1, 161, 0, 0, 52, 1144, 0, 4029, 813, 0, 0, 0, 0, 0, 0, 4087, 8, 81, 0, 1751, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 818, 0, 4574, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2719, 0, 0, 0, 1, 4126, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 63, 6049, 0, 0, 0, 0, 0, 0, 0, 0, 1116, 0, 197, 0, 0, 0, 0, 3090, 1787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 221, 488, 0, 336, 355, 2212, 2293, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6149, 6200, 5927, 5691, 6926, 6164, 6190, 5905] avg=6144.0000 max=6926 max_rank=4 min=5691 min_rank=3 imbalance_ratio=1.127279 max_load_reduction=42226 max_load_reduction_pct=0.859090 moved_to_echo=25185 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=19 rerouted_tokens_per_expert_current_ep_rank=[88, 0, 0, 0, 0, 44, 0, 0, 4368, 1540, 0, 0, 0, 0, 0, 0, 0, 0, 179, 0, 0, 61, 1347, 0, 3980, 809, 0, 0, 0, 0, 0, 0, 3717, 203, 135, 0, 1723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1967, 0, 4526, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3468, 0, 49, 0, 0, 2133, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 95, 6008, 0, 0, 0, 0, 0, 0, 0, 0, 914, 0, 243, 0, 0, 0, 0, 3041, 1767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 288, 467, 0, 331, 344, 2827, 2278, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6040, 6376, 5778, 6554, 5740, 6164, 5965, 6535] avg=6144.0000 max=6554 max_rank=3 min=5740 min_rank=4 imbalance_ratio=1.066732 max_load_reduction=42598 max_load_reduction_pct=0.866659 moved_to_echo=24404 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 8192, 128, 2688, 0, 512, 0, 128, 24064, 13952, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[256, 128, 384, 1920, 3840, 512, 3200, 2944, 18560, 18048, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[640, 128, 384, 0, 640, 640, 384, 128, 34560, 12288, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 1408, 128, 128, 512, 9472, 128, 31360, 6400, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[22912, 128, 384, 128, 256, 25088, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[33152, 1280, 896, 0, 13696, 128, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 1152, 11392, 0, 35840, 256, 640, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 128, 384, 128, 512, 128, 640, 47744, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=20 tokens_per_expert_current_ep_rank=[1, 0, 41, 3445, 13, 22, 0, 0, 4, 1, 2, 86, 18, 79, 2475, 29, 7, 2, 2146, 1506, 7804, 655, 4, 88, 7441, 23, 1, 7220, 473, 54, 1, 6, 0, 46, 231, 50, 52, 3, 0, 3358, 35, 811, 8, 52, 1, 0, 85, 0, 9, 8, 64, 1098, 4273, 5, 2090, 211, 1113, 93, 10, 0, 57, 1539, 165, 38] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 2911, 0, 0, 0, 0, 0, 0, 29, 88, 0, 88, 2631, 0, 0, 0, 3316, 1838, 8103, 720, 0, 88, 7586, 0, 0, 7451, 469, 62, 0, 0, 0, 0, 101, 0, 60, 0, 0, 2862, 0, 562, 0, 0, 0, 0, 88, 0, 0, 0, 0, 1474, 3448, 0, 1331, 168, 992, 88, 0, 0, 0, 2433, 165, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 3086, 0, 0, 0, 0, 0, 0, 0, 98, 0, 98, 2031, 0, 0, 0, 2086, 1382, 8090, 669, 0, 98, 7488, 0, 0, 7291, 323, 64, 0, 0, 0, 0, 204, 0, 57, 0, 0, 3463, 0, 1114, 0, 0, 0, 0, 96, 0, 0, 0, 0, 882, 4070, 0, 2924, 253, 1137, 97, 0, 0, 0, 1843, 206, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 2873, 0, 0, 0, 0, 0, 0, 0, 63, 0, 62, 2826, 0, 0, 0, 2434, 1835, 8124, 638, 0, 63, 7648, 0, 0, 7409, 440, 54, 0, 0, 0, 0, 134, 0, 39, 0, 0, 3443, 0, 795, 0, 0, 0, 0, 59, 0, 0, 0, 0, 1152, 3865, 0, 1840, 147, 1184, 61, 0, 0, 0, 1831, 131, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 2535, 0, 0, 0, 0, 0, 0, 1, 78, 0, 78, 2454, 0, 0, 0, 2206, 1745, 8113, 664, 0, 78, 7570, 0, 0, 7369, 403, 51, 0, 0, 0, 0, 159, 0, 51, 0, 0, 3684, 0, 933, 0, 0, 0, 0, 73, 0, 0, 0, 0, 1269, 3766, 0, 2340, 168, 1199, 78, 0, 0, 0, 1924, 158, 5] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=20 tokens_per_expert_current_ep_rank=[3, 0, 335, 4246, 136, 214, 1, 0, 35, 22, 0, 79, 238, 65, 2151, 175, 10, 19, 834, 884, 6731, 596, 60, 79, 6536, 194, 17, 6099, 292, 101, 13, 1, 0, 322, 273, 459, 47, 3, 0, 2371, 235, 1057, 47, 429, 30, 0, 68, 3, 93, 69, 183, 1058, 5591, 37, 3780, 215, 980, 104, 71, 12, 424, 567, 370, 88] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 3196, 0, 0, 0, 0, 0, 0, 8, 81, 0, 80, 2552, 0, 0, 0, 2807, 1715, 8107, 659, 0, 80, 7616, 0, 0, 7503, 461, 52, 0, 0, 0, 0, 109, 0, 52, 0, 0, 3205, 0, 683, 0, 0, 0, 0, 79, 0, 0, 0, 0, 1295, 3469, 0, 1475, 162, 1224, 80, 0, 0, 0, 2248, 152, 2] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 3218, 0, 0, 0, 0, 0, 0, 0, 90, 0, 88, 1759, 0, 0, 0, 2484, 1496, 8077, 612, 0, 89, 7588, 0, 0, 7326, 188, 61, 0, 0, 0, 0, 213, 0, 59, 0, 0, 3393, 0, 1235, 0, 0, 0, 0, 46, 0, 0, 0, 0, 792, 3754, 0, 3282, 257, 1207, 89, 0, 0, 0, 1476, 231, 42] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=20 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 41, 3445, 13, 22, 0, 0, 2223, 647, 0, 0, 0, 0, 0, 0, 4, 1, 2, 86, 18, 79, 2475, 29, 2370, 1169, 0, 0, 0, 0, 0, 0, 7, 2, 2146, 1506, 1541, 655, 4, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 1, 5678, 473, 54, 1, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 231, 50, 52, 3, 0, 3358, 1542, 1104, 0, 0, 0, 0, 0, 0, 35, 811, 8, 52, 1, 0, 85, 0, 5071, 0, 0, 0, 0, 0, 0, 0, 9, 8, 64, 1098, 2522, 5, 2090, 211, 0, 0, 0, 0, 0, 0, 0, 0, 1113, 93, 10, 0, 57, 1539, 165, 38, 2871, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6392, 6233, 5949, 6236, 6386, 6063, 6007, 5886] avg=6144.0000 max=6392 max_rank=0 min=5886 min_rank=7 imbalance_ratio=1.040365 max_load_reduction=42760 max_load_reduction_pct=0.869954 moved_to_echo=24342 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2911, 0, 0, 0, 0, 2308, 522, 0, 0, 0, 0, 0, 0, 0, 0, 29, 88, 0, 88, 2631, 0, 2416, 1214, 0, 0, 0, 0, 0, 0, 0, 0, 3316, 1838, 1600, 720, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5860, 469, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 0, 60, 0, 0, 2862, 1591, 890, 0, 0, 0, 0, 0, 0, 0, 562, 0, 0, 0, 0, 88, 0, 5170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1474, 2036, 0, 1331, 168, 0, 0, 0, 0, 0, 0, 0, 0, 992, 88, 0, 0, 0, 2433, 165, 0, 2981, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5741, 6466, 7562, 6391, 5504, 5820, 5009, 6659] avg=6144.0000 max=7562 max_rank=2 min=5009 min_rank=6 imbalance_ratio=1.230794 max_load_reduction=41590 max_load_reduction_pct=0.846151 moved_to_echo=22992 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2535, 0, 0, 0, 0, 2311, 570, 0, 0, 0, 0, 0, 0, 0, 0, 1, 78, 0, 78, 2454, 0, 2411, 1216, 0, 0, 0, 0, 0, 0, 0, 0, 2206, 1745, 1602, 664, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5795, 403, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 159, 0, 51, 0, 0, 3684, 1574, 973, 0, 0, 0, 0, 0, 0, 0, 933, 0, 0, 0, 0, 73, 0, 5159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1269, 2223, 0, 2340, 168, 0, 0, 0, 0, 0, 0, 0, 0, 1199, 78, 0, 0, 0, 1924, 158, 5, 2984, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5416, 6238, 6295, 6249, 6441, 6165, 6000, 6348] avg=6144.0000 max=6441 max_rank=4 min=5416 min_rank=0 imbalance_ratio=1.048340 max_load_reduction=42711 max_load_reduction_pct=0.868958 moved_to_echo=24954 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2873, 0, 0, 0, 0, 2314, 585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 62, 2826, 0, 2436, 1217, 0, 0, 0, 0, 0, 0, 0, 0, 2434, 1835, 1604, 638, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5827, 440, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, 0, 39, 0, 0, 3443, 1582, 998, 0, 0, 0, 0, 0, 0, 0, 795, 0, 0, 0, 0, 59, 0, 5212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1152, 2282, 0, 1840, 147, 0, 0, 0, 0, 0, 0, 0, 0, 1184, 61, 0, 0, 0, 1831, 131, 2, 2989, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5772, 6604, 6574, 6321, 6196, 6066, 5421, 6198] avg=6144.0000 max=6604 max_rank=1 min=5421 min_rank=6 imbalance_ratio=1.074870 max_load_reduction=42548 max_load_reduction_pct=0.865641 moved_to_echo=23881 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3086, 0, 0, 0, 0, 2304, 616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 98, 2031, 0, 2386, 1212, 0, 0, 0, 0, 0, 0, 0, 0, 2086, 1382, 1598, 669, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5734, 323, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 204, 0, 57, 0, 0, 3463, 1557, 1051, 0, 0, 0, 0, 0, 0, 0, 1114, 0, 0, 0, 0, 96, 0, 5102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 882, 2403, 0, 2924, 253, 0, 0, 0, 0, 0, 0, 0, 0, 1137, 97, 0, 0, 0, 1843, 206, 2, 2976, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6006, 5825, 5833, 6121, 6332, 6312, 6462, 6261] avg=6144.0000 max=6462 max_rank=6 min=5825 min_rank=1 imbalance_ratio=1.051758 max_load_reduction=42690 max_load_reduction_pct=0.868530 moved_to_echo=25367 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=20 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 335, 4246, 136, 214, 1, 0, 1917, 847, 0, 0, 0, 0, 0, 0, 35, 22, 0, 79, 238, 65, 2151, 175, 2082, 1008, 0, 0, 0, 0, 0, 0, 10, 19, 834, 884, 1330, 596, 60, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 194, 17, 4797, 292, 101, 13, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 322, 273, 459, 47, 3, 0, 2371, 1302, 1444, 0, 0, 0, 0, 0, 0, 235, 1057, 47, 429, 30, 0, 68, 3, 4454, 0, 0, 0, 0, 0, 0, 0, 93, 69, 183, 1058, 3300, 37, 3780, 215, 0, 0, 0, 0, 0, 0, 0, 0, 980, 104, 71, 12, 424, 567, 370, 88, 2476, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7699, 5855, 3812, 5415, 6221, 6323, 8735, 5092] avg=6144.0000 max=8735 max_rank=6 min=3812 min_rank=2 imbalance_ratio=1.421712 max_load_reduction=40417 max_load_reduction_pct=0.822286 moved_to_echo=26371 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3196, 0, 0, 0, 0, 2309, 525, 0, 0, 0, 0, 0, 0, 0, 0, 8, 81, 0, 80, 2552, 0, 2426, 1215, 0, 0, 0, 0, 0, 0, 0, 0, 2807, 1715, 1601, 659, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5901, 461, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 0, 52, 0, 0, 3205, 1602, 896, 0, 0, 0, 0, 0, 0, 0, 683, 0, 0, 0, 0, 79, 0, 5190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1295, 2048, 0, 1475, 162, 0, 0, 0, 0, 0, 0, 0, 0, 1224, 80, 0, 0, 0, 2248, 152, 2, 2982, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6030, 6362, 6862, 6414, 5864, 5952, 4980, 6688] avg=6144.0000 max=6862 max_rank=2 min=4980 min_rank=6 imbalance_ratio=1.116862 max_load_reduction=42290 max_load_reduction_pct=0.860392 moved_to_echo=23484 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3218, 0, 0, 0, 0, 2304, 574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 88, 1759, 0, 2418, 1214, 0, 0, 0, 0, 0, 0, 0, 0, 2484, 1496, 1584, 612, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5756, 188, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 213, 0, 59, 0, 0, 3393, 1570, 973, 0, 0, 0, 0, 0, 0, 0, 1235, 0, 0, 0, 0, 46, 0, 5170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 792, 2207, 0, 3282, 257, 0, 0, 0, 0, 0, 0, 0, 0, 1207, 89, 0, 0, 0, 1476, 231, 42, 2975, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6096, 5569, 6265, 6005, 6208, 6451, 6538, 6020] avg=6144.0000 max=6538 max_rank=6 min=5569 min_rank=1 imbalance_ratio=1.064128 max_load_reduction=42614 max_load_reduction_pct=0.866984 moved_to_echo=25217 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 0, 384, 25600, 256, 256, 128, 0, 18048, 4992, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 384, 1536, 512, 512, 128, 0, 25856, 12416, 8448, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 18432, 12416, 12544, 5248, 128, 768, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[9088, 768, 128, 128, 512, 13952, 1664, 256, 23296, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 128, 256, 9088, 19072, 128, 19072, 1664, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 128, 768, 256, 640, 18944, 256, 19072, 9472, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[384, 7296, 128, 512, 128, 0, 640, 128, 40576, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 256, 128, 45440, 3072, 512, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=21 tokens_per_expert_current_ep_rank=[0, 5923, 787, 1073, 7176, 7150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 768, 0, 0, 0, 0, 943, 538, 88, 0, 0, 0, 0, 0, 84, 0, 0, 6588, 3567, 139, 0, 8, 879, 0, 0, 0, 0, 0, 0, 1283, 0, 0, 0, 0, 72, 6415, 187, 5323, 0, 0, 88, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=21 tokens_per_expert_current_ep_rank=[0, 5086, 745, 1209, 6973, 6844, 13, 0, 5, 3, 7, 23, 6, 4, 29, 16, 59, 10, 83, 1, 35, 1002, 0, 0, 1, 0, 1012, 1071, 97, 0, 29, 8, 6, 6, 182, 1, 1, 6076, 4768, 113, 7, 0, 906, 1, 25, 19, 31, 36, 39, 1296, 20, 0, 7, 64, 63, 5350, 127, 5468, 2, 43, 87, 3, 6, 28] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=21 tokens_per_expert_current_ep_rank=[0, 5482, 691, 1437, 7459, 7046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 834, 0, 0, 0, 0, 660, 684, 61, 0, 0, 0, 0, 0, 96, 0, 0, 6316, 4589, 116, 0, 0, 856, 0, 0, 0, 0, 0, 0, 1238, 0, 0, 0, 0, 62, 5424, 129, 5833, 0, 0, 61, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=21 tokens_per_expert_current_ep_rank=[1, 4136, 637, 1120, 6450, 5983, 76, 0, 22, 34, 73, 110, 125, 37, 180, 155, 469, 169, 94, 13, 265, 1353, 1, 0, 8, 0, 878, 2535, 75, 0, 238, 21, 12, 23, 440, 22, 17, 5114, 3876, 126, 60, 10, 649, 42, 270, 106, 14, 423, 158, 1140, 322, 0, 59, 511, 75, 3673, 94, 5993, 39, 332, 72, 25, 19, 178] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=21 tokens_per_expert_current_ep_rank=[0, 4946, 670, 1299, 7448, 6849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 667, 0, 0, 0, 0, 640, 1244, 97, 0, 0, 0, 0, 0, 178, 0, 0, 5906, 5843, 149, 0, 0, 813, 0, 0, 0, 0, 0, 0, 1283, 3, 0, 0, 0, 82, 4393, 97, 6361, 0, 0, 97, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=21 tokens_per_expert_current_ep_rank=[0, 5616, 763, 1238, 7205, 7106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 838, 0, 0, 0, 0, 840, 679, 80, 0, 0, 0, 0, 0, 65, 0, 0, 6341, 3828, 114, 0, 7, 915, 0, 0, 0, 0, 0, 0, 1275, 0, 0, 0, 0, 62, 6178, 141, 5718, 0, 0, 80, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=21 tokens_per_expert_current_ep_rank=[0, 5106, 689, 1452, 7441, 6915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 0, 858, 0, 0, 0, 0, 638, 760, 78, 0, 0, 0, 0, 0, 120, 0, 0, 6116, 5275, 121, 0, 0, 795, 0, 0, 0, 0, 0, 0, 1298, 0, 0, 0, 0, 70, 4944, 93, 6161, 0, 0, 78, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=21 tokens_per_expert_current_ep_rank=[0, 4759, 664, 1350, 7416, 6852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 847, 0, 0, 0, 0, 672, 1430, 88, 0, 0, 0, 0, 0, 131, 0, 0, 5715, 6251, 130, 0, 0, 795, 0, 0, 0, 0, 0, 0, 1369, 10, 0, 0, 0, 69, 4056, 89, 6300, 0, 0, 89, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 4796, 787, 1073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 768, 0, 0, 2939, 2429, 0, 0, 0, 0, 0, 0, 0, 0, 943, 538, 88, 0, 0, 0, 2959, 1127, 509, 32, 0, 0, 0, 0, 0, 0, 84, 0, 0, 1200, 3567, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 879, 0, 0, 0, 0, 0, 4211, 1153, 0, 0, 0, 0, 0, 0, 0, 1283, 0, 0, 0, 0, 72, 5906, 0, 0, 0, 0, 0, 0, 0, 0, 187, 5291, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6656, 6023, 6209, 6196, 4990, 6251, 7261, 5566] avg=6144.0000 max=7261 max_rank=6 min=4990 min_rank=4 imbalance_ratio=1.181803 max_load_reduction=41891 max_load_reduction_pct=0.852275 moved_to_echo=24068 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 4118, 745, 1209, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3, 7, 23, 6, 4, 29, 16, 5853, 0, 0, 0, 0, 0, 0, 0, 59, 10, 83, 1, 35, 1002, 0, 0, 2813, 2240, 0, 0, 0, 0, 0, 0, 1, 0, 1012, 1071, 97, 0, 29, 8, 2729, 968, 424, 33, 0, 0, 0, 0, 6, 6, 182, 1, 1, 1107, 4768, 113, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 906, 1, 25, 19, 31, 36, 4031, 1120, 0, 0, 0, 0, 0, 0, 39, 1296, 20, 0, 7, 64, 63, 4926, 0, 0, 0, 0, 0, 0, 0, 0, 127, 5435, 2, 43, 87, 3, 6, 28, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6085, 5946, 6243, 6372, 6184, 6176, 6415, 5731] avg=6144.0000 max=6415 max_rank=6 min=5731 min_rank=7 imbalance_ratio=1.044108 max_load_reduction=42737 max_load_reduction_pct=0.869486 moved_to_echo=24506 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 4439, 691, 1437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 834, 0, 0, 2897, 2329, 0, 0, 0, 0, 0, 0, 0, 0, 660, 684, 61, 0, 0, 0, 2837, 1043, 430, 35, 0, 0, 0, 0, 0, 0, 96, 0, 0, 1150, 4589, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 856, 0, 0, 0, 0, 0, 4149, 1198, 0, 0, 0, 0, 0, 0, 0, 1238, 0, 0, 0, 0, 62, 4994, 0, 0, 0, 0, 0, 0, 0, 0, 129, 5798, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6567, 6261, 6138, 5750, 5951, 6203, 6294, 5988] avg=6144.0000 max=6567 max_rank=0 min=5750 min_rank=3 imbalance_ratio=1.068848 max_load_reduction=42585 max_load_reduction_pct=0.866394 moved_to_echo=24436 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 4005, 670, 1299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 667, 0, 0, 2816, 2178, 0, 0, 0, 0, 0, 0, 0, 0, 640, 1244, 97, 0, 0, 0, 2653, 941, 348, 38, 0, 0, 0, 0, 0, 0, 178, 0, 0, 1075, 5843, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 813, 0, 0, 0, 0, 0, 4033, 1196, 0, 0, 0, 0, 0, 0, 0, 1283, 3, 0, 0, 0, 82, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 97, 6323, 0, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5974, 6252, 5748, 5961, 7245, 6042, 5413, 6517] avg=6144.0000 max=7245 max_rank=4 min=5413 min_rank=6 imbalance_ratio=1.179199 max_load_reduction=41907 max_load_reduction_pct=0.852600 moved_to_echo=25217 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 4134, 689, 1452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 0, 858, 0, 0, 2842, 2255, 0, 0, 0, 0, 0, 0, 0, 0, 638, 760, 78, 0, 0, 0, 2747, 972, 392, 37, 0, 0, 0, 0, 0, 0, 120, 0, 0, 1114, 5275, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 795, 0, 0, 0, 0, 0, 4073, 1195, 0, 0, 0, 0, 0, 0, 0, 1298, 0, 0, 0, 0, 70, 4552, 0, 0, 0, 0, 0, 0, 0, 0, 93, 6124, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6275, 6246, 6099, 5624, 6630, 6063, 5920, 6295] avg=6144.0000 max=6630 max_rank=4 min=5624 min_rank=3 imbalance_ratio=1.079102 max_load_reduction=42522 max_load_reduction_pct=0.865112 moved_to_echo=24908 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=21 rerouted_tokens_per_expert_current_ep_rank=[1, 3349, 637, 1120, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 34, 73, 110, 125, 37, 180, 155, 5415, 0, 0, 0, 0, 0, 0, 0, 469, 169, 94, 13, 265, 1353, 1, 0, 2460, 1886, 0, 0, 0, 0, 0, 0, 8, 0, 878, 2535, 75, 0, 238, 21, 2297, 787, 291, 36, 0, 0, 0, 0, 12, 23, 440, 22, 17, 931, 3876, 126, 0, 0, 0, 0, 0, 0, 0, 0, 60, 10, 649, 42, 270, 106, 14, 423, 3523, 1035, 0, 0, 0, 0, 0, 0, 158, 1140, 322, 0, 59, 511, 75, 3382, 0, 0, 0, 0, 0, 0, 0, 0, 94, 5957, 39, 332, 72, 25, 19, 178, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5183, 6151, 6710, 7166, 5447, 6132, 5647, 6716] avg=6144.0000 max=7166 max_rank=3 min=5183 min_rank=0 imbalance_ratio=1.166341 max_load_reduction=41986 max_load_reduction_pct=0.854207 moved_to_echo=23942 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 4547, 763, 1238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 838, 0, 0, 2921, 2338, 0, 0, 0, 0, 0, 0, 0, 0, 840, 679, 80, 0, 0, 0, 2848, 1069, 490, 35, 0, 0, 0, 0, 0, 0, 65, 0, 0, 1155, 3828, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 915, 0, 0, 0, 0, 0, 4185, 1158, 0, 0, 0, 0, 0, 0, 0, 1275, 0, 0, 0, 0, 62, 5688, 0, 0, 0, 0, 0, 0, 0, 0, 141, 5683, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6548, 6047, 6160, 6041, 5162, 6265, 7025, 5904] avg=6144.0000 max=7025 max_rank=6 min=5162 min_rank=4 imbalance_ratio=1.143392 max_load_reduction=42127 max_load_reduction_pct=0.857076 moved_to_echo=24356 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 3850, 664, 1350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 847, 0, 0, 2817, 2111, 0, 0, 0, 0, 0, 0, 0, 0, 672, 1430, 88, 0, 0, 0, 2573, 909, 327, 43, 0, 0, 0, 0, 0, 0, 131, 0, 0, 1031, 6251, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 795, 0, 0, 0, 0, 0, 4035, 1190, 0, 0, 0, 0, 0, 0, 0, 1369, 10, 0, 0, 0, 69, 3729, 0, 0, 0, 0, 0, 0, 0, 0, 89, 6257, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5864, 6226, 5845, 6042, 7543, 6020, 5177, 6435] avg=6144.0000 max=7543 max_rank=4 min=5177 min_rank=6 imbalance_ratio=1.227702 max_load_reduction=41609 max_load_reduction_pct=0.846537 moved_to_echo=25175 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[640, 256, 768, 128, 384, 7168, 128, 0, 22528, 17792, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 6656, 128, 384, 128, 128, 512, 32256, 9344, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[256, 10240, 384, 0, 128, 640, 640, 37248, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=50176 observed_loss=-1024 tokens_per_expert=[128, 0, 6400, 8960, 768, 0, 384, 128, 21760, 7936, 3328, 384, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 1408, 128, 128, 8832, 38016, 1024, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 33280, 5760, 10240, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1024, 46976, 128, 384, 768, 128, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 128, 256, 256, 128, 256, 256, 48384, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=50176 slack_slots=48128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=22 tokens_per_expert_current_ep_rank=[5552, 65, 7, 6, 1, 1, 4, 2289, 539, 66, 311, 1, 8, 0, 107, 5126, 0, 15, 6, 3615, 266, 438, 0, 588, 0, 6, 0, 1, 2, 3, 0, 3, 1280, 56, 9, 0, 0, 0, 0, 4115, 0, 61, 7, 7163, 45, 56, 245, 2, 7, 24, 1, 3451, 16, 7571, 0, 36, 26, 15, 3, 5678, 149, 13, 10, 87] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=22 tokens_per_expert_current_ep_rank=[6301, 0, 0, 0, 0, 0, 0, 2500, 525, 0, 223, 0, 0, 0, 130, 5079, 0, 0, 0, 2372, 270, 402, 0, 622, 0, 0, 0, 0, 0, 0, 0, 0, 1090, 0, 0, 0, 0, 0, 0, 5110, 0, 0, 0, 7365, 0, 0, 248, 0, 0, 0, 0, 3773, 0, 7656, 0, 0, 0, 0, 0, 5211, 177, 0, 0, 98] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=22 tokens_per_expert_current_ep_rank=[5662, 0, 0, 0, 0, 0, 0, 2743, 556, 0, 163, 0, 0, 0, 64, 5368, 0, 0, 0, 3744, 248, 406, 0, 422, 0, 0, 0, 0, 0, 0, 0, 0, 1296, 0, 0, 0, 0, 0, 0, 4356, 0, 0, 0, 7436, 0, 0, 275, 0, 0, 0, 0, 3193, 0, 7871, 0, 0, 0, 0, 0, 5166, 120, 0, 0, 63] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=22 tokens_per_expert_current_ep_rank=[5850, 0, 0, 0, 0, 0, 0, 2672, 559, 0, 160, 0, 0, 0, 85, 5291, 0, 0, 0, 3012, 278, 409, 0, 446, 0, 0, 0, 0, 0, 0, 0, 0, 1467, 0, 0, 0, 0, 0, 0, 4582, 0, 0, 0, 7429, 0, 0, 268, 0, 0, 0, 0, 3643, 0, 7829, 0, 0, 0, 0, 0, 4946, 148, 0, 0, 78] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=22 tokens_per_expert_current_ep_rank=[4037, 0, 0, 0, 0, 0, 0, 3101, 484, 0, 119, 0, 0, 0, 83, 5556, 0, 0, 0, 5345, 246, 579, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 1654, 0, 0, 0, 0, 0, 0, 4696, 0, 0, 0, 7523, 0, 0, 288, 0, 0, 0, 0, 2517, 0, 7837, 0, 0, 0, 0, 0, 4765, 159, 0, 0, 88] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=22 tokens_per_expert_current_ep_rank=[6580, 440, 14, 8, 8, 19, 51, 1396, 436, 498, 1844, 38, 61, 2, 152, 4717, 4, 136, 46, 1369, 257, 320, 23, 685, 0, 23, 2, 9, 50, 56, 23, 49, 2272, 426, 71, 0, 5, 0, 1, 3662, 3, 500, 4, 5704, 235, 446, 286, 17, 11, 20, 4, 2624, 154, 6281, 0, 300, 5, 161, 7, 6346, 154, 62, 6, 69] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=22 tokens_per_expert_current_ep_rank=[5018, 0, 0, 0, 0, 0, 0, 2785, 550, 0, 133, 0, 0, 0, 80, 5501, 0, 0, 0, 4734, 285, 488, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 1759, 0, 0, 0, 0, 0, 0, 4386, 0, 0, 0, 7465, 0, 0, 279, 0, 0, 0, 0, 2468, 0, 7860, 0, 0, 0, 0, 0, 4953, 142, 0, 0, 80] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=22 tokens_per_expert_current_ep_rank=[6612, 0, 0, 0, 0, 0, 0, 2358, 500, 0, 231, 0, 0, 0, 92, 5157, 0, 0, 0, 1858, 259, 432, 0, 623, 0, 0, 0, 0, 0, 0, 0, 0, 882, 0, 0, 0, 0, 0, 0, 5194, 0, 0, 0, 7389, 0, 0, 292, 0, 0, 0, 0, 3558, 0, 7581, 0, 0, 0, 0, 0, 5892, 152, 0, 0, 90] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=22 rerouted_tokens_per_expert_current_ep_rank=[3492, 65, 7, 6, 1, 1, 4, 2289, 0, 0, 0, 0, 0, 0, 0, 0, 539, 66, 311, 1, 8, 0, 107, 4950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 6, 3615, 266, 438, 0, 588, 1060, 625, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1, 2, 3, 0, 3, 4648, 1435, 0, 0, 0, 0, 0, 0, 1280, 56, 9, 0, 0, 0, 0, 4115, 96, 0, 0, 0, 0, 0, 0, 0, 0, 61, 7, 5683, 45, 56, 245, 2, 0, 0, 0, 0, 0, 0, 0, 0, 7, 24, 1, 3451, 16, 2923, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 26, 15, 3, 5678, 149, 13, 10, 87, 420, 80, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5865, 5982, 6613, 6098, 5556, 6099, 6458, 6481] avg=6144.0000 max=6613 max_rank=2 min=5556 min_rank=4 imbalance_ratio=1.076335 max_load_reduction=42539 max_load_reduction_pct=0.865458 moved_to_echo=24594 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=22 rerouted_tokens_per_expert_current_ep_rank=[2540, 0, 0, 0, 0, 0, 0, 3101, 0, 0, 0, 0, 0, 0, 0, 0, 484, 0, 119, 0, 0, 0, 83, 5365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5345, 246, 579, 0, 75, 1113, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4811, 1043, 0, 0, 0, 0, 0, 0, 1654, 0, 0, 0, 0, 0, 0, 4696, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5969, 0, 0, 288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2517, 0, 3026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4765, 159, 0, 0, 88, 441, 87, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5641, 6051, 7812, 5854, 6454, 6257, 5543, 5540] avg=6144.0000 max=7812 max_rank=2 min=5540 min_rank=7 imbalance_ratio=1.271484 max_load_reduction=41340 max_load_reduction_pct=0.841064 moved_to_echo=23794 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=22 rerouted_tokens_per_expert_current_ep_rank=[3680, 0, 0, 0, 0, 0, 0, 2672, 0, 0, 0, 0, 0, 0, 0, 0, 559, 0, 160, 0, 0, 0, 85, 5109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3012, 278, 409, 0, 446, 1099, 658, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4806, 1512, 0, 0, 0, 0, 0, 0, 1467, 0, 0, 0, 0, 0, 0, 4582, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5895, 0, 0, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3643, 0, 3023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4946, 148, 0, 0, 78, 435, 83, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6352, 5913, 5902, 6318, 6148, 6163, 6666, 5690] avg=6144.0000 max=6666 max_rank=6 min=5690 min_rank=7 imbalance_ratio=1.084961 max_load_reduction=42486 max_load_reduction_pct=0.864380 moved_to_echo=24667 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=22 rerouted_tokens_per_expert_current_ep_rank=[3562, 0, 0, 0, 0, 0, 0, 2743, 0, 0, 0, 0, 0, 0, 0, 0, 556, 0, 163, 0, 0, 0, 64, 5184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3744, 248, 406, 0, 422, 1100, 637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4832, 1463, 0, 0, 0, 0, 0, 0, 1296, 0, 0, 0, 0, 0, 0, 4356, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5900, 0, 0, 275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3193, 0, 3039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5166, 120, 0, 0, 63, 436, 84, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6305, 5967, 6557, 6295, 5752, 6175, 6232, 5869] avg=6144.0000 max=6557 max_rank=2 min=5752 min_rank=4 imbalance_ratio=1.067220 max_load_reduction=42595 max_load_reduction_pct=0.866597 moved_to_echo=24028 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=22 rerouted_tokens_per_expert_current_ep_rank=[3964, 0, 0, 0, 0, 0, 0, 2500, 0, 0, 0, 0, 0, 0, 0, 0, 525, 0, 223, 0, 0, 0, 130, 4904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2372, 270, 402, 0, 622, 1090, 709, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4700, 1628, 0, 0, 0, 0, 0, 0, 1090, 0, 0, 0, 0, 0, 0, 5110, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5843, 0, 0, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3773, 0, 2956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5211, 177, 0, 0, 98, 432, 80, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6464, 5782, 5465, 6328, 6295, 6091, 6729, 5998] avg=6144.0000 max=6729 max_rank=6 min=5465 min_rank=2 imbalance_ratio=1.095215 max_load_reduction=42423 max_load_reduction_pct=0.863098 moved_to_echo=25113 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=22 rerouted_tokens_per_expert_current_ep_rank=[4139, 440, 14, 8, 8, 19, 51, 1396, 0, 0, 0, 0, 0, 0, 0, 0, 436, 498, 1844, 38, 61, 2, 152, 4555, 0, 0, 0, 0, 0, 0, 0, 0, 4, 136, 46, 1369, 257, 320, 23, 685, 844, 741, 0, 0, 0, 0, 0, 0, 0, 23, 2, 9, 50, 56, 23, 49, 3856, 1700, 0, 0, 0, 0, 0, 0, 2272, 426, 71, 0, 5, 0, 1, 3662, 88, 0, 0, 0, 0, 0, 0, 0, 3, 500, 4, 4526, 235, 446, 286, 17, 0, 0, 0, 0, 0, 0, 0, 0, 11, 20, 4, 2624, 154, 2425, 0, 300, 0, 0, 0, 0, 0, 0, 0, 0, 5, 161, 7, 6346, 154, 62, 6, 69, 334, 74, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6075, 7586, 4425, 5768, 6525, 6017, 5538, 7218] avg=6144.0000 max=7586 max_rank=1 min=4425 min_rank=2 imbalance_ratio=1.234701 max_load_reduction=41566 max_load_reduction_pct=0.845662 moved_to_echo=25298 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=22 rerouted_tokens_per_expert_current_ep_rank=[3156, 0, 0, 0, 0, 0, 0, 2785, 0, 0, 0, 0, 0, 0, 0, 0, 550, 0, 133, 0, 0, 0, 80, 5312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4734, 285, 488, 0, 186, 1104, 565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4825, 1297, 0, 0, 0, 0, 0, 0, 1759, 0, 0, 0, 0, 0, 0, 4386, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5923, 0, 0, 279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2468, 0, 3035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4953, 142, 0, 0, 80, 438, 86, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5941, 6075, 7362, 6122, 6248, 6202, 5503, 5699] avg=6144.0000 max=7362 max_rank=2 min=5503 min_rank=6 imbalance_ratio=1.198242 max_load_reduction=41790 max_load_reduction_pct=0.850220 moved_to_echo=23652 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=22 rerouted_tokens_per_expert_current_ep_rank=[4151, 0, 0, 0, 0, 0, 0, 2358, 0, 0, 0, 0, 0, 0, 0, 0, 500, 0, 231, 0, 0, 0, 92, 4973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1858, 259, 432, 0, 623, 1096, 748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4656, 1713, 0, 0, 0, 0, 0, 0, 882, 0, 0, 0, 0, 0, 0, 5194, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5856, 0, 0, 292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3558, 0, 2925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5892, 152, 0, 0, 90, 437, 86, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6509, 5796, 5016, 6369, 6174, 6148, 6483, 6657] avg=6144.0000 max=6657 max_rank=7 min=5016 min_rank=2 imbalance_ratio=1.083496 max_load_reduction=42495 max_load_reduction_pct=0.864563 moved_to_echo=25462 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[0, 128, 128, 128, 128, 128, 128, 128, 37248, 11904, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 256, 128, 26112, 2176, 3584, 128, 3712, 8576, 5248, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 128, 25344, 256, 23424, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[28800, 512, 128, 128, 128, 128, 128, 19968, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[11776, 512, 128, 0, 128, 0, 128, 36224, 896, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4224, 640, 3200, 128, 128, 128, 896, 40448, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 640, 128, 45696, 384, 512, 2304, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 256, 128, 43008, 1280, 128, 128, 768, 3456, 768, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 3771, 0, 0, 0, 0, 0, 0, 231, 0, 0, 7799, 0, 1768, 0, 75, 334, 64, 0, 0, 0, 362, 0, 954, 0, 0, 0, 0, 0, 0, 6963, 5010, 0, 265, 0, 0, 0, 201, 0, 1937, 2761, 0, 0, 0, 6695, 7895, 25, 0, 0, 0, 0, 231, 0, 98, 0, 0, 0, 1713, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=23 tokens_per_expert_current_ep_rank=[7, 2, 2518, 5, 7, 2, 4, 0, 59, 251, 8, 0, 7639, 2, 2438, 0, 95, 380, 67, 1, 0, 0, 322, 11, 798, 10, 9, 0, 8, 5, 0, 6878, 4779, 15, 229, 0, 59, 1, 197, 0, 2470, 3048, 29, 28, 71, 6568, 7553, 25, 72, 11, 0, 16, 331, 0, 89, 1, 0, 65, 1890, 57, 19, 2, 0, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 2186, 0, 0, 0, 0, 0, 0, 257, 0, 0, 7781, 0, 2101, 0, 0, 369, 62, 0, 0, 0, 304, 0, 536, 0, 0, 0, 0, 0, 0, 7099, 7140, 0, 312, 0, 0, 0, 184, 0, 1184, 3780, 0, 0, 0, 6546, 7329, 32, 0, 0, 0, 0, 257, 0, 88, 0, 0, 0, 1605, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=23 tokens_per_expert_current_ep_rank=[27, 24, 3502, 107, 30, 10, 71, 24, 292, 263, 37, 1, 6242, 9, 1993, 6, 144, 426, 108, 14, 0, 16, 218, 17, 1678, 97, 88, 28, 39, 22, 13, 5555, 2017, 7, 135, 0, 430, 1, 384, 13, 3100, 3983, 22, 252, 525, 5992, 6299, 38, 530, 46, 2, 172, 808, 0, 92, 4, 10, 517, 2056, 481, 93, 21, 12, 9] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 3110, 0, 0, 0, 0, 0, 0, 242, 0, 0, 7831, 0, 1761, 0, 12, 356, 54, 0, 0, 0, 328, 0, 581, 0, 0, 0, 0, 0, 0, 7114, 6156, 0, 217, 0, 0, 0, 177, 0, 1375, 3145, 0, 0, 0, 6893, 7884, 72, 0, 1, 0, 0, 242, 0, 63, 0, 0, 0, 1538, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 3207, 0, 0, 0, 0, 0, 0, 244, 0, 0, 7819, 0, 1673, 0, 25, 372, 51, 0, 0, 0, 389, 0, 784, 0, 0, 0, 0, 0, 0, 6986, 5956, 0, 296, 0, 0, 0, 200, 0, 1862, 2667, 0, 0, 0, 6719, 7882, 21, 0, 0, 0, 0, 244, 0, 78, 0, 0, 0, 1677, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 2493, 0, 0, 1, 0, 0, 0, 242, 0, 0, 7812, 0, 2224, 0, 0, 355, 52, 0, 0, 0, 350, 0, 536, 0, 0, 0, 0, 1, 0, 7110, 6768, 0, 306, 0, 0, 0, 164, 0, 1248, 3285, 0, 0, 0, 6761, 7541, 40, 0, 0, 0, 0, 242, 0, 80, 0, 0, 0, 1541, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 3444, 0, 0, 0, 0, 0, 0, 252, 0, 0, 7789, 0, 2382, 0, 78, 344, 61, 0, 0, 0, 354, 0, 1382, 0, 0, 0, 0, 0, 0, 6741, 4192, 0, 314, 0, 0, 0, 213, 0, 2282, 2329, 0, 0, 0, 6773, 7917, 6, 0, 0, 0, 0, 252, 0, 90, 0, 0, 0, 1957, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=23 rerouted_tokens_per_expert_current_ep_rank=[7, 2, 2518, 5, 7, 2, 4, 0, 1830, 1269, 0, 0, 0, 0, 0, 0, 59, 251, 8, 0, 3828, 2, 2438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 380, 67, 1, 0, 0, 322, 11, 3385, 1914, 0, 0, 0, 0, 0, 0, 798, 10, 9, 0, 8, 5, 0, 5254, 0, 0, 0, 0, 0, 0, 0, 0, 4779, 15, 229, 0, 59, 1, 197, 0, 355, 0, 0, 0, 0, 0, 0, 0, 2470, 3048, 29, 28, 71, 933, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 72, 11, 0, 16, 331, 0, 89, 1, 5639, 0, 0, 0, 0, 0, 0, 0, 0, 65, 1890, 57, 19, 2, 0, 1, 2250, 1981, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5644, 6586, 6175, 6084, 5635, 6604, 6159, 6265] avg=6144.0000 max=6604 max_rank=5 min=5635 min_rank=4 imbalance_ratio=1.074870 max_load_reduction=42548 max_load_reduction_pct=0.865641 moved_to_echo=24663 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2186, 0, 0, 0, 0, 0, 1864, 1310, 0, 0, 0, 0, 0, 0, 0, 257, 0, 0, 3900, 0, 2101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 369, 62, 0, 0, 0, 304, 0, 3374, 1858, 0, 0, 0, 0, 0, 0, 536, 0, 0, 0, 0, 0, 0, 5423, 0, 0, 0, 0, 0, 0, 0, 0, 7140, 0, 312, 0, 0, 0, 184, 0, 366, 0, 0, 0, 0, 0, 0, 0, 1184, 3780, 0, 0, 0, 929, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 257, 0, 88, 0, 5471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1605, 0, 0, 0, 0, 0, 2243, 2017, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5360, 6258, 5967, 5959, 8002, 5925, 5816, 5865] avg=6144.0000 max=8002 max_rank=4 min=5360 min_rank=0 imbalance_ratio=1.302409 max_load_reduction=41150 max_load_reduction_pct=0.837199 moved_to_echo=25608 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3771, 0, 0, 0, 0, 0, 1869, 1285, 0, 0, 0, 0, 0, 0, 0, 231, 0, 0, 3908, 0, 1768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 334, 64, 0, 0, 0, 362, 0, 3451, 2002, 0, 0, 0, 0, 0, 0, 954, 0, 0, 0, 0, 0, 0, 5319, 0, 0, 0, 0, 0, 0, 0, 0, 5010, 0, 265, 0, 0, 0, 201, 0, 359, 0, 0, 0, 0, 0, 0, 0, 1937, 2761, 0, 0, 0, 950, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 231, 0, 98, 0, 5893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1713, 0, 0, 0, 0, 0, 2294, 2022, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6925, 5907, 6288, 6273, 5835, 5673, 6222, 6029] avg=6144.0000 max=6925 max_rank=0 min=5673 min_rank=5 imbalance_ratio=1.127116 max_load_reduction=42227 max_load_reduction_pct=0.859111 moved_to_echo=23759 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3110, 0, 0, 0, 0, 0, 1876, 1313, 0, 0, 0, 0, 0, 0, 0, 242, 0, 0, 3925, 0, 1761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 356, 54, 0, 0, 0, 328, 0, 3553, 1999, 0, 0, 0, 0, 0, 0, 581, 0, 0, 0, 0, 0, 0, 5434, 0, 0, 0, 0, 0, 0, 0, 0, 6156, 0, 217, 0, 0, 0, 177, 0, 367, 0, 0, 0, 0, 0, 0, 0, 1375, 3145, 0, 0, 0, 979, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 242, 0, 63, 0, 5885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1538, 0, 0, 0, 0, 0, 2361, 2030, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6299, 5928, 6302, 6015, 6917, 5571, 6191, 5929] avg=6144.0000 max=6917 max_rank=4 min=5571 min_rank=5 imbalance_ratio=1.125814 max_load_reduction=42235 max_load_reduction_pct=0.859273 moved_to_echo=24608 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3207, 0, 0, 0, 0, 0, 1873, 1289, 0, 0, 0, 0, 0, 0, 0, 244, 0, 0, 3919, 0, 1673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 372, 51, 0, 0, 0, 389, 0, 3463, 1998, 0, 0, 0, 0, 0, 0, 784, 0, 0, 0, 0, 0, 0, 5336, 0, 0, 0, 0, 0, 0, 0, 0, 5956, 0, 296, 0, 0, 0, 200, 0, 361, 0, 0, 0, 0, 0, 0, 0, 1862, 2667, 0, 0, 0, 954, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 244, 0, 78, 0, 5884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1677, 0, 0, 0, 0, 0, 2302, 2027, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6369, 5836, 6298, 6120, 6813, 5504, 6206, 6006] avg=6144.0000 max=6813 max_rank=4 min=5504 min_rank=5 imbalance_ratio=1.108887 max_load_reduction=42339 max_load_reduction_pct=0.861389 moved_to_echo=24529 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=23 rerouted_tokens_per_expert_current_ep_rank=[27, 24, 3502, 107, 30, 10, 71, 24, 1496, 1025, 0, 0, 0, 0, 0, 0, 292, 263, 37, 1, 3128, 9, 1993, 6, 0, 0, 0, 0, 0, 0, 0, 0, 144, 426, 108, 14, 0, 16, 218, 17, 3088, 1597, 0, 0, 0, 0, 0, 0, 1678, 97, 88, 28, 39, 22, 13, 4243, 0, 0, 0, 0, 0, 0, 0, 0, 2017, 7, 135, 0, 430, 1, 384, 13, 287, 0, 0, 0, 0, 0, 0, 0, 3100, 3983, 22, 252, 525, 851, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 530, 46, 2, 172, 808, 0, 92, 4, 4702, 0, 0, 0, 0, 0, 0, 0, 10, 517, 2056, 481, 93, 21, 12, 9, 2053, 1618, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6316, 5729, 5628, 6208, 3274, 8771, 6356, 6870] avg=6144.0000 max=8771 max_rank=5 min=3274 min_rank=4 imbalance_ratio=1.427572 max_load_reduction=40381 max_load_reduction_pct=0.821554 moved_to_echo=25271 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2493, 0, 0, 1, 0, 0, 1872, 1312, 0, 0, 0, 0, 0, 0, 0, 242, 0, 0, 3915, 0, 2224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 355, 52, 0, 0, 0, 350, 0, 3485, 1912, 0, 0, 0, 0, 0, 0, 536, 0, 0, 0, 0, 1, 0, 5431, 0, 0, 0, 0, 0, 0, 0, 0, 6768, 0, 306, 0, 0, 0, 164, 0, 367, 0, 0, 0, 0, 0, 0, 0, 1248, 3285, 0, 0, 0, 960, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 242, 0, 80, 0, 5629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1541, 0, 0, 0, 0, 0, 2316, 2025, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5678, 6381, 6154, 5968, 7605, 5533, 5951, 5882] avg=6144.0000 max=7605 max_rank=4 min=5533 min_rank=5 imbalance_ratio=1.237793 max_load_reduction=41547 max_load_reduction_pct=0.845276 moved_to_echo=24971 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3444, 0, 0, 0, 0, 0, 1871, 1246, 0, 0, 0, 0, 0, 0, 0, 252, 0, 0, 3893, 0, 2382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 344, 61, 0, 0, 0, 354, 0, 3495, 2008, 0, 0, 0, 0, 0, 0, 1382, 0, 0, 0, 0, 0, 0, 5143, 0, 0, 0, 0, 0, 0, 0, 0, 4192, 0, 314, 0, 0, 0, 213, 0, 352, 0, 0, 0, 0, 0, 0, 0, 2282, 2329, 0, 0, 0, 954, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 252, 0, 90, 0, 5909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1957, 0, 0, 0, 0, 0, 2324, 2025, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6561, 6527, 6340, 6525, 5071, 5571, 6251, 6306] avg=6144.0000 max=6561 max_rank=0 min=5071 min_rank=4 imbalance_ratio=1.067871 max_load_reduction=42591 max_load_reduction_pct=0.866516 moved_to_echo=23199 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 24320, 128, 128, 128, 128, 128, 14592, 10112, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[512, 2944, 640, 128, 0, 128, 2688, 128, 27392, 15360, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[15488, 25088, 128, 384, 640, 7552, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[384, 2048, 128, 128, 30464, 128, 16384, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 640, 14080, 640, 128, 128, 128, 128, 18176, 15872, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[42112, 128, 2176, 0, 512, 128, 1792, 128, 2816, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7296, 128, 128, 128, 128, 128, 128, 41600, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[640, 128, 128, 256, 2688, 0, 768, 128, 45056, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=24 tokens_per_expert_current_ep_rank=[0, 71, 42, 64, 68, 49, 309, 36, 87, 7349, 504, 0, 0, 133, 7, 70, 11, 18, 0, 69, 58, 213, 43, 0, 0, 21, 0, 24, 945, 7371, 35, 0, 4, 69, 0, 2, 561, 6845, 5, 8025, 87, 1, 0, 3925, 409, 6, 188, 9, 16, 0, 4, 1, 0, 2928, 0, 0, 0, 0, 399, 7602, 7, 17, 0, 445] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=24 tokens_per_expert_current_ep_rank=[0, 62, 0, 0, 0, 0, 281, 0, 88, 7536, 519, 0, 0, 88, 0, 56, 0, 0, 0, 62, 0, 169, 0, 0, 0, 0, 0, 0, 784, 7648, 0, 0, 0, 0, 0, 0, 335, 7205, 0, 8104, 88, 0, 0, 4274, 95, 0, 91, 0, 0, 0, 0, 0, 0, 3118, 0, 0, 0, 0, 345, 7855, 0, 0, 0, 349] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=24 tokens_per_expert_current_ep_rank=[0, 54, 0, 0, 0, 0, 292, 0, 61, 7442, 557, 0, 0, 61, 0, 12, 0, 0, 0, 54, 0, 200, 0, 0, 0, 0, 0, 0, 845, 7673, 0, 0, 0, 0, 0, 0, 505, 7129, 0, 8131, 63, 0, 0, 4583, 67, 0, 66, 0, 0, 0, 0, 0, 0, 2760, 0, 0, 0, 0, 305, 7882, 0, 0, 0, 410] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=24 tokens_per_expert_current_ep_rank=[0, 65, 337, 499, 558, 304, 310, 250, 68, 6672, 486, 0, 1, 395, 31, 96, 38, 79, 0, 125, 448, 328, 292, 0, 2, 88, 11, 22, 1014, 5985, 413, 11, 69, 551, 0, 18, 947, 5274, 66, 7510, 69, 9, 22, 4232, 1336, 117, 818, 34, 130, 4, 10, 2, 1, 752, 1, 0, 12, 0, 577, 6096, 14, 34, 7, 1512] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=24 tokens_per_expert_current_ep_rank=[0, 51, 0, 0, 0, 0, 309, 0, 78, 7461, 577, 0, 0, 78, 0, 0, 0, 0, 0, 51, 0, 210, 0, 0, 0, 0, 0, 0, 842, 7646, 0, 0, 0, 0, 0, 0, 482, 7111, 0, 8114, 78, 0, 0, 4629, 83, 0, 96, 0, 0, 0, 0, 0, 0, 2551, 0, 0, 0, 0, 322, 7833, 0, 0, 0, 550] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=24 tokens_per_expert_current_ep_rank=[0, 56, 0, 0, 0, 0, 341, 1, 80, 7479, 488, 0, 0, 80, 0, 22, 0, 0, 0, 52, 0, 228, 0, 0, 0, 0, 0, 0, 838, 7641, 0, 0, 0, 0, 0, 0, 415, 7156, 0, 8112, 80, 0, 0, 4115, 81, 0, 80, 0, 0, 0, 0, 0, 0, 3353, 0, 0, 0, 0, 322, 7827, 0, 0, 0, 305] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=24 tokens_per_expert_current_ep_rank=[0, 59, 0, 0, 0, 0, 286, 0, 97, 7497, 635, 0, 0, 97, 0, 0, 0, 0, 0, 63, 0, 219, 0, 0, 0, 0, 0, 0, 806, 7613, 0, 0, 0, 0, 0, 0, 463, 7115, 0, 8095, 98, 0, 0, 4642, 102, 0, 137, 0, 0, 0, 0, 0, 0, 2405, 0, 0, 0, 0, 329, 7820, 0, 0, 0, 574] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=24 tokens_per_expert_current_ep_rank=[0, 57, 0, 0, 0, 0, 302, 0, 89, 7495, 573, 0, 0, 89, 0, 0, 0, 0, 0, 61, 0, 183, 0, 0, 0, 0, 0, 0, 826, 7591, 0, 0, 0, 0, 0, 0, 444, 7065, 0, 8103, 90, 0, 0, 4401, 124, 0, 150, 0, 0, 0, 0, 0, 0, 2755, 0, 0, 0, 0, 342, 7830, 0, 0, 0, 582] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 71, 42, 64, 68, 49, 309, 36, 2421, 2289, 793, 0, 0, 0, 0, 0, 87, 5344, 504, 0, 0, 133, 7, 70, 0, 0, 0, 0, 0, 0, 0, 0, 11, 18, 0, 69, 58, 213, 43, 0, 5736, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 24, 945, 5187, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 69, 0, 2, 561, 5513, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 1, 0, 3925, 409, 6, 188, 9, 1194, 0, 0, 0, 0, 0, 0, 0, 16, 0, 4, 1, 0, 2928, 0, 0, 2005, 1391, 138, 0, 0, 0, 0, 0, 0, 0, 399, 5181, 7, 17, 0, 445, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6142, 6145, 6148, 6212, 6154, 5819, 6483, 6049] avg=6144.0000 max=6483 max_rank=6 min=5819 min_rank=5 imbalance_ratio=1.055176 max_load_reduction=42669 max_load_reduction_pct=0.868103 moved_to_echo=24505 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 62, 0, 0, 0, 0, 281, 0, 2502, 2312, 823, 0, 0, 0, 0, 0, 88, 5480, 519, 0, 0, 88, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 169, 0, 0, 5792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 784, 5381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 335, 5804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 4274, 95, 0, 91, 0, 1256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3118, 0, 0, 2056, 1444, 145, 0, 0, 0, 0, 0, 0, 0, 345, 5353, 0, 0, 0, 349, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5980, 6231, 6023, 6165, 6139, 5804, 6763, 6047] avg=6144.0000 max=6763 max_rank=6 min=5804 min_rank=5 imbalance_ratio=1.100749 max_load_reduction=42389 max_load_reduction_pct=0.862406 moved_to_echo=24753 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 54, 0, 0, 0, 0, 292, 0, 2511, 2320, 826, 0, 0, 0, 0, 0, 61, 5412, 557, 0, 0, 61, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 200, 0, 0, 5811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 845, 5399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 5743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 4583, 67, 0, 66, 0, 1243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2760, 0, 0, 2030, 1448, 143, 0, 0, 0, 0, 0, 0, 0, 305, 5371, 0, 0, 0, 410, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6003, 6103, 6065, 6244, 6248, 6022, 6381, 6086] avg=6144.0000 max=6381 max_rank=6 min=6003 min_rank=0 imbalance_ratio=1.038574 max_load_reduction=42771 max_load_reduction_pct=0.870178 moved_to_echo=24737 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 51, 0, 0, 0, 0, 309, 0, 2495, 2315, 823, 0, 0, 0, 0, 0, 78, 5425, 577, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 210, 0, 0, 5799, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 842, 5380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 5728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 4629, 83, 0, 96, 0, 1240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2551, 0, 0, 2036, 1443, 143, 0, 0, 0, 0, 0, 0, 0, 322, 5338, 0, 0, 0, 550, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5993, 6158, 6060, 6222, 6210, 6126, 6173, 6210] avg=6144.0000 max=6222 max_rank=3 min=5993 min_rank=0 imbalance_ratio=1.012695 max_load_reduction=42930 max_load_reduction_pct=0.873413 moved_to_echo=24719 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 65, 337, 499, 558, 304, 310, 250, 1942, 2143, 644, 0, 0, 0, 0, 0, 68, 4852, 486, 0, 1, 395, 31, 96, 0, 0, 0, 0, 0, 0, 0, 0, 38, 79, 0, 125, 448, 328, 292, 0, 5367, 0, 0, 0, 0, 0, 0, 0, 2, 88, 11, 22, 1014, 4211, 413, 11, 0, 0, 0, 0, 0, 0, 0, 0, 69, 551, 0, 18, 947, 4248, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 9, 22, 4232, 1336, 117, 818, 34, 920, 0, 0, 0, 0, 0, 0, 0, 130, 4, 10, 2, 1, 752, 1, 0, 1820, 1130, 106, 0, 0, 0, 0, 0, 12, 0, 577, 4154, 14, 34, 7, 1512, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7052, 5929, 6677, 5772, 5899, 7557, 3956, 6310] avg=6144.0000 max=7557 max_rank=5 min=3956 min_rank=6 imbalance_ratio=1.229980 max_load_reduction=41595 max_load_reduction_pct=0.846252 moved_to_echo=23722 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 59, 0, 0, 0, 0, 286, 0, 2491, 2310, 819, 0, 0, 0, 0, 0, 97, 5452, 635, 0, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 219, 0, 0, 5785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 806, 5357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 463, 5731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 4642, 102, 0, 137, 0, 1241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2405, 0, 0, 2045, 1437, 143, 0, 0, 0, 0, 0, 0, 0, 329, 5329, 0, 0, 0, 574, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5965, 6281, 6067, 6163, 6194, 6220, 6030, 6232] avg=6144.0000 max=6281 max_rank=1 min=5965 min_rank=0 imbalance_ratio=1.022298 max_load_reduction=42871 max_load_reduction_pct=0.872213 moved_to_echo=24676 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 56, 0, 0, 0, 0, 341, 1, 2493, 2315, 822, 0, 0, 0, 0, 0, 80, 5439, 488, 0, 0, 80, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 228, 0, 0, 5797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 838, 5377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 415, 5764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 4115, 81, 0, 80, 0, 1248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3353, 0, 0, 2040, 1442, 144, 0, 0, 0, 0, 0, 0, 0, 322, 5334, 0, 0, 0, 305, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6028, 6109, 6077, 6215, 6179, 5604, 6979, 5961] avg=6144.0000 max=6979 max_rank=6 min=5604 min_rank=5 imbalance_ratio=1.135905 max_load_reduction=42173 max_load_reduction_pct=0.858012 moved_to_echo=24723 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 57, 0, 0, 0, 0, 302, 0, 2497, 2312, 821, 0, 0, 0, 0, 0, 89, 5445, 573, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 183, 0, 0, 5791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 826, 5333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 444, 5685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 4401, 124, 0, 150, 0, 1235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2755, 0, 0, 2050, 1437, 145, 0, 0, 0, 0, 0, 0, 0, 342, 5333, 0, 0, 0, 582, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5989, 6196, 6035, 6159, 6129, 6000, 6387, 6257] avg=6144.0000 max=6387 max_rank=6 min=5989 min_rank=0 imbalance_ratio=1.039551 max_load_reduction=42765 max_load_reduction_pct=0.870056 moved_to_echo=24773 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 512, 384, 640, 640, 384, 2432, 384, 19456, 18432, 6400, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=50176 observed_loss=-1024 tokens_per_expert=[256, 128, 128, 128, 128, 20736, 128, 0, 16128, 11264, 1152, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[768, 128, 128, 34816, 2304, 128, 1664, 128, 9600, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 0, 2944, 41472, 128, 128, 128, 4736, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 6912, 41728, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[768, 42880, 4352, 0, 128, 1024, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 128, 0, 640, 512, 1792, 384, 0, 45952, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 640, 0, 128, 4224, 44288, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=50176 slack_slots=48128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True - [2026-08-26 14:53:50] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 823.9 | throughput per GPU (TFLOP/s/GPU): 154.9 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.563369E+00 | loss scale: 1.0 | grad norm: 14.007 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1410, 339, 792, 859, 299, 981, 897, 778, 1358, 212, 692, 362, 278, 1515, 950, 1057, 719, 1044, 771, 669, 251, 553, 273, 1186, 447, 707, 683, 404, 722, 129, 764, 1271, 793, 760, 1045, 1167, 540, 191, 655, 972, 570, 578, 611, 713, 949, 1055, 515, 493, 771, 591, 800, 540, 998, 1502, 752, 865, 1457, 399, 1486, 727, 203, 771, 1344, 967] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1360, 341, 804, 847, 270, 831, 786, 949, 1325, 197, 671, 292, 366, 1236, 955, 1165, 668, 955, 808, 545, 325, 501, 316, 1409, 558, 694, 824, 328, 704, 89, 926, 1264, 841, 1011, 844, 1139, 515, 179, 622, 1105, 612, 567, 535, 825, 877, 950, 682, 624, 708, 580, 797, 527, 983, 1236, 798, 814, 1617, 592, 1460, 769, 132, 741, 1128, 1033] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1423, 288, 765, 765, 227, 1222, 984, 626, 1220, 317, 786, 370, 251, 1704, 915, 950, 694, 976, 759, 673, 337, 525, 300, 1245, 411, 650, 871, 401, 840, 166, 766, 1415, 893, 766, 1114, 1243, 372, 215, 670, 988, 585, 515, 641, 626, 872, 944, 470, 448, 826, 711, 719, 708, 1002, 1265, 691, 865, 1487, 313, 1453, 733, 206, 792, 1300, 877] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1425, 397, 873, 885, 298, 863, 809, 796, 1387, 260, 733, 357, 311, 1504, 863, 1137, 682, 971, 806, 577, 304, 557, 298, 1206, 427, 724, 761, 394, 702, 126, 768, 1361, 814, 864, 1193, 1245, 465, 199, 597, 906, 572, 555, 588, 744, 874, 959, 528, 568, 693, 573, 762, 641, 974, 1452, 726, 870, 1509, 276, 1482, 747, 190, 837, 1244, 943] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1346, 304, 776, 832, 298, 891, 816, 959, 1373, 241, 680, 351, 326, 1440, 912, 996, 807, 966, 762, 478, 304, 511, 345, 1337, 542, 678, 826, 385, 753, 154, 879, 1406, 854, 858, 1246, 1126, 400, 260, 500, 998, 594, 585, 495, 807, 910, 919, 569, 675, 766, 596, 867, 562, 1026, 1213, 773, 813, 1480, 320, 1403, 754, 186, 687, 1178, 1058] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1377, 360, 772, 936, 325, 1119, 846, 773, 1418, 186, 713, 325, 287, 1191, 892, 1099, 754, 943, 733, 623, 286, 534, 258, 1145, 588, 749, 721, 362, 691, 81, 851, 1444, 851, 808, 970, 1108, 537, 235, 578, 928, 673, 675, 569, 803, 907, 958, 584, 512, 774, 638, 767, 500, 1040, 1355, 790, 847, 1500, 299, 1454, 923, 165, 776, 1253, 993] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1415, 322, 708, 817, 228, 1324, 902, 625, 1192, 333, 938, 300, 213, 1738, 880, 883, 786, 1013, 962, 669, 257, 531, 317, 1110, 372, 658, 775, 426, 793, 124, 766, 1405, 929, 773, 1042, 1262, 486, 218, 649, 940, 502, 544, 676, 580, 911, 943, 498, 461, 734, 638, 677, 603, 984, 1551, 666, 744, 1480, 322, 1584, 769, 225, 786, 1357, 836] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1453, 286, 702, 872, 279, 1103, 917, 790, 1341, 254, 706, 336, 312, 1449, 992, 1136, 792, 891, 840, 715, 284, 569, 276, 1259, 508, 634, 768, 397, 780, 103, 707, 1157, 791, 645, 1053, 1040, 692, 228, 585, 966, 617, 559, 512, 796, 918, 977, 600, 607, 816, 644, 745, 497, 974, 1444, 790, 907, 1458, 267, 1506, 765, 214, 787, 1274, 870] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1221, 339, 792, 859, 299, 981, 897, 778, 0, 0, 0, 0, 0, 0, 0, 0, 1358, 212, 692, 362, 278, 1261, 950, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 719, 1044, 771, 669, 251, 553, 273, 1186, 254, 189, 187, 40, 0, 0, 0, 0, 447, 707, 683, 404, 722, 129, 764, 1271, 774, 0, 0, 0, 0, 0, 0, 0, 793, 760, 1045, 1127, 540, 191, 655, 972, 0, 0, 0, 0, 0, 0, 0, 0, 570, 578, 611, 713, 949, 1055, 515, 493, 402, 318, 0, 0, 0, 0, 0, 0, 771, 591, 800, 540, 998, 913, 752, 865, 0, 0, 0, 0, 0, 0, 0, 0, 365, 399, 1486, 727, 203, 771, 1344, 967, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6166, 6170, 6136, 5901, 6083, 6204, 6230, 6262] avg=6144.0000 max=6262 max_rank=7 min=5901 min_rank=3 imbalance_ratio=1.019206 max_load_reduction=42890 max_load_reduction_pct=0.872599 moved_to_echo=24779 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1234, 397, 873, 885, 298, 863, 809, 796, 0, 0, 0, 0, 0, 0, 0, 0, 1387, 260, 733, 357, 311, 1252, 863, 1137, 0, 0, 0, 0, 0, 0, 0, 0, 682, 971, 806, 577, 304, 557, 298, 1206, 252, 191, 181, 43, 0, 0, 0, 0, 427, 724, 761, 394, 702, 126, 768, 1361, 802, 0, 0, 0, 0, 0, 0, 0, 814, 864, 1193, 1202, 465, 199, 597, 906, 0, 0, 0, 0, 0, 0, 0, 0, 572, 555, 588, 744, 874, 959, 528, 568, 388, 329, 0, 0, 0, 0, 0, 0, 693, 573, 762, 641, 974, 883, 726, 870, 0, 0, 0, 0, 0, 0, 0, 0, 378, 276, 1482, 747, 190, 837, 1244, 943, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6155, 6300, 6068, 6065, 6240, 6105, 6122, 6097] avg=6144.0000 max=6300 max_rank=1 min=6065 min_rank=3 imbalance_ratio=1.025391 max_load_reduction=42852 max_load_reduction_pct=0.871826 moved_to_echo=24564 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1165, 304, 776, 832, 298, 891, 816, 959, 0, 0, 0, 0, 0, 0, 0, 0, 1373, 241, 680, 351, 326, 1199, 912, 996, 0, 0, 0, 0, 0, 0, 0, 0, 807, 966, 762, 478, 304, 511, 345, 1337, 241, 181, 151, 39, 0, 0, 0, 0, 542, 678, 826, 385, 753, 154, 879, 1406, 786, 0, 0, 0, 0, 0, 0, 0, 854, 858, 1246, 1087, 400, 260, 500, 998, 0, 0, 0, 0, 0, 0, 0, 0, 594, 585, 495, 807, 910, 919, 569, 675, 324, 323, 0, 0, 0, 0, 0, 0, 766, 596, 867, 562, 1026, 738, 773, 813, 0, 0, 0, 0, 0, 0, 0, 0, 371, 320, 1403, 754, 186, 687, 1178, 1058, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6041, 6078, 6122, 6409, 6203, 6201, 6141, 5957] avg=6144.0000 max=6409 max_rank=3 min=5957 min_rank=7 imbalance_ratio=1.043132 max_load_reduction=42743 max_load_reduction_pct=0.869609 moved_to_echo=24502 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1232, 288, 765, 765, 227, 1222, 984, 626, 0, 0, 0, 0, 0, 0, 0, 0, 1220, 317, 786, 370, 251, 1418, 915, 950, 0, 0, 0, 0, 0, 0, 0, 0, 694, 976, 759, 673, 337, 525, 300, 1245, 286, 191, 158, 43, 0, 0, 0, 0, 411, 650, 871, 401, 840, 166, 766, 1415, 790, 0, 0, 0, 0, 0, 0, 0, 893, 766, 1114, 1200, 372, 215, 670, 988, 0, 0, 0, 0, 0, 0, 0, 0, 585, 515, 641, 626, 872, 944, 470, 448, 338, 324, 0, 0, 0, 0, 0, 0, 826, 711, 719, 708, 1002, 769, 691, 865, 0, 0, 0, 0, 0, 0, 0, 0, 373, 313, 1453, 733, 206, 792, 1300, 877, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6109, 6227, 6187, 6310, 6218, 5763, 6291, 6047] avg=6144.0000 max=6310 max_rank=3 min=5763 min_rank=5 imbalance_ratio=1.027018 max_load_reduction=42842 max_load_reduction_pct=0.871623 moved_to_echo=24319 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1177, 341, 804, 847, 270, 831, 786, 949, 0, 0, 0, 0, 0, 0, 0, 0, 1325, 197, 671, 292, 366, 1029, 955, 1165, 0, 0, 0, 0, 0, 0, 0, 0, 668, 955, 808, 545, 325, 501, 316, 1409, 207, 183, 154, 39, 0, 0, 0, 0, 558, 694, 824, 328, 704, 89, 926, 1264, 859, 0, 0, 0, 0, 0, 0, 0, 841, 1011, 844, 1100, 515, 179, 622, 1105, 0, 0, 0, 0, 0, 0, 0, 0, 612, 567, 535, 825, 877, 950, 682, 624, 331, 353, 0, 0, 0, 0, 0, 0, 708, 580, 797, 527, 983, 751, 798, 814, 0, 0, 0, 0, 0, 0, 0, 0, 405, 592, 1460, 769, 132, 741, 1128, 1033, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6005, 6000, 6110, 6246, 6217, 6356, 5958, 6260] avg=6144.0000 max=6356 max_rank=5 min=5958 min_rank=6 imbalance_ratio=1.034505 max_load_reduction=42796 max_load_reduction_pct=0.870687 moved_to_echo=24791 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1192, 360, 772, 936, 325, 1119, 846, 773, 0, 0, 0, 0, 0, 0, 0, 0, 1418, 186, 713, 325, 287, 992, 892, 1099, 0, 0, 0, 0, 0, 0, 0, 0, 754, 943, 733, 623, 286, 534, 258, 1145, 199, 185, 169, 38, 0, 0, 0, 0, 588, 749, 721, 362, 691, 81, 851, 1444, 797, 0, 0, 0, 0, 0, 0, 0, 851, 808, 970, 1070, 537, 235, 578, 928, 0, 0, 0, 0, 0, 0, 0, 0, 673, 675, 569, 803, 907, 958, 584, 512, 362, 327, 0, 0, 0, 0, 0, 0, 774, 638, 767, 500, 1040, 824, 790, 847, 0, 0, 0, 0, 0, 0, 0, 0, 376, 299, 1454, 923, 165, 776, 1253, 993, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6323, 5912, 5867, 6284, 5977, 6370, 6180, 6239] avg=6144.0000 max=6370 max_rank=5 min=5867 min_rank=2 imbalance_ratio=1.036784 max_load_reduction=42782 max_load_reduction_pct=0.870402 moved_to_echo=24766 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1220, 322, 708, 817, 228, 1324, 902, 625, 0, 0, 0, 0, 0, 0, 0, 0, 1192, 333, 938, 300, 213, 1443, 880, 883, 0, 0, 0, 0, 0, 0, 0, 0, 786, 1013, 962, 669, 257, 531, 317, 1110, 295, 195, 198, 49, 0, 0, 0, 0, 372, 658, 775, 426, 793, 124, 766, 1405, 790, 0, 0, 0, 0, 0, 0, 0, 929, 773, 1042, 1213, 486, 218, 649, 940, 0, 0, 0, 0, 0, 0, 0, 0, 502, 544, 676, 580, 911, 943, 498, 461, 420, 328, 0, 0, 0, 0, 0, 0, 734, 638, 677, 603, 984, 933, 666, 744, 0, 0, 0, 0, 0, 0, 0, 0, 362, 322, 1584, 769, 225, 786, 1357, 836, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6146, 6182, 6382, 6109, 6250, 5863, 5979, 6241] avg=6144.0000 max=6382 max_rank=2 min=5863 min_rank=5 imbalance_ratio=1.038737 max_load_reduction=42770 max_load_reduction_pct=0.870158 moved_to_echo=24333 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1258, 286, 702, 872, 279, 1103, 917, 790, 0, 0, 0, 0, 0, 0, 0, 0, 1341, 254, 706, 336, 312, 1206, 992, 1136, 0, 0, 0, 0, 0, 0, 0, 0, 792, 891, 840, 715, 284, 569, 276, 1259, 243, 195, 180, 36, 0, 0, 0, 0, 508, 634, 768, 397, 780, 103, 707, 1157, 774, 0, 0, 0, 0, 0, 0, 0, 791, 645, 1053, 1004, 692, 228, 585, 966, 0, 0, 0, 0, 0, 0, 0, 0, 617, 559, 512, 796, 918, 977, 600, 607, 386, 318, 0, 0, 0, 0, 0, 0, 816, 644, 745, 497, 974, 878, 790, 907, 0, 0, 0, 0, 0, 0, 0, 0, 366, 267, 1506, 765, 214, 787, 1274, 870, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6207, 6283, 6280, 5828, 5964, 6290, 6251, 6049] avg=6144.0000 max=6290 max_rank=5 min=5828 min_rank=3 imbalance_ratio=1.023763 max_load_reduction=42862 max_load_reduction_pct=0.872030 moved_to_echo=24554 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[6784, 6528, 8576, 9088, 4096, 1792, 4864, 7808, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[10624, 2048, 6016, 2816, 2432, 9856, 7424, 8448, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[9728, 2688, 6272, 6912, 2304, 8448, 7040, 6400, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[4736, 4608, 4736, 6016, 7296, 7808, 4480, 4480, 3072, 2688, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3072, 2816, 11904, 6272, 1536, 6272, 10112, 7680, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[6144, 4992, 6144, 4608, 8064, 6784, 6016, 6784, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[6016, 7808, 6528, 4992, 2432, 4352, 2432, 9984, 2048, 1536, 1408, 384, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3968, 5504, 6272, 3200, 6016, 1024, 6528, 10752, 6400, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[669, 799, 806, 531, 973, 220, 283, 1302, 107, 938, 1358, 416, 629, 826, 800, 1502, 1149, 1279, 882, 483, 1408, 232, 1693, 1101, 709, 703, 811, 962, 116, 388, 203, 154, 472, 247, 466, 151, 1170, 1275, 978, 1042, 2138, 1427, 980, 1330, 361, 712, 36, 202, 969, 1280, 1134, 28, 171, 1625, 204, 1097, 931, 390, 865, 554, 1065, 569, 327, 524] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[650, 760, 871, 506, 884, 278, 387, 1061, 103, 967, 1294, 315, 672, 996, 913, 1494, 1174, 1259, 922, 365, 1535, 154, 1722, 959, 651, 691, 705, 822, 78, 322, 212, 115, 400, 200, 567, 206, 1256, 1835, 953, 995, 2344, 1358, 1286, 1444, 425, 675, 23, 189, 818, 1169, 801, 25, 137, 1526, 260, 1218, 839, 309, 832, 698, 1160, 513, 309, 545] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[577, 755, 770, 553, 1004, 248, 371, 1156, 82, 1199, 1486, 399, 667, 1046, 855, 1525, 1091, 1245, 967, 449, 1481, 165, 1664, 974, 637, 683, 854, 864, 58, 341, 221, 112, 399, 171, 554, 182, 1141, 1414, 1007, 931, 2252, 1444, 1062, 1540, 406, 658, 31, 203, 842, 1163, 985, 15, 130, 1472, 249, 1186, 932, 386, 831, 563, 1174, 536, 290, 504] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[684, 953, 730, 417, 1243, 227, 389, 1056, 78, 827, 1676, 376, 554, 844, 651, 1640, 862, 1323, 879, 478, 1120, 308, 1769, 1335, 946, 721, 806, 1123, 99, 425, 234, 151, 498, 289, 343, 82, 1069, 1166, 1129, 989, 2211, 1538, 705, 1098, 494, 975, 38, 278, 1142, 1021, 1399, 15, 168, 1426, 220, 1240, 919, 310, 709, 504, 995, 556, 324, 378] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[622, 857, 668, 575, 900, 306, 410, 1018, 105, 903, 1376, 340, 699, 1029, 811, 1525, 1077, 1216, 942, 352, 1510, 204, 1772, 1004, 697, 751, 763, 846, 95, 287, 203, 120, 458, 227, 541, 184, 1132, 1497, 1219, 1031, 2202, 1486, 1159, 1385, 469, 724, 27, 241, 766, 1260, 827, 18, 196, 1521, 244, 1249, 884, 366, 824, 554, 1086, 554, 326, 512] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[626, 831, 658, 413, 1183, 217, 305, 1201, 77, 988, 1590, 369, 655, 834, 682, 1683, 990, 1294, 845, 386, 1365, 257, 1704, 1161, 795, 703, 827, 1000, 73, 288, 199, 121, 382, 231, 439, 139, 1299, 1159, 1033, 947, 2364, 1556, 886, 1184, 368, 932, 51, 311, 1079, 1148, 1043, 14, 128, 1680, 228, 1237, 919, 307, 794, 553, 1057, 571, 310, 483] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[716, 828, 554, 479, 1345, 221, 293, 1351, 102, 920, 1835, 399, 546, 849, 529, 1774, 869, 1343, 914, 403, 1274, 325, 1730, 1567, 754, 802, 795, 1057, 101, 300, 157, 108, 364, 183, 381, 81, 1213, 860, 885, 843, 2070, 1306, 688, 1089, 329, 990, 20, 260, 1604, 1127, 1582, 3, 90, 1277, 244, 1040, 1068, 346, 1010, 549, 1061, 572, 337, 440] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[632, 854, 798, 497, 1052, 196, 336, 1245, 73, 969, 1478, 389, 562, 897, 866, 1544, 1163, 1194, 861, 590, 1351, 236, 1633, 1212, 760, 766, 815, 952, 123, 440, 175, 90, 423, 274, 450, 117, 1124, 1254, 950, 903, 2150, 1476, 901, 1212, 407, 926, 23, 224, 1047, 1195, 1101, 13, 188, 1504, 237, 1137, 856, 351, 1001, 568, 963, 599, 335, 494] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[669, 799, 806, 531, 973, 220, 283, 1302, 457, 126, 0, 0, 0, 0, 0, 0, 107, 938, 1358, 416, 629, 826, 800, 854, 0, 0, 0, 0, 0, 0, 0, 0, 1149, 1279, 882, 483, 1111, 232, 0, 1101, 0, 0, 0, 0, 0, 0, 0, 0, 709, 703, 811, 962, 116, 388, 203, 154, 1693, 432, 0, 0, 0, 0, 0, 0, 472, 247, 466, 151, 1170, 1275, 978, 1042, 271, 171, 0, 0, 0, 0, 0, 0, 918, 1427, 980, 1330, 361, 712, 36, 202, 0, 0, 0, 0, 0, 0, 0, 0, 969, 1280, 1134, 28, 171, 1354, 204, 1097, 0, 0, 0, 0, 0, 0, 0, 0, 931, 390, 865, 554, 1065, 569, 327, 524, 788, 191, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6166, 5928, 6237, 6171, 6243, 5966, 6237, 6204] avg=6144.0000 max=6243 max_rank=4 min=5928 min_rank=1 imbalance_ratio=1.016113 max_load_reduction=42909 max_load_reduction_pct=0.872986 moved_to_echo=24650 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[622, 857, 668, 575, 900, 306, 410, 1018, 464, 135, 0, 0, 0, 0, 0, 0, 105, 903, 1376, 340, 699, 1029, 811, 867, 0, 0, 0, 0, 0, 0, 0, 0, 1077, 1216, 942, 352, 1192, 204, 0, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 697, 751, 763, 846, 95, 287, 203, 120, 1772, 445, 0, 0, 0, 0, 0, 0, 458, 227, 541, 184, 1132, 1497, 1219, 1031, 254, 183, 0, 0, 0, 0, 0, 0, 945, 1486, 1159, 1385, 469, 724, 27, 241, 0, 0, 0, 0, 0, 0, 0, 0, 766, 1260, 827, 18, 196, 1267, 244, 1249, 0, 0, 0, 0, 0, 0, 0, 0, 884, 366, 824, 554, 1086, 554, 326, 512, 812, 194, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5955, 6130, 5987, 5979, 6726, 6436, 5827, 6112] avg=6144.0000 max=6726 max_rank=4 min=5827 min_rank=6 imbalance_ratio=1.094727 max_load_reduction=42426 max_load_reduction_pct=0.863159 moved_to_echo=25101 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[626, 831, 658, 413, 1183, 217, 305, 1201, 512, 122, 0, 0, 0, 0, 0, 0, 77, 988, 1590, 369, 655, 834, 682, 957, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1294, 845, 386, 1078, 257, 0, 1161, 0, 0, 0, 0, 0, 0, 0, 0, 795, 703, 827, 1000, 73, 288, 199, 121, 1704, 477, 0, 0, 0, 0, 0, 0, 382, 231, 439, 139, 1299, 1159, 1033, 947, 280, 165, 0, 0, 0, 0, 0, 0, 1015, 1556, 886, 1184, 368, 932, 51, 311, 0, 0, 0, 0, 0, 0, 0, 0, 1079, 1148, 1043, 14, 128, 1400, 228, 1237, 0, 0, 0, 0, 0, 0, 0, 0, 919, 307, 794, 553, 1057, 571, 310, 483, 872, 214, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6068, 6152, 6011, 6187, 6074, 6303, 6277, 6080] avg=6144.0000 max=6303 max_rank=5 min=6011 min_rank=2 imbalance_ratio=1.025879 max_load_reduction=42849 max_load_reduction_pct=0.871765 moved_to_echo=24734 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[577, 755, 770, 553, 1004, 248, 371, 1156, 464, 132, 0, 0, 0, 0, 0, 0, 82, 1199, 1486, 399, 667, 1046, 855, 867, 0, 0, 0, 0, 0, 0, 0, 0, 1091, 1245, 967, 449, 1170, 165, 0, 974, 0, 0, 0, 0, 0, 0, 0, 0, 637, 683, 854, 864, 58, 341, 221, 112, 1664, 455, 0, 0, 0, 0, 0, 0, 399, 171, 554, 182, 1141, 1414, 1007, 931, 245, 179, 0, 0, 0, 0, 0, 0, 967, 1444, 1062, 1540, 406, 658, 31, 203, 0, 0, 0, 0, 0, 0, 0, 0, 842, 1163, 985, 15, 130, 1227, 249, 1186, 0, 0, 0, 0, 0, 0, 0, 0, 932, 386, 831, 563, 1174, 536, 290, 504, 830, 194, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6030, 6601, 6061, 5889, 6223, 6311, 5797, 6240] avg=6144.0000 max=6601 max_rank=1 min=5797 min_rank=6 imbalance_ratio=1.074382 max_load_reduction=42551 max_load_reduction_pct=0.865702 moved_to_echo=24571 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[684, 953, 730, 417, 1243, 227, 389, 1056, 499, 100, 0, 0, 0, 0, 0, 0, 78, 827, 1676, 376, 554, 844, 651, 932, 0, 0, 0, 0, 0, 0, 0, 0, 862, 1323, 879, 478, 884, 308, 0, 1335, 0, 0, 0, 0, 0, 0, 0, 0, 946, 721, 806, 1123, 99, 425, 234, 151, 1769, 447, 0, 0, 0, 0, 0, 0, 498, 289, 343, 82, 1069, 1166, 1129, 989, 238, 136, 0, 0, 0, 0, 0, 0, 949, 1538, 705, 1098, 494, 975, 38, 278, 0, 0, 0, 0, 0, 0, 0, 0, 1142, 1021, 1399, 15, 168, 1188, 220, 1240, 0, 0, 0, 0, 0, 0, 0, 0, 919, 310, 709, 504, 995, 556, 324, 378, 815, 209, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6298, 5938, 6069, 6721, 5939, 6075, 6393, 5719] avg=6144.0000 max=6721 max_rank=3 min=5719 min_rank=7 imbalance_ratio=1.093913 max_load_reduction=42431 max_load_reduction_pct=0.863261 moved_to_echo=24126 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[650, 760, 871, 506, 884, 278, 387, 1061, 454, 137, 0, 0, 0, 0, 0, 0, 103, 967, 1294, 315, 672, 996, 913, 850, 0, 0, 0, 0, 0, 0, 0, 0, 1174, 1259, 922, 365, 1212, 154, 0, 959, 0, 0, 0, 0, 0, 0, 0, 0, 651, 691, 705, 822, 78, 322, 212, 115, 1722, 473, 0, 0, 0, 0, 0, 0, 400, 200, 567, 206, 1256, 1835, 953, 995, 254, 186, 0, 0, 0, 0, 0, 0, 1007, 1358, 1286, 1444, 425, 675, 23, 189, 0, 0, 0, 0, 0, 0, 0, 0, 818, 1169, 801, 25, 137, 1272, 260, 1218, 0, 0, 0, 0, 0, 0, 0, 0, 839, 309, 832, 698, 1160, 513, 309, 545, 864, 190, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5988, 6110, 6045, 5791, 6852, 6407, 5700, 6259] avg=6144.0000 max=6852 max_rank=4 min=5700 min_rank=6 imbalance_ratio=1.115234 max_load_reduction=42300 max_load_reduction_pct=0.860596 moved_to_echo=25218 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[716, 828, 554, 479, 1345, 221, 293, 1351, 542, 117, 0, 0, 0, 0, 0, 0, 102, 920, 1835, 399, 546, 849, 529, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 869, 1343, 914, 403, 999, 325, 0, 1567, 0, 0, 0, 0, 0, 0, 0, 0, 754, 802, 795, 1057, 101, 300, 157, 108, 1730, 422, 0, 0, 0, 0, 0, 0, 364, 183, 381, 81, 1213, 860, 885, 843, 217, 158, 0, 0, 0, 0, 0, 0, 880, 1306, 688, 1089, 329, 990, 20, 260, 0, 0, 0, 0, 0, 0, 0, 0, 1604, 1127, 1582, 3, 90, 1060, 244, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 346, 1010, 549, 1061, 572, 337, 440, 768, 230, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6446, 6182, 6420, 6226, 5185, 5562, 6750, 6381] avg=6144.0000 max=6750 max_rank=6 min=5185 min_rank=4 imbalance_ratio=1.098633 max_load_reduction=42402 max_load_reduction_pct=0.862671 moved_to_echo=23878 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[632, 854, 798, 497, 1052, 196, 336, 1245, 470, 121, 0, 0, 0, 0, 0, 0, 73, 969, 1478, 389, 562, 897, 866, 877, 0, 0, 0, 0, 0, 0, 0, 0, 1163, 1194, 861, 590, 1066, 236, 0, 1212, 0, 0, 0, 0, 0, 0, 0, 0, 760, 766, 815, 952, 123, 440, 175, 90, 1633, 434, 0, 0, 0, 0, 0, 0, 423, 274, 450, 117, 1124, 1254, 950, 903, 251, 164, 0, 0, 0, 0, 0, 0, 923, 1476, 901, 1212, 407, 926, 23, 224, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1195, 1101, 13, 188, 1253, 237, 1137, 0, 0, 0, 0, 0, 0, 0, 0, 856, 351, 1001, 568, 963, 599, 335, 494, 793, 197, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6201, 6111, 6322, 6188, 5910, 6092, 6171, 6157] avg=6144.0000 max=6322 max_rank=2 min=5910 min_rank=4 imbalance_ratio=1.028971 max_load_reduction=42830 max_load_reduction_pct=0.871379 moved_to_echo=24330 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[7680, 11648, 7680, 10368, 3328, 6656, 256, 1920, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[768, 7808, 12160, 3072, 4992, 7424, 6144, 7296, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7424, 2816, 6912, 4608, 8576, 4480, 2560, 3968, 6656, 1664, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6016, 5888, 6400, 7680, 768, 2816, 1664, 1024, 13696, 3712, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[8448, 10240, 7296, 3584, 8832, 1920, 0, 9344, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5248, 6656, 5888, 4096, 8704, 1920, 2816, 9472, 3968, 1024, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[8320, 9472, 8960, 256, 1280, 10112, 1920, 9472, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3456, 1920, 3840, 1152, 9472, 10496, 8192, 7808, 2048, 1408, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[172, 861, 484, 370, 78, 1063, 800, 290, 595, 410, 1149, 1255, 1772, 1239, 175, 1211, 1128, 1389, 250, 401, 246, 307, 1190, 966, 535, 794, 985, 805, 317, 531, 943, 310, 400, 1112, 1195, 700, 1871, 1861, 1659, 522, 575, 250, 379, 233, 1792, 1135, 169, 1301, 919, 216, 969, 826, 128, 319, 303, 769, 1534, 455, 82, 482, 110, 1248, 414, 2203] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[151, 1006, 428, 476, 83, 968, 881, 315, 582, 368, 1362, 1452, 1650, 1497, 293, 1077, 1184, 1410, 328, 437, 126, 320, 1239, 907, 459, 777, 1008, 885, 427, 609, 885, 258, 397, 1109, 980, 540, 2051, 1686, 1447, 467, 701, 231, 361, 249, 1591, 1240, 167, 1249, 1208, 213, 861, 686, 151, 361, 361, 645, 1457, 348, 112, 357, 182, 1317, 439, 2140] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[18, 1125, 377, 472, 12, 1273, 899, 211, 535, 356, 1507, 1562, 1462, 1125, 50, 886, 1204, 1573, 564, 421, 172, 175, 1350, 1069, 525, 855, 1553, 1080, 352, 503, 1121, 119, 297, 937, 904, 341, 2640, 1614, 1234, 442, 1047, 233, 355, 544, 1437, 1383, 201, 1214, 1180, 145, 355, 738, 21, 57, 318, 453, 1340, 258, 81, 593, 153, 1793, 154, 2184] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[163, 923, 433, 434, 125, 1050, 710, 318, 663, 388, 1297, 1099, 1680, 1373, 356, 1350, 1094, 1423, 310, 451, 103, 314, 1179, 866, 428, 782, 916, 701, 322, 510, 957, 408, 357, 976, 1212, 852, 1702, 1962, 1667, 524, 500, 284, 383, 263, 1655, 1103, 160, 1131, 1091, 245, 1037, 836, 168, 428, 262, 754, 1489, 420, 100, 469, 79, 1478, 335, 2104] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[50, 859, 488, 391, 26, 1308, 847, 311, 503, 430, 1425, 1255, 1787, 962, 145, 923, 1249, 1495, 426, 399, 81, 203, 1150, 1057, 581, 704, 1282, 1129, 336, 468, 891, 245, 377, 1202, 1393, 609, 2147, 1975, 1488, 423, 684, 219, 338, 231, 1499, 1366, 177, 1259, 1035, 155, 946, 775, 82, 226, 280, 559, 1541, 357, 80, 375, 162, 1505, 139, 2142] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[36, 1067, 487, 372, 30, 1185, 756, 263, 641, 412, 1336, 1349, 1770, 1068, 87, 1042, 1181, 1697, 431, 375, 198, 178, 1258, 1044, 647, 773, 1221, 951, 341, 496, 936, 210, 342, 1113, 1209, 489, 2333, 1898, 1326, 451, 723, 186, 347, 308, 1519, 1300, 173, 1205, 993, 132, 784, 836, 84, 118, 240, 539, 1574, 396, 103, 475, 87, 1626, 153, 2252] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[9, 1133, 429, 487, 8, 1229, 839, 304, 534, 371, 1389, 1521, 1865, 840, 35, 614, 1262, 1719, 413, 336, 293, 160, 1111, 899, 768, 874, 1342, 1222, 281, 423, 970, 112, 395, 1178, 1115, 440, 2645, 1986, 1244, 426, 761, 174, 357, 302, 1499, 1306, 166, 1104, 1064, 110, 493, 889, 74, 116, 260, 492, 1512, 453, 114, 309, 123, 1734, 105, 2414] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[236, 742, 411, 405, 100, 1081, 753, 314, 644, 334, 1300, 1081, 1701, 1751, 553, 1527, 1148, 1536, 268, 489, 117, 351, 1153, 863, 404, 803, 885, 547, 386, 537, 919, 382, 361, 911, 1081, 761, 1460, 1875, 1650, 499, 464, 235, 372, 230, 1823, 1151, 157, 1330, 980, 270, 1057, 770, 229, 402, 239, 829, 1477, 382, 62, 514, 95, 1248, 547, 1970] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[151, 1006, 428, 476, 83, 968, 881, 315, 1650, 95, 86, 0, 0, 0, 0, 0, 582, 368, 1346, 1452, 0, 1497, 293, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 1184, 1410, 328, 437, 126, 320, 1239, 907, 47, 16, 0, 0, 0, 0, 0, 0, 459, 777, 1008, 885, 427, 609, 885, 258, 615, 0, 0, 0, 0, 0, 0, 0, 397, 1109, 980, 540, 0, 955, 1447, 467, 0, 0, 0, 0, 0, 0, 0, 0, 701, 231, 361, 249, 1591, 1240, 167, 1249, 295, 30, 0, 0, 0, 0, 0, 0, 1208, 213, 861, 686, 151, 361, 361, 645, 1956, 0, 0, 0, 0, 0, 0, 0, 1457, 348, 112, 357, 182, 1317, 439, 1798, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6139, 6615, 6014, 5923, 5895, 6114, 6442, 6010] avg=6144.0000 max=6615 max_rank=1 min=5895 min_rank=4 imbalance_ratio=1.076660 max_load_reduction=42537 max_load_reduction_pct=0.865417 moved_to_echo=24461 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[163, 923, 433, 434, 125, 1050, 710, 318, 1680, 78, 100, 0, 0, 0, 0, 0, 663, 388, 1282, 1099, 0, 1373, 356, 1350, 0, 0, 0, 0, 0, 0, 0, 0, 1094, 1423, 310, 451, 103, 314, 1179, 866, 46, 15, 0, 0, 0, 0, 0, 0, 428, 782, 916, 701, 322, 510, 957, 408, 716, 0, 0, 0, 0, 0, 0, 0, 357, 976, 1212, 852, 0, 1111, 1667, 524, 0, 0, 0, 0, 0, 0, 0, 0, 500, 284, 383, 263, 1655, 1103, 160, 1131, 290, 35, 0, 0, 0, 0, 0, 0, 1091, 245, 1037, 836, 168, 428, 262, 754, 1624, 0, 0, 0, 0, 0, 0, 0, 1489, 420, 100, 469, 79, 1478, 335, 1768, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6014, 6511, 5801, 5740, 6699, 5804, 6445, 6138] avg=6144.0000 max=6699 max_rank=4 min=5740 min_rank=3 imbalance_ratio=1.090332 max_load_reduction=42453 max_load_reduction_pct=0.863708 moved_to_echo=25086 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[36, 1067, 487, 372, 30, 1185, 756, 263, 1770, 107, 97, 0, 0, 0, 0, 0, 641, 412, 1320, 1349, 0, 1068, 87, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1181, 1697, 431, 375, 198, 178, 1258, 1044, 49, 16, 0, 0, 0, 0, 0, 0, 647, 773, 1221, 951, 341, 496, 936, 210, 693, 0, 0, 0, 0, 0, 0, 0, 342, 1113, 1209, 489, 0, 1074, 1326, 451, 0, 0, 0, 0, 0, 0, 0, 0, 723, 186, 347, 308, 1519, 1300, 173, 1205, 310, 34, 0, 0, 0, 0, 0, 0, 993, 132, 784, 836, 84, 118, 240, 539, 2226, 0, 0, 0, 0, 0, 0, 0, 1574, 396, 103, 475, 87, 1626, 153, 1893, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6170, 5919, 6427, 6268, 6004, 6105, 5952, 6307] avg=6144.0000 max=6427 max_rank=2 min=5919 min_rank=1 imbalance_ratio=1.046061 max_load_reduction=42725 max_load_reduction_pct=0.869242 moved_to_echo=24368 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[172, 861, 484, 370, 78, 1063, 800, 290, 1772, 86, 95, 0, 0, 0, 0, 0, 595, 410, 1135, 1255, 0, 1239, 175, 1211, 0, 0, 0, 0, 0, 0, 0, 0, 1128, 1389, 250, 401, 246, 307, 1190, 966, 48, 14, 0, 0, 0, 0, 0, 0, 535, 794, 985, 805, 317, 531, 943, 310, 679, 0, 0, 0, 0, 0, 0, 0, 400, 1112, 1195, 700, 0, 1054, 1659, 522, 0, 0, 0, 0, 0, 0, 0, 0, 575, 250, 379, 233, 1792, 1135, 169, 1301, 303, 33, 0, 0, 0, 0, 0, 0, 919, 216, 969, 826, 128, 319, 303, 769, 1785, 0, 0, 0, 0, 0, 0, 0, 1534, 455, 82, 482, 110, 1248, 414, 1852, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6071, 6020, 5939, 5899, 6642, 6170, 6234, 6177] avg=6144.0000 max=6642 max_rank=4 min=5899 min_rank=3 imbalance_ratio=1.081055 max_load_reduction=42510 max_load_reduction_pct=0.864868 moved_to_echo=25223 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[18, 1125, 377, 472, 12, 1273, 899, 211, 1462, 122, 82, 0, 0, 0, 0, 0, 535, 356, 1489, 1562, 0, 1125, 50, 886, 0, 0, 0, 0, 0, 0, 0, 0, 1204, 1573, 564, 421, 172, 175, 1350, 1069, 48, 18, 0, 0, 0, 0, 0, 0, 525, 855, 1553, 1080, 352, 503, 1121, 119, 589, 0, 0, 0, 0, 0, 0, 0, 297, 937, 904, 341, 0, 914, 1234, 442, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 233, 355, 544, 1437, 1383, 201, 1214, 301, 29, 0, 0, 0, 0, 0, 0, 1180, 145, 355, 738, 21, 57, 318, 453, 2518, 0, 0, 0, 0, 0, 0, 0, 1340, 258, 81, 593, 153, 1793, 154, 1835, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6053, 6003, 6594, 6697, 5069, 6744, 5785, 6207] avg=6144.0000 max=6744 max_rank=5 min=5069 min_rank=4 imbalance_ratio=1.097656 max_load_reduction=42408 max_load_reduction_pct=0.862793 moved_to_echo=23805 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[236, 742, 411, 405, 100, 1081, 753, 314, 1701, 68, 96, 0, 0, 0, 0, 0, 644, 334, 1285, 1081, 0, 1751, 553, 1527, 0, 0, 0, 0, 0, 0, 0, 0, 1148, 1536, 268, 489, 117, 351, 1153, 863, 43, 15, 0, 0, 0, 0, 0, 0, 404, 803, 885, 547, 386, 537, 919, 382, 684, 0, 0, 0, 0, 0, 0, 0, 361, 911, 1081, 761, 0, 1061, 1650, 499, 0, 0, 0, 0, 0, 0, 0, 0, 464, 235, 372, 230, 1823, 1151, 157, 1330, 271, 34, 0, 0, 0, 0, 0, 0, 980, 270, 1057, 770, 229, 402, 239, 829, 1392, 0, 0, 0, 0, 0, 0, 0, 1477, 382, 62, 514, 95, 1248, 547, 1656, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5907, 7175, 5983, 5547, 6324, 6067, 6168, 5981] avg=6144.0000 max=7175 max_rank=1 min=5547 min_rank=3 imbalance_ratio=1.167806 max_load_reduction=41977 max_load_reduction_pct=0.854024 moved_to_echo=24540 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[50, 859, 488, 391, 26, 1308, 847, 311, 1787, 99, 101, 0, 0, 0, 0, 0, 503, 430, 1408, 1255, 0, 962, 145, 923, 0, 0, 0, 0, 0, 0, 0, 0, 1249, 1495, 426, 399, 81, 203, 1150, 1057, 47, 17, 0, 0, 0, 0, 0, 0, 581, 704, 1282, 1129, 336, 468, 891, 245, 721, 0, 0, 0, 0, 0, 0, 0, 377, 1202, 1393, 609, 0, 1117, 1488, 423, 0, 0, 0, 0, 0, 0, 0, 0, 684, 219, 338, 231, 1499, 1366, 177, 1259, 295, 36, 0, 0, 0, 0, 0, 0, 1035, 155, 946, 775, 82, 226, 280, 559, 2048, 0, 0, 0, 0, 0, 0, 0, 1541, 357, 80, 375, 162, 1505, 139, 1800, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6267, 5626, 6124, 6357, 6609, 6104, 6106, 5959] avg=6144.0000 max=6609 max_rank=4 min=5626 min_rank=1 imbalance_ratio=1.075684 max_load_reduction=42543 max_load_reduction_pct=0.865540 moved_to_echo=24778 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[9, 1133, 429, 487, 8, 1229, 839, 304, 1865, 123, 105, 0, 0, 0, 0, 0, 534, 371, 1368, 1521, 0, 840, 35, 614, 0, 0, 0, 0, 0, 0, 0, 0, 1262, 1719, 413, 336, 293, 160, 1111, 899, 56, 21, 0, 0, 0, 0, 0, 0, 768, 874, 1342, 1222, 281, 423, 970, 112, 729, 0, 0, 0, 0, 0, 0, 0, 395, 1178, 1115, 440, 0, 1112, 1244, 426, 0, 0, 0, 0, 0, 0, 0, 0, 761, 174, 357, 302, 1499, 1306, 166, 1104, 335, 40, 0, 0, 0, 0, 0, 0, 1064, 110, 493, 889, 74, 116, 260, 492, 2522, 0, 0, 0, 0, 0, 0, 0, 1512, 453, 114, 309, 123, 1734, 105, 2023, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6531, 5283, 6270, 6721, 5910, 6044, 6020, 6373] avg=6144.0000 max=6721 max_rank=3 min=5283 min_rank=1 imbalance_ratio=1.093913 max_load_reduction=42431 max_load_reduction_pct=0.863261 moved_to_echo=24347 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[8576, 1536, 6528, 6400, 1024, 2048, 2304, 5120, 16128, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4736, 3072, 10752, 10624, 0, 9856, 1792, 8704, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5504, 1920, 2944, 2432, 12928, 9984, 1408, 9856, 2432, 384, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[2944, 8576, 9216, 4736, 0, 8448, 11776, 3840, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4352, 6400, 9216, 7424, 2816, 4096, 7680, 2048, 5504, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[12032, 3072, 768, 3584, 1024, 12032, 2304, 14720, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[896, 7808, 3584, 3456, 512, 9216, 6528, 2432, 13696, 896, 768, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[9472, 12288, 3072, 3328, 1408, 2048, 9728, 7680, 384, 256, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[327, 715, 1175, 471, 657, 537, 678, 773, 1759, 2541, 194, 468, 1233, 530, 1153, 363, 1660, 117, 1922, 509, 473, 342, 13, 644, 1131, 1103, 676, 531, 517, 1164, 1506, 1130, 155, 652, 323, 335, 1000, 650, 1725, 492, 255, 2041, 192, 518, 342, 588, 433, 2623, 121, 357, 241, 972, 748, 1096, 359, 481, 414, 155, 1224, 577, 1001, 839, 532, 699] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[214, 419, 1495, 308, 636, 694, 746, 921, 1956, 3342, 367, 497, 1730, 412, 1268, 348, 1757, 73, 1919, 473, 658, 564, 1, 550, 1324, 1264, 600, 615, 537, 1015, 1242, 679, 264, 603, 179, 274, 890, 628, 1559, 638, 127, 2378, 101, 469, 267, 456, 341, 2894, 4, 253, 137, 1133, 493, 918, 166, 195, 369, 188, 1336, 531, 1088, 809, 431, 409] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[202, 381, 1667, 274, 754, 890, 732, 1011, 2245, 3306, 300, 475, 1339, 434, 1291, 359, 1707, 80, 2000, 490, 734, 460, 0, 591, 853, 1422, 587, 548, 482, 1192, 1371, 692, 294, 557, 235, 305, 975, 604, 1325, 613, 85, 2240, 142, 404, 235, 524, 350, 2554, 1, 283, 160, 1160, 424, 986, 184, 149, 463, 87, 1368, 509, 982, 922, 459, 704] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[381, 711, 1346, 293, 675, 558, 706, 848, 1636, 2732, 343, 520, 1748, 434, 966, 414, 1654, 70, 2203, 532, 528, 388, 24, 676, 1309, 1289, 667, 602, 523, 897, 1315, 764, 399, 785, 277, 318, 928, 632, 1439, 628, 382, 2267, 152, 484, 350, 545, 395, 2983, 46, 237, 225, 1003, 442, 1078, 243, 274, 289, 155, 1378, 565, 1037, 738, 392, 334] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[188, 357, 1591, 382, 803, 762, 761, 1003, 1790, 3183, 261, 455, 1139, 564, 1306, 370, 1926, 90, 1910, 402, 679, 502, 1, 537, 1245, 1291, 578, 424, 641, 1123, 1638, 801, 373, 421, 174, 313, 1128, 621, 1712, 647, 44, 2017, 131, 436, 301, 368, 342, 2700, 3, 259, 134, 1085, 562, 942, 194, 193, 523, 210, 1338, 510, 1103, 856, 395, 414] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[470, 763, 1448, 347, 703, 534, 823, 751, 1226, 2217, 287, 423, 1636, 554, 835, 415, 1575, 84, 2340, 536, 591, 339, 25, 676, 1129, 1212, 660, 629, 466, 919, 1272, 813, 220, 758, 279, 322, 871, 727, 1863, 619, 414, 2326, 128, 533, 402, 685, 486, 2903, 70, 374, 220, 968, 496, 1068, 298, 377, 284, 196, 1453, 577, 995, 656, 544, 342] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[240, 418, 1763, 286, 882, 756, 658, 1206, 1767, 3099, 475, 475, 1766, 436, 1251, 197, 1617, 56, 1707, 524, 754, 436, 4, 667, 1155, 1601, 608, 448, 552, 1009, 1239, 686, 430, 606, 225, 295, 832, 493, 1369, 665, 213, 2694, 145, 452, 300, 522, 318, 2866, 5, 265, 176, 988, 356, 966, 241, 234, 221, 142, 1218, 511, 961, 939, 334, 432] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[392, 814, 1411, 391, 722, 559, 866, 798, 1384, 2206, 278, 456, 1633, 413, 937, 424, 1590, 97, 2102, 552, 605, 401, 16, 650, 1156, 1163, 663, 612, 597, 937, 1302, 819, 219, 691, 322, 335, 836, 764, 1898, 546, 226, 2075, 135, 546, 359, 619, 423, 3267, 49, 358, 204, 1013, 503, 1326, 219, 308, 409, 145, 1275, 627, 1064, 652, 494, 299] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[214, 419, 1495, 308, 636, 694, 746, 921, 394, 0, 0, 0, 0, 0, 0, 0, 1956, 180, 367, 497, 1730, 412, 1268, 348, 0, 0, 0, 0, 0, 0, 0, 0, 1757, 73, 1919, 473, 658, 564, 1, 550, 171, 0, 0, 0, 0, 0, 0, 0, 1324, 1264, 600, 615, 537, 1015, 127, 679, 0, 0, 0, 0, 0, 0, 0, 0, 264, 603, 179, 274, 890, 628, 1559, 638, 493, 343, 0, 0, 0, 0, 0, 0, 127, 2378, 101, 469, 267, 456, 341, 1836, 0, 0, 0, 0, 0, 0, 0, 0, 4, 253, 137, 1133, 493, 918, 166, 195, 2973, 0, 0, 0, 0, 0, 0, 0, 369, 188, 1336, 531, 1088, 809, 431, 409, 772, 189, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5827, 6758, 6166, 6161, 5871, 5975, 6272, 6122] avg=6144.0000 max=6758 max_rank=1 min=5827 min_rank=0 imbalance_ratio=1.099935 max_load_reduction=42394 max_load_reduction_pct=0.862508 moved_to_echo=24240 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[327, 715, 1175, 471, 657, 537, 678, 773, 357, 0, 0, 0, 0, 0, 0, 0, 1759, 138, 194, 468, 1233, 530, 1153, 363, 0, 0, 0, 0, 0, 0, 0, 0, 1660, 117, 1922, 509, 473, 342, 13, 644, 155, 0, 0, 0, 0, 0, 0, 0, 1131, 1103, 676, 531, 517, 1164, 153, 1130, 0, 0, 0, 0, 0, 0, 0, 0, 155, 652, 323, 335, 1000, 650, 1725, 492, 446, 417, 0, 0, 0, 0, 0, 0, 255, 2041, 192, 518, 342, 588, 433, 1665, 0, 0, 0, 0, 0, 0, 0, 0, 121, 357, 241, 972, 748, 1096, 359, 481, 2260, 0, 0, 0, 0, 0, 0, 0, 414, 155, 1224, 577, 1001, 839, 532, 699, 936, 143, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5690, 5838, 5835, 6405, 6195, 6034, 6635, 6520] avg=6144.0000 max=6635 max_rank=6 min=5690 min_rank=0 imbalance_ratio=1.079915 max_load_reduction=42517 max_load_reduction_pct=0.865011 moved_to_echo=25384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[381, 711, 1346, 293, 675, 558, 706, 848, 406, 0, 0, 0, 0, 0, 0, 0, 1636, 148, 343, 520, 1748, 434, 966, 414, 0, 0, 0, 0, 0, 0, 0, 0, 1654, 70, 2203, 532, 528, 388, 24, 676, 176, 0, 0, 0, 0, 0, 0, 0, 1309, 1289, 667, 602, 523, 897, 134, 764, 0, 0, 0, 0, 0, 0, 0, 0, 399, 785, 277, 318, 928, 632, 1439, 628, 508, 364, 0, 0, 0, 0, 0, 0, 382, 2267, 152, 484, 350, 545, 395, 1893, 0, 0, 0, 0, 0, 0, 0, 0, 46, 237, 225, 1003, 442, 1078, 243, 274, 2430, 0, 0, 0, 0, 0, 0, 0, 289, 155, 1378, 565, 1037, 738, 392, 334, 817, 154, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5924, 6209, 6251, 6185, 6278, 6468, 5978, 5859] avg=6144.0000 max=6468 max_rank=5 min=5859 min_rank=7 imbalance_ratio=1.052734 max_load_reduction=42684 max_load_reduction_pct=0.868408 moved_to_echo=24583 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[392, 814, 1411, 391, 722, 559, 866, 798, 445, 0, 0, 0, 0, 0, 0, 0, 1384, 120, 278, 456, 1633, 413, 937, 424, 0, 0, 0, 0, 0, 0, 0, 0, 1590, 97, 2102, 552, 605, 401, 16, 650, 193, 0, 0, 0, 0, 0, 0, 0, 1156, 1163, 663, 612, 597, 937, 133, 819, 0, 0, 0, 0, 0, 0, 0, 0, 219, 691, 322, 335, 836, 764, 1898, 546, 556, 360, 0, 0, 0, 0, 0, 0, 226, 2075, 135, 546, 359, 619, 423, 2073, 0, 0, 0, 0, 0, 0, 0, 0, 49, 358, 204, 1013, 503, 1326, 219, 308, 1962, 0, 0, 0, 0, 0, 0, 0, 409, 145, 1275, 627, 1064, 652, 494, 299, 809, 124, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6398, 5645, 6206, 6080, 6527, 6456, 5942, 5898] avg=6144.0000 max=6527 max_rank=4 min=5645 min_rank=1 imbalance_ratio=1.062337 max_load_reduction=42625 max_load_reduction_pct=0.867208 moved_to_echo=24823 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[188, 357, 1591, 382, 803, 762, 761, 1003, 368, 0, 0, 0, 0, 0, 0, 0, 1790, 172, 261, 455, 1139, 564, 1306, 370, 0, 0, 0, 0, 0, 0, 0, 0, 1926, 90, 1910, 402, 679, 502, 1, 537, 159, 0, 0, 0, 0, 0, 0, 0, 1245, 1291, 578, 424, 641, 1123, 167, 801, 0, 0, 0, 0, 0, 0, 0, 0, 373, 421, 174, 313, 1128, 621, 1712, 647, 460, 453, 0, 0, 0, 0, 0, 0, 44, 2017, 131, 436, 301, 368, 342, 1713, 0, 0, 0, 0, 0, 0, 0, 0, 3, 259, 134, 1085, 562, 942, 194, 193, 2831, 0, 0, 0, 0, 0, 0, 0, 523, 210, 1338, 510, 1103, 856, 395, 414, 1018, 180, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6215, 6057, 6206, 6270, 6302, 5352, 6203, 6547] avg=6144.0000 max=6547 max_rank=7 min=5352 min_rank=5 imbalance_ratio=1.065592 max_load_reduction=42605 max_load_reduction_pct=0.866801 moved_to_echo=24404 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[470, 763, 1448, 347, 703, 534, 823, 751, 396, 0, 0, 0, 0, 0, 0, 0, 1226, 120, 287, 423, 1636, 554, 835, 415, 0, 0, 0, 0, 0, 0, 0, 0, 1575, 84, 2340, 536, 591, 339, 25, 676, 171, 0, 0, 0, 0, 0, 0, 0, 1129, 1212, 660, 629, 466, 919, 129, 813, 0, 0, 0, 0, 0, 0, 0, 0, 220, 758, 279, 322, 871, 727, 1863, 619, 494, 352, 0, 0, 0, 0, 0, 0, 414, 2326, 128, 533, 402, 685, 486, 1842, 0, 0, 0, 0, 0, 0, 0, 0, 70, 374, 220, 968, 496, 1068, 298, 377, 1972, 0, 0, 0, 0, 0, 0, 0, 284, 196, 1453, 577, 995, 656, 544, 342, 791, 125, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6235, 5496, 6337, 5957, 6505, 6816, 5843, 5963] avg=6144.0000 max=6816 max_rank=5 min=5496 min_rank=1 imbalance_ratio=1.109375 max_load_reduction=42336 max_load_reduction_pct=0.861328 moved_to_echo=25127 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[240, 418, 1763, 286, 882, 756, 658, 1206, 390, 0, 0, 0, 0, 0, 0, 0, 1767, 168, 475, 475, 1766, 436, 1251, 197, 0, 0, 0, 0, 0, 0, 0, 0, 1617, 56, 1707, 524, 754, 436, 4, 667, 169, 0, 0, 0, 0, 0, 0, 0, 1155, 1601, 608, 448, 552, 1009, 126, 686, 0, 0, 0, 0, 0, 0, 0, 0, 430, 606, 225, 295, 832, 493, 1369, 665, 488, 343, 0, 0, 0, 0, 0, 0, 213, 2694, 145, 452, 300, 522, 318, 1819, 0, 0, 0, 0, 0, 0, 0, 0, 5, 265, 176, 988, 356, 966, 241, 234, 2756, 0, 0, 0, 0, 0, 0, 0, 221, 142, 1218, 511, 961, 939, 334, 432, 770, 175, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6599, 6535, 5934, 6185, 5746, 6463, 5987, 5703] avg=6144.0000 max=6599 max_rank=0 min=5703 min_rank=7 imbalance_ratio=1.074056 max_load_reduction=42553 max_load_reduction_pct=0.865743 moved_to_echo=23899 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[202, 381, 1667, 274, 754, 890, 732, 1011, 353, 0, 0, 0, 0, 0, 0, 0, 2245, 171, 300, 475, 1339, 434, 1291, 359, 0, 0, 0, 0, 0, 0, 0, 0, 1707, 80, 2000, 490, 734, 460, 0, 591, 155, 0, 0, 0, 0, 0, 0, 0, 853, 1422, 587, 548, 482, 1192, 133, 692, 0, 0, 0, 0, 0, 0, 0, 0, 294, 557, 235, 305, 975, 604, 1325, 613, 438, 382, 0, 0, 0, 0, 0, 0, 85, 2240, 142, 404, 235, 524, 350, 1608, 0, 0, 0, 0, 0, 0, 0, 0, 1, 283, 160, 1160, 424, 986, 184, 149, 2945, 0, 0, 0, 0, 0, 0, 0, 463, 87, 1368, 509, 982, 922, 459, 704, 856, 190, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6264, 6614, 6217, 5909, 5728, 5588, 6292, 6540] avg=6144.0000 max=6614 max_rank=1 min=5588 min_rank=5 imbalance_ratio=1.076497 max_load_reduction=42538 max_load_reduction_pct=0.865438 moved_to_echo=24148 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[13568, 768, 16128, 4096, 5120, 3456, 128, 4992, 1408, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2432, 4608, 11904, 2816, 5888, 5376, 6016, 7424, 3200, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[1792, 18048, 1152, 3968, 2560, 4352, 3200, 14464, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[384, 2432, 1536, 8448, 4096, 8448, 1920, 2304, 20224, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2432, 5120, 2048, 2560, 7552, 5120, 12928, 4864, 3968, 3072, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[9344, 10368, 5120, 4480, 4352, 8320, 1152, 6400, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[13824, 1280, 2560, 3840, 12288, 3840, 9088, 2944, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3072, 1280, 10624, 4480, 8320, 6528, 3584, 3712, 6784, 1280, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[2, 1909, 1690, 884, 1130, 1346, 733, 1213, 451, 1559, 2485, 129, 170, 535, 355, 1120, 1279, 35, 131, 54, 132, 552, 256, 413, 956, 417, 1665, 565, 1710, 945, 627, 83, 251, 19, 33, 1256, 1124, 242, 181, 1214, 375, 450, 62, 881, 586, 12, 1703, 3, 243, 867, 350, 794, 428, 1254, 882, 1146, 927, 775, 167, 1155, 1171, 2837, 997, 1236] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[12, 1860, 1668, 813, 1083, 1113, 677, 1214, 632, 1341, 2565, 99, 159, 642, 368, 750, 1487, 24, 165, 65, 154, 643, 284, 327, 987, 385, 1594, 458, 1859, 924, 542, 74, 250, 5, 9, 1589, 1095, 220, 126, 1155, 287, 700, 87, 907, 584, 5, 1981, 8, 123, 819, 253, 755, 434, 1336, 781, 1112, 934, 839, 175, 1218, 1224, 2623, 1064, 1486] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[25, 2090, 1677, 515, 1063, 1562, 742, 977, 592, 1225, 2326, 193, 202, 625, 483, 883, 1384, 14, 248, 129, 145, 522, 325, 353, 797, 400, 1434, 600, 1838, 1212, 485, 112, 279, 86, 37, 1409, 967, 376, 187, 1454, 382, 719, 383, 989, 576, 15, 1692, 67, 330, 678, 320, 765, 465, 1281, 972, 881, 936, 826, 191, 938, 997, 2520, 957, 1299] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[3, 1997, 1879, 815, 1109, 1156, 693, 942, 401, 1285, 2681, 87, 105, 615, 349, 956, 1289, 73, 119, 87, 143, 544, 221, 411, 1042, 393, 1493, 619, 1518, 1077, 597, 64, 292, 6, 15, 1145, 1235, 328, 146, 1199, 325, 546, 146, 822, 639, 0, 1752, 2, 335, 840, 339, 1040, 511, 1235, 827, 1251, 1018, 702, 166, 1126, 1223, 2630, 1174, 1374] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 1142, 1402, 796, 1122, 2149, 599, 1486, 443, 795, 2442, 102, 134, 471, 295, 969, 1539, 3, 195, 60, 351, 610, 241, 228, 456, 346, 2577, 244, 2913, 643, 428, 71, 246, 1, 0, 2188, 720, 215, 55, 1401, 260, 345, 13, 938, 409, 0, 2227, 0, 19, 707, 232, 1140, 418, 1528, 1384, 1257, 441, 856, 145, 1385, 895, 2099, 1422, 954] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 1106, 1466, 1111, 1018, 1578, 663, 1304, 496, 1100, 3055, 79, 146, 574, 323, 741, 1453, 7, 74, 52, 173, 629, 322, 237, 634, 370, 2360, 254, 2858, 714, 408, 49, 204, 0, 0, 2152, 685, 95, 56, 1158, 233, 232, 15, 847, 454, 0, 2378, 0, 36, 1081, 186, 1055, 402, 1639, 1274, 1073, 536, 852, 121, 1290, 1096, 2203, 1095, 1350] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 1011, 1464, 898, 768, 1936, 600, 1464, 585, 1345, 2652, 109, 178, 616, 338, 546, 1736, 10, 121, 40, 176, 974, 335, 223, 670, 336, 1770, 185, 2956, 514, 329, 53, 231, 0, 0, 2635, 586, 48, 22, 1290, 174, 594, 11, 808, 348, 0, 2584, 1, 5, 1132, 167, 728, 406, 2227, 1343, 917, 423, 1054, 98, 1261, 888, 2011, 899, 1323] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 920, 1514, 1268, 1015, 1122, 630, 1258, 410, 1152, 2929, 77, 153, 831, 323, 621, 1175, 4, 146, 51, 202, 785, 375, 224, 792, 330, 2416, 264, 2750, 635, 349, 67, 254, 0, 2, 2442, 788, 149, 77, 1528, 267, 324, 5, 807, 535, 1, 2448, 0, 43, 1062, 204, 1048, 392, 1941, 1127, 986, 605, 714, 136, 1136, 1103, 2108, 931, 1201] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2, 1039, 0, 884, 1130, 1346, 733, 1213, 0, 0, 0, 0, 0, 0, 0, 0, 451, 1559, 2221, 129, 170, 535, 355, 1120, 0, 0, 0, 0, 0, 0, 0, 0, 1279, 35, 131, 54, 132, 552, 256, 413, 2837, 710, 0, 0, 0, 0, 0, 0, 956, 417, 1665, 565, 962, 945, 627, 83, 0, 0, 0, 0, 0, 0, 0, 0, 251, 19, 33, 1256, 1124, 242, 181, 1214, 870, 264, 166, 163, 116, 0, 0, 0, 375, 450, 62, 881, 586, 12, 1703, 3, 980, 632, 0, 0, 0, 0, 0, 0, 243, 867, 350, 794, 428, 1088, 882, 1146, 0, 0, 0, 0, 0, 0, 0, 0, 927, 775, 167, 1155, 1171, 0, 997, 1073, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6347, 6540, 6399, 6220, 5899, 5684, 5798, 6265] avg=6144.0000 max=6540 max_rank=1 min=5684 min_rank=5 imbalance_ratio=1.064453 max_load_reduction=42612 max_load_reduction_pct=0.866943 moved_to_echo=23646 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[25, 1137, 0, 515, 1063, 1562, 742, 977, 0, 0, 0, 0, 0, 0, 0, 0, 592, 1225, 2079, 193, 202, 625, 483, 883, 0, 0, 0, 0, 0, 0, 0, 0, 1384, 14, 248, 129, 145, 522, 325, 353, 2520, 704, 0, 0, 0, 0, 0, 0, 797, 400, 1434, 600, 1034, 1212, 485, 112, 0, 0, 0, 0, 0, 0, 0, 0, 279, 86, 37, 1409, 967, 376, 187, 1454, 953, 247, 170, 171, 124, 0, 0, 0, 382, 719, 383, 989, 576, 15, 1692, 67, 973, 680, 0, 0, 0, 0, 0, 0, 330, 678, 320, 765, 465, 1111, 972, 881, 0, 0, 0, 0, 0, 0, 0, 0, 936, 826, 191, 938, 997, 0, 957, 1128, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6021, 6282, 6344, 6074, 6460, 6476, 5522, 5973] avg=6144.0000 max=6476 max_rank=5 min=5522 min_rank=6 imbalance_ratio=1.054036 max_load_reduction=42676 max_load_reduction_pct=0.868245 moved_to_echo=24431 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[12, 1012, 0, 813, 1083, 1113, 677, 1214, 0, 0, 0, 0, 0, 0, 0, 0, 632, 1341, 2293, 99, 159, 642, 368, 750, 0, 0, 0, 0, 0, 0, 0, 0, 1487, 24, 165, 65, 154, 643, 284, 327, 2623, 700, 0, 0, 0, 0, 0, 0, 987, 385, 1594, 458, 1046, 924, 542, 74, 0, 0, 0, 0, 0, 0, 0, 0, 250, 5, 9, 1589, 1095, 220, 126, 1155, 848, 272, 177, 196, 126, 0, 0, 0, 287, 700, 87, 907, 584, 5, 1981, 8, 968, 687, 0, 0, 0, 0, 0, 0, 123, 819, 253, 755, 434, 1159, 781, 1112, 0, 0, 0, 0, 0, 0, 0, 0, 934, 839, 175, 1218, 1224, 0, 1064, 1290, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5924, 6284, 6472, 6010, 6068, 6214, 5436, 6744] avg=6144.0000 max=6744 max_rank=7 min=5436 min_rank=6 imbalance_ratio=1.097656 max_load_reduction=42408 max_load_reduction_pct=0.862793 moved_to_echo=24462 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 602, 0, 1111, 1018, 1578, 663, 1304, 0, 0, 0, 0, 0, 0, 0, 0, 496, 1100, 2731, 79, 146, 574, 323, 741, 0, 0, 0, 0, 0, 0, 0, 0, 1453, 7, 74, 52, 173, 629, 322, 237, 2203, 616, 0, 0, 0, 0, 0, 0, 634, 370, 2360, 254, 1607, 714, 408, 49, 0, 0, 0, 0, 0, 0, 0, 0, 204, 0, 0, 2152, 685, 95, 56, 1158, 504, 324, 217, 178, 194, 0, 0, 0, 233, 232, 15, 847, 454, 0, 2378, 0, 850, 1057, 0, 0, 0, 0, 0, 0, 36, 1081, 186, 1055, 402, 1422, 1274, 1073, 0, 0, 0, 0, 0, 0, 0, 0, 536, 852, 121, 1290, 1096, 0, 1095, 1172, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6276, 6190, 5766, 6396, 5767, 6066, 6529, 6162] avg=6144.0000 max=6529 max_rank=6 min=5766 min_rank=2 imbalance_ratio=1.062663 max_load_reduction=42623 max_load_reduction_pct=0.867167 moved_to_echo=24524 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 622, 0, 796, 1122, 2149, 599, 1486, 0, 0, 0, 0, 0, 0, 0, 0, 443, 795, 2183, 102, 134, 471, 295, 969, 0, 0, 0, 0, 0, 0, 0, 0, 1539, 3, 195, 60, 351, 610, 241, 228, 2099, 590, 0, 0, 0, 0, 0, 0, 456, 346, 2577, 244, 1637, 643, 428, 71, 0, 0, 0, 0, 0, 0, 0, 0, 246, 1, 0, 2188, 720, 215, 55, 1401, 520, 259, 203, 126, 198, 0, 0, 0, 260, 345, 13, 938, 409, 0, 2227, 0, 812, 1078, 0, 0, 0, 0, 0, 0, 19, 707, 232, 1140, 418, 1325, 1384, 1257, 0, 0, 0, 0, 0, 0, 0, 0, 441, 856, 145, 1385, 895, 0, 1422, 828, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6774, 5392, 5916, 6402, 6132, 6082, 6482, 5972] avg=6144.0000 max=6774 max_rank=0 min=5392 min_rank=1 imbalance_ratio=1.102539 max_load_reduction=42378 max_load_reduction_pct=0.862183 moved_to_echo=24668 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[3, 1087, 0, 815, 1109, 1156, 693, 942, 0, 0, 0, 0, 0, 0, 0, 0, 401, 1285, 2397, 87, 105, 615, 349, 956, 0, 0, 0, 0, 0, 0, 0, 0, 1289, 73, 119, 87, 143, 544, 221, 411, 2630, 790, 0, 0, 0, 0, 0, 0, 1042, 393, 1493, 619, 854, 1077, 597, 64, 0, 0, 0, 0, 0, 0, 0, 0, 292, 6, 15, 1145, 1235, 328, 146, 1199, 910, 284, 164, 181, 103, 0, 0, 0, 325, 546, 146, 822, 639, 0, 1752, 2, 1089, 561, 0, 0, 0, 0, 0, 0, 335, 840, 339, 1040, 511, 1071, 827, 1251, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 702, 166, 1126, 1223, 0, 1174, 1193, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5805, 6195, 6307, 6139, 6008, 5882, 6214, 6602] avg=6144.0000 max=6602 max_rank=7 min=5805 min_rank=0 imbalance_ratio=1.074544 max_load_reduction=42550 max_load_reduction_pct=0.865682 moved_to_echo=24706 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 501, 0, 1268, 1015, 1122, 630, 1258, 0, 0, 0, 0, 0, 0, 0, 0, 410, 1152, 2618, 77, 153, 831, 323, 621, 0, 0, 0, 0, 0, 0, 0, 0, 1175, 4, 146, 51, 202, 785, 375, 224, 2108, 636, 0, 0, 0, 0, 0, 0, 792, 330, 2416, 264, 1547, 635, 349, 67, 0, 0, 0, 0, 0, 0, 0, 0, 254, 0, 2, 2442, 788, 149, 77, 1528, 419, 311, 258, 158, 186, 0, 0, 0, 267, 324, 5, 807, 535, 1, 2448, 0, 878, 1017, 0, 0, 0, 0, 0, 0, 43, 1062, 204, 1048, 392, 1683, 1127, 986, 0, 0, 0, 0, 0, 0, 0, 0, 605, 714, 136, 1136, 1103, 0, 931, 1043, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5794, 6185, 5706, 6400, 6572, 6282, 6545, 5668] avg=6144.0000 max=6572 max_rank=4 min=5668 min_rank=7 imbalance_ratio=1.069661 max_load_reduction=42580 max_load_reduction_pct=0.866292 moved_to_echo=25067 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 545, 0, 898, 768, 1936, 600, 1464, 0, 0, 0, 0, 0, 0, 0, 0, 585, 1345, 2367, 109, 178, 616, 338, 546, 0, 0, 0, 0, 0, 0, 0, 0, 1736, 10, 121, 40, 176, 974, 335, 223, 2011, 616, 0, 0, 0, 0, 0, 0, 670, 336, 1770, 185, 1654, 514, 329, 53, 0, 0, 0, 0, 0, 0, 0, 0, 231, 0, 0, 2635, 586, 48, 22, 1290, 466, 285, 299, 180, 204, 0, 0, 0, 174, 594, 11, 808, 348, 0, 2584, 1, 848, 1098, 0, 0, 0, 0, 0, 0, 5, 1132, 167, 728, 406, 1928, 1343, 917, 0, 0, 0, 0, 0, 0, 0, 0, 423, 1054, 98, 1261, 888, 0, 899, 1143, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6211, 6084, 6242, 5511, 6246, 6466, 6626, 5766] avg=6144.0000 max=6626 max_rank=6 min=5511 min_rank=3 imbalance_ratio=1.078451 max_load_reduction=42526 max_load_reduction_pct=0.865194 moved_to_echo=25104 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5888, 6656, 1280, 9600, 8704, 0, 8576, 8960, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 6656, 0, 7168, 8320, 12032, 5376, 9984, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1152, 7296, 2176, 7424, 3456, 10880, 8704, 8704, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4096, 9856, 18944, 896, 1280, 4992, 2944, 6656, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2304, 3968, 768, 7040, 4224, 128, 16768, 128, 7424, 6912, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6400, 3072, 15360, 3200, 10368, 6784, 3840, 640, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2048, 128, 128, 14848, 7296, 1792, 896, 10496, 5504, 2304, 1664, 1408, 1280, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[11392, 256, 1280, 640, 1536, 5376, 2432, 2432, 19072, 5376, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[495, 2152, 795, 326, 64, 250, 1996, 743, 3, 0, 802, 740, 2612, 0, 0, 725, 507, 851, 3470, 20, 114, 1831, 296, 0, 75, 2124, 947, 111, 76, 139, 3705, 249, 1551, 432, 79, 1501, 765, 279, 1149, 850, 36, 0, 11, 1177, 92, 867, 1365, 656, 867, 1382, 1944, 1347, 142, 478, 0, 1288, 1639, 444, 409, 391, 245, 252, 656, 640] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[555, 1907, 1003, 231, 125, 311, 2191, 249, 59, 2, 542, 965, 1702, 0, 8, 913, 772, 838, 3303, 55, 213, 882, 338, 6, 130, 1623, 728, 135, 140, 217, 3623, 219, 1514, 480, 159, 1731, 813, 332, 1790, 1125, 109, 2, 34, 1600, 97, 566, 1528, 626, 965, 1779, 1274, 939, 168, 580, 1, 1360, 1546, 505, 483, 631, 384, 407, 855, 784] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[627, 2068, 998, 285, 102, 303, 2243, 418, 38, 8, 679, 845, 1788, 5, 8, 1206, 657, 953, 3296, 39, 173, 1132, 291, 10, 174, 2034, 860, 132, 133, 182, 3994, 238, 1254, 447, 109, 1509, 875, 322, 1437, 876, 70, 19, 30, 1062, 97, 654, 1583, 637, 945, 1495, 1413, 872, 161, 608, 17, 1363, 1519, 475, 466, 554, 438, 383, 761, 782] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[734, 2098, 1106, 353, 226, 294, 2121, 381, 135, 23, 399, 956, 1848, 1, 27, 1307, 781, 929, 3005, 96, 211, 943, 417, 38, 242, 1339, 797, 122, 185, 228, 3000, 333, 1384, 618, 201, 2138, 689, 398, 1715, 789, 125, 55, 44, 1214, 323, 459, 1729, 810, 1051, 1439, 1187, 977, 237, 558, 20, 1600, 1242, 526, 499, 510, 322, 339, 676, 603] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[687, 2464, 829, 242, 77, 261, 2422, 952, 4, 0, 728, 817, 2750, 0, 0, 967, 551, 835, 3628, 39, 51, 1665, 331, 0, 143, 2034, 946, 138, 68, 191, 3033, 401, 1762, 355, 68, 1647, 613, 297, 1063, 995, 45, 0, 6, 1177, 88, 707, 1118, 557, 824, 1308, 1891, 1265, 70, 480, 0, 1301, 1603, 353, 426, 305, 220, 210, 601, 543] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[543, 1952, 746, 295, 83, 234, 2419, 657, 5, 0, 706, 669, 2002, 0, 0, 812, 502, 828, 3459, 28, 83, 1620, 232, 0, 182, 2082, 916, 125, 63, 162, 3894, 223, 1550, 374, 92, 1420, 906, 285, 1047, 1165, 30, 0, 17, 1040, 130, 992, 1476, 800, 864, 1308, 1582, 1677, 147, 467, 0, 1423, 1626, 450, 399, 356, 285, 245, 706, 771] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[488, 1919, 1032, 285, 127, 241, 2366, 269, 43, 0, 592, 883, 1736, 0, 1, 1199, 952, 885, 3291, 67, 304, 884, 207, 0, 122, 1792, 760, 115, 177, 178, 3981, 303, 1356, 435, 128, 1611, 876, 308, 1654, 1103, 210, 0, 77, 1356, 123, 588, 1482, 547, 928, 1510, 1183, 961, 163, 700, 0, 1250, 1587, 451, 463, 546, 422, 477, 730, 728] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[411, 1966, 770, 425, 60, 285, 2074, 748, 1, 1, 746, 633, 2714, 0, 2, 676, 516, 797, 3522, 24, 68, 1929, 333, 3, 127, 1614, 852, 131, 74, 155, 3556, 198, 2044, 368, 46, 1782, 799, 325, 1158, 975, 21, 0, 11, 808, 85, 659, 1464, 775, 990, 1170, 1771, 1660, 196, 531, 2, 1335, 1858, 473, 427, 350, 263, 214, 568, 613] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[555, 1907, 1003, 231, 125, 311, 1341, 249, 0, 0, 0, 0, 0, 0, 0, 0, 59, 2, 542, 965, 1702, 0, 8, 913, 983, 289, 202, 0, 0, 0, 0, 0, 772, 838, 2697, 55, 213, 882, 338, 6, 0, 0, 0, 0, 0, 0, 0, 0, 130, 1623, 728, 135, 140, 217, 2640, 219, 0, 0, 0, 0, 0, 0, 0, 0, 1514, 480, 159, 594, 813, 332, 1790, 1125, 0, 0, 0, 0, 0, 0, 0, 0, 109, 2, 34, 1600, 97, 566, 1528, 626, 1137, 657, 0, 0, 0, 0, 0, 0, 965, 1779, 415, 939, 168, 580, 1, 1360, 0, 0, 0, 0, 0, 0, 0, 0, 1546, 505, 483, 631, 384, 407, 855, 784, 606, 561, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5722, 5665, 5801, 5832, 6807, 6356, 6207, 6762] avg=6144.0000 max=6807 max_rank=4 min=5665 min_rank=1 imbalance_ratio=1.107910 max_load_reduction=42345 max_load_reduction_pct=0.861511 moved_to_echo=26132 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[495, 2152, 795, 326, 64, 250, 1221, 743, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 802, 740, 2612, 0, 0, 725, 1006, 264, 308, 0, 0, 0, 0, 0, 507, 851, 2834, 20, 114, 1831, 296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 2124, 947, 111, 76, 139, 2699, 249, 0, 0, 0, 0, 0, 0, 0, 0, 1551, 432, 79, 515, 765, 279, 1149, 850, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 11, 1177, 92, 867, 1365, 656, 986, 1003, 0, 0, 0, 0, 0, 0, 867, 1382, 633, 1347, 142, 478, 0, 1288, 0, 0, 0, 0, 0, 0, 0, 0, 1639, 444, 409, 391, 245, 252, 656, 640, 636, 511, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6046, 6460, 6453, 6420, 5620, 6193, 6137, 5823] avg=6144.0000 max=6460 max_rank=1 min=5620 min_rank=4 imbalance_ratio=1.051432 max_load_reduction=42692 max_load_reduction_pct=0.868571 moved_to_echo=23773 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[627, 2068, 998, 285, 102, 303, 1373, 418, 0, 0, 0, 0, 0, 0, 0, 0, 38, 8, 679, 845, 1788, 5, 8, 1206, 1084, 296, 224, 0, 0, 0, 0, 0, 657, 953, 2692, 39, 173, 1132, 291, 10, 0, 0, 0, 0, 0, 0, 0, 0, 174, 2034, 860, 132, 133, 182, 2910, 238, 0, 0, 0, 0, 0, 0, 0, 0, 1254, 447, 109, 518, 875, 322, 1437, 876, 0, 0, 0, 0, 0, 0, 0, 0, 70, 19, 30, 1062, 97, 654, 1583, 637, 991, 729, 0, 0, 0, 0, 0, 0, 945, 1495, 460, 872, 161, 608, 17, 1363, 0, 0, 0, 0, 0, 0, 0, 0, 1519, 475, 466, 554, 438, 383, 761, 782, 604, 574, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6174, 6181, 5947, 6663, 5838, 5872, 5921, 6556] avg=6144.0000 max=6663 max_rank=3 min=5838 min_rank=4 imbalance_ratio=1.084473 max_load_reduction=42489 max_load_reduction_pct=0.864441 moved_to_echo=24187 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[734, 2098, 1106, 353, 226, 294, 1298, 381, 0, 0, 0, 0, 0, 0, 0, 0, 135, 23, 399, 956, 1848, 1, 27, 1307, 814, 280, 188, 0, 0, 0, 0, 0, 781, 929, 2454, 96, 211, 943, 417, 38, 0, 0, 0, 0, 0, 0, 0, 0, 242, 1339, 797, 122, 185, 228, 2186, 333, 0, 0, 0, 0, 0, 0, 0, 0, 1384, 618, 201, 733, 689, 398, 1715, 789, 0, 0, 0, 0, 0, 0, 0, 0, 125, 55, 44, 1214, 323, 459, 1729, 810, 1405, 612, 0, 0, 0, 0, 0, 0, 1051, 1439, 387, 977, 237, 558, 20, 1600, 0, 0, 0, 0, 0, 0, 0, 0, 1242, 526, 499, 510, 322, 339, 676, 603, 551, 543, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6490, 5978, 5869, 5432, 6527, 6776, 6269, 5811] avg=6144.0000 max=6776 max_rank=5 min=5432 min_rank=3 imbalance_ratio=1.102865 max_load_reduction=42376 max_load_reduction_pct=0.862142 moved_to_echo=25383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[687, 2464, 829, 242, 77, 261, 1482, 952, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 728, 817, 2750, 0, 0, 967, 823, 320, 299, 0, 0, 0, 0, 0, 551, 835, 2963, 39, 51, 1665, 331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 2034, 946, 138, 68, 191, 2210, 401, 0, 0, 0, 0, 0, 0, 0, 0, 1762, 355, 68, 565, 613, 297, 1063, 995, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 6, 1177, 88, 707, 1118, 557, 1082, 975, 0, 0, 0, 0, 0, 0, 824, 1308, 617, 1265, 70, 480, 0, 1301, 0, 0, 0, 0, 0, 0, 0, 0, 1603, 353, 426, 305, 220, 210, 601, 543, 665, 620, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6994, 6708, 6435, 6131, 5718, 5755, 5865, 5546] avg=6144.0000 max=6994 max_rank=0 min=5546 min_rank=7 imbalance_ratio=1.138346 max_load_reduction=42158 max_load_reduction_pct=0.857707 moved_to_echo=22884 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[488, 1919, 1032, 285, 127, 241, 1448, 269, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 592, 883, 1736, 0, 1, 1199, 1081, 312, 187, 0, 0, 0, 0, 0, 952, 885, 2688, 67, 304, 884, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 1792, 760, 115, 177, 178, 2900, 303, 0, 0, 0, 0, 0, 0, 0, 0, 1356, 435, 128, 553, 876, 308, 1654, 1103, 0, 0, 0, 0, 0, 0, 0, 0, 210, 0, 77, 1356, 123, 588, 1482, 547, 1058, 610, 0, 0, 0, 0, 0, 0, 928, 1510, 386, 961, 163, 700, 0, 1250, 0, 0, 0, 0, 0, 0, 0, 0, 1587, 451, 463, 546, 422, 477, 730, 728, 603, 606, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5809, 6034, 5987, 6347, 6413, 6051, 5898, 6613] avg=6144.0000 max=6613 max_rank=7 min=5809 min_rank=0 imbalance_ratio=1.076335 max_load_reduction=42539 max_load_reduction_pct=0.865458 moved_to_echo=24975 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[543, 1952, 746, 295, 83, 234, 1480, 657, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 706, 669, 2002, 0, 0, 812, 1057, 320, 250, 0, 0, 0, 0, 0, 502, 828, 2825, 28, 83, 1620, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, 2082, 916, 125, 63, 162, 2837, 223, 0, 0, 0, 0, 0, 0, 0, 0, 1550, 374, 92, 487, 906, 285, 1047, 1165, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 17, 1040, 130, 992, 1476, 800, 933, 816, 0, 0, 0, 0, 0, 0, 864, 1308, 516, 1677, 147, 467, 0, 1423, 0, 0, 0, 0, 0, 0, 0, 0, 1626, 450, 399, 356, 285, 245, 706, 771, 634, 619, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5990, 5821, 6118, 6590, 5906, 6234, 6402, 6091] avg=6144.0000 max=6590 max_rank=3 min=5821 min_rank=1 imbalance_ratio=1.072591 max_load_reduction=42562 max_load_reduction_pct=0.865926 moved_to_echo=24633 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[411, 1966, 770, 425, 60, 285, 1262, 748, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 746, 633, 2714, 0, 2, 676, 970, 278, 284, 0, 0, 0, 0, 0, 516, 797, 2872, 24, 68, 1929, 333, 3, 0, 0, 0, 0, 0, 0, 0, 0, 127, 1614, 852, 131, 74, 155, 2586, 198, 0, 0, 0, 0, 0, 0, 0, 0, 2044, 368, 46, 608, 799, 325, 1158, 975, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 11, 808, 85, 659, 1464, 775, 1174, 918, 0, 0, 0, 0, 0, 0, 990, 1170, 569, 1660, 196, 531, 2, 1335, 0, 0, 0, 0, 0, 0, 0, 0, 1858, 473, 427, 350, 263, 214, 568, 613, 650, 534, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5927, 6305, 6542, 5737, 6323, 5915, 6453, 5950] avg=6144.0000 max=6542 max_rank=2 min=5737 min_rank=3 imbalance_ratio=1.064779 max_load_reduction=42610 max_load_reduction_pct=0.866903 moved_to_echo=24641 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[7552, 11392, 4096, 9728, 1408, 4480, 128, 11008, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4608, 16640, 7296, 2560, 896, 2304, 11008, 4480, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[12416, 3584, 896, 4608, 6400, 2560, 11136, 7936, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[768, 128, 256, 9472, 1152, 5504, 11776, 5504, 8832, 6400, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[12672, 3712, 3584, 3712, 2688, 2560, 5632, 5504, 4992, 4608, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5248, 7040, 22144, 384, 1280, 11008, 2560, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[384, 128, 5248, 6528, 17152, 128, 128, 7808, 7936, 2432, 2048, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1280, 14720, 6912, 1024, 1024, 1536, 20992, 2176, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[631, 91, 0, 4220, 87, 0, 177, 78, 0, 1161, 473, 603, 882, 495, 404, 570, 0, 0, 712, 3, 0, 441, 375, 1292, 113, 61, 582, 0, 157, 233, 0, 0, 78, 1471, 101, 0, 0, 0, 0, 4222, 1229, 236, 177, 1842, 5724, 88, 985, 4306, 390, 307, 0, 1652, 317, 986, 6185, 0, 4540, 0, 0, 2, 0, 0, 322, 151] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[762, 113, 4, 3778, 93, 11, 203, 90, 1, 973, 643, 639, 870, 921, 431, 566, 0, 1, 778, 10, 13, 471, 428, 1982, 119, 65, 1040, 0, 310, 81, 1, 1, 96, 1551, 213, 0, 3, 19, 20, 2801, 1301, 251, 187, 1879, 5370, 86, 1187, 3505, 389, 464, 7, 1952, 358, 748, 5921, 18, 4918, 2, 16, 16, 12, 1, 318, 145] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[784, 130, 1, 3570, 69, 5, 276, 82, 0, 1006, 699, 699, 903, 956, 452, 763, 2, 3, 762, 18, 9, 417, 410, 2370, 135, 83, 1159, 5, 408, 110, 0, 3, 74, 1696, 257, 23, 1, 0, 0, 2078, 1230, 287, 220, 1916, 5369, 91, 1358, 3065, 417, 525, 0, 2127, 323, 649, 5892, 1, 4785, 1, 0, 19, 13, 2, 316, 128] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[571, 91, 0, 3861, 92, 0, 192, 76, 0, 1487, 600, 801, 876, 320, 375, 603, 0, 0, 833, 1, 0, 485, 415, 1504, 133, 73, 514, 0, 151, 95, 0, 0, 79, 1462, 94, 0, 0, 0, 0, 3917, 1380, 266, 193, 1846, 5684, 93, 1067, 4358, 351, 485, 0, 1181, 362, 1094, 6067, 0, 4406, 0, 0, 3, 0, 0, 465, 150] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[879, 171, 18, 3150, 149, 30, 260, 168, 40, 1208, 771, 798, 883, 788, 536, 610, 2, 23, 820, 68, 63, 604, 499, 1557, 106, 208, 1360, 6, 671, 210, 2, 53, 114, 1518, 288, 54, 11, 48, 22, 2045, 1422, 290, 231, 1995, 4658, 247, 1015, 3173, 394, 874, 75, 1853, 564, 842, 5422, 106, 4224, 38, 23, 108, 180, 0, 453, 154] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[653, 101, 0, 4151, 91, 0, 192, 83, 0, 1010, 473, 555, 835, 496, 386, 463, 0, 0, 719, 1, 1, 468, 342, 1367, 120, 68, 661, 0, 292, 226, 0, 0, 85, 1469, 136, 0, 0, 0, 0, 4007, 1280, 260, 191, 1859, 5524, 96, 834, 4363, 360, 163, 0, 2248, 383, 1066, 5964, 0, 4546, 0, 0, 2, 0, 0, 410, 152] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[694, 78, 0, 3583, 62, 0, 226, 52, 0, 850, 785, 781, 955, 940, 447, 665, 0, 0, 810, 2, 10, 424, 476, 2440, 117, 62, 1293, 0, 309, 264, 0, 0, 52, 1560, 214, 7, 0, 0, 0, 2434, 1202, 243, 183, 1947, 5439, 79, 1126, 3408, 404, 310, 0, 1898, 308, 573, 6142, 0, 4935, 0, 0, 7, 0, 0, 259, 97] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[696, 115, 0, 4026, 92, 0, 185, 100, 0, 1208, 443, 587, 866, 590, 378, 570, 0, 0, 783, 0, 1, 518, 408, 1044, 130, 54, 625, 0, 126, 91, 0, 1, 101, 1586, 64, 0, 0, 2, 1, 3531, 1163, 239, 184, 1991, 5614, 94, 1247, 4236, 337, 387, 0, 1637, 368, 1438, 6166, 1, 4502, 0, 0, 8, 4, 1, 448, 165] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[784, 130, 1, 3570, 69, 5, 276, 82, 907, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 699, 699, 903, 956, 452, 763, 703, 338, 0, 0, 0, 0, 0, 0, 2, 3, 762, 18, 9, 417, 410, 2370, 1856, 856, 0, 0, 0, 0, 0, 0, 135, 83, 1159, 5, 408, 110, 0, 3, 4513, 0, 0, 0, 0, 0, 0, 0, 74, 1696, 257, 23, 1, 0, 0, 2078, 1177, 0, 0, 0, 0, 0, 0, 0, 1230, 287, 220, 1916, 0, 91, 1358, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 417, 525, 0, 2127, 323, 649, 2156, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4785, 1, 0, 19, 13, 2, 316, 128, 776, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5824, 6519, 6703, 6416, 5306, 6146, 6198, 6040] avg=6144.0000 max=6703 max_rank=2 min=5306 min_rank=4 imbalance_ratio=1.090983 max_load_reduction=42449 max_load_reduction_pct=0.863627 moved_to_echo=23690 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[631, 91, 0, 4220, 87, 0, 177, 78, 1275, 0, 0, 0, 0, 0, 0, 0, 0, 1161, 473, 603, 882, 495, 404, 570, 738, 475, 0, 0, 0, 0, 0, 0, 0, 0, 712, 3, 0, 441, 375, 1292, 1949, 912, 0, 0, 0, 0, 0, 0, 113, 61, 582, 0, 157, 233, 0, 0, 4812, 0, 0, 0, 0, 0, 0, 0, 78, 1471, 101, 0, 0, 0, 0, 4222, 1236, 0, 0, 0, 0, 0, 0, 0, 1229, 236, 177, 1842, 0, 88, 985, 1466, 0, 0, 0, 0, 0, 0, 0, 0, 390, 307, 0, 1652, 317, 986, 2262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4540, 0, 0, 2, 0, 0, 322, 151, 1090, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6559, 5801, 5684, 5958, 7108, 6023, 5914, 6105] avg=6144.0000 max=7108 max_rank=4 min=5684 min_rank=2 imbalance_ratio=1.156901 max_load_reduction=42044 max_load_reduction_pct=0.855387 moved_to_echo=25150 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[762, 113, 4, 3778, 93, 11, 203, 90, 1038, 0, 0, 0, 0, 0, 0, 0, 1, 973, 643, 639, 870, 921, 431, 566, 706, 387, 0, 0, 0, 0, 0, 0, 0, 1, 778, 10, 13, 471, 428, 1982, 1865, 856, 0, 0, 0, 0, 0, 0, 119, 65, 1040, 0, 310, 81, 1, 1, 4514, 0, 0, 0, 0, 0, 0, 0, 96, 1551, 213, 0, 3, 19, 20, 2801, 1183, 0, 0, 0, 0, 0, 0, 0, 1301, 251, 187, 1879, 0, 86, 1187, 1193, 0, 0, 0, 0, 0, 0, 0, 0, 389, 464, 7, 1952, 358, 748, 2167, 18, 0, 0, 0, 0, 0, 0, 0, 0, 4918, 2, 16, 16, 12, 1, 318, 145, 887, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6092, 6137, 6404, 6131, 5886, 6084, 6103, 6315] avg=6144.0000 max=6404 max_rank=2 min=5886 min_rank=4 imbalance_ratio=1.042318 max_load_reduction=42748 max_load_reduction_pct=0.869710 moved_to_echo=24388 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[879, 171, 18, 3150, 149, 30, 260, 168, 939, 0, 0, 0, 0, 0, 0, 0, 40, 1208, 771, 798, 883, 788, 536, 610, 647, 350, 0, 0, 0, 0, 0, 0, 2, 23, 820, 68, 63, 604, 499, 1557, 1708, 742, 0, 0, 0, 0, 0, 0, 106, 208, 1360, 6, 671, 210, 2, 53, 3916, 0, 0, 0, 0, 0, 0, 0, 114, 1518, 288, 54, 11, 48, 22, 2045, 1083, 0, 0, 0, 0, 0, 0, 0, 1422, 290, 231, 1995, 0, 247, 1015, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 394, 874, 75, 1853, 564, 842, 1984, 106, 0, 0, 0, 0, 0, 0, 0, 0, 4224, 38, 23, 108, 180, 0, 453, 154, 803, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5764, 6631, 6086, 6532, 5183, 6281, 6692, 5983] avg=6144.0000 max=6692 max_rank=6 min=5183 min_rank=4 imbalance_ratio=1.089193 max_load_reduction=42460 max_load_reduction_pct=0.863851 moved_to_echo=24139 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[571, 91, 0, 3861, 92, 0, 192, 76, 1290, 0, 0, 0, 0, 0, 0, 0, 0, 1487, 600, 801, 876, 320, 375, 603, 724, 481, 0, 0, 0, 0, 0, 0, 0, 0, 833, 1, 0, 485, 415, 1504, 1911, 907, 0, 0, 0, 0, 0, 0, 133, 73, 514, 0, 151, 95, 0, 0, 4777, 0, 0, 0, 0, 0, 0, 0, 79, 1462, 94, 0, 0, 0, 0, 3917, 1212, 0, 0, 0, 0, 0, 0, 0, 1380, 266, 193, 1846, 0, 93, 1067, 1483, 0, 0, 0, 0, 0, 0, 0, 0, 351, 485, 0, 1181, 362, 1094, 2220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4406, 0, 0, 3, 0, 0, 465, 150, 1104, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6173, 6267, 6056, 5743, 6764, 6328, 5693, 6128] avg=6144.0000 max=6764 max_rank=4 min=5693 min_rank=6 imbalance_ratio=1.100911 max_load_reduction=42388 max_load_reduction_pct=0.862386 moved_to_echo=24913 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[694, 78, 0, 3583, 62, 0, 226, 52, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 850, 785, 781, 955, 940, 447, 665, 733, 376, 0, 0, 0, 0, 0, 0, 0, 0, 810, 2, 10, 424, 476, 2440, 1935, 868, 0, 0, 0, 0, 0, 0, 117, 62, 1293, 0, 309, 264, 0, 0, 4571, 0, 0, 0, 0, 0, 0, 0, 52, 1560, 214, 7, 0, 0, 0, 2434, 1227, 0, 0, 0, 0, 0, 0, 0, 1202, 243, 183, 1947, 0, 79, 1126, 1160, 0, 0, 0, 0, 0, 0, 0, 0, 404, 310, 0, 1898, 308, 573, 2247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4935, 0, 0, 7, 0, 0, 259, 97, 863, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5704, 6532, 6965, 6616, 5494, 5940, 5740, 6161] avg=6144.0000 max=6965 max_rank=2 min=5494 min_rank=4 imbalance_ratio=1.133626 max_load_reduction=42187 max_load_reduction_pct=0.858297 moved_to_echo=23335 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[653, 101, 0, 4151, 91, 0, 192, 83, 1292, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 473, 555, 835, 496, 386, 463, 712, 482, 0, 0, 0, 0, 0, 0, 0, 0, 719, 1, 1, 468, 342, 1367, 1879, 880, 0, 0, 0, 0, 0, 0, 120, 68, 661, 0, 292, 226, 0, 0, 4644, 0, 0, 0, 0, 0, 0, 0, 85, 1469, 136, 0, 0, 0, 0, 4007, 1192, 0, 0, 0, 0, 0, 0, 0, 1280, 260, 191, 1859, 0, 96, 834, 1484, 0, 0, 0, 0, 0, 0, 0, 0, 360, 163, 0, 2248, 383, 1066, 2181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4546, 0, 0, 2, 0, 0, 410, 152, 1105, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6563, 5412, 5657, 6011, 6889, 6004, 6401, 6215] avg=6144.0000 max=6889 max_rank=4 min=5412 min_rank=1 imbalance_ratio=1.121257 max_load_reduction=42263 max_load_reduction_pct=0.859843 moved_to_echo=25509 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[696, 115, 0, 4026, 92, 0, 185, 100, 1259, 0, 0, 0, 0, 0, 0, 0, 0, 1208, 443, 587, 866, 590, 378, 570, 739, 472, 0, 0, 0, 0, 0, 0, 0, 0, 783, 0, 1, 518, 408, 1044, 1947, 896, 0, 0, 0, 0, 0, 0, 130, 54, 625, 0, 126, 91, 0, 1, 4718, 0, 0, 0, 0, 0, 0, 0, 101, 1586, 64, 0, 0, 2, 1, 3531, 1237, 0, 0, 0, 0, 0, 0, 0, 1163, 239, 184, 1991, 0, 94, 1247, 1428, 0, 0, 0, 0, 0, 0, 0, 0, 337, 387, 0, 1637, 368, 1438, 2243, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4502, 0, 0, 8, 4, 1, 448, 165, 1077, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6473, 5853, 5597, 5745, 6522, 6346, 6411, 6205] avg=6144.0000 max=6522 max_rank=4 min=5597 min_rank=2 imbalance_ratio=1.061523 max_load_reduction=42630 max_load_reduction_pct=0.867310 moved_to_echo=25484 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 8960, 4992, 5504, 7168, 5632, 3456, 4864, 5760, 3456, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 6272, 128, 128, 3840, 3456, 13568, 15104, 7040, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[10240, 2176, 1664, 15360, 0, 896, 8832, 10368, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5760, 896, 128, 30464, 768, 128, 1792, 768, 9088, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[3072, 3584, 128, 14592, 3072, 7424, 17536, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[768, 12416, 1408, 128, 128, 128, 128, 25088, 9600, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[36864, 128, 128, 256, 256, 128, 3072, 1152, 7808, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1024, 768, 7296, 128, 2432, 1408, 128, 128, 36480, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[3021, 4953, 134, 199, 774, 84, 73, 158, 2, 509, 437, 697, 161, 395, 543, 811, 464, 1303, 68, 73, 407, 129, 698, 1434, 2532, 2650, 2081, 295, 92, 218, 319, 422, 81, 395, 792, 22, 103, 2727, 867, 192, 728, 9, 28, 101, 1871, 5266, 857, 16, 3502, 103, 765, 455, 526, 121, 347, 86, 309, 486, 78, 20, 21, 1365, 346, 431] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[3647, 7088, 0, 83, 503, 1, 68, 1, 0, 379, 92, 190, 74, 328, 564, 465, 164, 1769, 0, 0, 145, 90, 523, 1128, 4800, 2478, 2558, 213, 1, 11, 114, 381, 0, 392, 432, 0, 87, 2842, 747, 112, 623, 1, 0, 59, 1098, 6100, 151, 0, 4224, 0, 588, 545, 469, 80, 8, 0, 11, 393, 0, 0, 0, 1686, 351, 295] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[3370, 6729, 33, 112, 587, 15, 75, 30, 1, 408, 193, 297, 88, 349, 552, 853, 199, 1275, 23, 3, 210, 101, 1200, 1459, 3069, 2788, 2276, 245, 5, 16, 193, 307, 5, 393, 624, 3, 80, 2278, 677, 114, 660, 16, 1, 65, 1622, 6171, 558, 23, 4149, 31, 756, 483, 474, 82, 344, 0, 121, 410, 13, 4, 8, 1288, 308, 330] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[3070, 5940, 3, 166, 626, 41, 63, 62, 0, 456, 179, 352, 60, 324, 464, 929, 341, 1043, 1, 1, 246, 119, 1353, 1664, 2574, 3282, 2095, 281, 8, 61, 155, 375, 0, 298, 879, 1, 67, 2472, 628, 133, 654, 3, 1, 83, 2035, 6335, 603, 1, 4399, 2, 660, 482, 482, 77, 342, 2, 73, 436, 1, 0, 0, 1212, 174, 283] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[3585, 7041, 0, 84, 495, 0, 58, 3, 0, 450, 77, 213, 143, 301, 637, 387, 166, 1561, 0, 0, 201, 85, 319, 630, 4854, 2405, 1975, 217, 2, 5, 116, 420, 0, 309, 403, 0, 92, 3735, 776, 138, 722, 1, 0, 73, 1515, 6021, 330, 0, 4158, 0, 468, 517, 423, 2, 0, 0, 3, 290, 0, 0, 0, 2095, 299, 352] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[3364, 6395, 0, 197, 568, 38, 37, 47, 0, 389, 149, 232, 60, 306, 489, 904, 336, 966, 0, 0, 207, 68, 1839, 1404, 2147, 3408, 1791, 266, 0, 1, 162, 316, 0, 266, 1051, 0, 62, 2076, 562, 121, 693, 7, 0, 59, 2220, 6513, 554, 0, 4929, 0, 751, 439, 462, 66, 420, 0, 47, 417, 0, 0, 0, 982, 116, 253] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[3783, 7052, 4, 62, 552, 0, 77, 0, 0, 339, 148, 198, 120, 347, 691, 457, 190, 1614, 3, 0, 150, 107, 447, 1110, 4665, 2193, 2811, 214, 0, 0, 169, 291, 0, 571, 301, 0, 93, 2794, 825, 94, 687, 1, 0, 53, 889, 6134, 259, 0, 4080, 2, 534, 521, 427, 56, 12, 1, 40, 346, 2, 0, 0, 1804, 503, 329] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[3607, 7084, 0, 96, 526, 6, 69, 3, 0, 322, 126, 199, 146, 335, 571, 507, 174, 1549, 0, 0, 173, 95, 582, 1112, 4056, 2862, 2259, 236, 0, 13, 132, 371, 0, 506, 387, 1, 92, 2630, 737, 131, 696, 3, 0, 67, 1273, 6291, 227, 0, 4191, 0, 590, 563, 451, 87, 125, 0, 9, 356, 0, 0, 0, 1755, 377, 396] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3647, 2020, 0, 83, 503, 1, 68, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 379, 92, 190, 74, 328, 564, 465, 3898, 0, 0, 0, 0, 0, 0, 0, 164, 1769, 0, 0, 145, 90, 523, 1128, 1202, 1138, 0, 0, 0, 0, 0, 0, 435, 2478, 2558, 213, 1, 11, 114, 381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 392, 432, 0, 87, 2842, 747, 112, 1493, 0, 0, 0, 0, 0, 0, 0, 623, 1, 0, 59, 1098, 3369, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4224, 0, 588, 545, 469, 80, 8, 0, 36, 0, 0, 0, 0, 0, 0, 0, 11, 393, 0, 0, 0, 1686, 351, 295, 3227, 1170, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6323, 5990, 6159, 6191, 6105, 5301, 5950, 7133] avg=6144.0000 max=7133 max_rank=7 min=5301 min_rank=5 imbalance_ratio=1.160970 max_load_reduction=42019 max_load_reduction_pct=0.854879 moved_to_echo=24489 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3070, 1693, 3, 166, 626, 41, 63, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 456, 179, 352, 60, 324, 464, 929, 3267, 0, 0, 0, 0, 0, 0, 0, 341, 1043, 1, 1, 246, 119, 1353, 1664, 1248, 610, 0, 0, 0, 0, 0, 0, 234, 3282, 2095, 281, 8, 61, 155, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 298, 879, 1, 67, 2472, 628, 133, 1551, 0, 0, 0, 0, 0, 0, 0, 654, 3, 1, 83, 2035, 3498, 603, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4399, 2, 660, 482, 482, 77, 342, 2, 38, 0, 0, 0, 0, 0, 0, 0, 73, 436, 1, 0, 0, 1212, 174, 283, 1730, 980, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5724, 6031, 6626, 6491, 6029, 6878, 6484, 4889] avg=6144.0000 max=6878 max_rank=5 min=4889 min_rank=7 imbalance_ratio=1.119466 max_load_reduction=42274 max_load_reduction_pct=0.860067 moved_to_echo=24280 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3021, 1412, 134, 199, 774, 84, 73, 158, 0, 0, 0, 0, 0, 0, 0, 0, 2, 509, 437, 697, 161, 395, 543, 811, 2724, 0, 0, 0, 0, 0, 0, 0, 464, 1303, 68, 73, 407, 129, 698, 1434, 1038, 600, 0, 0, 0, 0, 0, 0, 230, 2650, 2081, 295, 92, 218, 319, 422, 0, 0, 0, 0, 0, 0, 0, 0, 81, 395, 792, 22, 103, 2727, 867, 192, 1289, 0, 0, 0, 0, 0, 0, 0, 728, 9, 28, 101, 1871, 2908, 857, 16, 0, 0, 0, 0, 0, 0, 0, 0, 3502, 103, 765, 455, 526, 121, 347, 86, 31, 0, 0, 0, 0, 0, 0, 0, 309, 486, 78, 20, 21, 1365, 346, 431, 1702, 817, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5855, 6279, 6214, 6307, 6468, 6518, 5936, 5575] avg=6144.0000 max=6518 max_rank=5 min=5575 min_rank=7 imbalance_ratio=1.060872 max_load_reduction=42634 max_load_reduction_pct=0.867391 moved_to_echo=24497 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3370, 1917, 33, 112, 587, 15, 75, 30, 0, 0, 0, 0, 0, 0, 0, 0, 1, 408, 193, 297, 88, 349, 552, 853, 3701, 0, 0, 0, 0, 0, 0, 0, 199, 1275, 23, 3, 210, 101, 1200, 1459, 1216, 727, 0, 0, 0, 0, 0, 0, 279, 2788, 2276, 245, 5, 16, 193, 307, 0, 0, 0, 0, 0, 0, 0, 0, 5, 393, 624, 3, 80, 2278, 677, 114, 1510, 0, 0, 0, 0, 0, 0, 0, 660, 16, 1, 65, 1622, 3408, 558, 23, 0, 0, 0, 0, 0, 0, 0, 0, 4149, 31, 756, 483, 474, 82, 344, 0, 37, 0, 0, 0, 0, 0, 0, 0, 121, 410, 13, 4, 8, 1288, 308, 330, 2063, 1111, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6139, 6442, 6413, 6109, 5684, 6353, 6356, 5656] avg=6144.0000 max=6442 max_rank=1 min=5656 min_rank=7 imbalance_ratio=1.048503 max_load_reduction=42710 max_load_reduction_pct=0.868937 moved_to_echo=24049 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3585, 2007, 0, 84, 495, 0, 58, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 450, 77, 213, 143, 301, 637, 387, 3872, 0, 0, 0, 0, 0, 0, 0, 166, 1561, 0, 0, 201, 85, 319, 630, 1186, 1151, 0, 0, 0, 0, 0, 0, 440, 2405, 1975, 217, 2, 5, 116, 420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 309, 403, 0, 92, 3735, 776, 138, 1474, 0, 0, 0, 0, 0, 0, 0, 722, 1, 0, 73, 1515, 3325, 330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4158, 0, 468, 517, 423, 2, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 3, 290, 0, 0, 0, 2095, 299, 352, 3263, 1162, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6232, 6080, 5299, 5580, 6927, 5966, 5604, 7464] avg=6144.0000 max=7464 max_rank=7 min=5299 min_rank=2 imbalance_ratio=1.214844 max_load_reduction=41688 max_load_reduction_pct=0.848145 moved_to_echo=25961 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3364, 1823, 0, 197, 568, 38, 37, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 389, 149, 232, 60, 306, 489, 904, 3517, 0, 0, 0, 0, 0, 0, 0, 336, 966, 0, 0, 207, 68, 1839, 1404, 1283, 509, 0, 0, 0, 0, 0, 0, 195, 3408, 1791, 266, 0, 1, 162, 316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 266, 1051, 0, 62, 2076, 562, 121, 1594, 0, 0, 0, 0, 0, 0, 0, 693, 7, 0, 59, 2220, 3597, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4929, 0, 751, 439, 462, 66, 420, 0, 39, 0, 0, 0, 0, 0, 0, 0, 47, 417, 0, 0, 0, 982, 116, 253, 1443, 1055, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6074, 6046, 6612, 6139, 5732, 7130, 7106, 4313] avg=6144.0000 max=7130 max_rank=5 min=4313 min_rank=7 imbalance_ratio=1.160482 max_load_reduction=42022 max_load_reduction_pct=0.854940 moved_to_echo=24281 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3607, 2019, 0, 96, 526, 6, 69, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 322, 126, 199, 146, 335, 571, 507, 3896, 0, 0, 0, 0, 0, 0, 0, 174, 1549, 0, 0, 173, 95, 582, 1112, 1240, 961, 0, 0, 0, 0, 0, 0, 368, 2862, 2259, 236, 0, 13, 132, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 387, 1, 92, 2630, 737, 131, 1540, 0, 0, 0, 0, 0, 0, 0, 696, 3, 0, 67, 1273, 3474, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4191, 0, 590, 563, 451, 87, 125, 0, 37, 0, 0, 0, 0, 0, 0, 0, 9, 356, 0, 0, 0, 1755, 377, 396, 2727, 1169, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6326, 6102, 5886, 6241, 6024, 5740, 6044, 6789] avg=6144.0000 max=6789 max_rank=7 min=5740 min_rank=5 imbalance_ratio=1.104980 max_load_reduction=42363 max_load_reduction_pct=0.861877 moved_to_echo=24597 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3783, 2001, 4, 62, 552, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 339, 148, 198, 120, 347, 691, 457, 3882, 0, 0, 0, 0, 0, 0, 0, 190, 1614, 3, 0, 150, 107, 447, 1110, 1213, 1109, 0, 0, 0, 0, 0, 0, 416, 2193, 2811, 214, 0, 0, 169, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 571, 301, 0, 93, 2794, 825, 94, 1505, 0, 0, 0, 0, 0, 0, 0, 687, 1, 0, 53, 889, 3377, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 2, 534, 521, 427, 56, 12, 1, 39, 0, 0, 0, 0, 0, 0, 0, 40, 346, 2, 0, 0, 1804, 503, 329, 3140, 1169, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6479, 6182, 5943, 6094, 6183, 5266, 5672, 7333] avg=6144.0000 max=7333 max_rank=7 min=5266 min_rank=5 imbalance_ratio=1.193522 max_load_reduction=41819 max_load_reduction_pct=0.850810 moved_to_echo=24454 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2048, 11136, 128, 128, 1792, 896, 7040, 9984, 9728, 6912, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5504, 128, 128, 640, 12544, 27008, 3584, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[27520, 14976, 256, 1024, 4736, 256, 640, 384, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[640, 3200, 128, 128, 128, 12288, 2560, 2688, 19328, 8704, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2688, 22144, 17920, 2048, 128, 384, 1408, 2944, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 3328, 1408, 2432, 896, 2688, 4608, 5376, 28800, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 3200, 4992, 128, 768, 21632, 5888, 1152, 12032, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[33664, 256, 5120, 4096, 3840, 640, 1664, 128, 384, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[576, 394, 3910, 4268, 951, 0, 5266, 2529, 0, 173, 0, 1029, 0, 2149, 0, 203, 258, 0, 1323, 23, 408, 83, 387, 132, 68, 86, 1150, 0, 0, 155, 0, 171, 5740, 1159, 0, 87, 260, 2558, 534, 0, 415, 3, 11, 0, 0, 0, 326, 137, 1689, 0, 246, 205, 155, 2104, 1026, 0, 1083, 469, 112, 2724, 2341, 0, 76, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[591, 529, 4297, 4500, 1350, 6, 5189, 3307, 6, 158, 25, 928, 29, 1761, 2, 274, 261, 1, 1545, 43, 712, 55, 397, 142, 66, 89, 1179, 4, 2, 150, 32, 191, 5429, 1122, 3, 81, 208, 1591, 544, 11, 424, 7, 55, 15, 0, 29, 357, 166, 1346, 4, 246, 360, 166, 2037, 741, 22, 1642, 501, 154, 2117, 1763, 0, 178, 12] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[589, 389, 3972, 4515, 537, 0, 5481, 2584, 0, 184, 0, 860, 0, 2063, 0, 141, 227, 0, 1285, 28, 506, 133, 353, 106, 58, 92, 1107, 0, 0, 150, 0, 178, 5642, 1377, 0, 98, 291, 2479, 572, 0, 394, 0, 16, 0, 0, 0, 339, 141, 1628, 0, 219, 187, 152, 1848, 747, 0, 766, 443, 293, 3234, 2620, 0, 128, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[576, 472, 4164, 4440, 1349, 0, 5462, 3724, 1, 134, 0, 812, 1, 1957, 0, 212, 251, 3, 1616, 70, 882, 73, 429, 143, 63, 67, 1290, 0, 4, 130, 1, 197, 5429, 1141, 0, 67, 196, 1095, 542, 1, 391, 0, 37, 0, 9, 1, 340, 129, 1191, 1, 247, 261, 183, 2590, 581, 0, 1745, 492, 91, 2355, 1360, 0, 154, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[548, 519, 4374, 4607, 1449, 0, 5318, 4020, 0, 124, 0, 801, 0, 1748, 0, 125, 255, 0, 1881, 36, 713, 77, 386, 102, 37, 62, 1393, 0, 0, 99, 0, 176, 5641, 1160, 0, 72, 200, 1077, 602, 0, 368, 0, 26, 0, 0, 0, 396, 119, 1582, 0, 260, 413, 133, 2099, 634, 0, 1630, 497, 101, 2064, 1133, 0, 95, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[565, 560, 4046, 4020, 582, 103, 4857, 2895, 35, 206, 24, 888, 147, 1938, 49, 431, 179, 48, 1201, 51, 633, 84, 614, 269, 68, 153, 1239, 25, 47, 194, 45, 196, 4830, 1264, 34, 123, 249, 1181, 549, 51, 454, 17, 254, 78, 111, 117, 431, 265, 891, 5, 386, 183, 930, 2337, 813, 80, 1280, 537, 127, 2694, 2009, 0, 433, 47] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[579, 465, 4017, 4025, 1127, 0, 4810, 2596, 0, 184, 0, 1126, 0, 2139, 0, 229, 252, 0, 1444, 31, 471, 85, 365, 143, 69, 92, 1140, 0, 0, 163, 0, 177, 5684, 1085, 0, 92, 273, 1909, 538, 0, 426, 3, 7, 0, 0, 0, 300, 241, 1652, 0, 252, 286, 125, 2340, 1180, 0, 1386, 454, 130, 2661, 2298, 0, 101, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[572, 426, 3826, 4143, 886, 1, 5528, 2112, 2, 182, 1, 1074, 2, 2242, 0, 170, 248, 0, 1268, 9, 485, 54, 337, 142, 77, 91, 994, 0, 0, 170, 0, 186, 5529, 1197, 1, 92, 256, 2442, 590, 0, 437, 0, 13, 3, 1, 2, 351, 200, 1504, 0, 247, 112, 177, 2405, 1065, 0, 1009, 429, 86, 2814, 2848, 1, 113, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[576, 394, 1044, 0, 951, 0, 0, 2529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 173, 0, 1029, 0, 2149, 0, 203, 2603, 61, 0, 0, 0, 0, 0, 0, 258, 0, 1323, 23, 408, 83, 387, 132, 3313, 7, 0, 0, 0, 0, 0, 0, 68, 86, 1150, 0, 0, 155, 0, 171, 4261, 140, 0, 0, 0, 0, 0, 0, 2366, 1159, 0, 87, 260, 2558, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 415, 3, 11, 0, 0, 0, 326, 137, 5126, 0, 0, 0, 0, 0, 0, 0, 1689, 0, 246, 205, 155, 2104, 1026, 0, 616, 263, 0, 0, 0, 0, 0, 0, 1083, 469, 112, 2108, 2341, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5494, 6218, 5934, 6031, 6964, 6018, 6304, 6189] avg=6144.0000 max=6964 max_rank=4 min=5494 min_rank=0 imbalance_ratio=1.133464 max_load_reduction=42188 max_load_reduction_pct=0.858317 moved_to_echo=25475 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[576, 472, 1111, 0, 1349, 0, 0, 3724, 0, 0, 0, 0, 0, 0, 0, 0, 1, 134, 0, 812, 1, 1957, 0, 212, 2772, 58, 0, 0, 0, 0, 0, 0, 251, 3, 1616, 70, 882, 73, 429, 143, 3133, 7, 0, 0, 0, 0, 0, 0, 63, 67, 1290, 0, 4, 130, 1, 197, 4433, 146, 0, 0, 0, 0, 0, 0, 2238, 1141, 0, 67, 196, 1095, 542, 1, 0, 0, 0, 0, 0, 0, 0, 0, 391, 0, 37, 0, 9, 1, 340, 129, 5316, 0, 0, 0, 0, 0, 0, 0, 1191, 1, 247, 261, 183, 2590, 581, 0, 533, 281, 0, 0, 0, 0, 0, 0, 1745, 492, 91, 1822, 1360, 0, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7232, 5947, 6607, 6331, 5280, 6223, 5868, 5664] avg=6144.0000 max=7232 max_rank=0 min=5280 min_rank=4 imbalance_ratio=1.177083 max_load_reduction=41920 max_load_reduction_pct=0.852865 moved_to_echo=23035 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[591, 529, 1146, 0, 1350, 6, 0, 3307, 0, 0, 0, 0, 0, 0, 0, 0, 6, 158, 25, 928, 29, 1761, 2, 274, 2861, 58, 0, 0, 0, 0, 0, 0, 261, 1, 1545, 43, 712, 55, 397, 142, 3133, 7, 0, 0, 0, 0, 0, 0, 66, 89, 1179, 4, 2, 150, 32, 191, 4493, 138, 0, 0, 0, 0, 0, 0, 2238, 1122, 3, 81, 208, 1591, 544, 11, 0, 0, 0, 0, 0, 0, 0, 0, 424, 7, 55, 15, 0, 29, 357, 166, 5051, 0, 0, 0, 0, 0, 0, 0, 1346, 4, 246, 360, 166, 2037, 741, 22, 479, 290, 0, 0, 0, 0, 0, 0, 1642, 501, 154, 1638, 1763, 0, 178, 12, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6929, 6102, 6296, 6344, 5798, 6104, 5691, 5888] avg=6144.0000 max=6929 max_rank=0 min=5691 min_rank=6 imbalance_ratio=1.127767 max_load_reduction=42223 max_load_reduction_pct=0.859029 moved_to_echo=23481 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[589, 389, 1060, 0, 537, 0, 0, 2584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 0, 860, 0, 2063, 0, 141, 2644, 60, 0, 0, 0, 0, 0, 0, 227, 0, 1285, 28, 506, 133, 353, 106, 3256, 8, 0, 0, 0, 0, 0, 0, 58, 92, 1107, 0, 0, 150, 0, 178, 4507, 147, 0, 0, 0, 0, 0, 0, 2326, 1377, 0, 98, 291, 2479, 572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 394, 0, 16, 0, 0, 0, 339, 141, 5334, 0, 0, 0, 0, 0, 0, 0, 1628, 0, 219, 187, 152, 1848, 747, 0, 732, 268, 0, 0, 0, 0, 0, 0, 766, 443, 293, 2502, 2620, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5159, 5952, 5902, 6239, 7143, 6224, 5781, 6752] avg=6144.0000 max=7143 max_rank=4 min=5159 min_rank=0 imbalance_ratio=1.162598 max_load_reduction=42009 max_load_reduction_pct=0.854675 moved_to_echo=25900 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[565, 560, 1079, 0, 582, 103, 0, 2895, 0, 0, 0, 0, 0, 0, 0, 0, 35, 206, 24, 888, 147, 1938, 49, 431, 2694, 51, 0, 0, 0, 0, 0, 0, 179, 48, 1201, 51, 633, 84, 614, 269, 2787, 7, 0, 0, 0, 0, 0, 0, 68, 153, 1239, 25, 47, 194, 45, 196, 4013, 130, 0, 0, 0, 0, 0, 0, 1992, 1264, 34, 123, 249, 1181, 549, 51, 0, 0, 0, 0, 0, 0, 0, 0, 454, 17, 254, 78, 111, 117, 431, 265, 4727, 0, 0, 0, 0, 0, 0, 0, 891, 5, 386, 183, 930, 2337, 813, 80, 609, 273, 0, 0, 0, 0, 0, 0, 1280, 537, 127, 2085, 2009, 0, 433, 47, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5784, 6463, 5873, 6110, 5443, 6454, 6507, 6518] avg=6144.0000 max=6518 max_rank=7 min=5443 min_rank=4 imbalance_ratio=1.060872 max_load_reduction=42634 max_load_reduction_pct=0.867391 moved_to_echo=24922 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[548, 519, 1167, 0, 1449, 0, 0, 4020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 801, 0, 1748, 0, 125, 2912, 60, 0, 0, 0, 0, 0, 0, 255, 0, 1881, 36, 713, 77, 386, 102, 3256, 9, 0, 0, 0, 0, 0, 0, 37, 62, 1393, 0, 0, 99, 0, 176, 4598, 143, 0, 0, 0, 0, 0, 0, 2325, 1160, 0, 72, 200, 1077, 602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 368, 0, 26, 0, 0, 0, 396, 119, 5175, 0, 0, 0, 0, 0, 0, 0, 1582, 0, 260, 413, 133, 2099, 634, 0, 467, 295, 0, 0, 0, 0, 0, 0, 1630, 497, 101, 1597, 1133, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7703, 5770, 6715, 6508, 5436, 6084, 5883, 5053] avg=6144.0000 max=7703 max_rank=0 min=5053 min_rank=7 imbalance_ratio=1.253743 max_load_reduction=41449 max_load_reduction_pct=0.843282 moved_to_echo=22456 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[579, 465, 1072, 0, 1127, 0, 0, 2596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 0, 1126, 0, 2139, 0, 229, 2674, 61, 0, 0, 0, 0, 0, 0, 252, 0, 1444, 31, 471, 85, 365, 143, 3280, 7, 0, 0, 0, 0, 0, 0, 69, 92, 1140, 0, 0, 163, 0, 177, 4018, 129, 0, 0, 0, 0, 0, 0, 2343, 1085, 0, 92, 273, 1909, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 426, 3, 7, 0, 0, 0, 300, 241, 4681, 0, 0, 0, 0, 0, 0, 0, 1652, 0, 252, 286, 125, 2340, 1180, 0, 602, 271, 0, 0, 0, 0, 0, 0, 1386, 454, 130, 2059, 2298, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5839, 6413, 6078, 5788, 6240, 5658, 6708, 6428] avg=6144.0000 max=6708 max_rank=6 min=5658 min_rank=5 imbalance_ratio=1.091797 max_load_reduction=42444 max_load_reduction_pct=0.863525 moved_to_echo=25034 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[572, 426, 1015, 0, 886, 1, 0, 2112, 0, 0, 0, 0, 0, 0, 0, 0, 2, 182, 1, 1074, 2, 2242, 0, 170, 2551, 63, 0, 0, 0, 0, 0, 0, 248, 0, 1268, 9, 485, 54, 337, 142, 3196, 8, 0, 0, 0, 0, 0, 0, 77, 91, 994, 0, 0, 170, 0, 186, 4135, 148, 0, 0, 0, 0, 0, 0, 2270, 1197, 1, 92, 256, 2442, 590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 437, 0, 13, 3, 1, 2, 351, 200, 5380, 0, 0, 0, 0, 0, 0, 0, 1504, 0, 247, 112, 177, 2405, 1065, 0, 640, 260, 0, 0, 0, 0, 0, 0, 1009, 429, 86, 2174, 2848, 1, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5012, 6287, 5747, 5801, 6848, 6387, 6410, 6660] avg=6144.0000 max=6848 max_rank=4 min=5012 min_rank=0 imbalance_ratio=1.114583 max_load_reduction=42304 max_load_reduction_pct=0.860677 moved_to_echo=26305 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[18176, 9600, 128, 768, 2048, 14336, 4480, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 1408, 128, 7552, 256, 16000, 128, 1792, 21760, 512, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[4608, 3840, 8704, 0, 8320, 128, 0, 23808, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[11520, 128, 2176, 2048, 2048, 17792, 6912, 128, 4736, 2304, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[10624, 3840, 1152, 16000, 16384, 128, 1280, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[2048, 128, 11648, 384, 4864, 768, 3328, 1280, 25472, 128, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[512, 768, 9600, 128, 128, 1280, 128, 1536, 34560, 1152, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3328, 128, 512, 128, 128, 256, 2944, 1408, 40832, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 7294, 1379, 12, 0, 68, 935, 3055, 0, 28, 0, 0, 0, 711, 2672, 155, 0, 834, 74, 251, 77, 0, 195, 688, 769, 337, 0, 112, 227, 90, 220, 0, 276, 7353, 87, 0, 3891, 0, 259, 0, 1864, 0, 0, 0, 1772, 251, 0, 0, 87, 0, 88, 815, 12, 440, 85, 0, 0, 386, 6412, 288, 248, 1999, 0, 2356] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 7032, 1213, 27, 0, 67, 1060, 3716, 0, 57, 15, 0, 1, 692, 2251, 146, 14, 699, 84, 256, 85, 29, 180, 720, 549, 346, 21, 164, 262, 93, 240, 7, 148, 7299, 81, 25, 2829, 10, 579, 1, 1728, 1, 41, 8, 1929, 272, 6, 41, 81, 26, 81, 1040, 15, 548, 82, 34, 2, 380, 6221, 112, 260, 2444, 5, 2797] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[33, 6234, 2333, 91, 4, 119, 1327, 2254, 6, 219, 52, 49, 48, 674, 2532, 185, 203, 686, 300, 185, 130, 66, 255, 785, 689, 421, 185, 239, 252, 107, 272, 73, 357, 6639, 153, 80, 2521, 159, 634, 9, 1023, 17, 133, 74, 1920, 273, 10, 100, 92, 109, 146, 906, 85, 584, 214, 207, 80, 400, 5788, 167, 297, 1915, 168, 2884] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 6922, 1314, 28, 0, 72, 964, 3844, 0, 4, 0, 0, 0, 646, 2413, 134, 39, 578, 60, 252, 71, 1, 168, 756, 586, 346, 56, 167, 253, 93, 135, 6, 84, 7273, 72, 1, 2705, 0, 717, 0, 1674, 0, 2, 0, 1946, 253, 0, 1, 66, 1, 71, 1028, 52, 525, 67, 1, 1, 414, 6341, 99, 258, 2483, 5, 3104] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 7022, 1186, 18, 0, 37, 1154, 4378, 0, 1, 0, 0, 0, 738, 2110, 100, 5, 458, 60, 253, 52, 0, 153, 814, 432, 317, 34, 159, 251, 58, 136, 3, 89, 7476, 66, 0, 1970, 0, 714, 0, 2006, 0, 0, 0, 2087, 240, 0, 0, 62, 0, 90, 998, 49, 494, 62, 0, 0, 400, 6358, 75, 215, 2930, 0, 2842] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 7272, 1423, 17, 0, 69, 968, 3447, 0, 20, 0, 0, 0, 670, 2574, 161, 0, 826, 146, 252, 83, 0, 209, 585, 594, 344, 0, 133, 232, 92, 301, 1, 286, 7368, 92, 0, 3675, 0, 312, 0, 1614, 0, 0, 0, 2105, 252, 0, 0, 92, 0, 110, 1055, 9, 453, 91, 0, 0, 358, 6263, 202, 250, 2028, 1, 2117] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 7294, 1540, 21, 0, 77, 1112, 2786, 0, 41, 1, 0, 0, 667, 2790, 170, 0, 859, 118, 245, 100, 4, 201, 580, 676, 337, 0, 73, 246, 107, 224, 0, 307, 7287, 93, 2, 3549, 0, 302, 1, 1779, 0, 4, 1, 1613, 247, 0, 3, 92, 2, 104, 1190, 3, 447, 90, 4, 0, 337, 6455, 224, 263, 1753, 1, 2730] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 7279, 1662, 7, 0, 59, 1022, 2500, 0, 44, 0, 0, 0, 688, 2409, 151, 0, 873, 143, 226, 76, 0, 201, 679, 1180, 319, 0, 136, 223, 85, 258, 5, 322, 7322, 93, 0, 3946, 0, 150, 0, 1402, 0, 0, 0, 2125, 227, 0, 0, 92, 0, 92, 966, 15, 433, 92, 0, 0, 350, 6406, 406, 263, 1303, 0, 2922] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 228, 1379, 12, 0, 68, 935, 3055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 711, 2672, 155, 2804, 0, 0, 0, 0, 0, 0, 0, 0, 834, 74, 251, 77, 0, 195, 688, 4077, 35, 0, 0, 0, 0, 0, 0, 769, 337, 0, 112, 227, 90, 220, 0, 4514, 0, 0, 0, 0, 0, 0, 0, 276, 2220, 87, 0, 3891, 0, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1864, 0, 0, 0, 1772, 251, 0, 0, 2294, 0, 0, 0, 0, 0, 0, 0, 87, 0, 88, 815, 12, 440, 85, 0, 2552, 1870, 0, 0, 0, 0, 0, 0, 0, 386, 465, 288, 248, 1999, 0, 2356, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5677, 6370, 6231, 6269, 6733, 6181, 5949, 5742] avg=6144.0000 max=6733 max_rank=4 min=5677 min_rank=0 imbalance_ratio=1.095866 max_load_reduction=42419 max_load_reduction_pct=0.863017 moved_to_echo=24605 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 217, 1314, 28, 0, 72, 964, 3844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 646, 2413, 134, 2773, 0, 0, 0, 0, 0, 0, 0, 39, 578, 60, 252, 71, 1, 168, 756, 4032, 34, 0, 0, 0, 0, 0, 0, 586, 346, 56, 167, 253, 93, 135, 6, 4284, 0, 0, 0, 0, 0, 0, 0, 84, 2196, 72, 1, 2705, 0, 717, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1674, 0, 2, 0, 1946, 253, 0, 1, 2270, 0, 0, 0, 0, 0, 0, 0, 66, 1, 71, 1028, 52, 525, 67, 1, 2421, 1850, 0, 0, 0, 0, 0, 0, 1, 414, 459, 99, 258, 2483, 5, 3104, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6439, 5970, 5991, 5926, 5775, 6146, 6082, 6823] avg=6144.0000 max=6823 max_rank=7 min=5775 min_rank=4 imbalance_ratio=1.110514 max_load_reduction=42329 max_load_reduction_pct=0.861186 moved_to_echo=24826 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[33, 195, 2333, 91, 4, 119, 1327, 2254, 0, 0, 0, 0, 0, 0, 0, 0, 6, 219, 52, 49, 48, 674, 2532, 185, 2532, 0, 0, 0, 0, 0, 0, 0, 203, 686, 300, 185, 130, 66, 255, 785, 3680, 31, 0, 0, 0, 0, 0, 0, 689, 421, 185, 239, 252, 107, 272, 73, 3858, 0, 0, 0, 0, 0, 0, 0, 357, 2004, 153, 80, 2521, 159, 634, 9, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 17, 133, 74, 1920, 273, 10, 100, 2072, 0, 0, 0, 0, 0, 0, 0, 92, 109, 146, 906, 85, 584, 214, 207, 2181, 1688, 0, 0, 0, 0, 0, 0, 80, 400, 420, 167, 297, 1915, 168, 2884, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6356, 6297, 6321, 6096, 5917, 5622, 6212, 6331] avg=6144.0000 max=6356 max_rank=0 min=5622 min_rank=5 imbalance_ratio=1.034505 max_load_reduction=42796 max_load_reduction_pct=0.870687 moved_to_echo=24082 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 220, 1213, 27, 0, 67, 1060, 3716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 15, 0, 1, 692, 2251, 146, 2783, 0, 0, 0, 0, 0, 0, 0, 14, 699, 84, 256, 85, 29, 180, 720, 3956, 34, 0, 0, 0, 0, 0, 0, 549, 346, 21, 164, 262, 93, 240, 7, 4352, 0, 0, 0, 0, 0, 0, 0, 148, 2204, 81, 25, 2829, 10, 579, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1728, 1, 41, 8, 1929, 272, 6, 41, 2278, 0, 0, 0, 0, 0, 0, 0, 81, 26, 81, 1040, 15, 548, 82, 34, 2460, 1815, 0, 0, 0, 0, 0, 0, 2, 380, 450, 112, 260, 2444, 5, 2797, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6303, 5945, 6057, 6034, 5877, 6304, 6182, 6450] avg=6144.0000 max=6450 max_rank=7 min=5877 min_rank=4 imbalance_ratio=1.049805 max_load_reduction=42702 max_load_reduction_pct=0.868774 moved_to_echo=24813 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 228, 1662, 7, 0, 59, 1022, 2500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 688, 2409, 151, 2792, 0, 0, 0, 0, 0, 0, 0, 0, 873, 143, 226, 76, 0, 201, 679, 4073, 34, 0, 0, 0, 0, 0, 0, 1180, 319, 0, 136, 223, 85, 258, 5, 4505, 0, 0, 0, 0, 0, 0, 0, 322, 2211, 93, 0, 3946, 0, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1402, 0, 0, 0, 2125, 227, 0, 0, 2285, 0, 0, 0, 0, 0, 0, 0, 92, 0, 92, 966, 15, 433, 92, 0, 2546, 1869, 0, 0, 0, 0, 0, 0, 0, 350, 464, 406, 263, 1303, 0, 2922, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5478, 6084, 6305, 6711, 6722, 6039, 6105, 5708] avg=6144.0000 max=6722 max_rank=4 min=5478 min_rank=0 imbalance_ratio=1.094076 max_load_reduction=42430 max_load_reduction_pct=0.863241 moved_to_echo=24574 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 220, 1186, 18, 0, 37, 1154, 4378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 738, 2110, 100, 2851, 0, 0, 0, 0, 0, 0, 0, 5, 458, 60, 253, 52, 0, 153, 814, 4043, 35, 0, 0, 0, 0, 0, 0, 432, 317, 34, 159, 251, 58, 136, 3, 4346, 0, 0, 0, 0, 0, 0, 0, 89, 2257, 66, 0, 1970, 0, 714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2006, 0, 0, 0, 2087, 240, 0, 0, 2333, 0, 0, 0, 0, 0, 0, 0, 62, 0, 90, 998, 49, 494, 62, 0, 2456, 1855, 0, 0, 0, 0, 0, 0, 0, 400, 460, 75, 215, 2930, 0, 2842, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6993, 5800, 5873, 5736, 5096, 6666, 6066, 6922] avg=6144.0000 max=6993 max_rank=0 min=5096 min_rank=4 imbalance_ratio=1.138184 max_load_reduction=42159 max_load_reduction_pct=0.857727 moved_to_echo=24750 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 227, 1423, 17, 0, 69, 968, 3447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 670, 2574, 161, 2810, 0, 0, 0, 0, 0, 0, 0, 0, 826, 146, 252, 83, 0, 209, 585, 3982, 35, 0, 0, 0, 0, 0, 0, 594, 344, 0, 133, 232, 92, 301, 1, 4501, 0, 0, 0, 0, 0, 0, 0, 286, 2224, 92, 0, 3675, 0, 312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1614, 0, 0, 0, 2105, 252, 0, 0, 2299, 0, 0, 0, 0, 0, 0, 0, 92, 0, 110, 1055, 9, 453, 91, 0, 2544, 1827, 0, 0, 0, 0, 0, 0, 0, 358, 454, 202, 250, 2028, 1, 2117, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6151, 6235, 6118, 6198, 6589, 6270, 6181, 5410] avg=6144.0000 max=6589 max_rank=4 min=5410 min_rank=7 imbalance_ratio=1.072428 max_load_reduction=42563 max_load_reduction_pct=0.865946 moved_to_echo=24450 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 219, 1540, 21, 0, 77, 1112, 2786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 1, 0, 0, 667, 2790, 170, 2782, 0, 0, 0, 0, 0, 0, 0, 0, 859, 118, 245, 100, 4, 201, 580, 4110, 39, 0, 0, 0, 0, 0, 0, 676, 337, 0, 73, 246, 107, 224, 0, 4519, 0, 0, 0, 0, 0, 0, 0, 307, 2189, 93, 2, 3549, 0, 302, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1779, 0, 4, 1, 1613, 247, 0, 3, 2277, 0, 0, 0, 0, 0, 0, 0, 92, 2, 104, 1190, 3, 447, 90, 4, 2556, 1887, 0, 0, 0, 0, 0, 0, 0, 337, 458, 224, 263, 1753, 1, 2730, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5755, 6451, 6256, 6182, 6443, 5924, 6375, 5766] avg=6144.0000 max=6451 max_rank=1 min=5755 min_rank=0 imbalance_ratio=1.049967 max_load_reduction=42701 max_load_reduction_pct=0.868754 moved_to_echo=24508 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[768, 256, 896, 8064, 256, 3968, 896, 256, 19840, 14720, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 3072, 3712, 1664, 2176, 16896, 256, 21760, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[1920, 17536, 768, 128, 25088, 256, 3712, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 512, 128, 128, 128, 5504, 19840, 1280, 22144, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5504, 2816, 384, 1280, 2048, 768, 1792, 128, 34944, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 1792, 12160, 256, 128, 640, 8576, 25984, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[13184, 128, 256, 128, 15616, 2048, 128, 256, 18176, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[384, 5888, 1024, 1920, 768, 128, 1664, 5632, 32000, 384, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 9, 0, 231, 13, 0, 5, 38, 411, 559, 1430, 23, 879, 346, 20, 79, 61, 10, 1, 1, 24, 82, 0, 6, 3, 2992, 0, 297, 255, 1780, 29, 29, 732, 2607, 0, 6, 0, 1449, 0, 1188, 3232, 807, 6478, 24, 565, 1126, 1233, 5, 0, 164, 41, 2632, 3, 29, 66, 13, 1217, 38, 4613, 5721, 2911, 2481, 0, 158] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[1, 0, 0, 223, 0, 0, 0, 0, 361, 553, 2171, 0, 825, 318, 0, 92, 73, 0, 0, 0, 0, 76, 0, 0, 1, 4126, 0, 292, 251, 1567, 2, 0, 585, 3495, 0, 0, 0, 1480, 0, 1113, 4561, 558, 5148, 0, 543, 1815, 1233, 0, 0, 246, 0, 2212, 0, 0, 73, 0, 1244, 0, 4284, 5777, 1832, 1839, 0, 182] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[96, 134, 6, 306, 210, 28, 80, 121, 427, 525, 1965, 153, 941, 438, 138, 122, 81, 74, 10, 17, 188, 116, 0, 179, 66, 3111, 22, 381, 396, 1410, 122, 182, 816, 2485, 1, 75, 10, 1529, 95, 1276, 3436, 1220, 5480, 162, 615, 958, 1466, 106, 3, 291, 160, 1789, 59, 98, 139, 47, 1490, 93, 3519, 4501, 2468, 2544, 5, 171] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 207, 0, 0, 0, 0, 397, 703, 1525, 0, 802, 336, 0, 86, 59, 0, 0, 0, 0, 78, 0, 0, 1, 3481, 0, 291, 245, 1767, 1, 0, 521, 3174, 0, 0, 0, 1455, 0, 1045, 4203, 543, 6022, 0, 563, 1792, 1109, 0, 0, 130, 0, 2230, 0, 0, 59, 0, 1278, 0, 5047, 5814, 2107, 1909, 0, 172] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[1, 0, 0, 346, 0, 0, 3, 1, 390, 413, 1327, 1, 788, 320, 1, 67, 68, 0, 13, 0, 0, 71, 0, 0, 5, 3058, 0, 238, 281, 1547, 1, 0, 745, 2290, 0, 0, 0, 1741, 1, 1222, 2651, 784, 6594, 0, 633, 731, 1052, 0, 0, 132, 1, 2703, 0, 0, 72, 0, 1460, 1, 4559, 5976, 3264, 3466, 0, 134] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 328, 0, 0, 0, 0, 347, 493, 1544, 0, 918, 316, 0, 62, 59, 0, 0, 0, 0, 52, 0, 0, 2, 3038, 0, 152, 212, 1643, 0, 0, 877, 2081, 0, 0, 0, 1481, 0, 1177, 2147, 765, 6823, 0, 584, 722, 970, 0, 0, 149, 0, 3251, 0, 0, 59, 0, 1124, 0, 4638, 6022, 3956, 3036, 0, 124] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 1, 211, 0, 0, 0, 4, 424, 653, 1886, 2, 806, 337, 0, 90, 53, 0, 0, 0, 4, 100, 0, 3, 2, 3741, 0, 345, 261, 1856, 1, 3, 345, 3526, 0, 0, 0, 1699, 0, 1118, 4139, 662, 5517, 0, 520, 1903, 1498, 6, 0, 142, 4, 2073, 0, 1, 53, 0, 1336, 3, 4365, 5776, 1903, 1598, 0, 182] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 220, 0, 0, 0, 0, 404, 618, 1530, 0, 794, 344, 0, 92, 67, 0, 0, 0, 0, 83, 0, 0, 10, 3464, 0, 306, 261, 2181, 1, 0, 465, 3140, 0, 0, 0, 1482, 0, 1104, 3863, 627, 5860, 0, 551, 1457, 1061, 0, 0, 220, 0, 2029, 0, 0, 67, 0, 1202, 0, 5275, 5823, 2314, 2054, 0, 183] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 207, 0, 0, 0, 0, 5814, 60, 61, 0, 0, 0, 0, 0, 397, 703, 1525, 0, 802, 336, 0, 86, 1401, 1132, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 78, 0, 0, 5962, 0, 0, 0, 0, 0, 0, 0, 1, 3481, 0, 291, 245, 1767, 1, 0, 236, 94, 66, 0, 0, 0, 0, 0, 521, 3108, 0, 0, 0, 1455, 0, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 2708, 543, 0, 0, 563, 1792, 1109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 0, 2230, 0, 0, 59, 0, 3854, 0, 0, 0, 0, 0, 0, 0, 1278, 0, 0, 0, 1871, 1909, 0, 172, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6142, 6382, 6099, 6182, 6129, 6715, 6273, 5230] avg=6144.0000 max=6715 max_rank=5 min=5230 min_rank=7 imbalance_ratio=1.092936 max_load_reduction=42437 max_load_reduction_pct=0.863383 moved_to_echo=24347 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 346, 0, 0, 3, 1, 5976, 65, 55, 0, 0, 0, 0, 0, 390, 413, 1327, 1, 788, 320, 1, 67, 883, 1024, 0, 0, 0, 0, 0, 0, 68, 0, 13, 0, 0, 71, 0, 0, 6529, 0, 0, 0, 0, 0, 0, 0, 5, 3058, 0, 238, 281, 1547, 1, 0, 365, 59, 48, 0, 0, 0, 0, 0, 745, 2242, 0, 0, 0, 1741, 1, 1222, 0, 0, 0, 0, 0, 0, 0, 0, 1709, 784, 0, 0, 633, 731, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 1, 2703, 0, 0, 72, 0, 3480, 0, 0, 0, 0, 0, 0, 0, 1460, 1, 0, 0, 2899, 3466, 0, 134, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6447, 5214, 6681, 5602, 5951, 4909, 6388, 7960] avg=6144.0000 max=7960 max_rank=7 min=4909 min_rank=5 imbalance_ratio=1.295573 max_load_reduction=41192 max_load_reduction_pct=0.838053 moved_to_echo=25208 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 9, 0, 231, 13, 0, 5, 38, 5721, 64, 55, 0, 0, 0, 0, 0, 411, 559, 1430, 23, 879, 346, 20, 79, 1077, 1035, 0, 0, 0, 0, 0, 0, 61, 10, 1, 1, 24, 82, 0, 6, 6414, 0, 0, 0, 0, 0, 0, 0, 3, 2992, 0, 297, 255, 1780, 29, 29, 326, 72, 54, 0, 0, 0, 0, 0, 732, 2553, 0, 6, 0, 1449, 0, 1188, 0, 0, 0, 0, 0, 0, 0, 0, 2083, 807, 0, 24, 565, 1126, 1233, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 41, 2632, 3, 29, 66, 13, 3523, 0, 0, 0, 0, 0, 0, 0, 1217, 38, 0, 0, 2585, 2481, 0, 158, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6136, 5859, 6599, 5837, 5928, 5843, 6471, 6479] avg=6144.0000 max=6599 max_rank=2 min=5837 min_rank=3 imbalance_ratio=1.074056 max_load_reduction=42553 max_load_reduction_pct=0.865743 moved_to_echo=24721 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[96, 134, 6, 306, 210, 28, 80, 121, 4501, 55, 42, 0, 0, 0, 0, 0, 427, 525, 1965, 153, 941, 438, 138, 122, 1145, 791, 0, 0, 0, 0, 0, 0, 81, 74, 10, 17, 188, 116, 0, 179, 5425, 0, 0, 0, 0, 0, 0, 0, 66, 3111, 22, 381, 396, 1410, 122, 182, 276, 77, 52, 0, 0, 0, 0, 0, 816, 2433, 1, 75, 10, 1529, 95, 1276, 0, 0, 0, 0, 0, 0, 0, 0, 2214, 1220, 0, 162, 615, 958, 1466, 106, 0, 0, 0, 0, 0, 0, 0, 0, 3, 291, 160, 1789, 59, 98, 139, 47, 2686, 0, 0, 0, 0, 0, 0, 0, 1490, 93, 0, 0, 2192, 2544, 5, 171, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5579, 6645, 6090, 6095, 6235, 6741, 5272, 6495] avg=6144.0000 max=6741 max_rank=5 min=5272 min_rank=6 imbalance_ratio=1.097168 max_load_reduction=42411 max_load_reduction_pct=0.862854 moved_to_echo=24743 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 223, 0, 0, 0, 0, 5777, 52, 53, 0, 0, 0, 0, 0, 361, 553, 2171, 0, 825, 318, 0, 92, 1520, 961, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 76, 0, 0, 5096, 0, 0, 0, 0, 0, 0, 0, 1, 4126, 0, 292, 251, 1567, 2, 0, 205, 102, 73, 0, 0, 0, 0, 0, 585, 3422, 0, 0, 0, 1480, 0, 1113, 0, 0, 0, 0, 0, 0, 0, 0, 2939, 558, 0, 0, 543, 1815, 1233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 0, 2212, 0, 0, 73, 0, 3270, 0, 0, 0, 0, 0, 0, 0, 1244, 0, 0, 0, 1627, 1839, 0, 182, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6106, 6801, 5245, 6619, 6600, 7088, 5801, 4892] avg=6144.0000 max=7088 max_rank=5 min=4892 min_rank=7 imbalance_ratio=1.153646 max_load_reduction=42064 max_load_reduction_pct=0.855794 moved_to_echo=24381 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 328, 0, 0, 0, 0, 6022, 69, 57, 0, 0, 0, 0, 0, 347, 493, 1544, 0, 918, 316, 0, 62, 715, 1041, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 52, 0, 0, 6754, 0, 0, 0, 0, 0, 0, 0, 2, 3038, 0, 152, 212, 1643, 0, 0, 443, 48, 43, 0, 0, 0, 0, 0, 877, 2038, 0, 0, 0, 1481, 0, 1177, 0, 0, 0, 0, 0, 0, 0, 0, 1384, 765, 0, 0, 584, 722, 970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 0, 3251, 0, 0, 59, 0, 3540, 0, 0, 0, 0, 0, 0, 0, 1124, 0, 0, 0, 3513, 3036, 0, 124, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6476, 5436, 6865, 5581, 5573, 4425, 6999, 7797] avg=6144.0000 max=7797 max_rank=7 min=4425 min_rank=5 imbalance_ratio=1.269043 max_load_reduction=41355 max_load_reduction_pct=0.841370 moved_to_echo=24794 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 220, 0, 0, 0, 0, 5823, 58, 63, 0, 0, 0, 0, 0, 404, 618, 1530, 0, 794, 344, 0, 92, 1287, 1185, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 0, 83, 0, 0, 5802, 0, 0, 0, 0, 0, 0, 0, 10, 3464, 0, 306, 261, 2181, 1, 0, 259, 87, 65, 0, 0, 0, 0, 0, 465, 3075, 0, 0, 0, 1482, 0, 1104, 0, 0, 0, 0, 0, 0, 0, 0, 2489, 627, 0, 0, 551, 1457, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 220, 0, 2029, 0, 0, 67, 0, 4027, 0, 0, 0, 0, 0, 0, 0, 1202, 0, 0, 0, 2055, 2054, 0, 183, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6164, 6254, 5952, 6634, 6126, 6185, 6343, 5494] avg=6144.0000 max=6634 max_rank=3 min=5494 min_rank=7 imbalance_ratio=1.079753 max_load_reduction=42518 max_load_reduction_pct=0.865031 moved_to_echo=24148 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 211, 0, 0, 0, 4, 5776, 56, 54, 0, 0, 0, 0, 0, 424, 653, 1886, 2, 806, 337, 0, 90, 1384, 979, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 4, 100, 0, 3, 5461, 0, 0, 0, 0, 0, 0, 0, 2, 3741, 0, 345, 261, 1856, 1, 3, 217, 98, 78, 0, 0, 0, 0, 0, 345, 3448, 0, 0, 0, 1699, 0, 1118, 0, 0, 0, 0, 0, 0, 0, 0, 2657, 662, 0, 0, 520, 1903, 1498, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 4, 2073, 0, 1, 53, 0, 3332, 0, 0, 0, 0, 0, 0, 0, 1336, 3, 0, 0, 1686, 1598, 0, 182, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6102, 6561, 5621, 6602, 6610, 7246, 5605, 4805] avg=6144.0000 max=7246 max_rank=5 min=4805 min_rank=7 imbalance_ratio=1.179362 max_load_reduction=41906 max_load_reduction_pct=0.852580 moved_to_echo=24266 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3200, 4608, 13440, 256, 6784, 2816, 256, 768, 9472, 8192, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[10368, 256, 0, 0, 18432, 18944, 128, 1408, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[18304, 6016, 0, 256, 4608, 10624, 9728, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5120, 22400, 128, 128, 128, 12416, 128, 9344, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 27136, 128, 2304, 2176, 13824, 256, 256, 2432, 640, 512, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 1536, 256, 18944, 128, 128, 640, 128, 27776, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 256, 128, 2176, 256, 128, 128, 256, 45440, 512, 512, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[640, 128, 128, 128, 256, 768, 0, 256, 47488, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 68, 310, 329, 142, 2803, 0, 418, 0, 0, 25, 0, 87, 0, 0, 0, 1912, 0, 0, 0, 0, 2740, 0, 1357, 0, 0, 144, 705, 0, 0, 0, 68, 6473, 7, 87, 5864, 0, 0, 164, 770, 235, 0, 0, 2300, 78, 0, 0, 0, 320, 0, 2814, 86, 4540, 66, 6327, 0, 0, 1850, 2799, 904, 1379, 820, 0, 161] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[114, 69, 455, 429, 361, 2070, 153, 570, 4, 100, 26, 85, 194, 215, 8, 49, 1336, 6, 20, 0, 57, 1836, 37, 1295, 42, 238, 320, 524, 69, 49, 24, 78, 6054, 89, 91, 4185, 2, 145, 192, 992, 306, 129, 342, 1856, 105, 60, 246, 96, 431, 38, 3206, 91, 3742, 66, 5836, 96, 125, 1921, 3272, 977, 1817, 1431, 116, 264] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 69, 319, 335, 123, 2617, 0, 410, 0, 0, 9, 0, 92, 0, 0, 0, 1370, 0, 0, 0, 0, 2682, 0, 1289, 0, 0, 84, 554, 0, 0, 0, 69, 6440, 0, 92, 6026, 0, 0, 175, 767, 250, 0, 0, 2301, 83, 0, 0, 0, 391, 0, 2820, 91, 4191, 68, 6354, 0, 0, 1843, 3416, 985, 1524, 1073, 0, 240] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 63, 321, 324, 320, 3167, 0, 408, 0, 1, 11, 0, 67, 0, 0, 0, 1988, 0, 0, 0, 0, 2192, 0, 1063, 0, 42, 24, 514, 0, 0, 0, 63, 6604, 4, 66, 5671, 0, 1, 138, 1355, 208, 1, 0, 1983, 71, 1, 1, 2, 399, 1, 3093, 67, 3511, 63, 6313, 1, 53, 1640, 3044, 905, 2075, 1185, 1, 127] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[9, 69, 327, 351, 383, 3019, 28, 427, 0, 7, 12, 9, 85, 32, 5, 5, 1889, 2, 0, 0, 3, 2037, 11, 1002, 4, 27, 116, 502, 1, 0, 0, 74, 6572, 33, 80, 5443, 0, 30, 161, 908, 237, 38, 60, 1726, 82, 2, 44, 15, 394, 1, 2852, 79, 3975, 66, 6219, 18, 11, 1724, 3623, 930, 2019, 1196, 16, 162] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 37, 315, 306, 472, 3144, 0, 488, 0, 0, 7, 0, 62, 0, 0, 0, 1716, 0, 0, 0, 0, 2061, 0, 983, 0, 59, 10, 488, 0, 0, 0, 37, 6670, 0, 62, 5624, 0, 0, 114, 1202, 156, 0, 0, 1402, 52, 0, 0, 0, 334, 0, 2870, 62, 3951, 37, 6493, 0, 44, 1797, 3581, 876, 2460, 1058, 0, 122] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 77, 300, 348, 291, 2574, 3, 438, 0, 0, 15, 1, 92, 0, 0, 2, 1820, 0, 0, 0, 0, 2481, 0, 1662, 1, 0, 204, 675, 0, 0, 2, 79, 6387, 6, 92, 5466, 0, 2, 190, 622, 276, 4, 4, 2649, 100, 0, 4, 0, 388, 0, 3290, 90, 3909, 77, 6239, 2, 0, 1545, 3167, 839, 1536, 992, 1, 210] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 58, 300, 303, 326, 2523, 0, 432, 0, 0, 16, 0, 92, 0, 0, 0, 1522, 0, 0, 0, 0, 2609, 0, 1832, 0, 0, 176, 532, 0, 0, 0, 58, 6517, 2, 92, 5817, 0, 0, 168, 756, 237, 0, 0, 2022, 76, 0, 0, 0, 289, 0, 3041, 92, 4333, 58, 6320, 0, 0, 1845, 3269, 939, 1409, 856, 0, 235] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 68, 310, 329, 142, 2803, 0, 418, 1342, 591, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 87, 0, 0, 0, 5974, 0, 0, 0, 0, 0, 0, 0, 1912, 0, 0, 0, 0, 2740, 0, 1357, 611, 245, 0, 0, 0, 0, 0, 0, 0, 0, 144, 705, 0, 0, 0, 68, 4863, 499, 0, 0, 0, 0, 0, 0, 0, 7, 87, 5253, 0, 0, 164, 770, 0, 0, 0, 0, 0, 0, 0, 0, 235, 0, 0, 2300, 78, 0, 0, 0, 1895, 1464, 0, 0, 0, 0, 0, 0, 320, 0, 2814, 86, 2953, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1850, 313, 904, 1379, 820, 0, 161, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6003, 6086, 6865, 6279, 6281, 5972, 6239, 5427] avg=6144.0000 max=6865 max_rank=2 min=5427 min_rank=7 imbalance_ratio=1.117350 max_load_reduction=42287 max_load_reduction_pct=0.860331 moved_to_echo=23919 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[114, 69, 455, 429, 361, 2070, 153, 570, 1106, 691, 0, 0, 0, 0, 0, 0, 4, 100, 26, 85, 194, 215, 8, 49, 5589, 0, 0, 0, 0, 0, 0, 0, 1336, 6, 20, 0, 57, 1836, 37, 1295, 436, 202, 0, 0, 0, 0, 0, 0, 42, 238, 320, 524, 69, 49, 24, 78, 4487, 465, 0, 0, 0, 0, 0, 0, 0, 89, 91, 3749, 2, 145, 192, 992, 0, 0, 0, 0, 0, 0, 0, 0, 306, 129, 342, 1856, 105, 60, 246, 96, 2216, 1349, 0, 0, 0, 0, 0, 0, 431, 38, 3206, 91, 2434, 66, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 125, 1921, 365, 977, 1817, 1431, 116, 264, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6018, 6270, 5225, 6296, 5260, 6705, 6362, 7016] avg=6144.0000 max=7016 max_rank=7 min=5225 min_rank=2 imbalance_ratio=1.141927 max_load_reduction=42136 max_load_reduction_pct=0.857259 moved_to_echo=25343 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 63, 321, 324, 320, 3167, 0, 408, 1038, 643, 0, 0, 0, 0, 0, 0, 0, 1, 11, 0, 67, 0, 0, 0, 6095, 0, 0, 0, 0, 0, 0, 0, 1988, 0, 0, 0, 0, 2192, 0, 1063, 591, 190, 0, 0, 0, 0, 0, 0, 0, 42, 24, 514, 0, 0, 0, 63, 4853, 509, 0, 0, 0, 0, 0, 0, 0, 4, 66, 5080, 0, 1, 138, 1355, 0, 0, 0, 0, 0, 0, 0, 0, 208, 1, 0, 1983, 71, 1, 1, 2, 2061, 1460, 0, 0, 0, 0, 0, 0, 399, 1, 3093, 67, 2283, 63, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 53, 1640, 340, 905, 2075, 1185, 1, 127, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6284, 6174, 6024, 6005, 6644, 5788, 5907, 6326] avg=6144.0000 max=6644 max_rank=4 min=5788 min_rank=5 imbalance_ratio=1.081380 max_load_reduction=42508 max_load_reduction_pct=0.864827 moved_to_echo=24665 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[9, 69, 327, 351, 383, 3019, 28, 427, 1175, 765, 0, 0, 0, 0, 0, 0, 0, 7, 12, 9, 85, 32, 5, 5, 6066, 0, 0, 0, 0, 0, 0, 0, 1889, 2, 0, 0, 3, 2037, 11, 1002, 567, 215, 0, 0, 0, 0, 0, 0, 4, 27, 116, 502, 1, 0, 0, 74, 4780, 506, 0, 0, 0, 0, 0, 0, 0, 33, 80, 4876, 0, 30, 161, 908, 0, 0, 0, 0, 0, 0, 0, 0, 237, 38, 60, 1726, 82, 2, 44, 15, 2454, 1439, 0, 0, 0, 0, 0, 0, 394, 1, 2852, 79, 2585, 66, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1724, 404, 930, 2019, 1196, 16, 162, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6553, 6221, 5726, 6010, 6088, 6097, 5995, 6462] avg=6144.0000 max=6553 max_rank=0 min=5726 min_rank=2 imbalance_ratio=1.066569 max_load_reduction=42599 max_load_reduction_pct=0.866679 moved_to_echo=24642 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 37, 315, 306, 472, 3144, 0, 488, 1168, 756, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 62, 0, 0, 0, 6157, 0, 0, 0, 0, 0, 0, 0, 1716, 0, 0, 0, 0, 2061, 0, 983, 586, 213, 0, 0, 0, 0, 0, 0, 0, 59, 10, 488, 0, 0, 0, 37, 4992, 513, 0, 0, 0, 0, 0, 0, 0, 0, 62, 5038, 0, 0, 114, 1202, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 0, 1402, 52, 0, 0, 0, 2425, 1501, 0, 0, 0, 0, 0, 0, 334, 0, 2870, 62, 2570, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 1797, 400, 876, 2460, 1058, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6686, 6226, 5559, 6099, 6416, 5536, 5873, 6757] avg=6144.0000 max=6757 max_rank=7 min=5536 min_rank=5 imbalance_ratio=1.099772 max_load_reduction=42395 max_load_reduction_pct=0.862528 moved_to_echo=24582 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 58, 300, 303, 326, 2523, 0, 432, 1281, 690, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 92, 0, 0, 0, 6016, 0, 0, 0, 0, 0, 0, 0, 1522, 0, 0, 0, 0, 2609, 0, 1832, 606, 234, 0, 0, 0, 0, 0, 0, 0, 0, 176, 532, 0, 0, 0, 58, 4859, 501, 0, 0, 0, 0, 0, 0, 0, 2, 92, 5211, 0, 0, 168, 756, 0, 0, 0, 0, 0, 0, 0, 0, 237, 0, 0, 2022, 76, 0, 0, 0, 2214, 1461, 0, 0, 0, 0, 0, 0, 289, 0, 3041, 92, 2818, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1845, 365, 939, 1409, 856, 0, 235, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5913, 6124, 6803, 6126, 6229, 6010, 6298, 5649] avg=6144.0000 max=6803 max_rank=2 min=5649 min_rank=7 imbalance_ratio=1.107259 max_load_reduction=42349 max_load_reduction_pct=0.861593 moved_to_echo=24186 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 69, 319, 335, 123, 2617, 0, 410, 1239, 721, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 92, 0, 0, 0, 5944, 0, 0, 0, 0, 0, 0, 0, 1370, 0, 0, 0, 0, 2682, 0, 1289, 628, 226, 0, 0, 0, 0, 0, 0, 0, 0, 84, 554, 0, 0, 0, 69, 4885, 496, 0, 0, 0, 0, 0, 0, 0, 0, 92, 5398, 0, 0, 175, 767, 0, 0, 0, 0, 0, 0, 0, 0, 250, 0, 0, 2301, 83, 0, 0, 0, 2313, 1469, 0, 0, 0, 0, 0, 0, 391, 0, 2820, 91, 2726, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1843, 382, 985, 1524, 1073, 0, 240, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5833, 6045, 6195, 6088, 6432, 6416, 6096, 6047] avg=6144.0000 max=6432 max_rank=4 min=5833 min_rank=0 imbalance_ratio=1.046875 max_load_reduction=42720 max_load_reduction_pct=0.869141 moved_to_echo=24991 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 77, 300, 348, 291, 2574, 3, 438, 1158, 673, 0, 0, 0, 0, 0, 0, 0, 0, 15, 1, 92, 0, 0, 2, 5896, 0, 0, 0, 0, 0, 0, 0, 1820, 0, 0, 0, 0, 2481, 0, 1662, 576, 216, 0, 0, 0, 0, 0, 0, 1, 0, 204, 675, 0, 0, 2, 79, 4797, 491, 0, 0, 0, 0, 0, 0, 0, 6, 92, 4890, 0, 2, 190, 622, 0, 0, 0, 0, 0, 0, 0, 0, 276, 4, 4, 2649, 100, 0, 4, 0, 2149, 1442, 0, 0, 0, 0, 0, 0, 388, 0, 3290, 90, 2535, 77, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1545, 345, 839, 1536, 992, 1, 210, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5862, 6006, 6755, 6249, 5802, 6628, 6382, 5468] avg=6144.0000 max=6755 max_rank=2 min=5468 min_rank=7 imbalance_ratio=1.099447 max_load_reduction=42397 max_load_reduction_pct=0.862569 moved_to_echo=24280 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[256, 14208, 2944, 7424, 14336, 8704, 256, 1536, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3072, 128, 24064, 768, 20992, 512, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[13568, 128, 128, 0, 128, 18688, 128, 10496, 4608, 1792, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 512, 2688, 2816, 2432, 22016, 256, 3712, 9600, 5632, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1920, 256, 512, 16256, 768, 128, 384, 128, 17792, 11648, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 256, 768, 39552, 128, 256, 1408, 7424, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 384, 1152, 4608, 128, 128, 128, 640, 38528, 4096, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 896, 256, 128, 128, 47744, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[55, 0, 68, 0, 71, 0, 0, 0, 0, 1484, 0, 162, 87, 150, 373, 1686, 2, 7133, 0, 475, 0, 0, 59, 0, 6813, 0, 357, 251, 0, 514, 86, 6078, 6940, 6473, 0, 714, 0, 0, 0, 0, 87, 57, 800, 0, 0, 1289, 251, 0, 96, 0, 0, 3229, 0, 0, 0, 0, 0, 3253, 0, 0, 0, 0, 0, 59] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[32, 0, 58, 0, 71, 0, 0, 0, 0, 1257, 0, 246, 92, 127, 349, 1562, 0, 7009, 0, 506, 0, 0, 73, 0, 6651, 0, 504, 227, 0, 584, 92, 5986, 6947, 6361, 0, 834, 0, 0, 0, 0, 92, 73, 775, 0, 0, 1523, 227, 0, 127, 0, 0, 3564, 0, 0, 0, 0, 0, 3130, 0, 0, 0, 0, 0, 73] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 63, 10, 107, 0, 0, 0, 0, 2023, 0, 217, 67, 509, 415, 1470, 0, 7067, 1, 491, 0, 0, 68, 1, 6764, 1, 216, 255, 1, 550, 67, 5600, 6904, 5936, 1, 589, 1, 0, 0, 0, 71, 68, 1905, 0, 0, 1122, 253, 0, 357, 0, 0, 2389, 0, 0, 0, 0, 0, 3524, 0, 0, 0, 0, 0, 69] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[28, 0, 66, 11, 97, 1, 2, 0, 20, 2525, 0, 172, 101, 527, 380, 1399, 0, 6956, 20, 458, 0, 0, 87, 19, 6688, 47, 198, 261, 27, 675, 79, 5681, 6848, 5700, 25, 503, 24, 5, 0, 4, 96, 62, 1345, 34, 0, 1224, 269, 47, 384, 1, 0, 2942, 6, 0, 4, 3, 24, 2959, 1, 2, 6, 37, 0, 72] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[6, 0, 37, 0, 61, 0, 0, 0, 0, 2531, 0, 177, 62, 532, 400, 1628, 0, 7187, 0, 433, 0, 0, 59, 0, 6815, 0, 156, 254, 0, 583, 62, 5834, 6948, 6003, 0, 346, 0, 0, 0, 0, 62, 59, 1255, 0, 0, 1079, 254, 0, 486, 0, 0, 2374, 0, 0, 0, 0, 0, 3410, 0, 0, 0, 0, 0, 59] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[175, 35, 100, 114, 98, 63, 50, 0, 75, 2189, 6, 269, 192, 1245, 520, 1620, 25, 6333, 159, 576, 11, 11, 241, 197, 5698, 186, 290, 249, 65, 804, 110, 5244, 6359, 3759, 223, 857, 69, 53, 6, 125, 249, 101, 1954, 236, 37, 1386, 287, 196, 427, 62, 21, 2723, 51, 30, 29, 24, 116, 2233, 17, 48, 161, 217, 25, 121] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[58, 0, 78, 7, 89, 0, 0, 0, 0, 1497, 0, 213, 92, 174, 338, 1339, 2, 7087, 2, 430, 0, 0, 53, 2, 6613, 4, 399, 247, 2, 556, 90, 5991, 6909, 6226, 0, 786, 3, 0, 0, 0, 93, 52, 1066, 2, 0, 1404, 249, 1, 235, 0, 0, 3324, 0, 0, 1, 0, 1, 3379, 0, 0, 1, 3, 0, 54] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[117, 0, 69, 0, 92, 0, 0, 0, 0, 1945, 0, 239, 91, 128, 359, 1622, 0, 7009, 0, 452, 0, 0, 67, 0, 6596, 0, 290, 252, 0, 558, 92, 6100, 6966, 6259, 0, 557, 0, 0, 0, 0, 92, 67, 837, 0, 0, 1353, 252, 0, 227, 0, 0, 3035, 0, 0, 0, 0, 0, 3361, 0, 0, 0, 0, 0, 68] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 63, 10, 107, 0, 0, 0, 5939, 0, 0, 0, 0, 0, 0, 0, 0, 2023, 0, 217, 67, 509, 415, 1470, 915, 418, 174, 0, 0, 0, 0, 0, 0, 5597, 1, 491, 0, 0, 68, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 216, 255, 1, 550, 67, 4685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5518, 1, 589, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 68, 1905, 0, 0, 1122, 253, 0, 2210, 965, 0, 0, 0, 0, 0, 0, 357, 0, 0, 2389, 0, 0, 0, 0, 2956, 0, 0, 0, 0, 0, 0, 0, 0, 3524, 0, 0, 0, 0, 0, 69, 1598, 1296, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6119, 6208, 6158, 5775, 6109, 6594, 5702, 6487] avg=6144.0000 max=6594 max_rank=5 min=5702 min_rank=6 imbalance_ratio=1.073242 max_load_reduction=42558 max_load_reduction_pct=0.865845 moved_to_echo=24892 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[55, 0, 68, 0, 71, 0, 0, 0, 5970, 0, 0, 0, 0, 0, 0, 0, 0, 1484, 0, 162, 87, 150, 373, 1686, 993, 455, 175, 0, 0, 0, 0, 0, 2, 5650, 0, 475, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 357, 251, 0, 514, 86, 5085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6018, 0, 714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 57, 800, 0, 0, 1289, 251, 0, 2226, 970, 0, 0, 0, 0, 0, 0, 96, 0, 0, 3229, 0, 0, 0, 0, 2977, 0, 0, 0, 0, 0, 0, 0, 0, 3253, 0, 0, 0, 0, 0, 59, 1610, 1308, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6164, 5565, 6186, 6293, 6732, 5680, 6302, 6230] avg=6144.0000 max=6732 max_rank=4 min=5565 min_rank=1 imbalance_ratio=1.095703 max_load_reduction=42420 max_load_reduction_pct=0.863037 moved_to_echo=24944 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[28, 0, 66, 11, 97, 1, 2, 0, 5891, 0, 0, 0, 0, 0, 0, 0, 20, 2525, 0, 172, 101, 527, 380, 1399, 928, 401, 171, 0, 0, 0, 0, 0, 0, 5509, 20, 458, 0, 0, 87, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 198, 261, 27, 675, 79, 4753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5299, 25, 503, 24, 5, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 96, 62, 1345, 34, 0, 1224, 269, 47, 2185, 957, 0, 0, 0, 0, 0, 0, 384, 1, 0, 2942, 6, 0, 4, 3, 2922, 0, 0, 0, 0, 0, 0, 0, 24, 2959, 1, 2, 6, 37, 0, 72, 1581, 1276, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6096, 6624, 6093, 6040, 5860, 6219, 6262, 5958] avg=6144.0000 max=6624 max_rank=1 min=5860 min_rank=4 imbalance_ratio=1.078125 max_load_reduction=42528 max_load_reduction_pct=0.865234 moved_to_echo=24299 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[175, 35, 100, 114, 98, 63, 50, 0, 5471, 0, 0, 0, 0, 0, 0, 0, 75, 2189, 6, 269, 192, 1245, 520, 1620, 857, 264, 156, 0, 0, 0, 0, 0, 25, 5016, 159, 576, 11, 11, 241, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 290, 249, 65, 804, 110, 4387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3495, 223, 857, 69, 53, 6, 125, 0, 0, 0, 0, 0, 0, 0, 0, 249, 101, 1954, 236, 37, 1386, 287, 196, 1861, 888, 0, 0, 0, 0, 0, 0, 427, 62, 21, 2723, 51, 30, 29, 24, 2491, 0, 0, 0, 0, 0, 0, 0, 116, 2233, 17, 48, 161, 217, 25, 121, 1346, 1161, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6106, 7393, 6236, 6091, 4828, 7195, 5858, 5445] avg=6144.0000 max=7393 max_rank=1 min=4828 min_rank=4 imbalance_ratio=1.203288 max_load_reduction=41759 max_load_reduction_pct=0.849589 moved_to_echo=23326 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[32, 0, 58, 0, 71, 0, 0, 0, 5977, 0, 0, 0, 0, 0, 0, 0, 0, 1257, 0, 246, 92, 127, 349, 1562, 978, 447, 172, 0, 0, 0, 0, 0, 0, 5552, 0, 506, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 227, 0, 584, 92, 5008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5914, 0, 834, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 73, 775, 0, 0, 1523, 227, 0, 2174, 970, 0, 0, 0, 0, 0, 0, 127, 0, 0, 3564, 0, 0, 0, 0, 2906, 0, 0, 0, 0, 0, 0, 0, 0, 3130, 0, 0, 0, 0, 0, 73, 1571, 1285, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6138, 5230, 6131, 6415, 6748, 5834, 6597, 6059] avg=6144.0000 max=6748 max_rank=4 min=5230 min_rank=1 imbalance_ratio=1.098307 max_load_reduction=42404 max_load_reduction_pct=0.862712 moved_to_echo=25238 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 37, 0, 61, 0, 0, 0, 5978, 0, 0, 0, 0, 0, 0, 0, 0, 2531, 0, 177, 62, 532, 400, 1628, 953, 422, 177, 0, 0, 0, 0, 0, 0, 5692, 0, 433, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 254, 0, 583, 62, 4881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5581, 0, 346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 59, 1255, 0, 0, 1079, 254, 0, 2228, 970, 0, 0, 0, 0, 0, 0, 486, 0, 0, 2374, 0, 0, 0, 0, 2977, 0, 0, 0, 0, 0, 0, 0, 0, 3410, 0, 0, 0, 0, 0, 59, 1610, 1318, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6082, 6882, 6184, 5936, 5927, 5907, 5837, 6397] avg=6144.0000 max=6882 max_rank=1 min=5837 min_rank=6 imbalance_ratio=1.120117 max_load_reduction=42270 max_load_reduction_pct=0.859985 moved_to_echo=24068 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[117, 0, 69, 0, 92, 0, 0, 0, 5993, 0, 0, 0, 0, 0, 0, 0, 0, 1945, 0, 239, 91, 128, 359, 1622, 997, 440, 172, 0, 0, 0, 0, 0, 0, 5552, 0, 452, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 290, 252, 0, 558, 92, 5103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5819, 0, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 67, 837, 0, 0, 1353, 252, 0, 2155, 973, 0, 0, 0, 0, 0, 0, 227, 0, 0, 3035, 0, 0, 0, 0, 2883, 0, 0, 0, 0, 0, 0, 0, 0, 3361, 0, 0, 0, 0, 0, 68, 1558, 1285, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6271, 5993, 6071, 6295, 6376, 5729, 6145, 6272] avg=6144.0000 max=6376 max_rank=4 min=5729 min_rank=5 imbalance_ratio=1.037760 max_load_reduction=42776 max_load_reduction_pct=0.870280 moved_to_echo=24522 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[58, 0, 78, 7, 89, 0, 0, 0, 5944, 0, 0, 0, 0, 0, 0, 0, 0, 1497, 0, 213, 92, 174, 338, 1339, 982, 443, 179, 0, 0, 0, 0, 0, 2, 5604, 2, 430, 0, 0, 53, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 399, 247, 2, 556, 90, 5009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5783, 0, 786, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 52, 1066, 2, 0, 1404, 249, 1, 2162, 965, 0, 0, 0, 0, 0, 0, 235, 0, 0, 3324, 0, 0, 1, 0, 2889, 0, 0, 0, 0, 0, 0, 0, 1, 3379, 0, 0, 1, 3, 0, 54, 1562, 1304, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6176, 5257, 6093, 6307, 6572, 5994, 6449, 6304] avg=6144.0000 max=6572 max_rank=4 min=5257 min_rank=1 imbalance_ratio=1.069661 max_load_reduction=42580 max_load_reduction_pct=0.866292 moved_to_echo=25319 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 15488, 128, 1792, 896, 3456, 3200, 12416, 7680, 3328, 1408, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[256, 25344, 128, 128, 256, 384, 128, 640, 12544, 10240, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[2432, 128, 128, 23680, 128, 128, 128, 128, 23040, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[896, 640, 9984, 384, 128, 10496, 2048, 256, 17280, 7680, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[0, 256, 2432, 2048, 128, 4864, 768, 38912, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 43520, 256, 5248, 128, 128, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[512, 128, 640, 256, 768, 128, 128, 0, 47232, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 44288, 256, 3840, 128, 128, 768, 256, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 60, 2738, 0, 0, 1164, 0, 0, 2144, 2361, 0, 0, 1, 731, 0, 71, 325, 0, 1450, 0, 67, 0, 1, 6402, 6, 0, 309, 0, 0, 0, 1, 89, 326, 0, 1, 2303, 746, 5, 67, 0, 1, 5641, 0, 0, 0, 251, 1, 532, 1, 67, 0, 5928, 45, 68, 68, 7029, 0, 0, 0, 6614, 752, 786, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[49, 22, 171, 1658, 94, 57, 1468, 8, 2, 1515, 3509, 12, 182, 313, 680, 30, 99, 377, 89, 1232, 49, 77, 2, 40, 5552, 108, 1, 305, 33, 28, 77, 60, 535, 318, 260, 200, 2115, 388, 258, 95, 183, 106, 5086, 0, 106, 47, 323, 326, 563, 149, 116, 23, 6389, 259, 117, 162, 5920, 25, 96, 102, 5432, 781, 764, 9] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 6, 84, 1737, 3, 1, 1776, 1, 0, 1627, 2910, 3, 36, 62, 701, 2, 82, 358, 19, 1175, 12, 61, 0, 3, 5949, 3, 0, 355, 13, 3, 0, 33, 367, 341, 32, 5, 2543, 655, 46, 79, 20, 40, 5798, 0, 5, 3, 300, 58, 477, 42, 79, 1, 6223, 98, 86, 84, 6852, 1, 4, 0, 6393, 706, 799, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 143, 2122, 0, 0, 1925, 0, 0, 1003, 2955, 0, 0, 0, 636, 0, 76, 303, 0, 1078, 0, 73, 0, 0, 6057, 0, 0, 301, 0, 0, 0, 0, 239, 319, 0, 0, 2342, 618, 0, 92, 71, 0, 5875, 0, 0, 0, 113, 0, 486, 0, 92, 0, 6621, 92, 76, 92, 7099, 0, 0, 0, 6389, 710, 1154, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 1, 118, 1413, 0, 2, 2178, 0, 0, 1052, 3775, 0, 2, 4, 661, 0, 100, 347, 0, 960, 0, 51, 0, 0, 5498, 0, 0, 306, 0, 0, 0, 1, 515, 361, 1, 3, 2048, 729, 3, 90, 44, 2, 5876, 0, 0, 4, 232, 4, 496, 3, 90, 0, 6501, 91, 100, 94, 7176, 0, 4, 0, 6469, 784, 963, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 74, 1827, 0, 0, 2122, 0, 0, 998, 3031, 0, 0, 0, 706, 0, 78, 329, 0, 981, 1, 59, 0, 0, 5763, 0, 0, 263, 0, 0, 0, 0, 433, 338, 0, 0, 2437, 701, 0, 86, 0, 0, 5836, 0, 0, 0, 198, 0, 469, 0, 86, 0, 6639, 86, 78, 87, 7214, 0, 0, 0, 6541, 717, 974, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 60, 2960, 0, 0, 1062, 0, 0, 2145, 2354, 0, 0, 0, 684, 0, 52, 306, 0, 1391, 0, 58, 0, 0, 6331, 47, 0, 240, 0, 0, 0, 0, 113, 324, 0, 0, 2826, 479, 0, 62, 0, 0, 5815, 0, 0, 0, 288, 0, 414, 0, 62, 0, 5943, 0, 5, 62, 6991, 0, 0, 0, 6603, 762, 713, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 150, 1931, 0, 0, 2069, 0, 0, 1097, 2764, 0, 0, 0, 679, 0, 83, 335, 0, 1190, 0, 67, 0, 0, 5795, 0, 0, 399, 0, 0, 0, 0, 467, 344, 0, 0, 2229, 839, 2, 92, 33, 0, 5923, 0, 0, 0, 245, 0, 472, 0, 91, 0, 6433, 92, 83, 91, 7084, 0, 0, 0, 6463, 676, 934, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 84, 1737, 3, 1, 1776, 1, 1879, 182, 178, 0, 0, 0, 0, 0, 0, 1627, 2910, 3, 36, 62, 701, 2, 859, 99, 0, 0, 0, 0, 0, 0, 82, 358, 19, 1175, 12, 61, 0, 3, 4411, 0, 0, 0, 0, 0, 0, 0, 5813, 3, 0, 355, 13, 3, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 367, 341, 32, 5, 2543, 655, 46, 79, 2259, 0, 0, 0, 0, 0, 0, 0, 20, 40, 5798, 0, 5, 3, 300, 58, 37, 0, 0, 0, 0, 0, 0, 0, 477, 42, 79, 1, 5186, 98, 86, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 4514, 706, 799, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5847, 6299, 6121, 6220, 6327, 6261, 6053, 6024] avg=6144.0000 max=6327 max_rank=4 min=5847 min_rank=0 imbalance_ratio=1.029785 max_load_reduction=42825 max_load_reduction_pct=0.871277 moved_to_echo=24665 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 74, 1827, 0, 0, 2122, 0, 1923, 192, 190, 0, 0, 0, 0, 0, 0, 998, 3031, 0, 0, 0, 706, 0, 916, 96, 0, 0, 0, 0, 0, 0, 78, 329, 0, 981, 1, 59, 0, 0, 4644, 0, 0, 0, 0, 0, 0, 0, 5631, 0, 0, 263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 433, 338, 0, 0, 2437, 701, 0, 86, 2378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5836, 0, 0, 0, 198, 0, 36, 0, 0, 0, 0, 0, 0, 0, 469, 0, 86, 0, 5533, 86, 78, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4618, 717, 974, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6328, 5747, 6092, 5894, 6373, 6070, 6339, 6309] avg=6144.0000 max=6373 max_rank=4 min=5747 min_rank=1 imbalance_ratio=1.037272 max_load_reduction=42779 max_load_reduction_pct=0.870341 moved_to_echo=25091 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[49, 22, 171, 1658, 94, 57, 1468, 8, 1597, 157, 183, 0, 0, 0, 0, 0, 2, 1515, 3509, 12, 182, 313, 680, 30, 882, 92, 0, 0, 0, 0, 0, 0, 99, 377, 89, 1232, 49, 77, 2, 40, 3813, 0, 0, 0, 0, 0, 0, 0, 5426, 108, 1, 305, 33, 28, 77, 60, 0, 0, 0, 0, 0, 0, 0, 0, 535, 318, 260, 200, 2115, 388, 258, 95, 1950, 0, 0, 0, 0, 0, 0, 0, 183, 106, 5086, 0, 106, 47, 323, 326, 34, 0, 0, 0, 0, 0, 0, 0, 563, 149, 116, 23, 5324, 259, 117, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 96, 102, 3835, 781, 764, 9, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5464, 7217, 5778, 6038, 6119, 6211, 6713, 5612] avg=6144.0000 max=7217 max_rank=1 min=5464 min_rank=0 imbalance_ratio=1.174642 max_load_reduction=41935 max_load_reduction_pct=0.853170 moved_to_echo=24655 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 143, 2122, 0, 0, 1925, 0, 1878, 190, 190, 0, 0, 0, 0, 0, 0, 1003, 2955, 0, 0, 0, 636, 0, 914, 100, 0, 0, 0, 0, 0, 0, 76, 303, 0, 1078, 0, 73, 0, 0, 4570, 0, 0, 0, 0, 0, 0, 0, 5919, 0, 0, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 239, 319, 0, 0, 2342, 618, 0, 92, 2339, 0, 0, 0, 0, 0, 0, 0, 71, 0, 5875, 0, 0, 0, 113, 0, 38, 0, 0, 0, 0, 0, 0, 0, 486, 0, 92, 0, 5517, 92, 76, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4511, 710, 1154, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6448, 5608, 6100, 6220, 5949, 6097, 6355, 6375] avg=6144.0000 max=6448 max_rank=0 min=5608 min_rank=1 imbalance_ratio=1.049479 max_load_reduction=42704 max_load_reduction_pct=0.868815 moved_to_echo=24776 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 60, 2960, 0, 0, 1062, 0, 1941, 186, 170, 0, 0, 0, 0, 0, 0, 2145, 2354, 0, 0, 0, 684, 0, 820, 105, 0, 0, 0, 0, 0, 0, 52, 306, 0, 1391, 0, 58, 0, 0, 4502, 0, 0, 0, 0, 0, 0, 0, 6187, 47, 0, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 324, 0, 0, 2826, 479, 0, 62, 2303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5815, 0, 0, 0, 288, 0, 39, 0, 0, 0, 0, 0, 0, 0, 414, 0, 62, 0, 4953, 0, 5, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4662, 762, 713, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6379, 6108, 6309, 6474, 6107, 6142, 5496, 6137] avg=6144.0000 max=6474 max_rank=3 min=5496 min_rank=6 imbalance_ratio=1.053711 max_load_reduction=42678 max_load_reduction_pct=0.868286 moved_to_echo=23882 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 118, 1413, 0, 2, 2178, 0, 1906, 192, 191, 0, 0, 0, 0, 0, 0, 1052, 3775, 0, 2, 4, 661, 0, 899, 95, 0, 0, 0, 0, 0, 0, 100, 347, 0, 960, 0, 51, 0, 0, 4620, 0, 0, 0, 0, 0, 0, 0, 5365, 0, 0, 306, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 515, 361, 1, 3, 2048, 729, 3, 90, 2364, 0, 0, 0, 0, 0, 0, 0, 44, 2, 5876, 0, 0, 4, 232, 4, 38, 0, 0, 0, 0, 0, 0, 0, 496, 3, 90, 0, 5411, 91, 100, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4563, 784, 963, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6001, 6488, 6078, 5672, 6114, 6200, 6285, 6314] avg=6144.0000 max=6488 max_rank=1 min=5672 min_rank=3 imbalance_ratio=1.055990 max_load_reduction=42664 max_load_reduction_pct=0.868001 moved_to_echo=24913 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 60, 2738, 0, 0, 1164, 0, 1944, 187, 170, 0, 0, 0, 0, 0, 0, 2144, 2361, 0, 0, 1, 731, 0, 818, 106, 0, 0, 0, 0, 0, 0, 71, 325, 0, 1450, 0, 67, 0, 1, 4525, 0, 0, 0, 0, 0, 0, 0, 6256, 6, 0, 309, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 89, 326, 0, 1, 2303, 746, 5, 67, 2317, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5641, 0, 0, 0, 251, 1, 40, 0, 0, 0, 0, 0, 0, 0, 532, 1, 67, 0, 4940, 45, 68, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4670, 752, 786, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6263, 6161, 6439, 6572, 5854, 5934, 5721, 6208] avg=6144.0000 max=6572 max_rank=3 min=5721 min_rank=6 imbalance_ratio=1.069661 max_load_reduction=42580 max_load_reduction_pct=0.866292 moved_to_echo=23717 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 150, 1931, 0, 0, 2069, 0, 1900, 188, 184, 0, 0, 0, 0, 0, 0, 1097, 2764, 0, 0, 0, 679, 0, 888, 96, 0, 0, 0, 0, 0, 0, 83, 335, 0, 1190, 0, 67, 0, 0, 4560, 0, 0, 0, 0, 0, 0, 0, 5663, 0, 0, 399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 344, 0, 0, 2229, 839, 2, 92, 2336, 0, 0, 0, 0, 0, 0, 0, 33, 0, 5923, 0, 0, 0, 245, 0, 36, 0, 0, 0, 0, 0, 0, 0, 472, 0, 91, 0, 5361, 92, 83, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4563, 676, 934, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6422, 5524, 6235, 6062, 6309, 6237, 6190, 6173] avg=6144.0000 max=6422 max_rank=0 min=5524 min_rank=1 imbalance_ratio=1.045247 max_load_reduction=42730 max_load_reduction_pct=0.869344 moved_to_echo=24909 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 11648, 23680, 128, 256, 384, 5504, 128, 7040, 896, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2816, 2688, 384, 256, 18944, 5248, 384, 768, 18304, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 896, 16512, 128, 128, 13824, 128, 14976, 1536, 1536, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[768, 2688, 128, 9472, 128, 640, 128, 128, 35712, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 128, 128, 128, 35968, 5888, 7168, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[3968, 256, 768, 128, 42240, 768, 640, 768, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[46336, 256, 128, 2560, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[384, 256, 45952, 0, 128, 128, 2048, 512, 384, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[2267, 22, 384, 88, 732, 5710, 207, 60, 1929, 151, 410, 29, 25, 29, 43, 32, 0, 87, 251, 200, 91, 119, 166, 52, 128, 17, 24, 130, 67, 96, 13, 132, 121, 0, 5243, 21, 12, 182, 3050, 874, 23, 6306, 725, 6493, 88, 150, 4138, 126, 37, 164, 48, 96, 47, 51, 216, 3197, 750, 354, 24, 526, 684, 222, 345, 1148] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[3118, 0, 1, 72, 784, 5147, 0, 0, 1329, 71, 415, 0, 0, 0, 0, 0, 0, 67, 129, 1, 0, 67, 1, 0, 90, 0, 0, 1, 0, 67, 0, 50, 0, 0, 6789, 13, 0, 0, 2456, 858, 0, 7230, 1061, 7598, 63, 1, 4590, 65, 0, 0, 0, 67, 14, 0, 0, 4700, 291, 1, 0, 340, 629, 129, 254, 593] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[2643, 0, 0, 92, 714, 5886, 0, 0, 1455, 76, 503, 0, 0, 0, 0, 0, 0, 92, 134, 0, 0, 92, 0, 0, 87, 0, 0, 0, 2, 92, 0, 0, 0, 0, 7010, 2, 0, 0, 3126, 877, 0, 7663, 1294, 7678, 58, 0, 3757, 87, 0, 0, 0, 92, 0, 0, 0, 3280, 280, 0, 0, 340, 609, 216, 229, 686] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[2599, 0, 70, 79, 719, 5708, 15, 7, 1267, 106, 446, 0, 5, 0, 2, 1, 0, 79, 166, 48, 28, 79, 25, 14, 87, 0, 0, 29, 1, 79, 0, 23, 15, 0, 6475, 0, 0, 8, 2430, 821, 1, 7335, 1217, 7435, 66, 46, 4523, 79, 11, 9, 7, 79, 0, 9, 15, 4352, 317, 71, 2, 344, 620, 143, 282, 758] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[2292, 0, 0, 87, 728, 6016, 0, 0, 1136, 77, 430, 0, 0, 0, 0, 0, 0, 87, 146, 0, 0, 86, 0, 0, 95, 0, 0, 0, 1, 87, 0, 1, 0, 0, 6977, 0, 0, 0, 2628, 806, 0, 7736, 1323, 7645, 68, 0, 4248, 81, 0, 0, 0, 87, 0, 0, 0, 3950, 262, 0, 0, 331, 586, 133, 254, 768] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[2169, 0, 0, 92, 716, 5824, 0, 0, 1026, 83, 428, 0, 0, 0, 0, 0, 0, 92, 152, 0, 0, 92, 0, 0, 110, 0, 0, 0, 0, 92, 0, 1, 0, 0, 6849, 0, 0, 0, 2341, 853, 0, 7611, 1626, 7566, 69, 0, 4475, 83, 0, 0, 0, 92, 0, 0, 0, 4233, 285, 0, 0, 318, 593, 213, 254, 814] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[2735, 0, 4, 92, 683, 5981, 1, 1, 1337, 100, 428, 0, 1, 1, 0, 0, 0, 92, 177, 4, 0, 90, 2, 0, 84, 0, 0, 3, 0, 92, 0, 3, 0, 0, 6896, 11, 0, 1, 2200, 860, 0, 7544, 1514, 7514, 77, 3, 3788, 79, 1, 0, 0, 92, 10, 0, 1, 4049, 324, 4, 0, 348, 632, 171, 248, 874] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[3076, 0, 0, 63, 725, 5449, 0, 0, 1262, 51, 391, 0, 0, 0, 0, 0, 0, 62, 87, 0, 0, 62, 0, 0, 92, 0, 0, 0, 0, 62, 0, 22, 0, 0, 6634, 0, 0, 0, 2691, 873, 0, 7422, 954, 7572, 37, 0, 4771, 62, 0, 0, 0, 62, 18, 0, 0, 4337, 262, 0, 0, 422, 658, 119, 256, 598] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3118, 0, 1, 72, 784, 2367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1329, 71, 415, 0, 0, 0, 0, 0, 2639, 1616, 0, 0, 0, 0, 0, 0, 0, 67, 129, 1, 0, 67, 1, 0, 4131, 1571, 0, 0, 0, 0, 0, 0, 90, 0, 0, 1, 0, 67, 0, 50, 6027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2700, 13, 0, 0, 2456, 858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 460, 1061, 0, 63, 1, 4590, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 14, 0, 0, 4700, 1770, 0, 0, 0, 0, 0, 0, 0, 291, 1, 0, 340, 629, 129, 254, 593, 2473, 1010, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6342, 6070, 5967, 6235, 6027, 6240, 6551, 5720] avg=6144.0000 max=6551 max_rank=6 min=5720 min_rank=7 imbalance_ratio=1.066243 max_load_reduction=42601 max_load_reduction_pct=0.866720 moved_to_echo=24538 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2292, 0, 0, 87, 728, 2766, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1136, 77, 430, 0, 0, 0, 0, 0, 2824, 1661, 0, 0, 0, 0, 0, 0, 0, 87, 146, 0, 0, 86, 0, 0, 4420, 1581, 0, 0, 0, 0, 0, 0, 95, 0, 0, 0, 1, 87, 0, 1, 6064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2775, 0, 0, 0, 2628, 806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 1323, 0, 68, 0, 4248, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 0, 3950, 2069, 0, 0, 0, 0, 0, 0, 0, 262, 0, 0, 331, 586, 133, 254, 768, 2541, 1181, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5873, 6128, 6320, 6248, 6209, 6212, 6106, 6056] avg=6144.0000 max=6320 max_rank=2 min=5873 min_rank=0 imbalance_ratio=1.028646 max_load_reduction=42832 max_load_reduction_pct=0.871419 moved_to_echo=24583 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2267, 22, 384, 88, 732, 2625, 207, 60, 0, 0, 0, 0, 0, 0, 0, 0, 1929, 151, 410, 29, 25, 29, 43, 32, 2302, 1248, 0, 0, 0, 0, 0, 0, 0, 87, 251, 200, 91, 119, 166, 52, 3603, 1343, 0, 0, 0, 0, 0, 0, 128, 17, 24, 130, 67, 96, 13, 132, 5150, 0, 0, 0, 0, 0, 0, 0, 121, 0, 2086, 21, 12, 182, 3050, 874, 0, 0, 0, 0, 0, 0, 0, 0, 23, 401, 725, 0, 88, 150, 4138, 126, 0, 0, 0, 0, 0, 0, 0, 0, 37, 164, 48, 96, 47, 51, 216, 3197, 1964, 0, 0, 0, 0, 0, 0, 0, 750, 354, 24, 526, 684, 222, 345, 1148, 1909, 1121, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6385, 6198, 5912, 5757, 6346, 5651, 5820, 7083] avg=6144.0000 max=7083 max_rank=7 min=5651 min_rank=5 imbalance_ratio=1.152832 max_load_reduction=42069 max_load_reduction_pct=0.855896 moved_to_echo=24900 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2599, 0, 70, 79, 719, 2625, 15, 7, 0, 0, 0, 0, 0, 0, 0, 0, 1267, 106, 446, 0, 5, 0, 2, 1, 2678, 1541, 0, 0, 0, 0, 0, 0, 0, 79, 166, 48, 28, 79, 25, 14, 4191, 1538, 0, 0, 0, 0, 0, 0, 87, 0, 0, 29, 1, 79, 0, 23, 5897, 0, 0, 0, 0, 0, 0, 0, 15, 0, 2576, 0, 0, 8, 2430, 821, 0, 0, 0, 0, 0, 0, 0, 0, 1, 466, 1217, 0, 66, 46, 4523, 79, 0, 0, 0, 0, 0, 0, 0, 0, 11, 9, 7, 79, 0, 9, 15, 4352, 1963, 0, 0, 0, 0, 0, 0, 0, 317, 71, 2, 344, 620, 143, 282, 758, 2358, 1120, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6114, 6046, 6168, 6116, 5850, 6398, 6445, 6015] avg=6144.0000 max=6445 max_rank=6 min=5850 min_rank=4 imbalance_ratio=1.048991 max_load_reduction=42707 max_load_reduction_pct=0.868876 moved_to_echo=24708 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2643, 0, 0, 92, 714, 2707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1455, 76, 503, 0, 0, 0, 0, 0, 2798, 1669, 0, 0, 0, 0, 0, 0, 0, 92, 134, 0, 0, 92, 0, 0, 4378, 1589, 0, 0, 0, 0, 0, 0, 87, 0, 0, 0, 2, 92, 0, 0, 6089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2788, 2, 0, 0, 3126, 877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 1294, 0, 58, 0, 3757, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 3280, 2024, 0, 0, 0, 0, 0, 0, 0, 280, 0, 0, 340, 609, 216, 229, 686, 2553, 1155, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6156, 6501, 6285, 6270, 6793, 5683, 5396, 6068] avg=6144.0000 max=6793 max_rank=4 min=5396 min_rank=6 imbalance_ratio=1.105632 max_load_reduction=42359 max_load_reduction_pct=0.861796 moved_to_echo=23940 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3076, 0, 0, 63, 725, 2506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1262, 51, 391, 0, 0, 0, 0, 0, 2710, 1579, 0, 0, 0, 0, 0, 0, 0, 62, 87, 0, 0, 62, 0, 0, 4240, 1567, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 62, 0, 22, 6005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2639, 0, 0, 0, 2691, 873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 954, 0, 37, 0, 4771, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 18, 0, 0, 4337, 1874, 0, 0, 0, 0, 0, 0, 0, 262, 0, 0, 422, 658, 119, 256, 598, 2416, 1069, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6370, 5993, 6018, 6181, 6203, 6296, 6291, 5800] avg=6144.0000 max=6370 max_rank=0 min=5800 min_rank=7 imbalance_ratio=1.036784 max_load_reduction=42782 max_load_reduction_pct=0.870402 moved_to_echo=24590 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2169, 0, 0, 92, 716, 2678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 83, 428, 0, 0, 0, 0, 0, 2779, 1630, 0, 0, 0, 0, 0, 0, 0, 92, 152, 0, 0, 92, 0, 0, 4348, 1565, 0, 0, 0, 0, 0, 0, 110, 0, 0, 0, 0, 92, 0, 1, 6001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2724, 0, 0, 0, 2341, 853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 1626, 0, 69, 0, 4475, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 4233, 2003, 0, 0, 0, 0, 0, 0, 0, 285, 0, 0, 318, 593, 213, 254, 814, 2495, 1143, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5655, 5946, 6249, 6204, 5918, 6737, 6328, 6115] avg=6144.0000 max=6737 max_rank=5 min=5655 min_rank=0 imbalance_ratio=1.096517 max_load_reduction=42415 max_load_reduction_pct=0.862935 moved_to_echo=25098 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2735, 0, 4, 92, 683, 2741, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1337, 100, 428, 0, 1, 1, 0, 0, 2757, 1646, 0, 0, 0, 0, 0, 0, 0, 92, 177, 4, 0, 90, 2, 0, 4313, 1555, 0, 0, 0, 0, 0, 0, 84, 0, 0, 3, 0, 92, 0, 3, 5959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2734, 11, 0, 1, 2200, 860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 1514, 0, 77, 3, 3788, 79, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 92, 10, 0, 1, 4049, 2062, 0, 0, 0, 0, 0, 0, 0, 324, 4, 0, 348, 632, 171, 248, 874, 2516, 1178, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6257, 6270, 6233, 6141, 5806, 5935, 6215, 6295] avg=6144.0000 max=6295 max_rank=7 min=5806 min_rank=4 imbalance_ratio=1.024577 max_load_reduction=42857 max_load_reduction_pct=0.871928 moved_to_echo=24251 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[2816, 512, 128, 3072, 5120, 1408, 2176, 6272, 19328, 9088, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[10752, 768, 3456, 128, 128, 128, 128, 128, 21504, 12672, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[256, 0, 21120, 128, 128, 256, 20992, 6912, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[20992, 128, 512, 768, 5888, 21120, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 256, 128, 768, 128, 128, 256, 32128, 15744, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 3840, 9728, 0, 640, 256, 34304, 768, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 768, 1280, 256, 128, 768, 256, 128, 33664, 12416, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[896, 128, 128, 256, 128, 768, 128, 256, 47232, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=17 tokens_per_expert_current_ep_rank=[4619, 1832, 0, 0, 0, 0, 2069, 86, 0, 2986, 0, 1423, 0, 0, 703, 0, 0, 173, 3567, 0, 0, 424, 0, 0, 0, 0, 3244, 638, 1750, 0, 1007, 448, 0, 0, 0, 0, 0, 0, 0, 3734, 0, 0, 297, 100, 4265, 153, 3898, 0, 6277, 27, 0, 168, 0, 86, 0, 4, 77, 0, 0, 0, 4168, 87, 0, 842] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=17 tokens_per_expert_current_ep_rank=[4145, 1677, 0, 0, 0, 0, 2303, 92, 0, 3007, 0, 1255, 0, 0, 704, 0, 0, 184, 3790, 0, 0, 442, 0, 0, 0, 0, 2509, 580, 1949, 0, 945, 539, 0, 0, 0, 0, 0, 0, 0, 3926, 0, 0, 386, 88, 4504, 164, 4488, 0, 6013, 44, 0, 174, 0, 92, 0, 0, 76, 0, 0, 0, 4011, 92, 0, 973] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=17 tokens_per_expert_current_ep_rank=[4852, 1451, 0, 0, 0, 0, 1387, 67, 0, 3474, 0, 1553, 54, 0, 851, 0, 0, 134, 3599, 4, 0, 479, 50, 1, 0, 0, 2635, 836, 1198, 1, 1083, 383, 1, 0, 0, 0, 1, 0, 0, 4585, 0, 1, 184, 98, 4214, 212, 4276, 1, 6700, 19, 0, 160, 0, 67, 0, 20, 71, 0, 0, 0, 3554, 67, 0, 829] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=17 tokens_per_expert_current_ep_rank=[4247, 2035, 17, 32, 55, 90, 1195, 146, 82, 3136, 127, 1294, 243, 102, 691, 13, 89, 188, 3047, 231, 63, 493, 121, 96, 69, 50, 2518, 827, 1004, 75, 1053, 560, 382, 242, 31, 39, 260, 25, 50, 3276, 41, 329, 323, 84, 4644, 350, 3978, 216, 5946, 102, 19, 249, 7, 102, 41, 109, 95, 51, 18, 319, 2762, 124, 53, 896] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=17 tokens_per_expert_current_ep_rank=[5186, 1607, 1, 0, 8, 4, 1539, 85, 6, 2979, 24, 1839, 95, 10, 773, 0, 14, 158, 3045, 17, 7, 452, 74, 10, 1, 5, 3080, 683, 1376, 42, 1394, 430, 68, 44, 7, 0, 58, 8, 0, 3707, 5, 56, 279, 72, 4005, 168, 4157, 28, 6636, 33, 2, 159, 0, 79, 0, 35, 82, 1, 0, 72, 3485, 79, 3, 880] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=17 tokens_per_expert_current_ep_rank=[5510, 1168, 0, 0, 0, 0, 1347, 60, 0, 2998, 0, 1857, 0, 0, 955, 0, 0, 124, 3466, 0, 0, 391, 42, 0, 0, 0, 3229, 743, 1189, 0, 1290, 330, 0, 0, 0, 0, 0, 0, 0, 4487, 0, 0, 216, 61, 4502, 172, 4242, 0, 6762, 32, 0, 128, 0, 62, 0, 1, 52, 0, 0, 0, 2820, 62, 0, 854] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=17 tokens_per_expert_current_ep_rank=[4761, 2268, 0, 0, 0, 0, 1990, 91, 0, 2943, 0, 1548, 0, 0, 697, 0, 0, 184, 3295, 0, 0, 433, 0, 0, 0, 0, 3590, 611, 1513, 0, 1031, 414, 0, 0, 0, 0, 0, 0, 0, 3677, 0, 0, 311, 73, 4384, 143, 3806, 0, 6417, 37, 0, 171, 0, 92, 0, 7, 83, 0, 0, 0, 3536, 91, 0, 955] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=17 tokens_per_expert_current_ep_rank=[4095, 2655, 0, 0, 0, 0, 2094, 90, 1, 2488, 1, 1406, 1, 0, 737, 0, 1, 182, 3490, 0, 0, 454, 0, 0, 0, 4, 3554, 583, 1145, 2, 1005, 440, 4, 2, 1, 0, 4, 0, 0, 3868, 0, 3, 477, 83, 4396, 132, 3627, 3, 6275, 3, 0, 153, 0, 90, 0, 1, 100, 0, 0, 4, 4463, 92, 0, 943] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=17 rerouted_tokens_per_expert_current_ep_rank=[2422, 1832, 0, 0, 0, 0, 2069, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2986, 0, 1423, 0, 0, 703, 0, 590, 204, 0, 0, 0, 0, 0, 0, 0, 173, 3567, 0, 0, 424, 0, 0, 1143, 840, 0, 0, 0, 0, 0, 0, 0, 0, 2654, 638, 1750, 0, 1007, 448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3734, 2035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 297, 100, 1390, 153, 3898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5731, 27, 0, 168, 0, 86, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 4168, 87, 0, 842, 1054, 342, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6409, 5906, 6147, 6497, 5769, 5838, 6016, 6570] avg=6144.0000 max=6570 max_rank=7 min=5769 min_rank=4 imbalance_ratio=1.069336 max_load_reduction=42582 max_load_reduction_pct=0.866333 moved_to_echo=24193 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=17 rerouted_tokens_per_expert_current_ep_rank=[2545, 1451, 0, 0, 0, 0, 1387, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3474, 0, 1553, 54, 0, 851, 0, 479, 218, 0, 0, 0, 0, 0, 0, 0, 134, 3599, 4, 0, 479, 50, 1, 1200, 830, 0, 0, 0, 0, 0, 0, 0, 0, 2156, 836, 1198, 1, 1083, 383, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 4585, 2011, 0, 0, 0, 0, 0, 0, 0, 0, 1, 184, 98, 1373, 212, 4276, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6117, 19, 0, 160, 0, 67, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 3554, 67, 0, 829, 1107, 365, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5450, 6629, 6297, 5657, 6598, 6145, 6383, 5993] avg=6144.0000 max=6629 max_rank=1 min=5450 min_rank=0 imbalance_ratio=1.078939 max_load_reduction=42523 max_load_reduction_pct=0.865133 moved_to_echo=25119 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=17 rerouted_tokens_per_expert_current_ep_rank=[2227, 2035, 17, 32, 55, 90, 1195, 146, 0, 0, 0, 0, 0, 0, 0, 0, 82, 3136, 127, 1294, 243, 102, 691, 13, 458, 193, 0, 0, 0, 0, 0, 0, 89, 188, 3047, 231, 63, 493, 121, 96, 1051, 914, 0, 0, 0, 0, 0, 0, 69, 50, 2060, 827, 1004, 75, 1053, 560, 0, 0, 0, 0, 0, 0, 0, 0, 382, 242, 31, 39, 260, 25, 50, 3276, 2216, 0, 0, 0, 0, 0, 0, 0, 41, 329, 323, 84, 1514, 350, 3978, 216, 0, 0, 0, 0, 0, 0, 0, 0, 5429, 102, 19, 249, 7, 102, 41, 109, 0, 0, 0, 0, 0, 0, 0, 0, 95, 51, 18, 319, 2762, 124, 53, 896, 969, 324, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5797, 6339, 6293, 5698, 6521, 6835, 6058, 5611] avg=6144.0000 max=6835 max_rank=5 min=5611 min_rank=7 imbalance_ratio=1.112467 max_load_reduction=42317 max_load_reduction_pct=0.860942 moved_to_echo=25025 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=17 rerouted_tokens_per_expert_current_ep_rank=[2719, 1607, 1, 0, 8, 4, 1539, 85, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2979, 24, 1839, 95, 10, 773, 0, 560, 216, 0, 0, 0, 0, 0, 0, 14, 158, 3045, 17, 7, 452, 74, 10, 1283, 788, 0, 0, 0, 0, 0, 0, 1, 5, 2520, 683, 1376, 42, 1394, 430, 0, 0, 0, 0, 0, 0, 0, 0, 68, 44, 7, 0, 58, 8, 0, 3707, 1911, 0, 0, 0, 0, 0, 0, 0, 5, 56, 279, 72, 1306, 168, 4157, 28, 0, 0, 0, 0, 0, 0, 0, 0, 6058, 33, 2, 159, 0, 79, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 82, 1, 0, 72, 3485, 79, 3, 880, 1184, 362, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5963, 6502, 5848, 6451, 5803, 6071, 6366, 6148] avg=6144.0000 max=6502 max_rank=1 min=5803 min_rank=4 imbalance_ratio=1.058268 max_load_reduction=42650 max_load_reduction_pct=0.867716 moved_to_echo=24388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=17 rerouted_tokens_per_expert_current_ep_rank=[2174, 1677, 0, 0, 0, 0, 2303, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3007, 0, 1255, 0, 0, 704, 0, 456, 195, 0, 0, 0, 0, 0, 0, 0, 184, 3790, 0, 0, 442, 0, 0, 1025, 887, 0, 0, 0, 0, 0, 0, 0, 0, 2053, 580, 1949, 0, 945, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3926, 2149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 386, 88, 1468, 164, 4488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5490, 44, 0, 174, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 4011, 92, 0, 973, 946, 328, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6246, 5617, 6328, 6066, 6075, 6594, 5800, 6426] avg=6144.0000 max=6594 max_rank=5 min=5617 min_rank=1 imbalance_ratio=1.073242 max_load_reduction=42558 max_load_reduction_pct=0.865845 moved_to_echo=24895 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=17 rerouted_tokens_per_expert_current_ep_rank=[2889, 1168, 0, 0, 0, 0, 1347, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2998, 0, 1857, 0, 0, 955, 0, 587, 220, 0, 0, 0, 0, 0, 0, 0, 124, 3466, 0, 0, 391, 42, 0, 1363, 886, 0, 0, 0, 0, 0, 0, 0, 0, 2642, 743, 1189, 0, 1290, 330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4487, 2148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 61, 1468, 172, 4242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6173, 32, 0, 128, 0, 62, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 2820, 62, 0, 854, 1258, 369, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5464, 6617, 6272, 6194, 6635, 6159, 6396, 5415] avg=6144.0000 max=6635 max_rank=4 min=5415 min_rank=7 imbalance_ratio=1.079915 max_load_reduction=42517 max_load_reduction_pct=0.865011 moved_to_echo=24605 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=17 rerouted_tokens_per_expert_current_ep_rank=[2496, 2268, 0, 0, 0, 0, 1990, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2943, 0, 1548, 0, 0, 697, 0, 653, 209, 0, 0, 0, 0, 0, 0, 0, 184, 3295, 0, 0, 433, 0, 0, 1178, 863, 0, 0, 0, 0, 0, 0, 0, 0, 2937, 611, 1513, 0, 1031, 414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3677, 2092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 311, 73, 1429, 143, 3806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5858, 37, 0, 171, 0, 92, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 3536, 91, 0, 955, 1087, 350, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6845, 6050, 5953, 6506, 5769, 5762, 6165, 6102] avg=6144.0000 max=6845 max_rank=0 min=5762 min_rank=5 imbalance_ratio=1.114095 max_load_reduction=42307 max_load_reduction_pct=0.860738 moved_to_echo=23798 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=17 rerouted_tokens_per_expert_current_ep_rank=[2139, 2655, 0, 0, 0, 0, 2094, 90, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2488, 1, 1406, 1, 0, 737, 0, 650, 208, 0, 0, 0, 0, 0, 0, 1, 182, 3490, 0, 0, 454, 0, 0, 1017, 870, 0, 0, 0, 0, 0, 0, 0, 4, 2904, 583, 1145, 2, 1005, 440, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 1, 0, 4, 0, 0, 3868, 2103, 0, 0, 0, 0, 0, 0, 0, 0, 3, 477, 83, 1423, 132, 3627, 3, 0, 0, 0, 0, 0, 0, 0, 0, 5721, 3, 0, 153, 0, 90, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 4, 4463, 92, 0, 943, 939, 346, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6978, 5492, 6014, 6083, 5982, 5748, 5968, 6887] avg=6144.0000 max=6978 max_rank=0 min=5492 min_rank=1 imbalance_ratio=1.135742 max_load_reduction=42174 max_load_reduction_pct=0.858032 moved_to_echo=24585 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[19712, 14720, 128, 128, 128, 128, 13952, 768, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 1408, 27392, 256, 128, 3584, 384, 128, 9344, 6912, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 24064, 256, 12288, 512, 128, 6144, 128, 4480, 1664, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[128, 128, 19968, 5504, 11136, 128, 8832, 3584, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[640, 128, 128, 512, 28800, 768, 128, 7296, 8576, 2816, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 512, 2560, 768, 11392, 1536, 32512, 256, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[512, 384, 128, 128, 384, 128, 128, 31360, 16768, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[46592, 384, 128, 1408, 128, 768, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=18 tokens_per_expert_current_ep_rank=[0, 67, 2989, 252, 0, 0, 4450, 0, 0, 0, 157, 1893, 7396, 0, 1, 7455, 2, 0, 0, 0, 0, 807, 1, 1, 1, 0, 0, 7808, 0, 1, 1244, 0, 9, 0, 130, 0, 0, 6633, 0, 0, 0, 0, 454, 0, 0, 1, 0, 170, 0, 0, 37, 0, 3, 628, 0, 67, 1, 6118, 0, 0, 274, 101, 0, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=18 tokens_per_expert_current_ep_rank=[1, 83, 3606, 269, 0, 2, 3750, 0, 0, 0, 145, 2078, 7159, 1, 78, 7289, 23, 0, 1, 6, 4, 426, 16, 49, 16, 0, 5, 7655, 4, 73, 1155, 6, 3, 3, 145, 0, 35, 6933, 8, 8, 0, 0, 428, 0, 5, 80, 54, 169, 0, 35, 25, 11, 6, 558, 56, 80, 14, 5771, 0, 17, 238, 534, 31, 5] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=18 tokens_per_expert_current_ep_rank=[6, 132, 3953, 351, 61, 31, 3401, 8, 5, 22, 215, 1278, 6538, 32, 273, 6582, 66, 24, 5, 31, 31, 524, 107, 207, 32, 48, 47, 6921, 105, 334, 1301, 50, 117, 10, 110, 7, 118, 6241, 251, 62, 33, 3, 519, 1, 128, 371, 171, 218, 41, 110, 33, 126, 41, 469, 323, 124, 72, 4306, 0, 100, 289, 1763, 233, 41] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=18 tokens_per_expert_current_ep_rank=[0, 87, 4008, 241, 0, 0, 4377, 0, 0, 0, 154, 1144, 7217, 0, 0, 7418, 1, 0, 0, 0, 0, 287, 10, 0, 0, 0, 0, 7782, 0, 0, 1529, 0, 0, 0, 141, 0, 0, 7386, 0, 0, 0, 0, 393, 0, 0, 0, 0, 281, 0, 0, 0, 0, 0, 641, 0, 87, 0, 5396, 0, 0, 231, 341, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=18 tokens_per_expert_current_ep_rank=[0, 92, 3880, 226, 0, 0, 4772, 0, 0, 0, 150, 740, 7101, 0, 0, 7338, 0, 0, 0, 0, 0, 318, 1, 0, 0, 0, 0, 7811, 0, 0, 1696, 0, 0, 0, 141, 0, 0, 7115, 0, 0, 0, 0, 484, 0, 0, 0, 0, 582, 0, 0, 0, 0, 0, 805, 0, 92, 0, 5360, 0, 0, 216, 232, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=18 tokens_per_expert_current_ep_rank=[0, 91, 3926, 234, 0, 0, 4370, 0, 0, 0, 161, 1352, 7106, 0, 0, 7326, 0, 0, 0, 0, 0, 328, 18, 0, 0, 0, 0, 7779, 0, 0, 1762, 0, 0, 0, 159, 0, 0, 7120, 0, 4, 0, 0, 500, 0, 0, 0, 0, 331, 0, 0, 0, 0, 0, 677, 0, 92, 0, 5211, 0, 0, 235, 370, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=18 tokens_per_expert_current_ep_rank=[1, 92, 4245, 246, 0, 0, 4297, 0, 0, 0, 167, 1260, 7145, 1, 5, 7327, 5, 0, 0, 0, 0, 319, 3, 3, 2, 0, 0, 7763, 0, 4, 1696, 0, 0, 1, 109, 0, 8, 7225, 0, 1, 1, 0, 479, 0, 0, 3, 1, 132, 0, 2, 0, 0, 0, 626, 4, 92, 0, 5101, 0, 0, 243, 541, 2, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=18 tokens_per_expert_current_ep_rank=[0, 62, 2879, 254, 0, 0, 4043, 0, 0, 0, 100, 2275, 7415, 0, 0, 7530, 2, 0, 0, 0, 0, 1004, 0, 0, 0, 0, 0, 7839, 0, 0, 1162, 0, 0, 0, 99, 0, 0, 6604, 0, 0, 0, 0, 350, 0, 0, 0, 0, 231, 0, 0, 48, 0, 0, 523, 0, 62, 0, 6274, 0, 0, 300, 96, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 67, 2989, 252, 0, 0, 2232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 1893, 4597, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 807, 1, 1, 5696, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4693, 0, 1, 1244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 130, 0, 0, 5699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 454, 0, 0, 1, 0, 170, 2229, 2218, 934, 127, 0, 0, 0, 0, 0, 0, 37, 0, 3, 628, 0, 67, 3115, 1759, 570, 0, 0, 0, 0, 0, 1, 5991, 0, 0, 274, 101, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5540, 6648, 6507, 5939, 5838, 6133, 6179, 6368] avg=6144.0000 max=6648 max_rank=1 min=5540 min_rank=0 imbalance_ratio=1.082031 max_load_reduction=42504 max_load_reduction_pct=0.864746 moved_to_echo=24518 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 87, 4008, 241, 0, 0, 2195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 1144, 4485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 287, 10, 0, 5668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4678, 0, 0, 1529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 0, 0, 6346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 393, 0, 0, 0, 0, 281, 2175, 2182, 1040, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 641, 0, 87, 3104, 1750, 557, 0, 0, 0, 0, 0, 0, 5284, 0, 0, 231, 341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6531, 5783, 5966, 6207, 6487, 6183, 6139, 5856] avg=6144.0000 max=6531 max_rank=0 min=5783 min_rank=1 imbalance_ratio=1.062988 max_load_reduction=42621 max_load_reduction_pct=0.867126 moved_to_echo=24665 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=18 rerouted_tokens_per_expert_current_ep_rank=[1, 83, 3606, 269, 0, 2, 1881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 2078, 4449, 1, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 1, 6, 4, 426, 16, 49, 5569, 0, 0, 0, 0, 0, 0, 0, 16, 0, 5, 4602, 4, 73, 1155, 6, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 145, 0, 35, 5957, 8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 428, 0, 5, 80, 54, 169, 2158, 1869, 976, 120, 0, 0, 0, 0, 0, 35, 25, 11, 6, 558, 56, 80, 3053, 1720, 552, 0, 0, 0, 0, 0, 14, 5651, 0, 17, 238, 534, 31, 5, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5842, 6751, 6094, 5861, 6159, 5859, 6096, 6490] avg=6144.0000 max=6751 max_rank=1 min=5842 min_rank=0 imbalance_ratio=1.098796 max_load_reduction=42401 max_load_reduction_pct=0.862651 moved_to_echo=24604 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=18 rerouted_tokens_per_expert_current_ep_rank=[6, 132, 3953, 351, 61, 31, 1706, 8, 0, 0, 0, 0, 0, 0, 0, 0, 5, 22, 215, 1278, 4064, 32, 273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 24, 5, 31, 31, 524, 107, 207, 5030, 0, 0, 0, 0, 0, 0, 0, 32, 48, 47, 4160, 105, 334, 1301, 50, 0, 0, 0, 0, 0, 0, 0, 0, 117, 10, 110, 7, 118, 5362, 251, 62, 0, 0, 0, 0, 0, 0, 0, 0, 33, 3, 519, 1, 128, 371, 171, 218, 1970, 1695, 879, 89, 0, 0, 0, 0, 41, 110, 33, 126, 41, 469, 323, 124, 2761, 1552, 504, 0, 0, 0, 0, 0, 72, 4217, 0, 100, 289, 1763, 233, 41, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6248, 5889, 6025, 6077, 6037, 6077, 6084, 6715] avg=6144.0000 max=6715 max_rank=7 min=5889 min_rank=1 imbalance_ratio=1.092936 max_load_reduction=42437 max_load_reduction_pct=0.863383 moved_to_echo=24913 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 92, 3880, 226, 0, 0, 2393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 740, 4413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 318, 1, 0, 5608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4695, 0, 0, 1696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 0, 0, 6113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 0, 0, 0, 0, 582, 2140, 2379, 1002, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 805, 0, 92, 3116, 1730, 548, 0, 0, 0, 0, 0, 0, 5249, 0, 0, 216, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6591, 5303, 5927, 6391, 6254, 6698, 6291, 5697] avg=6144.0000 max=6698 max_rank=5 min=5303 min_rank=1 imbalance_ratio=1.090169 max_load_reduction=42454 max_load_reduction_pct=0.863729 moved_to_echo=24940 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 62, 2879, 254, 0, 0, 2028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 2275, 4608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1004, 0, 0, 5754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4712, 0, 0, 1162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 0, 0, 5674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 350, 0, 0, 0, 0, 231, 2235, 2015, 930, 130, 0, 0, 0, 0, 0, 0, 48, 0, 0, 523, 0, 62, 3127, 1776, 572, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 300, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5223, 6983, 6760, 5874, 5773, 5891, 6108, 6540] avg=6144.0000 max=6983 max_rank=1 min=5223 min_rank=0 imbalance_ratio=1.136556 max_load_reduction=42169 max_load_reduction_pct=0.857931 moved_to_echo=24312 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=18 rerouted_tokens_per_expert_current_ep_rank=[0, 91, 3926, 234, 0, 0, 2192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 1352, 4416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 328, 18, 0, 5598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4676, 0, 0, 1762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 159, 0, 0, 6118, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 0, 0, 0, 0, 331, 2142, 2178, 1002, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 677, 0, 92, 3103, 1728, 548, 0, 0, 0, 0, 0, 0, 5103, 0, 0, 235, 370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6443, 5929, 5944, 6438, 6281, 6261, 6148, 5708] avg=6144.0000 max=6443 max_rank=0 min=5708 min_rank=7 imbalance_ratio=1.048665 max_load_reduction=42709 max_load_reduction_pct=0.868917 moved_to_echo=24398 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=18 rerouted_tokens_per_expert_current_ep_rank=[1, 92, 4245, 246, 0, 0, 2150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 167, 1260, 4433, 1, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 319, 3, 3, 5599, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4663, 0, 4, 1696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 109, 0, 8, 6204, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 479, 0, 0, 3, 1, 132, 2157, 2147, 1021, 109, 0, 0, 0, 0, 0, 2, 0, 0, 0, 626, 4, 92, 3100, 1728, 555, 0, 0, 0, 0, 0, 0, 4992, 0, 0, 243, 541, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6734, 5866, 5929, 6365, 6323, 6050, 6107, 5778] avg=6144.0000 max=6734 max_rank=0 min=5778 min_rank=7 imbalance_ratio=1.096029 max_load_reduction=42418 max_load_reduction_pct=0.862996 moved_to_echo=24258 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=50176 observed_loss=-1024 tokens_per_expert=[128, 128, 3712, 128, 256, 512, 256, 2176, 17280, 16768, 7808, 1024, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 1280, 12032, 35584, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 256, 256, 256, 128, 4992, 384, 768, 24576, 13824, 4480, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 128, 36992, 128, 512, 11648, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 768, 29568, 2176, 128, 128, 16896, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 4096, 256, 384, 44544, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 42752, 0, 128, 2048, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[256, 128, 1152, 128, 256, 47488, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=50176 slack_slots=48128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=19 tokens_per_expert_current_ep_rank=[53, 0, 51, 0, 76, 79, 59, 7, 0, 1, 186, 17, 0, 89, 2258, 1, 3315, 269, 94, 7473, 6742, 0, 46, 10, 0, 138, 2880, 0, 7169, 30, 85, 0, 5305, 20, 59, 9, 12, 1606, 7015, 79, 0, 12, 0, 59, 6, 66, 0, 61, 0, 811, 7, 704, 0, 65, 0, 0, 33, 0, 87, 240, 935, 76, 396, 361] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=19 tokens_per_expert_current_ep_rank=[90, 0, 0, 0, 0, 82, 0, 0, 0, 0, 173, 0, 0, 68, 2596, 0, 3485, 250, 91, 7608, 7185, 0, 0, 0, 0, 117, 3285, 0, 7281, 0, 68, 0, 5756, 0, 29, 0, 0, 725, 7182, 87, 0, 0, 0, 0, 0, 62, 0, 59, 0, 662, 0, 570, 0, 0, 0, 0, 0, 0, 8, 217, 753, 0, 332, 331] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=19 tokens_per_expert_current_ep_rank=[5, 0, 0, 0, 1, 67, 34, 0, 0, 0, 135, 0, 0, 73, 2481, 0, 3800, 251, 67, 7661, 7048, 0, 1, 0, 0, 128, 1689, 0, 7472, 0, 63, 0, 5758, 1, 1, 0, 2, 1582, 7169, 67, 0, 0, 0, 1, 0, 63, 0, 70, 0, 1035, 0, 507, 0, 1, 0, 0, 0, 0, 2, 253, 992, 1, 331, 340] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=19 tokens_per_expert_current_ep_rank=[94, 0, 0, 0, 0, 90, 0, 0, 0, 0, 187, 0, 0, 58, 2713, 0, 3468, 226, 92, 7546, 7052, 0, 0, 0, 0, 125, 3386, 0, 7245, 0, 58, 0, 5604, 0, 12, 0, 0, 704, 7201, 92, 0, 0, 0, 0, 0, 56, 0, 73, 0, 894, 0, 651, 0, 0, 0, 0, 0, 0, 1, 238, 603, 0, 343, 340] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=19 tokens_per_expert_current_ep_rank=[81, 5, 238, 0, 351, 104, 559, 142, 46, 45, 144, 110, 3, 276, 1898, 8, 3443, 372, 162, 6660, 5459, 7, 155, 89, 6, 479, 3533, 0, 6180, 161, 333, 105, 4239, 98, 310, 43, 125, 669, 6206, 191, 9, 85, 9, 288, 18, 38, 1, 84, 37, 628, 38, 1231, 31, 266, 0, 6, 158, 17, 253, 206, 1231, 383, 736, 364] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=19 tokens_per_expert_current_ep_rank=[13, 0, 0, 0, 0, 62, 0, 0, 0, 0, 126, 0, 0, 40, 2360, 0, 3728, 254, 62, 7656, 7130, 0, 0, 0, 0, 129, 1825, 0, 7414, 0, 37, 0, 5786, 0, 3, 0, 0, 1709, 7160, 62, 0, 0, 0, 0, 0, 37, 0, 59, 0, 991, 0, 578, 0, 0, 0, 0, 0, 0, 0, 211, 1015, 0, 283, 422] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=19 tokens_per_expert_current_ep_rank=[85, 0, 3, 0, 5, 63, 1, 0, 0, 0, 183, 0, 0, 77, 2596, 0, 3225, 248, 90, 7550, 6910, 0, 4, 2, 0, 149, 4078, 0, 7171, 1, 77, 0, 5156, 2, 160, 0, 1, 505, 7166, 92, 0, 1, 0, 3, 0, 77, 0, 53, 0, 922, 1, 663, 0, 4, 0, 0, 0, 0, 34, 223, 837, 4, 379, 351] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=19 tokens_per_expert_current_ep_rank=[89, 0, 0, 0, 0, 77, 0, 0, 0, 0, 184, 0, 0, 69, 2567, 0, 3534, 252, 94, 7512, 6959, 0, 0, 0, 0, 113, 3187, 0, 7123, 0, 69, 0, 6040, 0, 28, 0, 0, 815, 7200, 92, 0, 0, 0, 0, 0, 69, 0, 67, 0, 664, 0, 654, 0, 0, 0, 0, 0, 0, 2, 225, 796, 0, 353, 318] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=19 rerouted_tokens_per_expert_current_ep_rank=[90, 0, 0, 0, 0, 82, 0, 0, 4399, 1530, 0, 0, 0, 0, 0, 0, 0, 0, 173, 0, 0, 68, 2596, 0, 2922, 591, 0, 0, 0, 0, 0, 0, 3485, 250, 91, 0, 2371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 3285, 0, 2913, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5165, 0, 29, 0, 0, 725, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 59, 6078, 0, 0, 0, 0, 0, 0, 0, 0, 662, 0, 570, 0, 0, 0, 0, 2783, 1940, 0, 0, 0, 0, 0, 0, 0, 0, 8, 217, 753, 0, 332, 331, 2874, 1446, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6101, 6350, 6197, 6383, 6006, 6199, 5955, 5961] avg=6144.0000 max=6383 max_rank=3 min=5955 min_rank=6 imbalance_ratio=1.038900 max_load_reduction=42769 max_load_reduction_pct=0.870138 moved_to_echo=24121 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=19 rerouted_tokens_per_expert_current_ep_rank=[53, 0, 51, 0, 76, 79, 59, 7, 4297, 1502, 0, 0, 0, 0, 0, 0, 0, 1, 186, 17, 0, 89, 2258, 1, 2877, 545, 0, 0, 0, 0, 0, 0, 3315, 269, 94, 0, 2224, 0, 46, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 2880, 0, 2868, 30, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4760, 20, 59, 9, 12, 1606, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 59, 6, 66, 0, 61, 5971, 0, 0, 0, 0, 0, 0, 0, 0, 811, 7, 704, 0, 65, 0, 0, 2718, 1821, 0, 0, 0, 0, 0, 0, 33, 0, 87, 240, 935, 76, 396, 361, 2697, 1424, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6124, 5974, 5958, 6001, 6545, 6175, 6126, 6249] avg=6144.0000 max=6545 max_rank=4 min=5958 min_rank=2 imbalance_ratio=1.065267 max_load_reduction=42607 max_load_reduction_pct=0.866842 moved_to_echo=25095 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=19 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 0, 0, 1, 67, 34, 0, 4391, 1540, 0, 0, 0, 0, 0, 0, 0, 0, 135, 0, 0, 73, 2481, 0, 2999, 591, 0, 0, 0, 0, 0, 0, 3800, 251, 67, 0, 2325, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 1689, 0, 2989, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5167, 1, 1, 0, 2, 1582, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 63, 0, 70, 6121, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 0, 507, 0, 1, 0, 0, 2778, 1903, 0, 0, 0, 0, 0, 0, 0, 0, 2, 253, 992, 1, 331, 340, 2820, 1484, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6038, 6279, 6444, 4869, 6820, 6255, 6224, 6223] avg=6144.0000 max=6820 max_rank=4 min=4869 min_rank=3 imbalance_ratio=1.110026 max_load_reduction=42332 max_load_reduction_pct=0.861247 moved_to_echo=25522 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=19 rerouted_tokens_per_expert_current_ep_rank=[81, 5, 238, 0, 351, 104, 559, 142, 3802, 1340, 0, 0, 0, 0, 0, 0, 46, 45, 144, 110, 3, 276, 1898, 8, 2480, 435, 0, 0, 0, 0, 0, 0, 3443, 372, 162, 0, 1801, 7, 155, 89, 0, 0, 0, 0, 0, 0, 0, 0, 6, 479, 3533, 0, 2473, 161, 333, 105, 0, 0, 0, 0, 0, 0, 0, 0, 3804, 98, 310, 43, 125, 669, 0, 191, 0, 0, 0, 0, 0, 0, 0, 0, 9, 85, 9, 288, 18, 38, 1, 84, 5320, 0, 0, 0, 0, 0, 0, 0, 37, 628, 38, 1231, 31, 266, 0, 6, 2404, 1474, 0, 0, 0, 0, 0, 0, 158, 17, 253, 206, 1231, 383, 736, 364, 2184, 1227, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6622, 5445, 6029, 7090, 5240, 5852, 6115, 6759] avg=6144.0000 max=7090 max_rank=3 min=5240 min_rank=4 imbalance_ratio=1.153971 max_load_reduction=42062 max_load_reduction_pct=0.855754 moved_to_echo=23966 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=19 rerouted_tokens_per_expert_current_ep_rank=[94, 0, 0, 0, 0, 90, 0, 0, 4411, 1518, 0, 0, 0, 0, 0, 0, 0, 0, 187, 0, 0, 58, 2713, 0, 2907, 575, 0, 0, 0, 0, 0, 0, 3468, 226, 92, 0, 2327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 125, 3386, 0, 2899, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5029, 0, 12, 0, 0, 704, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 73, 6028, 0, 0, 0, 0, 0, 0, 0, 0, 894, 0, 651, 0, 0, 0, 0, 2790, 1904, 0, 0, 0, 0, 0, 0, 0, 0, 1, 238, 603, 0, 343, 340, 2821, 1439, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6113, 6440, 6113, 6468, 5837, 6157, 6239, 5785] avg=6144.0000 max=6468 max_rank=3 min=5785 min_rank=7 imbalance_ratio=1.052734 max_load_reduction=42684 max_load_reduction_pct=0.868408 moved_to_echo=24018 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=19 rerouted_tokens_per_expert_current_ep_rank=[13, 0, 0, 0, 0, 62, 0, 0, 4386, 1540, 0, 0, 0, 0, 0, 0, 0, 0, 126, 0, 0, 40, 2360, 0, 2975, 594, 0, 0, 0, 0, 0, 0, 3728, 254, 62, 0, 2352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 1825, 0, 2966, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5192, 0, 3, 0, 0, 1709, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 59, 6116, 0, 0, 0, 0, 0, 0, 0, 0, 991, 0, 578, 0, 0, 0, 0, 2774, 1926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 211, 1015, 0, 283, 422, 2852, 1473, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6001, 6095, 6396, 4957, 6966, 6212, 6269, 6256] avg=6144.0000 max=6966 max_rank=4 min=4957 min_rank=3 imbalance_ratio=1.133789 max_load_reduction=42186 max_load_reduction_pct=0.858276 moved_to_echo=25703 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=19 rerouted_tokens_per_expert_current_ep_rank=[89, 0, 0, 0, 0, 77, 0, 0, 4410, 1511, 0, 0, 0, 0, 0, 0, 0, 0, 184, 0, 0, 69, 2567, 0, 2858, 620, 0, 0, 0, 0, 0, 0, 3534, 252, 94, 0, 2296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 3187, 0, 2850, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5420, 0, 28, 0, 0, 815, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 67, 6001, 0, 0, 0, 0, 0, 0, 0, 0, 664, 0, 654, 0, 0, 0, 0, 2790, 1879, 0, 0, 0, 0, 0, 0, 0, 0, 2, 225, 796, 0, 353, 318, 2784, 1415, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6087, 6298, 6176, 6219, 6355, 6137, 5987, 5893] avg=6144.0000 max=6355 max_rank=4 min=5893 min_rank=7 imbalance_ratio=1.034342 max_load_reduction=42797 max_load_reduction_pct=0.870707 moved_to_echo=24372 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=19 rerouted_tokens_per_expert_current_ep_rank=[85, 0, 3, 0, 5, 63, 1, 0, 4390, 1519, 0, 0, 0, 0, 0, 0, 0, 0, 183, 0, 0, 77, 2596, 0, 2882, 533, 0, 0, 0, 0, 0, 0, 3225, 248, 90, 0, 2270, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 4078, 0, 2860, 1, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4623, 2, 160, 0, 1, 505, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 77, 0, 53, 6031, 0, 0, 0, 0, 0, 0, 0, 0, 922, 1, 663, 0, 4, 0, 0, 2776, 1871, 0, 0, 0, 0, 0, 0, 0, 0, 34, 223, 837, 4, 379, 351, 2769, 1429, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6066, 6271, 5839, 7165, 5383, 6165, 6237, 6026] avg=6144.0000 max=7165 max_rank=3 min=5383 min_rank=4 imbalance_ratio=1.166178 max_load_reduction=41987 max_load_reduction_pct=0.854228 moved_to_echo=23811 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[256, 128, 512, 1920, 7168, 512, 3200, 2944, 21888, 11392, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[512, 128, 384, 0, 512, 640, 768, 256, 34560, 12032, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 1408, 128, 128, 768, 19584, 128, 22912, 4608, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[28032, 2176, 768, 0, 18048, 128, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 1408, 23936, 0, 22912, 256, 896, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 6656, 128, 5632, 128, 384, 0, 128, 21888, 14720, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[39168, 128, 640, 128, 256, 8320, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 128, 384, 128, 512, 128, 640, 47744, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49408 slack_slots=48896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 4254, 0, 0, 0, 0, 0, 0, 0, 92, 0, 92, 1846, 0, 0, 0, 1855, 913, 7989, 639, 0, 92, 7219, 0, 0, 7226, 143, 58, 0, 0, 0, 0, 235, 0, 58, 0, 0, 3437, 0, 1069, 0, 0, 0, 0, 85, 0, 0, 0, 0, 392, 2504, 0, 4827, 293, 1013, 92, 0, 0, 0, 2271, 366, 92] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 3773, 0, 0, 0, 0, 0, 0, 0, 87, 0, 87, 2005, 0, 0, 0, 1820, 1112, 7907, 625, 0, 87, 7206, 0, 0, 7341, 74, 68, 0, 0, 0, 0, 160, 0, 67, 0, 0, 4024, 0, 1046, 0, 0, 0, 0, 63, 0, 0, 0, 0, 385, 2686, 0, 4692, 228, 991, 86, 1, 0, 0, 2139, 306, 86] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 52, 3897, 61, 44, 0, 0, 2, 0, 0, 79, 18, 78, 2613, 21, 1, 0, 1521, 1225, 7634, 619, 14, 79, 7093, 45, 0, 7048, 375, 72, 0, 1, 0, 61, 186, 77, 66, 1, 0, 4407, 58, 1066, 6, 76, 1, 0, 78, 0, 13, 4, 75, 570, 4010, 2, 2654, 239, 905, 81, 11, 0, 77, 1598, 153, 85] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=20 tokens_per_expert_current_ep_rank=[7, 0, 249, 4504, 340, 87, 7, 0, 30, 86, 5, 160, 187, 145, 2284, 32, 22, 52, 1332, 873, 6525, 647, 228, 95, 6354, 196, 10, 6154, 330, 86, 30, 14, 0, 150, 394, 334, 70, 38, 0, 3731, 154, 1019, 41, 272, 78, 5, 91, 38, 128, 76, 284, 489, 4566, 42, 3018, 348, 953, 128, 75, 22, 240, 696, 504, 97] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 3196, 0, 0, 0, 0, 0, 0, 0, 62, 0, 62, 2847, 0, 0, 0, 2025, 1390, 8004, 672, 0, 62, 7454, 0, 0, 7381, 453, 37, 0, 0, 0, 0, 120, 0, 36, 0, 0, 4544, 0, 684, 0, 0, 0, 0, 61, 0, 0, 0, 0, 561, 3141, 0, 2478, 159, 978, 62, 0, 0, 0, 2491, 149, 43] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 1, 3525, 0, 1, 0, 0, 0, 0, 8, 70, 0, 67, 2621, 0, 0, 0, 2181, 1347, 7919, 611, 1, 67, 7494, 0, 0, 7419, 399, 63, 0, 0, 0, 1, 126, 1, 63, 1, 0, 4385, 0, 710, 0, 1, 0, 0, 67, 0, 0, 0, 8, 539, 3152, 0, 2545, 190, 926, 67, 1, 0, 1, 2283, 224, 67] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 4, 4075, 3, 5, 0, 0, 0, 0, 0, 92, 1, 92, 1748, 1, 0, 0, 1909, 1090, 7854, 650, 1, 92, 7216, 1, 0, 7251, 155, 77, 0, 0, 0, 4, 208, 3, 77, 1, 0, 4036, 2, 1071, 1, 5, 2, 0, 73, 0, 1, 0, 3, 386, 2975, 0, 4635, 285, 939, 90, 0, 0, 5, 1708, 235, 90] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=20 tokens_per_expert_current_ep_rank=[0, 0, 0, 3752, 0, 0, 0, 0, 0, 0, 0, 92, 0, 91, 2004, 0, 0, 0, 2019, 1111, 7890, 611, 0, 91, 7245, 0, 0, 7341, 154, 69, 0, 0, 0, 0, 242, 0, 69, 0, 0, 4171, 0, 1129, 0, 0, 0, 0, 60, 0, 0, 0, 0, 393, 2604, 0, 4378, 310, 922, 92, 0, 0, 0, 2004, 217, 91] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3773, 0, 0, 0, 0, 1519, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 87, 2005, 0, 2168, 1550, 0, 0, 0, 0, 0, 0, 0, 0, 1820, 1112, 2483, 625, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5939, 74, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 67, 0, 0, 4024, 1402, 425, 0, 0, 0, 0, 0, 0, 0, 1046, 0, 0, 0, 0, 63, 0, 5038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 385, 2686, 0, 2748, 228, 0, 0, 0, 0, 0, 0, 0, 0, 991, 86, 1, 0, 0, 2139, 306, 86, 2869, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6297, 5897, 6127, 6081, 6078, 6147, 6047, 6478] avg=6144.0000 max=6478 max_rank=7 min=5897 min_rank=1 imbalance_ratio=1.054362 max_load_reduction=42674 max_load_reduction_pct=0.868205 moved_to_echo=24750 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 52, 3897, 61, 44, 0, 0, 859, 970, 0, 0, 0, 0, 0, 0, 2, 0, 0, 79, 18, 78, 2613, 21, 2134, 1497, 0, 0, 0, 0, 0, 0, 1, 0, 1521, 1225, 2397, 619, 14, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 5702, 375, 72, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 186, 77, 66, 1, 0, 4407, 1346, 240, 0, 0, 0, 0, 0, 0, 58, 1066, 6, 76, 1, 0, 78, 0, 4959, 0, 0, 0, 0, 0, 0, 0, 13, 4, 75, 570, 4010, 2, 1555, 239, 0, 0, 0, 0, 0, 0, 0, 0, 905, 81, 11, 0, 77, 1598, 153, 85, 2770, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5883, 6442, 5856, 6195, 6384, 6244, 6468, 5680] avg=6144.0000 max=6468 max_rank=6 min=5680 min_rank=7 imbalance_ratio=1.052734 max_load_reduction=42684 max_load_reduction_pct=0.868408 moved_to_echo=24776 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 3525, 0, 1, 0, 0, 824, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 8, 70, 0, 67, 2621, 0, 2256, 1552, 0, 0, 0, 0, 0, 0, 0, 0, 2181, 1347, 2488, 611, 1, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6002, 399, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 126, 1, 63, 1, 0, 4385, 1417, 230, 0, 0, 0, 0, 0, 0, 0, 710, 0, 1, 0, 0, 67, 0, 5238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 539, 3152, 0, 1491, 190, 0, 0, 0, 0, 0, 0, 0, 0, 926, 67, 1, 0, 1, 2283, 224, 67, 2873, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5357, 6574, 6695, 6464, 6224, 6016, 5380, 6442] avg=6144.0000 max=6695 max_rank=2 min=5357 min_rank=0 imbalance_ratio=1.089681 max_load_reduction=42457 max_load_reduction_pct=0.863790 moved_to_echo=24062 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4254, 0, 0, 0, 0, 1563, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 92, 1846, 0, 2173, 1566, 0, 0, 0, 0, 0, 0, 0, 0, 1855, 913, 2509, 639, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5846, 143, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 235, 0, 58, 0, 0, 3437, 1380, 437, 0, 0, 0, 0, 0, 0, 0, 1069, 0, 0, 0, 0, 85, 0, 5046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 392, 2504, 0, 2827, 293, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 92, 0, 0, 0, 2271, 366, 92, 2899, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6832, 5769, 6008, 6047, 5547, 6200, 6016, 6733] avg=6144.0000 max=6832 max_rank=0 min=5547 min_rank=4 imbalance_ratio=1.111979 max_load_reduction=42320 max_load_reduction_pct=0.861003 moved_to_echo=24496 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=20 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 249, 4504, 340, 87, 7, 0, 977, 829, 0, 0, 0, 0, 0, 0, 30, 86, 5, 160, 187, 145, 2284, 32, 1913, 1279, 0, 0, 0, 0, 0, 0, 22, 52, 1332, 873, 2050, 647, 228, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 196, 10, 4979, 330, 86, 30, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 394, 334, 70, 38, 0, 3731, 1175, 273, 0, 0, 0, 0, 0, 0, 154, 1019, 41, 272, 78, 5, 91, 38, 4441, 0, 0, 0, 0, 0, 0, 0, 128, 76, 284, 489, 4566, 42, 1768, 348, 0, 0, 0, 0, 0, 0, 0, 0, 953, 128, 75, 22, 240, 696, 504, 97, 2367, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7000, 6121, 5299, 5645, 6165, 6139, 7701, 5082] avg=6144.0000 max=7701 max_rank=6 min=5082 min_rank=7 imbalance_ratio=1.253418 max_load_reduction=41451 max_load_reduction_pct=0.843323 moved_to_echo=25087 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3196, 0, 0, 0, 0, 802, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 62, 2847, 0, 2244, 1569, 0, 0, 0, 0, 0, 0, 0, 0, 2025, 1390, 2514, 672, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5972, 453, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 0, 36, 0, 0, 4544, 1409, 224, 0, 0, 0, 0, 0, 0, 0, 684, 0, 0, 0, 0, 61, 0, 5210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 3141, 0, 1452, 159, 0, 0, 0, 0, 0, 0, 0, 0, 978, 62, 0, 0, 0, 2491, 149, 43, 2904, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5015, 6784, 6663, 6462, 6333, 5955, 5313, 6627] avg=6144.0000 max=6784 max_rank=1 min=5015 min_rank=0 imbalance_ratio=1.104167 max_load_reduction=42368 max_load_reduction_pct=0.861979 moved_to_echo=24228 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3752, 0, 0, 0, 0, 1418, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 91, 2004, 0, 2181, 1547, 0, 0, 0, 0, 0, 0, 0, 0, 2019, 1111, 2477, 611, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5939, 154, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 242, 0, 69, 0, 0, 4171, 1402, 396, 0, 0, 0, 0, 0, 0, 0, 1129, 0, 0, 0, 0, 60, 0, 5064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 393, 2604, 0, 2564, 310, 0, 0, 0, 0, 0, 0, 0, 0, 922, 92, 0, 0, 0, 2004, 217, 91, 2863, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6173, 5915, 6309, 6162, 6280, 6253, 5871, 6189] avg=6144.0000 max=6309 max_rank=2 min=5871 min_rank=6 imbalance_ratio=1.026855 max_load_reduction=42843 max_load_reduction_pct=0.871643 moved_to_echo=24593 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=20 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 4075, 3, 5, 0, 0, 1505, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 1, 92, 1748, 1, 2172, 1544, 0, 0, 0, 0, 0, 0, 0, 0, 1909, 1090, 2453, 650, 1, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5863, 155, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 208, 3, 77, 1, 0, 4036, 1388, 424, 0, 0, 0, 0, 0, 0, 2, 1071, 1, 5, 2, 0, 73, 0, 5044, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 386, 2975, 0, 2706, 285, 0, 0, 0, 0, 0, 0, 0, 0, 939, 90, 0, 0, 5, 1708, 235, 90, 2854, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6595, 5650, 6195, 6096, 6141, 6198, 6356, 5921] avg=6144.0000 max=6595 max_rank=0 min=5650 min_rank=1 imbalance_ratio=1.073405 max_load_reduction=42557 max_load_reduction_pct=0.865824 moved_to_echo=24616 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 128, 768, 256, 768, 18048, 128, 17280, 12160, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 0, 384, 30976, 512, 256, 128, 0, 9472, 7936, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7680, 768, 128, 128, 384, 15232, 2176, 768, 22400, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 14720, 9088, 19456, 5120, 256, 768, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[256, 128, 384, 3840, 25728, 128, 17152, 2176, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[0, 256, 1792, 512, 512, 128, 0, 32768, 11008, 2688, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[256, 7808, 128, 384, 128, 128, 640, 128, 40064, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[0, 256, 128, 46336, 2176, 640, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=21 tokens_per_expert_current_ep_rank=[0, 5083, 1784, 1163, 6904, 7254, 9, 0, 1, 5, 15, 183, 10, 5, 34, 5, 83, 14, 150, 0, 41, 1947, 3, 0, 0, 0, 1032, 320, 79, 0, 60, 0, 1, 1, 158, 0, 3, 5984, 4119, 189, 4, 0, 1113, 5, 37, 81, 11, 86, 19, 1348, 201, 0, 1, 82, 119, 2656, 79, 6475, 7, 75, 79, 6, 2, 27] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=21 tokens_per_expert_current_ep_rank=[44, 4245, 1997, 986, 6282, 6464, 195, 0, 29, 26, 31, 248, 215, 41, 71, 125, 309, 259, 147, 121, 140, 2172, 29, 0, 5, 0, 1455, 666, 99, 12, 122, 28, 12, 14, 443, 22, 18, 5263, 3125, 255, 63, 304, 1049, 76, 122, 140, 22, 295, 77, 1372, 381, 2, 53, 372, 187, 2468, 153, 5841, 29, 102, 109, 31, 46, 143] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=21 tokens_per_expert_current_ep_rank=[0, 5059, 1569, 1079, 7040, 7471, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 128, 0, 0, 1818, 0, 0, 0, 0, 1086, 309, 86, 0, 0, 0, 0, 0, 86, 0, 0, 5893, 5842, 154, 0, 0, 1176, 0, 0, 6, 0, 0, 0, 1432, 226, 0, 0, 0, 84, 1730, 86, 6705, 0, 0, 86, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=21 tokens_per_expert_current_ep_rank=[0, 5474, 2218, 996, 7063, 7453, 0, 0, 0, 0, 0, 81, 0, 0, 1, 0, 1, 0, 132, 2, 0, 1489, 0, 0, 0, 0, 997, 187, 67, 0, 1, 0, 0, 0, 74, 0, 0, 6468, 3922, 178, 0, 57, 1225, 0, 0, 3, 0, 0, 0, 1403, 31, 0, 1, 1, 94, 3282, 82, 6101, 0, 1, 67, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=21 tokens_per_expert_current_ep_rank=[0, 5279, 1549, 1185, 6972, 7360, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 132, 0, 0, 1596, 0, 0, 0, 0, 1105, 591, 92, 0, 0, 0, 0, 0, 95, 0, 0, 5675, 5473, 152, 0, 0, 1240, 0, 0, 9, 0, 0, 0, 1527, 215, 0, 0, 0, 76, 1758, 92, 6866, 0, 0, 92, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=21 tokens_per_expert_current_ep_rank=[0, 5484, 2000, 1100, 7084, 7507, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 116, 0, 0, 1555, 0, 0, 0, 0, 1048, 139, 62, 0, 0, 0, 0, 0, 88, 0, 0, 6448, 4049, 125, 0, 12, 1291, 0, 0, 0, 0, 0, 0, 1368, 15, 0, 0, 0, 62, 3041, 63, 6361, 0, 0, 62, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=21 tokens_per_expert_current_ep_rank=[0, 5331, 1642, 1058, 6983, 7393, 0, 0, 0, 1, 1, 20, 0, 1, 4, 1, 5, 0, 153, 0, 2, 2031, 0, 0, 0, 0, 1125, 442, 90, 0, 4, 0, 0, 0, 113, 0, 0, 5711, 5366, 194, 0, 0, 1151, 0, 3, 15, 1, 3, 0, 1651, 330, 0, 0, 5, 104, 1622, 90, 6411, 0, 3, 90, 0, 1, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=21 tokens_per_expert_current_ep_rank=[0, 4965, 1888, 1078, 7019, 7361, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 144, 0, 0, 1878, 0, 0, 0, 0, 1080, 393, 92, 0, 0, 0, 0, 0, 101, 0, 0, 5881, 5385, 164, 0, 0, 1217, 0, 0, 0, 0, 0, 0, 1514, 271, 0, 0, 0, 91, 1783, 92, 6654, 0, 0, 92, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 3427, 2218, 996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 1, 0, 6126, 0, 0, 0, 0, 0, 0, 0, 1, 0, 132, 2, 0, 1489, 0, 0, 3990, 449, 0, 0, 0, 0, 0, 0, 0, 0, 997, 187, 67, 0, 1, 0, 3494, 1216, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 1262, 3922, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 1225, 0, 0, 3, 0, 0, 3569, 1327, 0, 0, 0, 0, 0, 0, 0, 1403, 31, 0, 1, 1, 94, 3282, 1598, 492, 0, 0, 0, 0, 0, 0, 82, 5609, 0, 1, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6641, 6208, 6063, 5962, 5436, 6181, 6902, 5759] avg=6144.0000 max=6902 max_rank=6 min=5436 min_rank=4 imbalance_ratio=1.123372 max_load_reduction=42250 max_load_reduction_pct=0.859578 moved_to_echo=24278 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 3182, 1784, 1163, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 15, 183, 10, 5, 34, 5, 5963, 0, 0, 0, 0, 0, 0, 0, 83, 14, 150, 0, 41, 1947, 3, 0, 3691, 417, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 320, 79, 0, 60, 0, 3415, 1125, 0, 0, 0, 0, 0, 0, 1, 1, 158, 0, 3, 1168, 4119, 189, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1113, 5, 37, 81, 11, 86, 3489, 1291, 0, 0, 0, 0, 0, 0, 19, 1348, 201, 0, 1, 82, 119, 2656, 1484, 522, 0, 0, 0, 0, 0, 0, 79, 5953, 7, 75, 79, 6, 2, 27, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6138, 6221, 6346, 6031, 5639, 6117, 6432, 6228] avg=6144.0000 max=6432 max_rank=6 min=5639 min_rank=4 imbalance_ratio=1.046875 max_load_reduction=42720 max_load_reduction_pct=0.869141 moved_to_echo=24416 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=21 rerouted_tokens_per_expert_current_ep_rank=[44, 2658, 1997, 986, 0, 0, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 26, 31, 248, 215, 41, 71, 125, 5313, 0, 0, 0, 0, 0, 0, 0, 309, 259, 147, 121, 140, 2172, 29, 0, 3246, 348, 0, 0, 0, 0, 0, 0, 5, 0, 1455, 666, 99, 12, 122, 28, 3107, 989, 0, 0, 0, 0, 0, 0, 12, 14, 443, 22, 18, 1028, 3125, 255, 0, 0, 0, 0, 0, 0, 0, 0, 63, 304, 1049, 76, 122, 140, 22, 295, 3175, 1151, 0, 0, 0, 0, 0, 0, 77, 1372, 381, 2, 53, 372, 187, 2468, 1239, 471, 0, 0, 0, 0, 0, 0, 153, 5370, 29, 102, 109, 31, 46, 143, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5880, 6099, 6771, 6483, 4917, 6397, 6622, 5983] avg=6144.0000 max=6771 max_rank=2 min=4917 min_rank=4 imbalance_ratio=1.102051 max_load_reduction=42381 max_load_reduction_pct=0.862244 moved_to_echo=23919 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 3167, 1569, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 6141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 1818, 0, 0, 3635, 415, 0, 0, 0, 0, 0, 0, 0, 0, 1086, 309, 86, 0, 0, 0, 3482, 1108, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 1150, 5842, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1176, 0, 0, 6, 0, 0, 3558, 1330, 0, 0, 0, 0, 0, 0, 0, 1432, 226, 0, 0, 0, 84, 1730, 1477, 541, 0, 0, 0, 0, 0, 0, 86, 6164, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5815, 6142, 5996, 6071, 7232, 6070, 5490, 6336] avg=6144.0000 max=7232 max_rank=4 min=5490 min_rank=6 imbalance_ratio=1.177083 max_load_reduction=41920 max_load_reduction_pct=0.852865 moved_to_echo=25128 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 3305, 1549, 1185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 6051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 0, 1596, 0, 0, 3500, 433, 0, 0, 0, 0, 0, 0, 0, 0, 1105, 591, 92, 0, 0, 0, 3450, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 1108, 5473, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1240, 0, 0, 9, 0, 0, 3522, 1309, 0, 0, 0, 0, 0, 0, 0, 1527, 215, 0, 0, 0, 76, 1758, 1541, 554, 0, 0, 0, 0, 0, 0, 92, 6312, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6039, 6072, 5661, 6305, 6828, 6080, 5671, 6496] avg=6144.0000 max=6828 max_rank=4 min=5661 min_rank=2 imbalance_ratio=1.111328 max_load_reduction=42324 max_load_reduction_pct=0.861084 moved_to_echo=25075 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 3433, 2000, 1100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 6172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 0, 0, 1555, 0, 0, 3977, 450, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 139, 62, 0, 0, 0, 3505, 1212, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 1259, 4049, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1291, 0, 0, 0, 0, 0, 3579, 1335, 0, 0, 0, 0, 0, 0, 0, 1368, 15, 0, 0, 0, 62, 3041, 1601, 513, 0, 0, 0, 0, 0, 0, 63, 5848, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6533, 6244, 6098, 5966, 5521, 6217, 6600, 5973] avg=6144.0000 max=6600 max_rank=6 min=5521 min_rank=4 imbalance_ratio=1.074219 max_load_reduction=42552 max_load_reduction_pct=0.865723 moved_to_echo=24311 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 3331, 1642, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 20, 0, 1, 4, 1, 6078, 0, 0, 0, 0, 0, 0, 0, 5, 0, 153, 0, 2, 2031, 0, 0, 3527, 442, 0, 0, 0, 0, 0, 0, 0, 0, 1125, 442, 90, 0, 4, 0, 3455, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 113, 0, 0, 1108, 5366, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1151, 0, 3, 15, 1, 3, 3528, 1315, 0, 0, 0, 0, 0, 0, 0, 1651, 330, 0, 0, 5, 104, 1622, 1558, 520, 0, 0, 0, 0, 0, 0, 90, 5891, 0, 3, 90, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6031, 6106, 6160, 6192, 6781, 6016, 5790, 6076] avg=6144.0000 max=6781 max_rank=4 min=5790 min_rank=6 imbalance_ratio=1.103678 max_load_reduction=42371 max_load_reduction_pct=0.862040 moved_to_echo=24663 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=21 rerouted_tokens_per_expert_current_ep_rank=[0, 3109, 1888, 1078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 6051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 0, 1878, 0, 0, 3628, 407, 0, 0, 0, 0, 0, 0, 0, 0, 1080, 393, 92, 0, 0, 0, 3472, 1105, 0, 0, 0, 0, 0, 0, 0, 0, 101, 0, 0, 1148, 5385, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1217, 0, 0, 0, 0, 0, 3547, 1310, 0, 0, 0, 0, 0, 0, 0, 1514, 271, 0, 0, 0, 91, 1783, 1449, 537, 0, 0, 0, 0, 0, 0, 92, 6117, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6075, 6060, 6057, 6142, 6798, 6074, 5645, 6301] avg=6144.0000 max=6798 max_rank=4 min=5645 min_rank=6 imbalance_ratio=1.106445 max_load_reduction=42354 max_load_reduction_pct=0.861694 moved_to_echo=24818 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 384, 9472, 128, 256, 256, 128, 384, 28032, 10368, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 11648, 1792, 128, 128, 512, 896, 18432, 12032, 4224, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[512, 384, 1152, 128, 256, 14592, 128, 0, 29312, 3456, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[128, 25728, 14720, 8704, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 128, 1280, 128, 128, 9344, 37376, 1536, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 0, 8960, 3072, 768, 128, 256, 128, 27392, 8960, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 128, 640, 256, 128, 128, 256, 48000, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[768, 47360, 128, 256, 768, 128, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=22 tokens_per_expert_current_ep_rank=[5273, 53, 1, 2, 0, 0, 2, 3452, 533, 88, 496, 6, 11, 0, 421, 5989, 0, 42, 8, 1546, 259, 403, 15, 518, 4, 1, 0, 0, 1, 10, 0, 16, 1204, 65, 29, 0, 1, 0, 0, 2228, 0, 80, 0, 7130, 34, 73, 332, 2, 1, 0, 0, 4255, 32, 7225, 0, 55, 0, 10, 4, 6990, 155, 14, 4, 79] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=22 tokens_per_expert_current_ep_rank=[4238, 0, 0, 0, 0, 2, 0, 4166, 553, 1, 96, 0, 0, 0, 378, 6786, 0, 0, 0, 2534, 284, 413, 0, 155, 1, 0, 0, 0, 0, 1, 0, 0, 1151, 0, 0, 0, 0, 0, 0, 1965, 0, 1, 0, 7433, 0, 1, 293, 0, 0, 0, 0, 4028, 1, 7716, 0, 1, 0, 0, 0, 6816, 71, 0, 0, 67] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=22 tokens_per_expert_current_ep_rank=[5764, 0, 0, 0, 0, 0, 0, 3231, 545, 0, 172, 0, 0, 0, 240, 6428, 0, 0, 0, 1247, 258, 393, 0, 412, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 0, 0, 0, 0, 0, 0, 2441, 0, 0, 0, 7272, 0, 0, 383, 0, 0, 0, 0, 4357, 0, 7478, 0, 0, 0, 0, 0, 7323, 113, 0, 0, 87] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=22 tokens_per_expert_current_ep_rank=[6027, 340, 45, 35, 25, 192, 49, 2508, 616, 206, 1455, 34, 69, 40, 396, 5944, 12, 75, 55, 875, 334, 415, 12, 674, 9, 10, 102, 18, 57, 70, 105, 76, 1876, 247, 125, 7, 23, 7, 3, 1717, 72, 126, 7, 6305, 130, 337, 319, 82, 14, 24, 2, 2626, 189, 6331, 0, 182, 9, 104, 23, 6779, 194, 94, 232, 86] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=22 tokens_per_expert_current_ep_rank=[4006, 0, 0, 0, 0, 0, 0, 4182, 512, 0, 87, 0, 0, 0, 322, 6661, 0, 0, 0, 2530, 246, 366, 0, 181, 0, 0, 0, 0, 0, 0, 0, 0, 1236, 0, 0, 0, 0, 0, 0, 1810, 0, 0, 0, 7470, 0, 0, 296, 0, 0, 0, 0, 4531, 0, 7786, 0, 0, 0, 0, 0, 6802, 66, 0, 0, 62] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=22 tokens_per_expert_current_ep_rank=[5580, 2, 0, 0, 0, 0, 0, 3221, 528, 5, 187, 0, 0, 0, 200, 6648, 0, 3, 0, 1191, 275, 441, 0, 553, 0, 0, 0, 0, 0, 1, 0, 1, 760, 2, 1, 0, 0, 0, 0, 2920, 0, 5, 0, 7272, 1, 4, 363, 0, 0, 0, 0, 3987, 2, 7378, 0, 4, 0, 2, 1, 7414, 108, 0, 0, 92] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=22 tokens_per_expert_current_ep_rank=[5648, 0, 0, 0, 0, 0, 0, 3100, 519, 0, 309, 0, 0, 0, 197, 6805, 0, 0, 0, 1150, 285, 429, 0, 390, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 0, 0, 0, 0, 0, 0, 3169, 0, 0, 0, 7327, 0, 0, 378, 0, 0, 0, 0, 3598, 0, 7414, 0, 0, 0, 0, 0, 7193, 96, 0, 0, 92] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=22 tokens_per_expert_current_ep_rank=[5423, 0, 0, 0, 0, 0, 0, 3166, 534, 0, 183, 0, 0, 0, 249, 6451, 0, 0, 0, 1507, 276, 472, 0, 330, 0, 0, 0, 0, 0, 0, 0, 0, 952, 0, 0, 0, 0, 0, 0, 2424, 0, 0, 0, 7398, 0, 0, 409, 0, 0, 0, 0, 4511, 0, 7471, 0, 0, 0, 0, 0, 7206, 98, 0, 0, 92] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=22 rerouted_tokens_per_expert_current_ep_rank=[2161, 0, 0, 0, 0, 2, 0, 4166, 0, 0, 0, 0, 0, 0, 0, 0, 553, 1, 96, 0, 0, 0, 378, 5114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2534, 284, 413, 0, 155, 1409, 1672, 113, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 5517, 668, 0, 0, 0, 0, 0, 0, 1151, 0, 0, 0, 0, 0, 0, 1965, 1458, 1136, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5862, 0, 1, 293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4028, 1, 2199, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5680, 71, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6329, 6142, 6580, 6187, 5710, 6157, 6229, 5818] avg=6144.0000 max=6580 max_rank=2 min=5710 min_rank=4 imbalance_ratio=1.070964 max_load_reduction=42572 max_load_reduction_pct=0.866130 moved_to_echo=23914 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=22 rerouted_tokens_per_expert_current_ep_rank=[2687, 53, 1, 2, 0, 0, 2, 3452, 0, 0, 0, 0, 0, 0, 0, 0, 533, 88, 496, 6, 11, 0, 421, 4513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 8, 1546, 259, 403, 15, 518, 1754, 1476, 109, 0, 0, 0, 0, 0, 4, 1, 0, 0, 1, 10, 0, 16, 5166, 832, 0, 0, 0, 0, 0, 0, 1204, 65, 29, 0, 1, 0, 0, 2228, 1398, 1165, 0, 0, 0, 0, 0, 0, 0, 80, 0, 5623, 34, 73, 332, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4255, 32, 2059, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4, 5825, 155, 14, 4, 79, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6197, 6068, 6130, 6030, 6090, 6144, 6402, 6091] avg=6144.0000 max=6402 max_rank=6 min=6030 min_rank=3 imbalance_ratio=1.041992 max_load_reduction=42750 max_load_reduction_pct=0.869751 moved_to_echo=24727 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=22 rerouted_tokens_per_expert_current_ep_rank=[2938, 0, 0, 0, 0, 0, 0, 3231, 0, 0, 0, 0, 0, 0, 0, 0, 545, 0, 172, 0, 0, 0, 240, 4844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1247, 258, 393, 0, 412, 1917, 1584, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5347, 909, 0, 0, 0, 0, 0, 0, 1008, 0, 0, 0, 0, 0, 0, 2441, 1426, 1221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5735, 0, 0, 383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4357, 0, 2131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6102, 113, 0, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6169, 5801, 5922, 6256, 6096, 6118, 6488, 6302] avg=6144.0000 max=6488 max_rank=6 min=5801 min_rank=1 imbalance_ratio=1.055990 max_load_reduction=42664 max_load_reduction_pct=0.868001 moved_to_echo=25004 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=22 rerouted_tokens_per_expert_current_ep_rank=[3071, 340, 45, 35, 25, 192, 49, 2508, 0, 0, 0, 0, 0, 0, 0, 0, 616, 206, 1455, 34, 69, 40, 396, 4479, 0, 0, 0, 0, 0, 0, 0, 0, 12, 75, 55, 875, 334, 415, 12, 674, 2005, 1465, 96, 0, 0, 0, 0, 0, 9, 10, 102, 18, 57, 70, 105, 76, 4527, 951, 0, 0, 0, 0, 0, 0, 1876, 247, 125, 7, 23, 7, 3, 1717, 1236, 1130, 0, 0, 0, 0, 0, 0, 72, 126, 7, 4973, 130, 337, 319, 82, 0, 0, 0, 0, 0, 0, 0, 0, 14, 24, 2, 2626, 189, 1804, 0, 182, 0, 0, 0, 0, 0, 0, 0, 0, 9, 104, 23, 5649, 194, 94, 232, 86, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6265, 7295, 6018, 5925, 6371, 6046, 4841, 6391] avg=6144.0000 max=7295 max_rank=1 min=4841 min_rank=6 imbalance_ratio=1.187337 max_load_reduction=41857 max_load_reduction_pct=0.851583 moved_to_echo=23649 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=22 rerouted_tokens_per_expert_current_ep_rank=[2879, 0, 0, 0, 0, 0, 0, 3100, 0, 0, 0, 0, 0, 0, 0, 0, 519, 0, 309, 0, 0, 0, 197, 5128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1150, 285, 429, 0, 390, 1878, 1677, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5301, 891, 0, 0, 0, 0, 0, 0, 1053, 0, 0, 0, 0, 0, 0, 3169, 1437, 1199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5778, 0, 0, 378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3598, 0, 2113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5994, 96, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5979, 6153, 5921, 6192, 6858, 6156, 5711, 6182] avg=6144.0000 max=6858 max_rank=4 min=5711 min_rank=6 imbalance_ratio=1.116211 max_load_reduction=42294 max_load_reduction_pct=0.860474 moved_to_echo=24907 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=22 rerouted_tokens_per_expert_current_ep_rank=[2042, 0, 0, 0, 0, 0, 0, 4182, 0, 0, 0, 0, 0, 0, 0, 0, 512, 0, 87, 0, 0, 0, 322, 5019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2530, 246, 366, 0, 181, 1332, 1642, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5567, 632, 0, 0, 0, 0, 0, 0, 1236, 0, 0, 0, 0, 0, 0, 1810, 1465, 1134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5891, 0, 0, 296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4531, 0, 2219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5668, 66, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6224, 5940, 6411, 6199, 5645, 6187, 6750, 5796] avg=6144.0000 max=6750 max_rank=6 min=5645 min_rank=4 imbalance_ratio=1.098633 max_load_reduction=42402 max_load_reduction_pct=0.862671 moved_to_echo=24378 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=22 rerouted_tokens_per_expert_current_ep_rank=[2836, 2, 0, 0, 0, 0, 0, 3221, 0, 0, 0, 0, 0, 0, 0, 0, 528, 5, 187, 0, 0, 0, 200, 5006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1191, 275, 441, 0, 553, 1860, 1642, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 5280, 884, 0, 0, 0, 0, 0, 0, 760, 2, 1, 0, 0, 0, 0, 2920, 1431, 1240, 0, 0, 0, 0, 0, 0, 0, 5, 0, 5728, 1, 4, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3987, 2, 2098, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 6174, 108, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6059, 5926, 6078, 6166, 6354, 6101, 6091, 6377] avg=6144.0000 max=6377 max_rank=7 min=5926 min_rank=1 imbalance_ratio=1.037923 max_load_reduction=42775 max_load_reduction_pct=0.870260 moved_to_echo=24923 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=22 rerouted_tokens_per_expert_current_ep_rank=[2764, 0, 0, 0, 0, 0, 0, 3166, 0, 0, 0, 0, 0, 0, 0, 0, 534, 0, 183, 0, 0, 0, 249, 4861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1507, 276, 472, 0, 330, 1804, 1590, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5342, 855, 0, 0, 0, 0, 0, 0, 952, 0, 0, 0, 0, 0, 0, 2424, 1451, 1201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5834, 0, 0, 409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4511, 0, 2129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6005, 98, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5930, 5827, 6092, 6197, 6028, 6243, 6640, 6195] avg=6144.0000 max=6640 max_rank=6 min=5827 min_rank=1 imbalance_ratio=1.080729 max_load_reduction=42512 max_load_reduction_pct=0.864909 moved_to_echo=25106 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[128, 128, 128, 12672, 2304, 3456, 128, 3328, 14080, 12800, 896, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[9344, 384, 256, 128, 128, 128, 128, 18688, 11392, 9472, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 128, 32000, 256, 16768, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[21504, 512, 128, 128, 128, 256, 128, 27136, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 128, 42112, 6656, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4352, 384, 3072, 128, 128, 128, 2432, 39040, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 256, 128, 45440, 256, 512, 2816, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[128, 128, 128, 47104, 1024, 128, 256, 768, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 472, 0, 0, 0, 0, 0, 0, 251, 0, 0, 7168, 0, 5671, 0, 87, 331, 68, 0, 0, 0, 1110, 0, 2507, 0, 0, 0, 0, 0, 0, 6716, 1715, 0, 217, 0, 0, 0, 691, 0, 3441, 5247, 0, 0, 0, 6485, 5245, 59, 0, 0, 0, 0, 251, 0, 87, 0, 0, 0, 1333, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 632, 1, 0, 2, 0, 0, 1, 254, 0, 0, 7231, 0, 5452, 0, 56, 342, 107, 0, 0, 0, 1038, 0, 1632, 0, 1, 0, 0, 0, 0, 6669, 3186, 0, 230, 0, 1, 0, 805, 0, 2730, 6091, 0, 0, 1, 6509, 4484, 68, 1, 0, 0, 0, 254, 0, 67, 0, 0, 1, 1306, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=23 tokens_per_expert_current_ep_rank=[8, 1, 537, 22, 8, 0, 8, 0, 66, 261, 1, 0, 6800, 1, 5440, 5, 82, 349, 101, 0, 0, 0, 819, 2, 2554, 13, 14, 3, 1, 0, 0, 6454, 2089, 1, 219, 0, 80, 0, 967, 0, 3734, 5569, 5, 32, 87, 6358, 4143, 61, 88, 9, 0, 14, 453, 0, 84, 0, 0, 82, 1427, 67, 26, 4, 0, 3] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=23 tokens_per_expert_current_ep_rank=[55, 18, 698, 5, 17, 97, 147, 47, 64, 319, 46, 2, 5350, 13, 5512, 7, 179, 340, 131, 3, 4, 133, 680, 34, 3388, 109, 96, 109, 108, 25, 44, 5724, 1465, 28, 189, 0, 197, 4, 827, 147, 3825, 5240, 75, 99, 229, 5895, 2937, 85, 255, 56, 23, 245, 1053, 5, 123, 0, 22, 194, 1908, 242, 135, 46, 26, 73] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 675, 0, 0, 0, 0, 0, 0, 254, 0, 0, 7303, 0, 5338, 0, 23, 422, 60, 0, 0, 0, 1049, 0, 1423, 0, 0, 0, 0, 0, 0, 6691, 3470, 0, 220, 0, 0, 0, 884, 0, 2877, 6157, 0, 0, 0, 6629, 3988, 59, 0, 0, 0, 0, 255, 0, 80, 0, 0, 0, 1295, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 548, 0, 0, 0, 0, 0, 0, 227, 0, 0, 7196, 0, 5408, 0, 92, 340, 58, 0, 0, 0, 845, 0, 3019, 0, 0, 0, 0, 0, 0, 6725, 1989, 0, 307, 0, 0, 0, 681, 0, 3183, 5526, 0, 0, 0, 6452, 4646, 73, 0, 0, 0, 0, 229, 0, 92, 0, 0, 0, 1516, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 433, 0, 0, 0, 0, 0, 0, 252, 0, 0, 7141, 0, 5562, 0, 92, 318, 80, 0, 0, 0, 1324, 0, 2710, 0, 0, 0, 0, 0, 0, 6711, 1795, 0, 306, 0, 0, 0, 687, 0, 3941, 4912, 0, 0, 0, 6429, 4669, 67, 0, 0, 0, 0, 252, 0, 92, 0, 0, 0, 1379, 0, 0, 0, 0, 0] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=23 tokens_per_expert_current_ep_rank=[0, 0, 497, 2, 0, 0, 1, 0, 2, 247, 0, 0, 7141, 0, 5956, 0, 92, 351, 84, 0, 0, 0, 864, 1, 2646, 0, 2, 0, 0, 0, 0, 6716, 1600, 0, 260, 0, 5, 0, 585, 0, 3608, 5325, 0, 3, 5, 6459, 4749, 53, 5, 0, 0, 0, 253, 0, 93, 0, 0, 4, 1539, 3, 0, 0, 0, 1] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 472, 0, 0, 0, 0, 0, 4579, 1043, 0, 0, 0, 0, 0, 0, 0, 251, 0, 0, 330, 0, 5671, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 331, 68, 0, 0, 0, 1110, 0, 2620, 1906, 0, 0, 0, 0, 0, 0, 2507, 0, 0, 0, 0, 0, 0, 3685, 0, 0, 0, 0, 0, 0, 0, 0, 1715, 0, 217, 0, 0, 0, 691, 0, 2122, 1178, 0, 0, 0, 0, 0, 0, 3441, 0, 0, 0, 0, 0, 3123, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 0, 87, 0, 5795, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1333, 0, 0, 0, 0, 0, 2627, 1853, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6094, 6252, 6122, 6192, 5923, 6623, 6133, 5813] avg=6144.0000 max=6623 max_rank=5 min=5813 min_rank=7 imbalance_ratio=1.077962 max_load_reduction=42529 max_load_reduction_pct=0.865255 moved_to_echo=24492 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 632, 1, 0, 2, 0, 0, 4596, 1052, 0, 0, 0, 0, 0, 0, 1, 254, 0, 0, 333, 0, 5452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 342, 107, 0, 0, 0, 1038, 0, 3042, 1913, 0, 0, 0, 0, 0, 0, 1632, 0, 1, 0, 0, 0, 0, 3659, 0, 0, 0, 0, 0, 0, 0, 0, 3186, 0, 230, 0, 1, 0, 805, 0, 1814, 1170, 0, 0, 0, 0, 0, 0, 2730, 0, 0, 0, 1, 0, 2670, 68, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 254, 0, 67, 0, 5846, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1306, 0, 0, 0, 0, 0, 3049, 1840, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6283, 6040, 6498, 5292, 7206, 5469, 6168, 6196] avg=6144.0000 max=7206 max_rank=4 min=5292 min_rank=3 imbalance_ratio=1.172852 max_load_reduction=41946 max_load_reduction_pct=0.853394 moved_to_echo=25039 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=23 rerouted_tokens_per_expert_current_ep_rank=[55, 18, 698, 5, 17, 97, 147, 47, 4163, 779, 0, 0, 0, 0, 0, 0, 64, 319, 46, 2, 246, 13, 5512, 7, 0, 0, 0, 0, 0, 0, 0, 0, 179, 340, 131, 3, 4, 133, 680, 34, 2617, 1732, 0, 0, 0, 0, 0, 0, 3388, 109, 96, 109, 108, 25, 44, 3141, 0, 0, 0, 0, 0, 0, 0, 0, 1465, 28, 189, 0, 197, 4, 827, 147, 1188, 1004, 0, 0, 0, 0, 0, 0, 3825, 0, 75, 99, 229, 0, 1749, 85, 0, 0, 0, 0, 0, 0, 0, 0, 255, 56, 23, 245, 1053, 5, 123, 0, 4325, 0, 0, 0, 0, 0, 0, 0, 22, 194, 1908, 242, 135, 46, 26, 73, 2623, 1579, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6026, 6209, 5853, 7020, 5049, 6062, 6085, 6848] avg=6144.0000 max=7020 max_rank=3 min=5049 min_rank=4 imbalance_ratio=1.142578 max_load_reduction=42132 max_load_reduction_pct=0.857178 moved_to_echo=24044 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=23 rerouted_tokens_per_expert_current_ep_rank=[8, 1, 537, 22, 8, 0, 8, 0, 4489, 990, 0, 0, 0, 0, 0, 0, 66, 261, 1, 0, 313, 1, 5440, 5, 0, 0, 0, 0, 0, 0, 0, 0, 82, 349, 101, 0, 0, 0, 819, 2, 2781, 1869, 0, 0, 0, 0, 0, 0, 2554, 13, 14, 3, 1, 0, 0, 3541, 0, 0, 0, 0, 0, 0, 0, 0, 2089, 1, 219, 0, 80, 0, 967, 0, 1676, 1132, 0, 0, 0, 0, 0, 0, 3734, 0, 5, 32, 87, 0, 2467, 61, 0, 0, 0, 0, 0, 0, 0, 0, 88, 9, 0, 14, 453, 0, 84, 0, 5497, 0, 0, 0, 0, 0, 0, 0, 0, 82, 1427, 67, 26, 4, 0, 3, 2788, 1781, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6063, 6087, 6003, 6126, 6164, 6386, 6145, 6178] avg=6144.0000 max=6386 max_rank=5 min=6003 min_rank=2 imbalance_ratio=1.039388 max_load_reduction=42766 max_load_reduction_pct=0.870076 moved_to_echo=24873 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 548, 0, 0, 0, 0, 0, 4556, 1047, 0, 0, 0, 0, 0, 0, 0, 227, 0, 0, 332, 0, 5408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 340, 58, 0, 0, 0, 845, 0, 2760, 1896, 0, 0, 0, 0, 0, 0, 3019, 0, 0, 0, 0, 0, 0, 3690, 0, 0, 0, 0, 0, 0, 0, 0, 1989, 0, 307, 0, 0, 0, 681, 0, 1880, 1180, 0, 0, 0, 0, 0, 0, 3183, 0, 0, 0, 0, 0, 2766, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 229, 0, 92, 0, 5817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1516, 0, 0, 0, 0, 0, 2766, 1855, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6151, 5967, 5991, 6709, 6037, 6022, 6138, 6137] avg=6144.0000 max=6709 max_rank=3 min=5967 min_rank=1 imbalance_ratio=1.091960 max_load_reduction=42443 max_load_reduction_pct=0.863505 moved_to_echo=24334 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 675, 0, 0, 0, 0, 0, 4681, 1063, 0, 0, 0, 0, 0, 0, 0, 254, 0, 0, 336, 0, 5338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 422, 60, 0, 0, 0, 1049, 0, 3076, 1948, 0, 0, 0, 0, 0, 0, 1423, 0, 0, 0, 0, 0, 0, 3671, 0, 0, 0, 0, 0, 0, 0, 0, 3470, 0, 220, 0, 0, 0, 884, 0, 1614, 1174, 0, 0, 0, 0, 0, 0, 2877, 0, 0, 0, 0, 0, 2374, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 0, 80, 0, 5904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1295, 0, 0, 0, 0, 0, 3081, 1846, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6419, 5928, 6578, 5094, 7362, 5310, 6239, 6222] avg=6144.0000 max=7362 max_rank=4 min=5094 min_rank=3 imbalance_ratio=1.198242 max_load_reduction=41790 max_load_reduction_pct=0.850220 moved_to_echo=25133 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 497, 2, 0, 0, 1, 0, 4561, 1044, 0, 0, 0, 0, 0, 0, 2, 247, 0, 0, 321, 0, 5956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 351, 84, 0, 0, 0, 864, 1, 2660, 1898, 0, 0, 0, 0, 0, 0, 2646, 0, 2, 0, 0, 0, 0, 3678, 0, 0, 0, 0, 0, 0, 0, 0, 1600, 0, 260, 0, 5, 0, 585, 0, 1926, 1181, 0, 0, 0, 0, 0, 0, 3608, 0, 0, 3, 5, 0, 2823, 53, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 253, 0, 93, 0, 5776, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1539, 3, 0, 0, 0, 1, 2665, 1857, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6105, 6526, 5950, 6326, 5557, 6492, 6127, 6069] avg=6144.0000 max=6526 max_rank=1 min=5557 min_rank=4 imbalance_ratio=1.062174 max_load_reduction=42626 max_load_reduction_pct=0.867228 moved_to_echo=24245 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=23 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 433, 0, 0, 0, 0, 0, 4539, 1039, 0, 0, 0, 0, 0, 0, 0, 252, 0, 0, 329, 0, 5562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 318, 80, 0, 0, 0, 1324, 0, 2453, 1890, 0, 0, 0, 0, 0, 0, 2710, 0, 0, 0, 0, 0, 0, 3683, 0, 0, 0, 0, 0, 0, 0, 0, 1795, 0, 306, 0, 0, 0, 687, 0, 1889, 1177, 0, 0, 0, 0, 0, 0, 3941, 0, 0, 0, 0, 0, 2780, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 252, 0, 92, 0, 5773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1379, 0, 0, 0, 0, 0, 2459, 1851, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6011, 6143, 6157, 6393, 5854, 6788, 6117, 5689] avg=6144.0000 max=6788 max_rank=5 min=5689 min_rank=7 imbalance_ratio=1.104818 max_load_reduction=42364 max_load_reduction_pct=0.861898 moved_to_echo=24448 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[17408, 128, 2048, 0, 384, 128, 6144, 256, 14208, 9216, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 128, 4608, 128, 128, 128, 256, 128, 36224, 8064, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[19968, 128, 128, 128, 128, 128, 128, 28800, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[27392, 0, 128, 256, 384, 0, 20864, 640, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 384, 11776, 384, 256, 128, 128, 128, 22144, 14464, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[768, 2816, 768, 128, 128, 256, 7808, 128, 22016, 15104, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[256, 2176, 128, 128, 2560, 128, 44416, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[384, 128, 128, 384, 3072, 128, 768, 0, 44800, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=24 tokens_per_expert_current_ep_rank=[0, 68, 0, 0, 0, 0, 214, 125, 86, 7476, 184, 0, 0, 86, 0, 78, 2, 0, 0, 107, 0, 187, 0, 0, 0, 0, 0, 0, 859, 7362, 0, 0, 0, 0, 0, 0, 375, 6994, 0, 8106, 87, 0, 0, 3725, 97, 0, 271, 0, 0, 0, 0, 0, 0, 3571, 0, 0, 0, 0, 412, 7842, 0, 0, 0, 838] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=24 tokens_per_expert_current_ep_rank=[0, 268, 0, 1, 1, 1, 331, 7, 66, 7328, 331, 0, 0, 67, 0, 75, 31, 0, 0, 77, 1, 226, 1, 0, 0, 0, 0, 0, 768, 7303, 0, 0, 0, 1, 0, 0, 419, 7054, 0, 8123, 67, 0, 0, 4100, 82, 0, 78, 0, 0, 0, 0, 0, 0, 3420, 0, 0, 0, 0, 380, 7838, 0, 0, 0, 707] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=24 tokens_per_expert_current_ep_rank=[0, 119, 51, 78, 92, 50, 301, 384, 79, 7323, 196, 0, 0, 105, 5, 93, 57, 16, 0, 246, 82, 256, 67, 0, 1, 14, 0, 0, 929, 6669, 68, 0, 1, 87, 0, 0, 593, 6247, 8, 8013, 79, 2, 2, 4391, 417, 8, 354, 1, 30, 0, 1, 0, 1, 2310, 0, 0, 0, 0, 488, 7548, 1, 0, 6, 1283] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=24 tokens_per_expert_current_ep_rank=[0, 60, 0, 0, 0, 0, 274, 195, 92, 7499, 204, 0, 0, 92, 0, 76, 24, 0, 0, 271, 0, 277, 0, 0, 0, 0, 0, 0, 860, 7059, 0, 0, 0, 0, 0, 0, 185, 7010, 0, 8100, 92, 0, 0, 3674, 92, 0, 283, 0, 0, 0, 0, 0, 0, 3463, 0, 0, 0, 0, 456, 7815, 0, 0, 0, 999] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=24 tokens_per_expert_current_ep_rank=[0, 208, 0, 0, 0, 0, 278, 44, 62, 7408, 340, 0, 0, 62, 0, 52, 39, 0, 0, 39, 0, 184, 0, 0, 0, 0, 0, 0, 683, 7357, 0, 0, 0, 0, 0, 0, 405, 7093, 0, 8130, 62, 0, 0, 3761, 71, 0, 78, 0, 0, 0, 0, 0, 0, 3851, 0, 0, 0, 0, 389, 7884, 0, 0, 0, 672] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=24 tokens_per_expert_current_ep_rank=[0, 279, 200, 93, 282, 52, 447, 540, 87, 6299, 268, 13, 39, 160, 112, 225, 73, 21, 0, 470, 165, 452, 186, 0, 48, 115, 133, 69, 1131, 5481, 240, 3, 256, 197, 0, 41, 696, 5318, 51, 7451, 94, 14, 105, 3842, 1014, 23, 622, 37, 127, 23, 58, 5, 16, 1591, 5, 0, 179, 2, 718, 6437, 24, 32, 28, 2463] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=24 tokens_per_expert_current_ep_rank=[0, 77, 3, 5, 5, 4, 275, 392, 90, 7493, 47, 0, 0, 90, 0, 100, 11, 0, 0, 148, 4, 194, 3, 0, 0, 1, 0, 0, 954, 6937, 4, 0, 0, 5, 0, 0, 335, 6967, 0, 8097, 92, 0, 0, 3606, 129, 0, 278, 0, 1, 0, 0, 0, 0, 3557, 0, 0, 0, 0, 465, 7813, 0, 0, 0, 970] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=24 tokens_per_expert_current_ep_rank=[0, 146, 0, 0, 0, 0, 265, 151, 92, 7487, 186, 0, 0, 92, 0, 83, 2, 0, 0, 165, 0, 192, 0, 0, 0, 0, 0, 0, 895, 7228, 0, 0, 0, 0, 0, 0, 312, 6918, 0, 8100, 92, 0, 0, 3097, 105, 0, 211, 0, 0, 0, 0, 0, 0, 4153, 0, 0, 0, 0, 483, 7816, 0, 0, 0, 881] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 68, 0, 0, 0, 0, 214, 125, 3026, 2432, 0, 0, 0, 0, 0, 0, 86, 5775, 184, 0, 0, 86, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 107, 0, 187, 0, 0, 5674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 859, 5499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 375, 5896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 3725, 97, 0, 271, 0, 1098, 705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3571, 0, 0, 1863, 996, 151, 0, 0, 0, 0, 0, 0, 0, 412, 4665, 0, 0, 0, 838, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5865, 6209, 5970, 6358, 6271, 5983, 6581, 5915] avg=6144.0000 max=6581 max_rank=6 min=5865 min_rank=0 imbalance_ratio=1.071126 max_load_reduction=42571 max_load_reduction_pct=0.866109 moved_to_echo=24750 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 268, 0, 1, 1, 1, 331, 7, 3025, 2437, 0, 0, 0, 0, 0, 0, 66, 5660, 331, 0, 0, 67, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 77, 1, 226, 1, 0, 5686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 768, 5455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 419, 5947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 4100, 82, 0, 78, 0, 1107, 691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3420, 0, 0, 1848, 977, 150, 0, 0, 0, 0, 0, 0, 0, 380, 4663, 0, 0, 0, 707, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6071, 6199, 6022, 6223, 6367, 6125, 6395, 5750] avg=6144.0000 max=6395 max_rank=6 min=5750 min_rank=7 imbalance_ratio=1.040853 max_load_reduction=42757 max_load_reduction_pct=0.869893 moved_to_echo=24637 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 119, 51, 78, 92, 50, 301, 384, 2913, 2404, 0, 0, 0, 0, 0, 0, 79, 5657, 196, 0, 0, 105, 5, 93, 0, 0, 0, 0, 0, 0, 0, 0, 57, 16, 0, 246, 82, 256, 67, 0, 5609, 0, 0, 0, 0, 0, 0, 0, 1, 14, 0, 0, 929, 4982, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 87, 0, 0, 593, 5267, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 2, 2, 4391, 417, 8, 354, 1, 980, 690, 0, 0, 0, 0, 0, 0, 30, 0, 1, 0, 1, 2310, 0, 0, 1687, 976, 145, 0, 0, 0, 0, 0, 0, 0, 488, 4490, 1, 0, 6, 1283, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6392, 6135, 6333, 5994, 5956, 6924, 5150, 6268] avg=6144.0000 max=6924 max_rank=5 min=5150 min_rank=6 imbalance_ratio=1.126953 max_load_reduction=42228 max_load_reduction_pct=0.859131 moved_to_echo=24298 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 279, 200, 93, 282, 52, 447, 540, 2484, 2236, 0, 0, 0, 0, 0, 0, 87, 4866, 268, 13, 39, 160, 112, 225, 0, 0, 0, 0, 0, 0, 0, 0, 73, 21, 0, 470, 165, 452, 186, 0, 5215, 0, 0, 0, 0, 0, 0, 0, 48, 115, 133, 69, 1131, 4094, 240, 3, 0, 0, 0, 0, 0, 0, 0, 0, 256, 197, 0, 41, 696, 4484, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 14, 105, 3842, 1014, 23, 622, 37, 834, 594, 0, 0, 0, 0, 0, 0, 127, 23, 58, 5, 16, 1591, 5, 0, 1387, 839, 124, 0, 0, 0, 0, 0, 179, 2, 718, 3829, 24, 32, 28, 2463, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6613, 5770, 6582, 5833, 5725, 7179, 4175, 7275] avg=6144.0000 max=7275 max_rank=7 min=4175 min_rank=6 imbalance_ratio=1.184082 max_load_reduction=41877 max_load_reduction_pct=0.851990 moved_to_echo=24354 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 60, 0, 0, 0, 0, 274, 195, 3016, 2432, 0, 0, 0, 0, 0, 0, 92, 5793, 204, 0, 0, 92, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 271, 0, 277, 0, 0, 5668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 860, 5273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, 5910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 3674, 92, 0, 283, 0, 1100, 707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3463, 0, 0, 1786, 999, 150, 0, 0, 0, 0, 0, 0, 0, 456, 4649, 0, 0, 0, 999, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5977, 6257, 6240, 6133, 6095, 5948, 6398, 6104] avg=6144.0000 max=6398 max_rank=6 min=5948 min_rank=5 imbalance_ratio=1.041341 max_load_reduction=42754 max_load_reduction_pct=0.869832 moved_to_echo=24545 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 208, 0, 0, 0, 0, 278, 44, 3043, 2441, 0, 0, 0, 0, 0, 0, 62, 5723, 340, 0, 0, 62, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 39, 0, 184, 0, 0, 5689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 683, 5496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 405, 5980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 3761, 71, 0, 78, 0, 1113, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3851, 0, 0, 1861, 987, 151, 0, 0, 0, 0, 0, 0, 0, 389, 4690, 0, 0, 0, 672, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6014, 6239, 5951, 6179, 6385, 5783, 6850, 5751] avg=6144.0000 max=6850 max_rank=6 min=5751 min_rank=7 imbalance_ratio=1.114909 max_load_reduction=42302 max_load_reduction_pct=0.860636 moved_to_echo=24769 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 77, 3, 5, 5, 4, 275, 392, 3019, 2431, 0, 0, 0, 0, 0, 0, 90, 5780, 47, 0, 0, 90, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 148, 4, 194, 3, 0, 5666, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 954, 5179, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 335, 5869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 3606, 129, 0, 278, 0, 1098, 710, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3557, 0, 0, 1758, 1003, 154, 0, 0, 0, 0, 0, 0, 0, 465, 4640, 0, 0, 0, 970, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6211, 6107, 6026, 6138, 6209, 5913, 6473, 6075] avg=6144.0000 max=6473 max_rank=6 min=5913 min_rank=5 imbalance_ratio=1.053548 max_load_reduction=42679 max_load_reduction_pct=0.868306 moved_to_echo=24670 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=24 rerouted_tokens_per_expert_current_ep_rank=[0, 146, 0, 0, 0, 0, 265, 151, 3016, 2431, 0, 0, 0, 0, 0, 0, 92, 5783, 186, 0, 0, 92, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 165, 0, 192, 0, 0, 5669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 895, 5399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 312, 5832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 3097, 105, 0, 211, 0, 1086, 706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4153, 0, 0, 1829, 998, 150, 0, 0, 0, 0, 0, 0, 0, 483, 4650, 0, 0, 0, 881, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6009, 6236, 6028, 6294, 6144, 5297, 7130, 6014] avg=6144.0000 max=7130 max_rank=6 min=5297 min_rank=5 imbalance_ratio=1.160482 max_load_reduction=42022 max_load_reduction_pct=0.854940 moved_to_echo=24585 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=98304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 1280, 256, 256, 384, 128, 2432, 1920, 23552, 19328, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=98304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[256, 128, 128, 128, 128, 25984, 128, 0, 14080, 7808, 1280, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[768, 128, 128, 30208, 2048, 128, 2176, 128, 8448, 5504, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=98304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[256, 128, 3840, 36352, 128, 128, 128, 8832, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[768, 45056, 1792, 128, 128, 768, 128, 896, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=98304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[256, 128, 0, 1536, 256, 2048, 384, 0, 44928, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=98304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[128, 256, 256, 128, 7168, 41472, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=98304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[384, 384, 0, 128, 3328, 45312, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=98304 valid_num_tokens=50048 slack_slots=48256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=98304 valid_num_tokens=49792 slack_slots=48512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=98304 valid_num_tokens=49536 slack_slots=48768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=98304 valid_num_tokens=49920 slack_slots=48384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=98304 valid_num_tokens=49664 slack_slots=48640 device_initiated_grouped_gemm=True - [2026-08-26 14:53:51] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 821.1 | throughput per GPU (TFLOP/s/GPU): 155.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.509861E+00 | loss scale: 1.0 | grad norm: 83.626 | number of skipped iterations: 0 | number of nan iterations: 0 | -[after training is done] datetime: 2026-08-26 14:53:51 -[rank0]:[W826 14:53:53.072640381 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -W0826 14:53:53.478000 27578 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers -W0826 14:53:53.479000 27578 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 27669 closing signal SIGTERM -W0826 14:53:53.479000 27578 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 27670 closing signal SIGTERM -W0826 14:53:53.480000 27578 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 27671 closing signal SIGTERM -W0826 14:53:53.481000 27578 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 27672 closing signal SIGTERM -W0826 14:53:53.482000 27578 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 27673 closing signal SIGTERM -W0826 14:53:53.482000 27578 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 27674 closing signal SIGTERM -W0826 14:53:53.483000 27578 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 27675 closing signal SIGTERM -W0826 14:53:53.483000 27578 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 27676 closing signal SIGTERM +E0826 15:29:07.906000 93676 torch/distributed/elastic/multiprocessing/api.py:874] failed (exitcode: -9) local_rank: 0 (pid: 93774) of binary: /usr/bin/python Traceback (most recent call last): File "/usr/local/bin/torchrun", line 8, in sys.exit(main()) @@ -12635,17 +1422,69 @@ Traceback (most recent call last): File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ return launch_agent(self._config, self._entrypoint, list(args)) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent - result = agent.run() - ^^^^^^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper - result = f(*args, **kwargs) - ^^^^^^^^^^^^^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run - result = self._invoke_run(role) - ^^^^^^^^^^^^^^^^^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run - time.sleep(monitor_interval) - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler - raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 27578 got signal: 15 + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +pretrain_gpt.py FAILED +------------------------------------------------------------ +Failures: +[1]: + time : 2026-08-26_15:29:07 + host : sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 + rank : 1 (local_rank: 1) + exitcode : -9 (pid: 93775) + error_file: + traceback : Signal 9 (SIGKILL) received by PID 93775 +[2]: + time : 2026-08-26_15:29:07 + host : sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 + rank : 2 (local_rank: 2) + exitcode : -9 (pid: 93776) + error_file: + traceback : Signal 9 (SIGKILL) received by PID 93776 +[3]: + time : 2026-08-26_15:29:07 + host : sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 + rank : 3 (local_rank: 3) + exitcode : -9 (pid: 93777) + error_file: + traceback : Signal 9 (SIGKILL) received by PID 93777 +[4]: + time : 2026-08-26_15:29:07 + host : sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 + rank : 4 (local_rank: 4) + exitcode : -9 (pid: 93778) + error_file: + traceback : Signal 9 (SIGKILL) received by PID 93778 +[5]: + time : 2026-08-26_15:29:07 + host : sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 + rank : 5 (local_rank: 5) + exitcode : -9 (pid: 93779) + error_file: + traceback : Signal 9 (SIGKILL) received by PID 93779 +[6]: + time : 2026-08-26_15:29:07 + host : sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 + rank : 6 (local_rank: 6) + exitcode : -9 (pid: 93780) + error_file: + traceback : Signal 9 (SIGKILL) received by PID 93780 +[7]: + time : 2026-08-26_15:29:07 + host : sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 + rank : 7 (local_rank: 7) + exitcode : -9 (pid: 93781) + error_file: + traceback : Signal 9 (SIGKILL) received by PID 93781 +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-08-26_15:29:07 + host : sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 + rank : 0 (local_rank: 0) + exitcode : -9 (pid: 93774) + error_file: + traceback : Signal 9 (SIGKILL) received by PID 93774 +============================================================ diff --git a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1/mem_samples.csv b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1/mem_samples.csv index 488a3d27a783ca22c565fb5608022bccb1f19ae8..5e39d6e2b2e6bc5a833b8ad812ae306338f5199c 100644 --- a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1/mem_samples.csv +++ b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1/mem_samples.csv @@ -14,7 +14,7 @@ 5, 0 6, 0 7, 0 -0, 1 +0, 2 1, 1 2, 1 3, 1 @@ -54,46 +54,38 @@ 5, 821 6, 821 7, 821 -0, 1975 -1, 1975 -2, 1975 -3, 1975 -4, 1975 -5, 1975 -6, 1975 -7, 1975 -0, 2615 -1, 2615 -2, 2615 -3, 2615 -4, 2615 -5, 2615 -6, 2615 -7, 2615 -0, 47803 -1, 47803 -2, 47803 -3, 47803 -4, 47515 -5, 47803 -6, 47803 -7, 47803 -0, 49055 -1, 49055 -2, 49055 -3, 49055 -4, 48767 -5, 49055 -6, 49055 -7, 49055 -0, 49415 -1, 49425 -2, 49407 -3, 49423 -4, 49133 -5, 49421 -6, 49453 -7, 49421 +0, 1537 +1, 1537 +2, 1537 +3, 1537 +4, 1537 +5, 1537 +6, 1537 +7, 1537 +0, 2551 +1, 2551 +2, 2551 +3, 2551 +4, 2551 +5, 2551 +6, 2551 +7, 2551 +0, 48087 +1, 48119 +2, 48077 +3, 48113 +4, 47797 +5, 48139 +6, 48097 +7, 48119 +0, 49843 +1, 49833 +2, 49693 +3, 49693 +4, 49555 +5, 49843 +6, 49867 +7, 49863 0, 53697 1, 53697 2, 53697 @@ -110,22 +102,14 @@ 5, 54105 6, 54105 7, 54105 -0, 54105 -1, 54105 -2, 54105 -3, 54105 -4, 53817 -5, 54105 -6, 54105 -7, 54105 -0, 54943 -1, 54943 -2, 54943 -3, 54943 -4, 54655 -5, 54943 -6, 54943 -7, 54943 +0, 54405 +1, 54393 +2, 54417 +3, 54395 +4, 54107 +5, 54413 +6, 54409 +7, 54413 0, 55543 1, 55543 2, 55543 @@ -142,6 +126,14 @@ 5, 56567 6, 56567 7, 56567 +0, 58377 +1, 58377 +2, 58377 +3, 58377 +4, 58089 +5, 58377 +6, 58377 +7, 58377 0, 59799 1, 59799 2, 59799 @@ -150,22 +142,14 @@ 5, 59799 6, 59799 7, 59799 -0, 60573 -1, 60573 -2, 60573 -3, 60573 -4, 60285 -5, 60573 -6, 60573 -7, 60573 -0, 105833 -1, 105883 -2, 105879 -3, 105895 -4, 105891 -5, 105879 -6, 105919 -7, 105879 +0, 105853 +1, 105869 +2, 105869 +3, 105869 +4, 105821 +5, 105827 +6, 105825 +7, 105843 0, 106307 1, 106347 2, 106347 @@ -198,59 +182,75 @@ 5, 106349 6, 106349 7, 106349 -0, 106715 -1, 106757 -2, 106669 -3, 106763 -4, 106745 -5, 106729 -6, 106747 -7, 106751 -0, 110575 -1, 111009 -2, 110639 -3, 110625 -4, 110609 -5, 111001 -6, 110659 -7, 110985 -0, 111399 -1, 111823 -2, 111439 -3, 111439 -4, 111419 -5, 111823 -6, 111439 -7, 111823 -0, 141439 -1, 141943 +0, 110361 +1, 110401 +2, 110017 +3, 109633 +4, 109613 +5, 108481 +6, 109633 +7, 109633 +0, 111393 +1, 111049 +2, 111049 +3, 111433 +4, 111029 +5, 111433 +6, 111049 +7, 111433 +0, 112049 +1, 111699 +2, 111733 +3, 112073 +4, 111651 +5, 112085 +6, 111709 +7, 112061 +0, 141823 +1, 141559 2, 141559 -3, 141559 +3, 141943 4, 141539 5, 141943 6, 141559 7, 141943 -0, 141439 -1, 141943 +0, 141823 +1, 141559 2, 141559 -3, 141559 +3, 141943 4, 141539 5, 141943 6, 141559 7, 141943 -0, 141439 -1, 141943 +0, 141823 +1, 141559 2, 141559 -3, 141559 +3, 141943 4, 141539 5, 141943 6, 141559 7, 141943 -0, 3366 -1, 3622 -2, 140311 -3, 4646 -4, 3366 -5, 3366 -6, 3366 +0, 3622 +1, 140311 +2, 4646 +3, 3366 +4, 3622 +5, 140695 +6, 4646 7, 3366 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 diff --git a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1/train.log b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1/train.log index d473befa58cd58b382918d5f5824bd86f79d0a70..adf3001c0e49b9f24bfed24e6661f457360437b4 100644 --- a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1/train.log +++ b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1/train.log @@ -1,5 +1,5 @@ -[14:46:59] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) -[14:46:59] WARN: token-drop-analysis lacks --moe-router-fusion; QWEN3_235B runs unfused (timing only, routing identical) +[15:23:04] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +[15:23:04] WARN: token-drop-analysis lacks --moe-router-fusion; QWEN3_235B runs unfused (timing only, routing identical) ============================================================== QWEN3_235B / echo (slots/rank=16 policy=matched) E=128 K=8 H'=1536 m=16 @@ -8,12 +8,10 @@ memory_dump=none snapshots=/workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1/memory_snapshots/snapshot.pickle repo=/workspace/Megatron-MoE-Echo @ b099fd84 ============================================================== -W0826 14:47:01.327000 11720 torch/distributed/run.py:766] -W0826 14:47:01.327000 11720 torch/distributed/run.py:766] ***************************************** -W0826 14:47:01.327000 11720 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -W0826 14:47:01.327000 11720 torch/distributed/run.py:766] ***************************************** -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +W0826 15:23:06.174000 79898 torch/distributed/run.py:766] +W0826 15:23:06.174000 79898 torch/distributed/run.py:766] ***************************************** +W0826 15:23:06.174000 79898 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 15:23:06.174000 79898 torch/distributed/run.py:766] ***************************************** /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled @@ -577,51 +575,53 @@ WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it -[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : [Gloo] Rank 71 - is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank [Gloo] Rank 07 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 7 +7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank [Gloo] Rank 7 is connected to 76 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. -Expected number of connected peer ranks is : 7 -[Gloo] Rank [Gloo] Rank 0 is connected to 27 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +[Gloo] Rank 4[Gloo] Rank is connected to 76 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank [Gloo] Rank 3 is connected to 67 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : -7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 > initialized tensor model parallel with size 1 > initialized pipeline model parallel with size 1 > setting random seeds to 1234 ... [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[rank5]:[W826 14:47:19.631920282 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank2]:[W826 14:47:19.632008656 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank6]:[W826 14:47:19.632037782 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank1]:[W826 14:47:19.632088934 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank7]:[W826 14:47:19.632101936 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank4]:[W826 14:47:19.632101236 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank3]:[W826 14:47:19.633270648 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[rank2]:[W826 15:23:24.799034220 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 15:23:24.799056371 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 15:23:24.799058171 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 15:23:24.799061376 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 15:23:24.799120680 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 15:23:24.799205207 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 15:23:24.800705533 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. > compiling dataset index builder ... make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' make: Nothing to be done for 'default'. make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' ->>> done with dataset index builder. Compilation time: 0.035 seconds +>>> done with dataset index builder. Compilation time: 0.034 seconds WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. > compiling and loading fused kernels ... -[rank0]:[W826 14:47:19.788646935 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. ->>> done with compiling and loading fused kernels. Compilation time: 3.820 seconds -time to initialize megatron (seconds): 8.420 -[after megatron is initialized] datetime: 2026-08-26 14:47:27 +[rank0]:[W826 15:23:24.952668871 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 4.043 seconds +time to initialize megatron (seconds): 7.571 +[after megatron is initialized] datetime: 2026-08-26 15:23:30 building GPT model ... /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. warnings.warn( @@ -645,108 +645,108 @@ INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradi Params for bucket 1 (206045184 elements, 206045184 padded size): module.output_layer.weight Params for bucket 2 (71840000 elements, 71840000 padded size): - module.decoder.layers.11.self_attention.k_layernorm.weight - module.decoder.layers.11.self_attention.q_layernorm.weight module.decoder.final_layernorm.weight module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.k_layernorm.weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.11.self_attention.q_layernorm.weight module.decoder.layers.11.mlp.router.weight module.decoder.layers.11.self_attention.linear_qkv.weight module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.11.self_attention.linear_proj.weight Params for bucket 3 (71835904 elements, 71835904 padded size): + module.decoder.layers.10.mlp.router.weight module.decoder.layers.10.self_attention.q_layernorm.weight + module.decoder.layers.10.self_attention.linear_qkv.weight module.decoder.layers.10.pre_mlp_layernorm.weight module.decoder.layers.10.self_attention.k_layernorm.weight - module.decoder.layers.10.self_attention.linear_proj.weight - module.decoder.layers.10.mlp.router.weight - module.decoder.layers.10.self_attention.linear_qkv.weight module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.self_attention.linear_proj.weight Params for bucket 4 (71835904 elements, 71835904 padded size): module.decoder.layers.9.mlp.router.weight - module.decoder.layers.9.pre_mlp_layernorm.weight - module.decoder.layers.9.self_attention.k_layernorm.weight - module.decoder.layers.9.self_attention.linear_proj.weight module.decoder.layers.9.self_attention.q_layernorm.weight + module.decoder.layers.9.self_attention.k_layernorm.weight + module.decoder.layers.9.pre_mlp_layernorm.weight module.decoder.layers.9.self_attention.linear_qkv.weight module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.self_attention.linear_proj.weight Params for bucket 5 (71835904 elements, 71835904 padded size): - module.decoder.layers.8.self_attention.q_layernorm.weight - module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.weight module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.self_attention.linear_proj.weight module.decoder.layers.8.mlp.router.weight - module.decoder.layers.8.self_attention.linear_qkv.weight module.decoder.layers.8.self_attention.k_layernorm.weight - module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.8.self_attention.q_layernorm.weight + module.decoder.layers.8.pre_mlp_layernorm.weight Params for bucket 6 (71835904 elements, 71835904 padded size): - module.decoder.layers.7.self_attention.k_layernorm.weight - module.decoder.layers.7.self_attention.q_layernorm.weight + module.decoder.layers.7.self_attention.linear_qkv.weight module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.q_layernorm.weight module.decoder.layers.7.mlp.router.weight - module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.k_layernorm.weight module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.7.self_attention.linear_proj.weight Params for bucket 7 (71835904 elements, 71835904 padded size): - module.decoder.layers.6.self_attention.k_layernorm.weight - module.decoder.layers.6.pre_mlp_layernorm.weight - module.decoder.layers.6.self_attention.linear_proj.weight module.decoder.layers.6.mlp.router.weight module.decoder.layers.6.self_attention.linear_qkv.weight - module.decoder.layers.6.self_attention.q_layernorm.weight module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.6.self_attention.q_layernorm.weight + module.decoder.layers.6.self_attention.k_layernorm.weight + module.decoder.layers.6.pre_mlp_layernorm.weight Params for bucket 8 (71835904 elements, 71835904 padded size): + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.pre_mlp_layernorm.weight module.decoder.layers.5.self_attention.k_layernorm.weight module.decoder.layers.5.self_attention.q_layernorm.weight - module.decoder.layers.5.self_attention.linear_qkv.weight - module.decoder.layers.5.pre_mlp_layernorm.weight - module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.5.self_attention.linear_proj.weight Params for bucket 9 (71835904 elements, 71835904 padded size): - module.decoder.layers.4.self_attention.q_layernorm.weight module.decoder.layers.4.pre_mlp_layernorm.weight - module.decoder.layers.4.self_attention.k_layernorm.weight - module.decoder.layers.4.self_attention.linear_proj.weight - module.decoder.layers.4.mlp.router.weight module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.self_attention.q_layernorm.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.k_layernorm.weight module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.self_attention.linear_proj.weight Params for bucket 10 (71835904 elements, 71835904 padded size): module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_proj.weight module.decoder.layers.3.self_attention.linear_qkv.weight module.decoder.layers.3.self_attention.q_layernorm.weight - module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.3.self_attention.k_layernorm.weight - module.decoder.layers.3.pre_mlp_layernorm.weight - module.decoder.layers.3.self_attention.linear_proj.weight Params for bucket 11 (71835904 elements, 71835904 padded size): module.decoder.layers.2.self_attention.q_layernorm.weight module.decoder.layers.2.pre_mlp_layernorm.weight - module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.self_attention.k_layernorm.weight module.decoder.layers.2.mlp.router.weight module.decoder.layers.2.self_attention.linear_qkv.weight - module.decoder.layers.2.self_attention.k_layernorm.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.2.self_attention.linear_proj.weight Params for bucket 12 (71835904 elements, 71835904 padded size): - module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.weight module.decoder.layers.1.self_attention.k_layernorm.weight - module.decoder.layers.1.self_attention.q_layernorm.weight - module.decoder.layers.1.self_attention.linear_proj.weight module.decoder.layers.1.mlp.router.weight - module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.1.self_attention.q_layernorm.weight + module.decoder.layers.1.pre_mlp_layernorm.weight module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.self_attention.linear_proj.weight Params for bucket 13 (71835904 elements, 71835904 padded size): - module.decoder.layers.0.pre_mlp_layernorm.weight module.decoder.layers.0.self_attention.linear_qkv.weight - module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.0.mlp.router.weight module.decoder.layers.0.self_attention.q_layernorm.weight module.decoder.layers.0.self_attention.k_layernorm.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.pre_mlp_layernorm.weight module.decoder.layers.0.self_attention.linear_proj.weight Params for bucket 14 (206045184 elements, 206045184 padded size): module.embedding.word_embeddings.weight INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 76 Params for bucket 1 (44040192 elements, 44040192 padded size): - module.decoder.layers.11.mlp.experts.linear_fc2.weight14 module.decoder.layers.11.mlp.experts.linear_fc2.weight12 + module.decoder.layers.11.mlp.experts.linear_fc2.weight14 module.decoder.layers.11.mlp.experts.linear_fc2.weight10 module.decoder.layers.11.mlp.experts.linear_fc2.weight15 module.decoder.layers.11.mlp.experts.linear_fc2.weight11 @@ -754,59 +754,59 @@ Params for bucket 1 (44040192 elements, 44040192 padded size): module.decoder.layers.11.mlp.experts.linear_fc2.weight9 Params for bucket 2 (44040192 elements, 44040192 padded size): module.decoder.layers.11.mlp.experts.linear_fc2.weight8 - module.decoder.layers.11.mlp.experts.linear_fc2.weight4 module.decoder.layers.11.mlp.experts.linear_fc2.weight6 - module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 module.decoder.layers.11.mlp.experts.linear_fc2.weight3 module.decoder.layers.11.mlp.experts.linear_fc2.weight2 Params for bucket 3 (50331648 elements, 50331648 padded size): module.decoder.layers.11.mlp.experts.linear_fc2.weight1 module.decoder.layers.11.mlp.experts.linear_fc1.weight13 module.decoder.layers.11.mlp.experts.linear_fc1.weight15 - module.decoder.layers.11.mlp.experts.linear_fc2.weight0 module.decoder.layers.11.mlp.experts.linear_fc1.weight14 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 Params for bucket 4 (50331648 elements, 50331648 padded size): module.decoder.layers.11.mlp.experts.linear_fc1.weight12 - module.decoder.layers.11.mlp.experts.linear_fc1.weight10 module.decoder.layers.11.mlp.experts.linear_fc1.weight11 module.decoder.layers.11.mlp.experts.linear_fc1.weight9 + module.decoder.layers.11.mlp.experts.linear_fc1.weight10 Params for bucket 5 (50331648 elements, 50331648 padded size): module.decoder.layers.11.mlp.experts.linear_fc1.weight8 - module.decoder.layers.11.mlp.experts.linear_fc1.weight7 module.decoder.layers.11.mlp.experts.linear_fc1.weight5 module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 Params for bucket 6 (50331648 elements, 50331648 padded size): module.decoder.layers.11.mlp.experts.linear_fc1.weight4 module.decoder.layers.11.mlp.experts.linear_fc1.weight2 module.decoder.layers.11.mlp.experts.linear_fc1.weight3 module.decoder.layers.11.mlp.experts.linear_fc1.weight1 Params for bucket 7 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight12 + module.decoder.layers.10.mlp.experts.linear_fc2.weight14 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 module.decoder.layers.10.mlp.experts.linear_fc2.weight15 module.decoder.layers.10.mlp.experts.linear_fc2.weight13 module.decoder.layers.10.mlp.experts.linear_fc2.weight11 - module.decoder.layers.11.mlp.experts.linear_fc1.weight0 - module.decoder.layers.10.mlp.experts.linear_fc2.weight12 - module.decoder.layers.10.mlp.experts.linear_fc2.weight14 Params for bucket 8 (44040192 elements, 44040192 padded size): - module.decoder.layers.10.mlp.experts.linear_fc2.weight9 - module.decoder.layers.10.mlp.experts.linear_fc2.weight7 - module.decoder.layers.10.mlp.experts.linear_fc2.weight5 module.decoder.layers.10.mlp.experts.linear_fc2.weight10 - module.decoder.layers.10.mlp.experts.linear_fc2.weight8 module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight8 module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight9 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 Params for bucket 9 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight15 module.decoder.layers.10.mlp.experts.linear_fc2.weight3 module.decoder.layers.10.mlp.experts.linear_fc2.weight2 module.decoder.layers.10.mlp.experts.linear_fc2.weight0 module.decoder.layers.10.mlp.experts.linear_fc1.weight14 - module.decoder.layers.10.mlp.experts.linear_fc2.weight1 - module.decoder.layers.10.mlp.experts.linear_fc1.weight15 Params for bucket 10 (50331648 elements, 50331648 padded size): module.decoder.layers.10.mlp.experts.linear_fc1.weight13 - module.decoder.layers.10.mlp.experts.linear_fc1.weight10 module.decoder.layers.10.mlp.experts.linear_fc1.weight12 + module.decoder.layers.10.mlp.experts.linear_fc1.weight10 module.decoder.layers.10.mlp.experts.linear_fc1.weight11 Params for bucket 11 (50331648 elements, 50331648 padded size): module.decoder.layers.10.mlp.experts.linear_fc1.weight9 @@ -816,50 +816,50 @@ Params for bucket 11 (50331648 elements, 50331648 padded size): Params for bucket 12 (50331648 elements, 50331648 padded size): module.decoder.layers.10.mlp.experts.linear_fc1.weight5 module.decoder.layers.10.mlp.experts.linear_fc1.weight3 - module.decoder.layers.10.mlp.experts.linear_fc1.weight2 module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 Params for bucket 13 (44040192 elements, 44040192 padded size): - module.decoder.layers.9.mlp.experts.linear_fc2.weight14 - module.decoder.layers.10.mlp.experts.linear_fc1.weight0 module.decoder.layers.10.mlp.experts.linear_fc1.weight1 module.decoder.layers.9.mlp.experts.linear_fc2.weight13 module.decoder.layers.9.mlp.experts.linear_fc2.weight15 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight14 Params for bucket 14 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight9 + module.decoder.layers.9.mlp.experts.linear_fc2.weight11 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 module.decoder.layers.9.mlp.experts.linear_fc2.weight12 module.decoder.layers.9.mlp.experts.linear_fc2.weight10 module.decoder.layers.9.mlp.experts.linear_fc2.weight8 module.decoder.layers.9.mlp.experts.linear_fc2.weight6 - module.decoder.layers.9.mlp.experts.linear_fc2.weight9 - module.decoder.layers.9.mlp.experts.linear_fc2.weight11 - module.decoder.layers.9.mlp.experts.linear_fc2.weight7 Params for bucket 15 (50331648 elements, 50331648 padded size): - module.decoder.layers.9.mlp.experts.linear_fc2.weight4 - module.decoder.layers.9.mlp.experts.linear_fc2.weight1 module.decoder.layers.9.mlp.experts.linear_fc2.weight5 module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight15 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 module.decoder.layers.9.mlp.experts.linear_fc2.weight2 module.decoder.layers.9.mlp.experts.linear_fc2.weight0 - module.decoder.layers.9.mlp.experts.linear_fc1.weight15 Params for bucket 16 (50331648 elements, 50331648 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight14 + module.decoder.layers.9.mlp.experts.linear_fc1.weight12 module.decoder.layers.9.mlp.experts.linear_fc1.weight13 module.decoder.layers.9.mlp.experts.linear_fc1.weight11 - module.decoder.layers.9.mlp.experts.linear_fc1.weight12 Params for bucket 17 (50331648 elements, 50331648 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight10 - module.decoder.layers.9.mlp.experts.linear_fc1.weight7 module.decoder.layers.9.mlp.experts.linear_fc1.weight9 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 module.decoder.layers.9.mlp.experts.linear_fc1.weight8 Params for bucket 18 (50331648 elements, 50331648 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 module.decoder.layers.9.mlp.experts.linear_fc1.weight4 module.decoder.layers.9.mlp.experts.linear_fc1.weight3 - module.decoder.layers.9.mlp.experts.linear_fc1.weight5 Params for bucket 19 (44040192 elements, 44040192 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight15 module.decoder.layers.9.mlp.experts.linear_fc1.weight1 module.decoder.layers.9.mlp.experts.linear_fc1.weight0 - module.decoder.layers.8.mlp.experts.linear_fc2.weight15 Params for bucket 20 (44040192 elements, 44040192 padded size): module.decoder.layers.8.mlp.experts.linear_fc2.weight14 module.decoder.layers.8.mlp.experts.linear_fc2.weight12 @@ -878,93 +878,93 @@ Params for bucket 21 (44040192 elements, 44040192 padded size): module.decoder.layers.8.mlp.experts.linear_fc2.weight2 Params for bucket 22 (44040192 elements, 44040192 padded size): module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight15 module.decoder.layers.8.mlp.experts.linear_fc1.weight13 module.decoder.layers.8.mlp.experts.linear_fc1.weight14 - module.decoder.layers.8.mlp.experts.linear_fc1.weight15 Params for bucket 23 (50331648 elements, 50331648 padded size): module.decoder.layers.8.mlp.experts.linear_fc1.weight12 module.decoder.layers.8.mlp.experts.linear_fc1.weight9 - module.decoder.layers.8.mlp.experts.linear_fc1.weight10 module.decoder.layers.8.mlp.experts.linear_fc1.weight11 + module.decoder.layers.8.mlp.experts.linear_fc1.weight10 Params for bucket 24 (50331648 elements, 50331648 padded size): module.decoder.layers.8.mlp.experts.linear_fc1.weight8 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 module.decoder.layers.8.mlp.experts.linear_fc1.weight6 module.decoder.layers.8.mlp.experts.linear_fc1.weight7 - module.decoder.layers.8.mlp.experts.linear_fc1.weight5 Params for bucket 25 (50331648 elements, 50331648 padded size): module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 module.decoder.layers.8.mlp.experts.linear_fc1.weight1 module.decoder.layers.8.mlp.experts.linear_fc1.weight2 - module.decoder.layers.8.mlp.experts.linear_fc1.weight3 Params for bucket 26 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight12 module.decoder.layers.7.mlp.experts.linear_fc2.weight14 - module.decoder.layers.7.mlp.experts.linear_fc2.weight13 - module.decoder.layers.7.mlp.experts.linear_fc2.weight11 module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight13 module.decoder.layers.7.mlp.experts.linear_fc2.weight15 - module.decoder.layers.7.mlp.experts.linear_fc2.weight12 + module.decoder.layers.7.mlp.experts.linear_fc2.weight11 Params for bucket 27 (44040192 elements, 44040192 padded size): module.decoder.layers.7.mlp.experts.linear_fc2.weight10 module.decoder.layers.7.mlp.experts.linear_fc2.weight6 module.decoder.layers.7.mlp.experts.linear_fc2.weight8 module.decoder.layers.7.mlp.experts.linear_fc2.weight4 - module.decoder.layers.7.mlp.experts.linear_fc2.weight7 module.decoder.layers.7.mlp.experts.linear_fc2.weight9 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 module.decoder.layers.7.mlp.experts.linear_fc2.weight5 Params for bucket 28 (50331648 elements, 50331648 padded size): - module.decoder.layers.7.mlp.experts.linear_fc1.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight14 module.decoder.layers.7.mlp.experts.linear_fc2.weight3 - module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight15 module.decoder.layers.7.mlp.experts.linear_fc2.weight0 - module.decoder.layers.7.mlp.experts.linear_fc1.weight14 Params for bucket 29 (50331648 elements, 50331648 padded size): module.decoder.layers.7.mlp.experts.linear_fc1.weight13 - module.decoder.layers.7.mlp.experts.linear_fc1.weight10 module.decoder.layers.7.mlp.experts.linear_fc1.weight11 module.decoder.layers.7.mlp.experts.linear_fc1.weight12 + module.decoder.layers.7.mlp.experts.linear_fc1.weight10 Params for bucket 30 (50331648 elements, 50331648 padded size): module.decoder.layers.7.mlp.experts.linear_fc1.weight9 - module.decoder.layers.7.mlp.experts.linear_fc1.weight7 module.decoder.layers.7.mlp.experts.linear_fc1.weight8 module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 Params for bucket 31 (50331648 elements, 50331648 padded size): module.decoder.layers.7.mlp.experts.linear_fc1.weight5 module.decoder.layers.7.mlp.experts.linear_fc1.weight4 - module.decoder.layers.7.mlp.experts.linear_fc1.weight2 module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 Params for bucket 32 (44040192 elements, 44040192 padded size): - module.decoder.layers.7.mlp.experts.linear_fc1.weight1 - module.decoder.layers.6.mlp.experts.linear_fc2.weight14 - module.decoder.layers.7.mlp.experts.linear_fc1.weight0 module.decoder.layers.6.mlp.experts.linear_fc2.weight15 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 module.decoder.layers.6.mlp.experts.linear_fc2.weight13 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight14 Params for bucket 33 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc2.weight9 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 module.decoder.layers.6.mlp.experts.linear_fc2.weight12 module.decoder.layers.6.mlp.experts.linear_fc2.weight8 module.decoder.layers.6.mlp.experts.linear_fc2.weight10 module.decoder.layers.6.mlp.experts.linear_fc2.weight6 - module.decoder.layers.6.mlp.experts.linear_fc2.weight11 - module.decoder.layers.6.mlp.experts.linear_fc2.weight9 - module.decoder.layers.6.mlp.experts.linear_fc2.weight7 Params for bucket 34 (50331648 elements, 50331648 padded size): - module.decoder.layers.6.mlp.experts.linear_fc2.weight4 - module.decoder.layers.6.mlp.experts.linear_fc2.weight1 - module.decoder.layers.6.mlp.experts.linear_fc1.weight15 module.decoder.layers.6.mlp.experts.linear_fc2.weight5 - module.decoder.layers.6.mlp.experts.linear_fc2.weight2 module.decoder.layers.6.mlp.experts.linear_fc2.weight3 module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight15 Params for bucket 35 (50331648 elements, 50331648 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight14 + module.decoder.layers.6.mlp.experts.linear_fc1.weight12 module.decoder.layers.6.mlp.experts.linear_fc1.weight13 module.decoder.layers.6.mlp.experts.linear_fc1.weight11 - module.decoder.layers.6.mlp.experts.linear_fc1.weight12 Params for bucket 36 (50331648 elements, 50331648 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight10 - module.decoder.layers.6.mlp.experts.linear_fc1.weight7 - module.decoder.layers.6.mlp.experts.linear_fc1.weight9 module.decoder.layers.6.mlp.experts.linear_fc1.weight8 + module.decoder.layers.6.mlp.experts.linear_fc1.weight9 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 Params for bucket 37 (50331648 elements, 50331648 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight6 module.decoder.layers.6.mlp.experts.linear_fc1.weight5 @@ -972,25 +972,25 @@ Params for bucket 37 (50331648 elements, 50331648 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight4 Params for bucket 38 (44040192 elements, 44040192 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight2 - module.decoder.layers.6.mlp.experts.linear_fc1.weight0 module.decoder.layers.5.mlp.experts.linear_fc2.weight15 module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 Params for bucket 39 (44040192 elements, 44040192 padded size): module.decoder.layers.5.mlp.experts.linear_fc2.weight14 module.decoder.layers.5.mlp.experts.linear_fc2.weight12 module.decoder.layers.5.mlp.experts.linear_fc2.weight10 module.decoder.layers.5.mlp.experts.linear_fc2.weight8 - module.decoder.layers.5.mlp.experts.linear_fc2.weight11 module.decoder.layers.5.mlp.experts.linear_fc2.weight13 + module.decoder.layers.5.mlp.experts.linear_fc2.weight11 module.decoder.layers.5.mlp.experts.linear_fc2.weight9 Params for bucket 40 (44040192 elements, 44040192 padded size): - module.decoder.layers.5.mlp.experts.linear_fc2.weight4 module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 module.decoder.layers.5.mlp.experts.linear_fc2.weight1 module.decoder.layers.5.mlp.experts.linear_fc2.weight7 - module.decoder.layers.5.mlp.experts.linear_fc2.weight5 module.decoder.layers.5.mlp.experts.linear_fc2.weight3 - module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 Params for bucket 41 (44040192 elements, 44040192 padded size): module.decoder.layers.5.mlp.experts.linear_fc2.weight0 module.decoder.layers.5.mlp.experts.linear_fc1.weight14 @@ -1008,29 +1008,29 @@ Params for bucket 43 (50331648 elements, 50331648 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight7 Params for bucket 44 (50331648 elements, 50331648 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight4 - module.decoder.layers.5.mlp.experts.linear_fc1.weight2 module.decoder.layers.5.mlp.experts.linear_fc1.weight3 module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 Params for bucket 45 (44040192 elements, 44040192 padded size): - module.decoder.layers.4.mlp.experts.linear_fc2.weight12 - module.decoder.layers.4.mlp.experts.linear_fc2.weight14 module.decoder.layers.5.mlp.experts.linear_fc1.weight0 - module.decoder.layers.4.mlp.experts.linear_fc2.weight13 module.decoder.layers.4.mlp.experts.linear_fc2.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight13 module.decoder.layers.4.mlp.experts.linear_fc2.weight11 + module.decoder.layers.4.mlp.experts.linear_fc2.weight12 + module.decoder.layers.4.mlp.experts.linear_fc2.weight14 Params for bucket 46 (44040192 elements, 44040192 padded size): - module.decoder.layers.4.mlp.experts.linear_fc2.weight10 - module.decoder.layers.4.mlp.experts.linear_fc2.weight8 + module.decoder.layers.4.mlp.experts.linear_fc2.weight9 module.decoder.layers.4.mlp.experts.linear_fc2.weight7 module.decoder.layers.4.mlp.experts.linear_fc2.weight5 - module.decoder.layers.4.mlp.experts.linear_fc2.weight4 - module.decoder.layers.4.mlp.experts.linear_fc2.weight9 + module.decoder.layers.4.mlp.experts.linear_fc2.weight10 module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight8 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 Params for bucket 47 (50331648 elements, 50331648 padded size): - module.decoder.layers.4.mlp.experts.linear_fc2.weight1 - module.decoder.layers.4.mlp.experts.linear_fc1.weight15 module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight15 module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 module.decoder.layers.4.mlp.experts.linear_fc2.weight2 module.decoder.layers.4.mlp.experts.linear_fc1.weight14 Params for bucket 48 (50331648 elements, 50331648 padded size): @@ -1052,8 +1052,8 @@ Params for bucket 51 (44040192 elements, 44040192 padded size): module.decoder.layers.4.mlp.experts.linear_fc1.weight1 module.decoder.layers.3.mlp.experts.linear_fc2.weight14 module.decoder.layers.3.mlp.experts.linear_fc2.weight13 - module.decoder.layers.4.mlp.experts.linear_fc1.weight0 module.decoder.layers.3.mlp.experts.linear_fc2.weight15 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 Params for bucket 52 (44040192 elements, 44040192 padded size): module.decoder.layers.3.mlp.experts.linear_fc2.weight12 module.decoder.layers.3.mlp.experts.linear_fc2.weight10 @@ -1072,40 +1072,40 @@ Params for bucket 53 (50331648 elements, 50331648 padded size): module.decoder.layers.3.mlp.experts.linear_fc2.weight0 Params for bucket 54 (50331648 elements, 50331648 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight14 - module.decoder.layers.3.mlp.experts.linear_fc1.weight13 module.decoder.layers.3.mlp.experts.linear_fc1.weight11 module.decoder.layers.3.mlp.experts.linear_fc1.weight12 + module.decoder.layers.3.mlp.experts.linear_fc1.weight13 Params for bucket 55 (50331648 elements, 50331648 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight10 + module.decoder.layers.3.mlp.experts.linear_fc1.weight8 module.decoder.layers.3.mlp.experts.linear_fc1.weight9 module.decoder.layers.3.mlp.experts.linear_fc1.weight7 - module.decoder.layers.3.mlp.experts.linear_fc1.weight8 Params for bucket 56 (50331648 elements, 50331648 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 module.decoder.layers.3.mlp.experts.linear_fc1.weight3 module.decoder.layers.3.mlp.experts.linear_fc1.weight4 - module.decoder.layers.3.mlp.experts.linear_fc1.weight5 Params for bucket 57 (44040192 elements, 44040192 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight2 module.decoder.layers.3.mlp.experts.linear_fc1.weight0 - module.decoder.layers.2.mlp.experts.linear_fc2.weight15 module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.2.mlp.experts.linear_fc2.weight15 Params for bucket 58 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight13 + module.decoder.layers.2.mlp.experts.linear_fc2.weight11 + module.decoder.layers.2.mlp.experts.linear_fc2.weight9 module.decoder.layers.2.mlp.experts.linear_fc2.weight14 module.decoder.layers.2.mlp.experts.linear_fc2.weight10 module.decoder.layers.2.mlp.experts.linear_fc2.weight12 module.decoder.layers.2.mlp.experts.linear_fc2.weight8 - module.decoder.layers.2.mlp.experts.linear_fc2.weight11 - module.decoder.layers.2.mlp.experts.linear_fc2.weight13 - module.decoder.layers.2.mlp.experts.linear_fc2.weight9 Params for bucket 59 (44040192 elements, 44040192 padded size): - module.decoder.layers.2.mlp.experts.linear_fc2.weight4 - module.decoder.layers.2.mlp.experts.linear_fc2.weight6 - module.decoder.layers.2.mlp.experts.linear_fc2.weight1 module.decoder.layers.2.mlp.experts.linear_fc2.weight7 - module.decoder.layers.2.mlp.experts.linear_fc2.weight5 module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 Params for bucket 60 (44040192 elements, 44040192 padded size): module.decoder.layers.2.mlp.experts.linear_fc2.weight0 module.decoder.layers.2.mlp.experts.linear_fc1.weight13 @@ -1114,77 +1114,77 @@ Params for bucket 60 (44040192 elements, 44040192 padded size): Params for bucket 61 (50331648 elements, 50331648 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight12 module.decoder.layers.2.mlp.experts.linear_fc1.weight10 - module.decoder.layers.2.mlp.experts.linear_fc1.weight11 module.decoder.layers.2.mlp.experts.linear_fc1.weight9 + module.decoder.layers.2.mlp.experts.linear_fc1.weight11 Params for bucket 62 (50331648 elements, 50331648 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight8 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 module.decoder.layers.2.mlp.experts.linear_fc1.weight7 module.decoder.layers.2.mlp.experts.linear_fc1.weight5 - module.decoder.layers.2.mlp.experts.linear_fc1.weight6 Params for bucket 63 (50331648 elements, 50331648 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 module.decoder.layers.2.mlp.experts.linear_fc1.weight2 module.decoder.layers.2.mlp.experts.linear_fc1.weight3 - module.decoder.layers.2.mlp.experts.linear_fc1.weight1 Params for bucket 64 (44040192 elements, 44040192 padded size): - module.decoder.layers.1.mlp.experts.linear_fc2.weight15 - module.decoder.layers.1.mlp.experts.linear_fc2.weight13 - module.decoder.layers.1.mlp.experts.linear_fc2.weight11 module.decoder.layers.2.mlp.experts.linear_fc1.weight0 module.decoder.layers.1.mlp.experts.linear_fc2.weight14 module.decoder.layers.1.mlp.experts.linear_fc2.weight12 + module.decoder.layers.1.mlp.experts.linear_fc2.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight11 Params for bucket 65 (44040192 elements, 44040192 padded size): - module.decoder.layers.1.mlp.experts.linear_fc2.weight7 - module.decoder.layers.1.mlp.experts.linear_fc2.weight9 - module.decoder.layers.1.mlp.experts.linear_fc2.weight5 module.decoder.layers.1.mlp.experts.linear_fc2.weight10 module.decoder.layers.1.mlp.experts.linear_fc2.weight8 module.decoder.layers.1.mlp.experts.linear_fc2.weight6 module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight9 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 Params for bucket 66 (50331648 elements, 50331648 padded size): - module.decoder.layers.1.mlp.experts.linear_fc2.weight3 module.decoder.layers.1.mlp.experts.linear_fc1.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 module.decoder.layers.1.mlp.experts.linear_fc2.weight2 module.decoder.layers.1.mlp.experts.linear_fc2.weight0 - module.decoder.layers.1.mlp.experts.linear_fc2.weight1 module.decoder.layers.1.mlp.experts.linear_fc1.weight14 Params for bucket 67 (50331648 elements, 50331648 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight13 module.decoder.layers.1.mlp.experts.linear_fc1.weight11 - module.decoder.layers.1.mlp.experts.linear_fc1.weight10 module.decoder.layers.1.mlp.experts.linear_fc1.weight12 + module.decoder.layers.1.mlp.experts.linear_fc1.weight10 Params for bucket 68 (50331648 elements, 50331648 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight9 - module.decoder.layers.1.mlp.experts.linear_fc1.weight7 module.decoder.layers.1.mlp.experts.linear_fc1.weight8 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 module.decoder.layers.1.mlp.experts.linear_fc1.weight6 Params for bucket 69 (50331648 elements, 50331648 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight5 module.decoder.layers.1.mlp.experts.linear_fc1.weight4 - module.decoder.layers.1.mlp.experts.linear_fc1.weight2 module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 Params for bucket 70 (44040192 elements, 44040192 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight1 - module.decoder.layers.0.mlp.experts.linear_fc2.weight14 - module.decoder.layers.0.mlp.experts.linear_fc2.weight15 module.decoder.layers.0.mlp.experts.linear_fc2.weight13 + module.decoder.layers.0.mlp.experts.linear_fc2.weight15 + module.decoder.layers.0.mlp.experts.linear_fc2.weight14 module.decoder.layers.1.mlp.experts.linear_fc1.weight0 Params for bucket 71 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight9 + module.decoder.layers.0.mlp.experts.linear_fc2.weight11 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 module.decoder.layers.0.mlp.experts.linear_fc2.weight12 module.decoder.layers.0.mlp.experts.linear_fc2.weight10 module.decoder.layers.0.mlp.experts.linear_fc2.weight8 module.decoder.layers.0.mlp.experts.linear_fc2.weight6 - module.decoder.layers.0.mlp.experts.linear_fc2.weight11 - module.decoder.layers.0.mlp.experts.linear_fc2.weight9 - module.decoder.layers.0.mlp.experts.linear_fc2.weight7 Params for bucket 72 (50331648 elements, 50331648 padded size): - module.decoder.layers.0.mlp.experts.linear_fc2.weight4 - module.decoder.layers.0.mlp.experts.linear_fc2.weight2 - module.decoder.layers.0.mlp.experts.linear_fc2.weight1 module.decoder.layers.0.mlp.experts.linear_fc2.weight5 module.decoder.layers.0.mlp.experts.linear_fc2.weight3 module.decoder.layers.0.mlp.experts.linear_fc2.weight0 module.decoder.layers.0.mlp.experts.linear_fc1.weight15 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 Params for bucket 73 (50331648 elements, 50331648 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight14 module.decoder.layers.0.mlp.experts.linear_fc1.weight13 @@ -1197,16 +1197,16 @@ Params for bucket 74 (50331648 elements, 50331648 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight9 Params for bucket 75 (50331648 elements, 50331648 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight6 - module.decoder.layers.0.mlp.experts.linear_fc1.weight4 module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 module.decoder.layers.0.mlp.experts.linear_fc1.weight3 Params for bucket 76 (37748736 elements, 37748736 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight2 module.decoder.layers.0.mlp.experts.linear_fc1.weight0 module.decoder.layers.0.mlp.experts.linear_fc1.weight1 -INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine -[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 14:47:27 +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 15:23:31 > building train, validation, and test datasets ... > datasets target sizes (minimum size): train: 96 @@ -1214,41 +1214,47 @@ INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine test: 0 INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] > building train, validation, and test datasets for GPT ... -INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers INFO:megatron.core.datasets.indexed_dataset: Extract the document indices INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 -INFO:megatron.core.datasets.gpt_dataset:Build and save the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-shuffle_index.npy INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536 -INFO:megatron.core.datasets.gpt_dataset:> total number of epochs: 1 > finished creating GPT datasets ... -[after dataloaders are built] datetime: 2026-08-26 14:47:27 +[after dataloaders are built] datetime: 2026-08-26 15:23:31 done with setup ... (min, max) time across ranks (ms): - model-and-optimizer-setup ......................: (613.98, 628.58) - train/valid/test-data-iterators-setup ..........: (119.71, 126.23) + model-and-optimizer-setup ......................: (669.37, 687.81) + train/valid/test-data-iterators-setup ..........: (60.52, 70.03) training ... Overwriting rerun_state_machine.current_iteration from -1 to 0... -[before the start of training step] datetime: 2026-08-26 14:47:27 +[before the start of training step] datetime: 2026-08-26 15:23:31 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[3717, 45, 34, 300, 95, 653, 52, 51, 104, 192, 1954, 454, 2109, 99, 125, 334, 421, 1669, 139, 48, 884, 11, 59, 1539, 1204, 46, 11, 1777, 700, 2391, 70, 239, 782, 1437, 128, 12, 147, 177, 94, 45, 626, 24, 911, 3248, 27, 0, 201, 749, 265, 152, 6, 29, 1, 164, 75, 47, 55, 33, 48, 504, 1107, 40, 260, 135, 2559, 19, 1, 15, 1338, 7, 43, 1005, 1350, 92, 1655, 231, 451, 10, 624, 918, 107, 20, 8, 54, 19, 11, 1470, 689, 17, 40, 1188, 53, 37, 117, 145, 28, 1156, 254, 48, 1829, 139, 4, 28, 65, 223, 134, 710, 4519, 63, 123, 56, 17, 829, 28, 23, 16, 27, 895, 161, 162, 1302, 83, 1006, 435, 211, 188, 22, 3404] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[3582, 65, 17, 348, 173, 720, 6, 38, 93, 234, 1681, 554, 1835, 93, 233, 407, 142, 1767, 156, 15, 766, 17, 101, 1975, 1326, 75, 34, 1459, 303, 1912, 104, 216, 594, 1910, 110, 39, 109, 360, 58, 49, 1286, 52, 559, 2506, 31, 11, 243, 627, 184, 103, 0, 37, 1, 344, 78, 32, 81, 26, 23, 445, 1958, 51, 365, 198, 2233, 9, 3, 6, 1086, 21, 40, 1186, 763, 243, 1978, 168, 302, 7, 660, 925, 16, 10, 6, 46, 23, 2, 1051, 620, 14, 48, 1267, 4, 24, 145, 535, 142, 956, 143, 119, 2025, 93, 1, 53, 92, 209, 217, 753, 4732, 22, 151, 67, 75, 978, 43, 46, 88, 0, 405, 158, 486, 1208, 85, 1268, 431, 301, 219, 3, 3609] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[2774, 81, 20, 440, 113, 825, 1, 7, 136, 273, 1522, 352, 1570, 94, 177, 387, 235, 1345, 284, 33, 919, 36, 29, 1509, 489, 70, 25, 1199, 452, 1511, 12, 376, 878, 2834, 125, 78, 175, 272, 175, 99, 1378, 40, 576, 2598, 65, 9, 257, 644, 184, 315, 3, 56, 9, 660, 54, 34, 188, 28, 21, 756, 1127, 28, 279, 236, 2031, 54, 1, 19, 1611, 7, 76, 1269, 1083, 94, 1524, 249, 482, 5, 368, 923, 23, 18, 2, 51, 52, 11, 519, 705, 39, 18, 862, 9, 112, 282, 241, 92, 2628, 39, 39, 1753, 126, 7, 88, 169, 431, 338, 490, 4414, 32, 231, 104, 65, 1090, 24, 28, 48, 8, 783, 114, 301, 1128, 63, 1940, 366, 244, 369, 12, 3755] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[3419, 64, 28, 251, 137, 848, 33, 12, 76, 106, 1605, 391, 1987, 36, 318, 356, 166, 2384, 247, 50, 858, 11, 90, 1556, 781, 75, 13, 1523, 630, 1004, 48, 475, 733, 2475, 39, 61, 153, 188, 102, 42, 857, 33, 447, 3171, 69, 18, 216, 775, 179, 153, 0, 131, 1, 297, 102, 27, 105, 40, 56, 854, 1205, 22, 184, 346, 1707, 36, 2, 36, 2173, 1, 57, 967, 1056, 103, 1069, 219, 529, 22, 800, 1137, 39, 71, 10, 38, 84, 13, 1114, 765, 4, 45, 998, 5, 30, 137, 359, 119, 1929, 240, 33, 2444, 41, 1, 81, 127, 311, 147, 585, 4953, 244, 217, 66, 64, 797, 128, 64, 44, 3, 621, 94, 235, 1047, 107, 1257, 386, 289, 307, 13, 2257] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[3275, 77, 6, 177, 143, 360, 15, 19, 62, 166, 1593, 1553, 2007, 122, 228, 400, 246, 1894, 279, 10, 678, 29, 99, 1122, 518, 137, 56, 2124, 1089, 1664, 33, 110, 943, 2147, 54, 28, 94, 314, 68, 11, 745, 90, 518, 2679, 47, 21, 192, 487, 109, 347, 1, 61, 8, 693, 29, 8, 32, 24, 50, 523, 2127, 50, 559, 37, 1993, 45, 6, 5, 1362, 35, 12, 1426, 967, 121, 2164, 75, 535, 6, 627, 616, 46, 7, 0, 26, 191, 18, 568, 833, 16, 20, 925, 3, 12, 191, 265, 109, 1284, 218, 98, 3389, 66, 0, 41, 126, 759, 87, 349, 3809, 72, 111, 39, 68, 258, 23, 13, 40, 3, 359, 150, 626, 1393, 76, 1267, 386, 357, 121, 3, 3333] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[3717, 45, 34, 300, 95, 653, 52, 51, 104, 192, 1954, 454, 2109, 99, 125, 334, 421, 1669, 139, 48, 884, 11, 59, 1539, 1204, 46, 11, 1777, 700, 2391, 70, 239, 782, 1437, 128, 12, 147, 177, 94, 45, 626, 24, 911, 3248, 27, 0, 201, 749, 265, 152, 6, 29, 1, 164, 75, 47, 55, 33, 48, 504, 1107, 40, 260, 135, 2559, 19, 1, 15, 1338, 7, 43, 1005, 1350, 92, 1655, 231, 451, 10, 624, 918, 107, 20, 8, 54, 19, 11, 1470, 689, 17, 40, 1188, 53, 37, 117, 145, 28, 1156, 254, 48, 1829, 139, 4, 28, 65, 223, 134, 710, 4519, 63, 123, 56, 17, 829, 28, 23, 16, 27, 895, 161, 162, 1302, 83, 1006, 435, 211, 188, 22, 3404] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[2556, 42, 35, 461, 182, 450, 13, 35, 64, 376, 2194, 396, 1708, 186, 108, 858, 303, 1598, 273, 11, 372, 22, 43, 1516, 794, 17, 33, 1125, 451, 2609, 106, 431, 707, 1920, 113, 21, 105, 709, 184, 163, 904, 31, 727, 2372, 48, 26, 191, 651, 282, 290, 7, 29, 5, 238, 118, 61, 192, 35, 34, 463, 1327, 32, 595, 411, 2335, 15, 6, 11, 1851, 5, 145, 1681, 1183, 218, 1290, 394, 290, 12, 543, 713, 27, 26, 3, 50, 82, 4, 722, 1085, 68, 60, 784, 10, 50, 298, 226, 124, 839, 179, 99, 1323, 158, 32, 190, 60, 360, 307, 584, 4653, 22, 198, 126, 25, 820, 36, 33, 57, 5, 1137, 216, 182, 809, 74, 1406, 416, 267, 185, 8, 4060] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[3319, 91, 9, 325, 141, 771, 10, 0, 113, 131, 1583, 574, 1908, 98, 291, 400, 141, 1705, 44, 11, 526, 73, 84, 1479, 991, 46, 37, 1484, 600, 1555, 38, 361, 613, 2423, 39, 56, 153, 263, 95, 37, 1024, 35, 915, 2986, 44, 2, 246, 1124, 281, 240, 1, 18, 3, 576, 40, 30, 25, 37, 47, 586, 1487, 17, 275, 149, 2130, 74, 13, 2, 1225, 15, 54, 1289, 715, 56, 2134, 152, 477, 2, 779, 1073, 45, 21, 2, 34, 38, 3, 657, 745, 18, 56, 1257, 6, 64, 194, 280, 124, 954, 126, 99, 1887, 72, 13, 71, 122, 338, 133, 579, 5356, 46, 166, 60, 69, 339, 75, 5, 33, 11, 515, 109, 306, 1787, 70, 1401, 352, 186, 418, 3, 3295] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[2556, 42, 35, 461, 182, 450, 13, 35, 64, 376, 2194, 396, 1708, 186, 108, 858, 303, 1598, 273, 11, 372, 22, 43, 1516, 794, 17, 33, 1125, 451, 2609, 106, 431, 707, 1920, 113, 21, 105, 709, 184, 163, 904, 31, 727, 2372, 48, 26, 191, 651, 282, 290, 7, 29, 5, 238, 118, 61, 192, 35, 34, 463, 1327, 32, 595, 411, 2335, 15, 6, 11, 1851, 5, 145, 1681, 1183, 218, 1290, 394, 290, 12, 543, 713, 27, 26, 3, 50, 82, 4, 722, 1085, 68, 60, 784, 10, 50, 298, 226, 124, 839, 179, 99, 1323, 158, 32, 190, 60, 360, 307, 584, 4653, 22, 198, 126, 25, 820, 36, 33, 57, 5, 1137, 216, 182, 809, 74, 1406, 416, 267, 185, 8, 4060] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[2665, 133, 14, 471, 269, 921, 14, 52, 65, 336, 1992, 615, 1704, 128, 116, 437, 47, 1632, 291, 3, 442, 11, 160, 2050, 728, 41, 144, 1375, 716, 2000, 107, 271, 480, 1912, 60, 37, 102, 501, 132, 41, 1399, 67, 766, 2873, 85, 57, 156, 1053, 444, 246, 2, 44, 2, 326, 97, 16, 203, 15, 27, 598, 1585, 41, 434, 342, 2007, 8, 1, 12, 1186, 8, 83, 1486, 1110, 213, 1137, 96, 316, 19, 485, 778, 46, 2, 2, 57, 88, 24, 486, 839, 32, 43, 674, 2, 17, 340, 254, 78, 1434, 115, 34, 1751, 122, 0, 87, 102, 323, 122, 746, 4321, 4, 124, 80, 83, 903, 82, 67, 46, 8, 689, 261, 399, 909, 69, 1362, 557, 120, 550, 17, 4027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[3275, 77, 6, 177, 143, 360, 15, 19, 62, 166, 1593, 1553, 2007, 122, 228, 400, 246, 1894, 279, 10, 678, 29, 99, 1122, 518, 137, 56, 2124, 1089, 1664, 33, 110, 943, 2147, 54, 28, 94, 314, 68, 11, 745, 90, 518, 2679, 47, 21, 192, 487, 109, 347, 1, 61, 8, 693, 29, 8, 32, 24, 50, 523, 2127, 50, 559, 37, 1993, 45, 6, 5, 1362, 35, 12, 1426, 967, 121, 2164, 75, 535, 6, 627, 616, 46, 7, 0, 26, 191, 18, 568, 833, 16, 20, 925, 3, 12, 191, 265, 109, 1284, 218, 98, 3389, 66, 0, 41, 126, 759, 87, 349, 3809, 72, 111, 39, 68, 258, 23, 13, 40, 3, 359, 150, 626, 1393, 76, 1267, 386, 357, 121, 3, 3333] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[2774, 81, 20, 440, 113, 825, 1, 7, 136, 273, 1522, 352, 1570, 94, 177, 387, 235, 1345, 284, 33, 919, 36, 29, 1509, 489, 70, 25, 1199, 452, 1511, 12, 376, 878, 2834, 125, 78, 175, 272, 175, 99, 1378, 40, 576, 2598, 65, 9, 257, 644, 184, 315, 3, 56, 9, 660, 54, 34, 188, 28, 21, 756, 1127, 28, 279, 236, 2031, 54, 1, 19, 1611, 7, 76, 1269, 1083, 94, 1524, 249, 482, 5, 368, 923, 23, 18, 2, 51, 52, 11, 519, 705, 39, 18, 862, 9, 112, 282, 241, 92, 2628, 39, 39, 1753, 126, 7, 88, 169, 431, 338, 490, 4414, 32, 231, 104, 65, 1090, 24, 28, 48, 8, 783, 114, 301, 1128, 63, 1940, 366, 244, 369, 12, 3755] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1633, 91, 9, 325, 141, 771, 10, 0, 113, 131, 1583, 574, 1908, 98, 291, 400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 1705, 44, 11, 526, 73, 84, 1479, 991, 46, 37, 1484, 600, 127, 38, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 613, 2423, 39, 56, 153, 263, 95, 37, 1024, 35, 915, 1884, 44, 2, 246, 1124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 281, 240, 1, 18, 3, 576, 40, 30, 25, 37, 47, 586, 1487, 17, 275, 149, 1686, 1102, 902, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 383, 74, 13, 2, 1225, 15, 54, 1289, 715, 56, 2134, 152, 477, 2, 779, 1073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 21, 2, 34, 38, 3, 657, 745, 18, 56, 1257, 6, 64, 194, 280, 124, 2212, 1747, 871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 954, 126, 99, 1887, 72, 13, 71, 122, 338, 133, 579, 3144, 46, 166, 60, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 339, 75, 5, 33, 11, 515, 109, 306, 1787, 70, 1401, 352, 186, 418, 3, 2393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8078, 7747, 8953, 8059, 8443, 8374, 7879, 8003] avg=8192.0000 max=8953 max_rank=2 min=7747 min_rank=1 imbalance_ratio=1.092896 max_load_reduction=56583 max_load_reduction_pct=0.863388 moved_to_echo=32699 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1762, 65, 17, 348, 173, 720, 6, 38, 93, 234, 1681, 554, 1835, 93, 233, 407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 1767, 156, 15, 766, 17, 101, 1975, 1326, 75, 34, 1459, 303, 155, 104, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 594, 1910, 110, 39, 109, 360, 58, 49, 1286, 52, 559, 1581, 31, 11, 243, 627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 103, 0, 37, 1, 344, 78, 32, 81, 26, 23, 445, 1958, 51, 365, 198, 1820, 925, 988, 686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 401, 9, 3, 6, 1086, 21, 40, 1186, 763, 243, 1978, 168, 302, 7, 660, 925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 10, 6, 46, 23, 2, 1051, 620, 14, 48, 1267, 4, 24, 145, 535, 142, 1955, 1832, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 956, 143, 119, 2025, 93, 1, 53, 92, 209, 217, 753, 2777, 22, 151, 67, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 978, 43, 46, 88, 0, 405, 158, 486, 1208, 85, 1268, 431, 301, 219, 3, 2621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8259, 8611, 7619, 8345, 7798, 8811, 7753, 8340] avg=8192.0000 max=8811 max_rank=5 min=7619 min_rank=2 imbalance_ratio=1.075562 max_load_reduction=56725 max_load_reduction_pct=0.865555 moved_to_echo=32702 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1682, 64, 28, 251, 137, 848, 33, 12, 76, 106, 1605, 391, 1987, 36, 318, 356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 166, 2384, 247, 50, 858, 11, 90, 1556, 781, 75, 13, 1523, 630, 82, 48, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 733, 2475, 39, 61, 153, 188, 102, 42, 857, 33, 447, 2001, 69, 18, 216, 775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 153, 0, 131, 1, 297, 102, 27, 105, 40, 56, 854, 1205, 22, 184, 346, 1737, 1170, 618, 360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 307, 36, 2, 36, 2173, 1, 57, 967, 1056, 103, 1069, 219, 529, 22, 800, 1137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 71, 10, 38, 84, 13, 1114, 765, 4, 45, 998, 5, 30, 137, 359, 119, 2046, 1400, 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1929, 240, 33, 2444, 41, 1, 81, 127, 311, 147, 585, 2907, 244, 217, 66, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 797, 128, 64, 44, 3, 621, 94, 235, 1047, 107, 1257, 386, 289, 307, 13, 1639, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7930, 8989, 8209, 7587, 8514, 7839, 9437, 7031] avg=8192.0000 max=9437 max_rank=6 min=7031 min_rank=7 imbalance_ratio=1.151978 max_load_reduction=56099 max_load_reduction_pct=0.856003 moved_to_echo=32821 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1611, 77, 6, 177, 143, 360, 15, 19, 62, 166, 1593, 1553, 2007, 122, 228, 400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 1894, 279, 10, 678, 29, 99, 1122, 518, 137, 56, 2124, 1089, 135, 33, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 943, 2147, 54, 28, 94, 314, 68, 11, 745, 90, 518, 1690, 47, 21, 192, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 347, 1, 61, 8, 693, 29, 8, 32, 24, 50, 523, 2127, 50, 559, 37, 1664, 989, 912, 597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 358, 45, 6, 5, 1362, 35, 12, 1426, 967, 121, 2164, 75, 535, 6, 627, 616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 7, 0, 26, 191, 18, 568, 833, 16, 20, 925, 3, 12, 191, 265, 109, 1573, 1635, 932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1284, 218, 98, 3389, 66, 0, 41, 126, 759, 87, 349, 2236, 72, 111, 39, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 258, 23, 13, 40, 3, 359, 150, 626, 1393, 76, 1267, 386, 357, 121, 3, 2421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8539, 8559, 7449, 8820, 8360, 7370, 8943, 7496] avg=8192.0000 max=8943 max_rank=6 min=7370 min_rank=5 imbalance_ratio=1.091675 max_load_reduction=56593 max_load_reduction_pct=0.863541 moved_to_echo=32169 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1257, 42, 35, 461, 182, 450, 13, 35, 64, 376, 2194, 396, 1708, 186, 108, 858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 303, 1598, 273, 11, 372, 22, 43, 1516, 794, 17, 33, 1125, 451, 212, 106, 431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 707, 1920, 113, 21, 105, 709, 184, 163, 904, 31, 727, 1497, 48, 26, 191, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 282, 290, 7, 29, 5, 238, 118, 61, 192, 35, 34, 463, 1327, 32, 595, 411, 1299, 875, 1111, 936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 420, 15, 6, 11, 1851, 5, 145, 1681, 1183, 218, 1290, 394, 290, 12, 543, 713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 26, 3, 50, 82, 4, 722, 1085, 68, 60, 784, 10, 50, 298, 226, 124, 1922, 1915, 1461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 839, 179, 99, 1323, 158, 32, 190, 60, 360, 307, 584, 2731, 22, 198, 126, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 820, 36, 33, 57, 5, 1137, 216, 182, 809, 74, 1406, 416, 267, 185, 8, 2949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8365, 7307, 7997, 8340, 8777, 8917, 7233, 8600] avg=8192.0000 max=8917 max_rank=5 min=7233 min_rank=6 imbalance_ratio=1.088501 max_load_reduction=56619 max_load_reduction_pct=0.863937 moved_to_echo=33527 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1823, 45, 34, 300, 95, 653, 52, 51, 104, 192, 1954, 454, 2109, 99, 125, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 421, 1669, 139, 48, 884, 11, 59, 1539, 1204, 46, 11, 1777, 700, 189, 70, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 782, 1437, 128, 12, 147, 177, 94, 45, 626, 24, 911, 2044, 27, 0, 201, 749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 265, 152, 6, 29, 1, 164, 75, 47, 55, 33, 48, 504, 1107, 40, 260, 135, 1894, 1204, 935, 860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 456, 19, 1, 15, 1338, 7, 43, 1005, 1350, 92, 1655, 231, 451, 10, 624, 918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 20, 8, 54, 19, 11, 1470, 689, 17, 40, 1188, 53, 37, 117, 145, 28, 1871, 2103, 1342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1156, 254, 48, 1829, 139, 4, 28, 65, 223, 134, 710, 2648, 63, 123, 56, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 829, 28, 23, 16, 27, 895, 161, 162, 1302, 83, 1006, 435, 211, 188, 22, 2469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8424, 9006, 7404, 7814, 8215, 9319, 7497, 7857] avg=8192.0000 max=9319 max_rank=5 min=7404 min_rank=2 imbalance_ratio=1.137573 max_load_reduction=56217 max_load_reduction_pct=0.857803 moved_to_echo=32888 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1365, 81, 20, 440, 113, 825, 1, 7, 136, 273, 1522, 352, 1570, 94, 177, 387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 235, 1345, 284, 33, 919, 36, 29, 1509, 489, 70, 25, 1199, 452, 123, 12, 376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 878, 2834, 125, 78, 175, 272, 175, 99, 1378, 40, 576, 1639, 65, 9, 257, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 315, 3, 56, 9, 660, 54, 34, 188, 28, 21, 756, 1127, 28, 279, 236, 1409, 959, 1028, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 365, 54, 1, 19, 1611, 7, 76, 1269, 1083, 94, 1524, 249, 482, 5, 368, 923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 18, 2, 51, 52, 11, 519, 705, 39, 18, 862, 9, 112, 282, 241, 92, 1823, 1666, 846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2628, 39, 39, 1753, 126, 7, 88, 169, 431, 338, 490, 2591, 32, 231, 104, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1090, 24, 28, 48, 8, 783, 114, 301, 1128, 63, 1940, 366, 244, 369, 12, 2727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7363, 7136, 9244, 7916, 8130, 7371, 9131, 9245] avg=8192.0000 max=9245 max_rank=7 min=7136 min_rank=1 imbalance_ratio=1.128540 max_load_reduction=56291 max_load_reduction_pct=0.858932 moved_to_echo=33877 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1762, 65, 17, 348, 173, 720, 6, 38, 93, 234, 1681, 554, 1835, 93, 233, 407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 1767, 156, 15, 766, 17, 101, 1975, 1326, 75, 34, 1459, 303, 155, 104, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 594, 1910, 110, 39, 109, 360, 58, 49, 1286, 52, 559, 1581, 31, 11, 243, 627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 103, 0, 37, 1, 344, 78, 32, 81, 26, 23, 445, 1958, 51, 365, 198, 1820, 925, 988, 686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 401, 9, 3, 6, 1086, 21, 40, 1186, 763, 243, 1978, 168, 302, 7, 660, 925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 10, 6, 46, 23, 2, 1051, 620, 14, 48, 1267, 4, 24, 145, 535, 142, 1955, 1832, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 956, 143, 119, 2025, 93, 1, 53, 92, 209, 217, 753, 2777, 22, 151, 67, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 978, 43, 46, 88, 0, 405, 158, 486, 1208, 85, 1268, 431, 301, 219, 3, 2621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8259, 8611, 7619, 8345, 7798, 8811, 7753, 8340] avg=8192.0000 max=8811 max_rank=5 min=7619 min_rank=2 imbalance_ratio=1.075562 max_load_reduction=56725 max_load_reduction_pct=0.865555 moved_to_echo=32702 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1823, 45, 34, 300, 95, 653, 52, 51, 104, 192, 1954, 454, 2109, 99, 125, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 421, 1669, 139, 48, 884, 11, 59, 1539, 1204, 46, 11, 1777, 700, 189, 70, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 782, 1437, 128, 12, 147, 177, 94, 45, 626, 24, 911, 2044, 27, 0, 201, 749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 265, 152, 6, 29, 1, 164, 75, 47, 55, 33, 48, 504, 1107, 40, 260, 135, 1894, 1204, 935, 860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 456, 19, 1, 15, 1338, 7, 43, 1005, 1350, 92, 1655, 231, 451, 10, 624, 918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 20, 8, 54, 19, 11, 1470, 689, 17, 40, 1188, 53, 37, 117, 145, 28, 1871, 2103, 1342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1156, 254, 48, 1829, 139, 4, 28, 65, 223, 134, 710, 2648, 63, 123, 56, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 829, 28, 23, 16, 27, 895, 161, 162, 1302, 83, 1006, 435, 211, 188, 22, 2469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8424, 9006, 7404, 7814, 8215, 9319, 7497, 7857] avg=8192.0000 max=9319 max_rank=5 min=7404 min_rank=2 imbalance_ratio=1.137573 max_load_reduction=56217 max_load_reduction_pct=0.857803 moved_to_echo=32888 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1311, 133, 14, 471, 269, 921, 14, 52, 65, 336, 1992, 615, 1704, 128, 116, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 1632, 291, 3, 442, 11, 160, 2050, 728, 41, 144, 1375, 716, 163, 107, 271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 1912, 60, 37, 102, 501, 132, 41, 1399, 67, 766, 1813, 85, 57, 156, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 444, 246, 2, 44, 2, 326, 97, 16, 203, 15, 27, 598, 1585, 41, 434, 342, 1354, 1060, 1102, 717, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 361, 8, 1, 12, 1186, 8, 83, 1486, 1110, 213, 1137, 96, 316, 19, 485, 778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 2, 2, 57, 88, 24, 486, 839, 32, 43, 674, 2, 17, 340, 254, 78, 1785, 1646, 1120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1434, 115, 34, 1751, 122, 0, 87, 102, 323, 122, 746, 2536, 4, 124, 80, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 903, 82, 67, 46, 8, 689, 261, 399, 909, 69, 1362, 557, 120, 550, 17, 2925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8578, 8181, 8661, 8655, 7299, 7535, 7663, 8964] avg=8192.0000 max=8964 max_rank=7 min=7299 min_rank=4 imbalance_ratio=1.094238 max_load_reduction=56572 max_load_reduction_pct=0.863220 moved_to_echo=31461 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1257, 42, 35, 461, 182, 450, 13, 35, 64, 376, 2194, 396, 1708, 186, 108, 858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 303, 1598, 273, 11, 372, 22, 43, 1516, 794, 17, 33, 1125, 451, 212, 106, 431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 707, 1920, 113, 21, 105, 709, 184, 163, 904, 31, 727, 1497, 48, 26, 191, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 282, 290, 7, 29, 5, 238, 118, 61, 192, 35, 34, 463, 1327, 32, 595, 411, 1299, 875, 1111, 936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 420, 15, 6, 11, 1851, 5, 145, 1681, 1183, 218, 1290, 394, 290, 12, 543, 713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 26, 3, 50, 82, 4, 722, 1085, 68, 60, 784, 10, 50, 298, 226, 124, 1922, 1915, 1461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 839, 179, 99, 1323, 158, 32, 190, 60, 360, 307, 584, 2731, 22, 198, 126, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 820, 36, 33, 57, 5, 1137, 216, 182, 809, 74, 1406, 416, 267, 185, 8, 2949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8365, 7307, 7997, 8340, 8777, 8917, 7233, 8600] avg=8192.0000 max=8917 max_rank=5 min=7233 min_rank=6 imbalance_ratio=1.088501 max_load_reduction=56619 max_load_reduction_pct=0.863937 moved_to_echo=33527 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. @@ -1267,44 +1273,40 @@ WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] glob warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. - warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[2048, 1920, 128, 512, 128, 3328, 640, 256, 896, 256, 384, 4736, 12032, 384, 3072, 1920, 12928, 8320, 7680, 5376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[384, 256, 128, 384, 640, 128, 6656, 6400, 256, 384, 8064, 128, 384, 1792, 2432, 896, 15232, 13952, 8320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[11264, 1408, 640, 16512, 896, 128, 640, 896, 3072, 1536, 4864, 21632, 512, 1408, 640, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[5760, 17152, 768, 384, 1152, 2816, 1024, 512, 8320, 384, 5504, 14208, 512, 256, 1792, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[3072, 384, 128, 128, 11904, 128, 512, 10368, 8320, 1152, 13056, 1664, 3456, 128, 4992, 7168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[6016, 512, 384, 384, 128, 5504, 1280, 2816, 9600, 640, 11008, 3456, 2048, 2432, 128, 20224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[384, 256, 128, 384, 640, 128, 6656, 6400, 256, 384, 8064, 128, 384, 1792, 2432, 896, 15232, 13952, 8320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[12544, 640, 256, 2816, 1280, 5632, 256, 256, 768, 1920, 14208, 4992, 14848, 896, 1664, 3584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[2048, 1920, 128, 512, 128, 3328, 640, 256, 896, 256, 384, 4736, 12032, 384, 3072, 1920, 12928, 8320, 7680, 5376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[6016, 512, 384, 384, 128, 5504, 1280, 2816, 9600, 640, 11008, 3456, 2048, 2432, 128, 20224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[11264, 1408, 640, 16512, 896, 128, 640, 896, 3072, 1536, 4864, 21632, 512, 1408, 640, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[1792, 14080, 1792, 256, 5504, 256, 768, 12800, 6912, 512, 384, 12160, 4992, 1280, 640, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[3072, 384, 128, 128, 11904, 128, 512, 10368, 8320, 1152, 13056, 1664, 3456, 128, 4992, 7168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. @@ -1321,30 +1323,30 @@ WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] glob warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[324, 775, 0, 3049, 0, 0, 2, 7, 0, 168, 8068, 0, 0, 48, 0, 1, 0, 25, 2, 42, 3, 0, 24, 0, 0, 0, 1, 443, 5, 0, 1, 7, 0, 0, 0, 0, 0, 99, 1, 1, 0, 31, 22, 23, 150, 0, 4, 1, 1, 11, 0, 0, 1265, 0, 3, 6109, 0, 242, 5403, 0, 0, 92, 0, 571, 19, 0, 0, 0, 565, 7899, 46, 0, 0, 0, 0, 68, 0, 0, 3, 0, 0, 43, 154, 573, 37, 0, 53, 0, 5181, 0, 3309, 0, 42, 0, 6072, 15, 0, 0, 0, 266, 0, 5079, 0, 0, 3, 26, 27, 0, 0, 612, 0, 0, 0, 26, 10, 0, 2, 0, 1458, 27, 0, 0, 3, 0, 6894, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1429, 392, 0, 7003, 12, 0, 3, 1, 34, 86, 6511, 0, 1, 63, 0, 0, 0, 0, 216, 355, 0, 1, 155, 0, 87, 1, 148, 2942, 6, 0, 1, 0, 12, 1, 0, 0, 1, 1555, 0, 6, 0, 42, 5, 2, 4, 15, 0, 0, 0, 283, 4, 9, 1365, 7, 27, 6376, 2, 271, 102, 1, 0, 15, 0, 597, 13, 0, 3, 0, 1450, 8182, 0, 0, 2, 0, 0, 743, 0, 0, 1, 0, 0, 1, 10, 2, 0, 0, 0, 11, 5948, 0, 915, 10, 17, 0, 2610, 5, 0, 0, 0, 0, 5, 2277, 12, 1, 3, 8, 0, 15, 1, 10, 10, 7, 0, 80, 0, 0, 0, 1, 7628, 0, 0, 1, 7, 0, 5390, 8, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1144, 22, 2, 944, 0, 0, 7, 29, 0, 12, 7774, 0, 20, 5, 0, 99, 0, 3, 151, 911, 0, 0, 0, 0, 38, 0, 23, 57, 3, 0, 24, 248, 0, 0, 3, 15, 98, 3, 0, 0, 4, 262, 42, 25, 5, 184, 0, 0, 14, 23, 0, 0, 4475, 1, 310, 6856, 0, 1970, 362, 155, 0, 2834, 69, 869, 18, 0, 0, 0, 1314, 8136, 0, 0, 0, 52, 0, 66, 0, 17, 1, 0, 0, 143, 5, 0, 0, 0, 29, 0, 7982, 1, 871, 0, 57, 7, 1674, 284, 1, 36, 0, 0, 15, 486, 0, 0, 0, 0, 180, 2, 3, 0, 0, 0, 1, 166, 0, 0, 58, 0, 5496, 158, 6, 0, 1, 0, 8108, 67, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1331, 242, 0, 6709, 17, 0, 28, 2, 0, 480, 2779, 0, 0, 7, 0, 1, 0, 1, 0, 2008, 0, 0, 12, 0, 0, 0, 22, 119, 0, 0, 24, 101, 0, 0, 1, 0, 18, 243, 0, 0, 0, 2773, 4, 3, 0, 167, 0, 0, 0, 145, 0, 0, 681, 1, 202, 7580, 0, 7187, 550, 42, 0, 1664, 0, 2132, 224, 1, 0, 9, 1446, 8168, 10, 1, 0, 0, 0, 190, 0, 13, 20, 0, 0, 28, 0, 0, 0, 0, 0, 0, 5109, 0, 904, 0, 9, 0, 2149, 32, 0, 0, 9, 35, 0, 93, 0, 174, 28, 0, 0, 153, 0, 216, 165, 0, 0, 12, 0, 1, 667, 1, 138, 79, 0, 0, 196, 0, 7979, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[171, 212, 0, 647, 0, 0, 0, 0, 0, 0, 7648, 0, 2, 0, 0, 5, 5, 193, 97, 524, 0, 0, 1862, 0, 5, 0, 48, 5753, 2, 0, 0, 1, 194, 0, 0, 0, 32, 26, 0, 1, 148, 6, 24, 0, 239, 9, 3, 0, 0, 86, 49, 0, 780, 66, 212, 5369, 3, 287, 438, 15, 0, 516, 0, 1486, 0, 0, 0, 0, 1578, 6956, 9, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 283, 796, 4, 1, 0, 1876, 2, 8169, 0, 1026, 1, 373, 1, 6852, 16, 0, 0, 0, 0, 0, 1949, 0, 0, 98, 18, 180, 19, 1, 360, 64, 2, 0, 0, 1, 2, 0, 0, 40, 0, 0, 11, 6, 0, 7650, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1429, 392, 0, 7003, 12, 0, 3, 1, 34, 86, 6511, 0, 1, 63, 0, 0, 0, 0, 216, 355, 0, 1, 155, 0, 87, 1, 148, 2942, 6, 0, 1, 0, 12, 1, 0, 0, 1, 1555, 0, 6, 0, 42, 5, 2, 4, 15, 0, 0, 0, 283, 4, 9, 1365, 7, 27, 6376, 2, 271, 102, 1, 0, 15, 0, 597, 13, 0, 3, 0, 1450, 8182, 0, 0, 2, 0, 0, 743, 0, 0, 1, 0, 0, 1, 10, 2, 0, 0, 0, 11, 5948, 0, 915, 10, 17, 0, 2610, 5, 0, 0, 0, 0, 5, 2277, 12, 1, 3, 8, 0, 15, 1, 10, 10, 7, 0, 80, 0, 0, 0, 1, 7628, 0, 0, 1, 7, 0, 5390, 8, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[688, 184, 0, 2055, 0, 0, 5, 107, 0, 0, 3547, 0, 0, 269, 0, 61, 0, 109, 0, 1, 16, 0, 22, 0, 137, 4, 443, 700, 76, 1, 165, 119, 30, 0, 0, 1, 6, 0, 1, 0, 134, 0, 302, 1, 2, 0, 0, 4, 7, 969, 0, 1, 2935, 0, 6, 3686, 0, 166, 3327, 6612, 0, 36, 23, 5588, 0, 4, 0, 0, 1908, 8002, 95, 36, 3, 10, 0, 10, 3, 0, 3, 0, 0, 51, 126, 225, 0, 23, 29, 0, 7844, 0, 5135, 0, 9, 0, 7, 5, 27, 18, 2, 2, 4, 3366, 1, 0, 40, 0, 32, 422, 90, 0, 14, 0, 11, 4, 0, 0, 10, 2, 1437, 1, 2, 1, 0, 0, 3976, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[4190, 9, 13, 3708, 1, 0, 0, 2, 0, 2, 8052, 2, 0, 0, 1, 0, 0, 2, 0, 955, 0, 0, 0, 0, 1, 0, 50, 33, 10, 3, 34, 9, 0, 0, 1, 0, 0, 1, 6, 1, 3, 0, 0, 3, 1910, 0, 4, 0, 2, 1, 0, 0, 4502, 3, 156, 45, 0, 42, 1427, 4, 0, 143, 0, 2777, 15, 0, 0, 2, 75, 7645, 9, 1, 0, 0, 0, 0, 22, 0, 0, 57, 0, 13, 144, 5, 17, 183, 1012, 4, 7976, 0, 2294, 9, 41, 2, 2049, 743, 1, 0, 0, 0, 0, 5175, 0, 0, 0, 81, 1, 0, 0, 22, 0, 0, 0, 201, 20, 2, 1, 0, 1727, 0, 0, 0, 15, 25, 7821, 18, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[4, 814, 0, 2, 0, 0, 44, 62, 0, 0, 7372, 0, 0, 2024, 1, 1253, 0, 0, 29, 45, 0, 38, 0, 0, 0, 0, 2, 290, 0, 1, 1, 10, 1, 22, 0, 1, 3, 0, 0, 0, 0, 3706, 0, 4, 4, 0, 1, 0, 3, 0, 0, 4, 7678, 0, 0, 3976, 0, 2454, 1356, 1995, 0, 2, 93, 4201, 0, 0, 0, 109, 119, 7847, 0, 1, 71, 53, 0, 107, 0, 3, 201, 0, 4, 2491, 108, 0, 2, 3, 440, 0, 6177, 10, 2119, 2, 4, 0, 12, 365, 0, 0, 0, 0, 18, 1843, 0, 1, 0, 24, 38, 304, 30, 0, 0, 0, 0, 0, 0, 0, 33, 0, 513, 2, 0, 0, 20, 0, 4711, 0, 0, 255] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[688, 184, 0, 2055, 0, 0, 5, 107, 0, 0, 3547, 0, 0, 269, 0, 61, 0, 109, 0, 1, 16, 0, 22, 0, 137, 4, 443, 700, 76, 1, 165, 119, 30, 0, 0, 1, 6, 0, 1, 0, 134, 0, 302, 1, 2, 0, 0, 4, 7, 969, 0, 1, 2935, 0, 6, 3686, 0, 166, 3327, 6612, 0, 36, 23, 5588, 0, 4, 0, 0, 1908, 8002, 95, 36, 3, 10, 0, 10, 3, 0, 3, 0, 0, 51, 126, 225, 0, 23, 29, 0, 7844, 0, 5135, 0, 9, 0, 7, 5, 27, 18, 2, 2, 4, 3366, 1, 0, 40, 0, 32, 422, 90, 0, 14, 0, 11, 4, 0, 0, 10, 2, 1437, 1, 2, 1, 0, 0, 3976, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1331, 242, 0, 6709, 17, 0, 28, 2, 0, 480, 1317, 0, 0, 7, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2008, 0, 0, 12, 0, 0, 0, 22, 119, 0, 0, 24, 101, 2344, 1104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 18, 243, 0, 0, 0, 2773, 4, 3, 0, 167, 0, 0, 7580, 1209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 0, 139, 1, 202, 0, 0, 7187, 550, 42, 0, 1664, 0, 2132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 224, 1, 0, 9, 1446, 7112, 10, 1, 0, 0, 0, 190, 0, 13, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 1556, 0, 904, 0, 9, 0, 2149, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 35, 0, 93, 0, 174, 28, 0, 0, 153, 0, 216, 165, 0, 542, 1056, 1162, 358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 1, 667, 1, 138, 79, 0, 0, 196, 0, 6817, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10134, 5735, 11998, 12062, 9026, 4678, 3991, 7912] avg=8192.0000 max=12062 max_rank=3 min=3991 min_rank=6 imbalance_ratio=1.472412 max_load_reduction=53474 max_load_reduction_pct=0.815948 moved_to_echo=25607 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[324, 775, 0, 3049, 0, 0, 2, 7, 0, 168, 8068, 0, 0, 48, 0, 1, 0, 25, 2, 42, 3, 0, 24, 0, 0, 0, 1, 443, 5, 0, 1, 7, 0, 0, 0, 0, 0, 99, 1, 1, 0, 31, 22, 23, 150, 0, 4, 1, 1, 11, 0, 0, 1265, 0, 3, 6109, 0, 242, 5403, 0, 0, 92, 0, 571, 19, 0, 0, 0, 565, 7899, 46, 0, 0, 0, 0, 68, 0, 0, 3, 0, 0, 43, 154, 573, 37, 0, 53, 0, 5181, 0, 3309, 0, 42, 0, 6072, 15, 0, 0, 0, 266, 0, 5079, 0, 0, 3, 26, 27, 0, 0, 612, 0, 0, 0, 26, 10, 0, 2, 0, 1458, 27, 0, 0, 3, 0, 6894, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4190, 9, 13, 3708, 1, 0, 0, 2, 0, 2, 3805, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 955, 0, 0, 0, 0, 1, 0, 50, 33, 10, 3, 34, 9, 3665, 3203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 6, 1, 3, 0, 0, 3, 1910, 0, 4, 0, 45, 1894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 910, 3, 156, 0, 0, 42, 1427, 4, 0, 143, 0, 2777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 2, 75, 6653, 9, 1, 0, 0, 0, 0, 22, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 144, 5, 17, 183, 1012, 4, 2417, 0, 2294, 9, 41, 2, 2049, 743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 5175, 0, 0, 0, 81, 1, 0, 0, 22, 0, 0, 3592, 992, 1141, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 201, 20, 2, 1, 0, 1727, 0, 0, 0, 15, 25, 6680, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11733, 7965, 3868, 5465, 6834, 8933, 12049, 8689] avg=8192.0000 max=12049 max_rank=6 min=3868 min_rank=2 imbalance_ratio=1.470825 max_load_reduction=53487 max_load_reduction_pct=0.816147 moved_to_echo=36505 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1331, 242, 0, 6709, 17, 0, 28, 2, 0, 480, 1317, 0, 0, 7, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2008, 0, 0, 12, 0, 0, 0, 22, 119, 0, 0, 24, 101, 2344, 1104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 18, 243, 0, 0, 0, 2773, 4, 3, 0, 167, 0, 0, 7580, 1209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 0, 139, 1, 202, 0, 0, 7187, 550, 42, 0, 1664, 0, 2132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 224, 1, 0, 9, 1446, 7112, 10, 1, 0, 0, 0, 190, 0, 13, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 1556, 0, 904, 0, 9, 0, 2149, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 35, 0, 93, 0, 174, 28, 0, 0, 153, 0, 216, 165, 0, 542, 1056, 1162, 358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 1, 667, 1, 138, 79, 0, 0, 196, 0, 6817, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10134, 5735, 11998, 12062, 9026, 4678, 3991, 7912] avg=8192.0000 max=12062 max_rank=3 min=3991 min_rank=6 imbalance_ratio=1.472412 max_load_reduction=53474 max_load_reduction_pct=0.815948 moved_to_echo=25607 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1144, 22, 2, 944, 0, 0, 7, 29, 0, 12, 3681, 0, 20, 5, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 151, 911, 0, 0, 0, 0, 38, 0, 23, 57, 3, 0, 24, 248, 3663, 3090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 15, 98, 3, 0, 0, 4, 262, 42, 25, 5, 184, 0, 0, 6856, 1890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 23, 0, 0, 909, 1, 310, 0, 0, 1970, 362, 155, 0, 2834, 69, 869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 1314, 7084, 0, 0, 0, 52, 0, 66, 0, 17, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 5, 0, 0, 0, 29, 0, 2429, 1, 871, 0, 57, 7, 1674, 284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 36, 0, 0, 15, 486, 0, 0, 0, 0, 180, 2, 3, 0, 0, 0, 3566, 1052, 1181, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 166, 0, 0, 58, 0, 5496, 158, 6, 0, 1, 0, 6927, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5965, 8211, 9387, 7516, 8552, 5500, 7525, 12880] avg=8192.0000 max=12880 max_rank=7 min=5500 min_rank=5 imbalance_ratio=1.572266 max_load_reduction=52656 max_load_reduction_pct=0.803467 moved_to_echo=34457 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1429, 392, 0, 7003, 12, 0, 3, 1, 34, 86, 3083, 0, 1, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 355, 0, 1, 155, 0, 87, 1, 148, 2942, 6, 0, 1, 0, 2729, 2588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 0, 0, 1, 1555, 0, 6, 0, 42, 5, 2, 4, 15, 0, 0, 6376, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 283, 4, 9, 278, 7, 27, 0, 2, 271, 102, 1, 0, 15, 0, 597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 3, 0, 1450, 7124, 0, 0, 2, 0, 0, 743, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 10, 2, 0, 0, 0, 11, 1811, 0, 915, 10, 17, 0, 2610, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2277, 12, 1, 3, 8, 0, 15, 1, 10, 10, 7, 1087, 1058, 785, 840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 1, 7628, 0, 0, 1, 7, 0, 4605, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12107, 9229, 9427, 1596, 9336, 5392, 6119, 12330] avg=8192.0000 max=12330 max_rank=7 min=1596 min_rank=3 imbalance_ratio=1.505127 max_load_reduction=53206 max_load_reduction_pct=0.811859 moved_to_echo=33177 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[324, 775, 0, 3049, 0, 0, 2, 7, 0, 168, 3820, 0, 0, 48, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 2, 42, 3, 0, 24, 0, 0, 0, 1, 443, 5, 0, 1, 7, 2377, 3207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 1, 1, 0, 31, 22, 23, 150, 0, 4, 1, 6109, 1226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 11, 0, 0, 257, 0, 3, 0, 0, 242, 5403, 0, 0, 92, 0, 571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 565, 6878, 46, 0, 0, 0, 0, 68, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 154, 573, 37, 0, 53, 0, 1578, 0, 3309, 0, 42, 0, 6072, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 266, 0, 5079, 0, 0, 3, 26, 27, 0, 0, 612, 0, 0, 1008, 1021, 1004, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 10, 0, 2, 0, 1458, 27, 0, 0, 3, 0, 5890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 6137, 7667, 6580, 7579, 11876, 10087, 7416] avg=8192.0000 max=11876 max_rank=5 min=6137 min_rank=1 imbalance_ratio=1.449707 max_load_reduction=53660 max_load_reduction_pct=0.818787 moved_to_echo=36958 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[324, 775, 0, 3049, 0, 0, 2, 7, 0, 168, 3820, 0, 0, 48, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 2, 42, 3, 0, 24, 0, 0, 0, 1, 443, 5, 0, 1, 7, 2377, 3207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 1, 1, 0, 31, 22, 23, 150, 0, 4, 1, 6109, 1226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 11, 0, 0, 257, 0, 3, 0, 0, 242, 5403, 0, 0, 92, 0, 571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 565, 6878, 46, 0, 0, 0, 0, 68, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 154, 573, 37, 0, 53, 0, 1578, 0, 3309, 0, 42, 0, 6072, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 266, 0, 5079, 0, 0, 3, 26, 27, 0, 0, 612, 0, 0, 1008, 1021, 1004, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 10, 0, 2, 0, 1458, 27, 0, 0, 3, 0, 5890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 6137, 7667, 6580, 7579, 11876, 10087, 7416] avg=8192.0000 max=11876 max_rank=5 min=6137 min_rank=1 imbalance_ratio=1.449707 max_load_reduction=53660 max_load_reduction_pct=0.818787 moved_to_echo=36958 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[171, 212, 0, 647, 0, 0, 0, 0, 0, 0, 3621, 0, 2, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 193, 97, 524, 0, 0, 1862, 0, 5, 0, 48, 5753, 2, 0, 0, 1, 3748, 3040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 194, 0, 0, 0, 32, 26, 0, 1, 148, 6, 24, 0, 239, 9, 3, 0, 5369, 1934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 49, 0, 159, 66, 212, 0, 3, 287, 438, 15, 0, 516, 0, 1486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1578, 6057, 9, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 283, 796, 4, 1, 0, 1876, 2, 2487, 0, 1026, 1, 373, 1, 6852, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1949, 0, 0, 98, 18, 180, 19, 1, 360, 64, 2, 621, 899, 1114, 987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 40, 0, 0, 11, 6, 0, 6536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4658, 15278, 7985, 3317, 7672, 13718, 6312, 6596] avg=8192.0000 max=15278 max_rank=1 min=3317 min_rank=3 imbalance_ratio=1.864990 max_load_reduction=50258 max_load_reduction_pct=0.766876 moved_to_echo=34298 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[688, 184, 0, 2055, 0, 0, 5, 107, 0, 0, 1681, 0, 0, 269, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 0, 1, 16, 0, 22, 0, 137, 4, 443, 700, 76, 1, 165, 119, 3599, 1409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 1, 6, 0, 1, 0, 134, 0, 302, 1, 2, 0, 0, 4, 3686, 1857, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 969, 0, 1, 597, 0, 6, 0, 0, 166, 3327, 6612, 0, 36, 23, 5588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1908, 6968, 95, 36, 3, 10, 0, 10, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 126, 225, 0, 23, 29, 0, 2388, 0, 5135, 0, 9, 0, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 18, 2, 2, 4, 3366, 1, 0, 40, 0, 32, 422, 90, 0, 14, 0, 2338, 1034, 579, 457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4, 0, 0, 10, 2, 1437, 1, 2, 1, 0, 0, 3397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5050, 6801, 6024, 17332, 9040, 7998, 8426, 4865] avg=8192.0000 max=17332 max_rank=3 min=4865 min_rank=7 imbalance_ratio=2.115723 max_load_reduction=48204 max_load_reduction_pct=0.735535 moved_to_echo=30329 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1144, 22, 2, 944, 0, 0, 7, 29, 0, 12, 3681, 0, 20, 5, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 151, 911, 0, 0, 0, 0, 38, 0, 23, 57, 3, 0, 24, 248, 3663, 3090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 15, 98, 3, 0, 0, 4, 262, 42, 25, 5, 184, 0, 0, 6856, 1890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 23, 0, 0, 909, 1, 310, 0, 0, 1970, 362, 155, 0, 2834, 69, 869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 1314, 7084, 0, 0, 0, 52, 0, 66, 0, 17, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 5, 0, 0, 0, 29, 0, 2429, 1, 871, 0, 57, 7, 1674, 284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 36, 0, 0, 15, 486, 0, 0, 0, 0, 180, 2, 3, 0, 0, 0, 3566, 1052, 1181, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 166, 0, 0, 58, 0, 5496, 158, 6, 0, 1, 0, 6927, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5965, 8211, 9387, 7516, 8552, 5500, 7525, 12880] avg=8192.0000 max=12880 max_rank=7 min=5500 min_rank=5 imbalance_ratio=1.572266 max_load_reduction=52656 max_load_reduction_pct=0.803467 moved_to_echo=34457 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1429, 392, 0, 7003, 12, 0, 3, 1, 34, 86, 3083, 0, 1, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 355, 0, 1, 155, 0, 87, 1, 148, 2942, 6, 0, 1, 0, 2729, 2588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 0, 0, 1, 1555, 0, 6, 0, 42, 5, 2, 4, 15, 0, 0, 6376, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 283, 4, 9, 278, 7, 27, 0, 2, 271, 102, 1, 0, 15, 0, 597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 3, 0, 1450, 7124, 0, 0, 2, 0, 0, 743, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 10, 2, 0, 0, 0, 11, 1811, 0, 915, 10, 17, 0, 2610, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2277, 12, 1, 3, 8, 0, 15, 1, 10, 10, 7, 1087, 1058, 785, 840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 1, 7628, 0, 0, 1, 7, 0, 4605, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12107, 9229, 9427, 1596, 9336, 5392, 6119, 12330] avg=8192.0000 max=12330 max_rank=7 min=1596 min_rank=3 imbalance_ratio=1.505127 max_load_reduction=53206 max_load_reduction_pct=0.811859 moved_to_echo=33177 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4, 814, 0, 2, 0, 0, 44, 62, 0, 0, 3491, 0, 0, 2024, 1, 1253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 45, 0, 38, 0, 0, 0, 0, 2, 290, 0, 1, 1, 10, 2834, 2930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 22, 0, 1, 3, 0, 0, 0, 0, 3706, 0, 4, 4, 0, 1, 0, 3976, 1462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4, 1560, 0, 0, 0, 0, 2454, 1356, 1995, 0, 2, 93, 4201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 119, 6833, 0, 1, 71, 53, 0, 107, 0, 3, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2491, 108, 0, 2, 3, 440, 0, 1881, 10, 2119, 2, 4, 0, 12, 365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 1843, 0, 1, 0, 24, 38, 304, 30, 0, 0, 0, 6118, 1014, 686, 951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 513, 2, 0, 0, 20, 0, 4025, 0, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7695, 6180, 9180, 11668, 7497, 7441, 11027, 4848] avg=8192.0000 max=11668 max_rank=3 min=4848 min_rank=7 imbalance_ratio=1.424316 max_load_reduction=53868 max_load_reduction_pct=0.821960 moved_to_echo=30813 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[171, 212, 0, 647, 0, 0, 0, 0, 0, 0, 3621, 0, 2, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 193, 97, 524, 0, 0, 1862, 0, 5, 0, 48, 5753, 2, 0, 0, 1, 3748, 3040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 194, 0, 0, 0, 32, 26, 0, 1, 148, 6, 24, 0, 239, 9, 3, 0, 5369, 1934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 49, 0, 159, 66, 212, 0, 3, 287, 438, 15, 0, 516, 0, 1486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1578, 6057, 9, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 283, 796, 4, 1, 0, 1876, 2, 2487, 0, 1026, 1, 373, 1, 6852, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1949, 0, 0, 98, 18, 180, 19, 1, 360, 64, 2, 621, 899, 1114, 987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 40, 0, 0, 11, 6, 0, 6536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4658, 15278, 7985, 3317, 7672, 13718, 6312, 6596] avg=8192.0000 max=15278 max_rank=1 min=3317 min_rank=3 imbalance_ratio=1.864990 max_load_reduction=50258 max_load_reduction_pct=0.766876 moved_to_echo=34298 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. @@ -1369,17 +1371,17 @@ WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] glob warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 384, 512, 4864, 128, 128, 2176, 0, 384, 128, 768, 10368, 128, 128, 256, 512, 24960, 20608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 384, 128, 20352, 128, 256, 256, 256, 512, 1024, 128, 1280, 256, 128, 18944, 8192, 7680, 6784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 1536, 128, 128, 4864, 128, 1024, 0, 128, 12672, 13056, 8832, 0, 5376, 256, 18304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 384, 128, 20352, 128, 256, 256, 256, 512, 1024, 128, 1280, 256, 128, 18944, 8192, 7680, 6784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 3072, 1408, 896, 128, 256, 3456, 128, 16640, 128, 16640, 128, 640, 128, 21504, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[9344, 2688, 128, 24192, 128, 0, 128, 256, 128, 768, 24576, 128, 128, 2432, 128, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[256, 128, 128, 128, 256, 2048, 128, 128, 384, 6912, 512, 128, 2432, 384, 128, 128, 40064, 12928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] @@ -1390,153 +1392,153 @@ WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] glob WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[384, 128, 128, 128, 8576, 54784, 256, 128, 128, 128, 0, 1280, 128, 128, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 6172, 0, 0, 0, 3, 0, 0, 0, 13, 0, 0, 1, 1, 144, 6, 0, 0, 0, 0, 0, 6, 0, 138, 11, 2, 0, 1, 0, 3, 0, 0, 1, 0, 0, 8, 16, 0, 0, 0, 0, 0, 12, 11, 7, 71, 2, 0, 0, 0, 0, 50, 0, 6162, 1, 22, 0, 957, 7581, 0, 2757, 0, 0, 0, 0, 1, 0, 113, 0, 0, 323, 70, 0, 0, 8029, 2, 0, 0, 6, 8128, 0, 0, 0, 0, 24, 0, 0, 0, 7, 0, 0, 0, 0, 0, 7, 0, 5, 0, 1, 0, 251, 0, 8007, 0, 0, 0, 0, 8018, 2, 0, 0, 0, 0, 0, 243, 0, 0, 8135, 0, 1, 3, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[3, 17, 0, 0, 0, 8165, 24, 0, 1, 623, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 3, 0, 0, 0, 0, 1, 7, 29, 3, 172, 7, 3, 0, 0, 0, 0, 0, 16, 0, 35, 17, 0, 0, 0, 0, 0, 0, 0, 0, 874, 0, 0, 0, 0, 0, 3092, 0, 7906, 0, 0, 0, 12, 7106, 0, 1, 0, 0, 9, 0, 0, 0, 0, 0, 0, 2049, 0, 2, 9, 8123, 2, 0, 0, 0, 1620, 0, 0, 0, 0, 0, 9, 2, 0, 39, 2954, 0, 0, 155, 0, 0, 176, 0, 0, 0, 0, 36, 2440, 7437, 0, 0, 0, 0, 3729, 0, 0, 13, 0, 0, 2, 0, 0, 0, 8166, 0, 0, 229, 15, 197, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 5, 0, 0, 0, 8005, 0, 0, 0, 0, 3, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 4, 0, 7, 4, 0, 0, 0, 0, 0, 75, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 382, 10, 1, 0, 0, 0, 1, 8190, 0, 0, 0, 0, 7914, 0, 1458, 0, 0, 69, 0, 5, 0, 2, 0, 0, 0, 0, 32, 0, 8103, 0, 0, 0, 77, 7449, 0, 0, 0, 2, 0, 5, 0, 0, 33, 88, 0, 0, 69, 0, 0, 127, 0, 0, 0, 1, 0, 22, 7126, 12, 0, 0, 7, 8188, 0, 10, 0, 0, 0, 2, 0, 0, 0, 7962, 11, 0, 4, 0, 60, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 7713, 1, 7, 0, 0, 3, 159, 0, 0, 0, 0, 0, 0, 812, 3, 0, 0, 43, 3, 0, 14, 0, 0, 0, 185, 0, 0, 0, 0, 27, 0, 0, 0, 2, 0, 0, 8, 0, 0, 0, 0, 0, 0, 8, 171, 0, 0, 0, 0, 0, 550, 0, 7700, 0, 0, 0, 3, 8127, 0, 64, 1, 0, 272, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 7620, 101, 5, 2, 8, 8073, 0, 100, 0, 0, 0, 0, 0, 0, 2038, 0, 0, 3, 3652, 0, 1, 0, 0, 157, 0, 0, 4, 5, 1235, 2, 0, 0, 0, 7865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 1, 119, 0, 471, 12] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[10, 0, 3, 0, 2, 6231, 0, 0, 2, 6132, 0, 0, 0, 7, 65, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 27, 0, 1, 0, 7730, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 1, 0, 0, 1, 0, 0, 5, 0, 0, 1, 21, 0, 1069, 0, 1290, 2, 0, 0, 0, 8148, 0, 6, 348, 0, 0, 0, 3, 0, 44, 0, 0, 0, 0, 0, 43, 7140, 180, 0, 45, 16, 0, 69, 0, 0, 0, 0, 0, 0, 38, 0, 3361, 1, 8, 158, 0, 0, 0, 12, 32, 0, 0, 0, 5392, 4432, 0, 0, 3, 0, 4530, 0, 0, 50, 5, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 623, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 5, 0, 0, 0, 8005, 0, 0, 0, 0, 3, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 4, 0, 7, 4, 0, 0, 0, 0, 0, 75, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 382, 10, 1, 0, 0, 0, 1, 8190, 0, 0, 0, 0, 7914, 0, 1458, 0, 0, 69, 0, 5, 0, 2, 0, 0, 0, 0, 32, 0, 8103, 0, 0, 0, 77, 7449, 0, 0, 0, 2, 0, 5, 0, 0, 33, 88, 0, 0, 69, 0, 0, 127, 0, 0, 0, 1, 0, 22, 7126, 12, 0, 0, 7, 8188, 0, 10, 0, 0, 0, 2, 0, 0, 0, 7962, 11, 0, 4, 0, 60, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[7, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 5, 23, 0, 0, 26, 21, 0, 5092, 0, 14, 0, 0, 0, 0, 0, 152, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 1, 5, 0, 2, 1, 0, 0, 0, 0, 122, 0, 18, 0, 0, 0, 0, 0, 0, 0, 3158, 0, 21, 0, 8, 7978, 0, 2249, 0, 0, 62, 0, 0, 0, 0, 0, 0, 12, 145, 0, 0, 5422, 1107, 0, 0, 0, 7606, 0, 1839, 0, 0, 0, 0, 0, 0, 2801, 0, 0, 7, 1, 6, 0, 2, 0, 0, 0, 0, 7525, 5, 4688, 0, 85, 0, 0, 8181, 900, 0, 0, 0, 0, 17, 0, 0, 203, 5949, 0, 0, 0, 0, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[10, 0, 3, 0, 2, 6231, 0, 0, 2, 6132, 0, 0, 0, 7, 65, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 27, 0, 1, 0, 7730, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 1, 0, 0, 1, 0, 0, 5, 0, 0, 1, 21, 0, 1069, 0, 1290, 2, 0, 0, 0, 8148, 0, 6, 348, 0, 0, 0, 3, 0, 44, 0, 0, 0, 0, 0, 43, 7140, 180, 0, 45, 16, 0, 69, 0, 0, 0, 0, 0, 0, 38, 0, 3361, 1, 8, 158, 0, 0, 0, 12, 32, 0, 0, 0, 5392, 4432, 0, 0, 3, 0, 4530, 0, 0, 50, 5, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 623, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[82, 1, 1, 0, 0, 7967, 0, 4, 0, 0, 0, 4, 1, 27, 0, 0, 0, 39, 4364, 30, 11, 0, 23, 0, 11, 0, 1, 0, 0, 66, 2, 26, 212, 0, 14, 0, 2, 0, 0, 0, 0, 17, 0, 3, 0, 0, 9, 0, 11, 313, 0, 0, 0, 0, 0, 1, 0, 7987, 0, 0, 4, 0, 8123, 0, 1974, 0, 0, 76, 5, 0, 0, 0, 0, 0, 4, 0, 2, 0, 7608, 0, 0, 5, 0, 0, 67, 90, 0, 0, 0, 459, 0, 0, 68, 0, 0, 0, 280, 0, 0, 445, 6, 1, 77, 0, 2560, 0, 5914, 0, 0, 7, 0, 7110, 54, 0, 12, 16, 0, 66, 475, 0, 57, 8124, 3, 331, 160, 117, 7, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[12, 0, 0, 0, 0, 66, 0, 0, 7, 68, 0, 0, 0, 0, 31, 46, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 0, 4949, 0, 0, 12, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7426, 0, 8145, 0, 0, 0, 1374, 6463, 0, 2564, 0, 0, 11, 0, 0, 0, 1, 0, 0, 6, 0, 0, 4, 8190, 205, 0, 0, 0, 4434, 0, 0, 0, 5, 0, 0, 0, 1, 529, 0, 0, 0, 3407, 0, 3, 19, 0, 0, 0, 0, 1, 0, 737, 0, 2, 0, 0, 6971, 3, 0, 697, 13, 0, 0, 0, 2, 2, 7831, 0, 0, 12, 1, 1263, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[3, 17, 0, 0, 0, 8165, 24, 0, 1, 623, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 3, 0, 0, 0, 0, 1, 7, 29, 3, 172, 7, 3, 0, 0, 0, 0, 0, 16, 0, 35, 17, 0, 0, 0, 0, 0, 0, 0, 0, 874, 0, 0, 0, 0, 0, 3092, 0, 7906, 0, 0, 0, 12, 7106, 0, 1, 0, 0, 9, 0, 0, 0, 0, 0, 0, 2049, 0, 2, 9, 8123, 2, 0, 0, 0, 1620, 0, 0, 0, 0, 0, 9, 2, 0, 39, 2954, 0, 0, 155, 0, 0, 176, 0, 0, 0, 0, 36, 2440, 7437, 0, 0, 0, 0, 3729, 0, 0, 13, 0, 0, 2, 0, 0, 0, 8166, 0, 0, 229, 15, 197, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 7713, 1, 7, 0, 0, 3, 159, 0, 0, 0, 0, 1424, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 812, 3, 0, 0, 43, 3, 0, 14, 0, 0, 0, 185, 0, 0, 6441, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 2, 0, 0, 8, 0, 0, 0, 0, 0, 0, 7928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 171, 0, 0, 0, 0, 0, 550, 0, 7473, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 1, 0, 272, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 6218, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 8, 8073, 0, 100, 0, 0, 0, 0, 0, 0, 2038, 0, 0, 3, 918, 453, 227, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3652, 0, 1, 0, 0, 157, 0, 0, 4, 5, 1198, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7729, 0, 1, 119, 0, 471, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9791, 7700, 7965, 8205, 6660, 11864, 5019, 8332] avg=8192.0000 max=11864 max_rank=5 min=5019 min_rank=6 imbalance_ratio=1.448242 max_load_reduction=53672 max_load_reduction_pct=0.818970 moved_to_echo=31875 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 5, 23, 0, 0, 26, 1482, 344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 5092, 0, 14, 0, 0, 0, 0, 0, 152, 0, 0, 0, 2, 1, 6699, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 0, 2, 1, 0, 0, 0, 0, 122, 7781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 3065, 0, 21, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2249, 0, 0, 62, 0, 0, 0, 0, 0, 0, 12, 145, 0, 0, 4425, 1107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7606, 0, 1839, 0, 0, 0, 0, 0, 0, 2801, 0, 0, 7, 653, 329, 93, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 2, 0, 0, 0, 0, 7525, 5, 4546, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 900, 0, 0, 0, 0, 17, 0, 0, 203, 5620, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1951, 12178, 7912, 3112, 8000, 13470, 12170, 6743] avg=8192.0000 max=13470 max_rank=5 min=1951 min_rank=0 imbalance_ratio=1.644287 max_load_reduction=52066 max_load_reduction_pct=0.794464 moved_to_echo=40383 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3, 17, 0, 0, 0, 8165, 24, 0, 1, 623, 0, 0, 0, 0, 0, 0, 675, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 3, 0, 0, 0, 0, 1, 7, 29, 3, 172, 7, 3, 3054, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 35, 17, 0, 0, 0, 0, 0, 0, 0, 6932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 874, 0, 0, 0, 0, 0, 3092, 0, 7673, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 9, 0, 0, 0, 0, 0, 0, 2049, 0, 2, 9, 6628, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1620, 0, 0, 0, 0, 0, 9, 2, 0, 39, 2954, 0, 0, 979, 452, 233, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 0, 0, 176, 0, 0, 0, 0, 36, 2440, 7212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 2, 0, 0, 0, 7714, 0, 0, 229, 15, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10024, 3459, 7000, 11651, 8700, 6513, 10019, 8170] avg=8192.0000 max=11651 max_rank=3 min=3459 min_rank=1 imbalance_ratio=1.422241 max_load_reduction=53885 max_load_reduction_pct=0.822220 moved_to_echo=33402 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 6172, 0, 0, 0, 3, 0, 0, 0, 13, 0, 0, 1452, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 144, 6, 0, 0, 0, 0, 0, 6, 0, 138, 11, 2, 0, 1, 6566, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 8, 16, 0, 0, 0, 0, 0, 12, 11, 7395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 71, 2, 0, 0, 0, 0, 50, 0, 5981, 1, 22, 0, 957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2757, 0, 0, 0, 0, 1, 0, 113, 0, 0, 323, 70, 0, 0, 6551, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8128, 0, 0, 0, 0, 24, 0, 0, 0, 7, 0, 0, 0, 968, 450, 181, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 5, 0, 1, 0, 251, 0, 7765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 243, 0, 0, 7685, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 7062, 7446, 7091, 9817, 10006, 8029, 7934] avg=8192.0000 max=10006 max_rank=5 min=7062 min_rank=1 imbalance_ratio=1.221436 max_load_reduction=55530 max_load_reduction_pct=0.847321 moved_to_echo=35786 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[82, 1, 1, 0, 0, 7967, 0, 4, 0, 0, 0, 4, 1, 27, 0, 0, 1288, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 4364, 30, 11, 0, 23, 0, 11, 0, 1, 0, 0, 66, 2, 26, 5822, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 0, 14, 0, 2, 0, 0, 0, 0, 17, 0, 3, 0, 0, 9, 0, 7923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 313, 0, 0, 0, 0, 0, 1, 0, 7752, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1974, 0, 0, 76, 5, 0, 0, 0, 0, 0, 4, 0, 2, 0, 6207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 67, 90, 0, 0, 0, 459, 0, 0, 68, 0, 0, 0, 917, 449, 235, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 280, 0, 0, 445, 6, 1, 77, 0, 2560, 0, 5735, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 12, 16, 0, 66, 475, 0, 57, 7675, 3, 331, 160, 117, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9859, 10595, 8180, 8081, 8268, 2469, 9111, 8973] avg=8192.0000 max=10595 max_rank=1 min=2469 min_rank=5 imbalance_ratio=1.293335 max_load_reduction=54941 max_load_reduction_pct=0.838333 moved_to_echo=28821 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 0, 0, 0, 66, 0, 0, 7, 68, 0, 0, 0, 0, 31, 46, 1263, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 5708, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 0, 4949, 0, 0, 12, 0, 1, 0, 0, 0, 6304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7426, 0, 7905, 0, 0, 0, 1374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2564, 0, 0, 11, 0, 0, 0, 1, 0, 0, 6, 0, 0, 4, 6682, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4434, 0, 0, 0, 5, 0, 0, 0, 1, 529, 0, 0, 0, 987, 433, 240, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3407, 0, 3, 19, 0, 0, 0, 0, 1, 0, 715, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 697, 13, 0, 0, 0, 2, 2, 7398, 0, 0, 12, 1, 1263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2014, 5883, 11271, 16706, 9473, 6651, 4147, 9391] avg=8192.0000 max=16706 max_rank=3 min=2014 min_rank=0 imbalance_ratio=2.039307 max_load_reduction=48830 max_load_reduction_pct=0.745087 moved_to_echo=29662 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 0, 0, 0, 8005, 0, 0, 0, 0, 3, 0, 0, 0, 4, 0, 1483, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 7, 4, 0, 0, 0, 0, 0, 75, 6705, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 7720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 382, 10, 1, 0, 0, 0, 1, 7949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1458, 0, 0, 69, 0, 5, 0, 2, 0, 0, 0, 0, 32, 0, 6611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 7449, 0, 0, 0, 2, 0, 5, 0, 0, 33, 88, 0, 0, 977, 440, 241, 215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 127, 0, 0, 0, 1, 0, 22, 6911, 12, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 2, 0, 0, 0, 7522, 11, 0, 4, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10015, 6990, 7726, 8343, 8177, 9527, 7149, 7609] avg=8192.0000 max=10015 max_rank=0 min=6990 min_rank=1 imbalance_ratio=1.222534 max_load_reduction=55521 max_load_reduction_pct=0.847183 moved_to_echo=32462 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 3, 0, 2, 6231, 0, 0, 2, 6132, 0, 0, 0, 7, 65, 0, 821, 458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 27, 0, 1, 0, 7730, 0, 0, 3709, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 1, 0, 0, 1, 0, 7947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 1, 21, 0, 1069, 0, 1249, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 348, 0, 0, 0, 3, 0, 44, 0, 0, 0, 0, 0, 43, 5817, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 16, 0, 69, 0, 0, 0, 0, 0, 0, 38, 0, 3361, 1, 8, 865, 455, 41, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 0, 0, 0, 12, 32, 0, 0, 0, 5392, 4295, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 5, 0, 0, 0, 0, 0, 7706, 0, 0, 0, 0, 623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13731, 11669, 8036, 2347, 6441, 5036, 9892, 8384] avg=8192.0000 max=13731 max_rank=0 min=2347 min_rank=3 imbalance_ratio=1.676147 max_load_reduction=51805 max_load_reduction_pct=0.790482 moved_to_echo=29753 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3, 17, 0, 0, 0, 8165, 24, 0, 1, 623, 0, 0, 0, 0, 0, 0, 675, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 3, 0, 0, 0, 0, 1, 7, 29, 3, 172, 7, 3, 3054, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 35, 17, 0, 0, 0, 0, 0, 0, 0, 6932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 874, 0, 0, 0, 0, 0, 3092, 0, 7673, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 9, 0, 0, 0, 0, 0, 0, 2049, 0, 2, 9, 6628, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1620, 0, 0, 0, 0, 0, 9, 2, 0, 39, 2954, 0, 0, 979, 452, 233, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 0, 0, 176, 0, 0, 0, 0, 36, 2440, 7212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 2, 0, 0, 0, 7714, 0, 0, 229, 15, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10024, 3459, 7000, 11651, 8700, 6513, 10019, 8170] avg=8192.0000 max=11651 max_rank=3 min=3459 min_rank=1 imbalance_ratio=1.422241 max_load_reduction=53885 max_load_reduction_pct=0.822220 moved_to_echo=33402 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[82, 1, 1, 0, 0, 7967, 0, 4, 0, 0, 0, 4, 1, 27, 0, 0, 1288, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 4364, 30, 11, 0, 23, 0, 11, 0, 1, 0, 0, 66, 2, 26, 5822, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 0, 14, 0, 2, 0, 0, 0, 0, 17, 0, 3, 0, 0, 9, 0, 7923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 313, 0, 0, 0, 0, 0, 1, 0, 7752, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1974, 0, 0, 76, 5, 0, 0, 0, 0, 0, 4, 0, 2, 0, 6207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 67, 90, 0, 0, 0, 459, 0, 0, 68, 0, 0, 0, 917, 449, 235, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 280, 0, 0, 445, 6, 1, 77, 0, 2560, 0, 5735, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 12, 16, 0, 66, 475, 0, 57, 7675, 3, 331, 160, 117, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9859, 10595, 8180, 8081, 8268, 2469, 9111, 8973] avg=8192.0000 max=10595 max_rank=1 min=2469 min_rank=5 imbalance_ratio=1.293335 max_load_reduction=54941 max_load_reduction_pct=0.838333 moved_to_echo=28821 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 5, 23, 0, 0, 26, 1482, 344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 5092, 0, 14, 0, 0, 0, 0, 0, 152, 0, 0, 0, 2, 1, 6699, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 0, 2, 1, 0, 0, 0, 0, 122, 7781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 3065, 0, 21, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2249, 0, 0, 62, 0, 0, 0, 0, 0, 0, 12, 145, 0, 0, 4425, 1107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7606, 0, 1839, 0, 0, 0, 0, 0, 0, 2801, 0, 0, 7, 653, 329, 93, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 2, 0, 0, 0, 0, 7525, 5, 4546, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 900, 0, 0, 0, 0, 17, 0, 0, 203, 5620, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1951, 12178, 7912, 3112, 8000, 13470, 12170, 6743] avg=8192.0000 max=13470 max_rank=5 min=1951 min_rank=0 imbalance_ratio=1.644287 max_load_reduction=52066 max_load_reduction_pct=0.794464 moved_to_echo=40383 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 37376, 256, 2048, 0, 128, 128, 512, 128, 128, 5632, 6528, 128, 128, 7296, 3584, 1536, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 0, 128, 44416, 128, 128, 128, 6912, 128, 256, 128, 128, 128, 128, 9984, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 10496, 128, 128, 0, 128, 128, 128, 128, 256, 256, 128, 8192, 128, 128, 44800, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 1536, 512, 128, 128, 128, 0, 12288, 128, 49152, 128, 128, 128, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 10496, 128, 128, 0, 128, 128, 128, 128, 256, 256, 128, 8192, 128, 128, 44800, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[7808, 128, 128, 896, 128, 256, 128, 128, 10496, 7936, 38400, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[11136, 384, 0, 512, 128, 128, 0, 256, 0, 0, 2432, 256, 128, 128, 49152, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[1024, 128, 896, 128, 0, 128, 768, 128, 384, 59136, 128, 384, 640, 256, 2688, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[384, 128, 128, 0, 128, 128, 128, 5120, 128, 128, 128, 128, 128, 0, 128, 256, 60032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[384, 128, 128, 0, 128, 128, 128, 5120, 128, 128, 128, 128, 128, 0, 128, 256, 60032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4686, 0, 0, 3, 886, 0, 0, 5, 0, 5025, 0, 0, 1, 0, 0, 0, 0, 4, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 7121, 0, 4131, 6, 992, 0, 0, 0, 1, 0, 2, 55, 0, 193, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6782, 0, 0, 0, 2, 2, 0, 0, 0, 441, 0, 7828, 193, 1152, 49, 0, 661, 8105, 0, 0, 0, 0, 0, 0, 87, 1, 0, 7, 0, 0, 0, 67, 0, 0, 0, 1, 2, 394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 8181, 0, 0, 7340, 964, 0, 2, 84, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[2245, 0, 31, 402, 2, 887, 0, 0, 4, 6405, 0, 3, 0, 0, 0, 488, 1, 0, 56, 0, 1568, 0, 0, 3095, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 5, 0, 2930, 6, 6, 0, 0, 0, 0, 1, 0, 51, 0, 0, 0, 0, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6329, 0, 2, 0, 6987, 0, 0, 578, 6, 0, 4, 9, 0, 0, 0, 30, 0, 73, 0, 0, 6, 0, 0, 0, 3410, 0, 1, 2, 0, 1, 0, 8167, 0, 0, 0, 0, 0, 0, 983, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4525, 0, 23, 88, 0, 8046, 0, 0, 7812, 0, 0, 59, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 2, 0, 0, 0, 0, 49, 0, 7496, 2, 0, 0, 0, 0, 1, 0, 0, 7292, 0, 0, 1017, 3, 0, 0, 0, 0, 0, 12, 0, 89, 0, 0, 22, 0, 51, 332, 5911, 0, 0, 0, 92, 0, 0, 0, 0, 0, 154, 0, 275, 527, 0, 0, 0, 0, 0, 0, 61, 0, 331, 42, 0, 36, 0, 7915, 0, 0, 0, 0, 0, 0, 7639, 146, 0, 0, 0, 0, 7564, 0, 0, 0, 0, 0, 0, 7950, 0, 0, 93, 0, 1122, 0, 0, 0, 0, 2, 0, 6, 0, 7, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 22, 3, 8043, 0, 0, 11, 1174, 0, 0, 0, 26, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 93, 15, 0, 0, 125, 0, 309, 1175, 143, 306, 0, 2, 0, 2, 0, 0, 2, 0, 9, 370, 0, 0, 0, 0, 0, 0, 0, 0, 14, 217, 0, 0, 0, 0, 0, 0, 42, 17, 7737, 0, 765, 0, 8087, 0, 289, 0, 0, 0, 34, 0, 0, 783, 0, 73, 17, 8, 129, 26, 0, 0, 0, 7992, 0, 0, 2, 35, 143, 0, 0, 0, 2, 11, 5108, 0, 115, 33, 78, 35, 5591, 0, 0, 0, 0, 0, 0, 6763, 0, 24, 2, 45, 0, 0, 4279, 0, 0, 0, 0, 0, 0, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 9, 0, 0, 0, 1, 920, 0, 7954, 0, 9, 5429, 0, 0, 0, 0, 8, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[31, 0, 0, 0, 0, 1352, 4, 829, 0, 0, 0, 8037, 0, 2, 0, 0, 0, 0, 1, 0, 129, 0, 1, 0, 71, 0, 13, 0, 0, 0, 0, 2568, 1, 114, 53, 0, 12, 0, 0, 1446, 0, 0, 0, 699, 0, 0, 0, 12, 0, 7680, 0, 0, 2238, 0, 24, 0, 0, 2, 0, 2, 0, 1, 6474, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 8177, 0, 1566, 0, 5, 0, 8154, 444, 0, 0, 0, 0, 0, 5901, 28, 0, 0, 0, 0, 0, 618, 1, 0, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 3, 0, 0, 0, 0, 2, 26, 0, 50, 0, 0, 8137, 14, 0, 561, 0, 0, 44, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 3106, 0, 8, 0, 8151, 0, 4414, 0, 0, 13, 162, 0, 0, 8, 0, 1896, 0, 0, 0, 0, 0, 4, 0, 0, 1, 0, 966, 0, 1, 0, 0, 0, 4, 0, 8181, 0, 0, 0, 7149, 0, 0, 0, 0, 0, 2114, 0, 0, 1, 0, 0, 0, 0, 9, 1689, 0, 0, 0, 360, 0, 0, 0, 486, 864, 0, 0, 0, 0, 0, 6528, 31, 183, 0, 0, 0, 107, 0, 0, 1, 0, 0, 0, 2068, 0, 1, 0, 0, 0, 34, 7036, 0, 0, 0, 14, 0, 48, 0, 2, 24, 0, 0, 0, 0, 9, 0, 0, 1, 0, 0, 8, 0, 0, 0, 0, 7817, 0, 0, 2017, 6, 0, 0, 0, 13, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 2, 0, 0, 0, 0, 49, 0, 7496, 2, 0, 0, 0, 0, 1, 0, 0, 7292, 0, 0, 1017, 3, 0, 0, 0, 0, 0, 12, 0, 89, 0, 0, 22, 0, 51, 332, 5911, 0, 0, 0, 92, 0, 0, 0, 0, 0, 154, 0, 275, 527, 0, 0, 0, 0, 0, 0, 61, 0, 331, 42, 0, 36, 0, 7915, 0, 0, 0, 0, 0, 0, 7639, 146, 0, 0, 0, 0, 7564, 0, 0, 0, 0, 0, 0, 7950, 0, 0, 93, 0, 1122, 0, 0, 0, 0, 2, 0, 6, 0, 7, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 22, 3, 8043, 0, 0, 11, 1174, 0, 0, 0, 26, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[22, 0, 0, 69, 1, 556, 1, 7, 0, 0, 0, 8132, 0, 0, 0, 0, 0, 0, 0, 1, 6511, 0, 0, 0, 0, 0, 0, 0, 4, 94, 0, 1, 0, 0, 3, 0, 0, 0, 0, 7388, 0, 0, 0, 120, 0, 0, 0, 0, 0, 73, 0, 1, 0, 7, 0, 9, 0, 0, 0, 0, 0, 1652, 12, 0, 7, 0, 42, 0, 0, 0, 33, 20, 0, 4212, 0, 0, 0, 0, 0, 8157, 1000, 0, 0, 5, 0, 8, 7680, 0, 0, 0, 0, 290, 57, 3666, 0, 0, 0, 6108, 0, 21, 0, 0, 0, 0, 0, 7, 0, 0, 166, 0, 0, 0, 0, 1099, 0, 63, 1, 0, 7983, 0, 0, 4, 243, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4686, 0, 0, 3, 886, 0, 0, 5, 0, 5025, 0, 0, 1, 0, 0, 0, 0, 4, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 7121, 0, 4131, 6, 992, 0, 0, 0, 1, 0, 2, 55, 0, 193, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6782, 0, 0, 0, 2, 2, 0, 0, 0, 441, 0, 7828, 193, 1152, 49, 0, 661, 8105, 0, 0, 0, 0, 0, 0, 87, 1, 0, 7, 0, 0, 0, 67, 0, 0, 0, 1, 2, 394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 8181, 0, 0, 7340, 964, 0, 2, 84, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 93, 15, 0, 0, 125, 0, 309, 1175, 143, 306, 0, 2, 0, 2, 0, 0, 2, 0, 9, 370, 0, 0, 0, 0, 0, 0, 0, 0, 14, 217, 0, 0, 0, 0, 0, 0, 42, 17, 7737, 0, 765, 0, 8087, 0, 289, 0, 0, 0, 34, 0, 0, 783, 0, 73, 17, 8, 129, 26, 0, 0, 0, 7992, 0, 0, 2, 35, 143, 0, 0, 0, 2, 11, 5108, 0, 115, 33, 78, 35, 5591, 0, 0, 0, 0, 0, 0, 6763, 0, 24, 2, 45, 0, 0, 4279, 0, 0, 0, 0, 0, 0, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 9, 0, 0, 0, 1, 920, 0, 7954, 0, 9, 5429, 0, 0, 0, 0, 8, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[2245, 0, 31, 402, 2, 887, 0, 0, 4, 6405, 0, 3, 0, 0, 0, 488, 1, 0, 56, 0, 1568, 0, 0, 3095, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 5, 0, 2930, 6, 6, 0, 0, 0, 0, 1, 0, 51, 0, 0, 0, 0, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6329, 0, 2, 0, 6987, 0, 0, 578, 6, 0, 4, 9, 0, 0, 0, 30, 0, 73, 0, 0, 6, 0, 0, 0, 3410, 0, 1, 2, 0, 1, 0, 8167, 0, 0, 0, 0, 0, 0, 983, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4525, 0, 23, 88, 0, 8046, 0, 0, 7812, 0, 0, 59, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[4, 0, 57, 0, 0, 6330, 0, 0, 39, 1937, 312, 599, 0, 0, 0, 0, 1, 7, 0, 15, 349, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 81, 42, 0, 0, 1, 13, 0, 0, 8022, 1, 0, 0, 3, 0, 0, 0, 0, 0, 290, 0, 0, 2, 6, 0, 0, 0, 0, 0, 15, 0, 1, 4, 5, 0, 0, 6708, 1770, 36, 0, 0, 0, 0, 5072, 0, 1883, 0, 0, 0, 0, 0, 0, 5, 571, 1, 5, 6741, 0, 4, 0, 1, 313, 0, 3998, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 2, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 8183, 0, 7551, 4487, 0, 0, 0, 6, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 2, 0, 0, 0, 0, 49, 0, 6507, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7292, 0, 0, 1017, 3, 0, 0, 0, 0, 0, 12, 0, 3828, 681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 0, 22, 0, 51, 332, 5081, 0, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 0, 275, 527, 0, 0, 0, 0, 0, 0, 61, 0, 331, 42, 0, 830, 903, 989, 687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 7915, 0, 0, 0, 0, 0, 0, 0, 146, 0, 0, 0, 0, 6661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6582, 0, 0, 93, 0, 1122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 6, 0, 7, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7639, 1618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 3, 2597, 0, 0, 11, 1174, 0, 0, 0, 26, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6561, 12834, 5667, 4799, 14758, 7797, 9283, 3837] avg=8192.0000 max=14758 max_rank=4 min=3837 min_rank=7 imbalance_ratio=1.801514 max_load_reduction=50778 max_load_reduction_pct=0.774811 moved_to_echo=35675 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4068, 0, 0, 3, 886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 5025, 0, 0, 1, 0, 0, 0, 0, 4, 0, 0, 0, 3893, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 6122, 0, 4131, 6, 992, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 55, 0, 193, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6782, 0, 999, 968, 618, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 441, 0, 0, 193, 1152, 49, 0, 661, 7137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 1, 0, 7, 0, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7828, 1645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 2643, 0, 0, 7340, 964, 0, 2, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4958, 8935, 11270, 9625, 9637, 148, 9870, 11093] avg=8192.0000 max=11270 max_rank=2 min=148 min_rank=5 imbalance_ratio=1.375732 max_load_reduction=54266 max_load_reduction_pct=0.828033 moved_to_echo=30748 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2245, 0, 31, 402, 2, 887, 0, 0, 4, 6405, 0, 3, 0, 0, 0, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 56, 0, 1568, 0, 0, 3095, 0, 0, 0, 0, 0, 0, 0, 2, 3829, 292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 5, 0, 2930, 6, 6, 0, 0, 0, 0, 1, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6329, 0, 0, 8, 0, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 6987, 0, 0, 578, 6, 0, 4, 0, 0, 0, 0, 30, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 2824, 0, 1, 2, 0, 1, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 983, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 2, 9, 1618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4525, 0, 23, 88, 0, 2599, 0, 0, 7812, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10467, 8843, 3001, 6823, 7672, 11001, 2623, 15106] avg=8192.0000 max=15106 max_rank=7 min=2623 min_rank=6 imbalance_ratio=1.843994 max_load_reduction=50430 max_load_reduction_pct=0.769501 moved_to_echo=36402 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 3106, 0, 8, 0, 8151, 0, 3832, 0, 0, 13, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 1896, 0, 0, 0, 0, 0, 4, 0, 0, 1, 0, 966, 3720, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 7033, 0, 0, 0, 7149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2114, 0, 0, 1, 0, 0, 0, 0, 9, 1689, 0, 0, 0, 360, 0, 1148, 12, 582, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 864, 0, 0, 0, 0, 0, 0, 31, 183, 0, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1713, 0, 1, 0, 0, 0, 34, 7036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 48, 0, 2, 24, 0, 0, 0, 0, 9, 0, 0, 1, 0, 6528, 1572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 2525, 0, 0, 2017, 6, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15273, 6772, 14187, 6093, 1659, 8785, 8198, 4569] avg=8192.0000 max=15273 max_rank=0 min=1659 min_rank=4 imbalance_ratio=1.864380 max_load_reduction=50263 max_load_reduction_pct=0.766953 moved_to_echo=23211 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 93, 15, 0, 0, 125, 0, 309, 1175, 143, 306, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 9, 370, 0, 0, 0, 0, 0, 0, 0, 0, 14, 217, 0, 3785, 579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 17, 6651, 0, 765, 0, 8087, 0, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 783, 0, 73, 17, 8, 129, 26, 0, 0, 0, 7992, 0, 1086, 668, 0, 585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 35, 143, 0, 0, 0, 2, 11, 0, 0, 115, 33, 78, 35, 4923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5599, 0, 24, 2, 45, 0, 0, 4279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 9, 5108, 1600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 920, 0, 2569, 0, 9, 5429, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2170, 4976, 15851, 11401, 5377, 9949, 6876, 8936] avg=8192.0000 max=15851 max_rank=2 min=2170 min_rank=0 imbalance_ratio=1.934937 max_load_reduction=49685 max_load_reduction_pct=0.758133 moved_to_echo=31138 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[31, 0, 0, 0, 0, 1352, 4, 829, 0, 0, 0, 6977, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 129, 0, 1, 0, 71, 0, 13, 0, 0, 0, 0, 2568, 3872, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 114, 53, 0, 12, 0, 0, 1243, 0, 0, 0, 699, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7680, 0, 0, 2238, 0, 24, 0, 0, 2, 0, 2, 0, 1, 6474, 0, 203, 974, 1060, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 1566, 0, 5, 0, 7180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 444, 0, 0, 0, 0, 0, 4886, 28, 0, 0, 0, 0, 0, 618, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 3, 0, 0, 0, 0, 8177, 1637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 26, 0, 50, 0, 0, 2628, 14, 0, 561, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9195, 7160, 2134, 19168, 8755, 5977, 9822, 3325] avg=8192.0000 max=19168 max_rank=3 min=2134 min_rank=2 imbalance_ratio=2.339844 max_load_reduction=46368 max_load_reduction_pct=0.707520 moved_to_echo=27879 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 57, 0, 0, 6330, 0, 0, 39, 1937, 312, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7, 0, 15, 349, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 81, 3894, 577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 1, 13, 0, 0, 6896, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 290, 0, 0, 2, 6, 0, 0, 0, 0, 0, 15, 0, 1, 4, 5, 1126, 0, 79, 583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6708, 1770, 36, 0, 0, 0, 0, 0, 0, 1883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 571, 1, 5, 5581, 0, 4, 0, 1, 313, 0, 3998, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 2, 0, 0, 0, 0, 0, 5072, 1646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 2643, 0, 7551, 4487, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9199, 4937, 6956, 2111, 10397, 10504, 6741, 14691] avg=8192.0000 max=14691 max_rank=7 min=2111 min_rank=3 imbalance_ratio=1.793335 max_load_reduction=50845 max_load_reduction_pct=0.775833 moved_to_echo=42333 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[22, 0, 0, 69, 1, 556, 1, 7, 0, 0, 0, 7057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6511, 0, 0, 0, 0, 0, 0, 0, 4, 94, 0, 1, 3805, 663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 6347, 0, 0, 0, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 1, 0, 7, 0, 9, 0, 0, 0, 0, 0, 1652, 12, 0, 1041, 978, 1075, 668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 42, 0, 0, 0, 33, 20, 0, 0, 0, 0, 0, 0, 0, 7179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 0, 0, 5, 0, 8, 6349, 0, 0, 0, 0, 290, 57, 3666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6108, 0, 21, 0, 0, 0, 0, 0, 7, 0, 0, 166, 0, 0, 0, 4212, 1609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1099, 0, 63, 1, 0, 2569, 0, 0, 4, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7713, 11079, 6470, 5516, 7281, 11375, 12123, 3979] avg=8192.0000 max=12123 max_rank=6 min=3979 min_rank=7 imbalance_ratio=1.479858 max_load_reduction=53413 max_load_reduction_pct=0.815018 moved_to_echo=34758 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 93, 15, 0, 0, 125, 0, 309, 1175, 143, 306, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 9, 370, 0, 0, 0, 0, 0, 0, 0, 0, 14, 217, 0, 3785, 579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 17, 6651, 0, 765, 0, 8087, 0, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 783, 0, 73, 17, 8, 129, 26, 0, 0, 0, 7992, 0, 1086, 668, 0, 585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 35, 143, 0, 0, 0, 2, 11, 0, 0, 115, 33, 78, 35, 4923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5599, 0, 24, 2, 45, 0, 0, 4279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 9, 5108, 1600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 920, 0, 2569, 0, 9, 5429, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2170, 4976, 15851, 11401, 5377, 9949, 6876, 8936] avg=8192.0000 max=15851 max_rank=2 min=2170 min_rank=0 imbalance_ratio=1.934937 max_load_reduction=49685 max_load_reduction_pct=0.758133 moved_to_echo=31138 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 3106, 0, 8, 0, 8151, 0, 3832, 0, 0, 13, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 1896, 0, 0, 0, 0, 0, 4, 0, 0, 1, 0, 966, 3720, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 7033, 0, 0, 0, 7149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2114, 0, 0, 1, 0, 0, 0, 0, 9, 1689, 0, 0, 0, 360, 0, 1148, 12, 582, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 864, 0, 0, 0, 0, 0, 0, 31, 183, 0, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1713, 0, 1, 0, 0, 0, 34, 7036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 48, 0, 2, 24, 0, 0, 0, 0, 9, 0, 0, 1, 0, 6528, 1572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 2525, 0, 0, 2017, 6, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15273, 6772, 14187, 6093, 1659, 8785, 8198, 4569] avg=8192.0000 max=15273 max_rank=0 min=1659 min_rank=4 imbalance_ratio=1.864380 max_load_reduction=50263 max_load_reduction_pct=0.766953 moved_to_echo=23211 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4068, 0, 0, 3, 886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 5025, 0, 0, 1, 0, 0, 0, 0, 4, 0, 0, 0, 3893, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 6122, 0, 4131, 6, 992, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 55, 0, 193, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6782, 0, 999, 968, 618, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 441, 0, 0, 193, 1152, 49, 0, 661, 7137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 1, 0, 7, 0, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7828, 1645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 2643, 0, 0, 7340, 964, 0, 2, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4958, 8935, 11270, 9625, 9637, 148, 9870, 11093] avg=8192.0000 max=11270 max_rank=2 min=148 min_rank=5 imbalance_ratio=1.375732 max_load_reduction=54266 max_load_reduction_pct=0.828033 moved_to_echo=30748 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 57, 0, 0, 6330, 0, 0, 39, 1937, 312, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7, 0, 15, 349, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 81, 3894, 577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 1, 13, 0, 0, 6896, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 290, 0, 0, 2, 6, 0, 0, 0, 0, 0, 15, 0, 1, 4, 5, 1126, 0, 79, 583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6708, 1770, 36, 0, 0, 0, 0, 0, 0, 1883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 571, 1, 5, 5581, 0, 4, 0, 1, 313, 0, 3998, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 2, 0, 0, 0, 0, 0, 5072, 1646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 2643, 0, 7551, 4487, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9199, 4937, 6956, 2111, 10397, 10504, 6741, 14691] avg=8192.0000 max=14691 max_rank=7 min=2111 min_rank=3 imbalance_ratio=1.793335 max_load_reduction=50845 max_load_reduction_pct=0.775833 moved_to_echo=42333 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2245, 0, 31, 402, 2, 887, 0, 0, 4, 6405, 0, 3, 0, 0, 0, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 56, 0, 1568, 0, 0, 3095, 0, 0, 0, 0, 0, 0, 0, 2, 3829, 292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 5, 0, 2930, 6, 6, 0, 0, 0, 0, 1, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6329, 0, 0, 8, 0, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 6987, 0, 0, 578, 6, 0, 4, 0, 0, 0, 0, 30, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 2824, 0, 1, 2, 0, 1, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 983, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 2, 9, 1618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4525, 0, 23, 88, 0, 2599, 0, 0, 7812, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10467, 8843, 3001, 6823, 7672, 11001, 2623, 15106] avg=8192.0000 max=15106 max_rank=7 min=2623 min_rank=6 imbalance_ratio=1.843994 max_load_reduction=50430 max_load_reduction_pct=0.769501 moved_to_echo=36402 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 10368, 128, 384, 3968, 128, 128, 128, 128, 256, 1792, 128, 0, 2048, 28032, 128, 6528, 4608, 4480, 3584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 5760, 0, 256, 1152, 128, 20864, 128, 7680, 27776, 2432, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[256, 256, 128, 128, 128, 128, 3328, 39424, 128, 4992, 128, 17152, 0, 384, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[1536, 0, 128, 640, 128, 128, 33664, 128, 128, 128, 128, 1792, 128, 27904, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 23168, 0, 128, 4224, 128, 0, 128, 128, 128, 128, 256, 3712, 30720, 3584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[2304, 128, 128, 512, 128, 12416, 128, 1280, 1280, 16768, 640, 29056, 128, 128, 128, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 23168, 0, 128, 4224, 128, 0, 128, 128, 128, 128, 256, 3712, 30720, 3584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 6144, 128, 512, 1152, 128, 128, 128, 0, 128, 128, 128, 256, 0, 128, 128, 44672, 13056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 22272, 2816, 128, 640, 128, 512, 128, 0, 384, 4992, 128, 128, 768, 33280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[8, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 169, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 8175, 7564, 12, 0, 249, 0, 11, 0, 0, 1, 0, 0, 0, 0, 0, 9, 0, 0, 3, 385, 68, 0, 21, 0, 28, 1, 1, 0, 6, 102, 671, 7, 0, 0, 7876, 1, 0, 0, 0, 19, 0, 0, 32, 0, 0, 5, 1, 1, 0, 0, 395, 0, 0, 232, 3, 0, 2, 0, 0, 0, 8161, 0, 0, 1, 8141, 0, 0, 0, 0, 0, 0, 608, 0, 0, 304, 12, 0, 1, 98, 0, 0, 6848, 0, 5, 0, 0, 0, 7819, 0, 0, 6321, 0, 0, 0, 795, 296, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 17, 0, 0, 3259, 0, 0, 31, 0, 0, 4, 2923, 0, 3, 0, 0, 0, 5, 8182, 18, 0, 0, 0, 125, 0, 1, 1, 0, 0, 0, 0, 0, 8, 0, 0, 0, 1243, 0, 0, 5244, 0, 0, 0, 58, 0, 1, 0, 0, 0, 0, 0, 29, 0, 0, 18, 2218, 5415, 0, 0, 0, 0, 0, 0, 47, 2, 10, 1, 0, 17, 0, 0, 2097, 4698, 0, 0, 0, 21, 7453, 5, 0, 0, 0, 21, 0, 0, 0, 0, 6318, 63, 0, 65, 0, 0, 0, 0, 0, 0, 1866, 7845, 0, 0, 0, 0, 78, 1233, 0, 0, 4854, 0, 0, 0, 0, 0, 0, 0, 4, 34] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 65, 0, 0, 0, 66, 4, 0, 0, 2, 0, 0, 0, 18, 0, 8, 0, 0, 6458, 0, 271, 0, 0, 0, 0, 0, 0, 5261, 0, 0, 5, 18, 0, 0, 0, 1401, 6, 8, 0, 0, 2, 0, 0, 0, 0, 0, 0, 18, 0, 2, 1888, 0, 0, 0, 0, 0, 2, 0, 5392, 0, 1673, 0, 0, 0, 9, 8189, 0, 0, 0, 2, 0, 2, 0, 17, 0, 0, 0, 0, 21, 7, 0, 267, 0, 0, 0, 0, 8030, 0, 2, 0, 0, 0, 0, 7773, 0, 0, 1171, 0, 0, 0, 126, 0, 0, 1, 920, 0, 0, 8143, 0, 16, 0, 0, 0, 9, 79, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 0, 442, 0, 1, 0, 0, 0, 0, 0, 0, 3, 13, 5, 21, 0, 0, 0, 41, 9, 170, 0, 0, 0, 0, 0, 3, 7269, 7125, 0, 0, 7910, 0, 0, 1, 24, 1253, 1187, 2, 0, 0, 7365, 37, 24, 0, 10, 194, 58, 0, 0, 0, 498, 0, 0, 0, 0, 128, 2, 0, 0, 0, 240, 0, 1, 0, 14, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 41, 8, 0, 902, 4, 0, 0, 0, 8183, 0, 1, 33, 1796, 0, 0, 29, 0, 0, 1, 15, 40, 0, 242, 0, 28, 0, 0, 39, 0, 7125, 0, 0, 9, 0, 0, 5891, 0, 0, 6804, 0, 183, 0, 0, 66, 0, 0, 34, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 7, 0, 3, 0, 0, 0, 0, 1014, 0, 0, 394, 0, 0, 128, 0, 0, 0, 28, 0, 0, 0, 70, 0, 313, 7701, 3062, 0, 0, 0, 0, 50, 0, 2436, 0, 0, 0, 6, 243, 17, 0, 0, 0, 140, 0, 0, 6287, 0, 0, 0, 1, 0, 8, 0, 0, 0, 2378, 0, 0, 11, 3, 0, 0, 7365, 0, 0, 1365, 304, 0, 0, 0, 0, 0, 0, 328, 0, 0, 0, 51, 7882, 2, 0, 3, 0, 7933, 0, 0, 0, 7, 0, 0, 2, 0, 0, 7948, 0, 0, 0, 0, 0, 0, 1, 0, 9, 0, 2656, 0, 3, 0, 0, 1, 0, 0, 0, 5363, 0, 6, 3, 0, 4, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 2, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 5710, 0, 74, 0, 0, 11, 0, 4186, 0, 0, 0, 0, 0, 0, 8092, 33, 224, 0, 4, 0, 0, 0, 21, 2, 14, 0, 0, 0, 34, 0, 0, 3, 811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 309, 0, 6541, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 136, 0, 4, 0, 0, 0, 0, 0, 6, 0, 7950, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4820, 0, 5, 3, 2, 0, 0, 0, 7, 0, 36, 7996, 0, 0, 0, 0, 0, 8186, 0, 0, 2071, 0, 0, 0, 1, 0, 0, 0, 28, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 644, 0, 0, 0, 0, 0, 57, 0, 0, 0, 481, 0, 0, 0, 0, 0, 1, 0, 0, 0, 10, 176, 0, 0, 0, 895, 454, 0, 6593, 8092, 33, 6, 0, 0, 2, 376, 0, 0, 0, 18, 10, 0, 231, 6, 0, 0, 4, 0, 0, 4741, 367, 0, 2, 0, 0, 0, 2, 4983, 137, 7866, 33, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 31, 0, 0, 0, 1, 0, 11, 8, 388, 100, 53, 0, 0, 7823, 1, 0, 2, 33, 0, 0, 117, 0, 0, 16, 0, 0, 0, 6745, 1, 0, 0, 0, 0, 0, 7499, 0, 0, 0, 2, 0, 5, 0, 0, 6371, 0, 100, 0, 0, 0, 0, 7, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 65, 0, 0, 0, 66, 4, 0, 0, 2, 0, 0, 0, 18, 0, 8, 0, 0, 6458, 0, 271, 0, 0, 0, 0, 0, 0, 5261, 0, 0, 5, 18, 0, 0, 0, 1401, 6, 8, 0, 0, 2, 0, 0, 0, 0, 0, 0, 18, 0, 2, 1888, 0, 0, 0, 0, 0, 2, 0, 5392, 0, 1673, 0, 0, 0, 9, 8189, 0, 0, 0, 2, 0, 2, 0, 17, 0, 0, 0, 0, 21, 7, 0, 267, 0, 0, 0, 0, 8030, 0, 2, 0, 0, 0, 0, 7773, 0, 0, 1171, 0, 0, 0, 126, 0, 0, 1, 920, 0, 0, 8143, 0, 16, 0, 0, 0, 9, 79, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[8, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 169, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 8175, 7564, 12, 0, 249, 0, 11, 0, 0, 1, 0, 0, 0, 0, 0, 9, 0, 0, 3, 385, 68, 0, 21, 0, 28, 1, 1, 0, 6, 102, 671, 7, 0, 0, 7876, 1, 0, 0, 0, 19, 0, 0, 32, 0, 0, 5, 1, 1, 0, 0, 395, 0, 0, 232, 3, 0, 2, 0, 0, 0, 8161, 0, 0, 1, 8141, 0, 0, 0, 0, 0, 0, 608, 0, 0, 304, 12, 0, 1, 98, 0, 0, 6848, 0, 5, 0, 0, 0, 7819, 0, 0, 6321, 0, 0, 0, 795, 296, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 10, 0, 6, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 568, 0, 0, 0, 0, 1, 0, 325, 0, 0, 0, 0, 0, 3, 6910, 8, 5, 0, 0, 0, 1061, 0, 0, 0, 0, 0, 0, 3727, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 3, 7635, 0, 0, 0, 0, 0, 0, 0, 0, 403, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 1, 0, 10, 6794, 39, 0, 0, 0, 2, 0, 0, 1, 0, 443, 6364, 0, 17, 28, 0, 57, 0, 14, 0, 6297, 321, 0, 0, 0, 0, 0, 1288, 1, 0, 6796, 0, 0, 0, 0, 8162, 0, 1, 11, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 17, 0, 0, 3259, 4760, 730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 4, 2923, 0, 3, 0, 0, 0, 5, 3828, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 125, 0, 1, 1, 0, 0, 0, 0, 0, 8, 0, 0, 0, 1243, 3025, 803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5244, 0, 0, 0, 58, 0, 1, 0, 0, 0, 0, 0, 29, 0, 811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 2218, 5415, 0, 0, 0, 0, 0, 0, 47, 2, 10, 1, 0, 17, 3748, 1329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2097, 4698, 0, 0, 0, 21, 2693, 5, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6318, 63, 0, 65, 0, 0, 0, 0, 0, 0, 1866, 3367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 1233, 0, 0, 3240, 0, 0, 0, 0, 0, 0, 0, 4, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8767, 6812, 5206, 6143, 12805, 9535, 11679, 4589] avg=8192.0000 max=12805 max_rank=4 min=4589 min_rank=7 imbalance_ratio=1.563110 max_load_reduction=52731 max_load_reduction_pct=0.804611 moved_to_echo=38608 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 7, 0, 3, 0, 0, 0, 0, 1014, 0, 0, 394, 0, 0, 128, 0, 0, 0, 28, 0, 0, 0, 70, 0, 313, 7701, 3062, 0, 0, 0, 0, 50, 0, 2436, 0, 0, 0, 6, 243, 17, 0, 0, 0, 140, 0, 0, 6287, 0, 0, 0, 1, 0, 8, 0, 0, 0, 2378, 0, 0, 11, 3, 0, 0, 7365, 0, 0, 1365, 304, 0, 0, 0, 0, 0, 0, 328, 0, 0, 0, 51, 7882, 2, 0, 3, 0, 7933, 0, 0, 0, 7, 0, 0, 2, 0, 0, 7948, 0, 0, 0, 0, 0, 0, 1, 0, 9, 0, 2656, 0, 3, 0, 0, 1, 0, 0, 0, 5363, 0, 6, 3, 0, 4, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 17, 0, 0, 3259, 0, 0, 31, 0, 0, 4, 2923, 0, 3, 0, 0, 0, 5, 8182, 18, 0, 0, 0, 125, 0, 1, 1, 0, 0, 0, 0, 0, 8, 0, 0, 0, 1243, 0, 0, 5244, 0, 0, 0, 58, 0, 1, 0, 0, 0, 0, 0, 29, 0, 0, 18, 2218, 5415, 0, 0, 0, 0, 0, 0, 47, 2, 10, 1, 0, 17, 0, 0, 2097, 4698, 0, 0, 0, 21, 7453, 5, 0, 0, 0, 21, 0, 0, 0, 0, 6318, 63, 0, 65, 0, 0, 0, 0, 0, 0, 1866, 7845, 0, 0, 0, 0, 78, 1233, 0, 0, 4854, 0, 0, 0, 0, 0, 0, 0, 4, 34] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 644, 0, 0, 0, 0, 0, 57, 0, 0, 0, 481, 0, 0, 0, 0, 0, 1, 0, 0, 0, 10, 176, 0, 0, 0, 895, 454, 0, 6593, 8092, 33, 6, 0, 0, 2, 376, 0, 0, 0, 18, 10, 0, 231, 6, 0, 0, 4, 0, 0, 4741, 367, 0, 2, 0, 0, 0, 2, 4983, 137, 7866, 33, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 31, 0, 0, 0, 1, 0, 11, 8, 388, 100, 53, 0, 0, 7823, 1, 0, 2, 33, 0, 0, 117, 0, 0, 16, 0, 0, 0, 6745, 1, 0, 0, 0, 0, 0, 7499, 0, 0, 0, 2, 0, 5, 0, 0, 6371, 0, 100, 0, 0, 0, 0, 7, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 442, 0, 1, 0, 0, 0, 0, 0, 0, 3, 13, 5, 21, 5226, 663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 9, 170, 0, 0, 0, 0, 0, 3, 7269, 3333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7910, 0, 0, 1, 24, 1253, 1187, 2, 0, 0, 7365, 37, 24, 0, 10, 194, 2634, 1126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 498, 0, 0, 0, 0, 128, 2, 0, 0, 0, 240, 0, 1137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 14, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 3404, 1158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 8, 0, 902, 4, 0, 0, 0, 2957, 0, 1, 33, 1796, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 15, 40, 0, 242, 0, 28, 0, 0, 39, 0, 3058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 5891, 0, 0, 4541, 0, 183, 0, 0, 66, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6375, 10825, 21767, 2063, 4588, 5771, 3423, 10724] avg=8192.0000 max=21767 max_rank=2 min=2063 min_rank=3 imbalance_ratio=2.657104 max_load_reduction=43769 max_load_reduction_pct=0.667862 moved_to_echo=24506 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 10, 0, 6, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 568, 4339, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 325, 0, 0, 0, 0, 0, 3, 3232, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 0, 0, 0, 0, 0, 0, 3727, 0, 0, 0, 0, 0, 2555, 1125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 1136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 7635, 0, 0, 0, 0, 0, 0, 0, 0, 403, 0, 0, 0, 0, 153, 1123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 1, 0, 10, 2455, 39, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 443, 6364, 0, 17, 28, 0, 57, 0, 14, 0, 6297, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1288, 1, 0, 4535, 0, 0, 0, 0, 8162, 0, 1, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4955, 3574, 8468, 1166, 9317, 10698, 13360, 13998] avg=8192.0000 max=13998 max_rank=7 min=1166 min_rank=3 imbalance_ratio=1.708740 max_load_reduction=51538 max_load_reduction_pct=0.786407 moved_to_echo=47373 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 169, 5212, 637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 8175, 3539, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 249, 0, 11, 0, 0, 1, 0, 0, 0, 0, 0, 9, 0, 0, 3, 385, 2796, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 21, 0, 28, 1, 1, 0, 6, 102, 671, 7, 0, 0, 7876, 1, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 32, 0, 0, 5, 1, 1, 0, 0, 395, 0, 3272, 1229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 232, 3, 0, 2, 0, 0, 0, 2949, 0, 0, 1, 8141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 608, 0, 0, 304, 12, 0, 1, 98, 0, 0, 2939, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7819, 0, 0, 4218, 0, 0, 0, 795, 296, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6091, 11728, 4500, 9839, 4954, 11328, 3967, 13129] avg=8192.0000 max=13129 max_rank=7 min=3967 min_rank=6 imbalance_ratio=1.602661 max_load_reduction=52407 max_load_reduction_pct=0.799667 moved_to_echo=33378 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 65, 0, 0, 0, 66, 4, 0, 0, 2, 0, 0, 0, 18, 5128, 758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 6458, 0, 271, 0, 0, 0, 0, 0, 0, 2461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 18, 0, 0, 0, 1401, 6, 8, 0, 0, 2, 0, 0, 0, 0, 0, 1945, 1355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 2, 1888, 0, 0, 0, 0, 0, 2, 0, 5392, 0, 1673, 0, 1368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 8189, 0, 0, 0, 2, 0, 2, 0, 17, 0, 0, 0, 0, 3891, 855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 7, 0, 267, 0, 0, 0, 0, 2902, 0, 2, 0, 0, 0, 0, 7773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1171, 0, 0, 0, 126, 0, 0, 1, 920, 0, 0, 3494, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 79, 0, 5461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6041, 9198, 4740, 10343, 12965, 10972, 5728, 5549] avg=8192.0000 max=12965 max_rank=4 min=4740 min_rank=2 imbalance_ratio=1.582642 max_load_reduction=52571 max_load_reduction_pct=0.802170 moved_to_echo=35214 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 7, 0, 3, 0, 0, 0, 0, 1014, 0, 0, 394, 5066, 247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 28, 0, 0, 0, 70, 0, 313, 3603, 3062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 2436, 0, 0, 0, 6, 243, 17, 0, 0, 0, 140, 2847, 888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6287, 0, 0, 0, 1, 0, 8, 0, 0, 0, 2378, 0, 0, 11, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 7365, 0, 0, 1365, 304, 0, 0, 0, 0, 0, 0, 328, 0, 1269, 1251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 7882, 2, 0, 3, 0, 2867, 0, 0, 0, 7, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7948, 0, 0, 0, 0, 0, 0, 1, 0, 9, 0, 1140, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3579, 0, 6, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6731, 7204, 6627, 9581, 11885, 10814, 9101, 3593] avg=8192.0000 max=11885 max_rank=4 min=3593 min_rank=7 imbalance_ratio=1.450806 max_load_reduction=53651 max_load_reduction_pct=0.818649 moved_to_echo=35393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 5710, 5077, 744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 11, 0, 4186, 0, 0, 0, 0, 0, 0, 3785, 33, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 21, 2, 14, 0, 0, 0, 34, 0, 0, 3, 811, 2992, 342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 309, 0, 6541, 0, 346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 136, 0, 3820, 1315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 6, 0, 2873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4820, 0, 5, 3, 2, 0, 0, 0, 7, 0, 36, 3432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 1383, 0, 0, 0, 1, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19690, 8313, 4223, 7197, 5326, 2883, 8306, 9598] avg=8192.0000 max=19690 max_rank=0 min=2883 min_rank=5 imbalance_ratio=2.403564 max_load_reduction=45846 max_load_reduction_pct=0.699554 moved_to_echo=26113 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 17, 0, 0, 3259, 4760, 730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 4, 2923, 0, 3, 0, 0, 0, 5, 3828, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 125, 0, 1, 1, 0, 0, 0, 0, 0, 8, 0, 0, 0, 1243, 3025, 803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5244, 0, 0, 0, 58, 0, 1, 0, 0, 0, 0, 0, 29, 0, 811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 2218, 5415, 0, 0, 0, 0, 0, 0, 47, 2, 10, 1, 0, 17, 3748, 1329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2097, 4698, 0, 0, 0, 21, 2693, 5, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6318, 63, 0, 65, 0, 0, 0, 0, 0, 0, 1866, 3367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 1233, 0, 0, 3240, 0, 0, 0, 0, 0, 0, 0, 4, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8767, 6812, 5206, 6143, 12805, 9535, 11679, 4589] avg=8192.0000 max=12805 max_rank=4 min=4589 min_rank=7 imbalance_ratio=1.563110 max_load_reduction=52731 max_load_reduction_pct=0.804611 moved_to_echo=38608 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 644, 0, 0, 0, 0, 0, 57, 0, 0, 0, 481, 0, 0, 5001, 703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 10, 176, 0, 0, 0, 895, 207, 0, 6593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8092, 33, 6, 0, 0, 2, 376, 0, 0, 0, 18, 10, 0, 231, 6, 0, 171, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4741, 367, 0, 2, 0, 0, 0, 2, 4983, 137, 7866, 33, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 31, 0, 0, 0, 1, 3586, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 8, 388, 100, 53, 0, 0, 2822, 1, 0, 2, 33, 0, 0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 6745, 1, 0, 0, 0, 0, 0, 3210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5, 0, 0, 4242, 0, 100, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6886, 7882, 10005, 19204, 3696, 3535, 9972, 4356] avg=8192.0000 max=19204 max_rank=3 min=3535 min_rank=5 imbalance_ratio=2.344238 max_load_reduction=46332 max_load_reduction_pct=0.706970 moved_to_echo=21559 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 7, 0, 3, 0, 0, 0, 0, 1014, 0, 0, 394, 5066, 247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 28, 0, 0, 0, 70, 0, 313, 3603, 3062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 2436, 0, 0, 0, 6, 243, 17, 0, 0, 0, 140, 2847, 888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6287, 0, 0, 0, 1, 0, 8, 0, 0, 0, 2378, 0, 0, 11, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 7365, 0, 0, 1365, 304, 0, 0, 0, 0, 0, 0, 328, 0, 1269, 1251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 7882, 2, 0, 3, 0, 2867, 0, 0, 0, 7, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7948, 0, 0, 0, 0, 0, 0, 1, 0, 9, 0, 1140, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3579, 0, 6, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6731, 7204, 6627, 9581, 11885, 10814, 9101, 3593] avg=8192.0000 max=11885 max_rank=4 min=3593 min_rank=7 imbalance_ratio=1.450806 max_load_reduction=53651 max_load_reduction_pct=0.818649 moved_to_echo=35393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 65, 0, 0, 0, 66, 4, 0, 0, 2, 0, 0, 0, 18, 5128, 758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 6458, 0, 271, 0, 0, 0, 0, 0, 0, 2461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 18, 0, 0, 0, 1401, 6, 8, 0, 0, 2, 0, 0, 0, 0, 0, 1945, 1355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 2, 1888, 0, 0, 0, 0, 0, 2, 0, 5392, 0, 1673, 0, 1368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 8189, 0, 0, 0, 2, 0, 2, 0, 17, 0, 0, 0, 0, 3891, 855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 7, 0, 267, 0, 0, 0, 0, 2902, 0, 2, 0, 0, 0, 0, 7773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1171, 0, 0, 0, 126, 0, 0, 1, 920, 0, 0, 3494, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 79, 0, 5461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6041, 9198, 4740, 10343, 12965, 10972, 5728, 5549] avg=8192.0000 max=12965 max_rank=4 min=4740 min_rank=2 imbalance_ratio=1.582642 max_load_reduction=52571 max_load_reduction_pct=0.802170 moved_to_echo=35214 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 10, 0, 6, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 568, 4339, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 325, 0, 0, 0, 0, 0, 3, 3232, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 0, 0, 0, 0, 0, 0, 3727, 0, 0, 0, 0, 0, 2555, 1125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 1136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 7635, 0, 0, 0, 0, 0, 0, 0, 0, 403, 0, 0, 0, 0, 153, 1123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 1, 0, 10, 2455, 39, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 443, 6364, 0, 17, 28, 0, 57, 0, 14, 0, 6297, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1288, 1, 0, 4535, 0, 0, 0, 0, 8162, 0, 1, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4955, 3574, 8468, 1166, 9317, 10698, 13360, 13998] avg=8192.0000 max=13998 max_rank=7 min=1166 min_rank=3 imbalance_ratio=1.708740 max_load_reduction=51538 max_load_reduction_pct=0.786407 moved_to_echo=47373 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[16256, 128, 256, 1152, 128, 5120, 1664, 128, 0, 128, 11392, 128, 128, 256, 128, 2816, 19072, 7808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 9344, 0, 128, 0, 128, 128, 128, 0, 128, 1152, 512, 128, 10240, 39936, 4608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 9984, 20992, 0, 128, 1408, 384, 128, 128, 128, 512, 128, 128, 896, 128, 23168, 8448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 11648, 128, 7168, 384, 128, 128, 128, 256, 768, 128, 13184, 256, 24320, 128, 7936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 9984, 20992, 0, 128, 1408, 384, 128, 128, 128, 512, 128, 128, 896, 128, 23168, 8448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 24448, 128, 0, 31232, 0, 384, 128, 896, 8576, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[8320, 384, 2176, 14208, 128, 128, 128, 128, 22528, 128, 128, 128, 9984, 128, 0, 7936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 0, 20736, 7168, 128, 128, 7552, 128, 128, 128, 1152, 128, 8320, 20864, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 128, 256, 128, 6528, 256, 7808, 128, 256, 0, 128, 128, 16768, 24064, 3200, 6912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 7969, 0, 1333, 0, 2407, 5, 0, 0, 0, 0, 2082, 0, 0, 0, 3, 0, 0, 20, 0, 0, 283, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 2778, 1, 0, 0, 0, 6, 57, 0, 0, 0, 2, 7941, 22, 0, 725, 0, 331, 0, 70, 6321, 0, 6869, 3, 2, 0, 7, 0, 0, 58, 2605, 3203, 0, 90, 0, 20, 0, 0, 0, 0, 659, 0, 1, 8176, 5317, 0, 0, 0, 34, 0, 1, 0, 0, 984, 0, 0, 630, 0, 0, 0, 5, 0, 46, 0, 0, 0, 4236, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 9, 0, 0, 0, 8167, 1, 0, 0, 0, 2, 0, 0, 97, 0, 0, 7, 0, 8, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 133, 0, 0, 75, 0, 0, 0, 3126, 0, 1827, 0, 0, 0, 8128, 0, 2, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 41, 0, 0, 17, 0, 0, 8180, 0, 0, 5994, 0, 0, 0, 0, 47, 0, 2244, 0, 0, 0, 0, 0, 2, 0, 0, 4666, 0, 0, 0, 0, 0, 0, 0, 364, 1236, 0, 0, 1043, 7908, 0, 0, 0, 6495, 0, 0, 0, 0, 0, 1, 0, 5633, 0, 1, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 22, 0, 0, 0, 941, 0, 0, 5107, 0, 0, 0, 26, 0, 51, 1, 0, 8046, 2, 0, 0, 0, 1, 0, 0, 0, 4, 0, 0, 0, 357, 96, 0, 0, 0, 0, 0, 41, 197, 0, 0, 1103, 0, 0, 0, 6455, 0, 20, 0, 0, 0, 2391, 29, 0, 0, 0, 0, 12, 18, 237, 209, 0, 0, 0, 0, 0, 0, 92, 0, 0, 7955, 0, 149, 3, 0, 2, 0, 0, 8187, 0, 0, 0, 0, 1, 1080, 0, 0, 0, 2, 137, 0, 0, 0, 0, 59, 0, 0, 2701, 41, 37, 0, 952, 7614, 0, 83, 0, 59, 0, 0, 0, 7, 7, 0, 2351, 0, 0, 6980, 172, 0, 1, 0, 0, 0, 1498, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 0, 153, 1, 0, 8019, 0, 2552, 1715, 0, 0, 236, 0, 0, 4486, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 2, 33, 0, 0, 282, 4, 14, 0, 0, 7845, 0, 0, 0, 145, 3, 0, 0, 11, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 63, 0, 1, 0, 0, 7900, 0, 0, 209, 0, 43, 1, 0, 1785, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 31, 305, 4626, 1, 20, 0, 3492, 0, 0, 48, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 6367, 10, 0, 0, 0, 0, 0, 0, 7177, 0, 7837] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 2041, 0, 0, 1, 0, 0, 0, 3, 4570, 0, 0, 0, 9, 0, 0, 0, 0, 4996, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8, 6482, 0, 0, 18, 0, 708, 0, 129, 0, 0, 0, 0, 0, 0, 8172, 0, 372, 10, 1, 0, 1603, 46, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 9, 0, 23, 0, 0, 7575, 0, 0, 0, 222, 0, 0, 0, 6544, 0, 0, 0, 0, 0, 7, 0, 4, 0, 0, 57, 5, 9, 0, 0, 5129, 0, 0, 7784, 1266, 0, 0, 0, 7205, 0, 0, 0, 0, 12, 0, 0, 2, 0, 0, 1, 0, 0, 16, 55, 0, 15, 0, 379, 0, 0, 0, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 275, 2136, 7747, 0, 1, 15, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 16, 33, 0, 0, 0, 0, 0, 0, 0, 0, 379, 0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 0, 7630, 2, 0, 0, 0, 0, 0, 0, 2618, 0, 0, 12, 0, 8, 1162, 0, 0, 0, 0, 0, 1612, 0, 0, 215, 0, 8153, 521, 2370, 3, 0, 0, 0, 0, 6466, 0, 668, 0, 0, 0, 0, 2, 0, 0, 0, 6226, 0, 0, 0, 0, 0, 0, 0, 7018, 0, 6, 0, 326, 6255, 0, 2, 0, 55, 0, 0, 0, 0, 192, 0, 0, 0, 0, 3375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 7640, 0, 0, 0, 0, 0, 497, 0, 7190, 0, 0, 10, 9, 0, 0, 2, 0, 65, 0, 1, 0, 6606, 0, 0, 0, 2, 0, 0, 4, 0, 0, 6773, 0, 0, 3, 0, 8, 0, 1, 0, 0, 21, 0, 0, 0, 7997, 154, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 415, 0, 5, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8059, 0, 0, 14, 9, 6, 1, 8, 273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4240, 0, 0, 0, 0, 46, 0, 0, 0, 6230, 0, 0, 307, 0, 0, 0, 614, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 131] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 22, 0, 0, 0, 941, 0, 0, 5107, 0, 0, 0, 26, 0, 51, 1, 0, 8046, 2, 0, 0, 0, 1, 0, 0, 0, 4, 0, 0, 0, 357, 96, 0, 0, 0, 0, 0, 41, 197, 0, 0, 1103, 0, 0, 0, 6455, 0, 20, 0, 0, 0, 2391, 29, 0, 0, 0, 0, 12, 18, 237, 209, 0, 0, 0, 0, 0, 0, 92, 0, 0, 7955, 0, 149, 3, 0, 2, 0, 0, 8187, 0, 0, 0, 0, 1, 1080, 0, 0, 0, 2, 137, 0, 0, 0, 0, 59, 0, 0, 2701, 41, 37, 0, 952, 7614, 0, 83, 0, 59, 0, 0, 0, 7, 7, 0, 2351, 0, 0, 6980, 172, 0, 1, 0, 0, 0, 1498, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 70, 7112, 0, 0, 0, 3233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 0, 0, 0, 736, 2197, 0, 0, 0, 2434, 591, 0, 3487, 0, 0, 0, 0, 0, 0, 8142, 0, 0, 0, 2, 1, 0, 0, 0, 8191, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7870, 0, 0, 0, 7747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 4, 0, 515, 0, 0, 0, 1, 8093, 0, 0, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4796] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 2041, 0, 0, 1, 0, 0, 0, 3, 4570, 0, 0, 0, 9, 0, 0, 0, 0, 4996, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8, 6482, 0, 0, 18, 0, 708, 0, 129, 0, 0, 0, 0, 0, 0, 8172, 0, 372, 10, 1, 0, 1603, 46, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 9, 0, 23, 0, 0, 7575, 0, 0, 0, 222, 0, 0, 0, 6544, 0, 0, 0, 0, 0, 7, 0, 4, 0, 0, 57, 5, 9, 0, 0, 5129, 0, 0, 7784, 1266, 0, 0, 0, 7205, 0, 0, 0, 0, 12, 0, 0, 2, 0, 0, 1, 0, 0, 16, 55, 0, 15, 0, 379, 0, 0, 0, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 7969, 0, 1333, 0, 2407, 5, 0, 0, 0, 0, 2082, 0, 0, 0, 3, 0, 0, 20, 0, 0, 283, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 2778, 1, 0, 0, 0, 6, 57, 0, 0, 0, 2, 7941, 22, 0, 725, 0, 331, 0, 70, 6321, 0, 6869, 3, 2, 0, 7, 0, 0, 58, 2605, 3203, 0, 90, 0, 20, 0, 0, 0, 0, 659, 0, 1, 8176, 5317, 0, 0, 0, 34, 0, 1, 0, 0, 984, 0, 0, 630, 0, 0, 0, 5, 0, 46, 0, 0, 0, 4236, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 0, 153, 1, 0, 8019, 0, 2552, 1715, 0, 0, 236, 0, 0, 4486, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 2, 33, 0, 0, 282, 4, 14, 0, 0, 7845, 0, 0, 0, 145, 3, 0, 0, 11, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 63, 0, 1, 0, 0, 7900, 0, 0, 209, 0, 43, 1, 0, 1785, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 31, 305, 4626, 1, 20, 0, 3492, 0, 0, 48, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 6367, 10, 0, 0, 0, 0, 0, 0, 7177, 0, 7837] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 275, 2136, 7747, 0, 1, 9, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 33, 0, 0, 0, 0, 0, 0, 0, 0, 5083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 379, 0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 0, 6167, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2618, 0, 0, 12, 0, 8, 1162, 0, 0, 0, 3618, 1106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1612, 0, 0, 215, 0, 1964, 521, 2370, 3, 0, 0, 0, 0, 6466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 668, 0, 0, 0, 0, 2, 0, 0, 0, 6226, 0, 0, 0, 0, 0, 5, 1804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7018, 0, 6, 0, 326, 833, 0, 2, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 0, 0, 0, 0, 3375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1463, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10200, 5132, 6553, 8524, 13151, 8705, 8240, 5031] avg=8192.0000 max=13151 max_rank=4 min=5031 min_rank=7 imbalance_ratio=1.605347 max_load_reduction=52385 max_load_reduction_pct=0.799332 moved_to_echo=35127 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 7969, 0, 1333, 0, 1290, 5, 0, 0, 0, 0, 2082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 20, 0, 0, 283, 0, 1, 0, 0, 3, 0, 4954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 2778, 1, 0, 0, 3078, 1082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 57, 0, 0, 0, 2, 1905, 22, 0, 725, 0, 331, 0, 70, 6321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6869, 3, 2, 0, 7, 0, 0, 58, 2605, 3203, 0, 90, 0, 20, 0, 923, 1538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 659, 0, 1, 8176, 701, 0, 0, 0, 34, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 984, 0, 0, 630, 0, 0, 0, 5, 0, 46, 0, 0, 0, 4236, 0, 0, 0, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12682, 5264, 220, 6946, 9439, 15318, 9572, 6095] avg=8192.0000 max=15318 max_rank=5 min=220 min_rank=2 imbalance_ratio=1.869873 max_load_reduction=50218 max_load_reduction_pct=0.766266 moved_to_echo=40424 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 9, 0, 0, 0, 4399, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 7, 0, 8, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 133, 0, 0, 75, 0, 0, 0, 2527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1827, 0, 0, 0, 8128, 0, 2, 0, 0, 0, 0, 0, 0, 45, 0, 0, 4574, 1110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 17, 0, 0, 1970, 0, 0, 5994, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2244, 0, 0, 0, 0, 0, 2, 0, 0, 4666, 0, 0, 0, 0, 0, 3120, 2281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 364, 1236, 0, 0, 1043, 1053, 0, 0, 0, 6495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5633, 0, 1, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 599, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4413, 5213, 2738, 15686, 8069, 12313, 10191, 6913] avg=8192.0000 max=15686 max_rank=3 min=2738 min_rank=2 imbalance_ratio=1.914795 max_load_reduction=49850 max_load_reduction_pct=0.760651 moved_to_echo=37486 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 22, 0, 0, 0, 941, 0, 0, 2751, 0, 0, 0, 26, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8046, 2, 0, 0, 0, 1, 0, 0, 0, 4, 0, 0, 0, 357, 4960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 41, 197, 0, 0, 1103, 0, 0, 0, 5218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 2391, 29, 0, 0, 0, 0, 12, 18, 237, 209, 0, 0, 4404, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 1916, 0, 149, 3, 0, 2, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1080, 0, 0, 0, 2, 137, 0, 0, 0, 0, 59, 1951, 2196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2701, 41, 37, 0, 952, 1014, 0, 83, 0, 59, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 2351, 0, 0, 6980, 172, 0, 1, 0, 0, 0, 1498, 0, 0, 0, 1237, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3791, 13371, 6655, 8399, 10349, 5426, 4894, 12651] avg=8192.0000 max=13371 max_rank=1 min=3791 min_rank=0 imbalance_ratio=1.632202 max_load_reduction=52165 max_load_reduction_pct=0.795975 moved_to_echo=33320 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 70, 7112, 0, 0, 0, 1742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 0, 0, 0, 736, 4907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2197, 0, 0, 0, 2434, 591, 0, 3487, 0, 0, 0, 0, 0, 0, 6578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 8191, 0, 1, 0, 0, 0, 2, 0, 0, 4681, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1895, 0, 0, 0, 7747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 1235, 2335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 515, 0, 0, 0, 1, 1077, 0, 0, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4796, 1564, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8924, 5711, 15287, 13946, 9642, 3631, 1779, 6616] avg=8192.0000 max=15287 max_rank=2 min=1779 min_rank=6 imbalance_ratio=1.866089 max_load_reduction=50249 max_load_reduction_pct=0.766739 moved_to_echo=21668 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 7640, 0, 0, 0, 0, 0, 497, 0, 3873, 0, 0, 10, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 65, 0, 1, 0, 6606, 0, 0, 0, 2, 0, 0, 4, 0, 0, 5024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6773, 0, 0, 3, 0, 8, 0, 1, 0, 0, 21, 0, 0, 0, 6464, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 415, 0, 5, 0, 0, 2, 0, 1094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1941, 0, 0, 14, 9, 6, 1, 8, 273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4240, 0, 0, 0, 0, 46, 2747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6230, 0, 0, 307, 0, 0, 0, 614, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 131, 1533, 570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12029, 11704, 13424, 1519, 2252, 7033, 15339, 2236] avg=8192.0000 max=15339 max_rank=6 min=1519 min_rank=3 imbalance_ratio=1.872437 max_load_reduction=50197 max_load_reduction_pct=0.765945 moved_to_echo=26860 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 2041, 0, 0, 1, 0, 0, 0, 3, 2462, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4996, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8, 4723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6482, 0, 0, 18, 0, 708, 0, 129, 0, 0, 0, 0, 0, 0, 6606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 372, 10, 1, 0, 1603, 46, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 4167, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 9, 0, 23, 0, 0, 1824, 0, 0, 0, 222, 0, 0, 0, 6544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4, 0, 0, 57, 5, 9, 0, 0, 5129, 1746, 2078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7784, 1266, 0, 0, 0, 960, 0, 0, 0, 0, 12, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 16, 55, 0, 15, 0, 379, 0, 0, 0, 0, 3, 1566, 362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4516, 9728, 13943, 7264, 8629, 9035, 10024, 2397] avg=8192.0000 max=13943 max_rank=2 min=2397 min_rank=7 imbalance_ratio=1.702026 max_load_reduction=51593 max_load_reduction_pct=0.787247 moved_to_echo=30085 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 0, 153, 1, 0, 4320, 0, 2552, 1715, 0, 0, 236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4486, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 2, 33, 0, 0, 282, 4, 14, 0, 0, 6341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 3, 0, 0, 11, 1, 0, 0, 0, 0, 0, 0, 1, 0, 2019, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 1, 0, 0, 1903, 0, 0, 209, 0, 43, 1, 0, 1785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3063, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 305, 4626, 1, 20, 0, 466, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 6367, 10, 0, 0, 0, 0, 0, 0, 7177, 0, 7837, 1504, 636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8981, 9413, 6716, 3252, 4005, 4075, 5497, 23597] avg=8192.0000 max=23597 max_rank=7 min=3252 min_rank=3 imbalance_ratio=2.880493 max_load_reduction=41939 max_load_reduction_pct=0.639938 moved_to_echo=37174 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 7969, 0, 1333, 0, 1290, 5, 0, 0, 0, 0, 2082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 20, 0, 0, 283, 0, 1, 0, 0, 3, 0, 4954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 2778, 1, 0, 0, 3078, 1082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 57, 0, 0, 0, 2, 1905, 22, 0, 725, 0, 331, 0, 70, 6321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6869, 3, 2, 0, 7, 0, 0, 58, 2605, 3203, 0, 90, 0, 20, 0, 923, 1538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 659, 0, 1, 8176, 701, 0, 0, 0, 34, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 984, 0, 0, 630, 0, 0, 0, 5, 0, 46, 0, 0, 0, 4236, 0, 0, 0, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12682, 5264, 220, 6946, 9439, 15318, 9572, 6095] avg=8192.0000 max=15318 max_rank=5 min=220 min_rank=2 imbalance_ratio=1.869873 max_load_reduction=50218 max_load_reduction_pct=0.766266 moved_to_echo=40424 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 275, 2136, 7747, 0, 1, 9, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 33, 0, 0, 0, 0, 0, 0, 0, 0, 5083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 379, 0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 0, 6167, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2618, 0, 0, 12, 0, 8, 1162, 0, 0, 0, 3618, 1106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1612, 0, 0, 215, 0, 1964, 521, 2370, 3, 0, 0, 0, 0, 6466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 668, 0, 0, 0, 0, 2, 0, 0, 0, 6226, 0, 0, 0, 0, 0, 5, 1804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7018, 0, 6, 0, 326, 833, 0, 2, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 0, 0, 0, 0, 3375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1463, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10200, 5132, 6553, 8524, 13151, 8705, 8240, 5031] avg=8192.0000 max=13151 max_rank=4 min=5031 min_rank=7 imbalance_ratio=1.605347 max_load_reduction=52385 max_load_reduction_pct=0.799332 moved_to_echo=35127 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 7640, 0, 0, 0, 0, 0, 497, 0, 3873, 0, 0, 10, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 65, 0, 1, 0, 6606, 0, 0, 0, 2, 0, 0, 4, 0, 0, 5024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6773, 0, 0, 3, 0, 8, 0, 1, 0, 0, 21, 0, 0, 0, 6464, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 415, 0, 5, 0, 0, 2, 0, 1094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1941, 0, 0, 14, 9, 6, 1, 8, 273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4240, 0, 0, 0, 0, 46, 2747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6230, 0, 0, 307, 0, 0, 0, 614, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 131, 1533, 570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12029, 11704, 13424, 1519, 2252, 7033, 15339, 2236] avg=8192.0000 max=15339 max_rank=6 min=1519 min_rank=3 imbalance_ratio=1.872437 max_load_reduction=50197 max_load_reduction_pct=0.765945 moved_to_echo=26860 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 22, 0, 0, 0, 941, 0, 0, 2751, 0, 0, 0, 26, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8046, 2, 0, 0, 0, 1, 0, 0, 0, 4, 0, 0, 0, 357, 4960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 41, 197, 0, 0, 1103, 0, 0, 0, 5218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 2391, 29, 0, 0, 0, 0, 12, 18, 237, 209, 0, 0, 4404, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 1916, 0, 149, 3, 0, 2, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1080, 0, 0, 0, 2, 137, 0, 0, 0, 0, 59, 1951, 2196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2701, 41, 37, 0, 952, 1014, 0, 83, 0, 59, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 2351, 0, 0, 6980, 172, 0, 1, 0, 0, 0, 1498, 0, 0, 0, 1237, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3791, 13371, 6655, 8399, 10349, 5426, 4894, 12651] avg=8192.0000 max=13371 max_rank=1 min=3791 min_rank=0 imbalance_ratio=1.632202 max_load_reduction=52165 max_load_reduction_pct=0.795975 moved_to_echo=33320 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 70, 7112, 0, 0, 0, 1742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 0, 0, 0, 736, 4907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2197, 0, 0, 0, 2434, 591, 0, 3487, 0, 0, 0, 0, 0, 0, 6578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 8191, 0, 1, 0, 0, 0, 2, 0, 0, 4681, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1895, 0, 0, 0, 7747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 1235, 2335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 515, 0, 0, 0, 1, 1077, 0, 0, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4796, 1564, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8924, 5711, 15287, 13946, 9642, 3631, 1779, 6616] avg=8192.0000 max=15287 max_rank=2 min=1779 min_rank=6 imbalance_ratio=1.866089 max_load_reduction=50249 max_load_reduction_pct=0.766739 moved_to_echo=21668 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[1280, 128, 2432, 6272, 0, 16768, 256, 128, 128, 128, 384, 0, 1536, 11520, 0, 12800, 9472, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[2304, 128, 256, 128, 12160, 128, 2688, 8320, 0, 128, 512, 128, 4224, 384, 128, 128, 26624, 8704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[16000, 0, 0, 128, 2560, 1408, 128, 4096, 0, 512, 1280, 128, 0, 0, 39936, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] @@ -1548,34 +1550,34 @@ WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4814, 0, 1, 26, 2, 6, 11, 0, 318, 0, 4, 0, 0, 0, 2133, 0, 0, 2, 0, 0, 1, 0, 0, 2, 0, 17, 0, 0, 0, 228, 0, 0, 2, 0, 1, 0, 0, 0, 7825, 81, 0, 0, 0, 189, 0, 0, 7553, 0, 0, 0, 360, 0, 112, 0, 0, 7980, 0, 0, 0, 5, 0, 0, 0, 797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 290, 0, 0, 0, 0, 0, 0, 0, 0, 48, 1, 0, 2, 15, 269, 8133, 0, 8155, 0, 0, 66, 0, 0, 5017, 0, 0, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 2508, 0, 11, 0, 6964, 1273] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[70, 1, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 13, 0, 9, 7961, 0, 1, 0, 5534, 0, 0, 0, 0, 0, 2, 0, 0, 0, 12, 2, 0, 0, 14, 0, 77, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1175, 4730, 0, 0, 0, 2, 0, 0, 0, 0, 8, 0, 0, 0, 3509, 0, 1, 7548, 0, 0, 0, 1, 0, 0, 0, 0, 7894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 8142, 0, 0, 0, 0, 40, 0, 0, 1, 137, 0, 1315, 10, 1, 77, 0, 0, 1193, 462, 0, 7681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 91, 0, 0, 0, 7771, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4814, 0, 1, 26, 2, 6, 11, 0, 318, 0, 4, 0, 0, 0, 2133, 0, 0, 2, 0, 0, 1, 0, 0, 2, 0, 17, 0, 0, 0, 228, 0, 0, 2, 0, 1, 0, 0, 0, 7825, 81, 0, 0, 0, 189, 0, 0, 7553, 0, 0, 0, 360, 0, 112, 0, 0, 7980, 0, 0, 0, 5, 0, 0, 0, 797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 290, 0, 0, 0, 0, 0, 0, 0, 0, 48, 1, 0, 2, 15, 269, 8133, 0, 8155, 0, 0, 66, 0, 0, 5017, 0, 0, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 2508, 0, 11, 0, 6964, 1273] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 54, 0, 0, 24, 0, 0, 8101, 0, 0, 0, 0, 0, 5, 0, 0, 0, 190, 0, 0, 2687, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 104, 0, 0, 0, 0, 0, 0, 3, 0, 3, 8169, 0, 0, 4169, 0, 1, 0, 0, 0, 0, 0, 0, 7948, 0, 0, 20, 0, 0, 0, 0, 0, 2, 0, 0, 5, 0, 0, 53, 0, 0, 0, 0, 394, 0, 23, 15, 0, 0, 52, 7726, 0, 1372, 0, 0, 0, 0, 72, 5992, 0, 0, 20, 0, 0, 0, 0, 0, 0, 7579, 0, 0, 43, 0, 715, 0, 0, 0, 4857, 5089] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6248, 1, 0, 0, 740, 20, 15, 0, 0, 0, 0, 0, 1, 0, 0, 10, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 0, 0, 8147, 7839, 0, 8191, 0, 1, 131, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8165, 12, 0, 1875, 0, 3, 157, 0, 1546, 8017, 0, 3, 4926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 777, 0, 0, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[6, 0, 0, 0, 0, 0, 0, 3, 1019, 7, 0, 0, 134, 0, 0, 0, 6138, 0, 0, 0, 86, 6603, 3, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 757, 0, 3, 0, 1, 1, 0, 301, 0, 0, 0, 0, 0, 0, 0, 5, 40, 0, 0, 0, 0, 4865, 0, 133, 0, 8095, 38, 0, 49, 0, 7954, 19, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8, 0, 3, 21, 473, 0, 0, 3892, 0, 25, 0, 6494, 0, 1, 29, 0, 0, 0, 0, 0, 0, 0, 3, 5, 0, 2613, 0, 0, 105, 0, 0, 3653, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 32, 3, 0, 4032, 121, 2, 7, 7561, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 1139, 0, 0, 0, 0, 0, 0, 0, 4, 0, 5, 0, 0, 13, 0, 7141, 0, 0, 166, 0, 2296, 0, 0, 0, 0, 221, 0, 1, 0, 0, 14, 0, 694, 8, 36, 0, 465, 0, 0, 0, 5, 3, 0, 0, 106, 81, 0, 0, 0, 0, 0, 45, 178, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3592, 13, 0, 0, 0, 119, 130, 0, 0, 4268, 0, 0, 583, 0, 0, 0, 0, 0, 350, 0, 0, 10, 5920, 0, 7394, 0, 0, 0, 0, 13, 47, 2, 0, 0, 0, 0, 398, 0, 8043, 0, 0, 2643, 7031, 1, 17, 0, 0, 3583, 0, 0, 0, 0, 0, 48, 1, 0, 0, 0, 0, 0, 7, 0, 0, 754, 7947] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 336, 0, 0, 8189, 0, 0, 0, 103, 7233, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 4500, 0, 0, 1955, 0, 10, 3, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 39, 136, 8189, 0, 29, 0, 0, 0, 0, 0, 0, 777, 0, 6732, 6426, 0, 8081, 0, 0, 33, 571, 0, 7595, 0, 144, 0, 435, 0, 1, 0, 0, 38, 0, 13, 0, 0, 0, 0, 0, 0, 0, 1360, 1692] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[6, 0, 0, 0, 0, 0, 0, 3, 1019, 7, 0, 0, 134, 0, 0, 0, 6138, 0, 0, 0, 86, 6603, 3, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 757, 0, 3, 0, 1, 1, 0, 301, 0, 0, 0, 0, 0, 0, 0, 5, 40, 0, 0, 0, 0, 4865, 0, 133, 0, 8095, 38, 0, 49, 0, 7954, 19, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8, 0, 3, 21, 473, 0, 0, 3892, 0, 25, 0, 6494, 0, 1, 29, 0, 0, 0, 0, 0, 0, 0, 3, 5, 0, 2613, 0, 0, 105, 0, 0, 3653, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 32, 3, 0, 4032, 121, 2, 7, 7561, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 1815, 0, 0, 0, 0, 3, 0, 0, 7854, 17, 1, 0, 501, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 0, 0, 125, 0, 0, 0, 0, 183, 0, 0, 0, 0, 16, 0, 0, 8192, 4, 0, 0, 0, 0, 0, 0, 16, 790, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 4586, 0, 0, 0, 0, 3, 0, 807, 0, 0, 0, 0, 4733, 0, 4135, 0, 0, 0, 0, 0, 0, 3, 0, 8056, 0, 0, 1, 0, 0, 0, 28, 0, 0, 0, 0, 8159, 0, 0, 0, 7065, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6248, 1, 0, 0, 740, 20, 15, 0, 0, 0, 0, 0, 1, 0, 0, 10, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 0, 0, 8147, 7839, 0, 8191, 0, 1, 131, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8165, 12, 0, 1875, 0, 3, 157, 0, 1546, 8017, 0, 3, 4926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 777, 0, 0, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 1139, 0, 0, 0, 0, 0, 0, 0, 4, 0, 5, 0, 0, 13, 0, 7141, 0, 0, 166, 0, 2296, 0, 0, 0, 0, 221, 0, 1, 0, 0, 14, 0, 694, 8, 36, 0, 465, 0, 0, 0, 5, 3, 0, 0, 106, 81, 0, 0, 0, 0, 0, 45, 178, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3592, 13, 0, 0, 0, 119, 130, 0, 0, 4268, 0, 0, 583, 0, 0, 0, 0, 0, 350, 0, 0, 10, 5920, 0, 7394, 0, 0, 0, 0, 13, 47, 2, 0, 0, 0, 0, 398, 0, 8043, 0, 0, 2643, 7031, 1, 17, 0, 0, 3583, 0, 0, 0, 0, 0, 48, 1, 0, 0, 0, 0, 0, 7, 0, 0, 754, 7947] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 54, 0, 0, 24, 0, 0, 8101, 0, 0, 0, 0, 0, 5, 0, 5317, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 190, 0, 0, 2687, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 1130, 1494, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 0, 0, 0, 0, 3, 0, 3, 8169, 0, 0, 934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7948, 0, 0, 20, 0, 0, 0, 0, 3235, 675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 5, 0, 0, 53, 0, 0, 0, 0, 394, 0, 23, 15, 2967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 4737, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7579, 0, 0, 43, 0, 715, 0, 0, 0, 3363, 5089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13743, 2878, 2694, 9213, 11879, 3459, 4881, 16789] avg=8192.0000 max=16789 max_rank=7 min=2694 min_rank=2 imbalance_ratio=2.049438 max_load_reduction=48747 max_load_reduction_pct=0.743820 moved_to_echo=37008 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4814, 0, 1, 26, 2, 6, 11, 0, 4452, 1441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 4, 0, 0, 0, 2133, 0, 0, 2, 0, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 228, 0, 0, 2, 0, 1, 0, 0, 0, 7825, 81, 6714, 2142, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 0, 0, 7553, 0, 0, 0, 360, 0, 112, 0, 0, 1788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 797, 0, 0, 0, 0, 0, 0, 0, 0, 6192, 565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 290, 0, 0, 0, 0, 0, 0, 0, 0, 48, 1, 0, 3123, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 15, 269, 4986, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 2508, 0, 11, 0, 4822, 1273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10753, 2398, 17034, 10002, 7559, 3524, 5530, 8736] avg=8192.0000 max=17034 max_rank=2 min=2398 min_rank=1 imbalance_ratio=2.079346 max_load_reduction=48502 max_load_reduction_pct=0.740082 moved_to_echo=25349 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 336, 0, 0, 6740, 1427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6584, 0, 0, 0, 103, 7233, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 847, 0, 6654, 418, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 4500, 0, 0, 438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1517, 855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 39, 136, 8189, 0, 29, 0, 0, 0, 0, 0, 0, 2467, 1605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 777, 0, 6732, 3941, 0, 0, 0, 0, 33, 571, 0, 0, 0, 144, 0, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 38, 0, 13, 0, 0, 0, 0, 0, 0, 0, 942, 1692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8544, 13926, 7938, 4955, 2386, 12468, 12633, 2686] avg=8192.0000 max=13926 max_rank=1 min=2386 min_rank=4 imbalance_ratio=1.699951 max_load_reduction=51610 max_load_reduction_pct=0.787506 moved_to_echo=30173 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1139, 0, 0, 0, 0, 0, 0, 0, 4, 0, 5, 0, 0, 13, 0, 16, 1422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5739, 0, 0, 166, 0, 2296, 0, 0, 0, 0, 221, 0, 1, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 694, 8, 36, 0, 465, 0, 0, 0, 5, 3, 0, 0, 106, 81, 0, 6621, 234, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 178, 0, 0, 0, 0, 0, 0, 0, 1, 0, 803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 119, 130, 0, 0, 4268, 0, 0, 583, 0, 0, 0, 0, 2789, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 350, 0, 0, 10, 5920, 0, 7394, 0, 0, 0, 0, 13, 47, 2, 0, 157, 1402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 237, 0, 0, 0, 0, 2643, 7031, 1, 0, 0, 0, 3583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 1, 0, 0, 0, 0, 0, 7, 0, 0, 520, 7947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2599, 8437, 8257, 1027, 7903, 15295, 13495, 8523] avg=8192.0000 max=15295 max_rank=5 min=1027 min_rank=3 imbalance_ratio=1.867065 max_load_reduction=50241 max_load_reduction_pct=0.766617 moved_to_echo=45216 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[70, 1, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 13, 0, 9, 6816, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6401, 0, 1, 0, 5534, 0, 0, 0, 0, 0, 2, 0, 0, 0, 12, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 77, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1175, 4730, 63, 2390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8, 0, 0, 0, 3509, 0, 1, 1691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7894, 0, 0, 0, 0, 0, 0, 0, 5857, 865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 8142, 0, 0, 0, 0, 40, 0, 0, 1, 3, 1560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 1315, 7, 1, 0, 0, 0, 1193, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 91, 0, 0, 0, 5381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6952, 11952, 8453, 5211, 14617, 9754, 3115, 5482] avg=8192.0000 max=14617 max_rank=4 min=3115 min_rank=6 imbalance_ratio=1.784302 max_load_reduction=50919 max_load_reduction_pct=0.776962 moved_to_echo=32968 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8074, 0, 0, 0, 0, 0, 0, 0, 7116, 332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6248, 1, 0, 0, 740, 20, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 10, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 1543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 0, 0, 8147, 7839, 0, 1835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 131, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 6356, 901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 8, 0, 0, 0, 3, 157, 0, 1546, 0, 0, 3, 4926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 777, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15522, 7024, 1881, 17962, 7391, 164, 14808, 784] avg=8192.0000 max=17962 max_rank=3 min=164 min_rank=5 imbalance_ratio=2.192627 max_load_reduction=47574 max_load_reduction_pct=0.725922 moved_to_echo=23147 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 54, 0, 0, 24, 0, 0, 8101, 0, 0, 0, 0, 0, 5, 0, 5317, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 190, 0, 0, 2687, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 1130, 1494, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 0, 0, 0, 0, 3, 0, 3, 8169, 0, 0, 934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7948, 0, 0, 20, 0, 0, 0, 0, 3235, 675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 5, 0, 0, 53, 0, 0, 0, 0, 394, 0, 23, 15, 2967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 4737, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7579, 0, 0, 43, 0, 715, 0, 0, 0, 3363, 5089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13743, 2878, 2694, 9213, 11879, 3459, 4881, 16789] avg=8192.0000 max=16789 max_rank=7 min=2694 min_rank=2 imbalance_ratio=2.049438 max_load_reduction=48747 max_load_reduction_pct=0.743820 moved_to_echo=37008 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[70, 1, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 13, 0, 9, 6816, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6401, 0, 1, 0, 5534, 0, 0, 0, 0, 0, 2, 0, 0, 0, 12, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 77, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1175, 4730, 63, 2390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8, 0, 0, 0, 3509, 0, 1, 1691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7894, 0, 0, 0, 0, 0, 0, 0, 5857, 865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 8142, 0, 0, 0, 0, 40, 0, 0, 1, 3, 1560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 1315, 7, 1, 0, 0, 0, 1193, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 91, 0, 0, 0, 5381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6952, 11952, 8453, 5211, 14617, 9754, 3115, 5482] avg=8192.0000 max=14617 max_rank=4 min=3115 min_rank=6 imbalance_ratio=1.784302 max_load_reduction=50919 max_load_reduction_pct=0.776962 moved_to_echo=32968 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 0, 0, 0, 0, 0, 3, 1019, 7, 0, 0, 134, 0, 0, 0, 3243, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4935, 0, 0, 0, 86, 6603, 3, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 757, 0, 3, 0, 1, 1, 0, 301, 0, 0, 0, 0, 0, 0, 0, 2151, 2325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 40, 0, 0, 0, 0, 4865, 0, 133, 0, 8095, 38, 0, 49, 0, 1782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8, 0, 3, 21, 6172, 410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 0, 0, 3892, 0, 25, 0, 6494, 0, 1, 29, 0, 0, 0, 0, 0, 1, 1203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 32, 3, 0, 4032, 121, 2, 7, 5236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4874, 11796, 5539, 15007, 6634, 12118, 112, 9456] avg=8192.0000 max=15007 max_rank=3 min=112 min_rank=6 imbalance_ratio=1.831909 max_load_reduction=50529 max_load_reduction_pct=0.771011 moved_to_echo=28320 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 336, 0, 0, 6740, 1427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6584, 0, 0, 0, 103, 7233, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 847, 0, 6654, 418, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 4500, 0, 0, 438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1517, 855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 39, 136, 8189, 0, 29, 0, 0, 0, 0, 0, 0, 2467, 1605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 777, 0, 6732, 3941, 0, 0, 0, 0, 33, 571, 0, 0, 0, 144, 0, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 38, 0, 13, 0, 0, 0, 0, 0, 0, 0, 942, 1692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8544, 13926, 7938, 4955, 2386, 12468, 12633, 2686] avg=8192.0000 max=13926 max_rank=1 min=2386 min_rank=4 imbalance_ratio=1.699951 max_load_reduction=51610 max_load_reduction_pct=0.787506 moved_to_echo=30173 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 1815, 0, 0, 0, 0, 3, 0, 0, 0, 730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6315, 17, 1, 0, 501, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 3405, 2173, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 125, 0, 0, 0, 0, 183, 0, 0, 0, 0, 16, 0, 0, 1835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 16, 790, 0, 0, 0, 0, 0, 0, 0, 6357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 4586, 0, 0, 0, 0, 3, 0, 807, 0, 1817, 1539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2903, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 28, 0, 0, 0, 0, 8159, 0, 0, 0, 4892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2549, 7125, 13740, 2159, 7167, 8754, 10962, 13080] avg=8192.0000 max=13740 max_rank=2 min=2159 min_rank=3 imbalance_ratio=1.677246 max_load_reduction=51796 max_load_reduction_pct=0.790344 moved_to_echo=39963 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1139, 0, 0, 0, 0, 0, 0, 0, 4, 0, 5, 0, 0, 13, 0, 16, 1422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5739, 0, 0, 166, 0, 2296, 0, 0, 0, 0, 221, 0, 1, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 694, 8, 36, 0, 465, 0, 0, 0, 5, 3, 0, 0, 106, 81, 0, 6621, 234, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 178, 0, 0, 0, 0, 0, 0, 0, 1, 0, 803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 119, 130, 0, 0, 4268, 0, 0, 583, 0, 0, 0, 0, 2789, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 350, 0, 0, 10, 5920, 0, 7394, 0, 0, 0, 0, 13, 47, 2, 0, 157, 1402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 237, 0, 0, 0, 0, 2643, 7031, 1, 0, 0, 0, 3583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 1, 0, 0, 0, 0, 0, 7, 0, 0, 520, 7947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2599, 8437, 8257, 1027, 7903, 15295, 13495, 8523] avg=8192.0000 max=15295 max_rank=5 min=1027 min_rank=3 imbalance_ratio=1.867065 max_load_reduction=50241 max_load_reduction_pct=0.766617 moved_to_echo=45216 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 256, 256, 128, 256, 12544, 256, 256, 256, 8576, 128, 24576, 7936, 128, 11136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 1536, 128, 128, 128, 1024, 128, 0, 384, 128, 128, 0, 0, 128, 18048, 4864, 28288, 11264, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] @@ -1587,195 +1589,195 @@ WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 256, 128, 128, 256, 0, 896, 20992, 128, 0, 640, 128, 0, 128, 128, 38528, 4352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 1150, 3, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 154, 0, 0, 0, 51, 0, 0, 0, 1, 0, 0, 0, 6945, 0, 0, 0, 0, 6896, 0, 0, 633, 65, 0, 0, 3, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 4721, 2, 7, 17, 0, 0, 0, 6249, 1, 0, 0, 0, 0, 0, 0, 0, 1012, 7281, 0, 0, 0, 0, 2, 0, 0, 0, 39, 0, 0, 0, 152, 4589, 73, 1381, 0, 0, 0, 11, 0, 0, 0, 0, 0, 7651, 0, 0, 0, 0, 8147, 0, 0, 51, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4, 8125, 0, 2, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 450, 7908, 0, 0, 0, 0, 0, 0, 0, 0, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5061, 4, 0, 0, 5922, 0, 0, 0, 0, 0, 0, 5402, 0, 0, 8, 3749, 0, 0, 0, 244, 0, 7824, 2078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5023, 1, 3, 0, 0, 0, 7, 0, 1, 2, 1, 0, 0, 0, 1, 0, 2, 4754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 63, 0, 6, 0, 8176, 56, 0, 1, 5, 0, 11, 35, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 2, 0, 0, 0, 2, 0, 0, 0, 0, 8158, 0, 0, 7, 17] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 443, 0, 4, 0, 0, 0, 0, 0, 7941, 0, 0, 17, 0, 6, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 8186, 0, 0, 1233, 8146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 2, 7446, 0, 0, 0, 0, 17, 0, 0, 8182, 0, 0, 1, 3, 0, 0, 0, 297, 8139, 0, 0, 0, 0, 0, 0, 0, 0, 2, 9, 0, 0, 0, 0, 7, 0, 0, 0, 0, 6671, 5, 0, 0, 0, 0, 4150, 0, 0, 0, 0, 0, 121, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 1, 0, 4393, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1, 3306, 0, 0, 0, 24, 0, 0, 0, 0, 0, 8091, 0, 1, 14, 0, 4596, 2, 1123, 0, 0, 0, 0, 0, 18, 0, 46, 1, 1479, 68, 0, 0, 0, 0, 1, 0, 10, 7994, 68, 0, 0, 0, 0, 7564, 853, 0, 0, 0, 18, 2, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 5, 7023, 0, 0, 309, 0, 0, 0, 72, 452, 0, 0, 0, 3, 0, 6928, 0, 8152, 0, 0, 24, 0, 0, 0, 0, 0, 96, 0, 0, 0, 6, 81, 0, 0, 0, 1, 2033, 0, 0, 20, 0, 36, 0, 0, 0, 0, 0, 0, 15, 22, 0, 0, 0, 4901, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 34, 1302, 0, 17, 4, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 28, 0, 0, 0, 0, 7809, 0, 0, 0, 7005, 2, 0, 0, 0, 0, 114, 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 6806, 0, 0, 642, 0, 0, 0, 0, 43, 121, 0, 0, 662, 0, 0, 0, 0, 8132, 0, 0, 7534, 3639, 0, 0, 1236, 0, 0, 2, 0, 0, 0, 172, 0, 0, 0, 0, 0, 4325, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 0, 0, 3, 8191] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 1987, 0, 0, 0, 0, 0, 0, 0, 0, 6289, 14, 0, 0, 0, 0, 8155, 0, 0, 1316, 5025, 0, 9, 0, 5688, 0, 0, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 35, 7989, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 676, 0, 0, 4, 4256, 0, 0, 0, 0, 2857, 0, 0, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 14, 1, 0, 0, 0, 28, 0, 0, 0, 0, 2, 2710, 0, 0, 0, 0, 0, 0, 0, 8139, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 2, 119, 0, 0, 0, 0, 0, 3, 0, 0, 4322, 0, 0, 3526, 1, 0, 0, 0, 1, 0, 147, 2, 141, 0, 0, 0, 1219, 1445, 0, 0, 0, 0, 7774, 0, 0, 193, 12, 0, 0, 1, 3166, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 5309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4157, 0, 76, 7919, 0, 0, 0, 0, 0, 0, 5, 27, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1372, 0, 0, 0, 0, 2, 0, 3730, 406, 8155, 0, 0, 0, 0, 0, 0, 1, 675, 0, 0, 0, 81, 0, 2891, 0, 0, 8143, 0, 458, 0, 7] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1, 3306, 0, 0, 0, 24, 0, 0, 0, 0, 0, 8091, 0, 1, 14, 0, 4596, 2, 1123, 0, 0, 0, 0, 0, 18, 0, 46, 1, 1479, 68, 0, 0, 0, 0, 1, 0, 10, 7994, 68, 0, 0, 0, 0, 7564, 853, 0, 0, 0, 18, 2, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 5, 7023, 0, 0, 309, 0, 0, 0, 72, 452, 0, 0, 0, 3, 0, 6928, 0, 8152, 0, 0, 24, 0, 0, 0, 0, 0, 96, 0, 0, 0, 6, 81, 0, 0, 0, 1, 2033, 0, 0, 20, 0, 36, 0, 0, 0, 0, 0, 0, 15, 22, 0, 0, 0, 4901, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 0, 0, 3623, 0, 0, 0, 3918, 1, 0, 0, 0, 8180, 0, 36, 0, 0, 0, 404, 185, 3, 0, 12, 0, 0, 0, 0, 0, 0, 7929, 0, 241, 8110, 169, 0, 0, 0, 0, 0, 4346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 6248, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 10, 40, 0, 8137, 0, 0, 0, 0, 0, 1, 50, 0, 27, 0, 2286, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 2402, 1, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0, 7724, 0, 0, 0, 941, 0, 0, 0, 122, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 443, 0, 4, 0, 0, 0, 0, 0, 7941, 0, 0, 17, 0, 6, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 8186, 0, 0, 1233, 8146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 2, 7446, 0, 0, 0, 0, 17, 0, 0, 8182, 0, 0, 1, 3, 0, 0, 0, 297, 8139, 0, 0, 0, 0, 0, 0, 0, 0, 2, 9, 0, 0, 0, 0, 7, 0, 0, 0, 0, 6671, 5, 0, 0, 0, 0, 4150, 0, 0, 0, 0, 0, 121, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 1, 0, 4393, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 450, 7908, 0, 0, 0, 0, 0, 0, 0, 0, 518, 0, 0, 0, 0, 1173, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5061, 4, 0, 0, 5922, 0, 0, 0, 0, 0, 2062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 2514, 0, 0, 0, 244, 0, 7824, 2078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5023, 1, 3, 0, 0, 0, 7, 0, 1, 2, 1205, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 2, 3403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 63, 0, 6, 0, 8176, 56, 0, 1, 5, 0, 11, 35, 0, 1337, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 2, 0, 0, 0, 2, 0, 0, 0, 0, 8066, 0, 0, 7, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10141, 13049, 12668, 6304, 3407, 9837, 2007, 8123] avg=8192.0000 max=13049 max_rank=1 min=2007 min_rank=6 imbalance_ratio=1.592896 max_load_reduction=52487 max_load_reduction_pct=0.800888 moved_to_echo=23374 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 1572, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1987, 0, 0, 0, 0, 0, 0, 0, 0, 6289, 14, 0, 0, 0, 3113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1316, 3369, 0, 9, 0, 5688, 0, 0, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2025, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 35, 5719, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 676, 0, 0, 4, 4256, 0, 0, 0, 0, 2857, 0, 0, 0, 0, 2019, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 14, 1, 0, 0, 0, 28, 0, 0, 3023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2710, 0, 0, 0, 0, 0, 0, 0, 8047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9832, 11403, 10534, 3929, 5863, 10057, 3159, 10759] avg=8192.0000 max=11403 max_rank=1 min=3159 min_rank=6 imbalance_ratio=1.391968 max_load_reduction=54133 max_load_reduction_pct=0.826004 moved_to_echo=29838 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 119, 0, 0, 0, 0, 0, 3, 0, 0, 4322, 0, 0, 3526, 1, 3, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 147, 2, 141, 0, 0, 0, 1219, 1445, 0, 0, 0, 2968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 193, 9, 0, 0, 1, 3166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 5309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2976, 0, 76, 7919, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 27, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1924, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1372, 0, 0, 0, 0, 2, 0, 3730, 406, 8155, 0, 0, 0, 0, 0, 2882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 675, 0, 0, 0, 81, 0, 2891, 0, 0, 8051, 0, 458, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8068, 5923, 3369, 6368, 10972, 2125, 16547, 12164] avg=8192.0000 max=16547 max_rank=6 min=2125 min_rank=5 imbalance_ratio=2.019897 max_load_reduction=48989 max_load_reduction_pct=0.747513 moved_to_echo=41808 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 450, 7908, 0, 0, 0, 0, 0, 0, 0, 0, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5061, 4, 0, 0, 5922, 0, 0, 0, 0, 0, 0, 5402, 0, 0, 8, 3749, 0, 0, 0, 244, 0, 7824, 2078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5023, 1, 3, 0, 0, 0, 7, 0, 1, 2, 1, 0, 0, 0, 1, 0, 2, 4754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 63, 0, 6, 0, 8176, 56, 0, 1, 5, 0, 11, 35, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 2, 0, 0, 0, 2, 0, 0, 0, 0, 8158, 0, 0, 7, 17] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 443, 0, 4, 0, 0, 0, 0, 0, 7941, 0, 0, 17, 0, 2549, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 1233, 5461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 2, 7446, 0, 0, 0, 0, 17, 0, 0, 8182, 0, 2064, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 297, 5825, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 9, 0, 0, 0, 0, 7, 0, 0, 0, 0, 6671, 5, 0, 0, 0, 2028, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4150, 0, 0, 0, 0, 0, 121, 0, 0, 0, 0, 0, 2, 2, 0, 3033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 1, 0, 4344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11003, 3163, 6710, 17863, 6126, 8972, 7308, 4391] avg=8192.0000 max=17863 max_rank=3 min=3163 min_rank=1 imbalance_ratio=2.180542 max_load_reduction=47673 max_load_reduction_pct=0.727432 moved_to_echo=26797 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 0, 0, 3623, 0, 0, 0, 3918, 1, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 36, 0, 0, 0, 404, 185, 3, 0, 12, 0, 0, 0, 0, 0, 3028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 241, 8110, 115, 0, 0, 0, 0, 0, 4346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 6248, 37, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 8, 40, 0, 8137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 50, 0, 27, 0, 2286, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 1963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 2402, 1, 0, 123, 0, 0, 0, 0, 0, 0, 2938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7724, 0, 0, 0, 941, 0, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7597, 11848, 12812, 6418, 8195, 4359, 5520, 8787] avg=8192.0000 max=12812 max_rank=2 min=4359 min_rank=5 imbalance_ratio=1.563965 max_load_reduction=52724 max_load_reduction_pct=0.804504 moved_to_echo=26861 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 34, 1302, 0, 2192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 28, 0, 0, 0, 2981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4696, 2, 0, 0, 0, 0, 114, 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6806, 0, 0, 642, 0, 0, 0, 0, 43, 121, 2062, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 662, 0, 0, 0, 0, 5820, 0, 0, 7534, 3639, 0, 0, 1236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 172, 0, 0, 0, 0, 0, 4325, 0, 0, 0, 0, 1934, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 0, 0, 3, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3542, 3032, 4906, 9791, 18891, 6683, 10318, 8373] avg=8192.0000 max=18891 max_rank=4 min=3032 min_rank=1 imbalance_ratio=2.306030 max_load_reduction=46645 max_load_reduction_pct=0.711746 moved_to_echo=44265 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1150, 3, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 20, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 0, 0, 0, 51, 0, 0, 0, 1, 0, 0, 0, 6945, 0, 0, 0, 2634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 633, 44, 0, 0, 3, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4721, 2, 7, 17, 0, 0, 0, 6249, 1, 0, 1846, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 5211, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 152, 4589, 73, 1381, 0, 0, 0, 11, 0, 0, 0, 0, 1707, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7651, 0, 0, 0, 0, 8147, 0, 0, 51, 0, 0, 0, 0, 0, 0, 2555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4, 8034, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1352, 9785, 691, 12844, 6225, 8176, 18404, 8059] avg=8192.0000 max=18404 max_rank=6 min=691 min_rank=2 imbalance_ratio=2.246582 max_load_reduction=47132 max_load_reduction_pct=0.719177 moved_to_echo=40864 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 3306, 0, 0, 0, 24, 0, 0, 0, 0, 0, 8091, 0, 1, 14, 0, 2506, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4596, 2, 1123, 0, 0, 0, 0, 0, 18, 0, 46, 1, 1479, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 10, 5350, 68, 0, 0, 0, 0, 7564, 853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 2, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 0, 9, 0, 1784, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5019, 0, 0, 309, 0, 0, 0, 72, 452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 6928, 0, 8152, 0, 0, 24, 0, 0, 0, 0, 0, 0, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 0, 6, 81, 0, 0, 0, 1, 2033, 0, 0, 20, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 22, 0, 0, 0, 4843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14001, 7333, 13846, 2019, 5857, 15327, 2273, 4880] avg=8192.0000 max=15327 max_rank=5 min=2019 min_rank=3 imbalance_ratio=1.870972 max_load_reduction=50209 max_load_reduction_pct=0.766129 moved_to_echo=28337 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 34, 1302, 0, 2192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 28, 0, 0, 0, 2981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4696, 2, 0, 0, 0, 0, 114, 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6806, 0, 0, 642, 0, 0, 0, 0, 43, 121, 2062, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 662, 0, 0, 0, 0, 5820, 0, 0, 7534, 3639, 0, 0, 1236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 172, 0, 0, 0, 0, 0, 4325, 0, 0, 0, 0, 1934, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 0, 0, 3, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3542, 3032, 4906, 9791, 18891, 6683, 10318, 8373] avg=8192.0000 max=18891 max_rank=4 min=3032 min_rank=1 imbalance_ratio=2.306030 max_load_reduction=46645 max_load_reduction_pct=0.711746 moved_to_echo=44265 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 119, 0, 0, 0, 0, 0, 3, 0, 0, 4322, 0, 0, 3526, 1, 3, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 147, 2, 141, 0, 0, 0, 1219, 1445, 0, 0, 0, 2968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 193, 9, 0, 0, 1, 3166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 5309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2976, 0, 76, 7919, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 27, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1924, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1372, 0, 0, 0, 0, 2, 0, 3730, 406, 8155, 0, 0, 0, 0, 0, 2882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 675, 0, 0, 0, 81, 0, 2891, 0, 0, 8051, 0, 458, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8068, 5923, 3369, 6368, 10972, 2125, 16547, 12164] avg=8192.0000 max=16547 max_rank=6 min=2125 min_rank=5 imbalance_ratio=2.019897 max_load_reduction=48989 max_load_reduction_pct=0.747513 moved_to_echo=41808 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 450, 7908, 0, 0, 0, 0, 0, 0, 0, 0, 518, 0, 0, 0, 0, 1173, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5061, 4, 0, 0, 5922, 0, 0, 0, 0, 0, 2062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 2514, 0, 0, 0, 244, 0, 7824, 2078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5023, 1, 3, 0, 0, 0, 7, 0, 1, 2, 1205, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 2, 3403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 63, 0, 6, 0, 8176, 56, 0, 1, 5, 0, 11, 35, 0, 1337, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 2, 0, 0, 0, 2, 0, 0, 0, 0, 8066, 0, 0, 7, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10141, 13049, 12668, 6304, 3407, 9837, 2007, 8123] avg=8192.0000 max=13049 max_rank=1 min=2007 min_rank=6 imbalance_ratio=1.592896 max_load_reduction=52487 max_load_reduction_pct=0.800888 moved_to_echo=23374 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 1572, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1987, 0, 0, 0, 0, 0, 0, 0, 0, 6289, 14, 0, 0, 0, 3113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1316, 3369, 0, 9, 0, 5688, 0, 0, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2025, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 35, 5719, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 676, 0, 0, 4, 4256, 0, 0, 0, 0, 2857, 0, 0, 0, 0, 2019, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 14, 1, 0, 0, 0, 28, 0, 0, 3023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2710, 0, 0, 0, 0, 0, 0, 0, 8047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9832, 11403, 10534, 3929, 5863, 10057, 3159, 10759] avg=8192.0000 max=11403 max_rank=1 min=3159 min_rank=6 imbalance_ratio=1.391968 max_load_reduction=54133 max_load_reduction_pct=0.826004 moved_to_echo=29838 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 13312, 0, 0, 128, 128, 18048, 128, 3840, 640, 10240, 0, 0, 128, 128, 128, 19328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[13056, 128, 3200, 128, 128, 256, 5504, 384, 128, 0, 6016, 7552, 9984, 128, 0, 0, 19968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 768, 128, 256, 14080, 4352, 17792, 128, 0, 128, 13952, 128, 128, 128, 0, 12928, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 0, 128, 256, 37376, 128, 128, 768, 0, 128, 128, 6272, 8320, 128, 12160, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 256, 11520, 21632, 128, 128, 128, 9216, 0, 19968, 3200, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 0, 128, 256, 37376, 128, 128, 768, 0, 128, 128, 6272, 8320, 128, 12160, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 4992, 16256, 512, 128, 128, 0, 3712, 128, 0, 0, 24832, 128, 128, 4864, 128, 10112, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 768, 0, 128, 0, 1408, 34048, 128, 128, 23936, 3712, 128, 128, 1408, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 8448, 2816, 0, 128, 1152, 128, 2944, 128, 128, 41600, 0, 512, 128, 8320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 476, 0, 0, 0, 0, 1, 1267, 0, 1, 131, 0, 3, 0, 0, 7686, 0, 0, 0, 8009, 0, 0, 0, 0, 0, 47, 0, 8, 0, 7630, 3, 3, 0, 4, 1, 23, 0, 0, 0, 280, 0, 0, 0, 0, 0, 10, 0, 0, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 52, 4468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 6417, 3, 2, 92, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 329, 0, 0, 0, 0, 0, 0, 0, 3535, 4, 0, 0, 0, 0, 0, 8027, 0, 0, 0, 2, 8035, 0, 12, 0, 516, 39, 0, 0, 0, 0, 0, 0, 27, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 24, 5392, 0, 9, 0, 0, 0, 0, 0, 0, 161, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 8055, 0, 0, 0, 0, 0, 7258, 0, 0, 5, 8180, 0, 0, 0, 6268, 71, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 253, 0, 0, 7929, 0, 0, 0, 0, 0, 5, 0, 124, 0, 0, 0, 0, 70, 0, 0, 2134, 0, 0, 0, 572, 0, 3, 0, 0, 3, 0, 36, 0, 3, 0, 0, 0, 0, 2475, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 8019, 0, 0, 32, 0, 0, 0, 0, 0, 7777, 7, 0, 0, 0, 0, 0, 450, 0, 120] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 4710, 1872, 0, 0, 0, 0, 0, 0, 0, 0, 33, 6306, 0, 0, 0, 140, 8147, 18, 0, 0, 6827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7657, 40, 39, 0, 0, 0, 0, 0, 0, 13, 7861, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 1, 870, 477, 0, 0, 0, 3, 0, 7156, 0, 0, 0, 0, 3273, 0, 196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 144, 0, 0, 109, 8169, 0, 0, 976, 287, 0, 0, 0, 0, 0, 0, 5, 0, 67, 125, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 476, 0, 0, 0, 0, 1, 1267, 0, 1, 131, 0, 3, 0, 0, 7686, 0, 0, 0, 8009, 0, 0, 0, 0, 0, 47, 0, 8, 0, 7630, 3, 3, 0, 4, 1, 23, 0, 0, 0, 280, 0, 0, 0, 0, 0, 10, 0, 0, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 52, 4468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 6417, 3, 2, 92, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 329, 0, 0, 0, 0, 0, 0, 0, 3535, 4, 0, 0, 0, 0, 0, 8027, 0, 0, 0, 2, 8035, 0, 12, 0, 516, 39, 0, 0, 0, 0, 0, 0, 27, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 5986, 0, 0, 0, 777, 0, 8, 76, 0, 7, 69, 0, 0, 0, 0, 0, 4166, 0, 0, 0, 8, 2, 0, 0, 4, 0, 0, 8, 0, 534, 1728, 2, 0, 0, 0, 0, 7579, 0, 0, 1, 3419, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5619, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4742, 0, 0, 3376, 0, 2165, 0, 0, 0, 7631, 0, 0, 0, 11, 0, 0, 3461, 1, 0, 0, 0, 0, 0, 2765, 0, 0, 155, 0, 0, 6, 0, 43, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8187, 0, 0, 254, 0, 5, 0, 0, 0, 2592, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[4, 2237, 0, 0, 1, 148, 0, 6, 8, 171, 0, 3966, 0, 0, 0, 1, 0, 8129, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 4, 3181, 0, 0, 45, 0, 0, 0, 0, 0, 0, 8051, 0, 12, 0, 4231, 0, 726, 0, 0, 0, 0, 0, 11, 0, 0, 2, 0, 206, 255, 1, 5823, 13, 0, 11, 0, 2930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 10, 0, 0, 7245, 0, 0, 0, 2, 2, 7279, 0, 0, 0, 2, 0, 19, 0, 0, 0, 1, 8089, 0, 0, 0, 0, 2620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 4710, 1872, 0, 0, 0, 0, 0, 0, 0, 0, 33, 6306, 0, 0, 0, 140, 8147, 18, 0, 0, 6827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7657, 40, 39, 0, 0, 0, 0, 0, 0, 13, 7861, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 1, 870, 477, 0, 0, 0, 3, 0, 7156, 0, 0, 0, 0, 3273, 0, 196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 144, 0, 0, 109, 8169, 0, 0, 976, 287, 0, 0, 0, 0, 0, 0, 5, 0, 67, 125, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 580, 0, 0, 0, 30, 0, 0, 21, 0, 0, 0, 0, 0, 0, 2, 0, 4787, 48, 0, 0, 0, 0, 177, 4, 0, 0, 0, 0, 1, 0, 8186, 0, 2, 0, 0, 0, 240, 23, 0, 0, 6806, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 10, 0, 6, 321, 0, 37, 0, 688, 0, 8140, 0, 0, 0, 0, 118, 0, 22, 0, 7662, 3, 58, 0, 0, 7058, 0, 2, 3, 0, 3229, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 579, 0, 233, 0, 0, 0, 82, 3, 7296, 0, 0, 11, 0, 113, 0, 418, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 7999] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 4005, 2, 0, 0, 0, 3, 0, 1, 0, 0, 250, 0, 0, 0, 0, 4, 6192, 0, 0, 0, 4186, 0, 0, 0, 0, 0, 65, 235, 0, 0, 0, 24, 0, 0, 0, 0, 2214, 0, 0, 0, 8090, 0, 0, 0, 4272, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1, 1541, 0, 0, 5097, 0, 142, 0, 0, 0, 0, 0, 7879, 0, 0, 7885, 0, 18, 0, 0, 0, 0, 0, 0, 296, 0, 0, 155, 0, 258, 0, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 7161, 0, 0, 1659, 0, 0, 0, 0, 0, 2373, 0, 0, 0, 0, 0, 0, 0, 1435, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 249, 1788, 0, 0, 0, 0, 2, 0, 0, 13, 96, 0, 0, 0, 636, 0, 6800, 0, 0, 345, 6718, 0, 0, 0, 0, 0, 14, 1033, 0, 212, 2186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5812, 0, 0, 0, 8137, 0, 0, 0, 0, 10, 3929, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 920, 1, 0, 0, 4, 0, 2, 0, 0, 0, 5160, 0, 0, 0, 0, 2561, 0, 0, 0, 0, 0, 0, 1261, 0, 0, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 368, 0, 0, 0, 0, 0, 7957, 0, 3, 833, 0, 13, 0, 0, 0, 0, 0, 0, 0, 1, 8189, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 580, 0, 0, 0, 30, 0, 0, 21, 0, 0, 0, 0, 0, 0, 2, 7613, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 598, 48, 0, 0, 0, 0, 177, 4, 0, 0, 0, 0, 1, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 240, 23, 0, 0, 3418, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 10, 0, 6, 2951, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 226, 0, 37, 0, 688, 0, 8140, 0, 0, 0, 0, 118, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 58, 0, 0, 7058, 0, 2, 3, 0, 3229, 531, 0, 0, 0, 0, 0, 3684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 579, 0, 233, 0, 0, 0, 82, 3, 5872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 113, 0, 418, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 7999, 1424, 95, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8683, 9014, 3685, 3473, 9231, 14568, 6769, 10113] avg=8192.0000 max=14568 max_rank=5 min=3473 min_rank=3 imbalance_ratio=1.778320 max_load_reduction=50968 max_load_reduction_pct=0.777710 moved_to_echo=40681 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 249, 1788, 0, 0, 0, 0, 2, 0, 0, 13, 96, 0, 0, 0, 636, 0, 369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 859, 0, 0, 345, 6718, 0, 0, 0, 0, 0, 14, 1033, 0, 212, 2186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2925, 0, 0, 0, 8137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3929, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 920, 1, 2518, 711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2, 0, 0, 0, 5160, 0, 0, 0, 0, 2561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1261, 0, 0, 157, 0, 0, 0, 0, 0, 0, 0, 0, 5230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 368, 0, 0, 0, 0, 0, 6408, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 833, 0, 13, 0, 0, 0, 0, 0, 0, 0, 1, 8189, 0, 0, 0, 1, 1549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3153, 11367, 11062, 8214, 7727, 6648, 6779, 10586] avg=8192.0000 max=11367 max_rank=1 min=3153 min_rank=0 imbalance_ratio=1.387573 max_load_reduction=54169 max_load_reduction_pct=0.826553 moved_to_echo=31740 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 4005, 2, 0, 0, 0, 3, 0, 1, 0, 0, 250, 0, 0, 0, 0, 4, 6192, 0, 0, 0, 4186, 0, 0, 0, 0, 0, 65, 235, 0, 0, 0, 24, 0, 0, 0, 0, 2214, 0, 0, 0, 8090, 0, 0, 0, 4272, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1, 1541, 0, 0, 5097, 0, 142, 0, 0, 0, 0, 0, 7879, 0, 0, 7885, 0, 18, 0, 0, 0, 0, 0, 0, 296, 0, 0, 155, 0, 258, 0, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 7161, 0, 0, 1659, 0, 0, 0, 0, 0, 2373, 0, 0, 0, 0, 0, 0, 0, 1435, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 24, 5392, 0, 9, 0, 0, 0, 0, 0, 0, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 8055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7258, 0, 0, 5, 4117, 0, 0, 0, 6268, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 253, 0, 0, 3544, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5628, 0, 0, 0, 0, 0, 5, 0, 124, 0, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2134, 0, 0, 0, 572, 0, 3, 0, 0, 3, 0, 36, 0, 3, 0, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2475, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 6458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 7777, 7, 0, 0, 0, 0, 0, 450, 0, 120, 1561, 2301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5944, 8145, 17719, 3817, 5827, 2874, 8962, 12248] avg=8192.0000 max=17719 max_rank=2 min=2874 min_rank=5 imbalance_ratio=2.162964 max_load_reduction=47817 max_load_reduction_pct=0.729630 moved_to_echo=29911 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 5986, 0, 0, 0, 777, 0, 8, 76, 0, 7, 69, 0, 0, 0, 0, 0, 217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 0, 0, 0, 8, 2, 0, 0, 4, 0, 0, 8, 0, 534, 1728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 7579, 0, 0, 1, 1721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5619, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4742, 0, 0, 1481, 436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2396, 0, 2165, 0, 0, 0, 7631, 0, 0, 0, 11, 0, 0, 3461, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2765, 0, 0, 155, 0, 0, 6, 0, 43, 145, 0, 0, 3204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 6594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 254, 0, 5, 0, 0, 0, 2592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1593, 980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7140, 2810, 9303, 12280, 15665, 6318, 6596, 5424] avg=8192.0000 max=15665 max_rank=4 min=2810 min_rank=1 imbalance_ratio=1.912231 max_load_reduction=49871 max_load_reduction_pct=0.760971 moved_to_echo=34003 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 476, 0, 0, 0, 0, 1, 1267, 0, 1, 131, 0, 3, 0, 6376, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 971, 0, 0, 0, 8009, 0, 0, 0, 0, 0, 47, 0, 8, 0, 7630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 4, 1, 23, 0, 0, 0, 142, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 52, 121, 804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 92, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 329, 0, 5911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3535, 4, 0, 0, 0, 0, 0, 6465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8035, 0, 12, 0, 516, 39, 0, 0, 0, 0, 0, 0, 27, 0, 1562, 1297, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8272, 16665, 186, 9277, 3248, 6353, 10004, 11531] avg=8192.0000 max=16665 max_rank=1 min=186 min_rank=2 imbalance_ratio=2.034302 max_load_reduction=48871 max_load_reduction_pct=0.745712 moved_to_echo=31136 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 249, 1788, 0, 0, 0, 0, 2, 0, 0, 13, 96, 0, 0, 0, 636, 0, 369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 859, 0, 0, 345, 6718, 0, 0, 0, 0, 0, 14, 1033, 0, 212, 2186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2925, 0, 0, 0, 8137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3929, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 920, 1, 2518, 711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2, 0, 0, 0, 5160, 0, 0, 0, 0, 2561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1261, 0, 0, 157, 0, 0, 0, 0, 0, 0, 0, 0, 5230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 368, 0, 0, 0, 0, 0, 6408, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 833, 0, 13, 0, 0, 0, 0, 0, 0, 0, 1, 8189, 0, 0, 0, 1, 1549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3153, 11367, 11062, 8214, 7727, 6648, 6779, 10586] avg=8192.0000 max=11367 max_rank=1 min=3153 min_rank=0 imbalance_ratio=1.387573 max_load_reduction=54169 max_load_reduction_pct=0.826553 moved_to_echo=31740 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 4710, 1872, 0, 0, 0, 0, 0, 0, 0, 0, 33, 6306, 0, 0, 0, 7109, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 1029, 18, 0, 0, 6827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3854, 40, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 7861, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8, 0, 3317, 852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 870, 477, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3273, 0, 196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 144, 0, 0, 109, 6579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 976, 287, 0, 0, 0, 0, 0, 0, 5, 0, 67, 125, 0, 0, 0, 0, 1590, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20516, 8014, 3934, 12054, 1351, 9735, 6835, 3097] avg=8192.0000 max=20516 max_rank=0 min=1351 min_rank=4 imbalance_ratio=2.504395 max_load_reduction=45020 max_load_reduction_pct=0.686951 moved_to_echo=21018 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 4005, 2, 0, 0, 0, 3, 0, 1, 0, 0, 250, 0, 0, 0, 0, 0, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 782, 0, 0, 0, 4186, 0, 0, 0, 0, 0, 65, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 2214, 0, 0, 0, 4071, 0, 0, 0, 4272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1, 1541, 0, 0, 3505, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3618, 0, 142, 0, 0, 0, 0, 0, 7879, 0, 0, 7885, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 296, 0, 0, 155, 0, 258, 0, 10, 3, 0, 0, 0, 4762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 5767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1659, 0, 0, 0, 0, 0, 2373, 0, 0, 0, 0, 0, 0, 0, 1435, 0, 1394, 1479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4775, 5272, 10581, 5771, 19542, 5484, 5771, 8340] avg=8192.0000 max=19542 max_rank=4 min=4775 min_rank=0 imbalance_ratio=2.385498 max_load_reduction=45994 max_load_reduction_pct=0.701813 moved_to_echo=39137 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 476, 0, 0, 0, 0, 1, 1267, 0, 1, 131, 0, 3, 0, 6376, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 971, 0, 0, 0, 8009, 0, 0, 0, 0, 0, 47, 0, 8, 0, 7630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 4, 1, 23, 0, 0, 0, 142, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 52, 121, 804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 92, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 329, 0, 5911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3535, 4, 0, 0, 0, 0, 0, 6465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8035, 0, 12, 0, 516, 39, 0, 0, 0, 0, 0, 0, 27, 0, 1562, 1297, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8272, 16665, 186, 9277, 3248, 6353, 10004, 11531] avg=8192.0000 max=16665 max_rank=1 min=186 min_rank=2 imbalance_ratio=2.034302 max_load_reduction=48871 max_load_reduction_pct=0.745712 moved_to_echo=31136 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4, 2237, 0, 0, 1, 148, 0, 6, 8, 171, 0, 3966, 0, 0, 0, 1, 0, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 4, 3181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 4052, 0, 12, 0, 4231, 0, 726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 2, 0, 206, 255, 1, 5823, 13, 0, 3488, 851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 2930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 10, 0, 0, 7245, 0, 0, 0, 6252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 7279, 0, 0, 0, 2, 0, 19, 0, 0, 0, 1, 6515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1574, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7053, 4249, 9066, 10650, 2945, 13556, 13820, 4197] avg=8192.0000 max=13820 max_rank=6 min=2945 min_rank=4 imbalance_ratio=1.687012 max_load_reduction=51716 max_load_reduction_pct=0.789124 moved_to_echo=34518 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 580, 0, 0, 0, 30, 0, 0, 21, 0, 0, 0, 0, 0, 0, 2, 7613, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 598, 48, 0, 0, 0, 0, 177, 4, 0, 0, 0, 0, 1, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 240, 23, 0, 0, 3418, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 10, 0, 6, 2951, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 226, 0, 37, 0, 688, 0, 8140, 0, 0, 0, 0, 118, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 58, 0, 0, 7058, 0, 2, 3, 0, 3229, 531, 0, 0, 0, 0, 0, 3684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 579, 0, 233, 0, 0, 0, 82, 3, 5872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 113, 0, 418, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 7999, 1424, 95, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8683, 9014, 3685, 3473, 9231, 14568, 6769, 10113] avg=8192.0000 max=14568 max_rank=5 min=3473 min_rank=3 imbalance_ratio=1.778320 max_load_reduction=50968 max_load_reduction_pct=0.777710 moved_to_echo=40681 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 5986, 0, 0, 0, 777, 0, 8, 76, 0, 7, 69, 0, 0, 0, 0, 0, 217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 0, 0, 0, 8, 2, 0, 0, 4, 0, 0, 8, 0, 534, 1728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 7579, 0, 0, 1, 1721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5619, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4742, 0, 0, 1481, 436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2396, 0, 2165, 0, 0, 0, 7631, 0, 0, 0, 11, 0, 0, 3461, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2765, 0, 0, 155, 0, 0, 6, 0, 43, 145, 0, 0, 3204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 6594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 254, 0, 5, 0, 0, 0, 2592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1593, 980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7140, 2810, 9303, 12280, 15665, 6318, 6596, 5424] avg=8192.0000 max=15665 max_rank=4 min=2810 min_rank=1 imbalance_ratio=1.912231 max_load_reduction=49871 max_load_reduction_pct=0.760971 moved_to_echo=34003 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[15104, 0, 5376, 0, 768, 0, 15872, 128, 14080, 512, 128, 8064, 0, 6144, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[3840, 384, 10880, 0, 512, 0, 13312, 128, 128, 128, 128, 8320, 0, 512, 1536, 8192, 12288, 6272, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 128, 128, 17408, 128, 0, 128, 24320, 128, 128, 0, 22912, 128, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[3840, 384, 10880, 0, 512, 0, 13312, 128, 128, 128, 128, 8320, 0, 512, 1536, 8192, 12288, 6272, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[2176, 128, 128, 0, 15360, 0, 256, 512, 0, 3584, 640, 128, 7296, 256, 384, 0, 35456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 17664, 128, 128, 0, 128, 128, 128, 256, 256, 128, 20608, 1024, 128, 20992, 4864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 17792, 3712, 512, 128, 1024, 128, 128, 5504, 1536, 128, 4480, 6528, 0, 128, 640, 21120, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 17664, 128, 128, 0, 128, 128, 128, 256, 256, 128, 20608, 1024, 128, 20992, 4864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[256, 5888, 128, 128, 384, 25856, 128, 256, 128, 128, 128, 128, 1280, 128, 896, 30976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 9856, 128, 0, 640, 3584, 640, 128, 256, 0, 128, 128, 50688, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 3699, 0, 0, 0, 0, 0, 0, 0, 0, 7, 41, 0, 0, 0, 0, 0, 0, 3202, 0, 4705, 0, 0, 0, 1819, 0, 0, 0, 0, 5289, 0, 0, 0, 0, 7639, 0, 1195, 0, 0, 3, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 5205, 0, 6934, 0, 2, 8109, 0, 0, 0, 60, 37, 2139, 0, 0, 0, 0, 621, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 1339, 0, 0, 0, 0, 0, 0, 0, 2257, 0, 0, 0, 0, 2435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 296, 0, 4103, 0, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 0, 0, 58, 0, 3956] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7393, 0, 0, 0, 0, 688, 0, 0, 140, 0, 8081, 0, 0, 0, 0, 759, 0, 11, 0, 0, 0, 0, 0, 0, 6607, 0, 215, 0, 1393, 0, 0, 0, 7975, 0, 0, 100, 0, 0, 0, 83, 0, 0, 0, 0, 0, 3, 0, 8102, 0, 0, 12, 0, 0, 0, 721, 0, 0, 0, 3, 0, 0, 8, 0, 1, 1, 0, 101, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 7906, 4, 0, 0, 0, 6486, 1, 0, 0, 145, 0, 0, 0, 0, 15, 3, 0, 8136, 0, 0, 77, 0, 0, 14, 0, 0, 0, 0, 304, 0, 0, 0, 0, 14, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 98, 0, 0, 0, 2889, 0, 5635, 0, 0, 0, 8052, 0, 0, 8073, 0, 7844, 0, 0, 8115, 0, 30, 0, 0, 0, 2, 0, 0, 89, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1932, 0, 0, 0, 0, 150, 0, 0, 0, 0, 7904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1323, 0, 0, 0, 24, 0, 0, 5, 1130, 0, 0, 0, 0, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 15, 0, 0, 0, 0, 0, 1177, 0, 2109] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[6, 8082, 0, 0, 0, 58, 0, 0, 44, 13, 34, 75, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 1, 17, 0, 0, 0, 1, 0, 0, 7441, 0, 8135, 0, 0, 22, 0, 0, 0, 0, 64, 161, 0, 0, 0, 0, 3, 0, 0, 0, 0, 191, 6082, 0, 0, 7970, 0, 0, 0, 100, 8032, 0, 0, 0, 0, 0, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 72, 33, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 667, 593, 0, 0, 0, 19, 2144, 0, 7090] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 31, 42, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 80, 0, 0, 0, 536, 0, 7947, 0, 0, 0, 8174, 0, 0, 0, 65, 8084, 0, 9, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 8080, 0, 2, 5, 0, 1, 0, 0, 7211, 134, 2, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1607, 12, 0, 1, 0, 0, 1, 0, 1992, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 14, 0, 0, 0, 0, 8188, 0, 2, 0, 1, 0, 0, 0, 14, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 10, 0, 0, 4950] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 3699, 0, 0, 0, 0, 0, 0, 0, 0, 7, 41, 0, 0, 0, 0, 0, 0, 3202, 0, 4705, 0, 0, 0, 1819, 0, 0, 0, 0, 5289, 0, 0, 0, 0, 7639, 0, 1195, 0, 0, 3, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 5205, 0, 6934, 0, 2, 8109, 0, 0, 0, 60, 37, 2139, 0, 0, 0, 0, 621, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 1339, 0, 0, 0, 0, 0, 0, 0, 2257, 0, 0, 0, 0, 2435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 296, 0, 4103, 0, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 0, 0, 58, 0, 3956] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 216, 15, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 8192, 1016, 0, 0, 1, 0, 8191, 0, 389, 0, 8158, 0, 0, 4, 33, 22, 0, 0, 0, 0, 17, 0, 12, 0, 6443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5, 0, 0, 0, 42, 7964, 8172, 0, 0, 0, 0, 599, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 8084, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 8, 0, 0, 0, 0, 0, 79, 7826] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2618, 17, 0, 79, 6034, 0, 0, 0, 0, 3021, 0, 0, 37, 0, 8192, 2, 41, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 621, 0, 0, 0, 8154, 0, 0, 35, 3574, 8159, 0, 0, 0, 0, 651, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 1714, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 4636, 1336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8036] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7393, 0, 0, 0, 0, 688, 0, 0, 140, 0, 8081, 0, 0, 0, 0, 759, 0, 11, 0, 0, 0, 0, 0, 0, 6607, 0, 215, 0, 1393, 0, 0, 0, 7975, 0, 0, 100, 0, 0, 0, 83, 0, 0, 0, 0, 0, 3, 0, 8102, 0, 0, 12, 0, 0, 0, 721, 0, 0, 0, 3, 0, 0, 8, 0, 1, 1, 0, 101, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 7906, 4, 0, 0, 0, 6486, 1, 0, 0, 145, 0, 0, 0, 0, 15, 3, 0, 8136, 0, 0, 77, 0, 0, 14, 0, 0, 0, 0, 304, 0, 0, 0, 0, 14, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 98, 0, 0, 0, 2889, 0, 5635, 0, 0, 0, 8052, 0, 0, 8073, 0, 7844, 0, 0, 8115, 0, 30, 0, 0, 0, 2, 0, 0, 89, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1932, 0, 0, 0, 0, 150, 0, 0, 0, 0, 7904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1323, 0, 0, 0, 24, 0, 0, 5, 1130, 0, 0, 0, 0, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 15, 0, 0, 0, 0, 0, 1177, 0, 2109] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[11, 8, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 20, 0, 4, 0, 0, 4262, 0, 166, 0, 0, 0, 8074, 0, 10, 0, 0, 7419, 0, 0, 1130, 0, 0, 2, 0, 650, 1, 12, 0, 0, 7776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 8131, 28, 0, 0, 1, 6575, 0, 0, 0, 0, 77, 0, 67, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 254, 0, 0, 0, 0, 0, 0, 7992, 0, 8, 0, 0, 1126, 0, 0, 0, 0, 8, 0, 3, 0, 0, 0, 775, 0, 0, 0, 0, 0, 731, 3, 0, 0, 0, 0, 212, 1, 0, 0, 0, 0, 0, 254, 0, 1493] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 216, 15, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 8192, 1016, 0, 0, 1, 0, 8191, 0, 389, 0, 8158, 0, 0, 4, 33, 22, 0, 0, 0, 0, 17, 0, 12, 0, 6443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5, 0, 0, 0, 42, 7964, 8172, 0, 0, 0, 0, 599, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 8084, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 8, 0, 0, 0, 0, 0, 79, 7826] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 98, 0, 5470, 3566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2889, 0, 5347, 0, 0, 0, 0, 0, 0, 8073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8115, 0, 23, 0, 0, 0, 2, 0, 0, 89, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 4486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1323, 0, 0, 0, 24, 0, 0, 5, 1130, 0, 0, 0, 0, 896, 7, 673, 288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8025, 0, 0, 0, 0, 0, 0, 4322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 15, 0, 0, 0, 0, 0, 1177, 0, 2109, 2849, 1267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9140, 16309, 8239, 3099, 4636, 4347, 12347, 7419] avg=8192.0000 max=16309 max_rank=1 min=3099 min_rank=3 imbalance_ratio=1.990845 max_load_reduction=49227 max_load_reduction_pct=0.751144 moved_to_echo=28749 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 3699, 0, 0, 0, 0, 0, 0, 0, 0, 7, 41, 0, 0, 0, 0, 25, 805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3202, 0, 4465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5619, 0, 1195, 0, 0, 3, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5205, 0, 6934, 0, 2, 0, 0, 0, 0, 60, 7, 2139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 621, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 8109, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1339, 0, 0, 0, 0, 0, 0, 0, 2257, 0, 0, 0, 0, 2435, 2020, 454, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 296, 0, 4103, 0, 228, 0, 0, 2916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 0, 0, 58, 0, 3956, 1919, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4577, 7667, 6848, 14347, 9756, 8745, 7543, 6053] avg=8192.0000 max=14347 max_rank=3 min=4577 min_rank=0 imbalance_ratio=1.751343 max_load_reduction=51189 max_load_reduction_pct=0.781082 moved_to_echo=32097 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7393, 0, 0, 0, 0, 688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 7668, 0, 0, 0, 0, 759, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4860, 0, 215, 0, 1393, 0, 0, 0, 7975, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 721, 0, 0, 0, 3, 0, 0, 8, 0, 1, 1, 0, 101, 0, 0, 0, 8102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 7906, 4, 0, 0, 0, 6486, 1747, 0, 413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 145, 0, 0, 0, 0, 15, 3, 0, 8136, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 304, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8082, 8578, 14543, 98, 8937, 16589, 8377, 332] avg=8192.0000 max=16589 max_rank=5 min=98 min_rank=3 imbalance_ratio=2.025024 max_load_reduction=48947 max_load_reduction_pct=0.746872 moved_to_echo=34235 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 216, 15, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 34, 0, 5516, 3629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1016, 0, 0, 1, 0, 0, 0, 389, 0, 8158, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 22, 0, 0, 0, 0, 17, 0, 12, 0, 6443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 42, 1168, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 599, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5, 4562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 8084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 8, 0, 0, 0, 0, 0, 79, 7826, 0, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9412, 17760, 6527, 9384, 5167, 8088, 1, 9197] avg=8192.0000 max=17760 max_rank=1 min=1 min_rank=6 imbalance_ratio=2.167969 max_load_reduction=47776 max_load_reduction_pct=0.729004 moved_to_echo=22453 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 31, 42, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 80, 0, 4990, 3621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 0, 7541, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 8080, 0, 2, 0, 0, 1, 0, 0, 1065, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1607, 12, 0, 1, 0, 0, 1, 0, 1992, 0, 0, 0, 0, 8190, 0, 694, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 8188, 0, 2, 0, 1, 0, 0, 4456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 10, 0, 0, 4950, 2934, 1156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8788, 8151, 32, 9373, 4562, 12903, 12661, 9066] avg=8192.0000 max=12903 max_rank=5 min=32 min_rank=2 imbalance_ratio=1.575073 max_load_reduction=52633 max_load_reduction_pct=0.803116 moved_to_echo=39192 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[6, 8082, 0, 0, 0, 58, 0, 0, 44, 13, 34, 75, 0, 0, 0, 0, 5559, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7441, 0, 5984, 0, 0, 22, 0, 0, 0, 0, 64, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 191, 6082, 0, 0, 0, 0, 0, 0, 100, 1185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 72, 33, 0, 0, 2151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 667, 593, 0, 0, 0, 19, 2144, 0, 7090, 0, 1288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13872, 64, 13672, 7561, 8127, 2258, 8165, 11817] avg=8192.0000 max=13872 max_rank=0 min=64 min_rank=1 imbalance_ratio=1.693359 max_load_reduction=51664 max_load_reduction_pct=0.788330 moved_to_echo=30367 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 368, 0, 0, 0, 0, 0, 450, 2673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2481, 17, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 6023, 2, 41, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 621, 0, 0, 0, 8154, 0, 0, 35, 3574, 0, 0, 0, 0, 0, 97, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8159, 3361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 2169, 261, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1714, 0, 8190, 0, 0, 0, 0, 0, 0, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4636, 1336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8036, 1096, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3491, 2577, 6104, 12484, 11521, 2581, 11568, 15210] avg=8192.0000 max=15210 max_rank=7 min=2577 min_rank=1 imbalance_ratio=1.856689 max_load_reduction=50326 max_load_reduction_pct=0.767914 moved_to_echo=40880 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[6, 8082, 0, 0, 0, 58, 0, 0, 44, 13, 34, 75, 0, 0, 0, 0, 5559, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7441, 0, 5984, 0, 0, 22, 0, 0, 0, 0, 64, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 191, 6082, 0, 0, 0, 0, 0, 0, 100, 1185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 72, 33, 0, 0, 2151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 667, 593, 0, 0, 0, 19, 2144, 0, 7090, 0, 1288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13872, 64, 13672, 7561, 8127, 2258, 8165, 11817] avg=8192.0000 max=13872 max_rank=0 min=64 min_rank=1 imbalance_ratio=1.693359 max_load_reduction=51664 max_load_reduction_pct=0.788330 moved_to_echo=30367 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 3699, 0, 0, 0, 0, 0, 0, 0, 0, 7, 41, 0, 0, 0, 0, 25, 805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3202, 0, 4465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5619, 0, 1195, 0, 0, 3, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5205, 0, 6934, 0, 2, 0, 0, 0, 0, 60, 7, 2139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 621, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 8109, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1339, 0, 0, 0, 0, 0, 0, 0, 2257, 0, 0, 0, 0, 2435, 2020, 454, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 296, 0, 4103, 0, 228, 0, 0, 2916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 0, 0, 58, 0, 3956, 1919, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4577, 7667, 6848, 14347, 9756, 8745, 7543, 6053] avg=8192.0000 max=14347 max_rank=3 min=4577 min_rank=0 imbalance_ratio=1.751343 max_load_reduction=51189 max_load_reduction_pct=0.781082 moved_to_echo=32097 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 31, 42, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 80, 0, 4990, 3621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 0, 7541, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 8080, 0, 2, 0, 0, 1, 0, 0, 1065, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1607, 12, 0, 1, 0, 0, 1, 0, 1992, 0, 0, 0, 0, 8190, 0, 694, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 8188, 0, 2, 0, 1, 0, 0, 4456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 10, 0, 0, 4950, 2934, 1156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8788, 8151, 32, 9373, 4562, 12903, 12661, 9066] avg=8192.0000 max=12903 max_rank=5 min=32 min_rank=2 imbalance_ratio=1.575073 max_load_reduction=52633 max_load_reduction_pct=0.803116 moved_to_echo=39192 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[11, 8, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 20, 0, 4, 4550, 3576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4262, 0, 158, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1130, 0, 0, 2, 0, 650, 1, 12, 0, 0, 7776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 28, 0, 0, 1, 971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 67, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 8131, 4498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 254, 0, 0, 0, 0, 0, 0, 7992, 0, 8, 0, 0, 1126, 0, 637, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 3, 0, 0, 0, 775, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 731, 3, 0, 0, 0, 0, 212, 1, 0, 0, 0, 0, 0, 254, 0, 1493, 2694, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 4430, 9571, 9190, 12830, 10025, 4874, 6442] avg=8192.0000 max=12830 max_rank=4 min=4430 min_rank=1 imbalance_ratio=1.566162 max_load_reduction=52706 max_load_reduction_pct=0.804230 moved_to_echo=34171 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7393, 0, 0, 0, 0, 688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 7668, 0, 0, 0, 0, 759, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4860, 0, 215, 0, 1393, 0, 0, 0, 7975, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 721, 0, 0, 0, 3, 0, 0, 8, 0, 1, 1, 0, 101, 0, 0, 0, 8102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 7906, 4, 0, 0, 0, 6486, 1747, 0, 413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 145, 0, 0, 0, 0, 15, 3, 0, 8136, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 304, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8082, 8578, 14543, 98, 8937, 16589, 8377, 332] avg=8192.0000 max=16589 max_rank=5 min=98 min_rank=3 imbalance_ratio=2.025024 max_load_reduction=48947 max_load_reduction_pct=0.746872 moved_to_echo=34235 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 98, 0, 5470, 3566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2889, 0, 5347, 0, 0, 0, 0, 0, 0, 8073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8115, 0, 23, 0, 0, 0, 2, 0, 0, 89, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 4486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1323, 0, 0, 0, 24, 0, 0, 5, 1130, 0, 0, 0, 0, 896, 7, 673, 288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8025, 0, 0, 0, 0, 0, 0, 4322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 15, 0, 0, 0, 0, 0, 1177, 0, 2109, 2849, 1267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9140, 16309, 8239, 3099, 4636, 4347, 12347, 7419] avg=8192.0000 max=16309 max_rank=1 min=3099 min_rank=3 imbalance_ratio=1.990845 max_load_reduction=49227 max_load_reduction_pct=0.751144 moved_to_echo=28749 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[768, 4736, 1408, 0, 0, 128, 640, 768, 768, 0, 0, 128, 128, 3712, 128, 35584, 11520, 6272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 128, 4352, 384, 0, 128, 128, 0, 128, 128, 21376, 128, 128, 128, 128, 27264, 8192, 2816, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 19328, 1024, 27776, 128, 128, 128, 0, 896, 512, 8192, 8320, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 128, 4352, 384, 0, 128, 128, 0, 128, 128, 21376, 128, 128, 128, 128, 27264, 8192, 2816, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 12160, 128, 0, 128, 128, 128, 0, 128, 128, 7936, 128, 0, 128, 256, 768, 26624, 17920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[640, 128, 128, 0, 23552, 256, 21120, 128, 3584, 0, 128, 128, 128, 256, 5760, 10496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[16768, 128, 22528, 128, 1536, 768, 1536, 128, 128, 128, 22400, 256, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 256, 128, 0, 128, 1792, 128, 32896, 896, 12288, 0, 256, 128, 0, 17536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[768, 0, 0, 128, 1408, 128, 0, 128, 0, 128, 128, 0, 128, 128, 0, 0, 40704, 22528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1040, 0, 0, 0, 0, 0, 0, 0, 889, 0, 0, 0, 1, 94, 0, 0, 0, 7254, 35, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 3591, 0, 2937, 305, 63, 0, 8090, 0, 21, 118, 0, 0, 0, 0, 0, 0, 0, 7845, 61, 34, 0, 0, 7027, 4494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 2, 9, 0, 0, 0, 0, 0, 0, 0, 3003, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2112, 0, 8142, 91, 0, 0, 0, 115, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8163, 2494, 0, 0, 0, 0, 0, 0, 0, 476, 0, 0, 0, 3640, 0, 0, 0, 0, 0, 4986, 0, 0, 0, 0, 0, 1523, 0, 0, 0, 45, 0, 0, 0, 4539, 0, 0, 1248, 50, 0, 0, 7538, 0, 0, 0, 25, 0, 0, 0, 766, 0, 0, 366, 0, 0, 205, 7773, 264, 289, 0, 0, 0, 0, 0, 186, 0, 0, 0, 34, 0, 0, 3545, 0, 0, 469, 0, 0, 0, 0, 0, 7179, 0, 10, 1, 124, 0, 0, 0, 0, 0, 88, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 514, 0, 681, 469, 0, 0, 0, 0, 39, 0, 1, 0, 7756, 17, 0, 1, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 23, 0, 0, 3, 0, 0, 0, 1, 0, 0, 784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 12, 6745, 0, 8192, 406, 0, 0, 0, 0, 7167, 0, 0, 4144, 0, 7357, 0, 0, 381, 0, 8, 7509, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4170, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 485, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 65, 4951, 2, 0, 0, 0, 901, 0, 9, 0, 0, 496, 0, 0, 0, 2829, 7978, 0, 0, 0, 0, 902, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 202, 0, 0, 2660, 0, 0, 0, 342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3, 634, 0, 0, 0, 0, 0, 1381, 0, 199, 0, 1170, 0, 353, 0, 0, 0, 0, 5, 0, 0, 0, 0, 22, 3310, 0, 0, 0, 12, 346, 7525, 5, 43, 0, 0, 0, 0, 0, 32, 0, 0, 6620, 0, 0, 1076, 0, 2, 0, 5, 0, 0, 0, 0, 0, 0, 0, 35, 105, 0, 538, 0, 0, 0, 0, 18, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7914, 0, 11, 298, 0, 0, 17, 8148, 2655, 8188, 4251, 57, 0, 0, 6826, 0, 0, 524, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 5, 0, 0, 0, 0, 2651, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7971, 0, 0, 0, 0, 0, 0, 6682, 686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 18, 0, 7531, 5580, 7374, 0, 3, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 2049, 1036, 0, 6404, 48, 7, 7, 6268, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1930, 0, 0, 3696, 75, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 165, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1354, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2831, 0, 3, 0, 0, 0, 0, 10, 0, 492, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 539, 37, 0, 0, 0, 2, 0, 0, 17, 37, 0, 0, 0, 0, 4, 0, 0, 0, 44, 0, 4771, 196, 0, 0, 0, 0, 772, 0, 0, 0, 1817, 0, 2, 0, 0, 0, 7849, 0, 0, 1360, 377, 0, 3998, 0, 0, 39, 0, 0, 0, 0, 514, 0, 5123, 7571, 2, 0, 0, 0, 0, 8134, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 973, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 47, 21, 1, 5, 0, 0, 1, 0, 0, 1208, 0, 136, 0, 0, 3, 101, 0, 0, 16, 0, 0, 6039, 0, 706, 5874, 6966, 0, 0, 0, 0, 0, 176, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 23, 0, 0, 3, 0, 0, 0, 1, 0, 0, 784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 12, 6745, 0, 8192, 406, 0, 0, 0, 0, 7167, 0, 0, 4144, 0, 7357, 0, 0, 381, 0, 8, 7509, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4170, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 485, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 65, 4951, 2, 0, 0, 0, 901, 0, 9, 0, 0, 496, 0, 0, 0, 2829, 7978, 0, 0, 0, 0, 902, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 18, 0, 25, 66, 0, 8188, 0, 459, 0, 0, 158, 4, 3, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 1340, 8192, 0, 0, 0, 0, 0, 0, 0, 35, 4, 0, 0, 0, 0, 130, 79, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 4, 0, 0, 0, 207, 294, 0, 0, 0, 0, 2133, 8095, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 1, 12, 7, 7847, 0, 0, 0, 1, 0, 0, 0, 4028, 6945, 3, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1102, 0, 4, 7949, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8163, 2494, 0, 0, 0, 0, 0, 0, 0, 476, 0, 0, 0, 3640, 0, 0, 0, 0, 0, 4986, 0, 0, 0, 0, 0, 1523, 0, 0, 0, 45, 0, 0, 0, 4539, 0, 0, 1248, 50, 0, 0, 7538, 0, 0, 0, 25, 0, 0, 0, 766, 0, 0, 366, 0, 0, 205, 7773, 264, 289, 0, 0, 0, 0, 0, 186, 0, 0, 0, 34, 0, 0, 3545, 0, 0, 469, 0, 0, 0, 0, 0, 7179, 0, 10, 1, 124, 0, 0, 0, 0, 0, 88, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 514, 0, 681, 469, 0, 0, 0, 0, 39, 0, 1, 0, 7756, 17, 0, 1, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1040, 0, 0, 0, 0, 0, 0, 0, 889, 0, 0, 0, 1, 94, 0, 0, 0, 7254, 35, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 3591, 0, 2937, 305, 63, 0, 8090, 0, 21, 118, 0, 0, 0, 0, 0, 0, 0, 7845, 61, 34, 0, 0, 7027, 4494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 2, 9, 0, 0, 0, 0, 0, 0, 0, 3003, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2112, 0, 8142, 91, 0, 0, 0, 115, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2556, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7187, 0, 287, 0, 7909, 9, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 1, 0, 0, 6893, 8192, 0, 0, 0, 3, 0, 0, 0, 0, 27, 14, 3573, 0, 0, 7104, 0, 0, 0, 8, 0, 2, 0, 2, 4245, 0, 0, 0, 0, 871, 0, 0, 0, 0, 0, 0, 1, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 3200, 0, 7, 0, 0, 410, 2607, 13, 0, 0, 1442] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 0, 0, 0, 0, 2651, 552, 0, 0, 0, 0, 0, 0, 0, 0, 7531, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7971, 0, 0, 0, 0, 0, 0, 6682, 686, 0, 0, 0, 0, 0, 0, 1177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 18, 0, 0, 5580, 7374, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 2049, 479, 0, 6404, 48, 7, 7, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1930, 0, 0, 3696, 75, 0, 0, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 165, 0, 0, 0, 63, 0, 0, 0, 3, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1354, 18, 0, 0, 0, 0, 0, 0, 0, 5786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1654, 0, 0, 0, 0, 0, 0, 10, 0, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10841, 16516, 12976, 9019, 6156, 714, 7158, 2156] avg=8192.0000 max=16516 max_rank=1 min=714 min_rank=5 imbalance_ratio=2.016113 max_load_reduction=49020 max_load_reduction_pct=0.747986 moved_to_echo=16184 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 5, 0, 0, 0, 0, 2651, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7971, 0, 0, 0, 0, 0, 0, 6682, 686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 18, 0, 7531, 5580, 7374, 0, 3, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 2049, 1036, 0, 6404, 48, 7, 7, 6268, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1930, 0, 0, 3696, 75, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 165, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1354, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2831, 0, 3, 0, 0, 0, 0, 10, 0, 492, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 539, 37, 0, 0, 0, 2, 0, 0, 17, 37, 0, 0, 0, 7849, 732, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 44, 0, 4771, 196, 0, 0, 0, 0, 772, 0, 0, 293, 780, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1817, 0, 2, 0, 0, 0, 0, 0, 0, 1360, 377, 0, 2679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 514, 0, 5123, 3532, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 973, 0, 0, 3307, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 47, 21, 1, 5, 6800, 625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1208, 0, 136, 0, 0, 3, 101, 0, 0, 16, 0, 7509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6039, 0, 413, 5874, 0, 0, 0, 0, 0, 0, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9379, 6860, 6235, 9210, 4873, 7503, 8974, 12502] avg=8192.0000 max=12502 max_rank=7 min=4873 min_rank=4 imbalance_ratio=1.526123 max_load_reduction=53034 max_load_reduction_pct=0.809235 moved_to_echo=33852 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 202, 0, 0, 2660, 0, 0, 0, 342, 0, 0, 0, 0, 1381, 320, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3388, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 634, 0, 0, 0, 0, 0, 0, 0, 199, 0, 1170, 0, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 22, 1545, 0, 0, 0, 12, 346, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 32, 0, 0, 6620, 0, 0, 1076, 0, 2, 0, 1445, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 35, 105, 0, 538, 0, 0, 0, 0, 7993, 578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7914, 0, 11, 298, 6947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4760, 2655, 0, 4251, 57, 0, 0, 6826, 0, 0, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5100, 3458, 2244, 1935, 9265, 9254, 15190, 19090] avg=8192.0000 max=19090 max_rank=7 min=1935 min_rank=3 imbalance_ratio=2.330322 max_load_reduction=46446 max_load_reduction_pct=0.708710 moved_to_echo=52799 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8163, 2494, 0, 0, 0, 0, 0, 0, 0, 476, 4539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3640, 0, 0, 0, 0, 0, 4986, 0, 0, 0, 0, 0, 1523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 1248, 50, 0, 0, 7538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 766, 0, 0, 366, 0, 0, 205, 7773, 0, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 0, 0, 0, 34, 0, 0, 3545, 0, 0, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7179, 0, 10, 1, 124, 0, 0, 0, 0, 0, 88, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 514, 0, 681, 469, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 1, 0, 7756, 17, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15672, 10149, 8881, 9424, 4234, 7423, 1938, 7815] avg=8192.0000 max=15672 max_rank=0 min=1938 min_rank=6 imbalance_ratio=1.913086 max_load_reduction=49864 max_load_reduction_pct=0.760864 moved_to_echo=21410 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 23, 0, 0, 3, 0, 0, 0, 1, 0, 0, 784, 3, 711, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 6745, 0, 5486, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7167, 0, 0, 4144, 0, 3433, 0, 0, 381, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4170, 0, 0, 0, 1, 0, 0, 3213, 1107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 7788, 578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 65, 4951, 2, 0, 0, 0, 901, 0, 9, 0, 0, 6931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 0, 0, 0, 2829, 0, 0, 0, 0, 0, 902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1715, 1599, 12649, 15133, 8497, 8855, 12861, 4227] avg=8192.0000 max=15133 max_rank=3 min=1599 min_rank=1 imbalance_ratio=1.847290 max_load_reduction=50403 max_load_reduction_pct=0.769089 moved_to_echo=34440 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1040, 0, 0, 0, 0, 0, 0, 0, 889, 0, 3591, 758, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 94, 0, 0, 0, 7254, 35, 0, 0, 0, 0, 0, 0, 2, 878, 1579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2937, 305, 63, 0, 5419, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 0, 0, 0, 0, 0, 0, 0, 3661, 61, 34, 0, 0, 7027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 2, 9, 0, 0, 3426, 1092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3003, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 7948, 345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1234, 0, 0, 91, 0, 0, 0, 115, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6472, 9843, 8747, 10901, 12682, 11301, 4149, 1441] avg=8192.0000 max=12682 max_rank=4 min=1441 min_rank=7 imbalance_ratio=1.548096 max_load_reduction=52854 max_load_reduction_pct=0.806488 moved_to_echo=29573 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 539, 37, 0, 0, 0, 2, 0, 0, 17, 37, 0, 0, 0, 7849, 732, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 44, 0, 4771, 196, 0, 0, 0, 0, 772, 0, 0, 293, 780, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1817, 0, 2, 0, 0, 0, 0, 0, 0, 1360, 377, 0, 2679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 514, 0, 5123, 3532, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 973, 0, 0, 3307, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 47, 21, 1, 5, 6800, 625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1208, 0, 136, 0, 0, 3, 101, 0, 0, 16, 0, 7509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6039, 0, 413, 5874, 0, 0, 0, 0, 0, 0, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9379, 6860, 6235, 9210, 4873, 7503, 8974, 12502] avg=8192.0000 max=12502 max_rank=7 min=4873 min_rank=4 imbalance_ratio=1.526123 max_load_reduction=53034 max_load_reduction_pct=0.809235 moved_to_echo=33852 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8163, 2494, 0, 0, 0, 0, 0, 0, 0, 476, 4539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3640, 0, 0, 0, 0, 0, 4986, 0, 0, 0, 0, 0, 1523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 1248, 50, 0, 0, 7538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 766, 0, 0, 366, 0, 0, 205, 7773, 0, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 0, 0, 0, 34, 0, 0, 3545, 0, 0, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7179, 0, 10, 1, 124, 0, 0, 0, 0, 0, 88, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 514, 0, 681, 469, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 1, 0, 7756, 17, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15672, 10149, 8881, 9424, 4234, 7423, 1938, 7815] avg=8192.0000 max=15672 max_rank=0 min=1938 min_rank=6 imbalance_ratio=1.913086 max_load_reduction=49864 max_load_reduction_pct=0.760864 moved_to_echo=21410 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 0, 0, 0, 0, 2651, 552, 0, 0, 0, 0, 0, 0, 0, 0, 7531, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7971, 0, 0, 0, 0, 0, 0, 6682, 686, 0, 0, 0, 0, 0, 0, 1177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 18, 0, 0, 5580, 7374, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 2049, 479, 0, 6404, 48, 7, 7, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1930, 0, 0, 3696, 75, 0, 0, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 165, 0, 0, 0, 63, 0, 0, 0, 3, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1354, 18, 0, 0, 0, 0, 0, 0, 0, 5786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1654, 0, 0, 0, 0, 0, 0, 10, 0, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10841, 16516, 12976, 9019, 6156, 714, 7158, 2156] avg=8192.0000 max=16516 max_rank=1 min=714 min_rank=5 imbalance_ratio=2.016113 max_load_reduction=49020 max_load_reduction_pct=0.747986 moved_to_echo=16184 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7187, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2556, 0, 0, 0, 0, 0, 15, 0, 0, 3398, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 287, 0, 7909, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 1, 0, 0, 6893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 27, 14, 3573, 0, 0, 7104, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 2, 0, 2, 4245, 0, 0, 0, 0, 871, 0, 0, 0, 0, 0, 3123, 630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4767, 0, 0, 0, 7, 0, 0, 410, 2607, 13, 0, 0, 1442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7278, 5977, 8205, 7432, 10724, 8881, 7793, 9246] avg=8192.0000 max=10724 max_rank=4 min=5977 min_rank=1 imbalance_ratio=1.309082 max_load_reduction=54812 max_load_reduction_pct=0.836365 moved_to_echo=36644 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 18, 0, 25, 66, 0, 8188, 0, 459, 0, 0, 158, 4, 3, 0, 0, 130, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 1340, 8192, 0, 0, 0, 0, 0, 0, 1, 1597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 4, 0, 0, 0, 0, 0, 79, 0, 0, 0, 0, 5481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 207, 138, 0, 0, 0, 0, 2133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 1, 12, 7, 7847, 0, 0, 0, 128, 1104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4028, 6945, 3, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1102, 0, 3, 7949, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9079, 11134, 5599, 2482, 9105, 11605, 7473, 9059] avg=8192.0000 max=11605 max_rank=5 min=2482 min_rank=3 imbalance_ratio=1.416626 max_load_reduction=53931 max_load_reduction_pct=0.822922 moved_to_echo=37242 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 23, 0, 0, 3, 0, 0, 0, 1, 0, 0, 784, 3, 711, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 6745, 0, 5486, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7167, 0, 0, 4144, 0, 3433, 0, 0, 381, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4170, 0, 0, 0, 1, 0, 0, 3213, 1107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 7788, 578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 65, 4951, 2, 0, 0, 0, 901, 0, 9, 0, 0, 6931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 0, 0, 0, 2829, 0, 0, 0, 0, 0, 902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1715, 1599, 12649, 15133, 8497, 8855, 12861, 4227] avg=8192.0000 max=15133 max_rank=3 min=1599 min_rank=1 imbalance_ratio=1.847290 max_load_reduction=50403 max_load_reduction_pct=0.769089 moved_to_echo=34440 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[512, 7168, 128, 12928, 19456, 0, 4352, 128, 7808, 128, 8320, 2816, 512, 640, 0, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 8064, 128, 3840, 128, 128, 0, 14592, 8320, 13952, 0, 0, 0, 896, 0, 1536, 9216, 5632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 2560, 128, 128, 0, 128, 0, 0, 5760, 10880, 2944, 16384, 128, 19328, 8064, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 640, 384, 128, 20096, 5760, 512, 0, 0, 640, 128, 128, 896, 1280, 32256, 2688, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[256, 128, 128, 7296, 0, 640, 4992, 7424, 12800, 1024, 6528, 512, 256, 24192, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 256, 128, 0, 1280, 4992, 1536, 128, 0, 128, 1024, 8448, 128, 768, 768, 46720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 0, 128, 128, 128, 256, 128, 128, 128, 24576, 128, 128, 23296, 1152, 128, 512, 12032, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[256, 128, 128, 7296, 0, 640, 4992, 7424, 12800, 1024, 6528, 512, 256, 24192, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 0, 4608, 21376, 128, 128, 256, 256, 896, 640, 128, 128, 128, 128, 33664, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 7661, 0, 0, 0, 0, 0, 0, 0, 0, 3856, 0, 587, 0, 0, 0, 0, 0, 0, 0, 2, 32, 349, 0, 0, 0, 0, 8034, 0, 0, 0, 216, 5449, 0, 0, 12, 1, 0, 8065, 85, 0, 5, 0, 473, 0, 11, 0, 0, 0, 0, 0, 0, 0, 34, 239, 37, 0, 0, 0, 0, 2, 0, 0, 0, 0, 7621, 2, 0, 0, 7, 0, 7944, 0, 0, 0, 0, 8, 7330, 0, 5, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 8, 0, 2931, 0, 0, 5, 477] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 313, 8139, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 293, 3, 6423, 0, 0, 0, 5951, 0, 0, 0, 0, 0, 8120, 1798, 0, 2, 0, 0, 3, 0, 0, 0, 0, 25, 26, 73, 0, 0, 0, 0, 0, 0, 5126, 0, 0, 1, 64, 0, 0, 0, 0, 0, 0, 0, 0, 7886, 0, 0, 0, 0, 0, 7575, 1013, 0, 0, 641, 0, 0, 0, 3775, 91, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 440, 0, 1804, 1283, 0, 0, 0, 0, 0, 2049, 0, 188, 672, 0, 0, 0, 0, 21, 0, 2, 0, 10, 2863, 8192, 5, 0, 28, 29, 0, 0, 0, 0, 0, 7641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6899, 0, 0, 0, 0, 87, 0, 7635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 6546, 0, 9, 1, 0, 3351, 6892, 0, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 931, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7320, 47, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[3313, 0, 1, 0, 5, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 7689, 0, 0, 0, 4672, 0, 693, 0, 1223, 0, 92, 0, 0, 8043, 18, 353, 0, 485, 1049, 0, 14, 1778, 0, 0, 8018, 0, 0, 0, 0, 0, 0, 0, 0, 208, 0, 0, 0, 0, 0, 0, 0, 0, 138, 0, 3943, 2044, 0, 0, 0, 272, 0, 7, 0, 0, 0, 0, 1, 0, 0, 0, 0, 5, 0, 8188, 0, 0, 0, 0, 0, 249, 0, 0, 2760, 0, 0, 0, 0, 0, 2, 0, 0, 0, 154, 3, 0, 3676, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 25, 0, 6404, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4850, 0, 3, 0, 0, 0, 33, 0, 4, 30, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 7289, 2, 18, 0, 0, 610, 0, 0, 0, 0, 0, 8192, 16, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6101, 0, 0, 0, 0, 6179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 231, 0, 8147, 0, 0, 6, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 458, 1184, 0, 2082, 0, 0, 0, 0, 0, 2012, 8160, 0, 0, 22, 0, 0, 0, 0, 1711, 0, 0, 21, 0, 0, 1, 0, 0, 0, 8101, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6905, 6075, 0, 0, 1, 1360, 0, 0, 0, 0, 0, 7244, 0, 0, 0, 0, 0, 161, 92, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 10, 5262, 322, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1561, 1, 145, 0, 0, 24, 3759, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 8191, 0, 4693, 0, 0, 0, 1096, 0, 5035, 0, 0, 0, 50, 0, 412, 0, 745, 492, 49, 0, 105, 713, 47, 0, 0, 0, 0, 8083, 0, 0, 0, 2799] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[3182, 0, 0, 95, 0, 0, 5, 0, 0, 728, 5, 0, 0, 0, 0, 0, 8084, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 7667, 0, 0, 0, 0, 0, 8185, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 5837, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 9, 0, 163, 6634, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 613, 0, 0, 6, 0, 0, 0, 0, 900, 8029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 919, 0, 0, 0, 1, 8, 8063, 0, 0, 43, 6272] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 10, 0, 29, 0, 0, 0, 0, 0, 0, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7845, 0, 2607, 0, 0, 7471, 8, 0, 25, 0, 1, 8050, 0, 0, 0, 0, 0, 2, 2, 0, 2055, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 6184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 570, 0, 0, 0, 2, 6, 0, 2, 0, 8057, 1, 0, 1, 0, 0, 1, 0, 20, 486, 0, 0, 0, 0, 0, 0, 2, 6527, 0, 142, 0, 0, 0, 16, 6935, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 7, 0, 0, 90, 306, 0, 2, 7954, 0, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[3313, 0, 1, 0, 5, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 7689, 0, 0, 0, 4672, 0, 693, 0, 1223, 0, 92, 0, 0, 8043, 18, 353, 0, 485, 1049, 0, 14, 1778, 0, 0, 8018, 0, 0, 0, 0, 0, 0, 0, 0, 208, 0, 0, 0, 0, 0, 0, 0, 0, 138, 0, 3943, 2044, 0, 0, 0, 272, 0, 7, 0, 0, 0, 0, 1, 0, 0, 0, 0, 5, 0, 8188, 0, 0, 0, 0, 0, 249, 0, 0, 2760, 0, 0, 0, 0, 0, 2, 0, 0, 0, 154, 3, 0, 3676, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 25, 0, 6404, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 7661, 0, 0, 0, 0, 0, 0, 0, 0, 3856, 0, 587, 0, 0, 0, 0, 0, 0, 0, 2, 32, 349, 0, 0, 0, 0, 8034, 0, 0, 0, 216, 5449, 0, 0, 12, 1, 0, 8065, 85, 0, 5, 0, 473, 0, 11, 0, 0, 0, 0, 0, 0, 0, 34, 239, 37, 0, 0, 0, 0, 2, 0, 0, 0, 0, 7621, 2, 0, 0, 7, 0, 7944, 0, 0, 0, 0, 8, 7330, 0, 5, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 8, 0, 2931, 0, 0, 5, 477] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[3182, 0, 0, 95, 0, 0, 5, 0, 0, 728, 5, 0, 0, 0, 0, 0, 8084, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 7667, 0, 0, 0, 0, 0, 8185, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 5837, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 9, 0, 163, 6634, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 613, 0, 0, 6, 0, 0, 0, 0, 900, 8029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 919, 0, 0, 0, 1, 8, 8063, 0, 0, 43, 6272] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 440, 0, 1804, 1283, 0, 0, 0, 0, 0, 2049, 0, 188, 672, 0, 0, 0, 0, 21, 0, 2, 0, 10, 2863, 8192, 5, 0, 28, 29, 0, 0, 0, 0, 0, 7641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6899, 0, 0, 0, 0, 87, 0, 7635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 6546, 0, 9, 1, 0, 3351, 6892, 0, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 931, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7320, 47, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3313, 0, 1, 0, 5, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3396, 1485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7689, 0, 0, 0, 4672, 0, 693, 0, 1223, 0, 92, 0, 0, 3087, 18, 353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 1049, 0, 14, 1778, 0, 0, 3880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 208, 0, 0, 0, 0, 0, 0, 0, 0, 138, 0, 3943, 2044, 0, 0, 1560, 585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 272, 0, 7, 0, 0, 0, 0, 1, 0, 0, 0, 0, 5, 0, 3312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 249, 0, 0, 2760, 0, 0, 0, 0, 0, 2, 0, 4876, 2653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 3, 0, 3676, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 25, 0, 5686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 17827, 7206, 8478, 3597, 10540, 3972, 5713] avg=8192.0000 max=17827 max_rank=1 min=3597 min_rank=4 imbalance_ratio=2.176147 max_load_reduction=47709 max_load_reduction_pct=0.727982 moved_to_echo=23822 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 2915, 1341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2651, 6075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1360, 0, 0, 0, 0, 0, 3506, 0, 0, 0, 0, 0, 161, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 10, 5262, 322, 0, 0, 1339, 738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1561, 1, 145, 0, 0, 24, 1521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 2238, 2397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 4693, 0, 0, 0, 1096, 0, 5035, 0, 0, 0, 50, 0, 412, 0, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 745, 492, 49, 0, 105, 713, 47, 0, 0, 0, 0, 7177, 0, 0, 0, 2799, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4259, 8733, 5120, 7675, 3253, 4724, 19645, 12127] avg=8192.0000 max=19645 max_rank=6 min=3253 min_rank=4 imbalance_ratio=2.398071 max_load_reduction=45891 max_load_reduction_pct=0.700241 moved_to_echo=39749 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 7661, 0, 0, 0, 0, 0, 0, 0, 0, 3856, 0, 587, 0, 3396, 1496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 32, 349, 0, 0, 0, 0, 3076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 5449, 0, 0, 12, 1, 0, 3896, 85, 0, 5, 0, 473, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 239, 37, 0, 0, 0, 0, 2, 0, 1562, 272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7621, 2, 0, 0, 7, 0, 7944, 0, 0, 0, 0, 8, 2961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4369, 2673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3985, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 8, 0, 2594, 0, 0, 5, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16996, 3459, 10148, 2146, 18543, 7108, 4050, 3086] avg=8192.0000 max=18543 max_rank=4 min=2146 min_rank=3 imbalance_ratio=2.263550 max_load_reduction=46993 max_load_reduction_pct=0.717056 moved_to_echo=32787 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 313, 8139, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 293, 2, 6423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5951, 0, 0, 0, 0, 0, 3929, 1798, 0, 2, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 26, 73, 0, 0, 0, 0, 0, 0, 5126, 0, 0, 1, 0, 747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 7886, 0, 0, 0, 0, 0, 3064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 0, 0, 641, 0, 0, 0, 3775, 91, 0, 0, 0, 0, 0, 0, 0, 4511, 2687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7265, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9958, 6718, 11683, 5998, 11014, 12718, 181, 7266] avg=8192.0000 max=12718 max_rank=5 min=181 min_rank=6 imbalance_ratio=1.552490 max_load_reduction=52818 max_load_reduction_pct=0.805939 moved_to_echo=31179 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 10, 0, 29, 0, 0, 0, 0, 0, 0, 0, 113, 0, 0, 3312, 1491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3012, 0, 2607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7471, 8, 0, 25, 0, 1, 3895, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2055, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 6184, 0, 0, 0, 1521, 726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 570, 0, 0, 0, 2, 6, 0, 2, 0, 3259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 20, 486, 0, 0, 0, 0, 0, 0, 2, 4798, 2664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6527, 0, 142, 0, 0, 0, 16, 6935, 0, 0, 0, 0, 0, 0, 2, 0, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 90, 306, 0, 2, 7063, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4955, 5619, 11404, 10489, 3839, 7973, 13787, 7470] avg=8192.0000 max=13787 max_rank=6 min=3839 min_rank=4 imbalance_ratio=1.682983 max_load_reduction=51749 max_load_reduction_pct=0.789627 moved_to_echo=33069 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3182, 0, 0, 95, 0, 0, 5, 0, 0, 728, 5, 0, 0, 0, 0, 0, 0, 1516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8084, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7667, 0, 0, 0, 0, 0, 3960, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 5837, 0, 0, 0, 0, 736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 9, 0, 163, 2684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 613, 0, 3950, 2709, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 900, 8029, 0, 0, 0, 0, 0, 0, 0, 0, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 919, 0, 0, 0, 1, 8, 7159, 0, 0, 43, 6272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5531, 8086, 11628, 6652, 2860, 7274, 9103, 14402] avg=8192.0000 max=14402 max_rank=7 min=2860 min_rank=4 imbalance_ratio=1.758057 max_load_reduction=51134 max_load_reduction_pct=0.780243 moved_to_echo=33639 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 440, 0, 1804, 1283, 0, 0, 0, 0, 0, 1208, 1415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2049, 0, 188, 672, 0, 0, 0, 0, 21, 0, 2, 0, 10, 1100, 8192, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 29, 0, 0, 0, 0, 0, 3698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6899, 0, 0, 0, 0, 87, 0, 7635, 0, 0, 0, 555, 668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 6546, 0, 9, 1, 0, 3351, 2788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4104, 2528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 931, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6500, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6150, 12239, 3755, 15844, 12724, 7108, 1169, 6547] avg=8192.0000 max=15844 max_rank=3 min=1169 min_rank=6 imbalance_ratio=1.934082 max_load_reduction=49692 max_load_reduction_pct=0.758240 moved_to_echo=27548 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3313, 0, 1, 0, 5, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3396, 1485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7689, 0, 0, 0, 4672, 0, 693, 0, 1223, 0, 92, 0, 0, 3087, 18, 353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 1049, 0, 14, 1778, 0, 0, 3880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 208, 0, 0, 0, 0, 0, 0, 0, 0, 138, 0, 3943, 2044, 0, 0, 1560, 585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 272, 0, 7, 0, 0, 0, 0, 1, 0, 0, 0, 0, 5, 0, 3312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 249, 0, 0, 2760, 0, 0, 0, 0, 0, 2, 0, 4876, 2653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 3, 0, 3676, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 25, 0, 5686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 17827, 7206, 8478, 3597, 10540, 3972, 5713] avg=8192.0000 max=17827 max_rank=1 min=3597 min_rank=4 imbalance_ratio=2.176147 max_load_reduction=47709 max_load_reduction_pct=0.727982 moved_to_echo=23822 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4850, 0, 3, 0, 0, 0, 33, 0, 4, 3077, 1517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 2799, 2, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 610, 0, 0, 0, 0, 0, 3964, 16, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6101, 0, 0, 0, 1413, 740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 231, 0, 3296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 458, 4851, 2711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1184, 0, 2082, 0, 0, 0, 0, 0, 2012, 8160, 0, 0, 22, 0, 0, 0, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1711, 0, 0, 21, 0, 0, 1, 0, 0, 0, 7192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9484, 2855, 4592, 8254, 9706, 8091, 13629, 8925] avg=8192.0000 max=13629 max_rank=6 min=2855 min_rank=1 imbalance_ratio=1.663696 max_load_reduction=51907 max_load_reduction_pct=0.792038 moved_to_echo=40351 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 2915, 1341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2651, 6075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1360, 0, 0, 0, 0, 0, 3506, 0, 0, 0, 0, 0, 161, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 10, 5262, 322, 0, 0, 1339, 738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1561, 1, 145, 0, 0, 24, 1521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 2238, 2397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 4693, 0, 0, 0, 1096, 0, 5035, 0, 0, 0, 50, 0, 412, 0, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 745, 492, 49, 0, 105, 713, 47, 0, 0, 0, 0, 7177, 0, 0, 0, 2799, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4259, 8733, 5120, 7675, 3253, 4724, 19645, 12127] avg=8192.0000 max=19645 max_rank=6 min=3253 min_rank=4 imbalance_ratio=2.398071 max_load_reduction=45891 max_load_reduction_pct=0.700241 moved_to_echo=39749 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 7661, 0, 0, 0, 0, 0, 0, 0, 0, 3856, 0, 587, 0, 3396, 1496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 32, 349, 0, 0, 0, 0, 3076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 5449, 0, 0, 12, 1, 0, 3896, 85, 0, 5, 0, 473, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 239, 37, 0, 0, 0, 0, 2, 0, 1562, 272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7621, 2, 0, 0, 7, 0, 7944, 0, 0, 0, 0, 8, 2961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4369, 2673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3985, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 8, 0, 2594, 0, 0, 5, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16996, 3459, 10148, 2146, 18543, 7108, 4050, 3086] avg=8192.0000 max=18543 max_rank=4 min=2146 min_rank=3 imbalance_ratio=2.263550 max_load_reduction=46993 max_load_reduction_pct=0.717056 moved_to_echo=32787 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 440, 0, 1804, 1283, 0, 0, 0, 0, 0, 1208, 1415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2049, 0, 188, 672, 0, 0, 0, 0, 21, 0, 2, 0, 10, 1100, 8192, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 29, 0, 0, 0, 0, 0, 3698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6899, 0, 0, 0, 0, 87, 0, 7635, 0, 0, 0, 555, 668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 6546, 0, 9, 1, 0, 3351, 2788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4104, 2528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 931, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6500, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6150, 12239, 3755, 15844, 12724, 7108, 1169, 6547] avg=8192.0000 max=15844 max_rank=3 min=1169 min_rank=6 imbalance_ratio=1.934082 max_load_reduction=49692 max_load_reduction_pct=0.758240 moved_to_echo=27548 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[1024, 512, 128, 768, 0, 256, 128, 3840, 3456, 0, 0, 0, 128, 0, 640, 512, 33792, 21120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[16000, 128, 7168, 1024, 0, 3840, 2048, 19072, 7168, 8192, 0, 0, 128, 128, 512, 0, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] @@ -1826,35 +1828,35 @@ WARNING:megatron.core.rerun_state_machine:Result validation enabled warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") - [2026-08-26 14:48:09] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 41953.5 | throughput per GPU (TFLOP/s/GPU): 4.3 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.115992E+01 | loss scale: 1.0 | grad norm: 44.776 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 15:24:09] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 38112.7 | throughput per GPU (TFLOP/s/GPU): 4.7 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.115992E+01 | loss scale: 1.0 | grad norm: 44.778 | number of skipped iterations: 0 | number of nan iterations: 0 | Number of parameters in transformer block in billions: 29.85 Number of parameters in embedding layers in billions: 0.41 Total number of parameters in billions: 30.26 Number of parameters in most loaded shard in billions: 30.2590 Theoretical memory footprints: weight and optimizer=216428.96 MB -[Rank 0] (after 1 iterations) memory (MB) | allocated: 81277.55859375 | max allocated: 96153.076171875 | reserved: 101740.0 | max reserved: 101740.0 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[3707, 68, 25, 436, 236, 457, 12, 20, 63, 154, 2215, 412, 1272, 118, 112, 806, 148, 2480, 259, 9, 1318, 50, 172, 1656, 541, 60, 14, 1549, 506, 2068, 127, 270, 461, 1843, 175, 104, 250, 432, 98, 38, 1644, 51, 460, 3108, 25, 64, 94, 596, 204, 70, 4, 118, 8, 414, 53, 24, 170, 94, 41, 392, 1287, 11, 899, 195, 1338, 33, 8, 13, 1242, 13, 120, 1706, 748, 149, 1926, 142, 457, 19, 535, 772, 30, 13, 3, 24, 59, 7, 876, 548, 14, 23, 474, 12, 40, 166, 564, 75, 648, 123, 145, 1877, 177, 0, 103, 72, 549, 72, 509, 3597, 23, 196, 44, 39, 745, 109, 12, 131, 12, 751, 364, 191, 1297, 46, 2120, 408, 108, 237, 3, 3912] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[4396, 162, 2, 510, 136, 840, 9, 2, 52, 290, 1562, 547, 1308, 107, 203, 866, 144, 2237, 274, 13, 1033, 18, 98, 1615, 424, 66, 42, 1913, 390, 988, 53, 195, 428, 1255, 57, 47, 80, 467, 48, 4, 1189, 16, 719, 3433, 25, 30, 152, 818, 210, 82, 0, 19, 0, 354, 118, 18, 118, 19, 31, 502, 1077, 20, 340, 100, 2324, 26, 14, 19, 2136, 7, 33, 1883, 1048, 130, 1811, 76, 444, 2, 752, 1031, 26, 20, 2, 27, 42, 5, 761, 1036, 35, 24, 613, 3, 15, 185, 262, 79, 926, 115, 50, 1860, 102, 3, 37, 98, 248, 67, 660, 4953, 33, 55, 47, 95, 681, 37, 10, 81, 3, 683, 120, 295, 1193, 37, 1885, 413, 267, 310, 41, 2989] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +[Rank 0] (after 1 iterations) memory (MB) | allocated: 81277.55859375 | max allocated: 96153.076171875 | reserved: 102124.0 | max reserved: 102124.0 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[4466, 59, 13, 293, 131, 445, 7, 34, 33, 278, 1154, 284, 2095, 194, 227, 516, 189, 2176, 305, 6, 695, 39, 73, 1400, 589, 25, 53, 2019, 1037, 2078, 27, 378, 682, 1571, 61, 32, 119, 433, 65, 32, 842, 11, 717, 2891, 31, 8, 109, 882, 225, 94, 8, 74, 2, 374, 43, 16, 177, 73, 31, 620, 939, 13, 449, 298, 2401, 27, 0, 6, 1954, 10, 37, 1160, 938, 179, 1729, 158, 516, 6, 494, 798, 62, 4, 0, 43, 105, 12, 770, 432, 18, 61, 1014, 3, 74, 138, 547, 79, 1442, 171, 18, 1677, 123, 5, 21, 115, 567, 81, 905, 4220, 18, 135, 62, 37, 744, 71, 13, 75, 2, 904, 237, 224, 1129, 17, 1387, 413, 255, 210, 6, 3237] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[3882, 82, 42, 464, 292, 453, 14, 19, 52, 288, 1082, 1084, 1511, 110, 589, 281, 175, 2286, 193, 34, 1090, 68, 177, 1369, 739, 153, 68, 2335, 776, 1391, 93, 286, 843, 1450, 81, 12, 357, 415, 160, 11, 977, 26, 501, 2988, 25, 33, 152, 624, 223, 223, 7, 48, 7, 243, 40, 41, 94, 77, 57, 725, 1249, 25, 628, 159, 2166, 26, 5, 18, 1431, 79, 125, 1494, 777, 164, 1409, 89, 388, 15, 625, 881, 43, 31, 3, 26, 34, 1, 649, 938, 11, 43, 617, 10, 24, 225, 239, 129, 801, 120, 76, 3116, 157, 3, 91, 254, 566, 106, 762, 4275, 30, 192, 77, 25, 781, 35, 21, 58, 7, 278, 146, 263, 1309, 44, 1269, 339, 147, 249, 8, 3237] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[4024, 230, 10, 431, 54, 540, 53, 20, 50, 374, 1890, 410, 1445, 100, 201, 573, 81, 1542, 170, 15, 827, 27, 112, 1216, 691, 63, 50, 1491, 342, 1660, 30, 261, 641, 1675, 122, 36, 87, 357, 101, 35, 952, 19, 547, 2624, 22, 5, 136, 994, 272, 152, 1, 36, 2, 385, 150, 18, 113, 32, 18, 300, 1277, 68, 434, 128, 2704, 18, 15, 9, 1934, 2, 33, 1459, 980, 69, 1725, 261, 349, 10, 544, 771, 22, 16, 26, 28, 35, 36, 1197, 973, 27, 44, 1112, 16, 38, 103, 253, 67, 1349, 117, 17, 1989, 83, 6, 57, 82, 356, 93, 431, 5244, 27, 126, 53, 47, 923, 26, 14, 151, 1, 583, 147, 325, 1557, 39, 1758, 332, 377, 188, 5, 3435] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[3698, 93, 55, 467, 125, 699, 35, 52, 31, 249, 1681, 253, 1339, 106, 193, 387, 306, 2012, 209, 20, 489, 10, 58, 1558, 678, 60, 34, 1831, 692, 2065, 121, 633, 782, 1658, 100, 68, 166, 189, 183, 67, 1089, 21, 619, 2126, 39, 9, 207, 893, 183, 210, 4, 72, 6, 299, 103, 24, 129, 24, 17, 517, 1055, 16, 562, 287, 1992, 25, 8, 14, 1742, 3, 66, 1572, 865, 103, 1247, 225, 481, 6, 550, 1364, 55, 13, 2, 42, 52, 10, 1289, 676, 59, 48, 934, 2, 29, 336, 272, 66, 1545, 258, 77, 1387, 194, 2, 69, 82, 280, 143, 343, 4992, 30, 134, 114, 56, 813, 145, 9, 153, 8, 211, 254, 443, 1441, 88, 1838, 438, 83, 486, 5, 3304] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[4396, 162, 2, 510, 136, 840, 9, 2, 52, 290, 1562, 547, 1308, 107, 203, 866, 144, 2237, 274, 13, 1033, 18, 98, 1615, 424, 66, 42, 1913, 390, 988, 53, 195, 428, 1255, 57, 47, 80, 467, 48, 4, 1189, 16, 719, 3433, 25, 30, 152, 818, 210, 82, 0, 19, 0, 354, 118, 18, 118, 19, 31, 502, 1077, 20, 340, 100, 2324, 26, 14, 19, 2136, 7, 33, 1883, 1048, 130, 1811, 76, 444, 2, 752, 1031, 26, 20, 2, 27, 42, 5, 761, 1036, 35, 24, 613, 3, 15, 185, 262, 79, 926, 115, 50, 1860, 102, 3, 37, 98, 248, 67, 660, 4953, 33, 55, 47, 95, 681, 37, 10, 81, 3, 683, 120, 295, 1193, 37, 1885, 413, 267, 310, 41, 2989] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[3835, 120, 26, 419, 113, 348, 13, 12, 77, 263, 2564, 504, 1737, 90, 230, 435, 132, 2145, 484, 11, 1081, 28, 87, 1274, 508, 40, 76, 2035, 749, 2355, 82, 100, 510, 2885, 142, 30, 227, 427, 48, 69, 733, 63, 561, 2591, 40, 13, 141, 478, 388, 90, 1, 91, 2, 927, 33, 19, 240, 40, 50, 383, 924, 24, 519, 163, 1855, 20, 25, 1, 1356, 14, 73, 1108, 725, 45, 2149, 254, 659, 4, 444, 1094, 38, 9, 11, 16, 41, 5, 457, 562, 20, 31, 758, 15, 49, 100, 374, 112, 1227, 159, 107, 2249, 150, 0, 78, 300, 365, 114, 413, 3833, 25, 221, 41, 45, 674, 48, 18, 92, 11, 239, 190, 417, 1377, 54, 1997, 368, 155, 171, 10, 3139] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[3707, 68, 25, 436, 236, 457, 12, 20, 63, 154, 2215, 412, 1272, 118, 112, 806, 148, 2480, 259, 9, 1318, 50, 172, 1656, 541, 60, 14, 1549, 506, 2068, 127, 270, 461, 1843, 175, 104, 250, 432, 98, 38, 1644, 51, 460, 3108, 25, 64, 94, 596, 204, 70, 4, 118, 8, 414, 53, 24, 170, 94, 41, 392, 1287, 11, 899, 195, 1338, 33, 8, 13, 1242, 13, 120, 1706, 748, 149, 1926, 142, 457, 19, 535, 772, 30, 13, 3, 24, 59, 7, 876, 548, 14, 23, 474, 12, 40, 166, 564, 75, 648, 123, 145, 1877, 177, 0, 103, 72, 549, 72, 509, 3597, 23, 196, 44, 39, 745, 109, 12, 131, 12, 751, 364, 191, 1297, 46, 2120, 408, 108, 237, 3, 3912] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[4024, 230, 10, 431, 54, 540, 53, 20, 50, 374, 1890, 410, 1445, 100, 201, 573, 81, 1542, 170, 15, 827, 27, 112, 1216, 691, 63, 50, 1491, 342, 1660, 30, 261, 641, 1675, 122, 36, 87, 357, 101, 35, 952, 19, 547, 2624, 22, 5, 136, 994, 272, 152, 1, 36, 2, 385, 150, 18, 113, 32, 18, 300, 1277, 68, 434, 128, 2704, 18, 15, 9, 1934, 2, 33, 1459, 980, 69, 1725, 261, 349, 10, 544, 771, 22, 16, 26, 28, 35, 36, 1197, 973, 27, 44, 1112, 16, 38, 103, 253, 67, 1349, 117, 17, 1989, 83, 6, 57, 82, 356, 93, 431, 5244, 27, 126, 53, 47, 923, 26, 14, 151, 1, 583, 147, 325, 1557, 39, 1758, 332, 377, 188, 5, 3435] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[3882, 82, 42, 464, 292, 453, 14, 19, 52, 288, 1082, 1084, 1511, 110, 589, 281, 175, 2286, 193, 34, 1090, 68, 177, 1369, 739, 153, 68, 2335, 776, 1391, 93, 286, 843, 1450, 81, 12, 357, 415, 160, 11, 977, 26, 501, 2988, 25, 33, 152, 624, 223, 223, 7, 48, 7, 243, 40, 41, 94, 77, 57, 725, 1249, 25, 628, 159, 2166, 26, 5, 18, 1431, 79, 125, 1494, 777, 164, 1409, 89, 388, 15, 625, 881, 43, 31, 3, 26, 34, 1, 649, 938, 11, 43, 617, 10, 24, 225, 239, 129, 801, 120, 76, 3116, 157, 3, 91, 254, 566, 106, 762, 4275, 30, 192, 77, 25, 781, 35, 21, 58, 7, 278, 146, 263, 1309, 44, 1269, 339, 147, 249, 8, 3237] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[3495, 61, 42, 259, 178, 571, 6, 29, 70, 214, 1683, 441, 1991, 113, 353, 267, 114, 2618, 97, 44, 527, 74, 146, 1576, 766, 59, 36, 1579, 332, 1169, 91, 343, 461, 1856, 53, 87, 85, 415, 62, 71, 538, 71, 827, 3238, 39, 8, 326, 849, 260, 214, 4, 37, 6, 132, 79, 8, 33, 39, 51, 652, 2002, 32, 217, 93, 2814, 22, 3, 15, 1553, 16, 61, 1072, 765, 89, 2009, 122, 409, 18, 372, 803, 25, 18, 11, 66, 98, 17, 954, 848, 20, 75, 1080, 9, 12, 261, 275, 96, 1919, 179, 13, 2969, 56, 3, 11, 74, 424, 136, 697, 4726, 25, 86, 69, 75, 483, 25, 31, 22, 4, 555, 133, 408, 1145, 87, 1009, 365, 252, 263, 18, 3077] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[3698, 93, 55, 467, 125, 699, 35, 52, 31, 249, 1681, 253, 1339, 106, 193, 387, 306, 2012, 209, 20, 489, 10, 58, 1558, 678, 60, 34, 1831, 692, 2065, 121, 633, 782, 1658, 100, 68, 166, 189, 183, 67, 1089, 21, 619, 2126, 39, 9, 207, 893, 183, 210, 4, 72, 6, 299, 103, 24, 129, 24, 17, 517, 1055, 16, 562, 287, 1992, 25, 8, 14, 1742, 3, 66, 1572, 865, 103, 1247, 225, 481, 6, 550, 1364, 55, 13, 2, 42, 52, 10, 1289, 676, 59, 48, 934, 2, 29, 336, 272, 66, 1545, 258, 77, 1387, 194, 2, 69, 82, 280, 143, 343, 4992, 30, 134, 114, 56, 813, 145, 9, 153, 8, 211, 254, 443, 1441, 88, 1838, 438, 83, 486, 5, 3304] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1770, 68, 25, 436, 236, 457, 12, 20, 63, 154, 2215, 412, 1272, 118, 112, 806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 0, 259, 9, 1318, 50, 172, 1656, 541, 60, 14, 1536, 506, 2068, 127, 270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 1843, 175, 104, 250, 432, 98, 38, 1644, 51, 460, 2523, 25, 64, 94, 596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 204, 70, 4, 118, 8, 414, 53, 24, 170, 94, 41, 392, 1287, 11, 899, 195, 1285, 1395, 1112, 585, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 33, 8, 13, 1242, 13, 120, 1706, 748, 149, 1926, 142, 457, 19, 535, 772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 13, 3, 24, 59, 7, 876, 548, 14, 23, 474, 12, 40, 166, 564, 75, 2480, 1265, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 648, 123, 145, 1877, 177, 0, 103, 72, 549, 72, 509, 2312, 23, 196, 44, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 745, 109, 12, 131, 12, 751, 364, 191, 1297, 46, 2120, 408, 108, 237, 3, 2800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8734, 8858, 8374, 7956, 7215, 6889, 9334] avg=8192.0000 max=9334 max_rank=7 min=6889 min_rank=6 imbalance_ratio=1.139404 max_load_reduction=56202 max_load_reduction_pct=0.857574 moved_to_echo=31394 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2124, 59, 13, 293, 131, 445, 7, 34, 33, 278, 1154, 284, 2095, 194, 227, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 0, 305, 6, 695, 39, 73, 1400, 589, 25, 53, 1998, 1037, 2078, 27, 378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 682, 1571, 61, 32, 119, 433, 65, 32, 842, 11, 717, 2344, 31, 8, 109, 882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 225, 94, 8, 74, 2, 374, 43, 16, 177, 73, 31, 620, 939, 13, 449, 298, 1513, 1686, 925, 547, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 27, 0, 6, 1954, 10, 37, 1160, 938, 179, 1729, 158, 516, 6, 494, 798, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 4, 0, 43, 105, 12, 770, 432, 18, 61, 1014, 3, 74, 138, 547, 79, 2176, 2274, 656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1442, 171, 18, 1677, 123, 5, 21, 115, 567, 81, 905, 2707, 18, 135, 62, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 744, 71, 13, 75, 2, 904, 237, 224, 1129, 17, 1387, 413, 255, 210, 6, 2312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7887, 8892, 7939, 8128, 8139, 8468, 8084, 7999] avg=8192.0000 max=8892 max_rank=1 min=7887 min_rank=0 imbalance_ratio=1.085449 max_load_reduction=56644 max_load_reduction_pct=0.864319 moved_to_echo=32690 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1765, 93, 55, 467, 125, 699, 35, 52, 31, 249, 1681, 253, 1339, 106, 193, 387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 306, 0, 209, 20, 489, 10, 58, 1558, 678, 60, 34, 1815, 692, 2065, 121, 633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 782, 1658, 100, 68, 166, 189, 183, 67, 1089, 21, 619, 1726, 39, 9, 207, 893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 210, 4, 72, 6, 299, 103, 24, 129, 24, 17, 517, 1055, 16, 562, 287, 1783, 1392, 939, 400, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 25, 8, 14, 1742, 3, 66, 1572, 865, 103, 1247, 225, 481, 6, 550, 1364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 13, 2, 42, 52, 10, 1289, 676, 59, 48, 934, 2, 29, 336, 272, 66, 2012, 1884, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1545, 258, 77, 1387, 194, 2, 69, 82, 280, 143, 343, 3209, 30, 134, 114, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 813, 145, 9, 153, 8, 211, 254, 443, 1441, 88, 1838, 438, 83, 486, 5, 2365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7530, 8748, 7816, 8038, 8379, 8322, 7923, 8780] avg=8192.0000 max=8780 max_rank=7 min=7530 min_rank=0 imbalance_ratio=1.071777 max_load_reduction=56756 max_load_reduction_pct=0.866028 moved_to_echo=33404 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1831, 120, 26, 419, 113, 348, 13, 12, 77, 263, 2564, 504, 1737, 90, 230, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 484, 11, 1081, 28, 87, 1274, 508, 40, 76, 2017, 749, 2355, 82, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 2885, 142, 30, 227, 427, 48, 69, 733, 63, 561, 2104, 40, 13, 141, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 388, 90, 1, 91, 2, 927, 33, 19, 240, 40, 50, 383, 924, 24, 519, 163, 1369, 1443, 892, 487, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 20, 25, 1, 1356, 14, 73, 1108, 725, 45, 2149, 254, 659, 4, 444, 1094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 9, 11, 16, 41, 5, 457, 562, 20, 31, 758, 15, 49, 100, 374, 112, 2145, 1754, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1227, 159, 107, 2249, 150, 0, 78, 300, 365, 114, 413, 2464, 25, 221, 41, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 674, 48, 18, 92, 11, 239, 190, 417, 1377, 54, 1997, 368, 155, 171, 10, 2247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8782, 9024, 8471, 8103, 8072, 7058, 7958, 8068] avg=8192.0000 max=9024 max_rank=1 min=7058 min_rank=5 imbalance_ratio=1.101562 max_load_reduction=56512 max_load_reduction_pct=0.862305 moved_to_echo=31156 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1853, 82, 42, 464, 292, 453, 14, 19, 52, 288, 1082, 1084, 1511, 110, 589, 281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 175, 0, 193, 34, 1090, 68, 177, 1369, 739, 153, 68, 2315, 776, 1391, 93, 286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 843, 1450, 81, 12, 357, 415, 160, 11, 977, 26, 501, 2426, 25, 33, 152, 624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 223, 223, 7, 48, 7, 243, 40, 41, 94, 77, 57, 725, 1249, 25, 628, 159, 1527, 1461, 920, 562, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 26, 5, 18, 1431, 79, 125, 1494, 777, 164, 1409, 89, 388, 15, 625, 881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 31, 3, 26, 34, 1, 649, 938, 11, 43, 617, 10, 24, 225, 239, 129, 2286, 2048, 568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 801, 120, 76, 3116, 157, 3, 91, 254, 566, 106, 762, 2748, 30, 192, 77, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 781, 35, 21, 58, 7, 278, 146, 263, 1309, 44, 1269, 339, 147, 249, 8, 2317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8927, 8093, 8336, 7644, 7925, 9124, 7271] avg=8192.0000 max=9124 max_rank=6 min=7271 min_rank=7 imbalance_ratio=1.113770 max_load_reduction=56412 max_load_reduction_pct=0.860779 moved_to_echo=31964 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1922, 230, 10, 431, 54, 540, 53, 20, 50, 374, 1890, 410, 1445, 100, 201, 573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 170, 15, 827, 27, 112, 1216, 691, 63, 50, 1478, 342, 1660, 30, 261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 641, 1675, 122, 36, 87, 357, 101, 35, 952, 19, 547, 2131, 22, 5, 136, 994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 272, 152, 1, 36, 2, 385, 150, 18, 113, 32, 18, 300, 1277, 68, 434, 128, 1873, 1514, 976, 493, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 18, 15, 9, 1934, 2, 33, 1459, 980, 69, 1725, 261, 349, 10, 544, 771, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 16, 26, 28, 35, 36, 1197, 973, 27, 44, 1112, 16, 38, 103, 253, 67, 1542, 2557, 588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1349, 117, 17, 1989, 83, 6, 57, 82, 356, 93, 431, 3371, 27, 126, 53, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 923, 26, 14, 151, 1, 583, 147, 325, 1557, 39, 1758, 332, 377, 188, 5, 2459, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8303, 7023, 7860, 8255, 8326, 8680, 8204, 8885] avg=8192.0000 max=8885 max_rank=7 min=7023 min_rank=1 imbalance_ratio=1.084595 max_load_reduction=56651 max_load_reduction_pct=0.864426 moved_to_echo=34095 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2099, 162, 2, 510, 136, 840, 9, 2, 52, 290, 1562, 547, 1308, 107, 203, 866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 274, 13, 1033, 18, 98, 1615, 424, 66, 42, 1896, 390, 988, 53, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 428, 1255, 57, 47, 80, 467, 48, 4, 1189, 16, 719, 2787, 25, 30, 152, 818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 210, 82, 0, 19, 0, 354, 118, 18, 118, 19, 31, 502, 1077, 20, 340, 100, 1769, 1654, 849, 646, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 26, 14, 19, 2136, 7, 33, 1883, 1048, 130, 1811, 76, 444, 2, 752, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 20, 2, 27, 42, 5, 761, 1036, 35, 24, 613, 3, 15, 185, 262, 79, 2237, 2198, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 926, 115, 50, 1860, 102, 3, 37, 98, 248, 67, 660, 3184, 33, 55, 47, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 681, 37, 10, 81, 3, 683, 120, 295, 1193, 37, 1885, 413, 267, 310, 41, 2140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8695, 7249, 8122, 7943, 9538, 8213, 7580, 8196] avg=8192.0000 max=9538 max_rank=4 min=7249 min_rank=1 imbalance_ratio=1.164307 max_load_reduction=55998 max_load_reduction_pct=0.854462 moved_to_echo=33527 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1853, 82, 42, 464, 292, 453, 14, 19, 52, 288, 1082, 1084, 1511, 110, 589, 281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 175, 0, 193, 34, 1090, 68, 177, 1369, 739, 153, 68, 2315, 776, 1391, 93, 286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 843, 1450, 81, 12, 357, 415, 160, 11, 977, 26, 501, 2426, 25, 33, 152, 624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 223, 223, 7, 48, 7, 243, 40, 41, 94, 77, 57, 725, 1249, 25, 628, 159, 1527, 1461, 920, 562, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 26, 5, 18, 1431, 79, 125, 1494, 777, 164, 1409, 89, 388, 15, 625, 881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 31, 3, 26, 34, 1, 649, 938, 11, 43, 617, 10, 24, 225, 239, 129, 2286, 2048, 568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 801, 120, 76, 3116, 157, 3, 91, 254, 566, 106, 762, 2748, 30, 192, 77, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 781, 35, 21, 58, 7, 278, 146, 263, 1309, 44, 1269, 339, 147, 249, 8, 2317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8927, 8093, 8336, 7644, 7925, 9124, 7271] avg=8192.0000 max=9124 max_rank=6 min=7271 min_rank=7 imbalance_ratio=1.113770 max_load_reduction=56412 max_load_reduction_pct=0.860779 moved_to_echo=31964 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2124, 59, 13, 293, 131, 445, 7, 34, 33, 278, 1154, 284, 2095, 194, 227, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 0, 305, 6, 695, 39, 73, 1400, 589, 25, 53, 1998, 1037, 2078, 27, 378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 682, 1571, 61, 32, 119, 433, 65, 32, 842, 11, 717, 2344, 31, 8, 109, 882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 225, 94, 8, 74, 2, 374, 43, 16, 177, 73, 31, 620, 939, 13, 449, 298, 1513, 1686, 925, 547, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 27, 0, 6, 1954, 10, 37, 1160, 938, 179, 1729, 158, 516, 6, 494, 798, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 4, 0, 43, 105, 12, 770, 432, 18, 61, 1014, 3, 74, 138, 547, 79, 2176, 2274, 656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1442, 171, 18, 1677, 123, 5, 21, 115, 567, 81, 905, 2707, 18, 135, 62, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 744, 71, 13, 75, 2, 904, 237, 224, 1129, 17, 1387, 413, 255, 210, 6, 2312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7887, 8892, 7939, 8128, 8139, 8468, 8084, 7999] avg=8192.0000 max=8892 max_rank=1 min=7887 min_rank=0 imbalance_ratio=1.085449 max_load_reduction=56644 max_load_reduction_pct=0.864319 moved_to_echo=32690 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1770, 68, 25, 436, 236, 457, 12, 20, 63, 154, 2215, 412, 1272, 118, 112, 806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 0, 259, 9, 1318, 50, 172, 1656, 541, 60, 14, 1536, 506, 2068, 127, 270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 1843, 175, 104, 250, 432, 98, 38, 1644, 51, 460, 2523, 25, 64, 94, 596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 204, 70, 4, 118, 8, 414, 53, 24, 170, 94, 41, 392, 1287, 11, 899, 195, 1285, 1395, 1112, 585, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 33, 8, 13, 1242, 13, 120, 1706, 748, 149, 1926, 142, 457, 19, 535, 772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 13, 3, 24, 59, 7, 876, 548, 14, 23, 474, 12, 40, 166, 564, 75, 2480, 1265, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 648, 123, 145, 1877, 177, 0, 103, 72, 549, 72, 509, 2312, 23, 196, 44, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 745, 109, 12, 131, 12, 751, 364, 191, 1297, 46, 2120, 408, 108, 237, 3, 2800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8734, 8858, 8374, 7956, 7215, 6889, 9334] avg=8192.0000 max=9334 max_rank=7 min=6889 min_rank=6 imbalance_ratio=1.139404 max_load_reduction=56202 max_load_reduction_pct=0.857574 moved_to_echo=31394 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1669, 61, 42, 259, 178, 571, 6, 29, 70, 214, 1683, 441, 1991, 113, 353, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 0, 97, 44, 527, 74, 146, 1576, 766, 59, 36, 1565, 332, 1169, 91, 343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 1856, 53, 87, 85, 415, 62, 71, 538, 71, 827, 2629, 39, 8, 326, 849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 260, 214, 4, 37, 6, 132, 79, 8, 33, 39, 51, 652, 2002, 32, 217, 93, 1688, 1315, 874, 609, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 22, 3, 15, 1553, 16, 61, 1072, 765, 89, 2009, 122, 409, 18, 372, 803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 18, 11, 66, 98, 17, 954, 848, 20, 75, 1080, 9, 12, 261, 275, 96, 2618, 2661, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1919, 179, 13, 2969, 56, 3, 11, 74, 424, 136, 697, 3038, 25, 86, 69, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 25, 31, 22, 4, 555, 133, 408, 1145, 87, 1009, 365, 252, 263, 18, 2203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7947, 6939, 8377, 8359, 7482, 9655, 9774, 7003] avg=8192.0000 max=9774 max_rank=6 min=6939 min_rank=1 imbalance_ratio=1.193115 max_load_reduction=55762 max_load_reduction_pct=0.850861 moved_to_echo=33914 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1765, 93, 55, 467, 125, 699, 35, 52, 31, 249, 1681, 253, 1339, 106, 193, 387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 306, 0, 209, 20, 489, 10, 58, 1558, 678, 60, 34, 1815, 692, 2065, 121, 633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 782, 1658, 100, 68, 166, 189, 183, 67, 1089, 21, 619, 1726, 39, 9, 207, 893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 210, 4, 72, 6, 299, 103, 24, 129, 24, 17, 517, 1055, 16, 562, 287, 1783, 1392, 939, 400, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 25, 8, 14, 1742, 3, 66, 1572, 865, 103, 1247, 225, 481, 6, 550, 1364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 13, 2, 42, 52, 10, 1289, 676, 59, 48, 934, 2, 29, 336, 272, 66, 2012, 1884, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1545, 258, 77, 1387, 194, 2, 69, 82, 280, 143, 343, 3209, 30, 134, 114, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 813, 145, 9, 153, 8, 211, 254, 443, 1441, 88, 1838, 438, 83, 486, 5, 2365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7530, 8748, 7816, 8038, 8379, 8322, 7923, 8780] avg=8192.0000 max=8780 max_rank=7 min=7530 min_rank=0 imbalance_ratio=1.071777 max_load_reduction=56756 max_load_reduction_pct=0.866028 moved_to_echo=33404 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1831, 120, 26, 419, 113, 348, 13, 12, 77, 263, 2564, 504, 1737, 90, 230, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 484, 11, 1081, 28, 87, 1274, 508, 40, 76, 2017, 749, 2355, 82, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 2885, 142, 30, 227, 427, 48, 69, 733, 63, 561, 2104, 40, 13, 141, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 388, 90, 1, 91, 2, 927, 33, 19, 240, 40, 50, 383, 924, 24, 519, 163, 1369, 1443, 892, 487, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 20, 25, 1, 1356, 14, 73, 1108, 725, 45, 2149, 254, 659, 4, 444, 1094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 9, 11, 16, 41, 5, 457, 562, 20, 31, 758, 15, 49, 100, 374, 112, 2145, 1754, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1227, 159, 107, 2249, 150, 0, 78, 300, 365, 114, 413, 2464, 25, 221, 41, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 674, 48, 18, 92, 11, 239, 190, 417, 1377, 54, 1997, 368, 155, 171, 10, 2247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8782, 9024, 8471, 8103, 8072, 7058, 7958, 8068] avg=8192.0000 max=9024 max_rank=1 min=7058 min_rank=5 imbalance_ratio=1.101562 max_load_reduction=56512 max_load_reduction_pct=0.862305 moved_to_echo=31156 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1669, 61, 42, 259, 178, 571, 6, 29, 70, 214, 1683, 441, 1991, 113, 353, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 0, 97, 44, 527, 74, 146, 1576, 766, 59, 36, 1565, 332, 1169, 91, 343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 1856, 53, 87, 85, 415, 62, 71, 538, 71, 827, 2629, 39, 8, 326, 849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 260, 214, 4, 37, 6, 132, 79, 8, 33, 39, 51, 652, 2002, 32, 217, 93, 1688, 1315, 874, 609, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 22, 3, 15, 1553, 16, 61, 1072, 765, 89, 2009, 122, 409, 18, 372, 803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 18, 11, 66, 98, 17, 954, 848, 20, 75, 1080, 9, 12, 261, 275, 96, 2618, 2661, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1919, 179, 13, 2969, 56, 3, 11, 74, 424, 136, 697, 3038, 25, 86, 69, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 25, 31, 22, 4, 555, 133, 408, 1145, 87, 1009, 365, 252, 263, 18, 2203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7947, 6939, 8377, 8359, 7482, 9655, 9774, 7003] avg=8192.0000 max=9774 max_rank=6 min=6939 min_rank=1 imbalance_ratio=1.193115 max_load_reduction=55762 max_load_reduction_pct=0.850861 moved_to_echo=33914 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. @@ -1865,54 +1867,54 @@ WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] glob warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[2048, 1152, 128, 512, 128, 3200, 640, 256, 1152, 512, 384, 4096, 9856, 256, 4096, 1536, 12928, 11904, 7552, 4352, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[384, 128, 128, 384, 512, 128, 7040, 6016, 256, 384, 6656, 128, 384, 1536, 2816, 768, 17536, 16768, 4736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[5888, 512, 128, 768, 128, 4224, 1664, 2688, 10496, 512, 13312, 3200, 1664, 2176, 128, 18944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[1408, 0, 2048, 256, 7168, 384, 1024, 11776, 4992, 640, 384, 14720, 4864, 13824, 640, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[15104, 896, 256, 3328, 1280, 4480, 256, 256, 512, 2176, 13952, 3968, 12800, 1024, 2176, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[1408, 0, 2048, 256, 7168, 384, 1024, 11776, 4992, 640, 384, 14720, 4864, 13824, 640, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[1024, 256, 128, 128, 13440, 256, 640, 11520, 6912, 1024, 14080, 1408, 3712, 128, 4352, 7552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4864, 14208, 896, 512, 1408, 3200, 768, 384, 8064, 384, 4992, 18688, 256, 256, 1408, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[9984, 1280, 512, 17152, 1152, 128, 512, 1152, 3456, 896, 4736, 23040, 256, 1152, 512, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[5888, 512, 128, 768, 128, 4224, 1664, 2688, 10496, 512, 13312, 3200, 1664, 2176, 128, 18944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 282, 0, 1579, 0, 0, 0, 3, 17, 71, 4450, 0, 0, 2442, 0, 0, 0, 27, 0, 1220, 1, 0, 0, 0, 8, 2, 9, 3, 0, 0, 1236, 9, 0, 115, 0, 0, 7, 24, 0, 528, 0, 785, 542, 0, 6, 8, 48, 20, 0, 0, 7, 0, 809, 0, 99, 5615, 0, 4148, 611, 2819, 0, 1068, 5, 1206, 322, 0, 0, 0, 1325, 8144, 0, 2, 0, 4, 80, 0, 0, 97, 1, 0, 1, 0, 0, 174, 1, 733, 0, 0, 6540, 0, 540, 0, 0, 33, 5121, 0, 25, 54, 0, 95, 8, 3278, 0, 59, 2, 487, 74, 75, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 7099, 59, 0, 2, 367, 1, 901, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[134, 3721, 0, 3384, 2, 0, 3, 9, 2, 21, 6991, 0, 0, 6, 0, 0, 0, 0, 3, 1434, 0, 0, 924, 0, 0, 0, 0, 1820, 47, 2, 126, 0, 0, 14, 1, 9, 0, 0, 0, 660, 0, 17, 613, 1, 1, 55, 3, 0, 22, 9, 1, 0, 2338, 0, 0, 5784, 4, 189, 2567, 0, 0, 8, 5, 185, 1, 0, 0, 0, 210, 8022, 0, 0, 1, 0, 0, 0, 3, 13, 1, 1, 0, 0, 4, 0, 0, 0, 95, 1, 7693, 1, 3546, 1, 13, 0, 1231, 168, 0, 3, 30, 0, 0, 2581, 22, 1, 0, 0, 474, 162, 1, 0, 0, 0, 0, 124, 0, 6, 0, 4, 2526, 0, 1, 0, 11, 13, 7456, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[768, 333, 1, 3185, 1, 5, 1, 0, 47, 2, 7785, 0, 0, 1271, 2, 117, 0, 351, 137, 248, 0, 1, 0, 0, 1, 2, 13, 108, 0, 6, 33, 329, 0, 0, 26, 0, 0, 1, 0, 0, 0, 148, 56, 51, 11, 12, 0, 2, 131, 27, 17, 44, 1548, 2, 8, 2204, 0, 585, 1112, 2055, 0, 1425, 106, 3314, 93, 10, 1, 0, 1175, 7756, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 201, 2096, 742, 106, 1403, 4, 7915, 1, 89, 0, 1885, 47, 2579, 78, 0, 0, 0, 0, 7, 2012, 0, 0, 0, 6, 14, 375, 2, 7, 0, 3, 2, 226, 0, 89, 102, 1, 1385, 10, 0, 0, 21, 7, 7356, 89, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 282, 0, 1579, 0, 0, 0, 3, 17, 71, 4450, 0, 0, 2442, 0, 0, 0, 27, 0, 1220, 1, 0, 0, 0, 8, 2, 9, 3, 0, 0, 1236, 9, 0, 115, 0, 0, 7, 24, 0, 528, 0, 785, 542, 0, 6, 8, 48, 20, 0, 0, 7, 0, 809, 0, 99, 5615, 0, 4148, 611, 2819, 0, 1068, 5, 1206, 322, 0, 0, 0, 1325, 8144, 0, 2, 0, 4, 80, 0, 0, 97, 1, 0, 1, 0, 0, 174, 1, 733, 0, 0, 6540, 0, 540, 0, 0, 33, 5121, 0, 25, 54, 0, 95, 8, 3278, 0, 59, 2, 487, 74, 75, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 7099, 59, 0, 2, 367, 1, 901, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[83, 4768, 1, 3233, 85, 0, 15, 3, 0, 102, 7615, 0, 0, 575, 3, 12, 1, 0, 2, 456, 0, 0, 3, 0, 0, 0, 360, 1588, 2, 3, 0, 0, 0, 0, 3, 20, 18, 6, 0, 1, 0, 652, 12, 0, 65, 3, 0, 0, 0, 4, 58, 0, 94, 6, 0, 8142, 0, 4635, 2841, 69, 0, 40, 63, 363, 330, 16, 0, 8, 1177, 8013, 0, 0, 0, 0, 0, 5, 2, 0, 67, 0, 0, 0, 0, 1, 0, 1, 64, 0, 1053, 2, 175, 2, 18, 0, 4143, 0, 0, 0, 445, 0, 13, 835, 43, 517, 12, 6, 24, 79, 0, 0, 0, 0, 1, 0, 17, 0, 2, 0, 4548, 1, 9, 9, 200, 0, 7685, 7, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[52, 28, 62, 1689, 1, 1, 2, 23, 0, 0, 4889, 0, 27, 33, 3, 122, 0, 2, 15, 581, 0, 0, 11, 0, 0, 10, 4, 816, 1, 0, 35, 737, 2, 2, 3, 0, 0, 0, 1, 3, 0, 2967, 1367, 0, 4, 0, 0, 0, 52, 0, 127, 0, 4617, 0, 3, 7351, 0, 681, 354, 2129, 0, 4203, 74, 1914, 262, 0, 6, 0, 954, 7870, 0, 51, 0, 0, 0, 19, 0, 0, 0, 0, 0, 1, 104, 98, 3, 13, 705, 7, 4136, 0, 9, 0, 4, 7, 5171, 0, 0, 0, 0, 383, 0, 1544, 0, 12, 0, 9, 239, 1296, 0, 28, 0, 0, 0, 22, 0, 0, 0, 0, 13, 0, 51, 0, 4, 0, 7517, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[134, 3721, 0, 3384, 2, 0, 3, 9, 2, 21, 6991, 0, 0, 6, 0, 0, 0, 0, 3, 1434, 0, 0, 924, 0, 0, 0, 0, 1820, 47, 2, 126, 0, 0, 14, 1, 9, 0, 0, 0, 660, 0, 17, 613, 1, 1, 55, 3, 0, 22, 9, 1, 0, 2338, 0, 0, 5784, 4, 189, 2567, 0, 0, 8, 5, 185, 1, 0, 0, 0, 210, 8022, 0, 0, 1, 0, 0, 0, 3, 13, 1, 1, 0, 0, 4, 0, 0, 0, 95, 1, 7693, 1, 3546, 1, 13, 0, 1231, 168, 0, 3, 30, 0, 0, 2581, 22, 1, 0, 0, 474, 162, 1, 0, 0, 0, 0, 124, 0, 6, 0, 4, 2526, 0, 1, 0, 11, 13, 7456, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[171, 1250, 1, 34, 0, 0, 0, 0, 0, 216, 8019, 4, 22, 155, 0, 0, 40, 5, 1, 2873, 0, 0, 0, 0, 0, 0, 2, 865, 4, 0, 0, 84, 8, 0, 0, 19, 4, 11, 0, 2, 0, 103, 149, 0, 0, 683, 0, 0, 20, 79, 0, 0, 6417, 0, 12, 6010, 0, 928, 4586, 0, 0, 1021, 0, 644, 408, 0, 0, 0, 13, 8103, 22, 0, 0, 0, 0, 0, 0, 0, 2, 16, 0, 519, 41, 1, 106, 16, 3517, 0, 6692, 0, 277, 6, 1, 0, 677, 0, 27, 1, 0, 1, 0, 2594, 0, 0, 0, 66, 0, 0, 36, 4, 0, 0, 0, 0, 0, 0, 0, 6, 248, 7, 0, 0, 5, 0, 7682, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[391, 1862, 0, 307, 13, 0, 61, 273, 0, 0, 8065, 0, 9, 4612, 0, 35, 2, 0, 23, 2758, 0, 0, 266, 0, 0, 0, 2043, 27, 0, 0, 3, 0, 3, 2, 0, 0, 3, 591, 0, 0, 0, 39, 7, 0, 2, 51, 0, 0, 76, 1, 2, 1, 735, 68, 6, 6409, 0, 2, 1141, 63, 0, 49, 1, 665, 10, 0, 0, 0, 23, 8188, 20, 1, 0, 0, 0, 389, 0, 0, 0, 11, 0, 100, 0, 156, 0, 0, 0, 0, 6379, 1, 437, 0, 5, 0, 3538, 0, 1, 0, 0, 29, 0, 1449, 0, 5, 1, 0, 0, 0, 29, 0, 2, 0, 0, 0, 0, 0, 0, 0, 6157, 0, 0, 3, 82, 0, 7837, 16, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1546, 1823, 5, 431, 32, 0, 0, 0, 1, 1460, 5628, 0, 12, 496, 0, 4, 0, 1, 106, 627, 0, 0, 0, 0, 0, 0, 2, 154, 2, 0, 4, 2, 1, 97, 0, 0, 0, 0, 5, 482, 4, 1949, 6, 0, 223, 5, 0, 0, 86, 1220, 81, 7, 4274, 10, 138, 6892, 0, 5024, 5, 4, 0, 1957, 0, 1017, 2, 0, 0, 35, 351, 8125, 0, 15, 0, 29, 0, 102, 0, 0, 2, 0, 1, 0, 0, 1, 0, 1414, 98, 19, 7473, 0, 137, 0, 6, 0, 5817, 3, 0, 0, 2, 57, 11, 578, 108, 3, 2, 166, 9, 0, 45, 5, 0, 0, 0, 252, 0, 20, 0, 0, 840, 0, 0, 0, 37, 0, 3948, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[83, 4768, 1, 3233, 85, 0, 15, 3, 0, 102, 7615, 0, 0, 575, 3, 12, 1, 0, 2, 456, 0, 0, 3, 0, 0, 0, 360, 1588, 2, 3, 0, 0, 0, 0, 3, 20, 18, 6, 0, 1, 0, 652, 12, 0, 65, 3, 0, 0, 0, 4, 58, 0, 94, 6, 0, 8142, 0, 4635, 2841, 69, 0, 40, 63, 363, 330, 16, 0, 8, 1177, 8013, 0, 0, 0, 0, 0, 5, 2, 0, 67, 0, 0, 0, 0, 1, 0, 1, 64, 0, 1053, 2, 175, 2, 18, 0, 4143, 0, 0, 0, 445, 0, 13, 835, 43, 517, 12, 6, 24, 79, 0, 0, 0, 0, 1, 0, 17, 0, 2, 0, 4548, 1, 9, 9, 200, 0, 7685, 7, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[391, 1862, 0, 307, 13, 0, 61, 273, 0, 0, 8065, 0, 9, 4612, 0, 35, 2, 0, 23, 2758, 0, 0, 266, 0, 0, 0, 2043, 27, 0, 0, 3, 0, 3, 2, 0, 0, 3, 591, 0, 0, 0, 39, 7, 0, 2, 51, 0, 0, 76, 1, 2, 1, 735, 68, 6, 6409, 0, 2, 1141, 63, 0, 49, 1, 665, 10, 0, 0, 0, 23, 8188, 20, 1, 0, 0, 0, 389, 0, 0, 0, 11, 0, 100, 0, 156, 0, 0, 0, 0, 6379, 1, 437, 0, 5, 0, 3538, 0, 1, 0, 0, 29, 0, 1449, 0, 5, 1, 0, 0, 0, 29, 0, 2, 0, 0, 0, 0, 0, 0, 0, 6157, 0, 0, 3, 82, 0, 7837, 16, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[768, 333, 1, 3185, 1, 5, 1, 0, 47, 2, 3203, 0, 0, 1271, 2, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 351, 137, 248, 0, 1, 0, 0, 1, 2, 13, 108, 0, 6, 33, 329, 1010, 2211, 1394, 770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 1, 0, 0, 0, 148, 56, 51, 11, 12, 0, 2, 2204, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 27, 17, 44, 538, 2, 8, 0, 0, 585, 1112, 2055, 0, 1425, 106, 3314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 10, 1, 0, 1175, 6986, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 201, 2096, 742, 106, 1403, 4, 2847, 1, 89, 0, 1885, 47, 2579, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2012, 0, 0, 0, 6, 14, 375, 2, 7, 0, 3, 4031, 2857, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 226, 0, 89, 102, 1, 1385, 10, 0, 0, 21, 7, 5962, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8936, 6614, 3062, 9364, 8270, 12082, 9314, 7894] avg=8192.0000 max=12082 max_rank=5 min=3062 min_rank=2 imbalance_ratio=1.474854 max_load_reduction=53454 max_load_reduction_pct=0.815643 moved_to_echo=37560 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[134, 3721, 0, 3384, 2, 0, 3, 9, 2, 21, 2876, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1434, 0, 0, 924, 0, 0, 0, 0, 1820, 47, 2, 126, 0, 1526, 2149, 1413, 796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 1, 9, 0, 0, 0, 660, 0, 17, 613, 1, 1, 55, 3, 0, 5784, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 9, 1, 0, 812, 0, 0, 0, 4, 189, 2567, 0, 0, 8, 5, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 210, 7226, 0, 0, 1, 0, 0, 0, 3, 13, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 95, 1, 2767, 1, 3546, 1, 13, 0, 1231, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 30, 0, 0, 2581, 22, 1, 0, 0, 474, 162, 1, 0, 0, 0, 3620, 2777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 6, 0, 4, 2526, 0, 1, 0, 11, 13, 6043, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10158, 10240, 7653, 3802, 7456, 7827, 9671, 8729] avg=8192.0000 max=10240 max_rank=1 min=3802 min_rank=3 imbalance_ratio=1.250000 max_load_reduction=55296 max_load_reduction_pct=0.843750 moved_to_echo=33683 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 282, 0, 1579, 0, 0, 0, 3, 17, 71, 1831, 0, 0, 2442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 1220, 1, 0, 0, 0, 8, 2, 9, 3, 0, 0, 1236, 9, 528, 1827, 170, 808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 7, 24, 0, 528, 0, 785, 542, 0, 6, 8, 48, 20, 5615, 315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 281, 0, 99, 0, 0, 4148, 611, 2819, 0, 1068, 5, 1206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 322, 0, 0, 0, 1325, 7336, 0, 2, 0, 4, 80, 0, 0, 97, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 174, 1, 733, 0, 0, 2353, 0, 540, 0, 0, 33, 5121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 54, 0, 95, 8, 3278, 0, 59, 2, 487, 74, 75, 0, 0, 0, 1, 2304, 2360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 7099, 59, 0, 2, 367, 1, 731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6225, 5848, 8013, 10244, 9167, 8956, 8822, 8261] avg=8192.0000 max=10244 max_rank=3 min=5848 min_rank=1 imbalance_ratio=1.250488 max_load_reduction=55292 max_load_reduction_pct=0.843689 moved_to_echo=35206 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1546, 1823, 5, 431, 32, 0, 0, 0, 1, 1460, 2316, 0, 12, 496, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 106, 627, 0, 0, 0, 0, 0, 0, 2, 154, 2, 0, 4, 2, 2791, 2088, 748, 806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 97, 0, 0, 0, 0, 5, 482, 4, 1949, 6, 0, 223, 5, 0, 0, 6892, 398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 1220, 81, 7, 1483, 10, 138, 0, 0, 5024, 5, 4, 0, 1957, 0, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 35, 351, 7319, 0, 15, 0, 29, 0, 102, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1414, 98, 19, 2688, 0, 137, 0, 6, 0, 5817, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 57, 11, 578, 108, 3, 2, 166, 9, 0, 45, 5, 0, 0, 2914, 2697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 252, 0, 20, 0, 0, 840, 0, 0, 0, 37, 0, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8126, 7331, 10062, 11032, 7855, 10184, 6597, 4349] avg=8192.0000 max=11032 max_rank=3 min=4349 min_rank=7 imbalance_ratio=1.346680 max_load_reduction=54504 max_load_reduction_pct=0.831665 moved_to_echo=28985 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[52, 28, 62, 1689, 1, 1, 2, 23, 0, 0, 2012, 0, 27, 33, 3, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 15, 581, 0, 0, 11, 0, 0, 10, 4, 816, 1, 0, 35, 737, 3015, 1155, 1425, 781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 3, 0, 0, 0, 1, 3, 0, 2967, 1367, 0, 4, 0, 0, 0, 7351, 346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 127, 0, 1602, 0, 3, 0, 0, 681, 354, 2129, 0, 4203, 74, 1914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 262, 0, 6, 0, 954, 7089, 0, 51, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 104, 98, 3, 13, 705, 7, 1488, 0, 9, 0, 4, 7, 5171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 383, 0, 1544, 0, 12, 0, 9, 239, 1296, 0, 28, 0, 0, 2531, 1493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 13, 0, 51, 0, 4, 0, 6092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4055, 8588, 12046, 11139, 8381, 7610, 7535, 6182] avg=8192.0000 max=12046 max_rank=2 min=4055 min_rank=0 imbalance_ratio=1.470459 max_load_reduction=53490 max_load_reduction_pct=0.816193 moved_to_echo=29708 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[171, 1250, 1, 34, 0, 0, 0, 0, 0, 216, 3299, 4, 22, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 5, 1, 2873, 0, 0, 0, 0, 0, 0, 2, 865, 4, 0, 0, 84, 4190, 1870, 1456, 804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 19, 4, 11, 0, 2, 0, 103, 149, 0, 0, 683, 0, 0, 6010, 568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 79, 0, 0, 2227, 0, 12, 0, 0, 928, 4586, 0, 0, 1021, 0, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 408, 0, 0, 0, 13, 7299, 22, 0, 0, 0, 0, 0, 0, 0, 2, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 41, 1, 106, 16, 3517, 0, 2407, 0, 277, 6, 1, 0, 677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 1, 0, 1, 0, 2594, 0, 0, 0, 66, 0, 0, 36, 4, 0, 0, 4152, 2415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 248, 7, 0, 0, 5, 0, 6226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5152, 12194, 7557, 9517, 7760, 7568, 9296, 6492] avg=8192.0000 max=12194 max_rank=1 min=5152 min_rank=0 imbalance_ratio=1.488525 max_load_reduction=53342 max_load_reduction_pct=0.813934 moved_to_echo=31116 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[83, 4768, 1, 3233, 85, 0, 15, 3, 0, 102, 3133, 0, 0, 575, 3, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 456, 0, 0, 3, 0, 0, 0, 360, 1588, 2, 3, 0, 0, 61, 294, 1457, 795, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 20, 18, 6, 0, 1, 0, 652, 12, 0, 65, 3, 0, 0, 8142, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 58, 0, 33, 6, 0, 0, 0, 4635, 2841, 69, 0, 40, 63, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 330, 16, 0, 8, 1177, 7218, 0, 0, 0, 0, 0, 5, 2, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 64, 0, 379, 2, 175, 2, 18, 0, 4143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 445, 0, 13, 835, 43, 517, 12, 6, 24, 79, 0, 0, 0, 0, 3943, 380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 17, 0, 2, 0, 4548, 1, 9, 9, 200, 0, 6228, 7, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12013, 5022, 9461, 8112, 8823, 4785, 6297, 11023] avg=8192.0000 max=12013 max_rank=0 min=4785 min_rank=5 imbalance_ratio=1.466431 max_load_reduction=53523 max_load_reduction_pct=0.816696 moved_to_echo=30928 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[171, 1250, 1, 34, 0, 0, 0, 0, 0, 216, 3299, 4, 22, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 5, 1, 2873, 0, 0, 0, 0, 0, 0, 2, 865, 4, 0, 0, 84, 4190, 1870, 1456, 804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 19, 4, 11, 0, 2, 0, 103, 149, 0, 0, 683, 0, 0, 6010, 568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 79, 0, 0, 2227, 0, 12, 0, 0, 928, 4586, 0, 0, 1021, 0, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 408, 0, 0, 0, 13, 7299, 22, 0, 0, 0, 0, 0, 0, 0, 2, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 41, 1, 106, 16, 3517, 0, 2407, 0, 277, 6, 1, 0, 677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 1, 0, 1, 0, 2594, 0, 0, 0, 66, 0, 0, 36, 4, 0, 0, 4152, 2415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 248, 7, 0, 0, 5, 0, 6226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5152, 12194, 7557, 9517, 7760, 7568, 9296, 6492] avg=8192.0000 max=12194 max_rank=1 min=5152 min_rank=0 imbalance_ratio=1.488525 max_load_reduction=53342 max_load_reduction_pct=0.813934 moved_to_echo=31116 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1546, 1823, 5, 431, 32, 0, 0, 0, 1, 1460, 2316, 0, 12, 496, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 106, 627, 0, 0, 0, 0, 0, 0, 2, 154, 2, 0, 4, 2, 2791, 2088, 748, 806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 97, 0, 0, 0, 0, 5, 482, 4, 1949, 6, 0, 223, 5, 0, 0, 6892, 398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 1220, 81, 7, 1483, 10, 138, 0, 0, 5024, 5, 4, 0, 1957, 0, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 35, 351, 7319, 0, 15, 0, 29, 0, 102, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1414, 98, 19, 2688, 0, 137, 0, 6, 0, 5817, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 57, 11, 578, 108, 3, 2, 166, 9, 0, 45, 5, 0, 0, 2914, 2697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 252, 0, 20, 0, 0, 840, 0, 0, 0, 37, 0, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8126, 7331, 10062, 11032, 7855, 10184, 6597, 4349] avg=8192.0000 max=11032 max_rank=3 min=4349 min_rank=7 imbalance_ratio=1.346680 max_load_reduction=54504 max_load_reduction_pct=0.831665 moved_to_echo=28985 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[134, 3721, 0, 3384, 2, 0, 3, 9, 2, 21, 2876, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1434, 0, 0, 924, 0, 0, 0, 0, 1820, 47, 2, 126, 0, 1526, 2149, 1413, 796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 1, 9, 0, 0, 0, 660, 0, 17, 613, 1, 1, 55, 3, 0, 5784, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 9, 1, 0, 812, 0, 0, 0, 4, 189, 2567, 0, 0, 8, 5, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 210, 7226, 0, 0, 1, 0, 0, 0, 3, 13, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 95, 1, 2767, 1, 3546, 1, 13, 0, 1231, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 30, 0, 0, 2581, 22, 1, 0, 0, 474, 162, 1, 0, 0, 0, 3620, 2777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 6, 0, 4, 2526, 0, 1, 0, 11, 13, 6043, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10158, 10240, 7653, 3802, 7456, 7827, 9671, 8729] avg=8192.0000 max=10240 max_rank=1 min=3802 min_rank=3 imbalance_ratio=1.250000 max_load_reduction=55296 max_load_reduction_pct=0.843750 moved_to_echo=33683 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[391, 1862, 0, 307, 13, 0, 61, 273, 0, 0, 3308, 0, 9, 4612, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 23, 2758, 0, 0, 266, 0, 0, 0, 2043, 27, 0, 0, 3, 0, 484, 1786, 1489, 818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 0, 3, 591, 0, 0, 0, 39, 7, 0, 2, 51, 0, 0, 6409, 575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 1, 2, 1, 251, 68, 6, 0, 0, 2, 1141, 63, 0, 49, 1, 665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 23, 7370, 20, 1, 0, 0, 0, 389, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 156, 0, 0, 0, 0, 2287, 1, 437, 0, 5, 0, 3538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 29, 0, 1449, 0, 5, 1, 0, 0, 0, 29, 0, 2, 0, 4182, 2306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6157, 0, 0, 3, 82, 0, 6348, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10871, 9699, 7682, 2326, 7824, 6524, 8004, 12606] avg=8192.0000 max=12606 max_rank=7 min=2326 min_rank=3 imbalance_ratio=1.538818 max_load_reduction=52930 max_load_reduction_pct=0.807648 moved_to_echo=34958 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=67328 observed_loss=-1792 tokens_per_expert=[128, 512, 384, 10240, 128, 128, 1280, 0, 128, 128, 2560, 5504, 128, 128, 1536, 1280, 13696, 13440, 9600, 6400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 512, 640, 128, 14976, 256, 640, 128, 768, 896, 2048, 128, 128, 128, 128, 27776, 17408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[3200, 14080, 128, 13952, 256, 128, 128, 384, 128, 1920, 22016, 128, 128, 9600, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] @@ -1923,116 +1925,116 @@ WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] glob WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 256, 128, 128, 128, 640, 128, 1792, 128, 6784, 2816, 128, 384, 896, 128, 128, 48512, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[1536, 128, 128, 128, 5248, 57856, 128, 128, 128, 128, 128, 640, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[1536, 128, 128, 128, 5248, 57856, 128, 128, 128, 128, 128, 640, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 3532, 0, 5, 0, 0, 0, 15, 19, 9, 8, 0, 0, 0, 64, 0, 0, 0, 0, 238, 0, 0, 0, 0, 0, 7848, 0, 0, 0, 0, 2, 151, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 6, 0, 6, 0, 0, 0, 410, 10, 1889, 1, 0, 0, 0, 8104, 0, 3457, 0, 0, 0, 0, 0, 0, 0, 0, 2, 177, 0, 0, 0, 7238, 76, 0, 0, 0, 7989, 93, 0, 0, 0, 0, 0, 0, 0, 549, 5, 0, 15, 0, 0, 0, 0, 3, 0, 2, 0, 0, 218, 8147, 55, 0, 0, 41, 6381, 169, 15, 4, 91, 0, 14, 0, 0, 570, 7894, 0, 3, 0, 0, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 9, 0, 14, 7680, 66, 0, 0, 33, 0, 2, 0, 38, 42, 0, 0, 0, 0, 0, 24, 33, 0, 0, 0, 0, 4, 0, 0, 976, 25, 891, 0, 0, 0, 0, 0, 0, 11, 13, 1, 113, 0, 0, 0, 0, 0, 0, 9, 0, 0, 6, 0, 0, 1, 1, 0, 7997, 0, 0, 0, 2, 6866, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 275, 0, 6, 8103, 0, 0, 0, 2, 8149, 0, 0, 0, 0, 0, 0, 7, 0, 275, 0, 0, 0, 0, 0, 1, 20, 25, 1, 4, 0, 3, 0, 471, 1, 0, 0, 0, 8124, 0, 0, 69, 0, 0, 1, 1, 2, 0, 6932, 0, 0, 45, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 3, 12, 5, 0, 5744, 0, 22, 23, 57, 2, 259, 25, 4, 0, 41, 3, 0, 190, 0, 0, 249, 11, 0, 0, 1144, 0, 495, 1, 673, 278, 0, 0, 2, 0, 0, 0, 277, 0, 7, 0, 0, 2, 3, 0, 0, 0, 147, 0, 0, 2, 0, 0, 1, 0, 4951, 0, 4422, 0, 3, 0, 0, 7698, 0, 241, 0, 0, 0, 0, 0, 0, 239, 0, 0, 0, 0, 0, 0, 6800, 0, 0, 1, 0, 3116, 2, 743, 0, 0, 1, 3, 20, 0, 4416, 8, 0, 246, 0, 2, 0, 1, 0, 224, 715, 17, 2697, 0, 3997, 0, 0, 2, 3, 7524, 0, 2, 0, 146, 0, 0, 31, 0, 0, 7517, 0, 45, 18, 3, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 7, 0, 0, 0, 7195, 0, 1, 39, 35, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 180, 0, 8, 6, 1, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 833, 0, 0, 0, 0, 0, 0, 0, 44, 0, 1, 0, 0, 0, 0, 0, 36, 0, 8074, 0, 167, 1, 0, 4644, 0, 612, 0, 0, 0, 90, 0, 0, 0, 0, 0, 49, 0, 1, 0, 7444, 726, 0, 59, 51, 30, 0, 5, 0, 1, 0, 451, 0, 54, 46, 0, 5, 0, 140, 0, 0, 1958, 0, 36, 0, 0, 0, 6479, 5546, 1, 0, 1, 0, 7698, 393, 1, 3835, 0, 0, 0, 0, 0, 61, 8147, 1, 0, 258, 57, 21, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 1, 7499, 8, 0, 0, 0, 0, 0, 0, 66, 159, 0, 0, 81, 0, 0, 0, 289, 0, 0, 0, 0, 0, 2, 0, 6428, 69, 0, 0, 0, 0, 0, 0, 23, 0, 2, 3, 0, 0, 0, 0, 2, 0, 57, 0, 244, 29, 17, 0, 0, 0, 338, 0, 7926, 0, 0, 0, 0, 7850, 10, 6153, 1, 0, 0, 77, 0, 0, 0, 6, 15, 0, 8, 0, 0, 7987, 0, 0, 0, 15, 26, 0, 1192, 4, 0, 0, 0, 0, 0, 1547, 0, 0, 532, 1, 10, 0, 0, 0, 0, 241, 0, 0, 11, 830, 0, 0, 0, 0, 7576, 5, 0, 13, 8, 6, 0, 1, 1, 1, 8001, 0, 72, 87, 0, 4, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1, 9, 0, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 10, 0, 0, 0, 0, 0, 0, 8, 0, 0, 15, 5455, 0, 5, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 34, 0, 2, 0, 24, 0, 3, 2, 0, 6036, 0, 3, 0, 0, 8156, 0, 4149, 0, 0, 1, 1, 0, 0, 3, 0, 0, 0, 0, 0, 25, 6949, 1067, 0, 8, 0, 7667, 1, 0, 0, 24, 0, 216, 1, 0, 0, 0, 32, 0, 0, 0, 0, 2, 0, 525, 0, 1, 88, 441, 8144, 46, 0, 0, 0, 7764, 0, 114, 5, 3, 27, 0, 0, 229, 16, 6842, 1, 0, 480, 451, 9, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 7860, 0, 4, 0, 0, 12, 10, 0, 0, 0, 4, 9, 0, 0, 2, 58, 0, 37, 0, 0, 0, 1, 20, 1257, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 204, 0, 0, 128, 0, 0, 4, 19, 0, 730, 34, 0, 0, 0, 0, 0, 6296, 0, 6372, 0, 0, 0, 0, 6986, 0, 7819, 0, 0, 10, 0, 0, 1, 0, 3, 2, 3, 0, 0, 0, 6768, 769, 8, 39, 0, 1237, 0, 0, 0, 0, 0, 0, 0, 0, 106, 0, 0, 0, 0, 1, 0, 2152, 2, 0, 0, 0, 16, 2, 1146, 0, 0, 0, 37, 7240, 0, 5, 0, 0, 7, 1, 5, 0, 273, 7826, 0, 0, 0, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 7894, 0, 0, 0, 170, 2, 0, 0, 0, 1, 11, 0, 0, 0, 0, 16, 0, 0, 0, 6, 1, 0, 0, 0, 1340, 0, 19, 2, 0, 0, 8, 0, 0, 10, 31, 8, 411, 0, 0, 0, 0, 250, 0, 214, 0, 0, 0, 0, 0, 0, 520, 0, 4890, 0, 0, 0, 5, 7328, 0, 2037, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 5, 0, 0, 8191, 11, 0, 0, 0, 7945, 0, 0, 0, 0, 0, 0, 9, 0, 985, 1, 1, 0, 0, 0, 0, 7, 0, 0, 0, 0, 196, 970, 6255, 0, 0, 0, 0, 7676, 0, 16, 72, 110, 0, 14, 1, 0, 15, 7868, 0, 0, 0, 0, 9, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 3532, 0, 5, 0, 0, 0, 15, 19, 9, 8, 0, 0, 0, 64, 0, 0, 0, 0, 238, 0, 0, 0, 0, 0, 7848, 0, 0, 0, 0, 2, 151, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 6, 0, 6, 0, 0, 0, 410, 10, 1889, 1, 0, 0, 0, 8104, 0, 3457, 0, 0, 0, 0, 0, 0, 0, 0, 2, 177, 0, 0, 0, 7238, 76, 0, 0, 0, 7989, 93, 0, 0, 0, 0, 0, 0, 0, 549, 5, 0, 15, 0, 0, 0, 0, 3, 0, 2, 0, 0, 218, 8147, 55, 0, 0, 41, 6381, 169, 15, 4, 91, 0, 14, 0, 0, 570, 7894, 0, 3, 0, 0, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 9, 0, 14, 7680, 66, 0, 0, 33, 0, 2, 0, 38, 42, 0, 0, 0, 0, 0, 24, 33, 0, 0, 0, 0, 4, 0, 0, 976, 25, 891, 0, 0, 0, 0, 0, 0, 11, 13, 1, 113, 0, 0, 0, 0, 0, 0, 9, 0, 0, 6, 0, 0, 1, 1, 0, 7997, 0, 0, 0, 2, 6866, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 275, 0, 6, 8103, 0, 0, 0, 2, 8149, 0, 0, 0, 0, 0, 0, 7, 0, 275, 0, 0, 0, 0, 0, 1, 20, 25, 1, 4, 0, 3, 0, 471, 1, 0, 0, 0, 8124, 0, 0, 69, 0, 0, 1, 1, 2, 0, 6932, 0, 0, 45, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1, 9, 0, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 10, 0, 0, 0, 0, 0, 0, 8, 0, 0, 15, 5455, 0, 5, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 34, 0, 2, 0, 24, 0, 3, 2, 0, 6036, 0, 3, 0, 0, 8156, 0, 4149, 0, 0, 1, 1, 0, 0, 3, 0, 0, 0, 0, 0, 25, 6949, 1067, 0, 8, 0, 7667, 1, 0, 0, 24, 0, 216, 1, 0, 0, 0, 32, 0, 0, 0, 0, 2, 0, 525, 0, 1, 88, 441, 8144, 46, 0, 0, 0, 7764, 0, 114, 5, 3, 27, 0, 0, 229, 16, 6842, 1, 0, 480, 451, 9, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 7, 0, 0, 0, 7195, 0, 1, 39, 35, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 180, 0, 8, 6, 1, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 833, 0, 0, 0, 0, 0, 0, 0, 44, 0, 1, 0, 0, 0, 0, 0, 36, 0, 8074, 0, 167, 1, 0, 4644, 0, 612, 0, 0, 0, 90, 0, 0, 0, 0, 0, 49, 0, 1, 0, 7444, 726, 0, 59, 51, 30, 0, 5, 0, 1, 0, 451, 0, 54, 46, 0, 5, 0, 140, 0, 0, 1958, 0, 36, 0, 0, 0, 6479, 5546, 1, 0, 1, 0, 7698, 393, 1, 3835, 0, 0, 0, 0, 0, 61, 8147, 1, 0, 258, 57, 21, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 12, 5, 0, 5744, 0, 22, 23, 57, 2, 259, 25, 4, 0, 41, 1489, 420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 190, 0, 0, 249, 11, 0, 0, 1144, 0, 495, 1, 673, 278, 0, 4648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 277, 0, 7, 0, 0, 2, 3, 0, 0, 0, 147, 7188, 1119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 4951, 0, 4422, 0, 3, 0, 0, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 241, 0, 0, 0, 0, 0, 0, 239, 0, 0, 0, 0, 0, 0, 3346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3116, 2, 743, 0, 0, 1, 3, 20, 0, 4416, 8, 0, 246, 1965, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 224, 715, 17, 2697, 0, 3997, 0, 0, 2, 3, 1707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 146, 0, 0, 31, 0, 0, 7097, 0, 45, 18, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8106, 7692, 8745, 9889, 3826, 10571, 9365, 7342] avg=8192.0000 max=10571 max_rank=5 min=3826 min_rank=4 imbalance_ratio=1.290405 max_load_reduction=54965 max_load_reduction_pct=0.838699 moved_to_echo=31104 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 9, 0, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1522, 382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 10, 0, 0, 0, 0, 0, 0, 8, 0, 0, 15, 5455, 0, 5, 4796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 7616, 1154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 2, 0, 24, 0, 3, 2, 0, 6036, 0, 3, 0, 0, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4149, 0, 0, 1, 1, 0, 0, 3, 0, 0, 0, 0, 0, 25, 3419, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 7667, 1, 0, 0, 24, 0, 216, 1, 0, 0, 0, 32, 0, 2008, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 525, 0, 1, 88, 441, 8144, 46, 0, 0, 0, 1762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 5, 3, 27, 0, 0, 229, 16, 6460, 1, 0, 480, 451, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2343, 10292, 8779, 6644, 8665, 10009, 11009, 7795] avg=8192.0000 max=11009 max_rank=6 min=2343 min_rank=0 imbalance_ratio=1.343872 max_load_reduction=54527 max_load_reduction_pct=0.832016 moved_to_echo=37478 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 3532, 0, 5, 0, 0, 0, 15, 19, 9, 8, 0, 1585, 441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 238, 0, 0, 0, 0, 0, 7848, 0, 0, 3942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 151, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 7568, 949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 6, 0, 0, 0, 410, 10, 1889, 1, 0, 0, 0, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3457, 0, 0, 0, 0, 0, 0, 0, 0, 2, 177, 0, 0, 0, 3561, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7989, 93, 0, 0, 0, 0, 0, 0, 0, 549, 5, 0, 15, 2092, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 0, 218, 8147, 55, 0, 0, 41, 1448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 15, 4, 91, 0, 14, 0, 0, 570, 7453, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5615, 12092, 8677, 2858, 7273, 10785, 9914, 8322] avg=8192.0000 max=12092 max_rank=1 min=2858 min_rank=3 imbalance_ratio=1.476074 max_load_reduction=53444 max_load_reduction_pct=0.815491 moved_to_echo=36294 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 7860, 0, 4, 0, 0, 12, 10, 0, 0, 0, 4, 1482, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 2, 58, 0, 37, 0, 0, 0, 1, 20, 1257, 0, 0, 0, 4472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 204, 0, 0, 128, 0, 0, 4, 19, 0, 6524, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 730, 34, 0, 0, 0, 0, 0, 6296, 0, 6372, 0, 0, 0, 0, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7819, 0, 0, 10, 0, 0, 1, 0, 3, 2, 3, 0, 0, 0, 3330, 769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 39, 0, 1237, 0, 0, 0, 0, 0, 0, 0, 0, 106, 0, 0, 0, 1956, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2152, 2, 0, 0, 0, 16, 2, 1146, 0, 0, 0, 37, 1643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 7, 1, 5, 0, 273, 7389, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9810, 5856, 7964, 13894, 11937, 3394, 4999, 7682] avg=8192.0000 max=13894 max_rank=3 min=3394 min_rank=5 imbalance_ratio=1.696045 max_load_reduction=51642 max_load_reduction_pct=0.787994 moved_to_echo=28012 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 1, 7499, 8, 0, 0, 0, 0, 0, 0, 66, 159, 0, 1750, 447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 289, 0, 0, 0, 0, 0, 2, 0, 6428, 69, 0, 4680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 2, 3, 0, 0, 0, 0, 2, 0, 57, 7330, 1126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 244, 29, 17, 0, 0, 0, 338, 0, 7926, 0, 0, 0, 0, 520, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6153, 1, 0, 0, 77, 0, 0, 0, 6, 15, 0, 8, 0, 0, 3929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 26, 0, 1192, 4, 0, 0, 0, 0, 0, 1547, 0, 0, 532, 2308, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 10, 0, 0, 0, 0, 241, 0, 0, 11, 830, 0, 0, 0, 0, 1720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 13, 8, 6, 0, 1, 1, 1, 7554, 0, 72, 87, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9931, 11549, 8543, 9084, 10189, 5674, 2813, 7753] avg=8192.0000 max=11549 max_rank=1 min=2813 min_rank=6 imbalance_ratio=1.409790 max_load_reduction=53987 max_load_reduction_pct=0.823776 moved_to_echo=26429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 7894, 0, 0, 0, 170, 2, 0, 0, 0, 1, 11, 1799, 445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 6, 1, 0, 0, 0, 1340, 0, 19, 4745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8, 0, 0, 10, 31, 8, 411, 0, 0, 0, 0, 250, 0, 6848, 1145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 214, 0, 0, 0, 0, 0, 0, 520, 0, 4890, 0, 0, 0, 5, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2037, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 5, 0, 0, 4019, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7945, 0, 0, 0, 0, 0, 0, 9, 0, 985, 1, 1, 0, 2373, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 196, 970, 6255, 0, 0, 0, 0, 1730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 72, 110, 0, 14, 1, 0, 15, 7423, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10322, 6127, 8713, 6109, 6077, 11370, 9158, 7660] avg=8192.0000 max=11370 max_rank=5 min=6077 min_rank=4 imbalance_ratio=1.387939 max_load_reduction=54166 max_load_reduction_pct=0.826508 moved_to_echo=34265 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 9, 0, 14, 7680, 66, 0, 0, 33, 0, 2, 0, 38, 42, 0, 1775, 387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 33, 0, 0, 0, 0, 4, 0, 0, 976, 25, 891, 5018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 13, 1, 113, 0, 0, 0, 0, 0, 0, 6411, 1208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 6, 0, 0, 1, 1, 0, 7997, 0, 0, 0, 2, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 275, 0, 6, 3986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8149, 0, 0, 0, 0, 0, 0, 7, 0, 275, 0, 0, 0, 2342, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 20, 25, 1, 4, 0, 3, 0, 471, 1, 0, 0, 0, 1844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 1, 1, 2, 0, 6545, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10046, 6971, 7757, 8471, 12429, 10829, 2370, 6663] avg=8192.0000 max=12429 max_rank=4 min=2370 min_rank=6 imbalance_ratio=1.517212 max_load_reduction=53107 max_load_reduction_pct=0.810349 moved_to_echo=32291 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 7860, 0, 4, 0, 0, 12, 10, 0, 0, 0, 4, 1482, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 2, 58, 0, 37, 0, 0, 0, 1, 20, 1257, 0, 0, 0, 4472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 204, 0, 0, 128, 0, 0, 4, 19, 0, 6524, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 730, 34, 0, 0, 0, 0, 0, 6296, 0, 6372, 0, 0, 0, 0, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7819, 0, 0, 10, 0, 0, 1, 0, 3, 2, 3, 0, 0, 0, 3330, 769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 39, 0, 1237, 0, 0, 0, 0, 0, 0, 0, 0, 106, 0, 0, 0, 1956, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2152, 2, 0, 0, 0, 16, 2, 1146, 0, 0, 0, 37, 1643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 7, 1, 5, 0, 273, 7389, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9810, 5856, 7964, 13894, 11937, 3394, 4999, 7682] avg=8192.0000 max=13894 max_rank=3 min=3394 min_rank=5 imbalance_ratio=1.696045 max_load_reduction=51642 max_load_reduction_pct=0.787994 moved_to_echo=28012 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 3532, 0, 5, 0, 0, 0, 15, 19, 9, 8, 0, 1585, 441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 238, 0, 0, 0, 0, 0, 7848, 0, 0, 3942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 151, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 7568, 949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 6, 0, 0, 0, 410, 10, 1889, 1, 0, 0, 0, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3457, 0, 0, 0, 0, 0, 0, 0, 0, 2, 177, 0, 0, 0, 3561, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7989, 93, 0, 0, 0, 0, 0, 0, 0, 549, 5, 0, 15, 2092, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 0, 218, 8147, 55, 0, 0, 41, 1448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 15, 4, 91, 0, 14, 0, 0, 570, 7453, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5615, 12092, 8677, 2858, 7273, 10785, 9914, 8322] avg=8192.0000 max=12092 max_rank=1 min=2858 min_rank=3 imbalance_ratio=1.476074 max_load_reduction=53444 max_load_reduction_pct=0.815491 moved_to_echo=36294 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 0, 0, 0, 7195, 0, 1, 39, 35, 0, 0, 0, 0, 0, 0, 1631, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 180, 0, 8, 6, 1, 0, 0, 6, 0, 0, 4755, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 833, 0, 0, 0, 0, 0, 0, 0, 44, 4336, 1145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 36, 0, 8074, 0, 167, 1, 0, 308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 612, 0, 0, 0, 90, 0, 0, 0, 0, 0, 49, 0, 1, 0, 3662, 726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 51, 30, 0, 5, 0, 1, 0, 451, 0, 54, 46, 0, 5, 0, 2151, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 0, 1958, 0, 36, 0, 0, 0, 6479, 5546, 1, 0, 1, 0, 1747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 393, 1, 3835, 0, 0, 0, 0, 0, 61, 7692, 1, 0, 258, 57, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9363, 4957, 6358, 8587, 5140, 2904, 15908, 12319] avg=8192.0000 max=15908 max_rank=6 min=2904 min_rank=5 imbalance_ratio=1.941895 max_load_reduction=49628 max_load_reduction_pct=0.757263 moved_to_echo=36271 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 7894, 0, 0, 0, 170, 2, 0, 0, 0, 1, 11, 1799, 445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 6, 1, 0, 0, 0, 1340, 0, 19, 4745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8, 0, 0, 10, 31, 8, 411, 0, 0, 0, 0, 250, 0, 6848, 1145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 214, 0, 0, 0, 0, 0, 0, 520, 0, 4890, 0, 0, 0, 5, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2037, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 5, 0, 0, 4019, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7945, 0, 0, 0, 0, 0, 0, 9, 0, 985, 1, 1, 0, 2373, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 196, 970, 6255, 0, 0, 0, 0, 1730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 72, 110, 0, 14, 1, 0, 15, 7423, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10322, 6127, 8713, 6109, 6077, 11370, 9158, 7660] avg=8192.0000 max=11370 max_rank=5 min=6077 min_rank=4 imbalance_ratio=1.387939 max_load_reduction=54166 max_load_reduction_pct=0.826508 moved_to_echo=34265 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 9, 0, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1522, 382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 10, 0, 0, 0, 0, 0, 0, 8, 0, 0, 15, 5455, 0, 5, 4796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 7616, 1154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 2, 0, 24, 0, 3, 2, 0, 6036, 0, 3, 0, 0, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4149, 0, 0, 1, 1, 0, 0, 3, 0, 0, 0, 0, 0, 25, 3419, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 7667, 1, 0, 0, 24, 0, 216, 1, 0, 0, 0, 32, 0, 2008, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 525, 0, 1, 88, 441, 8144, 46, 0, 0, 0, 1762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 5, 3, 27, 0, 0, 229, 16, 6460, 1, 0, 480, 451, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2343, 10292, 8779, 6644, 8665, 10009, 11009, 7795] avg=8192.0000 max=11009 max_rank=6 min=2343 min_rank=0 imbalance_ratio=1.343872 max_load_reduction=54527 max_load_reduction_pct=0.832016 moved_to_echo=37478 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[256, 128, 128, 4224, 128, 896, 1024, 128, 3072, 8192, 34560, 128, 0, 128, 128, 13696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[32640, 128, 0, 128, 256, 0, 128, 256, 128, 128, 256, 384, 128, 128, 29312, 2688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 36224, 128, 2048, 128, 128, 128, 768, 128, 128, 7936, 128, 128, 896, 17280, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 128, 47872, 128, 128, 128, 384, 128, 384, 128, 128, 256, 128, 13056, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[1024, 384, 128, 128, 128, 128, 128, 12672, 128, 47616, 128, 256, 128, 128, 3840, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=67328 observed_loss=-1792 tokens_per_expert=[128, 128, 128, 256, 128, 384, 128, 1152, 128, 640, 256, 128, 128, 128, 384, 256, 53888, 8960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[640, 256, 4096, 384, 128, 128, 128, 256, 1024, 57728, 128, 128, 896, 512, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 384, 128, 128, 640, 256, 256, 128, 1280, 128, 640, 1280, 22784, 384, 1024, 37120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[640, 256, 4096, 384, 128, 128, 128, 256, 1024, 57728, 128, 128, 896, 512, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=67328 slack_slots=63744 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 384, 128, 128, 640, 256, 256, 128, 1280, 128, 640, 1280, 22784, 384, 1024, 37120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 1541, 0, 0, 7425, 5180, 0, 0, 0, 0, 0, 0, 864, 0, 5, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 2514, 0, 0, 0, 0, 0, 0, 7, 1463, 3, 299, 0, 0, 0, 0, 0, 0, 0, 6, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 0, 0, 0, 7639, 5, 62, 0, 0, 1, 7493, 0, 0, 0, 0, 238, 6803, 1, 0, 180, 0, 0, 0, 8187, 8, 0, 3, 0, 0, 0, 258, 283, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6573, 2, 150, 65, 0, 0, 0, 32, 0, 61] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[681, 20, 0, 10, 0, 1979, 0, 7, 0, 0, 206, 8168, 0, 11, 0, 0, 0, 123, 0, 6, 8040, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 7321, 0, 0, 0, 0, 0, 120, 0, 0, 1201, 0, 0, 0, 0, 71, 7, 3, 0, 0, 746, 0, 212, 0, 1233, 137, 0, 0, 4, 0, 3045, 6206, 1, 0, 7, 0, 58, 5444, 0, 76, 2, 0, 0, 0, 8172, 7, 0, 0, 3, 0, 0, 3943, 0, 0, 0, 5, 0, 2, 731, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 7, 0, 7482, 0, 0, 0, 0, 0, 19, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[155, 0, 0, 0, 0, 184, 0, 1372, 0, 27, 0, 8095, 0, 4, 0, 0, 0, 0, 53, 229, 1776, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 83, 0, 0, 4, 0, 0, 0, 767, 7463, 0, 0, 0, 48, 0, 0, 0, 0, 0, 7950, 0, 0, 123, 1, 0, 0, 0, 5, 0, 0, 0, 0, 0, 13, 0, 0, 431, 6936, 2637, 0, 0, 5, 0, 8136, 0, 2, 922, 0, 0, 1918, 2, 0, 0, 0, 0, 0, 6240, 0, 1, 0, 0, 2567, 438, 11, 0, 0, 0, 0, 0, 190, 28, 0, 0, 0, 0, 4, 0, 33, 0, 0, 0, 0, 0, 2, 0, 0, 3739, 0, 2938, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 0, 4, 0, 45, 0, 6011, 0, 35, 0, 0, 5, 0, 0, 44, 0, 1, 3080, 39, 0, 2, 0, 0, 0, 0, 37, 3, 71, 11, 0, 0, 0, 0, 0, 14, 9, 7494, 170, 0, 23, 2370, 3, 0, 10, 10, 0, 616, 0, 43, 0, 0, 0, 128, 3, 0, 0, 0, 0, 0, 3642, 0, 0, 0, 1625, 5603, 53, 670, 0, 129, 419, 5015, 0, 0, 35, 0, 0, 4302, 0, 0, 1, 2, 1, 597, 4368, 0, 0, 0, 0, 0, 0, 7941, 0, 0, 35, 0, 0, 555, 0, 0, 0, 0, 0, 0, 17, 0, 0, 190, 0, 8, 0, 40, 0, 0, 385, 0, 548, 8, 0, 2793, 6235, 0, 1, 0, 0, 35] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[119, 0, 101, 12, 0, 7291, 0, 0, 14, 0, 7, 3150, 0, 0, 1, 0, 0, 2, 1, 0, 1267, 1, 0, 0, 94, 18, 0, 0, 2, 0, 0, 10, 0, 347, 0, 0, 0, 4, 0, 8077, 2, 0, 2, 2, 0, 0, 0, 0, 0, 5215, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 2132, 1, 0, 0, 5, 8153, 0, 0, 55, 0, 0, 732, 0, 462, 954, 0, 0, 32, 0, 0, 0, 306, 2, 0, 5099, 0, 1241, 0, 0, 0, 1, 7178, 0, 0, 0, 0, 28, 3447, 86, 0, 6, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 48, 2, 0, 0, 8155, 0, 0, 1643, 12, 0, 0, 2, 8, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 1541, 0, 0, 7425, 5180, 0, 0, 0, 0, 0, 0, 864, 0, 5, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 2514, 0, 0, 0, 0, 0, 0, 7, 1463, 3, 299, 0, 0, 0, 0, 0, 0, 0, 6, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 0, 0, 0, 7639, 5, 62, 0, 0, 1, 7493, 0, 0, 0, 0, 238, 6803, 1, 0, 180, 0, 0, 0, 8187, 8, 0, 3, 0, 0, 0, 258, 283, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6573, 2, 150, 65, 0, 0, 0, 32, 0, 61] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[155, 0, 0, 0, 0, 184, 0, 1372, 0, 27, 0, 8095, 0, 4, 0, 0, 0, 0, 53, 229, 1776, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 83, 0, 0, 4, 0, 0, 0, 767, 7463, 0, 0, 0, 48, 0, 0, 0, 0, 0, 7950, 0, 0, 123, 1, 0, 0, 0, 5, 0, 0, 0, 0, 0, 13, 0, 0, 431, 6936, 2637, 0, 0, 5, 0, 8136, 0, 2, 922, 0, 0, 1918, 2, 0, 0, 0, 0, 0, 6240, 0, 1, 0, 0, 2567, 438, 11, 0, 0, 0, 0, 0, 190, 28, 0, 0, 0, 0, 4, 0, 33, 0, 0, 0, 0, 0, 2, 0, 0, 3739, 0, 2938, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 1, 5, 0, 0, 0, 2, 0, 0, 464, 40, 7323, 0, 8, 0, 0, 0, 0, 0, 263, 757, 0, 0, 1578, 0, 0, 0, 0, 36, 13, 0, 1945, 0, 4, 9, 0, 0, 1, 7, 23, 0, 0, 0, 7962, 0, 0, 4, 0, 0, 0, 283, 31, 0, 53, 0, 0, 0, 10, 0, 0, 0, 0, 169, 144, 0, 0, 0, 5626, 0, 559, 0, 9, 0, 5873, 0, 0, 0, 0, 0, 7370, 0, 0, 0, 0, 0, 40, 6333, 0, 1, 0, 19, 0, 0, 7992, 0, 0, 0, 0, 0, 0, 0, 0, 1, 67, 0, 0, 0, 0, 4, 0, 613, 0, 0, 0, 0, 2, 0, 5, 7665, 0, 140, 283, 1799, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2278, 0, 5, 0, 19, 0, 2770, 0, 0, 0, 0, 0, 2936, 0, 0, 7656, 0, 0, 1, 0, 2, 0, 0, 21, 0, 7, 1, 0, 0, 0, 2, 0, 0, 0, 8138, 1, 0, 32, 24, 0, 0, 0, 0, 1650, 0, 0, 0, 0, 0, 0, 6, 0, 0, 1, 46, 0, 0, 0, 6, 8, 227, 6106, 0, 0, 9, 0, 0, 63, 1, 0, 0, 426, 21, 0, 6876, 0, 0, 0, 6, 0, 0, 3292, 0, 1, 0, 0, 2158, 0, 8062, 0, 0, 5, 0, 0, 5356, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 10, 0, 4, 2, 0, 75, 55, 0, 6525, 0, 213, 213, 207, 2, 0, 0, 0, 9] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[2, 0, 5, 0, 0, 196, 0, 3, 0, 280, 0, 7861, 0, 0, 0, 787, 0, 0, 2, 0, 3348, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 23, 0, 3, 0, 0, 0, 0, 0, 634, 22, 0, 0, 416, 0, 0, 0, 0, 0, 4118, 1, 0, 1, 0, 169, 0, 0, 0, 72, 0, 0, 0, 8094, 0, 0, 0, 0, 50, 0, 0, 0, 2, 0, 2702, 7, 160, 868, 0, 0, 7147, 0, 0, 0, 0, 0, 0, 7966, 1, 145, 0, 1, 1, 26, 21, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 22, 0, 0, 0, 0, 28, 4, 3730, 0, 8188, 0, 40, 4, 8184, 0, 0, 0, 7, 183] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[119, 0, 101, 12, 0, 7291, 0, 0, 14, 0, 7, 3150, 0, 0, 1, 0, 0, 2, 1, 0, 1267, 1, 0, 0, 94, 18, 0, 0, 2, 0, 0, 10, 0, 347, 0, 0, 0, 4, 0, 8077, 2, 0, 2, 2, 0, 0, 0, 0, 0, 5215, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 2132, 1, 0, 0, 5, 8153, 0, 0, 55, 0, 0, 732, 0, 462, 954, 0, 0, 32, 0, 0, 0, 306, 2, 0, 5099, 0, 1241, 0, 0, 0, 1, 7178, 0, 0, 0, 0, 28, 3447, 86, 0, 6, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 48, 2, 0, 0, 8155, 0, 0, 1643, 12, 0, 0, 2, 8, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2278, 0, 5, 0, 19, 0, 2770, 0, 0, 0, 0, 0, 2936, 0, 0, 7656, 0, 0, 1, 0, 2, 0, 0, 21, 0, 7, 1, 0, 0, 0, 2, 0, 0, 0, 8138, 1, 0, 32, 24, 0, 0, 0, 0, 1650, 0, 0, 0, 0, 0, 0, 6, 0, 0, 1, 46, 0, 0, 0, 6, 8, 227, 6106, 0, 0, 9, 0, 0, 63, 1, 0, 0, 426, 21, 0, 6876, 0, 0, 0, 6, 0, 0, 3292, 0, 1, 0, 0, 2158, 0, 8062, 0, 0, 5, 0, 0, 5356, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 10, 0, 4, 2, 0, 75, 55, 0, 6525, 0, 213, 213, 207, 2, 0, 0, 0, 9] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[681, 20, 0, 10, 0, 1979, 0, 7, 0, 0, 206, 6562, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 0, 6, 8040, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2581, 2225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 7321, 0, 0, 0, 0, 1606, 615, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 0, 0, 1201, 0, 0, 0, 0, 71, 7, 3, 0, 0, 746, 0, 1967, 1410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 0, 1233, 137, 0, 0, 4, 0, 3045, 1658, 1, 0, 7, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 2, 0, 0, 0, 3621, 7, 0, 0, 3, 0, 0, 3943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2, 731, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 5444, 2326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 7, 0, 5457, 0, 0, 0, 0, 0, 19, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9476, 12976, 9553, 5525, 6355, 7652, 8512, 5487] avg=8192.0000 max=12976 max_rank=1 min=5487 min_rank=7 imbalance_ratio=1.583984 max_load_reduction=52560 max_load_reduction_pct=0.802002 moved_to_echo=28006 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 1541, 0, 0, 7425, 4162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 864, 0, 5, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 2514, 3116, 2229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1463, 3, 299, 0, 0, 0, 0, 0, 0, 1018, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 2375, 1239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7639, 5, 62, 0, 0, 1, 2002, 0, 0, 0, 0, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 180, 0, 0, 0, 3628, 8, 0, 3, 0, 0, 0, 258, 283, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 6803, 2330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4794, 2, 150, 65, 0, 0, 0, 32, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13129, 8732, 3330, 11744, 9947, 4363, 9187, 5104] avg=8192.0000 max=13129 max_rank=0 min=3330 min_rank=2 imbalance_ratio=1.602661 max_load_reduction=52407 max_load_reduction_pct=0.799667 moved_to_echo=28601 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2278, 0, 5, 0, 19, 0, 2226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2936, 0, 0, 7656, 0, 0, 1, 0, 2, 0, 0, 21, 0, 7, 1, 0, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8138, 1, 0, 32, 24, 0, 0, 0, 0, 544, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1650, 0, 0, 0, 0, 0, 0, 6, 0, 0, 1, 46, 0, 0, 0, 6, 0, 1230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 227, 6106, 0, 0, 9, 0, 0, 63, 1, 0, 0, 426, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 1459, 0, 1, 0, 0, 2158, 0, 8062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 5356, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 10, 0, 6876, 937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 75, 55, 0, 4759, 0, 213, 213, 207, 2, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4528, 11520, 9277, 2939, 6861, 11686, 13186, 5539] avg=8192.0000 max=13186 max_rank=6 min=2939 min_rank=3 imbalance_ratio=1.609619 max_load_reduction=52350 max_load_reduction_pct=0.798798 moved_to_echo=37272 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[155, 0, 0, 0, 0, 184, 0, 1372, 0, 27, 0, 6503, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 229, 1776, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 83, 3383, 1699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 767, 7463, 0, 0, 0, 48, 0, 0, 0, 0, 1592, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7950, 0, 0, 123, 1, 0, 0, 0, 5, 0, 0, 0, 0, 0, 13, 2579, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 431, 6936, 2637, 0, 0, 5, 0, 2174, 0, 2, 922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2765, 0, 1, 0, 0, 2567, 438, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 190, 28, 0, 0, 0, 0, 4, 0, 33, 0, 0, 0, 0, 1918, 1776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3739, 0, 2144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 7227, 10115, 11224, 13107, 5784, 3949, 5885] avg=8192.0000 max=13107 max_rank=4 min=3949 min_rank=6 imbalance_ratio=1.599976 max_load_reduction=52429 max_load_reduction_pct=0.800003 moved_to_echo=28725 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 0, 4, 0, 45, 0, 6011, 0, 29, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 1, 3080, 39, 0, 2, 0, 0, 0, 0, 37, 3, 71, 11, 2085, 1189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 9, 7494, 170, 0, 23, 2370, 3, 0, 10, 10, 6, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 616, 0, 43, 0, 0, 0, 128, 3, 0, 0, 0, 0, 0, 3642, 0, 1589, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1625, 5603, 53, 670, 0, 129, 419, 1341, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 1, 597, 1936, 0, 0, 0, 0, 0, 0, 7941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 555, 0, 0, 0, 0, 0, 0, 17, 0, 0, 190, 0, 8, 4302, 1243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 385, 0, 400, 8, 0, 2793, 6235, 0, 1, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6096, 6562, 10154, 6124, 9875, 10478, 6350, 9897] avg=8192.0000 max=10478 max_rank=5 min=6096 min_rank=0 imbalance_ratio=1.279053 max_load_reduction=55058 max_load_reduction_pct=0.840118 moved_to_echo=36600 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 5, 0, 0, 196, 0, 3, 0, 280, 0, 6310, 0, 0, 0, 787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3348, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 23, 1127, 2173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 634, 22, 0, 0, 416, 0, 0, 0, 0, 1551, 678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4118, 1, 0, 1, 0, 169, 0, 0, 0, 72, 0, 0, 0, 8094, 0, 860, 1548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 2, 0, 715, 7, 160, 868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3524, 1, 145, 0, 1, 1, 26, 21, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 22, 0, 0, 7147, 2269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 4, 3730, 0, 5962, 0, 40, 4, 8184, 0, 0, 0, 7, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7583, 6674, 3304, 14863, 1802, 3726, 9442, 18142] avg=8192.0000 max=18142 max_rank=7 min=1802 min_rank=4 imbalance_ratio=2.214600 max_load_reduction=47394 max_load_reduction_pct=0.723175 moved_to_echo=33112 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[155, 0, 0, 0, 0, 184, 0, 1372, 0, 27, 0, 6503, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 229, 1776, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 83, 3383, 1699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 767, 7463, 0, 0, 0, 48, 0, 0, 0, 0, 1592, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7950, 0, 0, 123, 1, 0, 0, 0, 5, 0, 0, 0, 0, 0, 13, 2579, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 431, 6936, 2637, 0, 0, 5, 0, 2174, 0, 2, 922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2765, 0, 1, 0, 0, 2567, 438, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 190, 28, 0, 0, 0, 0, 4, 0, 33, 0, 0, 0, 0, 1918, 1776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3739, 0, 2144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 7227, 10115, 11224, 13107, 5784, 3949, 5885] avg=8192.0000 max=13107 max_rank=4 min=3949 min_rank=6 imbalance_ratio=1.599976 max_load_reduction=52429 max_load_reduction_pct=0.800003 moved_to_echo=28725 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 5, 0, 0, 0, 2, 0, 0, 464, 40, 5883, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 263, 757, 0, 0, 1578, 0, 0, 0, 0, 36, 13, 0, 1945, 2442, 1724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 9, 0, 0, 1, 7, 23, 0, 0, 0, 7962, 0, 0, 4, 0, 1440, 630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 283, 31, 0, 53, 0, 0, 0, 10, 0, 0, 0, 0, 169, 144, 1861, 1445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5626, 0, 559, 0, 9, 0, 1570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 2807, 0, 1, 0, 19, 0, 0, 7992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 67, 0, 0, 0, 0, 4, 0, 613, 0, 7370, 1802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5, 5590, 0, 140, 283, 1799, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6403, 8758, 10080, 3996, 7764, 10859, 9857, 7819] avg=8192.0000 max=10859 max_rank=5 min=3996 min_rank=3 imbalance_ratio=1.325562 max_load_reduction=54677 max_load_reduction_pct=0.834305 moved_to_echo=36299 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[119, 0, 101, 12, 0, 7291, 0, 0, 14, 0, 7, 2531, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 1267, 1, 0, 0, 94, 18, 0, 0, 2, 0, 0, 10, 304, 1388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 347, 0, 0, 0, 4, 0, 8077, 2, 0, 2, 2, 0, 0, 0, 0, 619, 670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5215, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 2132, 1, 232, 1537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 8153, 0, 0, 55, 0, 0, 196, 0, 462, 954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 306, 2, 0, 2260, 0, 1241, 0, 0, 0, 1, 7178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 3447, 86, 0, 6, 0, 0, 0, 0, 3, 0, 0, 0, 0, 32, 1451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 2, 0, 0, 5948, 0, 0, 1643, 12, 0, 0, 2, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10076, 3087, 9723, 9121, 9825, 10988, 5053, 7663] avg=8192.0000 max=10988 max_rank=5 min=3087 min_rank=1 imbalance_ratio=1.341309 max_load_reduction=54548 max_load_reduction_pct=0.832336 moved_to_echo=33529 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 5, 0, 0, 196, 0, 3, 0, 280, 0, 6310, 0, 0, 0, 787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3348, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 23, 1127, 2173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 634, 22, 0, 0, 416, 0, 0, 0, 0, 1551, 678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4118, 1, 0, 1, 0, 169, 0, 0, 0, 72, 0, 0, 0, 8094, 0, 860, 1548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 2, 0, 715, 7, 160, 868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3524, 1, 145, 0, 1, 1, 26, 21, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 22, 0, 0, 7147, 2269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 4, 3730, 0, 5962, 0, 40, 4, 8184, 0, 0, 0, 7, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7583, 6674, 3304, 14863, 1802, 3726, 9442, 18142] avg=8192.0000 max=18142 max_rank=7 min=1802 min_rank=4 imbalance_ratio=2.214600 max_load_reduction=47394 max_load_reduction_pct=0.723175 moved_to_echo=33112 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2278, 0, 5, 0, 19, 0, 2226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2936, 0, 0, 7656, 0, 0, 1, 0, 2, 0, 0, 21, 0, 7, 1, 0, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8138, 1, 0, 32, 24, 0, 0, 0, 0, 544, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1650, 0, 0, 0, 0, 0, 0, 6, 0, 0, 1, 46, 0, 0, 0, 6, 0, 1230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 227, 6106, 0, 0, 9, 0, 0, 63, 1, 0, 0, 426, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 1459, 0, 1, 0, 0, 2158, 0, 8062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 5356, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 10, 0, 6876, 937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 75, 55, 0, 4759, 0, 213, 213, 207, 2, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4528, 11520, 9277, 2939, 6861, 11686, 13186, 5539] avg=8192.0000 max=13186 max_rank=6 min=2939 min_rank=3 imbalance_ratio=1.609619 max_load_reduction=52350 max_load_reduction_pct=0.798798 moved_to_echo=37272 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 3200, 1024, 512, 25984, 128, 128, 1664, 128, 128, 128, 128, 128, 128, 128, 4608, 15104, 13568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[1664, 18048, 384, 128, 1408, 128, 256, 256, 128, 128, 128, 128, 0, 0, 22912, 256, 11520, 9088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 7936, 128, 35072, 128, 640, 5120, 16512, 128, 128, 128, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 9600, 896, 0, 128, 128, 128, 128, 128, 128, 128, 256, 640, 128, 39936, 14208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 256, 384, 128, 640, 22016, 128, 1408, 128, 128, 4736, 512, 35456, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[256, 256, 9472, 34176, 2816, 1408, 128, 256, 3584, 9728, 128, 640, 3328, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 384, 128, 128, 0, 128, 896, 33408, 256, 384, 128, 18176, 128, 0, 128, 128, 8448, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[1024, 128, 128, 128, 0, 12032, 128, 3072, 128, 6912, 7680, 34304, 0, 128, 128, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 384, 128, 128, 0, 128, 896, 33408, 256, 384, 128, 18176, 128, 0, 128, 128, 8448, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 11, 0, 0, 0, 0, 0, 12, 1676, 0, 33, 24, 0, 4, 0, 56, 0, 1260, 0, 137, 0, 0, 0, 0, 37, 0, 0, 0, 0, 4326, 0, 0, 6478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 6, 2994, 0, 0, 0, 0, 27, 0, 0, 0, 0, 437, 0, 0, 2, 57, 0, 8045, 0, 0, 0, 0, 717, 0, 0, 0, 1, 5989, 2456, 0, 0, 9, 235, 0, 0, 0, 0, 4258, 0, 0, 0, 0, 27, 1, 0, 0, 0, 7546, 8186, 2, 2, 0, 7, 9, 110, 780, 421, 7650, 0, 0, 0, 1, 0, 0, 0, 0, 10] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 6, 2, 0, 24, 0, 0, 0, 0, 0, 7860, 0, 47, 0, 725, 78, 0, 5008, 0, 278, 0, 0, 1, 0, 77, 49, 0, 1780, 0, 0, 0, 149, 0, 0, 0, 0, 0, 11, 0, 20, 0, 0, 5, 17, 0, 102, 262, 972, 0, 0, 0, 0, 0, 0, 0, 240, 0, 919, 21, 0, 17, 2156, 178, 0, 0, 4, 8, 0, 0, 44, 0, 1668, 0, 0, 6, 0, 0, 7498, 0, 0, 0, 1842, 0, 7870, 1, 0, 0, 924, 0, 40, 0, 0, 0, 424, 88, 5, 29, 0, 0, 0, 0, 0, 6722, 820, 7490, 0, 2, 1, 13, 0, 0, 45, 0, 8172, 1, 1, 0, 736, 0, 0, 78, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3289, 0, 0, 7052, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 21, 7793, 0, 96, 2, 0, 0, 1, 0, 0, 1112, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 1, 0, 0, 0, 73, 0, 0, 168, 0, 4, 0, 0, 5439, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7935, 0, 0, 15, 4, 0, 22, 16, 0, 0, 7172, 404, 0, 1, 38, 0, 0, 0, 0, 6795, 0, 3814, 0, 0, 0, 0, 0, 5787, 0, 540, 7645, 0, 29, 0, 0, 0, 19, 51, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 1074, 0, 0, 2, 0, 0, 0, 0, 29, 0, 0, 0, 0, 2319, 6249, 12, 0, 8163, 0, 12, 7, 6760, 1, 0, 0, 0, 30, 8151, 26, 3, 0, 740, 0, 11, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 3603, 0, 0, 0, 5427, 0, 3, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 2660, 0, 7877, 0, 0, 160, 698, 0, 1, 0, 2, 0, 0, 106, 7, 0, 0, 0, 0, 0, 0, 3, 0, 552, 1090, 232, 0, 9, 0, 206, 1, 0, 8040, 0, 231, 0, 0, 969, 0, 0, 0, 9] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 17, 0, 1, 74, 0, 1, 0, 0, 0, 0, 3098, 0, 0, 0, 10, 0, 0, 0, 501, 132, 0, 0, 0, 7916, 213, 0, 0, 8157, 0, 0, 0, 0, 190, 1321, 0, 325, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 565, 0, 0, 0, 323, 0, 0, 0, 12, 0, 7629, 0, 0, 5, 23, 6774, 0, 0, 0, 111, 2, 0, 0, 657, 22, 0, 2994, 0, 0, 0, 0, 3326, 0, 0, 0, 0, 8168, 0, 234, 0, 268, 0, 13, 0, 1, 0, 13, 22, 1, 0, 0, 147, 2, 0, 0, 0, 64, 7069, 1, 0, 32, 0, 0, 311, 0, 5, 4523, 0, 166, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 1856, 12, 0, 0, 0, 0, 0, 0, 0, 0, 3031, 0, 971, 7222, 3801, 0, 104, 1084, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 1, 0, 187, 0, 1983, 2, 6, 0, 0, 0, 1, 0, 0, 0, 6680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 1265, 0, 0, 0, 7545, 8108, 0, 0, 0, 0, 8187, 0, 79, 0, 0, 5, 0, 0, 1, 1, 5622, 0, 0, 1, 0, 0, 13, 0, 0, 7, 0, 2825, 0, 0, 0, 0, 0, 0, 0, 12, 4517, 0, 0, 0, 38, 9, 0, 0, 0, 328] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 46, 1, 0, 2, 8, 1, 43, 2, 0, 7, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 2, 0, 0, 18, 0, 8128, 0, 0, 5339, 132, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 354, 598, 0, 8061, 0, 102, 191, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 387, 0, 0, 2, 0, 0, 0, 85, 71, 0, 0, 7468, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8041, 0, 0, 0, 1625, 0, 0, 0, 0, 0, 996, 0, 0, 0, 0, 0, 0, 0, 10, 1754, 1564, 803, 0, 0, 2, 0, 0, 5937, 0, 0, 5740, 0, 0, 0, 0, 7957, 0, 0, 0, 11] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 1856, 12, 0, 0, 0, 0, 0, 0, 0, 0, 3031, 0, 971, 7222, 3801, 0, 104, 1084, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 1, 0, 187, 0, 1983, 2, 6, 0, 0, 0, 1, 0, 0, 0, 6680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 1265, 0, 0, 0, 7545, 8108, 0, 0, 0, 0, 8187, 0, 79, 0, 0, 5, 0, 0, 1, 1, 5622, 0, 0, 1, 0, 0, 13, 0, 0, 7, 0, 2825, 0, 0, 0, 0, 0, 0, 0, 12, 4517, 0, 0, 0, 38, 9, 0, 0, 0, 328] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 3110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 82, 0, 933, 0, 0, 8, 0, 0, 0, 0, 0, 4912, 8172, 235, 850, 0, 0, 0, 5086, 0, 3, 13, 0, 0, 0, 160, 0, 0, 0, 0, 6, 0, 0, 312, 0, 238, 0, 0, 0, 0, 0, 0, 0, 444, 0, 3313, 0, 0, 0, 11, 529, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 7327, 0, 0, 5317, 0, 0, 0, 0, 0, 29, 8087, 0, 0, 0, 0, 0, 1132, 0, 0, 7641, 0, 0, 2271, 0, 21, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 1074, 0, 0, 2, 0, 0, 0, 0, 29, 0, 0, 0, 0, 2319, 6249, 12, 0, 8163, 0, 12, 7, 6760, 1, 0, 0, 0, 30, 8151, 26, 3, 0, 740, 0, 11, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 3603, 0, 0, 0, 5427, 0, 3, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 2660, 0, 7877, 0, 0, 160, 698, 0, 1, 0, 2, 0, 0, 106, 7, 0, 0, 0, 0, 0, 0, 3, 0, 552, 1090, 232, 0, 9, 0, 206, 1, 0, 8040, 0, 231, 0, 0, 969, 0, 0, 0, 9] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 11, 0, 0, 0, 0, 0, 12, 1676, 0, 33, 24, 0, 4, 0, 56, 3257, 1205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1260, 0, 137, 0, 0, 0, 0, 37, 0, 0, 0, 0, 4208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 6, 2994, 0, 5354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 437, 0, 0, 2, 57, 0, 8045, 0, 2367, 508, 118, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 717, 0, 0, 0, 1, 2732, 2456, 0, 0, 9, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4258, 0, 0, 0, 0, 27, 1, 0, 0, 0, 7546, 1441, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 9, 110, 780, 421, 5283, 0, 0, 0, 1, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6278, 5642, 7465, 8356, 11747, 6150, 13277, 6621] avg=8192.0000 max=13277 max_rank=6 min=5642 min_rank=1 imbalance_ratio=1.620728 max_load_reduction=52259 max_load_reduction_pct=0.797409 moved_to_echo=37795 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 6, 2, 0, 24, 0, 0, 0, 0, 0, 7860, 0, 47, 0, 725, 78, 0, 5008, 0, 278, 0, 0, 1, 0, 77, 49, 0, 1780, 0, 0, 0, 149, 0, 0, 0, 0, 0, 11, 0, 20, 0, 0, 5, 17, 0, 102, 262, 972, 0, 0, 0, 0, 0, 0, 0, 240, 0, 919, 21, 0, 17, 2156, 178, 0, 0, 4, 8, 0, 0, 44, 0, 1668, 0, 0, 6, 0, 0, 7498, 0, 0, 0, 1842, 0, 7870, 1, 0, 0, 924, 0, 40, 0, 0, 0, 424, 88, 5, 29, 0, 0, 0, 0, 0, 6722, 820, 7490, 0, 2, 1, 13, 0, 0, 45, 0, 8172, 1, 1, 0, 736, 0, 0, 78, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 11, 0, 0, 0, 0, 0, 12, 1676, 0, 33, 24, 0, 4, 0, 56, 0, 1260, 0, 137, 0, 0, 0, 0, 37, 0, 0, 0, 0, 4326, 0, 0, 6478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 6, 2994, 0, 0, 0, 0, 27, 0, 0, 0, 0, 437, 0, 0, 2, 57, 0, 8045, 0, 0, 0, 0, 717, 0, 0, 0, 1, 5989, 2456, 0, 0, 9, 235, 0, 0, 0, 0, 4258, 0, 0, 0, 0, 27, 1, 0, 0, 0, 7546, 8186, 2, 2, 0, 7, 9, 110, 780, 421, 7650, 0, 0, 0, 1, 0, 0, 0, 0, 10] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3289, 0, 0, 7052, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 21, 7793, 0, 96, 2, 0, 0, 1, 0, 0, 1112, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 1, 0, 0, 0, 73, 0, 0, 168, 0, 4, 0, 0, 5439, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7935, 0, 0, 15, 4, 0, 22, 16, 0, 0, 7172, 404, 0, 1, 38, 0, 0, 0, 0, 6795, 0, 3814, 0, 0, 0, 0, 0, 5787, 0, 540, 7645, 0, 29, 0, 0, 0, 19, 51, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 1074, 4283, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 29, 0, 0, 0, 0, 2319, 6078, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7525, 0, 12, 7, 6760, 1, 0, 0, 0, 30, 8151, 26, 3, 0, 740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 3603, 0, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5427, 0, 3, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 2, 2488, 638, 171, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2660, 0, 3594, 0, 0, 160, 698, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 106, 7, 0, 0, 0, 0, 0, 0, 3, 0, 98, 1090, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 206, 1, 0, 5552, 0, 231, 0, 0, 969, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5441, 8440, 23255, 3985, 8786, 7114, 1538, 6977] avg=8192.0000 max=23255 max_rank=2 min=1538 min_rank=6 imbalance_ratio=2.838745 max_load_reduction=42281 max_load_reduction_pct=0.645157 moved_to_echo=24415 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 6, 2, 0, 24, 0, 0, 0, 0, 0, 7860, 4280, 1102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 725, 78, 0, 5008, 0, 278, 0, 0, 1, 0, 75, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1641, 0, 0, 0, 149, 0, 0, 0, 0, 0, 11, 0, 20, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 102, 262, 972, 0, 0, 0, 0, 0, 0, 0, 240, 0, 919, 21, 4899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 2156, 178, 0, 0, 4, 8, 0, 0, 44, 0, 1668, 0, 0, 6, 2528, 139, 2, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7498, 0, 0, 0, 1842, 0, 3590, 1, 0, 0, 924, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 424, 88, 5, 29, 0, 0, 0, 0, 0, 6722, 820, 1319, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 13, 0, 0, 45, 0, 5644, 1, 1, 0, 736, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13274, 6261, 1826, 7432, 6920, 13895, 9409, 6519] avg=8192.0000 max=13895 max_rank=5 min=1826 min_rank=2 imbalance_ratio=1.696167 max_load_reduction=51641 max_load_reduction_pct=0.787979 moved_to_echo=36743 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 17, 0, 1, 74, 0, 1, 0, 0, 0, 0, 3098, 4442, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 501, 132, 0, 0, 0, 7916, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7520, 0, 0, 0, 0, 190, 1321, 0, 325, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 565, 0, 0, 0, 323, 0, 0, 0, 12, 0, 7629, 0, 4623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 23, 6774, 0, 0, 0, 111, 2, 0, 0, 657, 22, 0, 2994, 0, 1399, 637, 5, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3326, 0, 0, 0, 0, 3726, 0, 234, 0, 268, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 13, 22, 1, 0, 0, 147, 2, 0, 0, 0, 64, 1246, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 311, 0, 5, 3124, 0, 166, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8673, 8767, 9452, 13152, 12789, 7567, 1497, 3639] avg=8192.0000 max=13152 max_rank=3 min=1497 min_rank=6 imbalance_ratio=1.605469 max_load_reduction=52384 max_load_reduction_pct=0.799316 moved_to_echo=25492 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 1074, 4283, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 29, 0, 0, 0, 0, 2319, 6078, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7525, 0, 12, 7, 6760, 1, 0, 0, 0, 30, 8151, 26, 3, 0, 740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 3603, 0, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5427, 0, 3, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 2, 2488, 638, 171, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2660, 0, 3594, 0, 0, 160, 698, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 106, 7, 0, 0, 0, 0, 0, 0, 3, 0, 98, 1090, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 206, 1, 0, 5552, 0, 231, 0, 0, 969, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5441, 8440, 23255, 3985, 8786, 7114, 1538, 6977] avg=8192.0000 max=23255 max_rank=2 min=1538 min_rank=6 imbalance_ratio=2.838745 max_load_reduction=42281 max_load_reduction_pct=0.645157 moved_to_echo=24415 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 46, 1, 0, 2, 8, 1, 43, 2, 0, 7, 1, 0, 4373, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 2, 0, 0, 18, 0, 7905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4922, 132, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 598, 0, 8061, 0, 102, 191, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 387, 0, 0, 2, 0, 0, 0, 85, 71, 0, 0, 7468, 0, 1776, 417, 223, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3668, 0, 0, 0, 1625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 996, 0, 0, 0, 0, 0, 0, 0, 10, 1754, 1564, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 5937, 0, 0, 3964, 0, 0, 0, 0, 7957, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4603, 7928, 5445, 9481, 10447, 5295, 4466, 17871] avg=8192.0000 max=17871 max_rank=7 min=4466 min_rank=6 imbalance_ratio=2.181519 max_load_reduction=47665 max_load_reduction_pct=0.727310 moved_to_echo=38079 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3289, 0, 0, 7052, 4315, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 21, 7580, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 1112, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 1, 0, 0, 0, 73, 0, 0, 168, 0, 2494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 5439, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2365, 0, 213, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3620, 0, 0, 15, 4, 0, 22, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7172, 404, 0, 1, 38, 0, 0, 0, 0, 6795, 0, 673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5787, 0, 540, 5280, 0, 29, 0, 0, 0, 19, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15228, 7747, 1164, 2822, 8108, 3678, 15083, 11706] avg=8192.0000 max=15228 max_rank=0 min=1164 min_rank=2 imbalance_ratio=1.858887 max_load_reduction=50308 max_load_reduction_pct=0.767639 moved_to_echo=38575 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4452, 415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1856, 12, 0, 0, 0, 0, 0, 0, 0, 0, 2948, 0, 971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6658, 3801, 0, 104, 1084, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 187, 0, 1983, 2, 6, 0, 0, 0, 1, 0, 0, 0, 6680, 0, 1847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 1265, 0, 1397, 564, 83, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7545, 8108, 0, 0, 0, 0, 3735, 0, 79, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 5622, 0, 0, 1, 0, 0, 13, 0, 0, 7, 0, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 3120, 0, 0, 0, 38, 9, 0, 0, 0, 328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4875, 5787, 11661, 10707, 3383, 19472, 6144, 3507] avg=8192.0000 max=19472 max_rank=5 min=3383 min_rank=4 imbalance_ratio=2.376953 max_load_reduction=46064 max_load_reduction_pct=0.702881 moved_to_echo=32506 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 2825, 1195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 933, 0, 0, 8, 0, 0, 0, 0, 0, 4912, 7944, 235, 850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5086, 0, 3, 13, 0, 0, 0, 160, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 312, 0, 238, 0, 0, 0, 0, 0, 0, 0, 444, 0, 3313, 0, 5294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 529, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2369, 0, 228, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 7327, 0, 0, 5317, 0, 0, 0, 0, 0, 29, 1410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1132, 0, 0, 5272, 0, 0, 2271, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7164, 14964, 5268, 9601, 3356, 2365, 14122, 8696] avg=8192.0000 max=14964 max_rank=1 min=2365 min_rank=5 imbalance_ratio=1.826660 max_load_reduction=50572 max_load_reduction_pct=0.771667 moved_to_echo=28539 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4452, 415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1856, 12, 0, 0, 0, 0, 0, 0, 0, 0, 2948, 0, 971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6658, 3801, 0, 104, 1084, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 187, 0, 1983, 2, 6, 0, 0, 0, 1, 0, 0, 0, 6680, 0, 1847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 1265, 0, 1397, 564, 83, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7545, 8108, 0, 0, 0, 0, 3735, 0, 79, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 5622, 0, 0, 1, 0, 0, 13, 0, 0, 7, 0, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 3120, 0, 0, 0, 38, 9, 0, 0, 0, 328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4875, 5787, 11661, 10707, 3383, 19472, 6144, 3507] avg=8192.0000 max=19472 max_rank=5 min=3383 min_rank=4 imbalance_ratio=2.376953 max_load_reduction=46064 max_load_reduction_pct=0.702881 moved_to_echo=32506 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3289, 0, 0, 7052, 4315, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 21, 7580, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 1112, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 1, 0, 0, 0, 73, 0, 0, 168, 0, 2494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 5439, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2365, 0, 213, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3620, 0, 0, 15, 4, 0, 22, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7172, 404, 0, 1, 38, 0, 0, 0, 0, 6795, 0, 673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5787, 0, 540, 5280, 0, 29, 0, 0, 0, 19, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15228, 7747, 1164, 2822, 8108, 3678, 15083, 11706] avg=8192.0000 max=15228 max_rank=0 min=1164 min_rank=2 imbalance_ratio=1.858887 max_load_reduction=50308 max_load_reduction_pct=0.767639 moved_to_echo=38575 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 11, 0, 0, 0, 0, 0, 12, 1676, 0, 33, 24, 0, 4, 0, 56, 3257, 1205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1260, 0, 137, 0, 0, 0, 0, 37, 0, 0, 0, 0, 4208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 6, 2994, 0, 5354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 437, 0, 0, 2, 57, 0, 8045, 0, 2367, 508, 118, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 717, 0, 0, 0, 1, 2732, 2456, 0, 0, 9, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4258, 0, 0, 0, 0, 27, 1, 0, 0, 0, 7546, 1441, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 9, 110, 780, 421, 5283, 0, 0, 0, 1, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6278, 5642, 7465, 8356, 11747, 6150, 13277, 6621] avg=8192.0000 max=13277 max_rank=6 min=5642 min_rank=1 imbalance_ratio=1.620728 max_load_reduction=52259 max_load_reduction_pct=0.797409 moved_to_echo=37795 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[34304, 3968, 128, 5248, 8064, 256, 2560, 0, 384, 128, 8448, 128, 128, 1536, 896, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 18560, 8064, 128, 128, 5376, 256, 128, 0, 128, 15360, 10112, 6912, 1152, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 15104, 12160, 0, 128, 4608, 128, 27136, 2560, 384, 256, 3584, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] @@ -2042,237 +2044,237 @@ WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[640, 0, 8704, 384, 3968, 384, 128, 128, 384, 128, 128, 128, 768, 128, 25344, 128, 25472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 1408, 128, 3712, 128, 0, 5120, 640, 512, 0, 0, 128, 15232, 36992, 384, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 57, 0, 0, 1413, 0, 0, 0, 2721, 0, 10, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 0, 0, 0, 0, 1, 7, 0, 81, 0, 0, 0, 4, 0, 0, 131, 0, 0, 0, 0, 0, 0, 458, 5064, 26, 0, 0, 0, 0, 0, 0, 7996, 0, 5, 144, 0, 0, 0, 0, 0, 0, 0, 2, 0, 19, 0, 864, 7279, 0, 0, 8181, 0, 0, 0, 873, 6827, 0, 235, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 3, 0, 0, 0, 7267, 2870, 0, 2, 98, 8154, 0, 0, 8, 3443, 0, 0, 0, 6, 0, 0, 127, 0, 0, 2, 0, 0, 0, 0, 0, 3, 0, 548, 0, 369] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 374, 0, 122, 0, 0, 7249, 0, 0, 0, 0, 0, 960, 0, 133, 1, 0, 0, 1, 0, 13, 3, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 39, 2, 6005, 0, 13, 17, 539, 0, 0, 0, 4, 2046, 0, 0, 0, 1570, 146, 0, 0, 0, 2, 0, 0, 0, 3703, 0, 0, 8042, 6193, 0, 0, 0, 0, 293, 0, 8, 0, 5, 15, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8085, 2378, 0, 4, 0, 6270, 12, 1298, 0, 14, 0, 0, 0, 0, 3262, 36, 6606, 0, 0, 3, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 426, 0, 0, 3, 0, 3452, 10, 8, 5010, 0, 0, 0, 479, 0, 0, 0, 0, 420, 407, 1, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3029, 0, 0, 0, 142, 0, 0, 251, 0, 0, 0, 0, 2, 0, 2, 60, 4362, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4, 2, 0, 0, 0, 7359, 0, 0, 0, 0, 7230, 7588, 0, 0, 7, 0, 0, 0, 0, 0, 3, 0, 2, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 221, 0, 0, 0, 91, 8165, 0, 0, 0, 6192, 0, 2, 0, 1, 0, 20, 0, 2, 0, 0, 1771, 0, 0, 7999, 0, 0, 0, 0, 0, 0, 0, 709, 0, 32] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 7122, 0, 0, 0, 1570, 0, 0, 0, 4651, 0, 0, 0, 0, 0, 0, 0, 0, 7100, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 130, 2, 0, 0, 0, 45, 0, 196, 0, 0, 0, 0, 4301, 0, 2, 8151, 1, 5, 0, 0, 0, 0, 0, 0, 162, 21, 0, 0, 0, 0, 75, 0, 0, 0, 0, 13, 0, 7, 0, 0, 8099, 0, 12, 4127, 0, 57, 0, 0, 7055, 1135, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8031, 0, 0, 1, 3, 847, 2, 161, 46, 0, 0, 0, 0, 19, 517, 20, 1689, 0, 0, 0, 136, 0, 0, 0, 0, 0, 0, 0, 12, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[90, 8158, 0, 37, 0, 0, 5474, 0, 0, 10, 0, 0, 0, 0, 0, 2, 0, 1, 2299, 0, 0, 134, 347, 0, 0, 0, 0, 0, 0, 0, 19, 0, 3016, 18, 0, 0, 0, 301, 509, 0, 0, 0, 0, 0, 0, 0, 8135, 0, 10, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 9, 0, 0, 2, 0, 0, 3508, 0, 0, 0, 0, 0, 0, 0, 6596, 0, 3138, 0, 0, 0, 27, 0, 1, 0, 0, 0, 0, 5312, 0, 0, 912, 0, 0, 1668, 6930, 0, 0, 0, 7175, 0, 1578, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 13, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 5436, 0, 0, 0, 0, 2, 0, 32, 5260, 0, 3, 0, 0, 0, 0, 0, 0, 8184, 30, 3, 0, 1, 0, 0, 0, 0, 0, 8, 0, 21, 32, 8177, 53, 0, 0, 147, 43, 0, 0, 0, 14, 0, 1, 0, 0, 0, 0, 3217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1382, 0, 8175, 3791, 0, 0, 0, 0, 0, 0, 156, 0, 61, 0, 0, 0, 0, 1, 0, 0, 3, 7628, 0, 510, 0, 0, 1, 0, 0, 1775, 6087, 0, 0, 0, 4039, 0, 0, 0, 682, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 555] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 57, 0, 0, 1413, 0, 0, 0, 2721, 0, 10, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 0, 0, 0, 0, 1, 7, 0, 81, 0, 0, 0, 4, 0, 0, 131, 0, 0, 0, 0, 0, 0, 458, 5064, 26, 0, 0, 0, 0, 0, 0, 7996, 0, 5, 144, 0, 0, 0, 0, 0, 0, 0, 2, 0, 19, 0, 864, 7279, 0, 0, 8181, 0, 0, 0, 873, 6827, 0, 235, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 3, 0, 0, 0, 7267, 2870, 0, 2, 98, 8154, 0, 0, 8, 3443, 0, 0, 0, 6, 0, 0, 127, 0, 0, 2, 0, 0, 0, 0, 0, 3, 0, 548, 0, 369] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 374, 0, 122, 0, 0, 7249, 0, 0, 0, 0, 0, 960, 0, 133, 1, 0, 0, 1, 0, 13, 3, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 39, 2, 6005, 0, 13, 17, 539, 0, 0, 0, 4, 2046, 0, 0, 0, 1570, 146, 0, 0, 0, 2, 0, 0, 0, 3703, 0, 0, 8042, 6193, 0, 0, 0, 0, 293, 0, 8, 0, 5, 15, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8085, 2378, 0, 4, 0, 6270, 12, 1298, 0, 14, 0, 0, 0, 0, 3262, 36, 6606, 0, 0, 3, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 2259, 0, 0, 0, 0, 620, 0, 0, 7910, 0, 0, 0, 0, 0, 0, 0, 0, 2070, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 6298, 0, 1685, 0, 0, 1, 637, 43, 0, 264, 0, 0, 0, 0, 1, 12, 0, 0, 0, 67, 28, 1, 0, 0, 0, 5745, 0, 0, 8181, 0, 0, 0, 8, 8189, 0, 5696, 0, 12, 0, 0, 0, 0, 0, 16, 0, 0, 0, 1, 0, 0, 5, 0, 181, 1010, 0, 0, 1057, 7623, 0, 0, 1, 0, 1, 1, 0, 73, 0, 0, 30, 0, 0, 5715, 0, 0, 0, 0, 0, 0, 77, 8, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 22, 0, 0, 0, 0, 0, 556, 0, 7909, 0, 0, 0, 0, 0, 0, 0, 1, 7747, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 7023, 0, 5061, 0, 0, 0, 3115, 0, 563, 0, 0, 0, 0, 0, 0, 0, 0, 411, 0, 0, 32, 0, 2, 0, 48, 7925, 0, 1, 7241, 0, 0, 1, 0, 7200, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1238, 0, 0, 0, 0, 0, 0, 0, 90, 3369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 38, 0, 5356, 0, 0, 0, 0, 0, 0, 0, 476, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 426, 0, 0, 3, 0, 3452, 10, 8, 3301, 0, 0, 0, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 420, 407, 1, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2977, 2170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3029, 0, 0, 0, 142, 0, 0, 251, 0, 0, 0, 0, 2, 0, 2, 60, 336, 269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4362, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4, 2, 0, 3215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7359, 0, 0, 0, 0, 0, 7588, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 221, 0, 5060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 7896, 0, 0, 0, 0, 0, 2, 0, 1, 0, 20, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1771, 0, 0, 7999, 0, 0, 0, 0, 0, 0, 0, 709, 0, 32, 0, 1373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7679, 6037, 4091, 7585, 14954, 5294, 8012, 11884] avg=8192.0000 max=14954 max_rank=4 min=4091 min_rank=2 imbalance_ratio=1.825439 max_load_reduction=50582 max_load_reduction_pct=0.771820 moved_to_echo=40144 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[90, 8158, 0, 37, 0, 0, 5474, 0, 0, 10, 0, 0, 0, 0, 0, 2, 0, 1, 2299, 0, 0, 134, 347, 0, 0, 0, 0, 0, 0, 0, 19, 0, 3016, 18, 0, 0, 0, 301, 509, 0, 0, 0, 0, 0, 0, 0, 8135, 0, 10, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 9, 0, 0, 2, 0, 0, 3508, 0, 0, 0, 0, 0, 0, 0, 6596, 0, 3138, 0, 0, 0, 27, 0, 1, 0, 0, 0, 0, 5312, 0, 0, 912, 0, 0, 1668, 6930, 0, 0, 0, 7175, 0, 1578, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 13, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 2259, 0, 0, 0, 0, 620, 0, 0, 7910, 0, 0, 0, 0, 0, 0, 0, 0, 2070, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 6298, 0, 1685, 0, 0, 1, 637, 43, 0, 264, 0, 0, 0, 0, 1, 12, 0, 0, 0, 67, 28, 1, 0, 0, 0, 5745, 0, 0, 8181, 0, 0, 0, 8, 8189, 0, 5696, 0, 12, 0, 0, 0, 0, 0, 16, 0, 0, 0, 1, 0, 0, 5, 0, 181, 1010, 0, 0, 1057, 7623, 0, 0, 1, 0, 1, 1, 0, 73, 0, 0, 30, 0, 0, 5715, 0, 0, 0, 0, 0, 0, 77, 8, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 57, 0, 0, 1413, 0, 0, 0, 1793, 0, 10, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 0, 0, 0, 0, 1, 7, 0, 3919, 2184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 4, 0, 0, 131, 0, 0, 0, 0, 0, 0, 458, 5064, 182, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 7996, 0, 5, 144, 0, 0, 0, 0, 0, 4235, 2353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 19, 0, 864, 0, 0, 0, 8181, 0, 0, 0, 873, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 235, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 3, 0, 5095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7267, 2776, 0, 2, 98, 0, 0, 0, 8, 3443, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 0, 0, 2, 0, 0, 0, 0, 0, 3, 0, 548, 0, 369, 3442, 746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3296, 6248, 6014, 14759, 10971, 5411, 13600, 5237] avg=8192.0000 max=14759 max_rank=3 min=3296 min_rank=0 imbalance_ratio=1.801636 max_load_reduction=50777 max_load_reduction_pct=0.774796 moved_to_echo=35219 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 7122, 0, 0, 0, 1570, 0, 0, 0, 3064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7100, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 130, 408, 2431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 45, 0, 196, 0, 0, 0, 0, 4301, 0, 2, 8151, 1, 312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 162, 21, 0, 0, 0, 0, 75, 0, 0, 439, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 7, 0, 0, 0, 0, 12, 4127, 0, 57, 0, 0, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1135, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 1, 0, 0, 0, 0, 5668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8031, 0, 0, 1, 3, 0, 2, 161, 46, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 20, 1689, 0, 0, 0, 136, 0, 0, 0, 0, 0, 0, 0, 12, 0, 3557, 1275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11756, 10072, 13010, 3134, 5282, 6813, 8263, 7206] avg=8192.0000 max=13010 max_rank=2 min=3134 min_rank=3 imbalance_ratio=1.588135 max_load_reduction=52526 max_load_reduction_pct=0.801483 moved_to_echo=27564 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 374, 0, 122, 0, 0, 4775, 0, 0, 0, 0, 0, 960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 133, 1, 0, 0, 1, 0, 13, 3, 0, 0, 0, 1, 0, 0, 0, 3014, 2413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 39, 2, 6005, 0, 487, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 17, 539, 0, 0, 0, 4, 2046, 0, 0, 0, 1570, 146, 0, 0, 0, 3256, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3703, 0, 0, 0, 6193, 0, 0, 0, 0, 293, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 15, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8085, 2300, 0, 4, 0, 0, 12, 1298, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3262, 36, 6606, 0, 0, 3, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 4, 1987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6232, 5579, 6615, 7593, 10193, 5711, 11713, 11900] avg=8192.0000 max=11900 max_rank=7 min=5579 min_rank=1 imbalance_ratio=1.452637 max_load_reduction=53636 max_load_reduction_pct=0.818420 moved_to_echo=39517 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 5436, 0, 0, 0, 0, 2, 0, 32, 3465, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 30, 3, 0, 1, 0, 0, 0, 0, 0, 8, 0, 21, 32, 1941, 2452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 53, 0, 0, 147, 43, 0, 0, 0, 14, 0, 1, 0, 0, 0, 0, 353, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 2098, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1382, 0, 0, 3791, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 1, 0, 0, 3, 7628, 0, 510, 0, 0, 1, 5723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1775, 5887, 0, 0, 0, 0, 0, 0, 0, 682, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 555, 78, 1442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8938, 12672, 8988, 5377, 5198, 13927, 8359, 2077] avg=8192.0000 max=13927 max_rank=5 min=2077 min_rank=7 imbalance_ratio=1.700073 max_load_reduction=51609 max_load_reduction_pct=0.787491 moved_to_echo=29561 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[90, 8158, 0, 37, 0, 0, 5474, 0, 0, 8, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2299, 0, 0, 134, 347, 0, 0, 0, 0, 0, 0, 0, 19, 0, 3449, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3016, 18, 0, 0, 0, 301, 509, 0, 0, 0, 0, 0, 0, 0, 8135, 0, 0, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 3726, 2273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3138, 0, 0, 0, 27, 0, 1, 0, 0, 0, 0, 5312, 0, 0, 912, 2456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1668, 6702, 0, 0, 0, 0, 0, 1578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 13, 0, 0, 0, 3325, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13769, 7301, 12207, 6029, 1009, 11846, 9948, 3427] avg=8192.0000 max=13769 max_rank=0 min=1009 min_rank=4 imbalance_ratio=1.680786 max_load_reduction=51767 max_load_reduction_pct=0.789902 moved_to_echo=26230 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 22, 0, 0, 0, 0, 0, 556, 0, 5199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7747, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 2378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 7023, 0, 537, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5061, 0, 0, 0, 3115, 0, 563, 0, 0, 0, 0, 0, 0, 0, 0, 411, 0, 2485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 2, 0, 48, 0, 0, 1, 7241, 0, 0, 1, 0, 1082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1238, 0, 0, 0, 0, 0, 5547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 3255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 38, 0, 5356, 0, 0, 0, 0, 0, 0, 0, 476, 0, 0, 3633, 2173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5777, 10163, 7744, 11635, 8407, 6788, 3345, 11677] avg=8192.0000 max=11677 max_rank=7 min=3345 min_rank=6 imbalance_ratio=1.425415 max_load_reduction=53859 max_load_reduction_pct=0.821823 moved_to_echo=30217 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 5436, 0, 0, 0, 0, 2, 0, 32, 3465, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 30, 3, 0, 1, 0, 0, 0, 0, 0, 8, 0, 21, 32, 1941, 2452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 53, 0, 0, 147, 43, 0, 0, 0, 14, 0, 1, 0, 0, 0, 0, 353, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 2098, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1382, 0, 0, 3791, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 1, 0, 0, 3, 7628, 0, 510, 0, 0, 1, 5723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1775, 5887, 0, 0, 0, 0, 0, 0, 0, 682, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 555, 78, 1442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8938, 12672, 8988, 5377, 5198, 13927, 8359, 2077] avg=8192.0000 max=13927 max_rank=5 min=2077 min_rank=7 imbalance_ratio=1.700073 max_load_reduction=51609 max_load_reduction_pct=0.787491 moved_to_echo=29561 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 2259, 0, 0, 0, 0, 620, 0, 0, 5210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2070, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 3665, 1724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 6298, 0, 531, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1685, 0, 0, 1, 637, 43, 0, 264, 0, 0, 0, 0, 1, 12, 0, 0, 3958, 2823, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 28, 1, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 8, 1237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5696, 0, 12, 0, 0, 0, 0, 0, 16, 0, 0, 0, 1, 0, 0, 4021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 181, 977, 0, 0, 1057, 0, 0, 0, 1, 0, 1, 1, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 5715, 0, 0, 0, 0, 0, 0, 77, 8, 0, 0, 4129, 2169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8089, 7464, 6867, 9424, 9522, 9746, 2296, 12128] avg=8192.0000 max=12128 max_rank=7 min=2296 min_rank=6 imbalance_ratio=1.480469 max_load_reduction=53408 max_load_reduction_pct=0.814941 moved_to_echo=33692 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 426, 0, 0, 3, 0, 3452, 10, 8, 3301, 0, 0, 0, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 420, 407, 1, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2977, 2170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3029, 0, 0, 0, 142, 0, 0, 251, 0, 0, 0, 0, 2, 0, 2, 60, 336, 269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4362, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4, 2, 0, 3215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7359, 0, 0, 0, 0, 0, 7588, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 221, 0, 5060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 7896, 0, 0, 0, 0, 0, 2, 0, 1, 0, 20, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1771, 0, 0, 7999, 0, 0, 0, 0, 0, 0, 0, 709, 0, 32, 0, 1373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7679, 6037, 4091, 7585, 14954, 5294, 8012, 11884] avg=8192.0000 max=14954 max_rank=4 min=4091 min_rank=2 imbalance_ratio=1.825439 max_load_reduction=50582 max_load_reduction_pct=0.771820 moved_to_echo=40144 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[14464, 128, 640, 128, 3840, 128, 640, 10496, 128, 128, 256, 1664, 256, 128, 128, 512, 20992, 12544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[3840, 128, 10240, 128, 0, 19200, 256, 128, 0, 0, 128, 128, 128, 1792, 128, 1024, 18176, 11264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 256, 27904, 512, 128, 256, 640, 128, 128, 0, 0, 128, 128, 128, 128, 256, 19456, 16896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 7552, 128, 3840, 1408, 1024, 0, 17664, 128, 27776, 128, 128, 384, 896, 5504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 256, 27904, 512, 128, 256, 640, 128, 128, 0, 0, 128, 128, 128, 128, 256, 19456, 16896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 23424, 128, 128, 384, 3072, 9728, 640, 128, 26880, 0, 128, 128, 512, 0, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 27264, 29824, 0, 128, 1280, 0, 128, 3072, 128, 4224, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[14336, 128, 0, 128, 384, 384, 768, 512, 0, 128, 0, 4352, 128, 128, 36096, 5248, 2816, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[1152, 9216, 128, 128, 128, 128, 128, 128, 128, 128, 8960, 128, 5888, 128, 256, 1024, 39296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[14336, 128, 0, 128, 384, 384, 768, 512, 0, 128, 0, 4352, 128, 128, 36096, 5248, 2816, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[136, 731, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 17, 0, 4, 0, 0, 0, 21, 0, 0, 249, 0, 0, 0, 4120, 0, 0, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6078, 0, 433, 0, 1, 0, 0, 0, 0, 1, 0, 1, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 5, 0, 0, 0, 0, 139, 0, 0, 7588, 0, 0, 0, 1, 0, 0, 511, 0, 0, 0, 2288, 0, 0, 5038, 0, 0, 2324, 8175, 0, 7446, 2, 0, 0, 1, 0, 1, 0, 4, 0, 6, 13, 0, 103, 0, 3374, 0, 0, 0, 7787, 8148] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[11, 0, 2, 0, 417, 0, 0, 29, 1584, 0, 0, 19, 29, 0, 0, 6, 0, 0, 0, 128, 0, 7204, 0, 38, 0, 7458, 0, 0, 0, 0, 0, 0, 0, 27, 20, 0, 0, 0, 282, 80, 0, 1, 0, 0, 0, 2600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4523, 0, 465, 0, 0, 0, 0, 5971, 0, 0, 0, 0, 10, 1, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 1888, 0, 0, 0, 42, 0, 129, 0, 30, 1, 811, 2072, 1, 0, 0, 7714, 30, 0, 356, 0, 1, 0, 0, 0, 7873, 0, 0, 7597, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 297, 0, 0, 0, 884, 4825] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 0, 572, 0, 0, 0, 0, 2144, 0, 5, 0, 0, 0, 16, 539, 0, 0, 2, 0, 1042, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 34, 0, 0, 0, 100, 0, 0, 0, 2, 0, 12, 0, 0, 1, 8066, 0, 0, 0, 3865, 2, 8189, 2, 0, 5441, 0, 0, 0, 0, 0, 3, 0, 0, 29, 0, 0, 3, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 2, 5141, 3, 517, 18, 0, 0, 0, 0, 0, 0, 0, 6802, 0, 0, 0, 23, 0, 0, 9, 9, 6036, 0, 91, 0, 0, 0, 13, 60, 0, 0, 0, 7654, 0, 0, 1077, 0, 10, 0, 0, 7985, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 2139, 0, 0, 0, 0, 0, 9, 32, 0, 305, 0, 0, 649, 97, 0, 0, 0, 0, 0, 0, 981, 0, 0, 0, 2636, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 3, 0, 0, 0, 1, 0, 2, 0, 1, 23, 0, 7297, 0, 2, 0, 7200, 4433, 0, 6081, 0, 0, 0, 0, 2, 0, 403, 0, 0, 0, 0, 1, 0, 48, 0, 0, 0, 112, 0, 40, 0, 0, 0, 300, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 3271, 0, 0, 959, 80, 863, 8109, 0, 12, 0, 0, 0, 0, 0, 75, 0, 0, 1, 0, 0, 1095, 5744, 52, 1, 0, 4356, 8100] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[136, 731, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 17, 0, 4, 0, 0, 0, 21, 0, 0, 249, 0, 0, 0, 4120, 0, 0, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6078, 0, 433, 0, 1, 0, 0, 0, 0, 1, 0, 1, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 5, 0, 0, 0, 0, 139, 0, 0, 7588, 0, 0, 0, 1, 0, 0, 511, 0, 0, 0, 2288, 0, 0, 5038, 0, 0, 2324, 8175, 0, 7446, 2, 0, 0, 1, 0, 1, 0, 4, 0, 6, 13, 0, 103, 0, 3374, 0, 0, 0, 7787, 8148] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 538, 0, 0, 0, 0, 3860, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1127, 0, 4, 320, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 2, 0, 0, 7759, 0, 0, 0, 0, 0, 0, 0, 7221, 0, 0, 9, 0, 0, 8, 0, 8121, 0, 0, 0, 604, 124, 5, 0, 0, 6462, 0, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 156, 0, 0, 20, 0, 0, 889, 0, 34, 0, 0, 0, 0, 67, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 789, 7062, 0, 0, 5298, 0, 0, 0, 0, 0, 0, 6750, 8119, 0, 9, 0, 0, 0, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 3, 0, 10, 0, 0, 1724, 0, 7240, 0, 6038, 1, 0, 0, 0, 2, 6466, 579, 0, 0, 0, 0, 0, 0, 0, 0, 192, 0, 296, 0, 1, 0, 0, 0, 2734, 0, 14, 0, 0, 0, 0, 0, 43, 0, 1, 0, 0, 0, 0, 0, 18, 0, 422, 0, 0, 216, 0, 0, 0, 0, 0, 778, 0, 7604, 0, 0, 0, 0, 11, 34, 2, 5, 2790, 0, 0, 0, 11, 0, 3, 0, 0, 0, 13, 0, 0, 0, 0, 6903, 0, 1, 2, 0, 0, 2, 0, 0, 0, 0, 85, 0, 0, 7099, 0, 0, 1773, 3, 0, 5995, 1, 0, 1462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 409, 0, 78, 0, 12, 4460] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 6, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 5106, 0, 6576, 0, 0, 0, 0, 7657, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 3679, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 8180, 0, 0, 0, 0, 0, 0, 188, 0, 413, 0, 460, 1175, 0, 131, 0, 0, 4723, 48, 0, 1433, 0, 0, 356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 413, 0, 16, 0, 309, 66] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 4, 0, 0, 0, 1346, 0, 0, 0, 0, 20, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 5, 470, 1, 0, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7871, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 8165, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 89, 3, 0, 51, 83, 2, 0, 0, 0, 0, 0, 0, 1, 0, 5, 7447, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 7055, 1783, 51, 0, 0, 0, 0, 268, 0, 756, 124, 0, 3885, 7, 0, 0, 0, 0, 0, 8192, 176, 0, 0, 0, 7486, 0, 0, 0, 5094, 4862] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 6, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 5106, 0, 6576, 0, 0, 0, 0, 7657, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 3679, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 8180, 0, 0, 0, 0, 0, 0, 188, 0, 413, 0, 460, 1175, 0, 131, 0, 0, 4723, 48, 0, 1433, 0, 0, 356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 413, 0, 16, 0, 309, 66] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 3, 0, 10, 0, 0, 1724, 0, 7240, 0, 6038, 1, 0, 0, 0, 2, 6466, 579, 0, 0, 0, 0, 0, 0, 0, 0, 192, 0, 296, 0, 1, 0, 0, 0, 2734, 0, 14, 0, 0, 0, 0, 0, 43, 0, 1, 0, 0, 0, 0, 0, 18, 0, 422, 0, 0, 216, 0, 0, 0, 0, 0, 778, 0, 7604, 0, 0, 0, 0, 11, 34, 2, 5, 2790, 0, 0, 0, 11, 0, 3, 0, 0, 0, 13, 0, 0, 0, 0, 6903, 0, 1, 2, 0, 0, 2, 0, 0, 0, 0, 85, 0, 0, 7099, 0, 0, 1773, 3, 0, 5995, 1, 0, 1462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 409, 0, 78, 0, 12, 4460] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[11, 0, 2, 0, 417, 0, 0, 29, 1584, 0, 0, 19, 29, 0, 0, 6, 0, 0, 0, 128, 0, 7204, 0, 38, 0, 7458, 0, 0, 0, 0, 0, 0, 0, 27, 20, 0, 0, 0, 282, 80, 0, 1, 0, 0, 0, 2600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4523, 0, 465, 0, 0, 0, 0, 5971, 0, 0, 0, 0, 10, 1, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 1888, 0, 0, 0, 42, 0, 129, 0, 30, 1, 811, 2072, 1, 0, 0, 7714, 30, 0, 356, 0, 1, 0, 0, 0, 7873, 0, 0, 7597, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 297, 0, 0, 0, 884, 4825] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 0, 572, 0, 0, 0, 0, 2144, 0, 5, 0, 0, 0, 16, 539, 0, 0, 2, 0, 1042, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 34, 0, 0, 0, 100, 0, 0, 0, 2, 0, 12, 0, 0, 1, 8066, 0, 0, 0, 3865, 2, 8189, 2, 0, 5441, 0, 0, 0, 0, 0, 3, 0, 0, 29, 0, 0, 3, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 2, 5141, 3, 517, 18, 0, 0, 0, 0, 0, 0, 0, 6802, 0, 0, 0, 23, 0, 0, 9, 9, 6036, 0, 91, 0, 0, 0, 13, 60, 0, 0, 0, 7654, 0, 0, 1077, 0, 10, 0, 0, 7985, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 2139, 0, 0, 0, 0, 0, 9, 32, 0, 305, 0, 0, 649, 97, 0, 5294, 421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 981, 0, 0, 0, 2636, 0, 8, 0, 0, 0, 0, 0, 1567, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 3, 0, 0, 7446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 1, 23, 0, 7297, 0, 2, 0, 2540, 4433, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 403, 0, 0, 0, 0, 1, 0, 48, 0, 0, 8109, 787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 40, 0, 0, 0, 300, 0, 0, 0, 0, 0, 0, 6, 0, 1997, 3093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3271, 0, 0, 959, 80, 863, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 1, 0, 0, 1095, 5744, 52, 1, 0, 2359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8946, 5425, 7456, 14299, 9350, 5548, 5185, 9327] avg=8192.0000 max=14299 max_rank=3 min=5185 min_rank=6 imbalance_ratio=1.745483 max_load_reduction=51237 max_load_reduction_pct=0.781815 moved_to_echo=29410 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[136, 731, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 17, 0, 4, 0, 1, 423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 249, 0, 0, 0, 4120, 0, 0, 143, 0, 0, 0, 1604, 0, 236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 368, 0, 0, 0, 0, 0, 0, 7489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6078, 0, 433, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 5, 7446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 0, 0, 7588, 0, 0, 0, 1, 0, 0, 511, 0, 3571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 684, 0, 0, 5038, 0, 0, 2324, 8175, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 6, 13, 0, 103, 0, 3374, 0, 0, 0, 4216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1385, 6373, 7914, 6513, 7600, 11810, 16224, 7717] avg=8192.0000 max=16224 max_rank=6 min=1385 min_rank=0 imbalance_ratio=1.980469 max_load_reduction=49312 max_load_reduction_pct=0.752441 moved_to_echo=43351 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[11, 0, 2, 0, 417, 0, 0, 29, 1584, 0, 0, 19, 29, 0, 0, 6, 5198, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 7204, 0, 38, 0, 7458, 0, 0, 0, 0, 0, 0, 5409, 0, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 20, 0, 0, 0, 282, 80, 0, 1, 0, 0, 0, 2600, 0, 0, 4435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4523, 0, 465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 7873, 773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 1888, 0, 0, 0, 42, 0, 129, 0, 30, 1, 811, 2072, 1, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2305, 30, 0, 356, 0, 1, 0, 0, 0, 0, 0, 0, 7597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 297, 0, 0, 0, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7546, 20376, 7445, 4988, 8721, 5394, 10289, 777] avg=8192.0000 max=20376 max_rank=1 min=777 min_rank=7 imbalance_ratio=2.487305 max_load_reduction=45160 max_load_reduction_pct=0.689087 moved_to_echo=25181 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 0, 572, 0, 0, 0, 0, 2144, 0, 4738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 16, 539, 0, 0, 2, 0, 1042, 0, 0, 0, 0, 0, 4769, 1783, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 34, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 12, 0, 0, 1, 8066, 0, 0, 0, 3865, 2, 2887, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 29, 0, 0, 3, 0, 0, 0, 0, 6036, 703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 2, 5141, 3, 517, 18, 0, 0, 0, 0, 0, 3662, 3519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2033, 0, 0, 0, 23, 0, 0, 9, 9, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 60, 0, 0, 0, 7654, 0, 0, 1077, 0, 10, 0, 0, 4323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7455, 8156, 135, 14837, 6774, 12877, 2165, 13137] avg=8192.0000 max=14837 max_rank=3 min=135 min_rank=2 imbalance_ratio=1.811157 max_load_reduction=50699 max_load_reduction_pct=0.773605 moved_to_echo=34953 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 538, 0, 0, 0, 0, 3860, 0, 0, 0, 0, 0, 4, 5626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1127, 0, 4, 320, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 2, 0, 0, 7759, 0, 0, 0, 0, 0, 0, 0, 7221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 8, 0, 8121, 0, 0, 0, 604, 124, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 7062, 836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 0, 20, 0, 0, 889, 0, 34, 0, 0, 0, 0, 67, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 789, 0, 0, 0, 5298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6750, 8119, 0, 9, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10028, 1452, 15056, 8868, 7996, 1168, 6089, 14879] avg=8192.0000 max=15056 max_rank=2 min=1168 min_rank=5 imbalance_ratio=1.837891 max_load_reduction=50480 max_load_reduction_pct=0.770264 moved_to_echo=30132 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 4, 0, 0, 0, 1346, 0, 0, 0, 0, 20, 0, 0, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 5, 470, 1, 0, 178, 0, 0, 4947, 1777, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7871, 0, 4471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 2880, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 89, 3, 0, 51, 83, 2, 0, 0, 756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 7447, 0, 0, 0, 0, 0, 0, 0, 0, 2336, 3508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 2108, 1783, 51, 0, 0, 0, 0, 268, 0, 0, 124, 0, 3885, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 176, 0, 0, 0, 7486, 0, 0, 0, 2758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1625, 7519, 12342, 2915, 987, 13297, 8239, 18612] avg=8192.0000 max=18612 max_rank=7 min=987 min_rank=4 imbalance_ratio=2.271973 max_load_reduction=46924 max_load_reduction_pct=0.716003 moved_to_echo=41135 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 10, 0, 0, 1724, 0, 7240, 0, 6038, 1, 0, 0, 0, 2, 6621, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6466, 579, 0, 0, 0, 0, 0, 0, 0, 0, 192, 0, 296, 0, 1, 0, 59, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2734, 0, 14, 0, 0, 0, 0, 0, 43, 0, 1, 0, 0, 0, 4099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 422, 0, 0, 216, 0, 0, 0, 0, 0, 778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 34, 2, 5, 2790, 0, 0, 0, 11, 0, 3, 0, 5995, 983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 6903, 0, 1, 2, 0, 0, 2, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 7099, 0, 0, 1773, 3, 0, 0, 1, 0, 1462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 409, 0, 78, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21872, 7721, 6891, 1434, 9834, 6926, 10364, 494] avg=8192.0000 max=21872 max_rank=0 min=494 min_rank=7 imbalance_ratio=2.669922 max_load_reduction=43664 max_load_reduction_pct=0.666260 moved_to_echo=27618 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 6, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 6667, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 325, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8191, 3, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5106, 0, 6576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 3679, 0, 0, 0, 0, 1433, 990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 8180, 0, 0, 0, 0, 0, 0, 188, 0, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 413, 0, 135, 1175, 0, 131, 0, 0, 4723, 48, 0, 0, 0, 0, 356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 413, 0, 16, 0, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6679, 8514, 8297, 11682, 14274, 8516, 6981, 593] avg=8192.0000 max=14274 max_rank=4 min=593 min_rank=7 imbalance_ratio=1.742432 max_load_reduction=51262 max_load_reduction_pct=0.782196 moved_to_echo=30364 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 538, 0, 0, 0, 0, 3860, 0, 0, 0, 0, 0, 4, 5626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1127, 0, 4, 320, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 2, 0, 0, 7759, 0, 0, 0, 0, 0, 0, 0, 7221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 8, 0, 8121, 0, 0, 0, 604, 124, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 7062, 836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 0, 20, 0, 0, 889, 0, 34, 0, 0, 0, 0, 67, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 789, 0, 0, 0, 5298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6750, 8119, 0, 9, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10028, 1452, 15056, 8868, 7996, 1168, 6089, 14879] avg=8192.0000 max=15056 max_rank=2 min=1168 min_rank=5 imbalance_ratio=1.837891 max_load_reduction=50480 max_load_reduction_pct=0.770264 moved_to_echo=30132 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[11, 0, 2, 0, 417, 0, 0, 29, 1584, 0, 0, 19, 29, 0, 0, 6, 5198, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 7204, 0, 38, 0, 7458, 0, 0, 0, 0, 0, 0, 5409, 0, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 20, 0, 0, 0, 282, 80, 0, 1, 0, 0, 0, 2600, 0, 0, 4435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4523, 0, 465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 7873, 773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 1888, 0, 0, 0, 42, 0, 129, 0, 30, 1, 811, 2072, 1, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2305, 30, 0, 356, 0, 1, 0, 0, 0, 0, 0, 0, 7597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 297, 0, 0, 0, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7546, 20376, 7445, 4988, 8721, 5394, 10289, 777] avg=8192.0000 max=20376 max_rank=1 min=777 min_rank=7 imbalance_ratio=2.487305 max_load_reduction=45160 max_load_reduction_pct=0.689087 moved_to_echo=25181 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 10, 0, 0, 1724, 0, 7240, 0, 6038, 1, 0, 0, 0, 2, 6621, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6466, 579, 0, 0, 0, 0, 0, 0, 0, 0, 192, 0, 296, 0, 1, 0, 59, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2734, 0, 14, 0, 0, 0, 0, 0, 43, 0, 1, 0, 0, 0, 4099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 422, 0, 0, 216, 0, 0, 0, 0, 0, 778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 34, 2, 5, 2790, 0, 0, 0, 11, 0, 3, 0, 5995, 983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 6903, 0, 1, 2, 0, 0, 2, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 7099, 0, 0, 1773, 3, 0, 0, 1, 0, 1462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 409, 0, 78, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21872, 7721, 6891, 1434, 9834, 6926, 10364, 494] avg=8192.0000 max=21872 max_rank=0 min=494 min_rank=7 imbalance_ratio=2.669922 max_load_reduction=43664 max_load_reduction_pct=0.666260 moved_to_echo=27618 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 0, 572, 0, 0, 0, 0, 2144, 0, 4738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 16, 539, 0, 0, 2, 0, 1042, 0, 0, 0, 0, 0, 4769, 1783, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 34, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 12, 0, 0, 1, 8066, 0, 0, 0, 3865, 2, 2887, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 29, 0, 0, 3, 0, 0, 0, 0, 6036, 703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 2, 5141, 3, 517, 18, 0, 0, 0, 0, 0, 3662, 3519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2033, 0, 0, 0, 23, 0, 0, 9, 9, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 60, 0, 0, 0, 7654, 0, 0, 1077, 0, 10, 0, 0, 4323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7455, 8156, 135, 14837, 6774, 12877, 2165, 13137] avg=8192.0000 max=14837 max_rank=3 min=135 min_rank=2 imbalance_ratio=1.811157 max_load_reduction=50699 max_load_reduction_pct=0.773605 moved_to_echo=34953 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 0, 14976, 16000, 0, 128, 2176, 17792, 128, 128, 0, 14336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[14720, 640, 128, 256, 128, 10112, 128, 128, 384, 14336, 1792, 128, 512, 256, 128, 0, 17152, 5248, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 512, 128, 22400, 256, 17408, 0, 11520, 128, 8320, 5248, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[256, 2944, 128, 128, 1024, 128, 1792, 128, 8960, 5888, 6400, 128, 128, 768, 2304, 128, 34176, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 512, 128, 22400, 256, 17408, 0, 11520, 128, 8320, 5248, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=67328 observed_loss=-1792 tokens_per_expert=[0, 384, 128, 2048, 256, 128, 128, 36608, 128, 768, 128, 128, 128, 896, 2944, 128, 12160, 10240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[512, 128, 7296, 3072, 128, 16000, 128, 128, 9856, 8704, 1664, 0, 128, 128, 18688, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 128, 128, 128, 128, 128, 512, 128, 11264, 128, 0, 3840, 128, 256, 128, 128, 44800, 5120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 2816, 128, 128, 0, 8192, 128, 128, 384, 128, 0, 128, 2688, 23296, 128, 28032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=67328 slack_slots=63744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=67328 slack_slots=63744 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 3632, 0, 11, 0, 34, 0, 0, 9, 0, 0, 50, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 31, 695, 1977, 39, 0, 0, 7202, 0, 91, 1633, 245, 0, 0, 0, 6135, 0, 8123, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 832, 0, 0, 0, 0, 0, 1968, 6217, 0, 14, 64, 0, 0, 259, 0, 0, 7541, 0, 0, 0, 4, 0, 21, 708, 0, 0, 0, 0, 0, 0, 0, 0, 122, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2, 0, 0, 0, 0, 7667, 5021, 28, 171, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1, 1294, 0, 1, 0, 3, 0, 0, 0, 0, 3, 7729, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3012, 62, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 6559, 0, 7188, 0, 0, 0, 0, 0, 0, 1, 33, 0, 0, 412, 0, 0, 0, 0, 0, 0, 677, 21, 0, 3, 0, 4, 0, 775, 0, 63, 3559, 0, 244, 0, 0, 186, 0, 0, 0, 0, 0, 1, 7280, 10, 0, 0, 8121, 0, 0, 7, 0, 46, 0, 8, 0, 0, 0, 0, 0, 0, 1, 502, 0, 140, 38, 11, 0, 5, 2, 0, 0, 0, 0, 247, 1, 0, 1596, 0, 1, 0, 0, 0, 7535, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 28, 0, 0, 95, 0, 0, 0, 0, 0, 0, 7619, 0, 7798, 0, 0, 220, 0, 0, 0, 0, 37, 0, 352, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4232, 591, 0, 0, 0, 0, 1, 0, 0, 0, 7381, 0, 0, 0, 10, 12, 0, 5, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 632, 0, 0, 0, 0, 0, 0, 0, 0, 7951, 0, 1436, 3, 1622, 0, 1679, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 752, 112, 0, 0, 0, 0, 34, 0, 6968, 0, 303, 0, 0, 0, 4571, 0, 0, 0, 0, 1317, 7, 0, 0, 0, 0, 0, 0, 0, 0, 722, 43, 8025, 2, 970] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 4151, 0, 3, 1, 0, 0, 306, 4, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1856, 1, 0, 2, 0, 0, 0, 0, 1141, 3984, 0, 0, 438, 0, 0, 4016, 7105, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 6176, 0, 0, 0, 8023, 7, 0, 0, 0, 0, 0, 0, 0, 0, 1091, 0, 0, 1023, 0, 0, 8185, 1, 0, 0, 0, 0, 4, 1, 0, 0, 0, 0, 7, 4, 0, 0, 1661, 8192, 0, 0, 0, 0, 3, 0, 0, 7972, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 34, 0, 102] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3062, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 3257, 597, 22, 0, 0, 0, 7262, 7352, 0, 0, 0, 0, 5, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 82, 0, 0, 3, 0, 0, 0, 0, 8133, 0, 55, 0, 0, 0, 90, 0, 7, 6857, 0, 0, 0, 4803, 0, 31, 0, 0, 0, 33, 5, 0, 0, 0, 0, 7665, 0, 0, 11, 7261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8106, 0, 20, 0, 0, 0, 0, 676, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8, 0, 0, 0, 31] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 3, 3770, 0, 0, 23, 1, 0, 0, 0, 0, 5882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 849, 0, 0, 0, 0, 2566, 0, 0, 0, 0, 7635, 0, 1, 0, 0, 0, 152, 0, 0, 2, 0, 206, 0, 0, 0, 0, 8, 0, 0, 0, 0, 4160, 62, 0, 0, 1227, 0, 541, 0, 8189, 0, 58, 0, 0, 10, 0, 0, 0, 7887, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 0, 6393, 0, 0, 44, 1430, 0, 4, 9, 0, 0, 3093, 0, 0, 1, 1, 0, 0, 0, 0, 1480, 0, 0, 1740, 168, 6, 0, 949, 1, 0, 0, 0, 1, 0, 231, 0, 0, 6747, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 4151, 0, 3, 1, 0, 0, 306, 4, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1856, 1, 0, 2, 0, 0, 0, 0, 1141, 3984, 0, 0, 438, 0, 0, 4016, 7105, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 6176, 0, 0, 0, 8023, 7, 0, 0, 0, 0, 0, 0, 0, 0, 1091, 0, 0, 1023, 0, 0, 8185, 1, 0, 0, 0, 0, 4, 1, 0, 0, 0, 0, 7, 4, 0, 0, 1661, 8192, 0, 0, 0, 0, 3, 0, 0, 7972, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 34, 0, 102] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 3632, 0, 11, 0, 34, 0, 0, 9, 0, 0, 50, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 31, 695, 1977, 39, 0, 0, 7202, 0, 91, 1633, 245, 0, 0, 0, 6135, 0, 8123, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 832, 0, 0, 0, 0, 0, 1968, 6217, 0, 14, 64, 0, 0, 259, 0, 0, 7541, 0, 0, 0, 4, 0, 21, 708, 0, 0, 0, 0, 0, 0, 0, 0, 122, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2, 0, 0, 0, 0, 7667, 5021, 28, 171, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 11, 3334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7884, 37, 0, 0, 2, 243, 257, 0, 0, 83, 8187, 0, 0, 0, 3, 0, 3499, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 33, 0, 0, 0, 0, 0, 12, 0, 0, 1, 0, 0, 0, 1, 7988, 0, 0, 1, 0, 0, 0, 7567, 0, 240, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 8125, 0, 0, 0, 0, 0, 696, 0, 0, 0, 0, 0, 1, 0, 0, 21, 0, 0, 0, 0, 0, 0, 1, 0, 939, 0, 0, 1, 0, 0, 0, 0, 8182, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 156, 0, 0, 0, 0, 2, 7370, 2, 0, 0, 0, 1, 386, 1, 348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 20, 1, 9, 7941, 0, 0, 7932, 7405, 1, 3, 0, 0, 4, 442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7654, 0, 0, 0, 0, 0, 0, 0, 8006, 0, 0, 0, 0, 0, 9, 0, 0, 7984, 0, 0, 2869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 231, 0, 0, 50, 0, 123, 2, 170, 972, 1, 0, 5057, 0, 0, 0, 0, 0, 0, 0, 192, 0, 7, 0, 87, 4, 0, 0, 0, 0, 0, 0, 57, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 1294, 0, 1, 0, 3, 0, 0, 0, 0, 3, 7729, 0, 0, 0, 0, 639, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3012, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 6559, 0, 916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 33, 0, 0, 412, 0, 0, 0, 0, 0, 0, 677, 21, 0, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4, 0, 775, 0, 63, 2362, 0, 244, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7280, 10, 0, 0, 8121, 0, 0, 7, 0, 46, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 502, 0, 140, 38, 11, 0, 5, 2, 0, 0, 4153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 1, 0, 1596, 0, 1, 0, 0, 0, 7535, 0, 0, 0, 0, 2119, 424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9670, 3077, 15626, 1278, 3637, 15473, 4852, 11923] avg=8192.0000 max=15626 max_rank=2 min=1278 min_rank=3 imbalance_ratio=1.907471 max_load_reduction=49910 max_load_reduction_pct=0.761566 moved_to_echo=35885 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 4151, 0, 3, 1, 0, 0, 306, 4, 0, 0, 0, 0, 0, 1, 0, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1856, 1, 0, 2, 0, 0, 0, 0, 1141, 3984, 0, 0, 438, 0, 3702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7105, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6176, 0, 0, 0, 8023, 7, 0, 0, 0, 0, 38, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1091, 0, 0, 680, 0, 0, 8185, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 0, 7, 4, 0, 0, 1661, 8035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 7972, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 34, 0, 102, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4649, 11124, 7130, 14401, 9957, 9712, 8301, 262] avg=8192.0000 max=14401 max_rank=3 min=262 min_rank=7 imbalance_ratio=1.757935 max_load_reduction=51135 max_load_reduction_pct=0.780258 moved_to_echo=28232 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 3632, 0, 11, 0, 34, 0, 0, 9, 0, 0, 50, 0, 0, 0, 0, 1355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 31, 695, 1977, 39, 0, 6639, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 1633, 245, 0, 0, 0, 6135, 0, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 832, 0, 0, 0, 0, 0, 1968, 6217, 0, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 64, 0, 0, 259, 0, 0, 5001, 0, 0, 0, 4, 0, 21, 708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4864, 0, 0, 0, 0, 0, 0, 0, 4693, 563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2, 0, 0, 0, 0, 7667, 5021, 28, 171, 0, 0, 2395, 900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5091, 9474, 9139, 9303, 6071, 148, 10120, 16190] avg=8192.0000 max=16190 max_rank=7 min=148 min_rank=5 imbalance_ratio=1.976318 max_load_reduction=49346 max_load_reduction_pct=0.752960 moved_to_echo=32529 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 3770, 0, 0, 23, 1, 0, 0, 0, 0, 5882, 0, 0, 0, 0, 1417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 849, 0, 0, 0, 0, 2566, 0, 0, 0, 7037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 152, 0, 0, 2, 0, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 4160, 62, 0, 0, 1227, 0, 541, 0, 8189, 0, 298, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 10, 0, 0, 0, 5231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 0, 6393, 0, 0, 44, 1403, 0, 4, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3093, 0, 0, 1, 1, 0, 0, 0, 0, 1480, 0, 0, 1740, 168, 6, 0, 598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 949, 1, 0, 0, 0, 1, 0, 231, 0, 0, 6747, 1, 0, 0, 0, 0, 941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11096, 10452, 361, 14512, 5299, 7858, 7087, 8871] avg=8192.0000 max=14512 max_rank=3 min=361 min_rank=2 imbalance_ratio=1.771484 max_load_reduction=51024 max_load_reduction_pct=0.778564 moved_to_echo=29115 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3062, 0, 0, 0, 0, 1232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 3257, 597, 22, 0, 0, 6694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7352, 0, 0, 0, 0, 5, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 82, 0, 0, 3, 0, 0, 0, 0, 8133, 0, 259, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 90, 0, 7, 4548, 0, 0, 0, 4803, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 5, 0, 0, 0, 0, 7665, 0, 0, 11, 7122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8106, 0, 20, 0, 0, 0, 0, 568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 676, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8, 0, 0, 0, 31, 0, 818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4297, 10656, 7367, 8617, 9534, 14836, 8694, 1535] avg=8192.0000 max=14836 max_rank=5 min=1535 min_rank=7 imbalance_ratio=1.811035 max_load_reduction=50700 max_load_reduction_pct=0.773621 moved_to_echo=34599 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 3632, 0, 11, 0, 34, 0, 0, 9, 0, 0, 50, 0, 0, 0, 0, 1355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 31, 695, 1977, 39, 0, 6639, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 1633, 245, 0, 0, 0, 6135, 0, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 832, 0, 0, 0, 0, 0, 1968, 6217, 0, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 64, 0, 0, 259, 0, 0, 5001, 0, 0, 0, 4, 0, 21, 708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4864, 0, 0, 0, 0, 0, 0, 0, 4693, 563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2, 0, 0, 0, 0, 7667, 5021, 28, 171, 0, 0, 2395, 900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5091, 9474, 9139, 9303, 6071, 148, 10120, 16190] avg=8192.0000 max=16190 max_rank=7 min=148 min_rank=5 imbalance_ratio=1.976318 max_load_reduction=49346 max_load_reduction_pct=0.752960 moved_to_echo=32529 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 11, 3334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 1, 0, 1440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7884, 37, 0, 0, 2, 237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 0, 0, 0, 83, 8187, 0, 0, 0, 3, 0, 442, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 33, 0, 0, 0, 0, 0, 12, 308, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 5283, 0, 0, 1, 0, 0, 0, 7567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 7968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 696, 0, 0, 0, 0, 0, 1, 0, 0, 21, 0, 0, 0, 0, 0, 2023, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 939, 0, 0, 1, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 1034, 957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4799, 8161, 8959, 511, 12853, 16378, 2761, 11114] avg=8192.0000 max=16378 max_rank=5 min=511 min_rank=3 imbalance_ratio=1.999268 max_load_reduction=49158 max_load_reduction_pct=0.750092 moved_to_echo=43106 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 28, 0, 0, 95, 0, 0, 0, 0, 0, 0, 7619, 0, 7798, 0, 0, 1429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 220, 0, 0, 0, 0, 37, 0, 352, 0, 0, 0, 0, 0, 0, 2, 0, 3901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 591, 0, 0, 0, 0, 1, 0, 0, 0, 941, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 5, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 632, 0, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5272, 0, 1436, 3, 1622, 0, 1679, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 752, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 6968, 0, 303, 0, 0, 0, 4571, 0, 0, 0, 4264, 331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1317, 7, 0, 0, 0, 0, 0, 0, 0, 0, 722, 43, 8025, 2, 970, 2176, 949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16969, 4512, 1543, 952, 10013, 865, 16471, 14211] avg=8192.0000 max=16969 max_rank=0 min=865 min_rank=5 imbalance_ratio=2.071411 max_load_reduction=48567 max_load_reduction_pct=0.741074 moved_to_echo=41560 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 156, 0, 0, 0, 0, 2, 7370, 2, 0, 0, 0, 1435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 386, 1, 348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 20, 1, 7321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 7932, 7405, 1, 3, 0, 0, 4, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7654, 0, 0, 0, 0, 0, 0, 0, 8006, 0, 302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 5294, 0, 0, 2869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 123, 2, 170, 972, 1, 0, 5057, 0, 0, 0, 0, 0, 255, 620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 0, 7, 0, 87, 4, 0, 0, 0, 0, 0, 0, 57, 0, 130, 953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8965, 8080, 15411, 15962, 8172, 266, 7250, 1430] avg=8192.0000 max=15962 max_rank=3 min=266 min_rank=5 imbalance_ratio=1.948486 max_load_reduction=49574 max_load_reduction_pct=0.756439 moved_to_echo=17118 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 11, 3334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 1, 0, 1440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7884, 37, 0, 0, 2, 237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 0, 0, 0, 83, 8187, 0, 0, 0, 3, 0, 442, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 33, 0, 0, 0, 0, 0, 12, 308, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 5283, 0, 0, 1, 0, 0, 0, 7567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 7968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 696, 0, 0, 0, 0, 0, 1, 0, 0, 21, 0, 0, 0, 0, 0, 2023, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 939, 0, 0, 1, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 1034, 957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4799, 8161, 8959, 511, 12853, 16378, 2761, 11114] avg=8192.0000 max=16378 max_rank=5 min=511 min_rank=3 imbalance_ratio=1.999268 max_load_reduction=49158 max_load_reduction_pct=0.750092 moved_to_echo=43106 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 3770, 0, 0, 23, 1, 0, 0, 0, 0, 5882, 0, 0, 0, 0, 1417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 849, 0, 0, 0, 0, 2566, 0, 0, 0, 7037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 152, 0, 0, 2, 0, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 4160, 62, 0, 0, 1227, 0, 541, 0, 8189, 0, 298, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 10, 0, 0, 0, 5231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 0, 6393, 0, 0, 44, 1403, 0, 4, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3093, 0, 0, 1, 1, 0, 0, 0, 0, 1480, 0, 0, 1740, 168, 6, 0, 598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 949, 1, 0, 0, 0, 1, 0, 231, 0, 0, 6747, 1, 0, 0, 0, 0, 941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11096, 10452, 361, 14512, 5299, 7858, 7087, 8871] avg=8192.0000 max=14512 max_rank=3 min=361 min_rank=2 imbalance_ratio=1.771484 max_load_reduction=51024 max_load_reduction_pct=0.778564 moved_to_echo=29115 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 4151, 0, 3, 1, 0, 0, 306, 4, 0, 0, 0, 0, 0, 1, 0, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1856, 1, 0, 2, 0, 0, 0, 0, 1141, 3984, 0, 0, 438, 0, 3702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7105, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6176, 0, 0, 0, 8023, 7, 0, 0, 0, 0, 38, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1091, 0, 0, 680, 0, 0, 8185, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 0, 7, 4, 0, 0, 1661, 8035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 7972, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 34, 0, 102, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4649, 11124, 7130, 14401, 9957, 9712, 8301, 262] avg=8192.0000 max=14401 max_rank=3 min=262 min_rank=7 imbalance_ratio=1.757935 max_load_reduction=51135 max_load_reduction_pct=0.780258 moved_to_echo=28232 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[256, 128, 128, 7296, 128, 0, 8192, 22400, 128, 0, 1792, 24576, 128, 128, 896, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 0, 128, 18560, 896, 128, 128, 9344, 128, 640, 2688, 31232, 128, 2048, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 3072, 512, 1024, 128, 1664, 128, 128, 256, 0, 7680, 28288, 128, 8320, 128, 1152, 7936, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 9216, 7168, 128, 128, 256, 128, 384, 128, 128, 128, 31744, 128, 7808, 128, 0, 9216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[256, 0, 15104, 128, 17920, 15872, 128, 256, 0, 12800, 128, 3456, 256, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 3840, 128, 8064, 256, 128, 7680, 1024, 5376, 128, 14720, 0, 4608, 1792, 256, 128, 15488, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[256, 128, 128, 128, 2304, 0, 128, 33792, 0, 1792, 11136, 6528, 256, 1792, 8320, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[384, 512, 1920, 384, 128, 128, 128, 1280, 0, 128, 4352, 15232, 3968, 2048, 512, 128, 35584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 3072, 512, 1024, 128, 1664, 128, 128, 256, 0, 7680, 28288, 128, 8320, 128, 1152, 7936, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[256, 128, 128, 128, 2304, 0, 128, 33792, 0, 1792, 11136, 6528, 256, 1792, 8320, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 2452, 0, 0, 0, 0, 0, 0, 847, 12, 0, 0, 0, 0, 0, 0, 0, 3905, 0, 2475, 0, 1, 0, 0, 0, 9, 0, 2, 125, 15, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7612, 0, 8, 5, 7982, 0, 0, 200, 21, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 533, 290, 2946, 0, 0, 23, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 710, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 334, 0, 1, 0, 0, 0, 0, 0, 618, 0, 0, 0, 0, 94, 0, 0, 0, 0, 6307, 0, 0, 0, 0, 0, 2, 8084, 0, 0, 209, 0, 0, 0, 6126, 0, 0, 5365, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 4919, 19, 0, 0, 0, 0, 0, 0, 11, 21, 0, 1, 0, 0, 1, 0, 7608, 0, 144, 85, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7824, 0, 0, 0, 450, 1, 18, 0, 0, 0, 8114, 0, 0, 0, 0, 0, 0, 0, 1, 0, 385, 0, 0, 411, 0, 0, 1, 0, 0, 0, 4029, 0, 0, 0, 0, 0, 0, 0, 31, 39, 0, 5870, 0, 0, 0, 0, 211, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4, 2, 0, 0, 1179, 0, 0, 0, 336, 0, 0, 38, 0, 10, 0, 0, 0, 0, 0, 7938, 3257, 4, 4100, 0, 42, 0, 0, 0, 16, 0, 200, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 11, 512, 0, 0, 2, 0, 2, 28, 0, 0, 0, 1, 1, 0, 0, 0, 1886, 0, 2494, 0, 0, 0, 0, 0, 7468, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2214, 0, 282, 0, 6548, 0, 0, 0, 0, 6839, 683, 0, 0, 0, 1650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 94, 6, 2114, 31, 194, 0, 0, 7, 0, 2944, 0, 8192, 0, 0, 1, 0, 0, 1380, 0, 2231, 19, 0, 0, 0, 1169, 0, 0, 69, 0, 0, 6730, 0, 0, 0, 0, 0, 4, 535, 354, 0, 0, 8119, 567, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 75, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 7784, 0, 0, 6866, 0, 6, 3266, 0, 0, 12, 0, 0, 0, 0, 0, 7280, 0, 0, 0, 3338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 0, 0, 0, 0, 0, 26, 0, 0, 3, 0, 0, 0, 0, 58, 4, 0, 0, 0, 0, 257, 1423, 0, 0, 0, 0, 0, 4, 0, 0, 8130, 0, 0, 283, 2, 0, 3, 0, 0, 7514, 16, 11, 0, 0, 0, 0, 194, 0, 0, 15, 0, 0, 0, 0, 0, 30, 0, 73, 0, 0, 17, 1308, 0, 0, 0, 0, 0, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8093, 0, 0, 9, 0, 287, 0, 0, 0, 3, 0, 0, 0, 0, 114, 0, 0, 411, 7] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 7078, 0, 0, 0, 404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 235, 0, 0, 6888, 960, 0, 0, 0, 3711, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6659, 0, 0, 0, 130, 0, 0, 0, 86, 4, 1, 0, 10, 0, 0, 0, 0, 0, 19, 0, 0, 2, 113, 15, 991, 0, 60, 53, 0, 0, 0, 14, 0, 0, 0, 1064, 2033, 11, 6194, 0, 0, 0, 0, 0, 0, 0, 0, 7522, 1988, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1, 1, 5354, 0, 0, 0, 2246, 291, 0, 0, 0, 0, 8070, 47, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 3176] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 7588, 2, 0, 43, 722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4173, 1, 0, 4336, 1396, 0, 1, 0, 7222, 0, 0, 0, 1, 0, 1851, 0, 0, 0, 2209, 0, 5998, 0, 0, 2, 8076, 0, 0, 0, 0, 1, 11, 137, 0, 0, 0, 12, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 5, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 299, 938, 0, 0, 0, 0, 8030, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 409, 3249, 297, 275, 0, 64, 0, 2, 0, 0, 0, 7847, 0, 0, 0, 0, 2, 0, 0, 149, 9, 0, 0, 0, 0, 9, 2, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 2452, 0, 0, 0, 0, 0, 0, 847, 12, 0, 0, 0, 0, 0, 0, 0, 3905, 0, 2475, 0, 1, 0, 0, 0, 9, 0, 2, 125, 15, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7612, 0, 8, 5, 7982, 0, 0, 200, 21, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 533, 290, 2946, 0, 0, 23, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 710, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 334, 0, 1, 0, 0, 0, 0, 0, 618, 0, 0, 0, 0, 94, 0, 0, 0, 0, 6307, 0, 0, 0, 0, 0, 2, 8084, 0, 0, 209, 0, 0, 0, 6126, 0, 0, 5365, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 4919, 19, 0, 0, 0, 0, 0, 0, 11, 21, 0, 1, 0, 0, 1, 0, 7608, 0, 144, 85, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7824, 0, 0, 0, 450, 1, 18, 0, 0, 0, 8114, 0, 0, 0, 0, 0, 0, 0, 1, 0, 385, 0, 0, 411, 0, 0, 1, 0, 0, 0, 4029, 0, 0, 0, 0, 0, 0, 0, 31, 39, 0, 5870, 0, 0, 0, 0, 211, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4, 2, 0, 0, 1179, 0, 0, 0, 336, 0, 0, 38, 0, 10, 0, 0, 0, 0, 0, 7938, 3257, 4, 4100, 0, 42, 0, 0, 0, 16, 0, 200, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 126, 7, 54, 0, 0, 0, 0, 8, 0, 0, 8186, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 19, 0, 0, 6, 0, 0, 25, 1, 1, 2, 0, 0, 0, 0, 5, 50, 0, 0, 0, 1, 0, 3, 0, 0, 1816, 0, 0, 1, 0, 701, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 39, 18, 0, 3, 0, 0, 0, 408, 22, 17, 0, 0, 0, 0, 0, 7666, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3228, 0, 0, 0, 0, 0, 0, 0, 7800, 8188, 11, 0, 0, 0, 0, 8169, 1, 4, 0, 0, 6491, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4182, 97] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[2, 5779, 0, 254, 11, 0, 0, 0, 42, 20, 4, 0, 0, 0, 0, 0, 0, 927, 0, 1505, 0, 3, 42, 0, 0, 0, 0, 0, 0, 0, 0, 8089, 0, 0, 0, 0, 0, 7926, 0, 0, 0, 7941, 0, 0, 0, 0, 0, 0, 223, 0, 0, 0, 0, 0, 0, 330, 0, 0, 0, 0, 0, 6364, 1, 0, 0, 0, 0, 0, 0, 0, 6495, 0, 7905, 53, 0, 0, 0, 0, 0, 670, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7129, 0, 0, 0, 0, 1452, 0, 0, 1, 0, 1, 130, 5, 0, 50, 0, 0, 0, 2173] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 7784, 0, 0, 6866, 0, 6, 3266, 0, 0, 12, 0, 0, 0, 0, 0, 7280, 0, 0, 0, 3338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 0, 0, 0, 0, 0, 26, 0, 0, 3, 0, 0, 0, 0, 58, 4, 0, 0, 0, 0, 257, 1423, 0, 0, 0, 0, 0, 4, 0, 0, 8130, 0, 0, 283, 2, 0, 3, 0, 0, 7514, 16, 11, 0, 0, 0, 0, 194, 0, 0, 15, 0, 0, 0, 0, 0, 30, 0, 73, 0, 0, 17, 1308, 0, 0, 0, 0, 0, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8093, 0, 0, 9, 0, 287, 0, 0, 0, 3, 0, 0, 0, 0, 114, 0, 0, 411, 7] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 7588, 2, 0, 43, 722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4173, 1, 0, 4336, 1396, 0, 1, 0, 7222, 0, 0, 0, 1, 0, 1851, 0, 0, 0, 2209, 0, 5998, 0, 0, 2, 8076, 0, 0, 0, 0, 1, 11, 137, 0, 0, 0, 12, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 5, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 299, 938, 0, 0, 0, 0, 8030, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 409, 3249, 297, 275, 0, 64, 0, 2, 0, 0, 0, 7847, 0, 0, 0, 0, 2, 0, 0, 149, 9, 0, 0, 0, 0, 9, 2, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 11, 512, 0, 0, 2, 0, 2, 28, 0, 0, 0, 1, 1, 0, 0, 1601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2494, 0, 0, 0, 0, 0, 7468, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2214, 0, 282, 0, 3783, 0, 0, 0, 0, 6839, 683, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1886, 237, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 94, 6, 2114, 31, 194, 0, 0, 7, 0, 2944, 0, 6591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1380, 0, 2231, 19, 0, 0, 0, 1169, 0, 0, 69, 4855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6730, 0, 0, 0, 0, 0, 4, 535, 354, 0, 0, 3176, 567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 75, 0, 1, 0, 0, 0, 0, 2528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2158, 9963, 13801, 3861, 12057, 9724, 11366, 2606] avg=8192.0000 max=13801 max_rank=2 min=2158 min_rank=0 imbalance_ratio=1.684692 max_load_reduction=51735 max_load_reduction_pct=0.789413 moved_to_echo=35753 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 7588, 2, 0, 43, 722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4336, 1396, 0, 1, 0, 7222, 0, 0, 0, 1, 0, 1260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2209, 0, 5998, 0, 0, 2, 4658, 0, 0, 0, 0, 1, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 0, 12, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 5, 4173, 295, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 299, 752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8030, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 409, 3249, 297, 275, 0, 64, 0, 2, 0, 0, 0, 3069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 149, 9, 0, 0, 0, 0, 9, 2, 0, 0, 0, 3123, 413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8719, 14217, 12879, 4709, 1109, 12759, 7437, 3707] avg=8192.0000 max=14217 max_rank=1 min=1109 min_rank=4 imbalance_ratio=1.735474 max_load_reduction=51319 max_load_reduction_pct=0.783066 moved_to_echo=25012 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 126, 7, 54, 0, 0, 0, 0, 8, 0, 0, 8186, 0, 0, 0, 1499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 6, 0, 0, 25, 1, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 50, 0, 0, 0, 1, 0, 3, 0, 0, 1816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 701, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 8169, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 18, 0, 3, 0, 0, 0, 408, 22, 17, 0, 0, 0, 0, 0, 6167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3228, 0, 4885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7800, 8188, 11, 0, 0, 0, 0, 3196, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6491, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4182, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9880, 54, 1875, 8964, 6674, 8115, 19200, 10774] avg=8192.0000 max=19200 max_rank=6 min=54 min_rank=1 imbalance_ratio=2.343750 max_load_reduction=46336 max_load_reduction_pct=0.707031 moved_to_echo=44763 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 4919, 19, 0, 0, 0, 0, 0, 0, 11, 21, 0, 1, 0, 0, 1, 1594, 753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 85, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 450, 1, 18, 0, 0, 0, 4688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 385, 0, 0, 411, 0, 0, 1, 0, 0, 0, 4029, 0, 0, 7608, 293, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 39, 0, 5870, 0, 0, 0, 0, 211, 0, 6560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4, 2, 0, 0, 1179, 0, 4747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 336, 0, 0, 38, 0, 10, 0, 0, 0, 0, 0, 3105, 3257, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4100, 0, 42, 0, 0, 0, 16, 0, 200, 0, 0, 0, 0, 0, 0, 0, 3133, 1746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7319, 5614, 5157, 12814, 12711, 5934, 6750, 9237] avg=8192.0000 max=12814 max_rank=3 min=5157 min_rank=2 imbalance_ratio=1.564209 max_load_reduction=52722 max_load_reduction_pct=0.804474 moved_to_echo=34632 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2452, 0, 0, 0, 0, 0, 0, 847, 12, 0, 0, 0, 0, 0, 0, 138, 788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2475, 0, 1, 0, 0, 0, 9, 0, 2, 125, 15, 0, 5576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4398, 0, 8, 5, 7982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 21, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 533, 290, 2946, 0, 3905, 275, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 334, 0, 1, 0, 0, 3772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 618, 0, 0, 0, 0, 94, 0, 0, 0, 0, 2467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8084, 0, 0, 209, 0, 0, 0, 6126, 0, 0, 5365, 0, 2939, 1828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4237, 8203, 12393, 8257, 601, 4113, 3179, 24553] avg=8192.0000 max=24553 max_rank=7 min=601 min_rank=4 imbalance_ratio=2.997192 max_load_reduction=40983 max_load_reduction_pct=0.625351 moved_to_echo=32446 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 7784, 0, 0, 6866, 0, 6, 3266, 0, 0, 12, 0, 0, 0, 0, 0, 786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 3, 0, 0, 0, 0, 58, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 257, 1423, 0, 0, 0, 0, 0, 4, 0, 0, 8130, 0, 0, 7280, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 283, 2, 0, 3, 0, 0, 7514, 16, 11, 0, 0, 0, 0, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 30, 0, 73, 0, 0, 17, 1308, 0, 0, 0, 4843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 287, 0, 0, 0, 3, 0, 0, 0, 0, 114, 0, 0, 411, 7, 0, 1819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18720, 8869, 91, 17186, 8023, 6286, 3711, 2650] avg=8192.0000 max=18720 max_rank=0 min=91 min_rank=2 imbalance_ratio=2.285156 max_load_reduction=46816 max_load_reduction_pct=0.714355 moved_to_echo=20670 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 7078, 0, 0, 0, 404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6888, 960, 0, 0, 0, 3711, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6659, 0, 0, 0, 76, 0, 0, 0, 86, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 19, 0, 0, 2, 113, 15, 991, 0, 60, 235, 4, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 14, 0, 0, 0, 1064, 2033, 11, 6194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7522, 1988, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4826, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 5354, 0, 0, 0, 2246, 291, 0, 0, 0, 0, 3157, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 3176, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7482, 11560, 6826, 1536, 9369, 14340, 11097, 3326] avg=8192.0000 max=14340 max_rank=5 min=1536 min_rank=3 imbalance_ratio=1.750488 max_load_reduction=51196 max_load_reduction_pct=0.781189 moved_to_echo=38132 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 126, 7, 54, 0, 0, 0, 0, 8, 0, 0, 8186, 0, 0, 0, 1499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 6, 0, 0, 25, 1, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 50, 0, 0, 0, 1, 0, 3, 0, 0, 1816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 701, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 8169, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 18, 0, 3, 0, 0, 0, 408, 22, 17, 0, 0, 0, 0, 0, 6167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3228, 0, 4885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7800, 8188, 11, 0, 0, 0, 0, 3196, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6491, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4182, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9880, 54, 1875, 8964, 6674, 8115, 19200, 10774] avg=8192.0000 max=19200 max_rank=6 min=54 min_rank=1 imbalance_ratio=2.343750 max_load_reduction=46336 max_load_reduction_pct=0.707031 moved_to_echo=44763 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2, 5779, 0, 254, 11, 0, 0, 0, 42, 20, 4, 0, 0, 0, 0, 0, 131, 778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1505, 0, 3, 42, 0, 0, 0, 0, 0, 0, 0, 0, 5506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7926, 0, 0, 0, 4588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 223, 0, 0, 0, 0, 0, 0, 330, 0, 0, 0, 0, 0, 6364, 1, 0, 927, 287, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6495, 0, 7905, 53, 0, 0, 0, 0, 0, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1452, 0, 0, 1, 0, 1, 130, 5, 0, 50, 0, 0, 0, 2173, 3066, 1805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7021, 7056, 12514, 8209, 14992, 4265, 2796, 8683] avg=8192.0000 max=14992 max_rank=4 min=2796 min_rank=6 imbalance_ratio=1.830078 max_load_reduction=50544 max_load_reduction_pct=0.771240 moved_to_echo=30736 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 7784, 0, 0, 6866, 0, 6, 3266, 0, 0, 12, 0, 0, 0, 0, 0, 786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 3, 0, 0, 0, 0, 58, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 257, 1423, 0, 0, 0, 0, 0, 4, 0, 0, 8130, 0, 0, 7280, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 283, 2, 0, 3, 0, 0, 7514, 16, 11, 0, 0, 0, 0, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 30, 0, 73, 0, 0, 17, 1308, 0, 0, 0, 4843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 287, 0, 0, 0, 3, 0, 0, 0, 0, 114, 0, 0, 411, 7, 0, 1819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18720, 8869, 91, 17186, 8023, 6286, 3711, 2650] avg=8192.0000 max=18720 max_rank=0 min=91 min_rank=2 imbalance_ratio=2.285156 max_load_reduction=46816 max_load_reduction_pct=0.714355 moved_to_echo=20670 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 7588, 2, 0, 43, 722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4336, 1396, 0, 1, 0, 7222, 0, 0, 0, 1, 0, 1260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2209, 0, 5998, 0, 0, 2, 4658, 0, 0, 0, 0, 1, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 0, 12, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 5, 4173, 295, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 299, 752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8030, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 409, 3249, 297, 275, 0, 64, 0, 2, 0, 0, 0, 3069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 149, 9, 0, 0, 0, 0, 9, 2, 0, 0, 0, 3123, 413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8719, 14217, 12879, 4709, 1109, 12759, 7437, 3707] avg=8192.0000 max=14217 max_rank=1 min=1109 min_rank=4 imbalance_ratio=1.735474 max_load_reduction=51319 max_load_reduction_pct=0.783066 moved_to_echo=25012 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 0, 8320, 128, 8192, 256, 128, 384, 384, 128, 128, 6400, 128, 0, 9984, 5504, 14848, 7680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 0, 15616, 3456, 128, 2304, 128, 0, 128, 384, 2560, 128, 4480, 128, 36992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 2688, 128, 22912, 0, 384, 128, 22272, 0, 128, 128, 8192, 8704, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 8064, 9344, 384, 384, 7808, 10624, 512, 640, 384, 0, 0, 24192, 3968, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 2688, 128, 22912, 0, 384, 128, 22272, 0, 128, 128, 8192, 8704, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[640, 128, 0, 3072, 1536, 0, 512, 384, 0, 128, 128, 128, 640, 19840, 3072, 128, 34304, 1408, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[384, 128, 128, 128, 128, 128, 16256, 512, 15104, 2176, 128, 6272, 0, 3456, 384, 21248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 128, 6656, 11392, 5760, 128, 128, 0, 18432, 0, 128, 256, 128, 128, 23296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 27904, 8448, 384, 128, 8064, 0, 128, 4224, 128, 128, 128, 8192, 128, 0, 128, 5248, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[2746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1406, 0, 1, 0, 0, 1793, 44, 0, 0, 19, 134, 0, 0, 0, 8191, 0, 0, 0, 23, 1055, 0, 0, 2, 0, 1282, 0, 30, 0, 0, 2523, 0, 0, 6832, 0, 0, 0, 3829, 1, 0, 0, 0, 0, 343, 0, 5841, 2319, 0, 3978, 0, 0, 0, 9, 8130, 4, 0, 0, 0, 0, 0, 0, 0, 130, 0, 0, 0, 0, 0, 36, 0, 0, 8131, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 258, 5, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1378, 0, 0, 0, 3343, 0, 1, 0, 0, 0, 4, 0, 0, 0, 868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 841] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2, 39, 0, 171, 0, 6, 133, 0, 0, 0, 0, 0, 0, 6602, 1, 88, 0, 50, 0, 0, 2031, 0, 5, 0, 84, 2, 0, 4, 12, 0, 48, 0, 1184, 0, 0, 7702, 1, 1, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 1737, 0, 781, 7224, 137, 0, 0, 0, 501, 0, 0, 0, 0, 0, 0, 0, 0, 4519, 5684, 57, 0, 0, 0, 0, 41, 0, 6, 0, 0, 0, 0, 0, 0, 0, 96, 0, 229, 0, 0, 0, 3, 0, 0, 0, 303, 0, 0, 0, 16, 7, 0, 0, 0, 6694, 0, 0, 1, 13, 1674, 730, 0, 0, 205, 4, 8002, 0, 0, 0, 0, 347, 0, 147, 3, 36] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[7241, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6526, 380, 0, 0, 0, 6050, 0, 0, 0, 55, 8056, 0, 0, 0, 0, 0, 86, 0, 8173, 0, 0, 5, 0, 0, 15, 0, 0, 0, 8135, 0, 118, 0, 0, 6992, 0, 0, 167, 0, 30, 0, 0, 0, 0, 0, 1402, 0, 0, 0, 23, 74, 0, 0, 0, 0, 19, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 1, 0, 0, 1393, 0, 0, 946, 0, 0, 0, 63, 0, 0, 0, 0, 64, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1577, 0, 0, 0, 7903] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 61, 0, 0, 243, 3799, 0, 577, 0, 0, 0, 0, 0, 0, 1, 6117, 0, 0, 0, 7512, 0, 0, 8052, 11, 0, 0, 0, 6773, 0, 0, 2, 0, 0, 8, 1, 3, 0, 7140, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 102, 4001, 0, 0, 1, 0, 0, 0, 0, 0, 0, 20, 2, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 2917, 0, 0, 0, 0, 0, 97, 0, 0, 0, 0, 0, 60, 0, 7524, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 2, 8, 0, 0, 0, 23, 0, 0, 0, 0, 1941, 588, 0, 0, 247, 0, 0, 0, 0, 0, 3099, 0, 0, 0, 19, 4573] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[2746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1406, 0, 1, 0, 0, 1793, 44, 0, 0, 19, 134, 0, 0, 0, 8191, 0, 0, 0, 23, 1055, 0, 0, 2, 0, 1282, 0, 30, 0, 0, 2523, 0, 0, 6832, 0, 0, 0, 3829, 1, 0, 0, 0, 0, 343, 0, 5841, 2319, 0, 3978, 0, 0, 0, 9, 8130, 4, 0, 0, 0, 0, 0, 0, 0, 130, 0, 0, 0, 0, 0, 36, 0, 0, 8131, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 258, 5, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1378, 0, 0, 0, 3343, 0, 1, 0, 0, 0, 4, 0, 0, 0, 868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 841] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 8063, 0, 0, 0, 0, 0, 0, 1, 0, 0, 387, 780, 65, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 7823, 0, 127, 0, 0, 1582, 7, 0, 5439, 1, 0, 0, 2, 0, 0, 0, 0, 0, 55, 0, 0, 331, 0, 7382, 0, 0, 0, 0, 7550, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 74, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4211, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 33, 1, 6328, 0, 0, 0, 0, 0, 338, 0, 0, 42, 0, 8036, 0, 601, 0, 0, 0, 0, 0, 0, 6132] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[8161, 7670, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8143, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 46, 8037, 0, 0, 0, 394, 1, 7, 0, 692, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 5354, 0, 0, 6463, 0, 0, 0, 0, 88, 0, 0, 10, 0, 0, 0, 7, 0, 0, 0, 0, 0, 2, 0, 161, 0, 0, 28, 796, 0, 0, 0, 0, 0, 0, 0, 1, 1, 10, 0, 0, 2433, 0, 0, 0, 0, 5, 85, 0, 0, 27, 0, 0, 81, 5047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 2664, 0, 0, 0, 0, 0, 5076] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 9, 0, 776, 0, 0, 0, 1, 0, 961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4911, 0, 966, 0, 8171, 0, 0, 0, 0, 6477, 0, 0, 7, 8, 2783, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4451, 0, 3, 0, 0, 37, 0, 3, 0, 0, 7448, 451, 0, 0, 0, 8, 0, 3, 0, 0, 0, 0, 0, 0, 233, 0, 0, 0, 3, 0, 0, 73, 0, 0, 0, 0, 2688, 29, 8140, 0, 0, 0, 652, 0, 7750, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 304, 0, 8104, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 61, 0, 0, 243, 3799, 0, 577, 0, 0, 0, 0, 0, 0, 1, 6117, 0, 0, 0, 7512, 0, 0, 8052, 11, 0, 0, 0, 6773, 0, 0, 2, 0, 0, 8, 1, 3, 0, 7140, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 102, 4001, 0, 0, 1, 0, 0, 0, 0, 0, 0, 20, 2, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 2917, 0, 0, 0, 0, 0, 97, 0, 0, 0, 0, 0, 60, 0, 7524, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 2, 8, 0, 0, 0, 23, 0, 0, 0, 0, 1941, 588, 0, 0, 247, 0, 0, 0, 0, 0, 3099, 0, 0, 0, 19, 4573] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 806, 0, 0, 3, 0, 913, 0, 0, 0, 5846, 198, 0, 7782, 0, 5, 0, 0, 0, 0, 2123, 0, 0, 0, 0, 0, 0, 16, 7800, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 1660, 0, 0, 8012, 0, 0, 0, 0, 3666, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 129, 3, 0, 0, 0, 108, 0, 6459, 1250, 0, 0, 0, 5173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1194, 198, 8047, 0, 0, 0, 517, 6, 13, 0, 0, 0, 0, 0, 0, 7, 3395, 0, 73, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 9, 0, 776, 0, 0, 0, 1, 0, 961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4911, 0, 966, 0, 8171, 0, 0, 0, 0, 6477, 0, 0, 7, 8, 2783, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4451, 0, 3, 0, 0, 37, 0, 3, 0, 0, 7448, 451, 0, 0, 0, 8, 0, 3, 0, 0, 0, 0, 0, 0, 233, 0, 0, 0, 3, 0, 0, 73, 0, 0, 0, 0, 2688, 29, 8140, 0, 0, 0, 652, 0, 7750, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 304, 0, 8104, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[7241, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6526, 380, 0, 0, 0, 6050, 0, 0, 0, 55, 8056, 0, 0, 0, 0, 0, 86, 0, 8173, 0, 0, 5, 0, 0, 15, 0, 0, 0, 8135, 0, 118, 0, 0, 6992, 0, 0, 167, 0, 30, 0, 0, 0, 0, 0, 1402, 0, 0, 0, 23, 74, 0, 0, 0, 0, 19, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 1, 0, 0, 1393, 0, 0, 946, 0, 0, 0, 63, 0, 0, 0, 0, 64, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1577, 0, 0, 0, 7903] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2, 39, 0, 171, 0, 6, 133, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6602, 1, 88, 0, 8, 0, 0, 2031, 0, 5, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4, 12, 0, 48, 0, 1184, 0, 0, 1844, 1, 1, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1737, 0, 781, 364, 137, 0, 0, 0, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4519, 5684, 57, 0, 0, 0, 0, 41, 0, 6860, 2114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 96, 0, 229, 0, 0, 0, 3, 0, 12, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 303, 0, 0, 0, 16, 7, 0, 0, 0, 6694, 0, 0, 1, 13, 3744, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1674, 730, 0, 0, 205, 4, 8002, 0, 0, 0, 0, 347, 0, 147, 3, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[379, 8819, 11269, 3520, 19275, 354, 10780, 11140] avg=8192.0000 max=19275 max_rank=4 min=354 min_rank=5 imbalance_ratio=2.352905 max_load_reduction=46261 max_load_reduction_pct=0.705887 moved_to_echo=41549 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1406, 0, 1, 0, 0, 1793, 4694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 19, 134, 0, 0, 0, 948, 0, 0, 0, 23, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1282, 0, 30, 0, 0, 2523, 0, 0, 1636, 0, 0, 0, 3829, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 343, 0, 5841, 2319, 0, 201, 0, 0, 0, 9, 8130, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 0, 0, 0, 0, 0, 36, 0, 0, 3777, 1875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8131, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 258, 5, 0, 0, 2126, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1378, 0, 0, 0, 3343, 0, 1, 0, 0, 3321, 423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10640, 2223, 9303, 16847, 5818, 10730, 8470, 1505] avg=8192.0000 max=16847 max_rank=3 min=1505 min_rank=7 imbalance_ratio=2.056519 max_load_reduction=48689 max_load_reduction_pct=0.742935 moved_to_echo=26523 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 9, 0, 776, 0, 0, 0, 1, 0, 961, 0, 0, 0, 0, 0, 0, 4682, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4911, 0, 966, 0, 946, 0, 0, 0, 0, 6477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8, 2783, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4451, 0, 3, 0, 0, 2, 0, 3, 0, 0, 7448, 451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 3, 0, 0, 0, 0, 0, 0, 233, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 73, 0, 0, 0, 0, 2688, 29, 8140, 0, 0, 0, 652, 0, 2121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7750, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 304, 0, 8104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6429, 13300, 2801, 12358, 279, 13706, 8255, 8408] avg=8192.0000 max=13706 max_rank=5 min=279 min_rank=4 imbalance_ratio=1.673096 max_load_reduction=51830 max_load_reduction_pct=0.790863 moved_to_echo=30648 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[7241, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 6, 0, 0, 0, 0, 3740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 755, 380, 0, 0, 0, 6050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 8056, 0, 0, 0, 0, 0, 86, 0, 1957, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 8135, 0, 118, 0, 0, 352, 0, 0, 167, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1402, 0, 0, 0, 23, 74, 0, 0, 0, 0, 19, 3, 6640, 2243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 1, 0, 0, 1393, 0, 1694, 1961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 946, 0, 0, 0, 63, 0, 0, 0, 0, 64, 0, 0, 0, 2, 0, 3973, 337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1577, 0, 0, 0, 5942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11003, 7185, 10159, 8817, 10404, 5064, 5385, 7519] avg=8192.0000 max=11003 max_rank=0 min=5064 min_rank=5 imbalance_ratio=1.343140 max_load_reduction=54533 max_load_reduction_pct=0.832108 moved_to_echo=28372 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 8063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 387, 780, 65, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7823, 0, 127, 0, 0, 1582, 7, 0, 1302, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 331, 0, 372, 0, 0, 0, 0, 7550, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 74, 0, 0, 2, 0, 2, 7010, 1493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4211, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 33, 1, 6328, 0, 0, 0, 0, 2644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 338, 0, 0, 42, 0, 8036, 0, 601, 0, 0, 0, 0, 0, 0, 4610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8066, 1245, 10844, 8311, 8588, 5737, 9118, 13627] avg=8192.0000 max=13627 max_rank=7 min=1245 min_rank=1 imbalance_ratio=1.663452 max_load_reduction=51909 max_load_reduction_pct=0.792068 moved_to_echo=37070 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2, 39, 0, 171, 0, 6, 133, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6602, 1, 88, 0, 8, 0, 0, 2031, 0, 5, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4, 12, 0, 48, 0, 1184, 0, 0, 1844, 1, 1, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1737, 0, 781, 364, 137, 0, 0, 0, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4519, 5684, 57, 0, 0, 0, 0, 41, 0, 6860, 2114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 96, 0, 229, 0, 0, 0, 3, 0, 12, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 303, 0, 0, 0, 16, 7, 0, 0, 0, 6694, 0, 0, 1, 13, 3744, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1674, 730, 0, 0, 205, 4, 8002, 0, 0, 0, 0, 347, 0, 147, 3, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[379, 8819, 11269, 3520, 19275, 354, 10780, 11140] avg=8192.0000 max=19275 max_rank=4 min=354 min_rank=5 imbalance_ratio=2.352905 max_load_reduction=46261 max_load_reduction_pct=0.705887 moved_to_echo=41549 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 61, 0, 0, 243, 3799, 0, 577, 0, 0, 0, 4305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6117, 0, 0, 0, 869, 0, 0, 8052, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6773, 0, 0, 2, 0, 0, 8, 1, 3, 0, 1709, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 4001, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 2, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 2917, 0, 0, 0, 0, 1960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 0, 0, 0, 60, 0, 7524, 0, 0, 0, 1, 3, 1950, 1137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8, 0, 0, 0, 23, 0, 0, 0, 0, 3471, 388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1941, 588, 0, 0, 247, 0, 0, 0, 0, 0, 3099, 0, 0, 0, 19, 3436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8985, 15050, 8497, 4104, 4906, 10772, 3892, 9330] avg=8192.0000 max=15050 max_rank=1 min=3892 min_rank=6 imbalance_ratio=1.837158 max_load_reduction=50486 max_load_reduction_pct=0.770355 moved_to_echo=28900 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[8161, 7670, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8143, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 46, 8037, 0, 0, 0, 394, 1, 7, 0, 167, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5354, 0, 0, 326, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 7, 0, 0, 0, 0, 0, 2, 0, 161, 0, 0, 6137, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 796, 0, 0, 0, 0, 0, 0, 0, 1, 1, 10, 0, 0, 2433, 0, 0, 1260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 85, 0, 0, 27, 0, 0, 81, 5047, 0, 0, 0, 0, 336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 2664, 0, 0, 0, 0, 0, 3816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15832, 8144, 8663, 5768, 6506, 4529, 5581, 10513] avg=8192.0000 max=15832 max_rank=0 min=4529 min_rank=5 imbalance_ratio=1.932617 max_load_reduction=49704 max_load_reduction_pct=0.758423 moved_to_echo=27129 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 806, 3353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 913, 0, 0, 0, 669, 198, 0, 7782, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2123, 0, 0, 0, 0, 0, 0, 16, 1861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 1660, 0, 0, 399, 0, 0, 0, 0, 3666, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 7613, 2144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 129, 3, 0, 0, 0, 108, 0, 6459, 1250, 0, 0, 0, 5173, 1520, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1194, 198, 8047, 0, 0, 0, 517, 3795, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 13, 0, 0, 0, 0, 0, 0, 7, 3395, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4202, 9570, 4000, 5811, 9760, 14644, 14055, 3494] avg=8192.0000 max=14644 max_rank=5 min=3494 min_rank=7 imbalance_ratio=1.787598 max_load_reduction=50892 max_load_reduction_pct=0.776550 moved_to_echo=41953 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[7241, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 6, 0, 0, 0, 0, 3740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 755, 380, 0, 0, 0, 6050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 8056, 0, 0, 0, 0, 0, 86, 0, 1957, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 8135, 0, 118, 0, 0, 352, 0, 0, 167, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1402, 0, 0, 0, 23, 74, 0, 0, 0, 0, 19, 3, 6640, 2243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 1, 0, 0, 1393, 0, 1694, 1961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 946, 0, 0, 0, 63, 0, 0, 0, 0, 64, 0, 0, 0, 2, 0, 3973, 337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1577, 0, 0, 0, 5942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11003, 7185, 10159, 8817, 10404, 5064, 5385, 7519] avg=8192.0000 max=11003 max_rank=0 min=5064 min_rank=5 imbalance_ratio=1.343140 max_load_reduction=54533 max_load_reduction_pct=0.832108 moved_to_echo=28372 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 61, 0, 0, 243, 3799, 0, 577, 0, 0, 0, 4305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6117, 0, 0, 0, 869, 0, 0, 8052, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6773, 0, 0, 2, 0, 0, 8, 1, 3, 0, 1709, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 4001, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 2, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 2917, 0, 0, 0, 0, 1960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 0, 0, 0, 60, 0, 7524, 0, 0, 0, 1, 3, 1950, 1137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8, 0, 0, 0, 23, 0, 0, 0, 0, 3471, 388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1941, 588, 0, 0, 247, 0, 0, 0, 0, 0, 3099, 0, 0, 0, 19, 3436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8985, 15050, 8497, 4104, 4906, 10772, 3892, 9330] avg=8192.0000 max=15050 max_rank=1 min=3892 min_rank=6 imbalance_ratio=1.837158 max_load_reduction=50486 max_load_reduction_pct=0.770355 moved_to_echo=28900 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 9, 0, 776, 0, 0, 0, 1, 0, 961, 0, 0, 0, 0, 0, 0, 4682, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4911, 0, 966, 0, 946, 0, 0, 0, 0, 6477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8, 2783, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4451, 0, 3, 0, 0, 2, 0, 3, 0, 0, 7448, 451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 3, 0, 0, 0, 0, 0, 0, 233, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 73, 0, 0, 0, 0, 2688, 29, 8140, 0, 0, 0, 652, 0, 2121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7750, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 304, 0, 8104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6429, 13300, 2801, 12358, 279, 13706, 8255, 8408] avg=8192.0000 max=13706 max_rank=5 min=279 min_rank=4 imbalance_ratio=1.673096 max_load_reduction=51830 max_load_reduction_pct=0.790863 moved_to_echo=30648 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[8192, 896, 128, 256, 128, 0, 128, 4224, 3072, 128, 22400, 1280, 384, 128, 4608, 5248, 9472, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[18176, 7680, 0, 896, 0, 0, 128, 128, 128, 1280, 13568, 128, 640, 256, 0, 2688, 20864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 128, 1408, 128, 0, 4736, 5760, 256, 128, 128, 3200, 256, 128, 128, 38144, 12032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 27264, 896, 1152, 0, 4224, 640, 0, 17920, 128, 13696, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 0, 13056, 128, 18816, 2688, 896, 2048, 256, 128, 256, 128, 27520, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[3712, 1792, 0, 0, 896, 896, 24192, 4096, 640, 6144, 3200, 2048, 0, 256, 128, 18560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[7808, 1152, 384, 128, 128, 128, 128, 1536, 0, 1280, 384, 29568, 0, 128, 128, 640, 21376, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[3712, 1792, 0, 0, 896, 896, 24192, 4096, 640, 6144, 3200, 2048, 0, 256, 128, 18560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[6912, 128, 30208, 128, 256, 128, 512, 5376, 128, 128, 10496, 128, 128, 8192, 3840, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 5364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 283, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 8118, 0, 0, 0, 709, 0, 1, 6, 8178, 0, 69, 0, 362, 0, 0, 0, 8, 101, 0, 1, 0, 1, 0, 0, 0, 4386, 18, 0, 0, 0, 356, 0, 2, 0, 0, 1929, 0, 0, 1, 0, 0, 0, 4907, 0, 0, 736, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 736, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6219, 0, 0, 0, 0, 0, 8169, 0, 1762, 0, 0, 0, 0, 7283, 7, 3302, 0, 0, 979, 0, 0, 0, 0, 0, 0, 1245] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[6, 0, 0, 2, 0, 0, 0, 0, 3, 0, 0, 0, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 0, 2895, 0, 8160, 0, 7343, 0, 0, 0, 0, 28, 8192, 0, 0, 0, 2135, 5799, 9, 0, 0, 0, 7975, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 1, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 188, 0, 0, 0, 237, 1, 0, 3395, 239, 0, 0, 6533, 0, 0, 1699, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 92, 0, 0, 0, 2, 0, 132, 33, 0, 0, 0, 0, 0, 31, 0, 0, 1, 0, 0, 61, 0, 5489, 0, 8081, 0, 0, 0, 0, 0, 1102, 0, 8081, 6, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 15, 0, 1099, 0, 0, 0, 0, 5, 0, 43, 0, 0, 0, 0, 6, 0, 0, 0, 6600, 0, 0, 0, 2342, 4261, 3, 0, 0, 0, 0, 0, 0, 3, 318, 8, 0, 0, 2, 0, 0, 36, 0, 5178, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 8151, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1050, 0, 0, 0, 1051, 237, 0, 0, 2006, 0, 519, 0, 0, 0, 69, 0, 0, 1238, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 128, 0, 166, 0, 0, 0, 7789, 0, 0, 0, 0, 0, 586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 496, 7868, 0, 0, 0, 0, 2, 0, 0, 7394, 0, 307, 0, 0, 0, 8114, 0, 0, 9, 0, 28, 8173, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7829, 6832, 0, 0, 0, 0, 9, 2, 0, 0, 0, 0, 172, 0, 5, 2, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 3, 4, 0, 0, 0, 0, 2, 15, 0, 0, 1, 3, 0, 0, 454, 0, 22, 1, 0, 0, 275, 33, 0, 0, 0, 0, 0, 7265, 0, 0, 364, 1047, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 967, 3, 0, 0, 4646, 0, 0, 0, 0, 0, 1, 0, 0, 22, 0, 427, 0, 0, 0, 0, 56, 67, 0, 0, 0, 0, 0, 0, 0, 1030, 1, 5035, 0, 0, 0, 0, 1, 7235, 0, 8182, 0, 0, 0, 0, 238, 5, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7718, 3015, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 6056, 0, 0, 7990, 0, 0, 0, 0, 0, 0, 56, 0, 4, 0, 2, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 4002, 1, 0, 0, 0, 0, 7, 282, 0, 4, 0, 1, 0, 0, 14, 4, 0, 8046, 0, 0, 0, 0, 0, 1, 392, 4, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 6470, 0, 0, 0, 0, 0, 0, 0, 1406, 14, 0, 0, 0, 0, 0, 0, 1, 690, 0, 0, 0, 457, 0, 0, 78, 0, 0, 54, 5793, 1, 501, 348, 196, 0, 0, 0, 0, 0, 0, 0, 78, 38, 0, 0, 0, 0, 0, 0, 10, 0, 0, 1588, 0, 0, 211, 5, 0, 0, 11, 8109, 0, 0, 6267, 0, 0, 7120, 0, 0, 25, 0, 0, 0, 0, 7033, 0, 0, 0, 2, 2100, 0, 0, 0, 0, 106, 0, 0, 0, 0, 2, 456, 0, 295, 0, 0, 0, 0, 7947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8067, 0, 1, 54, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8039, 337, 0, 0, 0, 145, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 119, 8190, 0, 0, 0, 0, 2, 0, 0, 1, 368, 8094, 0, 1, 0, 1, 0, 0, 3, 2, 7731, 6844, 0, 1587, 0, 450, 0, 0, 0, 2704, 0, 0, 0, 30, 261, 0, 0, 0, 0, 0, 2392, 0, 0, 40, 0, 0, 0, 0, 13, 9, 26, 5409, 0, 0, 807, 286, 0, 0, 0, 0, 0, 0, 0, 1443, 0, 4405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8, 88, 262, 0, 0, 0, 0, 0, 0, 0, 0, 5319, 0, 0, 0, 0, 102, 15, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 167, 1174, 0, 0, 0, 0, 0, 4, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2392, 0, 7318, 8108, 0, 0, 1, 0, 1, 2, 0, 0, 6382, 0, 0, 7803, 7, 7594, 0, 0, 0, 231, 144, 7549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 398, 0, 0, 0, 0, 0, 0, 0, 0, 23, 6363, 1, 1, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 1425, 0, 0, 7649, 0, 0, 0, 0, 0, 0, 30, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[6, 0, 0, 2, 0, 0, 0, 0, 3, 0, 0, 0, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 0, 2895, 0, 8160, 0, 7343, 0, 0, 0, 0, 28, 8192, 0, 0, 0, 2135, 5799, 9, 0, 0, 0, 7975, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 1, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 188, 0, 0, 0, 237, 1, 0, 3395, 239, 0, 0, 6533, 0, 0, 1699, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 92, 0, 0, 0, 2, 0, 132, 33, 0, 0, 0, 0, 0, 31, 0, 0, 1, 0, 0, 61, 0, 5489, 0, 8081, 0, 0, 0, 0, 0, 1102, 0, 8081, 6, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 15, 0, 1099, 0, 0, 0, 0, 5, 0, 43, 0, 0, 0, 0, 6, 0, 0, 0, 6600, 0, 0, 0, 2342, 4261, 3, 0, 0, 0, 0, 0, 0, 3, 318, 8, 0, 0, 2, 0, 0, 36, 0, 5178, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 8151, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1050, 0, 0, 0, 1051, 237, 0, 0, 2006, 0, 519, 0, 0, 0, 69, 0, 0, 1238, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8039, 337, 0, 0, 0, 145, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 119, 8190, 0, 0, 0, 0, 2, 0, 0, 1, 368, 8094, 0, 1, 0, 1, 0, 0, 3, 2, 7731, 6844, 0, 1587, 0, 450, 0, 0, 0, 2704, 0, 0, 0, 30, 261, 0, 0, 0, 0, 0, 2392, 0, 0, 40, 0, 0, 0, 0, 13, 9, 26, 5409, 0, 0, 807, 286, 0, 0, 0, 0, 0, 0, 0, 1443, 0, 4405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8, 88, 262, 0, 0, 0, 0, 0, 0, 0, 0, 5319, 0, 0, 0, 0, 102, 15, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 967, 3, 0, 0, 4646, 0, 0, 0, 0, 0, 1, 0, 0, 22, 0, 427, 0, 0, 0, 0, 56, 67, 0, 0, 0, 0, 0, 0, 0, 1030, 1, 5035, 0, 0, 0, 0, 1, 7235, 0, 8182, 0, 0, 0, 0, 238, 5, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7718, 3015, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 6056, 0, 0, 7990, 0, 0, 0, 0, 0, 0, 56, 0, 4, 0, 2, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 4002, 1, 0, 0, 0, 0, 7, 282, 0, 4, 0, 1, 0, 0, 14, 4, 0, 8046, 0, 0, 0, 0, 0, 1, 392, 4, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 5364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 0, 634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 283, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 8118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 709, 0, 1, 6, 0, 0, 34, 0, 362, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 0, 1, 0, 1, 0, 0, 0, 4386, 18, 0, 0, 0, 292, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1929, 0, 0, 1, 0, 0, 0, 4907, 0, 0, 736, 0, 0, 0, 19, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 736, 0, 0, 34, 0, 0, 0, 0, 7544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6219, 0, 0, 0, 0, 0, 8169, 0, 1762, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7283, 7, 3302, 0, 0, 979, 0, 0, 0, 0, 0, 0, 1245, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6186, 8464, 1120, 4801, 7593, 8327, 16166, 12879] avg=8192.0000 max=16166 max_rank=6 min=1120 min_rank=2 imbalance_ratio=1.973389 max_load_reduction=49370 max_load_reduction_pct=0.753326 moved_to_echo=44965 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 967, 3, 0, 0, 4646, 0, 0, 0, 0, 0, 1, 0, 0, 22, 4376, 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 427, 0, 0, 0, 0, 56, 67, 0, 0, 0, 0, 0, 0, 0, 1030, 1855, 659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 3872, 0, 0, 0, 0, 238, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 6056, 0, 0, 7990, 0, 0, 0, 2362, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 4, 0, 2, 0, 0, 0, 0, 0, 0, 0, 15, 6673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4002, 1, 0, 0, 0, 0, 7, 282, 0, 4, 0, 1, 0, 0, 1160, 1948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4, 0, 8046, 0, 0, 0, 0, 0, 1, 392, 4, 0, 0, 0, 0, 1371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10577, 4094, 4117, 6321, 16440, 6750, 7405, 9832] avg=8192.0000 max=16440 max_rank=4 min=4094 min_rank=1 imbalance_ratio=2.006836 max_load_reduction=49096 max_load_reduction_pct=0.749146 moved_to_echo=40427 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 92, 0, 0, 0, 2, 0, 132, 33, 0, 0, 0, 0, 0, 31, 7022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 61, 0, 5489, 0, 8081, 0, 0, 0, 0, 0, 1102, 4059, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 3864, 0, 0, 15, 0, 1099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 43, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2342, 4261, 3, 0, 0, 0, 0, 0, 0, 3, 318, 8, 0, 0, 2357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 36, 0, 5178, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1050, 0, 0, 2541, 1944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 237, 0, 0, 2006, 0, 519, 0, 0, 0, 69, 0, 0, 1238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7312, 19852, 4984, 54, 9292, 5231, 13691, 5120] avg=8192.0000 max=19852 max_rank=1 min=54 min_rank=3 imbalance_ratio=2.423340 max_load_reduction=45684 max_load_reduction_pct=0.697083 moved_to_echo=33334 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 128, 0, 166, 0, 0, 0, 7789, 0, 0, 0, 0, 0, 586, 0, 0, 6426, 630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 496, 7868, 0, 0, 0, 0, 2, 0, 4202, 968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 307, 0, 0, 0, 0, 0, 0, 9, 0, 28, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2, 0, 0, 0, 0, 172, 0, 5, 2, 0, 0, 1, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 3, 7484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 2, 15, 0, 0, 1, 3, 0, 0, 454, 0, 22, 2630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 275, 33, 0, 0, 0, 0, 0, 7265, 0, 0, 364, 1047, 0, 1391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15725, 13582, 8517, 6414, 220, 7571, 3131, 10376] avg=8192.0000 max=15725 max_rank=0 min=220 min_rank=4 imbalance_ratio=1.919556 max_load_reduction=49811 max_load_reduction_pct=0.760056 moved_to_echo=21298 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 0, 2, 0, 0, 0, 0, 3, 0, 0, 0, 1012, 0, 0, 0, 0, 632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1371, 0, 8160, 0, 7343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 8192, 0, 0, 0, 2135, 5799, 9, 0, 0, 0, 6529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 1, 0, 11, 0, 0, 0, 835, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 7505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 7, 689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 0, 0, 237, 1, 0, 3395, 239, 0, 0, 6533, 0, 0, 1699, 0, 1417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1655, 409, 16874, 22692, 928, 8570, 699, 13709] avg=8192.0000 max=22692 max_rank=3 min=409 min_rank=1 imbalance_ratio=2.770020 max_load_reduction=42844 max_load_reduction_pct=0.653748 moved_to_echo=23906 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 167, 1174, 0, 0, 0, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 552, 0, 0, 0, 4644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3455, 8108, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 6382, 0, 0, 7803, 7, 7594, 0, 0, 0, 231, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 398, 0, 0, 0, 0, 0, 2118, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 6363, 1, 1, 0, 0, 0, 0, 61, 0, 0, 0, 0, 2206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 1425, 0, 0, 7649, 0, 0, 0, 2905, 1745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1605, 5203, 11565, 22133, 2581, 8655, 13732, 62] avg=8192.0000 max=22133 max_rank=3 min=62 min_rank=7 imbalance_ratio=2.701782 max_load_reduction=43403 max_load_reduction_pct=0.662277 moved_to_echo=25030 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 6470, 0, 0, 0, 0, 0, 450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1406, 14, 0, 0, 0, 0, 0, 0, 1, 690, 0, 0, 0, 976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 457, 0, 0, 78, 0, 0, 54, 0, 1, 238, 348, 196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 38, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 211, 5, 0, 0, 11, 8109, 0, 0, 6267, 0, 0, 7120, 0, 0, 25, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7033, 0, 0, 0, 2, 2100, 0, 0, 0, 0, 106, 0, 5343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 456, 0, 295, 0, 0, 0, 0, 7947, 0, 0, 0, 0, 612, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8067, 0, 1, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6922, 3087, 1372, 126, 21892, 14584, 9431, 8122] avg=8192.0000 max=21892 max_rank=4 min=126 min_rank=3 imbalance_ratio=2.672363 max_load_reduction=43644 max_load_reduction_pct=0.665955 moved_to_echo=54029 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 0, 2, 0, 0, 0, 0, 3, 0, 0, 0, 1012, 0, 0, 0, 0, 632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1371, 0, 8160, 0, 7343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 8192, 0, 0, 0, 2135, 5799, 9, 0, 0, 0, 6529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 1, 0, 11, 0, 0, 0, 835, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 7505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 7, 689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 0, 0, 237, 1, 0, 3395, 239, 0, 0, 6533, 0, 0, 1699, 0, 1417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1655, 409, 16874, 22692, 928, 8570, 699, 13709] avg=8192.0000 max=22692 max_rank=3 min=409 min_rank=1 imbalance_ratio=2.770020 max_load_reduction=42844 max_load_reduction_pct=0.653748 moved_to_echo=23906 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8039, 337, 0, 0, 0, 145, 0, 0, 0, 0, 7033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 119, 8190, 0, 0, 0, 0, 2, 0, 0, 1, 1471, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 368, 0, 0, 1, 0, 1, 0, 0, 3, 2, 7731, 6844, 0, 1587, 0, 450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2704, 0, 0, 0, 30, 261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 13, 9, 26, 5409, 0, 0, 807, 286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1443, 0, 4405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8, 88, 262, 0, 0, 0, 921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5319, 0, 0, 0, 0, 102, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15554, 10845, 16987, 2995, 6590, 5848, 1281, 5436] avg=8192.0000 max=16987 max_rank=2 min=1281 min_rank=6 imbalance_ratio=2.073608 max_load_reduction=48549 max_load_reduction_pct=0.740799 moved_to_echo=19155 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 167, 1174, 0, 0, 0, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 552, 0, 0, 0, 4644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3455, 8108, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 6382, 0, 0, 7803, 7, 7594, 0, 0, 0, 231, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 398, 0, 0, 0, 0, 0, 2118, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 6363, 1, 1, 0, 0, 0, 0, 61, 0, 0, 0, 0, 2206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 1425, 0, 0, 7649, 0, 0, 0, 2905, 1745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1605, 5203, 11565, 22133, 2581, 8655, 13732, 62] avg=8192.0000 max=22133 max_rank=3 min=62 min_rank=7 imbalance_ratio=2.701782 max_load_reduction=43403 max_load_reduction_pct=0.662277 moved_to_echo=25030 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 967, 3, 0, 0, 4646, 0, 0, 0, 0, 0, 1, 0, 0, 22, 4376, 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 427, 0, 0, 0, 0, 56, 67, 0, 0, 0, 0, 0, 0, 0, 1030, 1855, 659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 3872, 0, 0, 0, 0, 238, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 6056, 0, 0, 7990, 0, 0, 0, 2362, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 4, 0, 2, 0, 0, 0, 0, 0, 0, 0, 15, 6673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4002, 1, 0, 0, 0, 0, 7, 282, 0, 4, 0, 1, 0, 0, 1160, 1948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4, 0, 8046, 0, 0, 0, 0, 0, 1, 392, 4, 0, 0, 0, 0, 1371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10577, 4094, 4117, 6321, 16440, 6750, 7405, 9832] avg=8192.0000 max=16440 max_rank=4 min=4094 min_rank=1 imbalance_ratio=2.006836 max_load_reduction=49096 max_load_reduction_pct=0.749146 moved_to_echo=40427 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 17408, 128, 128, 7808, 2176, 18048, 128, 128, 128, 256, 19712, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 256, 4352, 4352, 128, 128, 8192, 128, 128, 23040, 512, 128, 17024, 384, 0, 128, 7936, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[896, 0, 128, 1152, 0, 128, 128, 0, 128, 12928, 16256, 15616, 128, 17152, 1408, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] @@ -2280,4896 +2282,4865 @@ WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[384, 512, 128, 1408, 128, 512, 256, 13824, 512, 16000, 0, 128, 1280, 0, 128, 10368, 17280, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 1152, 5632, 0, 128, 12800, 8192, 256, 128, 256, 7808, 1024, 640, 256, 128, 24960, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[256, 1152, 256, 15744, 384, 10752, 0, 3968, 1280, 128, 15872, 6656, 128, 512, 4096, 1280, 4352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 128, 13440, 7808, 128, 4480, 768, 2176, 0, 128, 128, 0, 128, 128, 36864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 128, 13440, 7808, 128, 4480, 768, 2176, 0, 128, 128, 0, 128, 128, 36864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[6148, 0, 0, 0, 0, 0, 0, 64, 0, 1641, 0, 0, 0, 1, 0, 0, 57, 0, 0, 0, 1192, 0, 0, 0, 0, 0, 262, 0, 0, 0, 1220, 0, 0, 4786, 0, 134, 0, 7825, 0, 0, 7984, 4, 0, 0, 0, 617, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3872, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 6807, 0, 0, 0, 2, 0, 7885, 95, 0, 0, 1, 0, 0, 1260, 2944, 757, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 1212, 0, 1, 0, 0, 0, 1, 0, 22, 0, 0, 0, 0, 211, 0, 0, 0, 0, 195, 0, 8192, 0, 78, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[22, 0, 183, 0, 1, 9, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2077, 0, 0, 0, 0, 7257, 5667, 0, 151, 45, 4605, 0, 0, 7648, 537, 22, 5811, 0, 0, 0, 0, 0, 0, 641, 0, 0, 0, 0, 25, 7948, 13, 5, 343, 0, 0, 0, 0, 0, 29, 0, 85, 0, 21, 0, 0, 0, 0, 0, 64, 1171, 0, 0, 0, 12, 0, 7816, 0, 0, 0, 0, 0, 16, 0, 0, 33, 0, 0, 0, 0, 0, 5, 520, 0, 0, 0, 0, 14, 3976, 56, 655, 0, 0, 0, 0, 12, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8031, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[22, 28, 0, 0, 1, 0, 0, 39, 0, 0, 0, 0, 0, 439, 3, 8, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 9, 637, 0, 7832, 0, 0, 38, 0, 707, 34, 0, 8188, 0, 76, 8108, 0, 0, 0, 0, 0, 0, 1, 0, 19, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3899, 0, 0, 0, 0, 0, 0, 7429, 0, 0, 0, 0, 819, 0, 0, 0, 0, 0, 0, 7417, 0, 0, 0, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 7, 156, 0, 0, 0, 0, 0, 0, 0, 0, 29, 323, 0, 0, 0, 0, 0, 0, 4098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8094, 0, 7053, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 11, 0, 0, 175, 0, 0, 0, 0, 47, 0, 0, 0, 0, 2, 0, 0, 0, 5613, 234, 0, 0, 7839, 0, 0, 0, 0, 1, 8040, 0, 3, 0, 2, 0, 0, 7, 12, 0, 23, 0, 1750, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 432, 0, 0, 0, 0, 0, 1084, 2, 4813, 117, 0, 0, 0, 12, 0, 0, 8116, 0, 0, 0, 0, 0, 2227, 0, 0, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1537, 0, 0, 8009, 0, 0, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 7016, 0, 0, 0, 56] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[6848, 0, 0, 5, 0, 0, 215, 0, 239, 0, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3849, 38, 0, 0, 0, 6865, 7787, 0, 0, 0, 0, 23, 0, 43, 0, 5, 6118, 0, 0, 0, 0, 0, 904, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 5945, 0, 0, 7, 0, 0, 8, 12, 1, 0, 2209, 34, 0, 0, 0, 0, 0, 0, 0, 7698, 0, 373, 0, 0, 0, 2838, 0, 0, 0, 0, 5134, 0, 0, 0, 0, 1, 161, 0, 0, 0, 0, 0, 0, 937, 470, 0, 0, 0, 0, 0, 0, 1687, 0, 0, 113, 0, 0, 38, 163, 0, 0, 558, 0, 3815, 0, 11, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 140, 0, 0, 0, 0, 0, 0, 1, 0, 1058, 0, 0, 0, 0, 1833, 0, 43, 0, 6072, 7549, 0, 0, 0, 0, 0, 6454, 0, 0, 0, 933, 0, 2308, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3056, 0, 0, 0, 0, 0, 1, 0, 0, 0, 159, 0, 0, 7415, 0, 0, 0, 0, 0, 3744, 5, 0, 0, 5144, 0, 0, 0, 0, 6826, 0, 2982, 0, 0, 0, 0, 0, 266, 0, 0, 0, 0, 0, 1, 0, 935, 2, 0, 0, 222, 23, 0, 0, 0, 0, 0, 0, 0, 1, 0, 270, 0, 0, 0, 8083, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[7384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 108, 0, 0, 0, 3528, 0, 1, 0, 1871, 552, 0, 0, 0, 0, 4856, 0, 0, 0, 2488, 0, 0, 0, 0, 0, 0, 1, 0, 581, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 14, 3074, 0, 122, 0, 0, 107, 0, 0, 0, 0, 0, 7905, 0, 0, 0, 3972, 6738, 0, 2, 0, 186, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8008, 0, 1, 5, 6, 645, 0, 1, 484, 0, 0, 0, 3902, 0, 4788, 0, 4118] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[6848, 0, 0, 5, 0, 0, 215, 0, 239, 0, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3849, 38, 0, 0, 0, 6865, 7787, 0, 0, 0, 0, 23, 0, 43, 0, 5, 6118, 0, 0, 0, 0, 0, 904, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 5945, 0, 0, 7, 0, 0, 8, 12, 1, 0, 2209, 34, 0, 0, 0, 0, 0, 0, 0, 7698, 0, 373, 0, 0, 0, 2838, 0, 0, 0, 0, 5134, 0, 0, 0, 0, 1, 161, 0, 0, 0, 0, 0, 0, 937, 470, 0, 0, 0, 0, 0, 0, 1687, 0, 0, 113, 0, 0, 38, 163, 0, 0, 558, 0, 3815, 0, 11, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 11, 0, 0, 175, 0, 0, 0, 0, 47, 0, 0, 0, 0, 2, 0, 0, 0, 5613, 234, 0, 0, 7839, 0, 0, 0, 0, 1, 8040, 0, 3, 0, 2, 0, 0, 7, 12, 0, 23, 0, 1750, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 432, 0, 0, 0, 0, 0, 1084, 2, 4813, 117, 0, 0, 0, 12, 0, 0, 8116, 0, 0, 0, 0, 0, 2227, 0, 0, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1537, 0, 0, 8009, 0, 0, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 7016, 0, 0, 0, 56] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[8168, 8142, 0, 0, 4, 0, 0, 0, 2, 0, 0, 2, 0, 0, 323, 0, 0, 0, 4, 0, 0, 0, 0, 0, 46, 40, 0, 108, 0, 7252, 0, 0, 0, 0, 0, 3, 0, 228, 0, 0, 8067, 0, 1632, 0, 0, 9, 0, 0, 0, 0, 8, 1, 0, 0, 4, 0, 0, 0, 0, 4, 7290, 0, 0, 0, 153, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4948, 4, 0, 0, 0, 0, 0, 46, 13, 0, 0, 0, 0, 396, 0, 0, 3233, 0, 0, 1253, 0, 0, 0, 0, 1, 0, 2, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7854, 0, 6158, 0, 0, 0, 93] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[6148, 0, 0, 0, 0, 0, 0, 64, 0, 1641, 0, 0, 0, 1, 0, 0, 57, 0, 0, 0, 1192, 0, 0, 0, 0, 0, 262, 0, 0, 0, 1220, 0, 0, 4786, 0, 134, 0, 7825, 0, 0, 7984, 4, 0, 0, 0, 617, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3872, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 6807, 0, 0, 0, 2, 0, 7885, 95, 0, 0, 1, 0, 0, 1260, 2944, 757, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 1212, 0, 1, 0, 0, 0, 1, 0, 22, 0, 0, 0, 0, 211, 0, 0, 0, 0, 195, 0, 8192, 0, 78, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 140, 0, 0, 0, 0, 0, 0, 1, 0, 1058, 0, 0, 0, 0, 1833, 0, 43, 0, 6072, 7549, 0, 0, 0, 0, 0, 6454, 0, 0, 0, 933, 0, 2308, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3056, 0, 0, 0, 0, 0, 1, 0, 0, 0, 159, 0, 0, 7415, 0, 0, 0, 0, 0, 3744, 5, 0, 0, 5144, 0, 0, 0, 0, 6826, 0, 2982, 0, 0, 0, 0, 0, 266, 0, 0, 0, 0, 0, 1, 0, 935, 2, 0, 0, 222, 23, 0, 0, 0, 0, 0, 0, 0, 1, 0, 270, 0, 0, 0, 8083, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[22, 28, 0, 0, 1, 0, 0, 39, 0, 0, 0, 0, 0, 439, 3, 8, 2310, 1876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 9, 637, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 707, 34, 0, 8188, 0, 76, 528, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3899, 0, 0, 0, 5270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7429, 0, 0, 0, 0, 819, 0, 0, 0, 0, 0, 0, 4312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 7, 156, 0, 1646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 323, 0, 0, 0, 0, 0, 0, 4098, 3780, 1229, 311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8094, 0, 5096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4726, 4712, 9572, 9189, 12560, 1817, 9770, 13190] avg=8192.0000 max=13190 max_rank=7 min=1817 min_rank=5 imbalance_ratio=1.610107 max_load_reduction=52346 max_load_reduction_pct=0.798737 moved_to_echo=37337 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[8168, 8142, 0, 0, 4, 0, 0, 0, 2, 0, 0, 2, 0, 0, 323, 0, 2299, 1251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 46, 40, 0, 108, 0, 3748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 228, 0, 0, 525, 0, 1632, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 0, 0, 4, 0, 0, 0, 0, 4, 7290, 0, 0, 0, 5243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 2877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 46, 13, 0, 0, 0, 0, 396, 0, 0, 3233, 1437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1253, 0, 0, 0, 0, 1, 0, 2, 38, 0, 0, 0, 0, 0, 3504, 820, 271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7854, 0, 4450, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20191, 3946, 2397, 12550, 3036, 5129, 5889, 12398] avg=8192.0000 max=20191 max_rank=0 min=2397 min_rank=2 imbalance_ratio=2.464722 max_load_reduction=45345 max_load_reduction_pct=0.691910 moved_to_echo=26452 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 11, 0, 0, 175, 0, 0, 0, 0, 47, 0, 0, 0, 0, 6, 563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 5613, 234, 0, 0, 7839, 0, 0, 0, 0, 1, 8040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 0, 7, 12, 0, 3, 0, 1750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 432, 0, 0, 0, 0, 0, 1084, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4813, 117, 0, 0, 0, 12, 0, 0, 8116, 0, 0, 0, 0, 0, 1295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1537, 1637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8009, 0, 0, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 369, 309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5070, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[802, 21729, 1777, 1581, 14355, 3288, 16876, 5128] avg=8192.0000 max=21729 max_rank=1 min=802 min_rank=0 imbalance_ratio=2.652466 max_load_reduction=43807 max_load_reduction_pct=0.668442 moved_to_echo=39647 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[6148, 0, 0, 0, 0, 0, 0, 64, 0, 1641, 0, 0, 0, 1, 0, 0, 2279, 1998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 1192, 0, 0, 0, 0, 0, 262, 0, 0, 0, 1220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4786, 0, 134, 0, 7825, 0, 0, 512, 4, 0, 0, 0, 617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3872, 0, 0, 0, 5193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 6807, 0, 0, 0, 2, 0, 4577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 1, 0, 0, 1260, 2944, 757, 0, 50, 0, 0, 0, 0, 0, 1914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 1212, 0, 1, 0, 0, 0, 1, 0, 22, 0, 1310, 365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 211, 0, 0, 0, 0, 195, 0, 5913, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12131, 2731, 13878, 9066, 11392, 7021, 2920, 6397] avg=8192.0000 max=13878 max_rank=2 min=2731 min_rank=1 imbalance_ratio=1.694092 max_load_reduction=51658 max_load_reduction_pct=0.788239 moved_to_echo=27730 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[7384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1383, 777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 108, 0, 0, 0, 3528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1871, 552, 0, 0, 0, 0, 317, 0, 0, 0, 2488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 581, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 3156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 14, 1788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 0, 0, 107, 0, 0, 0, 0, 0, 7905, 0, 0, 0, 3972, 6738, 910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 186, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8008, 0, 509, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 6, 645, 0, 1, 484, 0, 0, 0, 2820, 0, 4788, 0, 4118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9544, 3642, 5229, 3807, 1804, 19754, 8888, 12868] avg=8192.0000 max=19754 max_rank=5 min=1804 min_rank=4 imbalance_ratio=2.411377 max_load_reduction=45782 max_load_reduction_pct=0.698578 moved_to_echo=43314 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[6848, 0, 0, 5, 0, 0, 215, 0, 239, 0, 368, 0, 0, 0, 0, 0, 1743, 1947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3849, 38, 0, 0, 0, 3552, 7787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 43, 0, 5, 399, 0, 0, 0, 0, 0, 904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 5945, 0, 0, 7, 3976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 12, 1, 0, 2209, 34, 0, 0, 0, 0, 0, 0, 0, 4476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 373, 0, 0, 0, 2838, 0, 0, 0, 0, 5134, 0, 0, 0, 0, 1, 890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 0, 0, 0, 0, 0, 0, 937, 470, 0, 0, 0, 0, 0, 0, 1687, 3313, 1275, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 0, 0, 38, 163, 0, 0, 558, 0, 2757, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11365, 15226, 1374, 9944, 6740, 9236, 8011, 3640] avg=8192.0000 max=15226 max_rank=1 min=1374 min_rank=2 imbalance_ratio=1.858643 max_load_reduction=50310 max_load_reduction_pct=0.767670 moved_to_echo=27627 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[22, 0, 183, 0, 1, 9, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1656, 1977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2077, 0, 0, 0, 0, 3755, 5667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 45, 4605, 0, 0, 7648, 537, 22, 378, 0, 0, 0, 0, 0, 0, 641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 7948, 13, 5, 343, 0, 0, 0, 0, 0, 29, 0, 3777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 21, 0, 0, 0, 0, 0, 64, 1171, 0, 0, 0, 12, 0, 4544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 33, 0, 0, 0, 0, 0, 5, 520, 1874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 3976, 56, 655, 0, 0, 0, 0, 12, 0, 9, 0, 3502, 1295, 354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3849, 11499, 14027, 12140, 5897, 2448, 9873, 5803] avg=8192.0000 max=14027 max_rank=2 min=2448 min_rank=5 imbalance_ratio=1.712280 max_load_reduction=51509 max_load_reduction_pct=0.785965 moved_to_echo=24021 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[7384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1383, 777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 108, 0, 0, 0, 3528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1871, 552, 0, 0, 0, 0, 317, 0, 0, 0, 2488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 581, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 3156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 14, 1788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 0, 0, 107, 0, 0, 0, 0, 0, 7905, 0, 0, 0, 3972, 6738, 910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 186, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8008, 0, 509, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 6, 645, 0, 1, 484, 0, 0, 0, 2820, 0, 4788, 0, 4118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9544, 3642, 5229, 3807, 1804, 19754, 8888, 12868] avg=8192.0000 max=19754 max_rank=5 min=1804 min_rank=4 imbalance_ratio=2.411377 max_load_reduction=45782 max_load_reduction_pct=0.698578 moved_to_echo=43314 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 140, 1839, 947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1058, 0, 0, 0, 0, 949, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6072, 7549, 0, 0, 0, 0, 0, 420, 0, 0, 0, 933, 0, 2308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3056, 0, 0, 0, 4195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 159, 0, 0, 7415, 0, 0, 0, 0, 0, 2177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 5144, 0, 0, 0, 0, 6826, 0, 2982, 0, 0, 0, 0, 0, 1886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 266, 0, 0, 0, 0, 0, 1, 0, 935, 2, 0, 0, 222, 23, 0, 0, 884, 620, 356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 270, 0, 0, 0, 5841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2928, 2051, 17282, 7259, 9752, 16843, 3309, 6112] avg=8192.0000 max=17282 max_rank=2 min=2051 min_rank=1 imbalance_ratio=2.109619 max_load_reduction=48254 max_load_reduction_pct=0.736298 moved_to_echo=36016 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[6148, 0, 0, 0, 0, 0, 0, 64, 0, 1641, 0, 0, 0, 1, 0, 0, 2279, 1998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 1192, 0, 0, 0, 0, 0, 262, 0, 0, 0, 1220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4786, 0, 134, 0, 7825, 0, 0, 512, 4, 0, 0, 0, 617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3872, 0, 0, 0, 5193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 6807, 0, 0, 0, 2, 0, 4577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 1, 0, 0, 1260, 2944, 757, 0, 50, 0, 0, 0, 0, 0, 1914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 1212, 0, 1, 0, 0, 0, 1, 0, 22, 0, 1310, 365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 211, 0, 0, 0, 0, 195, 0, 5913, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12131, 2731, 13878, 9066, 11392, 7021, 2920, 6397] avg=8192.0000 max=13878 max_rank=2 min=2731 min_rank=1 imbalance_ratio=1.694092 max_load_reduction=51658 max_load_reduction_pct=0.788239 moved_to_echo=27730 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 11, 0, 0, 175, 0, 0, 0, 0, 47, 0, 0, 0, 0, 6, 563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 5613, 234, 0, 0, 7839, 0, 0, 0, 0, 1, 8040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 0, 7, 12, 0, 3, 0, 1750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 432, 0, 0, 0, 0, 0, 1084, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4813, 117, 0, 0, 0, 12, 0, 0, 8116, 0, 0, 0, 0, 0, 1295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1537, 1637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8009, 0, 0, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 369, 309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5070, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[802, 21729, 1777, 1581, 14355, 3288, 16876, 5128] avg=8192.0000 max=21729 max_rank=1 min=802 min_rank=0 imbalance_ratio=2.652466 max_load_reduction=43807 max_load_reduction_pct=0.668442 moved_to_echo=39647 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[8168, 8142, 0, 0, 4, 0, 0, 0, 2, 0, 0, 2, 0, 0, 323, 0, 2299, 1251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 46, 40, 0, 108, 0, 3748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 228, 0, 0, 525, 0, 1632, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 0, 0, 4, 0, 0, 0, 0, 4, 7290, 0, 0, 0, 5243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 2877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 46, 13, 0, 0, 0, 0, 396, 0, 0, 3233, 1437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1253, 0, 0, 0, 0, 1, 0, 2, 38, 0, 0, 0, 0, 0, 3504, 820, 271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7854, 0, 4450, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20191, 3946, 2397, 12550, 3036, 5129, 5889, 12398] avg=8192.0000 max=20191 max_rank=0 min=2397 min_rank=2 imbalance_ratio=2.464722 max_load_reduction=45345 max_load_reduction_pct=0.691910 moved_to_echo=26452 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 512, 0, 5248, 256, 2944, 1408, 3072, 7680, 0, 16128, 0, 512, 128, 4224, 12032, 12288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 128, 0, 6912, 256, 128, 0, 14976, 128, 384, 768, 0, 16128, 26368, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[512, 128, 9344, 256, 128, 4096, 128, 11136, 1792, 128, 128, 0, 256, 128, 128, 13824, 15104, 7552, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 128, 0, 6912, 256, 128, 0, 14976, 128, 384, 768, 0, 16128, 26368, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[28672, 8192, 256, 128, 128, 128, 512, 128, 256, 1664, 384, 128, 128, 512, 384, 256, 13568, 11392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[256, 11008, 14848, 768, 0, 24064, 640, 128, 3200, 128, 3456, 0, 3456, 640, 3328, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 128, 8576, 128, 128, 384, 512, 0, 128, 24064, 128, 128, 1152, 30848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[256, 4864, 256, 7552, 128, 0, 2432, 128, 896, 23552, 128, 0, 0, 128, 128, 26112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 896, 128, 256, 768, 0, 16768, 0, 37760, 0, 4992, 0, 4352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True - [2026-08-26 14:48:10] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 697.9 | throughput per GPU (TFLOP/s/GPU): 259.1 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.117604E+01 | loss scale: 1.0 | grad norm: 44.306 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[3515, 69, 4, 230, 222, 626, 111, 3, 28, 505, 1274, 204, 1288, 111, 322, 212, 106, 1661, 1, 28, 875, 11, 52, 1719, 1410, 81, 10, 4590, 2355, 3651, 106, 629, 510, 1037, 184, 4, 84, 930, 9, 64, 1082, 8, 640, 2851, 4, 73, 19, 3605, 48, 147, 2, 22, 9, 139, 15, 5, 190, 12, 40, 343, 267, 130, 242, 223, 2800, 5, 3, 0, 1206, 0, 27, 472, 938, 22, 1285, 614, 372, 26, 506, 225, 27, 5, 9, 12, 44, 1, 71, 1220, 4, 120, 252, 3, 33, 159, 268, 13, 332, 191, 16, 941, 41, 4, 70, 102, 160, 325, 334, 3360, 2, 296, 485, 103, 380, 960, 1, 125, 0, 245, 544, 142, 31, 26, 4694, 413, 84, 46, 3, 1391] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[3821, 66, 2, 132, 142, 359, 59, 0, 15, 403, 1517, 117, 1630, 144, 148, 460, 169, 1649, 29, 12, 996, 23, 18, 606, 636, 35, 1, 4013, 1763, 3381, 112, 266, 928, 1806, 160, 16, 92, 1116, 23, 129, 1219, 2, 538, 2689, 0, 45, 29, 4062, 8, 41, 3, 13, 1, 193, 17, 8, 238, 17, 10, 206, 293, 22, 155, 144, 2483, 2, 4, 1, 1570, 2, 22, 761, 1435, 88, 2123, 420, 420, 70, 592, 74, 23, 17, 0, 22, 30, 1, 148, 1613, 4, 80, 203, 42, 106, 67, 163, 3, 162, 271, 13, 409, 59, 9, 320, 87, 167, 453, 235, 4032, 2, 274, 382, 103, 214, 761, 0, 37, 0, 360, 380, 65, 67, 26, 4785, 322, 85, 47, 5, 1868] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[3735, 18, 1, 88, 111, 317, 118, 1, 18, 394, 1078, 175, 1546, 91, 337, 397, 62, 2330, 5, 22, 789, 9, 27, 811, 540, 38, 12, 3920, 2756, 3668, 22, 472, 1085, 2407, 146, 1, 149, 1044, 4, 75, 422, 23, 216, 2407, 3, 17, 15, 3430, 59, 82, 3, 5, 8, 376, 3, 9, 166, 16, 25, 185, 246, 11, 221, 57, 2945, 10, 1, 0, 1132, 2, 21, 704, 1028, 13, 1789, 837, 459, 7, 374, 576, 3, 4, 0, 102, 46, 1, 67, 1641, 5, 66, 162, 4, 12, 67, 302, 19, 237, 385, 55, 891, 27, 10, 47, 19, 197, 112, 371, 4063, 11, 177, 434, 44, 60, 976, 0, 53, 0, 125, 472, 403, 115, 5, 4667, 302, 27, 52, 2, 1974] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[5001, 33, 5, 432, 63, 708, 83, 0, 36, 224, 1107, 69, 1557, 59, 219, 438, 113, 2035, 7, 25, 1006, 12, 31, 830, 1064, 44, 0, 4044, 2093, 2755, 32, 417, 1017, 1304, 145, 19, 125, 398, 5, 14, 536, 8, 698, 3850, 1, 19, 30, 3533, 10, 53, 12, 19, 0, 241, 22, 4, 95, 10, 31, 542, 227, 41, 125, 125, 2804, 11, 2, 2, 1498, 0, 41, 615, 1275, 2, 2360, 317, 517, 3, 322, 234, 8, 22, 0, 14, 17, 10, 85, 617, 3, 97, 255, 18, 27, 35, 323, 14, 309, 391, 11, 818, 37, 3, 44, 30, 158, 83, 671, 4032, 20, 108, 305, 34, 195, 913, 1, 120, 0, 204, 348, 231, 106, 28, 4688, 375, 58, 80, 25, 1761] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[4187, 72, 2, 131, 120, 480, 84, 0, 8, 336, 1226, 119, 1754, 283, 92, 430, 59, 1360, 35, 27, 833, 3, 14, 1762, 1424, 35, 3, 4112, 2316, 3503, 49, 493, 544, 1565, 172, 52, 28, 363, 11, 132, 1109, 1, 797, 2251, 96, 30, 17, 3472, 14, 89, 5, 12, 1, 310, 46, 7, 177, 17, 15, 450, 369, 13, 94, 278, 2967, 27, 1, 0, 807, 0, 9, 1098, 1507, 32, 1924, 221, 622, 11, 174, 217, 31, 1, 4, 21, 16, 0, 157, 1400, 3, 55, 189, 7, 51, 145, 377, 8, 496, 232, 13, 319, 99, 4, 49, 25, 416, 190, 408, 3558, 20, 148, 683, 98, 144, 589, 0, 93, 0, 125, 699, 297, 49, 19, 4632, 294, 57, 42, 31, 1736] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[4036, 22, 1, 94, 176, 596, 209, 0, 1, 531, 1522, 252, 1302, 149, 212, 740, 120, 1910, 5, 7, 1280, 0, 43, 656, 1282, 25, 1, 4151, 2209, 3240, 178, 372, 745, 1058, 152, 1, 39, 1197, 24, 39, 530, 4, 390, 2287, 0, 26, 37, 4167, 8, 118, 4, 11, 4, 90, 5, 3, 199, 11, 12, 190, 263, 7, 257, 322, 2528, 4, 0, 0, 971, 2, 31, 983, 962, 8, 2408, 400, 604, 0, 504, 276, 8, 0, 0, 14, 25, 0, 67, 1508, 10, 52, 73, 1, 17, 216, 210, 7, 123, 369, 35, 417, 145, 2, 66, 5, 226, 151, 399, 4193, 3, 140, 365, 65, 94, 818, 0, 35, 1, 164, 905, 134, 49, 24, 5023, 396, 50, 59, 9, 1660] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[3773, 4, 1, 197, 78, 463, 131, 0, 30, 381, 2161, 192, 1502, 103, 319, 380, 44, 2076, 11, 36, 754, 3, 54, 1373, 930, 48, 1, 4470, 2088, 2719, 35, 626, 961, 1359, 226, 7, 46, 892, 7, 96, 715, 12, 954, 3254, 10, 79, 78, 3331, 15, 78, 0, 20, 1, 283, 23, 10, 48, 8, 22, 200, 381, 7, 260, 398, 2823, 3, 2, 0, 890, 4, 31, 827, 1067, 24, 2345, 277, 438, 4, 396, 152, 12, 12, 0, 27, 45, 0, 108, 1361, 9, 66, 164, 8, 103, 196, 145, 32, 279, 201, 11, 741, 56, 12, 114, 38, 146, 165, 360, 3452, 10, 166, 371, 111, 135, 886, 4, 26, 1, 222, 516, 233, 69, 8, 4539, 489, 46, 60, 11, 1723] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[4197, 30, 0, 258, 90, 690, 66, 0, 39, 417, 590, 184, 1496, 63, 211, 297, 46, 2023, 7, 85, 1104, 7, 32, 1430, 1147, 53, 6, 4766, 2365, 3587, 86, 625, 821, 1370, 243, 43, 136, 517, 8, 71, 1088, 12, 884, 3666, 0, 18, 40, 3526, 18, 45, 1, 6, 16, 260, 6, 10, 164, 46, 36, 206, 260, 111, 330, 185, 2539, 7, 0, 1, 1222, 10, 53, 397, 952, 16, 2706, 148, 525, 15, 363, 220, 28, 2, 0, 58, 32, 0, 113, 479, 8, 74, 385, 20, 136, 73, 97, 19, 164, 116, 23, 460, 33, 14, 140, 30, 127, 193, 421, 3280, 10, 161, 353, 74, 187, 911, 2, 85, 1, 204, 416, 93, 105, 62, 4848, 387, 84, 97, 15, 1602] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2626, 69, 4, 230, 222, 626, 111, 3, 28, 505, 1274, 204, 1288, 111, 322, 212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 1661, 1, 28, 875, 11, 52, 1719, 1410, 81, 10, 0, 2355, 135, 106, 629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 1037, 184, 4, 84, 930, 9, 64, 1082, 8, 640, 2851, 4, 73, 19, 218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 147, 2, 22, 9, 139, 15, 5, 190, 12, 40, 343, 267, 130, 242, 223, 4590, 2275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1433, 5, 3, 0, 1206, 0, 27, 472, 938, 22, 1285, 614, 372, 26, 506, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 5, 9, 12, 44, 1, 71, 1220, 4, 120, 252, 3, 33, 159, 268, 13, 3516, 1367, 1112, 330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 332, 191, 16, 941, 41, 4, 70, 102, 160, 325, 334, 3360, 2, 296, 485, 103, 896, 559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 380, 960, 1, 125, 0, 245, 544, 142, 31, 26, 3798, 413, 84, 46, 3, 1391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7835, 9179, 7717, 8699, 7134, 8566, 8217, 8189] avg=8192.0000 max=9179 max_rank=1 min=7134 min_rank=4 imbalance_ratio=1.120483 max_load_reduction=56357 max_load_reduction_pct=0.859940 moved_to_echo=32106 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2790, 18, 1, 88, 111, 317, 118, 1, 18, 394, 1078, 175, 1546, 91, 337, 397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 2330, 5, 22, 789, 9, 27, 811, 540, 38, 12, 0, 2756, 136, 22, 472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1085, 2407, 146, 1, 149, 1044, 4, 75, 422, 23, 216, 2407, 3, 17, 15, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 82, 3, 5, 8, 376, 3, 9, 166, 16, 25, 185, 246, 11, 221, 57, 3920, 2164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1508, 10, 1, 0, 1132, 2, 21, 704, 1028, 13, 1789, 837, 459, 7, 374, 576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 102, 46, 1, 67, 1641, 5, 66, 162, 4, 12, 67, 302, 19, 3532, 1437, 1058, 351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 237, 385, 55, 891, 27, 10, 47, 19, 197, 112, 371, 4063, 11, 177, 434, 44, 891, 594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 976, 0, 53, 0, 125, 472, 403, 115, 5, 3776, 302, 27, 52, 2, 1974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7480, 8031, 8222, 7556, 8461, 8879, 8565, 8342] avg=8192.0000 max=8879 max_rank=5 min=7480 min_rank=0 imbalance_ratio=1.083862 max_load_reduction=56657 max_load_reduction_pct=0.864517 moved_to_echo=34247 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2854, 66, 2, 132, 142, 359, 59, 0, 15, 403, 1517, 117, 1630, 144, 148, 460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 1649, 29, 12, 996, 23, 18, 606, 636, 35, 1, 0, 1763, 125, 112, 266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 928, 1806, 160, 16, 92, 1116, 23, 129, 1219, 2, 538, 2689, 0, 45, 29, 246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 41, 3, 13, 1, 193, 17, 8, 238, 17, 10, 206, 293, 22, 155, 144, 4013, 2563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1271, 2, 4, 1, 1570, 2, 22, 761, 1435, 88, 2123, 420, 420, 70, 592, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 17, 0, 22, 30, 1, 148, 1613, 4, 80, 203, 42, 106, 67, 163, 3, 3256, 1212, 1253, 359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 162, 271, 13, 409, 59, 9, 320, 87, 167, 453, 235, 4032, 2, 274, 382, 103, 913, 608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 214, 761, 0, 37, 0, 360, 380, 65, 67, 26, 3872, 322, 85, 47, 5, 1868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8048, 6440, 9038, 7945, 8855, 8602, 8499, 8109] avg=8192.0000 max=9038 max_rank=2 min=6440 min_rank=1 imbalance_ratio=1.103271 max_load_reduction=56498 max_load_reduction_pct=0.862091 moved_to_echo=34065 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3015, 22, 1, 94, 176, 596, 209, 0, 1, 531, 1522, 252, 1302, 149, 212, 740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 1910, 5, 7, 1280, 0, 43, 656, 1282, 25, 1, 0, 2209, 120, 178, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 745, 1058, 152, 1, 39, 1197, 24, 39, 530, 4, 390, 2287, 0, 26, 37, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 118, 4, 11, 4, 90, 5, 3, 199, 11, 12, 190, 263, 7, 257, 322, 4151, 2630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1294, 4, 0, 0, 971, 2, 31, 983, 962, 8, 2408, 400, 604, 0, 504, 276, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 14, 25, 0, 67, 1508, 10, 52, 73, 1, 17, 216, 210, 7, 3120, 1234, 1286, 379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 369, 35, 417, 145, 2, 66, 5, 226, 151, 399, 4193, 3, 140, 365, 65, 959, 642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 818, 0, 35, 1, 164, 905, 134, 49, 24, 4064, 396, 50, 59, 9, 1660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8822, 8208, 6780, 8285, 8447, 8227, 8305, 8462] avg=8192.0000 max=8822 max_rank=0 min=6780 min_rank=2 imbalance_ratio=1.076904 max_load_reduction=56714 max_load_reduction_pct=0.865387 moved_to_echo=33441 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3128, 72, 2, 131, 120, 480, 84, 0, 8, 336, 1226, 119, 1754, 283, 92, 430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 1360, 35, 27, 833, 3, 14, 1762, 1424, 35, 3, 0, 2316, 130, 49, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 1565, 172, 52, 28, 363, 11, 132, 1109, 1, 797, 2251, 96, 30, 17, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 89, 5, 12, 1, 310, 46, 7, 177, 17, 15, 450, 369, 13, 94, 278, 4112, 2191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1519, 27, 1, 0, 807, 0, 9, 1098, 1507, 32, 1924, 221, 622, 11, 174, 217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 1, 4, 21, 16, 0, 157, 1400, 3, 55, 189, 7, 51, 145, 377, 8, 3373, 1448, 1071, 393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 232, 13, 319, 99, 4, 49, 25, 416, 190, 408, 3558, 20, 148, 683, 98, 884, 666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 589, 0, 93, 0, 125, 699, 297, 49, 19, 3748, 294, 57, 42, 31, 1736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8265, 8543, 7378, 8200, 8169, 8750, 8308, 7923] avg=8192.0000 max=8750 max_rank=5 min=7378 min_rank=2 imbalance_ratio=1.068115 max_load_reduction=56786 max_load_reduction_pct=0.866486 moved_to_echo=33150 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2818, 4, 1, 197, 78, 463, 131, 0, 30, 381, 2161, 192, 1502, 103, 319, 380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 2076, 11, 36, 754, 3, 54, 1373, 930, 48, 1, 0, 2088, 101, 35, 626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 961, 1359, 226, 7, 46, 892, 7, 96, 715, 12, 954, 3254, 10, 79, 78, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 78, 0, 20, 1, 283, 23, 10, 48, 8, 22, 200, 381, 7, 260, 398, 4470, 2102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1445, 3, 2, 0, 890, 4, 31, 827, 1067, 24, 2345, 277, 438, 4, 396, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 12, 0, 27, 45, 0, 108, 1361, 9, 66, 164, 8, 103, 196, 145, 32, 2618, 1378, 1028, 355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 279, 201, 11, 741, 56, 12, 114, 38, 146, 165, 360, 3452, 10, 166, 371, 111, 866, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 886, 4, 26, 1, 222, 516, 233, 69, 8, 3673, 489, 46, 60, 11, 1723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8760, 8180, 8897, 8326, 7905, 7667, 7699, 8102] avg=8192.0000 max=8897 max_rank=2 min=7667 min_rank=5 imbalance_ratio=1.086060 max_load_reduction=56639 max_load_reduction_pct=0.864243 moved_to_echo=31373 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3727, 33, 5, 432, 63, 708, 83, 0, 36, 224, 1107, 69, 1557, 59, 219, 438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 2035, 7, 25, 1006, 12, 31, 830, 1064, 44, 0, 0, 2093, 97, 32, 417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1304, 145, 19, 125, 398, 5, 14, 536, 8, 698, 3850, 1, 19, 30, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 53, 12, 19, 0, 241, 22, 4, 95, 10, 31, 542, 227, 41, 125, 125, 4044, 2233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1431, 11, 2, 2, 1498, 0, 41, 615, 1275, 2, 2360, 317, 517, 3, 322, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 22, 0, 14, 17, 10, 85, 617, 3, 97, 255, 18, 27, 35, 323, 14, 2658, 1373, 1095, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 309, 391, 11, 818, 37, 3, 44, 30, 158, 83, 671, 4032, 20, 108, 305, 34, 898, 799, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 195, 913, 1, 120, 0, 204, 348, 231, 106, 28, 3790, 375, 58, 80, 25, 1761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8760, 7806, 8374, 7834, 8630, 7146, 8751, 8235] avg=8192.0000 max=8760 max_rank=0 min=7146 min_rank=5 imbalance_ratio=1.069336 max_load_reduction=56776 max_load_reduction_pct=0.866333 moved_to_echo=32762 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3135, 30, 0, 258, 90, 690, 66, 0, 39, 417, 590, 184, 1496, 63, 211, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 2023, 7, 85, 1104, 7, 32, 1430, 1147, 53, 6, 0, 2365, 133, 86, 625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 821, 1370, 243, 43, 136, 517, 8, 71, 1088, 12, 884, 3666, 0, 18, 40, 213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 45, 1, 6, 16, 260, 6, 10, 164, 46, 36, 206, 260, 111, 330, 185, 4766, 2225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1300, 7, 0, 1, 1222, 10, 53, 397, 952, 16, 2706, 148, 525, 15, 363, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 2, 0, 58, 32, 0, 113, 479, 8, 74, 385, 20, 136, 73, 97, 19, 3454, 1239, 1088, 394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 116, 23, 460, 33, 14, 140, 30, 127, 193, 421, 3280, 10, 161, 353, 74, 925, 668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 187, 911, 2, 85, 1, 204, 416, 93, 105, 62, 3923, 387, 84, 97, 15, 1602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7566, 9149, 9130, 8691, 7935, 7699, 7192, 8174] avg=8192.0000 max=9149 max_rank=1 min=7192 min_rank=6 imbalance_ratio=1.116821 max_load_reduction=56387 max_load_reduction_pct=0.860397 moved_to_echo=31000 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] + [2026-08-26 15:24:10] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 693.3 | throughput per GPU (TFLOP/s/GPU): 260.8 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.117604E+01 | loss scale: 1.0 | grad norm: 44.306 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[4024, 22, 1, 92, 170, 594, 205, 0, 1, 533, 1541, 249, 1291, 144, 216, 744, 115, 1911, 4, 7, 1278, 0, 41, 661, 1284, 25, 1, 4140, 2229, 3238, 175, 366, 745, 1055, 149, 1, 40, 1202, 23, 38, 536, 4, 388, 2279, 0, 26, 38, 4181, 7, 116, 3, 11, 4, 89, 5, 3, 200, 11, 12, 190, 265, 6, 258, 315, 2536, 5, 0, 0, 981, 1, 30, 978, 953, 9, 2401, 403, 597, 0, 501, 277, 8, 0, 0, 14, 25, 0, 66, 1523, 11, 51, 74, 1, 18, 214, 206, 7, 123, 365, 37, 418, 142, 2, 67, 5, 222, 146, 394, 4198, 3, 139, 365, 66, 94, 819, 0, 35, 1, 164, 915, 135, 47, 25, 5037, 399, 50, 58, 9, 1664] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[3817, 65, 1, 133, 142, 360, 61, 0, 14, 409, 1521, 119, 1629, 146, 155, 460, 165, 1657, 29, 12, 990, 25, 18, 611, 637, 36, 1, 4003, 1764, 3391, 115, 263, 924, 1798, 162, 17, 91, 1114, 23, 127, 1222, 2, 541, 2681, 0, 44, 31, 4058, 8, 42, 4, 14, 1, 185, 15, 8, 238, 17, 10, 204, 297, 21, 156, 144, 2470, 2, 4, 1, 1566, 2, 22, 765, 1438, 84, 2117, 418, 414, 71, 595, 74, 22, 17, 0, 22, 31, 1, 148, 1604, 5, 79, 208, 39, 105, 70, 163, 3, 161, 270, 16, 414, 59, 9, 321, 88, 160, 436, 236, 4027, 2, 278, 389, 104, 216, 762, 0, 38, 0, 365, 376, 66, 69, 26, 4793, 325, 86, 49, 6, 1881] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[4177, 74, 3, 128, 121, 473, 87, 0, 8, 334, 1221, 122, 1749, 287, 95, 427, 61, 1353, 32, 26, 827, 3, 12, 1757, 1433, 35, 3, 4102, 2332, 3507, 51, 491, 548, 1558, 170, 52, 28, 360, 10, 130, 1102, 2, 793, 2248, 96, 31, 17, 3463, 13, 91, 6, 10, 1, 310, 47, 7, 181, 17, 16, 455, 369, 13, 97, 278, 2960, 28, 1, 0, 805, 0, 10, 1096, 1505, 32, 1912, 227, 622, 11, 173, 216, 31, 1, 4, 21, 14, 0, 156, 1406, 3, 56, 187, 7, 49, 148, 373, 8, 495, 233, 13, 323, 98, 4, 47, 25, 416, 188, 410, 3561, 20, 147, 691, 98, 146, 600, 0, 96, 0, 126, 705, 296, 48, 20, 4639, 297, 58, 43, 31, 1750] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[3753, 4, 1, 195, 78, 462, 136, 0, 30, 385, 2164, 192, 1493, 103, 325, 381, 43, 2086, 12, 37, 755, 3, 50, 1375, 944, 46, 2, 4481, 2100, 2719, 37, 621, 952, 1352, 223, 9, 45, 883, 7, 95, 713, 11, 960, 3249, 10, 81, 81, 3323, 15, 75, 0, 22, 1, 281, 23, 11, 49, 8, 22, 197, 380, 7, 262, 397, 2817, 3, 2, 0, 891, 4, 33, 823, 1063, 23, 2329, 272, 440, 4, 401, 153, 12, 12, 0, 28, 44, 0, 107, 1374, 9, 65, 163, 8, 99, 198, 143, 33, 276, 197, 12, 735, 53, 12, 114, 38, 141, 165, 358, 3468, 11, 168, 372, 111, 137, 901, 4, 26, 1, 220, 516, 232, 68, 8, 4546, 490, 47, 59, 11, 1729] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[3527, 68, 4, 228, 226, 629, 116, 3, 28, 503, 1279, 204, 1268, 110, 323, 211, 110, 1678, 1, 28, 876, 12, 52, 1718, 1403, 83, 11, 4594, 2363, 3636, 110, 629, 500, 1032, 183, 4, 84, 931, 10, 62, 1082, 8, 638, 2845, 4, 75, 20, 3587, 49, 150, 2, 22, 9, 141, 15, 6, 191, 12, 40, 340, 268, 134, 243, 227, 2795, 5, 3, 0, 1205, 0, 29, 473, 936, 23, 1275, 611, 375, 26, 505, 222, 28, 5, 10, 13, 44, 1, 76, 1222, 4, 120, 249, 3, 36, 161, 261, 13, 336, 191, 16, 939, 41, 4, 68, 102, 159, 328, 333, 3369, 2, 299, 485, 102, 380, 961, 1, 125, 0, 241, 542, 143, 31, 27, 4699, 413, 84, 43, 3, 1395] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[4179, 30, 0, 258, 87, 690, 66, 0, 39, 418, 598, 186, 1483, 63, 213, 300, 47, 2031, 7, 85, 1107, 7, 30, 1438, 1150, 54, 6, 4779, 2372, 3590, 89, 621, 819, 1359, 238, 44, 136, 515, 6, 69, 1088, 12, 888, 3666, 0, 18, 41, 3520, 18, 46, 1, 6, 15, 260, 5, 10, 162, 45, 34, 205, 261, 108, 333, 187, 2556, 7, 0, 1, 1217, 10, 53, 400, 952, 15, 2701, 153, 518, 15, 362, 216, 28, 2, 0, 60, 30, 0, 113, 491, 8, 74, 384, 21, 133, 72, 95, 18, 158, 116, 23, 457, 34, 14, 140, 30, 127, 190, 424, 3277, 9, 158, 351, 72, 191, 921, 2, 83, 1, 201, 414, 91, 104, 59, 4859, 389, 82, 96, 15, 1605] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[3734, 16, 1, 91, 109, 319, 118, 1, 18, 396, 1081, 178, 1545, 90, 341, 391, 66, 2345, 5, 22, 790, 9, 26, 802, 544, 39, 12, 3921, 2777, 3661, 22, 468, 1079, 2392, 144, 0, 150, 1040, 4, 75, 423, 24, 215, 2404, 3, 18, 16, 3422, 59, 82, 3, 4, 8, 372, 3, 9, 166, 15, 25, 185, 249, 9, 221, 57, 2951, 10, 1, 0, 1124, 2, 21, 709, 1029, 12, 1778, 842, 462, 7, 365, 570, 3, 3, 0, 100, 46, 1, 66, 1646, 7, 66, 155, 4, 13, 72, 302, 19, 232, 389, 53, 890, 28, 10, 47, 18, 196, 111, 372, 4075, 11, 176, 435, 45, 65, 977, 0, 54, 0, 126, 472, 402, 119, 4, 4673, 302, 27, 51, 2, 1974] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[5003, 32, 5, 425, 62, 708, 84, 0, 36, 228, 1108, 68, 1550, 60, 224, 437, 114, 2041, 8, 25, 1005, 12, 31, 829, 1063, 45, 0, 4041, 2109, 2765, 32, 412, 1015, 1297, 145, 20, 123, 398, 6, 14, 540, 8, 693, 3847, 1, 19, 31, 3540, 11, 56, 11, 19, 0, 246, 21, 4, 95, 11, 31, 545, 226, 44, 129, 128, 2814, 11, 2, 2, 1497, 0, 41, 607, 1263, 2, 2353, 315, 512, 3, 322, 232, 8, 20, 1, 14, 20, 10, 83, 620, 3, 97, 255, 20, 26, 35, 321, 14, 299, 387, 10, 813, 38, 3, 40, 30, 157, 83, 672, 4040, 20, 107, 304, 35, 194, 913, 1, 120, 0, 199, 356, 231, 103, 26, 4689, 374, 57, 81, 25, 1770] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3010, 22, 1, 92, 170, 594, 205, 0, 1, 533, 1541, 249, 1291, 144, 216, 744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 1911, 4, 7, 1278, 0, 41, 661, 1284, 25, 1, 0, 2229, 100, 175, 366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 745, 1055, 149, 1, 40, 1202, 23, 38, 536, 4, 388, 2279, 0, 26, 38, 274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 116, 3, 11, 4, 89, 5, 3, 200, 11, 12, 190, 265, 6, 258, 315, 4140, 2639, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1314, 5, 0, 0, 981, 1, 30, 978, 953, 9, 2401, 403, 597, 0, 501, 277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 14, 25, 0, 66, 1523, 11, 51, 74, 1, 18, 214, 206, 7, 3138, 1222, 1268, 392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 365, 37, 418, 142, 2, 67, 5, 222, 146, 394, 4198, 3, 139, 365, 66, 984, 622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 819, 0, 35, 1, 164, 915, 135, 47, 25, 4053, 399, 50, 58, 9, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8813, 8197, 6798, 8274, 8450, 8238, 8298, 8468] avg=8192.0000 max=8813 max_rank=0 min=6798 min_rank=2 imbalance_ratio=1.075806 max_load_reduction=56723 max_load_reduction_pct=0.865524 moved_to_echo=33454 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3125, 74, 3, 128, 121, 473, 87, 0, 8, 334, 1221, 122, 1749, 287, 95, 427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 1353, 32, 26, 827, 3, 12, 1757, 1433, 35, 3, 0, 2332, 109, 51, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 1558, 170, 52, 28, 360, 10, 130, 1102, 2, 793, 2248, 96, 31, 17, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 91, 6, 10, 1, 310, 47, 7, 181, 17, 16, 455, 369, 13, 97, 278, 4102, 2186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1534, 28, 1, 0, 805, 0, 10, 1096, 1505, 32, 1912, 227, 622, 11, 173, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 1, 4, 21, 14, 0, 156, 1406, 3, 56, 187, 7, 49, 148, 373, 8, 3398, 1426, 1051, 407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 233, 13, 323, 98, 4, 47, 25, 416, 188, 410, 3561, 20, 147, 691, 98, 906, 645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 600, 0, 96, 0, 126, 705, 296, 48, 20, 3733, 297, 58, 43, 31, 1750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8254, 8525, 7371, 8199, 8172, 8746, 8320, 7949] avg=8192.0000 max=8746 max_rank=5 min=7371 min_rank=2 imbalance_ratio=1.067627 max_load_reduction=56790 max_load_reduction_pct=0.866547 moved_to_echo=33187 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2638, 68, 4, 228, 226, 629, 116, 3, 28, 503, 1279, 204, 1268, 110, 323, 211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 1678, 1, 28, 876, 12, 52, 1718, 1403, 83, 11, 0, 2363, 113, 110, 629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 1032, 183, 4, 84, 931, 10, 62, 1082, 8, 638, 2845, 4, 75, 20, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 150, 2, 22, 9, 141, 15, 6, 191, 12, 40, 340, 268, 134, 243, 227, 4594, 2264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1449, 5, 3, 0, 1205, 0, 29, 473, 936, 23, 1275, 611, 375, 26, 505, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 5, 10, 13, 44, 1, 76, 1222, 4, 120, 249, 3, 36, 161, 261, 13, 3523, 1346, 1088, 344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 336, 191, 16, 939, 41, 4, 68, 102, 159, 328, 333, 3369, 2, 299, 485, 102, 918, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 380, 961, 1, 125, 0, 241, 542, 143, 31, 27, 3781, 413, 84, 43, 3, 1395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7838, 9187, 7713, 8707, 7137, 8547, 8237, 8170] avg=8192.0000 max=9187 max_rank=1 min=7137 min_rank=4 imbalance_ratio=1.121460 max_load_reduction=56349 max_load_reduction_pct=0.859818 moved_to_echo=32091 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2855, 65, 1, 133, 142, 360, 61, 0, 14, 409, 1521, 119, 1629, 146, 155, 460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 165, 1657, 29, 12, 990, 25, 18, 611, 637, 36, 1, 0, 1764, 105, 115, 263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 924, 1798, 162, 17, 91, 1114, 23, 127, 1222, 2, 541, 2681, 0, 44, 31, 266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 42, 4, 14, 1, 185, 15, 8, 238, 17, 10, 204, 297, 21, 156, 144, 4003, 2561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1280, 2, 4, 1, 1566, 2, 22, 765, 1438, 84, 2117, 418, 414, 71, 595, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 17, 0, 22, 31, 1, 148, 1604, 5, 79, 208, 39, 105, 70, 163, 3, 3286, 1190, 1231, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 270, 16, 414, 59, 9, 321, 88, 160, 436, 236, 4027, 2, 278, 389, 104, 936, 590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 762, 0, 38, 0, 365, 376, 66, 69, 26, 3857, 325, 86, 49, 6, 1881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8070, 6428, 9043, 7928, 8853, 8596, 8496, 8122] avg=8192.0000 max=9043 max_rank=2 min=6428 min_rank=1 imbalance_ratio=1.103882 max_load_reduction=56493 max_load_reduction_pct=0.862015 moved_to_echo=34067 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2807, 4, 1, 195, 78, 462, 136, 0, 30, 385, 2164, 192, 1493, 103, 325, 381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 2086, 12, 37, 755, 3, 50, 1375, 944, 46, 2, 0, 2100, 84, 37, 621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 952, 1352, 223, 9, 45, 883, 7, 95, 713, 11, 960, 3249, 10, 81, 81, 218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 75, 0, 22, 1, 281, 23, 11, 49, 8, 22, 197, 380, 7, 262, 397, 4481, 2097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1460, 3, 2, 0, 891, 4, 33, 823, 1063, 23, 2329, 272, 440, 4, 401, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 12, 0, 28, 44, 0, 107, 1374, 9, 65, 163, 8, 99, 198, 143, 33, 2635, 1357, 1008, 366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 276, 197, 12, 735, 53, 12, 114, 38, 141, 165, 358, 3468, 11, 168, 372, 111, 888, 580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 901, 4, 26, 1, 220, 516, 232, 68, 8, 3658, 490, 47, 59, 11, 1729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8756, 8195, 8889, 8328, 7901, 7661, 7699, 8107] avg=8192.0000 max=8889 max_rank=2 min=7661 min_rank=5 imbalance_ratio=1.085083 max_load_reduction=56647 max_load_reduction_pct=0.864365 moved_to_echo=31368 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3733, 32, 5, 425, 62, 708, 84, 0, 36, 228, 1108, 68, 1550, 60, 224, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 2041, 8, 25, 1005, 12, 31, 829, 1063, 45, 0, 0, 2109, 83, 32, 412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1297, 145, 20, 123, 398, 6, 14, 540, 8, 693, 3847, 1, 19, 31, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 56, 11, 19, 0, 246, 21, 4, 95, 11, 31, 545, 226, 44, 129, 128, 4041, 2237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1455, 11, 2, 2, 1497, 0, 41, 607, 1263, 2, 2353, 315, 512, 3, 322, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 20, 1, 14, 20, 10, 83, 620, 3, 97, 255, 20, 26, 35, 321, 14, 2682, 1359, 1078, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 299, 387, 10, 813, 38, 3, 40, 30, 157, 83, 672, 4040, 20, 107, 304, 35, 918, 777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 194, 913, 1, 120, 0, 199, 356, 231, 103, 26, 3771, 374, 57, 81, 25, 1770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8760, 7809, 8382, 7855, 8617, 7159, 8733, 8221] avg=8192.0000 max=8760 max_rank=0 min=7159 min_rank=5 imbalance_ratio=1.069336 max_load_reduction=56776 max_load_reduction_pct=0.866333 moved_to_echo=32730 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3126, 30, 0, 258, 87, 690, 66, 0, 39, 418, 598, 186, 1483, 63, 213, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 2031, 7, 85, 1107, 7, 30, 1438, 1150, 54, 6, 0, 2372, 111, 89, 621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 819, 1359, 238, 44, 136, 515, 6, 69, 1088, 12, 888, 3666, 0, 18, 41, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 46, 1, 6, 15, 260, 5, 10, 162, 45, 34, 205, 261, 108, 333, 187, 4779, 2222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1325, 7, 0, 1, 1217, 10, 53, 400, 952, 15, 2701, 153, 518, 15, 362, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 2, 0, 60, 30, 0, 113, 491, 8, 74, 384, 21, 133, 72, 95, 18, 3479, 1231, 1068, 407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 116, 23, 457, 34, 14, 140, 30, 127, 190, 424, 3277, 9, 158, 351, 72, 949, 646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 191, 921, 2, 83, 1, 201, 414, 91, 104, 59, 3910, 389, 82, 96, 15, 1605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7557, 9155, 9129, 8697, 7945, 7714, 7175, 8164] avg=8192.0000 max=9155 max_rank=1 min=7175 min_rank=6 imbalance_ratio=1.117554 max_load_reduction=56381 max_load_reduction_pct=0.860306 moved_to_echo=30998 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2793, 16, 1, 91, 109, 319, 118, 1, 18, 396, 1081, 178, 1545, 90, 341, 391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 2345, 5, 22, 790, 9, 26, 802, 544, 39, 12, 0, 2777, 113, 22, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1079, 2392, 144, 0, 150, 1040, 4, 75, 423, 24, 215, 2404, 3, 18, 16, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 82, 3, 4, 8, 372, 3, 9, 166, 15, 25, 185, 249, 9, 221, 57, 3921, 2160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1529, 10, 1, 0, 1124, 2, 21, 709, 1029, 12, 1778, 842, 462, 7, 365, 570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 100, 46, 1, 66, 1646, 7, 66, 155, 4, 13, 72, 302, 19, 3548, 1422, 1038, 364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 232, 389, 53, 890, 28, 10, 47, 18, 196, 111, 372, 4075, 11, 176, 435, 45, 913, 577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 977, 0, 54, 0, 126, 472, 402, 119, 4, 3760, 302, 27, 51, 2, 1974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7488, 8040, 8211, 7548, 8461, 8875, 8578, 8335] avg=8192.0000 max=8875 max_rank=5 min=7488 min_rank=0 imbalance_ratio=1.083374 max_load_reduction=56661 max_load_reduction_pct=0.864578 moved_to_echo=34249 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[768, 15104, 128, 256, 7680, 128, 384, 9216, 8448, 384, 128, 0, 18048, 1024, 640, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[256, 128, 128, 384, 256, 128, 896, 9856, 128, 640, 1792, 128, 512, 1024, 1920, 128, 25600, 10752, 9088, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[24192, 384, 128, 1664, 1024, 4352, 896, 128, 256, 3200, 10496, 1408, 12160, 1024, 1920, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[6656, 12032, 1536, 256, 768, 6528, 128, 640, 6784, 128, 5120, 23296, 128, 384, 384, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[2176, 2176, 256, 5120, 512, 128, 896, 384, 1664, 1792, 3200, 30080, 128, 1536, 3456, 640, 7296, 5248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[11264, 128, 128, 128, 9344, 128, 256, 5888, 9216, 256, 17024, 3328, 3968, 256, 3328, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[1536, 6912, 128, 640, 128, 1664, 4352, 1664, 640, 256, 30720, 3072, 512, 512, 128, 13824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[256, 768, 128, 128, 128, 1920, 256, 128, 1280, 256, 256, 2432, 2432, 384, 1792, 1792, 34176, 18432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[768, 15104, 128, 256, 7680, 128, 384, 9216, 8576, 384, 128, 0, 18048, 896, 640, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[256, 128, 128, 384, 256, 128, 896, 9984, 128, 640, 1792, 128, 512, 1024, 1920, 128, 25728, 10624, 8832, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[11392, 128, 128, 128, 9344, 128, 256, 5888, 9216, 256, 16896, 3328, 3968, 256, 3328, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[6656, 11904, 1536, 256, 768, 6528, 128, 640, 6784, 128, 5120, 23296, 128, 384, 384, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[24192, 384, 128, 1664, 1024, 4352, 896, 128, 256, 3328, 10624, 1408, 12032, 1024, 1920, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[2176, 2176, 256, 4992, 512, 128, 896, 384, 1664, 1664, 3200, 30080, 128, 1536, 3456, 640, 7424, 4992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[256, 768, 128, 128, 128, 1920, 256, 128, 1408, 256, 256, 2432, 2432, 384, 1792, 1792, 34176, 18432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[1536, 6912, 128, 640, 128, 1664, 4352, 1664, 640, 256, 30592, 3072, 512, 512, 128, 13824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 24, 0, 224, 1, 2, 2, 0, 0, 0, 0, 0, 7872, 169, 0, 11, 0, 0, 7396, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 7, 0, 1, 65, 0, 0, 0, 89, 0, 0, 2, 0, 0, 0, 6, 0, 11, 0, 0, 0, 0, 4411, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 34, 0, 0, 601, 8033, 0, 0, 1, 0, 8151, 143, 8191, 0, 1, 74, 0, 10, 0, 574, 0, 3602, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 1, 0, 2, 7626, 0, 0, 0, 8185, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 0, 23, 1, 0, 0, 4, 1, 0, 0, 8, 0, 0, 8150, 0, 0, 2, 6, 0, 7140, 0, 20, 16, 1, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 2, 0, 106, 0, 0, 0, 0, 0, 0, 3, 0, 0, 235, 34, 4, 0, 0, 0, 0, 5, 0, 415, 0, 84, 361, 5342, 0, 0, 0, 0, 0, 0, 1, 3, 1, 0, 4, 0, 0, 404, 0, 0, 8, 7978, 0, 0, 0, 0, 7541, 13, 8191, 0, 3, 31, 0, 229, 0, 2441, 0, 7793, 0, 0, 1, 0, 3, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 61, 5, 2, 624, 0, 0, 0, 8179, 2, 28, 11, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 9, 0, 0, 0, 4, 3, 2, 0, 18, 0, 0, 0, 0, 1, 0, 7978, 0, 0, 0, 0, 0, 2707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 5, 0, 0, 0, 9, 6, 0, 0, 3, 0, 2, 1, 106, 0, 16, 0, 1, 0, 0, 34, 0, 1, 0, 7482, 1, 0, 0, 0, 11, 0, 16, 0, 0, 0, 0, 1, 0, 790, 0, 0, 27, 8152, 4, 0, 1, 0, 7745, 0, 8174, 1, 0, 523, 0, 0, 0, 509, 114, 8142, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 3, 7, 0, 0, 0, 0, 0, 0, 0, 1, 4618, 0, 0, 0, 8172, 34, 55, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 18, 0, 16, 21, 5, 0, 0, 11, 0, 0, 0, 0, 0, 7609, 0, 0, 1, 0, 0, 7354, 0, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 2, 1, 14, 2, 0, 15, 26, 0, 0, 127, 0, 0, 1, 338, 0, 72, 0, 0, 0, 0, 7799, 0, 0, 0, 14, 0, 0, 0, 3, 0, 0, 23, 18, 0, 3, 0, 0, 69, 7989, 0, 0, 0, 0, 5360, 9, 8160, 0, 1172, 0, 0, 0, 0, 84, 0, 8042, 0, 0, 32, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 335, 0, 0, 3095, 0, 0, 0, 7675, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[9, 1, 0, 0, 0, 1, 0, 0, 76, 189, 0, 0, 0, 276, 10, 0, 7852, 3, 0, 0, 0, 0, 7350, 0, 2, 0, 0, 0, 5, 0, 31, 0, 0, 0, 15, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3, 31, 108, 1, 0, 781, 0, 28, 248, 0, 1, 0, 154, 0, 1, 0, 1, 1, 2, 5554, 0, 1, 0, 0, 3, 0, 33, 0, 0, 0, 18, 0, 0, 0, 0, 3, 461, 5678, 1, 3, 2, 1, 8112, 0, 8115, 0, 5, 3, 1, 3, 0, 13, 1, 5143, 2, 0, 0, 21, 0, 0, 0, 2, 0, 0, 36, 0, 0, 0, 2, 0, 3, 33, 0, 0, 8136, 0, 1, 0, 6960, 2, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 1, 0, 0, 5, 7936, 1, 0, 0, 0, 0, 4765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 59, 0, 3, 334, 0, 2, 0, 62, 0, 0, 0, 0, 18, 38, 7, 1, 15, 0, 0, 0, 0, 6132, 8, 0, 0, 12, 1, 0, 0, 0, 0, 0, 0, 0, 2, 1284, 0, 0, 86, 8089, 0, 0, 2, 2, 8091, 0, 8162, 0, 0, 34, 0, 0, 0, 2049, 0, 8188, 0, 1, 8, 4, 0, 6, 0, 0, 0, 2, 3, 4, 0, 0, 0, 3, 0, 2, 2, 80, 1957, 9, 0, 0, 8054, 0, 0, 0, 6, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[13, 0, 0, 1, 0, 27, 2, 0, 6, 13, 0, 0, 0, 0, 0, 0, 8146, 0, 0, 0, 0, 7, 3325, 2, 0, 0, 0, 0, 0, 7, 0, 16, 0, 7, 0, 0, 0, 9, 0, 1, 7, 0, 0, 0, 0, 63, 0, 1, 0, 0, 0, 1, 176, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 6048, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 7, 0, 790, 0, 0, 1452, 8084, 0, 0, 0, 0, 8155, 0, 8171, 0, 2, 111, 15, 0, 0, 1113, 2, 6773, 0, 1, 0, 0, 0, 11, 0, 0, 0, 3, 183, 21, 0, 13, 0, 2, 0, 241, 0, 202, 4106, 0, 0, 0, 8190, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 6, 0, 5, 7937, 15, 0, 0, 0, 0, 8039, 0, 685, 19, 1, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 1, 2, 6, 1, 1, 0, 0, 0, 0, 0, 2, 0, 0, 5, 58, 9, 0, 0, 0, 0, 0, 4, 0, 0, 30, 0, 3646, 8, 0, 0, 0, 0, 1, 29, 1, 0, 0, 0, 0, 1, 11, 1, 0, 10, 7902, 2, 22, 116, 0, 8168, 0, 8177, 0, 11, 37, 8, 45, 3, 4604, 10, 6702, 0, 1, 0, 3, 0, 5, 0, 0, 0, 1, 8, 1, 0, 1, 0, 0, 0, 640, 1, 0, 1427, 0, 0, 1, 7089, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 24, 0, 224, 1, 2, 2, 0, 0, 0, 0, 0, 8033, 228, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2038, 169, 0, 11, 0, 0, 7396, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 7, 0, 1, 65, 0, 0, 8100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 0, 2, 0, 0, 0, 6, 0, 11, 0, 0, 0, 0, 4411, 1774, 257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 34, 0, 0, 5606, 2196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 601, 0, 0, 0, 1, 0, 6486, 143, 0, 0, 1, 74, 0, 10, 0, 574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3602, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 7626, 0, 0, 0, 4215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8605, 9615, 8175, 6550, 7840, 7890, 5017, 11844] avg=8192.0000 max=11844 max_rank=7 min=5017 min_rank=6 imbalance_ratio=1.445801 max_load_reduction=53692 max_load_reduction_pct=0.819275 moved_to_echo=32591 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 0, 23, 1, 0, 0, 4, 1, 0, 0, 8, 0, 0, 7978, 236, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2110, 0, 0, 2, 6, 0, 7140, 0, 20, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 2, 0, 106, 0, 0, 0, 0, 0, 0, 3, 8100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 235, 34, 4, 0, 0, 0, 0, 5, 0, 415, 0, 84, 361, 5342, 1773, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 1, 0, 4, 0, 0, 404, 0, 0, 5804, 2195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 6000, 13, 0, 0, 3, 31, 0, 229, 0, 2441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7793, 0, 0, 1, 0, 3, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 61, 5, 2, 624, 0, 0, 0, 4211, 2, 28, 11, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8346, 9295, 8218, 8491, 8412, 8725, 9102, 4947] avg=8192.0000 max=9295 max_rank=1 min=4947 min_rank=7 imbalance_ratio=1.134644 max_load_reduction=56241 max_load_reduction_pct=0.858170 moved_to_echo=31186 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 1, 0, 0, 5, 8089, 230, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2055, 1, 0, 0, 0, 0, 4765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 59, 0, 3, 334, 0, 2, 8071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 18, 38, 7, 1, 15, 0, 0, 0, 0, 6132, 1746, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 12, 1, 0, 0, 0, 0, 0, 0, 0, 2, 1284, 0, 0, 5651, 2161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 2, 2, 6438, 0, 0, 0, 0, 34, 0, 0, 0, 2049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 1, 8, 4, 0, 6, 0, 0, 0, 2, 3, 4, 0, 0, 1398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 2, 80, 1957, 9, 0, 0, 4147, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8421, 6821, 8470, 8274, 9119, 8611, 9614, 6206] avg=8192.0000 max=9614 max_rank=6 min=6206 min_rank=7 imbalance_ratio=1.173584 max_load_reduction=55922 max_load_reduction_pct=0.853302 moved_to_echo=33550 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 9, 0, 0, 0, 4, 3, 2, 0, 18, 0, 0, 0, 0, 1, 0, 8152, 231, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2066, 0, 0, 0, 0, 0, 2707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 5, 0, 0, 0, 9, 6, 0, 0, 8084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 1, 106, 0, 16, 0, 1, 0, 0, 34, 0, 1, 0, 7482, 1771, 244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 11, 0, 16, 0, 0, 0, 0, 1, 0, 790, 0, 0, 5681, 2193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 4, 0, 1, 0, 6163, 0, 0, 1, 0, 523, 0, 0, 0, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 8142, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 3, 7, 0, 0, 1338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4618, 0, 0, 0, 4208, 34, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8510, 4773, 8120, 9661, 8693, 7228, 9635, 8916] avg=8192.0000 max=9661 max_rank=3 min=4773 min_rank=1 imbalance_ratio=1.179321 max_load_reduction=55875 max_load_reduction_pct=0.852585 moved_to_echo=34472 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 18, 0, 16, 21, 5, 0, 0, 11, 0, 0, 0, 0, 0, 7989, 220, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1971, 0, 0, 1, 0, 0, 7354, 0, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 2, 1, 14, 2, 0, 8069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 26, 0, 0, 127, 0, 0, 1, 338, 0, 72, 0, 0, 0, 0, 7799, 1664, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 3, 0, 0, 23, 18, 0, 3, 0, 0, 5418, 2060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 4265, 9, 0, 0, 1172, 0, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8042, 0, 0, 32, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 335, 0, 0, 3095, 0, 0, 0, 3951, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8371, 9335, 8098, 10211, 7539, 5599, 9001, 7382] avg=8192.0000 max=10211 max_rank=3 min=5599 min_rank=5 imbalance_ratio=1.246460 max_load_reduction=55325 max_load_reduction_pct=0.844193 moved_to_echo=29521 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[13, 0, 0, 1, 0, 27, 2, 0, 6, 13, 0, 0, 0, 0, 0, 0, 8084, 236, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2109, 0, 0, 0, 0, 7, 3325, 2, 0, 0, 0, 0, 0, 7, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 9, 0, 1, 7, 0, 0, 0, 0, 63, 0, 1, 8081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 176, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 6048, 1775, 257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 7, 0, 790, 0, 0, 5801, 2198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1452, 0, 0, 0, 0, 0, 6489, 0, 0, 0, 2, 111, 15, 0, 0, 1113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6773, 0, 1, 0, 0, 0, 11, 0, 0, 0, 3, 183, 21, 0, 13, 1409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 241, 0, 202, 4106, 0, 0, 0, 4217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8472, 5466, 8169, 8261, 8802, 9182, 8416, 8768] avg=8192.0000 max=9182 max_rank=5 min=5466 min_rank=1 imbalance_ratio=1.120850 max_load_reduction=56354 max_load_reduction_pct=0.859894 moved_to_echo=35168 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[9, 1, 0, 0, 0, 1, 0, 0, 76, 189, 0, 0, 0, 276, 10, 0, 5678, 227, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2034, 3, 0, 0, 0, 0, 7350, 0, 2, 0, 0, 0, 5, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3, 31, 108, 1, 8024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 781, 0, 28, 248, 0, 1, 0, 154, 0, 1, 0, 1, 1, 2, 5554, 1509, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 33, 0, 0, 0, 18, 0, 0, 0, 0, 3, 5591, 1868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 0, 1, 3, 2, 1, 6455, 0, 0, 0, 5, 3, 1, 3, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5143, 2, 0, 0, 21, 0, 0, 0, 2, 0, 0, 36, 0, 0, 0, 1401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 33, 0, 0, 8136, 0, 1, 0, 3583, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6558, 9425, 8185, 8536, 7517, 6948, 6606, 11761] avg=8192.0000 max=11761 max_rank=7 min=6558 min_rank=0 imbalance_ratio=1.435669 max_load_reduction=53775 max_load_reduction_pct=0.820541 moved_to_echo=32832 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 6, 0, 5, 7902, 235, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2046, 15, 0, 0, 0, 0, 8039, 0, 685, 19, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 2, 6, 1, 1, 0, 0, 0, 0, 0, 2, 8086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 58, 9, 0, 0, 0, 0, 0, 4, 0, 0, 30, 0, 3646, 1540, 260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 1, 29, 1, 0, 0, 0, 0, 1, 11, 1, 0, 5656, 1906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 2, 22, 116, 0, 6495, 0, 0, 0, 11, 37, 8, 45, 3, 4604, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 6702, 0, 1, 0, 3, 0, 5, 0, 0, 0, 1, 8, 1, 0, 1, 1413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 640, 1, 0, 1427, 0, 0, 1, 3643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8253, 10806, 8101, 5552, 7614, 11353, 8145, 5712] avg=8192.0000 max=11353 max_rank=5 min=5552 min_rank=3 imbalance_ratio=1.385864 max_load_reduction=54183 max_load_reduction_pct=0.826767 moved_to_echo=32824 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 24, 0, 221, 1, 2, 2, 0, 0, 0, 0, 0, 7876, 164, 0, 10, 0, 0, 7348, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 6, 0, 0, 72, 0, 0, 0, 86, 0, 0, 3, 0, 0, 1, 8, 0, 10, 0, 0, 0, 0, 4435, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 37, 0, 0, 594, 8029, 0, 0, 1, 0, 8153, 151, 8190, 0, 1, 69, 0, 12, 0, 577, 0, 3638, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 1, 0, 0, 7613, 0, 0, 0, 8185, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 6, 0, 5, 7940, 15, 0, 0, 0, 0, 8033, 0, 692, 18, 1, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 1, 2, 7, 1, 1, 0, 0, 0, 0, 0, 2, 0, 0, 5, 58, 8, 0, 0, 0, 0, 0, 5, 0, 0, 32, 0, 3652, 9, 0, 0, 0, 0, 1, 29, 1, 0, 0, 0, 0, 1, 11, 1, 0, 10, 7899, 2, 22, 118, 0, 8167, 0, 8178, 0, 13, 33, 7, 43, 4, 4610, 10, 6680, 0, 1, 0, 2, 0, 5, 0, 0, 0, 1, 7, 1, 0, 0, 0, 0, 0, 647, 1, 0, 1431, 0, 0, 1, 7087, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 21, 1, 0, 0, 3, 1, 0, 0, 9, 0, 0, 8143, 0, 0, 2, 6, 0, 7122, 0, 20, 14, 1, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 2, 0, 98, 0, 0, 0, 0, 0, 0, 5, 0, 0, 234, 36, 4, 0, 0, 0, 0, 5, 0, 426, 0, 84, 363, 5342, 0, 0, 0, 0, 1, 0, 1, 3, 1, 0, 4, 0, 1, 406, 0, 0, 5, 7977, 0, 0, 0, 0, 7529, 13, 8191, 0, 3, 26, 0, 233, 0, 2478, 0, 7783, 0, 0, 1, 0, 3, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 57, 4, 2, 634, 0, 0, 0, 8179, 2, 32, 10, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[11, 0, 0, 1, 0, 26, 2, 0, 5, 12, 0, 0, 0, 0, 0, 0, 8147, 0, 0, 0, 0, 7, 3275, 2, 0, 0, 0, 0, 0, 7, 0, 15, 0, 7, 0, 0, 0, 10, 0, 1, 9, 0, 0, 0, 0, 63, 0, 1, 0, 0, 0, 0, 168, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 6101, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 7, 0, 800, 0, 0, 1387, 8092, 0, 0, 0, 0, 8153, 0, 8170, 0, 2, 108, 16, 0, 0, 1116, 2, 6790, 0, 1, 0, 0, 0, 11, 0, 0, 1, 3, 182, 24, 0, 14, 0, 2, 0, 249, 0, 204, 4132, 0, 0, 0, 8190, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 21, 0, 17, 24, 7, 0, 0, 10, 0, 0, 0, 0, 0, 7584, 0, 0, 1, 0, 0, 7325, 0, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 2, 1, 14, 2, 0, 14, 26, 0, 0, 126, 0, 0, 1, 344, 0, 70, 0, 0, 0, 0, 7802, 0, 0, 0, 15, 0, 0, 0, 3, 0, 0, 24, 17, 0, 3, 0, 0, 75, 7979, 0, 0, 0, 0, 5373, 10, 8161, 0, 1164, 0, 0, 0, 0, 86, 0, 8052, 0, 0, 32, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 337, 0, 1, 3124, 0, 0, 0, 7670, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 10, 0, 0, 0, 4, 3, 1, 0, 22, 0, 0, 0, 0, 1, 0, 7978, 0, 0, 0, 0, 0, 2684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 5, 0, 0, 0, 9, 5, 0, 0, 3, 0, 2, 1, 112, 0, 15, 0, 1, 0, 0, 34, 0, 1, 0, 7475, 1, 0, 0, 0, 11, 0, 15, 0, 0, 0, 0, 1, 0, 788, 0, 0, 24, 8152, 3, 0, 1, 0, 7758, 0, 8175, 1, 0, 501, 0, 0, 0, 518, 114, 8144, 0, 30, 0, 0, 0, 0, 0, 0, 0, 1, 5, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4640, 0, 0, 0, 8173, 32, 59, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[10, 1, 0, 0, 0, 1, 0, 0, 79, 185, 0, 0, 0, 279, 10, 0, 7844, 3, 0, 0, 0, 0, 7320, 0, 2, 0, 0, 0, 5, 0, 30, 0, 0, 0, 13, 0, 0, 0, 0, 0, 5, 0, 0, 0, 5, 32, 108, 1, 0, 774, 0, 27, 243, 0, 1, 0, 160, 0, 1, 0, 1, 0, 2, 5573, 0, 1, 0, 0, 3, 0, 32, 0, 0, 0, 21, 0, 0, 0, 0, 3, 442, 5670, 1, 3, 1, 1, 8109, 0, 8114, 0, 5, 2, 1, 3, 0, 12, 0, 5190, 2, 0, 0, 22, 0, 0, 0, 2, 0, 0, 35, 0, 0, 0, 2, 0, 3, 33, 0, 0, 8137, 0, 1, 0, 6967, 2, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 0, 1, 0, 0, 4, 7935, 0, 0, 0, 0, 0, 4689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 64, 0, 3, 334, 0, 2, 0, 64, 0, 0, 1, 0, 19, 36, 8, 1, 15, 0, 0, 0, 0, 6125, 7, 0, 0, 11, 1, 0, 0, 0, 0, 0, 0, 0, 2, 1309, 0, 0, 91, 8088, 0, 0, 1, 2, 8089, 0, 8161, 0, 0, 28, 0, 0, 0, 2069, 0, 8188, 0, 1, 8, 3, 0, 6, 0, 0, 0, 2, 3, 4, 0, 0, 0, 3, 0, 0, 2, 77, 1996, 11, 0, 0, 8058, 0, 0, 0, 6, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 24, 0, 221, 1, 2, 2, 0, 0, 0, 0, 0, 8029, 228, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2076, 164, 0, 10, 0, 0, 7348, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 6, 0, 0, 72, 0, 0, 8098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 3, 0, 0, 1, 8, 0, 10, 0, 0, 0, 0, 4435, 1755, 262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 37, 0, 0, 5572, 2233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 594, 0, 0, 0, 1, 0, 6494, 151, 0, 0, 1, 69, 0, 12, 0, 577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3638, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 1397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7613, 0, 0, 0, 4197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8599, 9599, 8180, 6560, 7846, 7899, 5042, 11811] avg=8192.0000 max=11811 max_rank=7 min=5042 min_rank=6 imbalance_ratio=1.441772 max_load_reduction=53725 max_load_reduction_pct=0.819778 moved_to_echo=32598 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 10, 0, 0, 0, 4, 3, 1, 0, 22, 0, 0, 0, 0, 1, 0, 8152, 231, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2103, 0, 0, 0, 0, 0, 2684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 5, 0, 0, 0, 9, 5, 0, 0, 8083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 1, 112, 0, 15, 0, 1, 0, 0, 34, 0, 1, 0, 7475, 1753, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 11, 0, 15, 0, 0, 0, 0, 1, 0, 788, 0, 0, 5644, 2230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 3, 0, 1, 0, 6178, 0, 0, 1, 0, 501, 0, 0, 0, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 8144, 0, 30, 0, 0, 0, 0, 0, 0, 0, 1, 5, 6, 0, 0, 1330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4640, 0, 0, 0, 4190, 32, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8516, 4787, 8119, 9647, 8690, 7226, 9630, 8921] avg=8192.0000 max=9647 max_rank=3 min=4787 min_rank=1 imbalance_ratio=1.177612 max_load_reduction=55889 max_load_reduction_pct=0.852798 moved_to_echo=34467 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 21, 0, 17, 24, 7, 0, 0, 10, 0, 0, 0, 0, 0, 7979, 220, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1999, 0, 0, 1, 0, 0, 7325, 0, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 2, 1, 14, 2, 0, 8069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 26, 0, 0, 126, 0, 0, 1, 344, 0, 70, 0, 0, 0, 0, 7802, 1645, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 3, 0, 0, 24, 17, 0, 3, 0, 0, 5365, 2092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 0, 0, 4279, 10, 0, 0, 1164, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8052, 0, 0, 32, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 337, 0, 1, 3124, 0, 0, 0, 3933, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8370, 9333, 8097, 10201, 7519, 5614, 9006, 7396] avg=8192.0000 max=10201 max_rank=3 min=5614 min_rank=5 imbalance_ratio=1.245239 max_load_reduction=55335 max_load_reduction_pct=0.844345 moved_to_echo=29535 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 21, 1, 0, 0, 3, 1, 0, 0, 9, 0, 0, 7977, 236, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2146, 0, 0, 2, 6, 0, 7122, 0, 20, 14, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 2, 0, 98, 0, 0, 0, 0, 0, 0, 5, 8099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 234, 36, 4, 0, 0, 0, 0, 5, 0, 426, 0, 84, 363, 5342, 1754, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 3, 1, 0, 4, 0, 1, 406, 0, 0, 5761, 2231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 5997, 13, 0, 0, 3, 26, 0, 233, 0, 2478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7783, 0, 0, 1, 0, 3, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 57, 4, 2, 634, 0, 0, 0, 4194, 2, 32, 10, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8343, 9311, 8211, 8490, 8409, 8755, 9079, 4938] avg=8192.0000 max=9311 max_rank=1 min=4938 min_rank=7 imbalance_ratio=1.136597 max_load_reduction=56225 max_load_reduction_pct=0.857925 moved_to_echo=31181 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[11, 0, 0, 1, 0, 26, 2, 0, 5, 12, 0, 0, 0, 0, 0, 0, 8092, 236, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2148, 0, 0, 0, 0, 7, 3275, 2, 0, 0, 0, 0, 0, 7, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 10, 0, 1, 9, 0, 0, 0, 0, 63, 0, 1, 8078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 6101, 1756, 262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 7, 0, 800, 0, 0, 5763, 2234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1387, 0, 0, 0, 0, 0, 6494, 0, 0, 0, 2, 108, 16, 0, 0, 1116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6790, 0, 1, 0, 0, 0, 11, 0, 0, 1, 3, 182, 24, 0, 14, 1397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 249, 0, 204, 4132, 0, 0, 0, 4200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8477, 5454, 8169, 8291, 8810, 9123, 8425, 8787] avg=8192.0000 max=9123 max_rank=5 min=5454 min_rank=1 imbalance_ratio=1.113647 max_load_reduction=56413 max_load_reduction_pct=0.860794 moved_to_echo=35145 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[10, 1, 0, 0, 0, 1, 0, 0, 79, 185, 0, 0, 0, 279, 10, 0, 5670, 227, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2068, 3, 0, 0, 0, 0, 7320, 0, 2, 0, 0, 0, 5, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 5, 0, 0, 0, 5, 32, 108, 1, 8023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 774, 0, 27, 243, 0, 1, 0, 160, 0, 1, 0, 1, 0, 2, 5573, 1494, 261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 32, 0, 0, 0, 21, 0, 0, 0, 0, 3, 5549, 1900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 442, 0, 1, 3, 1, 1, 6458, 0, 0, 0, 5, 2, 1, 3, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5190, 2, 0, 0, 22, 0, 0, 0, 2, 0, 0, 35, 0, 0, 0, 1390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 33, 0, 0, 8137, 0, 1, 0, 3573, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6553, 9428, 8187, 8537, 7509, 6929, 6641, 11752] avg=8192.0000 max=11752 max_rank=7 min=6553 min_rank=0 imbalance_ratio=1.434570 max_load_reduction=53784 max_load_reduction_pct=0.820679 moved_to_echo=32831 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 6, 0, 5, 7899, 236, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2083, 15, 0, 0, 0, 0, 8033, 0, 692, 18, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 2, 7, 1, 1, 0, 0, 0, 0, 0, 2, 8085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 58, 8, 0, 0, 0, 0, 0, 5, 0, 0, 32, 0, 3652, 1525, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 1, 29, 1, 0, 0, 0, 0, 1, 11, 1, 0, 5621, 1938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 2, 22, 118, 0, 6494, 0, 0, 0, 13, 33, 7, 43, 4, 4610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 6680, 0, 1, 0, 2, 0, 5, 0, 0, 0, 1, 7, 1, 0, 0, 1405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 647, 1, 0, 1431, 0, 0, 1, 3624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8255, 10843, 8101, 5553, 7612, 11356, 8112, 5704] avg=8192.0000 max=11356 max_rank=5 min=5553 min_rank=3 imbalance_ratio=1.386230 max_load_reduction=54180 max_load_reduction_pct=0.826721 moved_to_echo=32784 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 0, 1, 0, 0, 4, 8088, 230, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2092, 0, 0, 0, 0, 0, 4689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 64, 0, 3, 334, 0, 2, 8068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 1, 0, 19, 36, 8, 1, 15, 0, 0, 0, 0, 6125, 1728, 260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 11, 1, 0, 0, 0, 0, 0, 0, 0, 2, 1309, 0, 0, 5613, 2198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 1, 2, 6443, 0, 0, 0, 0, 28, 0, 0, 0, 2069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 1, 8, 3, 0, 6, 0, 0, 0, 2, 3, 4, 0, 0, 1386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 2, 77, 1996, 11, 0, 0, 4132, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8423, 6781, 8472, 8257, 9141, 8634, 9601, 6227] avg=8192.0000 max=9601 max_rank=6 min=6227 min_rank=7 imbalance_ratio=1.171997 max_load_reduction=55935 max_load_reduction_pct=0.853500 moved_to_echo=33603 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[128, 128, 0, 128, 128, 128, 128, 128, 128, 0, 128, 128, 128, 3328, 128, 128, 45312, 16896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 1408, 128, 384, 31616, 128, 128, 128, 32256, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[16512, 256, 0, 128, 128, 128, 48128, 128, 768, 128, 128, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 1024, 256, 128, 768, 0, 128, 128, 512, 128, 128, 512, 128, 128, 384, 46464, 13568, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[2816, 0, 128, 128, 128, 128, 48896, 256, 0, 128, 1280, 896, 128, 384, 128, 11392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 54400, 128, 128, 128, 128, 128, 128, 0, 128, 0, 128, 256, 128, 0, 128, 10624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[128, 128, 0, 128, 128, 128, 128, 128, 128, 0, 128, 128, 128, 3456, 128, 128, 44928, 17152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 1408, 128, 384, 31744, 128, 128, 128, 32128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[16768, 256, 0, 128, 128, 128, 47872, 128, 768, 128, 128, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 1024, 256, 128, 768, 0, 128, 128, 640, 128, 128, 512, 128, 128, 384, 46592, 13440, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[2688, 0, 128, 128, 128, 128, 48896, 256, 0, 128, 1280, 768, 128, 384, 128, 11520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 54528, 128, 128, 128, 128, 128, 128, 0, 128, 128, 128, 256, 128, 0, 128, 10624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 0, 128, 128, 256, 128, 256, 128, 0, 128, 384, 128, 128, 61952, 1920, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 0, 128, 128, 256, 128, 256, 128, 0, 128, 384, 128, 128, 61952, 1920, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 0, 0, 128, 128, 128, 256, 128, 128, 128, 128, 640, 128, 128, 64640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 0, 0, 128, 128, 128, 128, 128, 128, 128, 128, 640, 128, 128, 64640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 4196, 8179, 0, 0, 0, 7995, 0, 0, 0, 0, 0, 0, 7, 0, 0, 650, 0, 0, 0, 0, 0, 25, 0, 0, 0, 5131, 0, 0, 0, 0, 0, 0, 2, 542, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 7492, 2, 0, 0, 0, 0, 2829, 0, 0, 2149, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 56, 0, 0, 0, 6, 0, 12, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8170, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 351, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 31, 7694, 0, 0, 6907, 8145, 0, 3, 0, 3976, 0, 0, 2, 0, 157, 0, 0, 0, 0, 466, 0, 0, 0, 37, 2, 109, 0, 0, 0, 2026, 0, 0, 2, 0, 0, 10, 0, 26, 5, 0, 0, 0, 0, 0, 8119, 0, 0, 0, 0, 0, 0, 7579, 179, 241, 0, 0, 0, 8059, 0, 0, 0, 0, 0, 3, 0, 0, 0, 95, 1, 1, 1, 0, 0, 0, 1, 0, 1, 13, 2949, 1, 0, 0, 4, 2, 4, 0, 0, 1, 0, 0, 0, 318, 8005, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 1, 4, 0, 0, 0, 1, 0, 2007, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 42, 1, 53, 8182, 0, 0, 0, 7673, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 2, 423, 0, 0, 0, 3, 0, 3, 18, 6502, 0, 0, 0, 0, 0, 0, 8145, 9, 0, 0, 0, 0, 2, 8148, 2, 0, 0, 0, 0, 7661, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 16, 120, 0, 16, 2, 0, 0, 1, 0, 0, 0, 0, 1, 1, 8189, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 14, 0, 0, 315, 0, 17, 1, 0, 1, 0, 0, 145, 0, 0, 1, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 8131, 0, 1, 1540, 8065, 0, 0, 0, 7843, 10, 0, 0, 0, 0, 1, 8, 0, 0, 872, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3896, 0, 0, 0, 0, 0, 0, 0, 2277, 0, 1, 0, 0, 0, 0, 8107, 2, 0, 0, 5, 0, 0, 7190, 0, 0, 2, 0, 2, 7852, 0, 0, 388, 0, 0, 1, 0, 32, 0, 3, 2, 3, 16, 1, 47, 6, 0, 3, 37, 32, 138, 299, 0, 26, 0, 6, 0, 1, 0, 0, 0, 0, 0, 0, 8188, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 5, 1, 25, 0, 1858, 0, 1, 0, 0, 0, 0, 0, 8, 5, 0, 0, 32, 0, 0, 0, 0, 0, 0, 7919, 0, 0, 2571, 8162, 0, 0, 0, 6410, 0, 0, 0, 64, 0, 0, 12, 4, 0, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0, 4750, 0, 0, 14, 0, 0, 0, 0, 351, 0, 2, 0, 0, 0, 0, 8029, 0, 0, 9, 9, 0, 0, 7982, 362, 1, 0, 0, 0, 8159, 0, 0, 317, 0, 0, 0, 0, 4, 0, 0, 0, 0, 1, 0, 23, 0, 15, 0, 47, 0, 597, 115, 0, 4, 0, 0, 0, 0, 0, 22, 2, 0, 0, 0, 7453, 181, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 61, 0, 0, 2, 7955, 0, 1, 0, 0, 0, 1, 35, 0, 1, 0, 1, 0, 3, 0, 0, 0, 76, 0, 8128, 170, 0, 37, 8057, 0, 0, 21, 7372, 0, 0, 0, 12, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4, 0, 0, 0, 48, 48, 0, 0, 0, 0, 0, 0, 0, 3842, 0, 4, 0, 0, 1, 8, 5628, 0, 3, 4, 0, 0, 22, 7634, 29, 0, 0, 0, 0, 4389, 0, 13, 2458, 0, 0, 60, 0, 1, 0, 0, 0, 0, 2, 169, 883, 0, 98, 0, 8, 15, 1, 3, 0, 20, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 8179, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 101, 1, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 13, 2, 4, 0, 0, 2, 10, 0, 0, 0, 1, 0, 8182, 0, 0, 0, 7953, 0, 0, 0, 5554, 0, 0, 0, 0, 0, 0, 0, 0, 9, 153, 0, 0, 3, 0, 0, 2, 0, 1, 0, 6382, 0, 0, 0, 0, 0, 1, 0, 458, 0, 0, 3, 0, 0, 0, 7829, 0, 0, 0, 0, 1, 0, 6763, 0, 0, 0, 1, 0, 8094, 0, 5, 3019, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 282, 0, 569, 0, 1313, 0, 0, 234, 28, 0, 0, 0, 273, 0, 0, 0, 0, 0, 70, 0, 8187, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 4328, 0, 0, 0, 0, 0, 3, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8130, 0, 0, 1591, 7850, 0, 0, 0, 7022, 0, 1, 0, 95, 39, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 2738, 1, 0, 1, 0, 0, 0, 0, 8167, 6, 3, 0, 0, 0, 0, 7534, 0, 1, 2, 0, 5, 0, 7200, 418, 11, 0, 0, 0, 776, 0, 0, 1080, 38, 0, 0, 0, 0, 0, 0, 0, 119, 12, 0, 0, 0, 1, 1, 25, 0, 189, 0, 0, 0, 0, 0, 0, 0, 0, 118, 0, 0, 2, 0, 7993, 0, 14] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1692, 5053, 778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4196, 0, 0, 0, 0, 7114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 650, 0, 0, 0, 0, 0, 25, 0, 0, 0, 5131, 0, 2435, 1585, 881, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 542, 0, 0, 0, 0, 0, 1, 5756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 854, 2, 0, 0, 0, 0, 2829, 0, 0, 2149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 56, 0, 0, 0, 6, 0, 12, 7302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8073, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7524, 8280, 11310, 10811, 6301, 5834, 7378, 8098] avg=8192.0000 max=11310 max_rank=2 min=5834 min_rank=5 imbalance_ratio=1.380615 max_load_reduction=54226 max_load_reduction_pct=0.827423 moved_to_echo=27611 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 14, 0, 0, 315, 0, 17, 1, 0, 1, 0, 0, 145, 4850, 774, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 1, 8065, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1540, 0, 0, 0, 0, 6979, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8, 0, 0, 872, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3896, 0, 2410, 1521, 864, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2277, 0, 1, 0, 0, 0, 0, 5697, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 819, 0, 0, 2, 0, 2, 7852, 0, 0, 388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 32, 0, 3, 2, 3, 16, 1, 47, 6, 0, 3, 37, 32, 138, 7256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 299, 0, 26, 0, 6, 0, 1, 0, 0, 0, 0, 0, 0, 8091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6117, 8173, 8531, 9670, 7977, 9068, 7577, 8423] avg=8192.0000 max=9670 max_rank=3 min=6117 min_rank=0 imbalance_ratio=1.180420 max_load_reduction=55866 max_load_reduction_pct=0.852448 moved_to_echo=33045 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 351, 5112, 731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8145, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 31, 0, 0, 0, 6907, 0, 0, 3, 0, 3538, 0, 0, 2, 0, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 466, 0, 0, 0, 37, 2, 109, 0, 0, 0, 2026, 0, 2414, 1603, 438, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 10, 0, 26, 5, 0, 0, 0, 0, 0, 5705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 864, 179, 241, 0, 0, 0, 8059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 95, 1, 1, 1, 0, 0, 0, 1, 0, 1, 13, 2949, 6866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 2, 4, 0, 0, 1, 0, 0, 0, 318, 7910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6199, 8245, 10640, 7190, 5748, 9343, 9931, 8240] avg=8192.0000 max=10640 max_rank=2 min=5748 min_rank=4 imbalance_ratio=1.298828 max_load_reduction=54896 max_load_reduction_pct=0.837646 moved_to_echo=33262 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 1, 4, 0, 0, 0, 1, 0, 2007, 5496, 777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 1, 53, 0, 0, 0, 0, 6828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 2, 423, 0, 2422, 1724, 845, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 18, 6502, 0, 0, 0, 0, 0, 0, 5723, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 928, 2, 0, 0, 0, 0, 7661, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 16, 7296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 0, 16, 2, 0, 0, 1, 0, 0, 0, 0, 1, 1, 8092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8287, 8288, 6924, 5517, 12258, 8707, 7322, 8233] avg=8192.0000 max=12258 max_rank=4 min=5517 min_rank=3 imbalance_ratio=1.496338 max_load_reduction=53278 max_load_reduction_pct=0.812958 moved_to_echo=36520 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 61, 0, 0, 2, 7955, 5149, 773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 35, 0, 1, 0, 1, 0, 3, 0, 0, 0, 8057, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 170, 0, 37, 0, 0, 0, 21, 6560, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4, 0, 0, 0, 48, 48, 0, 1673, 1615, 812, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3842, 0, 4, 0, 0, 1, 8, 3955, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 22, 870, 29, 0, 0, 0, 0, 4389, 0, 13, 2458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 1, 0, 0, 0, 0, 2, 169, 883, 0, 98, 0, 8, 15, 1, 7253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 20, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 8082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13944, 8201, 6876, 4299, 7813, 7785, 8490, 8128] avg=8192.0000 max=13944 max_rank=0 min=4299 min_rank=3 imbalance_ratio=1.702148 max_load_reduction=51592 max_load_reduction_pct=0.787231 moved_to_echo=32216 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 5, 1, 25, 0, 1858, 5384, 753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8, 5, 0, 0, 32, 0, 0, 0, 0, 8162, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2571, 0, 0, 0, 0, 5704, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4, 0, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0, 4750, 0, 2387, 1689, 706, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 351, 0, 2, 0, 0, 0, 0, 5642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 0, 0, 909, 362, 1, 0, 0, 0, 8159, 0, 0, 317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 1, 0, 23, 0, 15, 0, 47, 0, 597, 7068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 4, 0, 0, 0, 0, 0, 22, 2, 0, 0, 0, 7365, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8030, 8306, 8339, 9642, 6009, 9766, 7755, 7689] avg=8192.0000 max=9766 max_rank=5 min=6009 min_rank=4 imbalance_ratio=1.192139 max_load_reduction=55770 max_load_reduction_pct=0.850983 moved_to_echo=31219 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 101, 1, 0, 0, 0, 0, 0, 0, 13, 0, 0, 4562, 779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 2, 4, 0, 0, 2, 10, 0, 0, 0, 7953, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 153, 0, 0, 3, 0, 0, 2, 0, 1, 0, 6382, 0, 2328, 1431, 612, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 458, 0, 0, 3, 0, 0, 0, 5501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 770, 0, 0, 0, 1, 0, 8094, 0, 5, 3019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 282, 0, 569, 0, 1313, 0, 0, 7301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 234, 28, 0, 0, 0, 273, 0, 0, 0, 0, 0, 70, 0, 8090, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5459, 8086, 4943, 11018, 5963, 11890, 9481, 8696] avg=8192.0000 max=11890 max_rank=5 min=4943 min_rank=2 imbalance_ratio=1.451416 max_load_reduction=53646 max_load_reduction_pct=0.818573 moved_to_echo=36030 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 4328, 4860, 774, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7850, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1591, 0, 0, 0, 0, 6245, 0, 1, 0, 95, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 2738, 1, 2245, 1527, 777, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8167, 6, 3, 0, 0, 0, 0, 5289, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5, 0, 813, 418, 11, 0, 0, 0, 776, 0, 0, 1080, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 12, 0, 0, 0, 1, 1, 25, 0, 189, 7255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 0, 0, 2, 0, 7895, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9976, 7957, 7973, 7389, 13467, 3143, 7602, 8029] avg=8192.0000 max=13467 max_rank=4 min=3143 min_rank=5 imbalance_ratio=1.643921 max_load_reduction=52069 max_load_reduction_pct=0.794510 moved_to_echo=32241 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 360, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 34, 7680, 0, 0, 6866, 8144, 0, 3, 0, 3973, 0, 0, 1, 0, 158, 0, 0, 0, 0, 468, 0, 0, 0, 36, 1, 112, 0, 0, 0, 2044, 0, 0, 2, 0, 0, 8, 0, 29, 4, 0, 0, 0, 0, 0, 8116, 0, 0, 0, 0, 0, 0, 7580, 177, 247, 0, 0, 0, 8055, 0, 0, 0, 0, 0, 4, 0, 0, 0, 89, 1, 2, 2, 0, 0, 0, 1, 0, 1, 13, 2972, 1, 0, 0, 4, 2, 4, 0, 1, 2, 0, 0, 0, 328, 8001, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 4249, 8185, 0, 0, 0, 7984, 0, 0, 0, 0, 0, 0, 7, 0, 0, 628, 0, 0, 0, 0, 0, 24, 0, 0, 0, 5177, 0, 0, 0, 0, 0, 0, 2, 524, 0, 0, 0, 0, 0, 1, 8190, 0, 0, 0, 0, 0, 0, 7516, 2, 0, 0, 0, 0, 2799, 0, 0, 2167, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 56, 0, 0, 0, 8, 0, 10, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8171, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 6, 1, 25, 0, 1858, 0, 1, 0, 0, 0, 0, 0, 9, 4, 0, 0, 31, 0, 0, 0, 0, 0, 0, 7914, 0, 0, 2596, 8161, 0, 0, 0, 6375, 0, 0, 0, 63, 0, 0, 11, 3, 0, 0, 0, 4, 0, 0, 0, 3, 0, 0, 0, 4765, 0, 0, 15, 0, 0, 0, 0, 362, 0, 2, 0, 0, 0, 0, 8020, 0, 0, 9, 8, 0, 0, 7982, 345, 1, 0, 0, 0, 8162, 0, 0, 309, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 25, 0, 21, 0, 56, 0, 574, 130, 0, 4, 0, 0, 0, 0, 0, 21, 2, 0, 0, 0, 7468, 182, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 1983, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 32, 1, 59, 8182, 0, 0, 0, 7665, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 2, 409, 0, 0, 0, 4, 0, 3, 19, 6541, 0, 0, 0, 0, 0, 0, 8140, 9, 0, 0, 0, 0, 2, 8145, 2, 0, 0, 0, 0, 7659, 0, 0, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 20, 130, 0, 19, 2, 0, 0, 1, 0, 0, 0, 0, 1, 1, 8188, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 4314, 0, 0, 0, 0, 0, 2, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8130, 0, 0, 1618, 7851, 0, 0, 0, 7038, 0, 3, 0, 96, 38, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 2772, 1, 0, 1, 0, 0, 0, 0, 8163, 8, 3, 0, 0, 0, 0, 7533, 0, 1, 1, 0, 5, 0, 7202, 407, 14, 0, 0, 0, 759, 0, 0, 1049, 35, 0, 0, 0, 0, 0, 0, 0, 121, 12, 0, 0, 0, 2, 1, 21, 0, 189, 0, 0, 0, 0, 0, 0, 0, 0, 118, 0, 0, 2, 0, 7990, 0, 16] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 12, 0, 0, 311, 0, 18, 1, 0, 2, 0, 0, 143, 0, 0, 1, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 8128, 0, 1, 1531, 8069, 0, 0, 0, 7808, 9, 0, 0, 0, 0, 1, 8, 0, 0, 866, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3890, 0, 0, 0, 0, 0, 0, 0, 2371, 0, 3, 0, 0, 0, 0, 8110, 2, 0, 0, 4, 0, 0, 7167, 2, 0, 2, 0, 1, 7846, 0, 0, 390, 0, 0, 1, 0, 33, 0, 3, 2, 2, 17, 1, 44, 6, 0, 4, 38, 32, 130, 298, 0, 26, 0, 6, 0, 1, 0, 0, 0, 0, 0, 0, 8188, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 104, 1, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 10, 2, 4, 0, 0, 2, 11, 0, 0, 0, 1, 0, 8182, 0, 0, 0, 7947, 0, 0, 0, 5563, 0, 0, 0, 0, 0, 0, 0, 0, 7, 146, 0, 0, 3, 0, 0, 2, 0, 0, 0, 6362, 0, 0, 0, 0, 0, 2, 0, 444, 0, 0, 3, 0, 0, 0, 7839, 0, 0, 0, 0, 1, 0, 6789, 0, 0, 0, 1, 0, 8083, 0, 5, 3026, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 298, 0, 572, 0, 1303, 0, 0, 230, 27, 0, 0, 0, 277, 0, 0, 0, 0, 0, 69, 0, 8187, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 0, 62, 0, 0, 2, 7964, 0, 1, 0, 0, 0, 2, 30, 0, 1, 0, 1, 0, 3, 0, 0, 0, 75, 0, 8133, 171, 0, 44, 8054, 0, 0, 21, 7375, 0, 0, 0, 14, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 48, 55, 0, 0, 0, 0, 0, 0, 0, 3811, 0, 4, 0, 0, 1, 10, 5645, 0, 4, 5, 0, 0, 22, 7628, 29, 0, 0, 0, 0, 4379, 0, 13, 2452, 0, 0, 57, 0, 0, 0, 0, 0, 0, 2, 167, 893, 0, 104, 0, 7, 15, 1, 3, 0, 18, 0, 0, 0, 0, 22, 0, 0, 0, 0, 1, 8178, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1628, 5079, 778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4249, 0, 0, 0, 0, 7104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 628, 0, 0, 0, 0, 0, 24, 0, 0, 0, 5177, 0, 2448, 1566, 880, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 524, 0, 0, 0, 0, 0, 1, 5742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 871, 2, 0, 0, 0, 0, 2799, 0, 0, 2167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 56, 0, 0, 0, 8, 0, 10, 7299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8069, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7486, 8289, 11353, 10832, 6269, 5839, 7375, 8093] avg=8192.0000 max=11353 max_rank=2 min=5839 min_rank=5 imbalance_ratio=1.385864 max_load_reduction=54183 max_load_reduction_pct=0.826767 moved_to_echo=27576 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 6, 1, 25, 0, 1858, 5394, 753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 9, 4, 0, 0, 31, 0, 0, 0, 0, 8161, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2596, 0, 0, 0, 0, 5672, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 3, 0, 0, 0, 4, 0, 0, 0, 3, 0, 0, 0, 4765, 0, 2397, 1664, 703, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 362, 0, 2, 0, 0, 0, 0, 5623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 8, 0, 0, 924, 345, 1, 0, 0, 0, 8162, 0, 0, 309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 25, 0, 21, 0, 56, 0, 574, 7062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 0, 4, 0, 0, 0, 0, 0, 21, 2, 0, 0, 0, 7375, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8041, 8305, 8331, 9643, 6002, 9758, 7742, 7714] avg=8192.0000 max=9758 max_rank=5 min=6002 min_rank=4 imbalance_ratio=1.191162 max_load_reduction=55778 max_load_reduction_pct=0.851105 moved_to_echo=31216 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 360, 5122, 731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8144, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 34, 0, 0, 0, 6866, 0, 0, 3, 0, 3535, 0, 0, 1, 0, 158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 0, 0, 0, 36, 1, 112, 0, 0, 0, 2044, 0, 2426, 1580, 438, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8, 0, 29, 4, 0, 0, 0, 0, 0, 5690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 878, 177, 247, 0, 0, 0, 8055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 89, 1, 2, 2, 0, 0, 0, 1, 0, 1, 13, 2972, 6852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 2, 4, 0, 1, 2, 0, 0, 0, 328, 7901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6218, 8244, 10599, 7205, 5733, 9357, 9937, 8243] avg=8192.0000 max=10599 max_rank=2 min=5733 min_rank=4 imbalance_ratio=1.293823 max_load_reduction=54937 max_load_reduction_pct=0.838272 moved_to_echo=33270 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 1983, 5504, 778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 1, 59, 0, 0, 0, 0, 6820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 2, 409, 0, 2433, 1698, 845, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3, 19, 6541, 0, 0, 0, 0, 0, 0, 5707, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 943, 2, 0, 0, 0, 0, 7659, 0, 0, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 20, 7296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 0, 19, 2, 0, 0, 1, 0, 0, 0, 0, 1, 1, 8086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8271, 8289, 6912, 5494, 12283, 8722, 7325, 8240] avg=8192.0000 max=12283 max_rank=4 min=5494 min_rank=3 imbalance_ratio=1.499390 max_load_reduction=53253 max_load_reduction_pct=0.812576 moved_to_echo=36570 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 4314, 4870, 775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7851, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1618, 0, 0, 0, 0, 6257, 0, 3, 0, 96, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 2772, 1, 2255, 1504, 781, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8163, 8, 3, 0, 0, 0, 0, 5278, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 0, 828, 407, 14, 0, 0, 0, 759, 0, 0, 1049, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 121, 12, 0, 0, 0, 2, 1, 21, 0, 189, 7253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 0, 0, 2, 0, 7884, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9970, 7958, 8014, 7423, 13454, 3098, 7599, 8020] avg=8192.0000 max=13454 max_rank=4 min=3098 min_rank=5 imbalance_ratio=1.642334 max_load_reduction=52082 max_load_reduction_pct=0.794708 moved_to_echo=32171 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 0, 62, 0, 0, 2, 7964, 5154, 774, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 30, 0, 1, 0, 1, 0, 3, 0, 0, 0, 8054, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 171, 0, 44, 0, 0, 0, 21, 6562, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 48, 55, 0, 1687, 1590, 813, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3811, 0, 4, 0, 0, 1, 10, 3958, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 22, 884, 29, 0, 0, 0, 0, 4379, 0, 13, 2452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 2, 167, 893, 0, 104, 0, 7, 15, 1, 7256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 18, 0, 0, 0, 0, 22, 0, 0, 0, 0, 1, 8076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13961, 8195, 6887, 4299, 7788, 7784, 8502, 8120] avg=8192.0000 max=13961 max_rank=0 min=4299 min_rank=3 imbalance_ratio=1.704224 max_load_reduction=51575 max_load_reduction_pct=0.786972 moved_to_echo=32194 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 104, 1, 0, 0, 0, 0, 0, 0, 10, 0, 0, 4587, 779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 2, 4, 0, 0, 2, 11, 0, 0, 0, 7947, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 146, 0, 0, 3, 0, 0, 2, 0, 0, 0, 6362, 0, 2343, 1415, 613, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 444, 0, 0, 3, 0, 0, 0, 5496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 787, 0, 0, 0, 1, 0, 8083, 0, 5, 3026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 298, 0, 572, 0, 1303, 0, 0, 7300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 230, 27, 0, 0, 0, 277, 0, 0, 0, 0, 0, 69, 0, 8085, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5484, 8079, 4951, 10993, 5945, 11903, 9492, 8689] avg=8192.0000 max=11903 max_rank=5 min=4951 min_rank=2 imbalance_ratio=1.453003 max_load_reduction=53633 max_load_reduction_pct=0.818375 moved_to_echo=36029 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 12, 0, 0, 311, 0, 18, 1, 0, 2, 0, 0, 143, 4843, 775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1, 8069, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1531, 0, 0, 0, 0, 6947, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8, 0, 0, 866, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3890, 0, 2424, 1494, 861, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2371, 0, 3, 0, 0, 0, 0, 5686, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 830, 2, 0, 2, 0, 1, 7846, 0, 0, 390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 33, 0, 3, 2, 2, 17, 1, 44, 6, 0, 4, 38, 32, 130, 7251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 298, 0, 26, 0, 6, 0, 1, 0, 0, 0, 0, 0, 0, 8086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6105, 8177, 8489, 9647, 8062, 9075, 7564, 8417] avg=8192.0000 max=9647 max_rank=3 min=6105 min_rank=0 imbalance_ratio=1.177612 max_load_reduction=55889 max_load_reduction_pct=0.852798 moved_to_echo=33118 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=67584 observed_loss=-2048 tokens_per_expert=[128, 128, 128, 128, 2176, 128, 128, 128, 128, 128, 256, 128, 128, 128, 25472, 128, 18432, 12800, 6016, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 0, 128, 128, 128, 128, 128, 384, 128, 128, 128, 128, 128, 128, 128, 18432, 40576, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 128, 128, 0, 128, 128, 22272, 128, 128, 128, 0, 128, 128, 43392, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 6912, 1024, 256, 128, 128, 128, 47872, 0, 128, 9600, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 256, 128, 16896, 0, 0, 128, 128, 48000, 128, 128, 128, 256, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=67456 observed_loss=-1920 tokens_per_expert=[128, 128, 128, 128, 2176, 128, 128, 128, 128, 128, 256, 128, 0, 128, 25600, 128, 18432, 12544, 6016, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 0, 128, 128, 128, 128, 128, 384, 0, 128, 128, 128, 128, 128, 128, 18304, 40576, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 128, 128, 0, 128, 128, 22272, 128, 128, 128, 0, 128, 128, 43264, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 7040, 1024, 384, 128, 128, 128, 47744, 0, 128, 9600, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 256, 128, 17024, 0, 0, 128, 128, 47872, 128, 128, 128, 256, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 0, 128, 128, 128, 128, 256, 1408, 128, 768, 128, 1536, 128, 3968, 57600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=67584 slack_slots=63488 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 128, 128, 0, 0, 128, 128, 128, 128, 0, 128, 128, 128, 128, 128, 128, 64640, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=67456 slack_slots=63616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 128, 128, 0, 0, 128, 128, 128, 128, 0, 128, 128, 128, 0, 128, 128, 64640, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[896, 128, 128, 128, 128, 384, 128, 128, 256, 128, 0, 128, 384, 63616, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 7489, 0, 0, 0, 0, 0, 0, 1, 41, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 140, 8184, 0, 0, 0, 0, 0, 13, 0, 7318, 25, 0, 8, 0, 0, 0, 8189, 0, 0, 0, 0, 163, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 8115, 24, 0, 0, 0, 0, 0, 0, 22, 0, 2, 0, 8182, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 8130, 0, 0, 0, 179, 0, 0, 0, 8, 0, 0, 0, 0, 0, 2675, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 6521, 14, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[5, 0, 0, 148, 0, 25, 0, 2411, 0, 1537, 0, 0, 0, 0, 452, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 9, 6080, 6140, 0, 3, 26, 0, 0, 7, 0, 8114, 0, 0, 0, 0, 0, 1, 7665, 5, 0, 0, 0, 0, 22, 0, 0, 137, 1, 0, 0, 2, 12, 0, 8181, 0, 0, 0, 0, 0, 5, 0, 4, 0, 1, 8, 6000, 0, 0, 11, 2, 0, 0, 0, 0, 0, 2, 3, 21, 0, 0, 0, 1, 0, 0, 0, 8191, 0, 0, 0, 2694, 0, 0, 0, 226, 0, 0, 0, 0, 0, 5284, 0, 0, 0, 0, 202, 0, 0, 0, 0, 0, 3, 0, 172, 1218, 0, 0, 0, 0, 442, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 0, 0, 6376, 0, 200, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7474, 8163, 0, 0, 2, 2, 0, 12, 0, 3748, 7, 0, 2, 0, 1, 0, 2774, 0, 0, 1, 0, 18, 0, 0, 0, 2824, 0, 20, 0, 0, 95, 0, 8132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8139, 0, 0, 0, 57, 0, 0, 0, 5, 0, 0, 0, 0, 0, 872, 0, 0, 0, 0, 1, 0, 0, 420, 0, 0, 0, 0, 0, 7997, 0, 1, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8167, 7924, 3, 0, 3, 0, 0, 0, 0, 49, 0, 1, 0, 0, 0, 15, 7980, 0, 0, 57, 0, 0, 5, 0, 0, 0, 340, 0, 0, 0, 626, 0, 7605, 0, 0, 0, 24, 0, 0, 0, 0, 0, 162, 0, 8144, 0, 0, 18, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7867, 0, 5, 9, 62, 0, 299, 0, 0, 0, 0, 1, 5, 12, 3148, 0, 1, 0, 0, 1, 0, 5, 0, 18, 0, 0, 0, 0, 8013, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 28, 0, 0, 0, 8082, 0, 0, 49, 5, 0, 0, 0, 0, 2, 0, 0, 0, 0, 48, 0, 0, 0, 792, 0, 0, 0, 6163, 5768, 0, 2, 0, 0, 0, 179, 0, 4531, 0, 0, 0, 0, 0, 0, 7969, 0, 0, 0, 0, 52, 0, 0, 0, 75, 798, 0, 0, 0, 0, 0, 8124, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8192, 11, 0, 0, 1, 42, 0, 2, 58, 0, 0, 0, 30, 28, 0, 0, 0, 0, 9, 0, 6963, 3, 0, 0, 2750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 1, 3, 0, 1, 0, 0, 0, 0, 0, 0, 2, 6, 33, 4634, 0, 0, 0, 0, 0, 11] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 15, 0, 0, 0, 7895, 0, 6, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 7329, 1250, 0, 0, 0, 0, 0, 977, 0, 7691, 19, 0, 0, 0, 2, 0, 7748, 0, 0, 23, 0, 372, 0, 0, 5, 0, 353, 0, 0, 7, 16, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 8144, 0, 0, 0, 0, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 7207, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 8, 0, 0, 0, 0, 8180, 0, 0, 2, 8, 0, 3, 0, 32, 0, 5, 44, 3, 0, 0, 0, 0, 7, 0, 27, 0, 2, 6901, 7419, 16, 0, 0, 0, 0, 228, 2, 5448, 1, 0, 11, 0, 21, 0, 8089, 0, 0, 0, 0, 217, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 8104, 0, 10, 0, 0, 0, 24, 0, 0, 0, 0, 0, 1, 0, 15, 0, 0, 1, 57, 0, 7933, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 3, 0, 0, 0, 0, 1, 0, 569, 0, 0, 0, 0, 0, 3915, 0, 0, 2, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 1, 0, 0, 8081, 0, 57, 0, 0, 0, 1, 70, 0, 2, 8, 0, 0, 0, 0, 1, 0, 6, 8, 0, 0, 0, 8025, 8094, 0, 0, 0, 0, 1, 89, 0, 8046, 3, 0, 0, 0, 0, 2, 7969, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 3, 8004, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 18, 0, 0, 39, 0, 0, 0, 0, 2, 0, 17, 0, 0, 474, 0, 0, 0, 1, 0, 0, 0, 136, 0, 0, 0, 0, 0, 28, 0, 3, 0, 0, 34, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 7489, 0, 0, 0, 0, 0, 0, 1, 41, 1209, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 140, 2191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 7318, 25, 0, 8, 0, 0, 0, 2654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 163, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 7457, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 2, 0, 8182, 0, 0, 0, 49, 0, 0, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 8130, 0, 0, 212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 0, 0, 0, 8, 0, 0, 0, 0, 0, 2675, 0, 1, 0, 0, 5535, 1087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 6521, 14, 0, 0, 0, 0, 0, 3697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8932, 2357, 10018, 7646, 8509, 8354, 9485, 10235] avg=8192.0000 max=10235 max_rank=7 min=2357 min_rank=1 imbalance_ratio=1.249390 max_load_reduction=55301 max_load_reduction_pct=0.843826 moved_to_echo=36583 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 0, 0, 6376, 0, 200, 0, 0, 0, 0, 1, 1, 1206, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7474, 2184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 12, 0, 3748, 7, 0, 2, 0, 1, 0, 900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 18, 0, 0, 0, 2824, 0, 20, 0, 0, 95, 0, 7472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 3, 0, 0, 0, 0, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8139, 0, 0, 213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 5, 0, 0, 0, 0, 0, 872, 0, 0, 0, 0, 1874, 1085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 420, 0, 0, 0, 0, 0, 7997, 0, 1, 1, 0, 0, 0, 3688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7982, 9659, 4674, 10430, 8438, 8352, 3893, 12108] avg=8192.0000 max=12108 max_rank=7 min=3893 min_rank=6 imbalance_ratio=1.478027 max_load_reduction=53428 max_load_reduction_pct=0.815247 moved_to_echo=32791 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4965, 0, 0, 0, 0, 0, 0, 0, 0, 1170, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8167, 2121, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 49, 0, 1, 0, 0, 0, 15, 2587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 5, 0, 0, 0, 340, 0, 0, 0, 626, 0, 6988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 162, 0, 8144, 0, 0, 18, 1, 0, 0, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7867, 0, 5, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 62, 0, 299, 0, 0, 0, 0, 1, 5, 12, 3148, 0, 1, 0, 0, 5393, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 0, 18, 0, 0, 0, 0, 8013, 0, 0, 0, 0, 0, 0, 3580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6315, 10292, 2655, 8016, 8587, 8071, 9983, 11617] avg=8192.0000 max=11617 max_rank=7 min=2655 min_rank=2 imbalance_ratio=1.418091 max_load_reduction=53919 max_load_reduction_pct=0.822739 moved_to_echo=38258 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8, 0, 0, 0, 0, 8180, 0, 0, 2, 8, 0, 3, 0, 32, 1096, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 44, 3, 0, 0, 0, 0, 7, 0, 27, 0, 2, 6901, 1986, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 228, 2, 5448, 1, 0, 11, 0, 21, 0, 2622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 217, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 7497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 8104, 0, 10, 0, 0, 0, 24, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 15, 0, 0, 1, 57, 0, 7933, 0, 9, 213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 3, 0, 0, 0, 5467, 986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 569, 0, 0, 0, 0, 0, 3915, 0, 0, 2, 1, 0, 0, 3351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9522, 8991, 8333, 7718, 8403, 8229, 6501, 7839] avg=8192.0000 max=9522 max_rank=0 min=6501 min_rank=6 imbalance_ratio=1.162354 max_load_reduction=56014 max_load_reduction_pct=0.854706 moved_to_echo=30972 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 28, 0, 0, 0, 8082, 0, 0, 49, 5, 0, 0, 0, 0, 855, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 48, 0, 0, 0, 792, 0, 0, 0, 6163, 1535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 179, 0, 4531, 0, 0, 0, 0, 0, 0, 2578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 75, 798, 0, 0, 0, 0, 0, 7452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8192, 11, 0, 0, 1, 42, 0, 258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 58, 0, 0, 0, 30, 28, 0, 0, 0, 0, 9, 0, 6963, 3, 0, 217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 1, 3, 0, 1, 5391, 769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6, 33, 4634, 0, 0, 0, 0, 0, 11, 2609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9216, 8540, 7290, 8377, 8506, 7310, 9002, 7295] avg=8192.0000 max=9216 max_rank=0 min=7290 min_rank=2 imbalance_ratio=1.125000 max_load_reduction=56320 max_load_reduction_pct=0.859375 moved_to_echo=32113 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 15, 0, 0, 0, 7895, 0, 6, 0, 0, 0, 0, 1, 0, 184, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 7329, 336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 977, 0, 7691, 19, 0, 0, 0, 2, 0, 2512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 372, 0, 0, 5, 0, 353, 0, 0, 7, 16, 0, 7518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 8144, 0, 0, 0, 0, 10, 1, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 5236, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 7207, 0, 0, 0, 0, 0, 0, 564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8294, 7684, 11201, 8294, 8419, 8392, 5478, 7774] avg=8192.0000 max=11201 max_rank=2 min=5478 min_rank=6 imbalance_ratio=1.367310 max_load_reduction=54335 max_load_reduction_pct=0.829086 moved_to_echo=30063 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 0, 148, 0, 25, 0, 2411, 0, 1537, 0, 0, 0, 0, 452, 0, 907, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 9, 6080, 1643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 26, 0, 0, 7, 0, 8114, 0, 0, 0, 0, 0, 1, 2485, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 137, 1, 0, 0, 2, 12, 0, 7518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4, 0, 1, 8, 6000, 0, 0, 11, 2, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 21, 0, 0, 0, 1, 0, 0, 0, 8191, 0, 0, 214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2694, 0, 0, 0, 226, 0, 0, 0, 0, 0, 5284, 0, 0, 0, 0, 5180, 816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 202, 0, 0, 0, 0, 0, 3, 0, 172, 1218, 0, 0, 0, 0, 442, 0, 2774, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5678, 7795, 10641, 7692, 6287, 8432, 14200, 4811] avg=8192.0000 max=14200 max_rank=6 min=4811 min_rank=7 imbalance_ratio=1.733398 max_load_reduction=51336 max_load_reduction_pct=0.783325 moved_to_echo=33730 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 1, 0, 0, 8081, 0, 57, 0, 0, 0, 1, 70, 0, 1195, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8, 0, 0, 0, 0, 1, 0, 6, 8, 0, 0, 0, 8025, 2168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 89, 0, 8046, 3, 0, 0, 0, 0, 2, 2583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 3, 7355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8135, 0, 0, 0, 0, 0, 0, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 18, 0, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 2, 0, 17, 0, 0, 474, 0, 0, 0, 1, 5386, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 0, 0, 0, 0, 0, 28, 0, 3, 0, 0, 34, 0, 3656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9597, 10218, 10724, 7363, 8387, 8396, 6994, 3857] avg=8192.0000 max=10724 max_rank=2 min=3857 min_rank=7 imbalance_ratio=1.309082 max_load_reduction=54812 max_load_reduction_pct=0.836365 moved_to_echo=27634 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 14, 0, 0, 0, 7894, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 7275, 1369, 0, 0, 0, 0, 0, 997, 0, 7679, 21, 0, 0, 0, 2, 0, 7731, 0, 0, 22, 0, 375, 0, 0, 5, 0, 344, 0, 0, 6, 19, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 8144, 0, 0, 0, 0, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 7162, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 0, 0, 6306, 0, 199, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7432, 8175, 0, 0, 2, 2, 0, 10, 0, 3690, 5, 0, 1, 0, 3, 0, 2873, 0, 0, 0, 0, 14, 0, 0, 0, 2800, 0, 22, 0, 0, 104, 0, 8129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8140, 0, 0, 0, 57, 0, 0, 0, 5, 0, 0, 0, 0, 0, 943, 0, 0, 0, 0, 1, 0, 0, 427, 0, 0, 0, 0, 0, 8000, 0, 1, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[4, 0, 0, 146, 0, 27, 0, 2351, 0, 1556, 0, 0, 0, 0, 423, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 8, 6092, 6208, 0, 3, 27, 0, 0, 6, 0, 8116, 0, 0, 0, 0, 0, 1, 7661, 5, 0, 0, 0, 0, 22, 0, 0, 150, 1, 0, 0, 2, 11, 0, 8179, 0, 0, 0, 0, 0, 6, 0, 5, 0, 1, 9, 6021, 0, 0, 9, 2, 0, 1, 0, 0, 0, 2, 3, 18, 0, 0, 0, 1, 0, 0, 0, 8191, 0, 0, 0, 2657, 0, 0, 0, 222, 0, 0, 0, 0, 0, 5200, 0, 0, 0, 0, 201, 1, 0, 0, 0, 0, 3, 0, 182, 1297, 0, 0, 0, 0, 442, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 29, 0, 0, 0, 8081, 0, 0, 45, 5, 0, 0, 0, 0, 2, 0, 0, 0, 0, 45, 0, 0, 0, 801, 0, 0, 0, 6092, 5737, 0, 2, 0, 0, 0, 189, 0, 4670, 0, 0, 0, 0, 0, 0, 7959, 0, 0, 0, 0, 61, 0, 0, 0, 75, 779, 0, 0, 0, 0, 0, 8126, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8192, 11, 0, 0, 0, 46, 0, 2, 54, 0, 0, 0, 29, 29, 0, 0, 0, 0, 8, 0, 6986, 2, 0, 0, 2794, 0, 0, 0, 1, 0, 0, 0, 0, 0, 83, 0, 3, 0, 1, 0, 1, 0, 0, 0, 0, 2, 6, 33, 4541, 0, 0, 0, 0, 0, 12] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8167, 7928, 5, 0, 3, 1, 0, 0, 0, 53, 0, 1, 0, 0, 0, 14, 7997, 0, 0, 58, 0, 0, 5, 0, 0, 0, 345, 0, 0, 0, 608, 0, 7601, 0, 0, 0, 24, 0, 0, 0, 0, 0, 160, 0, 8141, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7860, 0, 6, 6, 67, 0, 292, 0, 0, 0, 0, 1, 5, 11, 3253, 0, 1, 0, 0, 1, 0, 4, 0, 18, 0, 0, 0, 0, 8014, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 7, 0, 0, 0, 0, 8181, 0, 0, 2, 10, 0, 1, 0, 31, 0, 7, 39, 3, 0, 0, 0, 0, 6, 0, 28, 0, 2, 6889, 7429, 16, 0, 0, 1, 0, 238, 2, 5545, 1, 0, 11, 0, 26, 0, 8101, 0, 0, 0, 0, 218, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 8101, 0, 10, 0, 0, 0, 22, 0, 0, 0, 0, 0, 1, 0, 13, 0, 0, 1, 56, 0, 7928, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 56, 3, 0, 0, 0, 0, 1, 0, 565, 0, 0, 0, 0, 0, 3801, 0, 0, 2, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 7464, 0, 0, 0, 0, 0, 0, 2, 41, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 107, 8184, 0, 0, 0, 0, 0, 11, 0, 7421, 23, 0, 8, 0, 0, 0, 8189, 0, 0, 0, 0, 169, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 8116, 23, 0, 0, 0, 0, 0, 0, 22, 0, 2, 0, 8183, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 8125, 0, 0, 0, 194, 0, 0, 0, 10, 0, 0, 0, 0, 0, 2637, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6496, 11, 0, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 1, 0, 0, 8078, 0, 52, 0, 0, 0, 1, 74, 0, 2, 8, 0, 0, 0, 0, 1, 0, 8, 11, 0, 0, 0, 8032, 8106, 0, 0, 0, 0, 0, 77, 0, 8036, 1, 0, 0, 0, 0, 1, 7971, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 3, 8008, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 0, 0, 0, 0, 0, 8166, 18, 0, 0, 39, 0, 0, 0, 0, 3, 0, 17, 0, 0, 484, 0, 0, 0, 1, 0, 0, 0, 130, 0, 0, 0, 0, 0, 24, 0, 2, 0, 0, 31, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 14, 0, 0, 0, 7894, 0, 8, 0, 0, 0, 0, 0, 0, 208, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 7275, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 997, 0, 7679, 21, 0, 0, 0, 2, 0, 2463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 375, 0, 0, 5, 0, 344, 0, 0, 6, 19, 0, 7520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 8144, 0, 0, 0, 0, 6, 1, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 1, 0, 0, 0, 5268, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 7162, 0, 0, 0, 0, 0, 0, 621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8315, 7665, 11162, 8291, 8413, 8392, 5513, 7785] avg=8192.0000 max=11162 max_rank=2 min=5513 min_rank=6 imbalance_ratio=1.362549 max_load_reduction=54374 max_load_reduction_pct=0.829681 moved_to_echo=30103 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 0, 0, 6306, 0, 199, 0, 0, 0, 0, 1, 1, 1247, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7432, 2209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 10, 0, 3690, 5, 0, 1, 0, 3, 0, 916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 2800, 0, 22, 0, 0, 104, 0, 7474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 3, 0, 0, 0, 0, 0, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8140, 0, 0, 213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 5, 0, 0, 0, 0, 0, 943, 0, 0, 0, 0, 1957, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 427, 0, 0, 0, 0, 0, 8000, 0, 1, 1, 0, 0, 0, 3710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7950, 9642, 4629, 10414, 8435, 8355, 3971, 12140] avg=8192.0000 max=12140 max_rank=7 min=3971 min_rank=6 imbalance_ratio=1.481934 max_load_reduction=53396 max_load_reduction_pct=0.814758 moved_to_echo=32901 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 1, 0, 0, 8078, 0, 52, 0, 0, 0, 1, 74, 0, 1237, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8, 0, 0, 0, 0, 1, 0, 8, 11, 0, 0, 0, 8032, 2190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 8036, 1, 0, 0, 0, 0, 1, 2540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 3, 7363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8135, 0, 0, 0, 0, 0, 0, 249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 0, 0, 0, 0, 0, 8166, 18, 0, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 3, 0, 17, 0, 0, 484, 0, 0, 0, 1, 5431, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 0, 0, 0, 0, 0, 24, 0, 2, 0, 0, 31, 0, 3679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9634, 10252, 10655, 7371, 8385, 8398, 6975, 3866] avg=8192.0000 max=10655 max_rank=2 min=3866 min_rank=7 imbalance_ratio=1.300659 max_load_reduction=54881 max_load_reduction_pct=0.837418 moved_to_echo=27624 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 29, 0, 0, 0, 8081, 0, 0, 45, 5, 0, 0, 0, 0, 879, 196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 45, 0, 0, 0, 801, 0, 0, 0, 6092, 1538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 189, 0, 4670, 0, 0, 0, 0, 0, 0, 2533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 75, 779, 0, 0, 0, 0, 0, 7456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8192, 11, 0, 0, 0, 46, 0, 257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 54, 0, 0, 0, 29, 29, 0, 0, 0, 0, 8, 0, 6986, 2, 0, 217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2794, 0, 0, 0, 1, 0, 0, 0, 0, 0, 83, 0, 3, 0, 1, 5426, 712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 6, 33, 4541, 0, 0, 0, 0, 0, 12, 2608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9235, 8478, 7394, 8371, 8508, 7327, 9020, 7203] avg=8192.0000 max=9235 max_rank=0 min=7203 min_rank=7 imbalance_ratio=1.127319 max_load_reduction=56301 max_load_reduction_pct=0.859085 moved_to_echo=32058 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 7464, 0, 0, 0, 0, 0, 0, 2, 41, 1249, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 107, 2211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 7421, 23, 0, 8, 0, 0, 0, 2609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 7462, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 2, 0, 8183, 0, 0, 0, 54, 0, 0, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 8125, 0, 0, 212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 194, 0, 0, 0, 10, 0, 0, 0, 0, 0, 2637, 0, 1, 0, 0, 5580, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6496, 11, 0, 0, 0, 1, 0, 3714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8946, 2345, 10072, 7656, 8513, 8349, 9432, 10223] avg=8192.0000 max=10223 max_rank=7 min=2345 min_rank=1 imbalance_ratio=1.247925 max_load_reduction=55313 max_load_reduction_pct=0.844009 moved_to_echo=36517 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4867, 0, 0, 0, 0, 0, 0, 0, 0, 1210, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8167, 2142, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 53, 0, 1, 0, 0, 0, 14, 2548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 5, 0, 0, 0, 345, 0, 0, 0, 608, 0, 6988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 160, 0, 8141, 0, 0, 18, 0, 0, 0, 236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7860, 0, 6, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 67, 0, 292, 0, 0, 0, 0, 1, 5, 11, 3253, 0, 1, 0, 0, 5449, 978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 18, 0, 0, 0, 0, 8014, 0, 0, 0, 0, 0, 0, 3598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6255, 10315, 2620, 8004, 8579, 8065, 10063, 11635] avg=8192.0000 max=11635 max_rank=7 min=2620 min_rank=2 imbalance_ratio=1.420288 max_load_reduction=53901 max_load_reduction_pct=0.822464 moved_to_echo=38342 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 0, 146, 0, 27, 0, 2351, 0, 1556, 0, 0, 0, 0, 423, 0, 947, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 8, 6092, 1678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 27, 0, 0, 6, 0, 8116, 0, 0, 0, 0, 0, 1, 2441, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 150, 1, 0, 0, 2, 11, 0, 7520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 5, 0, 1, 9, 6021, 0, 0, 9, 2, 0, 1, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 18, 0, 0, 0, 1, 0, 0, 0, 8191, 0, 0, 214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2657, 0, 0, 0, 222, 0, 0, 0, 0, 0, 5200, 0, 0, 0, 0, 5220, 766, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 201, 1, 0, 0, 0, 0, 3, 0, 182, 1297, 0, 0, 0, 0, 442, 0, 2817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5645, 7841, 10599, 7706, 6308, 8429, 14065, 4943] avg=8192.0000 max=14065 max_rank=6 min=4943 min_rank=7 imbalance_ratio=1.716919 max_load_reduction=51471 max_load_reduction_pct=0.785385 moved_to_echo=33745 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 7, 0, 0, 0, 0, 8181, 0, 0, 2, 10, 0, 1, 0, 31, 1133, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 39, 3, 0, 0, 0, 0, 6, 0, 28, 0, 2, 6889, 2008, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 238, 2, 5545, 1, 0, 11, 0, 26, 0, 2581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 218, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 7501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 8101, 0, 10, 0, 0, 0, 22, 253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 13, 0, 0, 1, 56, 0, 7928, 0, 9, 213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 56, 3, 0, 0, 0, 5520, 917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 565, 0, 0, 0, 0, 0, 3801, 0, 0, 2, 1, 0, 0, 3371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9556, 8998, 8405, 7723, 8395, 8221, 6497, 7741] avg=8192.0000 max=9556 max_rank=0 min=6497 min_rank=6 imbalance_ratio=1.166504 max_load_reduction=55980 max_load_reduction_pct=0.854187 moved_to_echo=30854 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 5888, 0, 384, 0, 256, 128, 0, 128, 128, 128, 12672, 128, 128, 0, 128, 39552, 7040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[256, 128, 128, 1152, 128, 0, 128, 128, 256, 39552, 128, 128, 128, 128, 512, 128, 23936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 1536, 128, 45056, 128, 128, 128, 0, 128, 128, 18944, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 0, 128, 128, 128, 128, 896, 128, 0, 128, 50304, 14208, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 256, 128, 128, 0, 53504, 0, 1920, 128, 128, 0, 128, 640, 128, 7936, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 896, 128, 0, 128, 3072, 1536, 128, 128, 128, 768, 128, 59264, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 5888, 0, 384, 0, 256, 128, 0, 128, 128, 128, 12800, 128, 128, 0, 128, 39936, 6656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 0, 1536, 128, 45312, 128, 128, 128, 0, 128, 128, 18688, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[256, 128, 128, 1152, 128, 0, 128, 128, 256, 39424, 128, 128, 128, 128, 512, 128, 24192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 0, 128, 128, 128, 128, 896, 128, 0, 128, 50176, 14464, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 256, 128, 128, 0, 53248, 0, 1920, 128, 128, 0, 128, 512, 128, 8192, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 896, 128, 0, 128, 3072, 1536, 128, 128, 128, 768, 128, 59392, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 0, 128, 128, 128, 128, 128, 0, 128, 128, 128, 0, 63616, 128, 128, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 128, 128, 128, 0, 128, 128, 256, 128, 63104, 128, 128, 128, 128, 128, 128, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 103, 0, 0, 0, 0, 6, 0, 298, 0, 0, 0, 0, 0, 0, 4, 0, 6416, 0, 0, 0, 2681, 0, 3, 2, 0, 0, 0, 0, 5166, 0, 0, 0, 8191, 8144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1533, 7984, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8028, 0, 0, 0, 0, 34, 0, 0, 0, 7473, 0, 15, 0, 1, 0, 0, 0, 0, 2, 0, 0, 9, 0, 0, 0, 271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 776, 0, 0, 0, 0, 0, 0, 2, 0, 8136, 0, 0, 246, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6234, 0, 0, 0, 0, 2, 0, 6, 0, 6, 1, 3162, 8, 0, 122, 0, 0, 0, 0, 8076, 0, 0, 0, 8024, 8189, 0, 0, 2, 0, 125, 0, 0, 0, 0, 0, 0, 0, 0, 7422, 1850, 0, 4, 1, 0, 1, 0, 0, 0, 0, 0, 71, 8158, 0, 0, 0, 0, 1, 0, 0, 0, 7939, 0, 0, 0, 0, 0, 3, 0, 1, 1, 0, 73, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 288, 46, 10, 0, 0, 0, 26, 5, 0, 0, 0, 3326, 6, 0, 2, 0, 0, 0, 0, 0, 1983, 0, 0, 306, 0, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 0, 1, 0, 0, 0, 5, 0, 1420, 0, 0, 0, 0, 0, 4, 1127, 0, 0, 0, 3485, 7, 1, 2, 0, 1, 0, 397, 6206, 0, 0, 0, 8171, 8097, 0, 0, 0, 12, 0, 0, 0, 0, 0, 2, 6, 0, 4, 381, 7129, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 3, 7841, 0, 0, 0, 0, 32, 0, 0, 0, 7633, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 2381, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 202, 0, 0, 0, 0, 4, 13, 1, 0, 0, 3676, 0, 0, 0, 0, 0, 0, 5, 0, 7087, 0, 15, 11, 0, 5, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 0, 587, 0, 0, 0, 0, 0, 7109, 0, 0, 0, 0, 1, 7, 1562, 1, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 513, 0, 0, 0, 8188, 8108, 0, 0, 0, 307, 0, 0, 0, 0, 0, 20, 0, 0, 0, 7909, 4155, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 7625, 0, 0, 0, 0, 76, 0, 0, 0, 8109, 4, 0, 0, 13, 0, 6, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 2, 0, 5, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6932, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 103] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 164, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 6874, 0, 0, 0, 2337, 1, 0, 728, 275, 27, 0, 467, 462, 0, 0, 4, 8039, 8133, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 2, 0, 1, 7709, 7226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7130, 0, 0, 0, 2, 339, 0, 0, 8, 3306, 0, 117, 0, 43, 0, 0, 0, 6, 1, 12, 464, 2, 0, 24, 8, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 41, 282, 0, 51, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 7, 0, 2, 7052, 0, 0, 4021, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 312, 80, 34, 0, 21, 1, 0, 838, 0, 0, 0, 1643, 0, 0, 0, 24, 0, 0, 0, 5301, 11, 0, 0, 8173, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4922, 8117, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 424, 8116, 0, 0, 0, 1, 695, 0, 0, 0, 7980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 801, 0, 0, 0, 0, 797, 0, 14, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 954, 0, 0, 0, 0, 0, 0, 0, 0, 8058, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 847, 0, 0, 3, 6, 0, 1, 0, 0, 0, 0, 0, 0, 7407, 0, 245, 69, 0, 0, 0, 0, 7720, 0, 0, 0, 8145, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 7993, 7249, 0, 0, 0, 28, 30, 0, 0, 0, 0, 0, 0, 7998, 0, 0, 0, 0, 11, 0, 0, 0, 1091, 0, 0, 0, 0, 0, 0, 33, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 147, 2, 0, 0, 0, 0, 0, 0, 0, 0, 152, 23, 0, 0, 0, 12, 0, 0, 25, 7441, 3, 635, 41, 0, 0, 0, 0, 22, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 485, 0, 0, 0, 0, 0, 0, 774, 0, 1, 0, 28, 0, 0, 0, 0, 666, 0, 0, 0, 158, 0, 233, 0, 0, 1, 0, 0, 7160, 0, 0, 0, 7974, 8106, 0, 0, 0, 0, 1, 0, 0, 73, 0, 1, 0, 0, 0, 1960, 7730, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8, 0, 7936, 0, 0, 1, 0, 1, 4, 1, 1, 8153, 0, 15, 30, 0, 0, 2, 0, 1, 7, 0, 0, 0, 1, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0, 1522, 0, 0, 0, 0, 0, 0, 0, 0, 8085, 19, 8, 4153, 0, 21, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 164, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 3705, 1557, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6874, 0, 0, 0, 2337, 1, 0, 728, 275, 27, 0, 467, 279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8039, 9, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7709, 3521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 339, 0, 0, 8, 3306, 0, 117, 0, 43, 0, 0, 0, 6, 105, 385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 12, 464, 2, 0, 24, 8, 43, 0, 0, 0, 0, 0, 0, 0, 0, 7555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 41, 282, 0, 51, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5573, 569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 7, 0, 2, 6667, 0, 0, 4021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5526, 10988, 8056, 11231, 4311, 8109, 6615, 10700] avg=8192.0000 max=11231 max_rank=3 min=4311 min_rank=4 imbalance_ratio=1.370972 max_load_reduction=54305 max_load_reduction_pct=0.828629 moved_to_echo=29735 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 0, 587, 0, 0, 0, 0, 0, 7109, 0, 0, 0, 0, 2130, 1664, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7, 1562, 1, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 9, 0, 0, 0, 307, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7909, 2025, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 8109, 4, 0, 0, 13, 0, 6, 0, 0, 117, 378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 2, 0, 5, 0, 7532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 5961, 567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6554, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11579, 1959, 8524, 9937, 8703, 7561, 6531, 10742] avg=8192.0000 max=11579 max_rank=0 min=1959 min_rank=1 imbalance_ratio=1.413452 max_load_reduction=53957 max_load_reduction_pct=0.823318 moved_to_echo=33537 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 103, 0, 0, 0, 0, 6, 0, 298, 0, 0, 0, 0, 0, 0, 4094, 1752, 875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 6416, 0, 0, 0, 2681, 0, 3, 2, 0, 0, 0, 0, 3111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1533, 3890, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 7473, 0, 15, 0, 1, 0, 0, 0, 0, 1180, 444, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 9, 0, 0, 0, 271, 0, 0, 0, 0, 0, 0, 0, 0, 7565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 776, 0, 0, 0, 6276, 569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 7692, 0, 0, 246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7128, 12217, 8202, 5434, 9147, 7847, 7621, 7940] avg=8192.0000 max=12217 max_rank=1 min=5434 min_rank=3 imbalance_ratio=1.491333 max_load_reduction=53319 max_load_reduction_pct=0.813583 moved_to_echo=32555 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 0, 1, 0, 0, 0, 5, 0, 1420, 0, 0, 0, 0, 3656, 1712, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1127, 0, 0, 0, 3485, 7, 1, 2, 0, 1, 0, 397, 3737, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 10, 0, 0, 0, 12, 0, 0, 0, 0, 0, 2, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 381, 3473, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 7633, 0, 87, 0, 0, 0, 0, 0, 0, 1417, 387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 2381, 0, 0, 0, 1, 0, 0, 0, 0, 7521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 202, 0, 0, 0, 0, 4, 13, 1, 0, 0, 3676, 0, 0, 0, 6129, 566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 6700, 0, 15, 11, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7852, 8761, 8201, 3917, 9556, 9921, 10592, 6736] avg=8192.0000 max=10592 max_rank=6 min=3917 min_rank=3 imbalance_ratio=1.292969 max_load_reduction=54944 max_load_reduction_pct=0.838379 moved_to_echo=36805 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 312, 80, 34, 0, 21, 4162, 1772, 898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 838, 0, 0, 0, 1643, 0, 0, 0, 24, 0, 0, 0, 3192, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4922, 3955, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 695, 0, 0, 0, 7980, 0, 0, 0, 0, 0, 0, 0, 0, 1211, 440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 801, 0, 0, 0, 0, 797, 0, 14, 0, 0, 0, 0, 0, 0, 7584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 954, 0, 0, 0, 6344, 571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7618, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7281, 5709, 8183, 9302, 10327, 9196, 7917, 7621] avg=8192.0000 max=10327 max_rank=4 min=5709 min_rank=1 imbalance_ratio=1.260620 max_load_reduction=55209 max_load_reduction_pct=0.842422 moved_to_echo=35061 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6234, 0, 0, 0, 0, 948, 1782, 1369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 6, 0, 6, 1, 3162, 8, 0, 122, 0, 0, 0, 0, 4863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8024, 9, 0, 0, 2, 0, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7422, 902, 0, 4, 1, 0, 1, 0, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7939, 0, 0, 0, 0, 0, 3, 0, 1, 1844, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 73, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 7607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 288, 46, 10, 0, 0, 0, 26, 5, 0, 0, 0, 3326, 6, 0, 2, 6376, 573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1875, 0, 0, 306, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10333, 8170, 8160, 8401, 9896, 7736, 10658, 2182] avg=8192.0000 max=10658 max_rank=6 min=2182 min_rank=7 imbalance_ratio=1.301025 max_load_reduction=54878 max_load_reduction_pct=0.837372 moved_to_echo=30472 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 847, 0, 0, 3, 6, 0, 1, 3717, 1747, 1308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7407, 0, 245, 69, 0, 0, 0, 0, 4649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7993, 3532, 0, 0, 0, 28, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 1091, 0, 0, 0, 0, 0, 0, 33, 0, 1763, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 2, 0, 0, 0, 0, 0, 0, 0, 0, 152, 23, 0, 0, 6251, 570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 25, 7035, 3, 635, 41, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7630, 12370, 8158, 11583, 3304, 7573, 7145, 7773] avg=8192.0000 max=12370 max_rank=1 min=3304 min_rank=4 imbalance_ratio=1.510010 max_load_reduction=53166 max_load_reduction_pct=0.811249 moved_to_echo=25795 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 485, 0, 0, 0, 0, 0, 0, 774, 0, 1, 0, 28, 0, 0, 3969, 1733, 1217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 666, 0, 0, 0, 158, 0, 233, 0, 0, 1, 0, 0, 4304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7974, 3, 0, 0, 0, 0, 1, 0, 0, 73, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1960, 3761, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4, 1, 1, 8153, 0, 15, 30, 0, 0, 2, 0, 1, 1639, 444, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 1, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 7533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0, 1522, 0, 0, 0, 6203, 570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7641, 19, 8, 4153, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 5362, 8052, 5731, 10292, 7593, 8457, 11842] avg=8192.0000 max=11842 max_rank=7 min=5362 min_rank=1 imbalance_ratio=1.445557 max_load_reduction=53694 max_load_reduction_pct=0.819305 moved_to_echo=38184 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6261, 0, 0, 0, 0, 1, 0, 4, 0, 5, 0, 3163, 8, 0, 123, 0, 0, 0, 0, 8073, 0, 0, 0, 8020, 8187, 0, 0, 2, 0, 124, 0, 0, 0, 0, 0, 0, 0, 0, 7416, 1849, 0, 6, 1, 0, 1, 0, 0, 0, 0, 0, 54, 8160, 0, 0, 0, 0, 1, 0, 0, 0, 7950, 0, 0, 0, 0, 0, 3, 0, 2, 1, 0, 70, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 286, 41, 9, 0, 0, 0, 29, 6, 0, 0, 0, 3365, 9, 0, 2, 0, 0, 0, 0, 0, 1974, 0, 0, 278, 0, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 503, 0, 0, 0, 0, 0, 0, 763, 0, 1, 0, 26, 0, 0, 0, 0, 696, 0, 0, 0, 136, 0, 226, 0, 0, 1, 0, 0, 7128, 0, 0, 0, 7973, 8107, 0, 0, 0, 0, 2, 0, 0, 73, 0, 1, 0, 0, 0, 1962, 7746, 0, 3, 0, 0, 0, 0, 0, 0, 0, 7, 0, 7937, 0, 0, 1, 0, 1, 4, 1, 1, 8150, 0, 15, 32, 0, 0, 2, 0, 2, 7, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 1491, 0, 0, 0, 0, 0, 0, 0, 0, 8081, 19, 7, 4187, 0, 20, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 7, 0, 0, 0, 0, 0, 3, 0, 1441, 0, 0, 0, 0, 0, 4, 1146, 0, 0, 0, 3391, 7, 2, 2, 0, 2, 0, 412, 6166, 0, 0, 0, 8171, 8096, 0, 0, 0, 13, 0, 0, 0, 0, 0, 2, 6, 0, 4, 426, 7115, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 3, 7848, 0, 0, 0, 0, 29, 0, 0, 0, 7640, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 2443, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 201, 0, 0, 0, 0, 4, 13, 1, 0, 0, 3650, 0, 0, 0, 0, 0, 0, 5, 0, 7085, 0, 17, 13, 0, 5, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 0, 579, 0, 0, 0, 0, 0, 7138, 0, 0, 0, 0, 0, 6, 1565, 1, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 492, 0, 0, 0, 8188, 8112, 0, 0, 0, 311, 0, 0, 0, 0, 0, 23, 0, 0, 0, 7908, 4190, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 7621, 0, 0, 0, 0, 80, 0, 0, 0, 8104, 4, 0, 0, 14, 0, 6, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 3, 0, 6, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6962, 0, 0, 4010, 0, 0, 0, 0, 0, 0, 102] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 349, 79, 36, 0, 21, 1, 0, 832, 0, 0, 0, 1701, 0, 0, 0, 25, 0, 0, 0, 5369, 11, 0, 0, 8173, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4790, 8112, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 443, 8105, 0, 0, 0, 1, 690, 0, 0, 0, 7976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 787, 0, 0, 0, 0, 765, 0, 14, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 980, 0, 0, 0, 0, 0, 0, 0, 0, 8051, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 101, 0, 0, 0, 0, 4, 0, 292, 0, 0, 0, 0, 0, 0, 4, 0, 6306, 0, 0, 0, 2739, 0, 3, 2, 0, 0, 0, 0, 5147, 0, 0, 0, 8190, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1528, 7989, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8029, 0, 0, 0, 0, 34, 0, 0, 0, 7469, 0, 12, 0, 1, 0, 0, 0, 0, 2, 0, 0, 8, 0, 0, 0, 276, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 874, 0, 0, 0, 0, 0, 0, 5, 0, 8144, 0, 0, 226, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 852, 0, 0, 3, 6, 0, 0, 0, 0, 0, 0, 0, 0, 7431, 0, 241, 72, 0, 0, 0, 0, 7751, 0, 0, 0, 8147, 8150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 7985, 7254, 0, 0, 0, 21, 30, 0, 0, 0, 0, 0, 0, 8007, 0, 0, 0, 0, 9, 0, 0, 0, 1073, 0, 0, 0, 0, 0, 0, 35, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 147, 2, 0, 0, 0, 0, 0, 0, 0, 0, 157, 23, 0, 0, 0, 14, 0, 0, 25, 7395, 4, 634, 41, 0, 0, 0, 0, 22, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 168, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 6854, 0, 0, 0, 2326, 1, 0, 736, 287, 27, 0, 471, 481, 0, 0, 1, 8040, 8133, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 2, 0, 1, 7706, 7286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7125, 0, 0, 0, 3, 316, 0, 0, 7, 3265, 0, 138, 0, 42, 0, 0, 0, 5, 1, 10, 461, 2, 0, 23, 7, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 39, 282, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 6, 0, 2, 7012, 0, 0, 4037, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6261, 0, 0, 0, 0, 945, 1801, 1320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 5, 0, 3163, 8, 0, 123, 0, 0, 0, 0, 4869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8020, 9, 0, 0, 2, 0, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7416, 904, 0, 6, 1, 0, 1, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7950, 0, 0, 0, 0, 0, 3, 0, 2, 1884, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 70, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 7602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 286, 41, 9, 0, 0, 0, 29, 6, 0, 0, 0, 3365, 9, 0, 2, 6359, 576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1871, 0, 0, 278, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10327, 8173, 8155, 8382, 9943, 7724, 10682, 2150] avg=8192.0000 max=10682 max_rank=6 min=2150 min_rank=7 imbalance_ratio=1.303955 max_load_reduction=54854 max_load_reduction_pct=0.837006 moved_to_echo=30499 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 7, 0, 0, 0, 0, 0, 3, 0, 1441, 0, 0, 0, 0, 3639, 1731, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1146, 0, 0, 0, 3391, 7, 2, 2, 0, 2, 0, 412, 3719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 10, 0, 0, 0, 13, 0, 0, 0, 0, 0, 2, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 426, 3476, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 7640, 0, 85, 0, 0, 0, 0, 0, 0, 1439, 369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 2443, 0, 0, 0, 1, 0, 0, 0, 0, 7517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 201, 0, 0, 0, 0, 4, 13, 1, 0, 0, 3650, 0, 0, 0, 6117, 569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 6716, 0, 17, 13, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7829, 8685, 8202, 3963, 9562, 9983, 10556, 6756] avg=8192.0000 max=10556 max_rank=6 min=3963 min_rank=3 imbalance_ratio=1.288574 max_load_reduction=54980 max_load_reduction_pct=0.838928 moved_to_echo=36857 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 349, 79, 36, 0, 21, 4149, 1788, 878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 832, 0, 0, 0, 1701, 0, 0, 0, 25, 0, 0, 0, 3238, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4790, 3963, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 690, 0, 0, 0, 7976, 0, 0, 0, 0, 0, 0, 0, 0, 1253, 420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 787, 0, 0, 0, 0, 765, 0, 14, 0, 0, 0, 0, 0, 0, 7581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 980, 0, 0, 0, 6317, 574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7631, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7303, 5808, 8183, 9197, 10340, 9147, 7924, 7634] avg=8192.0000 max=10340 max_rank=4 min=5808 min_rank=1 imbalance_ratio=1.262207 max_load_reduction=55196 max_load_reduction_pct=0.842224 moved_to_echo=35045 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 0, 579, 0, 0, 0, 0, 0, 7138, 0, 0, 0, 0, 2143, 1682, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1565, 1, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 10, 0, 0, 0, 311, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7908, 2047, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 8104, 4, 0, 0, 14, 0, 6, 0, 0, 114, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 3, 0, 6, 0, 7532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 5939, 570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6599, 0, 0, 4010, 0, 0, 0, 0, 0, 0, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11625, 1950, 8532, 9958, 8685, 7563, 6511, 10712] avg=8192.0000 max=11625 max_rank=0 min=1950 min_rank=1 imbalance_ratio=1.419067 max_load_reduction=53911 max_load_reduction_pct=0.822617 moved_to_echo=33471 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 101, 0, 0, 0, 0, 4, 0, 292, 0, 0, 0, 0, 0, 0, 4086, 1771, 842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 6306, 0, 0, 0, 2739, 0, 3, 2, 0, 0, 0, 0, 3104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1528, 3903, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 7469, 0, 12, 0, 1, 0, 0, 0, 0, 1201, 425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8, 0, 0, 0, 276, 0, 0, 0, 0, 0, 0, 0, 0, 7560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 874, 0, 0, 0, 6258, 572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 7719, 0, 0, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7096, 12158, 8201, 5439, 9142, 7846, 7704, 7950] avg=8192.0000 max=12158 max_rank=1 min=5439 min_rank=3 imbalance_ratio=1.484131 max_load_reduction=53378 max_load_reduction_pct=0.814484 moved_to_echo=32642 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 852, 0, 0, 3, 6, 0, 0, 3710, 1768, 1268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7431, 0, 241, 72, 0, 0, 0, 0, 4674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8147, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7985, 3544, 0, 0, 0, 21, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 1073, 0, 0, 0, 0, 0, 0, 35, 0, 1809, 385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 2, 0, 0, 0, 0, 0, 0, 0, 0, 157, 23, 0, 0, 6239, 573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 25, 7010, 4, 634, 41, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7608, 12418, 8158, 11580, 3311, 7570, 7141, 7750] avg=8192.0000 max=12418 max_rank=1 min=3311 min_rank=4 imbalance_ratio=1.515869 max_load_reduction=53118 max_load_reduction_pct=0.810516 moved_to_echo=25772 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 503, 0, 0, 0, 0, 0, 0, 763, 0, 1, 0, 26, 0, 0, 3967, 1752, 1170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 696, 0, 0, 0, 136, 0, 226, 0, 0, 1, 0, 0, 4292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7973, 2, 0, 0, 0, 0, 2, 0, 0, 73, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1962, 3779, 0, 3, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4, 1, 1, 8150, 0, 15, 32, 0, 0, 2, 0, 2, 1666, 425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 7531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 1491, 0, 0, 0, 6185, 574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7656, 19, 7, 4187, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 5351, 8051, 5751, 10300, 7595, 8417, 11889] avg=8192.0000 max=11889 max_rank=7 min=5351 min_rank=1 imbalance_ratio=1.451294 max_load_reduction=53647 max_load_reduction_pct=0.818588 moved_to_echo=38201 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 168, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 3727, 1573, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6854, 0, 0, 0, 2326, 1, 0, 736, 287, 27, 0, 471, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8040, 9, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7706, 3559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 316, 0, 0, 7, 3265, 0, 138, 0, 42, 0, 0, 0, 5, 112, 365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 10, 461, 2, 0, 23, 7, 52, 0, 0, 0, 0, 0, 0, 0, 0, 7552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 39, 282, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5552, 572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 6, 0, 2, 6647, 0, 0, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5566, 10993, 8054, 11266, 4253, 8108, 6601, 10695] avg=8192.0000 max=11266 max_rank=3 min=4253 min_rank=4 imbalance_ratio=1.375244 max_load_reduction=54270 max_load_reduction_pct=0.828094 moved_to_echo=29657 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 128, 640, 128, 128, 768, 0, 128, 0, 2048, 0, 15104, 128, 128, 0, 128, 26496, 13824, 6912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 39936, 25088, 0, 128, 128, 128, 128, 0, 128, 0, 0, 128, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 512, 512, 384, 0, 128, 0, 128, 128, 128, 256, 128, 10496, 128, 0, 128, 49152, 4608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 17536, 128, 128, 128, 20992, 128, 512, 1024, 384, 128, 0, 896, 24448, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 128, 128, 1280, 128, 128, 128, 51712, 128, 256, 128, 128, 0, 128, 128, 128, 9344, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 128, 51840, 128, 768, 12928, 128, 128, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 1408, 128, 128, 128, 128, 3712, 0, 128, 128, 128, 128, 0, 128, 0, 60544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 640, 128, 128, 768, 0, 128, 0, 1920, 0, 15232, 128, 128, 0, 128, 26368, 13952, 6656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 39808, 25216, 0, 128, 128, 128, 128, 0, 128, 0, 0, 128, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 512, 512, 384, 0, 128, 0, 128, 128, 128, 256, 128, 10624, 128, 0, 128, 49024, 4608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 17408, 128, 128, 0, 20992, 128, 512, 1024, 512, 128, 0, 896, 24576, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 128, 51968, 128, 768, 12800, 128, 128, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[0, 128, 128, 1280, 128, 128, 128, 51712, 128, 256, 128, 128, 0, 128, 128, 128, 9600, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 1408, 128, 0, 128, 128, 3712, 0, 128, 128, 128, 128, 0, 128, 0, 60544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 128, 65024, 128, 0, 0, 128, 384, 128, 0, 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8183, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 644, 0, 0, 0, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 65, 0, 13, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 5, 7321, 0, 7923, 0, 215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8118, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 0, 0, 4007, 3, 0, 8192, 0, 0, 0, 0, 0, 0, 3, 0, 20, 4189, 0, 0, 0, 0, 0, 0, 8145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 1, 0, 0, 10, 0, 1, 0, 3633, 43, 165, 92, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3729, 0, 0, 0, 0, 0, 0, 7932, 3, 0, 153, 0, 210, 3215, 0, 7998, 0, 1, 0, 0, 0, 0, 323, 0, 0, 0, 0, 49, 0, 5562, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7723, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 0, 7702, 0, 0, 8117, 0, 9, 0, 0, 4, 0, 163, 0, 0, 7118, 0, 0, 0, 0, 0, 71, 8151, 0, 26, 14, 428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 228, 0, 1782, 0, 0, 0, 14, 0, 6, 0, 0, 0, 1, 1, 5937, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 617, 0, 7741, 0, 0, 30, 0, 0, 0, 97, 0, 0, 0, 0, 0, 0, 8070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4, 0, 34, 66, 1, 914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 142, 0, 0, 0, 0, 4037, 0, 0, 8186, 0, 0, 2, 0, 0, 0, 0, 0, 0, 686, 0, 0, 0, 0, 68, 0, 2219, 0, 77, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 7958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 7925, 0, 0, 0, 0, 0, 0, 8191, 0, 5, 0, 0, 0, 8192, 0, 8099, 0, 125, 0, 0, 0, 0, 590, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 134, 170, 61, 0, 2, 0, 0, 8, 0, 0, 0, 407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1, 0, 0, 6, 0, 0, 4054, 0, 0, 8072, 0, 0, 0, 1, 0, 1, 0, 0, 0, 5574, 0, 0, 0, 3, 0, 0, 4371, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 133, 0, 0, 8018, 0, 141, 0, 0, 2, 0, 8180, 0, 0, 0, 0, 0, 5876, 0, 8167, 0, 3098, 3, 17, 0, 0, 90, 0, 19, 3, 4, 0, 0, 3755, 0, 0, 0, 0, 0, 440, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 66, 0, 1944, 0, 0, 0, 0, 71, 0, 13, 0, 163, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8180, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 0, 573, 0, 0, 0, 0, 3, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 85, 0, 0, 8, 0, 1, 4, 0, 12, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 7564, 0, 0, 0, 0, 0, 0, 8050, 3, 0, 0, 0, 37, 8123, 0, 7900, 0, 102, 0, 0, 0, 0, 20, 0, 15, 0, 1, 0, 0, 8169, 0, 0, 11, 0, 65, 0, 0, 0, 0, 4, 1, 9, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 16, 0, 86, 2, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[25, 0, 0, 0, 0, 0, 4744, 0, 0, 8150, 0, 0, 0, 0, 0, 0, 11, 0, 73, 8154, 1, 4, 1, 0, 1, 20, 2700, 0, 0, 0, 200, 0, 0, 42, 0, 0, 0, 0, 0, 7, 0, 72, 0, 0, 0, 0, 0, 1444, 0, 349, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 7516, 0, 6, 0, 0, 0, 0, 8083, 0, 0, 0, 20, 0, 3686, 0, 7670, 7, 1255, 0, 0, 0, 0, 3719, 0, 0, 0, 0, 0, 0, 1468, 0, 0, 0, 4, 523, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 5424, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 6, 0, 4655, 91, 3, 8156, 0, 0, 0, 9, 1, 0, 0, 3, 1, 4195, 0, 0, 0, 0, 0, 11, 7796, 0, 0, 2, 729, 0, 0, 0, 0, 1, 6, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 320, 0, 1, 0, 0, 0, 0, 0, 0, 45, 0, 0, 4611, 0, 0, 2, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 1454, 0, 7838, 0, 3182, 0, 0, 1, 0, 2, 0, 0, 0, 6, 0, 0, 6403, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 21, 0, 0, 0, 2, 0, 0, 0, 1, 7798, 0, 5, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8183, 0, 0, 2467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 378, 0, 0, 0, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 65, 5719, 321, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8177, 0, 0, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 7321, 0, 2327, 0, 215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8118, 0, 0, 0, 0, 10, 133, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10650, 8763, 6167, 8324, 9868, 8444, 8136, 5184] avg=8192.0000 max=10650 max_rank=0 min=5184 min_rank=7 imbalance_ratio=1.300049 max_load_reduction=54886 max_load_reduction_pct=0.837494 moved_to_echo=31632 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 142, 0, 0, 0, 0, 4037, 0, 0, 2467, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 686, 0, 0, 0, 0, 68, 0, 1302, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 7958, 5719, 328, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 7925, 0, 0, 458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 8192, 0, 2379, 0, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 590, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 1, 459, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, 170, 61, 0, 2, 0, 0, 8, 0, 0, 8134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6648, 2133, 14078, 8456, 10701, 9399, 8509, 5612] avg=8192.0000 max=14078 max_rank=2 min=2133 min_rank=1 imbalance_ratio=1.718506 max_load_reduction=51458 max_load_reduction_pct=0.785187 moved_to_echo=34221 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8180, 0, 0, 2468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 573, 0, 0, 0, 0, 3, 0, 4793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 85, 0, 0, 8, 0, 1, 4, 0, 12, 0, 5723, 320, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 7564, 0, 0, 1691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 37, 8123, 0, 2320, 0, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 15, 0, 1, 0, 0, 8169, 0, 0, 11, 0, 65, 1695, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 9, 0, 0, 0, 0, 0, 2, 0, 0, 1, 7994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 86, 2, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 5077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10648, 5370, 6211, 9361, 10585, 10159, 8011, 5191] avg=8192.0000 max=10648 max_rank=0 min=5191 min_rank=7 imbalance_ratio=1.299805 max_load_reduction=54888 max_load_reduction_pct=0.837524 moved_to_echo=33946 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 0, 0, 4007, 3, 0, 2469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 20, 4189, 0, 0, 0, 0, 0, 0, 4773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 1, 0, 0, 10, 0, 1, 0, 3633, 5723, 324, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 165, 92, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3729, 0, 0, 1684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 153, 0, 210, 3215, 0, 2349, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 323, 0, 0, 0, 0, 49, 0, 5562, 0, 0, 0, 0, 42, 1688, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7723, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6485, 8985, 9822, 5714, 5931, 7849, 7879, 12871] avg=8192.0000 max=12871 max_rank=7 min=5714 min_rank=3 imbalance_ratio=1.571167 max_load_reduction=52665 max_load_reduction_pct=0.803604 moved_to_echo=34530 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 0, 7702, 0, 0, 2446, 0, 9, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 163, 0, 0, 7118, 0, 0, 0, 0, 0, 71, 4777, 0, 26, 14, 428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 228, 5671, 313, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1782, 0, 0, 0, 14, 0, 6, 0, 0, 0, 1, 1, 5937, 0, 0, 1685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 617, 0, 2274, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 0, 0, 0, 0, 8070, 0, 0, 0, 0, 0, 1689, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4, 0, 8127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 66, 1, 914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10164, 12597, 6271, 9426, 2921, 10035, 8132, 5990] avg=8192.0000 max=12597 max_rank=1 min=2921 min_rank=4 imbalance_ratio=1.537720 max_load_reduction=52939 max_load_reduction_pct=0.807785 moved_to_echo=27078 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[25, 0, 0, 0, 0, 0, 4744, 0, 0, 2456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 73, 8154, 1, 4, 1, 0, 1, 20, 1583, 0, 0, 0, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 7, 0, 72, 0, 0, 0, 0, 0, 1444, 5694, 310, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 349, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 7516, 0, 6, 558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 3686, 0, 2254, 7, 1255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3719, 0, 0, 0, 0, 0, 0, 1468, 0, 0, 0, 4, 523, 559, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 8027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5424, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7225, 10048, 7625, 8466, 7222, 6450, 8105, 10395] avg=8192.0000 max=10395 max_rank=7 min=6450 min_rank=5 imbalance_ratio=1.268921 max_load_reduction=55141 max_load_reduction_pct=0.841385 moved_to_echo=32172 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 6, 0, 4655, 91, 3, 2458, 0, 0, 0, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 4195, 0, 0, 0, 0, 0, 11, 4568, 0, 0, 2, 729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 5698, 317, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 320, 0, 1, 0, 0, 0, 0, 0, 0, 45, 0, 0, 4611, 0, 0, 1612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1454, 0, 2303, 0, 3182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 6, 0, 0, 6403, 0, 0, 0, 0, 0, 1616, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 21, 0, 0, 0, 2, 0, 0, 0, 8103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7798, 0, 5, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 5037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7224, 9509, 6081, 6589, 6941, 8209, 8132, 12851] avg=8192.0000 max=12851 max_rank=7 min=6081 min_rank=2 imbalance_ratio=1.568726 max_load_reduction=52685 max_load_reduction_pct=0.803909 moved_to_echo=36133 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 6, 0, 0, 4054, 0, 0, 2429, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5574, 0, 0, 0, 3, 0, 0, 2553, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3238, 5643, 335, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 133, 0, 0, 8018, 0, 141, 908, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 5876, 0, 2388, 0, 3098, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 90, 0, 19, 3, 4, 0, 0, 3755, 0, 0, 0, 0, 0, 910, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 440, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 66, 8123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1944, 0, 0, 0, 0, 71, 0, 13, 0, 163, 0, 0, 0, 0, 0, 5251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6492, 8131, 9281, 9200, 11367, 4991, 8632, 7442] avg=8192.0000 max=11367 max_rank=4 min=4991 min_rank=5 imbalance_ratio=1.387573 max_load_reduction=54169 max_load_reduction_pct=0.826553 moved_to_echo=32432 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 140, 0, 0, 0, 0, 4200, 0, 0, 8182, 0, 0, 2, 0, 0, 0, 0, 0, 0, 662, 0, 0, 0, 0, 79, 0, 2032, 0, 77, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 7961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 7925, 0, 0, 0, 0, 0, 0, 8190, 0, 6, 0, 0, 0, 8192, 0, 8100, 0, 125, 0, 0, 0, 0, 615, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 177, 60, 0, 2, 0, 0, 8, 0, 0, 0, 425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8176, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 0, 545, 0, 0, 0, 0, 3, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 90, 0, 0, 6, 0, 3, 5, 0, 15, 2, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 7637, 0, 0, 0, 0, 0, 0, 8051, 3, 0, 0, 0, 34, 8123, 0, 7874, 0, 98, 0, 0, 0, 0, 23, 0, 15, 0, 1, 0, 0, 8169, 0, 0, 9, 0, 60, 0, 0, 0, 0, 4, 1, 9, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 13, 0, 81, 2, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5, 0, 4667, 89, 3, 8155, 0, 0, 0, 7, 2, 0, 0, 2, 1, 4037, 0, 0, 0, 0, 0, 9, 7797, 0, 0, 2, 741, 0, 0, 0, 0, 1, 7, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 328, 0, 1, 0, 0, 0, 0, 0, 0, 46, 0, 0, 4683, 0, 2, 2, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 1612, 0, 7836, 0, 3104, 0, 0, 1, 0, 1, 0, 0, 0, 7, 0, 0, 6390, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 0, 0, 0, 21, 0, 0, 0, 2, 0, 0, 0, 2, 7794, 0, 3, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[25, 0, 0, 0, 0, 0, 4686, 0, 0, 8150, 0, 0, 0, 0, 0, 0, 13, 0, 77, 8150, 1, 5, 1, 0, 2, 23, 2593, 0, 0, 0, 198, 0, 0, 44, 0, 0, 0, 0, 0, 7, 0, 73, 0, 0, 0, 0, 0, 1551, 0, 368, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 7507, 0, 6, 0, 0, 0, 0, 8081, 0, 0, 0, 19, 0, 3586, 0, 7649, 6, 1278, 0, 0, 0, 0, 3767, 0, 0, 0, 0, 0, 0, 1406, 0, 0, 0, 3, 523, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 5581, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 0, 0, 3992, 8, 0, 8192, 0, 0, 0, 0, 0, 0, 4, 0, 19, 4168, 0, 0, 0, 0, 0, 0, 8137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 1, 0, 0, 11, 0, 1, 0, 3693, 42, 179, 102, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3854, 0, 0, 0, 0, 0, 0, 7930, 3, 0, 154, 0, 191, 3115, 0, 7995, 0, 1, 0, 0, 0, 0, 357, 0, 0, 0, 0, 49, 0, 5484, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7728, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8182, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8138, 0, 0, 0, 0, 0, 0, 624, 0, 0, 0, 253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 74, 0, 13, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 4, 7310, 0, 7934, 0, 211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8115, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 0, 7711, 0, 0, 8116, 0, 9, 0, 0, 4, 0, 164, 0, 0, 7042, 0, 0, 0, 0, 0, 72, 8146, 0, 27, 14, 431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 221, 0, 1759, 0, 0, 0, 14, 0, 7, 0, 0, 0, 1, 1, 5978, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 648, 0, 7750, 0, 0, 32, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 8067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 33, 65, 1, 911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1, 0, 0, 7, 0, 0, 3979, 0, 0, 8074, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5707, 0, 0, 0, 3, 0, 0, 4255, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, 0, 0, 8003, 0, 143, 0, 0, 2, 0, 8181, 0, 0, 0, 0, 0, 5891, 0, 8163, 0, 3134, 2, 22, 0, 0, 91, 0, 18, 3, 4, 0, 0, 3655, 0, 0, 0, 0, 0, 458, 1, 0, 0, 0, 0, 2, 0, 3, 0, 0, 0, 0, 0, 0, 62, 0, 2032, 0, 0, 0, 0, 70, 0, 8, 0, 174, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 140, 0, 0, 0, 0, 4200, 0, 0, 2462, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 662, 0, 0, 0, 0, 79, 0, 1211, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 7961, 5720, 298, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 7925, 0, 0, 409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 8192, 0, 2388, 0, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 615, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 412, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 177, 60, 0, 2, 0, 0, 8, 0, 0, 8130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6804, 2029, 14051, 8397, 10711, 9435, 8509, 5600] avg=8192.0000 max=14051 max_rank=2 min=2029 min_rank=1 imbalance_ratio=1.715210 max_load_reduction=51485 max_load_reduction_pct=0.785599 moved_to_echo=34255 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8176, 0, 0, 2465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 545, 0, 0, 0, 0, 3, 0, 4870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 90, 0, 0, 6, 0, 3, 5, 0, 15, 2, 5726, 290, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 7637, 0, 0, 1649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 34, 8123, 0, 2321, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 15, 0, 1, 0, 0, 8169, 0, 0, 9, 0, 60, 1658, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 9, 0, 0, 0, 0, 0, 2, 0, 0, 1, 7992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 81, 2, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 5031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10641, 5419, 6197, 9388, 10579, 10167, 8009, 5136] avg=8192.0000 max=10641 max_rank=0 min=5136 min_rank=7 imbalance_ratio=1.298950 max_load_reduction=54895 max_load_reduction_pct=0.837631 moved_to_echo=33891 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5, 0, 4667, 89, 3, 2454, 0, 0, 0, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 4037, 0, 0, 0, 0, 0, 9, 4643, 0, 0, 2, 741, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 5701, 288, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 328, 0, 1, 0, 0, 0, 0, 0, 0, 46, 0, 0, 4683, 0, 2, 1573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1612, 0, 2311, 0, 3104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 7, 0, 0, 6390, 0, 0, 0, 0, 0, 1581, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 0, 0, 0, 21, 0, 0, 0, 2, 0, 0, 0, 8100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7794, 0, 3, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 5006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7227, 9435, 6060, 6633, 7029, 8211, 8129, 12812] avg=8192.0000 max=12812 max_rank=7 min=6060 min_rank=2 imbalance_ratio=1.563965 max_load_reduction=52724 max_load_reduction_pct=0.804504 moved_to_echo=36181 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 0, 0, 3992, 8, 0, 2465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 19, 4168, 0, 0, 0, 0, 0, 0, 4846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 1, 0, 0, 11, 0, 1, 0, 3693, 5727, 294, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 179, 102, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3854, 0, 0, 1641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 154, 0, 191, 3115, 0, 2357, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 357, 0, 0, 0, 0, 49, 0, 5484, 0, 0, 0, 0, 40, 1650, 236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7728, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6471, 9037, 9854, 5819, 5821, 7816, 7875, 12843] avg=8192.0000 max=12843 max_rank=7 min=5819 min_rank=3 imbalance_ratio=1.567749 max_load_reduction=52693 max_load_reduction_pct=0.804031 moved_to_echo=34355 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8182, 0, 0, 2463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8138, 0, 0, 0, 0, 0, 0, 373, 0, 0, 0, 253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 74, 5722, 292, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8176, 0, 0, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 7310, 0, 2339, 0, 211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8115, 0, 0, 0, 0, 10, 126, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10645, 8764, 6155, 8315, 9864, 8485, 8133, 5175] avg=8192.0000 max=10645 max_rank=0 min=5175 min_rank=7 imbalance_ratio=1.299438 max_load_reduction=54891 max_load_reduction_pct=0.837570 moved_to_echo=31657 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[25, 0, 0, 0, 0, 0, 4686, 0, 0, 2453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 77, 8150, 1, 5, 1, 0, 2, 23, 1545, 0, 0, 0, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 7, 0, 73, 0, 0, 0, 0, 0, 1551, 5697, 281, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 368, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 7507, 0, 6, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 3586, 0, 2255, 6, 1278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3767, 0, 0, 0, 0, 0, 0, 1406, 0, 0, 0, 3, 523, 525, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 8022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5581, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7164, 10015, 7712, 8438, 7144, 6450, 8107, 10506] avg=8192.0000 max=10506 max_rank=7 min=6450 min_rank=5 imbalance_ratio=1.282471 max_load_reduction=55030 max_load_reduction_pct=0.839691 moved_to_echo=32207 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 0, 7711, 0, 0, 2442, 0, 9, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 0, 0, 7042, 0, 0, 0, 0, 0, 72, 4851, 0, 27, 14, 431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 221, 5674, 285, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1759, 0, 0, 0, 14, 0, 7, 0, 0, 0, 1, 1, 5978, 0, 0, 1643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 648, 0, 2285, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 8067, 0, 0, 0, 0, 0, 1652, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 8123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 65, 1, 911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10170, 12601, 6242, 9403, 2965, 10067, 8127, 5961] avg=8192.0000 max=12601 max_rank=1 min=2965 min_rank=4 imbalance_ratio=1.538208 max_load_reduction=52935 max_load_reduction_pct=0.807724 moved_to_echo=27120 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 7, 0, 0, 3979, 0, 0, 2426, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5707, 0, 0, 0, 3, 0, 0, 2525, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3243, 5648, 305, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, 0, 0, 8003, 0, 143, 863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 5891, 0, 2394, 0, 3134, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 91, 0, 18, 3, 4, 0, 0, 3655, 0, 0, 0, 0, 0, 867, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 458, 1, 0, 0, 0, 0, 2, 0, 3, 0, 0, 0, 0, 0, 0, 62, 8121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2032, 0, 0, 0, 0, 70, 0, 8, 0, 174, 0, 0, 0, 0, 0, 5219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6414, 8236, 9265, 9143, 11423, 4905, 8647, 7503] avg=8192.0000 max=11423 max_rank=4 min=4905 min_rank=5 imbalance_ratio=1.394409 max_load_reduction=54113 max_load_reduction_pct=0.825699 moved_to_echo=32478 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 256, 128, 128, 128, 0, 45568, 128, 128, 19712, 0, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 23552, 128, 1024, 128, 0, 128, 128, 128, 0, 256, 0, 0, 0, 0, 0, 40832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 256, 128, 128, 128, 0, 45696, 128, 128, 19712, 0, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 128, 128, 0, 256, 128, 128, 128, 0, 128, 128, 128, 128, 16768, 45696, 2432, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 23808, 128, 1024, 128, 0, 128, 128, 128, 0, 256, 0, 0, 0, 0, 0, 40448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 0, 0, 128, 128, 256, 128, 256, 38528, 0, 18688, 128, 8064, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=67328 observed_loss=-1792 tokens_per_expert=[128, 128, 128, 128, 128, 0, 256, 128, 128, 128, 0, 128, 128, 128, 128, 16640, 45696, 2688, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[256, 128, 128, 38656, 128, 128, 128, 128, 128, 128, 24832, 0, 128, 128, 1664, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 4864, 0, 128, 128, 128, 128, 0, 49792, 0, 0, 128, 128, 768, 8832, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 2816, 128, 128, 128, 128, 0, 128, 128, 0, 384, 128, 128, 53504, 0, 256, 8832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 4992, 0, 128, 128, 128, 128, 0, 49536, 0, 0, 128, 128, 640, 8576, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[256, 128, 128, 38528, 128, 128, 128, 128, 128, 128, 24960, 0, 128, 128, 1664, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 2816, 128, 128, 128, 128, 0, 128, 128, 0, 384, 128, 128, 53888, 0, 256, 8448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[512, 128, 128, 128, 128, 128, 256, 256, 128, 128, 128, 0, 128, 128, 128, 128, 64512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[512, 0, 128, 128, 128, 128, 256, 256, 128, 128, 128, 0, 128, 128, 128, 128, 64640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=67328 slack_slots=63744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 7872, 0, 0, 0, 0, 0, 120, 0, 0, 0, 0, 0, 0, 0, 0, 190, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2029, 0, 790, 15, 0, 4980, 0, 0, 0, 8192, 45, 0, 1444, 0, 0, 0, 0, 0, 0, 2476, 0, 0, 0, 8186, 7430, 0, 0, 9, 0, 0, 0, 8076, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[5111, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 17, 23, 22, 0, 0, 0, 0, 0, 0, 2299, 0, 0, 0, 0, 0, 132, 0, 0, 0, 1, 233, 0, 0, 16, 529, 7528, 1, 0, 1, 0, 0, 10, 0, 0, 0, 0, 1, 0, 0, 28, 17, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7, 9, 0, 3, 0, 0, 0, 0, 3504, 0, 30, 0, 0, 0, 0, 3, 0, 3, 0, 743, 129, 0, 0, 0, 7768, 0, 5, 0, 8068, 5383, 0, 1, 0, 0, 0, 0, 0, 0, 3130, 4, 0, 0, 8141, 8154, 0, 95, 0, 1, 0, 0, 4377, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[436, 393, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 6517, 0, 0, 0, 0, 2, 7, 0, 0, 0, 0, 151, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 296, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 5, 8034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7834, 0, 515, 3146, 4, 6845, 2, 10, 108, 8192, 47, 0, 2807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1369, 7484, 0, 0, 0, 46, 0, 0, 3035, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[171, 2526, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 227, 0, 0, 0, 0, 0, 276, 0, 613, 0, 0, 0, 47, 0, 0, 0, 31, 0, 1, 0, 0, 7970, 0, 101, 20, 0, 0, 0, 0, 2, 1, 0, 0, 1, 0, 8059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 3558, 0, 0, 9, 0, 11, 0, 0, 7, 0, 0, 8069, 1, 0, 0, 0, 2531, 0, 0, 0, 8130, 17, 47, 0, 0, 0, 113, 0, 0, 0, 4, 0, 0, 0, 8145, 8021, 0, 0, 1261, 0, 3, 0, 5504, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[6117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 318, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 6, 0, 0, 0, 0, 0, 0, 0, 7343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 641, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 7917, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 7120, 0, 0, 272, 0, 0, 0, 0, 0, 8192, 155, 0, 772, 0, 0, 0, 0, 0, 0, 573, 0, 0, 0, 8187, 8027, 0, 523, 0, 0, 0, 0, 7837, 1504, 0, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[5415, 538, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 8, 0, 0, 0, 29, 3, 0, 0, 0, 0, 0, 0, 31, 8134, 0, 0, 0, 0, 0, 47, 5, 0, 56, 0, 0, 0, 0, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 6426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4797, 98, 0, 0, 1, 6159, 0, 21, 0, 7247, 5353, 0, 142, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8069, 8058, 0, 0, 0, 0, 0, 3, 2968, 1638, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 389, 0, 0, 0, 0, 3, 0, 29, 36, 1, 1, 0, 0, 0, 0, 1, 0, 0, 0, 524, 478, 0, 1, 0, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 235, 4543, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 760, 0, 7661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 2, 0, 0, 121, 0, 4, 0, 57, 0, 0, 6744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2210, 0, 42, 129, 0, 39, 0, 26, 0, 7609, 155, 0, 7252, 0, 1, 0, 1, 0, 0, 1311, 0, 0, 0, 8108, 7842, 0, 0, 619, 46, 0, 0, 8097, 2, 0, 0, 302, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2103, 2190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7872, 0, 0, 0, 0, 0, 120, 0, 0, 0, 0, 0, 5836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 190, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3838, 1452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7571, 898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2029, 0, 790, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4980, 0, 0, 0, 621, 45, 0, 1444, 0, 0, 0, 0, 0, 0, 2476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1402, 0, 0, 9, 0, 0, 0, 8076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4324, 13829, 9707, 5294, 8469, 4860, 9566, 9487] avg=8192.0000 max=13829 max_rank=1 min=4324 min_rank=0 imbalance_ratio=1.688110 max_load_reduction=51707 max_load_reduction_pct=0.788986 moved_to_echo=32382 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[5111, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 17, 23, 22, 0, 0, 1345, 2403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2299, 0, 0, 0, 0, 0, 132, 0, 0, 0, 1, 233, 5805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 529, 7528, 1, 0, 1, 0, 0, 10, 0, 0, 0, 0, 1, 863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 17, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4212, 1444, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7, 9, 0, 3, 0, 0, 0, 0, 7456, 892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1296, 0, 30, 0, 0, 0, 0, 3, 0, 3, 0, 743, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7768, 0, 5, 0, 612, 5383, 0, 1, 0, 0, 0, 0, 0, 0, 3130, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1539, 0, 95, 0, 1, 0, 0, 4377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8922, 8470, 8949, 5708, 8368, 2204, 16903, 6012] avg=8192.0000 max=16903 max_rank=6 min=2204 min_rank=5 imbalance_ratio=2.063354 max_load_reduction=48633 max_load_reduction_pct=0.742081 moved_to_echo=33487 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[436, 393, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 3084, 2206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 6517, 0, 0, 0, 0, 2, 7, 0, 0, 0, 0, 151, 976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 296, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 3865, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 5, 7571, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7834, 0, 515, 3146, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6845, 2, 10, 108, 621, 47, 0, 2807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1413, 0, 0, 0, 46, 0, 0, 3035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6158, 7658, 10142, 4445, 7728, 14471, 10440, 4494] avg=8192.0000 max=14471 max_rank=5 min=4445 min_rank=3 imbalance_ratio=1.766479 max_load_reduction=51065 max_load_reduction_pct=0.779190 moved_to_echo=37133 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[171, 2526, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 227, 1366, 2364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 276, 0, 613, 0, 0, 0, 47, 0, 0, 0, 31, 5807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7970, 0, 101, 20, 0, 0, 0, 0, 2, 1, 0, 0, 876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4143, 1446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 7513, 892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1316, 0, 0, 9, 0, 11, 0, 0, 7, 0, 0, 8069, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2531, 0, 0, 0, 617, 17, 47, 0, 0, 0, 113, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1514, 0, 0, 1261, 0, 3, 0, 5504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6711, 6774, 8971, 13649, 8407, 9413, 3329, 8282] avg=8192.0000 max=13649 max_rank=3 min=3329 min_rank=6 imbalance_ratio=1.666138 max_load_reduction=51887 max_load_reduction_pct=0.791733 moved_to_echo=29431 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[5415, 538, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 2467, 2375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 8, 0, 0, 0, 29, 3, 0, 0, 0, 5753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 8134, 0, 0, 0, 0, 0, 47, 5, 0, 56, 0, 0, 1582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 4162, 1431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 6697, 885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4797, 98, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6159, 0, 21, 0, 550, 5353, 0, 142, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1521, 0, 0, 0, 0, 0, 3, 2968, 1638, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10800, 5797, 9855, 5861, 7594, 7273, 12226, 6130] avg=8192.0000 max=12226 max_rank=6 min=5797 min_rank=1 imbalance_ratio=1.492432 max_load_reduction=53310 max_load_reduction_pct=0.813446 moved_to_echo=33223 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[6117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 318, 0, 0, 0, 4, 3039, 2366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 6, 0, 0, 0, 5837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 641, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4146, 1452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 7571, 898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2929, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 7120, 0, 0, 272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 621, 155, 0, 772, 0, 0, 0, 0, 0, 0, 573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1515, 0, 523, 0, 0, 0, 0, 7837, 1504, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11844, 5859, 9292, 6244, 8471, 10324, 2121, 11381] avg=8192.0000 max=11844 max_rank=0 min=2121 min_rank=6 imbalance_ratio=1.445801 max_load_reduction=53692 max_load_reduction_pct=0.819275 moved_to_echo=32297 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 389, 0, 0, 0, 0, 3, 0, 29, 36, 1, 1, 0, 0, 0, 0, 2592, 2314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 524, 478, 0, 1, 0, 0, 146, 0, 0, 0, 0, 0, 5782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 235, 4543, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 760, 0, 7661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 1438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 2, 0, 0, 121, 0, 4, 0, 57, 0, 0, 7035, 888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2210, 0, 42, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 26, 0, 574, 155, 0, 7252, 0, 1, 0, 1, 0, 0, 1311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1475, 0, 0, 619, 46, 0, 0, 8097, 2, 0, 0, 302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5365, 6932, 6447, 13912, 8111, 4869, 9359, 10541] avg=8192.0000 max=13912 max_rank=3 min=4869 min_rank=5 imbalance_ratio=1.698242 max_load_reduction=51624 max_load_reduction_pct=0.787720 moved_to_echo=32880 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 28, 13, 1, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4437, 0, 66, 0, 5, 0, 105, 0, 91, 0, 0, 1, 0, 0, 0, 0, 418, 0, 0, 41, 1, 35, 0, 0, 0, 0, 0, 0, 0, 0, 8064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6814, 16, 0, 0, 3, 0, 0, 0, 0, 4, 0, 8133, 0, 0, 1446, 0, 505, 0, 0, 0, 8154, 0, 0, 336, 0, 0, 0, 0, 0, 0, 132, 0, 0, 0, 7731, 1909, 0, 52, 524, 156, 0, 0, 8110, 3, 0, 3, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 28, 13, 1, 8192, 0, 0, 0, 0, 0, 0, 2616, 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4437, 0, 66, 0, 5, 0, 105, 0, 91, 0, 0, 1, 5512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 418, 0, 0, 41, 1, 35, 0, 0, 0, 0, 0, 0, 1678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 986, 1372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7535, 847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2520, 16, 0, 0, 3, 0, 0, 0, 0, 4, 0, 8133, 0, 0, 1446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 0, 0, 0, 619, 0, 0, 336, 0, 0, 0, 0, 0, 0, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 361, 0, 52, 524, 156, 0, 0, 8110, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11412, 10217, 2173, 10423, 8388, 12122, 1592, 9209] avg=8192.0000 max=12122 max_rank=5 min=1592 min_rank=6 imbalance_ratio=1.479736 max_load_reduction=53414 max_load_reduction_pct=0.815033 moved_to_echo=31311 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 7874, 0, 0, 0, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 176, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5482, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2073, 0, 814, 15, 0, 5003, 0, 0, 0, 8192, 54, 0, 1461, 0, 0, 0, 0, 0, 0, 2389, 0, 0, 0, 8187, 7433, 0, 0, 9, 1, 0, 0, 8055, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[160, 2499, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 202, 0, 0, 0, 0, 0, 290, 0, 594, 0, 0, 0, 52, 0, 0, 0, 32, 0, 0, 0, 0, 7968, 0, 97, 19, 0, 0, 0, 0, 2, 1, 0, 0, 3, 0, 8077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3483, 0, 0, 10, 0, 11, 0, 0, 7, 0, 0, 8061, 0, 0, 0, 0, 2632, 0, 0, 0, 8130, 19, 47, 0, 0, 0, 117, 0, 0, 0, 6, 0, 0, 0, 8146, 8017, 0, 0, 1302, 0, 3, 0, 5489, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 28, 11, 2, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4573, 0, 69, 0, 7, 0, 103, 0, 93, 0, 0, 1, 0, 0, 0, 0, 473, 0, 0, 51, 1, 32, 0, 0, 0, 0, 0, 0, 1, 0, 8061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6802, 18, 0, 0, 3, 0, 0, 0, 0, 4, 0, 8135, 0, 0, 1429, 0, 481, 0, 0, 0, 8155, 0, 0, 304, 0, 0, 0, 0, 0, 0, 147, 0, 0, 0, 7751, 1780, 0, 49, 505, 155, 0, 0, 8110, 1, 0, 3, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[5517, 573, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 6, 0, 0, 0, 34, 3, 0, 0, 0, 0, 0, 0, 37, 8133, 0, 0, 0, 0, 0, 47, 3, 1, 42, 0, 0, 0, 0, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 6307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4793, 100, 0, 0, 0, 6116, 0, 21, 0, 7234, 5251, 0, 124, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8049, 8030, 0, 0, 0, 0, 0, 2, 3075, 1731, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[6208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 323, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 6, 0, 0, 0, 0, 0, 0, 0, 7321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 618, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 7909, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 7233, 0, 0, 250, 0, 0, 0, 0, 0, 8192, 146, 0, 758, 0, 0, 0, 0, 0, 0, 579, 0, 0, 0, 8187, 8032, 0, 547, 0, 0, 0, 0, 7839, 1356, 0, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[449, 402, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 6549, 0, 0, 0, 0, 1, 5, 0, 0, 0, 0, 157, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 276, 0, 1, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4, 8044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7788, 0, 506, 2947, 4, 6768, 2, 13, 110, 8192, 48, 0, 2907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1501, 7472, 0, 0, 0, 40, 0, 0, 3093, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[5048, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 15, 23, 21, 0, 0, 0, 0, 0, 0, 2391, 0, 0, 0, 0, 0, 108, 0, 0, 0, 1, 232, 0, 0, 17, 557, 7497, 1, 0, 1, 0, 0, 10, 0, 0, 0, 0, 1, 0, 0, 29, 18, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7, 5, 0, 3, 0, 0, 0, 0, 3560, 0, 34, 0, 0, 0, 0, 4, 0, 3, 0, 751, 130, 0, 0, 0, 7798, 0, 4, 0, 8070, 5209, 0, 2, 0, 0, 0, 0, 0, 0, 3334, 3, 0, 0, 8145, 8153, 0, 98, 0, 1, 0, 0, 4244, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 442, 0, 0, 0, 0, 2, 0, 29, 40, 1, 2, 0, 0, 0, 0, 1, 0, 0, 0, 463, 485, 0, 3, 0, 0, 165, 0, 0, 0, 0, 0, 0, 0, 0, 223, 4782, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 753, 0, 7673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 2, 0, 0, 124, 1, 4, 0, 59, 0, 0, 6612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2112, 0, 41, 120, 0, 34, 0, 37, 0, 7600, 118, 0, 7239, 0, 1, 0, 1, 0, 0, 1359, 0, 0, 0, 8107, 7849, 0, 0, 605, 41, 0, 0, 8096, 2, 0, 0, 298, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2087, 2205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7874, 0, 0, 0, 0, 0, 114, 0, 0, 0, 0, 0, 5794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3815, 1487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7563, 906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2064, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2073, 0, 814, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5003, 0, 0, 0, 629, 54, 0, 1461, 0, 0, 0, 0, 0, 0, 2389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1413, 0, 0, 9, 1, 0, 0, 8055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4321, 13782, 9675, 5308, 8469, 4967, 9536, 9478] avg=8192.0000 max=13782 max_rank=1 min=4321 min_rank=0 imbalance_ratio=1.682373 max_load_reduction=51754 max_load_reduction_pct=0.789703 moved_to_echo=32450 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[6208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 323, 0, 0, 0, 4, 3011, 2383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 6, 0, 0, 0, 5794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 618, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4122, 1487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 7563, 906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2977, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 7233, 0, 0, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 629, 146, 0, 758, 0, 0, 0, 0, 0, 0, 579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1527, 0, 547, 0, 0, 0, 0, 7839, 1356, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11929, 5816, 9242, 6231, 8472, 10463, 2112, 11271] avg=8192.0000 max=11929 max_rank=0 min=2112 min_rank=6 imbalance_ratio=1.456177 max_load_reduction=53607 max_load_reduction_pct=0.817978 moved_to_echo=32318 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[160, 2499, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 202, 1326, 2379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 290, 0, 594, 0, 0, 0, 52, 0, 0, 0, 32, 5765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7968, 0, 97, 19, 0, 0, 0, 0, 2, 1, 0, 0, 846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4114, 1479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7506, 902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1311, 0, 0, 10, 0, 11, 0, 0, 7, 0, 0, 8061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2632, 0, 0, 0, 624, 19, 47, 0, 0, 0, 117, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1524, 0, 0, 1302, 0, 3, 0, 5489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6625, 6733, 8933, 13673, 8409, 9400, 3445, 8318] avg=8192.0000 max=13673 max_rank=3 min=3445 min_rank=6 imbalance_ratio=1.669067 max_load_reduction=51863 max_load_reduction_pct=0.791367 moved_to_echo=29572 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[449, 402, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 3062, 2217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 6549, 0, 0, 0, 0, 1, 5, 0, 0, 0, 0, 157, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 276, 0, 1, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 3835, 273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4, 7563, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7788, 0, 506, 2947, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6768, 2, 13, 110, 629, 48, 0, 2907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1420, 0, 0, 0, 40, 0, 0, 3093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6180, 7780, 10114, 4425, 7734, 14273, 10477, 4553] avg=8192.0000 max=14273 max_rank=5 min=4425 min_rank=3 imbalance_ratio=1.742310 max_load_reduction=51263 max_load_reduction_pct=0.782211 moved_to_echo=37037 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[5517, 573, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 2401, 2382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 6, 0, 0, 0, 34, 3, 0, 0, 0, 5696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 8133, 0, 0, 0, 0, 0, 47, 3, 1, 42, 0, 0, 1532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 4121, 1462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 6678, 891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4793, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6116, 0, 21, 0, 556, 5251, 0, 124, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1527, 0, 0, 0, 0, 0, 2, 3075, 1731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10879, 5742, 9795, 5865, 7584, 7267, 12069, 6335] avg=8192.0000 max=12069 max_rank=6 min=5742 min_rank=1 imbalance_ratio=1.473267 max_load_reduction=53467 max_load_reduction_pct=0.815842 moved_to_echo=33255 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 28, 11, 2, 8192, 0, 0, 0, 0, 0, 0, 2590, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4573, 0, 69, 0, 7, 0, 103, 0, 93, 0, 0, 1, 5485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 0, 0, 51, 1, 32, 0, 0, 0, 0, 0, 0, 1652, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 913, 1409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7529, 857, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2560, 18, 0, 0, 3, 0, 0, 0, 0, 4, 0, 8135, 0, 0, 1429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 0, 0, 0, 626, 0, 0, 304, 0, 0, 0, 0, 0, 0, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 339, 0, 49, 505, 155, 0, 0, 8110, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11351, 10331, 2209, 10385, 8391, 12149, 1558, 9162] avg=8192.0000 max=12149 max_rank=5 min=1558 min_rank=6 imbalance_ratio=1.483032 max_load_reduction=53387 max_load_reduction_pct=0.814621 moved_to_echo=31260 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 442, 0, 0, 0, 0, 2, 0, 29, 40, 1, 2, 0, 0, 0, 0, 2521, 2332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 463, 485, 0, 3, 0, 0, 165, 0, 0, 0, 0, 0, 5739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 223, 4782, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 1611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 753, 0, 7673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 1472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 2, 0, 0, 124, 1, 4, 0, 59, 0, 0, 7020, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2112, 0, 41, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 37, 0, 580, 118, 0, 7239, 0, 1, 0, 1, 0, 0, 1359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1484, 0, 0, 605, 41, 0, 0, 8096, 2, 0, 0, 298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5369, 6856, 6620, 13931, 8111, 4754, 9369, 10526] avg=8192.0000 max=13931 max_rank=3 min=4754 min_rank=5 imbalance_ratio=1.700562 max_load_reduction=51605 max_load_reduction_pct=0.787430 moved_to_echo=32760 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[5048, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 15, 23, 21, 0, 0, 1355, 2419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2391, 0, 0, 0, 0, 0, 108, 0, 0, 0, 1, 232, 5764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 557, 7497, 1, 0, 1, 0, 0, 10, 0, 0, 0, 0, 1, 864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 18, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4184, 1481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7, 5, 0, 3, 0, 0, 0, 0, 7450, 900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1341, 0, 34, 0, 0, 0, 0, 4, 0, 3, 0, 751, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7798, 0, 4, 0, 620, 5209, 0, 2, 0, 0, 0, 0, 0, 0, 3334, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1550, 0, 98, 0, 1, 0, 0, 4244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8882, 8496, 8948, 5718, 8366, 2263, 16970, 5893] avg=8192.0000 max=16970 max_rank=6 min=2263 min_rank=5 imbalance_ratio=2.071533 max_load_reduction=48566 max_load_reduction_pct=0.741058 moved_to_echo=33492 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[17280, 3968, 0, 128, 128, 0, 128, 128, 128, 8320, 128, 384, 128, 128, 0, 256, 18688, 16896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[28928, 128, 128, 128, 4864, 11264, 128, 12800, 0, 128, 128, 128, 128, 0, 7680, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[768, 0, 24960, 128, 0, 0, 128, 0, 0, 128, 0, 0, 128, 0, 128, 0, 29440, 10368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[18048, 128, 128, 128, 128, 128, 0, 128, 128, 128, 0, 40960, 256, 1408, 5120, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 128, 21760, 768, 128, 640, 128, 128, 640, 128, 128, 0, 128, 512, 41344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 10752, 0, 768, 2432, 256, 128, 128, 48128, 3200, 0, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[17536, 3968, 0, 128, 128, 0, 128, 128, 128, 8320, 128, 384, 128, 128, 0, 256, 18432, 16896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[28928, 128, 128, 128, 4992, 10880, 128, 12800, 0, 128, 128, 128, 128, 0, 7936, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[768, 0, 24960, 128, 128, 0, 128, 0, 0, 128, 0, 0, 128, 128, 128, 0, 29184, 10624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[18176, 128, 128, 128, 128, 128, 0, 128, 128, 128, 128, 40960, 256, 1408, 4864, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 0, 128, 21888, 896, 128, 640, 128, 128, 512, 128, 128, 0, 128, 512, 41216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 10880, 0, 768, 2432, 256, 128, 128, 48128, 3200, 0, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 128, 128, 1024, 52352, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 12032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 0, 128, 1024, 52608, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 11776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 0, 128, 128, 128, 0, 128, 0, 128, 256, 0, 128, 0, 128, 0, 128, 59008, 6400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 8146, 8189, 0, 0, 0, 1421, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 3455, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 8192, 4, 0, 0, 4346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6513] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 8164, 0, 0, 7376, 0, 0, 0, 20, 0, 0, 8192, 8178, 0, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 11, 0, 0, 441, 0, 0, 0, 6928, 0, 0, 0, 2, 0, 0, 337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 210, 0, 7783, 2199, 5, 0, 107, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 214, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 742, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 177, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6172] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 8180, 0, 0, 8184, 0, 0, 0, 1, 0, 0, 8151, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 1522, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 21, 0, 0, 8181, 610, 0, 0, 7816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7861, 0, 0, 36, 0, 6, 5571, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 220, 0, 0, 3, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 15, 21, 0, 0, 0, 0, 0, 59, 808] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 8110, 0, 0, 7951, 0, 10, 0, 0, 0, 0, 5041, 8049, 6, 2, 122, 2912, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 8002, 0, 0, 1, 1, 6, 0, 1, 0, 0, 0, 0, 0, 0, 0, 31, 0, 10, 0, 8108, 0, 0, 0, 7525, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 10, 8127, 0, 672, 46, 0, 5, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 23, 1, 0, 2, 0, 0, 0, 157, 167, 158, 0, 3, 0, 0, 0, 0, 0, 83, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 8159, 0, 0, 8174, 0, 0, 0, 248, 0, 4, 7963, 8181, 297, 0, 0, 3, 0, 4, 0, 0, 0, 17, 0, 0, 3, 0, 0, 0, 0, 0, 0, 130, 0, 0, 0, 0, 0, 0, 9, 0, 7, 0, 7651, 5, 0, 0, 0, 0, 0, 0, 19, 0, 13, 0, 57, 0, 0, 0, 0, 6, 0, 8183, 7, 0, 0, 9, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6380, 0, 1, 0, 0, 1612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 615, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 0, 7684] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1, 8189, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 8168, 8136, 0, 0, 0, 2, 0, 0, 0, 0, 838, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2880, 36, 6, 0, 0, 0, 0, 313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7930, 4, 2, 0, 6428, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7109, 0, 0, 0, 0, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 2, 0, 0, 7216] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 8182, 0, 0, 8116, 73, 0, 0, 0, 0, 0, 8076, 8151, 0, 0, 0, 2021, 2, 0, 2, 0, 0, 0, 0, 1, 2, 0, 0, 13, 45, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7893, 2, 0, 0, 0, 1, 5, 0, 0, 0, 0, 0, 0, 5, 0, 0, 3, 12, 0, 7011, 567, 0, 3, 4435, 1, 0, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 7648, 0, 1, 0, 0, 0, 216, 0, 1, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 22, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 458, 231, 2159] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 7889, 0, 0, 6331, 0, 0, 0, 0, 0, 0, 8029, 7984, 236, 0, 0, 0, 0, 0, 0, 0, 6876, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 213, 0, 43, 116, 0, 22, 0, 0, 0, 0, 1524, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1778, 1, 8190, 703, 5, 0, 6416, 0, 5, 22, 926, 0, 0, 0, 0, 0, 29, 0, 0, 0, 8, 2, 0, 0, 0, 7590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 357] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 362, 0, 0, 0, 0, 0, 0, 8146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1421, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 818, 0, 0, 0, 0, 6225, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 3455, 3, 0, 0, 0, 1954, 641, 606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 7551, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6513, 2291, 1964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8508, 8583, 6674, 7565, 7922, 7441, 8074, 10769] avg=8192.0000 max=10769 max_rank=7 min=6674 min_rank=2 imbalance_ratio=1.314575 max_load_reduction=54767 max_load_reduction_pct=0.835678 moved_to_echo=34206 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 325, 0, 0, 0, 20, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6218, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 11, 0, 0, 441, 0, 0, 0, 6928, 0, 0, 0, 2, 1762, 609, 617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 210, 0, 7174, 2199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 107, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 214, 0, 0, 1, 7574, 0, 0, 0, 0, 0, 742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 177, 0, 0, 0, 0, 8046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6172, 2065, 1960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8537, 6414, 10371, 9920, 3337, 8531, 8226, 10200] avg=8192.0000 max=10371 max_rank=2 min=3337 min_rank=4 imbalance_ratio=1.265991 max_load_reduction=55165 max_load_reduction_pct=0.841751 moved_to_echo=30294 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 361, 0, 0, 0, 1, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 6189, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 1522, 0, 0, 0, 0, 1955, 641, 592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 21, 0, 0, 7540, 610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7269, 0, 0, 36, 0, 6, 5571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 220, 0, 0, 3, 0, 0, 0, 0, 0, 36, 8062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 21, 0, 0, 0, 0, 0, 59, 808, 2291, 1953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8513, 6364, 4723, 8177, 11393, 12882, 8337, 5147] avg=8192.0000 max=12882 max_rank=5 min=4723 min_rank=2 imbalance_ratio=1.572510 max_load_reduction=52654 max_load_reduction_pct=0.803436 moved_to_echo=37759 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 359, 73, 0, 0, 0, 0, 0, 8076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2021, 2, 0, 2, 0, 0, 0, 0, 1, 2, 0, 0, 13, 45, 2, 0, 6197, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7893, 2, 0, 0, 0, 1938, 549, 576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 0, 0, 0, 0, 0, 0, 5, 0, 0, 3, 12, 0, 6462, 567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4435, 1, 0, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 23, 3547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7072, 0, 1, 0, 0, 0, 216, 0, 1, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 458, 231, 2159, 2272, 1954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8508, 8404, 10958, 7055, 8098, 7302, 8064, 7147] avg=8192.0000 max=10958 max_rank=2 min=7055 min_rank=3 imbalance_ratio=1.337646 max_load_reduction=54578 max_load_reduction_pct=0.832794 moved_to_echo=30611 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 280, 0, 0, 0, 0, 0, 0, 8029, 0, 236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6876, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 6069, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 213, 0, 43, 116, 0, 22, 0, 0, 0, 0, 1524, 0, 0, 0, 1, 1512, 641, 571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1778, 1, 7549, 703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 6416, 0, 5, 22, 926, 0, 0, 0, 0, 0, 29, 0, 0, 0, 2767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 2, 0, 0, 0, 7019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 235, 0, 0, 0, 0, 0, 0, 0, 7775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 357, 1772, 1915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8545, 13061, 4643, 10031, 10170, 7029, 8010, 4047] avg=8192.0000 max=13061 max_rank=1 min=4047 min_rank=7 imbalance_ratio=1.594360 max_load_reduction=52475 max_load_reduction_pct=0.800705 moved_to_echo=29256 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 362, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 838, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 6185, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2880, 36, 6, 0, 0, 1955, 621, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7309, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 6428, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6574, 0, 0, 0, 0, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 8071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 2, 0, 0, 7216, 2292, 1951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8531, 7175, 6039, 7627, 10024, 6584, 8082, 11474] avg=8192.0000 max=11474 max_rank=7 min=6039 min_rank=2 imbalance_ratio=1.400635 max_load_reduction=54062 max_load_reduction_pct=0.824921 moved_to_echo=36164 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 361, 0, 0, 0, 248, 0, 4, 7963, 0, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4, 0, 0, 0, 17, 0, 0, 3, 0, 0, 0, 0, 0, 0, 6220, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 0, 0, 0, 0, 0, 0, 9, 0, 7, 0, 7651, 5, 0, 0, 0, 1952, 641, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 13, 0, 57, 0, 0, 0, 0, 6, 0, 7542, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5900, 0, 1, 0, 0, 1612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 615, 0, 0, 0, 8041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 0, 7684, 2288, 1961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8873, 6365, 10875, 7644, 3584, 7513, 8656, 12026] avg=8192.0000 max=12026 max_rank=7 min=3584 min_rank=4 imbalance_ratio=1.468018 max_load_reduction=53510 max_load_reduction_pct=0.816498 moved_to_echo=31779 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 340, 0, 10, 0, 0, 0, 0, 5041, 0, 6, 2, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2912, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6120, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 8002, 0, 0, 1, 1, 1903, 639, 615, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1, 0, 0, 0, 0, 0, 0, 0, 31, 0, 10, 0, 7469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7525, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 3479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 7512, 0, 672, 46, 0, 5, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 23, 1, 0, 2, 7992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 167, 158, 0, 3, 0, 0, 0, 0, 0, 83, 0, 0, 2229, 1929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5521, 9170, 11253, 7517, 11008, 8254, 8087, 4726] avg=8192.0000 max=11253 max_rank=2 min=4726 min_rank=7 imbalance_ratio=1.373657 max_load_reduction=54283 max_load_reduction_pct=0.828293 moved_to_echo=32075 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 128, 128, 128, 128, 0, 128, 0, 128, 256, 128, 128, 0, 128, 0, 128, 58880, 6528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 8154, 8190, 0, 0, 0, 1359, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 3561, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 8192, 5, 0, 0, 4302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6469] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1, 8188, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 8168, 8140, 0, 0, 0, 2, 0, 0, 0, 0, 853, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2961, 41, 4, 0, 0, 0, 0, 316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7930, 4, 4, 0, 6523, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6973, 1, 0, 0, 0, 13, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 2, 0, 0, 7153] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 8164, 0, 0, 7392, 0, 0, 0, 20, 0, 0, 8192, 8179, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 12, 0, 0, 397, 0, 0, 0, 6956, 0, 0, 0, 1, 0, 0, 323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 206, 0, 7791, 2275, 5, 0, 103, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 225, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 758, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6073] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 8180, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 8154, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 13, 0, 0, 1575, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0, 0, 0, 21, 0, 0, 8182, 579, 0, 0, 7824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7841, 0, 0, 35, 0, 4, 5523, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 215, 0, 0, 3, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 21, 21, 0, 0, 0, 0, 0, 63, 841] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 7890, 0, 0, 6285, 0, 0, 0, 0, 0, 0, 8040, 7987, 255, 0, 0, 1, 0, 0, 0, 0, 6832, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 212, 0, 52, 116, 0, 20, 0, 0, 0, 0, 1696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1723, 1, 8189, 776, 6, 0, 6377, 0, 5, 21, 922, 0, 0, 0, 0, 0, 31, 0, 0, 0, 7, 3, 0, 0, 0, 7584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 266] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 8110, 0, 0, 7946, 0, 10, 0, 0, 0, 0, 5073, 8049, 4, 2, 118, 2885, 0, 0, 0, 0, 21, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 8006, 0, 0, 1, 1, 6, 0, 1, 0, 0, 0, 0, 0, 0, 0, 33, 0, 8, 0, 8106, 0, 0, 0, 7512, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 9, 8129, 0, 681, 45, 0, 5, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 17, 1, 0, 3, 0, 0, 0, 148, 178, 159, 0, 3, 0, 0, 0, 0, 0, 78, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 8158, 0, 0, 8174, 0, 0, 1, 248, 0, 4, 7966, 8180, 297, 0, 0, 2, 0, 8, 0, 0, 0, 16, 0, 0, 4, 0, 0, 0, 0, 0, 0, 129, 0, 0, 0, 0, 0, 0, 9, 0, 6, 0, 7667, 5, 1, 0, 0, 0, 0, 0, 18, 0, 11, 0, 58, 0, 0, 0, 0, 7, 0, 8184, 10, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6340, 0, 1, 0, 0, 1644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 7682] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 8181, 0, 0, 8117, 74, 0, 0, 0, 0, 0, 8071, 8153, 0, 0, 0, 1992, 2, 0, 2, 0, 0, 0, 0, 3, 2, 0, 0, 17, 34, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7904, 2, 0, 0, 0, 1, 5, 0, 0, 0, 0, 0, 0, 7, 0, 0, 3, 11, 0, 7030, 536, 0, 2, 4469, 1, 0, 0, 0, 0, 0, 0, 97, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 7636, 0, 1, 0, 0, 0, 190, 0, 2, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 20, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 456, 228, 2195] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 353, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1359, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 850, 0, 0, 0, 0, 6243, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 3561, 3, 0, 0, 0, 1918, 648, 580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 7544, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6469, 2338, 1947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8507, 8567, 6720, 7563, 7875, 7471, 8078, 10755] avg=8192.0000 max=10755 max_rank=7 min=6720 min_rank=2 imbalance_ratio=1.312866 max_load_reduction=54781 max_load_reduction_pct=0.835892 moved_to_echo=34179 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 353, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 853, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 6205, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2961, 41, 4, 0, 0, 1919, 627, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7303, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 6523, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6471, 1, 0, 0, 0, 13, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 8075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 2, 0, 0, 7153, 2340, 1935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8522, 7204, 6060, 7624, 10112, 6486, 8087, 11441] avg=8192.0000 max=11441 max_rank=7 min=6060 min_rank=2 imbalance_ratio=1.396606 max_load_reduction=54095 max_load_reduction_pct=0.825424 moved_to_echo=36126 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 353, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 6206, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 13, 0, 0, 1575, 0, 0, 0, 0, 1918, 647, 565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0, 0, 0, 21, 0, 0, 7535, 579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7276, 0, 0, 35, 0, 4, 5523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 215, 0, 0, 3, 0, 0, 0, 0, 0, 33, 8067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 21, 0, 0, 0, 0, 0, 63, 841, 2340, 1936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8507, 6374, 4719, 8142, 11399, 12839, 8334, 5222] avg=8192.0000 max=12839 max_rank=5 min=4719 min_rank=2 imbalance_ratio=1.567261 max_load_reduction=52697 max_load_reduction_pct=0.804092 moved_to_echo=37794 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 319, 0, 0, 0, 20, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6236, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 12, 0, 0, 397, 0, 0, 0, 6956, 0, 0, 0, 1, 1732, 616, 590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 206, 0, 7175, 2275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 103, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 225, 0, 0, 1, 7601, 0, 0, 0, 0, 0, 758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, 0, 0, 0, 0, 8050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6073, 2113, 1943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8531, 6433, 10306, 9979, 3337, 8585, 8233, 10132] avg=8192.0000 max=10306 max_rank=2 min=3337 min_rank=4 imbalance_ratio=1.258057 max_load_reduction=55230 max_load_reduction_pct=0.842743 moved_to_echo=30287 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 272, 0, 0, 0, 0, 0, 0, 8040, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 6832, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 6088, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 0, 52, 116, 0, 20, 0, 0, 0, 0, 1696, 0, 0, 0, 0, 1473, 648, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1723, 1, 7541, 776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 6377, 0, 5, 21, 922, 0, 0, 0, 0, 0, 31, 0, 0, 0, 2744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3, 0, 0, 0, 7038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 233, 0, 0, 0, 0, 0, 0, 0, 7780, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 266, 1796, 1899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8567, 13034, 4763, 10041, 10106, 7048, 8013, 3964] avg=8192.0000 max=13034 max_rank=1 min=3964 min_rank=7 imbalance_ratio=1.591064 max_load_reduction=52502 max_load_reduction_pct=0.801117 moved_to_echo=29131 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 353, 0, 0, 1, 248, 0, 4, 7966, 0, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8, 0, 0, 0, 16, 0, 0, 4, 0, 0, 0, 0, 0, 0, 6236, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 0, 0, 0, 0, 0, 0, 9, 0, 6, 0, 7667, 5, 1, 0, 0, 1916, 647, 456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 11, 0, 58, 0, 0, 0, 0, 7, 0, 7537, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5884, 0, 1, 0, 0, 1644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 594, 0, 0, 0, 8044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 7682, 2336, 1944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8869, 6380, 10836, 7641, 3581, 7529, 8638, 12062] avg=8192.0000 max=12062 max_rank=7 min=3581 min_rank=4 imbalance_ratio=1.472412 max_load_reduction=53474 max_load_reduction_pct=0.815948 moved_to_echo=31810 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 331, 0, 10, 0, 0, 0, 0, 5073, 0, 4, 2, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2885, 0, 0, 0, 0, 21, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6137, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 8006, 0, 0, 1, 1, 1866, 645, 589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1, 0, 0, 0, 0, 0, 0, 0, 33, 0, 8, 0, 7461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7512, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 3474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 7540, 0, 681, 45, 0, 5, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 17, 1, 0, 3, 7997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 178, 159, 0, 3, 0, 0, 0, 0, 0, 78, 0, 0, 2275, 1912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5538, 9157, 11218, 7509, 10990, 8287, 8084, 4753] avg=8192.0000 max=11218 max_rank=2 min=4753 min_rank=7 imbalance_ratio=1.369385 max_load_reduction=54318 max_load_reduction_pct=0.828827 moved_to_echo=32114 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 350, 74, 0, 0, 0, 0, 0, 8071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1992, 2, 0, 2, 0, 0, 0, 0, 3, 2, 0, 0, 17, 34, 5, 0, 6216, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7904, 2, 0, 0, 0, 1902, 556, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 0, 0, 0, 0, 0, 0, 7, 0, 0, 3, 11, 0, 6474, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4469, 1, 0, 0, 0, 0, 0, 0, 97, 0, 0, 0, 0, 22, 3545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7086, 0, 1, 0, 0, 0, 190, 0, 2, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 456, 228, 2195, 2320, 1937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8495, 8387, 10914, 7037, 8136, 7291, 8069, 7207] avg=8192.0000 max=10914 max_rank=2 min=7037 min_rank=3 imbalance_ratio=1.332275 max_load_reduction=54622 max_load_reduction_pct=0.833466 moved_to_echo=30703 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 256, 256, 256, 0, 256, 128, 0, 0, 0, 128, 640, 384, 30976, 17536, 15616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[256, 256, 128, 128, 128, 0, 128, 512, 128, 128, 0, 39936, 128, 128, 128, 128, 14976, 4992, 4608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 37120, 128, 128, 128, 1024, 0, 0, 128, 128, 0, 128, 0, 128, 128, 27392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[6528, 128, 128, 128, 0, 7808, 128, 0, 128, 128, 0, 896, 128, 128, 128, 0, 49536, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 256, 256, 256, 0, 256, 128, 0, 0, 0, 128, 640, 384, 30720, 17920, 15488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[256, 256, 128, 128, 128, 0, 128, 512, 128, 128, 0, 40448, 128, 128, 128, 128, 14720, 5120, 4480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 37248, 128, 128, 128, 1024, 0, 0, 128, 128, 0, 128, 0, 128, 128, 27392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[6400, 128, 128, 128, 0, 7808, 128, 0, 128, 128, 128, 896, 128, 128, 128, 0, 49664, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 256, 0, 0, 128, 56448, 0, 768, 128, 0, 1664, 6656, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 0, 0, 2816, 128, 128, 0, 384, 0, 128, 61824, 0, 640, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 768, 128, 128, 128, 0, 128, 128, 0, 128, 128, 2048, 128, 58624, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 0, 0, 384, 0, 256, 128, 0, 256, 640, 128, 0, 128, 64128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 256, 0, 0, 128, 56448, 128, 768, 128, 0, 1792, 6528, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 640, 128, 128, 128, 0, 128, 128, 0, 128, 128, 2048, 128, 58624, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 0, 0, 2688, 128, 128, 128, 384, 0, 128, 61824, 0, 640, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 0, 0, 0, 384, 0, 256, 128, 0, 256, 640, 128, 0, 128, 64256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[8192, 0, 0, 0, 0, 0, 0, 0, 0, 418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8163, 7946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 48, 0, 0, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7293, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 7969, 0, 0, 0, 8120, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 716, 0, 3, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[7907, 0, 0, 0, 0, 0, 0, 7585, 0, 0, 90, 0, 0, 0, 0, 14, 0, 0, 10, 0, 102, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 6, 0, 0, 0, 0, 1, 69, 228, 0, 0, 0, 0, 0, 0, 7989, 0, 0, 0, 0, 5605, 7511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1142, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 183, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1635, 2, 8099, 0, 0, 0, 638, 0, 0, 0, 8, 8180, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 292, 22, 0, 0, 3, 0, 5, 0, 0, 0, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[8191, 0, 0, 0, 0, 0, 0, 0, 0, 182, 0, 0, 5, 0, 1142, 0, 0, 5, 1, 6, 0, 2, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8182, 0, 0, 0, 0, 8165, 7145, 2, 2, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 43, 135, 0, 0, 0, 0, 0, 239, 0, 0, 0, 8127, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8086, 4, 0, 0, 7646, 0, 0, 1, 0, 8149, 0, 0, 0, 0, 3, 0, 0, 53, 0, 0, 0, 0, 0, 6, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[8177, 0, 0, 0, 0, 0, 0, 4785, 22, 49, 5, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4, 0, 0, 5, 0, 0, 0, 0, 0, 9, 0, 503, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 8176, 7622, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 76, 1106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 1, 8055, 0, 0, 0, 1423, 15, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 1, 0, 589, 1, 107, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[8160, 0, 0, 0, 12, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 7031, 0, 7, 0, 0, 337, 0, 2, 8159, 0, 0, 0, 40, 4526, 3590, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 1609, 0, 0, 0, 1, 1, 3, 0, 0, 0, 8153, 0, 7, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 7, 7139, 0, 0, 0, 8149, 0, 0, 88, 0, 8082, 5, 0, 5, 0, 0, 0, 0, 0, 8, 0, 0, 0, 15, 0, 0, 0, 0, 8, 3, 3, 0, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[8029, 0, 0, 158, 0, 0, 0, 0, 0, 3108, 0, 0, 3, 0, 74, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 5, 0, 3, 0, 0, 0, 45, 0, 39, 25, 0, 0, 8168, 0, 14, 2, 9, 8038, 8173, 0, 0, 0, 0, 0, 0, 54, 0, 20, 0, 0, 0, 0, 0, 0, 0, 523, 143, 0, 260, 0, 0, 0, 0, 1, 0, 0, 2, 7533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7721, 0, 0, 2, 5102, 0, 0, 103, 0, 7915, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 1, 0, 199, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[8192, 0, 0, 0, 0, 0, 0, 0, 0, 266, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8132, 6255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1831, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8116, 0, 0, 0, 8165, 0, 0, 0, 0, 7995, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[8171, 0, 0, 0, 1, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 6, 0, 0, 8, 0, 0, 0, 0, 0, 64, 0, 2, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 8133, 0, 0, 0, 43, 8041, 8175, 1, 5, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 24, 17, 0, 12, 0, 0, 0, 0, 0, 0, 59, 8153, 0, 21, 0, 0, 1, 2, 0, 0, 0, 4, 0, 0, 77, 0, 8109, 0, 18, 0, 8018, 0, 0, 0, 0, 8088, 0, 0, 0, 6, 0, 0, 0, 0, 129, 0, 0, 0, 34, 0, 2, 0, 0, 0, 0, 17, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[5784, 0, 0, 0, 0, 0, 0, 7585, 0, 0, 90, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 102, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 6, 8139, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 69, 228, 0, 0, 0, 0, 0, 0, 7066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 846, 7511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1142, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 183, 4120, 2123, 923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1635, 2, 8099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 638, 0, 0, 0, 8, 2275, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 292, 22, 0, 0, 3, 0, 5, 0, 0, 0, 0, 2, 0, 4660, 1785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13473, 8358, 7364, 8357, 8505, 9788, 2922, 6769] avg=8192.0000 max=13473 max_rank=0 min=2922 min_rank=6 imbalance_ratio=1.644653 max_load_reduction=52063 max_load_reduction_pct=0.794418 moved_to_echo=27984 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[5992, 0, 0, 0, 0, 0, 0, 0, 0, 266, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8139, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 7246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1227, 6255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1831, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4027, 2200, 946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 2223, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 0, 0, 6761, 1745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6278, 8284, 7303, 7482, 9007, 8168, 10393, 8621] avg=8192.0000 max=10393 max_rank=6 min=6278 min_rank=0 imbalance_ratio=1.268677 max_load_reduction=55143 max_load_reduction_pct=0.841415 moved_to_echo=36189 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[5992, 0, 0, 0, 0, 0, 0, 0, 0, 182, 0, 0, 5, 0, 1142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 6, 0, 2, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8076, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1233, 7145, 2, 2, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 135, 0, 0, 0, 0, 0, 239, 0, 0, 0, 4104, 2199, 945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 7646, 0, 0, 1, 0, 2266, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 6, 0, 1, 0, 0, 0, 0, 0, 0, 0, 6788, 1779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7321, 8237, 7242, 8386, 7665, 8138, 9920, 8627] avg=8192.0000 max=9920 max_rank=6 min=7242 min_rank=2 imbalance_ratio=1.210938 max_load_reduction=55616 max_load_reduction_pct=0.848633 moved_to_echo=34350 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[5981, 0, 0, 0, 0, 0, 0, 4785, 22, 49, 5, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 5, 0, 0, 0, 0, 0, 9, 0, 503, 8137, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 7221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1233, 7622, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 76, 1106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4115, 2196, 943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 1, 8055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1423, 15, 0, 0, 0, 2272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 1, 0, 589, 1, 107, 0, 0, 6798, 1783, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10844, 8803, 7227, 8860, 8581, 8127, 3710, 9384] avg=8192.0000 max=10844 max_rank=0 min=3710 min_rank=6 imbalance_ratio=1.323730 max_load_reduction=54692 max_load_reduction_pct=0.834534 moved_to_echo=29802 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[5977, 0, 0, 0, 1, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 8, 0, 0, 0, 0, 0, 64, 0, 2, 0, 0, 10, 0, 8101, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 7194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 1214, 8175, 1, 5, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 24, 17, 0, 12, 0, 0, 0, 0, 0, 0, 59, 4073, 2194, 939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 21, 0, 0, 1, 2, 0, 0, 0, 4, 0, 0, 77, 0, 8109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 8018, 0, 0, 0, 0, 2250, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 0, 0, 0, 34, 0, 2, 0, 0, 0, 0, 17, 0, 0, 0, 6685, 1765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6014, 8333, 7199, 9479, 7321, 8266, 10292, 8632] avg=8192.0000 max=10292 max_rank=6 min=6014 min_rank=0 imbalance_ratio=1.256348 max_load_reduction=55244 max_load_reduction_pct=0.842957 moved_to_echo=34511 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[5992, 0, 0, 0, 0, 0, 0, 0, 0, 418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7247, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1232, 7946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4125, 2200, 946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 7969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8120, 0, 0, 0, 0, 2278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 716, 0, 3, 0, 0, 6787, 1788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6410, 7391, 7246, 9186, 7567, 8043, 10398, 9295] avg=8192.0000 max=10398 max_rank=6 min=6410 min_rank=0 imbalance_ratio=1.269287 max_load_reduction=55138 max_load_reduction_pct=0.841339 moved_to_echo=35303 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[5969, 0, 0, 0, 12, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8101, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 7031, 0, 7, 0, 0, 337, 0, 2, 7217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 683, 3590, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 1609, 0, 0, 0, 1, 1, 3, 0, 0, 0, 4070, 2191, 942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 7, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 7, 7139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 88, 0, 2248, 5, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 15, 0, 0, 0, 0, 8, 3, 3, 0, 0, 3, 3763, 1764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5984, 8458, 14603, 4324, 8863, 7242, 10495, 5567] avg=8192.0000 max=14603 max_rank=2 min=4324 min_rank=3 imbalance_ratio=1.782593 max_load_reduction=50933 max_load_reduction_pct=0.777176 moved_to_echo=32167 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[5869, 0, 0, 158, 0, 0, 0, 0, 0, 3108, 0, 0, 3, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 7490, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3, 0, 0, 0, 45, 0, 39, 25, 0, 0, 7221, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 9, 1204, 8173, 0, 0, 0, 0, 0, 0, 54, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 143, 0, 260, 0, 0, 0, 0, 1, 0, 0, 2, 3991, 2160, 947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5102, 0, 0, 103, 0, 2192, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 1, 0, 199, 0, 6687, 1732, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9212, 7672, 7352, 9462, 8027, 7764, 7406, 8641] avg=8192.0000 max=9462 max_rank=3 min=7352 min_rank=2 imbalance_ratio=1.155029 max_load_reduction=56074 max_load_reduction_pct=0.855621 moved_to_echo=31838 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[8192, 0, 0, 0, 0, 0, 0, 0, 0, 264, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8133, 6240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1854, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8112, 0, 0, 0, 8163, 0, 0, 0, 0, 7987, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[8192, 0, 0, 0, 0, 0, 0, 0, 0, 390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8161, 7937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 46, 0, 0, 257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7256, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 7961, 0, 0, 0, 8121, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 792, 0, 3, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[8191, 0, 0, 0, 0, 0, 0, 0, 0, 201, 0, 0, 5, 0, 1143, 0, 0, 6, 4, 5, 0, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 8182, 0, 0, 0, 0, 8162, 7118, 2, 2, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 39, 123, 0, 0, 0, 0, 0, 229, 0, 0, 0, 8126, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 8089, 4, 0, 0, 7679, 0, 0, 0, 0, 8148, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 4, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[7923, 0, 0, 0, 0, 0, 0, 7585, 0, 0, 100, 0, 0, 0, 0, 21, 0, 0, 13, 0, 100, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 9, 0, 0, 0, 0, 1, 68, 234, 0, 0, 0, 0, 0, 0, 7986, 0, 0, 0, 0, 5620, 7553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1177, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 170, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1518, 13, 8099, 0, 0, 0, 625, 0, 0, 0, 11, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 292, 21, 0, 0, 2, 0, 5, 0, 0, 0, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[8029, 0, 0, 167, 0, 0, 0, 0, 0, 3072, 0, 0, 3, 0, 89, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 5, 0, 4, 0, 0, 0, 46, 0, 36, 13, 0, 0, 8168, 0, 13, 2, 9, 8040, 8174, 0, 0, 0, 0, 0, 0, 54, 0, 20, 0, 0, 0, 0, 0, 0, 0, 518, 118, 0, 241, 0, 0, 0, 0, 1, 0, 0, 4, 7555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7742, 0, 0, 2, 5126, 0, 0, 86, 0, 7918, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 1, 0, 209, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[8177, 0, 0, 0, 0, 0, 0, 4788, 24, 58, 4, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3, 0, 0, 6, 0, 0, 0, 0, 0, 7, 0, 495, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 8175, 7632, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 0, 79, 1140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 1, 8058, 0, 0, 0, 1366, 15, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 99, 0, 0, 0, 0, 0, 0, 0, 3, 0, 606, 1, 100, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[8161, 0, 0, 0, 11, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 7064, 0, 4, 0, 0, 347, 0, 2, 8164, 0, 0, 0, 37, 4664, 3412, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 1548, 0, 0, 0, 2, 1, 5, 0, 0, 0, 8149, 0, 7, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 5, 7190, 0, 0, 0, 8153, 0, 0, 90, 0, 8083, 4, 0, 7, 0, 0, 0, 0, 0, 6, 0, 0, 0, 15, 0, 0, 0, 0, 7, 5, 3, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[8171, 0, 0, 0, 2, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4, 0, 0, 0, 0, 0, 63, 0, 2, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 8131, 0, 0, 0, 41, 8038, 8171, 1, 5, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 29, 10, 0, 11, 0, 0, 0, 0, 0, 0, 59, 8154, 0, 21, 0, 0, 1, 1, 0, 0, 0, 4, 0, 0, 88, 0, 8111, 0, 23, 0, 8016, 0, 0, 0, 0, 8087, 0, 0, 0, 10, 0, 0, 0, 0, 128, 0, 0, 0, 34, 0, 2, 0, 0, 0, 0, 18, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[5989, 0, 0, 0, 0, 0, 0, 0, 0, 264, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8136, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 7243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1250, 6240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1854, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4022, 2203, 949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 2223, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 0, 0, 0, 0, 6735, 1742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6279, 8285, 7299, 7490, 9030, 8167, 10391, 8595] avg=8192.0000 max=10391 max_rank=6 min=6279 min_rank=0 imbalance_ratio=1.268433 max_load_reduction=55145 max_load_reduction_pct=0.841446 moved_to_echo=36183 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[5989, 0, 0, 0, 0, 0, 0, 0, 0, 390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7207, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1254, 7937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4124, 2203, 949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 7961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8121, 0, 0, 0, 0, 2280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 792, 0, 3, 0, 0, 6758, 1786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6379, 7356, 7243, 9199, 7579, 8039, 10401, 9340] avg=8192.0000 max=10401 max_rank=6 min=6379 min_rank=0 imbalance_ratio=1.269653 max_load_reduction=55135 max_load_reduction_pct=0.841293 moved_to_echo=35359 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[5988, 0, 0, 0, 0, 0, 0, 0, 0, 201, 0, 0, 5, 0, 1143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4, 5, 0, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8071, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 7234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1254, 7118, 2, 2, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 123, 0, 0, 0, 0, 0, 229, 0, 0, 0, 4103, 2203, 948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 8089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 7679, 0, 0, 0, 0, 2268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 4, 0, 1, 0, 0, 0, 0, 0, 0, 0, 6759, 1777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7337, 8241, 7240, 8382, 7645, 8146, 9951, 8594] avg=8192.0000 max=9951 max_rank=6 min=7240 min_rank=2 imbalance_ratio=1.214722 max_load_reduction=55585 max_load_reduction_pct=0.848160 moved_to_echo=34336 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[5792, 0, 0, 0, 0, 0, 0, 7585, 0, 0, 100, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 100, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 9, 8137, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 68, 234, 0, 0, 0, 0, 0, 0, 7061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 864, 7553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1177, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 170, 4118, 2131, 925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1518, 13, 8099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 625, 0, 0, 0, 11, 2277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 292, 21, 0, 0, 2, 0, 5, 0, 0, 0, 0, 2, 0, 4654, 1784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13498, 8363, 7364, 8417, 8536, 9685, 2913, 6760] avg=8192.0000 max=13498 max_rank=0 min=2913 min_rank=6 imbalance_ratio=1.647705 max_load_reduction=52038 max_load_reduction_pct=0.794037 moved_to_echo=27894 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[5864, 0, 0, 167, 0, 0, 0, 0, 0, 3072, 0, 0, 3, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 7506, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4, 0, 0, 0, 46, 0, 36, 13, 0, 0, 7216, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 9, 1230, 8174, 0, 0, 0, 0, 0, 0, 54, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 118, 0, 241, 0, 0, 0, 0, 1, 0, 0, 4, 3990, 2165, 952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5126, 0, 0, 86, 0, 2197, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 1, 0, 209, 0, 6660, 1731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9195, 7696, 7333, 9489, 7989, 7791, 7421, 8622] avg=8192.0000 max=9489 max_rank=3 min=7333 min_rank=2 imbalance_ratio=1.158325 max_load_reduction=56047 max_load_reduction_pct=0.855209 moved_to_echo=31823 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[5978, 0, 0, 0, 0, 0, 0, 4788, 24, 58, 4, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 6, 0, 0, 0, 0, 0, 7, 0, 495, 8134, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 7217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1257, 7632, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 0, 79, 1140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4114, 2199, 946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 1, 8058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1366, 15, 0, 0, 0, 2274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 0, 0, 0, 0, 0, 0, 0, 3, 0, 606, 1, 100, 0, 0, 6769, 1782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10854, 8794, 7222, 8893, 8625, 8133, 3655, 9360] avg=8192.0000 max=10854 max_rank=0 min=3655 min_rank=6 imbalance_ratio=1.324951 max_load_reduction=54682 max_load_reduction_pct=0.834381 moved_to_echo=29773 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[5966, 0, 0, 0, 11, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8094, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 7064, 0, 4, 0, 0, 347, 0, 2, 7218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 717, 3412, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 1548, 0, 0, 0, 2, 1, 5, 0, 0, 0, 4070, 2195, 946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 7, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 5, 7190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 90, 0, 2250, 4, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 15, 0, 0, 0, 0, 7, 5, 3, 0, 0, 2, 3862, 1763, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5980, 8471, 14639, 4178, 8807, 7294, 10504, 5663] avg=8192.0000 max=14639 max_rank=2 min=4178 min_rank=3 imbalance_ratio=1.786987 max_load_reduction=50897 max_load_reduction_pct=0.776627 moved_to_echo=32268 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[5973, 0, 0, 0, 2, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4, 0, 0, 0, 0, 0, 63, 0, 2, 0, 0, 12, 0, 8099, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 7189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 1236, 8171, 1, 5, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 29, 10, 0, 11, 0, 0, 0, 0, 0, 0, 59, 4072, 2198, 942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 21, 0, 0, 1, 1, 0, 0, 0, 4, 0, 0, 88, 0, 8111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 8016, 0, 0, 0, 0, 2251, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 34, 0, 2, 0, 0, 0, 0, 18, 0, 0, 0, 6656, 1764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6014, 8330, 7196, 9488, 7325, 8281, 10300, 8602] avg=8192.0000 max=10300 max_rank=6 min=6014 min_rank=0 imbalance_ratio=1.257324 max_load_reduction=55236 max_load_reduction_pct=0.842834 moved_to_echo=34508 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 128, 128, 1408, 640, 640, 4736, 384, 128, 0, 128, 128, 256, 0, 0, 256, 32640, 17536, 7552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 128, 128, 1408, 640, 640, 4736, 256, 128, 0, 128, 128, 256, 0, 0, 256, 32640, 17536, 7680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 47360, 128, 0, 256, 128, 18048, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 256, 0, 384, 128, 128, 128, 128, 128, 128, 128, 1536, 128, 128, 256, 128, 49024, 14208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 256, 0, 384, 128, 128, 128, 128, 128, 128, 128, 1536, 128, 128, 256, 128, 48896, 14208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[47616, 0, 0, 256, 128, 0, 128, 12416, 128, 4096, 128, 0, 128, 128, 1280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 8960, 56448, 128, 128, 0, 0, 128, 128, 128, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 128, 128, 7168, 256, 128, 0, 128, 384, 128, 128, 57728, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[512, 0, 128, 0, 0, 128, 128, 0, 0, 0, 128, 128, 0, 1792, 128, 63360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 9088, 56320, 128, 128, 0, 0, 128, 128, 128, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 128, 128, 7296, 256, 128, 0, 128, 384, 128, 128, 57728, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[512, 0, 128, 0, 0, 128, 128, 0, 0, 0, 128, 128, 0, 1664, 128, 63488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[128, 128, 384, 128, 128, 128, 0, 128, 0, 128, 128, 128, 128, 128, 128, 640, 63488, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 917, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 6, 0, 7869, 0, 5, 0, 0, 0, 0, 6571, 0, 0, 8090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 389, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 294, 0, 0, 0, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 96, 82, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 8103, 0, 8192, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 1676, 0, 0, 0, 9, 0, 0, 0, 0, 11, 0, 16, 0, 0, 0, 0, 0, 8126, 0, 0, 0, 0, 0, 9, 1572, 0, 126, 7673, 0, 0, 2, 0, 0, 0, 0, 0, 10, 30, 0, 0, 0, 0, 0, 0, 0, 77, 0, 15, 2, 0, 43, 0, 0, 1, 0, 0, 109, 0, 0, 189, 0, 538, 1, 0, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8175, 3, 0, 4821, 0, 2, 7, 489, 0, 0, 0, 0, 0, 1628, 0, 0, 0, 0, 0, 0, 0, 8107, 0, 8185, 0, 0, 5574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 3, 0, 0, 0, 0, 8040, 10, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 683, 0, 0, 0, 0, 164, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 3, 24, 0, 0, 8043, 0, 13, 0, 0, 0, 0, 62, 0, 0, 7889, 0, 0, 12, 0, 0, 0, 1, 28, 0, 0, 10, 0, 288, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 347, 0, 2, 0, 0, 0, 1, 0, 2892, 0, 0, 0, 0, 0, 2, 1, 0, 125, 0, 0, 0, 7, 6, 70, 0, 0, 8060, 0, 0, 127, 0, 0, 9, 7632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4739, 48, 8051, 0, 0, 8109, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8070, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 47, 0, 4, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 7044, 1, 0, 3, 0, 0, 0, 4805, 0, 0, 6638, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 3, 0, 5, 0, 0, 0, 0, 0, 8141, 0, 0, 0, 0, 0, 0, 11, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 163, 0, 0, 0, 0, 0, 377, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 8185, 7, 0, 5176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 1, 0, 16, 0, 5, 0, 0, 0, 0, 0, 3, 0, 0, 31, 0, 0, 0, 0, 0, 127, 0, 301, 0, 0, 0, 0, 0, 9, 187, 0, 0, 7099, 0, 0, 26, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 88, 0, 0, 10, 0, 0, 0, 0, 0, 6, 0, 4762, 0, 12, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2282, 0, 0, 0, 0, 8010, 0, 0, 2253, 0, 10, 1, 7446, 1, 0, 0, 0, 0, 213, 317, 0, 0, 0, 0, 0, 0, 8192, 0, 8153, 1, 0, 7981, 0, 2, 0, 0, 0, 0, 9, 0, 0, 0, 6, 0, 109, 0, 0, 0, 7779, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 3751, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 2211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 1173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4776, 0, 17, 0, 0, 388, 0, 0, 0, 0, 0, 581, 0, 0, 4, 0, 115, 0, 95, 0, 0, 8192, 0, 0, 362, 0, 0, 83, 3033, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7974, 0, 8190, 0, 16, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 20, 0, 0, 0, 64, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3341, 0, 0, 3, 0, 0, 0, 1, 10, 0, 5280, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1, 0, 193, 0, 99, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 14, 0, 2, 0, 0, 0, 7573, 0, 0, 7, 1, 0, 0, 1, 0, 130, 8103, 19, 0, 1, 0, 0, 110, 7423, 0, 56, 0, 0, 8, 0, 2, 0, 0, 7, 0, 0, 0, 8124, 0, 8174, 3, 0, 17, 25, 0, 0, 16, 0, 0, 0, 0, 0, 9, 327, 0, 0, 0, 0, 0, 8122, 29, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 1072, 0, 0, 194, 0, 0, 0, 6861, 2, 2, 7506, 0, 0, 0, 51, 2, 0, 0, 0, 0, 0, 0, 9, 3347, 0, 0, 0, 0, 0, 0, 22, 653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 327, 0, 0, 15, 0, 7997, 0, 0, 0, 0, 0, 5051, 25, 0, 0, 24, 35, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8051, 0, 8106, 0, 0, 7674, 0, 0, 0, 0, 0, 0, 0, 0, 225, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 917, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 7289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 7869, 0, 5, 0, 0, 0, 0, 6571, 0, 0, 862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 162, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 389, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 5739, 903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 294, 0, 0, 0, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 4172, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2996, 0, 0, 0, 0, 0, 96, 82, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1878, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 15313, 8272, 7051, 5058, 3235, 10051, 8344] avg=8192.0000 max=15313 max_rank=1 min=3235 min_rank=5 imbalance_ratio=1.869263 max_load_reduction=50223 max_load_reduction_pct=0.766342 moved_to_echo=26688 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 683, 0, 0, 0, 0, 164, 0, 0, 0, 0, 4, 0, 0, 0, 0, 7164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 24, 0, 0, 8043, 0, 13, 0, 0, 0, 0, 62, 0, 0, 848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7889, 0, 0, 12, 0, 0, 0, 1, 28, 0, 0, 10, 0, 288, 0, 0, 159, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 347, 0, 2, 0, 0, 0, 1, 0, 3356, 887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2892, 0, 0, 0, 0, 0, 2, 1, 0, 125, 0, 0, 0, 7, 6, 70, 4105, 284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2948, 0, 0, 127, 0, 0, 9, 7632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1099, 48, 0, 0, 0, 8109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8015, 8993, 8407, 4599, 7492, 10716, 9263, 8051] avg=8192.0000 max=10716 max_rank=5 min=4599 min_rank=3 imbalance_ratio=1.308105 max_load_reduction=54820 max_load_reduction_pct=0.836487 moved_to_echo=35522 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 1676, 0, 0, 0, 9, 0, 0, 0, 0, 11, 0, 7283, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 8126, 0, 0, 0, 0, 0, 9, 1572, 0, 126, 860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7673, 0, 0, 2, 0, 0, 0, 0, 0, 10, 30, 0, 0, 0, 0, 0, 162, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 15, 2, 0, 43, 0, 0, 1, 0, 0, 109, 0, 0, 5742, 902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 0, 538, 1, 0, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4163, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2990, 3, 0, 4821, 0, 2, 7, 489, 0, 0, 0, 0, 0, 1628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1879, 0, 0, 0, 0, 5574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 3, 0, 0, 0, 0, 8020, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8980, 10709, 7897, 6891, 5606, 9944, 7453, 8056] avg=8192.0000 max=10709 max_rank=1 min=5606 min_rank=4 imbalance_ratio=1.307251 max_load_reduction=54827 max_load_reduction_pct=0.836594 moved_to_echo=31059 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 16, 0, 5, 0, 0, 0, 0, 0, 3, 0, 0, 31, 0, 7254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 0, 301, 0, 0, 0, 0, 0, 9, 187, 0, 0, 843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7099, 0, 0, 26, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 158, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 88, 0, 0, 10, 0, 0, 0, 0, 0, 6, 0, 5802, 899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4762, 0, 12, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2282, 0, 0, 4079, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2930, 0, 0, 2253, 0, 10, 1, 7446, 1, 0, 0, 0, 0, 213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 317, 0, 0, 0, 0, 0, 0, 1899, 0, 0, 1, 0, 7981, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 6, 0, 109, 0, 0, 0, 7760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7310, 1467, 7388, 6806, 11627, 12854, 10200, 7884] avg=8192.0000 max=12854 max_rank=5 min=1467 min_rank=1 imbalance_ratio=1.569092 max_load_reduction=52682 max_load_reduction_pct=0.803864 moved_to_echo=42565 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 47, 0, 4, 0, 0, 0, 0, 0, 0, 2, 7284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7044, 1, 0, 3, 0, 0, 0, 4805, 0, 0, 862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6638, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 354, 162, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 5782, 901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 5, 0, 0, 0, 0, 0, 8141, 0, 0, 0, 0, 0, 0, 11, 4172, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2996, 0, 0, 0, 0, 0, 0, 163, 0, 0, 0, 0, 0, 377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1893, 0, 0, 7, 0, 5176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7337, 12715, 7178, 6708, 12821, 3537, 7076, 8164] avg=8192.0000 max=12821 max_rank=4 min=3537 min_rank=5 imbalance_ratio=1.565063 max_load_reduction=52715 max_load_reduction_pct=0.804367 moved_to_echo=31598 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 3751, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 7287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 1173, 0, 0, 162, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4776, 0, 17, 5648, 903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 388, 0, 0, 0, 0, 0, 581, 0, 0, 4, 0, 115, 0, 95, 4172, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2996, 0, 0, 362, 0, 0, 83, 3033, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1848, 0, 0, 0, 16, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11087, 3073, 9534, 11344, 5833, 6476, 10021, 8168] avg=8192.0000 max=11344 max_rank=3 min=3073 min_rank=1 imbalance_ratio=1.384766 max_load_reduction=54192 max_load_reduction_pct=0.826904 moved_to_echo=30498 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 20, 0, 0, 0, 64, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 7274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3341, 0, 0, 3, 0, 0, 0, 1, 10, 0, 853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5280, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1, 0, 193, 0, 99, 160, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 5754, 900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 2, 0, 0, 0, 7573, 0, 0, 7, 1, 0, 0, 1, 4126, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 2964, 19, 0, 1, 0, 0, 110, 7423, 0, 56, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 7, 0, 0, 0, 1883, 0, 0, 3, 0, 17, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 9, 327, 0, 0, 0, 0, 0, 8102, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7367, 4208, 5758, 14861, 12211, 10711, 1937, 8483] avg=8192.0000 max=14861 max_rank=3 min=1937 min_rank=6 imbalance_ratio=1.814087 max_load_reduction=50675 max_load_reduction_pct=0.773239 moved_to_echo=33342 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 7213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 1072, 0, 0, 194, 0, 0, 0, 6861, 2, 2, 846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7506, 0, 0, 0, 51, 2, 0, 0, 0, 0, 0, 0, 9, 3347, 0, 0, 162, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5708, 893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 327, 0, 0, 4076, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 2913, 0, 0, 0, 0, 0, 5051, 25, 0, 0, 24, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1858, 0, 0, 0, 0, 7674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 225, 0, 0, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7228, 9058, 11102, 7276, 4888, 8063, 9535, 8386] avg=8192.0000 max=11102 max_rank=2 min=4888 min_rank=4 imbalance_ratio=1.355225 max_load_reduction=54434 max_load_reduction_pct=0.830597 moved_to_echo=30872 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4, 0, 7866, 0, 6, 0, 0, 0, 0, 6565, 0, 0, 8076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 380, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 281, 0, 0, 0, 0, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 81, 89, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 8111, 0, 8192, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 3874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 2256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4890, 0, 20, 0, 0, 392, 1, 0, 0, 0, 0, 611, 0, 0, 4, 0, 180, 0, 92, 0, 0, 8192, 0, 0, 372, 0, 0, 86, 2782, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7968, 0, 8190, 0, 19, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 1042, 0, 0, 187, 0, 0, 0, 6811, 2, 3, 7552, 0, 0, 0, 50, 2, 0, 0, 0, 0, 0, 0, 10, 3329, 0, 0, 0, 0, 0, 0, 27, 568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 272, 0, 0, 21, 0, 7998, 0, 0, 0, 0, 0, 5254, 27, 0, 0, 23, 28, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8049, 0, 8104, 0, 0, 7672, 0, 0, 0, 0, 0, 0, 0, 0, 221, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 49, 0, 5, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 6995, 2, 0, 2, 0, 0, 0, 5032, 0, 0, 6648, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 2, 0, 3, 0, 0, 0, 0, 0, 8139, 0, 0, 0, 0, 0, 0, 11, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 157, 0, 0, 0, 0, 0, 388, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 8185, 12, 0, 4981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 1744, 0, 0, 0, 9, 0, 0, 0, 0, 8, 0, 20, 0, 0, 0, 0, 0, 8111, 0, 0, 0, 0, 0, 11, 1618, 0, 137, 7657, 0, 0, 2, 0, 0, 0, 0, 0, 9, 30, 0, 0, 0, 0, 0, 0, 0, 78, 0, 12, 5, 0, 43, 0, 0, 1, 0, 0, 105, 0, 0, 177, 0, 544, 1, 0, 223, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 8175, 3, 0, 4755, 0, 2, 3, 541, 0, 0, 0, 0, 0, 1600, 0, 0, 0, 0, 0, 0, 0, 8120, 0, 8185, 0, 0, 5531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 3, 0, 0, 0, 0, 8033, 10, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 2, 0, 15, 0, 8, 0, 0, 0, 0, 0, 3, 0, 0, 30, 0, 0, 0, 0, 0, 114, 0, 341, 0, 0, 0, 0, 0, 3, 188, 0, 0, 7049, 0, 0, 25, 0, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 60, 0, 0, 9, 0, 0, 0, 0, 0, 6, 0, 4780, 0, 13, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2316, 0, 0, 0, 0, 8012, 0, 0, 2261, 0, 10, 1, 7402, 1, 0, 0, 0, 0, 208, 335, 0, 0, 0, 0, 0, 0, 8192, 0, 8154, 1, 0, 7988, 0, 2, 0, 0, 0, 0, 9, 0, 0, 0, 5, 0, 105, 0, 0, 0, 7795, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 22, 0, 0, 0, 59, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3490, 0, 0, 3, 0, 0, 0, 0, 10, 0, 5087, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1, 0, 169, 0, 102, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 8, 0, 2, 0, 0, 0, 7571, 0, 0, 8, 1, 0, 0, 1, 0, 127, 8105, 22, 0, 0, 0, 0, 115, 7461, 0, 60, 0, 0, 7, 0, 2, 0, 0, 5, 0, 0, 0, 8141, 0, 8176, 2, 0, 19, 21, 0, 0, 19, 0, 0, 0, 0, 0, 9, 351, 0, 0, 0, 0, 0, 8121, 30, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 660, 0, 0, 0, 0, 161, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 3, 25, 0, 0, 8057, 0, 17, 0, 0, 0, 0, 59, 0, 0, 7884, 0, 0, 12, 0, 0, 0, 3, 23, 0, 0, 8, 0, 266, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 353, 0, 1, 0, 0, 0, 1, 0, 2950, 0, 0, 0, 1, 0, 2, 1, 0, 125, 0, 0, 0, 7, 4, 66, 0, 0, 8057, 0, 0, 110, 0, 0, 17, 7590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4773, 50, 8053, 0, 0, 8115, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8061, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 7257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 7866, 0, 6, 0, 0, 0, 0, 6565, 0, 0, 817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 380, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 5706, 935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 281, 0, 0, 0, 0, 0, 109, 0, 0, 0, 0, 0, 0, 0, 4143, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3016, 0, 0, 0, 0, 0, 81, 89, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1905, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8267, 15258, 8313, 7041, 5033, 3216, 10080, 8328] avg=8192.0000 max=15258 max_rank=1 min=3216 min_rank=5 imbalance_ratio=1.862549 max_load_reduction=50278 max_load_reduction_pct=0.767181 moved_to_echo=26657 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 3874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 7255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 1033, 0, 0, 216, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4890, 0, 20, 5605, 935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 392, 1, 0, 0, 0, 0, 611, 0, 0, 4, 0, 180, 0, 92, 4143, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3016, 0, 0, 372, 0, 0, 86, 2782, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1871, 0, 0, 0, 19, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11177, 3073, 9448, 11450, 5915, 6258, 10048, 8167] avg=8192.0000 max=11450 max_rank=3 min=3073 min_rank=1 imbalance_ratio=1.397705 max_load_reduction=54086 max_load_reduction_pct=0.825287 moved_to_echo=30388 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 49, 0, 5, 0, 0, 0, 0, 0, 0, 2, 7252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6995, 2, 0, 2, 0, 0, 0, 5032, 0, 0, 817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6648, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 350, 216, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 5745, 933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 0, 0, 0, 0, 0, 8139, 0, 0, 0, 0, 0, 0, 11, 4143, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3016, 0, 0, 0, 0, 0, 0, 157, 0, 0, 0, 0, 0, 388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1917, 0, 0, 12, 0, 4981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7308, 12848, 7239, 6706, 12802, 3562, 6910, 8161] avg=8192.0000 max=12848 max_rank=1 min=3562 min_rank=5 imbalance_ratio=1.568359 max_load_reduction=52688 max_load_reduction_pct=0.803955 moved_to_echo=31435 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 1744, 0, 0, 0, 9, 0, 0, 0, 0, 8, 0, 7251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 8111, 0, 0, 0, 0, 0, 11, 1618, 0, 137, 815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7657, 0, 0, 2, 0, 0, 0, 0, 0, 9, 30, 0, 0, 0, 0, 0, 215, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 12, 5, 0, 43, 0, 0, 1, 0, 0, 105, 0, 0, 5712, 934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 177, 0, 544, 1, 0, 223, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4134, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3011, 3, 0, 4755, 0, 2, 3, 541, 0, 0, 0, 0, 0, 1600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1907, 0, 0, 0, 0, 5531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 3, 0, 0, 0, 0, 8013, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9013, 10712, 7933, 6890, 5581, 9918, 7438, 8051] avg=8192.0000 max=10712 max_rank=1 min=5581 min_rank=4 imbalance_ratio=1.307617 max_load_reduction=54824 max_load_reduction_pct=0.836548 moved_to_echo=30988 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 14, 0, 0, 0, 0, 0, 7180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 1042, 0, 0, 187, 0, 0, 0, 6811, 2, 3, 800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7552, 0, 0, 0, 50, 2, 0, 0, 0, 0, 0, 0, 10, 3329, 0, 0, 215, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5666, 924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 272, 0, 0, 4048, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 2935, 0, 0, 0, 0, 0, 5254, 27, 0, 0, 23, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1882, 0, 0, 0, 0, 7672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 221, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7195, 8925, 11182, 7185, 4821, 8288, 9557, 8383] avg=8192.0000 max=11182 max_rank=2 min=4821 min_rank=4 imbalance_ratio=1.364990 max_load_reduction=54354 max_load_reduction_pct=0.829376 moved_to_echo=31049 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 15, 0, 8, 0, 0, 0, 0, 0, 3, 0, 0, 30, 0, 7224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 0, 341, 0, 0, 0, 0, 0, 3, 188, 0, 0, 799, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7049, 0, 0, 25, 0, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 211, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 60, 0, 0, 9, 0, 0, 0, 0, 0, 6, 0, 5763, 930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4780, 0, 13, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2316, 0, 0, 4052, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2950, 0, 0, 2261, 0, 10, 1, 7402, 1, 0, 0, 0, 0, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 335, 0, 0, 0, 0, 0, 0, 1924, 0, 0, 1, 0, 7988, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 5, 0, 105, 0, 0, 0, 7775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7282, 1445, 7395, 6770, 11667, 12833, 10250, 7894] avg=8192.0000 max=12833 max_rank=5 min=1445 min_rank=1 imbalance_ratio=1.566528 max_load_reduction=52703 max_load_reduction_pct=0.804184 moved_to_echo=42644 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 22, 0, 0, 0, 59, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 7244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3490, 0, 0, 3, 0, 0, 0, 0, 10, 0, 808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5087, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1, 0, 169, 0, 102, 213, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 5727, 932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 2, 0, 0, 0, 7571, 0, 0, 8, 1, 0, 0, 1, 4099, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 2985, 22, 0, 0, 0, 0, 115, 7461, 0, 60, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 5, 0, 0, 0, 1912, 0, 0, 2, 0, 19, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 9, 351, 0, 0, 0, 0, 0, 8100, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7334, 4311, 5598, 14854, 12192, 10777, 1961, 8509] avg=8192.0000 max=14854 max_rank=3 min=1961 min_rank=6 imbalance_ratio=1.813232 max_load_reduction=50682 max_load_reduction_pct=0.773346 moved_to_echo=33439 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 660, 0, 0, 0, 0, 161, 0, 0, 0, 0, 4, 0, 0, 0, 0, 7135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 25, 0, 0, 8057, 0, 17, 0, 0, 0, 0, 59, 0, 0, 803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7884, 0, 0, 12, 0, 0, 0, 3, 23, 0, 0, 8, 0, 266, 0, 0, 212, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 353, 0, 1, 0, 0, 0, 1, 0, 3358, 918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2950, 0, 0, 0, 1, 0, 2, 1, 0, 125, 0, 0, 0, 7, 4, 66, 4075, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2967, 0, 0, 110, 0, 0, 17, 7590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1121, 50, 0, 0, 0, 8115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7960, 8964, 8428, 4640, 7525, 10684, 9292, 8043] avg=8192.0000 max=10684 max_rank=5 min=4640 min_rank=3 imbalance_ratio=1.304199 max_load_reduction=54852 max_load_reduction_pct=0.836975 moved_to_echo=35544 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 256, 23808, 128, 0, 7680, 0, 128, 5376, 26368, 128, 128, 128, 128, 128, 2304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 128, 256, 128, 38016, 128, 128, 256, 0, 0, 128, 20096, 128, 128, 6912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[7936, 0, 1280, 128, 128, 256, 128, 128, 16512, 128, 0, 128, 128, 2816, 128, 256, 33152, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[384, 0, 128, 128, 0, 0, 128, 14336, 128, 0, 128, 128, 50944, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 128, 128, 128, 9344, 0, 128, 384, 0, 128, 0, 0, 4992, 128, 128, 43648, 7296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 4480, 0, 128, 0, 2688, 256, 128, 128, 128, 128, 128, 0, 0, 128, 128, 58112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[58368, 0, 0, 128, 128, 128, 0, 128, 128, 128, 128, 128, 128, 5120, 0, 512, 1408, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 23936, 128, 0, 7552, 0, 128, 5632, 26112, 128, 128, 128, 128, 128, 2304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 128, 128, 128, 38272, 128, 128, 256, 0, 0, 128, 20352, 128, 256, 6528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[7936, 0, 1280, 128, 128, 256, 128, 128, 16512, 128, 0, 128, 128, 2816, 128, 256, 32896, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[384, 0, 128, 128, 0, 0, 128, 14464, 128, 0, 128, 128, 50688, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 128, 128, 128, 9216, 0, 128, 384, 0, 128, 0, 0, 5120, 128, 128, 43392, 7552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 4608, 0, 128, 0, 2944, 256, 128, 128, 128, 128, 128, 0, 0, 128, 128, 57856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[58112, 0, 0, 128, 128, 128, 0, 128, 128, 128, 128, 128, 128, 4864, 0, 512, 1792, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 128, 0, 256, 128, 640, 128, 128, 0, 0, 0, 64640, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[2, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 1, 6, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 45, 0, 0, 32, 6578, 0, 2, 0, 0, 0, 5843, 0, 0, 0, 0, 404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8111, 1077, 0, 0, 0, 0, 0, 0, 0, 36, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 84, 0, 0, 302, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 17, 0, 5, 14, 0, 0, 0, 8175, 6, 17, 14, 0, 26, 0, 0, 0, 0, 0, 0, 0, 3153, 7046, 0, 0, 0, 16] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 12, 0, 8167, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8099, 0, 0, 0, 4, 0, 5391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 5, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 8189, 0, 0, 0, 0, 1224, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 8192, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 2, 148, 0, 0, 0, 0, 0, 0, 0, 6121, 3532, 0, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 7773, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 469, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8058, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8192, 0, 0, 0, 0, 17, 1, 0, 0, 0, 0, 0, 0, 35, 8170, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 2, 6, 0, 0, 0, 0, 7718, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 32, 0, 0, 0, 60, 0, 0, 0, 0, 8015, 348, 0, 0, 1, 0, 7406, 0, 0, 81, 0, 0, 0, 0, 8, 0, 0, 0, 0, 181, 1, 0, 0, 0, 0, 0, 8082, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 580, 34, 0, 0, 0, 0, 272, 1, 0, 0, 7682, 0, 0, 2, 0, 2388, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 6761, 0, 0, 0, 13, 0, 0, 89, 0, 1, 0, 0, 8144, 0, 0, 0, 3, 1461, 0, 0, 0, 0, 72, 0, 0, 10, 5886, 0, 0, 0, 129] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 7772, 0, 0, 0, 0, 0, 7772, 0, 0, 0, 0, 1811, 168, 0, 7, 0, 0, 0, 97, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 110, 3, 0, 0, 0, 0, 291, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 18, 8003, 0, 400, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7735, 0, 0, 0, 0, 0, 0, 9, 16, 1, 0, 7, 8192, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 6734, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 18, 0, 0, 0, 7, 3, 0, 0, 0, 25, 0, 8186, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 1, 4, 0, 0, 0, 7607, 0, 0, 35, 0, 0, 6616, 0, 10, 0, 0, 502, 0, 1515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8017, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 7449, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8174, 0, 0, 3, 0, 541, 6, 0, 0, 13, 0, 0, 8171, 0, 0, 0, 7, 119, 0, 0, 0, 0, 0, 0, 0, 0, 268, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 1, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7491, 0, 0, 0, 0, 0, 7220, 0, 0, 0, 0, 0, 0, 770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8157, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5389, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 173, 0, 0, 0, 0, 0, 0, 0, 279, 3248, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 123, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 91, 8042, 117, 1, 0, 41, 0, 6068, 0, 2356, 30, 1, 2, 0, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 63, 0, 0, 0, 2, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5877, 1, 0, 5, 2, 25, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 8066, 0, 0, 0, 11, 6, 0, 0, 0, 4, 0, 0, 8166, 0, 4, 12, 0, 272, 0, 0, 0, 13, 0, 0, 0, 8043, 1559, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 18, 0, 0, 0, 7, 3, 0, 0, 0, 25, 0, 8186, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 1, 4, 0, 0, 0, 7649, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 334, 0, 0, 35, 0, 0, 6616, 0, 10, 0, 0, 502, 0, 1515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 7449, 219, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 3, 0, 541, 6, 0, 0, 13, 0, 0, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 119, 0, 0, 0, 0, 0, 0, 0, 0, 268, 1, 0, 0, 0, 1110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8296, 8182, 9012, 7876, 7761, 13645, 9259, 1505] avg=8192.0000 max=13645 max_rank=5 min=1505 min_rank=7 imbalance_ratio=1.665649 max_load_reduction=51891 max_load_reduction_pct=0.791794 moved_to_echo=32170 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 7669, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 341, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8048, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 224, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8058, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 1, 0, 0, 0, 0, 0, 0, 35, 8170, 0, 0, 0, 0, 1134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8215, 8547, 8050, 8501, 6094, 8582, 9357] avg=8192.0000 max=9357 max_rank=7 min=6094 min_rank=5 imbalance_ratio=1.142212 max_load_reduction=56179 max_load_reduction_pct=0.857224 moved_to_echo=32534 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8153, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 7669, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 341, 0, 0, 0, 0, 0, 7772, 0, 0, 0, 0, 1811, 168, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 0, 0, 0, 0, 0, 8023, 0, 0, 110, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 291, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 18, 8003, 224, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 400, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7735, 0, 0, 0, 0, 0, 0, 9, 16, 1, 0, 7, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 6734, 0, 0, 0, 0, 0, 1134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8192, 10099, 8233, 8639, 5997, 8291, 7872] avg=8192.0000 max=10099 max_rank=2 min=5997 min_rank=5 imbalance_ratio=1.232788 max_load_reduction=55437 max_load_reduction_pct=0.845901 moved_to_echo=30799 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 12, 0, 8167, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7668, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 354, 0, 0, 0, 4, 0, 5391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8044, 0, 5, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 8189, 234, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1224, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 5828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8192, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 148, 0, 0, 0, 0, 0, 0, 0, 6121, 3532, 0, 0, 0, 2, 1182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8239, 8170, 5749, 8083, 8511, 7055, 8742, 10987] avg=8192.0000 max=10987 max_rank=7 min=5749 min_rank=2 imbalance_ratio=1.341187 max_load_reduction=54549 max_load_reduction_pct=0.832352 moved_to_echo=35295 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 1, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7666, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 328, 0, 0, 0, 0, 0, 7220, 0, 0, 0, 0, 0, 0, 770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 216, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5389, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 173, 0, 0, 0, 0, 0, 0, 0, 279, 3248, 0, 0, 0, 0, 1093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 8133, 8318, 8045, 8494, 13593, 5915, 4793] avg=8192.0000 max=13593 max_rank=5 min=4793 min_rank=7 imbalance_ratio=1.659302 max_load_reduction=51943 max_load_reduction_pct=0.792587 moved_to_echo=32795 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 2, 6, 0, 0, 0, 0, 7718, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 32, 0, 0, 0, 60, 0, 0, 0, 0, 7624, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 351, 348, 0, 0, 1, 0, 7406, 0, 0, 81, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 1, 0, 0, 0, 0, 0, 7940, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 580, 34, 0, 0, 0, 0, 272, 1, 0, 0, 7682, 231, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2388, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 5768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6761, 0, 0, 0, 13, 0, 0, 89, 0, 1, 0, 0, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1461, 0, 0, 0, 0, 72, 0, 0, 10, 5886, 0, 0, 0, 129, 1170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7783, 8241, 8195, 8128, 8885, 8189, 7384, 8731] avg=8192.0000 max=8885 max_rank=4 min=7384 min_rank=6 imbalance_ratio=1.084595 max_load_reduction=56651 max_load_reduction_pct=0.864426 moved_to_echo=33189 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 1, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 45, 0, 0, 32, 7653, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 288, 0, 2, 0, 0, 0, 5843, 0, 0, 0, 0, 404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7968, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8166, 190, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 302, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 17, 0, 5, 14, 0, 0, 0, 522, 6, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 26, 0, 0, 0, 0, 0, 0, 0, 3153, 7046, 0, 0, 0, 16, 960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8223, 8161, 6537, 9045, 8479, 5124, 8752, 11215] avg=8192.0000 max=11215 max_rank=7 min=5124 min_rank=5 imbalance_ratio=1.369019 max_load_reduction=54321 max_load_reduction_pct=0.828873 moved_to_echo=33570 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 123, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 91, 7646, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 337, 117, 1, 0, 41, 0, 6068, 0, 2356, 30, 1, 2, 0, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8013, 0, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5877, 236, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 2, 25, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 5790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8066, 0, 0, 0, 11, 6, 0, 0, 0, 4, 0, 0, 520, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 272, 0, 0, 0, 13, 0, 0, 0, 8043, 1559, 0, 0, 0, 0, 1177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8347, 8242, 9079, 8076, 6266, 5839, 8611, 11076] avg=8192.0000 max=11076 max_rank=7 min=5839 min_rank=5 imbalance_ratio=1.352051 max_load_reduction=54460 max_load_reduction_pct=0.830994 moved_to_echo=31792 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 8164, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8097, 0, 0, 0, 4, 0, 5451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 4, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 8188, 0, 0, 0, 0, 1317, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 8191, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 2, 148, 0, 0, 0, 0, 0, 0, 0, 6109, 3399, 0, 0, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 1, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7502, 0, 0, 0, 0, 0, 7185, 0, 0, 0, 0, 0, 0, 802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8167, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5461, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 165, 0, 0, 0, 0, 0, 0, 0, 286, 3158, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 2, 15, 0, 0, 0, 0, 7715, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 18, 0, 0, 0, 61, 0, 0, 0, 0, 8011, 298, 0, 0, 0, 0, 7385, 0, 0, 89, 0, 0, 0, 0, 8, 0, 0, 0, 2, 204, 0, 0, 0, 0, 0, 0, 8079, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 582, 25, 0, 0, 0, 0, 297, 0, 0, 0, 7687, 0, 0, 2, 0, 2265, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 6777, 0, 0, 0, 13, 0, 0, 85, 0, 2, 0, 0, 8144, 0, 0, 0, 3, 1466, 0, 0, 0, 0, 70, 0, 0, 12, 6015, 0, 0, 0, 129] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 7767, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 478, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8060, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8192, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 28, 8169, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[1, 0, 0, 17, 0, 0, 0, 1, 0, 0, 0, 0, 8147, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 45, 0, 0, 32, 6412, 0, 2, 0, 0, 0, 5839, 0, 0, 0, 0, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8115, 1132, 0, 0, 0, 0, 0, 0, 0, 35, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 89, 0, 0, 296, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 0, 8169, 0, 0, 0, 0, 17, 0, 6, 14, 0, 0, 0, 8176, 6, 18, 14, 0, 19, 0, 0, 0, 0, 0, 0, 0, 3318, 7000, 0, 0, 0, 16] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 100, 8045, 115, 1, 0, 38, 0, 6128, 0, 2372, 25, 1, 2, 0, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 72, 0, 0, 0, 1, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5853, 0, 0, 5, 2, 36, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 8072, 0, 0, 0, 9, 9, 0, 0, 0, 5, 0, 0, 8165, 0, 4, 12, 1, 218, 0, 0, 0, 14, 0, 0, 0, 8055, 1500, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8150, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 7778, 0, 0, 0, 0, 0, 7746, 0, 1, 0, 0, 1752, 163, 0, 6, 0, 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 114, 3, 0, 0, 0, 0, 285, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 14, 8026, 0, 393, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7737, 0, 0, 0, 0, 0, 0, 9, 7, 1, 0, 10, 8192, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 6838, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 18, 0, 0, 0, 7, 5, 0, 0, 0, 25, 0, 8184, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 7624, 0, 0, 34, 0, 0, 6607, 0, 8, 0, 0, 479, 0, 1531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8014, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 7456, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8179, 0, 0, 3, 0, 524, 7, 0, 0, 15, 0, 0, 8170, 0, 0, 0, 5, 95, 0, 0, 0, 0, 0, 0, 0, 0, 316, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 1, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7675, 458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 333, 0, 0, 0, 0, 0, 7185, 0, 0, 0, 0, 0, 0, 802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 205, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5461, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 165, 0, 0, 0, 0, 0, 0, 0, 286, 3158, 0, 0, 0, 0, 1094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 8137, 8320, 8034, 8495, 13627, 5975, 4703] avg=8192.0000 max=13627 max_rank=5 min=4703 min_rank=7 imbalance_ratio=1.663452 max_load_reduction=51909 max_load_reduction_pct=0.792068 moved_to_echo=32800 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 8164, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7677, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 359, 0, 0, 0, 4, 0, 5451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8035, 0, 4, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 8188, 222, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1317, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 5841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 148, 0, 0, 0, 0, 0, 0, 0, 6109, 3399, 0, 0, 0, 3, 1180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8234, 8173, 5814, 8072, 8511, 7161, 8730, 10841] avg=8192.0000 max=10841 max_rank=7 min=5814 min_rank=2 imbalance_ratio=1.323364 max_load_reduction=54695 max_load_reduction_pct=0.834579 moved_to_echo=35243 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 7678, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 344, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8037, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 213, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5603, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8060, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 28, 8169, 0, 0, 0, 0, 1132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8224, 8548, 8039, 8501, 6110, 8576, 9347] avg=8192.0000 max=9347 max_rank=7 min=6110 min_rank=5 imbalance_ratio=1.140991 max_load_reduction=56189 max_load_reduction_pct=0.857376 moved_to_echo=32534 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 17, 0, 0, 0, 1, 0, 0, 0, 0, 8147, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 45, 0, 0, 32, 7663, 392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 284, 0, 2, 0, 0, 0, 5839, 0, 0, 0, 0, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7961, 1132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8165, 175, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 0, 296, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 4626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 17, 0, 6, 14, 0, 0, 0, 513, 6, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 19, 0, 0, 0, 0, 0, 0, 0, 3318, 7000, 0, 0, 0, 16, 935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8224, 8155, 6530, 9093, 8473, 5016, 8743, 11302] avg=8192.0000 max=11302 max_rank=7 min=5016 min_rank=5 imbalance_ratio=1.379639 max_load_reduction=54234 max_load_reduction_pct=0.827545 moved_to_echo=33534 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8150, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 7678, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 345, 0, 0, 0, 0, 0, 7746, 0, 1, 0, 0, 1752, 163, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 8013, 0, 0, 114, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 285, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 14, 8026, 213, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 393, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7737, 0, 0, 0, 0, 0, 0, 9, 7, 1, 0, 10, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 6838, 0, 0, 0, 0, 0, 1134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8180, 10013, 8224, 8646, 6008, 8278, 7975] avg=8192.0000 max=10013 max_rank=2 min=6008 min_rank=5 imbalance_ratio=1.222290 max_load_reduction=55523 max_load_reduction_pct=0.847214 moved_to_echo=30907 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 2, 15, 0, 0, 0, 0, 7715, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 18, 0, 0, 0, 61, 0, 0, 0, 0, 7633, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 356, 298, 0, 0, 0, 0, 7385, 0, 0, 89, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 204, 0, 0, 0, 0, 0, 0, 7926, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 582, 25, 0, 0, 0, 0, 297, 0, 0, 0, 7687, 219, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2265, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 5779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6777, 0, 0, 0, 13, 0, 0, 85, 0, 2, 0, 0, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1466, 0, 0, 0, 0, 70, 0, 0, 12, 6015, 0, 0, 0, 129, 1168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7789, 8237, 8136, 8139, 8906, 8078, 7388, 8863] avg=8192.0000 max=8906 max_rank=4 min=7388 min_rank=6 imbalance_ratio=1.087158 max_load_reduction=56630 max_load_reduction_pct=0.864105 moved_to_echo=33235 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 121, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 100, 7658, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 341, 115, 1, 0, 38, 0, 6128, 0, 2372, 25, 1, 2, 0, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8001, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5853, 224, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 36, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 5808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8072, 0, 0, 0, 9, 9, 0, 0, 0, 5, 0, 0, 507, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 218, 0, 0, 0, 14, 0, 0, 0, 8055, 1500, 0, 0, 0, 0, 1177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8345, 8255, 9178, 8073, 6235, 5867, 8606, 10977] avg=8192.0000 max=10977 max_rank=7 min=5867 min_rank=5 imbalance_ratio=1.339966 max_load_reduction=54559 max_load_reduction_pct=0.832504 moved_to_echo=31685 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 18, 0, 0, 0, 7, 5, 0, 0, 0, 25, 0, 8184, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 7658, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 338, 0, 0, 34, 0, 0, 6607, 0, 8, 0, 0, 479, 0, 1531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 7456, 209, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 3, 0, 524, 7, 0, 0, 15, 0, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 95, 0, 0, 0, 0, 0, 0, 0, 0, 316, 1, 0, 0, 0, 1111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8296, 8175, 8997, 7862, 7769, 13669, 9240, 1528] avg=8192.0000 max=13669 max_rank=5 min=1528 min_rank=7 imbalance_ratio=1.668579 max_load_reduction=51867 max_load_reduction_pct=0.791428 moved_to_echo=32206 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 2304, 128, 0, 0, 128, 128, 0, 0, 24448, 29824, 128, 0, 0, 256, 8960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 512, 128, 128, 20736, 128, 128, 0, 128, 0, 0, 128, 128, 128, 0, 0, 44288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[2688, 512, 128, 128, 128, 0, 54528, 0, 2432, 128, 128, 2816, 256, 2432, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 128, 128, 0, 0, 128, 128, 128, 0, 128, 128, 0, 0, 128, 61312, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 2176, 0, 0, 0, 128, 128, 0, 0, 24704, 29568, 128, 0, 0, 256, 8960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 512, 128, 128, 20736, 128, 128, 0, 128, 0, 0, 128, 128, 128, 128, 0, 44288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[2816, 512, 128, 128, 128, 0, 54528, 0, 2432, 128, 128, 2688, 256, 2560, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 128, 128, 0, 0, 128, 128, 128, 0, 128, 128, 0, 0, 256, 61440, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 60672, 0, 0, 128, 128, 640, 128, 128, 128, 128, 0, 128, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 128, 128, 384, 128, 640, 128, 128, 0, 128, 0, 384, 128, 128, 128, 61824, 1792, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 256, 128, 0, 0, 0, 0, 0, 64000, 1152, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 128, 128, 384, 128, 640, 128, 128, 0, 128, 0, 384, 0, 128, 128, 61824, 1792, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 256, 0, 0, 0, 0, 0, 0, 63872, 1152, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 128, 0, 128, 128, 128, 0, 0, 128, 0, 64896, 128, 0, 128, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 8127, 0, 2, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 17, 0, 0, 77, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 1, 0, 0, 0, 1, 8182, 0, 22, 0, 0, 8141, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 14, 0, 8069, 21, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7676, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 527, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 5, 0, 100, 0, 670, 0, 0, 0, 0, 893, 0, 0, 306, 0, 1, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7851, 0, 0, 0, 7851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 7527, 0, 0, 0, 0, 72, 7952, 8002, 0, 0, 3245, 43, 0, 0, 0, 132, 0, 0, 0, 12, 344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4176, 0, 4, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8038, 0, 0, 0, 0, 11, 0, 0, 28, 0, 0, 8157, 0, 0, 22, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4112, 0, 0, 0, 0, 0, 0, 0, 0, 8126, 0, 0, 0, 5337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 227, 8192, 3573, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 3, 0, 0, 2794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 7882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 670, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8150, 3, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 7, 0, 0, 0, 0, 0, 0, 7181, 0, 3, 0, 3620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 0, 1, 0, 0, 863, 8090, 8, 0, 11, 301, 36, 0, 0, 0, 101, 0, 0, 0, 0, 7345, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 672, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 3322, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 4, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 732, 0, 0, 21, 0, 0, 0, 0, 0, 0, 6022, 0, 577, 0, 0, 0, 0, 0, 22, 7992, 0, 0, 17, 7326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 73, 0, 0, 62, 0, 0, 0, 5883, 8171, 1, 0, 0, 0, 1, 0, 0, 0, 0, 11, 268, 0, 0, 8189, 0, 0, 5, 0, 0, 44, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 2176, 1620, 0, 0, 0, 0, 0, 0, 1, 96, 0, 1, 6, 0, 1, 0, 0, 8120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7965, 27, 11, 0, 81] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 10, 0, 11, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 11, 3, 0, 0, 0, 0, 67, 0, 0, 7341, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 11, 2, 0, 0, 0, 0, 3, 5, 0, 0, 131, 1, 0, 0, 0, 0, 0, 0, 0, 7571, 8109, 1803, 0, 0, 0, 0, 0, 45, 0, 2599, 0, 290, 0, 0, 4482, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 26, 0, 1, 0, 7, 0, 4, 0, 6, 7520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 8169, 943, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 838, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2893, 0, 0, 0, 0, 0, 0, 4, 0, 7010, 0, 0, 0, 0, 0, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 99, 8171, 1323, 0, 0, 0, 0, 0, 0, 0, 3740, 0, 260, 0, 0, 8133, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 6653, 78, 0, 0, 0, 0, 5, 0, 1, 0, 0, 60, 0, 5118, 0, 0, 4, 8155, 0, 0, 0, 40, 0, 72, 0, 0, 1, 2, 8014, 4628, 0, 20, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 1343, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 11, 0, 0, 8128, 2, 0, 1, 0, 5, 0, 0, 0, 8176, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 5, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 7999, 5250, 0, 0, 664, 56, 0, 0, 0, 875, 0, 0, 0, 0, 615, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 143, 0, 146, 0, 7849, 0, 111, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 11, 0, 0, 10, 8150, 0, 1, 0, 0, 0, 0, 0, 11, 0, 0, 7615, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 91, 0, 0, 0, 7521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4112, 0, 0, 0, 0, 0, 0, 0, 0, 3589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 227, 2873, 3573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 3, 0, 0, 2542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 7882, 679, 252, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 670, 0, 0, 0, 5319, 912, 668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7471, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7615, 7701, 8587, 6673, 10726, 9191, 7569, 7474] avg=8192.0000 max=10726 max_rank=4 min=6673 min_rank=3 imbalance_ratio=1.309326 max_load_reduction=54810 max_load_reduction_pct=0.836334 moved_to_echo=34960 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 732, 0, 0, 21, 7458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6022, 0, 577, 0, 0, 0, 0, 0, 22, 3530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 7326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 73, 0, 0, 62, 0, 0, 0, 5883, 2866, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 11, 268, 0, 0, 7451, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 2176, 663, 738, 369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1620, 0, 0, 0, 0, 0, 0, 1, 96, 0, 1, 6, 0, 1, 0, 0, 5305, 897, 662, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7302, 27, 11, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8211, 10151, 10539, 8886, 7736, 4003, 8589, 7421] avg=8192.0000 max=10539 max_rank=2 min=4003 min_rank=5 imbalance_ratio=1.286499 max_load_reduction=54997 max_load_reduction_pct=0.839188 moved_to_echo=27749 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 3590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 17, 0, 0, 77, 0, 0, 0, 0, 0, 2872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 8182, 0, 22, 0, 0, 7407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 14, 0, 8069, 679, 734, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5317, 912, 626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7477, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7055, 11758, 3300, 2971, 15613, 9889, 6881, 8069] avg=8192.0000 max=15613 max_rank=4 min=2971 min_rank=3 imbalance_ratio=1.905884 max_load_reduction=49923 max_load_reduction_pct=0.761765 moved_to_echo=40452 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 5, 0, 100, 0, 670, 0, 0, 0, 0, 893, 0, 0, 306, 0, 7383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 7527, 0, 0, 0, 0, 72, 2789, 8002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3245, 43, 0, 0, 0, 132, 0, 0, 0, 12, 313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4176, 679, 31, 362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5163, 881, 655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 28, 0, 0, 7478, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9357, 3487, 10991, 18462, 3745, 5251, 6704, 7539] avg=8192.0000 max=18462 max_rank=3 min=3487 min_rank=1 imbalance_ratio=2.253662 max_load_reduction=47074 max_load_reduction_pct=0.718292 moved_to_echo=23239 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 0, 0, 0, 7489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 7, 0, 0, 0, 0, 0, 0, 3172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 0, 1, 0, 0, 863, 2837, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 301, 36, 0, 0, 0, 101, 0, 0, 0, 0, 6683, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 672, 0, 8173, 681, 662, 331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 3322, 0, 0, 0, 5253, 806, 666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7489, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8542, 11372, 6495, 3792, 7132, 10614, 10096, 7493] avg=8192.0000 max=11372 max_rank=1 min=3792 min_rank=3 imbalance_ratio=1.388184 max_load_reduction=54164 max_load_reduction_pct=0.826477 moved_to_echo=35335 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 10, 0, 11, 0, 0, 0, 0, 0, 0, 0, 4, 7515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 11, 3, 0, 0, 0, 0, 67, 0, 0, 3243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 11, 2, 0, 0, 0, 0, 3, 2936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 131, 1, 0, 0, 0, 0, 0, 0, 0, 7571, 2844, 1803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 2599, 0, 290, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 26, 0, 1, 0, 7, 0, 4, 0, 6, 7520, 680, 404, 339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 5265, 823, 668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 7489, 943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7541, 3325, 11115, 12355, 7012, 8988, 6757, 8443] avg=8192.0000 max=12355 max_rank=3 min=3325 min_rank=1 imbalance_ratio=1.508179 max_load_reduction=53181 max_load_reduction_pct=0.811478 moved_to_echo=31200 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 1343, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 19, 0, 0, 0, 7486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 8128, 2, 0, 1, 0, 5, 0, 0, 0, 3612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 5, 0, 0, 0, 72, 0, 0, 3270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 2805, 5250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 664, 56, 0, 0, 0, 875, 0, 0, 0, 0, 560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 143, 0, 146, 0, 7849, 634, 55, 377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 11, 0, 0, 10, 5194, 917, 664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 11, 0, 0, 6981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8884, 11759, 11517, 8112, 2155, 9206, 6910, 6993] avg=8192.0000 max=11759 max_rank=1 min=2155 min_rank=4 imbalance_ratio=1.435425 max_load_reduction=53777 max_load_reduction_pct=0.820572 moved_to_echo=25264 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 838, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 7490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2893, 0, 0, 0, 0, 0, 0, 4, 0, 3086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 99, 2861, 1323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3740, 0, 260, 0, 0, 7396, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 6653, 673, 737, 328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0, 5, 0, 1, 0, 0, 60, 0, 5118, 0, 0, 4, 5310, 789, 665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 72, 0, 0, 1, 2, 7341, 4628, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8331, 5983, 2992, 4285, 11417, 8394, 12030, 12104] avg=8192.0000 max=12104 max_rank=7 min=2992 min_rank=2 imbalance_ratio=1.477539 max_load_reduction=53432 max_load_reduction_pct=0.815308 moved_to_echo=43945 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 8126, 0, 2, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 15, 0, 0, 83, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 2, 0, 0, 0, 1, 8180, 0, 29, 0, 0, 8140, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 18, 0, 8075, 18, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7662, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 528, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 6, 0, 0, 0, 0, 0, 0, 7208, 0, 3, 0, 3559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 1, 0, 0, 827, 8089, 8, 0, 9, 278, 41, 0, 0, 0, 128, 0, 0, 0, 0, 7300, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 670, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 3459, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3880, 0, 0, 0, 0, 0, 0, 0, 0, 8135, 0, 0, 0, 5738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 203, 8192, 3603, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 3, 0, 0, 2596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 7879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 683, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8150, 3, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 10, 0, 8, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 19, 3, 0, 0, 0, 0, 67, 0, 0, 7282, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 13, 2, 0, 0, 0, 0, 3, 4, 0, 0, 132, 8, 0, 0, 0, 0, 0, 0, 0, 7657, 8106, 1742, 0, 0, 0, 0, 0, 53, 0, 2656, 0, 270, 0, 0, 4391, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 27, 0, 1, 0, 7, 0, 5, 0, 4, 7580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 8165, 951, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 92, 0, 705, 0, 0, 0, 0, 791, 0, 0, 319, 0, 1, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7846, 0, 0, 0, 7826, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 7577, 0, 0, 0, 0, 57, 7987, 8021, 0, 0, 3212, 40, 0, 0, 0, 114, 0, 0, 0, 20, 362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4278, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8006, 0, 0, 0, 0, 13, 0, 0, 28, 0, 0, 8158, 0, 0, 22, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 1303, 0, 1, 0, 0, 0, 39, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 12, 0, 0, 8127, 2, 0, 1, 0, 5, 0, 0, 0, 8172, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 5, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 8006, 5382, 0, 0, 754, 52, 0, 0, 0, 806, 0, 0, 0, 0, 512, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 142, 0, 141, 0, 7851, 0, 110, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 13, 0, 0, 9, 8147, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 7615, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 849, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2710, 0, 0, 0, 0, 0, 0, 4, 0, 7078, 0, 0, 0, 0, 0, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 95, 8173, 1620, 0, 0, 0, 0, 0, 0, 0, 3421, 0, 237, 0, 0, 8137, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 6646, 65, 0, 0, 0, 0, 7, 0, 3, 0, 0, 58, 0, 5266, 0, 0, 4, 8156, 0, 0, 0, 38, 0, 70, 0, 0, 1, 1, 8015, 4651, 0, 17, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 636, 0, 0, 20, 0, 0, 0, 0, 0, 0, 6002, 0, 592, 0, 0, 0, 0, 0, 22, 8010, 0, 0, 19, 7457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 70, 0, 0, 63, 0, 0, 0, 5981, 8171, 3, 0, 0, 0, 1, 0, 0, 0, 0, 10, 271, 0, 0, 8189, 0, 0, 5, 0, 0, 43, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 2078, 1577, 0, 0, 0, 0, 0, 0, 1, 101, 0, 1, 5, 0, 1, 0, 0, 8118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7961, 17, 10, 0, 84] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 0, 0, 0, 7518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 6, 0, 0, 0, 0, 0, 0, 3230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 1, 0, 0, 827, 2769, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 278, 41, 0, 0, 0, 128, 0, 0, 0, 0, 6768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 670, 0, 8173, 683, 409, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 3459, 0, 0, 0, 5320, 742, 635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7487, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8561, 11429, 6389, 3685, 7224, 10552, 10206, 7490] avg=8192.0000 max=11429 max_rank=1 min=3685 min_rank=3 imbalance_ratio=1.395142 max_load_reduction=54107 max_load_reduction_pct=0.825607 moved_to_echo=35472 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 90, 0, 0, 0, 7553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3880, 0, 0, 0, 0, 0, 0, 0, 0, 3647, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 203, 2804, 3603, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 3, 0, 0, 2407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 7879, 681, 461, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 683, 0, 0, 0, 5388, 837, 638, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7469, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7646, 7527, 8928, 6610, 10594, 9213, 7546, 7472] avg=8192.0000 max=10594 max_rank=4 min=6610 min_rank=3 imbalance_ratio=1.293213 max_load_reduction=54942 max_load_reduction_pct=0.838348 moved_to_echo=34825 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 3642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 15, 0, 0, 83, 0, 0, 0, 0, 0, 2803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 8180, 0, 29, 0, 0, 7547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 18, 0, 8075, 682, 461, 593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5386, 836, 597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7475, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7070, 11801, 3239, 2906, 15759, 9852, 6840, 8069] avg=8192.0000 max=15759 max_rank=4 min=2906 min_rank=3 imbalance_ratio=1.923706 max_load_reduction=49777 max_load_reduction_pct=0.759537 moved_to_echo=40520 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 10, 0, 8, 0, 0, 0, 0, 0, 0, 0, 4, 7547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 19, 3, 0, 0, 0, 0, 67, 0, 0, 3264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 13, 2, 0, 0, 0, 0, 3, 2856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 132, 8, 0, 0, 0, 0, 0, 0, 0, 7657, 2775, 1742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 2656, 0, 270, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 27, 0, 1, 0, 7, 0, 5, 0, 4, 7580, 682, 413, 320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 5331, 749, 637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 7483, 951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7569, 3354, 11041, 12318, 7050, 9040, 6718, 8446] avg=8192.0000 max=12318 max_rank=3 min=3354 min_rank=1 imbalance_ratio=1.503662 max_load_reduction=53218 max_load_reduction_pct=0.812042 moved_to_echo=31254 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 92, 0, 705, 0, 0, 0, 0, 791, 0, 0, 319, 0, 7383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7826, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 7577, 0, 0, 0, 0, 57, 2734, 8021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3212, 40, 0, 0, 0, 114, 0, 0, 0, 20, 336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4278, 682, 445, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5253, 807, 623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 28, 0, 0, 7476, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9293, 3532, 10903, 18428, 3722, 5432, 6687, 7539] avg=8192.0000 max=18428 max_rank=3 min=3532 min_rank=1 imbalance_ratio=2.249512 max_load_reduction=47108 max_load_reduction_pct=0.718811 moved_to_echo=23380 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 1303, 0, 1, 0, 0, 0, 39, 0, 0, 0, 0, 23, 0, 0, 0, 7514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 8127, 2, 0, 1, 0, 5, 0, 0, 0, 3663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 5, 0, 0, 0, 74, 0, 0, 3205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 2740, 5382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 754, 52, 0, 0, 0, 806, 0, 0, 0, 0, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 142, 0, 141, 0, 7851, 636, 463, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 13, 0, 0, 9, 5266, 841, 633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 6979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8880, 11810, 11454, 8163, 2087, 9274, 6875, 6993] avg=8192.0000 max=11810 max_rank=1 min=2087 min_rank=4 imbalance_ratio=1.441650 max_load_reduction=53726 max_load_reduction_pct=0.819794 moved_to_echo=25229 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 849, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 7522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2710, 0, 0, 0, 0, 0, 0, 4, 0, 3162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2780, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 95, 2793, 1620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3421, 0, 237, 0, 0, 7542, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 6646, 673, 404, 595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 7, 0, 3, 0, 0, 58, 0, 5266, 0, 0, 4, 5380, 732, 634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 70, 0, 0, 1, 1, 7342, 4651, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8374, 5876, 2965, 4510, 11221, 8321, 12149, 12120] avg=8192.0000 max=12149 max_rank=6 min=2965 min_rank=2 imbalance_ratio=1.483032 max_load_reduction=53387 max_load_reduction_pct=0.814621 moved_to_echo=43811 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 636, 0, 0, 20, 7487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6002, 0, 592, 0, 0, 0, 0, 0, 22, 3591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 7457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 70, 0, 0, 63, 0, 0, 0, 5981, 2797, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 10, 271, 0, 0, 7592, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 2078, 665, 454, 597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1577, 0, 0, 0, 0, 0, 0, 1, 101, 0, 1, 5, 0, 1, 0, 0, 5374, 824, 631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7296, 17, 10, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8143, 10207, 10617, 8916, 7879, 3852, 8515, 7407] avg=8192.0000 max=10617 max_rank=2 min=3852 min_rank=5 imbalance_ratio=1.296021 max_load_reduction=54919 max_load_reduction_pct=0.837997 moved_to_echo=27653 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 256, 256, 128, 0, 7680, 128, 128, 0, 0, 14720, 22784, 19968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 0, 128, 0, 128, 37632, 128, 640, 128, 0, 128, 128, 128, 128, 27392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[1792, 128, 128, 0, 0, 128, 128, 128, 128, 0, 128, 128, 9216, 128, 0, 128, 42240, 7040, 5376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 40576, 0, 256, 0, 0, 0, 128, 128, 0, 0, 128, 0, 128, 24832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 4224, 256, 0, 128, 128, 23936, 128, 896, 0, 128, 36480, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 128, 128, 0, 128, 128, 128, 128, 128, 0, 128, 256, 128, 896, 128, 52608, 5376, 3712, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 2304, 128, 0, 128, 128, 768, 128, 0, 128, 0, 896, 1920, 0, 384, 128, 59392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 128, 128, 0, 128, 128, 128, 0, 128, 128, 128, 59136, 6144, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 256, 256, 128, 0, 7808, 128, 128, 0, 0, 14848, 22272, 20480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 128, 0, 128, 37120, 128, 640, 128, 0, 128, 128, 128, 128, 27776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 41088, 0, 256, 0, 0, 0, 128, 128, 0, 0, 128, 0, 128, 24320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[1664, 128, 128, 0, 0, 128, 0, 128, 128, 0, 128, 128, 9472, 128, 0, 128, 42752, 6400, 5120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 4352, 256, 0, 128, 128, 23552, 128, 896, 0, 128, 36864, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 128, 128, 0, 128, 128, 128, 128, 128, 0, 128, 256, 128, 896, 128, 52608, 5504, 3584, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 2176, 128, 128, 128, 128, 768, 128, 0, 128, 0, 896, 1792, 0, 384, 128, 59648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 0, 128, 128, 128, 0, 128, 128, 128, 59008, 6272, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True - [2026-08-26 14:48:11] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 436.9 | throughput per GPU (TFLOP/s/GPU): 413.9 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.694162E+00 | loss scale: 1.0 | grad norm: 66.444 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[315, 4, 0, 60, 31, 228, 671, 0, 0, 575, 659, 21, 430, 73, 180, 218, 30, 1091, 0, 13, 943, 8, 86, 1195, 1535, 12, 1, 7684, 4716, 5628, 8, 719, 91, 478, 211, 2, 11, 1496, 0, 108, 176, 4, 534, 1158, 0, 125, 23, 7533, 0, 17, 6, 6, 1, 84, 0, 0, 97, 4, 17, 181, 2, 4, 14, 40, 2034, 1, 15, 0, 611, 1, 7, 160, 653, 1, 1506, 1313, 66, 12, 212, 11, 0, 23, 2, 22, 122, 0, 1, 959, 0, 213, 50, 111, 36, 9, 66, 8, 11, 109, 3, 10, 107, 35, 44, 14, 217, 837, 285, 869, 2, 80, 1917, 30, 2, 3641, 0, 18, 1, 82, 866, 20, 6, 25, 8024, 381, 32, 10, 12, 64] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[340, 0, 0, 88, 123, 366, 660, 0, 0, 887, 197, 31, 222, 78, 134, 262, 12, 1932, 0, 14, 1147, 0, 5, 1287, 1936, 4, 0, 7605, 5020, 5990, 33, 577, 107, 372, 311, 2, 20, 776, 0, 5, 184, 1, 309, 1130, 0, 153, 3, 7919, 0, 14, 17, 0, 0, 103, 0, 6, 219, 0, 19, 220, 1, 3, 0, 54, 2641, 0, 2, 1, 504, 0, 7, 61, 810, 0, 1104, 1057, 136, 10, 258, 21, 0, 3, 0, 2, 69, 1, 1, 723, 1, 335, 9, 22, 61, 6, 57, 2, 5, 288, 4, 15, 33, 8, 51, 1, 273, 347, 428, 633, 0, 100, 1955, 16, 7, 2917, 0, 16, 0, 92, 813, 5, 2, 12, 8073, 526, 7, 15, 0, 92] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[245, 0, 0, 127, 109, 168, 589, 0, 0, 580, 280, 27, 160, 68, 270, 135, 12, 1022, 0, 14, 1035, 0, 0, 1015, 1374, 32, 0, 7477, 4651, 5895, 6, 1238, 294, 371, 330, 2, 52, 1506, 0, 0, 107, 0, 448, 1216, 0, 230, 35, 7779, 0, 49, 2, 0, 1, 53, 1, 6, 209, 1, 14, 129, 2, 0, 6, 25, 1951, 1, 1, 0, 601, 3, 16, 88, 715, 0, 2100, 1278, 95, 3, 206, 6, 0, 1, 0, 11, 32, 0, 1, 683, 3, 148, 17, 4, 20, 4, 57, 3, 2, 284, 0, 15, 23, 12, 69, 17, 472, 264, 314, 837, 0, 152, 2571, 18, 0, 2667, 0, 7, 0, 139, 1633, 5, 1, 2, 8129, 378, 9, 10, 1, 60] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[263, 1, 0, 66, 122, 248, 497, 1, 2, 679, 355, 15, 207, 80, 162, 130, 11, 1268, 0, 8, 832, 1, 0, 1176, 1845, 14, 0, 7585, 5321, 6584, 2, 484, 239, 558, 204, 1, 25, 777, 0, 37, 116, 1, 421, 1066, 0, 161, 3, 7865, 0, 15, 17, 3, 0, 82, 4, 14, 45, 1, 19, 277, 17, 4, 5, 73, 2058, 1, 0, 0, 451, 0, 29, 196, 698, 0, 1205, 1761, 60, 17, 176, 10, 1, 6, 0, 22, 95, 1, 0, 773, 0, 85, 29, 6, 2, 4, 29, 5, 7, 103, 7, 19, 16, 10, 32, 13, 226, 500, 310, 604, 4, 73, 2563, 4, 0, 3563, 0, 21, 0, 31, 1245, 3, 1, 43, 8005, 336, 9, 5, 4, 50] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[168, 2, 0, 67, 93, 307, 448, 0, 5, 555, 272, 9, 147, 90, 92, 165, 8, 589, 0, 33, 821, 0, 15, 1443, 3244, 5, 0, 7441, 4414, 6071, 25, 525, 71, 356, 178, 0, 5, 937, 0, 48, 140, 1, 464, 886, 0, 141, 15, 7849, 0, 14, 5, 0, 0, 28, 1, 11, 107, 0, 37, 299, 1, 2, 3, 49, 1774, 0, 0, 0, 613, 0, 11, 163, 883, 0, 1833, 1897, 56, 20, 296, 1, 1, 0, 1, 4, 67, 0, 0, 676, 1, 106, 10, 6, 43, 9, 64, 0, 11, 131, 0, 3, 21, 3, 15, 7, 102, 701, 421, 718, 0, 137, 2208, 9, 2, 3201, 0, 4, 1, 99, 1713, 2, 1, 63, 8117, 487, 6, 12, 6, 98] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[99, 1, 0, 83, 100, 334, 549, 0, 1, 1035, 540, 44, 227, 56, 85, 174, 2, 1073, 0, 19, 1161, 0, 1, 1214, 2167, 7, 0, 7831, 5103, 6176, 13, 891, 106, 335, 197, 1, 20, 1145, 0, 7, 73, 0, 386, 1217, 0, 234, 4, 7969, 1, 12, 7, 2, 0, 75, 0, 6, 188, 0, 20, 348, 0, 0, 8, 33, 1626, 1, 0, 0, 807, 0, 13, 127, 521, 1, 1264, 981, 53, 2, 198, 7, 0, 2, 0, 0, 30, 0, 0, 588, 0, 138, 8, 34, 13, 16, 30, 5, 32, 139, 3, 9, 10, 5, 23, 5, 265, 423, 209, 524, 0, 54, 2084, 12, 1, 4100, 0, 6, 0, 45, 1143, 6, 0, 21, 8168, 310, 2, 12, 2, 78] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[288, 1, 0, 173, 62, 168, 481, 0, 0, 852, 249, 19, 142, 42, 108, 66, 6, 861, 1, 24, 810, 3, 1, 1140, 1751, 13, 0, 7507, 5114, 6585, 9, 1065, 138, 249, 354, 1, 16, 646, 0, 47, 181, 0, 603, 1287, 0, 72, 7, 7764, 1, 51, 11, 1, 0, 30, 2, 6, 148, 4, 26, 229, 1, 4, 38, 218, 2147, 1, 5, 0, 504, 0, 44, 18, 652, 0, 1306, 1152, 59, 8, 112, 4, 2, 6, 1, 3, 30, 0, 4, 633, 1, 80, 21, 32, 14, 13, 18, 0, 0, 112, 0, 7, 32, 11, 75, 13, 162, 1096, 220, 596, 0, 46, 2972, 7, 14, 3821, 0, 6, 0, 70, 1110, 4, 3, 37, 8025, 411, 8, 5, 8, 119] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[323, 2, 0, 94, 131, 299, 605, 1, 3, 1194, 195, 41, 141, 35, 89, 169, 7, 858, 3, 7, 1103, 11, 2, 1685, 2071, 9, 0, 7411, 4844, 6558, 4, 824, 66, 397, 198, 1, 3, 877, 2, 13, 163, 8, 677, 1440, 0, 68, 4, 7884, 14, 8, 24, 3, 0, 98, 0, 13, 261, 7, 40, 232, 5, 1, 3, 41, 2116, 10, 0, 0, 473, 0, 19, 39, 856, 0, 1262, 998, 94, 2, 182, 7, 0, 2, 0, 1, 32, 0, 7, 546, 1, 109, 24, 19, 14, 8, 141, 6, 28, 116, 10, 46, 29, 3, 42, 3, 233, 309, 422, 687, 2, 76, 1670, 22, 4, 3190, 0, 7, 1, 123, 1322, 22, 0, 24, 8085, 751, 9, 13, 10, 39] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[263, 1, 0, 66, 122, 248, 497, 1, 2, 679, 355, 15, 207, 80, 162, 130, 4216, 922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1268, 0, 8, 832, 1, 0, 1176, 1845, 14, 0, 0, 2179, 0, 2, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 239, 558, 204, 1, 25, 777, 0, 37, 116, 1, 421, 1066, 0, 161, 3, 4408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 17, 3, 0, 82, 4, 14, 45, 1, 19, 277, 17, 4, 5, 73, 7585, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2058, 1, 0, 0, 451, 0, 29, 196, 698, 0, 1205, 1761, 60, 17, 176, 10, 1816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 22, 95, 1, 0, 773, 0, 85, 29, 6, 2, 4, 29, 5, 6582, 927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 103, 7, 19, 16, 10, 32, 13, 226, 500, 310, 604, 4, 73, 2563, 4, 2535, 1326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3563, 0, 21, 0, 31, 1245, 3, 1, 43, 2862, 336, 9, 5, 4, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7966, 7820, 8017, 8163, 8478, 8567, 8352, 8173] avg=8192.0000 max=8567 max_rank=5 min=7820 min_rank=1 imbalance_ratio=1.045776 max_load_reduction=56969 max_load_reduction_pct=0.869278 moved_to_echo=33570 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[315, 4, 0, 60, 31, 228, 671, 0, 0, 575, 659, 21, 430, 73, 180, 218, 4226, 883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 1091, 0, 13, 943, 8, 86, 1195, 1535, 12, 1, 0, 1930, 0, 8, 719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 478, 211, 2, 11, 1496, 0, 108, 176, 4, 534, 1158, 0, 125, 23, 4222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 6, 6, 1, 84, 0, 0, 97, 4, 17, 181, 2, 4, 14, 40, 7684, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2034, 1, 15, 0, 611, 1, 7, 160, 653, 1, 1506, 1313, 66, 12, 212, 11, 1610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 2, 22, 122, 0, 1, 959, 0, 213, 50, 111, 36, 9, 66, 8, 5626, 929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 109, 3, 10, 107, 35, 44, 14, 217, 837, 285, 869, 2, 80, 1917, 30, 2428, 1176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3641, 0, 18, 1, 82, 866, 20, 6, 25, 2869, 381, 32, 10, 12, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8574, 7571, 8639, 8159, 8213, 8177, 8174, 8029] avg=8192.0000 max=8639 max_rank=2 min=7571 min_rank=1 imbalance_ratio=1.054565 max_load_reduction=56897 max_load_reduction_pct=0.868179 moved_to_echo=32593 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[245, 0, 0, 127, 109, 168, 589, 0, 0, 580, 280, 27, 160, 68, 270, 135, 4281, 912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1022, 0, 14, 1035, 0, 0, 1015, 1374, 32, 0, 0, 1905, 0, 6, 1238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 294, 371, 330, 2, 52, 1506, 0, 0, 107, 0, 448, 1216, 0, 230, 35, 4360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 2, 0, 1, 53, 1, 6, 209, 1, 14, 129, 2, 0, 6, 25, 7477, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1951, 1, 1, 0, 601, 3, 16, 88, 715, 0, 2100, 1278, 95, 3, 206, 6, 1587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 11, 32, 0, 1, 683, 3, 148, 17, 4, 20, 4, 57, 3, 5893, 941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 284, 0, 15, 23, 12, 69, 17, 472, 264, 314, 837, 0, 152, 2571, 18, 2507, 1159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2667, 0, 7, 0, 139, 1633, 5, 1, 2, 2907, 378, 9, 10, 1, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7951, 7653, 8951, 7977, 8651, 7818, 8716, 7819] avg=8192.0000 max=8951 max_rank=2 min=7653 min_rank=1 imbalance_ratio=1.092651 max_load_reduction=56585 max_load_reduction_pct=0.863419 moved_to_echo=33004 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[340, 0, 0, 88, 123, 366, 660, 0, 0, 887, 197, 31, 222, 78, 134, 262, 4252, 928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1932, 0, 14, 1147, 0, 5, 1287, 1936, 4, 0, 0, 2056, 0, 33, 577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 372, 311, 2, 20, 776, 0, 5, 184, 1, 309, 1130, 0, 153, 3, 4439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 17, 0, 0, 103, 0, 6, 219, 0, 19, 220, 1, 3, 0, 54, 7605, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2641, 0, 2, 1, 504, 0, 7, 61, 810, 0, 1104, 1057, 136, 10, 258, 21, 1713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 69, 1, 1, 723, 1, 335, 9, 22, 61, 6, 57, 2, 5988, 935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 288, 4, 15, 33, 8, 51, 1, 273, 347, 428, 633, 0, 100, 1955, 16, 2552, 1251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2917, 0, 16, 0, 92, 813, 5, 2, 12, 2886, 526, 7, 15, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8568, 9003, 7812, 8263, 8325, 8215, 7960, 7390] avg=8192.0000 max=9003 max_rank=1 min=7390 min_rank=7 imbalance_ratio=1.098999 max_load_reduction=56533 max_load_reduction_pct=0.862625 moved_to_echo=31890 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[99, 1, 0, 83, 100, 334, 549, 0, 1, 1035, 540, 44, 227, 56, 85, 174, 4302, 934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1073, 0, 19, 1161, 0, 1, 1214, 2167, 7, 0, 0, 2089, 0, 13, 891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 335, 197, 1, 20, 1145, 0, 7, 73, 0, 386, 1217, 0, 234, 4, 4467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 12, 7, 2, 0, 75, 0, 6, 188, 0, 20, 348, 0, 0, 8, 33, 7831, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1626, 1, 0, 0, 807, 0, 13, 127, 521, 1, 1264, 981, 53, 2, 198, 7, 1742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 30, 0, 0, 588, 0, 138, 8, 34, 13, 16, 30, 5, 6174, 946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 139, 3, 9, 10, 5, 23, 5, 265, 423, 209, 524, 0, 54, 2084, 12, 2568, 1272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4100, 0, 6, 0, 45, 1143, 6, 0, 21, 2920, 310, 2, 12, 2, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8564, 8637, 8192, 8533, 7343, 7984, 7637, 8646] avg=8192.0000 max=8646 max_rank=7 min=7343 min_rank=4 imbalance_ratio=1.055420 max_load_reduction=56890 max_load_reduction_pct=0.868073 moved_to_echo=31610 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[168, 2, 0, 67, 93, 307, 448, 0, 5, 555, 272, 9, 147, 90, 92, 165, 4275, 920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 589, 0, 33, 821, 0, 15, 1443, 3244, 5, 0, 0, 1808, 0, 25, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 356, 178, 0, 5, 937, 0, 48, 140, 1, 464, 886, 0, 141, 15, 4399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 5, 0, 0, 28, 1, 11, 107, 0, 37, 299, 1, 2, 3, 49, 7441, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1774, 0, 0, 0, 613, 0, 11, 163, 883, 0, 1833, 1897, 56, 20, 296, 1, 1506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4, 67, 0, 0, 676, 1, 106, 10, 6, 43, 9, 64, 0, 6070, 940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 131, 0, 3, 21, 3, 15, 7, 102, 701, 421, 718, 0, 137, 2208, 9, 2530, 1100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3201, 0, 4, 1, 99, 1713, 2, 1, 63, 2902, 487, 6, 12, 6, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7615, 8516, 7641, 7999, 9053, 7998, 8117, 8597] avg=8192.0000 max=9053 max_rank=4 min=7615 min_rank=0 imbalance_ratio=1.105103 max_load_reduction=56483 max_load_reduction_pct=0.861862 moved_to_echo=33765 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[288, 1, 0, 173, 62, 168, 481, 0, 0, 852, 249, 19, 142, 42, 108, 66, 4227, 910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 861, 1, 24, 810, 3, 1, 1140, 1751, 13, 0, 0, 2094, 0, 9, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 249, 354, 1, 16, 646, 0, 47, 181, 0, 603, 1287, 0, 72, 7, 4352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 51, 11, 1, 0, 30, 2, 6, 148, 4, 26, 229, 1, 4, 38, 218, 7507, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2147, 1, 5, 0, 504, 0, 44, 18, 652, 0, 1306, 1152, 59, 8, 112, 4, 1745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6, 1, 3, 30, 0, 4, 633, 1, 80, 21, 32, 14, 13, 18, 0, 6583, 929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 7, 32, 11, 75, 13, 162, 1096, 220, 596, 0, 46, 2972, 7, 2502, 1275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 3821, 0, 6, 0, 70, 1110, 4, 3, 37, 2869, 411, 8, 5, 8, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7788, 7778, 7953, 8279, 7757, 8370, 9126, 8485] avg=8192.0000 max=9126 max_rank=6 min=7757 min_rank=4 imbalance_ratio=1.114014 max_load_reduction=56410 max_load_reduction_pct=0.860748 moved_to_echo=33738 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[323, 2, 0, 94, 131, 299, 605, 1, 3, 1194, 195, 41, 141, 35, 89, 169, 4262, 926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 858, 3, 7, 1103, 11, 2, 1685, 2071, 9, 0, 0, 1974, 0, 4, 824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 397, 198, 1, 3, 877, 2, 13, 163, 8, 677, 1440, 0, 68, 4, 4414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 8, 24, 3, 0, 98, 0, 13, 261, 7, 40, 232, 5, 1, 3, 41, 7411, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2116, 10, 0, 0, 473, 0, 19, 39, 856, 0, 1262, 998, 94, 2, 182, 7, 1658, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 32, 0, 7, 546, 1, 109, 24, 19, 14, 8, 141, 6, 6556, 941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 116, 10, 46, 29, 3, 42, 3, 233, 309, 422, 687, 2, 76, 1670, 22, 2544, 1212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3190, 0, 7, 1, 123, 1322, 22, 0, 24, 2882, 751, 9, 13, 10, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8510, 8558, 8331, 8163, 7716, 8407, 7454, 8397] avg=8192.0000 max=8558 max_rank=1 min=7454 min_rank=6 imbalance_ratio=1.044678 max_load_reduction=56978 max_load_reduction_pct=0.869415 moved_to_echo=31974 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] + [2026-08-26 15:24:11] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 1009.1 | throughput per GPU (TFLOP/s/GPU): 179.2 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.693994E+00 | loss scale: 1.0 | grad norm: 66.383 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[345, 0, 0, 87, 122, 361, 671, 0, 0, 901, 195, 32, 221, 75, 134, 262, 11, 1940, 0, 15, 1149, 0, 5, 1289, 1925, 3, 0, 7611, 5012, 5967, 33, 574, 110, 369, 308, 1, 20, 773, 0, 5, 186, 1, 311, 1129, 0, 163, 3, 7922, 0, 15, 18, 0, 0, 105, 0, 6, 213, 0, 19, 223, 1, 3, 0, 54, 2631, 0, 2, 1, 504, 0, 7, 60, 814, 0, 1114, 1061, 135, 10, 258, 20, 0, 3, 0, 2, 69, 1, 1, 724, 1, 329, 8, 22, 60, 6, 53, 2, 6, 286, 4, 16, 34, 8, 50, 1, 275, 339, 426, 628, 0, 101, 1966, 16, 9, 2921, 0, 16, 0, 90, 815, 5, 2, 12, 8072, 524, 7, 15, 0, 96] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[245, 0, 0, 125, 113, 167, 599, 0, 0, 591, 288, 27, 160, 69, 279, 134, 13, 1008, 0, 14, 1028, 0, 0, 1028, 1381, 35, 0, 7483, 4641, 5893, 6, 1231, 299, 381, 322, 2, 52, 1510, 0, 0, 107, 0, 452, 1220, 0, 239, 33, 7781, 0, 50, 3, 0, 1, 52, 1, 7, 211, 1, 14, 126, 2, 0, 6, 25, 1942, 1, 1, 0, 594, 3, 17, 90, 708, 0, 2096, 1277, 100, 2, 208, 6, 0, 1, 0, 9, 33, 0, 1, 672, 3, 149, 17, 4, 20, 4, 56, 3, 2, 276, 0, 15, 24, 11, 67, 17, 490, 259, 312, 820, 0, 151, 2581, 18, 0, 2660, 0, 7, 0, 140, 1622, 4, 1, 2, 8133, 372, 9, 10, 1, 60] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[267, 1, 0, 64, 119, 246, 498, 1, 2, 675, 350, 17, 202, 83, 169, 132, 11, 1253, 0, 7, 829, 1, 0, 1178, 1865, 14, 0, 7590, 5317, 6585, 2, 488, 245, 569, 207, 1, 25, 779, 0, 36, 119, 1, 426, 1056, 0, 164, 3, 7865, 0, 16, 17, 3, 0, 78, 4, 13, 43, 1, 19, 288, 16, 4, 5, 73, 2027, 1, 0, 0, 451, 0, 30, 198, 703, 0, 1210, 1766, 60, 17, 179, 10, 1, 6, 0, 21, 93, 1, 0, 767, 0, 87, 31, 5, 1, 4, 28, 5, 8, 100, 7, 19, 16, 10, 35, 13, 227, 499, 307, 602, 4, 72, 2559, 4, 0, 3556, 0, 19, 0, 31, 1244, 3, 1, 42, 8004, 339, 8, 6, 4, 53] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[286, 1, 0, 169, 64, 164, 481, 0, 0, 852, 252, 18, 138, 42, 113, 64, 6, 859, 1, 24, 813, 3, 1, 1144, 1748, 12, 0, 7513, 5099, 6564, 10, 1062, 141, 255, 353, 1, 16, 647, 0, 47, 182, 0, 602, 1295, 0, 78, 7, 7761, 1, 50, 12, 1, 0, 30, 2, 6, 150, 4, 27, 231, 1, 4, 37, 224, 2140, 1, 6, 0, 510, 0, 43, 16, 654, 0, 1316, 1147, 55, 8, 112, 4, 2, 7, 1, 4, 30, 0, 4, 625, 1, 81, 22, 32, 14, 15, 18, 0, 0, 111, 0, 7, 32, 11, 78, 13, 159, 1097, 214, 596, 0, 48, 2975, 7, 14, 3834, 0, 7, 0, 70, 1103, 3, 3, 39, 8026, 410, 9, 5, 7, 122] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[332, 1, 0, 95, 127, 296, 611, 1, 3, 1204, 189, 41, 142, 33, 93, 170, 6, 849, 3, 7, 1113, 10, 2, 1702, 2087, 10, 0, 7428, 4829, 6524, 4, 818, 67, 406, 197, 1, 4, 873, 2, 11, 164, 8, 684, 1433, 0, 70, 5, 7882, 14, 8, 23, 3, 0, 100, 0, 13, 256, 7, 43, 235, 5, 1, 3, 40, 2105, 9, 0, 0, 479, 0, 19, 40, 865, 0, 1262, 1000, 91, 2, 181, 7, 0, 2, 0, 1, 33, 0, 5, 548, 1, 109, 25, 20, 16, 8, 138, 6, 27, 115, 10, 46, 31, 3, 43, 3, 241, 302, 429, 672, 2, 77, 1676, 20, 4, 3173, 0, 8, 1, 120, 1320, 23, 0, 21, 8087, 754, 8, 13, 9, 38] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[318, 5, 0, 55, 31, 223, 673, 0, 0, 580, 656, 23, 433, 66, 184, 219, 30, 1081, 0, 15, 954, 8, 88, 1204, 1538, 11, 1, 7686, 4703, 5634, 7, 719, 92, 483, 203, 2, 10, 1505, 0, 108, 177, 3, 540, 1148, 0, 125, 21, 7533, 0, 19, 5, 5, 1, 81, 0, 0, 96, 4, 15, 179, 2, 4, 15, 40, 2032, 1, 15, 0, 607, 1, 7, 157, 644, 1, 1513, 1319, 67, 12, 211, 11, 0, 23, 2, 22, 123, 0, 1, 959, 0, 209, 51, 111, 36, 9, 66, 9, 11, 112, 3, 10, 107, 37, 45, 15, 218, 844, 282, 877, 2, 81, 1919, 29, 2, 3620, 0, 17, 1, 84, 862, 20, 7, 25, 8024, 382, 36, 11, 12, 61] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[168, 2, 0, 66, 91, 304, 448, 0, 5, 561, 275, 9, 147, 90, 94, 166, 7, 590, 0, 34, 817, 0, 15, 1458, 3249, 5, 0, 7440, 4412, 6051, 25, 524, 69, 362, 174, 0, 5, 939, 0, 50, 139, 1, 471, 884, 0, 145, 14, 7852, 0, 14, 7, 0, 0, 25, 1, 10, 106, 0, 35, 303, 1, 1, 3, 49, 1753, 0, 0, 0, 604, 0, 11, 169, 900, 0, 1837, 1902, 57, 20, 293, 1, 1, 0, 1, 4, 67, 0, 0, 677, 1, 108, 11, 6, 44, 11, 64, 0, 12, 129, 0, 3, 20, 2, 14, 7, 106, 693, 417, 710, 0, 140, 2208, 10, 2, 3201, 0, 4, 1, 95, 1716, 2, 1, 64, 8115, 490, 6, 12, 5, 96] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[102, 1, 0, 82, 103, 333, 555, 0, 1, 1042, 542, 46, 224, 54, 85, 167, 2, 1071, 0, 19, 1163, 0, 1, 1225, 2162, 7, 0, 7831, 5096, 6164, 13, 898, 113, 332, 198, 1, 23, 1157, 0, 7, 70, 0, 391, 1224, 0, 234, 4, 7973, 1, 12, 7, 2, 0, 76, 0, 6, 187, 0, 20, 348, 0, 0, 8, 34, 1612, 1, 0, 0, 806, 0, 13, 131, 524, 1, 1262, 983, 53, 2, 197, 7, 0, 2, 0, 0, 30, 0, 0, 577, 0, 137, 8, 34, 14, 16, 29, 5, 32, 136, 3, 9, 11, 5, 23, 6, 276, 420, 208, 516, 0, 53, 2083, 12, 1, 4087, 0, 5, 0, 45, 1141, 6, 0, 23, 8168, 312, 2, 12, 2, 78] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[267, 1, 0, 64, 119, 246, 498, 1, 2, 675, 350, 17, 202, 83, 169, 132, 4196, 931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1253, 0, 7, 829, 1, 0, 1178, 1865, 14, 0, 0, 2172, 0, 2, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, 569, 207, 1, 25, 779, 0, 36, 119, 1, 426, 1056, 0, 164, 3, 4392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 17, 3, 0, 78, 4, 13, 43, 1, 19, 288, 16, 4, 5, 73, 7585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2027, 1, 0, 0, 451, 0, 30, 198, 703, 0, 1210, 1766, 60, 17, 179, 10, 1823, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 21, 93, 1, 0, 767, 0, 87, 31, 5, 1, 4, 28, 5, 6585, 938, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 100, 7, 19, 16, 10, 35, 13, 227, 499, 307, 602, 4, 72, 2559, 4, 2542, 1322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3556, 0, 19, 0, 31, 1244, 3, 1, 42, 2870, 339, 8, 6, 4, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7953, 7820, 8023, 8165, 8475, 8578, 8346, 8176] avg=8192.0000 max=8578 max_rank=5 min=7820 min_rank=1 imbalance_ratio=1.047119 max_load_reduction=56958 max_load_reduction_pct=0.869110 moved_to_echo=33575 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[245, 0, 0, 125, 113, 167, 599, 0, 0, 591, 288, 27, 160, 69, 279, 134, 4264, 921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1008, 0, 14, 1028, 0, 0, 1028, 1381, 35, 0, 0, 1896, 0, 6, 1231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 299, 381, 322, 2, 52, 1510, 0, 0, 107, 0, 452, 1220, 0, 239, 33, 4346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 3, 0, 1, 52, 1, 7, 211, 1, 14, 126, 2, 0, 6, 25, 7479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1942, 1, 1, 0, 594, 3, 17, 90, 708, 0, 2096, 1277, 100, 2, 208, 6, 1591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 9, 33, 0, 1, 672, 3, 149, 17, 4, 20, 4, 56, 3, 5893, 953, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 276, 0, 15, 24, 11, 67, 17, 490, 259, 312, 820, 0, 151, 2581, 18, 2514, 1154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2660, 0, 7, 0, 140, 1622, 4, 1, 2, 2916, 372, 9, 10, 1, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7982, 7640, 8963, 7978, 8636, 7822, 8711, 7804] avg=8192.0000 max=8963 max_rank=2 min=7640 min_rank=1 imbalance_ratio=1.094116 max_load_reduction=56573 max_load_reduction_pct=0.863235 moved_to_echo=32973 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[345, 0, 0, 87, 122, 361, 671, 0, 0, 901, 195, 32, 221, 75, 134, 262, 4232, 938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1940, 0, 15, 1149, 0, 5, 1289, 1925, 3, 0, 0, 2047, 0, 33, 574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 369, 308, 1, 20, 773, 0, 5, 186, 1, 311, 1129, 0, 163, 3, 4424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 18, 0, 0, 105, 0, 6, 213, 0, 19, 223, 1, 3, 0, 54, 7607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2631, 0, 2, 1, 504, 0, 7, 60, 814, 0, 1114, 1061, 135, 10, 258, 20, 1718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 69, 1, 1, 724, 1, 329, 8, 22, 60, 6, 53, 2, 5967, 946, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 286, 4, 16, 34, 8, 50, 1, 275, 339, 426, 628, 0, 101, 1966, 16, 2560, 1247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2921, 0, 16, 0, 90, 815, 5, 2, 12, 2894, 524, 7, 15, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8576, 8991, 7803, 8264, 8335, 8198, 7963, 7406] avg=8192.0000 max=8991 max_rank=1 min=7406 min_rank=7 imbalance_ratio=1.097534 max_load_reduction=56545 max_load_reduction_pct=0.862808 moved_to_echo=31902 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[168, 2, 0, 66, 91, 304, 448, 0, 5, 561, 275, 9, 147, 90, 94, 166, 4254, 929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 590, 0, 34, 817, 0, 15, 1458, 3249, 5, 0, 0, 1802, 0, 25, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 362, 174, 0, 5, 939, 0, 50, 139, 1, 471, 884, 0, 145, 14, 4386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 7, 0, 0, 25, 1, 10, 106, 0, 35, 303, 1, 1, 3, 49, 7435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1753, 0, 0, 0, 604, 0, 11, 169, 900, 0, 1837, 1902, 57, 20, 293, 1, 1513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4, 67, 0, 0, 677, 1, 108, 11, 6, 44, 11, 64, 0, 6051, 951, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 129, 0, 3, 20, 2, 14, 7, 106, 693, 417, 710, 0, 140, 2208, 10, 2537, 1097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3201, 0, 4, 1, 95, 1716, 2, 1, 64, 2910, 490, 6, 12, 5, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7609, 8526, 7639, 7990, 9060, 8002, 8105, 8605] avg=8192.0000 max=9060 max_rank=4 min=7609 min_rank=0 imbalance_ratio=1.105957 max_load_reduction=56476 max_load_reduction_pct=0.861755 moved_to_echo=33772 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[286, 1, 0, 169, 64, 164, 481, 0, 0, 852, 252, 18, 138, 42, 113, 64, 4208, 919, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 859, 1, 24, 813, 3, 1, 1144, 1748, 12, 0, 0, 2083, 0, 10, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 255, 353, 1, 16, 647, 0, 47, 182, 0, 602, 1295, 0, 78, 7, 4334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 50, 12, 1, 0, 30, 2, 6, 150, 4, 27, 231, 1, 4, 37, 224, 7509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2140, 1, 6, 0, 510, 0, 43, 16, 654, 0, 1316, 1147, 55, 8, 112, 4, 1748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7, 1, 4, 30, 0, 4, 625, 1, 81, 22, 32, 14, 15, 18, 0, 6564, 941, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 0, 7, 32, 11, 78, 13, 159, 1097, 214, 596, 0, 48, 2975, 7, 2508, 1268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 3834, 0, 7, 0, 70, 1103, 3, 3, 39, 2877, 410, 9, 5, 7, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7771, 7766, 7958, 8289, 7760, 8365, 9124, 8503] avg=8192.0000 max=9124 max_rank=6 min=7760 min_rank=4 imbalance_ratio=1.113770 max_load_reduction=56412 max_load_reduction_pct=0.860779 moved_to_echo=33752 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[332, 1, 0, 95, 127, 296, 611, 1, 3, 1204, 189, 41, 142, 33, 93, 170, 4243, 936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 849, 3, 7, 1113, 10, 2, 1702, 2087, 10, 0, 0, 1962, 0, 4, 818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 406, 197, 1, 4, 873, 2, 11, 164, 8, 684, 1433, 0, 70, 5, 4394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 8, 23, 3, 0, 100, 0, 13, 256, 7, 43, 235, 5, 1, 3, 40, 7424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2105, 9, 0, 0, 479, 0, 19, 40, 865, 0, 1262, 1000, 91, 2, 181, 7, 1661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 33, 0, 5, 548, 1, 109, 25, 20, 16, 8, 138, 6, 6524, 953, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 115, 10, 46, 31, 3, 43, 3, 241, 302, 429, 672, 2, 77, 1676, 20, 2552, 1206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3173, 0, 8, 1, 120, 1320, 23, 0, 21, 2891, 754, 8, 13, 9, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8517, 8573, 8319, 8175, 7721, 8393, 7455, 8383] avg=8192.0000 max=8573 max_rank=1 min=7455 min_rank=6 imbalance_ratio=1.046509 max_load_reduction=56963 max_load_reduction_pct=0.869186 moved_to_echo=31952 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[318, 5, 0, 55, 31, 223, 673, 0, 0, 580, 656, 23, 433, 66, 184, 219, 4207, 892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 1081, 0, 15, 954, 8, 88, 1204, 1538, 11, 1, 0, 1921, 0, 7, 719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 483, 203, 2, 10, 1505, 0, 108, 177, 3, 540, 1148, 0, 125, 21, 4207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 5, 5, 1, 81, 0, 0, 96, 4, 15, 179, 2, 4, 15, 40, 7681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2032, 1, 15, 0, 607, 1, 7, 157, 644, 1, 1513, 1319, 67, 12, 211, 11, 1612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 2, 22, 123, 0, 1, 959, 0, 209, 51, 111, 36, 9, 66, 9, 5634, 941, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 112, 3, 10, 107, 37, 45, 15, 218, 844, 282, 877, 2, 81, 1919, 29, 2434, 1170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3620, 0, 17, 1, 84, 862, 20, 7, 25, 2876, 382, 36, 11, 12, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8565, 7577, 8624, 8147, 8210, 8201, 8196, 8016] avg=8192.0000 max=8624 max_rank=2 min=7577 min_rank=1 imbalance_ratio=1.052734 max_load_reduction=56912 max_load_reduction_pct=0.868408 moved_to_echo=32623 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[102, 1, 0, 82, 103, 333, 555, 0, 1, 1042, 542, 46, 224, 54, 85, 167, 4282, 944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1071, 0, 19, 1163, 0, 1, 1225, 2162, 7, 0, 0, 2082, 0, 13, 898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 332, 198, 1, 23, 1157, 0, 7, 70, 0, 391, 1224, 0, 234, 4, 4453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 12, 7, 2, 0, 76, 0, 6, 187, 0, 20, 348, 0, 0, 8, 34, 7827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1612, 1, 0, 0, 806, 0, 13, 131, 524, 1, 1262, 983, 53, 2, 197, 7, 1747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 30, 0, 0, 577, 0, 137, 8, 34, 14, 16, 29, 5, 6164, 957, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 136, 3, 9, 11, 5, 23, 6, 276, 420, 208, 516, 0, 53, 2083, 12, 2576, 1267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4087, 0, 5, 0, 45, 1141, 6, 0, 23, 2929, 312, 2, 12, 2, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8563, 8643, 8207, 8528, 7339, 7977, 7636, 8643] avg=8192.0000 max=8643 max_rank=1 min=7339 min_rank=4 imbalance_ratio=1.055054 max_load_reduction=56893 max_load_reduction_pct=0.868118 moved_to_echo=31595 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 8704, 128, 256, 7936, 128, 128, 10240, 16000, 128, 128, 0, 16128, 0, 128, 6400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[2048, 128, 0, 768, 896, 2176, 4608, 128, 128, 6400, 2816, 256, 1792, 640, 1152, 1408, 34048, 7424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 1408, 128, 128, 384, 128, 384, 128, 2048, 4480, 2688, 5504, 128, 768, 18048, 128, 20224, 9856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[2176, 128, 0, 768, 896, 2176, 4608, 128, 128, 6528, 2816, 256, 1792, 512, 1152, 1408, 33920, 7424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 1280, 128, 128, 384, 128, 384, 128, 2048, 4480, 2688, 5504, 128, 768, 18048, 128, 20224, 9856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 8704, 128, 256, 7936, 128, 128, 10240, 16000, 128, 128, 0, 16000, 0, 128, 6400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[1152, 3200, 2048, 128, 256, 8192, 128, 384, 1152, 128, 3968, 9472, 0, 1280, 128, 34944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[16256, 128, 128, 128, 4608, 128, 256, 896, 5888, 128, 11648, 10496, 640, 128, 1664, 128, 13440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 27136, 0, 128, 128, 768, 9856, 128, 128, 256, 23168, 3584, 128, 128, 128, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[16384, 128, 128, 128, 4608, 128, 256, 896, 5888, 128, 11648, 10496, 640, 128, 1664, 128, 13440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[1152, 3200, 2048, 128, 256, 8192, 128, 384, 1152, 128, 3968, 9472, 0, 1280, 128, 35072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 512, 128, 128, 5632, 128, 1280, 256, 256, 256, 128, 512, 128, 49536, 7552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 512, 128, 128, 5632, 128, 1280, 256, 256, 256, 128, 512, 128, 49408, 7680, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 256, 128, 128, 128, 640, 128, 128, 1280, 128, 256, 2048, 128, 128, 128, 640, 60672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 256, 128, 128, 128, 640, 128, 128, 1280, 128, 256, 1920, 128, 128, 128, 640, 60544, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 8185, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 8188, 0, 0, 0, 1, 0, 0, 30, 8107, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8185, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8189, 0, 1, 3, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 43, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 11, 0, 0, 0, 0, 0, 8029, 1, 0, 0, 3, 0, 215, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 7986, 0, 0, 16, 16, 2, 0, 0, 0, 0, 0, 8188, 4, 0, 0, 5, 0, 0, 79, 7906, 0, 0, 0, 0, 0, 0, 7, 2, 1, 0, 0, 0, 16, 0, 12, 4, 6, 3, 2, 2, 6, 1, 0, 0, 21, 7996, 4, 2, 1, 0, 0, 0, 0, 0, 0, 7926, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 8, 1, 0, 9, 8161, 0, 0, 2, 0, 134, 0, 8169, 2, 10, 217, 1, 0, 0, 0, 4, 0, 7, 0, 0, 0, 0, 1, 0, 0, 11, 0, 1860, 0, 0, 4, 0, 0, 0, 0, 0, 6510, 0, 3, 4, 0, 3, 190, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[2, 0, 0, 1, 0, 8101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 1, 0, 0, 8006, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 1, 0, 0, 2, 1, 1, 0, 0, 0, 0, 0, 8053, 5, 0, 0, 0, 0, 2, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8188, 0, 0, 0, 0, 6, 0, 8190, 0, 0, 0, 0, 0, 0, 3, 0, 142, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 296, 0, 0, 2, 0, 0, 0, 0, 0, 8076, 1, 0, 0, 1, 0, 25, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8174, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 6, 0, 0, 213, 7771, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 18, 0, 0, 0, 0, 6, 0, 0, 29, 0, 0, 0, 3, 7999, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8191, 0, 0, 3, 0, 9, 0, 8192, 0, 0, 47, 0, 0, 0, 7, 0, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 7491, 0, 0, 0, 0, 0, 812, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 7, 0, 0, 309, 8121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 8177, 0, 0, 0, 0, 0, 16, 0, 10, 0, 7586, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 1, 0, 80, 0, 8187, 0, 0, 7, 0, 0, 0, 2, 0, 120, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 47, 0, 1, 0, 0, 1, 0, 0, 0, 7355, 9, 1, 0, 0, 0, 924, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 4, 5664, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7737, 0, 0, 0, 0, 0, 4, 0, 0, 0, 8170, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 3, 0, 6, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3435, 0, 0, 2, 0, 0, 0, 0, 0, 7703, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 8163, 0, 0, 17, 0, 0, 0, 0, 5, 0, 0, 8178, 0, 0, 0, 21, 0, 0, 3, 7301, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 1, 0, 1, 0, 0, 3, 0, 0, 2, 0, 4622, 29, 0, 0, 0, 1, 67, 2, 0, 0, 7768, 0, 0, 13, 0, 10, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 4, 0, 0, 1, 8151, 0, 1, 0, 0, 52, 0, 8192, 0, 0, 22, 0, 3, 0, 36, 0, 3862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 796, 0, 0, 4, 0, 0, 0, 0, 6, 8146, 0, 0, 0, 0, 1, 17, 0, 0, 1, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[16, 0, 0, 0, 0, 8182, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 9, 5815, 11, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8148, 1, 0, 0, 0, 1, 0, 0, 0, 0, 8084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 15, 0, 8190, 0, 1, 0, 0, 0, 0, 0, 0, 828, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1361, 0, 0, 0, 0, 0, 0, 0, 0, 8011, 3, 0, 0, 0, 0, 450, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8174, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 760, 0, 0, 0, 6, 0, 0, 213, 7771, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 18, 0, 0, 0, 0, 6, 0, 0, 29, 0, 0, 7545, 726, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 7999, 0, 0, 0, 0, 0, 0, 0, 1, 0, 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 0, 0, 3, 0, 9, 0, 0, 0, 0, 47, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 6550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7491, 0, 0, 0, 0, 0, 812, 0, 0, 0, 0, 154, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8753, 8326, 8565, 8194, 8183, 6808, 8495] avg=8192.0000 max=8753 max_rank=1 min=6808 min_rank=6 imbalance_ratio=1.068481 max_load_reduction=56783 max_load_reduction_pct=0.866440 moved_to_echo=31680 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 8185, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 760, 0, 0, 0, 1, 0, 0, 30, 8107, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7540, 726, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8185, 0, 0, 0, 0, 0, 2, 0, 0, 0, 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8115, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 11, 6548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8029, 1, 0, 0, 3, 0, 215, 0, 0, 0, 0, 154, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8224, 8923, 8269, 8751, 8193, 8120, 6616, 8440] avg=8192.0000 max=8923 max_rank=1 min=6616 min_rank=6 imbalance_ratio=1.089233 max_load_reduction=56613 max_load_reduction_pct=0.863846 moved_to_echo=31369 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 761, 0, 0, 0, 0, 0, 0, 4, 5664, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7603, 726, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7737, 0, 0, 0, 0, 0, 4, 0, 0, 0, 567, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3435, 0, 0, 2, 6551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7703, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 154, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8217, 6431, 8344, 8309, 8190, 8126, 10023, 7896] avg=8192.0000 max=10023 max_rank=6 min=6431 min_rank=1 imbalance_ratio=1.223511 max_load_reduction=55513 max_load_reduction_pct=0.847061 moved_to_echo=34235 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 1, 0, 8101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 760, 0, 0, 0, 0, 1, 0, 0, 8006, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 1, 0, 0, 2, 1, 1, 0, 0, 0, 7588, 726, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8053, 5, 0, 0, 0, 0, 2, 0, 0, 0, 566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8114, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 296, 0, 0, 2, 6548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8076, 1, 0, 0, 1, 0, 25, 0, 0, 0, 0, 154, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8140, 8778, 8394, 8626, 8190, 8124, 6989, 8295] avg=8192.0000 max=8778 max_rank=1 min=6989 min_rank=6 imbalance_ratio=1.071533 max_load_reduction=56758 max_load_reduction_pct=0.866058 moved_to_echo=31598 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[16, 0, 0, 0, 0, 8182, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 760, 0, 0, 0, 0, 0, 0, 9, 5815, 11, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7523, 726, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8148, 1, 0, 0, 0, 1, 0, 0, 0, 0, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8112, 0, 0, 0, 0, 15, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 828, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1361, 0, 0, 0, 6550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8011, 3, 0, 0, 0, 0, 450, 1, 0, 0, 0, 154, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 6597, 8277, 8711, 8188, 8128, 8740, 8657] avg=8192.0000 max=8740 max_rank=6 min=6597 min_rank=1 imbalance_ratio=1.066895 max_load_reduction=56796 max_load_reduction_pct=0.866638 moved_to_echo=33713 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 760, 0, 0, 0, 7, 0, 0, 309, 8121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7060, 726, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8177, 0, 0, 0, 0, 0, 16, 0, 10, 0, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 0, 1, 0, 80, 0, 0, 0, 0, 7, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 47, 0, 1, 0, 6550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7355, 9, 1, 0, 0, 0, 924, 0, 0, 0, 0, 154, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 9197, 7789, 8730, 8190, 8204, 6719, 8482] avg=8192.0000 max=9197 max_rank=1 min=6719 min_rank=6 imbalance_ratio=1.122681 max_load_reduction=56339 max_load_reduction_pct=0.859665 moved_to_echo=31595 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 8163, 0, 0, 17, 0, 0, 0, 0, 5, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 760, 0, 0, 0, 21, 0, 0, 3, 7301, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 1, 0, 1, 0, 0, 3, 0, 0, 7229, 725, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4622, 29, 0, 0, 0, 1, 67, 2, 0, 0, 539, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 4, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8078, 0, 1, 0, 0, 52, 0, 0, 0, 0, 22, 0, 3, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 796, 0, 0, 4, 6540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8146, 0, 0, 0, 0, 1, 17, 0, 0, 1, 1, 153, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8105, 7970, 5275, 8207, 8193, 11202, 8363] avg=8192.0000 max=11202 max_rank=6 min=5275 min_rank=3 imbalance_ratio=1.367432 max_load_reduction=54334 max_load_reduction_pct=0.829071 moved_to_echo=35965 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 7986, 0, 0, 16, 16, 2, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 751, 4, 0, 0, 5, 0, 0, 79, 7906, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 16, 0, 12, 4, 6, 3, 2, 2, 6, 1, 0, 7381, 729, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 7996, 4, 2, 1, 0, 0, 0, 0, 0, 0, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 8, 1, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 8083, 0, 0, 2, 0, 134, 0, 0, 2, 10, 217, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 7, 0, 0, 0, 0, 1, 0, 0, 11, 0, 1860, 0, 0, 4, 6552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6510, 0, 3, 4, 0, 3, 190, 1, 0, 0, 0, 156, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8059, 8752, 8167, 8569, 8184, 8458, 8439, 6908] avg=8192.0000 max=8752 max_rank=1 min=6908 min_rank=7 imbalance_ratio=1.068359 max_load_reduction=56784 max_load_reduction_pct=0.866455 moved_to_echo=31989 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 0, 8105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 1, 0, 0, 7972, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 1, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 8047, 4, 0, 0, 0, 0, 2, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8188, 0, 0, 0, 0, 7, 0, 8190, 0, 0, 0, 0, 0, 0, 3, 0, 147, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 317, 0, 0, 1, 0, 0, 0, 0, 0, 8089, 1, 0, 0, 1, 0, 31, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8174, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 7, 0, 0, 214, 7768, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 17, 0, 0, 0, 0, 6, 0, 0, 27, 0, 0, 0, 1, 8002, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8190, 0, 0, 4, 0, 9, 0, 8192, 0, 0, 51, 0, 0, 0, 7, 0, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 0, 0, 0, 0, 0, 0, 0, 0, 7509, 0, 0, 0, 0, 0, 786, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 7, 0, 0, 313, 8102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 8179, 0, 0, 0, 0, 0, 20, 0, 10, 0, 7576, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 1, 0, 72, 0, 8187, 0, 0, 8, 0, 0, 0, 2, 0, 122, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 51, 0, 1, 0, 0, 1, 0, 0, 0, 7391, 8, 1, 0, 0, 0, 908, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 8183, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 8188, 0, 0, 0, 1, 0, 0, 31, 8106, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8185, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8189, 0, 1, 3, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 47, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 12, 0, 0, 0, 0, 0, 8033, 1, 0, 0, 2, 0, 208, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[17, 0, 0, 0, 0, 8182, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 10, 5592, 10, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8150, 1, 0, 0, 0, 1, 0, 0, 0, 0, 8079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 17, 0, 8190, 0, 1, 1, 0, 0, 0, 0, 0, 850, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1559, 0, 0, 0, 0, 0, 0, 0, 0, 8025, 3, 0, 0, 0, 0, 438, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 8166, 0, 0, 15, 0, 0, 0, 0, 5, 0, 0, 8178, 0, 0, 0, 18, 0, 0, 4, 7291, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 1, 0, 1, 0, 0, 3, 0, 0, 2, 0, 4617, 28, 0, 0, 0, 1, 71, 3, 0, 0, 7754, 0, 0, 13, 0, 11, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4, 0, 0, 1, 8151, 0, 2, 0, 0, 50, 0, 8192, 0, 0, 20, 0, 3, 0, 35, 0, 3851, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 832, 0, 0, 4, 0, 0, 0, 0, 5, 8150, 0, 0, 0, 0, 1, 18, 0, 0, 1, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 7994, 0, 0, 16, 16, 2, 0, 0, 0, 0, 0, 8187, 4, 0, 0, 7, 0, 0, 75, 7866, 0, 0, 0, 0, 0, 0, 7, 2, 1, 0, 0, 0, 16, 0, 10, 4, 6, 2, 2, 2, 8, 1, 0, 0, 21, 7982, 4, 3, 1, 0, 0, 0, 0, 0, 0, 7893, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 7, 2, 0, 9, 8161, 0, 0, 2, 0, 136, 0, 8169, 2, 10, 227, 1, 0, 0, 0, 4, 0, 7, 0, 0, 0, 0, 1, 0, 0, 11, 0, 1865, 0, 0, 3, 0, 0, 0, 0, 0, 6602, 0, 3, 4, 0, 3, 164, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 3, 5452, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7702, 0, 0, 0, 0, 0, 4, 0, 0, 0, 8169, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 3, 0, 8, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3630, 0, 0, 2, 0, 0, 0, 0, 0, 7756, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8174, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 828, 0, 0, 0, 7, 0, 0, 214, 7768, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 17, 0, 0, 0, 0, 6, 0, 0, 27, 0, 0, 7535, 741, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8002, 0, 0, 0, 0, 0, 0, 0, 1, 0, 569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8115, 0, 0, 4, 0, 9, 0, 0, 0, 0, 51, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 0, 0, 0, 6489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7509, 0, 0, 0, 0, 0, 786, 0, 0, 0, 0, 132, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8820, 8328, 8573, 8194, 8186, 6757, 8468] avg=8192.0000 max=8820 max_rank=1 min=6757 min_rank=6 imbalance_ratio=1.076660 max_load_reduction=56716 max_load_reduction_pct=0.865417 moved_to_echo=31605 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 828, 0, 0, 0, 7, 0, 0, 313, 8102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7044, 741, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8179, 0, 0, 0, 0, 0, 20, 0, 10, 0, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8113, 0, 0, 1, 0, 72, 0, 0, 0, 0, 8, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 51, 0, 1, 0, 6489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7391, 8, 1, 0, 0, 0, 908, 0, 0, 0, 0, 132, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8224, 9250, 7788, 8742, 8190, 8196, 6664, 8482] avg=8192.0000 max=9250 max_rank=1 min=6664 min_rank=6 imbalance_ratio=1.129150 max_load_reduction=56286 max_load_reduction_pct=0.858856 moved_to_echo=31532 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 0, 8105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 828, 0, 0, 0, 0, 1, 0, 0, 7972, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 1, 0, 0, 2, 0, 1, 0, 0, 0, 7579, 741, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8047, 4, 0, 0, 0, 0, 2, 0, 0, 0, 572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8113, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 317, 0, 0, 1, 6487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8089, 1, 0, 0, 1, 0, 31, 0, 0, 0, 0, 132, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8141, 8811, 8397, 8625, 8190, 8124, 6953, 8295] avg=8192.0000 max=8811 max_rank=1 min=6953 min_rank=6 imbalance_ratio=1.075562 max_load_reduction=56725 max_load_reduction_pct=0.865555 moved_to_echo=31562 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[17, 0, 0, 0, 0, 8182, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 828, 0, 0, 0, 0, 0, 0, 10, 5592, 10, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7512, 741, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8150, 1, 0, 0, 0, 1, 0, 0, 0, 0, 567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8110, 0, 0, 0, 0, 17, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 850, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1559, 0, 0, 0, 6489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8025, 3, 0, 0, 0, 0, 438, 1, 0, 0, 0, 132, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 6442, 8281, 8719, 8188, 8129, 8899, 8640] avg=8192.0000 max=8899 max_rank=6 min=6442 min_rank=1 imbalance_ratio=1.086304 max_load_reduction=56637 max_load_reduction_pct=0.864212 moved_to_echo=33856 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 8183, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 828, 0, 0, 0, 1, 0, 0, 31, 8106, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7535, 741, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8185, 0, 0, 0, 0, 0, 3, 0, 0, 0, 569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 12, 6487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8033, 1, 0, 0, 2, 0, 208, 0, 0, 0, 0, 132, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8220, 8991, 8279, 8759, 8193, 8119, 6558, 8417] avg=8192.0000 max=8991 max_rank=1 min=6558 min_rank=6 imbalance_ratio=1.097534 max_load_reduction=56545 max_load_reduction_pct=0.862808 moved_to_echo=31287 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 8166, 0, 0, 15, 0, 0, 0, 0, 5, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 827, 0, 0, 0, 18, 0, 0, 4, 7291, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 1, 0, 1, 0, 0, 3, 0, 0, 7210, 740, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4617, 28, 0, 0, 0, 1, 71, 3, 0, 0, 544, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8076, 0, 2, 0, 0, 50, 0, 0, 0, 0, 20, 0, 3, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3851, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 832, 0, 0, 4, 6480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 8150, 0, 0, 0, 0, 1, 18, 0, 0, 1, 1, 131, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8160, 7967, 5279, 8206, 8187, 11168, 8348] avg=8192.0000 max=11168 max_rank=6 min=5279 min_rank=3 imbalance_ratio=1.363281 max_load_reduction=54368 max_load_reduction_pct=0.829590 moved_to_echo=35909 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 7994, 0, 0, 16, 16, 2, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 818, 4, 0, 0, 7, 0, 0, 75, 7866, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 16, 0, 10, 4, 6, 2, 2, 2, 8, 1, 0, 7345, 745, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 7982, 4, 3, 1, 0, 0, 0, 0, 0, 0, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 7, 2, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 8081, 0, 0, 2, 0, 136, 0, 0, 2, 10, 227, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 7, 0, 0, 0, 0, 1, 0, 0, 11, 0, 1865, 0, 0, 3, 6491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6602, 0, 3, 4, 0, 3, 164, 1, 0, 0, 0, 133, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8065, 8777, 8146, 8559, 8184, 8468, 8382, 6955] avg=8192.0000 max=8777 max_rank=1 min=6955 min_rank=7 imbalance_ratio=1.071411 max_load_reduction=56759 max_load_reduction_pct=0.866074 moved_to_echo=31989 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 828, 0, 0, 0, 0, 0, 0, 3, 5452, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7596, 741, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7702, 0, 0, 0, 0, 0, 4, 0, 0, 0, 573, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3630, 0, 0, 2, 6491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7756, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 132, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8217, 6285, 8350, 8280, 8191, 8127, 10155, 7931] avg=8192.0000 max=10155 max_rank=6 min=6285 min_rank=1 imbalance_ratio=1.239624 max_load_reduction=55381 max_load_reduction_pct=0.845047 moved_to_echo=34404 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 5248, 128, 128, 0, 0, 128, 128, 0, 0, 128, 0, 7936, 0, 128, 128, 52480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[6144, 128, 0, 0, 128, 128, 0, 768, 58752, 128, 128, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=67328 observed_loss=-1792 tokens_per_expert=[128, 128, 128, 128, 0, 256, 0, 128, 128, 128, 128, 128, 128, 128, 128, 0, 59520, 5888, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 5248, 128, 128, 0, 0, 128, 128, 0, 0, 128, 0, 8448, 0, 128, 128, 51968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[6656, 128, 0, 0, 128, 128, 0, 768, 58240, 128, 128, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=67328 observed_loss=-1792 tokens_per_expert=[128, 128, 128, 128, 0, 256, 0, 128, 128, 128, 128, 128, 128, 128, 128, 0, 59392, 6016, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 60928, 128, 128, 128, 0, 128, 128, 128, 128, 0, 4480, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 128, 0, 0, 128, 61440, 128, 128, 128, 128, 128, 2688, 128, 0, 128, 128, 1280, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 0, 128, 128, 65280, 0, 128, 128, 128, 128, 128, 0, 128, 0, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 0, 0, 128, 0, 128, 128, 128, 128, 128, 0, 128, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 128, 0, 0, 128, 61568, 128, 128, 128, 128, 128, 2560, 128, 0, 128, 128, 1152, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 64896, 0, 128, 128, 0, 384, 0, 0, 128, 128, 384, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 0, 128, 128, 65280, 0, 128, 128, 128, 128, 128, 0, 128, 0, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 0, 0, 128, 0, 128, 0, 128, 128, 128, 0, 128, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=67328 slack_slots=63744 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[8192, 0, 0, 0, 10, 0, 0, 20, 0, 0, 0, 7690, 7, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2, 0, 0, 8163, 1, 0, 0, 0, 7988, 0, 0, 0, 0, 0, 153, 0, 8072, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 399, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 47] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[8186, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 4925, 107, 0, 0, 13, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 22, 0, 0, 8088, 0, 0, 0, 1, 8123, 0, 0, 0, 0, 0, 2940, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 0, 0, 0, 0, 1, 4, 0, 0, 0, 0, 0, 0, 0, 6, 0, 8191, 0, 175] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[8192, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4740, 10, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8075, 0, 0, 0, 8191, 20, 0, 0, 0, 8144, 0, 0, 0, 0, 0, 1665, 0, 8139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 273, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1662, 0, 0, 4, 0, 3, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[8191, 0, 0, 0, 10, 0, 0, 116, 1, 1, 0, 3247, 22, 1, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8100, 0, 0, 0, 8190, 0, 0, 0, 3, 8161, 0, 0, 0, 1, 0, 3618, 0, 8174, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 19, 0, 0, 0, 0, 0, 0, 7, 0, 0, 229, 0, 0, 0, 0, 8184, 0, 2, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1028, 0, 0, 0, 0, 2, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[8189, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 6207, 40, 0, 0, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 46, 0, 0, 8117, 101, 0, 0, 0, 8185, 0, 0, 0, 8, 0, 1762, 0, 8095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 3, 92, 4, 0, 0, 1, 8187, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 88, 0, 0, 44, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 15] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[8189, 0, 0, 1, 2, 0, 0, 15, 0, 0, 0, 7820, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 8189, 5, 0, 0, 7998, 0, 0, 0, 3, 8090, 2, 0, 0, 0, 0, 245, 0, 8089, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 16, 0, 0, 0, 0, 0, 0, 0, 0, 3, 11, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 1, 27, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 1, 0, 0, 411, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 22] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[8175, 0, 0, 0, 72, 0, 0, 1195, 0, 0, 0, 5199, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8062, 0, 0, 0, 8186, 0, 0, 0, 0, 7958, 0, 1, 0, 0, 0, 678, 0, 7009, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 1, 196, 3, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 0, 1, 0, 0, 8190, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 1, 0, 0, 0, 5, 0, 2118, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 24, 51, 8190, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[7953, 6, 0, 11, 248, 0, 0, 0, 0, 0, 0, 6462, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8154, 284, 0, 0, 8083, 0, 0, 0, 0, 8117, 0, 0, 0, 0, 0, 387, 0, 8043, 4, 3, 0, 0, 0, 0, 1, 0, 0, 14, 0, 0, 0, 0, 25, 3, 0, 4, 0, 1, 0, 0, 0, 201, 0, 7, 0, 19, 0, 0, 0, 0, 0, 129, 0, 0, 2, 0, 8162, 0, 1, 0, 0, 0, 0, 21, 48, 0, 1, 0, 0, 0, 3, 0, 7, 0, 10, 0, 0, 0, 0, 0, 859, 5, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 13, 0, 12, 8175, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2167, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4740, 10, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8075, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1665, 0, 6884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8111, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 273, 0, 0, 0, 0, 8029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1662, 0, 0, 5951, 1255, 163, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6920, 8084, 8569, 8224, 8208, 8302, 9077, 8152] avg=8192.0000 max=9077 max_rank=6 min=6920 min_rank=0 imbalance_ratio=1.108032 max_load_reduction=56459 max_load_reduction_pct=0.861496 moved_to_echo=33739 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2167, 0, 0, 0, 10, 0, 0, 20, 0, 0, 0, 7690, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 0, 6827, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7957, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 399, 0, 0, 0, 0, 8029, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 5951, 1245, 163, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9894, 8204, 6982, 8194, 8155, 8430, 7487, 8190] avg=8192.0000 max=9894 max_rank=0 min=6982 min_rank=2 imbalance_ratio=1.207764 max_load_reduction=55642 max_load_reduction_pct=0.849030 moved_to_echo=32262 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2166, 0, 0, 0, 10, 0, 0, 116, 1, 1, 0, 3247, 22, 1, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8100, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 3618, 0, 6914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 19, 0, 0, 0, 0, 0, 0, 7, 0, 0, 8128, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 229, 0, 0, 0, 0, 8021, 0, 2, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1028, 0, 0, 5951, 1260, 163, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5582, 8109, 10536, 8224, 8229, 8255, 8449, 8152] avg=8192.0000 max=10536 max_rank=2 min=5582 min_rank=0 imbalance_ratio=1.286133 max_load_reduction=55000 max_load_reduction_pct=0.839233 moved_to_echo=33085 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2165, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 4925, 107, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 22, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2940, 0, 6899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8088, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8091, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 8028, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 0, 0, 5947, 1258, 163, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 0, 0, 0, 0, 0, 6, 0, 8136, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7213, 8210, 9840, 8121, 8168, 8032, 7630, 8322] avg=8192.0000 max=9840 max_rank=2 min=7213 min_rank=0 imbalance_ratio=1.201172 max_load_reduction=55696 max_load_reduction_pct=0.849854 moved_to_echo=32152 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2166, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 6207, 40, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 46, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 0, 0, 0, 0, 0, 0, 0, 8, 0, 1762, 0, 6847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 3, 8152, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 4, 0, 0, 1, 8024, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 88, 0, 0, 5949, 1248, 163, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8418, 8203, 8718, 8150, 8231, 8122, 7496, 8198] avg=8192.0000 max=8718 max_rank=2 min=7496 min_rank=6 imbalance_ratio=1.064209 max_load_reduction=56818 max_load_reduction_pct=0.866974 moved_to_echo=32047 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2166, 0, 0, 1, 2, 0, 0, 15, 0, 0, 0, 7820, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 8189, 5, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 0, 0, 0, 245, 0, 6842, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7998, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 16, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8058, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 8027, 0, 0, 0, 0, 0, 0, 1, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 1, 0, 0, 411, 0, 3, 5949, 1247, 163, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10007, 8205, 7093, 8032, 8152, 8066, 7823, 8158] avg=8192.0000 max=10007 max_rank=0 min=7093 min_rank=2 imbalance_ratio=1.221558 max_load_reduction=55529 max_load_reduction_pct=0.847305 moved_to_echo=32199 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2162, 0, 0, 0, 72, 0, 0, 1195, 0, 0, 0, 5199, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8062, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 678, 0, 5928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 1, 196, 3, 8, 0, 0, 0, 8186, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7926, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 0, 1, 0, 0, 8027, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 1, 0, 0, 0, 5, 0, 2118, 1, 0, 5939, 1081, 163, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 24, 51, 8135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8630, 8073, 6607, 8429, 8000, 8131, 9448, 8218] avg=8192.0000 max=9448 max_rank=6 min=6607 min_rank=2 imbalance_ratio=1.153320 max_load_reduction=56088 max_load_reduction_pct=0.855835 moved_to_echo=33797 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2093, 6, 0, 11, 248, 0, 0, 0, 0, 0, 0, 6462, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8154, 284, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 387, 0, 6800, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 0, 14, 0, 0, 0, 0, 25, 3, 0, 8083, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 0, 0, 201, 0, 7, 0, 19, 0, 0, 0, 0, 0, 8084, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 0, 0, 2, 0, 7996, 0, 1, 0, 0, 0, 0, 21, 48, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 7, 0, 10, 0, 0, 0, 0, 0, 859, 5, 0, 5783, 1243, 166, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 13, 0, 12, 8116, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8872, 8448, 7191, 8162, 8393, 8198, 8126, 8146] avg=8192.0000 max=8872 max_rank=0 min=7191 min_rank=2 imbalance_ratio=1.083008 max_load_reduction=56664 max_load_reduction_pct=0.864624 moved_to_echo=32863 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[8192, 0, 0, 0, 12, 0, 0, 20, 0, 0, 0, 7656, 7, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2, 0, 0, 8162, 1, 0, 0, 0, 7994, 0, 0, 0, 0, 0, 146, 0, 8101, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 413, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 45] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[8187, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 4904, 96, 0, 0, 15, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 22, 0, 0, 8096, 0, 0, 0, 1, 8130, 0, 0, 0, 0, 0, 2962, 0, 8161, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 0, 0, 0, 0, 1, 4, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8191, 0, 169] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[8191, 0, 0, 0, 10, 0, 0, 102, 1, 1, 0, 3152, 18, 1, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8100, 0, 0, 0, 8190, 0, 0, 0, 3, 8158, 0, 0, 0, 1, 0, 3638, 0, 8176, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 17, 0, 0, 0, 0, 0, 0, 7, 0, 0, 226, 0, 0, 0, 0, 8184, 0, 2, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1129, 0, 0, 0, 0, 2, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[7953, 5, 0, 10, 226, 0, 0, 0, 0, 0, 0, 6374, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8155, 275, 0, 0, 8083, 0, 0, 0, 0, 8122, 0, 0, 0, 0, 0, 457, 0, 8049, 5, 3, 0, 0, 0, 0, 1, 0, 0, 16, 0, 0, 0, 0, 24, 3, 0, 4, 0, 1, 0, 0, 0, 200, 0, 7, 0, 19, 0, 0, 0, 0, 0, 133, 0, 0, 2, 0, 8163, 0, 0, 0, 0, 0, 0, 21, 47, 0, 2, 0, 0, 0, 3, 0, 7, 0, 10, 0, 0, 0, 0, 0, 893, 7, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 13, 0, 16, 8175, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[8189, 0, 0, 1, 3, 0, 0, 15, 0, 0, 0, 7801, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8189, 5, 0, 0, 7998, 0, 0, 0, 2, 8103, 2, 0, 0, 0, 0, 262, 0, 8084, 2, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 2, 13, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 1, 25, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 1, 0, 0, 409, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 21] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[8189, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 6144, 35, 0, 0, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8143, 45, 0, 0, 8118, 108, 0, 0, 0, 8185, 0, 0, 0, 7, 0, 1799, 0, 8104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 4, 94, 4, 0, 0, 1, 8187, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 104, 0, 0, 44, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 15] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[8176, 0, 0, 0, 70, 0, 0, 1182, 0, 0, 0, 5179, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8064, 0, 0, 0, 8187, 0, 0, 0, 0, 7959, 0, 1, 0, 0, 0, 686, 0, 7026, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 1, 200, 3, 8, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 0, 1, 0, 0, 8190, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 88, 1, 0, 0, 0, 5, 0, 2105, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 27, 51, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[8192, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4718, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8072, 0, 0, 0, 8191, 23, 0, 0, 0, 8148, 0, 0, 0, 0, 0, 1587, 0, 8139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 292, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1742, 0, 0, 5, 0, 2, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2221, 0, 0, 0, 12, 0, 0, 20, 0, 0, 0, 7656, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 0, 6833, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7965, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 413, 0, 0, 0, 0, 8022, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 5902, 1268, 170, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8138, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9916, 8207, 6981, 8191, 8159, 8437, 7457, 8188] avg=8192.0000 max=9916 max_rank=0 min=6981 min_rank=2 imbalance_ratio=1.210449 max_load_reduction=55620 max_load_reduction_pct=0.848694 moved_to_echo=32241 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2221, 0, 0, 1, 3, 0, 0, 15, 0, 0, 0, 7801, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8189, 5, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 262, 0, 6818, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7998, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8073, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 8020, 0, 0, 0, 0, 0, 0, 1, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 1, 0, 0, 409, 0, 3, 5899, 1266, 170, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10043, 8206, 7086, 8031, 8159, 8059, 7794, 8158] avg=8192.0000 max=10043 max_rank=0 min=7086 min_rank=2 imbalance_ratio=1.225952 max_load_reduction=55493 max_load_reduction_pct=0.846756 moved_to_echo=32170 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2221, 0, 0, 0, 10, 0, 0, 102, 1, 1, 0, 3152, 18, 1, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8100, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 3638, 0, 6896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 17, 0, 0, 0, 0, 0, 0, 7, 0, 0, 8127, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 226, 0, 0, 0, 0, 8014, 0, 2, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1129, 0, 0, 5901, 1280, 170, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5522, 8111, 10538, 8222, 8221, 8245, 8524, 8153] avg=8192.0000 max=10538 max_rank=2 min=5522 min_rank=0 imbalance_ratio=1.286377 max_load_reduction=54998 max_load_reduction_pct=0.839203 moved_to_echo=33143 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2221, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 6144, 35, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8143, 45, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 0, 0, 0, 0, 0, 0, 0, 7, 0, 1799, 0, 6835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8118, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 4, 8154, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 4, 0, 0, 1, 8017, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 104, 0, 0, 5899, 1269, 170, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8138, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8404, 8202, 8749, 8149, 8229, 8118, 7487, 8198] avg=8192.0000 max=8749 max_rank=2 min=7487 min_rank=6 imbalance_ratio=1.067993 max_load_reduction=56787 max_load_reduction_pct=0.866501 moved_to_echo=32032 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2152, 5, 0, 10, 226, 0, 0, 0, 0, 0, 0, 6374, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8155, 275, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 457, 0, 6785, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 0, 16, 0, 0, 0, 0, 24, 3, 0, 8083, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 0, 0, 200, 0, 7, 0, 19, 0, 0, 0, 0, 0, 8091, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 133, 0, 0, 2, 0, 7989, 0, 0, 0, 0, 0, 0, 21, 47, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 7, 0, 10, 0, 0, 0, 0, 0, 893, 7, 0, 5733, 1264, 174, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 13, 0, 16, 8115, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8813, 8443, 7247, 8161, 8390, 8194, 8139, 8149] avg=8192.0000 max=8813 max_rank=0 min=7247 min_rank=2 imbalance_ratio=1.075806 max_load_reduction=56723 max_load_reduction_pct=0.865524 moved_to_echo=32872 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2217, 0, 0, 0, 70, 0, 0, 1182, 0, 0, 0, 5179, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8064, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 686, 0, 5926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 1, 200, 3, 8, 0, 0, 0, 8187, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7929, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 0, 1, 0, 0, 8020, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 1, 0, 0, 0, 5, 0, 2105, 1, 0, 5890, 1100, 170, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 27, 51, 8137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8650, 8076, 6613, 8432, 7999, 8142, 9403, 8221] avg=8192.0000 max=9403 max_rank=6 min=6613 min_rank=2 imbalance_ratio=1.147827 max_load_reduction=56133 max_load_reduction_pct=0.856522 moved_to_echo=33765 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2220, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 4904, 96, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 22, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2962, 0, 6883, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8096, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8100, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 0, 8021, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 0, 0, 5898, 1278, 170, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8137, 0, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7237, 8208, 9847, 8128, 8172, 8027, 7601, 8316] avg=8192.0000 max=9847 max_rank=2 min=7237 min_rank=0 imbalance_ratio=1.202026 max_load_reduction=55689 max_load_reduction_pct=0.849747 moved_to_echo=32116 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2221, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4718, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8072, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1587, 0, 6865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 292, 0, 0, 0, 0, 8022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1742, 0, 0, 5902, 1274, 170, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8138, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6951, 8083, 8475, 8222, 8207, 8314, 9131, 8153] avg=8192.0000 max=9131 max_rank=6 min=6951 min_rank=0 imbalance_ratio=1.114624 max_load_reduction=56405 max_load_reduction_pct=0.860672 moved_to_echo=33805 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 0, 0, 128, 0, 128, 128, 128, 0, 128, 128, 128, 0, 6528, 128, 128, 47488, 9856, 1408, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[17280, 128, 0, 128, 384, 128, 0, 1408, 128, 128, 0, 46336, 256, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 128, 0, 0, 128, 0, 128, 128, 0, 128, 0, 11520, 0, 54016, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[1280, 128, 128, 128, 128, 64256, 0, 128, 0, 0, 128, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 128, 128, 128, 0, 0, 128, 0, 128, 128, 128, 65152, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 128, 0, 128, 512, 0, 128, 128, 128, 0, 0, 128, 0, 128, 64512, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 0, 128, 128, 0, 128, 0, 128, 128, 128, 256, 128, 128, 128, 128, 0, 65024, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 0, 128, 0, 0, 0, 128, 128, 128, 0, 0, 65152, 384, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8190, 8192, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 8191, 8188, 7284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 8186, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 855, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8187, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 8190, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 8190, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 1, 6, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 0, 5, 5, 0, 3, 0, 0, 0, 0, 0, 33, 8158, 4, 0, 15, 0, 0, 0, 0, 0, 0, 6, 0, 0, 2, 0, 1, 0, 8171, 8147, 6837, 0, 0, 1, 5, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 8151, 0, 0, 1, 0, 0, 0, 0, 0, 0, 11, 6788, 4, 0, 0, 0, 0, 0, 0, 0, 3, 1, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 58, 24, 0, 0, 0, 0, 15, 7, 7, 0, 0, 17, 0, 0, 39, 0, 0, 0, 0, 1188, 0, 1423, 14, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8125, 8192, 7193, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 5, 0, 0, 0, 8192, 0, 0, 8192, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 6684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2377, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 218, 2, 0, 0, 0, 0, 0, 8191, 5, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8183, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 8186, 6215, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 2, 1922, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8030, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 282, 8192, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8062, 8192, 7468, 0, 0, 0, 0, 0, 1, 4, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8188, 0, 750, 0, 0, 0, 0, 0, 0, 0, 0, 7587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 8, 0, 0, 0, 0, 216, 0, 307, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 8100, 8187, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 3, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 6, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 89, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 753, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 297, 52, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 7140, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8139, 8924, 8190, 8542, 8187, 8184, 7175] avg=8192.0000 max=8924 max_rank=2 min=7175 min_rank=7 imbalance_ratio=1.089355 max_load_reduction=56612 max_load_reduction_pct=0.863831 moved_to_echo=32088 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 8140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 747, 0, 7193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 6684, 0, 0, 0, 0, 0, 295, 52, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2377, 0, 0, 7083, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8163, 8212, 7940, 8254, 7118, 8187, 8181, 9481] avg=8192.0000 max=9481 max_rank=7 min=7118 min_rank=4 imbalance_ratio=1.157349 max_load_reduction=56055 max_load_reduction_pct=0.855331 moved_to_echo=32967 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8135, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 753, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 297, 52, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8174, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 1, 6, 0, 0, 7135, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8140, 8931, 8188, 8546, 8190, 8181, 7175] avg=8192.0000 max=8931 max_rank=2 min=7175 min_rank=7 imbalance_ratio=1.090210 max_load_reduction=56605 max_load_reduction_pct=0.863724 moved_to_echo=32092 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 8134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 754, 0, 7284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 297, 52, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 3, 0, 8172, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 855, 0, 0, 7140, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8161, 8040, 8184, 8543, 8216, 8191, 8012] avg=8192.0000 max=8543 max_rank=4 min=8012 min_rank=7 imbalance_ratio=1.042847 max_load_reduction=56993 max_load_reduction_pct=0.869644 moved_to_echo=32962 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 218, 2, 0, 0, 0, 0, 0, 8186, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8131, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 752, 0, 6215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8029, 0, 0, 0, 0, 0, 296, 52, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 4, 0, 0, 0, 8170, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 2, 1922, 0, 0, 7121, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8411, 8138, 6967, 8182, 8384, 8187, 8183, 9084] avg=8192.0000 max=9084 max_rank=7 min=6967 min_rank=2 imbalance_ratio=1.108887 max_load_reduction=56452 max_load_reduction_pct=0.861389 moved_to_echo=33838 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 0, 3, 0, 0, 0, 0, 0, 33, 8100, 4, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 2, 0, 1, 0, 743, 0, 6837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 11, 6788, 4, 0, 0, 0, 0, 301, 58, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 58, 24, 0, 0, 0, 0, 15, 7, 8132, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 17, 0, 0, 39, 0, 0, 0, 0, 1188, 0, 1423, 14, 3, 7127, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8165, 7589, 8161, 7174, 8189, 8245, 9833] avg=8192.0000 max=9833 max_rank=7 min=7174 min_rank=4 imbalance_ratio=1.200317 max_load_reduction=55703 max_load_reduction_pct=0.849960 moved_to_echo=33441 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 282, 8140, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 742, 0, 7468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 750, 0, 0, 0, 0, 0, 0, 0, 0, 7587, 0, 0, 0, 0, 0, 292, 52, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 8176, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 8, 0, 0, 0, 0, 216, 0, 307, 0, 0, 7028, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8026, 8439, 8210, 8191, 8688, 8189, 8186, 7607] avg=8192.0000 max=8688 max_rank=4 min=7607 min_rank=7 imbalance_ratio=1.060547 max_load_reduction=56848 max_load_reduction_pct=0.867432 moved_to_echo=32670 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8139, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 745, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 294, 52, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 6, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8171, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 89, 1, 0, 7061, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8142, 8935, 8186, 8541, 8191, 8185, 7169] avg=8192.0000 max=8935 max_rank=2 min=7169 min_rank=7 imbalance_ratio=1.090698 max_load_reduction=56601 max_load_reduction_pct=0.863663 moved_to_echo=32086 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 0, 0, 128, 0, 128, 128, 128, 0, 128, 128, 128, 0, 6784, 128, 128, 47104, 10112, 1408, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[17792, 128, 0, 128, 384, 128, 0, 1408, 128, 128, 0, 45952, 256, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 256, 0, 0, 128, 0, 128, 128, 0, 128, 0, 11648, 0, 53888, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 0, 128, 128, 0, 128, 0, 128, 128, 128, 256, 128, 128, 128, 128, 0, 65152, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[1408, 128, 128, 128, 128, 64128, 0, 128, 0, 0, 128, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 128, 0, 128, 384, 0, 128, 128, 128, 0, 0, 128, 0, 128, 64640, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 128, 0, 0, 0, 0, 128, 128, 0, 0, 65152, 384, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 128, 128, 128, 0, 0, 128, 0, 128, 128, 128, 65152, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8187, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 8189, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 8191, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 1, 5, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 8191, 8189, 7361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 8187, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 777, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8124, 0, 5, 4, 0, 3, 0, 0, 0, 0, 0, 35, 8158, 3, 0, 14, 0, 0, 0, 0, 0, 0, 6, 0, 0, 2, 0, 1, 0, 8171, 8147, 6843, 0, 0, 1, 7, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 8151, 0, 0, 2, 0, 0, 0, 0, 0, 0, 12, 6806, 4, 0, 0, 0, 0, 0, 0, 0, 3, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5, 0, 60, 24, 0, 0, 0, 0, 15, 7, 7, 0, 0, 16, 0, 0, 41, 0, 0, 0, 0, 1178, 0, 1408, 14, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 8103, 8187, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 3, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 84, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8028, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 290, 8192, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8052, 8192, 7491, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8188, 0, 753, 0, 0, 0, 0, 0, 0, 0, 0, 7592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 9, 0, 0, 0, 0, 211, 0, 284, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 214, 3, 0, 0, 0, 0, 0, 8191, 5, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8183, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 8186, 6241, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 2, 1894, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8125, 8192, 7208, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 6, 0, 0, 0, 8192, 0, 0, 8192, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 6520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2522, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8190, 8192, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 8133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 734, 0, 7361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 313, 53, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 3, 0, 8172, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 777, 0, 0, 7144, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8160, 8097, 8184, 8561, 8215, 8189, 7938] avg=8192.0000 max=8561 max_rank=4 min=7938 min_rank=7 imbalance_ratio=1.045044 max_load_reduction=56975 max_load_reduction_pct=0.869370 moved_to_echo=32903 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8134, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 734, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 313, 53, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8173, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 1, 5, 0, 0, 7138, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8139, 8914, 8188, 8563, 8190, 8180, 7175] avg=8192.0000 max=8914 max_rank=2 min=7175 min_rank=7 imbalance_ratio=1.088135 max_load_reduction=56622 max_load_reduction_pct=0.863983 moved_to_echo=32108 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 8139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 729, 0, 7208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 6520, 0, 0, 0, 0, 0, 310, 53, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2522, 0, 0, 7086, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8213, 7937, 8254, 6972, 8187, 8180, 9629] avg=8192.0000 max=9629 max_rank=7 min=6972 min_rank=4 imbalance_ratio=1.175415 max_load_reduction=55907 max_load_reduction_pct=0.853073 moved_to_echo=32968 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4, 0, 3, 0, 0, 0, 0, 0, 35, 8099, 3, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 2, 0, 1, 0, 724, 0, 6843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 12, 6806, 4, 0, 0, 0, 0, 316, 59, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5, 0, 60, 24, 0, 0, 0, 0, 15, 7, 8131, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 16, 0, 0, 41, 0, 0, 0, 0, 1178, 0, 1408, 14, 4, 7131, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8163, 7576, 8165, 7210, 8188, 8247, 9815] avg=8192.0000 max=9815 max_rank=7 min=7210 min_rank=4 imbalance_ratio=1.198120 max_load_reduction=55721 max_load_reduction_pct=0.850235 moved_to_echo=33460 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8138, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 727, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 309, 53, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8171, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 84, 1, 0, 7067, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8141, 8916, 8186, 8557, 8191, 8186, 7170] avg=8192.0000 max=8916 max_rank=2 min=7170 min_rank=7 imbalance_ratio=1.088379 max_load_reduction=56620 max_load_reduction_pct=0.863953 moved_to_echo=32104 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 214, 3, 0, 0, 0, 0, 0, 8188, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8130, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 733, 0, 6241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8033, 0, 0, 0, 0, 0, 312, 53, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 4, 0, 0, 0, 8169, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 2, 1894, 0, 0, 7126, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8410, 8137, 6974, 8181, 8404, 8187, 8182, 9061] avg=8192.0000 max=9061 max_rank=7 min=6974 min_rank=2 imbalance_ratio=1.106079 max_load_reduction=56475 max_load_reduction_pct=0.861740 moved_to_echo=33834 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 290, 8139, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 723, 0, 7491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 753, 0, 0, 0, 0, 0, 0, 0, 0, 7592, 0, 0, 0, 0, 0, 307, 53, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 8176, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 9, 0, 0, 0, 0, 211, 0, 284, 0, 0, 7022, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8025, 8445, 8214, 8189, 8711, 8190, 8187, 7575] avg=8192.0000 max=8711 max_rank=4 min=7575 min_rank=7 imbalance_ratio=1.063354 max_load_reduction=56825 max_load_reduction_pct=0.867081 moved_to_echo=32663 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 734, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 313, 53, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 7143, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8138, 8908, 8189, 8558, 8188, 8185, 7173] avg=8192.0000 max=8908 max_rank=2 min=7173 min_rank=7 imbalance_ratio=1.087402 max_load_reduction=56628 max_load_reduction_pct=0.864075 moved_to_echo=32104 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 0, 128, 0, 128, 128, 128, 0, 0, 0, 1536, 128, 7040, 128, 128, 56960, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 128, 128, 0, 128, 0, 128, 0, 6016, 0, 59648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 896, 128, 0, 0, 0, 0, 0, 0, 128, 61952, 128, 0, 0, 0, 0, 2432, 512, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 128, 0, 0, 0, 128, 0, 0, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 128, 0, 0, 0, 128, 128, 0, 128, 128, 128, 128, 128, 0, 128, 128, 65408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 0, 0, 0, 256, 128, 128, 0, 0, 0, 0, 65280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 0, 0, 128, 0, 0, 512, 65152, 128, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 0, 0, 128, 0, 128, 128, 128, 0, 0, 0, 1536, 128, 7040, 128, 128, 56960, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 128, 128, 0, 128, 0, 128, 0, 5888, 0, 59776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 896, 128, 0, 0, 0, 0, 0, 0, 128, 61824, 128, 0, 0, 0, 0, 2560, 512, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 0, 0, 0, 256, 128, 128, 0, 0, 0, 0, 65280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 0, 0, 0, 128, 128, 0, 128, 128, 128, 128, 128, 0, 128, 128, 65408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 0, 0, 128, 0, 0, 512, 65152, 128, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 128, 0, 0, 0, 128, 0, 0, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 4, 0, 0, 0, 0, 8192, 0, 7735, 0, 0, 0, 0, 0, 0, 0, 0, 8148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1226, 4098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 3622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 14, 0, 0, 8145, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7867, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 2, 0, 0, 0, 8192, 0, 7225, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4, 1380, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8185, 0, 0, 0, 0, 0, 0, 0, 7774, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 27, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 1, 0, 0, 1, 23, 0, 0, 0, 4, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8177, 3, 0, 0, 0, 0, 1, 0, 8112, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 7, 1, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8160, 0, 1, 0, 4, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 8191, 0, 0, 0, 4, 0, 0, 0, 0, 8191, 0, 7276, 0, 3, 0, 0, 0, 0, 0, 0, 7973, 0, 0, 0, 0, 2, 0, 0, 0, 0, 10, 0, 0, 0, 115, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 3276, 1551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 12, 8185, 0, 0, 0, 0, 1, 3, 0, 4812, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 77, 0, 0, 99, 0, 0, 8018, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7700, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1, 0, 0, 0, 8192, 0, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 15, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 3, 0, 0, 52, 3, 0, 0, 8182, 0, 7872, 0, 0, 0, 0, 0, 0, 0, 0, 8110, 0, 0, 0, 0, 0, 0, 0, 0, 100, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2861, 4727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 17, 1008, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 7991, 0, 0, 223, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 7982, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 8188, 0, 8119, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 285, 831, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 7180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 4, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 8162, 0, 127, 0, 0, 0, 0, 0, 1, 8162, 0, 8168, 4, 0, 0, 21, 0, 2, 18, 25, 8008, 0, 0, 0, 0, 3, 0, 0, 0, 83, 0, 5, 1, 0, 1, 0, 8, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 791, 1, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 8172, 3, 1, 14, 0, 1, 10, 4, 7907, 0, 19, 0, 2, 0, 0, 0, 0, 38, 0, 0, 0, 1, 0, 0, 4, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7535, 0, 0, 0, 29, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7735, 0, 0, 0, 0, 0, 0, 0, 0, 342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7806, 355, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1226, 4098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2122, 0, 0, 0, 0, 0, 0, 0, 3622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 14, 0, 0, 8137, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7867, 0, 0, 0, 0, 0, 183, 19, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8077, 8167, 10856, 5744, 8187, 8251, 8077] avg=8192.0000 max=10856 max_rank=3 min=5744 min_rank=4 imbalance_ratio=1.325195 max_load_reduction=54680 max_load_reduction_pct=0.834351 moved_to_echo=30259 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 7842, 355, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 15, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 5532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2122, 0, 0, 0, 0, 0, 0, 0, 8083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 183, 19, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 8493, 8207, 5690, 10205, 8186, 8184, 8397] avg=8192.0000 max=10205 max_rank=4 min=5690 min_rank=3 imbalance_ratio=1.245728 max_load_reduction=55331 max_load_reduction_pct=0.844284 moved_to_echo=34972 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 344, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 23, 0, 0, 0, 4, 0, 2, 0, 0, 0, 0, 0, 0, 7846, 355, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2117, 3, 0, 0, 0, 0, 1, 0, 8112, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 7, 1, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8160, 0, 1, 0, 4, 2, 183, 19, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 8531, 8236, 5598, 10236, 8187, 8199, 8378] avg=8192.0000 max=10236 max_rank=4 min=5598 min_rank=3 imbalance_ratio=1.249512 max_load_reduction=55300 max_load_reduction_pct=0.843811 moved_to_echo=35000 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7276, 0, 3, 0, 0, 0, 0, 0, 0, 335, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 115, 0, 5, 0, 0, 0, 0, 0, 0, 7638, 355, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 3276, 1551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 2119, 0, 0, 0, 0, 1, 3, 0, 4812, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 99, 0, 0, 8010, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7700, 0, 0, 0, 0, 0, 183, 19, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 7616, 8128, 10377, 6947, 8194, 8187, 7910] avg=8192.0000 max=10377 max_rank=3 min=6947 min_rank=4 imbalance_ratio=1.266724 max_load_reduction=55159 max_load_reduction_pct=0.841660 moved_to_echo=31238 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7225, 0, 0, 0, 0, 0, 0, 0, 0, 344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7845, 355, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1380, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2119, 0, 0, 0, 0, 0, 0, 0, 7774, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 27, 0, 0, 8150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 1, 0, 0, 183, 19, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 7569, 8213, 6916, 9894, 8188, 8179, 8402] avg=8192.0000 max=9894 max_rank=4 min=6916 min_rank=3 imbalance_ratio=1.207764 max_load_reduction=55642 max_load_reduction_pct=0.849030 moved_to_echo=34663 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8119, 0, 0, 0, 0, 0, 0, 0, 0, 344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7, 0, 2, 0, 0, 0, 0, 7842, 355, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 285, 831, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2121, 0, 0, 0, 0, 0, 0, 0, 7180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 4, 0, 0, 0, 0, 183, 19, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8170, 8463, 8212, 6649, 9301, 8186, 8179, 8376] avg=8192.0000 max=9301 max_rank=4 min=6649 min_rank=3 imbalance_ratio=1.135376 max_load_reduction=56235 max_load_reduction_pct=0.858078 moved_to_echo=34042 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 52, 3, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7872, 0, 0, 0, 0, 0, 0, 0, 0, 341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7769, 355, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2861, 4727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2121, 0, 0, 0, 0, 0, 0, 17, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7983, 0, 0, 223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 7982, 0, 0, 0, 0, 0, 183, 19, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8213, 8232, 13119, 3146, 8190, 8206, 8209] avg=8192.0000 max=13119 max_rank=3 min=3146 min_rank=4 imbalance_ratio=1.601440 max_load_reduction=52417 max_load_reduction_pct=0.799820 moved_to_echo=27751 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 0, 0, 127, 0, 0, 0, 0, 0, 1, 8139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 4, 0, 0, 21, 0, 2, 18, 25, 333, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 5, 1, 0, 1, 0, 8, 0, 0, 2, 0, 0, 0, 7675, 360, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 791, 1, 0, 0, 0, 0, 0, 0, 0, 15, 0, 5523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2102, 3, 1, 14, 0, 1, 10, 4, 7907, 0, 19, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 1, 0, 0, 4, 0, 0, 0, 0, 19, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8148, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7535, 0, 0, 0, 29, 0, 187, 23, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8271, 8574, 8141, 6331, 10063, 8218, 8151, 7787] avg=8192.0000 max=10063 max_rank=4 min=6331 min_rank=3 imbalance_ratio=1.228394 max_load_reduction=55473 max_load_reduction_pct=0.846451 moved_to_echo=34219 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 8164, 0, 124, 0, 0, 0, 0, 0, 2, 8162, 0, 8168, 6, 0, 0, 21, 0, 1, 19, 24, 8001, 0, 0, 0, 0, 3, 0, 0, 0, 78, 0, 3, 1, 0, 0, 0, 8, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 750, 1, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 8172, 3, 2, 14, 0, 1, 12, 6, 7906, 0, 18, 0, 2, 0, 0, 0, 0, 36, 0, 0, 0, 1, 0, 0, 5, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7586, 0, 0, 0, 28, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 8188, 0, 8122, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 292, 921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 7082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 4, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 4, 0, 0, 0, 0, 8192, 0, 7705, 0, 0, 0, 0, 0, 0, 0, 0, 8147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1285, 4241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 3422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 13, 0, 0, 8147, 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7890, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1, 0, 0, 0, 8192, 0, 8143, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 204, 16, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 1, 0, 0, 1, 19, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8178, 3, 0, 0, 0, 0, 1, 0, 8119, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 7, 1, 2, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 4, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 2, 0, 0, 52, 3, 0, 0, 8182, 0, 7827, 0, 0, 0, 0, 0, 0, 0, 0, 8107, 0, 0, 0, 0, 0, 0, 0, 0, 101, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3064, 4709, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 17, 841, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 7988, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 8006, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 8191, 0, 0, 0, 4, 0, 0, 0, 0, 8191, 0, 7210, 0, 4, 0, 0, 0, 0, 0, 0, 7968, 0, 0, 0, 0, 2, 0, 0, 0, 0, 10, 0, 0, 0, 110, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 3519, 1546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 13, 8185, 0, 0, 0, 0, 1, 3, 0, 4630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 74, 0, 0, 88, 0, 0, 8017, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7736, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 2, 0, 0, 0, 8192, 0, 7187, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 5, 1559, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8185, 0, 0, 0, 0, 0, 0, 0, 7630, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 24, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7705, 0, 0, 0, 0, 0, 0, 0, 0, 365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7782, 381, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1285, 4241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2226, 0, 0, 0, 0, 0, 0, 0, 3422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 13, 0, 0, 8139, 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7890, 0, 0, 0, 0, 0, 168, 19, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8070, 8169, 10943, 5648, 8186, 8258, 8085] avg=8192.0000 max=10943 max_rank=3 min=5648 min_rank=4 imbalance_ratio=1.335815 max_load_reduction=54593 max_load_reduction_pct=0.833023 moved_to_echo=30177 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8122, 0, 0, 0, 0, 0, 0, 0, 0, 367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8, 0, 2, 0, 0, 0, 0, 7819, 381, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 292, 921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2226, 0, 0, 0, 0, 0, 0, 0, 7082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 4, 0, 0, 0, 0, 167, 19, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8170, 8489, 8216, 6631, 9308, 8186, 8178, 8358] avg=8192.0000 max=9308 max_rank=4 min=6631 min_rank=3 imbalance_ratio=1.136230 max_load_reduction=56228 max_load_reduction_pct=0.857971 moved_to_echo=34030 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 0, 0, 124, 0, 0, 0, 0, 0, 2, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 6, 0, 0, 21, 0, 1, 19, 24, 354, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 3, 1, 0, 0, 0, 8, 0, 0, 2, 0, 0, 0, 7647, 386, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 750, 1, 0, 0, 0, 0, 0, 0, 0, 16, 0, 5407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2207, 3, 2, 14, 0, 1, 12, 6, 7906, 0, 18, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 1, 0, 0, 5, 0, 0, 0, 0, 23, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7586, 0, 0, 0, 28, 0, 172, 20, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8271, 8596, 8131, 6175, 10171, 8223, 8152, 7817] avg=8192.0000 max=10171 max_rank=4 min=6175 min_rank=3 imbalance_ratio=1.241577 max_load_reduction=55365 max_load_reduction_pct=0.844803 moved_to_echo=34363 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7187, 0, 0, 0, 0, 0, 0, 0, 0, 367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7821, 381, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1559, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2224, 0, 0, 0, 0, 0, 0, 0, 7630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 24, 0, 0, 8150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 167, 19, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 7554, 8219, 6983, 9855, 8187, 8176, 8387] avg=8192.0000 max=9855 max_rank=4 min=6983 min_rank=3 imbalance_ratio=1.203003 max_load_reduction=55681 max_load_reduction_pct=0.849625 moved_to_echo=34605 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8143, 0, 0, 0, 0, 0, 0, 0, 0, 367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 7820, 381, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 204, 16, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 5417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2226, 0, 0, 0, 0, 0, 0, 0, 8025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 168, 19, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 8510, 8211, 5638, 10251, 8186, 8184, 8382] avg=8192.0000 max=10251 max_rank=4 min=5638 min_rank=3 imbalance_ratio=1.251343 max_load_reduction=55285 max_load_reduction_pct=0.843582 moved_to_echo=35003 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 52, 3, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7827, 0, 0, 0, 0, 0, 0, 0, 0, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7744, 381, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3064, 4709, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2226, 0, 0, 0, 0, 0, 0, 17, 841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7981, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 8006, 0, 0, 0, 0, 0, 167, 18, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8190, 8234, 13190, 3084, 8189, 8212, 8216] avg=8192.0000 max=13190 max_rank=3 min=3084 min_rank=4 imbalance_ratio=1.610107 max_load_reduction=52346 max_load_reduction_pct=0.798737 moved_to_echo=27701 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 367, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 19, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 7823, 381, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2222, 3, 0, 0, 0, 0, 1, 0, 8119, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 7, 1, 2, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 4, 2, 167, 19, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 8554, 8234, 5478, 10348, 8186, 8203, 8362] avg=8192.0000 max=10348 max_rank=4 min=5478 min_rank=3 imbalance_ratio=1.263184 max_load_reduction=55188 max_load_reduction_pct=0.842102 moved_to_echo=35099 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7210, 0, 4, 0, 0, 0, 0, 0, 0, 357, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 110, 0, 5, 0, 0, 0, 0, 0, 0, 7611, 381, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 3519, 1546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 2224, 0, 0, 0, 0, 1, 3, 0, 4630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 88, 0, 0, 8010, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7736, 0, 0, 0, 0, 0, 167, 19, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 7573, 8122, 10498, 6871, 8193, 8173, 7929] avg=8192.0000 max=10498 max_rank=3 min=6871 min_rank=4 imbalance_ratio=1.281494 max_load_reduction=55038 max_load_reduction_pct=0.839813 moved_to_echo=31166 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 0, 128, 9344, 12032, 128, 0, 0, 128, 0, 0, 0, 0, 128, 128, 44288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 17024, 128, 128, 128, 0, 128, 128, 128, 48512, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[62848, 128, 128, 0, 128, 0, 128, 128, 128, 2816, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 256, 128, 128, 128, 0, 128, 0, 128, 0, 128, 128, 0, 0, 0, 62336, 2944, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 0, 128, 128, 0, 0, 0, 128, 0, 0, 63872, 128, 128, 128, 128, 128, 1536, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 0, 0, 0, 0, 256, 0, 128, 128, 128, 0, 128, 65408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 128, 10112, 12288, 128, 0, 0, 128, 0, 0, 0, 0, 128, 128, 43392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 17792, 128, 128, 128, 0, 128, 128, 128, 47744, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 256, 128, 128, 128, 0, 128, 0, 128, 0, 128, 128, 0, 0, 0, 62080, 3072, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[62592, 128, 128, 0, 128, 0, 128, 128, 128, 2944, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 0, 0, 0, 128, 0, 128, 128, 128, 0, 128, 65408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 0, 128, 128, 0, 0, 0, 128, 0, 0, 64000, 128, 0, 128, 128, 128, 1408, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 0, 256, 128, 0, 65024, 0, 128, 384, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 128, 128, 0, 256, 128, 128, 65024, 0, 128, 384, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 128, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 1, 3, 0, 0, 1, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 8192, 0, 48, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 89, 0, 0, 1, 0, 8002, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1193, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 8183, 0, 5, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 464, 0, 0, 7, 0, 6371, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 126, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 1, 0, 8184, 0, 1, 1, 0, 8107, 0, 15, 0, 2, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 113, 0, 0, 0, 0, 5, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 8, 0, 0, 0, 1, 4376, 1, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3662, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 6793, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8188, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 8156, 38, 0, 0, 5, 21, 0, 0, 8166, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 6, 0, 0, 0, 8161, 0, 17, 16, 0, 8170, 0, 5121, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4, 19, 20, 0, 0, 0, 0, 0, 8156, 0, 0, 1, 5, 1, 0, 0, 0, 0, 0, 1, 0, 0, 3, 8151, 0, 0, 0, 12, 21, 1356, 0, 24, 0, 0, 202, 0, 0, 0, 9, 0, 0, 0, 272, 0, 0, 0, 0, 15, 0, 0, 10, 0, 0, 0, 0, 2, 0, 0, 0, 1176, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 8163, 1953, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 161, 0, 0, 0, 0, 0, 0, 7799, 0, 0, 0, 0, 7961, 0, 3702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 624, 10, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 974, 0, 0, 6, 0, 48, 0, 0, 19, 0, 0, 0, 0, 1357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 471, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 8111, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 193, 0, 0, 1, 0, 6468, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 886, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1, 0, 0, 8190, 5700, 0, 0, 1, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 2, 0, 8117, 0, 0, 0, 0, 8180, 0, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 8190, 0, 0, 0, 0, 21, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 0, 3, 1, 0, 0, 768, 0, 1, 0, 0, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 315, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6163, 1193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7046, 0, 5, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8007, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7703, 0, 0, 0, 0, 0, 464, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6371, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2029, 1137, 722, 489, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 0, 0, 7421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7356, 8192, 8239, 7052, 8071, 8174, 10939, 7513] avg=8192.0000 max=10939 max_rank=6 min=7052 min_rank=3 imbalance_ratio=1.335327 max_load_reduction=54597 max_load_reduction_pct=0.833084 moved_to_echo=34697 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6163, 6793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7051, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8007, 0, 0, 0, 0, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7703, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2029, 1137, 722, 489, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 7421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12956, 8192, 8241, 7091, 8412, 7744, 5474, 7426] avg=8192.0000 max=12956 max_rank=0 min=5474 min_rank=6 imbalance_ratio=1.581543 max_load_reduction=52580 max_load_reduction_pct=0.802307 moved_to_echo=29056 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 1, 3, 0, 0, 1, 0, 6153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7054, 0, 48, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8006, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7702, 0, 0, 0, 0, 0, 89, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8002, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2026, 1138, 721, 489, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 1, 0, 0, 0, 7411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6159, 8192, 8235, 7128, 8009, 7792, 12562, 7459] avg=8192.0000 max=12562 max_rank=6 min=6159 min_rank=0 imbalance_ratio=1.533447 max_load_reduction=52974 max_load_reduction_pct=0.808319 moved_to_echo=35822 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6139, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 1, 0, 8184, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6981, 0, 15, 0, 2, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8006, 0, 0, 0, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 7701, 0, 0, 0, 0, 0, 8, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4376, 1, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2021, 1126, 721, 489, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3662, 0, 0, 0, 0, 0, 0, 7419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6265, 8194, 8237, 7003, 8119, 7715, 8922, 11081] avg=8192.0000 max=11081 max_rank=7 min=6265 min_rank=0 imbalance_ratio=1.352661 max_load_reduction=54455 max_load_reduction_pct=0.830917 moved_to_echo=35837 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6162, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 8111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8007, 0, 0, 0, 0, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7701, 0, 0, 0, 0, 0, 193, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6468, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2029, 1137, 721, 489, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 886, 0, 0, 0, 0, 0, 0, 7414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6634, 8192, 8162, 7046, 8259, 7895, 11048, 8300] avg=8192.0000 max=11048 max_rank=6 min=6634 min_rank=0 imbalance_ratio=1.348633 max_load_reduction=54488 max_load_reduction_pct=0.831421 moved_to_echo=35502 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1, 0, 0, 6162, 5700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 2, 0, 8117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7044, 0, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8005, 0, 0, 0, 0, 21, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7702, 0, 3, 1, 0, 0, 768, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2028, 1136, 721, 489, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 315, 0, 0, 0, 0, 0, 0, 7413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11874, 8192, 8169, 7482, 8029, 8475, 5585, 7730] avg=8192.0000 max=11874 max_rank=0 min=5585 min_rank=6 imbalance_ratio=1.449463 max_load_reduction=53662 max_load_reduction_pct=0.818817 moved_to_echo=29819 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 6141, 1953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 161, 0, 0, 0, 0, 0, 0, 7799, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6855, 0, 3702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8005, 0, 0, 0, 0, 624, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7682, 0, 0, 0, 0, 0, 974, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 19, 0, 0, 0, 0, 1357, 0, 0, 0, 0, 0, 0, 0, 2022, 1106, 722, 488, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 7420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8096, 8187, 8009, 10557, 8639, 8662, 5947, 7439] avg=8192.0000 max=10557 max_rank=3 min=5947 min_rank=6 imbalance_ratio=1.288696 max_load_reduction=54979 max_load_reduction_pct=0.838913 moved_to_echo=30687 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 6133, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 21, 0, 0, 8162, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 6, 0, 0, 0, 8161, 0, 17, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7032, 0, 5121, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 7971, 0, 0, 1, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 7662, 0, 0, 0, 12, 21, 1356, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 202, 0, 0, 0, 9, 0, 0, 0, 272, 0, 0, 0, 0, 15, 0, 0, 2023, 1138, 722, 489, 185, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 2, 0, 0, 0, 1176, 0, 0, 0, 0, 0, 1, 7399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6196, 8195, 8244, 12177, 7998, 9079, 5059, 8588] avg=8192.0000 max=12177 max_rank=3 min=5059 min_rank=6 imbalance_ratio=1.486450 max_load_reduction=53359 max_load_reduction_pct=0.814194 moved_to_echo=30724 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 116, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 1, 0, 8185, 0, 1, 1, 0, 8105, 0, 9, 0, 1, 0, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 89, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1, 4310, 1, 1, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3774, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 0, 0, 8190, 5524, 0, 0, 1, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 2, 0, 8118, 0, 0, 0, 0, 8180, 0, 350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 8190, 0, 0, 0, 0, 22, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 0, 3, 1, 0, 0, 804, 0, 1, 0, 0, 1206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 359, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1186, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 8185, 0, 5, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 439, 0, 0, 8, 0, 6400, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 1, 3, 0, 0, 1, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 8192, 0, 42, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 67, 0, 0, 1, 0, 8039, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 6568, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8187, 0, 39, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 1136, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 1853, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8192, 0, 169, 0, 0, 0, 0, 0, 0, 7791, 0, 0, 0, 0, 7967, 0, 3593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 631, 12, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 1011, 0, 0, 7, 0, 53, 0, 0, 18, 0, 0, 0, 0, 1517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 8155, 30, 0, 0, 7, 20, 0, 0, 8166, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 5, 0, 0, 0, 8162, 0, 11, 15, 0, 8172, 0, 4851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 19, 23, 0, 0, 0, 0, 0, 8156, 0, 0, 2, 4, 1, 0, 0, 0, 0, 0, 1, 0, 0, 4, 8152, 0, 0, 0, 13, 21, 1432, 0, 23, 0, 0, 258, 0, 0, 0, 9, 0, 0, 0, 274, 0, 0, 0, 0, 16, 0, 0, 10, 0, 0, 0, 0, 1, 0, 0, 0, 1321, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 376, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 8111, 0, 0, 0, 0, 8184, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 202, 0, 0, 1, 0, 6488, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 953, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6241, 1186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7107, 0, 5, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8011, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7690, 0, 0, 0, 0, 0, 439, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6400, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1951, 1078, 766, 502, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 0, 0, 0, 7375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7427, 8192, 8241, 7113, 8072, 8137, 10888, 7466] avg=8192.0000 max=10888 max_rank=6 min=7113 min_rank=3 imbalance_ratio=1.329102 max_load_reduction=54648 max_load_reduction_pct=0.833862 moved_to_echo=34563 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 1, 3, 0, 0, 1, 0, 6231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7113, 0, 42, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8010, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7689, 0, 0, 0, 0, 0, 67, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8039, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1947, 1079, 765, 502, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 1, 0, 0, 0, 7365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6237, 8192, 8237, 7182, 8013, 7757, 12514, 7404] avg=8192.0000 max=12514 max_rank=6 min=6237 min_rank=0 imbalance_ratio=1.527588 max_load_reduction=53022 max_load_reduction_pct=0.809052 moved_to_echo=35688 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6215, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 1, 0, 8185, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7038, 0, 9, 0, 1, 0, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8010, 0, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 7688, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4310, 1, 1, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1942, 1067, 766, 502, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3774, 0, 0, 0, 0, 0, 0, 7374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6331, 8194, 8238, 7055, 8099, 7697, 8774, 11148] avg=8192.0000 max=11148 max_rank=7 min=6331 min_rank=0 imbalance_ratio=1.360840 max_load_reduction=54388 max_load_reduction_pct=0.829895 moved_to_echo=35718 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 0, 0, 6240, 5524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 2, 0, 8118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7103, 0, 350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8009, 0, 0, 0, 0, 22, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7689, 0, 3, 1, 0, 0, 804, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1950, 1077, 766, 502, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 359, 0, 0, 0, 0, 0, 0, 7368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11777, 8192, 8170, 7454, 8034, 8498, 5682, 7729] avg=8192.0000 max=11777 max_rank=0 min=5682 min_rank=6 imbalance_ratio=1.437622 max_load_reduction=53759 max_load_reduction_pct=0.820297 moved_to_echo=29943 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6241, 6568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7109, 0, 39, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8011, 0, 0, 0, 0, 399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7690, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1136, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1951, 1078, 766, 502, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 7375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12809, 8192, 8243, 7149, 8410, 7729, 5615, 7389] avg=8192.0000 max=12809 max_rank=0 min=5615 min_rank=6 imbalance_ratio=1.563599 max_load_reduction=52727 max_load_reduction_pct=0.804550 moved_to_echo=29143 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6241, 376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 8111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7106, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8011, 0, 0, 0, 0, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7687, 0, 0, 0, 0, 0, 202, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6488, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1950, 1078, 766, 502, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 953, 0, 0, 0, 0, 0, 0, 7369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6618, 8192, 8162, 7108, 8259, 7890, 10985, 8322] avg=8192.0000 max=10985 max_rank=6 min=6618 min_rank=0 imbalance_ratio=1.340942 max_load_reduction=54551 max_load_reduction_pct=0.832382 moved_to_echo=35456 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 6209, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 20, 0, 0, 8162, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 5, 0, 0, 0, 8162, 0, 11, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7091, 0, 4851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 7968, 0, 0, 2, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 7647, 0, 0, 0, 13, 21, 1432, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 258, 0, 0, 0, 9, 0, 0, 0, 274, 0, 0, 0, 0, 16, 0, 0, 1946, 1081, 768, 505, 188, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 1, 0, 0, 0, 1321, 0, 0, 0, 0, 0, 1, 7357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6264, 8196, 8234, 11964, 7998, 9141, 5049, 8690] avg=8192.0000 max=11964 max_rank=3 min=5049 min_rank=6 imbalance_ratio=1.460449 max_load_reduction=53572 max_load_reduction_pct=0.817444 moved_to_echo=30878 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6220, 1853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 169, 0, 0, 0, 0, 0, 0, 7791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6918, 0, 3593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8008, 0, 0, 0, 0, 631, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7669, 0, 0, 0, 0, 0, 1011, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 18, 0, 0, 0, 0, 1517, 0, 0, 0, 0, 0, 0, 0, 1944, 1049, 766, 501, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 7375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8073, 8186, 8011, 10511, 8651, 8687, 6029, 7388] avg=8192.0000 max=10511 max_rank=3 min=6029 min_rank=6 imbalance_ratio=1.283081 max_load_reduction=55025 max_load_reduction_pct=0.839615 moved_to_echo=30755 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[27520, 128, 128, 128, 128, 0, 0, 0, 1664, 0, 0, 0, 0, 128, 0, 0, 16256, 9088, 5888, 3968, 1536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[27904, 128, 128, 128, 128, 0, 0, 0, 1920, 0, 0, 0, 0, 128, 0, 0, 15616, 8704, 6144, 4096, 1536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 0, 128, 0, 0, 128, 128, 128, 128, 128, 0, 49920, 15744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 56704, 0, 8960, 0, 128, 0, 0, 128, 128, 128, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 0, 0, 0, 128, 128, 0, 128, 128, 128, 128, 128, 128, 49280, 16384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 56192, 0, 9472, 0, 128, 0, 0, 128, 128, 128, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 0, 128, 0, 128, 0, 0, 0, 6272, 0, 0, 128, 0, 0, 128, 59392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 0, 128, 61568, 0, 128, 128, 128, 128, 3968, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 0, 0, 128, 0, 64128, 0, 0, 128, 128, 1536, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 0, 65536, 0, 128, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 0, 128, 0, 128, 0, 0, 0, 6656, 0, 0, 128, 0, 0, 128, 59008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 0, 0, 128, 61568, 0, 128, 128, 128, 128, 4096, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 0, 512, 0, 256, 0, 0, 128, 0, 128, 0, 65024, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 0, 65536, 0, 128, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 0, 0, 128, 0, 64128, 0, 0, 128, 128, 1536, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 12, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 5763, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 579, 8192, 0, 8188, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 11, 0, 0, 8182, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 2832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5353, 8192, 0, 8035, 0, 0, 0, 52, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 1, 8183, 0, 0, 0, 0, 17, 0, 0, 0, 0, 3, 0, 0, 0, 9, 0, 0, 6, 0, 8189, 0, 7420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 117, 85, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1348, 8191, 0, 7706, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8, 0, 7519, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7956, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 1, 20, 0, 0, 8149, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 8192, 0, 7144, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 13, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 8192, 0, 8178, 0, 1, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 8062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 28, 0, 8191, 0, 8184, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 9, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 154, 8190, 0, 8191, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8118, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 8168, 4, 3522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3, 0, 8, 5, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4621, 8190, 0, 8151, 0, 0, 0, 0, 8188, 0, 1, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 27, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 46, 6333, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 24, 0, 0, 0, 39, 3, 8137, 0, 8160, 0, 14, 0, 1, 25, 0, 0, 0, 0, 8, 0, 3, 1, 0, 0, 0, 0, 895, 0, 0, 0, 0, 0, 0, 40, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 90, 8125, 0, 8171, 0, 0, 0, 0, 8136, 12, 0, 25, 0, 0, 0, 0, 0, 2, 14, 0, 0, 0, 0, 0, 0, 0, 0, 827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 302, 0, 7883, 0, 0, 0, 3, 17, 0, 0, 0, 0, 0, 0, 18, 8164, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8192, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 306, 0, 0, 0, 0, 0, 0, 0, 433, 695, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7616, 8191, 0, 7186, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7925, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7, 0, 0, 7656, 0, 0, 420, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 12, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2796, 0, 5763, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1853, 0, 0, 0, 5395, 2190, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 579, 5592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 7952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8446, 8559, 9479, 6171, 8392, 7954, 8298, 8237] avg=8192.0000 max=9479 max_rank=2 min=6171 min_rank=3 imbalance_ratio=1.157104 max_load_reduction=56057 max_load_reduction_pct=0.855362 moved_to_echo=32881 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 11, 0, 0, 8182, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2796, 0, 2832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 52, 0, 0, 0, 5396, 2190, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5353, 5592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8035, 0, 0, 0, 52, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 7952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 0, 0, 1, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8437, 5628, 7767, 10945, 8291, 7968, 8314, 8186] avg=8192.0000 max=10945 max_rank=3 min=5628 min_rank=1 imbalance_ratio=1.336060 max_load_reduction=54591 max_load_reduction_pct=0.832993 moved_to_echo=32759 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 20, 0, 0, 8149, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 2796, 0, 7144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 13, 32, 0, 0, 0, 5396, 2190, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 5592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 1, 0, 0, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 0, 0, 0, 0, 0, 0, 7947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8182, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 2, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8414, 9949, 7669, 6625, 8383, 7952, 8306, 8238] avg=8192.0000 max=9949 max_rank=1 min=6625 min_rank=3 imbalance_ratio=1.214478 max_load_reduction=55587 max_load_reduction_pct=0.848190 moved_to_echo=32879 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 1, 8183, 0, 0, 0, 0, 17, 0, 0, 0, 0, 3, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 6, 0, 2795, 0, 7420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 117, 85, 0, 0, 0, 5394, 2190, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1348, 5591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7706, 0, 0, 0, 0, 204, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 580, 0, 0, 0, 0, 0, 0, 7951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8, 0, 7519, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7956, 1, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8447, 10230, 7823, 6940, 7911, 8531, 7651, 8003] avg=8192.0000 max=10230 max_rank=1 min=6940 min_rank=3 imbalance_ratio=1.248779 max_load_reduction=55306 max_load_reduction_pct=0.843903 moved_to_echo=32096 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 8062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 28, 0, 2796, 0, 8184, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 9, 0, 0, 0, 5395, 2189, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 154, 5591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 7949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8118, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8305, 11010, 7632, 5764, 8395, 7956, 8246, 8228] avg=8192.0000 max=11010 max_rank=1 min=5764 min_rank=3 imbalance_ratio=1.343994 max_load_reduction=54526 max_load_reduction_pct=0.832001 moved_to_echo=32825 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 2788, 4, 3522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3, 0, 8, 5, 0, 0, 0, 5380, 2189, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4621, 5591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 204, 0, 1, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 1, 0, 0, 0, 0, 0, 7948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 37, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 27, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8427, 6322, 7623, 10213, 8366, 7970, 8352, 8263] avg=8192.0000 max=10213 max_rank=3 min=6322 min_rank=1 imbalance_ratio=1.246704 max_load_reduction=55323 max_load_reduction_pct=0.844162 moved_to_echo=32951 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2796, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 306, 0, 0, 0, 0, 0, 0, 0, 433, 695, 0, 0, 0, 5396, 2190, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7616, 5591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7186, 0, 0, 0, 0, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 0, 0, 0, 0, 0, 0, 7949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7925, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7, 0, 0, 7656, 0, 0, 420, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8434, 2800, 9056, 13214, 7390, 8465, 8048, 8129] avg=8192.0000 max=13214 max_rank=3 min=2800 min_rank=1 imbalance_ratio=1.613037 max_load_reduction=52322 max_load_reduction_pct=0.798370 moved_to_echo=32032 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 46, 6333, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 39, 3, 2772, 0, 8160, 0, 14, 0, 1, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 3, 1, 0, 0, 0, 0, 895, 0, 0, 0, 5365, 2176, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 90, 5527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 198, 12, 0, 25, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 827, 0, 0, 0, 0, 0, 0, 7899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 302, 0, 7883, 0, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 17, 0, 0, 0, 0, 0, 0, 18, 8164, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6626, 11038, 8487, 5664, 8408, 8740, 8321, 8252] avg=8192.0000 max=11038 max_rank=1 min=5664 min_rank=3 imbalance_ratio=1.347412 max_load_reduction=54498 max_load_reduction_pct=0.831573 moved_to_echo=33721 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 8169, 4, 3500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3, 0, 11, 13, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4639, 8190, 0, 8152, 0, 0, 0, 0, 8188, 0, 1, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 27, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 1, 8184, 0, 0, 0, 0, 18, 0, 0, 0, 0, 3, 0, 0, 0, 9, 0, 0, 7, 0, 8190, 0, 7427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 107, 98, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1284, 8191, 0, 7710, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 0, 7533, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7973, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 8070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 30, 0, 8191, 0, 8166, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 27, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 143, 8190, 0, 8191, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8120, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 1, 21, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 8192, 0, 7042, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 13, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1098, 8192, 0, 8176, 0, 1, 0, 0, 8187, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 45, 6323, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 24, 0, 0, 0, 38, 2, 8139, 0, 8155, 0, 14, 0, 1, 25, 0, 0, 0, 0, 9, 0, 4, 0, 0, 0, 0, 0, 941, 1, 0, 0, 0, 0, 0, 42, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 83, 8125, 0, 8171, 0, 0, 0, 0, 8135, 11, 0, 25, 0, 0, 0, 0, 0, 1, 16, 0, 0, 0, 0, 0, 0, 0, 0, 806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 293, 0, 7886, 0, 0, 0, 5, 16, 0, 0, 0, 0, 0, 0, 19, 8163, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 13, 0, 0, 8181, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 2801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5380, 8192, 0, 8034, 0, 0, 0, 56, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8143, 0, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8192, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 275, 0, 0, 0, 0, 0, 0, 0, 396, 772, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7557, 8191, 0, 7158, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7930, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7, 0, 0, 7667, 0, 0, 433, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 11, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 5620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 577, 8192, 0, 8189, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 2827, 4, 3500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3, 0, 11, 13, 0, 0, 0, 5342, 2194, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4639, 5594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 0, 0, 0, 0, 207, 0, 1, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 1, 0, 0, 0, 0, 0, 7943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 35, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 27, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8428, 6337, 7603, 10234, 8370, 7959, 8348, 8257] avg=8192.0000 max=10234 max_rank=3 min=6337 min_rank=1 imbalance_ratio=1.249268 max_load_reduction=55302 max_load_reduction_pct=0.843842 moved_to_echo=32934 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 13, 0, 0, 8181, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2835, 0, 2801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 59, 0, 0, 0, 5357, 2194, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5380, 5595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8034, 0, 0, 0, 56, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 7947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8143, 0, 0, 2, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8437, 5636, 7731, 10975, 8297, 7963, 8312, 8185] avg=8192.0000 max=10975 max_rank=3 min=5636 min_rank=1 imbalance_ratio=1.339722 max_load_reduction=54561 max_load_reduction_pct=0.832535 moved_to_echo=32757 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 1, 8184, 0, 0, 0, 0, 18, 0, 0, 0, 0, 3, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 7, 0, 2835, 0, 7427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 107, 98, 0, 0, 0, 5355, 2194, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1284, 5595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7710, 0, 0, 0, 0, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 598, 0, 0, 0, 0, 0, 0, 7946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 0, 7533, 0, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7973, 1, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8449, 10278, 7793, 6880, 7917, 8544, 7660, 8015] avg=8192.0000 max=10278 max_rank=1 min=6880 min_rank=3 imbalance_ratio=1.254639 max_load_reduction=55258 max_load_reduction_pct=0.843170 moved_to_echo=32136 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 21, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 2835, 0, 7042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 13, 65, 0, 0, 0, 5357, 2194, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1098, 5595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 1, 0, 0, 207, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 7942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8182, 0, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 2, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8419, 9886, 7668, 6694, 8385, 7947, 8304, 8233] avg=8192.0000 max=9886 max_rank=1 min=6694 min_rank=3 imbalance_ratio=1.206787 max_load_reduction=55650 max_load_reduction_pct=0.849152 moved_to_echo=32869 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 45, 6323, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 38, 2, 2813, 0, 8155, 0, 14, 0, 1, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4, 0, 0, 0, 0, 0, 941, 1, 0, 0, 5326, 2182, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 83, 5531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 200, 11, 0, 25, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 806, 0, 0, 0, 0, 0, 0, 7895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 293, 0, 7886, 0, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 16, 0, 0, 0, 0, 0, 0, 19, 8163, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6613, 11072, 8503, 5664, 8408, 8717, 8315, 8244] avg=8192.0000 max=11072 max_rank=1 min=5664 min_rank=3 imbalance_ratio=1.351562 max_load_reduction=54464 max_load_reduction_pct=0.831055 moved_to_echo=33684 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 8070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 30, 0, 2835, 0, 8166, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 27, 0, 0, 0, 5356, 2194, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 143, 5594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 7944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8120, 0, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8312, 11033, 7618, 5757, 8398, 7949, 8247, 8222] avg=8192.0000 max=11033 max_rank=1 min=5757 min_rank=3 imbalance_ratio=1.346802 max_load_reduction=54503 max_load_reduction_pct=0.831650 moved_to_echo=32816 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2835, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 275, 0, 0, 0, 0, 0, 0, 0, 396, 772, 0, 0, 0, 5357, 2194, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7557, 5595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7158, 0, 0, 0, 0, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 564, 0, 0, 0, 0, 0, 0, 7944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7930, 0, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7, 0, 0, 7667, 0, 0, 433, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8433, 2839, 9032, 13160, 7365, 8508, 8051, 8148] avg=8192.0000 max=13160 max_rank=3 min=2839 min_rank=1 imbalance_ratio=1.606445 max_load_reduction=52376 max_load_reduction_pct=0.799194 moved_to_echo=32072 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 11, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2835, 0, 5620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1995, 0, 0, 0, 5356, 2194, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 577, 5595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 7947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8445, 8455, 9588, 6172, 8396, 7949, 8299, 8232] avg=8192.0000 max=9588 max_rank=2 min=6172 min_rank=3 imbalance_ratio=1.170410 max_load_reduction=55948 max_load_reduction_pct=0.853699 moved_to_echo=32876 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 0, 384, 128, 0, 128, 128, 0, 128, 128, 768, 3712, 0, 0, 0, 43136, 17536, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 0, 128, 128, 0, 128, 128, 128, 22400, 128, 43136, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 128, 128, 0, 0, 0, 0, 128, 0, 0, 0, 20864, 44672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 128, 0, 0, 128, 0, 0, 2048, 128, 0, 0, 0, 0, 0, 63616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 0, 0, 384, 128, 0, 128, 0, 0, 128, 128, 640, 4096, 128, 0, 0, 42880, 17664, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 128, 128, 0, 128, 128, 128, 22656, 128, 42752, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 128, 128, 128, 0, 0, 0, 128, 128, 0, 0, 0, 20864, 44800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 0, 128, 63488, 0, 0, 0, 0, 128, 0, 128, 128, 0, 128, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 63872, 0, 128, 0, 128, 1664, 128, 128, 128, 0, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 0, 128, 63488, 0, 0, 128, 0, 128, 0, 128, 128, 0, 128, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 0, 128, 128, 0, 128, 0, 0, 128, 0, 128, 384, 128, 64256, 128, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 128, 0, 0, 0, 0, 0, 2048, 128, 0, 0, 0, 0, 0, 63616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 0, 128, 128, 0, 128, 0, 0, 128, 0, 128, 384, 128, 64256, 128, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 128, 128, 128, 128, 0, 0, 128, 0, 128, 64768, 128, 0, 512, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 455, 0, 0, 7939, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8188, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 8003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 229, 1, 0, 7978, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 8123, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 206, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 354, 0, 0, 8160, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8164, 13, 0, 1, 0, 3, 0, 0, 32, 0, 667, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1463, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 3, 0, 122, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 2, 0, 0, 0, 8185, 0, 0, 46, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 0, 0, 0, 7697, 0, 0, 0, 3, 0, 0, 0, 0, 2, 13, 0, 0, 46, 0, 0, 8028, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 0, 8177, 0, 0, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8185, 0, 1, 0, 8190, 0, 1, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 1, 0, 0, 4, 32, 8, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 7, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 5969, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2594, 0, 0, 7670, 8192, 0, 0, 14, 0, 0, 0, 0, 1, 0, 0, 0, 0, 537, 0, 7189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 10, 0, 0, 8031, 10, 0, 0, 4, 0, 0, 0, 0, 0, 644, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 29, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8134, 0, 0, 3, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8090, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 8191, 1, 0, 7, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 47, 7, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 30, 11, 0, 0, 0, 0, 0, 343, 0, 0, 4, 88, 0, 0, 0, 0, 0, 0, 0, 8088, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 3, 1, 0, 0, 0, 0, 33, 0, 5, 0, 0, 0, 0, 2, 28, 0, 0, 0, 0, 0, 0, 6541, 0, 0, 0, 1, 0, 0, 0, 23, 0, 0, 0, 0, 1, 2, 0, 7976, 7946, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 493, 0, 8093, 0, 0, 139, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 17, 8056, 0, 0, 2, 8144, 0, 36, 0, 0, 0, 8142, 28, 0, 0, 0, 0, 0, 0, 22, 37, 1011, 118, 62] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7940, 0, 0, 0, 0, 0, 0, 0, 362, 276, 269, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 455, 0, 0, 7663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8193, 8192, 8873, 8118, 7923, 8201, 7879] avg=8192.0000 max=8873 max_rank=3 min=7879 min_rank=7 imbalance_ratio=1.083130 max_load_reduction=56663 max_load_reduction_pct=0.864609 moved_to_echo=32121 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7132, 0, 0, 0, 0, 0, 0, 0, 362, 284, 269, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 354, 0, 0, 7876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 7916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7802, 13, 0, 1, 0, 3, 0, 0, 32, 0, 667, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8159, 8193, 8193, 8073, 8230, 7966, 8202, 8520] avg=8192.0000 max=8520 max_rank=7 min=7966 min_rank=5 imbalance_ratio=1.040039 max_load_reduction=57016 max_load_reduction_pct=0.869995 moved_to_echo=32918 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6605, 0, 0, 0, 0, 0, 0, 0, 363, 285, 269, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1463, 0, 7922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7829, 0, 0, 0, 0, 0, 0, 0, 3, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8158, 8193, 8192, 7547, 7906, 9385, 8201, 7954] avg=8192.0000 max=9385 max_rank=5 min=7547 min_rank=3 imbalance_ratio=1.145630 max_load_reduction=56151 max_load_reduction_pct=0.856796 moved_to_echo=33446 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8183, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 8003, 0, 0, 0, 0, 0, 0, 0, 363, 277, 267, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 229, 1, 0, 7701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 7855, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8156, 8193, 8193, 8940, 7931, 7893, 8200, 8030] avg=8192.0000 max=8940 max_rank=3 min=7893 min_rank=5 imbalance_ratio=1.091309 max_load_reduction=56596 max_load_reduction_pct=0.863586 moved_to_echo=32054 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 8116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8185, 0, 0, 46, 14, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 0, 0, 0, 7697, 0, 0, 0, 3, 0, 0, 0, 362, 279, 269, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 13, 0, 0, 46, 0, 0, 7749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 0, 7907, 0, 0, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8185, 0, 1, 0, 9, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7819, 0, 0, 0, 0, 0, 1, 0, 0, 4, 32, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8153, 8248, 8190, 8641, 7810, 8432, 8198, 7864] avg=8192.0000 max=8641 max_rank=3 min=7810 min_rank=4 imbalance_ratio=1.054810 max_load_reduction=56895 max_load_reduction_pct=0.868149 moved_to_echo=32304 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 29, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8134, 0, 0, 3, 0, 0, 1, 0, 363, 281, 269, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 7922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 7828, 1, 0, 0, 0, 0, 0, 0, 0, 0, 47, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 8222, 8191, 9078, 7809, 7995, 8200, 7890] avg=8192.0000 max=9078 max_rank=3 min=7809 min_rank=4 imbalance_ratio=1.108154 max_load_reduction=56458 max_load_reduction_pct=0.861481 moved_to_echo=31894 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 7, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 5969, 0, 0, 17, 0, 0, 0, 0, 356, 267, 236, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2594, 0, 0, 7403, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 537, 0, 6952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1, 0, 0, 0, 0, 8192, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 7675, 10, 0, 0, 4, 0, 0, 0, 0, 0, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8202, 8192, 6871, 10011, 7490, 8215, 8343] avg=8192.0000 max=10011 max_rank=4 min=6871 min_rank=3 imbalance_ratio=1.222046 max_load_reduction=55525 max_load_reduction_pct=0.847244 moved_to_echo=34059 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 30, 11, 0, 0, 0, 0, 0, 343, 0, 0, 4, 88, 0, 0, 0, 7914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8088, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 1, 0, 0, 0, 0, 33, 0, 5, 0, 0, 0, 0, 8142, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 28, 0, 0, 0, 0, 0, 0, 6541, 0, 0, 0, 1, 0, 0, 0, 364, 282, 271, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 1, 2, 0, 7694, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 0, 7820, 0, 0, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 17, 8056, 0, 0, 2, 2, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7778, 28, 0, 0, 0, 0, 0, 0, 22, 37, 1011, 118, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8390, 8092, 8193, 7513, 7721, 8452, 8119, 9056] avg=8192.0000 max=9056 max_rank=7 min=7513 min_rank=3 imbalance_ratio=1.105469 max_load_reduction=56480 max_load_reduction_pct=0.861816 moved_to_echo=33348 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8189, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 7914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 214, 1, 0, 7978, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 8126, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 305, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 28, 14, 0, 0, 0, 0, 0, 329, 0, 0, 3, 88, 0, 0, 0, 0, 0, 0, 0, 8090, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 33, 0, 6, 0, 0, 0, 0, 2, 27, 0, 0, 0, 0, 0, 0, 6496, 0, 0, 0, 2, 0, 0, 0, 23, 0, 0, 0, 0, 0, 2, 0, 7976, 7956, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 466, 0, 8088, 0, 0, 135, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 15, 8059, 0, 0, 1, 8145, 0, 35, 0, 0, 0, 8143, 26, 0, 0, 0, 0, 0, 0, 26, 37, 1098, 113, 61] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 350, 0, 0, 8154, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8165, 14, 0, 0, 0, 2, 0, 0, 37, 0, 677, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 445, 0, 0, 7916, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 25, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8146, 0, 0, 3, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8087, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 8191, 2, 0, 6, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 49, 6, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 2, 0, 0, 0, 8186, 0, 0, 40, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 0, 0, 0, 7719, 0, 0, 0, 4, 0, 0, 0, 0, 2, 13, 0, 0, 38, 0, 0, 8027, 8147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 0, 8177, 0, 0, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8184, 0, 1, 0, 8190, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 1, 0, 0, 2, 42, 12, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 4, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 5920, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2598, 0, 0, 7650, 8192, 0, 0, 14, 0, 0, 0, 0, 1, 0, 0, 0, 0, 551, 0, 7162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 1, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 10, 0, 0, 8014, 13, 0, 0, 3, 0, 0, 0, 0, 0, 739, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1560, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 2, 0, 127, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7974, 0, 0, 0, 0, 0, 0, 0, 399, 275, 263, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 445, 0, 0, 7651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8159, 8194, 8193, 8934, 8096, 7917, 8201, 7842] avg=8192.0000 max=8934 max_rank=3 min=7842 min_rank=7 imbalance_ratio=1.090576 max_load_reduction=56602 max_load_reduction_pct=0.863678 moved_to_echo=32056 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 25, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8146, 0, 0, 3, 0, 0, 1, 0, 400, 275, 268, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 7917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 7791, 1, 0, 0, 0, 0, 0, 0, 0, 0, 49, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 8218, 8190, 9119, 7818, 7983, 8204, 7853] avg=8192.0000 max=9119 max_rank=3 min=7818 min_rank=4 imbalance_ratio=1.113159 max_load_reduction=56417 max_load_reduction_pct=0.860855 moved_to_echo=31858 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7123, 0, 0, 0, 0, 0, 0, 0, 399, 275, 271, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 350, 0, 0, 7881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 7911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7766, 14, 0, 0, 0, 2, 0, 0, 37, 0, 677, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8195, 8193, 8092, 8231, 7963, 8202, 8499] avg=8192.0000 max=8499 max_rank=7 min=7963 min_rank=5 imbalance_ratio=1.037476 max_load_reduction=57037 max_load_reduction_pct=0.870316 moved_to_echo=32895 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8183, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 7914, 0, 0, 0, 0, 0, 0, 0, 400, 273, 265, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 214, 1, 0, 7711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 7853, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8158, 8195, 8193, 8881, 7926, 7892, 8199, 8092] avg=8192.0000 max=8881 max_rank=3 min=7892 min_rank=5 imbalance_ratio=1.084106 max_load_reduction=56655 max_load_reduction_pct=0.864487 moved_to_echo=32109 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 28, 14, 0, 0, 0, 0, 0, 329, 0, 0, 3, 88, 0, 0, 0, 7926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8090, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 33, 0, 6, 0, 0, 0, 0, 8144, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 27, 0, 0, 0, 0, 0, 0, 6496, 0, 0, 0, 2, 0, 0, 0, 401, 275, 268, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 2, 0, 7704, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 466, 0, 7813, 0, 0, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 15, 8059, 0, 0, 1, 1, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7742, 26, 0, 0, 0, 0, 0, 0, 26, 37, 1098, 113, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8388, 8096, 8195, 7493, 7730, 8414, 8117, 9103] avg=8192.0000 max=9103 max_rank=7 min=7493 min_rank=3 imbalance_ratio=1.111206 max_load_reduction=56433 max_load_reduction_pct=0.861099 moved_to_echo=33364 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 8115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8186, 0, 0, 40, 14, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 0, 0, 0, 7719, 0, 0, 0, 4, 0, 0, 0, 400, 275, 266, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 13, 0, 0, 38, 0, 0, 7759, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 0, 7902, 0, 0, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8184, 0, 1, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7781, 0, 0, 0, 0, 0, 1, 0, 0, 2, 42, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8148, 8244, 8190, 8692, 7812, 8416, 8196, 7838] avg=8192.0000 max=8692 max_rank=3 min=7812 min_rank=4 imbalance_ratio=1.061035 max_load_reduction=56844 max_load_reduction_pct=0.867371 moved_to_echo=32262 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 4, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 5920, 0, 0, 21, 0, 0, 0, 0, 391, 240, 254, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2598, 0, 0, 7394, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 551, 0, 6922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 1, 0, 0, 0, 0, 8192, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 7623, 13, 0, 0, 3, 0, 0, 0, 0, 0, 739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8200, 8191, 6851, 10006, 7474, 8216, 8388] avg=8192.0000 max=10006 max_rank=4 min=6851 min_rank=3 imbalance_ratio=1.221436 max_load_reduction=55530 max_load_reduction_pct=0.847321 moved_to_echo=34084 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6504, 0, 0, 0, 0, 0, 0, 0, 400, 275, 272, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1560, 0, 7917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7792, 0, 0, 0, 0, 0, 0, 0, 2, 0, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8194, 8191, 7474, 7917, 9477, 8201, 7921] avg=8192.0000 max=9477 max_rank=5 min=7474 min_rank=3 imbalance_ratio=1.156860 max_load_reduction=56059 max_load_reduction_pct=0.855392 moved_to_echo=33516 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 128, 128, 0, 0, 0, 58112, 0, 0, 128, 128, 0, 128, 0, 2944, 2304, 2176, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 0, 62464, 128, 0, 128, 128, 128, 128, 0, 128, 128, 2816, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 0, 0, 3200, 0, 62336, 128, 0, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 0, 3712, 128, 0, 61824, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 0, 0, 65536, 128, 0, 128, 128, 0, 128, 128, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 128, 128, 128, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 128, 128, 0, 0, 0, 57856, 0, 0, 128, 128, 0, 128, 0, 3200, 2176, 2176, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 62080, 128, 0, 0, 128, 128, 128, 0, 128, 128, 3200, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 0, 0, 3328, 0, 62208, 128, 0, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 0, 0, 3712, 128, 0, 61952, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 0, 128, 128, 128, 0, 384, 0, 0, 128, 128, 0, 128, 0, 65024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 128, 128, 128, 0, 0, 128, 65408, 0, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 0, 0, 65536, 128, 0, 128, 128, 0, 128, 128, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 8186, 0, 0, 0, 0, 0, 1, 8189, 537, 0, 0, 0, 2, 0, 0, 0, 0, 7918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8173, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 81, 0, 0, 8160, 0, 7854, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 4, 0, 0, 0, 0, 15, 0, 8180, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 32, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8192, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 3, 0, 0, 0, 0, 0, 0, 7844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 8149, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8185, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 37, 8188, 0, 0, 0, 0, 2, 0, 8191, 11, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 8190, 0, 7624, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 19, 0, 8189, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 7883, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7855, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 7, 0, 0, 0, 0, 0, 0, 8189, 0, 6, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8188, 37, 0, 0, 0, 0, 0, 0, 15, 8, 0, 0, 0, 0, 8154, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1, 8188, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[23, 0, 0, 8136, 0, 0, 0, 0, 0, 4, 7432, 850, 11, 0, 0, 0, 0, 0, 0, 0, 7794, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 1, 8103, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 10, 8061, 0, 8153, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 1, 24, 39, 0, 0, 0, 4, 0, 8144, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 13, 0, 264, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8134, 85, 0, 0, 16, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 2, 0, 0, 0, 0, 10, 0, 28, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 13, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 8192, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8, 0, 0, 2, 0, 0, 0, 2, 0, 0, 8185, 0, 8100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 46, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8098, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 28, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 8110, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8103, 7976, 8172, 8326, 8200, 8339, 8226, 8194] avg=8192.0000 max=8339 max_rank=5 min=7976 min_rank=1 imbalance_ratio=1.017944 max_load_reduction=57197 max_load_reduction_pct=0.872757 moved_to_echo=32959 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 28, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8109, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8096, 8324, 8174, 8340, 8200, 8077, 8140, 8185] avg=8192.0000 max=8340 max_rank=3 min=8077 min_rank=5 imbalance_ratio=1.018066 max_load_reduction=57196 max_load_reduction_pct=0.872742 moved_to_echo=32602 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8095, 537, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 94, 28, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 81, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7854, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 4, 0, 0, 0, 0, 15, 0, 8063, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 32, 0, 0, 0, 1, 8092, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8635, 8051, 8224, 8025, 8197, 8089, 8133, 8182] avg=8192.0000 max=8635 max_rank=0 min=8025 min_rank=3 imbalance_ratio=1.054077 max_load_reduction=56901 max_load_reduction_pct=0.868240 moved_to_echo=32601 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 37, 0, 0, 0, 0, 0, 2, 0, 8097, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 28, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7624, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 89, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 8051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 8109, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8147, 8292, 8175, 7798, 8200, 8612, 8129, 8183] avg=8192.0000 max=8612 max_rank=5 min=7798 min_rank=3 imbalance_ratio=1.051270 max_load_reduction=56924 max_load_reduction_pct=0.868591 moved_to_echo=33124 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8098, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 28, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 2, 0, 0, 0, 2, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 46, 8074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8110, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8102, 8286, 8179, 8271, 8200, 8196, 8117, 8185] avg=8192.0000 max=8286 max_rank=1 min=8102 min_rank=0 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32698 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 8097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 27, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7883, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 447, 0, 0, 0, 0, 0, 0, 8104, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8116, 8323, 8339, 8053, 8176, 7756, 8577, 8196] avg=8192.0000 max=8577 max_rank=6 min=7756 min_rank=5 imbalance_ratio=1.046997 max_load_reduction=56959 max_load_reduction_pct=0.869125 moved_to_echo=32705 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8098, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 6, 0, 1, 0, 0, 0, 0, 0, 0, 0, 94, 28, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 15, 8, 0, 0, 0, 0, 8136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 89, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1, 8071, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8106, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8105, 8328, 8196, 8324, 8200, 8083, 8112, 8188] avg=8192.0000 max=8328 max_rank=1 min=8083 min_rank=5 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=32583 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[23, 0, 0, 0, 0, 0, 0, 0, 0, 4, 7344, 850, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7794, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 1, 88, 33, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 10, 8037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 1, 24, 39, 0, 94, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8144, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 264, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8007, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 2, 0, 0, 0, 8021, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 28, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 13, 0, 8129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8232, 7956, 8077, 8399, 8163, 8384, 8102, 8223] avg=8192.0000 max=8399 max_rank=3 min=7956 min_rank=1 imbalance_ratio=1.025269 max_load_reduction=57137 max_load_reduction_pct=0.871841 moved_to_echo=32872 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 2, 0, 0, 0, 0, 0, 0, 7864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8190, 0, 8146, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8183, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 4, 0, 0, 0, 0, 0, 0, 8189, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 37, 0, 0, 0, 0, 0, 0, 13, 12, 0, 0, 0, 0, 8155, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1, 8188, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 8192, 5, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8, 0, 0, 2, 0, 0, 0, 5, 0, 0, 8185, 0, 8095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 51, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 33, 8189, 0, 0, 0, 0, 2, 0, 8191, 11, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8190, 0, 7605, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 569, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 14, 0, 8186, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 7892, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7864, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 8186, 0, 0, 0, 0, 0, 1, 8189, 507, 0, 0, 0, 2, 0, 0, 0, 0, 7955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 8159, 0, 7848, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 0, 0, 0, 4, 0, 0, 0, 0, 14, 0, 8181, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 34, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[24, 0, 0, 8139, 0, 0, 0, 0, 0, 4, 7427, 851, 11, 0, 0, 0, 0, 0, 0, 0, 7809, 0, 0, 0, 0, 0, 0, 29, 1, 0, 0, 0, 8104, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 14, 8056, 0, 8149, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 1, 24, 39, 0, 0, 0, 3, 0, 8146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 255, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8127, 86, 0, 0, 19, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3, 0, 29, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 10, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8192, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8104, 3, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 25, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8146, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 8109, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8109, 7986, 8174, 8326, 8200, 8324, 8223, 8194] avg=8192.0000 max=8326 max_rank=3 min=7986 min_rank=1 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32941 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8104, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 25, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 13, 12, 0, 0, 0, 0, 8138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 92, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1, 8071, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8105, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8108, 8319, 8200, 8328, 8201, 8080, 8110, 8190] avg=8192.0000 max=8328 max_rank=3 min=8080 min_rank=5 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=32581 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8104, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 25, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 2, 0, 0, 0, 5, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 51, 8074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8109, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8109, 8276, 8183, 8270, 8200, 8197, 8115, 8186] avg=8192.0000 max=8276 max_rank=1 min=8109 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32698 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 33, 0, 0, 0, 0, 0, 2, 0, 8103, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 24, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7605, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 91, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 569, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 8052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 8108, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8149, 8284, 8175, 7783, 8200, 8632, 8128, 8185] avg=8192.0000 max=8632 max_rank=5 min=7783 min_rank=3 imbalance_ratio=1.053711 max_load_reduction=56904 max_load_reduction_pct=0.868286 moved_to_echo=33145 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 8104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 24, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7892, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 445, 0, 0, 0, 0, 0, 0, 8106, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8118, 8313, 8337, 8065, 8176, 7758, 8574, 8195] avg=8192.0000 max=8574 max_rank=6 min=7758 min_rank=5 imbalance_ratio=1.046631 max_load_reduction=56962 max_load_reduction_pct=0.869171 moved_to_echo=32703 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[24, 0, 0, 0, 0, 0, 0, 0, 0, 4, 7340, 851, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7809, 0, 0, 0, 0, 0, 0, 29, 1, 0, 0, 0, 87, 26, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 14, 8029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 1, 24, 39, 0, 94, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 255, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8007, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 2, 0, 0, 0, 8021, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 29, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 10, 0, 8133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 7966, 8072, 8399, 8162, 8377, 8112, 8218] avg=8192.0000 max=8399 max_rank=3 min=7966 min_rank=1 imbalance_ratio=1.025269 max_load_reduction=57137 max_load_reduction_pct=0.871841 moved_to_echo=32869 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8101, 507, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 88, 25, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7848, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 0, 0, 0, 4, 0, 0, 0, 0, 14, 0, 8065, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 34, 0, 0, 0, 0, 8092, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8611, 8078, 8220, 8024, 8197, 8091, 8133, 8182] avg=8192.0000 max=8611 max_rank=0 min=8024 min_rank=3 imbalance_ratio=1.051147 max_load_reduction=56925 max_load_reduction_pct=0.868607 moved_to_echo=32603 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 25, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8108, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8102, 8314, 8175, 8341, 8200, 8077, 8141, 8186] avg=8192.0000 max=8341 max_rank=3 min=8077 min_rank=5 imbalance_ratio=1.018188 max_load_reduction=57195 max_load_reduction_pct=0.872726 moved_to_echo=32604 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[64128, 0, 0, 128, 0, 0, 128, 0, 0, 128, 0, 0, 128, 128, 128, 0, 768, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[64128, 0, 0, 128, 0, 128, 128, 0, 0, 128, 128, 0, 128, 128, 128, 0, 768, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 0, 64640, 0, 128, 0, 0, 0, 0, 128, 128, 0, 128, 0, 768, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 1024, 384, 128, 0, 0, 128, 128, 0, 128, 0, 128, 128, 64256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 0, 64512, 0, 128, 0, 128, 0, 0, 128, 0, 0, 128, 128, 768, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 0, 0, 0, 128, 0, 128, 128, 512, 0, 128, 128, 0, 0, 128, 64768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 0, 0, 0, 0, 0, 128, 128, 64128, 1408, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 0, 256, 128, 128, 128, 128, 128, 0, 128, 0, 128, 65280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 128, 128, 0, 128, 0, 0, 128, 128, 0, 128, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 128, 0, 65536, 128, 128, 0, 0, 0, 0, 0, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 0, 0, 128, 0, 128, 128, 512, 0, 128, 128, 0, 0, 0, 64768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 0, 65536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 0, 0, 256, 128, 0, 128, 128, 128, 0, 128, 0, 128, 65280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 6831, 262, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 7, 0, 1, 0, 0, 0, 0, 7876, 0, 589, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 2, 0, 0, 0, 8, 0, 0, 8191, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6386, 0, 4, 209, 0, 0, 1, 0, 0, 0, 0, 2355, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 6981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 8190, 0, 0, 0, 0, 0, 1, 0, 8041, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 1, 12, 0, 1, 8188, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7787, 2, 395, 0, 0, 0, 4, 0, 0, 0, 0, 1352, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8183] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 6565, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6863, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 2611, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 7440, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7712, 0, 538, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 6946, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8106, 0, 1, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 5, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7346, 0, 3, 5, 0, 0, 0, 0, 0, 0, 0, 2064, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8084] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 7220, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 105, 23, 0, 0, 0, 0, 0, 0, 7924, 0, 397, 1, 0, 0, 0, 0, 8183, 3, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 2, 0, 0, 8192, 0, 0, 0, 2, 0, 0, 0, 1, 0, 8116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7297, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[22, 0, 0, 2107, 30, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 9, 0, 0, 0, 0, 17, 6, 0, 0, 6, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 229, 0, 0, 0, 2, 0, 21, 7613, 0, 0, 50, 0, 0, 0, 0, 7839, 0, 0, 1, 0, 0, 0, 8146, 0, 0, 0, 0, 0, 0, 0, 0, 8135, 0, 0, 0, 0, 0, 2056, 0, 26, 0, 8145, 36, 0, 0, 0, 0, 0, 12, 0, 1, 0, 0, 0, 0, 24, 7387, 28, 8, 0, 0, 59, 0, 0, 34, 10, 27, 5257, 9, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4, 0, 8106] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 2986, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 5552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6566, 0, 839, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 5, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2037, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 6565, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1220, 707, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7366, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5643, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 2611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8715, 7755, 8114, 8291, 8199, 7969, 8284, 8209] avg=8192.0000 max=8715 max_rank=0 min=7755 min_rank=1 imbalance_ratio=1.063843 max_load_reduction=56821 max_load_reduction_pct=0.867020 moved_to_echo=32661 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 6831, 262, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1135, 707, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7361, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 7, 0, 1, 0, 0, 8104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7876, 0, 589, 0, 0, 0, 0, 0, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 7969, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5251, 0, 4, 209, 0, 0, 1, 0, 0, 0, 0, 2355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9147, 7481, 8114, 8583, 8202, 7979, 7820, 8210] avg=8192.0000 max=9147 max_rank=0 min=7481 min_rank=1 imbalance_ratio=1.116577 max_load_reduction=56389 max_load_reduction_pct=0.860428 moved_to_echo=32211 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 7440, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1442, 707, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7366, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7712, 0, 538, 0, 0, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6672, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9892, 7934, 8202, 8369, 8201, 7969, 6764, 8205] avg=8192.0000 max=9892 max_rank=0 min=6764 min_rank=6 imbalance_ratio=1.207520 max_load_reduction=55644 max_load_reduction_pct=0.849060 moved_to_echo=31139 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 6981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1384, 694, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 1, 0, 0, 0, 0, 0, 1, 0, 0, 7230, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8190, 0, 0, 0, 0, 0, 1, 0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 12, 0, 1, 7966, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6403, 2, 395, 0, 0, 0, 4, 0, 0, 0, 0, 1352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8183, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9268, 7306, 8112, 8309, 8198, 7983, 8156, 8204] avg=8192.0000 max=9268 max_rank=0 min=7306 min_rank=1 imbalance_ratio=1.131348 max_load_reduction=56268 max_load_reduction_pct=0.858582 moved_to_echo=32541 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 6946, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1306, 707, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7364, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8106, 0, 1, 0, 0, 0, 0, 0, 119, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 5, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6040, 0, 3, 5, 0, 0, 0, 0, 0, 0, 0, 2064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8084, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9341, 7435, 8109, 8227, 8203, 7968, 8112, 8141] avg=8192.0000 max=9341 max_rank=0 min=7435 min_rank=1 imbalance_ratio=1.140259 max_load_reduction=56195 max_load_reduction_pct=0.857468 moved_to_echo=32424 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 7220, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1297, 706, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7358, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 105, 23, 0, 0, 0, 0, 8110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7924, 0, 397, 1, 0, 0, 0, 0, 119, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 2, 0, 0, 8192, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6000, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9698, 7431, 8239, 8444, 8205, 7897, 7421, 8201] avg=8192.0000 max=9698 max_rank=0 min=7421 min_rank=6 imbalance_ratio=1.183838 max_load_reduction=55838 max_load_reduction_pct=0.852020 moved_to_echo=31724 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2986, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1171, 707, 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7366, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6566, 0, 839, 0, 0, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 5, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5069, 12989, 8344, 7524, 8193, 7725, 7453, 8239] avg=8192.0000 max=12989 max_rank=1 min=5069 min_rank=0 imbalance_ratio=1.585571 max_load_reduction=52547 max_load_reduction_pct=0.801804 moved_to_echo=31610 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[22, 0, 0, 2107, 30, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 1317, 682, 214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 17, 6, 0, 0, 6, 0, 0, 39, 0, 0, 7053, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 229, 0, 0, 0, 2, 8071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 7613, 0, 0, 50, 0, 0, 0, 0, 104, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2056, 0, 26, 0, 7915, 36, 0, 0, 0, 0, 0, 12, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 6070, 28, 8, 0, 0, 59, 0, 0, 34, 10, 27, 5257, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4, 0, 8106, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4406, 7205, 8302, 7789, 8135, 10046, 11526, 8127] avg=8192.0000 max=11526 max_rank=6 min=4406 min_rank=0 imbalance_ratio=1.406982 max_load_reduction=54010 max_load_reduction_pct=0.824127 moved_to_echo=37834 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 7027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 8190, 0, 0, 0, 0, 0, 1, 0, 8051, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 1, 11, 0, 1, 8187, 1, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 7720, 2, 413, 0, 0, 0, 4, 0, 0, 0, 0, 1344, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8183] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 7459, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7699, 0, 537, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8102, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 7022, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8108, 0, 1, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 5, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7197, 0, 3, 7, 0, 0, 0, 0, 0, 0, 0, 2134, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8059] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 3147, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 5477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6542, 0, 846, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 5, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2005, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 7221, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 23, 0, 0, 0, 0, 0, 0, 7921, 0, 389, 1, 0, 0, 0, 0, 8185, 3, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 1, 0, 0, 8192, 0, 0, 0, 2, 0, 0, 0, 1, 0, 8116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7291, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 6842, 279, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 7, 0, 1, 0, 0, 0, 0, 7869, 0, 592, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 0, 6, 0, 0, 8191, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6426, 0, 4, 211, 0, 0, 2, 0, 0, 0, 0, 2296, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 6653, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6739, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 2662, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[23, 0, 0, 2459, 22, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 10, 0, 0, 0, 0, 16, 8, 0, 0, 4, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 0, 0, 0, 2, 0, 24, 7540, 0, 0, 49, 0, 0, 0, 0, 7864, 0, 0, 1, 0, 0, 0, 8146, 0, 0, 0, 0, 0, 0, 0, 0, 8135, 0, 0, 0, 0, 0, 2041, 0, 26, 0, 8145, 35, 0, 0, 0, 0, 0, 12, 0, 1, 0, 0, 0, 0, 24, 7415, 27, 11, 0, 0, 58, 0, 0, 35, 11, 23, 4936, 9, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 8106] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 7459, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1367, 691, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7368, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7699, 0, 537, 0, 0, 0, 0, 0, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6735, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9803, 7950, 8199, 8369, 8200, 7973, 6833, 8209] avg=8192.0000 max=9803 max_rank=0 min=6833 min_rank=6 imbalance_ratio=1.196655 max_load_reduction=55733 max_load_reduction_pct=0.850418 moved_to_echo=31215 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 7221, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1230, 690, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7362, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 23, 0, 0, 0, 0, 8108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7921, 0, 389, 1, 0, 0, 0, 0, 133, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 1, 0, 0, 8192, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6061, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9601, 7439, 8239, 8447, 8203, 7900, 7502, 8205] avg=8192.0000 max=9601 max_rank=0 min=7439 min_rank=1 imbalance_ratio=1.171997 max_load_reduction=55935 max_load_reduction_pct=0.853500 moved_to_echo=31810 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 7022, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1214, 690, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7366, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8108, 0, 1, 0, 0, 0, 0, 0, 134, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 5, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5983, 0, 3, 7, 0, 0, 0, 0, 0, 0, 0, 2134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8059, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9325, 7440, 8107, 8244, 8202, 7971, 8127, 8120] avg=8192.0000 max=9325 max_rank=0 min=7440 min_rank=1 imbalance_ratio=1.138306 max_load_reduction=56211 max_load_reduction_pct=0.857712 moved_to_echo=32420 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 7027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1303, 679, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 1, 0, 0, 0, 0, 0, 1, 0, 0, 7241, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8190, 0, 0, 0, 0, 0, 1, 0, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 11, 0, 1, 7968, 1, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6417, 2, 413, 0, 0, 0, 4, 0, 0, 0, 0, 1344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8183, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9211, 7321, 8111, 8324, 8197, 7985, 8180, 8207] avg=8192.0000 max=9211 max_rank=0 min=7321 min_rank=1 imbalance_ratio=1.124390 max_load_reduction=56325 max_load_reduction_pct=0.859451 moved_to_echo=32569 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 6653, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1137, 691, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7368, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5602, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 2662, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8695, 7747, 8110, 8303, 8200, 7972, 8297, 8212] avg=8192.0000 max=8695 max_rank=0 min=7747 min_rank=1 imbalance_ratio=1.061401 max_load_reduction=56841 max_load_reduction_pct=0.867325 moved_to_echo=32681 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3147, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1105, 691, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7368, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6542, 0, 846, 0, 0, 0, 0, 0, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 5, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5140, 12919, 8352, 7521, 8193, 7720, 7450, 8241] avg=8192.0000 max=12919 max_rank=1 min=5140 min_rank=0 imbalance_ratio=1.577026 max_load_reduction=52617 max_load_reduction_pct=0.802872 moved_to_echo=31604 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 6842, 279, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1084, 690, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7363, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 7, 0, 1, 0, 0, 8101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7869, 0, 592, 0, 0, 0, 0, 0, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 7972, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5342, 0, 4, 211, 0, 0, 2, 0, 0, 0, 0, 2296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9100, 7485, 8111, 8594, 8201, 7980, 7855, 8210] avg=8192.0000 max=9100 max_rank=0 min=7485 min_rank=1 imbalance_ratio=1.110840 max_load_reduction=56436 max_load_reduction_pct=0.861145 moved_to_echo=32246 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[23, 0, 0, 2459, 22, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 1257, 666, 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 16, 8, 0, 0, 4, 0, 0, 43, 0, 0, 7078, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 0, 0, 0, 2, 8065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 7540, 0, 0, 49, 0, 0, 0, 0, 120, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 8135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2041, 0, 26, 0, 7920, 35, 0, 0, 0, 0, 0, 12, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 6158, 27, 11, 0, 0, 58, 0, 0, 35, 11, 23, 4936, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 8106, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4661, 7235, 8307, 7734, 8140, 10035, 11292, 8132] avg=8192.0000 max=11292 max_rank=6 min=4661 min_rank=0 imbalance_ratio=1.378418 max_load_reduction=54244 max_load_reduction_pct=0.827698 moved_to_echo=37599 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 47104, 896, 0, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 10368, 5632, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 6528, 0, 0, 128, 128, 0, 0, 128, 0, 0, 128, 0, 0, 58496, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 0, 128, 47616, 128, 512, 256, 0, 128, 128, 0, 128, 128, 128, 17280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 62208, 0, 2432, 128, 0, 0, 128, 0, 1024, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[2176, 128, 128, 128, 63488, 128, 128, 0, 0, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 65536, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 128, 0, 0, 0, 0, 128, 0, 0, 0, 128, 128, 0, 65408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 47872, 896, 0, 0, 0, 0, 0, 0, 128, 0, 128, 0, 128, 9728, 5504, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 128, 47744, 128, 512, 256, 0, 128, 128, 0, 128, 128, 128, 17024, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 6400, 0, 0, 128, 128, 0, 0, 128, 0, 0, 128, 0, 0, 58624, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[2048, 128, 128, 128, 63488, 128, 128, 0, 0, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 62080, 0, 2432, 128, 0, 0, 128, 0, 1152, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 128, 128, 128, 384, 0, 0, 128, 0, 128, 384, 0, 128, 0, 128, 64896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 65536, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 128, 0, 0, 0, 0, 128, 0, 0, 0, 128, 128, 0, 65280, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 11, 0, 4, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 51, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8172, 0, 0, 0, 8190, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 8187, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 8191, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8183, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8150, 0, 0, 0, 8191, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 11, 0, 1, 0, 0, 0, 0, 0, 94, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8094, 0, 0, 0, 8189, 0, 0, 8, 8130, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 8, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 19, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 8184, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7841, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 367, 25, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8126, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8097, 0, 0, 0, 8180, 0, 0, 0, 8187, 0, 0, 0, 85, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 88, 0, 0, 0, 0, 8133, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 337, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 7945, 0, 0, 41, 0, 11, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 47, 8142, 0, 2, 0, 0, 0, 19, 2, 0, 0, 0, 0, 0, 0, 3, 0, 8159, 2, 0, 0, 8097, 0, 0, 0, 8106, 0, 0, 3, 0, 25, 0, 43, 0, 0, 0, 0, 0, 8, 0, 0, 7847, 0, 24, 0, 0, 0, 0, 0, 19, 0, 3, 34, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8042, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8246, 8054, 8213, 8197, 8227, 8194, 8229] avg=8192.0000 max=8246 max_rank=1 min=8054 min_rank=2 imbalance_ratio=1.006592 max_load_reduction=57290 max_load_reduction_pct=0.874176 moved_to_echo=32847 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 11, 0, 4, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 8041, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 5, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 7, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8172, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8217, 8120, 8202, 8200, 8211, 8168, 8226] avg=8192.0000 max=8226 max_rank=7 min=8120 min_rank=2 imbalance_ratio=1.004150 max_load_reduction=57310 max_load_reduction_pct=0.874481 moved_to_echo=32805 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8043, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8241, 8084, 8213, 8167, 8230, 8194, 8229] avg=8192.0000 max=8241 max_rank=1 min=8084 min_rank=2 imbalance_ratio=1.005981 max_load_reduction=57295 max_load_reduction_pct=0.874252 moved_to_echo=32820 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8042, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8150, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8245, 8102, 8214, 8191, 8189, 8189, 8229] avg=8192.0000 max=8245 max_rank=1 min=8102 min_rank=2 imbalance_ratio=1.006470 max_load_reduction=57291 max_load_reduction_pct=0.874191 moved_to_echo=32798 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 11, 0, 1, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 8040, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8094, 0, 0, 0, 38, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8130, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8235, 8210, 8208, 8198, 8141, 8136, 8229] avg=8192.0000 max=8235 max_rank=1 min=8136 min_rank=6 imbalance_ratio=1.005249 max_load_reduction=57301 max_load_reduction_pct=0.874344 moved_to_echo=32704 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 8, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 8035, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 10, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 19, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8240, 8066, 8223, 8213, 8216, 8192, 8197] avg=8192.0000 max=8240 max_rank=1 min=8066 min_rank=2 imbalance_ratio=1.005859 max_load_reduction=57296 max_load_reduction_pct=0.874268 moved_to_echo=32818 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7841, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 367, 25, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8031, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8126, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8097, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 85, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 7923, 8440, 8224, 8133, 8135, 8275, 8229] avg=8192.0000 max=8440 max_rank=2 min=7923 min_rank=1 imbalance_ratio=1.030273 max_load_reduction=57096 max_load_reduction_pct=0.871216 moved_to_echo=32772 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 88, 0, 0, 0, 8130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 337, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 7955, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 7945, 0, 0, 41, 0, 11, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 47, 1, 0, 2, 0, 0, 0, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8159, 2, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8106, 0, 0, 3, 0, 25, 0, 43, 0, 0, 0, 0, 0, 8, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7847, 0, 24, 0, 0, 0, 0, 0, 19, 0, 3, 34, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8268, 8189, 8460, 8039, 8237, 8187, 8188, 7968] avg=8192.0000 max=8460 max_rank=2 min=7968 min_rank=7 imbalance_ratio=1.032715 max_load_reduction=57076 max_load_reduction_pct=0.870911 moved_to_echo=32580 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7827, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 373, 27, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8131, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8104, 0, 0, 0, 8180, 0, 0, 0, 8189, 0, 0, 0, 81, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 12, 0, 1, 0, 0, 0, 0, 0, 93, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8093, 0, 0, 0, 8189, 0, 0, 6, 8135, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 8, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 14, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 19, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 8184, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 2, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8148, 0, 0, 0, 8191, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 84, 0, 0, 0, 0, 8133, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 332, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 7951, 0, 0, 40, 0, 11, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 48, 8142, 0, 2, 0, 0, 0, 19, 2, 0, 0, 0, 0, 0, 0, 3, 0, 8159, 2, 0, 0, 8098, 0, 0, 0, 8106, 0, 0, 3, 0, 24, 0, 43, 0, 0, 0, 0, 0, 7, 0, 0, 7854, 0, 25, 0, 0, 0, 0, 0, 19, 0, 3, 36, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8191, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 13, 0, 4, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 50, 0, 10, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8174, 0, 0, 0, 8191, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 8187, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8044, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8240, 8082, 8212, 8171, 8230, 8194, 8228] avg=8192.0000 max=8240 max_rank=1 min=8082 min_rank=2 imbalance_ratio=1.005859 max_load_reduction=57296 max_load_reduction_pct=0.874268 moved_to_echo=32823 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 8, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 14, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 0, 0, 0, 8036, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 10, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 19, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8242, 8066, 8222, 8212, 8216, 8193, 8197] avg=8192.0000 max=8242 max_rank=1 min=8066 min_rank=2 imbalance_ratio=1.006104 max_load_reduction=57294 max_load_reduction_pct=0.874237 moved_to_echo=32818 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 12, 0, 1, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 8041, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8093, 0, 0, 0, 38, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8135, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8237, 8209, 8207, 8197, 8138, 8141, 8228] avg=8192.0000 max=8237 max_rank=1 min=8138 min_rank=5 imbalance_ratio=1.005493 max_load_reduction=57299 max_load_reduction_pct=0.874313 moved_to_echo=32704 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7827, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 373, 27, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8032, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8131, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8104, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 81, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 7909, 8447, 8222, 8137, 8142, 8273, 8228] avg=8192.0000 max=8447 max_rank=2 min=7909 min_rank=1 imbalance_ratio=1.031128 max_load_reduction=57089 max_load_reduction_pct=0.871109 moved_to_echo=32780 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8043, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8248, 8055, 8212, 8194, 8228, 8194, 8228] avg=8192.0000 max=8248 max_rank=1 min=8055 min_rank=2 imbalance_ratio=1.006836 max_load_reduction=57288 max_load_reduction_pct=0.874146 moved_to_echo=32844 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8043, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 2, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8148, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8245, 8104, 8212, 8190, 8187, 8191, 8228] avg=8192.0000 max=8245 max_rank=1 min=8104 min_rank=2 imbalance_ratio=1.006470 max_load_reduction=57291 max_load_reduction_pct=0.874191 moved_to_echo=32796 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 84, 0, 0, 0, 8131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 332, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 7958, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 7951, 0, 0, 40, 0, 11, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 48, 0, 0, 2, 0, 0, 0, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8159, 2, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8106, 0, 0, 3, 0, 24, 0, 43, 0, 0, 0, 0, 0, 7, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7854, 0, 25, 0, 0, 0, 0, 0, 19, 0, 3, 36, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8262, 8194, 8453, 8048, 8237, 8182, 8186, 7974] avg=8192.0000 max=8453 max_rank=2 min=7974 min_rank=7 imbalance_ratio=1.031860 max_load_reduction=57083 max_load_reduction_pct=0.871017 moved_to_echo=32579 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 13, 0, 4, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 10, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4, 3, 0, 0, 8043, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 5, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8174, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8221, 8120, 8201, 8198, 8213, 8164, 8225] avg=8192.0000 max=8225 max_rank=7 min=8120 min_rank=2 imbalance_ratio=1.004028 max_load_reduction=57311 max_load_reduction_pct=0.874496 moved_to_echo=32800 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 0, 0, 0, 0, 128, 128, 128, 0, 128, 0, 0, 0, 65408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[65152, 0, 128, 128, 0, 0, 0, 0, 128, 0, 128, 128, 128, 0, 128, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[512, 768, 128, 128, 0, 0, 0, 0, 0, 0, 128, 0, 128, 128, 0, 0, 64256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 128, 128, 0, 0, 0, 0, 65280, 128, 0, 128, 0, 128, 0, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 0, 0, 128, 128, 0, 65280, 128, 0, 0, 384, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[65408, 0, 0, 128, 128, 128, 0, 128, 0, 0, 128, 128, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[384, 768, 128, 128, 0, 0, 128, 0, 0, 0, 128, 0, 128, 128, 0, 0, 64256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 65152, 0, 0, 0, 128, 128, 0, 0, 128, 128, 128, 0, 0, 0, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 65408, 128, 0, 128, 128, 128, 0, 128, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 0, 0, 128, 128, 0, 65280, 128, 0, 0, 384, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 0, 0, 0, 0, 128, 128, 128, 0, 128, 0, 0, 0, 65408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[65280, 0, 128, 128, 0, 0, 0, 0, 128, 0, 128, 128, 128, 0, 128, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 65408, 128, 0, 128, 128, 128, 0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 128, 0, 0, 0, 0, 65280, 128, 0, 128, 0, 128, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 6258, 0, 8192, 0, 1272, 0, 0, 0, 8192, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7901, 1901, 0, 0, 0, 0, 0, 0, 0, 6951, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8189, 0, 0, 0, 0, 8192, 0, 8187, 0, 23, 0, 0, 0, 8191, 0, 0, 36, 814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 8187, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7997, 0, 0, 2, 0, 0, 0, 0, 0, 7478, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 5741, 0, 8187, 0, 2072, 0, 0, 0, 8181, 1, 0, 0, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 5, 8190, 0, 16, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8118, 253, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8121, 0, 8192, 0, 115, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 22, 0, 0, 0, 0, 0, 0, 0, 8119, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8185, 0, 0, 0, 16, 8134, 0, 8192, 0, 57, 0, 0, 0, 8187, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7605, 441, 0, 0, 0, 3, 0, 0, 0, 8189, 0, 0, 4, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 7747, 0, 8190, 0, 510, 0, 0, 0, 8162, 0, 0, 0, 186, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 7957, 4, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 8, 0, 0, 8188, 0, 8179, 0, 0, 0, 8103, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 7601, 1195, 0, 0, 0, 1, 0, 0, 1, 7010, 0, 6, 0, 0, 0, 0, 0, 0, 8192, 0, 192, 0, 0, 0, 87, 0, 0, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 8, 0, 0, 10, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 4, 0, 0, 0, 24, 0, 0, 7747, 0, 26, 20, 0, 8179, 0, 7784, 0, 0, 0, 0, 0, 7859, 0, 0, 108, 2005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 173, 0, 8136, 0, 316, 0, 0, 0, 0, 0, 0, 0, 21, 0, 17, 32, 0, 0, 7995, 0, 0, 0, 0, 0, 0, 0, 0, 6642, 13, 0, 0, 0, 25, 0, 3, 0, 8068, 0, 0, 0, 16, 192, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7689, 301, 94, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8140, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5957, 0, 0, 0, 1272, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8125, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 1901, 0, 0, 0, 0, 0, 0, 0, 6951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8098, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8099, 8240, 8233, 7272, 8244, 8128, 8967, 8353] avg=8192.0000 max=8967 max_rank=6 min=7272 min_rank=3 imbalance_ratio=1.094604 max_load_reduction=56569 max_load_reduction_pct=0.863174 moved_to_echo=33692 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7972, 391, 94, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8141, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7730, 0, 0, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8125, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 22, 0, 0, 0, 0, 0, 0, 0, 8119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8098, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8472, 8242, 8244, 7845, 8243, 8127, 8260, 8103] avg=8192.0000 max=8472 max_rank=0 min=7845 min_rank=3 imbalance_ratio=1.034180 max_load_reduction=57064 max_load_reduction_pct=0.870728 moved_to_echo=32733 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7782, 394, 94, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 8139, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8189, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7798, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 36, 814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 8120, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 0, 0, 2, 0, 0, 0, 0, 0, 7478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8060, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8292, 8241, 8242, 8671, 8239, 8146, 7597, 8108] avg=8192.0000 max=8671 max_rank=3 min=7597 min_rank=6 imbalance_ratio=1.058472 max_load_reduction=56865 max_load_reduction_pct=0.867691 moved_to_echo=32090 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 7900, 276, 94, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8129, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5465, 0, 0, 0, 2072, 0, 0, 0, 0, 1, 0, 0, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 8187, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 8123, 0, 16, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 253, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8098, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8293, 8229, 8236, 7731, 8245, 8145, 8557, 8100] avg=8192.0000 max=8557 max_rank=6 min=7731 min_rank=3 imbalance_ratio=1.044556 max_load_reduction=56979 max_load_reduction_pct=0.869431 moved_to_echo=33047 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7401, 392, 94, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 8135, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8185, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 7742, 0, 0, 0, 57, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 441, 0, 0, 0, 3, 0, 0, 0, 8189, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8098, 0, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7902, 8278, 8245, 7816, 8244, 8121, 8748, 8182] avg=8192.0000 max=8748 max_rank=6 min=7816 min_rank=3 imbalance_ratio=1.067871 max_load_reduction=56788 max_load_reduction_pct=0.866516 moved_to_echo=33295 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7743, 373, 94, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 0, 0, 1, 0, 0, 0, 8110, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7374, 0, 0, 0, 510, 0, 0, 0, 0, 0, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8190, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 116, 4, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8096, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 8215, 8243, 8070, 8246, 8120, 8313, 8103] avg=8192.0000 max=8313 max_rank=6 min=8070 min_rank=3 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32782 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 397, 0, 0, 0, 0, 7397, 0, 94, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8051, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 111, 1195, 0, 0, 0, 1, 0, 0, 1, 7010, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8098, 0, 192, 0, 0, 0, 87, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7903, 8144, 8225, 8190, 8240, 8125, 8329, 8380] avg=8192.0000 max=8380 max_rank=7 min=7903 min_rank=0 imbalance_ratio=1.022949 max_load_reduction=57156 max_load_reduction_pct=0.872131 moved_to_echo=33074 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 0, 0, 10, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7783, 397, 97, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 10, 0, 0, 0, 0, 7810, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 4, 0, 0, 0, 24, 0, 0, 7747, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 20, 0, 7782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 2005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 7784, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 173, 0, 8065, 0, 316, 0, 0, 0, 0, 0, 0, 0, 21, 0, 17, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 6642, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 3, 0, 7971, 0, 0, 0, 16, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8349, 7947, 7868, 9941, 7835, 8624, 6765, 8207] avg=8192.0000 max=9941 max_rank=3 min=6765 min_rank=6 imbalance_ratio=1.213501 max_load_reduction=55595 max_load_reduction_pct=0.848312 moved_to_echo=31431 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8186, 0, 0, 0, 15, 8131, 0, 8192, 0, 35, 0, 0, 0, 8187, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7622, 426, 0, 0, 0, 3, 0, 0, 0, 8188, 0, 0, 3, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 101, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 7, 0, 0, 8188, 0, 8178, 0, 0, 0, 8104, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 7609, 1179, 0, 0, 0, 1, 0, 0, 2, 6987, 0, 7, 0, 0, 0, 0, 0, 0, 8192, 0, 190, 0, 0, 0, 88, 0, 0, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8189, 0, 0, 0, 0, 8192, 0, 8187, 0, 17, 0, 0, 0, 8191, 0, 0, 37, 806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 8187, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8012, 0, 0, 2, 0, 0, 0, 0, 0, 7485, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 5769, 0, 8188, 0, 2038, 0, 0, 0, 8183, 1, 0, 0, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 4, 8190, 0, 19, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8115, 253, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 6, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 4, 0, 0, 0, 24, 0, 0, 7769, 0, 26, 19, 0, 8179, 0, 7785, 0, 0, 0, 0, 0, 7853, 0, 0, 109, 2001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 175, 0, 8136, 0, 330, 0, 0, 0, 0, 0, 0, 0, 17, 0, 16, 30, 0, 0, 8000, 0, 0, 0, 0, 0, 0, 0, 0, 6620, 17, 0, 0, 0, 25, 1, 2, 0, 8064, 0, 0, 0, 16, 189, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 6200, 0, 8192, 0, 1290, 0, 0, 0, 8192, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7912, 1935, 0, 0, 0, 0, 0, 0, 0, 6950, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 253, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8114, 0, 8192, 0, 108, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 28, 0, 0, 0, 0, 0, 0, 0, 8124, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 5, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 7736, 0, 8190, 0, 518, 0, 0, 0, 8161, 0, 0, 0, 183, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 7960, 8, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 1, 0, 0, 6, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7416, 375, 94, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 8135, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8186, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 7756, 0, 0, 0, 35, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 426, 0, 0, 0, 3, 0, 0, 0, 8188, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8098, 0, 0, 0, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7905, 8286, 8241, 7807, 8244, 8121, 8733, 8199] avg=8192.0000 max=8733 max_rank=6 min=7807 min_rank=3 imbalance_ratio=1.066040 max_load_reduction=56803 max_load_reduction_pct=0.866745 moved_to_echo=33297 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7698, 286, 94, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8140, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5914, 0, 0, 0, 1290, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8123, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 1935, 0, 0, 0, 0, 0, 0, 0, 6950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8098, 0, 253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8098, 8238, 8233, 7243, 8244, 8126, 9003, 8351] avg=8192.0000 max=9003 max_rank=6 min=7243 min_rank=3 imbalance_ratio=1.098999 max_load_reduction=56533 max_load_reduction_pct=0.862625 moved_to_echo=33724 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7795, 378, 93, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 8139, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8189, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7814, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 37, 806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 8118, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 0, 0, 2, 0, 0, 0, 0, 0, 7485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8061, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8293, 8240, 8239, 8674, 8239, 8145, 7607, 8099] avg=8192.0000 max=8674 max_rank=3 min=7607 min_rank=6 imbalance_ratio=1.058838 max_load_reduction=56862 max_load_reduction_pct=0.867645 moved_to_echo=32090 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7971, 374, 94, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8141, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7740, 0, 0, 0, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8123, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 121, 28, 0, 0, 0, 0, 0, 0, 0, 8124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8098, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8459, 8241, 8241, 7848, 8243, 8128, 8273, 8103] avg=8192.0000 max=8459 max_rank=0 min=7848 min_rank=3 imbalance_ratio=1.032593 max_load_reduction=57077 max_load_reduction_pct=0.870926 moved_to_echo=32747 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7745, 357, 94, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 5, 0, 0, 1, 0, 0, 0, 8109, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7379, 0, 0, 0, 518, 0, 0, 0, 0, 0, 0, 0, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8190, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 118, 8, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8096, 0, 1, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8217, 8215, 8240, 8080, 8245, 8117, 8319, 8103] avg=8192.0000 max=8319 max_rank=6 min=8080 min_rank=3 imbalance_ratio=1.015503 max_load_reduction=57217 max_load_reduction_pct=0.873062 moved_to_echo=32784 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 429, 0, 0, 0, 0, 7403, 0, 94, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8052, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 114, 1179, 0, 0, 0, 1, 0, 0, 2, 6987, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8098, 0, 190, 0, 0, 0, 88, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7946, 8144, 8222, 8187, 8240, 8123, 8295, 8379] avg=8192.0000 max=8379 max_rank=7 min=7946 min_rank=0 imbalance_ratio=1.022827 max_load_reduction=57157 max_load_reduction_pct=0.872147 moved_to_echo=33037 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 7896, 266, 94, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8131, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5503, 0, 0, 0, 2038, 0, 0, 0, 0, 1, 0, 0, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 8188, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8121, 0, 19, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 253, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8098, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8283, 8230, 8233, 7742, 8245, 8145, 8558, 8100] avg=8192.0000 max=8558 max_rank=6 min=7742 min_rank=3 imbalance_ratio=1.044678 max_load_reduction=56978 max_load_reduction_pct=0.869415 moved_to_echo=33048 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 6, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7788, 379, 95, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 10, 0, 0, 0, 0, 7804, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 4, 0, 0, 0, 24, 0, 0, 7769, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 19, 0, 7800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 2001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 7785, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 175, 0, 8063, 0, 330, 0, 0, 0, 0, 0, 0, 0, 17, 0, 16, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 0, 0, 0, 0, 0, 0, 0, 0, 6620, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 1, 2, 0, 7969, 0, 0, 0, 16, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8335, 7942, 7887, 9955, 7836, 8631, 6748, 8202] avg=8192.0000 max=9955 max_rank=3 min=6748 min_rank=6 imbalance_ratio=1.215210 max_load_reduction=55581 max_load_reduction_pct=0.848099 moved_to_echo=31417 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 0, 128, 128, 128, 0, 0, 128, 0, 512, 0, 0, 0, 0, 61696, 2560, 768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 1024, 3840, 0, 128, 0, 128, 0, 0, 128, 60800, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 0, 0, 128, 128, 128, 0, 0, 128, 0, 512, 0, 0, 0, 0, 61824, 2432, 768, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 49920, 0, 0, 0, 12288, 0, 0, 0, 0, 128, 0, 256, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 0, 0, 0, 0, 128, 0, 128, 128, 0, 128, 128, 0, 0, 0, 64768, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 0, 64640, 128, 512, 0, 128, 384, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[256, 128, 65024, 0, 384, 128, 0, 0, 128, 0, 128, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 0, 128, 0, 128, 0, 128, 128, 0, 128, 128, 0, 0, 0, 64768, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 0, 64640, 128, 512, 0, 128, 384, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 128, 0, 65152, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 0, 0, 0, 128, 128, 0, 128, 0, 128, 0, 0, 65152, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[256, 128, 65024, 0, 384, 128, 0, 0, 128, 0, 128, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True - [2026-08-26 14:48:11] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 438.2 | throughput per GPU (TFLOP/s/GPU): 412.7 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.338553E+01 | loss scale: 1.0 | grad norm: 19.943 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 10, 180, 98, 1564, 0, 0, 3005, 3, 2, 21, 5, 120, 32, 1, 294, 0, 18, 807, 0, 4, 1556, 4115, 2, 2, 8173, 1285, 7404, 0, 804, 36, 31, 153, 0, 7, 298, 0, 13, 82, 1, 1376, 184, 1, 667, 34, 8172, 0, 1, 63, 0, 0, 0, 0, 1, 147, 0, 19, 364, 0, 1, 0, 17, 1027, 0, 0, 0, 576, 0, 9, 2, 781, 0, 1227, 1238, 19, 4, 153, 3, 18, 8, 2, 0, 55, 0, 3, 783, 5, 337, 0, 3, 35, 4, 11, 0, 2, 305, 4, 1, 21, 6, 1, 0, 207, 900, 688, 15, 1, 162, 3722, 0, 1, 4597, 0, 0, 0, 23, 833, 0, 0, 14, 6160, 372, 2, 18, 2, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 228, 208, 1378, 4, 7, 2025, 1, 2, 13, 0, 102, 12, 1, 121, 0, 13, 1150, 0, 0, 1927, 4072, 3, 0, 8165, 1441, 7543, 6, 807, 11, 34, 234, 0, 2, 204, 1, 16, 52, 3, 559, 133, 2, 1526, 4, 8170, 0, 1, 23, 2, 0, 6, 0, 6, 218, 1, 12, 231, 0, 3, 0, 26, 679, 0, 0, 0, 497, 0, 7, 11, 1038, 0, 765, 1289, 40, 19, 448, 0, 0, 0, 0, 0, 84, 0, 0, 1327, 0, 191, 1, 32, 1, 4, 0, 6, 1, 573, 0, 4, 14, 2, 0, 5, 122, 1128, 433, 12, 4, 154, 3549, 0, 0, 4813, 3, 0, 0, 7, 1179, 0, 2, 2, 6044, 322, 1, 0, 5, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1, 0, 0, 8, 191, 235, 1821, 0, 8, 3740, 1, 0, 25, 2, 35, 4, 0, 137, 0, 7, 949, 0, 0, 2074, 4365, 1, 0, 8174, 1285, 7497, 1, 233, 22, 46, 149, 0, 14, 423, 0, 6, 140, 2, 1019, 142, 0, 680, 39, 8179, 0, 1, 51, 1, 0, 5, 0, 3, 91, 0, 32, 245, 0, 2, 0, 9, 794, 0, 0, 0, 566, 0, 45, 2, 950, 0, 907, 849, 3, 9, 323, 0, 1, 1, 0, 3, 34, 0, 0, 561, 1, 304, 0, 6, 17, 2, 0, 3, 0, 205, 3, 0, 14, 9, 9, 6, 36, 938, 422, 23, 0, 93, 4083, 0, 0, 4208, 2, 0, 1, 37, 1632, 0, 2, 1, 5738, 592, 0, 2, 2, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 43, 141, 222, 1608, 0, 0, 3226, 12, 0, 2, 5, 114, 22, 0, 209, 0, 1, 760, 0, 3, 1787, 4391, 0, 0, 8189, 1889, 7839, 0, 1140, 2, 20, 259, 0, 6, 301, 0, 17, 82, 0, 665, 468, 0, 632, 0, 8172, 0, 2, 44, 0, 0, 4, 0, 1, 322, 0, 34, 83, 0, 0, 0, 11, 630, 0, 0, 0, 684, 0, 2, 4, 770, 0, 699, 775, 7, 5, 269, 0, 0, 1, 0, 3, 58, 0, 0, 854, 0, 211, 0, 4, 4, 0, 8, 0, 0, 199, 0, 0, 20, 5, 0, 0, 205, 527, 646, 3, 1, 59, 3641, 1, 0, 4951, 0, 0, 0, 10, 662, 0, 0, 0, 6506, 384, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 11, 178, 215, 1911, 0, 0, 2559, 2, 0, 7, 2, 74, 16, 0, 61, 0, 3, 1252, 0, 13, 2404, 3172, 0, 0, 8158, 1865, 7318, 3, 1438, 11, 39, 332, 0, 9, 171, 0, 2, 207, 0, 629, 106, 0, 621, 6, 8172, 0, 6, 24, 0, 0, 2, 0, 0, 235, 0, 19, 121, 0, 0, 4, 12, 853, 0, 0, 0, 565, 0, 4, 3, 953, 0, 867, 494, 22, 2, 245, 0, 0, 1, 0, 0, 41, 0, 2, 528, 0, 113, 0, 16, 3, 1, 2, 0, 0, 255, 1, 0, 15, 2, 2, 0, 119, 290, 454, 20, 0, 130, 4761, 1, 2, 5284, 0, 0, 0, 16, 723, 0, 0, 3, 6982, 367, 0, 4, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[3, 0, 0, 11, 156, 355, 1588, 0, 0, 2575, 2, 3, 5, 4, 137, 13, 0, 139, 0, 22, 631, 0, 5, 1771, 4345, 0, 2, 8163, 1649, 7817, 11, 1128, 14, 57, 101, 0, 0, 187, 0, 12, 29, 2, 1072, 340, 0, 631, 2, 8158, 2, 2, 61, 0, 5, 1, 0, 2, 105, 3, 59, 242, 1, 1, 0, 14, 1086, 0, 1, 0, 401, 1, 19, 1, 914, 1, 1152, 1228, 15, 8, 143, 0, 2, 3, 1, 0, 103, 0, 0, 896, 6, 160, 0, 5, 2, 0, 0, 5, 0, 166, 7, 3, 10, 6, 0, 0, 152, 1106, 547, 10, 1, 72, 3396, 2, 3, 4167, 0, 2, 0, 10, 1195, 0, 2, 3, 6345, 536, 0, 1, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[5, 0, 0, 26, 150, 235, 1816, 0, 0, 3094, 3, 3, 20, 2, 96, 2, 0, 151, 0, 26, 727, 2, 0, 2506, 3511, 0, 0, 8170, 1824, 7347, 6, 1021, 5, 58, 235, 2, 2, 271, 0, 24, 65, 0, 470, 150, 0, 863, 8, 8173, 0, 1, 20, 0, 0, 5, 0, 0, 188, 1, 28, 280, 0, 0, 0, 27, 1220, 4, 0, 0, 724, 0, 3, 2, 812, 0, 809, 1009, 29, 5, 128, 0, 0, 0, 0, 1, 25, 0, 0, 929, 0, 258, 1, 3, 17, 2, 0, 6, 0, 93, 0, 0, 21, 4, 9, 2, 194, 960, 367, 1, 4, 90, 4375, 2, 1, 4230, 0, 3, 0, 13, 992, 0, 1, 8, 5885, 670, 0, 4, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[4, 1, 0, 6, 90, 200, 1782, 0, 0, 2099, 1, 0, 5, 3, 61, 5, 0, 183, 0, 10, 923, 0, 2, 1998, 3777, 1, 0, 8176, 1524, 7513, 1, 1222, 4, 35, 227, 0, 1, 87, 0, 4, 63, 0, 665, 185, 0, 673, 4, 8144, 0, 0, 33, 1, 0, 1, 0, 0, 203, 0, 40, 171, 1, 0, 0, 29, 621, 0, 0, 0, 576, 0, 11, 1, 605, 0, 1420, 737, 13, 16, 214, 0, 0, 0, 2, 1, 76, 0, 1, 763, 1, 270, 0, 1, 10, 0, 3, 0, 0, 315, 1, 0, 13, 1, 2, 1, 215, 815, 623, 2, 0, 118, 4371, 0, 1, 5366, 0, 2, 0, 9, 1342, 0, 0, 0, 6324, 514, 0, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 10, 180, 98, 1564, 0, 0, 3005, 3, 2, 21, 5, 120, 32, 1601, 1564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 294, 0, 18, 807, 0, 4, 1556, 2631, 2, 2, 0, 1285, 0, 0, 804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 31, 153, 0, 7, 298, 0, 13, 82, 1, 1376, 184, 1, 667, 34, 5762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 63, 0, 0, 0, 0, 1, 147, 0, 19, 364, 0, 1, 0, 17, 7690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 0, 0, 0, 576, 0, 9, 2, 781, 0, 1227, 1238, 19, 4, 153, 3, 2664, 965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 8, 2, 0, 55, 0, 3, 783, 5, 337, 0, 3, 35, 4, 11, 0, 6439, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 305, 4, 1, 21, 6, 1, 0, 207, 900, 688, 15, 1, 162, 3722, 0, 1484, 846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4597, 0, 0, 0, 23, 833, 0, 0, 14, 1895, 372, 2, 18, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 7404, 8645, 8303, 8668, 8186, 8365, 7760] avg=8192.0000 max=8668 max_rank=4 min=7404 min_rank=1 imbalance_ratio=1.058105 max_load_reduction=56868 max_load_reduction_pct=0.867737 moved_to_echo=32979 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 228, 208, 1378, 4, 7, 2025, 1, 2, 13, 0, 102, 12, 1571, 1564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 121, 0, 13, 1150, 0, 0, 1927, 2603, 3, 0, 0, 1441, 0, 6, 807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 34, 234, 0, 2, 204, 1, 16, 52, 3, 559, 133, 2, 1526, 4, 5760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 23, 2, 0, 6, 0, 6, 218, 1, 12, 231, 0, 3, 0, 26, 7683, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 679, 0, 0, 0, 497, 0, 7, 11, 1038, 0, 765, 1289, 40, 19, 448, 0, 2614, 982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 1327, 0, 191, 1, 32, 1, 4, 0, 6, 6561, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 573, 0, 4, 14, 2, 0, 5, 122, 1128, 433, 12, 4, 154, 3549, 0, 1469, 846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4813, 3, 0, 0, 7, 1179, 0, 2, 2, 1859, 322, 1, 0, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7123, 8072, 8541, 8212, 8389, 8689, 8316, 8194] avg=8192.0000 max=8689 max_rank=5 min=7123 min_rank=0 imbalance_ratio=1.060669 max_load_reduction=56847 max_load_reduction_pct=0.867416 moved_to_echo=33588 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 43, 141, 222, 1608, 0, 0, 3226, 12, 0, 2, 5, 114, 22, 1691, 1564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 209, 0, 1, 760, 0, 3, 1787, 2807, 0, 0, 0, 1889, 0, 0, 1140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 20, 259, 0, 6, 301, 0, 17, 82, 0, 665, 468, 0, 632, 0, 5762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 44, 0, 0, 4, 0, 1, 322, 0, 34, 83, 0, 0, 0, 11, 7704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 630, 0, 0, 0, 684, 0, 2, 4, 770, 0, 699, 775, 7, 5, 269, 0, 2814, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 58, 0, 0, 854, 0, 211, 0, 4, 4, 0, 8, 0, 6818, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 199, 0, 0, 20, 5, 0, 0, 205, 527, 646, 3, 1, 59, 3641, 1, 1584, 846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4951, 0, 0, 0, 10, 662, 0, 0, 0, 2001, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8650, 8596, 8214, 8205, 7680, 8446, 7737, 8008] avg=8192.0000 max=8650 max_rank=0 min=7680 min_rank=4 imbalance_ratio=1.055908 max_load_reduction=56886 max_load_reduction_pct=0.868011 moved_to_echo=31871 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 11, 156, 355, 1588, 0, 0, 2575, 2, 3, 5, 4, 137, 13, 1649, 1562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 0, 22, 631, 0, 5, 1771, 2778, 0, 2, 0, 1649, 0, 11, 1128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 57, 101, 0, 0, 187, 0, 12, 29, 2, 1072, 340, 0, 631, 2, 5751, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 61, 0, 5, 1, 0, 2, 105, 3, 59, 242, 1, 1, 0, 14, 7680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1086, 0, 1, 0, 401, 1, 19, 1, 914, 1, 1152, 1228, 15, 8, 143, 0, 2744, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 1, 0, 103, 0, 0, 896, 6, 160, 0, 5, 2, 0, 0, 5, 6799, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 166, 7, 3, 10, 6, 0, 0, 152, 1106, 547, 10, 1, 72, 3396, 2, 1567, 845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4167, 0, 2, 0, 10, 1195, 0, 2, 3, 1952, 536, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8063, 8136, 8198, 8178, 8732, 8465, 7890, 7874] avg=8192.0000 max=8732 max_rank=4 min=7874 min_rank=7 imbalance_ratio=1.065918 max_load_reduction=56804 max_load_reduction_pct=0.866760 moved_to_echo=32961 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 0, 26, 150, 235, 1816, 0, 0, 3094, 3, 3, 20, 2, 96, 2, 1529, 1565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 0, 26, 727, 2, 0, 2506, 2245, 0, 0, 0, 1824, 0, 6, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 58, 235, 2, 2, 271, 0, 24, 65, 0, 470, 150, 0, 863, 8, 5762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 20, 0, 0, 5, 0, 0, 188, 1, 28, 280, 0, 0, 0, 27, 7686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1220, 4, 0, 0, 724, 0, 3, 2, 812, 0, 809, 1009, 29, 5, 128, 0, 2545, 957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 25, 0, 0, 929, 0, 258, 1, 3, 17, 2, 0, 6, 6390, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 0, 21, 4, 9, 2, 194, 960, 367, 1, 4, 90, 4375, 2, 1266, 846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4230, 0, 3, 0, 13, 992, 0, 1, 8, 1811, 670, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8546, 8508, 7915, 8236, 8247, 8116, 8234, 7734] avg=8192.0000 max=8546 max_rank=0 min=7734 min_rank=7 imbalance_ratio=1.043213 max_load_reduction=56990 max_load_reduction_pct=0.869598 moved_to_echo=32331 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 8, 191, 235, 1821, 0, 8, 3740, 1, 0, 25, 2, 35, 4, 1491, 1566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 7, 949, 0, 0, 2074, 2791, 1, 0, 0, 1285, 0, 1, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 46, 149, 0, 14, 423, 0, 6, 140, 2, 1019, 142, 0, 680, 39, 5766, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 51, 1, 0, 5, 0, 3, 91, 0, 32, 245, 0, 2, 0, 9, 7691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 794, 0, 0, 0, 566, 0, 45, 2, 950, 0, 907, 849, 3, 9, 323, 0, 2482, 977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 3, 34, 0, 0, 561, 1, 304, 0, 6, 17, 2, 0, 3, 6520, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 205, 3, 0, 14, 9, 9, 6, 36, 938, 422, 23, 0, 93, 4083, 0, 1574, 847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4208, 2, 0, 1, 37, 1632, 0, 2, 1, 1765, 592, 0, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9128, 7478, 8448, 8131, 7907, 7936, 8262, 8246] avg=8192.0000 max=9128 max_rank=0 min=7478 min_rank=1 imbalance_ratio=1.114258 max_load_reduction=56408 max_load_reduction_pct=0.860718 moved_to_echo=32351 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[4, 1, 0, 6, 90, 200, 1782, 0, 0, 2099, 1, 0, 5, 3, 61, 5, 1644, 1559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 0, 10, 923, 0, 2, 1998, 2415, 1, 0, 0, 1524, 0, 1, 1222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 35, 227, 0, 1, 87, 0, 4, 63, 0, 665, 185, 0, 673, 4, 5742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 1, 0, 1, 0, 0, 203, 0, 40, 171, 1, 0, 0, 29, 7693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 621, 0, 0, 0, 576, 0, 11, 1, 605, 0, 1420, 737, 13, 16, 214, 0, 2735, 979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 76, 0, 1, 763, 1, 270, 0, 1, 10, 0, 3, 0, 6534, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 315, 1, 0, 13, 1, 2, 1, 215, 815, 623, 2, 0, 118, 4371, 0, 1362, 843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5366, 0, 2, 0, 9, 1342, 0, 0, 0, 1945, 514, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7460, 8279, 7690, 8172, 7928, 8145, 8682, 9180] avg=8192.0000 max=9180 max_rank=7 min=7460 min_rank=0 imbalance_ratio=1.120605 max_load_reduction=56356 max_load_reduction_pct=0.859924 moved_to_echo=33935 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 11, 178, 215, 1911, 0, 0, 2559, 2, 0, 7, 2, 74, 16, 1818, 1568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 3, 1252, 0, 13, 2404, 2024, 0, 0, 0, 1865, 0, 3, 1438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 39, 332, 0, 9, 171, 0, 2, 207, 0, 629, 106, 0, 621, 6, 5752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 24, 0, 0, 2, 0, 0, 235, 0, 19, 121, 0, 0, 4, 12, 7676, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 853, 0, 0, 0, 565, 0, 4, 3, 953, 0, 867, 494, 22, 2, 245, 0, 3023, 954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 41, 0, 2, 528, 0, 113, 0, 16, 3, 1, 2, 0, 6364, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 1, 0, 15, 2, 2, 0, 119, 290, 454, 20, 0, 130, 4761, 1, 1148, 852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5284, 0, 0, 0, 16, 723, 0, 0, 3, 2141, 367, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8361, 9063, 7885, 8099, 7985, 7553, 8050, 8540] avg=8192.0000 max=9063 max_rank=1 min=7553 min_rank=5 imbalance_ratio=1.106323 max_load_reduction=56473 max_load_reduction_pct=0.861710 moved_to_echo=32128 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True + [2026-08-26 15:24:11] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 445.6 | throughput per GPU (TFLOP/s/GPU): 405.8 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.338528E+01 | loss scale: 1.0 | grad norm: 19.948 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[4, 0, 0, 26, 156, 232, 1846, 0, 0, 3096, 3, 3, 17, 2, 99, 2, 0, 148, 0, 28, 718, 2, 0, 2554, 3475, 0, 0, 8171, 1795, 7355, 6, 1010, 5, 55, 233, 2, 2, 277, 0, 24, 64, 0, 469, 148, 0, 868, 8, 8172, 0, 1, 20, 0, 0, 5, 0, 0, 188, 1, 28, 286, 0, 0, 0, 27, 1210, 3, 0, 0, 706, 0, 3, 2, 814, 0, 806, 981, 30, 6, 124, 0, 0, 0, 0, 1, 25, 0, 0, 926, 0, 259, 1, 3, 16, 2, 0, 7, 0, 87, 0, 0, 18, 4, 9, 2, 198, 955, 375, 1, 4, 92, 4360, 2, 1, 4230, 0, 4, 0, 13, 1007, 0, 1, 8, 5930, 675, 0, 3, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 45, 154, 219, 1638, 0, 0, 3249, 11, 0, 3, 5, 126, 21, 1, 201, 0, 1, 743, 0, 4, 1814, 4368, 0, 0, 8189, 1851, 7840, 0, 1127, 2, 17, 259, 0, 6, 302, 0, 16, 78, 0, 668, 475, 0, 648, 0, 8174, 0, 2, 44, 0, 0, 4, 0, 1, 314, 0, 36, 85, 0, 0, 0, 10, 621, 0, 0, 0, 662, 0, 1, 4, 772, 0, 708, 758, 6, 6, 268, 0, 0, 1, 0, 3, 61, 0, 0, 852, 0, 221, 0, 4, 4, 0, 7, 0, 0, 190, 0, 0, 19, 5, 0, 0, 217, 520, 649, 3, 1, 61, 3609, 1, 0, 4943, 0, 0, 0, 12, 677, 0, 0, 0, 6541, 378, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 234, 205, 1398, 4, 7, 2025, 1, 2, 13, 0, 116, 10, 1, 126, 0, 13, 1111, 0, 1, 1963, 4049, 3, 0, 8165, 1414, 7544, 6, 813, 11, 36, 224, 0, 0, 204, 1, 15, 51, 2, 565, 135, 2, 1534, 4, 8168, 0, 1, 22, 2, 0, 6, 0, 6, 219, 1, 14, 241, 0, 3, 0, 25, 662, 0, 0, 0, 478, 0, 6, 12, 1041, 0, 775, 1273, 40, 21, 447, 0, 0, 0, 0, 0, 87, 0, 0, 1322, 0, 193, 1, 31, 1, 6, 0, 6, 1, 569, 0, 4, 14, 2, 0, 4, 133, 1113, 432, 12, 4, 159, 3536, 0, 0, 4800, 3, 0, 0, 7, 1201, 0, 2, 2, 6087, 318, 1, 0, 5, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[3, 0, 0, 10, 163, 353, 1612, 0, 0, 2597, 2, 3, 5, 2, 148, 15, 0, 134, 0, 20, 624, 0, 5, 1807, 4312, 0, 2, 8163, 1628, 7818, 11, 1119, 15, 58, 95, 0, 0, 185, 0, 10, 28, 2, 1091, 338, 0, 646, 2, 8158, 2, 2, 62, 0, 4, 1, 0, 1, 98, 3, 60, 248, 0, 1, 0, 14, 1076, 0, 1, 0, 392, 1, 17, 1, 915, 1, 1152, 1205, 15, 8, 142, 0, 2, 3, 0, 0, 104, 0, 0, 882, 6, 166, 0, 5, 2, 0, 0, 6, 0, 159, 8, 3, 10, 6, 0, 0, 160, 1088, 553, 11, 1, 75, 3387, 2, 3, 4165, 0, 2, 0, 10, 1195, 0, 2, 3, 6370, 536, 0, 1, 0, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[4, 1, 0, 6, 92, 197, 1803, 0, 0, 2113, 1, 0, 5, 2, 64, 5, 0, 181, 0, 10, 915, 0, 3, 2035, 3730, 1, 0, 8176, 1494, 7514, 1, 1209, 3, 36, 217, 0, 1, 88, 0, 4, 63, 0, 673, 192, 0, 688, 4, 8143, 0, 0, 32, 1, 0, 1, 0, 0, 201, 0, 40, 176, 1, 0, 0, 30, 607, 0, 0, 0, 569, 0, 11, 1, 607, 0, 1422, 703, 13, 17, 210, 0, 0, 0, 2, 1, 75, 0, 1, 762, 1, 278, 0, 1, 9, 0, 3, 0, 0, 306, 1, 0, 12, 2, 2, 1, 215, 803, 624, 1, 0, 125, 4363, 0, 1, 5366, 0, 2, 0, 9, 1365, 0, 0, 0, 6375, 513, 0, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 12, 188, 213, 1949, 0, 0, 2570, 2, 0, 7, 2, 76, 16, 0, 59, 0, 3, 1228, 0, 13, 2428, 3152, 0, 0, 8159, 1848, 7329, 2, 1429, 12, 41, 320, 0, 9, 176, 0, 2, 201, 0, 630, 105, 0, 632, 5, 8171, 0, 7, 25, 0, 0, 2, 0, 0, 234, 0, 20, 121, 0, 0, 4, 13, 833, 0, 0, 0, 559, 0, 4, 2, 968, 0, 876, 475, 22, 2, 238, 0, 0, 1, 0, 0, 41, 0, 2, 525, 0, 112, 0, 15, 4, 1, 2, 0, 0, 249, 1, 0, 14, 2, 2, 0, 119, 286, 463, 19, 0, 130, 4743, 1, 2, 5288, 0, 0, 0, 17, 729, 0, 0, 3, 7007, 359, 0, 4, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1, 0, 0, 8, 201, 231, 1838, 0, 8, 3759, 1, 0, 26, 2, 38, 3, 0, 130, 0, 7, 936, 0, 0, 2110, 4336, 1, 0, 8174, 1271, 7502, 1, 225, 23, 48, 142, 0, 12, 420, 0, 6, 140, 2, 1019, 148, 0, 695, 38, 8179, 0, 1, 48, 1, 0, 5, 0, 3, 91, 0, 32, 259, 0, 2, 0, 10, 774, 0, 0, 0, 563, 0, 46, 3, 950, 0, 899, 830, 3, 9, 321, 0, 1, 1, 0, 3, 38, 0, 0, 559, 1, 309, 0, 6, 15, 2, 0, 3, 0, 201, 3, 0, 15, 8, 9, 4, 38, 921, 424, 22, 0, 97, 4050, 0, 0, 4215, 2, 0, 1, 37, 1647, 0, 2, 1, 5780, 584, 0, 2, 2, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 9, 189, 101, 1586, 0, 0, 3009, 4, 2, 22, 4, 128, 31, 1, 289, 0, 17, 795, 0, 4, 1584, 4090, 2, 2, 8172, 1290, 7400, 0, 792, 32, 29, 153, 0, 5, 297, 0, 14, 77, 1, 1383, 181, 1, 674, 31, 8170, 0, 1, 65, 0, 0, 1, 0, 1, 143, 0, 19, 377, 0, 1, 0, 18, 1008, 0, 0, 0, 576, 0, 9, 1, 783, 0, 1218, 1221, 19, 4, 151, 3, 16, 8, 2, 0, 56, 0, 3, 779, 6, 339, 0, 3, 35, 4, 12, 0, 2, 296, 4, 1, 21, 6, 1, 0, 209, 889, 694, 15, 2, 165, 3707, 0, 1, 4584, 0, 0, 0, 24, 846, 0, 0, 13, 6208, 371, 1, 18, 2, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 0, 26, 156, 232, 1846, 0, 0, 3096, 3, 3, 17, 2, 99, 2, 1542, 1504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 0, 28, 718, 2, 0, 2554, 2252, 0, 0, 0, 1795, 0, 6, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 55, 233, 2, 2, 277, 0, 24, 64, 0, 469, 148, 0, 868, 8, 5752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 20, 0, 0, 5, 0, 0, 188, 1, 28, 286, 0, 0, 0, 27, 7682, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1210, 3, 0, 0, 706, 0, 3, 2, 814, 0, 806, 981, 30, 6, 124, 0, 2582, 966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 25, 0, 0, 926, 0, 259, 1, 3, 16, 2, 0, 7, 6389, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 18, 4, 9, 2, 198, 955, 375, 1, 4, 92, 4360, 2, 1223, 916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4230, 0, 4, 0, 13, 1007, 0, 1, 8, 1806, 675, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8532, 8513, 7907, 8238, 8233, 8118, 8246, 7749] avg=8192.0000 max=8532 max_rank=0 min=7749 min_rank=7 imbalance_ratio=1.041504 max_load_reduction=57004 max_load_reduction_pct=0.869812 moved_to_echo=32346 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 45, 154, 219, 1638, 0, 0, 3249, 11, 0, 3, 5, 126, 21, 1701, 1504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 201, 0, 1, 743, 0, 4, 1814, 2830, 0, 0, 0, 1851, 0, 0, 1127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 17, 259, 0, 6, 302, 0, 16, 78, 0, 668, 475, 0, 648, 0, 5754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 44, 0, 0, 4, 0, 1, 314, 0, 36, 85, 0, 0, 0, 10, 7699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 621, 0, 0, 0, 662, 0, 1, 4, 772, 0, 708, 758, 6, 6, 268, 0, 2849, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 61, 0, 0, 852, 0, 221, 0, 4, 4, 0, 7, 0, 6810, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 190, 0, 0, 19, 5, 0, 0, 217, 520, 649, 3, 1, 61, 3609, 1, 1538, 916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4943, 0, 0, 0, 12, 677, 0, 0, 0, 1991, 378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8676, 8572, 8225, 8195, 7685, 8453, 7729, 8001] avg=8192.0000 max=8676 max_rank=0 min=7685 min_rank=4 imbalance_ratio=1.059082 max_load_reduction=56860 max_load_reduction_pct=0.867615 moved_to_echo=31868 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 12, 188, 213, 1949, 0, 0, 2570, 2, 0, 7, 2, 76, 16, 1827, 1508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 3, 1228, 0, 13, 2428, 2039, 0, 0, 0, 1848, 0, 2, 1429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 41, 320, 0, 9, 176, 0, 2, 201, 0, 630, 105, 0, 632, 5, 5743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 25, 0, 0, 2, 0, 0, 234, 0, 20, 121, 0, 0, 4, 13, 7672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 833, 0, 0, 0, 559, 0, 4, 2, 968, 0, 876, 475, 22, 2, 238, 0, 3056, 964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 41, 0, 2, 525, 0, 112, 0, 15, 4, 1, 2, 0, 6365, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 249, 1, 0, 14, 2, 2, 0, 119, 286, 463, 19, 0, 130, 4743, 1, 1113, 920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5288, 0, 0, 0, 17, 729, 0, 0, 3, 2124, 359, 0, 4, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8370, 9049, 7876, 8098, 7999, 7555, 8062, 8527] avg=8192.0000 max=9049 max_rank=1 min=7555 min_rank=5 imbalance_ratio=1.104614 max_load_reduction=56487 max_load_reduction_pct=0.861923 moved_to_echo=32143 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 10, 163, 353, 1612, 0, 0, 2597, 2, 3, 5, 2, 148, 15, 1657, 1501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, 0, 20, 624, 0, 5, 1807, 2794, 0, 2, 0, 1628, 0, 11, 1119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 58, 95, 0, 0, 185, 0, 10, 28, 2, 1091, 338, 0, 646, 2, 5743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 62, 0, 4, 1, 0, 1, 98, 3, 60, 248, 0, 1, 0, 14, 7675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1076, 0, 1, 0, 392, 1, 17, 1, 915, 1, 1152, 1205, 15, 8, 142, 0, 2774, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 104, 0, 0, 882, 6, 166, 0, 5, 2, 0, 0, 6, 6791, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 159, 8, 3, 10, 6, 0, 0, 160, 1088, 553, 11, 1, 75, 3387, 2, 1518, 914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4165, 0, 2, 0, 10, 1195, 0, 2, 3, 1939, 536, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8071, 8144, 8213, 8171, 8727, 8455, 7895, 7860] avg=8192.0000 max=8727 max_rank=4 min=7860 min_rank=7 imbalance_ratio=1.065308 max_load_reduction=56809 max_load_reduction_pct=0.866837 moved_to_echo=32937 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[4, 1, 0, 6, 92, 197, 1803, 0, 0, 2113, 1, 0, 5, 2, 64, 5, 1658, 1499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 0, 10, 915, 0, 3, 2035, 2417, 1, 0, 0, 1494, 0, 1, 1209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 36, 217, 0, 1, 88, 0, 4, 63, 0, 673, 192, 0, 688, 4, 5732, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 1, 0, 1, 0, 0, 201, 0, 40, 176, 1, 0, 0, 30, 7688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 607, 0, 0, 0, 569, 0, 11, 1, 607, 0, 1422, 703, 13, 17, 210, 0, 2776, 988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 75, 0, 1, 762, 1, 278, 0, 1, 9, 0, 3, 0, 6526, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 306, 1, 0, 12, 2, 2, 1, 215, 803, 624, 1, 0, 125, 4363, 0, 1313, 912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5366, 0, 2, 0, 9, 1365, 0, 0, 0, 1941, 513, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7450, 8266, 7701, 8170, 7924, 8147, 8680, 9198] avg=8192.0000 max=9198 max_rank=7 min=7450 min_rank=0 imbalance_ratio=1.122803 max_load_reduction=56338 max_load_reduction_pct=0.859650 moved_to_echo=33949 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 234, 205, 1398, 4, 7, 2025, 1, 2, 13, 0, 116, 10, 1583, 1503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 126, 0, 13, 1111, 0, 1, 1963, 2624, 3, 0, 0, 1414, 0, 6, 813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 36, 224, 0, 0, 204, 1, 15, 51, 2, 565, 135, 2, 1534, 4, 5750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 22, 2, 0, 6, 0, 6, 219, 1, 14, 241, 0, 3, 0, 25, 7676, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 662, 0, 0, 0, 478, 0, 6, 12, 1041, 0, 775, 1273, 40, 21, 447, 0, 2651, 991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 1322, 0, 193, 1, 31, 1, 6, 0, 6, 6553, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 569, 0, 4, 14, 2, 0, 4, 133, 1113, 432, 12, 4, 159, 3536, 0, 1425, 915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4800, 3, 0, 0, 7, 1201, 0, 2, 2, 1853, 318, 1, 0, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7109, 8075, 8534, 8216, 8397, 8689, 8323, 8193] avg=8192.0000 max=8689 max_rank=5 min=7109 min_rank=0 imbalance_ratio=1.060669 max_load_reduction=56847 max_load_reduction_pct=0.867416 moved_to_echo=33602 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 9, 189, 101, 1586, 0, 0, 3009, 4, 2, 22, 4, 128, 31, 1615, 1504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 289, 0, 17, 795, 0, 4, 1584, 2650, 2, 2, 0, 1290, 0, 0, 792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 29, 153, 0, 5, 297, 0, 14, 77, 1, 1383, 181, 1, 674, 31, 5751, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 65, 0, 0, 1, 0, 1, 143, 0, 19, 377, 0, 1, 0, 18, 7684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 0, 0, 0, 576, 0, 9, 1, 783, 0, 1218, 1221, 19, 4, 151, 3, 2703, 973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 8, 2, 0, 56, 0, 3, 779, 6, 339, 0, 3, 35, 4, 12, 0, 6427, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 296, 4, 1, 21, 6, 1, 0, 209, 889, 694, 15, 2, 165, 3707, 0, 1440, 915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4584, 0, 0, 0, 24, 846, 0, 0, 13, 1890, 371, 1, 18, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 7426, 8629, 8310, 8669, 8178, 8367, 7753] avg=8192.0000 max=8669 max_rank=4 min=7426 min_rank=1 imbalance_ratio=1.058228 max_load_reduction=56867 max_load_reduction_pct=0.867722 moved_to_echo=32967 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 8, 201, 231, 1838, 0, 8, 3759, 1, 0, 26, 2, 38, 3, 1503, 1505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 0, 7, 936, 0, 0, 2110, 2810, 1, 0, 0, 1271, 0, 1, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 48, 142, 0, 12, 420, 0, 6, 140, 2, 1019, 148, 0, 695, 38, 5758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 48, 1, 0, 5, 0, 3, 91, 0, 32, 259, 0, 2, 0, 10, 7686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 774, 0, 0, 0, 563, 0, 46, 3, 950, 0, 899, 830, 3, 9, 321, 0, 2517, 987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 3, 38, 0, 0, 559, 1, 309, 0, 6, 15, 2, 0, 3, 6515, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 201, 3, 0, 15, 8, 9, 4, 38, 921, 424, 22, 0, 97, 4050, 0, 1526, 916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4215, 2, 0, 1, 37, 1647, 0, 2, 1, 1760, 584, 0, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9124, 7491, 8451, 8138, 7902, 7941, 8234, 8255] avg=8192.0000 max=9124 max_rank=0 min=7491 min_rank=1 imbalance_ratio=1.113770 max_load_reduction=56412 max_load_reduction_pct=0.860779 moved_to_echo=32332 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 128, 1408, 1792, 13568, 128, 128, 22400, 128, 128, 128, 128, 768, 128, 13056, 12544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 1408, 0, 128, 7296, 128, 128, 16128, 20352, 128, 128, 0, 12800, 0, 128, 7808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[6912, 128, 128, 0, 4608, 128, 128, 128, 6912, 128, 7936, 7680, 256, 128, 2048, 128, 21632, 7936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 2176, 128, 128, 128, 128, 128, 128, 1280, 6784, 4224, 128, 128, 896, 32000, 128, 11520, 6784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 37632, 128, 128, 128, 128, 8576, 0, 128, 128, 15488, 3840, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 384, 1792, 128, 128, 2048, 128, 128, 768, 128, 6528, 1792, 128, 6400, 128, 46080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 128, 1408, 1792, 13696, 128, 128, 22528, 128, 128, 128, 128, 896, 128, 13184, 12032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 1280, 0, 128, 7168, 128, 128, 16384, 20480, 128, 128, 0, 12672, 0, 128, 7808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[128, 2176, 128, 128, 128, 128, 128, 128, 1408, 6656, 4224, 128, 128, 1024, 31872, 128, 11136, 7424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[6912, 128, 128, 0, 4608, 128, 128, 128, 6912, 128, 7936, 7552, 256, 128, 1920, 128, 22016, 7936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 37632, 128, 128, 128, 256, 8704, 0, 128, 128, 15360, 3840, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 384, 1664, 128, 128, 2048, 128, 128, 768, 128, 6528, 1792, 128, 6400, 128, 46080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 512, 0, 128, 6656, 128, 1920, 128, 128, 128, 128, 128, 128, 52480, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 384, 128, 128, 128, 0, 128, 1536, 128, 256, 1792, 128, 128, 128, 256, 61568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[128, 128, 384, 128, 128, 128, 0, 128, 1536, 128, 256, 1920, 128, 128, 128, 256, 61568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[8099, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 1, 0, 0, 3, 0, 16, 0, 0, 0, 8191, 0, 0, 8181, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 38, 0, 13, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 8190, 0, 1015, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2510, 0, 0, 8146, 0, 4, 0, 0, 0, 31, 0, 1, 0, 0, 0, 4675, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[8139, 0, 14, 0, 1, 0, 0, 1, 1, 5, 0, 11, 0, 0, 0, 0, 0, 0, 1, 0, 8181, 14, 0, 8171, 0, 0, 2, 3, 0, 0, 0, 2, 1, 0, 0, 0, 0, 34, 0, 0, 8, 0, 0, 0, 1, 0, 7, 1, 4, 2, 100, 0, 0, 0, 0, 0, 9, 2, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 1, 4, 1, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 8175, 0, 1, 2, 0, 0, 1, 8179, 0, 3703, 0, 0, 3, 0, 1, 0, 63, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 3028, 0, 3, 7947, 2, 0, 3, 0, 0, 78, 0, 1, 3, 0, 0, 1453, 0, 0, 0, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[8183, 0, 1, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 8182, 19, 7, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 4, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8146, 6, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8189, 0, 1, 2, 0, 0, 4, 8192, 0, 2474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2447, 0, 0, 8086, 0, 1, 0, 0, 0, 294, 0, 0, 0, 1, 0, 3089, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[8181, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 8189, 38, 1, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 1, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 8188, 0, 2825, 0, 0, 0, 0, 0, 0, 8, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 4204, 0, 0, 8151, 0, 2, 0, 1, 0, 7, 1, 1, 0, 0, 0, 1158, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 8186, 140, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 2266, 9, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4285, 0, 0, 8192, 0, 1, 0, 0, 0, 109, 0, 0, 0, 0, 0, 1370, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[8191, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 8, 0, 2, 0, 8192, 0, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3696, 0, 0, 8075, 0, 0, 0, 0, 0, 151, 0, 0, 0, 0, 0, 4152, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[8164, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8191, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 4, 71, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 9, 0, 1, 0, 8185, 0, 425, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 881, 0, 0, 8140, 0, 23, 0, 0, 0, 362, 0, 0, 0, 0, 0, 6488, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[8145, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 2, 0, 17, 0, 0, 0, 8190, 0, 0, 8182, 10, 0, 1, 0, 0, 1, 0, 1, 0, 0, 1, 7, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 13, 0, 0, 0, 8186, 0, 234, 21, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1744, 0, 0, 8184, 0, 13, 0, 0, 0, 46, 1, 2, 0, 0, 0, 6052, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8099, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 1, 0, 0, 3, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 8151, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 38, 0, 13, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6530, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2510, 0, 0, 5353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 31, 0, 1, 0, 0, 0, 4675, 0, 0, 0, 0, 2793, 1650, 33, 15, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8125, 8198, 8233, 8162, 8193, 7549, 7863, 9213] avg=8192.0000 max=9213 max_rank=7 min=7549 min_rank=5 imbalance_ratio=1.124634 max_load_reduction=56323 max_load_reduction_pct=0.859421 moved_to_echo=32818 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8139, 0, 14, 0, 1, 0, 0, 1, 1, 5, 0, 11, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 14, 0, 8141, 0, 0, 2, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 34, 0, 0, 8, 0, 0, 0, 1, 0, 7, 1, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 100, 0, 0, 0, 0, 0, 9, 2, 0, 0, 8124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 1, 0, 0, 4, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6526, 0, 1, 2, 0, 0, 1, 0, 0, 3703, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 3028, 0, 3, 5222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 0, 0, 78, 0, 1, 3, 0, 0, 1453, 0, 0, 0, 4, 2725, 1649, 33, 17, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8163, 8214, 8241, 8191, 10238, 8319, 5979] avg=8192.0000 max=10238 max_rank=5 min=5979 min_rank=7 imbalance_ratio=1.249756 max_load_reduction=55298 max_load_reduction_pct=0.843781 moved_to_echo=32727 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8183, 0, 1, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 19, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8113, 6, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8187, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6538, 0, 1, 2, 0, 0, 4, 0, 0, 2474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2447, 0, 0, 5314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 294, 0, 0, 0, 1, 0, 3089, 0, 0, 0, 0, 2772, 1651, 33, 15, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8178, 8182, 8129, 8190, 9019, 7761, 7867] avg=8192.0000 max=9019 max_rank=5 min=7761 min_rank=6 imbalance_ratio=1.100952 max_load_reduction=56517 max_load_reduction_pct=0.862381 moved_to_echo=32837 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8191, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6535, 0, 0, 8, 0, 2, 0, 0, 0, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3696, 0, 0, 5306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 0, 0, 0, 0, 0, 4152, 0, 0, 0, 0, 2769, 1651, 33, 15, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8215, 8162, 8176, 8216, 8193, 6790, 9002, 8782] avg=8192.0000 max=9002 max_rank=6 min=6790 min_rank=5 imbalance_ratio=1.098877 max_load_reduction=56534 max_load_reduction_pct=0.862640 moved_to_echo=32767 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8181, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8159, 38, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 1, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6535, 0, 0, 0, 0, 0, 0, 0, 0, 2825, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 4204, 0, 0, 5356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 7, 1, 1, 0, 0, 0, 1158, 0, 0, 0, 0, 2795, 1651, 33, 16, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8199, 8182, 8152, 8193, 9360, 9570, 5676] avg=8192.0000 max=9570 max_rank=6 min=5676 min_rank=7 imbalance_ratio=1.168213 max_load_reduction=55966 max_load_reduction_pct=0.853973 moved_to_echo=32799 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 140, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6539, 0, 0, 0, 0, 0, 0, 0, 0, 2266, 9, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4285, 0, 0, 5383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 109, 0, 0, 0, 0, 0, 1370, 0, 0, 0, 0, 2809, 1652, 33, 15, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8297, 8195, 8171, 8194, 8815, 9668, 6000] avg=8192.0000 max=9668 max_rank=6 min=6000 min_rank=7 imbalance_ratio=1.180176 max_load_reduction=55868 max_load_reduction_pct=0.852478 moved_to_echo=32677 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8164, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 71, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6526, 0, 0, 9, 0, 1, 0, 0, 0, 425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 881, 0, 0, 5349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 362, 0, 0, 0, 0, 0, 6488, 0, 0, 0, 0, 2791, 1649, 33, 15, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8163, 8172, 8259, 8191, 6961, 6233, 11372] avg=8192.0000 max=11372 max_rank=7 min=6233 min_rank=6 imbalance_ratio=1.388184 max_load_reduction=54164 max_load_reduction_pct=0.826477 moved_to_echo=32757 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8145, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 2, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 8146, 10, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6534, 0, 0, 13, 0, 0, 0, 0, 0, 234, 21, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1744, 0, 0, 5375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 46, 1, 2, 0, 0, 0, 6052, 1, 0, 0, 0, 2809, 1653, 39, 15, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8176, 8182, 8206, 8191, 6804, 7120, 10647] avg=8192.0000 max=10647 max_rank=7 min=6804 min_rank=5 imbalance_ratio=1.299683 max_load_reduction=54889 max_load_reduction_pct=0.837540 moved_to_echo=32762 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[8181, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 8189, 44, 1, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 1, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 8188, 0, 3242, 0, 0, 0, 0, 0, 0, 8, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 3642, 0, 0, 8146, 0, 2, 0, 2, 0, 20, 1, 1, 0, 0, 0, 1285, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 8186, 143, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 2504, 9, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3757, 0, 0, 8192, 0, 1, 0, 0, 0, 185, 0, 0, 0, 0, 0, 1571, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[8184, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 8183, 20, 7, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 5, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8150, 6, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8189, 0, 1, 1, 0, 0, 4, 8192, 0, 2739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1956, 0, 0, 8085, 0, 1, 0, 0, 0, 444, 0, 0, 0, 1, 0, 3158, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[8164, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8191, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 4, 74, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 10, 0, 1, 0, 8184, 0, 390, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 648, 0, 0, 8134, 0, 22, 0, 0, 0, 432, 0, 0, 0, 0, 0, 6674, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[8098, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 1, 0, 0, 3, 0, 15, 0, 0, 0, 8191, 0, 0, 8181, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 40, 0, 15, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 8190, 0, 1077, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2052, 0, 0, 8146, 0, 4, 0, 0, 0, 61, 0, 1, 0, 0, 0, 5038, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[8140, 0, 15, 0, 1, 0, 0, 1, 1, 5, 0, 11, 0, 0, 0, 0, 0, 0, 1, 0, 8181, 16, 0, 8171, 0, 0, 2, 3, 0, 0, 0, 2, 1, 0, 1, 0, 0, 29, 0, 0, 7, 0, 0, 0, 1, 0, 7, 1, 4, 2, 105, 0, 0, 0, 0, 0, 8, 2, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 1, 4, 2, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 8175, 0, 1, 2, 0, 0, 1, 8179, 0, 4088, 0, 0, 4, 0, 1, 0, 60, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 2511, 0, 3, 7955, 2, 0, 3, 0, 0, 73, 0, 1, 3, 0, 0, 1579, 0, 0, 0, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[8191, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 9, 0, 2, 0, 8192, 0, 263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3062, 0, 0, 8081, 0, 0, 0, 0, 0, 173, 0, 0, 0, 0, 0, 4734, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[8145, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 2, 0, 13, 0, 0, 0, 8191, 0, 0, 8182, 7, 0, 1, 0, 0, 0, 0, 2, 0, 0, 1, 6, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 12, 0, 0, 0, 8186, 0, 250, 20, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1463, 0, 0, 8185, 0, 13, 0, 0, 0, 61, 2, 2, 0, 0, 0, 6290, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8181, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 44, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 1, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6365, 0, 0, 0, 0, 0, 0, 0, 0, 3242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 3642, 0, 0, 5818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 20, 1, 1, 0, 0, 0, 1285, 0, 0, 0, 0, 2328, 1822, 41, 17, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8203, 8182, 8146, 8192, 9607, 9471, 5532] avg=8192.0000 max=9607 max_rank=5 min=5532 min_rank=7 imbalance_ratio=1.172729 max_load_reduction=55929 max_load_reduction_pct=0.853409 moved_to_echo=32802 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8098, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 1, 0, 0, 3, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 8150, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 40, 0, 15, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6360, 0, 0, 0, 0, 0, 0, 0, 0, 1077, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2052, 0, 0, 5818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 61, 0, 1, 0, 0, 0, 5038, 0, 0, 0, 0, 2328, 1820, 41, 15, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8123, 8198, 8236, 8154, 8193, 7441, 7870, 9321] avg=8192.0000 max=9321 max_rank=7 min=7441 min_rank=5 imbalance_ratio=1.137817 max_load_reduction=56215 max_load_reduction_pct=0.857773 moved_to_echo=32825 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 143, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6368, 0, 0, 0, 0, 0, 0, 0, 0, 2504, 9, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3757, 0, 0, 5851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 185, 0, 0, 0, 0, 0, 1571, 0, 0, 0, 0, 2341, 1823, 41, 15, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8299, 8195, 8173, 8194, 8882, 9608, 5990] avg=8192.0000 max=9608 max_rank=6 min=5990 min_rank=7 imbalance_ratio=1.172852 max_load_reduction=55928 max_load_reduction_pct=0.853394 moved_to_echo=32674 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8184, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 20, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8109, 6, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8186, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6367, 0, 1, 1, 0, 0, 4, 0, 0, 2739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1956, 0, 0, 5774, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 444, 0, 0, 0, 1, 0, 3158, 0, 0, 0, 0, 2311, 1822, 41, 15, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8179, 8184, 8126, 8189, 9112, 7730, 7806] avg=8192.0000 max=9112 max_rank=5 min=7730 min_rank=6 imbalance_ratio=1.112305 max_load_reduction=56424 max_load_reduction_pct=0.860962 moved_to_echo=32837 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8140, 0, 15, 0, 1, 0, 0, 1, 1, 5, 0, 11, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 16, 0, 8140, 0, 0, 2, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 29, 0, 0, 7, 0, 0, 0, 1, 0, 7, 1, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 105, 0, 0, 0, 0, 0, 8, 2, 0, 0, 8117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 2, 0, 0, 4, 0, 0, 0, 0, 0, 0, 8181, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6356, 0, 1, 2, 0, 0, 1, 0, 0, 4088, 0, 0, 4, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 2511, 0, 3, 5681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 0, 0, 73, 0, 1, 3, 0, 0, 1579, 0, 0, 0, 4, 2274, 1819, 41, 16, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8164, 8209, 8238, 8193, 10454, 8258, 5828] avg=8192.0000 max=10454 max_rank=5 min=5828 min_rank=7 imbalance_ratio=1.276123 max_load_reduction=55082 max_load_reduction_pct=0.840485 moved_to_echo=32733 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8191, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6364, 0, 0, 9, 0, 2, 0, 0, 0, 263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3062, 0, 0, 5771, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 173, 0, 0, 0, 0, 0, 4734, 0, 0, 0, 0, 2310, 1821, 41, 15, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8215, 8161, 8177, 8213, 8192, 6638, 8833, 9107] avg=8192.0000 max=9107 max_rank=7 min=6638 min_rank=5 imbalance_ratio=1.111694 max_load_reduction=56429 max_load_reduction_pct=0.861038 moved_to_echo=32770 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8145, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 2, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 8147, 7, 0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6361, 0, 0, 12, 0, 0, 0, 0, 0, 250, 20, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1463, 0, 0, 5842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 61, 2, 2, 0, 0, 0, 6290, 1, 0, 0, 0, 2343, 1826, 44, 15, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8170, 8182, 8218, 8192, 6644, 7306, 10611] avg=8192.0000 max=10611 max_rank=7 min=6644 min_rank=5 imbalance_ratio=1.295288 max_load_reduction=54925 max_load_reduction_pct=0.838089 moved_to_echo=32753 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8164, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 74, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6357, 0, 0, 10, 0, 1, 0, 0, 0, 390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 648, 0, 0, 5809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 432, 0, 0, 0, 0, 0, 6674, 0, 0, 0, 0, 2325, 1819, 41, 15, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8162, 8171, 8268, 8191, 6758, 6460, 11341] avg=8192.0000 max=11341 max_rank=7 min=6460 min_rank=6 imbalance_ratio=1.384399 max_load_reduction=54195 max_load_reduction_pct=0.826950 moved_to_echo=32750 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 128, 0, 1152, 128, 128, 128, 128, 0, 28544, 128, 0, 0, 128, 22272, 13312, 384, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 128, 128, 0, 128, 0, 0, 0, 128, 0, 22912, 0, 128, 42752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 52352, 0, 128, 128, 0, 128, 128, 0, 0, 13312, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 0, 1536, 128, 128, 128, 128, 0, 30336, 128, 0, 0, 128, 18560, 14592, 384, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 128, 128, 0, 128, 0, 0, 0, 128, 0, 19200, 0, 128, 46464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 50944, 0, 128, 128, 0, 128, 128, 0, 0, 14592, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 256, 256, 128, 0, 128, 0, 0, 128, 128, 0, 0, 65152, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[128, 0, 128, 128, 128, 128, 128, 128, 128, 0, 128, 128, 128, 128, 128, 128, 65408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 0, 0, 128, 0, 65280, 256, 128, 128, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 256, 256, 128, 0, 128, 0, 0, 128, 128, 0, 0, 65280, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[65280, 0, 128, 0, 128, 128, 0, 128, 128, 128, 128, 128, 0, 128, 128, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 128, 0, 0, 128, 0, 65280, 256, 128, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 0, 128, 128, 128, 0, 0, 128, 0, 0, 0, 0, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[8074, 3, 0, 0, 0, 0, 0, 6, 0, 0, 0, 7, 0, 0, 3, 0, 0, 0, 7, 0, 0, 0, 4, 2, 0, 0, 0, 0, 0, 8134, 8172, 0, 1, 1, 0, 0, 4, 8163, 0, 0, 0, 2, 0, 0, 0, 0, 0, 36, 10, 0, 0, 0, 12, 0, 0, 0, 0, 1, 15, 0, 8171, 0, 0, 0, 0, 0, 2, 4, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 8167, 6, 1, 8159, 0, 0, 0, 0, 11, 0, 0, 0, 0, 4, 0, 0, 0, 0, 110, 0, 11, 0, 8, 0, 1, 0, 0, 0, 0, 19, 0, 2, 9, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 6] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[8192, 0, 5, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 8192, 0, 0, 0, 0, 0, 0, 8133, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 8190, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 8192, 0, 0, 8088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[8157, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 2, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 8108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[8163, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 131, 0, 0, 0, 0, 0, 0, 0, 0, 8106, 8183, 0, 0, 1, 1, 0, 0, 8143, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 1, 0, 0, 3, 0, 8184, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8144, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 8185, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[8188, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 0, 1, 0, 1, 1, 0, 0, 0, 8179, 8190, 0, 1, 0, 0, 0, 0, 8189, 3, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 5, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 8087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 226, 1, 0, 0, 0, 0, 0, 0, 0, 8182, 8191, 0, 0, 1, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8181, 0, 0, 3, 0, 2, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8188, 0, 0, 7964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8189, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 7804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[8139, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 168, 0, 1, 0, 0, 0, 0, 0, 0, 8186, 8185, 0, 0, 0, 0, 0, 3, 8180, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 8190, 9, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 1, 8049, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 7, 0, 4, 0, 1, 0, 2, 0, 1, 4, 0, 8189, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[8074, 3, 0, 0, 0, 0, 0, 6, 0, 0, 0, 7, 0, 0, 3, 0, 18, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4, 2, 0, 0, 0, 0, 0, 8044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 4, 8163, 0, 0, 0, 2, 0, 0, 0, 0, 0, 36, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 12, 0, 0, 0, 0, 1, 15, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 8172, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 137, 6, 1, 8159, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 110, 0, 11, 0, 8, 0, 1, 0, 0, 8029, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 2, 9, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8117, 8057, 8218, 8199, 8183, 8317, 8235, 8210] avg=8192.0000 max=8317 max_rank=5 min=8057 min_rank=1 imbalance_ratio=1.015259 max_load_reduction=57219 max_load_reduction_pct=0.873093 moved_to_echo=32945 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 8101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 7804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 314, 0, 0, 8054, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8217, 8179, 8195, 8186, 8193, 7941, 8440, 8185] avg=8192.0000 max=8440 max_rank=6 min=7941 min_rank=5 imbalance_ratio=1.030273 max_load_reduction=57096 max_load_reduction_pct=0.871216 moved_to_echo=32759 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[8163, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 131, 0, 0, 0, 0, 0, 0, 0, 0, 8016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 8143, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 1, 0, 0, 3, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 8144, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 28, 0, 0, 8054, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8148, 8174, 8195, 8188, 8297, 8155, 8189] avg=8192.0000 max=8297 max_rank=5 min=8148 min_rank=1 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32829 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[8157, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 8102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 8108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 8052, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8182, 8202, 8208, 8194, 8245, 8133, 8185] avg=8192.0000 max=8245 max_rank=5 min=8133 min_rank=6 imbalance_ratio=1.006470 max_load_reduction=57291 max_load_reduction_pct=0.874191 moved_to_echo=32757 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 226, 1, 0, 0, 0, 0, 0, 0, 0, 8092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8171, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 7964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 8050, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8182, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8319, 8192, 8175, 8200, 8101, 8156, 8188] avg=8192.0000 max=8319 max_rank=1 min=8101 min_rank=5 imbalance_ratio=1.015503 max_load_reduction=57217 max_load_reduction_pct=0.873062 moved_to_echo=32645 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[8188, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 0, 1, 0, 1, 1, 0, 0, 0, 8089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8189, 3, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 5, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 8087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8053, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8215, 8200, 8205, 8191, 8191, 8224, 8126, 8184] avg=8192.0000 max=8224 max_rank=5 min=8126 min_rank=6 imbalance_ratio=1.003906 max_load_reduction=57312 max_load_reduction_pct=0.874512 moved_to_echo=32725 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[8192, 0, 5, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 8098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 8180, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 137, 0, 0, 8088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 8054, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 8193, 8145, 8199, 8194, 8226, 8157, 8196] avg=8192.0000 max=8226 max_rank=0 min=8145 min_rank=2 imbalance_ratio=1.004150 max_load_reduction=57310 max_load_reduction_pct=0.874481 moved_to_echo=32773 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[8139, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 20, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 168, 0, 1, 0, 0, 0, 0, 0, 0, 8088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8180, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 9, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 8167, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 0, 1, 8049, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 6, 0, 0, 8047, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4, 0, 1, 0, 2, 0, 1, 4, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8258, 8205, 8183, 8193, 8185, 8134, 8199] avg=8192.0000 max=8258 max_rank=1 min=8134 min_rank=6 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32711 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[8188, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 0, 1, 0, 1, 1, 0, 0, 0, 8180, 8191, 0, 1, 0, 0, 0, 0, 8188, 3, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 4, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 8076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[8139, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 182, 0, 1, 0, 0, 0, 0, 0, 0, 8186, 8185, 0, 0, 0, 0, 0, 3, 8179, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 8190, 9, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 1, 8036, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 7, 0, 4, 0, 1, 0, 2, 0, 1, 4, 0, 8189, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[8162, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 134, 0, 0, 0, 0, 0, 0, 0, 0, 8097, 8183, 0, 0, 0, 1, 0, 0, 8144, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 1, 0, 0, 3, 0, 8185, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8149, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 8186, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[8192, 0, 5, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 8192, 0, 0, 0, 0, 0, 0, 8135, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 8190, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 8192, 0, 0, 8082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[8075, 3, 0, 0, 0, 0, 0, 5, 0, 0, 0, 8, 0, 0, 3, 0, 0, 0, 7, 0, 0, 0, 4, 2, 0, 0, 0, 0, 0, 8133, 8172, 0, 1, 1, 0, 0, 4, 8163, 0, 0, 0, 3, 0, 0, 0, 0, 0, 39, 10, 0, 0, 0, 12, 0, 0, 0, 0, 1, 16, 0, 8171, 0, 0, 0, 0, 0, 2, 4, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 8167, 6, 1, 8155, 0, 0, 0, 0, 11, 0, 0, 0, 0, 3, 0, 0, 0, 0, 109, 0, 11, 0, 8, 0, 1, 0, 0, 0, 0, 20, 0, 2, 9, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 6] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 7796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 234, 1, 0, 0, 0, 0, 0, 0, 0, 8183, 8191, 0, 0, 1, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8180, 0, 0, 3, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8189, 0, 0, 7956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8189, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[8160, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 2, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 8104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[8188, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 0, 1, 0, 1, 1, 0, 0, 0, 8086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8188, 3, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 4, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 0, 0, 8076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8047, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8215, 8207, 8204, 8190, 8192, 8219, 8125, 8184] avg=8192.0000 max=8219 max_rank=5 min=8125 min_rank=6 imbalance_ratio=1.003296 max_load_reduction=57317 max_load_reduction_pct=0.874588 moved_to_echo=32720 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[8162, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 134, 0, 0, 0, 0, 0, 0, 0, 0, 8004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8144, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 1, 0, 0, 3, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 0, 0, 8149, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 27, 0, 0, 8048, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8139, 8174, 8196, 8188, 8308, 8153, 8190] avg=8192.0000 max=8308 max_rank=5 min=8139 min_rank=1 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32839 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[8139, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 19, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 182, 0, 1, 0, 0, 0, 0, 0, 0, 8087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8179, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 10, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 8172, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 0, 1, 8036, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 5, 0, 0, 8039, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4, 0, 1, 0, 2, 0, 1, 4, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8271, 8201, 8190, 8194, 8178, 8127, 8198] avg=8192.0000 max=8271 max_rank=1 min=8127 min_rank=6 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32697 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 8096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 0, 0, 7796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 327, 0, 0, 8048, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8217, 8169, 8195, 8185, 8193, 7939, 8453, 8185] avg=8192.0000 max=8453 max_rank=6 min=7939 min_rank=5 imbalance_ratio=1.031860 max_load_reduction=57083 max_load_reduction_pct=0.871017 moved_to_echo=32770 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[8075, 3, 0, 0, 0, 0, 0, 5, 0, 0, 0, 8, 0, 0, 3, 0, 17, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4, 2, 0, 0, 0, 0, 0, 8039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 4, 8163, 0, 0, 0, 3, 0, 0, 0, 0, 0, 39, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 12, 0, 0, 0, 0, 1, 16, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8172, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 143, 6, 1, 8155, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 109, 0, 11, 0, 8, 0, 1, 0, 0, 8023, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 2, 9, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8118, 8052, 8222, 8199, 8182, 8320, 8232, 8211] avg=8192.0000 max=8320 max_rank=5 min=8052 min_rank=1 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32945 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[8160, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 8097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 0, 0, 8104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 8046, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8181, 8201, 8205, 8194, 8247, 8132, 8185] avg=8192.0000 max=8247 max_rank=5 min=8132 min_rank=6 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32758 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 234, 1, 0, 0, 0, 0, 0, 0, 0, 8088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8169, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 0, 0, 7956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 8045, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8182, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8323, 8192, 8173, 8199, 8099, 8157, 8188] avg=8192.0000 max=8323 max_rank=1 min=8099 min_rank=5 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32643 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[8192, 0, 5, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 8094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8135, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 8179, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 143, 0, 0, 8082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 8048, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 8194, 8147, 8198, 8194, 8226, 8157, 8195] avg=8192.0000 max=8226 max_rank=5 min=8147 min_rank=2 imbalance_ratio=1.004150 max_load_reduction=57310 max_load_reduction_pct=0.874481 moved_to_echo=32772 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 0, 0, 0, 128, 0, 128, 0, 128, 128, 512, 0, 0, 64512, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 0, 0, 128, 1152, 128, 128, 64512, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 896, 128, 128, 128, 128, 128, 0, 0, 0, 64640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 128, 128, 0, 0, 0, 128, 0, 128, 0, 128, 128, 512, 0, 0, 64384, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[65408, 128, 128, 128, 128, 0, 0, 128, 0, 0, 0, 128, 0, 0, 128, 128, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 0, 128, 128, 128, 0, 128, 0, 128, 128, 0, 65536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 0, 128, 1152, 128, 128, 64384, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 128, 128, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 128, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 128, 128, 1024, 128, 128, 128, 128, 128, 0, 0, 0, 64640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 0, 128, 65408, 128, 0, 0, 128, 0, 128, 0, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 0, 128, 128, 128, 0, 128, 0, 128, 128, 0, 65536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 0, 128, 128, 0, 0, 128, 0, 128, 128, 128, 65408, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 128, 128, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 128, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 7919, 0, 0, 0, 7972, 0, 0, 0, 0, 8176, 0, 8185, 4, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8, 10, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8133, 0, 0, 0, 0, 186, 0, 0, 67, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 0, 0, 0, 0, 8056, 0, 0, 0, 0, 0, 0, 8190, 288, 0, 0, 0, 0, 9, 0, 8183, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7740, 0, 0, 0, 8171, 0, 0, 0, 0, 8180, 0, 8185, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 86, 0, 0, 526, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 0, 0, 0, 7939, 0, 0, 0, 0, 0, 0, 8191, 92, 0, 0, 0, 0, 0, 0, 8182, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8121, 0, 0, 0, 8140, 0, 0, 0, 0, 8165, 0, 8182, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 1, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 68, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 8147, 0, 0, 0, 0, 0, 0, 8191, 9, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 8188, 0, 0, 0, 8151, 0, 0, 0, 0, 7976, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 8126, 0, 1, 0, 0, 64, 0, 0, 17, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 1, 0, 3, 8191, 5, 0, 0, 0, 0, 0, 2, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7920, 0, 0, 0, 8157, 0, 0, 0, 0, 7809, 0, 8184, 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 374, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 8180, 0, 0, 0, 0, 5, 0, 2, 272, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 8141, 0, 0, 0, 0, 0, 1, 8188, 50, 0, 0, 1, 0, 0, 3, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 7, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8181, 0, 0, 0, 8175, 10, 0, 0, 0, 7907, 0, 8140, 0, 0, 1, 10, 3, 0, 0, 0, 0, 0, 13, 0, 4, 0, 0, 12, 0, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 9, 2, 8123, 0, 0, 0, 0, 16, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 9, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7900, 0, 0, 0, 1, 0, 75, 8170, 0, 0, 0, 1, 3, 0, 57, 8163, 0, 8] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7831, 0, 0, 0, 7705, 0, 0, 0, 0, 7740, 0, 8192, 244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 571, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 17, 0, 0, 563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8089, 0, 0, 0, 0, 0, 0, 8192, 6, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7947, 0, 0, 1, 8030, 0, 0, 0, 0, 8058, 0, 8189, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 8192, 0, 0, 0, 0, 14, 0, 0, 299, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 8086, 0, 0, 0, 0, 0, 0, 8191, 181, 1, 0, 0, 0, 0, 3, 8190, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 7919, 0, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 0, 0, 0, 0, 8176, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8, 10, 0, 0, 0, 7969, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8133, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 0, 0, 67, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7590, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 18, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 0, 0, 0, 0, 8056, 0, 69, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 288, 0, 0, 0, 0, 9, 0, 8091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8127, 8330, 7992, 8158, 8060, 8207, 8274, 8388] avg=8192.0000 max=8388 max_rank=7 min=7992 min_rank=2 imbalance_ratio=1.023926 max_load_reduction=57148 max_load_reduction_pct=0.872009 moved_to_echo=32929 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8121, 0, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 0, 0, 0, 0, 8165, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 1, 0, 0, 0, 7966, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7750, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 8147, 0, 69, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 8100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8328, 8349, 8008, 8201, 8100, 8191, 8250, 8109] avg=8192.0000 max=8349 max_rank=1 min=8008 min_rank=2 imbalance_ratio=1.019165 max_load_reduction=57187 max_load_reduction_pct=0.872604 moved_to_echo=32650 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 8188, 0, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 0, 0, 0, 0, 7976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 5, 0, 0, 0, 7971, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 8126, 0, 1, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 17, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7761, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 69, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 5, 0, 0, 0, 0, 0, 2, 8099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8397, 8129, 8217, 8156, 8060, 8191, 8276, 8110] avg=8192.0000 max=8397 max_rank=0 min=8060 min_rank=4 imbalance_ratio=1.025024 max_load_reduction=57139 max_load_reduction_pct=0.871872 moved_to_echo=32637 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7740, 0, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 0, 0, 0, 0, 8180, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 7969, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 526, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7780, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 0, 0, 0, 7939, 0, 69, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 0, 0, 8090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7946, 8351, 7995, 8216, 8609, 8190, 8047, 8182] avg=8192.0000 max=8609 max_rank=4 min=7946 min_rank=0 imbalance_ratio=1.050903 max_load_reduction=56927 max_load_reduction_pct=0.868637 moved_to_echo=33028 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8181, 0, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 10, 0, 0, 0, 7907, 0, 0, 0, 0, 1, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 4, 0, 0, 12, 0, 371, 0, 0, 0, 0, 7925, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 9, 2, 8123, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 7783, 215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 9, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7900, 0, 69, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 75, 0, 0, 0, 0, 1, 3, 0, 57, 8071, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8428, 8085, 8327, 8158, 8119, 8191, 8012, 8216] avg=8192.0000 max=8428 max_rank=0 min=8012 min_rank=6 imbalance_ratio=1.028809 max_load_reduction=57108 max_load_reduction_pct=0.871399 moved_to_echo=32538 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7920, 0, 211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 0, 0, 0, 0, 7809, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 374, 1, 0, 0, 0, 7969, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 8180, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2, 272, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 7770, 215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 8141, 0, 74, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 50, 0, 0, 1, 0, 0, 3, 8087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8135, 7951, 8347, 8212, 8268, 8186, 8295, 8142] avg=8192.0000 max=8347 max_rank=2 min=7951 min_rank=1 imbalance_ratio=1.018921 max_load_reduction=57189 max_load_reduction_pct=0.872635 moved_to_echo=32891 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7831, 0, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 0, 0, 0, 7740, 0, 0, 244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 571, 2, 0, 0, 0, 7976, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7336, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8089, 0, 69, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 8100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8025, 8129, 8551, 8215, 8132, 8190, 8188, 8106] avg=8192.0000 max=8551 max_rank=2 min=8025 min_rank=0 imbalance_ratio=1.043823 max_load_reduction=56985 max_load_reduction_pct=0.869522 moved_to_echo=32616 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7947, 0, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 152, 0, 0, 0, 0, 8058, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 0, 0, 7973, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 8192, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 299, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 2, 7645, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 8086, 0, 69, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 1, 0, 0, 0, 0, 3, 8098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8150, 8212, 8099, 8220, 8188, 8190, 8194, 8283] avg=8192.0000 max=8283 max_rank=7 min=8099 min_rank=2 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32855 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 8188, 0, 0, 0, 8155, 0, 0, 0, 0, 8023, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 224, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 8094, 0, 0, 0, 0, 61, 0, 0, 21, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 1, 0, 3, 8191, 1, 0, 0, 0, 0, 0, 1, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 7914, 0, 0, 0, 7967, 0, 0, 0, 0, 8178, 0, 8184, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 6, 10, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 8133, 0, 0, 0, 0, 181, 0, 0, 90, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 0, 0, 0, 0, 8049, 0, 0, 0, 0, 0, 2, 8190, 289, 0, 0, 0, 0, 6, 0, 8182, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8117, 0, 0, 0, 8142, 0, 0, 0, 0, 8156, 0, 8182, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 1, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 72, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 8147, 0, 0, 0, 0, 0, 0, 8191, 7, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7907, 0, 0, 0, 8151, 0, 1, 0, 0, 7702, 0, 8187, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 479, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 8181, 0, 0, 0, 0, 3, 0, 1, 290, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 8141, 0, 0, 0, 0, 0, 1, 8189, 51, 0, 0, 1, 0, 0, 3, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7945, 0, 0, 1, 8031, 0, 0, 0, 0, 8047, 0, 8189, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 8192, 0, 0, 0, 0, 12, 0, 0, 314, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 8075, 0, 0, 0, 0, 0, 0, 8191, 175, 1, 0, 0, 0, 0, 3, 8190, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7728, 0, 0, 0, 8171, 0, 0, 0, 0, 8175, 0, 8185, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 87, 0, 0, 579, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 0, 0, 0, 7893, 0, 0, 0, 0, 0, 0, 8191, 92, 0, 0, 0, 0, 0, 0, 8183, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7831, 0, 0, 0, 7688, 0, 0, 0, 0, 7580, 0, 8192, 244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 704, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 17, 0, 0, 621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8078, 0, 0, 0, 0, 0, 0, 8192, 5, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 7, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8181, 0, 0, 0, 8175, 10, 0, 0, 0, 7918, 0, 8137, 0, 0, 1, 10, 3, 0, 0, 0, 0, 0, 14, 0, 4, 2, 0, 12, 0, 374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8, 2, 8122, 0, 0, 0, 0, 19, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 9, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7889, 0, 0, 0, 1, 0, 71, 8170, 0, 0, 0, 0, 2, 0, 57, 8161, 0, 9] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 7914, 0, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 178, 0, 0, 0, 0, 8178, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 6, 10, 0, 0, 0, 7934, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 8133, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 0, 0, 90, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7533, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 18, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 0, 0, 0, 0, 8049, 0, 60, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 289, 0, 0, 0, 0, 6, 0, 8092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8128, 8359, 7955, 8166, 8055, 8207, 8277, 8389] avg=8192.0000 max=8389 max_rank=7 min=7955 min_rank=2 imbalance_ratio=1.024048 max_load_reduction=57147 max_load_reduction_pct=0.871994 moved_to_echo=32928 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 8188, 0, 211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, 0, 0, 0, 0, 8023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 224, 5, 0, 0, 0, 7937, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 8094, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 21, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7711, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 60, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 1, 0, 0, 0, 0, 0, 1, 8101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8403, 8205, 8168, 8131, 8046, 8190, 8286, 8107] avg=8192.0000 max=8403 max_rank=0 min=8046 min_rank=4 imbalance_ratio=1.025757 max_load_reduction=57133 max_load_reduction_pct=0.871780 moved_to_echo=32629 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8117, 0, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 0, 0, 0, 0, 8156, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 1, 0, 0, 0, 7932, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7698, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 8147, 0, 60, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 8102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8328, 8370, 7995, 8193, 8090, 8191, 8260, 8109] avg=8192.0000 max=8370 max_rank=1 min=7995 min_rank=2 imbalance_ratio=1.021729 max_load_reduction=57166 max_load_reduction_pct=0.872284 moved_to_echo=32650 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7945, 0, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 180, 0, 0, 0, 0, 8047, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 0, 0, 0, 7939, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 8192, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 314, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 2, 7593, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 8075, 0, 60, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 175, 1, 0, 0, 0, 0, 3, 8100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8153, 8229, 8081, 8226, 8183, 8190, 8195, 8279] avg=8192.0000 max=8279 max_rank=7 min=8081 min_rank=2 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32847 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7831, 0, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 173, 0, 0, 0, 0, 7580, 0, 0, 244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 704, 3, 0, 0, 0, 7943, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7269, 249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8078, 0, 60, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 8102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8029, 7997, 8652, 8219, 8156, 8190, 8186, 8107] avg=8192.0000 max=8652 max_rank=2 min=7997 min_rank=1 imbalance_ratio=1.056152 max_load_reduction=56884 max_load_reduction_pct=0.867981 moved_to_echo=32639 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8181, 0, 211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 10, 0, 0, 0, 7918, 0, 0, 0, 0, 1, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 4, 2, 0, 12, 0, 374, 0, 0, 0, 0, 7888, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8, 2, 8122, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 7729, 249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 9, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7889, 0, 60, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 71, 0, 0, 0, 0, 0, 2, 0, 57, 8071, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8431, 8126, 8296, 8163, 8106, 8191, 8012, 8211] avg=8192.0000 max=8431 max_rank=0 min=8012 min_rank=6 imbalance_ratio=1.029175 max_load_reduction=57105 max_load_reduction_pct=0.871353 moved_to_echo=32520 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7728, 0, 211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 0, 0, 0, 0, 8175, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 7936, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 579, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7726, 249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 0, 0, 0, 7893, 0, 60, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 0, 0, 8093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7939, 8377, 7968, 8223, 8642, 8191, 8011, 8185] avg=8192.0000 max=8642 max_rank=4 min=7939 min_rank=0 imbalance_ratio=1.054932 max_load_reduction=56894 max_load_reduction_pct=0.868134 moved_to_echo=33029 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7907, 0, 215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 0, 1, 0, 0, 7702, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 479, 0, 0, 0, 0, 7938, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 8181, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 290, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 7711, 249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 8141, 0, 66, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 51, 0, 0, 1, 0, 0, 3, 8093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8125, 7873, 8421, 8215, 8258, 8186, 8309, 8149] avg=8192.0000 max=8421 max_rank=2 min=7873 min_rank=1 imbalance_ratio=1.027954 max_load_reduction=57115 max_load_reduction_pct=0.871506 moved_to_echo=32902 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 0, 128, 0, 128, 128, 0, 128, 0, 1792, 128, 0, 0, 0, 63744, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 1280, 128, 0, 0, 0, 64128, 0, 0, 384, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 0, 128, 0, 640, 128, 0, 128, 128, 128, 128, 64768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 128, 256, 128, 0, 0, 0, 64640, 0, 640, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 512, 0, 128, 1920, 128, 0, 0, 0, 0, 128, 0, 128, 128, 0, 128, 61440, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 128, 0, 0, 0, 0, 128, 128, 0, 128, 0, 128, 63872, 0, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 0, 128, 128, 0, 0, 0, 128, 128, 128, 128, 65408, 0, 128, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 0, 128, 0, 128, 128, 0, 128, 0, 2048, 128, 0, 0, 0, 63488, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 128, 256, 128, 0, 0, 0, 64512, 0, 512, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 1536, 128, 128, 0, 0, 63872, 0, 0, 384, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 512, 0, 128, 2176, 128, 0, 0, 0, 0, 128, 0, 128, 128, 0, 128, 61056, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 128, 0, 0, 0, 0, 128, 128, 0, 128, 0, 128, 63872, 0, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 128, 0, 0, 0, 128, 128, 128, 128, 65280, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 0, 128, 0, 0, 0, 0, 0, 128, 128, 128, 0, 0, 128, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 0, 128, 0, 640, 128, 0, 128, 128, 128, 128, 64768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 7, 1, 5, 0, 8150, 0, 16, 0, 21, 0, 0, 0, 2767, 8154, 0, 8166, 8013, 0, 10, 0, 0, 5, 0, 6, 19, 1, 0, 0, 0, 0, 0, 0, 0, 98, 17, 0, 0, 0, 0, 1, 0, 0, 0, 7, 0, 0, 0, 0, 14, 0, 0, 8170, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 21, 8128, 0, 0, 0, 0, 0, 0, 1697, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 7, 3861, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8142, 0, 0, 0, 0, 6] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 4650, 8192, 0, 8192, 8188, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 2, 0, 801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 2657, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 1, 8182, 0, 0, 0, 0, 0, 0, 1, 7650, 8191, 0, 8191, 8125, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 83, 0, 151, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 8190, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8016, 8191, 0, 8189, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 2, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 38, 0, 0, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 1, 7439, 8186, 0, 8192, 8167, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1, 0, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 574, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 2, 0, 8043, 8155, 0, 8190, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 40, 0, 0, 0, 8192, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 2988, 8178, 0, 8190, 8190, 0, 0, 0, 2, 1, 0, 1, 0, 0, 0, 0, 28, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 1, 0, 43, 0, 412, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4725, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5, 0, 14, 2, 0, 0, 8190, 0, 0, 0, 6, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 8115, 8192, 0, 8191, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 32, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1981, 0, 0, 0, 0, 0, 0, 0, 4650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 8188, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7753, 0, 0, 0, 0, 2, 0, 801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 2657, 0, 0, 0, 0, 0, 0, 0, 0, 6195, 438, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6631, 8209, 8178, 8190, 8556, 8202, 9381, 8189] avg=8192.0000 max=9381 max_rank=6 min=6631 min_rank=0 imbalance_ratio=1.145142 max_load_reduction=56155 max_load_reduction_pct=0.856857 moved_to_echo=34328 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 1, 1978, 0, 0, 0, 0, 0, 0, 1, 7650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 8125, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 8161, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7736, 0, 0, 0, 0, 83, 0, 151, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 365, 0, 0, 0, 0, 0, 0, 0, 0, 6188, 437, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9632, 8149, 8189, 8190, 7972, 8205, 7003, 8196] avg=8192.0000 max=9632 max_rank=0 min=7003 min_rank=6 imbalance_ratio=1.175781 max_load_reduction=55904 max_load_reduction_pct=0.853027 moved_to_echo=31376 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1974, 0, 0, 0, 0, 0, 2, 0, 8043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7752, 0, 0, 0, 0, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6176, 438, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 40, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10019, 8201, 8176, 8210, 7903, 8165, 6625, 8237] avg=8192.0000 max=10019 max_rank=0 min=6625 min_rank=6 imbalance_ratio=1.223022 max_load_reduction=55517 max_load_reduction_pct=0.847122 moved_to_echo=30930 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 1, 5, 0, 1970, 0, 16, 0, 21, 0, 0, 0, 2767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 8013, 0, 10, 0, 0, 5, 0, 6, 19, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 17, 0, 0, 0, 0, 1, 0, 0, 0, 7, 0, 0, 0, 8136, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 8169, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 7694, 0, 0, 0, 0, 0, 0, 1697, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 8154, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 7, 3861, 0, 0, 0, 4, 0, 0, 0, 0, 6164, 434, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8137, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4787, 8074, 8275, 8187, 9420, 8168, 10481, 8144] avg=8192.0000 max=10481 max_rank=6 min=4787 min_rank=0 imbalance_ratio=1.279419 max_load_reduction=55055 max_load_reduction_pct=0.840073 moved_to_echo=36213 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 1981, 0, 0, 0, 0, 0, 0, 0, 8016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 2, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7748, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8191, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 120, 0, 0, 0, 0, 0, 0, 0, 0, 6195, 437, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9998, 8211, 8175, 8192, 7773, 8203, 6797, 8187] avg=8192.0000 max=9998 max_rank=0 min=6797 min_rank=6 imbalance_ratio=1.220459 max_load_reduction=55538 max_load_reduction_pct=0.847443 moved_to_echo=30960 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1980, 0, 0, 0, 0, 0, 1, 0, 8115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7752, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8192, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6195, 438, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10096, 8204, 8178, 8190, 7802, 8205, 6673, 8188] avg=8192.0000 max=10096 max_rank=0 min=6673 min_rank=6 imbalance_ratio=1.232422 max_load_reduction=55440 max_load_reduction_pct=0.845947 moved_to_echo=30868 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1979, 0, 0, 0, 0, 0, 0, 1, 7439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 8167, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7754, 0, 0, 0, 0, 1, 0, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 574, 0, 0, 3, 0, 0, 0, 0, 0, 6190, 438, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9419, 8249, 8180, 8191, 7902, 8196, 7212, 8187] avg=8192.0000 max=9419 max_rank=0 min=7212 min_rank=6 imbalance_ratio=1.149780 max_load_reduction=56117 max_load_reduction_pct=0.856277 moved_to_echo=31497 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1966, 0, 0, 0, 0, 0, 0, 0, 2988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 8190, 0, 0, 0, 2, 1, 0, 1, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 8161, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7750, 0, 0, 1, 0, 43, 0, 412, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8178, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4725, 2, 1, 0, 0, 0, 0, 0, 0, 6181, 441, 11, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5, 0, 14, 2, 0, 0, 8179, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4954, 8239, 8185, 8186, 8208, 8192, 11364, 8208] avg=8192.0000 max=11364 max_rank=6 min=4954 min_rank=0 imbalance_ratio=1.387207 max_load_reduction=54172 max_load_reduction_pct=0.826599 moved_to_echo=35972 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 1, 8183, 0, 0, 0, 0, 0, 0, 1, 7506, 8191, 0, 8191, 8126, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 87, 0, 300, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 8190, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8011, 8191, 0, 8189, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 2, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 8185, 0, 0, 0, 0, 24, 0, 5, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 41, 0, 0, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 2963, 8177, 0, 8190, 8190, 0, 0, 0, 2, 1, 0, 1, 0, 0, 0, 0, 32, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 1, 0, 46, 0, 450, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4704, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 0, 13, 2, 0, 0, 8190, 0, 0, 0, 6, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 2, 0, 8032, 8158, 0, 8190, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 38, 0, 0, 0, 8192, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 8, 1, 7, 0, 8150, 0, 16, 0, 22, 0, 0, 0, 2548, 8154, 0, 8166, 8011, 0, 9, 0, 0, 5, 0, 6, 19, 1, 0, 0, 0, 0, 0, 0, 0, 96, 17, 0, 0, 0, 0, 1, 0, 0, 0, 7, 0, 0, 0, 0, 14, 0, 0, 8170, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 0, 20, 8128, 0, 0, 0, 0, 0, 0, 2039, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 7, 3741, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8143, 0, 0, 0, 0, 7] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 8116, 8192, 0, 8191, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 34, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 4634, 8192, 0, 8192, 8188, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 2, 0, 1170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 2325, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 7274, 8186, 0, 8192, 8169, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1, 0, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 657, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 2053, 0, 0, 0, 0, 0, 0, 0, 8011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 2, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7623, 0, 0, 0, 0, 24, 0, 5, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8191, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 116, 0, 0, 0, 0, 0, 0, 0, 0, 6123, 562, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10065, 8211, 8175, 8192, 7654, 8203, 6849, 8187] avg=8192.0000 max=10065 max_rank=0 min=6849 min_rank=6 imbalance_ratio=1.228638 max_load_reduction=55471 max_load_reduction_pct=0.846420 moved_to_echo=30893 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2047, 0, 0, 0, 0, 0, 2, 0, 8032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7628, 0, 0, 0, 0, 158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 6103, 562, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 38, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10081, 8201, 8175, 8211, 7786, 8167, 6680, 8235] avg=8192.0000 max=10081 max_rank=0 min=6680 min_rank=6 imbalance_ratio=1.230591 max_load_reduction=55455 max_load_reduction_pct=0.846176 moved_to_echo=30868 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 1, 2051, 0, 0, 0, 0, 0, 0, 1, 7506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 8126, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 8161, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7611, 0, 0, 0, 0, 87, 0, 300, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 354, 0, 0, 0, 0, 0, 0, 0, 0, 6116, 561, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9561, 8152, 8189, 8190, 8000, 8204, 7047, 8193] avg=8192.0000 max=9561 max_rank=0 min=7047 min_rank=6 imbalance_ratio=1.167114 max_load_reduction=55975 max_load_reduction_pct=0.854111 moved_to_echo=31444 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2053, 0, 0, 0, 0, 0, 1, 0, 8116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7628, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8192, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6122, 562, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10170, 8205, 8178, 8190, 7675, 8204, 6726, 8188] avg=8192.0000 max=10170 max_rank=0 min=6726 min_rank=6 imbalance_ratio=1.241455 max_load_reduction=55366 max_load_reduction_pct=0.844818 moved_to_echo=30793 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2053, 0, 0, 0, 0, 0, 0, 0, 4634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 8188, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7628, 0, 0, 0, 0, 2, 0, 1170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 2325, 0, 0, 0, 0, 0, 0, 0, 0, 6123, 562, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6687, 8210, 8178, 8190, 8800, 8201, 9081, 8189] avg=8192.0000 max=9081 max_rank=6 min=6687 min_rank=0 imbalance_ratio=1.108521 max_load_reduction=56455 max_load_reduction_pct=0.861435 moved_to_echo=34271 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 1, 7, 0, 2043, 0, 16, 0, 22, 0, 0, 0, 2548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 8011, 0, 9, 0, 0, 5, 0, 6, 19, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 17, 0, 0, 0, 0, 1, 0, 0, 0, 7, 0, 0, 0, 8136, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 8169, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 7570, 0, 0, 0, 0, 0, 0, 2039, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 8154, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 7, 3741, 0, 0, 0, 4, 0, 0, 0, 0, 6092, 558, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8138, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4645, 8072, 8272, 8186, 9636, 8167, 10412, 8146] avg=8192.0000 max=10412 max_rank=6 min=4645 min_rank=0 imbalance_ratio=1.270996 max_load_reduction=55124 max_load_reduction_pct=0.841125 moved_to_echo=36361 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2051, 0, 0, 0, 0, 0, 0, 0, 7274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 8169, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7630, 0, 0, 0, 0, 1, 0, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 657, 0, 0, 3, 0, 0, 0, 0, 0, 6116, 562, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9325, 8252, 8180, 8191, 7860, 8195, 7346, 8187] avg=8192.0000 max=9325 max_rank=0 min=7346 min_rank=6 imbalance_ratio=1.138306 max_load_reduction=56211 max_load_reduction_pct=0.857712 moved_to_echo=31588 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2039, 0, 0, 0, 0, 0, 0, 0, 2963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8190, 0, 0, 0, 2, 1, 0, 1, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 8167, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7626, 0, 0, 1, 0, 46, 0, 450, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8177, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4704, 2, 4, 0, 0, 0, 0, 0, 0, 6111, 565, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 0, 13, 2, 0, 0, 8184, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5002, 8233, 8189, 8186, 8125, 8195, 11395, 8211] avg=8192.0000 max=11395 max_rank=6 min=5002 min_rank=0 imbalance_ratio=1.390991 max_load_reduction=54141 max_load_reduction_pct=0.826126 moved_to_echo=35926 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 0, 0, 0, 256, 0, 128, 12416, 128, 128, 128, 128, 0, 0, 0, 0, 49536, 3584, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 128, 128, 128, 128, 15872, 0, 128, 0, 128, 0, 128, 128, 49792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 61952, 0, 0, 128, 0, 384, 0, 3328, 128, 0, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 128, 128, 128, 128, 16512, 0, 128, 0, 128, 0, 128, 128, 49152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 0, 0, 0, 256, 0, 128, 11904, 128, 128, 128, 128, 0, 0, 0, 0, 49024, 4608, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[256, 65280, 0, 128, 0, 128, 128, 0, 128, 128, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 128, 0, 0, 0, 128, 128, 0, 0, 128, 0, 0, 128, 65280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 0, 65536, 128, 0, 0, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 128, 128, 0, 128, 128, 0, 0, 65536, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 0, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 128, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[256, 65280, 0, 128, 0, 128, 128, 0, 128, 128, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 128, 128, 0, 128, 128, 0, 0, 65536, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 128, 128, 0, 0, 0, 128, 128, 0, 0, 128, 0, 0, 128, 65280, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 61056, 0, 0, 128, 0, 384, 0, 4224, 128, 0, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 8123, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 8139, 0, 9, 7996, 0, 0, 0, 0, 0, 0, 0, 138, 0, 0, 0, 0, 0, 2, 0, 6, 1, 0, 0, 99, 0, 0, 37, 0, 0, 0, 1, 0, 0, 0, 0, 8191, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 7648, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8189, 62, 0, 8118, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 54, 0, 24, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 1, 0, 8187, 0, 0, 8168, 1, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 38, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 8028, 0, 0, 0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 6, 2, 0, 8177, 0, 6, 7833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 1, 0, 0, 77, 424, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8164, 0, 4, 0, 14, 0, 0, 0, 8165, 0, 0, 10, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 6, 1, 8138, 1, 0, 0, 0, 0, 0, 8162, 0, 0, 7916, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 3, 0, 37, 0, 0, 0, 0, 0, 0, 339, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7, 0, 0, 36, 41, 0, 3, 14, 0, 0, 8037, 0, 2, 0, 16, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 0, 14, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 8105, 0, 0, 0, 0, 0, 0, 0, 8029, 0, 0, 2, 0, 0, 0, 8192, 0, 0, 0, 3, 0, 0, 8113, 83, 36, 8027, 0, 0, 0, 0, 0, 3, 0, 167, 0, 0, 0, 0, 0, 142, 0, 67, 0, 0, 4, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 1, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8189, 0, 1, 2, 0, 0, 0, 0, 8132, 0, 0, 1, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8189, 0, 41, 8030, 0, 0, 0, 1, 0, 2, 0, 80, 0, 0, 0, 4, 0, 80, 0, 1, 0, 0, 16, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 17, 0, 0, 0, 1, 0, 8172, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 0, 0, 7648, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 0, 0, 0, 0, 0, 7948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8098, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8228, 8222, 7797, 8490, 8193, 8221, 8197, 8188] avg=8192.0000 max=8490 max_rank=3 min=7797 min_rank=2 imbalance_ratio=1.036377 max_load_reduction=57046 max_load_reduction_pct=0.870453 moved_to_echo=32799 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 8123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 0, 9, 7996, 0, 0, 0, 0, 0, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 6, 1, 0, 0, 99, 0, 0, 37, 0, 0, 7897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8191, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 8096, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8207, 8286, 8042, 8196, 8221, 8189, 8213] avg=8192.0000 max=8286 max_rank=2 min=8042 min_rank=3 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32819 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 8105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8029, 0, 0, 2, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 143, 83, 36, 8027, 0, 0, 0, 0, 0, 3, 0, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 0, 67, 0, 0, 4, 0, 0, 0, 57, 0, 0, 7871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8120, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8097, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8061, 8462, 8141, 8193, 8130, 8197, 8188] avg=8192.0000 max=8462 max_rank=2 min=8061 min_rank=1 imbalance_ratio=1.032959 max_load_reduction=57074 max_load_reduction_pct=0.870880 moved_to_echo=32708 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 62, 0, 8118, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 24, 1, 0, 0, 1, 0, 0, 0, 0, 0, 7945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8098, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8203, 8325, 8025, 8194, 8202, 8197, 8187] avg=8192.0000 max=8325 max_rank=2 min=8025 min_rank=3 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32780 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 8028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2, 0, 144, 0, 6, 7833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 1, 0, 0, 77, 424, 0, 0, 1, 0, 0, 7933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 8092, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8087, 8185, 7991, 8482, 8194, 8210, 8196, 8191] avg=8192.0000 max=8482 max_rank=3 min=7991 min_rank=2 imbalance_ratio=1.035400 max_load_reduction=57054 max_load_reduction_pct=0.870575 moved_to_echo=32791 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 144, 0, 0, 8168, 1, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 8097, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8244, 8222, 8328, 7983, 8194, 8176, 8199, 8190] avg=8192.0000 max=8328 max_rank=2 min=7983 min_rank=3 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=32759 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 59, 0, 4, 0, 14, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 6, 1, 30, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 0, 7916, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 37, 0, 0, 0, 0, 0, 0, 339, 0, 0, 0, 0, 21, 7919, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 36, 41, 0, 3, 14, 0, 0, 8037, 0, 2, 0, 16, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 8108, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 0, 14, 0, 0, 0, 0, 0, 0, 8070, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8242, 8224, 8069, 8319, 8175, 8166, 8159, 8182] avg=8192.0000 max=8319 max_rank=3 min=8069 min_rank=2 imbalance_ratio=1.015503 max_load_reduction=57217 max_load_reduction_pct=0.873062 moved_to_echo=32682 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 51, 0, 1, 2, 0, 0, 0, 0, 8132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 41, 8030, 0, 0, 0, 1, 0, 2, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 80, 0, 1, 0, 0, 16, 3, 0, 0, 0, 0, 0, 7950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 17, 0, 0, 0, 1, 0, 8172, 0, 0, 1, 1, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8162, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8100, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8212, 8278, 8054, 8197, 8210, 8202, 8197] avg=8192.0000 max=8278 max_rank=2 min=8054 min_rank=3 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32806 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8143, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8189, 62, 0, 8117, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 56, 0, 24, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8164, 0, 4, 0, 14, 0, 0, 0, 8166, 0, 0, 10, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 6, 1, 8139, 1, 0, 0, 0, 0, 0, 8163, 0, 0, 7882, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 2, 0, 35, 0, 0, 0, 0, 0, 0, 369, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7, 0, 0, 36, 46, 0, 3, 15, 0, 0, 8034, 0, 2, 0, 16, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 0, 15, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 8186, 1, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 2, 0, 8186, 0, 0, 8172, 1, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 37, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 8106, 0, 0, 0, 0, 0, 0, 0, 8029, 0, 0, 2, 0, 0, 0, 8192, 0, 0, 0, 3, 0, 0, 8114, 80, 35, 8029, 0, 0, 0, 0, 0, 2, 0, 168, 0, 0, 0, 0, 0, 141, 0, 70, 0, 0, 4, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 1, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 8124, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 8139, 0, 10, 8009, 0, 0, 0, 0, 0, 0, 0, 127, 0, 0, 0, 0, 0, 1, 0, 2, 2, 0, 0, 96, 0, 0, 37, 0, 0, 0, 1, 0, 0, 0, 0, 8191, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 7615, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8189, 0, 0, 2, 0, 0, 0, 0, 8131, 0, 0, 1, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8189, 0, 39, 8020, 0, 0, 0, 1, 0, 2, 0, 89, 0, 0, 0, 4, 0, 81, 0, 1, 0, 0, 18, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 18, 0, 0, 0, 1, 0, 8174, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 8032, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 5, 5, 0, 8176, 0, 6, 7805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 1, 0, 0, 78, 450, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 8186, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 156, 0, 0, 8172, 1, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 8097, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 8223, 8343, 7971, 8193, 8174, 8198, 8189] avg=8192.0000 max=8343 max_rank=2 min=7971 min_rank=3 imbalance_ratio=1.018433 max_load_reduction=57193 max_load_reduction_pct=0.872696 moved_to_echo=32754 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 8106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8029, 0, 0, 2, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 154, 80, 35, 8029, 0, 0, 0, 0, 0, 2, 0, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 0, 70, 0, 0, 4, 0, 0, 0, 55, 0, 0, 7863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8120, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8098, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8062, 8471, 8133, 8193, 8129, 8196, 8188] avg=8192.0000 max=8471 max_rank=2 min=8062 min_rank=1 imbalance_ratio=1.034058 max_load_reduction=57065 max_load_reduction_pct=0.870743 moved_to_echo=32706 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 8143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 62, 0, 8117, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 24, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8099, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8205, 8336, 8016, 8193, 8201, 8196, 8188] avg=8192.0000 max=8336 max_rank=2 min=8016 min_rank=3 imbalance_ratio=1.017578 max_load_reduction=57200 max_load_reduction_pct=0.872803 moved_to_echo=32778 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 58, 0, 4, 0, 14, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 6, 1, 30, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 0, 0, 7882, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 35, 0, 0, 0, 0, 0, 0, 369, 0, 0, 0, 0, 21, 7910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 36, 46, 0, 3, 15, 0, 0, 8034, 0, 2, 0, 16, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 8109, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 0, 15, 0, 0, 0, 0, 0, 0, 8071, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8242, 8224, 8046, 8337, 8175, 8168, 8160, 8184] avg=8192.0000 max=8337 max_rank=3 min=8046 min_rank=2 imbalance_ratio=1.017700 max_load_reduction=57199 max_load_reduction_pct=0.872787 moved_to_echo=32687 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 8124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 0, 10, 8009, 0, 0, 0, 0, 0, 0, 0, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 2, 0, 0, 96, 0, 0, 37, 0, 0, 7887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8191, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 8097, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8207, 8301, 8025, 8196, 8223, 8189, 8213] avg=8192.0000 max=8301 max_rank=2 min=8025 min_rank=3 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32821 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 0, 7615, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 575, 0, 0, 0, 0, 0, 7938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8099, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8227, 8223, 7777, 8513, 8192, 8220, 8196, 8188] avg=8192.0000 max=8513 max_rank=3 min=7777 min_rank=2 imbalance_ratio=1.039185 max_load_reduction=57023 max_load_reduction_pct=0.870102 moved_to_echo=32796 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 52, 0, 0, 2, 0, 0, 0, 0, 8131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 0, 39, 8020, 0, 0, 0, 1, 0, 2, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 81, 0, 1, 0, 0, 18, 1, 0, 0, 0, 0, 0, 7939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 3, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 18, 0, 0, 0, 1, 0, 8174, 0, 0, 1, 1, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8166, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8097, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8208, 8286, 8044, 8200, 8211, 8208, 8194] avg=8192.0000 max=8286 max_rank=2 min=8044 min_rank=3 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32813 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 8032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 0, 155, 0, 6, 7805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 1, 0, 0, 78, 450, 0, 0, 1, 0, 0, 7923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 8094, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8090, 8184, 7976, 8497, 8194, 8210, 8193, 8192] avg=8192.0000 max=8497 max_rank=3 min=7976 min_rank=2 imbalance_ratio=1.037231 max_load_reduction=57039 max_load_reduction_pct=0.870346 moved_to_echo=32789 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 128, 384, 0, 128, 128, 0, 128, 1536, 0, 0, 128, 0, 128, 63488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 1152, 256, 128, 63744, 128, 0, 0, 128, 0, 128, 0, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 0, 128, 128, 128, 0, 128, 0, 128, 0, 0, 0, 0, 64768, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 65280, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 384, 0, 128, 128, 0, 128, 1536, 0, 0, 128, 0, 128, 63360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 128, 0, 128, 128, 128, 128, 128, 0, 128, 0, 0, 0, 0, 64768, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 0, 512, 0, 128, 128, 128, 0, 0, 0, 65152, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 0, 65280, 0, 0, 128, 0, 128, 128, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 0, 128, 65536, 128, 0, 0, 0, 128, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 1280, 256, 128, 63744, 128, 0, 0, 128, 0, 128, 0, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 128, 0, 0, 128, 256, 0, 128, 128, 128, 0, 65280, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 512, 0, 128, 128, 128, 0, 0, 0, 65152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 65280, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 0, 128, 65536, 128, 0, 0, 0, 128, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 7800, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 325, 0, 18, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 17, 2, 0, 0, 1, 8190, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 148, 5965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2222, 0, 32, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 8069, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 29] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 84, 5, 0, 5, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 193, 6838, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1177, 0, 120, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7, 0, 0, 8052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 7198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 731, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 249] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 74, 7617, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 525, 0, 12, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 57, 1, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 6703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1489, 0, 26, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 31, 7136, 0, 0, 0, 0, 3, 0, 0, 2, 0, 1, 0, 0, 950, 0, 19, 0, 0, 0, 0, 0, 8190, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8188, 0, 0, 0, 8189, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 155] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8145, 1, 0, 0, 29, 0, 0, 0, 0, 40, 0, 0, 6668, 0, 35, 0, 287, 0, 0, 10, 0, 0, 14, 0, 0, 1144, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 15, 7, 22, 8, 0, 5, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 17, 1, 0, 0, 0, 8102, 0, 0, 0, 8157, 0, 5, 0, 0, 0, 0, 0, 0, 83, 8170, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 8130, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 7800, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1053, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 325, 0, 18, 0, 0, 0, 0, 0, 7096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 42, 20, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 8123, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 8917, 7440, 8192, 8224, 8184, 8253, 8152] avg=8192.0000 max=8917 max_rank=1 min=7440 min_rank=2 imbalance_ratio=1.088501 max_load_reduction=56619 max_load_reduction_pct=0.863937 moved_to_echo=32813 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 84, 5, 0, 5, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 193, 6838, 15, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1177, 0, 120, 0, 0, 0, 0, 0, 7097, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8084, 0, 0, 0, 0, 0, 0, 0, 0, 42, 20, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8123, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8268, 8111, 8397, 8189, 8109, 8176, 8153, 8133] avg=8192.0000 max=8397 max_rank=2 min=8109 min_rank=4 imbalance_ratio=1.025024 max_load_reduction=57139 max_load_reduction_pct=0.871872 moved_to_echo=32571 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 7198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 731, 0, 1, 0, 0, 0, 0, 0, 7097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8089, 0, 0, 0, 0, 0, 0, 0, 0, 42, 20, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 249, 8123, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8421, 7829, 8189, 8201, 8185, 8158, 8375] avg=8192.0000 max=8421 max_rank=1 min=7829 min_rank=2 imbalance_ratio=1.027954 max_load_reduction=57115 max_load_reduction_pct=0.871506 moved_to_echo=32919 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 17, 2, 0, 0, 1, 8170, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 5965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2222, 0, 32, 0, 0, 0, 0, 0, 7094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8069, 0, 0, 3, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8081, 0, 0, 0, 0, 0, 0, 0, 0, 42, 20, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 29, 8123, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 7178, 9348, 8193, 8129, 8184, 8152, 8160] avg=8192.0000 max=9348 max_rank=2 min=7178 min_rank=1 imbalance_ratio=1.141113 max_load_reduction=56188 max_load_reduction_pct=0.857361 moved_to_echo=32625 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8125, 1, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 6668, 0, 35, 0, 287, 0, 0, 10, 0, 0, 1048, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 1144, 0, 0, 0, 0, 0, 0, 0, 7064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 7, 22, 8, 0, 5, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 17, 1, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 83, 8163, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 8130, 0, 0, 0, 0, 0, 0, 0, 24, 42, 20, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 8034, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8156, 8099, 8222, 8211, 8243, 8253, 8234, 8118] avg=8192.0000 max=8253 max_rank=5 min=8099 min_rank=1 imbalance_ratio=1.007446 max_load_reduction=57283 max_load_reduction_pct=0.874069 moved_to_echo=32848 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 74, 7617, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1053, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 0, 12, 0, 0, 0, 0, 0, 7097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 2, 0, 0, 0, 42, 20, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 8122, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8759, 7634, 8187, 8229, 8188, 8226, 8141] avg=8192.0000 max=8759 max_rank=1 min=7634 min_rank=2 imbalance_ratio=1.069214 max_load_reduction=56777 max_load_reduction_pct=0.866348 moved_to_echo=32784 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 57, 1, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 6703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1489, 0, 26, 0, 0, 0, 0, 0, 7095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8124, 0, 0, 0, 0, 0, 0, 0, 0, 42, 20, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 8123, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 7805, 8610, 8186, 8155, 8185, 8193, 8172] avg=8192.0000 max=8610 max_rank=2 min=7805 min_rank=1 imbalance_ratio=1.051025 max_load_reduction=56926 max_load_reduction_pct=0.868622 moved_to_echo=32705 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 31, 7136, 0, 0, 0, 0, 3, 0, 0, 2, 0, 1057, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 950, 0, 19, 0, 0, 0, 0, 0, 7085, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8177, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8080, 0, 0, 0, 0, 0, 0, 0, 0, 48, 25, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 155, 8121, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8166, 8246, 8056, 8189, 8246, 8181, 8167, 8285] avg=8192.0000 max=8285 max_rank=7 min=8056 min_rank=2 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32879 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 76, 7557, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 586, 0, 12, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 82, 5, 0, 5, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 206, 6751, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1254, 0, 120, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 6, 0, 0, 8051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 1, 0, 0, 29, 0, 0, 0, 0, 41, 0, 0, 6690, 0, 35, 0, 299, 0, 0, 9, 0, 0, 15, 0, 0, 1091, 0, 0, 0, 0, 0, 0, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 0, 15, 7, 22, 8, 0, 7, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 16, 2, 0, 0, 0, 8103, 0, 0, 0, 8158, 0, 5, 0, 0, 0, 0, 0, 0, 79, 8170, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 8128, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 17, 3, 0, 0, 2, 8190, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 154, 6033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2152, 0, 31, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 8071, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 33] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 6743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1450, 0, 27, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 7805, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 309, 0, 20, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 35, 7121, 0, 0, 0, 0, 3, 0, 0, 2, 0, 1, 0, 0, 975, 0, 21, 0, 0, 0, 0, 0, 8190, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8188, 0, 0, 0, 8189, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 141] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 174, 7210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 696, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 272] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 82, 5, 0, 5, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 206, 6751, 13, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1254, 0, 120, 0, 0, 0, 0, 0, 7102, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 6, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8084, 0, 0, 0, 0, 0, 0, 0, 0, 47, 20, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8117, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8266, 8030, 8481, 8189, 8109, 8175, 8158, 8128] avg=8192.0000 max=8481 max_rank=2 min=8030 min_rank=1 imbalance_ratio=1.035278 max_load_reduction=57055 max_load_reduction_pct=0.870590 moved_to_echo=32570 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 76, 7557, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1043, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 586, 0, 12, 0, 0, 0, 0, 0, 7102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 0, 0, 2, 0, 0, 0, 47, 20, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 8116, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8696, 7700, 8186, 8229, 8188, 8225, 8140] avg=8192.0000 max=8696 max_rank=1 min=7700 min_rank=2 imbalance_ratio=1.061523 max_load_reduction=56840 max_load_reduction_pct=0.867310 moved_to_echo=32782 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8125, 1, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 6690, 0, 35, 0, 299, 0, 0, 9, 0, 0, 1038, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 1091, 0, 0, 0, 0, 0, 0, 0, 7067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 7, 22, 8, 0, 7, 0, 0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 16, 2, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 79, 8163, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 8128, 0, 0, 0, 0, 0, 0, 0, 25, 47, 20, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 8029, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8155, 8129, 8173, 8214, 8243, 8249, 8238, 8135] avg=8192.0000 max=8249 max_rank=5 min=8129 min_rank=1 imbalance_ratio=1.006958 max_load_reduction=57287 max_load_reduction_pct=0.874130 moved_to_echo=32865 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 17, 3, 0, 0, 2, 8170, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 6033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2152, 0, 31, 0, 0, 0, 0, 0, 7101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8071, 0, 0, 2, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8071, 0, 0, 0, 0, 0, 0, 0, 0, 47, 20, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 33, 8117, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 7246, 9284, 8193, 8131, 8184, 8146, 8158] avg=8192.0000 max=9284 max_rank=2 min=7246 min_rank=1 imbalance_ratio=1.133301 max_load_reduction=56252 max_load_reduction_pct=0.858337 moved_to_echo=32619 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 7805, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1043, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 309, 0, 20, 0, 0, 0, 0, 0, 7101, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 47, 20, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 8117, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8173, 8922, 7431, 8192, 8224, 8184, 8257, 8153] avg=8192.0000 max=8922 max_rank=1 min=7431 min_rank=2 imbalance_ratio=1.089111 max_load_reduction=56614 max_load_reduction_pct=0.863861 moved_to_echo=32818 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 35, 7121, 0, 0, 0, 0, 3, 0, 0, 2, 0, 1045, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 975, 0, 21, 0, 0, 0, 0, 0, 7092, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8182, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8081, 0, 0, 0, 0, 0, 0, 0, 0, 53, 26, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 141, 8117, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8162, 8229, 8090, 8188, 8244, 8186, 8169, 8268] avg=8192.0000 max=8268 max_rank=7 min=8090 min_rank=2 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32867 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 6743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1450, 0, 27, 0, 0, 0, 0, 0, 7101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8122, 0, 0, 0, 0, 0, 0, 0, 0, 47, 20, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 8117, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8236, 7840, 8578, 8185, 8154, 8185, 8196, 8162] avg=8192.0000 max=8578 max_rank=2 min=7840 min_rank=1 imbalance_ratio=1.047119 max_load_reduction=56958 max_load_reduction_pct=0.869110 moved_to_echo=32697 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 174, 7210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 696, 0, 1, 0, 0, 0, 0, 0, 7102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8073, 0, 0, 0, 0, 0, 0, 0, 0, 47, 20, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 272, 8117, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8444, 7799, 8189, 8202, 8185, 8147, 8392] avg=8192.0000 max=8444 max_rank=1 min=7799 min_rank=2 imbalance_ratio=1.030762 max_load_reduction=57092 max_load_reduction_pct=0.871155 moved_to_echo=32926 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 8576, 0, 256, 0, 0, 0, 0, 0, 56832, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 128, 0, 768, 55936, 128, 128, 0, 384, 128, 0, 128, 128, 0, 8448, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 128, 0, 128, 0, 768, 55936, 128, 128, 0, 384, 128, 0, 128, 128, 0, 8448, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 128, 0, 0, 0, 128, 65536, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 768, 64896, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 128, 0, 0, 64896, 0, 0, 0, 0, 128, 0, 0, 128, 384, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 65152, 0, 0, 128, 0, 128, 128, 128, 0, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 128, 0, 0, 128, 128, 128, 128, 0, 128, 128, 128, 0, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 128, 0, 0, 0, 128, 65536, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 128, 0, 0, 65024, 0, 0, 0, 0, 128, 0, 0, 128, 384, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 640, 64896, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 0, 0, 128, 128, 256, 128, 0, 128, 128, 65408, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 0, 0, 128, 256, 128, 0, 128, 128, 65408, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 8192, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8103, 0, 0, 0, 840, 8192, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 6864, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 8192, 0, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 3552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7950, 1, 0, 0, 489, 8184, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 0, 0, 4151, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 8188, 1, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1170, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 7876, 5, 0, 0, 1270, 8192, 0, 0, 8127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 5553, 0, 0, 8182, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 25, 8190, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 1412, 8192, 0, 0, 8183, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4, 0, 0, 0, 8191, 1, 0, 0, 6522, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 8192, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 208, 8192, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 8192, 0, 0, 0, 6595, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 8191, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 0, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 191, 8191, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 6912, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 8161, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 1, 11, 4, 0, 39, 0, 12, 0, 4, 0, 7, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7773, 0, 0, 15, 2, 8168, 0, 0, 8093, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 7316, 22, 0, 0, 0, 827, 0, 0, 0, 0, 0, 438, 1, 0, 0, 8143, 0, 9, 0, 8177, 0, 0, 8163, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 197, 8190, 0, 64, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 401, 0, 1, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 8117, 0, 0, 1, 38, 8189, 0, 0, 8189, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 8187, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 7550, 0, 0, 8190, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 8019, 0, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 3552, 0, 0, 0, 0, 0, 0, 6581, 407, 173, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7543, 1, 0, 0, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1595, 0, 0, 0, 4151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8271, 10733, 8187, 8185, 8033, 8191, 5747] avg=8192.0000 max=10733 max_rank=2 min=5747 min_rank=7 imbalance_ratio=1.310181 max_load_reduction=54803 max_load_reduction_pct=0.836227 moved_to_echo=30156 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 8019, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 529, 0, 0, 0, 0, 0, 0, 6581, 414, 173, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7689, 0, 0, 0, 840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1595, 0, 0, 0, 6864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8063, 7702, 8208, 8194, 8529, 8191, 8459] avg=8192.0000 max=8529 max_rank=5 min=7702 min_rank=2 imbalance_ratio=1.041138 max_load_reduction=57007 max_load_reduction_pct=0.869858 moved_to_echo=33373 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 8019, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1256, 0, 0, 0, 0, 0, 0, 6581, 419, 173, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7768, 0, 0, 0, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 1595, 0, 0, 0, 6595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8156, 8430, 8197, 8193, 7976, 8190, 8204] avg=8192.0000 max=8430 max_rank=2 min=7976 min_rank=5 imbalance_ratio=1.029053 max_load_reduction=57106 max_load_reduction_pct=0.871368 moved_to_echo=32563 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 8015, 1, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1170, 0, 0, 0, 0, 0, 0, 6580, 403, 173, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8127, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7473, 5, 0, 0, 1270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1595, 0, 0, 0, 5553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8586, 8327, 8147, 8208, 8748, 8192, 7148] avg=8192.0000 max=8748 max_rank=5 min=7148 min_rank=7 imbalance_ratio=1.067871 max_load_reduction=56788 max_load_reduction_pct=0.866516 moved_to_echo=32296 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7988, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 1, 6541, 397, 173, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4, 0, 39, 0, 12, 0, 4, 0, 7, 0, 3, 1, 0, 0, 0, 8093, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7376, 0, 0, 15, 2, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 7315, 22, 0, 0, 0, 827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 438, 1, 0, 0, 1586, 0, 9, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8220, 7990, 7129, 8192, 8177, 7420, 8197, 10211] avg=8192.0000 max=10211 max_rank=7 min=7129 min_rank=2 imbalance_ratio=1.246460 max_load_reduction=55325 max_load_reduction_pct=0.844193 moved_to_echo=34005 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 8018, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 0, 1029, 0, 0, 0, 0, 0, 0, 6581, 417, 173, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7742, 0, 0, 0, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1595, 0, 0, 0, 6912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8129, 8205, 8189, 8192, 7933, 8191, 8507] avg=8192.0000 max=8507 max_rank=7 min=7933 min_rank=5 imbalance_ratio=1.038452 max_load_reduction=57029 max_load_reduction_pct=0.870193 moved_to_echo=32823 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 25, 8017, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 249, 0, 0, 0, 0, 0, 0, 6580, 417, 173, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7739, 0, 0, 0, 1412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1595, 1, 0, 0, 6522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8067, 7420, 8201, 8194, 9152, 8191, 8122] avg=8192.0000 max=9152 max_rank=5 min=7420 min_rank=2 imbalance_ratio=1.117188 max_load_reduction=56384 max_load_reduction_pct=0.860352 moved_to_echo=33659 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 197, 8011, 0, 64, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 401, 0, 1, 1, 0, 0, 0, 6582, 419, 179, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 8189, 20, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7698, 0, 0, 1, 38, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 8180, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1588, 0, 0, 0, 7550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8274, 7590, 8215, 8193, 7745, 8193, 9138] avg=8192.0000 max=9138 max_rank=7 min=7590 min_rank=2 imbalance_ratio=1.115479 max_load_reduction=56398 max_load_reduction_pct=0.860565 moved_to_echo=33269 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 8192, 0, 237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 3550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7953, 1, 0, 0, 423, 8186, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 0, 0, 4219, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 8189, 1, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1122, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 7878, 5, 0, 0, 1623, 8192, 0, 0, 8132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 5239, 0, 0, 8183, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 22, 8190, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 1337, 8192, 0, 0, 8183, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4, 0, 0, 0, 8191, 1, 0, 0, 6586, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 1, 12, 2, 0, 40, 0, 12, 0, 4, 0, 8, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7780, 0, 0, 15, 0, 8168, 0, 0, 8093, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 7323, 22, 0, 0, 0, 812, 0, 0, 0, 0, 0, 443, 1, 0, 0, 8142, 0, 9, 0, 8178, 0, 0, 8162, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 8192, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 638, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8102, 0, 0, 0, 700, 8192, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 6891, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 227, 8190, 0, 62, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 427, 0, 1, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 8118, 0, 0, 1, 35, 8189, 0, 0, 8188, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 8187, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 7498, 0, 0, 8190, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 8191, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 1, 0, 0, 185, 8191, 0, 0, 8173, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 6863, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 8192, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 127, 8192, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 8192, 0, 0, 0, 6647, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 8015, 0, 237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 3550, 0, 0, 0, 0, 0, 0, 6557, 406, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7547, 1, 0, 0, 423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1620, 0, 0, 0, 4219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8262, 10710, 8184, 8188, 7971, 8192, 5840] avg=8192.0000 max=10710 max_rank=2 min=5840 min_rank=7 imbalance_ratio=1.307373 max_load_reduction=54826 max_load_reduction_pct=0.836578 moved_to_echo=30191 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 22, 8013, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 261, 0, 0, 0, 0, 0, 0, 6556, 416, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7740, 0, 0, 0, 1337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1620, 1, 0, 0, 6586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8062, 7410, 8200, 8194, 9078, 8192, 8211] avg=8192.0000 max=9078 max_rank=5 min=7410 min_rank=2 imbalance_ratio=1.108154 max_load_reduction=56458 max_load_reduction_pct=0.861481 moved_to_echo=33675 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 8012, 1, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1122, 0, 0, 0, 0, 0, 0, 6556, 402, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 15, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7476, 5, 0, 0, 1623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1620, 0, 0, 0, 5239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8583, 8257, 8152, 8208, 9104, 8192, 6859] avg=8192.0000 max=9104 max_rank=5 min=6859 min_rank=7 imbalance_ratio=1.111328 max_load_reduction=56432 max_load_reduction_pct=0.861084 moved_to_echo=32363 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7985, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 1, 6517, 397, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 2, 0, 40, 0, 12, 0, 4, 0, 8, 0, 3, 1, 0, 0, 0, 8093, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7383, 0, 0, 15, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 7323, 22, 0, 0, 0, 812, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 443, 1, 0, 0, 1610, 0, 9, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 7986, 7107, 8191, 8178, 7425, 8189, 10241] avg=8192.0000 max=10241 max_rank=7 min=7107 min_rank=2 imbalance_ratio=1.250122 max_load_reduction=55295 max_load_reduction_pct=0.843735 moved_to_echo=34033 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 227, 8007, 0, 62, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 427, 0, 1, 1, 0, 0, 0, 6557, 418, 183, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 8188, 20, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7700, 0, 0, 1, 35, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 8183, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1613, 0, 0, 0, 7498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8298, 7591, 8214, 8192, 7745, 8196, 9111] avg=8192.0000 max=9111 max_rank=7 min=7591 min_rank=2 imbalance_ratio=1.112183 max_load_reduction=56425 max_load_reduction_pct=0.860977 moved_to_echo=33244 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 8015, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 638, 0, 0, 0, 0, 0, 0, 6557, 413, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 15, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7689, 0, 0, 0, 700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1620, 0, 0, 0, 6891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8065, 7788, 8209, 8193, 8389, 8192, 8511] avg=8192.0000 max=8511 max_rank=7 min=7788 min_rank=2 imbalance_ratio=1.038940 max_load_reduction=57025 max_load_reduction_pct=0.870132 moved_to_echo=33285 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 8014, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1083, 0, 0, 0, 0, 0, 0, 6557, 416, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8173, 15, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7741, 1, 0, 0, 185, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1620, 0, 0, 0, 6863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8126, 8235, 8192, 8191, 7928, 8192, 8483] avg=8192.0000 max=8483 max_rank=7 min=7928 min_rank=5 imbalance_ratio=1.035522 max_load_reduction=57053 max_load_reduction_pct=0.870560 moved_to_echo=32794 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 8015, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1286, 0, 0, 0, 0, 0, 0, 6557, 418, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7769, 0, 0, 0, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 1620, 0, 0, 0, 6647, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8154, 8438, 8194, 8192, 7896, 8191, 8280] avg=8192.0000 max=8438 max_rank=2 min=7896 min_rank=5 imbalance_ratio=1.030029 max_load_reduction=57098 max_load_reduction_pct=0.871246 moved_to_echo=32559 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 0, 0, 0, 128, 0, 0, 8192, 0, 128, 128, 0, 0, 128, 52608, 3328, 1408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 512, 128, 0, 0, 12800, 128, 128, 0, 52352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 0, 0, 0, 0, 0, 0, 8448, 0, 128, 128, 0, 0, 128, 52480, 3328, 1536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 128, 512, 128, 0, 0, 13056, 128, 128, 0, 52224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 61056, 128, 0, 128, 4480, 0, 0, 0, 0, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 128, 1024, 64128, 128, 512, 128, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 128, 0, 128, 0, 64640, 128, 0, 0, 0, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 1024, 64128, 128, 512, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 0, 128, 0, 128, 0, 128, 0, 128, 128, 128, 128, 0, 0, 128, 65408, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 0, 128, 0, 128, 0, 128, 0, 128, 0, 128, 128, 0, 0, 128, 65408, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 0, 128, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 0, 128, 0, 64768, 128, 0, 0, 0, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 0, 128, 128, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 17, 8172, 0, 0, 1691, 0, 0, 0, 0, 0, 60, 0, 0, 0, 8167, 0, 0, 0, 0, 62, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 395, 0, 0, 0, 0, 0, 0, 10, 0, 7782, 123, 0, 7223, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8109, 0, 6, 0, 16, 0, 0, 0, 5044, 0, 0, 0, 0, 0, 1, 0, 0, 8140, 0, 0, 0, 17, 0, 0, 0, 6, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 28, 0, 3, 8135, 0, 0, 0, 0, 1607, 0, 0, 19, 13, 0, 0, 7, 0, 0, 604, 0, 0, 0, 19, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7826, 4, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 3380, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 701, 0, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5637, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[35, 0, 8052, 0, 0, 4348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 1, 0, 0, 7686, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7864, 0, 0, 0, 0, 0, 0, 0, 0, 2871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 544, 0, 243, 6, 0, 0, 0, 0, 0, 0, 0, 0, 1946, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[43, 1, 8175, 0, 0, 7019, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8126, 0, 1, 8143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 4807, 0, 0, 0, 1, 0, 0, 0, 0, 7294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 85, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1144, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[7, 0, 8190, 0, 0, 4199, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8061, 0, 0, 8083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8192, 0, 0, 0, 0, 0, 0, 0, 7063, 0, 0, 0, 0, 0, 0, 0, 0, 3658, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 619, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1494, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[32, 0, 8136, 0, 0, 456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7630, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 18, 8192, 0, 0, 0, 0, 0, 0, 0, 5125, 0, 0, 0, 0, 0, 0, 0, 0, 8146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 191, 0, 425, 2, 0, 0, 0, 0, 0, 0, 0, 0, 5051, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[5, 0, 8191, 0, 0, 4791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 1, 0, 0, 0, 0, 0, 0, 8020, 7, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 8034, 0, 0, 0, 0, 0, 0, 0, 0, 5225, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 290, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 1, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 8166, 0, 0, 2059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7299, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 33, 0, 0, 3, 0, 0, 0, 0, 7880, 8, 0, 8078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 1, 0, 0, 0, 0, 1, 6861, 0, 0, 0, 0, 0, 0, 0, 0, 7628, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 2, 0, 0, 0, 0, 8190, 0, 0, 490, 0, 206, 0, 0, 0, 1, 0, 0, 0, 6, 1, 419, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7826, 4, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2170, 0, 0, 0, 0, 0, 0, 0, 3380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1462, 327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7487, 0, 0, 701, 0, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5999, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5300, 8215, 8062, 8335, 5550, 9980, 8308, 11786] avg=8192.0000 max=11786 max_rank=7 min=5300 min_rank=0 imbalance_ratio=1.438721 max_load_reduction=53750 max_load_reduction_pct=0.820160 moved_to_echo=35624 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[35, 0, 0, 0, 0, 4348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8052, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 1, 0, 0, 7686, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2170, 0, 0, 0, 0, 0, 0, 0, 7864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1602, 327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7487, 0, 0, 544, 0, 243, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5999, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9919, 8075, 7980, 8339, 10034, 4800, 8280, 8109] avg=8192.0000 max=10034 max_rank=4 min=4800 min_rank=5 imbalance_ratio=1.224854 max_load_reduction=55502 max_load_reduction_pct=0.846893 moved_to_echo=31223 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 17, 0, 0, 0, 1691, 0, 0, 0, 0, 0, 60, 0, 0, 0, 6191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 395, 0, 0, 0, 0, 0, 0, 10, 0, 7782, 123, 0, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7223, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2149, 0, 6, 0, 16, 0, 0, 0, 5044, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 0, 0, 0, 17, 0, 0, 0, 6, 0, 0, 0, 0, 9, 0, 1792, 325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 3, 7434, 0, 0, 0, 0, 1607, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 7, 0, 0, 604, 0, 0, 0, 19, 0, 0, 0, 0, 0, 5938, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7959, 8268, 8540, 7408, 7216, 10289, 9091, 6765] avg=8192.0000 max=10289 max_rank=5 min=6765 min_rank=7 imbalance_ratio=1.255981 max_load_reduction=55247 max_load_reduction_pct=0.843002 moved_to_echo=33361 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[43, 1, 0, 0, 0, 7019, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8126, 0, 1, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2170, 0, 0, 0, 0, 0, 0, 0, 4807, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 936, 327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7487, 0, 0, 85, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5998, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10300, 8198, 8362, 8304, 6978, 8557, 7599, 7238] avg=8192.0000 max=10300 max_rank=0 min=6978 min_rank=4 imbalance_ratio=1.257324 max_load_reduction=55236 max_load_reduction_pct=0.842834 moved_to_echo=30372 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 0, 0, 0, 4199, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8061, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2170, 0, 0, 0, 0, 0, 0, 0, 7063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3658, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1703, 327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7487, 0, 0, 619, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5999, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10093, 8213, 8292, 8234, 9233, 5688, 8115, 7668] avg=8192.0000 max=10093 max_rank=0 min=5688 min_rank=5 imbalance_ratio=1.232056 max_load_reduction=55443 max_load_reduction_pct=0.845993 moved_to_echo=30704 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[32, 0, 0, 0, 0, 456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7630, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 18, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2170, 0, 0, 0, 0, 0, 0, 0, 5125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1261, 327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7487, 0, 0, 191, 0, 425, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5999, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4848, 8159, 7861, 8355, 7295, 9734, 8105, 11179] avg=8192.0000 max=11179 max_rank=7 min=4848 min_rank=0 imbalance_ratio=1.364624 max_load_reduction=54357 max_load_reduction_pct=0.829422 moved_to_echo=36313 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 0, 0, 0, 4791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 1, 0, 0, 0, 0, 0, 0, 8020, 7, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2170, 0, 0, 0, 0, 0, 0, 0, 8034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5225, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1370, 327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 7487, 0, 0, 290, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 1, 0, 0, 0, 1, 0, 0, 5998, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9532, 8214, 8276, 8331, 10204, 6924, 7859, 6196] avg=8192.0000 max=10204 max_rank=4 min=6196 min_rank=7 imbalance_ratio=1.245605 max_load_reduction=55332 max_load_reduction_pct=0.844299 moved_to_echo=31183 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8166, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 33, 0, 0, 3, 0, 0, 0, 0, 7880, 8, 0, 237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2163, 0, 1, 0, 0, 0, 0, 1, 6861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7628, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1605, 329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 2, 0, 0, 0, 0, 7472, 0, 0, 490, 0, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 6, 1, 419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6000, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7585, 8194, 8163, 8230, 9026, 9564, 8179, 6595] avg=8192.0000 max=9564 max_rank=5 min=6595 min_rank=7 imbalance_ratio=1.167480 max_load_reduction=55972 max_load_reduction_pct=0.854065 moved_to_echo=33364 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[36, 0, 8053, 0, 0, 4227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 1, 0, 0, 7687, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8087, 0, 0, 0, 0, 0, 0, 0, 0, 2567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 548, 0, 217, 6, 0, 0, 0, 0, 0, 0, 0, 0, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[5, 0, 8191, 0, 0, 5176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 1, 0, 0, 0, 0, 0, 0, 8024, 7, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 8095, 0, 0, 0, 0, 0, 0, 0, 0, 4193, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 330, 0, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 1, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 8172, 0, 0, 2081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7306, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 31, 0, 0, 3, 0, 0, 0, 0, 7896, 6, 0, 8084, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 1, 0, 0, 0, 0, 1, 6907, 0, 0, 0, 0, 0, 0, 0, 0, 7568, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 9, 0, 2, 0, 0, 0, 0, 8190, 0, 0, 506, 0, 193, 0, 0, 0, 1, 0, 0, 0, 6, 1, 377, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[46, 1, 8175, 0, 0, 6535, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8132, 0, 2, 8145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 5139, 0, 0, 0, 1, 0, 0, 0, 0, 7166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 78, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1426, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[33, 0, 8141, 0, 0, 269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7623, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 19, 8192, 0, 0, 0, 0, 0, 0, 0, 5645, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 184, 0, 423, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4751, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[7, 0, 8190, 0, 0, 3949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8080, 0, 0, 8093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 8192, 0, 0, 0, 0, 0, 0, 0, 7401, 0, 0, 0, 0, 0, 0, 0, 0, 3304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 597, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1758, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 15, 8172, 0, 0, 1515, 0, 0, 0, 0, 0, 60, 0, 0, 0, 8166, 0, 0, 0, 0, 67, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 384, 0, 0, 0, 0, 0, 0, 11, 0, 7782, 123, 0, 7208, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8105, 0, 6, 0, 16, 0, 0, 0, 5531, 0, 0, 0, 0, 0, 3, 0, 0, 8137, 0, 0, 0, 18, 0, 0, 0, 6, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 26, 0, 3, 8135, 0, 0, 0, 0, 1394, 0, 0, 20, 12, 0, 0, 7, 0, 0, 536, 0, 0, 0, 18, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7818, 5, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 635, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5127, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[36, 0, 0, 0, 0, 4227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8053, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 1, 0, 0, 7687, 0, 0, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1836, 0, 0, 0, 0, 0, 0, 0, 8087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1733, 297, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7520, 0, 0, 548, 0, 217, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9954, 8074, 7976, 8338, 9923, 4751, 8291, 8229] avg=8192.0000 max=9954 max_rank=0 min=4751 min_rank=5 imbalance_ratio=1.215088 max_load_reduction=55582 max_load_reduction_pct=0.848114 moved_to_echo=31194 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 0, 0, 0, 5176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 1, 0, 0, 0, 0, 0, 0, 8024, 7, 0, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1835, 0, 0, 0, 0, 0, 0, 0, 8095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4193, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1571, 297, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 7520, 0, 0, 330, 0, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 1, 0, 0, 0, 1, 0, 0, 6181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10340, 8212, 8279, 8330, 9930, 6217, 7963, 6265] avg=8192.0000 max=10340 max_rank=0 min=6217 min_rank=5 imbalance_ratio=1.262207 max_load_reduction=55196 max_load_reduction_pct=0.842224 moved_to_echo=30375 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8172, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 31, 0, 0, 3, 0, 0, 0, 0, 7896, 6, 0, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8084, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1821, 0, 1, 0, 0, 0, 0, 1, 6907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7568, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 1709, 298, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 2, 0, 0, 0, 0, 7509, 0, 0, 506, 0, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 6, 1, 377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7678, 8197, 8171, 8234, 8730, 9739, 8219, 6568] avg=8192.0000 max=9739 max_rank=5 min=6568 min_rank=7 imbalance_ratio=1.188843 max_load_reduction=55797 max_load_reduction_pct=0.851395 moved_to_echo=33256 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 0, 0, 0, 3949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8080, 0, 0, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1836, 0, 0, 0, 0, 0, 0, 0, 7401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1811, 297, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7520, 0, 0, 597, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9903, 8211, 8309, 8244, 9237, 5566, 8127, 7939] avg=8192.0000 max=9903 max_rank=0 min=5566 min_rank=5 imbalance_ratio=1.208862 max_load_reduction=55633 max_load_reduction_pct=0.848892 moved_to_echo=30869 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[46, 1, 0, 0, 0, 6535, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8132, 0, 2, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1835, 0, 0, 0, 0, 0, 0, 0, 5139, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 995, 297, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7520, 0, 0, 78, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9855, 8196, 8367, 8305, 6975, 8612, 7620, 7606] avg=8192.0000 max=9855 max_rank=0 min=6975 min_rank=4 imbalance_ratio=1.203003 max_load_reduction=55681 max_load_reduction_pct=0.849625 moved_to_echo=30813 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 15, 0, 0, 0, 1515, 0, 0, 0, 0, 0, 60, 0, 0, 0, 6260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 384, 0, 0, 0, 0, 0, 0, 11, 0, 7782, 123, 0, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7208, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1815, 0, 6, 0, 16, 0, 0, 0, 5531, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 0, 0, 0, 18, 0, 0, 0, 6, 0, 0, 0, 0, 7, 0, 1906, 295, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 3, 7467, 0, 0, 0, 0, 1394, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 7, 0, 0, 536, 0, 0, 0, 18, 0, 0, 0, 0, 0, 6116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7850, 8271, 8529, 7394, 7371, 10522, 8910, 6689] avg=8192.0000 max=10522 max_rank=5 min=6689 min_rank=7 imbalance_ratio=1.284424 max_load_reduction=55014 max_load_reduction_pct=0.839447 moved_to_echo=33492 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[33, 0, 0, 0, 0, 269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8141, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7623, 0, 0, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 19, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1836, 0, 0, 0, 0, 0, 0, 0, 5645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1337, 297, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7520, 0, 0, 184, 0, 423, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4751, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4695, 8162, 7852, 8357, 7481, 9928, 8129, 10932] avg=8192.0000 max=10932 max_rank=7 min=4695 min_rank=0 imbalance_ratio=1.334473 max_load_reduction=54604 max_load_reduction_pct=0.833191 moved_to_echo=36470 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7818, 5, 0, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1836, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1561, 297, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7520, 0, 0, 635, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5261, 8213, 8053, 8334, 5889, 10201, 8277, 11308] avg=8192.0000 max=11308 max_rank=7 min=5261 min_rank=0 imbalance_ratio=1.380371 max_load_reduction=54228 max_load_reduction_pct=0.827454 moved_to_echo=35675 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 128, 128, 128, 16384, 0, 0, 128, 128, 0, 0, 128, 0, 0, 48000, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 0, 0, 0, 24832, 0, 0, 128, 0, 0, 128, 0, 128, 0, 40576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[17408, 0, 128, 0, 128, 0, 0, 128, 48256, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 51200, 0, 0, 0, 128, 0, 128, 0, 128, 0, 0, 128, 0, 128, 0, 11776, 2688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 128, 0, 128, 0, 128, 59904, 0, 0, 2944, 0, 2816, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[64384, 0, 0, 128, 0, 0, 0, 0, 0, 0, 128, 0, 128, 0, 128, 128, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 65280, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 0, 512, 128, 128, 0, 128, 0, 128, 128, 0, 63104, 256, 128, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 128, 0, 0, 0, 23936, 0, 0, 128, 0, 0, 128, 0, 0, 0, 41472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 0, 128, 128, 128, 16128, 0, 0, 128, 128, 0, 0, 128, 0, 0, 49408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[14720, 0, 128, 0, 128, 0, 0, 128, 50944, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 49280, 0, 0, 0, 128, 0, 128, 0, 128, 0, 0, 128, 0, 128, 0, 12672, 2432, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 128, 0, 128, 0, 128, 60160, 0, 0, 2944, 0, 2560, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[64384, 0, 0, 128, 0, 0, 0, 0, 128, 0, 128, 0, 128, 0, 128, 128, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 128, 128, 128, 512, 128, 128, 0, 128, 0, 128, 128, 0, 63104, 256, 128, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 65408, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7773, 0, 0, 6678, 0, 0, 0, 8113, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 1803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 7633, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 558, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 7320, 1, 0, 8137, 199, 0, 0, 8164, 0, 0, 48, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 8159, 0, 10, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 8082, 2, 0, 0, 0, 15, 8015, 6, 0, 0, 0, 683, 3, 0, 0, 0, 0, 0, 0, 0, 8124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 26, 22, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 109, 74, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 18] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8079, 0, 0, 6797, 0, 0, 0, 8170, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 1488, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 7439, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 755, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 6631, 0, 0, 0, 8163, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 1541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8148, 0, 0, 0, 8189, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 7434, 2, 0, 0, 8187, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8165, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8189, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 8189, 0, 0, 2, 0, 0, 8180, 1, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 791, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 793, 0, 0, 6678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8100, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 8105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 6980, 84, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7472, 8120, 8182, 8235, 8208, 8112, 8323, 8884] avg=8192.0000 max=8884 max_rank=7 min=7472 min_rank=0 imbalance_ratio=1.084473 max_load_reduction=56652 max_load_reduction_pct=0.864441 moved_to_echo=33527 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 836, 0, 0, 7439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 755, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 7356, 84, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8275, 8178, 8204, 8189, 8205, 8108, 8169, 8208] avg=8192.0000 max=8275 max_rank=0 min=8108 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32690 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 836, 0, 0, 7633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 558, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7356, 84, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8469, 8163, 8203, 8189, 8207, 8109, 8185, 8011] avg=8192.0000 max=8469 max_rank=0 min=8011 min_rank=7 imbalance_ratio=1.033813 max_load_reduction=57067 max_load_reduction_pct=0.870773 moved_to_echo=32512 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 747, 1, 0, 8137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 199, 0, 0, 8151, 0, 0, 48, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 0, 0, 23, 0, 0, 0, 0, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 10, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 17, 2, 0, 0, 0, 15, 8015, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 683, 3, 0, 0, 0, 0, 0, 0, 0, 8040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 26, 22, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 74, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 18, 6573, 84, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8983, 8410, 8148, 8171, 8112, 8726, 8112, 6874] avg=8192.0000 max=8983 max_rank=0 min=6874 min_rank=7 imbalance_ratio=1.096558 max_load_reduction=56553 max_load_reduction_pct=0.862930 moved_to_echo=31824 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 824, 0, 0, 6797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 8106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1488, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7255, 84, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7622, 8161, 8202, 8189, 8197, 8141, 8185, 8839] avg=8192.0000 max=8839 max_rank=7 min=7622 min_rank=0 imbalance_ratio=1.078979 max_load_reduction=56697 max_load_reduction_pct=0.865128 moved_to_echo=33362 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 836, 0, 0, 8148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7356, 84, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8984, 8180, 8208, 8186, 8209, 8108, 8169, 7492] avg=8192.0000 max=8984 max_rank=0 min=7492 min_rank=7 imbalance_ratio=1.096680 max_load_reduction=56552 max_load_reduction_pct=0.862915 moved_to_echo=31978 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 836, 0, 0, 6631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8150, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7354, 84, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7467, 8156, 8203, 8192, 8200, 8109, 8219, 8990] avg=8192.0000 max=8990 max_rank=7 min=7467 min_rank=0 imbalance_ratio=1.097412 max_load_reduction=56546 max_load_reduction_pct=0.862823 moved_to_echo=33518 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 828, 0, 0, 7434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8165, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8165, 0, 0, 0, 0, 1, 0, 0, 0, 12, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 11, 0, 0, 2, 0, 0, 8180, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 8100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 791, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 1, 7332, 91, 10, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8264, 8168, 8186, 8185, 8198, 8123, 8174, 8238] avg=8192.0000 max=8264 max_rank=0 min=8123 min_rank=5 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32733 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7772, 0, 0, 6824, 0, 0, 0, 8108, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 0, 0, 0, 0, 1666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 7683, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 508, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 97, 0, 0, 0, 0, 0, 1, 0, 7314, 2, 0, 8138, 199, 0, 0, 8164, 0, 0, 48, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 8159, 0, 10, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 8083, 2, 0, 0, 0, 16, 8018, 6, 0, 0, 0, 719, 4, 0, 0, 0, 0, 0, 0, 0, 8124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 21, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 75, 72, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 16] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8131, 0, 0, 6915, 0, 0, 0, 8174, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 1317, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 7430, 2, 0, 0, 8187, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8165, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8189, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 8189, 0, 0, 1, 0, 0, 8182, 1, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 783, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 8148, 0, 0, 0, 8189, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 6727, 0, 0, 0, 8162, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 1442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 7470, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 739, 0, 0, 6824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8095, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 8100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 7033, 90, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7564, 8113, 8185, 8237, 8207, 8109, 8318, 8803] avg=8192.0000 max=8803 max_rank=7 min=7564 min_rank=0 imbalance_ratio=1.074585 max_load_reduction=56733 max_load_reduction_pct=0.865677 moved_to_echo=33437 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 779, 0, 0, 7683, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7413, 90, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8462, 8163, 8202, 8188, 8207, 8102, 8188, 8024] avg=8192.0000 max=8462 max_rank=0 min=8024 min_rank=7 imbalance_ratio=1.032959 max_load_reduction=57074 max_load_reduction_pct=0.870880 moved_to_echo=32521 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 779, 0, 0, 8148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7412, 90, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8927, 8181, 8207, 8185, 8209, 8102, 8170, 7555] avg=8192.0000 max=8927 max_rank=0 min=7555 min_rank=7 imbalance_ratio=1.089722 max_load_reduction=56609 max_load_reduction_pct=0.863785 moved_to_echo=32036 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 97, 0, 0, 0, 0, 0, 1, 0, 696, 2, 0, 8138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 199, 0, 0, 8151, 0, 0, 48, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 0, 0, 24, 0, 0, 0, 0, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 10, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 17, 2, 0, 0, 0, 16, 8018, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 719, 4, 0, 0, 0, 0, 0, 0, 0, 8035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 21, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 72, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 16, 6618, 89, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8934, 8410, 8148, 8172, 8116, 8758, 8112, 6886] avg=8192.0000 max=8934 max_rank=0 min=6886 min_rank=7 imbalance_ratio=1.090576 max_load_reduction=56602 max_load_reduction_pct=0.863678 moved_to_echo=31872 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 773, 0, 0, 7430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8166, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8165, 0, 0, 0, 0, 1, 0, 0, 0, 11, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 8, 0, 0, 1, 0, 0, 8182, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 8097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 783, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 7398, 94, 10, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8169, 8186, 8186, 8196, 8121, 8176, 8297] avg=8192.0000 max=8297 max_rank=7 min=8121 min_rank=5 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32790 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 773, 0, 0, 6915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 8100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1317, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7358, 90, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7689, 8166, 8202, 8189, 8192, 8140, 8181, 8777] avg=8192.0000 max=8777 max_rank=7 min=7689 min_rank=0 imbalance_ratio=1.071411 max_load_reduction=56759 max_load_reduction_pct=0.866074 moved_to_echo=33290 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 779, 0, 0, 6727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7411, 90, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7506, 8155, 8203, 8191, 8204, 8102, 8221, 8954] avg=8192.0000 max=8954 max_rank=7 min=7506 min_rank=0 imbalance_ratio=1.093018 max_load_reduction=56582 max_load_reduction_pct=0.863373 moved_to_echo=33481 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 779, 0, 0, 7470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 7413, 90, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8249, 8179, 8203, 8188, 8205, 8102, 8170, 8240] avg=8192.0000 max=8249 max_rank=0 min=8102 min_rank=5 imbalance_ratio=1.006958 max_load_reduction=57287 max_load_reduction_pct=0.874130 moved_to_echo=32717 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 0, 7168, 128, 0, 0, 128, 128, 0, 0, 128, 0, 128, 128, 0, 128, 57600, 768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 6656, 128, 0, 59008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 65536, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 128, 0, 65408, 0, 0, 0, 128, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[768, 128, 128, 0, 0, 0, 0, 0, 0, 64896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 128, 0, 6144, 128, 0, 59392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 0, 6656, 128, 0, 0, 128, 128, 0, 0, 128, 0, 128, 128, 0, 128, 58112, 768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[768, 128, 128, 0, 0, 0, 0, 0, 0, 64768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 0, 128, 0, 65536, 0, 0, 0, 128, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 0, 0, 65280, 0, 128, 128, 128, 128, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 0, 0, 0, 256, 128, 0, 128, 0, 128, 65408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 256, 128, 0, 65280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 0, 0, 128, 128, 0, 128, 0, 128, 65408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 256, 128, 0, 65280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 65536, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 0, 0, 27, 8192, 0, 0, 0, 8085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 352, 0, 0, 0, 8094, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 26, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 0, 47, 0, 8127, 3, 1164, 12, 7007, 0, 38, 0, 113, 18, 15, 0, 0, 0, 7978, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 11, 0, 0, 33, 1, 8150, 0, 1, 0, 0, 134, 0, 12, 8161, 0, 0, 0, 0, 0, 0, 5, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8017, 0, 0, 0, 0, 0, 0, 12, 19, 0, 6, 18, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 6, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 0, 0, 16, 0, 0, 0, 7997, 0, 0, 0, 0, 1, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 5, 8192, 0, 0, 0, 8122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 58, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 21, 0, 0, 0, 8160, 0, 0, 0, 0, 1, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 1, 0, 9, 0, 0, 63, 8190, 0, 0, 0, 8143, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 0, 0, 0, 0, 0, 0, 7945, 0, 0, 0, 0, 1, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 8192, 0, 0, 0, 7596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 684, 0, 0, 92, 0, 0, 0, 8049, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 15, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8134, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 8190, 4, 0, 0, 8077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 1, 1, 0, 8190, 0, 0, 0, 0, 0, 0, 2, 8189, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 8187, 0, 0, 0, 2, 0, 0, 0, 2, 0, 1, 0, 8190, 0, 0, 0, 2, 0, 114, 0, 0, 0, 9, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7804, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 0, 0, 27, 73, 0, 0, 0, 8085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 352, 0, 0, 0, 7945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 8116, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 0, 0, 0, 26, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7883, 8293, 8203, 8329, 8239, 8206, 8190, 8193] avg=8192.0000 max=8329 max_rank=3 min=7883 min_rank=0 imbalance_ratio=1.016724 max_load_reduction=57207 max_load_reduction_pct=0.872910 moved_to_echo=32828 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 0, 0, 16, 0, 0, 0, 7850, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8116, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8269, 8231, 8203, 8079, 8180, 8206, 8173, 8195] avg=8192.0000 max=8269 max_rank=0 min=8079 min_rank=3 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32754 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 0, 47, 0, 73, 3, 1164, 12, 7007, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 18, 15, 0, 0, 0, 7978, 0, 0, 0, 0, 0, 1, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 33, 1, 8000, 0, 1, 0, 0, 134, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 22, 0, 0, 0, 0, 0, 0, 0, 8051, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8017, 0, 0, 0, 0, 0, 0, 12, 19, 0, 6, 18, 8, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8244, 8411, 8136, 8192, 8143, 8093, 8146, 8171] avg=8192.0000 max=8411 max_rank=1 min=8093 min_rank=5 imbalance_ratio=1.026733 max_load_reduction=57125 max_load_reduction_pct=0.871658 moved_to_echo=32553 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 0, 0, 0, 0, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 5, 73, 0, 0, 0, 8122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 58, 0, 0, 0, 8006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8116, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8204, 8203, 8135, 8182, 8206, 8195, 8195] avg=8192.0000 max=8216 max_rank=0 min=8135 min_rank=3 imbalance_ratio=1.002930 max_load_reduction=57320 max_load_reduction_pct=0.874634 moved_to_echo=32778 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 9, 0, 0, 63, 73, 0, 0, 0, 8143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 0, 0, 0, 0, 0, 0, 7798, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8114, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 5, 8185, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8269, 8290, 8190, 8045, 8179, 8203, 8166, 8194] avg=8192.0000 max=8290 max_rank=1 min=8045 min_rank=3 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32742 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 8, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 2, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 21, 0, 0, 0, 8010, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8116, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8267, 8264, 8203, 8036, 8188, 8205, 8178, 8195] avg=8192.0000 max=8267 max_rank=0 min=8036 min_rank=3 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32766 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8093, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 73, 0, 0, 0, 7596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 684, 0, 0, 92, 0, 0, 0, 7901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 15, 8116, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 7694, 8197, 8677, 8199, 8206, 8198, 8194] avg=8192.0000 max=8677 max_rank=3 min=7694 min_rank=1 imbalance_ratio=1.059204 max_load_reduction=56859 max_load_reduction_pct=0.867599 moved_to_echo=32797 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8134, 0, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 65, 4, 0, 0, 8077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 4, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 1, 1, 0, 8030, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 8117, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 2, 0, 0, 0, 2, 0, 1, 0, 12, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 2, 0, 114, 0, 0, 0, 9, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8217, 8149, 8201, 8043, 8226, 8211, 8290, 8199] avg=8192.0000 max=8290 max_rank=6 min=8043 min_rank=3 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32926 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 24, 0, 0, 0, 8159, 0, 0, 0, 0, 1, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 8190, 4, 0, 0, 8076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1, 0, 6, 3, 1, 1, 0, 8190, 0, 0, 0, 0, 0, 0, 2, 8189, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 8187, 0, 0, 0, 2, 0, 0, 0, 2, 0, 1, 0, 8190, 0, 0, 0, 2, 0, 115, 0, 0, 0, 7, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 0, 45, 0, 8127, 2, 1176, 8, 6996, 0, 44, 0, 111, 18, 16, 0, 0, 0, 7971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 34, 1, 8151, 0, 2, 0, 0, 139, 0, 12, 8160, 0, 0, 0, 0, 0, 0, 6, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8021, 0, 0, 0, 0, 0, 0, 12, 18, 0, 8, 18, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 7, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 219, 0, 0, 16, 0, 0, 0, 7991, 0, 0, 0, 0, 1, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 1, 0, 8, 0, 0, 62, 8190, 0, 0, 0, 8145, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 248, 0, 0, 0, 0, 0, 0, 7942, 0, 0, 0, 0, 1, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 0, 0, 27, 8192, 0, 0, 0, 8089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 356, 0, 0, 0, 8099, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 22, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 8192, 0, 0, 0, 7513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 761, 0, 0, 97, 0, 0, 0, 8052, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 16, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 5, 8192, 0, 0, 0, 8117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 58, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 8, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 2, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 24, 0, 0, 0, 7993, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8105, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8269, 8275, 8204, 8021, 8188, 8204, 8180, 8195] avg=8192.0000 max=8275 max_rank=1 min=8021 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32767 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7789, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 0, 0, 27, 84, 0, 0, 0, 8089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 356, 0, 0, 0, 7935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 8105, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8147, 0, 0, 0, 0, 0, 22, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7871, 8306, 8204, 8336, 8241, 8205, 8180, 8193] avg=8192.0000 max=8336 max_rank=3 min=7871 min_rank=0 imbalance_ratio=1.017578 max_load_reduction=57200 max_load_reduction_pct=0.872803 moved_to_echo=32819 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 0, 0, 0, 0, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 77, 4, 0, 0, 8076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 3, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 3, 1, 1, 0, 8015, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 8105, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 2, 0, 0, 0, 2, 0, 1, 0, 10, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 2, 0, 115, 0, 0, 0, 7, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8189, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8220, 8160, 8204, 8029, 8224, 8214, 8286, 8199] avg=8192.0000 max=8286 max_rank=6 min=8029 min_rank=3 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32923 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 219, 0, 0, 16, 0, 0, 0, 7829, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8105, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8272, 8241, 8204, 8065, 8179, 8205, 8175, 8195] avg=8192.0000 max=8272 max_rank=0 min=8065 min_rank=3 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32754 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 0, 45, 0, 84, 2, 1176, 8, 6996, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 18, 16, 0, 0, 0, 7971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 34, 1, 7985, 0, 2, 0, 0, 139, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 22, 0, 0, 0, 0, 0, 0, 0, 8040, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8021, 0, 0, 0, 0, 0, 0, 12, 18, 0, 8, 18, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8244, 8422, 8128, 8182, 8144, 8097, 8148, 8171] avg=8192.0000 max=8422 max_rank=1 min=8097 min_rank=5 imbalance_ratio=1.028076 max_load_reduction=57114 max_load_reduction_pct=0.871490 moved_to_echo=32560 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 5, 84, 0, 0, 0, 8117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 58, 0, 0, 0, 7993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8105, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 8209, 8204, 8127, 8182, 8205, 8195, 8195] avg=8192.0000 max=8219 max_rank=0 min=8127 min_rank=3 imbalance_ratio=1.003296 max_load_reduction=57317 max_load_reduction_pct=0.874588 moved_to_echo=32777 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 8, 0, 0, 62, 84, 0, 0, 0, 8145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 248, 0, 0, 0, 0, 0, 0, 7781, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8103, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 5, 8185, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8272, 8301, 8190, 8030, 8178, 8201, 8170, 8194] avg=8192.0000 max=8301 max_rank=1 min=8030 min_rank=3 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32743 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8088, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 84, 0, 0, 0, 7513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 761, 0, 0, 97, 0, 0, 0, 7888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 16, 8105, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8169, 7622, 8198, 8746, 8200, 8205, 8202, 8194] avg=8192.0000 max=8746 max_rank=3 min=7622 min_rank=1 imbalance_ratio=1.067627 max_load_reduction=56790 max_load_reduction_pct=0.866547 moved_to_echo=32801 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 0, 128, 0, 128, 128, 640, 128, 1280, 128, 63488, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 0, 128, 0, 128, 128, 768, 128, 1280, 128, 63360, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[1408, 0, 128, 640, 128, 128, 128, 63488, 0, 128, 0, 0, 256, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 0, 0, 0, 128, 128, 0, 128, 0, 0, 0, 128, 0, 128, 128, 64896, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[65408, 0, 0, 0, 128, 0, 256, 0, 0, 0, 128, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 65024, 0, 0, 0, 0, 0, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 0, 0, 65408, 0, 128, 128, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[1280, 0, 128, 640, 128, 128, 128, 63616, 0, 128, 0, 0, 256, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 128, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 64896, 0, 0, 0, 0, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 0, 0, 65408, 0, 128, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 128, 0, 128, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 128, 65408, 0, 0, 0, 128, 0, 0, 128, 128, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[65408, 0, 0, 0, 128, 0, 256, 0, 0, 0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 792, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 81, 0, 0, 0, 0, 0, 0, 7933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 182, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7444, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 256, 0, 0, 0, 0, 0, 0, 7913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 485, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 16, 0, 0, 0, 0, 0, 0, 7984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7717, 8191, 0, 4, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 90, 2, 0, 1, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 166, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 10, 0, 0, 0, 8057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8051, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 115, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 93, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 12, 0, 0, 0, 0, 0, 0, 8102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8126, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 96, 0, 0, 0, 0, 0, 0, 8114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 326, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7876, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 121, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 8138, 0, 0, 0, 0, 0, 0, 0, 0, 239, 0, 0, 6, 0, 0, 0, 0, 0, 26, 0, 4, 0, 0, 0, 0, 0, 0, 34, 0, 0, 7982, 0, 0, 0, 0, 0, 0, 0, 0, 8142, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 2, 36, 0, 0, 18, 0, 0, 3, 0, 0, 4, 0, 0, 0, 7, 23, 8115, 8120, 0, 0, 15, 0, 14, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 19, 0, 10, 0, 0, 0, 0, 32, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 228, 0, 0, 8190, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 10, 0, 2, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 1, 0, 8099, 1, 0, 0, 0, 0, 2, 0, 0, 0, 15, 0, 0, 0, 0, 138, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 2, 0, 0, 0, 1, 7872, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 203, 0, 0, 0, 0, 0, 0, 8006, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 444, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 32, 0, 0, 0, 0, 0, 0, 7955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7755, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 98, 0, 0, 0, 0, 0, 0, 8095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 792, 0, 0, 7877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8065, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 0, 81, 0, 0, 0, 0, 0, 0, 7933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 182, 0, 0, 7760, 307, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7444, 316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8053, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8669, 8181, 8141, 8278, 7761, 8309, 7997, 8200] avg=8192.0000 max=8669 max_rank=0 min=7761 min_rank=4 imbalance_ratio=1.058228 max_load_reduction=56867 max_load_reduction_pct=0.867722 moved_to_echo=32267 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 326, 0, 0, 7884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8065, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 0, 0, 0, 0, 0, 0, 0, 0, 8069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 7760, 308, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7876, 316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8083, 0, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8181, 8196, 8101, 8192, 8204, 8260, 8192] avg=8192.0000 max=8260 max_rank=6 min=8101 min_rank=3 imbalance_ratio=1.008301 max_load_reduction=57276 max_load_reduction_pct=0.873962 moved_to_echo=32848 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 485, 0, 0, 7879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8063, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 0, 16, 0, 0, 0, 0, 0, 0, 7984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 0, 0, 7759, 308, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7717, 316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 8062, 0, 90, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8364, 8180, 8127, 8234, 8033, 8159, 8245, 8194] avg=8192.0000 max=8364 max_rank=0 min=8033 min_rank=4 imbalance_ratio=1.020996 max_load_reduction=57172 max_load_reduction_pct=0.872375 moved_to_echo=32631 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 93, 0, 0, 7884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8065, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 0, 12, 0, 0, 0, 0, 0, 0, 8102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 7760, 308, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8126, 316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8070, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7977, 8181, 8241, 8139, 8442, 8166, 8198, 8192] avg=8192.0000 max=8442 max_rank=4 min=7977 min_rank=0 imbalance_ratio=1.030518 max_load_reduction=57094 max_load_reduction_pct=0.871185 moved_to_echo=32998 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 7832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 239, 0, 0, 6, 0, 0, 0, 0, 0, 26, 0, 4, 0, 0, 0, 0, 7858, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 124, 0, 0, 0, 0, 0, 0, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 2, 36, 7691, 306, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 3, 0, 0, 4, 0, 0, 0, 7, 23, 8115, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 14, 0, 0, 0, 0, 8054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 19, 0, 10, 0, 0, 0, 0, 32, 0, 0, 0, 15, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7832, 8248, 8300, 8092, 8484, 8083, 8321, 8176] avg=8192.0000 max=8484 max_rank=4 min=7832 min_rank=0 imbalance_ratio=1.035645 max_load_reduction=57052 max_load_reduction_pct=0.870544 moved_to_echo=33064 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 166, 0, 0, 7884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8062, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 0, 0, 0, 0, 10, 0, 0, 0, 8057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 7760, 308, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8051, 316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8083, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8050, 8178, 8194, 8115, 8368, 8198, 8240, 8193] avg=8192.0000 max=8368 max_rank=4 min=8050 min_rank=0 imbalance_ratio=1.021484 max_load_reduction=57168 max_load_reduction_pct=0.872314 moved_to_echo=32999 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 444, 0, 0, 7880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8065, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 0, 32, 0, 0, 0, 0, 0, 0, 7955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 229, 0, 0, 7760, 308, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7755, 316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8051, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8324, 8181, 8114, 8326, 8071, 8149, 8179, 8192] avg=8192.0000 max=8326 max_rank=3 min=8071 min_rank=4 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32591 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 228, 0, 0, 7880, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 2, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8068, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 0, 0, 0, 0, 0, 0, 1, 0, 8099, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 15, 0, 0, 0, 0, 138, 0, 0, 7760, 310, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 2, 0, 0, 0, 1, 7872, 307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8065, 0, 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8006, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8110, 8206, 8223, 8251, 8185, 8268, 8096, 8197] avg=8192.0000 max=8268 max_rank=5 min=8096 min_rank=6 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32746 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 164, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 10, 0, 0, 0, 8043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8059, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 122, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 438, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 32, 0, 0, 0, 0, 0, 0, 7946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7764, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 98, 0, 0, 0, 0, 0, 0, 8094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 734, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 80, 0, 0, 0, 0, 0, 0, 7909, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 200, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7505, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 266, 0, 0, 0, 0, 0, 0, 7904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 8137, 0, 0, 0, 0, 0, 0, 0, 0, 243, 0, 0, 7, 0, 0, 0, 0, 0, 26, 0, 4, 0, 0, 0, 0, 0, 0, 40, 0, 0, 7981, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 1, 39, 0, 0, 19, 0, 0, 3, 0, 0, 4, 0, 0, 0, 7, 23, 8114, 8117, 0, 0, 15, 0, 15, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 18, 0, 10, 0, 0, 0, 0, 32, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 319, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7886, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 122, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 221, 0, 0, 8190, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 9, 0, 2, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 1, 0, 8093, 1, 0, 0, 0, 0, 1, 0, 0, 0, 18, 0, 0, 0, 0, 138, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 2, 0, 0, 0, 1, 7874, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 221, 0, 0, 0, 0, 0, 0, 8000, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 463, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 14, 0, 0, 0, 0, 0, 0, 7973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7737, 8191, 0, 4, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 99, 1, 0, 1, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 92, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 14, 0, 0, 0, 0, 0, 0, 8094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8130, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 103, 0, 0, 0, 0, 0, 0, 8115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 734, 0, 0, 7890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8054, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 80, 0, 0, 0, 0, 0, 0, 7909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 200, 0, 0, 7764, 294, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7505, 302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8049, 0, 266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8624, 8180, 8126, 8291, 7808, 8315, 7991, 8201] avg=8192.0000 max=8624 max_rank=0 min=7808 min_rank=4 imbalance_ratio=1.052734 max_load_reduction=56912 max_load_reduction_pct=0.868408 moved_to_echo=32315 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 164, 0, 0, 7897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8052, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 0, 0, 10, 0, 0, 0, 8043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 7764, 295, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8059, 302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8076, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8061, 8178, 8190, 8112, 8362, 8198, 8242, 8193] avg=8192.0000 max=8362 max_rank=4 min=8061 min_rank=0 imbalance_ratio=1.020752 max_load_reduction=57174 max_load_reduction_pct=0.872406 moved_to_echo=32995 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 319, 0, 0, 7897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8055, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 0, 0, 0, 0, 0, 0, 8063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 7764, 295, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7886, 302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8076, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8181, 8200, 8098, 8188, 8198, 8263, 8192] avg=8192.0000 max=8263 max_rank=6 min=8098 min_rank=3 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32841 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 438, 0, 0, 7894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8055, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 32, 0, 0, 0, 0, 0, 0, 7946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 236, 0, 0, 7764, 294, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7764, 302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8045, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8332, 8181, 8115, 8326, 8066, 8143, 8181, 8192] avg=8192.0000 max=8332 max_rank=0 min=8066 min_rank=4 imbalance_ratio=1.017090 max_load_reduction=57204 max_load_reduction_pct=0.872864 moved_to_echo=32582 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 92, 0, 0, 7897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8055, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 14, 0, 0, 0, 0, 0, 0, 8094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 7764, 295, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8130, 302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8063, 0, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7989, 8181, 8245, 8129, 8432, 8166, 8202, 8192] avg=8192.0000 max=8432 max_rank=4 min=7989 min_rank=0 imbalance_ratio=1.029297 max_load_reduction=57104 max_load_reduction_pct=0.871338 moved_to_echo=32992 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 7844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 0, 0, 7, 0, 0, 0, 0, 0, 26, 0, 4, 0, 0, 0, 0, 7848, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 133, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 1, 39, 7693, 293, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 3, 0, 0, 4, 0, 0, 0, 7, 23, 8114, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 15, 0, 0, 0, 0, 8048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 18, 0, 10, 0, 0, 0, 0, 32, 0, 0, 0, 12, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7844, 8253, 8310, 8086, 8469, 8078, 8320, 8176] avg=8192.0000 max=8469 max_rank=4 min=7844 min_rank=0 imbalance_ratio=1.033813 max_load_reduction=57067 max_load_reduction_pct=0.870773 moved_to_echo=33043 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 221, 0, 0, 7892, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 2, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8059, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 0, 0, 0, 0, 0, 0, 1, 0, 8093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 18, 0, 0, 0, 0, 138, 0, 0, 7769, 298, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 2, 0, 0, 0, 1, 7874, 292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8056, 0, 221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8000, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8114, 8202, 8226, 8257, 8172, 8277, 8093, 8195] avg=8192.0000 max=8277 max_rank=5 min=8093 min_rank=6 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32737 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 463, 0, 0, 7893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8053, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 14, 0, 0, 0, 0, 0, 0, 7973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 0, 0, 7763, 294, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7737, 302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 8056, 0, 99, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8356, 8180, 8124, 8237, 8039, 8161, 8244, 8195] avg=8192.0000 max=8356 max_rank=0 min=8039 min_rank=4 imbalance_ratio=1.020020 max_load_reduction=57180 max_load_reduction_pct=0.872498 moved_to_echo=32639 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 128, 128, 0, 0, 0, 128, 0, 0, 128, 0, 768, 128, 128, 62080, 2560, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 0, 128, 128, 0, 0, 128, 128, 0, 0, 128, 128, 62976, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 2560, 0, 0, 63104, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[256, 128, 0, 128, 0, 128, 128, 0, 0, 128, 0, 128, 0, 128, 0, 0, 64384, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 128, 0, 0, 0, 0, 64640, 0, 1024, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 0, 0, 1024, 0, 256, 0, 0, 128, 0, 128, 0, 64384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 0, 0, 65536, 0, 128, 0, 0, 0, 128, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 64896, 128, 0, 128, 0, 0, 0, 0, 128, 128, 128, 0, 128, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 0, 128, 128, 0, 0, 128, 128, 0, 0, 128, 128, 63104, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 128, 128, 0, 0, 0, 128, 0, 0, 128, 0, 896, 128, 128, 62080, 2432, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[256, 128, 0, 128, 0, 128, 128, 0, 0, 128, 0, 128, 0, 128, 0, 0, 64256, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 2432, 0, 0, 63104, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 128, 0, 0, 1152, 0, 256, 0, 0, 128, 0, 128, 0, 64384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 64768, 128, 0, 128, 0, 0, 0, 0, 128, 128, 128, 0, 128, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 128, 0, 0, 0, 0, 64512, 0, 1152, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 0, 65536, 0, 128, 0, 0, 0, 128, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True - [2026-08-26 14:48:11] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 436.7 | throughput per GPU (TFLOP/s/GPU): 414.1 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.009987E+01 | loss scale: 1.0 | grad norm: 83.590 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 1, 2, 1049, 265, 5515, 0, 0, 6045, 0, 0, 1, 1, 458, 0, 0, 7, 0, 1, 232, 0, 3, 2558, 246, 0, 0, 8188, 1134, 8179, 2, 3426, 0, 0, 1026, 0, 0, 0, 0, 0, 39, 0, 2311, 0, 0, 5802, 0, 115, 0, 6, 1038, 0, 0, 0, 0, 0, 172, 0, 54, 1668, 0, 1, 0, 6, 5, 0, 0, 0, 810, 2, 1, 1, 808, 0, 696, 1, 1, 14, 213, 0, 1, 3, 0, 0, 131, 0, 0, 1900, 1, 137, 1, 26, 11, 0, 0, 1, 0, 2586, 0, 0, 1, 3, 1, 2, 12, 2468, 354, 0, 0, 18, 411, 0, 0, 4259, 1, 0, 0, 6, 82, 2, 0, 2, 9, 1004, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 739, 329, 4493, 0, 1, 5660, 5, 0, 0, 0, 614, 0, 2, 5, 0, 8, 358, 0, 4, 2028, 293, 0, 0, 8192, 667, 8172, 3, 3585, 0, 1, 388, 0, 1, 0, 0, 2, 15, 0, 2625, 2, 0, 5225, 2, 165, 0, 6, 915, 0, 0, 0, 0, 0, 384, 4, 55, 1353, 0, 0, 0, 51, 0, 0, 0, 0, 936, 0, 0, 1, 852, 0, 1240, 8, 33, 24, 235, 0, 0, 2, 2, 0, 333, 0, 0, 2718, 1, 149, 0, 49, 85, 1, 5, 0, 0, 1757, 2, 0, 1, 13, 0, 0, 24, 4492, 986, 0, 0, 44, 444, 0, 0, 3407, 0, 0, 0, 0, 60, 0, 0, 3, 48, 1225, 0, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 1600, 301, 3831, 0, 0, 6287, 0, 1, 1, 0, 224, 0, 1, 11, 0, 5, 334, 0, 1, 2868, 214, 1, 0, 8189, 738, 8175, 1, 3647, 0, 0, 958, 0, 2, 1, 0, 1, 25, 0, 3075, 0, 0, 5342, 0, 62, 0, 1, 724, 2, 9, 0, 0, 2, 331, 1, 68, 1967, 0, 1, 0, 5, 0, 0, 0, 0, 630, 0, 4, 0, 1095, 0, 818, 0, 1, 13, 283, 0, 0, 0, 0, 0, 87, 0, 0, 1639, 3, 79, 0, 50, 78, 1, 0, 0, 0, 2916, 2, 0, 3, 23, 1, 0, 13, 4057, 503, 0, 0, 74, 774, 3, 0, 2373, 0, 2, 0, 3, 64, 0, 0, 0, 4, 929, 0, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 549, 513, 5190, 0, 0, 6656, 8, 0, 0, 0, 351, 3, 0, 26, 0, 11, 403, 0, 2, 2602, 319, 0, 0, 8188, 842, 8172, 0, 2981, 12, 15, 585, 0, 2, 4, 1, 0, 9, 1, 3023, 4, 4, 5159, 9, 97, 0, 0, 763, 0, 0, 4, 0, 0, 418, 2, 55, 1485, 2, 0, 0, 14, 0, 2, 0, 2, 1066, 0, 0, 0, 753, 0, 914, 18, 1, 12, 317, 0, 0, 0, 0, 1, 177, 0, 0, 1570, 0, 190, 4, 13, 113, 0, 0, 0, 0, 2302, 0, 1, 3, 39, 0, 0, 44, 4624, 930, 0, 0, 60, 359, 0, 0, 2251, 0, 5, 0, 16, 84, 0, 1, 3, 22, 1145, 1, 1, 5, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 823, 382, 4829, 0, 0, 5781, 0, 5, 3, 0, 452, 2, 0, 5, 0, 28, 95, 0, 3, 2268, 416, 1, 0, 8192, 895, 8162, 2, 4209, 3, 4, 610, 0, 1, 5, 0, 3, 20, 1, 2428, 1, 0, 4956, 3, 65, 0, 0, 713, 0, 4, 1, 0, 0, 271, 0, 69, 1751, 0, 0, 0, 31, 0, 1, 0, 0, 834, 0, 0, 0, 916, 0, 1044, 12, 18, 24, 193, 0, 0, 0, 1, 6, 176, 0, 0, 2154, 0, 146, 0, 42, 18, 0, 3, 0, 0, 2571, 3, 0, 4, 9, 0, 0, 27, 3590, 803, 0, 2, 12, 386, 0, 0, 3946, 0, 0, 1, 0, 43, 0, 0, 0, 15, 1027, 4, 0, 9, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 9, 759, 241, 4591, 0, 0, 6528, 0, 0, 0, 0, 414, 2, 0, 17, 0, 0, 337, 0, 0, 1989, 194, 0, 0, 8191, 640, 8188, 0, 3478, 1, 0, 932, 0, 4, 24, 0, 0, 19, 0, 2591, 26, 0, 5768, 1, 60, 0, 0, 1042, 0, 0, 4, 0, 1, 367, 0, 104, 1272, 0, 0, 0, 15, 4, 1, 0, 0, 1223, 0, 1, 0, 932, 0, 724, 20, 0, 5, 443, 0, 0, 0, 0, 0, 240, 0, 0, 2684, 0, 247, 0, 5, 21, 2, 0, 0, 0, 3766, 0, 0, 6, 12, 0, 0, 19, 3483, 585, 0, 0, 119, 471, 0, 0, 1944, 0, 0, 0, 3, 71, 0, 0, 0, 8, 686, 0, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1, 0, 0, 8, 954, 342, 4774, 2, 0, 6979, 0, 0, 0, 1, 120, 0, 1, 3, 0, 1, 245, 0, 1, 2110, 185, 0, 0, 8190, 610, 8188, 0, 4050, 0, 2, 1177, 0, 0, 1, 0, 0, 2, 0, 3368, 15, 1, 5699, 0, 76, 0, 0, 686, 1, 0, 1, 0, 2, 437, 0, 57, 1471, 0, 0, 0, 20, 0, 2, 0, 0, 655, 0, 3, 0, 914, 0, 670, 6, 11, 16, 363, 0, 0, 0, 0, 1, 35, 0, 0, 2554, 2, 118, 0, 16, 72, 0, 0, 3, 0, 2145, 2, 0, 4, 39, 3, 2, 5, 2495, 433, 0, 0, 32, 663, 1, 0, 3019, 0, 0, 0, 0, 39, 0, 0, 2, 6, 1424, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 1298, 181, 4122, 0, 0, 7138, 0, 0, 0, 0, 329, 0, 0, 20, 0, 25, 328, 0, 11, 2683, 274, 0, 0, 8192, 881, 8184, 0, 3062, 1, 27, 1076, 1, 7, 14, 0, 4, 43, 0, 2620, 3, 0, 4052, 0, 96, 0, 0, 1405, 0, 0, 0, 0, 0, 385, 0, 80, 1263, 0, 0, 0, 18, 0, 1, 0, 0, 1030, 0, 1, 1, 1010, 0, 858, 11, 7, 53, 180, 0, 0, 2, 1, 1, 230, 0, 0, 2086, 3, 221, 0, 17, 104, 1, 0, 0, 0, 2659, 0, 0, 4, 81, 1, 0, 6, 4716, 686, 0, 0, 42, 508, 0, 0, 1891, 0, 1, 0, 2, 146, 0, 0, 0, 15, 1125, 0, 1, 5, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 1600, 301, 3831, 0, 0, 1827, 0, 1, 1, 0, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 11, 0, 5, 334, 0, 1, 2868, 214, 1, 0, 0, 738, 863, 1, 3647, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 958, 0, 2, 1, 0, 1, 25, 0, 3075, 0, 0, 4531, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 724, 2, 9, 0, 0, 2, 331, 1, 68, 1967, 0, 1, 0, 5, 2716, 2594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 630, 0, 4, 0, 1095, 0, 818, 0, 1, 13, 283, 0, 4596, 585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 1639, 3, 79, 0, 50, 78, 1, 0, 0, 5595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2916, 2, 0, 3, 23, 1, 0, 13, 4057, 503, 0, 0, 74, 774, 3, 609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2373, 0, 2, 0, 3, 64, 0, 0, 0, 4, 929, 0, 0, 0, 2, 3875, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7787, 8684, 8655, 8421, 8025, 7532, 8978, 7454] avg=8192.0000 max=8978 max_rank=6 min=7454 min_rank=7 imbalance_ratio=1.095947 max_load_reduction=56558 max_load_reduction_pct=0.863007 moved_to_echo=31989 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 549, 513, 5190, 0, 0, 1934, 8, 0, 0, 0, 351, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 11, 403, 0, 2, 2602, 319, 0, 0, 0, 842, 863, 0, 2981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 15, 585, 0, 2, 4, 1, 0, 9, 1, 3023, 4, 4, 4376, 9, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 763, 0, 0, 4, 0, 0, 418, 2, 55, 1485, 2, 0, 0, 14, 2715, 2594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 1066, 0, 0, 0, 753, 0, 914, 18, 1, 12, 317, 0, 4594, 619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 177, 0, 0, 1570, 0, 190, 4, 13, 113, 0, 0, 0, 5594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2302, 0, 1, 3, 39, 0, 0, 44, 4624, 930, 0, 0, 60, 359, 0, 588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2251, 0, 5, 0, 16, 84, 0, 1, 3, 22, 1145, 1, 1, 5, 0, 4103, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8551, 8049, 8142, 8052, 8298, 7662, 8950, 7832] avg=8192.0000 max=8950 max_rank=6 min=7662 min_rank=5 imbalance_ratio=1.092529 max_load_reduction=56586 max_load_reduction_pct=0.863434 moved_to_echo=32742 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 739, 329, 4493, 0, 1, 1644, 5, 0, 0, 0, 614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5, 0, 8, 358, 0, 4, 2028, 293, 0, 0, 0, 667, 863, 3, 3585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 388, 0, 1, 0, 0, 2, 15, 0, 2625, 2, 0, 4432, 2, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 915, 0, 0, 0, 0, 0, 384, 4, 55, 1353, 0, 0, 0, 51, 2715, 2595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 936, 0, 0, 1, 852, 0, 1240, 8, 33, 24, 235, 0, 4594, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 333, 0, 0, 2718, 1, 149, 0, 49, 85, 1, 5, 0, 5597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1757, 2, 0, 1, 13, 0, 0, 24, 4492, 986, 0, 0, 44, 444, 0, 595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3407, 0, 0, 0, 0, 60, 0, 0, 3, 48, 1225, 0, 0, 2, 0, 3489, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7827, 7816, 7633, 8078, 8450, 8942, 8358, 8432] avg=8192.0000 max=8942 max_rank=5 min=7633 min_rank=2 imbalance_ratio=1.091553 max_load_reduction=56594 max_load_reduction_pct=0.863556 moved_to_echo=34182 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 2, 1049, 265, 5515, 0, 0, 1756, 0, 0, 1, 1, 458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 1, 232, 0, 3, 2558, 246, 0, 0, 0, 1134, 864, 2, 3426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 0, 0, 0, 0, 0, 39, 0, 2311, 0, 0, 4921, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1038, 0, 0, 0, 0, 0, 172, 0, 54, 1668, 0, 1, 0, 6, 2717, 2594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 810, 2, 1, 1, 808, 0, 696, 1, 1, 14, 213, 0, 4598, 563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 131, 0, 0, 1900, 1, 137, 1, 26, 11, 0, 0, 1, 5594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2586, 0, 0, 1, 3, 1, 2, 12, 2468, 354, 0, 0, 18, 411, 0, 661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4259, 1, 0, 0, 6, 82, 2, 0, 2, 9, 1004, 0, 0, 0, 0, 3726, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9048, 8473, 8412, 8256, 7713, 7806, 6517, 9311] avg=8192.0000 max=9311 max_rank=7 min=6517 min_rank=6 imbalance_ratio=1.136597 max_load_reduction=56225 max_load_reduction_pct=0.857925 moved_to_echo=31347 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 8, 954, 342, 4774, 2, 0, 2027, 0, 0, 0, 1, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 1, 245, 0, 1, 2110, 185, 0, 0, 0, 610, 865, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1177, 0, 0, 1, 0, 0, 2, 0, 3368, 15, 1, 4833, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 686, 1, 0, 1, 0, 2, 437, 0, 57, 1471, 0, 0, 0, 20, 2720, 2595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 655, 0, 3, 0, 914, 0, 670, 6, 11, 16, 363, 0, 4603, 650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 35, 0, 0, 2554, 2, 118, 0, 16, 72, 0, 0, 3, 5595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2145, 2, 0, 4, 39, 3, 2, 5, 2495, 433, 0, 0, 32, 663, 1, 650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3019, 0, 0, 0, 0, 39, 0, 0, 2, 6, 1424, 0, 0, 0, 0, 4302, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8229, 8071, 9475, 7990, 7893, 8396, 6474, 9008] avg=8192.0000 max=9475 max_rank=2 min=6474 min_rank=6 imbalance_ratio=1.156616 max_load_reduction=56061 max_load_reduction_pct=0.855423 moved_to_echo=31771 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 823, 382, 4829, 0, 0, 1680, 0, 5, 3, 0, 452, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 28, 95, 0, 3, 2268, 416, 1, 0, 0, 895, 861, 2, 4209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 610, 0, 1, 5, 0, 3, 20, 1, 2428, 1, 0, 4203, 3, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 713, 0, 4, 1, 0, 0, 271, 0, 69, 1751, 0, 0, 0, 31, 2712, 2596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 834, 0, 0, 0, 916, 0, 1044, 12, 18, 24, 193, 0, 4589, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 176, 0, 0, 2154, 0, 146, 0, 42, 18, 0, 3, 0, 5596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2571, 3, 0, 4, 9, 0, 0, 27, 3590, 803, 0, 2, 12, 386, 0, 565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3946, 0, 0, 1, 0, 43, 0, 0, 0, 15, 1027, 4, 0, 9, 0, 3563, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8783, 7347, 8148, 8169, 8142, 7972, 8796] avg=8192.0000 max=8796 max_rank=7 min=7347 min_rank=2 imbalance_ratio=1.073730 max_load_reduction=56740 max_load_reduction_pct=0.865784 moved_to_echo=33079 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 9, 759, 241, 4591, 0, 0, 1896, 0, 0, 0, 0, 414, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 337, 0, 0, 1989, 194, 0, 0, 0, 640, 865, 0, 3478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 932, 0, 4, 24, 0, 0, 19, 0, 2591, 26, 0, 4893, 1, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 0, 0, 4, 0, 1, 367, 0, 104, 1272, 0, 0, 0, 15, 2720, 2596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 1223, 0, 1, 0, 932, 0, 724, 20, 0, 5, 443, 0, 4603, 608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 0, 0, 2684, 0, 247, 0, 5, 21, 2, 0, 0, 5595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3766, 0, 0, 6, 12, 0, 0, 19, 3483, 585, 0, 0, 119, 471, 0, 657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1944, 0, 0, 0, 3, 71, 0, 0, 0, 8, 686, 0, 0, 2, 0, 4024, 218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7912, 7520, 8551, 8121, 8564, 8794, 9118, 6956] avg=8192.0000 max=9118 max_rank=6 min=6956 min_rank=7 imbalance_ratio=1.113037 max_load_reduction=56418 max_load_reduction_pct=0.860870 moved_to_echo=33432 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 1298, 181, 4122, 0, 0, 2067, 0, 0, 0, 0, 329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 25, 328, 0, 11, 2683, 274, 0, 0, 0, 881, 856, 0, 3062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 27, 1076, 1, 7, 14, 0, 4, 43, 0, 2620, 3, 0, 3429, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1405, 0, 0, 0, 0, 0, 385, 0, 80, 1263, 0, 0, 0, 18, 2723, 2596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1030, 0, 1, 1, 1010, 0, 858, 11, 7, 53, 180, 0, 4605, 667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 230, 0, 0, 2086, 3, 221, 0, 17, 104, 1, 0, 0, 5596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2659, 0, 0, 4, 81, 1, 0, 6, 4716, 686, 0, 0, 42, 508, 0, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1891, 0, 1, 0, 2, 146, 0, 0, 0, 15, 1125, 0, 1, 5, 0, 4404, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8003, 8140, 7321, 8470, 8424, 8262, 9169, 7747] avg=8192.0000 max=9169 max_rank=6 min=7321 min_rank=2 imbalance_ratio=1.119263 max_load_reduction=56367 max_load_reduction_pct=0.860092 moved_to_echo=33602 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 + [2026-08-26 15:24:12] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 447.8 | throughput per GPU (TFLOP/s/GPU): 403.8 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.010174E+01 | loss scale: 1.0 | grad norm: 83.769 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1, 0, 0, 9, 999, 356, 4405, 2, 0, 6981, 0, 0, 0, 1, 141, 0, 1, 3, 0, 1, 230, 0, 1, 2176, 188, 0, 0, 8190, 614, 8188, 0, 4088, 0, 2, 1163, 0, 0, 1, 0, 0, 2, 0, 3387, 14, 1, 5808, 0, 78, 0, 0, 709, 1, 0, 1, 0, 2, 421, 0, 57, 1496, 0, 0, 0, 21, 0, 2, 0, 0, 653, 0, 3, 0, 937, 0, 672, 7, 12, 18, 366, 0, 0, 0, 0, 1, 42, 0, 0, 2582, 2, 114, 0, 15, 60, 0, 0, 4, 0, 2099, 2, 0, 4, 40, 3, 2, 5, 2465, 431, 0, 0, 36, 668, 1, 0, 3062, 0, 0, 0, 0, 43, 0, 0, 2, 6, 1438, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 1, 2, 1089, 270, 5260, 0, 0, 6042, 0, 0, 1, 1, 500, 0, 0, 7, 0, 1, 208, 0, 3, 2625, 245, 0, 0, 8188, 1145, 8181, 2, 3461, 0, 1, 1006, 0, 0, 0, 0, 1, 36, 0, 2311, 0, 0, 5898, 0, 125, 0, 6, 1067, 0, 0, 0, 0, 0, 166, 0, 54, 1682, 0, 1, 0, 7, 5, 0, 0, 0, 795, 2, 1, 1, 828, 0, 690, 1, 2, 14, 219, 0, 1, 3, 0, 0, 140, 0, 0, 1910, 1, 129, 1, 22, 10, 0, 0, 1, 0, 2492, 0, 0, 0, 3, 1, 2, 12, 2441, 351, 0, 0, 19, 420, 0, 0, 4300, 0, 0, 0, 6, 82, 2, 0, 3, 8, 1025, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 1654, 307, 3498, 0, 0, 6286, 0, 1, 1, 0, 254, 0, 1, 11, 0, 6, 313, 0, 1, 2946, 218, 1, 0, 8189, 750, 8175, 1, 3676, 0, 0, 979, 0, 2, 1, 0, 1, 25, 0, 3096, 0, 0, 5423, 0, 67, 0, 1, 745, 2, 9, 0, 0, 2, 320, 1, 72, 2006, 0, 1, 0, 8, 0, 0, 0, 0, 621, 0, 4, 0, 1102, 0, 810, 0, 1, 15, 285, 0, 0, 0, 0, 0, 90, 0, 0, 1647, 3, 73, 0, 47, 65, 1, 0, 0, 0, 2840, 2, 0, 2, 22, 1, 0, 15, 4024, 507, 0, 0, 76, 775, 3, 0, 2414, 0, 2, 0, 3, 71, 0, 0, 0, 4, 956, 0, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 754, 344, 4119, 0, 1, 5643, 5, 0, 0, 0, 654, 0, 2, 6, 0, 9, 343, 0, 4, 2091, 297, 0, 0, 8192, 673, 8175, 3, 3626, 0, 1, 380, 0, 1, 0, 0, 2, 13, 0, 2637, 2, 0, 5326, 1, 174, 0, 8, 938, 0, 0, 0, 0, 0, 370, 4, 53, 1368, 0, 0, 0, 52, 0, 0, 0, 0, 937, 0, 0, 1, 888, 0, 1237, 8, 31, 25, 237, 0, 0, 2, 2, 0, 352, 0, 0, 2757, 1, 130, 0, 48, 71, 1, 4, 0, 0, 1695, 2, 0, 1, 11, 0, 0, 25, 4497, 985, 0, 0, 48, 436, 0, 0, 3450, 0, 0, 0, 0, 64, 0, 0, 4, 52, 1259, 0, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 10, 788, 245, 4209, 0, 0, 6535, 0, 0, 0, 0, 446, 1, 0, 17, 0, 0, 324, 0, 0, 2051, 189, 0, 0, 8191, 657, 8188, 0, 3528, 2, 0, 934, 0, 4, 24, 0, 0, 18, 0, 2603, 25, 0, 5849, 1, 57, 0, 0, 1058, 0, 0, 4, 0, 1, 351, 0, 116, 1291, 0, 0, 0, 14, 4, 1, 0, 0, 1224, 0, 1, 0, 961, 0, 734, 19, 0, 5, 460, 0, 0, 0, 0, 0, 245, 0, 0, 2748, 0, 220, 0, 4, 18, 2, 0, 0, 0, 3693, 0, 0, 5, 12, 0, 0, 18, 3457, 584, 0, 0, 124, 476, 0, 0, 1980, 0, 0, 0, 4, 85, 0, 0, 0, 7, 712, 0, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 1347, 189, 3798, 0, 0, 7135, 0, 0, 0, 0, 359, 0, 0, 21, 0, 24, 321, 0, 12, 2734, 275, 0, 0, 8192, 886, 8184, 0, 3092, 1, 28, 1088, 1, 7, 14, 0, 4, 49, 0, 2612, 3, 0, 4158, 0, 96, 0, 0, 1438, 0, 0, 0, 0, 0, 384, 0, 83, 1282, 0, 0, 0, 18, 0, 1, 0, 0, 1029, 0, 1, 1, 1025, 0, 864, 14, 7, 54, 180, 0, 0, 2, 1, 1, 242, 0, 0, 2122, 3, 202, 0, 18, 85, 1, 0, 0, 0, 2582, 0, 0, 4, 84, 1, 0, 6, 4698, 683, 0, 0, 41, 509, 0, 0, 1898, 0, 1, 0, 3, 157, 0, 0, 0, 15, 1155, 0, 1, 4, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 850, 391, 4449, 0, 0, 5772, 0, 5, 3, 0, 480, 2, 0, 5, 0, 30, 70, 0, 3, 2334, 420, 1, 0, 8192, 908, 8162, 2, 4270, 3, 4, 588, 0, 1, 6, 0, 3, 18, 1, 2442, 1, 0, 5053, 3, 68, 0, 0, 725, 0, 4, 1, 0, 0, 262, 0, 78, 1783, 0, 0, 0, 31, 0, 1, 0, 0, 825, 0, 0, 0, 947, 0, 1029, 13, 18, 26, 189, 0, 0, 0, 0, 7, 182, 0, 0, 2204, 0, 141, 0, 40, 15, 0, 3, 0, 0, 2518, 3, 0, 4, 8, 0, 0, 26, 3586, 800, 0, 2, 15, 393, 0, 0, 3994, 0, 0, 1, 0, 47, 0, 0, 0, 14, 1046, 4, 0, 8, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 571, 528, 4840, 0, 0, 6644, 8, 0, 0, 0, 384, 2, 0, 25, 0, 11, 391, 0, 2, 2682, 308, 0, 0, 8188, 852, 8172, 0, 3002, 12, 18, 589, 0, 2, 5, 1, 0, 9, 1, 3037, 3, 4, 5277, 8, 104, 0, 0, 774, 0, 0, 4, 0, 0, 387, 2, 59, 1499, 2, 0, 0, 12, 0, 2, 0, 2, 1054, 0, 0, 0, 777, 0, 916, 20, 1, 16, 322, 0, 0, 0, 0, 1, 191, 0, 0, 1598, 0, 180, 4, 12, 90, 0, 0, 0, 0, 2240, 0, 1, 3, 41, 0, 0, 44, 4637, 929, 0, 0, 66, 358, 0, 0, 2288, 0, 5, 0, 15, 88, 0, 1, 3, 22, 1179, 1, 1, 5, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 1654, 307, 3498, 0, 0, 2092, 0, 1, 1, 0, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 11, 0, 6, 313, 0, 1, 2946, 218, 1, 0, 0, 750, 764, 1, 3676, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 979, 0, 2, 1, 0, 1, 25, 0, 3096, 0, 0, 4503, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 745, 2, 9, 0, 0, 2, 320, 1, 72, 2006, 0, 1, 0, 8, 2664, 2612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 621, 0, 4, 0, 1102, 0, 810, 0, 1, 15, 285, 0, 4747, 415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 1647, 3, 73, 0, 47, 65, 1, 0, 0, 5577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2840, 2, 0, 2, 22, 1, 0, 15, 4024, 507, 0, 0, 76, 775, 3, 686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2414, 0, 2, 0, 3, 71, 0, 0, 0, 4, 956, 0, 0, 0, 2, 3779, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7810, 8688, 8674, 8443, 8000, 7503, 8953, 7465] avg=8192.0000 max=8953 max_rank=6 min=7465 min_rank=7 imbalance_ratio=1.092896 max_load_reduction=56583 max_load_reduction_pct=0.863388 moved_to_echo=31921 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 9, 999, 356, 4405, 2, 0, 2324, 0, 0, 0, 1, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 1, 230, 0, 1, 2176, 188, 0, 0, 0, 614, 766, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1163, 0, 0, 1, 0, 0, 2, 0, 3387, 14, 1, 4823, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 709, 1, 0, 1, 0, 2, 421, 0, 57, 1496, 0, 0, 0, 21, 2668, 2613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 653, 0, 3, 0, 937, 0, 672, 7, 12, 18, 366, 0, 4754, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 42, 0, 0, 2582, 2, 114, 0, 15, 60, 0, 0, 4, 5577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2099, 2, 0, 4, 40, 3, 2, 5, 2465, 431, 0, 0, 36, 668, 1, 735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3062, 0, 0, 0, 0, 43, 0, 0, 2, 6, 1438, 0, 0, 0, 0, 4196, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8068, 9471, 7989, 7885, 8397, 6491, 8997] avg=8192.0000 max=9471 max_rank=2 min=6491 min_rank=6 imbalance_ratio=1.156128 max_load_reduction=56065 max_load_reduction_pct=0.855484 moved_to_echo=31770 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 2, 1089, 270, 5260, 0, 0, 2011, 0, 0, 1, 1, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 1, 208, 0, 3, 2625, 245, 0, 0, 0, 1145, 765, 2, 3461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1006, 0, 0, 0, 0, 1, 36, 0, 2311, 0, 0, 4897, 0, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1067, 0, 0, 0, 0, 0, 166, 0, 54, 1682, 0, 1, 0, 7, 2666, 2611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 795, 2, 1, 1, 828, 0, 690, 1, 2, 14, 219, 0, 4750, 399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 140, 0, 0, 1910, 1, 129, 1, 22, 10, 0, 0, 1, 5577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2492, 0, 0, 0, 3, 1, 2, 12, 2441, 351, 0, 0, 19, 420, 0, 747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4300, 0, 0, 0, 6, 82, 2, 0, 3, 8, 1025, 0, 0, 0, 0, 3632, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9135, 8462, 8377, 8260, 7707, 7795, 6488, 9312] avg=8192.0000 max=9312 max_rank=7 min=6488 min_rank=6 imbalance_ratio=1.136719 max_load_reduction=56224 max_load_reduction_pct=0.857910 moved_to_echo=31302 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 754, 344, 4119, 0, 1, 1879, 5, 0, 0, 0, 654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6, 0, 9, 343, 0, 4, 2091, 297, 0, 0, 0, 673, 764, 3, 3626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 380, 0, 1, 0, 0, 2, 13, 0, 2637, 2, 0, 4423, 1, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 938, 0, 0, 0, 0, 0, 370, 4, 53, 1368, 0, 0, 0, 52, 2664, 2613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 937, 0, 0, 1, 888, 0, 1237, 8, 31, 25, 237, 0, 4747, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 352, 0, 0, 2757, 1, 130, 0, 48, 71, 1, 4, 0, 5579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1695, 2, 0, 1, 11, 0, 0, 25, 4497, 985, 0, 0, 48, 436, 0, 674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3450, 0, 0, 0, 0, 64, 0, 0, 4, 52, 1259, 0, 0, 2, 0, 3392, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7758, 7818, 7634, 8070, 8483, 8947, 8374, 8452] avg=8192.0000 max=8947 max_rank=5 min=7634 min_rank=2 imbalance_ratio=1.092163 max_load_reduction=56589 max_load_reduction_pct=0.863480 moved_to_echo=34256 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 1347, 189, 3798, 0, 0, 2368, 0, 0, 0, 0, 359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 24, 321, 0, 12, 2734, 275, 0, 0, 0, 886, 759, 0, 3092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 28, 1088, 1, 7, 14, 0, 4, 49, 0, 2612, 3, 0, 3444, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1438, 0, 0, 0, 0, 0, 384, 0, 83, 1282, 0, 0, 0, 18, 2670, 2614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1029, 0, 1, 1, 1025, 0, 864, 14, 7, 54, 180, 0, 4755, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 242, 0, 0, 2122, 3, 202, 0, 18, 85, 1, 0, 0, 5578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2582, 0, 0, 4, 84, 1, 0, 6, 4698, 683, 0, 0, 41, 509, 0, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1898, 0, 1, 0, 3, 157, 0, 0, 0, 15, 1155, 0, 1, 4, 0, 4292, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8067, 8124, 7347, 8489, 8406, 8255, 9139, 7709] avg=8192.0000 max=9139 max_rank=6 min=7347 min_rank=2 imbalance_ratio=1.115601 max_load_reduction=56397 max_load_reduction_pct=0.860550 moved_to_echo=33509 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 10, 788, 245, 4209, 0, 0, 2176, 0, 0, 0, 0, 446, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 324, 0, 0, 2051, 189, 0, 0, 0, 657, 766, 0, 3528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 934, 0, 4, 24, 0, 0, 18, 0, 2603, 25, 0, 4857, 1, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 0, 0, 4, 0, 1, 351, 0, 116, 1291, 0, 0, 0, 14, 2668, 2613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 1224, 0, 1, 0, 961, 0, 734, 19, 0, 5, 460, 0, 4754, 431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, 0, 0, 2748, 0, 220, 0, 4, 18, 2, 0, 0, 5578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3693, 0, 0, 5, 12, 0, 0, 18, 3457, 584, 0, 0, 124, 476, 0, 740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1980, 0, 0, 0, 4, 85, 0, 0, 0, 7, 712, 0, 0, 2, 0, 3928, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7875, 7532, 8525, 8116, 8594, 8815, 9109, 6970] avg=8192.0000 max=9109 max_rank=6 min=6970 min_rank=7 imbalance_ratio=1.111938 max_load_reduction=56427 max_load_reduction_pct=0.861008 moved_to_echo=33488 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 571, 528, 4840, 0, 0, 2212, 8, 0, 0, 0, 384, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 11, 391, 0, 2, 2682, 308, 0, 0, 0, 852, 764, 0, 3002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 18, 589, 0, 2, 5, 1, 0, 9, 1, 3037, 3, 4, 4382, 8, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 774, 0, 0, 4, 0, 0, 387, 2, 59, 1499, 2, 0, 0, 12, 2663, 2611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 1054, 0, 0, 0, 777, 0, 916, 20, 1, 16, 322, 0, 4745, 438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 191, 0, 0, 1598, 0, 180, 4, 12, 90, 0, 0, 0, 5577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2240, 0, 1, 3, 41, 0, 0, 44, 4637, 929, 0, 0, 66, 358, 0, 668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2288, 0, 5, 0, 15, 88, 0, 1, 3, 22, 1179, 1, 1, 5, 0, 3994, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8549, 8037, 8175, 8013, 8293, 7653, 8987, 7829] avg=8192.0000 max=8987 max_rank=6 min=7653 min_rank=5 imbalance_ratio=1.097046 max_load_reduction=56549 max_load_reduction_pct=0.862869 moved_to_echo=32762 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 850, 391, 4449, 0, 0, 1921, 0, 5, 3, 0, 480, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 30, 70, 0, 3, 2334, 420, 1, 0, 0, 908, 764, 2, 4270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 588, 0, 1, 6, 0, 3, 18, 1, 2442, 1, 0, 4195, 3, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 725, 0, 4, 1, 0, 0, 262, 0, 78, 1783, 0, 0, 0, 31, 2659, 2613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 825, 0, 0, 0, 947, 0, 1029, 13, 18, 26, 189, 0, 4739, 381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 182, 0, 0, 2204, 0, 141, 0, 40, 15, 0, 3, 0, 5579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2518, 3, 0, 4, 8, 0, 0, 26, 3586, 800, 0, 2, 15, 393, 0, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3994, 0, 0, 1, 0, 47, 0, 0, 0, 14, 1046, 4, 0, 8, 0, 3470, 218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8104, 8807, 7333, 8156, 8168, 8171, 7995, 8802] avg=8192.0000 max=8807 max_rank=1 min=7333 min_rank=2 imbalance_ratio=1.075073 max_load_reduction=56729 max_load_reduction_pct=0.865616 moved_to_echo=33136 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 128, 7296, 128, 128, 128, 0, 128, 2816, 128, 640, 12288, 128, 128, 0, 256, 21760, 20864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 128, 2432, 0, 128, 19200, 2176, 128, 0, 0, 6528, 6912, 128, 28544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 23168, 128, 128, 128, 128, 640, 128, 128, 128, 128, 8576, 128, 128, 128, 128, 31488, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 0, 128, 128, 7808, 2560, 37376, 128, 128, 14848, 128, 128, 128, 128, 3072, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 128, 7552, 128, 128, 128, 0, 128, 2688, 128, 640, 12416, 128, 128, 0, 256, 21376, 20992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 23424, 0, 128, 128, 128, 640, 128, 128, 128, 128, 8832, 128, 128, 128, 128, 30720, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 0, 128, 2304, 0, 128, 19712, 2176, 128, 0, 0, 6528, 6144, 128, 28800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 0, 128, 128, 8064, 2688, 34688, 128, 128, 17024, 128, 128, 128, 128, 3328, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 20736, 128, 128, 128, 256, 128, 128, 256, 29952, 5376, 0, 128, 512, 4096, 128, 4864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 0, 128, 7296, 128, 128, 128, 7296, 0, 7040, 128, 128, 256, 2304, 0, 36864, 4864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 6784, 128, 128, 128, 128, 128, 256, 128, 22144, 128, 128, 35712, 128, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 1536, 0, 0, 17408, 128, 1408, 128, 256, 512, 128, 128, 128, 44800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 20224, 128, 128, 128, 256, 128, 128, 256, 29824, 5376, 0, 128, 512, 4096, 128, 5504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 6784, 128, 128, 128, 128, 128, 256, 128, 22144, 128, 128, 35584, 128, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 128, 7168, 128, 128, 128, 7552, 0, 7040, 128, 128, 256, 2304, 0, 38016, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 1536, 0, 0, 17664, 128, 1280, 128, 256, 512, 128, 128, 128, 44672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[8189, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 8153, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 8182, 2, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 11, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[8185, 0, 7958, 5, 0, 0, 0, 0, 4, 0, 0, 1, 0, 0, 1, 9, 0, 2, 0, 3, 8186, 8160, 0, 8185, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 112, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3, 0, 0, 1, 0, 3, 0, 1, 8180, 1, 0, 0, 0, 0, 4, 0, 4, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8188, 0, 132, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 7989, 0, 0, 0, 0, 0, 5, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8186, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 205, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 4, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 7882, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8189, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 232, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 8192, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 8100, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8191, 8110, 0, 8185, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 8184, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[8189, 0, 7793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 7676, 0, 8188, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 257, 0, 0, 8188, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 8188, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7, 1, 0, 8191, 0, 647, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 2, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8170, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 8190, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 11, 0, 0, 8191, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 7266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8185, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 896, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8192, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[273, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 7959, 2, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8191, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8103, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 7831, 223, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8441, 8240, 8019, 8198, 8191, 8189, 8188, 8070] avg=8192.0000 max=8441 max_rank=0 min=8019 min_rank=2 imbalance_ratio=1.030396 max_load_reduction=57095 max_load_reduction_pct=0.871201 moved_to_echo=32638 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[273, 0, 7793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7676, 0, 87, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 257, 0, 0, 7965, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 8188, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 8101, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7, 1, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 647, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 2, 0, 1, 0, 7831, 223, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8066, 7766, 8224, 8192, 8191, 8191, 8196, 8710] avg=8192.0000 max=8710 max_rank=7 min=7766 min_rank=1 imbalance_ratio=1.063232 max_load_reduction=56826 max_load_reduction_pct=0.867096 moved_to_echo=33288 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[273, 0, 7958, 5, 0, 0, 0, 0, 4, 0, 0, 1, 0, 0, 1, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 0, 8160, 0, 87, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 0, 7962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3, 0, 0, 1, 0, 3, 0, 1, 8180, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 8098, 85, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7827, 223, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8251, 8255, 8074, 8196, 8194, 8187, 8189, 8190] avg=8192.0000 max=8255 max_rank=1 min=8074 min_rank=2 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32760 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[272, 0, 7882, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 232, 0, 0, 7968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8105, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7834, 223, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8276, 8200, 8203, 8192, 8192, 8205, 8111] avg=8192.0000 max=8276 max_rank=1 min=8111 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32700 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[272, 0, 7989, 0, 0, 0, 0, 0, 5, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 205, 0, 0, 7968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8105, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7834, 223, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8268, 8273, 8173, 8187, 8192, 8191, 8188, 8064] avg=8192.0000 max=8273 max_rank=1 min=8064 min_rank=7 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32635 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[272, 0, 8100, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8110, 0, 87, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 7961, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8174, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 8098, 86, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7834, 223, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8373, 8204, 8007, 8178, 8192, 8189, 8186, 8207] avg=8192.0000 max=8373 max_rank=0 min=8007 min_rank=2 imbalance_ratio=1.022095 max_load_reduction=57163 max_load_reduction_pct=0.872238 moved_to_echo=32774 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[272, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 7967, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 8102, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 11, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7834, 223, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8451, 8257, 7981, 8186, 8192, 8196, 8199, 8074] avg=8192.0000 max=8451 max_rank=0 min=7981 min_rank=2 imbalance_ratio=1.031616 max_load_reduction=57085 max_load_reduction_pct=0.871048 moved_to_echo=32661 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[263, 0, 7266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 896, 0, 0, 7962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8112, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7840, 228, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7529, 8265, 8858, 8196, 8192, 8201, 8185, 8110] avg=8192.0000 max=8858 max_rank=2 min=7529 min_rank=0 imbalance_ratio=1.081299 max_load_reduction=56678 max_load_reduction_pct=0.864838 moved_to_echo=32688 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[8185, 0, 7987, 5, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 9, 0, 2, 0, 3, 8186, 8160, 0, 8185, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 106, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3, 0, 0, 1, 0, 3, 0, 1, 8180, 1, 0, 0, 0, 0, 4, 0, 4, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8188, 0, 109, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 7898, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8190, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 233, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 8192, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 7313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8186, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 849, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[8189, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 8157, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 8182, 2, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 12, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 8183, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8171, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 8190, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6, 0, 0, 8191, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 7978, 0, 0, 0, 0, 0, 5, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8186, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[8189, 0, 7855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 7687, 0, 8188, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 236, 0, 0, 8189, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 8190, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6, 1, 0, 8191, 0, 594, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 2, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 8116, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8192, 8112, 0, 8185, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 8184, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[252, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 7968, 2, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8106, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 7854, 214, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8419, 8241, 8024, 8197, 8191, 8190, 8190, 8084] avg=8192.0000 max=8419 max_rank=0 min=8024 min_rank=2 imbalance_ratio=1.027710 max_load_reduction=57117 max_load_reduction_pct=0.871536 moved_to_echo=32655 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[252, 0, 7987, 5, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 0, 8160, 0, 84, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 0, 0, 7971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3, 0, 0, 1, 0, 3, 0, 1, 8180, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 8101, 83, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7850, 214, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8259, 8252, 8077, 8195, 8194, 8188, 8191, 8180] avg=8192.0000 max=8259 max_rank=0 min=8077 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=57277 max_load_reduction_pct=0.873978 moved_to_echo=32753 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[252, 0, 7898, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 233, 0, 0, 7977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8108, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7857, 214, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8153, 8274, 8210, 8203, 8192, 8192, 8199, 8113] avg=8192.0000 max=8274 max_rank=1 min=8113 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32696 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[252, 0, 7855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7687, 0, 84, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 236, 0, 0, 7975, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 8190, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 8104, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6, 1, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 594, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 2, 0, 1, 0, 7854, 214, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8107, 7774, 8213, 8193, 8190, 8192, 8197, 8670] avg=8192.0000 max=8670 max_rank=7 min=7774 min_rank=1 imbalance_ratio=1.058350 max_load_reduction=56866 max_load_reduction_pct=0.867706 moved_to_echo=33249 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[252, 0, 8116, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8112, 0, 84, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 7970, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8174, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8101, 83, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7857, 214, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8369, 8202, 8009, 8177, 8193, 8188, 8188, 8210] avg=8192.0000 max=8369 max_rank=0 min=8009 min_rank=2 imbalance_ratio=1.021606 max_load_reduction=57167 max_load_reduction_pct=0.872299 moved_to_echo=32779 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[252, 0, 8183, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 7976, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 8105, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 7857, 214, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8436, 8255, 7987, 8188, 8192, 8196, 8196, 8086] avg=8192.0000 max=8436 max_rank=0 min=7987 min_rank=2 imbalance_ratio=1.029785 max_load_reduction=57100 max_load_reduction_pct=0.871277 moved_to_echo=32670 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[252, 0, 7978, 0, 0, 0, 0, 0, 5, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 0, 0, 7977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8108, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7857, 214, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8237, 8270, 8193, 8187, 8192, 8191, 8190, 8076] avg=8192.0000 max=8270 max_rank=1 min=8076 min_rank=7 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32649 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[243, 0, 7313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 849, 0, 0, 7974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8110, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7860, 217, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7556, 8268, 8823, 8196, 8192, 8199, 8185, 8117] avg=8192.0000 max=8823 max_rank=2 min=7556 min_rank=0 imbalance_ratio=1.077026 max_load_reduction=56713 max_load_reduction_pct=0.865372 moved_to_echo=32693 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[2176, 0, 63360, 128, 0, 0, 0, 0, 128, 0, 128, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 1152, 128, 0, 128, 0, 0, 128, 128, 0, 0, 128, 128, 0, 128, 0, 62720, 1792, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[2048, 0, 63616, 128, 0, 0, 128, 0, 128, 0, 128, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 1024, 128, 0, 128, 0, 0, 128, 128, 0, 0, 128, 128, 0, 128, 0, 62848, 1792, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 1792, 0, 0, 63872, 128, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 0, 64896, 0, 768, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 0, 128, 128, 128, 0, 0, 128, 0, 0, 128, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 128, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 128, 128, 128, 0, 128, 0, 64896, 768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 1920, 0, 0, 63744, 128, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 0, 128, 0, 64896, 0, 768, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 128, 0, 0, 128, 128, 128, 0, 128, 65536, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 128, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 0, 128, 128, 128, 0, 0, 128, 0, 0, 128, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 1, 0, 0, 35, 0, 8070, 0, 0, 0, 0, 0, 0, 3, 0, 8186, 0, 0, 14, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6717, 0, 0, 0, 0, 0, 0, 0, 0, 1539, 0, 8166, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 3, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 1, 0, 0, 0, 0, 0, 0, 0, 6816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 38, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 7470, 0, 2, 1, 184, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8188, 0, 20, 2, 0, 0, 0, 0, 0, 2, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4996, 0, 0, 0, 3, 1, 0, 2, 0, 5086, 0, 8167, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 13, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 8178, 0, 6, 0, 0, 0, 0, 0, 0, 0, 8106, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 7, 29, 0, 8183, 0, 0, 0, 0, 0, 6, 0, 0, 8110, 0, 0, 1, 36, 0, 0, 7, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 6, 0, 0, 0, 0, 0, 7, 0, 0, 0, 8183, 0, 53, 0, 0, 4, 8, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 6994, 0, 0, 3, 8, 0, 0, 4, 0, 1206, 0, 8157, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 657, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 16, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 14, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6471, 0, 0, 0, 0, 0, 0, 0, 0, 1086, 0, 8170, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7745, 0, 0, 0, 0, 0, 1073, 0, 7876, 0, 0, 0, 0, 0, 0, 24, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 6423, 0, 1, 0, 0, 0, 0, 0, 0, 1431, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7638, 0, 0, 0, 0, 0, 516, 0, 8144, 0, 0, 0, 0, 0, 0, 36, 0, 8182, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6175, 0, 0, 0, 0, 0, 0, 0, 0, 2062, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6933, 0, 0, 0, 0, 0, 1863, 0, 7985, 0, 0, 0, 0, 0, 0, 32, 0, 8189, 0, 0, 8, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7121, 0, 0, 0, 0, 0, 0, 0, 0, 637, 0, 8185, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7416, 0, 0, 0, 0, 0, 1344, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 8125, 0, 0, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7684, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 0, 1, 0, 0, 0, 0, 0, 0, 0, 6816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 330, 15, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6914, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5, 0, 7903, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8173, 0, 20, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4985, 0, 0, 0, 3, 1, 0, 2, 0, 5086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8286, 7006, 8513, 6955, 8319, 8196, 8184, 10077] avg=8192.0000 max=10077 max_rank=7 min=6955 min_rank=3 imbalance_ratio=1.230103 max_load_reduction=55459 max_load_reduction_pct=0.846237 moved_to_echo=34776 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8057, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 1, 0, 362, 15, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 7576, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7906, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6702, 0, 0, 0, 0, 0, 0, 0, 0, 1539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8155, 8371, 8558, 7628, 8201, 8203, 8179, 8241] avg=8192.0000 max=8558 max_rank=2 min=7628 min_rank=3 imbalance_ratio=1.044678 max_load_reduction=56978 max_load_reduction_pct=0.869415 moved_to_echo=32824 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 13, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 3, 2, 0, 8170, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 0, 6, 0, 0, 0, 0, 0, 0, 0, 8106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 358, 15, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 29, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 7507, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 7, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 6, 0, 7893, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 8168, 0, 53, 0, 0, 4, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8157, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6979, 0, 0, 3, 8, 0, 0, 4, 0, 1206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8287, 8301, 8591, 7550, 8189, 8240, 8178, 8200] avg=8192.0000 max=8591 max_rank=2 min=7550 min_rank=3 imbalance_ratio=1.048706 max_load_reduction=56945 max_load_reduction_pct=0.868912 moved_to_echo=32807 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7863, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7745, 0, 0, 0, 0, 362, 15, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1073, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 7582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7906, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8192, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6409, 0, 1, 0, 0, 0, 0, 0, 0, 1431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7962, 8381, 8137, 8679, 8154, 8176, 8206, 7841] avg=8192.0000 max=8679 max_rank=3 min=7841 min_rank=7 imbalance_ratio=1.059448 max_load_reduction=56857 max_load_reduction_pct=0.867569 moved_to_echo=32377 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7638, 0, 0, 0, 0, 362, 15, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 7573, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7906, 247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6162, 0, 0, 0, 0, 0, 0, 0, 0, 2062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8229, 8381, 8028, 8136, 8153, 8181, 8204, 8224] avg=8192.0000 max=8381 max_rank=1 min=8028 min_rank=2 imbalance_ratio=1.023071 max_load_reduction=57155 max_load_reduction_pct=0.872116 moved_to_echo=32762 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 361, 15, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7570, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7906, 247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6457, 0, 0, 0, 0, 0, 0, 0, 0, 1086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8270, 8345, 8566, 8243, 8195, 8190, 8184, 7543] avg=8192.0000 max=8566 max_rank=2 min=7543 min_rank=7 imbalance_ratio=1.045654 max_load_reduction=56970 max_load_reduction_pct=0.869293 moved_to_echo=32112 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7416, 0, 0, 0, 0, 359, 15, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7520, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7906, 246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7667, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8275, 8380, 7807, 8871, 8153, 8175, 8204, 7671] avg=8192.0000 max=8871 max_rank=3 min=7671 min_rank=7 imbalance_ratio=1.082886 max_load_reduction=56665 max_load_reduction_pct=0.864639 moved_to_echo=32203 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7973, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6933, 0, 0, 0, 0, 366, 18, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1863, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 7571, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7910, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7102, 0, 0, 0, 0, 0, 0, 0, 0, 637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8072, 8371, 7336, 9474, 8172, 8175, 8197, 7739] avg=8192.0000 max=9474 max_rank=3 min=7336 min_rank=2 imbalance_ratio=1.156494 max_load_reduction=56062 max_load_reduction_pct=0.855438 moved_to_echo=32283 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 1, 0, 0, 24, 0, 8057, 0, 0, 0, 0, 0, 0, 2, 0, 8184, 0, 0, 14, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7138, 0, 0, 0, 0, 0, 0, 0, 0, 1143, 0, 8164, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 324, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 16, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 14, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6743, 0, 0, 0, 0, 0, 0, 0, 0, 1155, 0, 8170, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 12, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4, 2, 0, 0, 0, 0, 0, 8178, 0, 6, 0, 0, 0, 0, 0, 0, 0, 8107, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 7, 10, 0, 8183, 0, 0, 0, 0, 0, 5, 0, 0, 8115, 0, 0, 1, 38, 0, 0, 7, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 6, 0, 0, 0, 0, 0, 7, 0, 0, 0, 8183, 0, 44, 0, 0, 4, 8, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 7161, 0, 0, 3, 8, 0, 0, 3, 0, 1065, 0, 8157, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7025, 0, 0, 0, 0, 0, 1592, 0, 7968, 0, 0, 0, 0, 0, 0, 31, 0, 8190, 0, 0, 9, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7509, 0, 0, 0, 0, 0, 0, 0, 0, 446, 0, 8182, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7514, 0, 0, 0, 0, 0, 403, 0, 8146, 0, 0, 0, 0, 0, 0, 37, 0, 8190, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6849, 0, 0, 0, 0, 0, 0, 0, 0, 1615, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7550, 0, 0, 0, 0, 0, 950, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7901, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 3, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7079, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 19, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 7457, 0, 2, 1, 181, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8188, 0, 18, 2, 0, 0, 0, 0, 0, 2, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5216, 0, 0, 0, 3, 1, 0, 2, 0, 4642, 0, 8164, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7717, 0, 0, 0, 0, 0, 948, 0, 7887, 0, 0, 0, 0, 0, 0, 20, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7068, 0, 0, 0, 0, 0, 0, 0, 0, 932, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8043, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 1, 0, 234, 137, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 7733, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7936, 217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7001, 0, 0, 0, 0, 0, 0, 0, 0, 1143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8144, 8338, 8553, 7773, 8200, 8206, 8178, 8144] avg=8192.0000 max=8553 max_rank=2 min=7773 min_rank=3 imbalance_ratio=1.044067 max_load_reduction=56983 max_load_reduction_pct=0.869492 moved_to_echo=32728 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 233, 130, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7728, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7936, 217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6613, 0, 0, 0, 0, 0, 0, 0, 0, 1155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8268, 8314, 8548, 8068, 8193, 8191, 8186, 7768] avg=8192.0000 max=8548 max_rank=2 min=7768 min_rank=7 imbalance_ratio=1.043457 max_load_reduction=56988 max_load_reduction_pct=0.869568 moved_to_echo=32338 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7514, 0, 0, 0, 0, 234, 132, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 403, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 7739, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7936, 217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6717, 0, 0, 0, 0, 0, 0, 0, 0, 1615, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8233, 8348, 7893, 8191, 8153, 8181, 8205, 8332] avg=8192.0000 max=8348 max_rank=1 min=7893 min_rank=2 imbalance_ratio=1.019043 max_load_reduction=57188 max_load_reduction_pct=0.872620 moved_to_echo=32871 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 12, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4, 2, 0, 8169, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 6, 0, 0, 0, 0, 0, 0, 0, 8107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 232, 138, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 10, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 7668, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 7, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 6, 0, 7922, 215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 8170, 0, 44, 0, 0, 4, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8157, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7023, 0, 0, 3, 8, 0, 0, 3, 0, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8290, 8269, 8582, 7691, 8190, 8233, 8179, 8102] avg=8192.0000 max=8582 max_rank=2 min=7691 min_rank=3 imbalance_ratio=1.047607 max_load_reduction=56954 max_load_reduction_pct=0.869049 moved_to_echo=32704 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7955, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7025, 0, 0, 0, 0, 237, 147, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1592, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 7731, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7941, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7362, 0, 0, 0, 0, 0, 0, 0, 0, 446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8058, 8336, 7427, 9363, 8175, 8174, 8195, 7808] avg=8192.0000 max=9363 max_rank=3 min=7427 min_rank=2 imbalance_ratio=1.142944 max_load_reduction=56173 max_load_reduction_pct=0.857132 moved_to_echo=32352 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 213, 100, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7046, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5, 0, 7933, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8175, 0, 18, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5116, 0, 0, 0, 3, 1, 0, 2, 0, 4642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8288, 7236, 8484, 7068, 8318, 8196, 8182, 9764] avg=8192.0000 max=9764 max_rank=7 min=7068 min_rank=3 imbalance_ratio=1.191895 max_load_reduction=55772 max_load_reduction_pct=0.851013 moved_to_echo=34460 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7550, 0, 0, 0, 0, 233, 152, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7716, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7936, 217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7749, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8278, 8347, 7948, 8673, 8154, 8177, 8205, 7754] avg=8192.0000 max=8673 max_rank=3 min=7754 min_rank=7 imbalance_ratio=1.058716 max_load_reduction=56863 max_load_reduction_pct=0.867661 moved_to_echo=32290 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7874, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7717, 0, 0, 0, 0, 234, 136, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 948, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 7741, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7936, 217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8192, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6932, 0, 0, 0, 0, 0, 0, 0, 0, 932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7977, 8348, 8101, 8709, 8153, 8178, 8206, 7864] avg=8192.0000 max=8709 max_rank=3 min=7864 min_rank=7 imbalance_ratio=1.063110 max_load_reduction=56827 max_load_reduction_pct=0.867111 moved_to_echo=32401 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 5632, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 59904, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 52480, 0, 128, 128, 128, 128, 0, 128, 0, 13056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[384, 0, 0, 128, 0, 0, 0, 0, 128, 0, 128, 0, 128, 0, 128, 0, 63360, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 62464, 0, 0, 128, 0, 2944, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 4352, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 61184, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 62592, 0, 0, 128, 0, 1920, 1152, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 54528, 0, 0, 128, 128, 128, 0, 128, 0, 11008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 1280, 0, 128, 0, 0, 0, 0, 0, 0, 0, 64384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[384, 0, 0, 128, 0, 0, 0, 0, 128, 0, 128, 0, 128, 0, 128, 0, 63488, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 128, 128, 0, 64768, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 1536, 0, 128, 0, 0, 0, 0, 0, 0, 0, 64128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 65408, 0, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 0, 128, 128, 0, 0, 0, 0, 128, 0, 0, 128, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 0, 0, 0, 0, 0, 0, 0, 257, 0, 0, 0, 0, 0, 0, 0, 5468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 4, 0, 1550, 0, 0, 0, 0, 8192, 124, 0, 0, 0, 0, 8189, 1, 657, 8186, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 0, 0, 1, 0, 0, 0, 0, 2, 17, 0, 0, 8185, 0, 7998, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 5, 5, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 17, 3880, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4, 8178, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 5, 0, 4541, 0, 0, 0, 0, 8162, 24, 0, 0, 0, 2, 8076, 0, 0, 8189, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 8086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 6941, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 4, 0, 1165, 0, 0, 0, 0, 8168, 184, 0, 0, 0, 0, 8040, 0, 76, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 8116, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 5792, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 43, 0, 2360, 0, 0, 0, 0, 8191, 90, 0, 0, 0, 0, 8109, 0, 3, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 162, 0, 0, 0, 0, 0, 0, 0, 4566, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 2, 0, 1551, 0, 0, 0, 0, 8192, 339, 0, 0, 0, 0, 8119, 0, 1629, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 5714, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 511, 0, 1906, 0, 0, 0, 0, 8191, 140, 0, 0, 0, 0, 8105, 0, 1, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 29, 0, 0, 0, 0, 0, 13, 0, 10, 8180, 0, 8178, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 4, 0, 2, 1, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6684, 0, 0, 0, 0, 6, 3, 6, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 12, 8061, 8, 1, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 362, 0, 1101, 0, 0, 0, 0, 8169, 5, 0, 0, 7, 0, 8153, 0, 54, 8180, 10] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 3121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 229, 0, 881, 0, 0, 0, 0, 8183, 189, 0, 0, 0, 0, 8157, 0, 3794, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 0, 0, 1, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 17, 0, 0, 47, 0, 7998, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 0, 0, 0, 0, 25, 0, 0, 0, 8148, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 8138, 14, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 17, 3880, 0, 0, 0, 2702, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4, 8178, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8046, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4541, 0, 0, 0, 0, 0, 24, 0, 0, 0, 2, 5367, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8066, 8188, 8164, 6741, 8196, 8052, 9938] avg=8192.0000 max=9938 max_rank=7 min=6741 min_rank=4 imbalance_ratio=1.213135 max_load_reduction=55598 max_load_reduction_pct=0.848358 moved_to_echo=32927 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8177, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 0, 0, 0, 8131, 15, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 257, 0, 0, 0, 0, 0, 0, 0, 5468, 0, 0, 0, 2740, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8049, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1550, 0, 0, 0, 0, 0, 124, 0, 0, 0, 0, 5441, 1, 657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8239, 8184, 8354, 8604, 8148, 8053, 7773] avg=8192.0000 max=8604 max_rank=4 min=7773 min_rank=7 imbalance_ratio=1.050293 max_load_reduction=56932 max_load_reduction_pct=0.868713 moved_to_echo=32578 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 8145, 15, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 5714, 0, 0, 0, 2712, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8049, 0, 0, 0, 0, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1906, 0, 0, 0, 0, 0, 140, 0, 0, 0, 0, 5386, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8213, 8182, 8201, 8582, 8178, 8560, 7433] avg=8192.0000 max=8582 max_rank=4 min=7433 min_rank=7 imbalance_ratio=1.047607 max_load_reduction=56954 max_load_reduction_pct=0.869049 moved_to_echo=32753 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 29, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 10, 47, 0, 8178, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2, 1, 0, 0, 0, 0, 82, 0, 0, 0, 8154, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2, 0, 0, 8133, 15, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6684, 0, 0, 0, 2728, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 6, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 12, 8061, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 0, 0, 0, 0, 0, 0, 0, 8035, 0, 0, 0, 0, 362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1101, 0, 0, 0, 0, 0, 5, 0, 0, 7, 0, 5417, 0, 54, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8214, 8257, 8249, 8164, 9551, 8101, 8406, 6594] avg=8192.0000 max=9551 max_rank=4 min=6594 min_rank=7 imbalance_ratio=1.165894 max_load_reduction=55985 max_load_reduction_pct=0.854263 moved_to_echo=32652 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 8146, 14, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 162, 0, 0, 0, 0, 0, 0, 0, 4566, 0, 0, 0, 2722, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8040, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1551, 0, 0, 0, 0, 0, 339, 0, 0, 0, 0, 5385, 0, 1629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8235, 8184, 8189, 7595, 8201, 8042, 8904] avg=8192.0000 max=8904 max_rank=7 min=7595 min_rank=4 imbalance_ratio=1.086914 max_load_reduction=56632 max_load_reduction_pct=0.864136 moved_to_echo=32742 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 8169, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 8130, 14, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 3121, 0, 0, 0, 2729, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8191, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8049, 0, 0, 0, 0, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 881, 0, 0, 0, 0, 0, 189, 0, 0, 0, 0, 5420, 0, 3794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8230, 8186, 8159, 6015, 8196, 8278, 10284] avg=8192.0000 max=10284 max_rank=7 min=6015 min_rank=4 imbalance_ratio=1.255371 max_load_reduction=55252 max_load_reduction_pct=0.843079 moved_to_echo=32773 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 8086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 8153, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 15, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 6941, 0, 0, 0, 2690, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 8180, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8049, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1165, 0, 0, 0, 0, 0, 184, 0, 0, 0, 0, 5343, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8133, 8178, 8154, 9796, 8260, 8053, 6768] avg=8192.0000 max=9796 max_rank=4 min=6768 min_rank=7 imbalance_ratio=1.195801 max_load_reduction=55740 max_load_reduction_pct=0.850525 moved_to_echo=32877 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 8116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8176, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8128, 15, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 5792, 0, 0, 0, 2713, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 8181, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8049, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2360, 0, 0, 0, 0, 0, 90, 0, 0, 0, 0, 5389, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8163, 8185, 8151, 8652, 8256, 8092, 7842] avg=8192.0000 max=8652 max_rank=4 min=7842 min_rank=7 imbalance_ratio=1.056152 max_load_reduction=56884 max_load_reduction_pct=0.867981 moved_to_echo=32842 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 29, 0, 0, 0, 0, 0, 13, 0, 10, 8180, 0, 8178, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 4, 0, 2, 1, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6181, 0, 0, 0, 0, 6, 2, 3, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 12, 8058, 8, 1, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 364, 0, 1564, 0, 1, 0, 0, 8170, 3, 0, 0, 7, 0, 8152, 0, 99, 8180, 10] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 8087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 6691, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 3, 0, 1403, 0, 0, 0, 0, 8169, 192, 0, 0, 0, 0, 8038, 0, 83, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 188, 0, 0, 0, 0, 0, 0, 0, 3968, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 2336, 0, 0, 0, 0, 8192, 345, 0, 0, 0, 0, 8120, 0, 1417, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 2483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 186, 0, 1369, 0, 0, 0, 0, 8184, 183, 0, 0, 0, 0, 8155, 0, 3989, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 5688, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 293, 0, 2147, 0, 0, 0, 0, 8192, 138, 0, 0, 0, 0, 8104, 0, 5, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 0, 0, 1, 0, 0, 0, 0, 2, 17, 0, 0, 8185, 0, 7986, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 5, 5, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 17, 3303, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4, 8178, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 1, 0, 5155, 0, 0, 0, 0, 8161, 25, 0, 0, 0, 1, 8051, 0, 0, 8189, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 0, 0, 0, 0, 0, 0, 0, 324, 0, 0, 0, 0, 0, 0, 0, 4602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 3, 0, 2311, 0, 0, 0, 0, 8192, 122, 0, 0, 0, 0, 8188, 1, 706, 8185, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 8117, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 5080, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 22, 0, 3098, 0, 0, 0, 0, 8191, 90, 0, 0, 0, 0, 8112, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 8087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 8153, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 16, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 6691, 0, 0, 0, 3233, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 8180, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8084, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1403, 0, 0, 0, 0, 0, 192, 0, 0, 0, 0, 4796, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8135, 8180, 8157, 10051, 8257, 8087, 6474] avg=8192.0000 max=10051 max_rank=4 min=6474 min_rank=7 imbalance_ratio=1.226929 max_load_reduction=55485 max_load_reduction_pct=0.846634 moved_to_echo=32869 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 29, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 10, 48, 0, 8178, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2, 1, 0, 0, 0, 0, 83, 0, 0, 0, 8154, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2, 0, 0, 8132, 16, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6181, 0, 0, 0, 3279, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2, 3, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 12, 8058, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 0, 0, 0, 0, 0, 0, 0, 8071, 0, 0, 0, 0, 364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1564, 0, 1, 0, 0, 0, 3, 0, 0, 7, 0, 4864, 0, 99, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8214, 8258, 8250, 8165, 9561, 8096, 8444, 6548] avg=8192.0000 max=9561 max_rank=4 min=6548 min_rank=7 imbalance_ratio=1.167114 max_load_reduction=55975 max_load_reduction_pct=0.854111 moved_to_echo=32649 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 8144, 16, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 5688, 0, 0, 0, 3259, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8084, 0, 0, 0, 0, 293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2147, 0, 0, 0, 0, 0, 138, 0, 0, 0, 0, 4836, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8218, 8182, 8202, 9068, 8177, 8377, 7126] avg=8192.0000 max=9068 max_rank=4 min=7126 min_rank=7 imbalance_ratio=1.106934 max_load_reduction=56468 max_load_reduction_pct=0.861633 moved_to_echo=32748 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 0, 0, 1, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 17, 0, 0, 48, 0, 7986, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 0, 0, 0, 0, 28, 0, 0, 0, 8146, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 8137, 15, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 17, 3303, 0, 0, 0, 3238, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4, 8178, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8082, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5155, 0, 0, 0, 0, 0, 25, 0, 0, 0, 1, 4804, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8055, 8189, 8166, 6662, 8199, 8085, 9989] avg=8192.0000 max=9989 max_rank=7 min=6662 min_rank=4 imbalance_ratio=1.219360 max_load_reduction=55547 max_load_reduction_pct=0.847580 moved_to_echo=32935 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8177, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 8146, 15, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 0, 0, 0, 0, 0, 0, 3968, 0, 0, 0, 3270, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2336, 0, 0, 0, 0, 0, 345, 0, 0, 0, 0, 4840, 0, 1417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8234, 8183, 8183, 7529, 8202, 8080, 8938] avg=8192.0000 max=8938 max_rank=7 min=7529 min_rank=4 imbalance_ratio=1.091064 max_load_reduction=56598 max_load_reduction_pct=0.863617 moved_to_echo=32749 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 8169, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 8131, 15, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 2483, 0, 0, 0, 3280, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8191, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8084, 0, 0, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1369, 0, 0, 0, 0, 0, 183, 0, 0, 0, 0, 4866, 0, 3989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8231, 8185, 8161, 5895, 8199, 8270, 10407] avg=8192.0000 max=10407 max_rank=7 min=5895 min_rank=4 imbalance_ratio=1.270386 max_load_reduction=55129 max_load_reduction_pct=0.841202 moved_to_echo=32771 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 8117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8175, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8127, 16, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 5080, 0, 0, 0, 3263, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8182, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8084, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3098, 0, 0, 0, 0, 0, 90, 0, 0, 0, 0, 4840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8165, 8183, 8153, 8450, 8255, 8106, 8028] avg=8192.0000 max=8450 max_rank=4 min=8028 min_rank=7 imbalance_ratio=1.031494 max_load_reduction=57086 max_load_reduction_pct=0.871063 moved_to_echo=32839 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8176, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 0, 0, 0, 8132, 16, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 324, 0, 0, 0, 0, 0, 0, 0, 4602, 0, 0, 0, 3293, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8084, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2311, 0, 0, 0, 0, 0, 122, 0, 0, 0, 0, 4886, 1, 706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8240, 8184, 8349, 8320, 8151, 8087, 8026] avg=8192.0000 max=8349 max_rank=3 min=8026 min_rank=7 imbalance_ratio=1.019165 max_load_reduction=57187 max_load_reduction_pct=0.872604 moved_to_echo=32584 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 512, 0, 0, 0, 128, 0, 0, 128, 42240, 0, 0, 0, 21760, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[15104, 0, 0, 0, 0, 0, 1152, 0, 0, 128, 128, 43264, 128, 6272, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 128, 0, 0, 0, 0, 0, 0, 0, 64384, 0, 0, 0, 0, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 128, 0, 128, 384, 0, 65152, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 384, 128, 0, 0, 65152, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 256, 0, 0, 0, 0, 0, 0, 128, 0, 128, 0, 128, 65408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 640, 0, 0, 0, 128, 0, 0, 128, 38016, 0, 0, 0, 26240, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[19456, 0, 128, 0, 0, 0, 1152, 0, 0, 128, 128, 38784, 128, 6400, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 128, 0, 0, 0, 0, 0, 0, 0, 64768, 0, 0, 0, 0, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 0, 128, 0, 128, 128, 0, 0, 0, 0, 256, 0, 0, 0, 65408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 256, 0, 0, 0, 0, 0, 0, 128, 0, 128, 0, 128, 65408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 256, 128, 0, 0, 65152, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 128, 0, 128, 384, 0, 65152, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8114, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 8116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8093, 0, 8192, 0, 0, 0, 8186, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 12, 0, 0, 0, 0, 0, 0, 1950, 0, 176, 1113, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 11, 8, 0, 0, 0, 5, 0, 0, 0, 0, 0, 16, 0, 0, 2, 0, 0, 6626, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 8151, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 6, 0, 6, 0, 0, 8166, 12, 0, 0, 0, 0, 0, 0, 0, 13, 6484, 2, 8167, 0, 0, 0, 8180, 2, 9, 0, 0, 9] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 946, 0, 0, 175, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 7821, 0, 0, 0, 22, 0, 0, 2, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 8192, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 8072, 0, 0, 0, 4, 0, 0, 0, 0, 0, 7403, 0, 8192, 0, 0, 1, 8163, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3872, 0, 0, 719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 7721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4260, 0, 8192, 0, 0, 0, 8188, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 866, 0, 2, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 7913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7541, 0, 8192, 0, 0, 0, 8185, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 236, 1967, 0, 0, 0, 0, 0, 3, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4, 7, 0, 15, 2, 0, 0, 5910, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 2, 0, 8186, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 4, 1, 0, 0, 0, 8184, 3, 0, 0, 0, 0, 0, 0, 0, 5, 8184, 4, 8182, 0, 2, 2, 8186, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 896, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7283, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 0, 8192, 0, 0, 0, 8177, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4882, 0, 0, 1828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 7711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1982, 0, 8192, 0, 0, 0, 8160, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 0, 5852, 627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7382, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8114, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 656, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 0, 0, 0, 0, 0, 8116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8093, 0, 0, 0, 0, 0, 1678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6653, 7428, 8186, 8815, 8194, 8192, 8297, 9771] avg=8192.0000 max=9771 max_rank=7 min=6653 min_rank=0 imbalance_ratio=1.192749 max_load_reduction=55765 max_load_reduction_pct=0.850906 moved_to_echo=34454 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 12, 0, 0, 0, 0, 0, 0, 1950, 0, 5848, 626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 1113, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7368, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 11, 8, 0, 0, 0, 5, 0, 0, 0, 0, 0, 16, 0, 8158, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 6626, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 655, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8150, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, 6, 0, 6, 0, 0, 8166, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 6484, 2, 0, 0, 0, 0, 1677, 2, 9, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8437, 8668, 8222, 7286, 8176, 8181, 8370, 8196] avg=8192.0000 max=8668 max_rank=1 min=7286 min_rank=3 imbalance_ratio=1.058105 max_load_reduction=56868 max_load_reduction_pct=0.867737 moved_to_echo=32923 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 866, 0, 5851, 627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7384, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 656, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 0, 0, 0, 0, 0, 7913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7541, 0, 0, 0, 0, 0, 1678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7345, 7474, 8187, 8840, 8191, 8186, 8094, 9219] avg=8192.0000 max=9219 max_rank=7 min=7345 min_rank=0 imbalance_ratio=1.125366 max_load_reduction=56317 max_load_reduction_pct=0.859329 moved_to_echo=33690 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 5852, 626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 236, 1967, 0, 0, 0, 0, 0, 3, 5, 0, 0, 0, 0, 0, 0, 0, 7365, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4, 7, 0, 8172, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 2, 0, 0, 5910, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 656, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8185, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, 4, 1, 0, 0, 0, 8184, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 8184, 4, 0, 0, 2, 2, 1678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6494, 9586, 8193, 6633, 8189, 8194, 8372, 9875] avg=8192.0000 max=9875 max_rank=7 min=6494 min_rank=0 imbalance_ratio=1.205444 max_load_reduction=55661 max_load_reduction_pct=0.849319 moved_to_echo=34630 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3872, 0, 5853, 627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7384, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 656, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 0, 0, 0, 0, 0, 7721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4260, 0, 0, 0, 0, 0, 1679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10370, 8113, 8187, 8642, 8191, 8192, 7902, 5939] avg=8192.0000 max=10370 max_rank=0 min=5939 min_rank=7 imbalance_ratio=1.265869 max_load_reduction=55166 max_load_reduction_pct=0.841766 moved_to_echo=30224 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 946, 0, 5836, 626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 175, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 7370, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 7821, 0, 0, 0, 22, 0, 0, 2, 0, 0, 0, 0, 654, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, 0, 0, 0, 0, 0, 8072, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7403, 0, 0, 0, 0, 1, 1673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7490, 7561, 8187, 8533, 8215, 8217, 8256, 9077] avg=8192.0000 max=9077 max_rank=7 min=7490 min_rank=0 imbalance_ratio=1.108032 max_load_reduction=56459 max_load_reduction_pct=0.861496 moved_to_echo=33765 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 896, 0, 5846, 627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7384, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7283, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 655, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 0, 0, 0, 0, 0, 1676, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7375, 7480, 8187, 7942, 8191, 8187, 8362, 9812] avg=8192.0000 max=9812 max_rank=7 min=7375 min_rank=0 imbalance_ratio=1.197754 max_load_reduction=55724 max_load_reduction_pct=0.850281 moved_to_echo=34552 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4882, 0, 5837, 632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7388, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 658, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 0, 0, 0, 0, 0, 7711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1982, 0, 0, 0, 0, 0, 1665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11372, 9226, 8187, 8845, 8189, 8187, 7883, 3647] avg=8192.0000 max=11372 max_rank=0 min=3647 min_rank=7 imbalance_ratio=1.388184 max_load_reduction=54164 max_load_reduction_pct=0.826477 moved_to_echo=27906 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 591, 0, 0, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 7354, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 8140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 8192, 0, 0, 0, 8187, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 1652, 0, 0, 534, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 6459, 0, 0, 0, 22, 0, 0, 2, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 8192, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 8124, 0, 0, 0, 4, 0, 0, 0, 0, 0, 7646, 0, 8192, 0, 0, 1, 8162, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1175, 0, 1, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 7902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7601, 0, 8192, 0, 0, 0, 8186, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2552, 0, 0, 1340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4320, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 8192, 0, 0, 0, 8179, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 13, 0, 0, 0, 0, 0, 0, 3804, 0, 176, 2240, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 11, 7, 0, 0, 0, 5, 0, 0, 0, 0, 0, 17, 0, 0, 3, 0, 0, 2836, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 8152, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8175, 6, 0, 5, 0, 0, 8169, 12, 0, 0, 0, 0, 0, 0, 0, 13, 7292, 2, 8167, 0, 0, 0, 8180, 2, 9, 0, 0, 8] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5128, 0, 0, 1682, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7351, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 7945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2477, 0, 8192, 0, 0, 0, 8161, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4913, 0, 0, 701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 7788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 7698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3453, 0, 8192, 0, 0, 0, 8189, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 316, 5645, 0, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8, 0, 18, 3, 0, 0, 2132, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 2, 0, 8186, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 4, 1, 0, 0, 0, 8184, 3, 0, 0, 0, 0, 0, 0, 0, 5, 8185, 4, 8182, 0, 2, 1, 8186, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 1652, 0, 4195, 1483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 6547, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 6459, 0, 0, 0, 22, 0, 0, 2, 0, 0, 0, 0, 2410, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 0, 0, 0, 0, 8124, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7646, 0, 0, 0, 0, 1, 1557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7421, 7097, 8187, 8924, 8216, 8214, 8273, 9204] avg=8192.0000 max=9204 max_rank=7 min=7097 min_rank=1 imbalance_ratio=1.123535 max_load_reduction=56332 max_load_reduction_pct=0.859558 moved_to_echo=33907 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 591, 0, 4207, 1486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6559, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 7354, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 2417, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 0, 0, 0, 0, 8140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 1563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6305, 6807, 8186, 9825, 8193, 8192, 8285, 9743] avg=8192.0000 max=9825 max_rank=3 min=6305 min_rank=0 imbalance_ratio=1.199341 max_load_reduction=55711 max_load_reduction_pct=0.850082 moved_to_echo=34413 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2552, 0, 4203, 1487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6561, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4320, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 2415, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 1561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8249, 7911, 8187, 6739, 8191, 8187, 8333, 9739] avg=8192.0000 max=9739 max_rank=7 min=6739 min_rank=3 imbalance_ratio=1.188843 max_load_reduction=55797 max_load_reduction_pct=0.851395 moved_to_echo=34450 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 13, 0, 0, 0, 0, 0, 0, 3804, 0, 4204, 1483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 2240, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6547, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 11, 7, 0, 0, 0, 5, 0, 0, 0, 0, 0, 17, 0, 8158, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 2836, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2415, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 6, 0, 5, 0, 0, 8169, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 7292, 2, 0, 0, 0, 0, 1561, 2, 9, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9505, 8973, 8221, 5256, 8175, 8182, 8337, 8887] avg=8192.0000 max=9505 max_rank=0 min=5256 min_rank=3 imbalance_ratio=1.160278 max_load_reduction=56031 max_load_reduction_pct=0.854965 moved_to_echo=33581 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5128, 0, 4198, 1490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1682, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6563, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7351, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2414, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 0, 0, 0, 0, 0, 7945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2477, 0, 0, 0, 0, 0, 1549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10839, 8255, 8187, 9768, 8190, 8187, 8084, 4026] avg=8192.0000 max=10839 max_rank=0 min=4026 min_rank=7 imbalance_ratio=1.323120 max_load_reduction=54697 max_load_reduction_pct=0.834610 moved_to_echo=28487 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1175, 0, 4207, 1487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6561, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2417, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 0, 0, 0, 0, 7902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7601, 0, 0, 0, 0, 0, 1562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6871, 6699, 8187, 10193, 8191, 8186, 8046, 9163] avg=8192.0000 max=10193 max_rank=3 min=6699 min_rank=1 imbalance_ratio=1.244263 max_load_reduction=55343 max_load_reduction_pct=0.844467 moved_to_echo=33586 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4913, 0, 4209, 1487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6561, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 7788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2418, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 0, 0, 0, 0, 7698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3453, 0, 0, 0, 0, 0, 1562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10628, 7272, 8187, 10209, 8191, 8192, 7842, 5015] avg=8192.0000 max=10628 max_rank=0 min=5015 min_rank=7 imbalance_ratio=1.297363 max_load_reduction=54908 max_load_reduction_pct=0.837830 moved_to_echo=29240 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 4207, 1483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 316, 5645, 0, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0, 6544, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8, 0, 8172, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 3, 0, 0, 2132, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 2417, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8185, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 4, 1, 0, 0, 0, 8184, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 8185, 4, 0, 0, 2, 1, 1562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5718, 12522, 8194, 4622, 8189, 8196, 8336, 9759] avg=8192.0000 max=12522 max_rank=1 min=4622 min_rank=3 imbalance_ratio=1.528564 max_load_reduction=53014 max_load_reduction_pct=0.808929 moved_to_echo=34480 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[256, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 128, 0, 13696, 0, 46848, 5120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 128, 52096, 128, 0, 0, 128, 128, 13440, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 0, 0, 60160, 0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 5248, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[512, 6016, 0, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 59136, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 0, 65536, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 0, 1536, 128, 128, 128, 0, 0, 64128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 128, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 128, 0, 19968, 0, 33664, 11904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 0, 0, 46080, 0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 19328, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 0, 128, 53120, 128, 0, 0, 128, 128, 12544, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[512, 12544, 0, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 52480, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 1152, 128, 128, 128, 0, 0, 64384, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 128, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 0, 65536, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 5, 0, 0, 5, 8180, 0, 0, 0, 0, 0, 0, 0, 7286, 8183, 22, 5, 0, 0, 0, 3, 0, 0, 5, 0, 0, 2, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 6, 0, 2, 0, 0, 0, 2, 0, 0, 0, 8184, 0, 929, 0, 4, 8169, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 5, 0, 11, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 8180, 8176, 1, 11, 0, 0, 0, 6, 47, 0, 16, 0, 0, 10, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 8108, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 8094, 0, 0, 0, 0, 17, 15, 0, 0, 0, 0, 2, 0, 8169, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 10, 0, 0, 0, 0, 7, 89, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5, 8192, 0, 0, 0, 0, 0, 0, 0, 8053, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 8192, 0, 139, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8154, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 37, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[12, 0, 0, 0, 15, 8182, 0, 1, 0, 0, 0, 0, 0, 8002, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 8192, 0, 190, 0, 0, 8181, 0, 0, 4, 1, 0, 0, 0, 0, 0, 2, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8161, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 28, 0, 0, 8192, 0, 0, 0, 11, 0, 0, 0, 0, 0, 1, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8089, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8192, 0, 102, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8181, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 3, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8192, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 5, 0, 11, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 8180, 8176, 1, 11, 0, 0, 0, 6, 48, 0, 16, 0, 0, 11, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 8105, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 8093, 0, 0, 0, 0, 16, 15, 0, 0, 0, 0, 2, 0, 8170, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 10, 0, 0, 0, 0, 7, 91, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[12, 0, 0, 0, 16, 8182, 0, 1, 0, 0, 0, 0, 0, 8003, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 8192, 0, 189, 0, 0, 8181, 0, 0, 4, 1, 0, 0, 0, 0, 0, 2, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8092, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8192, 0, 99, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5, 8192, 0, 0, 0, 0, 0, 0, 0, 8061, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 8192, 0, 131, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8155, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 37, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8161, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8192, 0, 28, 0, 0, 8192, 0, 0, 0, 10, 0, 0, 0, 0, 0, 1, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 7286, 167, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 3, 0, 0, 5, 0, 0, 2, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 6, 0, 2, 0, 0, 0, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 929, 0, 4, 7999, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 0, 5, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4, 8016, 153, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7485, 8191, 8182, 8946, 8169, 8183, 8187, 8193] avg=8192.0000 max=8946 max_rank=3 min=7485 min_rank=0 imbalance_ratio=1.092041 max_load_reduction=56590 max_load_reduction_pct=0.863495 moved_to_echo=32732 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 167, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 6, 48, 0, 16, 0, 0, 11, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8105, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7925, 0, 0, 0, 0, 16, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8170, 0, 0, 0, 0, 0, 3, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 10, 0, 0, 0, 0, 7, 91, 12, 8009, 151, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8364, 8257, 8127, 7956, 8190, 8294, 8169, 8179] avg=8192.0000 max=8364 max_rank=0 min=7956 min_rank=3 imbalance_ratio=1.020996 max_load_reduction=57172 max_load_reduction_pct=0.872375 moved_to_echo=32832 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 8061, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 131, 0, 0, 8020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8018, 153, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8233, 8183, 8203, 8158, 8191, 8173, 8195, 8200] avg=8192.0000 max=8233 max_rank=0 min=8158 min_rank=3 imbalance_ratio=1.005005 max_load_reduction=57303 max_load_reduction_pct=0.874374 moved_to_echo=32759 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8092, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 99, 0, 0, 8022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8025, 153, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8259, 8183, 8203, 8124, 8204, 8180, 8198, 8185] avg=8192.0000 max=8259 max_rank=0 min=8124 min_rank=3 imbalance_ratio=1.008179 max_load_reduction=57277 max_load_reduction_pct=0.873978 moved_to_echo=32767 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 0, 0, 16, 0, 0, 1, 0, 0, 0, 0, 0, 8003, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 189, 0, 0, 8011, 0, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8006, 153, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8178, 8203, 8214, 8168, 8161, 8195, 8218] avg=8192.0000 max=8218 max_rank=7 min=8161 min_rank=5 imbalance_ratio=1.003174 max_load_reduction=57318 max_load_reduction_pct=0.874603 moved_to_echo=32742 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8022, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8025, 153, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8348, 8183, 8206, 8026, 8207, 8180, 8200, 8186] avg=8192.0000 max=8348 max_rank=0 min=8026 min_rank=3 imbalance_ratio=1.019043 max_load_reduction=57188 max_load_reduction_pct=0.872620 moved_to_echo=32773 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 28, 0, 0, 8022, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8023, 153, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8328, 8183, 8203, 8061, 8196, 8178, 8197, 8190] avg=8192.0000 max=8328 max_rank=0 min=8061 min_rank=3 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=32761 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 8014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8027, 156, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8320, 8178, 8209, 8051, 8211, 8187, 8195, 8185] avg=8192.0000 max=8320 max_rank=0 min=8051 min_rank=3 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32778 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 5, 0, 0, 6, 8180, 0, 0, 0, 0, 0, 0, 0, 7268, 8183, 23, 6, 0, 0, 0, 3, 0, 0, 5, 0, 0, 2, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 7, 0, 2, 0, 0, 0, 2, 0, 0, 0, 8183, 0, 947, 0, 4, 8169, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 8053, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 139, 0, 0, 8017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8014, 157, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8229, 8183, 8202, 8163, 8190, 8173, 8196, 8200] avg=8192.0000 max=8229 max_rank=0 min=8163 min_rank=3 imbalance_ratio=1.004517 max_load_reduction=57307 max_load_reduction_pct=0.874435 moved_to_echo=32759 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 0, 0, 15, 0, 0, 1, 0, 0, 0, 0, 0, 8002, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 190, 0, 0, 8008, 0, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8003, 157, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8178, 8202, 8212, 8168, 8162, 8195, 8218] avg=8192.0000 max=8218 max_rank=7 min=8162 min_rank=5 imbalance_ratio=1.003174 max_load_reduction=57318 max_load_reduction_pct=0.874603 moved_to_echo=32743 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 171, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 6, 47, 0, 16, 0, 0, 10, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8108, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7923, 0, 0, 0, 0, 17, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8169, 0, 0, 0, 0, 0, 3, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 10, 0, 0, 0, 0, 7, 89, 12, 8005, 155, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8368, 8255, 8129, 7955, 8189, 8292, 8169, 8179] avg=8192.0000 max=8368 max_rank=0 min=7955 min_rank=3 imbalance_ratio=1.021484 max_load_reduction=57168 max_load_reduction_pct=0.872314 moved_to_echo=32829 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8089, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 102, 0, 0, 8019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8021, 157, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8260, 8183, 8202, 8124, 8204, 8180, 8197, 8186] avg=8192.0000 max=8260 max_rank=0 min=8124 min_rank=3 imbalance_ratio=1.008301 max_load_reduction=57276 max_load_reduction_pct=0.873962 moved_to_echo=32767 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 8019, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8019, 157, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8333, 8183, 8202, 8058, 8196, 8178, 8196, 8190] avg=8192.0000 max=8333 max_rank=0 min=8058 min_rank=3 imbalance_ratio=1.017212 max_load_reduction=57203 max_load_reduction_pct=0.872849 moved_to_echo=32760 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 8003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8026, 161, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8320, 8179, 8212, 8040, 8213, 8190, 8196, 8186] avg=8192.0000 max=8320 max_rank=0 min=8040 min_rank=3 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32785 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8019, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8021, 157, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8352, 8183, 8205, 8023, 8207, 8180, 8200, 8186] avg=8192.0000 max=8352 max_rank=0 min=8023 min_rank=3 imbalance_ratio=1.019531 max_load_reduction=57184 max_load_reduction_pct=0.872559 moved_to_echo=32773 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 7268, 171, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 3, 0, 0, 5, 0, 0, 2, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 7, 0, 2, 0, 0, 0, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 947, 0, 4, 7996, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 0, 5, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 3, 8012, 157, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7473, 8192, 8182, 8961, 8169, 8181, 8187, 8191] avg=8192.0000 max=8961 max_rank=3 min=7473 min_rank=0 imbalance_ratio=1.093872 max_load_reduction=56575 max_load_reduction_pct=0.863266 moved_to_echo=32728 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 64128, 1408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 128, 128, 128, 64256, 1280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 0, 0, 0, 1536, 0, 128, 64128, 0, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 0, 0, 128, 128, 0, 0, 128, 0, 0, 0, 128, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 128, 128, 0, 128, 0, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 128, 0, 65408, 0, 0, 0, 128, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 64128, 1408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 128, 128, 128, 64128, 1280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 128, 0, 65408, 0, 0, 0, 128, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 128, 128, 0, 128, 0, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 0, 65536, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 0, 0, 128, 128, 0, 0, 128, 0, 0, 0, 128, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 734, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 1585, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 7648, 0, 0, 0, 8174, 12, 0, 0, 25, 0, 0, 0, 0, 8102, 6, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 3, 0, 0, 0, 2, 8168, 0, 0, 0, 0, 7, 8133, 14, 0, 111, 8173, 0, 0, 0, 0, 11, 0, 15, 13, 0, 0, 0, 0, 0, 0, 121, 2, 0, 3, 0, 0, 0, 1648, 8, 0, 0, 0, 0, 298, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 9, 0, 8178, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 14, 0, 0, 0, 0, 0, 6601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 195, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 165, 0, 0, 0, 0, 0, 0, 109, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 0, 8180, 0, 0, 0, 8179, 3, 4, 0, 0, 0, 0, 10, 0, 7861, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 13, 0, 0, 0, 0, 0, 8183, 0, 0, 43, 0, 4, 8151, 0, 0, 568, 8182, 0, 0, 0, 0, 5, 5, 4, 3, 0, 0, 0, 0, 0, 0, 701, 1, 0, 0, 0, 0, 0, 47, 0, 11, 0, 0, 0, 296, 0, 0, 0, 0, 0, 0, 0, 0, 8, 5, 0, 5, 0, 8168, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 6866, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 3, 0, 0, 8192, 0, 0, 156, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 1870, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 2, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 401, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8184, 0, 6, 0, 0, 0, 0, 0, 0, 8113, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 18, 0, 0, 0, 0, 0, 8192, 0, 0, 26, 0, 0, 8174, 0, 0, 72, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 281, 0, 0, 0, 0, 0, 0, 82, 0, 11, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 1663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7915, 0, 0, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 165, 0, 7998, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 6933, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8046, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1194, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8257, 8240, 8110, 8173, 7247, 8119, 8203, 9187] avg=8192.0000 max=9187 max_rank=7 min=7247 min_rank=4 imbalance_ratio=1.121460 max_load_reduction=56349 max_load_reduction_pct=0.859818 moved_to_echo=32756 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7915, 0, 0, 734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 7999, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1585, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 6933, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1194, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8257, 8213, 8649, 8081, 8713, 8187, 8184, 7252] avg=8192.0000 max=8713 max_rank=4 min=7252 min_rank=7 imbalance_ratio=1.063599 max_load_reduction=56823 max_load_reduction_pct=0.867050 moved_to_echo=32336 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 0, 8180, 0, 0, 0, 65, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 7861, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 13, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 4, 7876, 0, 0, 568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 4, 3, 0, 0, 0, 0, 0, 0, 701, 1, 7990, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 11, 0, 0, 0, 296, 0, 0, 0, 0, 6922, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 5, 0, 5, 0, 8168, 0, 0, 8, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 6866, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 1192, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8255, 7949, 8491, 8720, 7469, 8266, 8174, 8212] avg=8192.0000 max=8720 max_rank=3 min=7469 min_rank=4 imbalance_ratio=1.064453 max_load_reduction=56816 max_load_reduction_pct=0.866943 moved_to_echo=32121 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 7648, 0, 0, 0, 65, 12, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8102, 6, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 7, 7858, 14, 0, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 15, 13, 0, 0, 0, 0, 0, 0, 121, 2, 7974, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1648, 8, 0, 0, 0, 0, 298, 0, 0, 0, 0, 6918, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 9, 0, 8178, 0, 0, 10, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 14, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1191, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7752, 8177, 7995, 8146, 9069, 8276, 8189, 7932] avg=8192.0000 max=9069 max_rank=4 min=7752 min_rank=0 imbalance_ratio=1.107056 max_load_reduction=56467 max_load_reduction_pct=0.861618 moved_to_echo=33466 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 65, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7915, 0, 0, 401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7998, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6933, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1194, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8259, 8255, 8316, 8009, 7942, 8235, 8182, 8338] avg=8192.0000 max=8338 max_rank=7 min=7942 min_rank=4 imbalance_ratio=1.017822 max_load_reduction=57198 max_load_reduction_pct=0.872772 moved_to_echo=32697 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 65, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 7915, 0, 0, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 7999, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1870, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 6933, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1194, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8258, 8249, 8074, 8091, 9080, 8244, 8188, 7352] avg=8192.0000 max=9080 max_rank=4 min=7352 min_rank=7 imbalance_ratio=1.108398 max_load_reduction=56456 max_load_reduction_pct=0.861450 moved_to_echo=32864 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 64, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8113, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 18, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 7898, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 281, 0, 7998, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 11, 0, 0, 0, 7, 0, 0, 0, 0, 6927, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7909, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1193, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8261, 8195, 7996, 8289, 7221, 7982, 8235, 9357] avg=8192.0000 max=9357 max_rank=7 min=7221 min_rank=4 imbalance_ratio=1.142212 max_load_reduction=56179 max_load_reduction_pct=0.857224 moved_to_echo=32795 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 7999, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6939, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1197, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8237, 8258, 7905, 8027, 8795, 8227, 8181, 7906] avg=8192.0000 max=8795 max_rank=4 min=7905 min_rank=2 imbalance_ratio=1.073608 max_load_reduction=56741 max_load_reduction_pct=0.865799 moved_to_echo=33109 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 120, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 672, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 0, 1428, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 2, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 220, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 1764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 7687, 0, 0, 0, 8175, 12, 0, 0, 24, 0, 0, 0, 0, 8098, 6, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 3, 0, 0, 0, 3, 8168, 0, 0, 0, 0, 7, 8131, 13, 0, 80, 8173, 0, 0, 0, 0, 11, 0, 15, 13, 0, 0, 0, 0, 0, 0, 115, 2, 0, 2, 0, 0, 0, 1555, 8, 0, 0, 0, 0, 299, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 9, 0, 8178, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 14, 0, 0, 0, 0, 0, 6700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8184, 0, 6, 0, 0, 0, 0, 0, 0, 8114, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 19, 0, 0, 0, 0, 0, 8192, 0, 0, 23, 0, 0, 8173, 0, 0, 52, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 275, 0, 0, 0, 0, 0, 0, 34, 0, 11, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 2, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 8192, 0, 0, 94, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0, 2184, 0, 0, 0, 0, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 0, 0, 8180, 0, 0, 0, 8179, 3, 4, 0, 0, 0, 0, 10, 0, 7862, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 14, 0, 0, 0, 0, 0, 8183, 0, 0, 43, 0, 4, 8150, 0, 0, 492, 8182, 0, 0, 0, 0, 5, 5, 4, 3, 0, 0, 0, 0, 0, 0, 732, 1, 0, 0, 0, 0, 0, 62, 0, 11, 0, 0, 0, 300, 0, 0, 0, 0, 0, 0, 0, 0, 8, 5, 0, 4, 0, 8166, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 6894, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7980, 0, 0, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 0, 7994, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 6897, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8041, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1236, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8251, 8240, 8100, 8173, 7191, 8115, 8205, 9261] avg=8192.0000 max=9261 max_rank=7 min=7191 min_rank=4 imbalance_ratio=1.130493 max_load_reduction=56275 max_load_reduction_pct=0.858688 moved_to_echo=32772 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 7995, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6903, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1242, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8239, 8256, 7968, 8023, 8864, 8229, 8181, 7776] avg=8192.0000 max=8864 max_rank=4 min=7776 min_rank=7 imbalance_ratio=1.082031 max_load_reduction=56672 max_load_reduction_pct=0.864746 moved_to_echo=33050 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7980, 0, 0, 672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 7995, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1428, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 6897, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8113, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1236, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8251, 8214, 8652, 8081, 8524, 8187, 8184, 7443] avg=8192.0000 max=8652 max_rank=2 min=7443 min_rank=7 imbalance_ratio=1.056152 max_load_reduction=56884 max_load_reduction_pct=0.867981 moved_to_echo=32338 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 58, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 19, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 7962, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 275, 0, 7994, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 11, 0, 0, 0, 6, 0, 0, 0, 0, 6891, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7914, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1235, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8255, 8197, 8037, 8279, 7140, 7988, 8234, 9406] avg=8192.0000 max=9406 max_rank=7 min=7140 min_rank=4 imbalance_ratio=1.148193 max_load_reduction=56130 max_load_reduction_pct=0.856476 moved_to_echo=32768 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 7687, 0, 0, 0, 58, 12, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8098, 6, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 7, 7920, 13, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 15, 13, 0, 0, 0, 0, 0, 0, 115, 2, 7971, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1555, 8, 0, 0, 0, 0, 299, 0, 0, 0, 0, 6883, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 9, 0, 8178, 0, 0, 10, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 14, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1234, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7783, 8172, 8026, 8137, 8944, 8278, 8188, 8008] avg=8192.0000 max=8944 max_rank=4 min=7783 min_rank=0 imbalance_ratio=1.091797 max_load_reduction=56592 max_load_reduction_pct=0.863525 moved_to_echo=33418 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 59, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7980, 0, 0, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 7994, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6897, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1236, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8253, 8256, 8200, 8007, 8042, 8227, 8182, 8369] avg=8192.0000 max=8369 max_rank=7 min=8007 min_rank=3 imbalance_ratio=1.021606 max_load_reduction=57167 max_load_reduction_pct=0.872299 moved_to_echo=32820 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 59, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7980, 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 7995, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2184, 0, 0, 0, 0, 0, 97, 0, 0, 0, 0, 6897, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1236, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8253, 8250, 8075, 8089, 9375, 8247, 8188, 7059] avg=8192.0000 max=9375 max_rank=4 min=7059 min_rank=7 imbalance_ratio=1.144409 max_load_reduction=56161 max_load_reduction_pct=0.856949 moved_to_echo=32869 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 0, 0, 8180, 0, 0, 0, 59, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 7862, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 14, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 4, 7939, 0, 0, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 4, 3, 0, 0, 0, 0, 0, 0, 732, 1, 7986, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 11, 0, 0, 0, 300, 0, 0, 0, 0, 6886, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 5, 0, 4, 0, 8166, 0, 0, 8, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 6894, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 1234, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8251, 7951, 8478, 8747, 7456, 8265, 8174, 8214] avg=8192.0000 max=8747 max_rank=3 min=7456 min_rank=4 imbalance_ratio=1.067749 max_load_reduction=56789 max_load_reduction_pct=0.866531 moved_to_echo=32109 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 54912, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 9600, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 0, 0, 7936, 128, 128, 0, 0, 0, 768, 0, 0, 0, 0, 55552, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 128, 0, 0, 0, 128, 0, 128, 63232, 128, 0, 2304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 1536, 128, 64000, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 65024, 128, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 128, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 128, 0, 65024, 0, 0, 128, 0, 0, 0, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 128, 0, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 55040, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 9984, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 8064, 128, 128, 0, 0, 0, 768, 0, 0, 0, 0, 55168, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 128, 0, 0, 0, 128, 0, 128, 63744, 128, 0, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 128, 128, 0, 0, 65024, 0, 0, 0, 512, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 128, 0, 64896, 0, 0, 128, 0, 0, 0, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 128, 0, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 65024, 128, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 128, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 1536, 128, 64000, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1513, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 35, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1208, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 3, 0, 0, 0, 0, 0, 0, 21, 8183, 0, 8177, 0, 0, 2, 0, 0, 0, 8183, 0, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4, 5, 0, 0, 0, 8170, 0, 0, 11, 33, 0, 0, 3, 0, 11, 0, 0, 2, 0, 0, 0, 4, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8060, 0, 7, 8179, 0, 0, 1, 0, 0, 7, 0, 0, 0, 0, 0, 0, 12, 0, 0, 5, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5752, 8192, 0, 8160, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 8187, 0, 8174, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 78, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 10, 0, 0, 0, 8109, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 18, 0, 0, 0, 0, 5336, 8173, 0, 8129, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8, 16, 0, 0, 35, 7966, 0, 26, 1, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 12, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8109, 0, 1, 8167, 3, 0, 0, 10, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 2821, 4, 0, 0, 0, 19, 0, 14, 0, 0, 0, 3, 0, 0, 0, 0, 0, 12, 0, 190, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2956, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7028, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1513, 0, 0, 5166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6678, 0, 0, 0, 0, 0, 0, 0, 3012, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8045, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 6679, 8204, 8202, 8187, 8194, 9797, 8081] avg=8192.0000 max=9797 max_rank=6 min=6679 min_rank=1 imbalance_ratio=1.195923 max_load_reduction=55739 max_load_reduction_pct=0.850510 moved_to_echo=34259 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 3, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 5157, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4, 8148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 8170, 0, 0, 11, 33, 0, 0, 3, 0, 11, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8060, 0, 7, 39, 0, 0, 1, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 5, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 3006, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8033, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 5180, 8247, 8243, 8193, 8114, 11293, 8075] avg=8192.0000 max=11293 max_rank=6 min=5180 min_rank=1 imbalance_ratio=1.378540 max_load_reduction=54243 max_load_reduction_pct=0.827682 moved_to_echo=35675 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1208, 0, 0, 5166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6986, 0, 0, 0, 0, 0, 0, 0, 3012, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8045, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 6374, 8162, 8202, 8195, 8227, 10105, 8080] avg=8192.0000 max=10105 max_rank=6 min=6374 min_rank=1 imbalance_ratio=1.233521 max_load_reduction=55431 max_load_reduction_pct=0.845810 moved_to_echo=34607 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5752, 0, 0, 5147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 40, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2458, 0, 0, 0, 0, 0, 0, 0, 3000, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 8045, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 10899, 8158, 8178, 8195, 8234, 5565, 8116] avg=8192.0000 max=10899 max_rank=1 min=5565 min_rank=6 imbalance_ratio=1.330444 max_load_reduction=54637 max_load_reduction_pct=0.833694 moved_to_echo=30110 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 18, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5336, 0, 0, 5128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8, 8138, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 35, 7966, 0, 26, 1, 54, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 12, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8109, 0, 1, 39, 3, 0, 0, 10, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 2821, 4, 0, 0, 0, 19, 0, 14, 2988, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 12, 0, 190, 0, 0, 0, 0, 8021, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 10464, 8177, 8108, 8177, 8181, 5969, 8261] avg=8192.0000 max=10464 max_rank=1 min=5969 min_rank=6 imbalance_ratio=1.277344 max_load_reduction=55072 max_load_reduction_pct=0.840332 moved_to_echo=30588 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 0, 0, 5156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 10, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8109, 0, 0, 39, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7710, 0, 0, 0, 0, 0, 0, 0, 3005, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 8036, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 5634, 8257, 8206, 8196, 8154, 10822, 8077] avg=8192.0000 max=10822 max_rank=6 min=5634 min_rank=1 imbalance_ratio=1.321045 max_load_reduction=54714 max_load_reduction_pct=0.834869 moved_to_echo=35249 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7028, 0, 0, 5166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 0, 0, 0, 0, 0, 0, 0, 3012, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 0, 0, 0, 0, 8045, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 12194, 8169, 8193, 8194, 8207, 4150, 8238] avg=8192.0000 max=12194 max_rank=1 min=4150 min_rank=6 imbalance_ratio=1.488525 max_load_reduction=53342 max_load_reduction_pct=0.813934 moved_to_echo=28789 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2956, 0, 0, 5156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4711, 0, 0, 0, 0, 0, 0, 0, 3015, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 0, 0, 0, 0, 8050, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8112, 8162, 8204, 8199, 8225, 7835, 8608] avg=8192.0000 max=8608 max_rank=7 min=7835 min_rank=6 imbalance_ratio=1.050781 max_load_reduction=56928 max_load_reduction_pct=0.868652 moved_to_echo=32867 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 787, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 35, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 313, 8187, 0, 8174, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 77, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 10, 0, 0, 0, 8110, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5169, 8192, 0, 8156, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2062, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 841, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 784, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6409, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 10, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 326, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 17, 0, 0, 0, 0, 4391, 8173, 0, 8134, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8, 16, 0, 0, 32, 7969, 0, 25, 1, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 12, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8108, 0, 1, 8167, 3, 0, 0, 10, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 3759, 4, 0, 0, 0, 18, 0, 14, 0, 0, 0, 3, 0, 0, 0, 0, 0, 12, 0, 203, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 3, 0, 0, 0, 0, 0, 0, 6, 8183, 0, 8177, 0, 0, 2, 0, 0, 0, 8183, 0, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4, 5, 0, 0, 0, 8170, 0, 0, 11, 33, 0, 0, 3, 0, 11, 0, 0, 2, 0, 0, 0, 4, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8060, 0, 7, 8179, 0, 0, 1, 0, 0, 7, 0, 0, 0, 0, 0, 0, 12, 0, 0, 4, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 787, 0, 0, 5715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7400, 0, 0, 0, 0, 0, 0, 0, 2464, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 7976, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 6502, 8205, 8202, 8186, 8193, 10041, 8015] avg=8192.0000 max=10041 max_rank=6 min=6502 min_rank=1 imbalance_ratio=1.225708 max_load_reduction=55495 max_load_reduction_pct=0.846786 moved_to_echo=34435 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 313, 0, 0, 5702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 10, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8110, 0, 0, 40, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7877, 0, 0, 0, 0, 0, 0, 0, 2459, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 7966, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 6015, 8256, 8207, 8196, 8156, 10512, 8003] avg=8192.0000 max=10512 max_rank=6 min=6015 min_rank=1 imbalance_ratio=1.283203 max_load_reduction=55024 max_load_reduction_pct=0.839600 moved_to_echo=34867 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5169, 0, 0, 5690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 39, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2984, 0, 0, 0, 0, 0, 0, 0, 2453, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 0, 0, 0, 0, 7976, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 10859, 8159, 8173, 8195, 8233, 5614, 8112] avg=8192.0000 max=10859 max_rank=1 min=5614 min_rank=6 imbalance_ratio=1.325562 max_load_reduction=54677 max_load_reduction_pct=0.834305 moved_to_echo=30154 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2062, 0, 0, 5698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5287, 0, 0, 0, 0, 0, 0, 0, 2469, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 841, 0, 0, 0, 0, 7978, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 7760, 8161, 8206, 8202, 8226, 7936, 8853] avg=8192.0000 max=8853 max_rank=7 min=7760 min_rank=1 imbalance_ratio=1.080688 max_load_reduction=56683 max_load_reduction_pct=0.864914 moved_to_echo=33217 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 784, 0, 0, 5715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7410, 0, 0, 0, 0, 0, 0, 0, 2464, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7976, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 6499, 8163, 8202, 8194, 8226, 10051, 8010] avg=8192.0000 max=10051 max_rank=6 min=6499 min_rank=1 imbalance_ratio=1.226929 max_load_reduction=55485 max_load_reduction_pct=0.846634 moved_to_echo=34481 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6409, 0, 0, 5715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1476, 0, 0, 0, 0, 0, 0, 0, 2464, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 326, 0, 0, 0, 0, 7976, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 12124, 8172, 8197, 8193, 8206, 4117, 8336] avg=8192.0000 max=12124 max_rank=1 min=4117 min_rank=6 imbalance_ratio=1.479980 max_load_reduction=53412 max_load_reduction_pct=0.815002 moved_to_echo=28852 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 17, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4391, 0, 0, 5674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8, 8139, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 32, 7969, 0, 25, 1, 53, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 12, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8108, 0, 1, 40, 3, 0, 0, 10, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 3759, 4, 0, 0, 0, 18, 0, 14, 2447, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 12, 0, 203, 0, 0, 0, 0, 7951, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 10065, 8172, 8106, 8178, 8181, 6434, 8203] avg=8192.0000 max=10065 max_rank=1 min=6434 min_rank=6 imbalance_ratio=1.228638 max_load_reduction=55471 max_load_reduction_pct=0.846420 moved_to_echo=30996 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 3, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 5704, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 8170, 0, 0, 11, 33, 0, 0, 3, 0, 11, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8060, 0, 7, 40, 0, 0, 1, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 4, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 1, 2460, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 7963, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 5712, 8248, 8243, 8192, 8115, 10831, 8004] avg=8192.0000 max=10831 max_rank=6 min=5712 min_rank=1 imbalance_ratio=1.322144 max_load_reduction=54705 max_load_reduction_pct=0.834732 moved_to_echo=35142 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 0, 0, 128, 0, 0, 40576, 128, 0, 0, 0, 128, 0, 128, 24064, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 24320, 0, 0, 41344, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 1024, 0, 0, 0, 0, 64384, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 128, 0, 0, 128, 0, 0, 44416, 128, 0, 0, 0, 128, 0, 128, 19712, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 19968, 0, 0, 45696, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 1536, 0, 0, 0, 0, 63872, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 65280, 0, 128, 384, 128, 0, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 128, 65280, 0, 128, 128, 128, 0, 0, 128, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 128, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 256, 0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 0, 128, 0, 128, 65280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 128, 65280, 0, 128, 128, 128, 0, 0, 128, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 128, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 178, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3132, 8192, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 5057, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 3, 28, 0, 0, 8028, 0, 0, 0, 0, 0, 19, 0, 0, 0, 7584, 8142, 0, 0, 0, 40, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 670, 0, 0, 8167, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 2, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 7, 0, 8161, 0, 0, 5, 6, 0, 0, 0, 8043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 21, 0, 16, 0, 2, 0, 0, 0, 0, 0, 0, 0, 144, 4, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 16, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1321, 8174, 0, 0, 91, 12, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 7322, 0, 0, 8148, 0, 0, 0, 8186, 0, 0, 2, 1, 0, 0, 0, 27, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8175, 0, 5, 1, 4, 0, 0, 0, 7734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 10, 0, 0, 0, 0, 0, 8184, 5, 0, 26, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3236, 8192, 0, 0, 25, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 4972, 0, 0, 8190, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7717, 8192, 0, 0, 11, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 471, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 2, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8122, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5631, 8192, 0, 0, 121, 0, 0, 0, 0, 0, 51, 0, 0, 9, 0, 0, 0, 0, 2646, 0, 0, 8161, 0, 0, 0, 8191, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 7945, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8134, 0, 0, 91, 12, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7322, 0, 0, 5491, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7989, 174, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 0, 5, 1, 4, 0, 0, 0, 7734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5334, 2657, 40, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 10, 0, 0, 0, 0, 0, 8180, 5, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3865, 8244, 12816, 8215, 7930, 8193, 8044, 8229] avg=8192.0000 max=12816 max_rank=2 min=3865 min_rank=0 imbalance_ratio=1.564453 max_load_reduction=52720 max_load_reduction_pct=0.804443 moved_to_echo=32396 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 5521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8006, 177, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 8016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 178, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 5427, 2671, 40, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10759, 8152, 5536, 8189, 8202, 8186, 8324, 8188] avg=8192.0000 max=10759 max_rank=0 min=5536 min_rank=2 imbalance_ratio=1.313354 max_load_reduction=54777 max_load_reduction_pct=0.835831 moved_to_echo=32900 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 0, 0, 25, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4972, 0, 0, 5520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8006, 177, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 8138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 5413, 2670, 40, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5816, 8189, 10492, 8189, 8324, 8187, 8153, 8186] avg=8192.0000 max=10492 max_rank=2 min=5816 min_rank=0 imbalance_ratio=1.280762 max_load_reduction=55044 max_load_reduction_pct=0.839905 moved_to_echo=32850 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5057, 0, 0, 5521, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8006, 177, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 8007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 5425, 2671, 40, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5719, 8155, 10659, 8189, 8193, 8186, 8247, 8188] avg=8192.0000 max=10659 max_rank=2 min=5719 min_rank=0 imbalance_ratio=1.301147 max_load_reduction=54877 max_load_reduction_pct=0.837357 moved_to_echo=32814 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 28, 0, 0, 2536, 0, 0, 0, 0, 0, 19, 0, 0, 0, 7584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8102, 0, 0, 0, 40, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 670, 0, 0, 5504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 2, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 7, 0, 7975, 174, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 0, 0, 5, 6, 0, 0, 0, 8043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 21, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 2, 0, 0, 0, 0, 0, 0, 0, 144, 4, 0, 0, 0, 0, 5318, 2663, 40, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10170, 8147, 6174, 8189, 8240, 8226, 8191, 8199] avg=8192.0000 max=10170 max_rank=0 min=6174 min_rank=2 imbalance_ratio=1.241455 max_load_reduction=55366 max_load_reduction_pct=0.844818 moved_to_echo=32856 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7717, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 0, 0, 11, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 0, 0, 5521, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8006, 177, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 2, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 5422, 2671, 40, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10302, 8168, 6001, 8189, 8340, 8186, 8164, 8186] avg=8192.0000 max=10302 max_rank=0 min=6001 min_rank=2 imbalance_ratio=1.257568 max_load_reduction=55234 max_load_reduction_pct=0.842804 moved_to_echo=32876 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 0, 0, 121, 0, 0, 0, 0, 0, 51, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2646, 0, 0, 5500, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7997, 176, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 7945, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 5397, 2661, 40, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8333, 8185, 8179, 8132, 8186, 8141, 8176] avg=8192.0000 max=8333 max_rank=1 min=8132 min_rank=4 imbalance_ratio=1.017212 max_load_reduction=57203 max_load_reduction_pct=0.872849 moved_to_echo=32635 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 0, 0, 5518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8009, 182, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 0, 0, 0, 0, 0, 0, 0, 7992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 5431, 2674, 44, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10701, 8148, 5673, 8197, 8175, 8186, 8272, 8184] avg=8192.0000 max=10701 max_rank=0 min=5673 min_rank=2 imbalance_ratio=1.306274 max_load_reduction=54835 max_load_reduction_pct=0.836716 moved_to_echo=32817 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3086, 8192, 0, 0, 28, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 5111, 0, 0, 8190, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5638, 8192, 0, 0, 121, 0, 0, 0, 0, 0, 51, 0, 0, 9, 0, 0, 0, 0, 2586, 0, 0, 8163, 0, 0, 0, 8191, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 7990, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 3, 29, 0, 0, 8024, 0, 0, 0, 0, 0, 18, 0, 0, 0, 7385, 8143, 0, 0, 0, 41, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 826, 0, 0, 8167, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 2, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 7, 0, 8161, 0, 0, 5, 5, 0, 0, 0, 8107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 21, 0, 13, 0, 2, 0, 0, 0, 0, 0, 0, 0, 129, 4, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 16, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7785, 8192, 0, 0, 12, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 403, 0, 0, 8191, 0, 0, 0, 8192, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 2, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8118, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3871, 8192, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4309, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 159, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1394, 8175, 0, 0, 92, 12, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 7179, 0, 0, 8148, 0, 0, 0, 8186, 0, 0, 2, 3, 0, 0, 0, 23, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8175, 0, 5, 1, 4, 0, 0, 0, 7805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 10, 0, 0, 0, 0, 0, 8184, 5, 0, 29, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 28, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5111, 0, 0, 5607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8034, 148, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 5506, 2583, 41, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5601, 8191, 10718, 8189, 8307, 8186, 8159, 8185] avg=8192.0000 max=10718 max_rank=2 min=5601 min_rank=0 imbalance_ratio=1.308350 max_load_reduction=54818 max_load_reduction_pct=0.836456 moved_to_echo=32837 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5638, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 121, 0, 0, 0, 0, 0, 51, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2586, 0, 0, 5588, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8026, 148, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 0, 0, 0, 0, 0, 0, 0, 7990, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 5490, 2575, 41, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8144, 8332, 8222, 8181, 8149, 8185, 8146, 8177] avg=8192.0000 max=8332 max_rank=1 min=8144 min_rank=0 imbalance_ratio=1.017090 max_load_reduction=57204 max_load_reduction_pct=0.872864 moved_to_echo=32657 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 29, 0, 0, 2469, 0, 0, 0, 0, 0, 18, 0, 0, 0, 7385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8102, 0, 0, 0, 41, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 826, 0, 0, 5591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 2, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 7, 0, 8004, 146, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 0, 0, 5, 5, 0, 0, 0, 8107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 21, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 2, 0, 0, 0, 0, 0, 0, 0, 129, 4, 0, 0, 0, 0, 5409, 2576, 41, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9904, 8147, 6417, 8189, 8274, 8228, 8178, 8199] avg=8192.0000 max=9904 max_rank=0 min=6417 min_rank=2 imbalance_ratio=1.208984 max_load_reduction=55632 max_load_reduction_pct=0.848877 moved_to_echo=32879 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4309, 0, 0, 5608, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8034, 149, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 0, 0, 0, 0, 0, 0, 0, 8009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 5520, 2584, 41, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6391, 8154, 10015, 8189, 8167, 8186, 8247, 8187] avg=8192.0000 max=10015 max_rank=2 min=6391 min_rank=0 imbalance_ratio=1.222534 max_load_reduction=55521 max_load_reduction_pct=0.847183 moved_to_echo=32787 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 0, 0, 5605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8039, 153, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 0, 0, 0, 0, 0, 0, 0, 8011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 5525, 2587, 45, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10632, 8147, 5748, 8199, 8164, 8185, 8277, 8184] avg=8192.0000 max=10632 max_rank=0 min=5748 min_rank=2 imbalance_ratio=1.297852 max_load_reduction=54904 max_load_reduction_pct=0.837769 moved_to_echo=32810 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 12, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 403, 0, 0, 5608, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8033, 149, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 2, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 5517, 2583, 41, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10303, 8170, 6029, 8188, 8311, 8186, 8164, 8185] avg=8192.0000 max=10303 max_rank=0 min=6029 min_rank=2 imbalance_ratio=1.257690 max_load_reduction=55233 max_load_reduction_pct=0.842789 moved_to_echo=32846 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 5608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8034, 149, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 0, 0, 0, 0, 0, 0, 0, 8034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 159, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 0, 0, 0, 5522, 2584, 41, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10690, 8151, 5625, 8189, 8192, 8186, 8316, 8187] avg=8192.0000 max=10690 max_rank=0 min=5625 min_rank=2 imbalance_ratio=1.304932 max_load_reduction=54846 max_load_reduction_pct=0.836884 moved_to_echo=32881 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8134, 0, 0, 92, 12, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7179, 0, 0, 5578, 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8018, 146, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 0, 5, 1, 4, 0, 0, 0, 7805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 5, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5425, 2570, 41, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 10, 0, 0, 0, 0, 0, 8180, 5, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3871, 8244, 12762, 8212, 7972, 8194, 8049, 8232] avg=8192.0000 max=12762 max_rank=2 min=3871 min_rank=0 imbalance_ratio=1.557861 max_load_reduction=52774 max_load_reduction_pct=0.805267 moved_to_echo=32447 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[384, 0, 128, 128, 0, 0, 0, 0, 0, 256, 256, 128, 128, 0, 0, 0, 43264, 21376, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 128, 0, 0, 20608, 0, 0, 0, 0, 0, 128, 0, 0, 0, 44928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 21376, 0, 0, 44160, 0, 0, 0, 0, 0, 0, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[384, 0, 128, 128, 0, 0, 0, 0, 0, 128, 256, 128, 128, 0, 0, 0, 44032, 20736, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 0, 0, 20096, 0, 0, 0, 0, 0, 128, 0, 0, 0, 45568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 20608, 0, 0, 44800, 0, 0, 0, 0, 0, 0, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 0, 128, 0, 0, 128, 0, 128, 0, 0, 128, 0, 0, 128, 0, 64000, 1536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1536, 0, 128, 128, 128, 0, 0, 0, 64128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 0, 128, 0, 0, 128, 0, 128, 0, 0, 128, 0, 0, 128, 0, 64256, 1280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[65152, 0, 0, 256, 128, 0, 0, 0, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1280, 0, 128, 128, 128, 0, 0, 0, 64384, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 128, 0, 0, 0, 0, 0, 65536, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 30, 0, 0, 0, 0, 0, 5, 14, 0, 0, 0, 0, 5, 0, 0, 0, 14, 0, 6, 8155, 0, 8168, 0, 0, 6055, 0, 1, 0, 0, 0, 0, 0, 13, 3, 8137, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 8036, 0, 0, 0, 0, 9, 0, 0, 1, 0, 0, 19, 0, 0, 8, 5, 0, 0, 7, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 15, 0, 0, 12, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 104, 14, 0, 0, 0, 0, 0, 2006, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 1903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8086, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 1, 0, 0, 0, 0, 0, 1146, 0, 0, 0, 0, 0, 0, 5257, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 17, 1, 0, 0, 0, 0, 0, 0, 8176, 9, 3, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 29, 8178, 0, 0, 1861, 0, 0, 0, 0, 0, 0, 0, 7, 0, 8157, 0, 8, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 46, 0, 0, 0, 7769, 10, 0, 0, 0, 0, 0, 15, 0, 1, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 8109, 0, 0, 0, 4, 0, 2, 2, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 2253, 5, 0, 0, 0, 0, 1, 4433, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 3, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 2206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8116, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8015, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 34, 0, 0, 0, 0, 0, 893, 0, 0, 0, 0, 0, 0, 5261, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 18, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8148, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 44, 8183, 0, 0, 571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 8078, 0, 0, 0, 0, 0, 0, 0, 0, 120, 0, 0, 86, 0, 0, 0, 0, 0, 1079, 0, 0, 0, 0, 0, 0, 6948, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 3403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4789, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8100, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 5, 0, 0, 0, 0, 0, 182, 0, 0, 0, 0, 0, 0, 7625, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 4005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 0, 0, 0, 0, 0, 0, 4101, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 17, 1, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 3, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5612, 29, 0, 0, 0, 1861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 8157, 0, 8, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 46, 0, 0, 0, 7769, 10, 0, 0, 0, 0, 0, 15, 0, 1, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 8109, 0, 0, 0, 4, 0, 2, 2, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 2253, 5, 0, 0, 0, 0, 1, 4433, 0, 0, 2369, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8188, 7502, 8183, 7972, 8181, 8179, 9142] avg=8192.0000 max=9142 max_rank=7 min=7502 min_rank=2 imbalance_ratio=1.115967 max_load_reduction=56394 max_load_reduction_pct=0.860504 moved_to_echo=33474 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 5, 14, 0, 0, 0, 0, 5, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 6, 5602, 0, 0, 0, 0, 6055, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 3, 8137, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 8036, 0, 0, 0, 0, 9, 0, 0, 1, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 8, 5, 0, 0, 7, 0, 8152, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 15, 0, 0, 12, 0, 0, 1, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 104, 14, 0, 0, 0, 0, 0, 2006, 0, 2, 2365, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8283, 8211, 11678, 8165, 8249, 8192, 8244, 4514] avg=8192.0000 max=11678 max_rank=2 min=4514 min_rank=7 imbalance_ratio=1.425537 max_load_reduction=53858 max_load_reduction_pct=0.821808 moved_to_echo=29199 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5628, 0, 0, 0, 0, 2206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8116, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8015, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 893, 0, 0, 0, 0, 0, 0, 5261, 0, 0, 2375, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8168, 8212, 7834, 8193, 8255, 8195, 8105, 8574] avg=8192.0000 max=8574 max_rank=7 min=7834 min_rank=2 imbalance_ratio=1.046631 max_load_reduction=56962 max_load_reduction_pct=0.869171 moved_to_echo=33129 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5628, 0, 0, 0, 0, 1903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8086, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1146, 0, 0, 0, 0, 0, 0, 5257, 0, 0, 2375, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8191, 7531, 8201, 8293, 8193, 8152, 8790] avg=8192.0000 max=8790 max_rank=7 min=7531 min_rank=2 imbalance_ratio=1.072998 max_load_reduction=56746 max_load_reduction_pct=0.865875 moved_to_echo=33428 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5628, 0, 0, 0, 0, 3403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4789, 0, 0, 2375, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8183, 9031, 8202, 8321, 8193, 8242, 7177] avg=8192.0000 max=9031 max_rank=2 min=7177 min_rank=7 imbalance_ratio=1.102417 max_load_reduction=56505 max_load_reduction_pct=0.862198 moved_to_echo=31933 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 18, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5628, 44, 0, 0, 0, 571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 5, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8078, 0, 0, 0, 0, 0, 0, 0, 0, 120, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 1079, 0, 0, 0, 0, 0, 0, 6948, 0, 0, 2375, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8162, 8184, 6243, 8180, 7824, 8193, 8251, 10499] avg=8192.0000 max=10499 max_rank=7 min=6243 min_rank=2 imbalance_ratio=1.281616 max_load_reduction=55037 max_load_reduction_pct=0.839798 moved_to_echo=34767 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5628, 0, 0, 0, 0, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8100, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 182, 0, 0, 0, 0, 0, 0, 7625, 0, 0, 2375, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8185, 6105, 8199, 8296, 8193, 8179, 10197] avg=8192.0000 max=10197 max_rank=7 min=6105 min_rank=2 imbalance_ratio=1.244751 max_load_reduction=55339 max_load_reduction_pct=0.844406 moved_to_echo=34865 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5607, 0, 0, 0, 0, 4005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 0, 0, 0, 0, 0, 0, 4101, 0, 0, 2380, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8182, 9612, 8213, 8326, 8196, 8184, 6643] avg=8192.0000 max=9612 max_rank=2 min=6643 min_rank=7 imbalance_ratio=1.173340 max_load_reduction=55924 max_load_reduction_pct=0.853333 moved_to_echo=31349 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 2, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 2060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8129, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8005, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 36, 0, 0, 0, 0, 0, 854, 0, 0, 0, 0, 0, 0, 5443, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 8192, 0, 0, 2719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8078, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 1, 0, 0, 0, 0, 0, 1105, 0, 0, 0, 0, 0, 0, 4484, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 3831, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 0, 0, 0, 0, 0, 0, 4268, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 18, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8147, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 45, 8183, 0, 0, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 8080, 0, 0, 0, 0, 0, 0, 0, 0, 118, 0, 0, 89, 0, 0, 0, 0, 0, 1017, 0, 0, 0, 0, 0, 0, 7025, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8177, 30, 0, 0, 0, 0, 0, 6, 13, 0, 0, 0, 0, 5, 0, 0, 0, 13, 0, 6, 8156, 0, 8168, 0, 0, 6144, 0, 1, 0, 0, 0, 0, 0, 13, 2, 8138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 8037, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 18, 0, 0, 7, 4, 0, 0, 9, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8165, 15, 0, 0, 12, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 93, 14, 0, 0, 0, 0, 0, 1928, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 759, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8094, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 6, 0, 0, 0, 0, 0, 199, 0, 0, 0, 0, 0, 0, 7331, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 19, 1, 0, 0, 0, 0, 0, 0, 8176, 9, 3, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 29, 8178, 0, 0, 2633, 0, 0, 0, 0, 0, 0, 0, 6, 0, 8156, 0, 8, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 47, 0, 0, 0, 7811, 10, 0, 0, 0, 0, 0, 15, 0, 2, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 8109, 0, 0, 0, 4, 0, 2, 1, 0, 0, 0, 0, 70, 0, 0, 0, 0, 0, 2232, 5, 0, 0, 0, 0, 2, 3640, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 4136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4056, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5333, 0, 0, 0, 0, 2060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8129, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8005, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 854, 0, 0, 0, 0, 0, 0, 5443, 0, 0, 2675, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8169, 8207, 7393, 8194, 8259, 8195, 8100, 9019] avg=8192.0000 max=9019 max_rank=7 min=7393 min_rank=2 imbalance_ratio=1.100952 max_load_reduction=56517 max_load_reduction_pct=0.862381 moved_to_echo=33573 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5333, 1, 0, 0, 0, 2719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8078, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1105, 0, 0, 0, 0, 0, 0, 4484, 0, 0, 2675, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8195, 8053, 8202, 8289, 8193, 8144, 8275] avg=8192.0000 max=8289 max_rank=4 min=8053 min_rank=2 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32901 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 18, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5333, 45, 0, 0, 0, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 5, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8080, 0, 0, 0, 0, 0, 0, 0, 0, 118, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 0, 1017, 0, 0, 0, 0, 0, 0, 7025, 0, 0, 2675, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8162, 8183, 5920, 8181, 7826, 8193, 8254, 10817] avg=8192.0000 max=10817 max_rank=7 min=5920 min_rank=2 imbalance_ratio=1.320435 max_load_reduction=54719 max_load_reduction_pct=0.834946 moved_to_echo=35090 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5323, 0, 0, 0, 0, 3831, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 0, 0, 0, 0, 0, 0, 4268, 0, 0, 2676, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8182, 9154, 8210, 8315, 8199, 8186, 7112] avg=8192.0000 max=9154 max_rank=2 min=7112 min_rank=7 imbalance_ratio=1.117432 max_load_reduction=56382 max_load_reduction_pct=0.860321 moved_to_echo=31812 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 6, 13, 0, 0, 0, 0, 5, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 6, 5311, 0, 0, 0, 0, 6144, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 2, 8138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 8037, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 7, 4, 0, 0, 9, 0, 8152, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8165, 15, 0, 0, 12, 0, 0, 1, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 93, 14, 0, 0, 0, 0, 0, 1928, 0, 2, 2663, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8282, 8211, 11475, 8164, 8240, 8191, 8249, 4724] avg=8192.0000 max=11475 max_rank=2 min=4724 min_rank=7 imbalance_ratio=1.400757 max_load_reduction=54061 max_load_reduction_pct=0.824905 moved_to_echo=29404 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5333, 0, 0, 0, 0, 4136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4056, 0, 0, 2675, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8185, 9469, 8203, 8312, 8193, 8245, 6742] avg=8192.0000 max=9469 max_rank=2 min=6742 min_rank=7 imbalance_ratio=1.155884 max_load_reduction=56067 max_load_reduction_pct=0.855515 moved_to_echo=31492 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5333, 0, 0, 0, 0, 759, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8094, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 199, 0, 0, 0, 0, 0, 0, 7331, 0, 0, 2675, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8185, 6092, 8200, 8287, 8193, 8176, 10221] avg=8192.0000 max=10221 max_rank=7 min=6092 min_rank=2 imbalance_ratio=1.247681 max_load_reduction=55315 max_load_reduction_pct=0.844040 moved_to_echo=34877 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 19, 1, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 3, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5318, 29, 0, 0, 0, 2633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 8156, 0, 8, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 47, 0, 0, 0, 7811, 10, 0, 0, 0, 0, 0, 15, 0, 2, 0, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 8109, 0, 0, 0, 4, 0, 2, 1, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 0, 2232, 5, 0, 0, 0, 0, 2, 3640, 0, 0, 2667, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8188, 7980, 8182, 8008, 8179, 8182, 8626] avg=8192.0000 max=8626 max_rank=7 min=7980 min_rank=2 imbalance_ratio=1.052979 max_load_reduction=56910 max_load_reduction_pct=0.868378 moved_to_echo=32995 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[256, 0, 0, 128, 0, 0, 5888, 128, 0, 0, 0, 0, 128, 40448, 0, 128, 19072, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 45056, 128, 0, 0, 0, 20608, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 0, 0, 0, 64384, 128, 0, 0, 0, 128, 0, 128, 128, 128, 0, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 0, 128, 128, 0, 0, 128, 0, 65536, 0, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[256, 0, 0, 128, 0, 0, 5760, 128, 0, 0, 0, 0, 128, 38272, 0, 128, 21504, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 42624, 128, 0, 0, 0, 22912, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 0, 0, 64512, 128, 0, 0, 0, 128, 0, 128, 0, 128, 0, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 128, 0, 0, 128, 128, 0, 0, 0, 0, 128, 0, 65408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 0, 128, 0, 128, 128, 128, 0, 0, 0, 128, 0, 0, 0, 65408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 65536, 0, 128, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 65408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 64896, 128, 0, 0, 128, 0, 128, 128, 0, 256, 0, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 65536, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 64896, 128, 0, 0, 128, 0, 128, 128, 0, 256, 0, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 0, 128, 128, 0, 0, 128, 0, 65536, 0, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 8122, 0, 0, 0, 9, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 62, 9, 10, 0, 32, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 10, 0, 0, 0, 0, 7, 0, 0, 0, 8154, 0, 0, 0, 4, 0, 0, 0, 0, 20, 0, 0, 2, 1, 0, 0, 0, 8171, 0, 8155, 0, 32, 0, 0, 0, 3, 0, 1, 0, 0, 4, 0, 2, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 8110, 0, 0, 1, 0, 8144, 0, 0, 13, 20, 20, 2, 95, 0, 25, 0, 0, 0, 0, 0, 8153, 0, 0, 13, 0, 0, 37, 0, 0, 0, 0, 0, 8163, 0, 0, 16, 0, 13, 25, 19, 0, 10, 0, 0, 4, 727, 0, 0, 0, 0, 18, 0, 10, 9, 0, 0, 0, 0, 0, 0, 0, 5756, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 11, 0, 0, 0, 4, 8169, 0, 8131, 0, 0, 0, 0, 0, 6, 0, 1817, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7957, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 1098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 36, 0, 4, 0, 24, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 8192, 0, 11, 0, 0, 0, 0, 0, 3967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1359, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 6840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8104, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 9, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8055, 0, 62, 9, 10, 0, 32, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 10, 0, 0, 0, 0, 7, 0, 0, 0, 8154, 3520, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 20, 0, 0, 2, 1, 0, 0, 0, 8105, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4627, 0, 0, 0, 32, 0, 0, 0, 3, 0, 1, 0, 0, 4, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 8196, 8171, 8174, 11773, 8156, 4669, 8226] avg=8192.0000 max=11773 max_rank=4 min=4669 min_rank=6 imbalance_ratio=1.437134 max_load_reduction=53763 max_load_reduction_pct=0.820358 moved_to_echo=32824 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 8131, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8110, 0, 0, 1, 0, 10, 0, 0, 13, 20, 20, 2, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 8153, 0, 0, 13, 0, 0, 37, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8046, 0, 0, 16, 0, 13, 25, 19, 0, 10, 0, 0, 4, 727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 10, 9, 0, 0, 0, 0, 0, 0, 0, 5756, 3519, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 11, 0, 0, 0, 4, 8126, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4626, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1817, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7957, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8159, 8271, 8236, 8860, 9393, 8175, 6457, 7985] avg=8192.0000 max=9393 max_rank=4 min=6457 min_rank=6 imbalance_ratio=1.146606 max_load_reduction=56143 max_load_reduction_pct=0.856674 moved_to_echo=32010 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8074, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7100, 3523, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8192, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8202, 8199, 8080, 10705, 8198, 5729, 8223] avg=8192.0000 max=10705 max_rank=4 min=5729 min_rank=6 imbalance_ratio=1.306763 max_load_reduction=54831 max_load_reduction_pct=0.836655 moved_to_echo=32855 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8087, 3528, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8197, 8182, 8074, 11697, 8199, 4767, 8220] avg=8192.0000 max=11697 max_rank=4 min=4767 min_rank=6 imbalance_ratio=1.427856 max_load_reduction=53839 max_load_reduction_pct=0.821518 moved_to_echo=32883 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 10, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 3529, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4638, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8167, 8183, 8074, 4640, 8204, 11852, 8216] avg=8192.0000 max=11852 max_rank=6 min=4640 min_rank=4 imbalance_ratio=1.446777 max_load_reduction=53684 max_load_reduction_pct=0.819153 moved_to_echo=32912 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1359, 3527, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8202, 8196, 8074, 4968, 8200, 11475, 8221] avg=8192.0000 max=11475 max_rank=6 min=4968 min_rank=4 imbalance_ratio=1.400757 max_load_reduction=54061 max_load_reduction_pct=0.824905 moved_to_echo=32864 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8066, 0, 36, 0, 4, 0, 24, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4225, 3514, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4620, 0, 0, 0, 11, 0, 0, 0, 0, 0, 3967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 8192, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8197, 8169, 8131, 7820, 8197, 8598, 8224] avg=8192.0000 max=8598 max_rank=6 min=7820 min_rank=4 imbalance_ratio=1.049561 max_load_reduction=56938 max_load_reduction_pct=0.868805 moved_to_echo=32839 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 924, 3531, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8104, 8200, 8069, 4540, 8207, 11989, 8221] avg=8192.0000 max=11989 max_rank=6 min=4540 min_rank=4 imbalance_ratio=1.463501 max_load_reduction=53547 max_load_reduction_pct=0.817062 moved_to_echo=32957 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 35, 0, 4, 0, 26, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 8192, 0, 12, 0, 0, 0, 0, 0, 3389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8104, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 8109, 0, 0, 0, 0, 8144, 0, 0, 13, 20, 11, 4, 93, 0, 25, 0, 0, 0, 0, 0, 8154, 0, 0, 13, 0, 0, 35, 0, 0, 0, 0, 0, 8164, 0, 0, 16, 0, 11, 30, 19, 0, 10, 0, 0, 4, 707, 0, 0, 0, 0, 18, 0, 11, 9, 0, 0, 0, 0, 0, 0, 0, 5725, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 11, 0, 0, 0, 3, 8169, 0, 8133, 0, 0, 0, 0, 0, 6, 0, 1876, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7955, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1067, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1352, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 6846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 8124, 0, 0, 0, 9, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 60, 9, 10, 0, 34, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 9, 0, 0, 0, 0, 7, 0, 0, 0, 8155, 0, 0, 0, 4, 0, 0, 0, 0, 20, 0, 2, 2, 1, 0, 0, 0, 8171, 0, 8155, 0, 31, 0, 0, 0, 3, 0, 0, 0, 0, 4, 0, 2, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8077, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7594, 3387, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4780, 0, 0, 0, 0, 0, 0, 0, 602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8192, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8203, 8200, 8083, 11059, 8198, 5382, 8223] avg=8192.0000 max=11059 max_rank=4 min=5382 min_rank=6 imbalance_ratio=1.349976 max_load_reduction=54477 max_load_reduction_pct=0.831253 moved_to_echo=32862 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8068, 0, 35, 0, 4, 0, 26, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4803, 3387, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4780, 0, 12, 0, 0, 0, 0, 0, 3389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8192, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8198, 8168, 8134, 8268, 8197, 8181, 8223] avg=8192.0000 max=8268 max_rank=4 min=8134 min_rank=3 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32869 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 11, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1067, 3387, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4780, 0, 0, 0, 0, 0, 0, 0, 7181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8166, 8182, 8077, 4532, 8203, 11961, 8215] avg=8192.0000 max=11961 max_rank=6 min=4532 min_rank=4 imbalance_ratio=1.460083 max_load_reduction=53575 max_load_reduction_pct=0.817490 moved_to_echo=32911 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8104, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 934, 3390, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4770, 0, 0, 0, 0, 0, 0, 0, 7347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8109, 8203, 8067, 4404, 8208, 12117, 8225] avg=8192.0000 max=12117 max_rank=6 min=4404 min_rank=4 imbalance_ratio=1.479126 max_load_reduction=53419 max_load_reduction_pct=0.815109 moved_to_echo=32954 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 3387, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4780, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8198, 8179, 8077, 11631, 8198, 4835, 8220] avg=8192.0000 max=11631 max_rank=4 min=4835 min_rank=6 imbalance_ratio=1.419800 max_load_reduction=53905 max_load_reduction_pct=0.822525 moved_to_echo=32884 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 8169, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8109, 0, 0, 0, 0, 11, 0, 0, 13, 20, 11, 4, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 8154, 0, 0, 13, 0, 0, 35, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8049, 0, 0, 16, 0, 11, 30, 19, 0, 10, 0, 0, 4, 707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 11, 9, 0, 0, 0, 0, 0, 0, 0, 5725, 3362, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 11, 0, 0, 0, 3, 8125, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4746, 0, 0, 0, 0, 0, 6, 0, 1876, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7955, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8261, 8236, 8846, 9203, 8174, 6636, 7983] avg=8192.0000 max=9203 max_rank=4 min=6636 min_rank=6 imbalance_ratio=1.123413 max_load_reduction=56333 max_load_reduction_pct=0.859573 moved_to_echo=31996 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1352, 3387, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4780, 0, 0, 0, 0, 0, 0, 0, 6846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8203, 8197, 8077, 4817, 8199, 11626, 8221] avg=8192.0000 max=11626 max_rank=6 min=4817 min_rank=4 imbalance_ratio=1.419189 max_load_reduction=53910 max_load_reduction_pct=0.822601 moved_to_echo=32863 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 11, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8057, 0, 60, 9, 10, 0, 34, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 9, 0, 0, 0, 0, 7, 0, 0, 0, 8155, 3372, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 20, 0, 2, 2, 1, 0, 0, 0, 8105, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4758, 0, 31, 0, 0, 0, 3, 0, 0, 0, 0, 4, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8198, 8171, 8175, 11622, 8159, 4798, 8226] avg=8192.0000 max=11622 max_rank=4 min=4798 min_rank=6 imbalance_ratio=1.418701 max_load_reduction=53914 max_load_reduction_pct=0.822662 moved_to_echo=32805 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 128, 128, 0, 128, 128, 0, 0, 0, 128, 0, 0, 0, 36736, 28288, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[37120, 0, 0, 0, 128, 0, 0, 0, 128, 0, 28544, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 38272, 0, 128, 0, 0, 0, 128, 0, 27392, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 128, 0, 128, 128, 0, 0, 0, 128, 0, 0, 0, 37888, 27136, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 65408, 0, 0, 128, 0, 128, 0, 0, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 128, 0, 0, 0, 128, 0, 0, 128, 128, 0, 0, 128, 65280, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 65408, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 64640, 0, 128, 128, 128, 128, 128, 128, 0, 128, 0, 128, 128, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 0, 0, 65408, 0, 0, 128, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 65408, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 128, 128, 0, 0, 0, 128, 0, 128, 128, 128, 0, 0, 128, 65280, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 65408, 0, 0, 0, 0, 128, 0, 0, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 1795, 0, 8191, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 6308, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 43, 0, 8192, 0, 17, 0, 0, 80, 0, 0, 0, 0, 0, 8112, 0, 0, 0, 0, 8192, 0, 8149, 0, 0, 8176, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 6, 8158, 0, 0, 0, 0, 30, 0, 0, 1045, 0, 7727, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 8083, 0, 127, 0, 0, 8160, 2, 7010, 0, 0, 8136, 0, 0, 0, 16, 7941, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 14, 1, 0, 0, 6, 19, 0, 2, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8, 12, 8170, 0, 522, 21, 0, 11, 0, 0, 0, 0, 238, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 84, 0, 8192, 0, 2, 0, 0, 30, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 8192, 0, 8108, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 613, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 8192, 0, 7274, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8, 8179, 0, 0, 0, 0, 0, 0, 0, 203, 225, 8171, 0, 14, 0, 0, 0, 4, 0, 2, 0, 0, 7845, 0, 1, 0, 0, 8182, 0, 8158, 0, 0, 8105, 0, 8, 0, 0, 8111, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2, 0, 0, 0, 1, 0, 0, 6, 0, 0, 8178, 0, 0, 10, 0, 9, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 5, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 17, 0, 8192, 0, 176, 0, 0, 64, 0, 0, 0, 0, 0, 8009, 0, 0, 0, 0, 8192, 0, 8180, 0, 0, 8130, 5, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 52, 0, 8192, 0, 11, 0, 0, 9, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 8192, 0, 8136, 0, 0, 8190, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 7684, 0, 17, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7546, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 7824, 8658, 8223, 8190, 8053, 8199, 8195] avg=8192.0000 max=8658 max_rank=2 min=7824 min_rank=1 imbalance_ratio=1.056885 max_load_reduction=56878 max_load_reduction_pct=0.867889 moved_to_echo=32637 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 6, 8158, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 1045, 0, 7248, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 0, 127, 0, 0, 0, 2, 7010, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 14, 1, 0, 0, 6, 19, 7915, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8, 12, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 21, 0, 11, 0, 0, 0, 0, 238, 0, 0, 0, 0, 0, 2, 7519, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 8160, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8331, 7662, 8034, 8204, 8791, 8166, 8170] avg=8192.0000 max=8791 max_rank=5 min=7662 min_rank=2 imbalance_ratio=1.073120 max_load_reduction=56745 max_load_reduction_pct=0.865860 moved_to_echo=33331 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8, 8179, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 203, 225, 7664, 0, 14, 0, 0, 0, 4, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 0, 1, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 8086, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2, 0, 0, 0, 1, 0, 0, 6, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 9, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 7298, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 5, 0, 20, 0, 0, 8102, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 8182, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8112, 8659, 8167, 8194, 7857, 8160, 8194] avg=8192.0000 max=8659 max_rank=2 min=7857 min_rank=5 imbalance_ratio=1.057007 max_load_reduction=56877 max_load_reduction_pct=0.867874 moved_to_echo=32405 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1795, 0, 7683, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 0, 0, 0, 0, 0, 0, 6308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7621, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 9480, 6822, 8223, 8190, 8216, 8215, 8195] avg=8192.0000 max=9480 max_rank=1 min=6822 min_rank=2 imbalance_ratio=1.157227 max_load_reduction=56056 max_load_reduction_pct=0.855347 moved_to_echo=32816 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 7684, 0, 2, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 0, 0, 0, 0, 0, 0, 8108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7593, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 7800, 8620, 8223, 8190, 8100, 8215, 8195] avg=8192.0000 max=8620 max_rank=2 min=7800 min_rank=1 imbalance_ratio=1.052246 max_load_reduction=56916 max_load_reduction_pct=0.868469 moved_to_echo=32700 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 7684, 0, 176, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7451, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8126, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 7941, 8687, 8215, 8191, 7958, 8154, 8197] avg=8192.0000 max=8687 max_rank=2 min=7941 min_rank=1 imbalance_ratio=1.060425 max_load_reduction=56849 max_load_reduction_pct=0.867447 moved_to_echo=32500 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 7684, 0, 11, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 0, 0, 0, 0, 0, 0, 8136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 7607, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 7756, 8649, 8223, 8190, 8120, 8213, 8195] avg=8192.0000 max=8649 max_rank=2 min=7756 min_rank=1 imbalance_ratio=1.055786 max_load_reduction=56887 max_load_reduction_pct=0.868027 moved_to_echo=32718 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 613, 0, 7679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 0, 0, 0, 0, 0, 0, 7274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 7617, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8292, 7779, 8228, 8187, 8441, 8214, 8195] avg=8192.0000 max=8441 max_rank=5 min=7779 min_rank=2 imbalance_ratio=1.030396 max_load_reduction=57095 max_load_reduction_pct=0.871201 moved_to_echo=33037 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 7, 0, 8192, 0, 17, 0, 0, 83, 0, 0, 0, 0, 0, 8109, 0, 0, 0, 0, 8192, 0, 8185, 0, 0, 8176, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 50, 0, 8192, 0, 2, 0, 0, 29, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 8192, 0, 8142, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 6, 8158, 0, 0, 0, 0, 30, 0, 0, 910, 0, 7740, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 8078, 0, 122, 0, 0, 8161, 3, 7136, 0, 0, 8137, 0, 0, 0, 17, 7943, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 15, 1, 0, 0, 6, 18, 0, 2, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8, 12, 8170, 0, 523, 21, 0, 10, 0, 0, 0, 0, 239, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8, 8178, 0, 0, 0, 0, 0, 0, 0, 215, 226, 8171, 0, 15, 0, 0, 0, 4, 0, 2, 0, 0, 7833, 0, 2, 0, 0, 8182, 0, 8157, 0, 0, 8102, 0, 8, 0, 0, 8113, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 8178, 0, 0, 10, 0, 9, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 6, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 9, 1, 8192, 0, 177, 0, 0, 66, 0, 0, 0, 0, 0, 8007, 0, 0, 0, 0, 8192, 0, 8188, 0, 0, 8128, 6, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 789, 0, 8191, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 7164, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 23, 0, 8192, 0, 12, 0, 0, 9, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 8192, 0, 8169, 0, 0, 8191, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 314, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 8192, 0, 7685, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 7875, 0, 17, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 320, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7732, 316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 7982, 8505, 8223, 8190, 8048, 8198, 8196] avg=8192.0000 max=8505 max_rank=2 min=7982 min_rank=1 imbalance_ratio=1.038208 max_load_reduction=57031 max_load_reduction_pct=0.870224 moved_to_echo=32632 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 7875, 0, 2, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 323, 0, 0, 0, 0, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7783, 316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 7956, 8465, 8223, 8190, 8099, 8215, 8195] avg=8192.0000 max=8465 max_rank=2 min=7956 min_rank=1 imbalance_ratio=1.033325 max_load_reduction=57071 max_load_reduction_pct=0.870834 moved_to_echo=32699 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 7875, 0, 177, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 317, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7634, 316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8125, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8128, 8511, 8214, 8192, 7950, 8154, 8196] avg=8192.0000 max=8511 max_rank=2 min=7950 min_rank=5 imbalance_ratio=1.038940 max_load_reduction=57025 max_load_reduction_pct=0.870132 moved_to_echo=32492 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 6, 8158, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 910, 0, 7441, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 320, 0, 122, 0, 0, 0, 3, 7136, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 15, 1, 0, 0, 6, 18, 7917, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8, 12, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 21, 0, 10, 0, 0, 0, 0, 239, 0, 0, 0, 0, 0, 1, 7702, 298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8134, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 8161, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8389, 7598, 8035, 8205, 8794, 8168, 8170] avg=8192.0000 max=8794 max_rank=5 min=7598 min_rank=2 imbalance_ratio=1.073486 max_load_reduction=56742 max_load_reduction_pct=0.865814 moved_to_echo=33337 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 5, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 314, 0, 7866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 316, 0, 0, 0, 0, 0, 0, 7685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 7806, 318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8180, 8001, 8227, 8187, 8326, 8215, 8195] avg=8192.0000 max=8326 max_rank=5 min=8001 min_rank=2 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32923 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 7875, 0, 12, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 323, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7796, 316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 7919, 8492, 8223, 8190, 8113, 8214, 8195] avg=8192.0000 max=8492 max_rank=2 min=7919 min_rank=1 imbalance_ratio=1.036621 max_load_reduction=57044 max_load_reduction_pct=0.870422 moved_to_echo=32712 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 789, 0, 7874, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 324, 0, 0, 0, 0, 0, 0, 7164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7811, 316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8665, 7488, 8223, 8190, 8365, 8215, 8195] avg=8192.0000 max=8665 max_rank=1 min=7488 min_rank=2 imbalance_ratio=1.057739 max_load_reduction=56871 max_load_reduction_pct=0.867783 moved_to_echo=32965 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8, 8178, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 215, 226, 7855, 0, 15, 0, 0, 0, 4, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 309, 0, 2, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 8087, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 9, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 7469, 315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 6, 0, 18, 0, 0, 8100, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 8182, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8317, 8476, 8168, 8192, 7841, 8157, 8194] avg=8192.0000 max=8476 max_rank=2 min=7841 min_rank=5 imbalance_ratio=1.034668 max_load_reduction=57060 max_load_reduction_pct=0.870667 moved_to_echo=32384 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 1024, 128, 0, 128, 0, 0, 0, 0, 384, 0, 128, 0, 0, 0, 128, 60288, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4096, 0, 128, 0, 0, 0, 128, 61440, 0, 0, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 3968, 256, 61056, 0, 256, 0, 0, 256, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 1024, 128, 0, 128, 0, 0, 0, 0, 384, 0, 128, 0, 0, 0, 128, 61824, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[2560, 0, 128, 0, 0, 0, 128, 62848, 0, 0, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 2432, 256, 62592, 0, 256, 0, 0, 256, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 128, 0, 128, 0, 128, 65408, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 0, 128, 128, 0, 0, 0, 128, 0, 0, 128, 128, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 128, 128, 65024, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 128, 128, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 65536, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 128, 0, 128, 0, 128, 65408, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 128, 128, 65024, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True - [2026-08-26 14:48:12] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 438.9 | throughput per GPU (TFLOP/s/GPU): 412.0 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.353778E+01 | loss scale: 1.0 | grad norm: 121.538 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 31, 719, 34, 0, 0, 8152, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 552, 0, 0, 3064, 2, 0, 0, 8191, 0, 8191, 3, 2, 0, 0, 15, 0, 0, 0, 0, 0, 9, 0, 19, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 12, 3, 0, 7911, 0, 0, 0, 2, 1, 0, 0, 0, 3802, 0, 0, 0, 3720, 0, 6924, 0, 0, 0, 1907, 0, 0, 0, 0, 0, 0, 0, 0, 5901, 0, 16, 0, 0, 5, 0, 41, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 6, 5808, 0, 0, 7, 0, 0, 0, 64, 0, 1, 0, 1, 2, 0, 0, 0, 1, 405, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 14, 861, 25, 0, 0, 8064, 0, 0, 1, 0, 0, 0, 2, 1, 1, 2, 539, 0, 0, 2318, 8, 0, 0, 8190, 3, 8190, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 23, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 1, 0, 0, 0, 7889, 1, 0, 0, 0, 0, 1, 0, 0, 2914, 0, 0, 0, 4072, 0, 6660, 1, 0, 0, 2081, 0, 0, 0, 0, 0, 0, 0, 0, 6273, 0, 13, 0, 0, 8, 0, 3, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 4, 6937, 0, 0, 2, 0, 0, 1, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 326, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 47, 741, 42, 0, 0, 8061, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 536, 0, 0, 2626, 34, 0, 0, 8192, 1, 8192, 0, 14, 0, 1, 7, 1, 1, 1, 0, 0, 3, 0, 54, 2, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5, 0, 0, 7813, 0, 0, 0, 2, 0, 0, 0, 0, 3700, 0, 0, 0, 4116, 0, 6722, 1, 0, 0, 2130, 0, 0, 0, 0, 0, 3, 0, 0, 6003, 0, 3, 0, 1, 5, 0, 8, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 19, 5878, 0, 0, 3, 4, 0, 0, 112, 0, 0, 0, 3, 7, 0, 0, 1, 0, 413, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 30, 1066, 33, 0, 0, 8070, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0, 489, 0, 0, 3269, 7, 2, 0, 8191, 4, 8190, 0, 3, 0, 0, 6, 0, 0, 4, 0, 1, 1, 0, 14, 0, 0, 0, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 0, 7887, 0, 0, 0, 0, 0, 0, 0, 0, 3513, 0, 0, 2, 3588, 0, 6534, 2, 2, 0, 1922, 0, 0, 0, 0, 0, 1, 0, 0, 6324, 0, 1, 0, 0, 0, 2, 20, 0, 0, 7, 1, 1, 0, 0, 0, 0, 0, 14, 5700, 0, 0, 4, 7, 0, 0, 211, 0, 0, 0, 0, 0, 0, 0, 1, 0, 391, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 73, 575, 47, 0, 1, 8095, 0, 0, 0, 0, 3, 5, 0, 1, 0, 1, 560, 0, 0, 3422, 3, 1, 0, 8191, 13, 8190, 0, 1, 0, 0, 11, 0, 4, 1, 0, 3, 0, 0, 91, 4, 0, 0, 11, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 8016, 0, 0, 0, 0, 0, 0, 0, 0, 3687, 0, 3, 0, 3204, 0, 7173, 7, 0, 0, 1803, 0, 0, 1, 0, 0, 4, 0, 0, 5792, 0, 64, 0, 0, 19, 0, 69, 0, 0, 47, 0, 0, 0, 1, 0, 0, 0, 21, 5890, 0, 0, 10, 0, 0, 0, 27, 0, 0, 0, 9, 0, 1, 0, 0, 0, 372, 3, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 25, 514, 45, 0, 0, 7993, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 441, 0, 0, 3720, 7, 1, 0, 8185, 4, 8184, 0, 0, 1, 3, 5, 0, 1, 0, 0, 0, 0, 2, 5, 0, 0, 0, 1, 0, 0, 5, 23, 0, 0, 0, 3, 0, 4, 5, 0, 8028, 0, 2, 1, 0, 2, 0, 0, 1, 3461, 0, 0, 0, 3542, 2, 6724, 8, 0, 3, 1607, 0, 0, 2, 0, 2, 10, 0, 1, 5711, 1, 2, 0, 0, 4, 0, 10, 0, 1, 37, 3, 0, 0, 0, 0, 0, 1, 3, 6648, 1, 0, 8, 3, 2, 0, 131, 0, 2, 0, 6, 1, 0, 0, 0, 0, 384, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 14, 65, 578, 6, 0, 0, 8094, 0, 0, 0, 2, 1, 4, 0, 1, 0, 1, 576, 0, 0, 1937, 15, 0, 0, 8191, 4, 8190, 0, 5, 1, 0, 6, 0, 2, 2, 0, 1, 0, 0, 8, 1, 0, 1, 0, 0, 0, 3, 4, 1, 0, 0, 1, 0, 2, 0, 0, 7816, 0, 0, 0, 1, 1, 0, 0, 0, 3865, 0, 0, 0, 4291, 0, 6790, 7, 0, 0, 2326, 0, 1, 0, 0, 0, 1, 0, 0, 5716, 0, 17, 0, 0, 0, 0, 6, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 22, 6513, 0, 0, 3, 0, 0, 0, 23, 0, 2, 0, 0, 0, 0, 0, 1, 0, 411, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[2, 1, 0, 3, 19, 620, 23, 0, 0, 8053, 0, 0, 0, 1, 0, 5, 0, 0, 0, 1, 474, 1, 2, 2674, 12, 0, 0, 8192, 2, 8191, 0, 19, 0, 0, 16, 1, 1, 0, 0, 6, 1, 0, 28, 3, 0, 0, 2, 1, 0, 1, 3, 0, 0, 3, 3, 0, 0, 0, 0, 7835, 0, 0, 0, 3, 2, 0, 0, 0, 3629, 0, 2, 0, 3756, 1, 6294, 3, 1, 0, 2759, 0, 0, 2, 0, 0, 1, 0, 0, 6369, 0, 33, 0, 1, 2, 2, 2, 0, 0, 10, 0, 0, 0, 2, 0, 0, 0, 40, 6029, 0, 0, 2, 1, 0, 0, 31, 0, 0, 0, 1, 1, 0, 0, 0, 2, 356, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 47, 741, 42, 0, 0, 7394, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 536, 0, 0, 2626, 34, 0, 0, 0, 1, 4772, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7, 1, 1, 1, 0, 0, 3, 0, 54, 2, 0, 0, 1, 0, 8144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5, 0, 0, 7813, 0, 0, 0, 2, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3700, 0, 0, 0, 2770, 0, 0, 1, 0, 0, 2130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 6003, 0, 3, 0, 1, 5, 0, 8, 0, 2132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 19, 5878, 0, 0, 3, 4, 0, 1346, 389, 346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 0, 0, 3, 7, 0, 0, 1, 0, 413, 0, 0, 0, 0, 6722, 942, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8233, 7985, 8215, 8099, 8601, 8155, 8000, 8248] avg=8192.0000 max=8601 max_rank=4 min=7985 min_rank=1 imbalance_ratio=1.049927 max_load_reduction=56935 max_load_reduction_pct=0.868759 moved_to_echo=33004 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 14, 861, 25, 0, 0, 7397, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 2, 539, 0, 0, 2318, 8, 0, 0, 0, 3, 4770, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 23, 0, 0, 0, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 1, 0, 0, 0, 7889, 1, 0, 0, 0, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2914, 0, 0, 0, 2740, 0, 0, 1, 0, 0, 2081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6273, 0, 13, 0, 0, 8, 0, 3, 0, 2132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 4, 6937, 0, 0, 2, 0, 0, 1332, 389, 346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 326, 0, 0, 0, 0, 6660, 942, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8298, 7645, 8167, 8172, 7737, 8429, 9023, 8065] avg=8192.0000 max=9023 max_rank=6 min=7645 min_rank=1 imbalance_ratio=1.101440 max_load_reduction=56513 max_load_reduction_pct=0.862320 moved_to_echo=33254 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 31, 719, 34, 0, 0, 7478, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 552, 0, 0, 3064, 2, 0, 0, 0, 0, 4771, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 9, 0, 19, 0, 0, 0, 0, 0, 8143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 12, 3, 0, 7911, 0, 0, 0, 2, 281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3802, 0, 0, 0, 2503, 0, 0, 0, 0, 0, 1907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5901, 0, 16, 0, 0, 5, 0, 41, 1, 2132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 6, 5808, 0, 0, 7, 0, 0, 1217, 393, 346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 1, 0, 1, 2, 0, 0, 0, 1, 405, 0, 0, 0, 0, 6924, 942, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8266, 8395, 8186, 8210, 8213, 8096, 7782, 8388] avg=8192.0000 max=8395 max_rank=1 min=7782 min_rank=6 imbalance_ratio=1.024780 max_load_reduction=57141 max_load_reduction_pct=0.871902 moved_to_echo=32479 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 25, 514, 45, 0, 0, 7332, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 441, 0, 0, 3720, 7, 1, 0, 0, 4, 4768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 5, 0, 1, 0, 0, 0, 0, 2, 5, 0, 0, 0, 1, 0, 8137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 23, 0, 0, 0, 3, 0, 4, 5, 0, 8028, 0, 2, 1, 0, 276, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 3461, 0, 0, 0, 2383, 2, 0, 8, 0, 3, 1607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 10, 0, 1, 5711, 1, 2, 0, 0, 4, 0, 10, 0, 2130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 37, 3, 0, 0, 0, 0, 0, 1, 3, 6648, 1, 0, 8, 3, 2, 1159, 385, 345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 0, 2, 0, 6, 1, 0, 0, 0, 0, 384, 0, 0, 0, 0, 6724, 941, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7920, 8941, 8155, 8347, 7467, 7873, 8596, 8237] avg=8192.0000 max=8941 max_rank=1 min=7467 min_rank=4 imbalance_ratio=1.091431 max_load_reduction=56595 max_load_reduction_pct=0.863571 moved_to_echo=32173 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 73, 575, 47, 0, 1, 7426, 0, 0, 0, 0, 3, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 560, 0, 0, 3422, 3, 1, 0, 0, 13, 4770, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 4, 1, 0, 3, 0, 0, 91, 4, 0, 0, 11, 1, 8143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 8016, 0, 0, 0, 0, 279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3687, 0, 3, 0, 2156, 0, 0, 7, 0, 0, 1803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 0, 0, 5792, 0, 64, 0, 0, 19, 0, 69, 0, 2132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 1, 0, 0, 0, 21, 5890, 0, 0, 10, 0, 0, 1048, 390, 346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 9, 0, 1, 0, 0, 0, 372, 3, 0, 0, 0, 7173, 942, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8131, 8772, 8269, 8299, 7656, 8081, 7753, 8575] avg=8192.0000 max=8772 max_rank=1 min=7656 min_rank=4 imbalance_ratio=1.070801 max_load_reduction=56764 max_load_reduction_pct=0.866150 moved_to_echo=32065 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 14, 65, 578, 6, 0, 0, 7425, 0, 0, 0, 2, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 576, 0, 0, 1937, 15, 0, 0, 0, 4, 4770, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 0, 2, 2, 0, 1, 0, 0, 8, 1, 0, 1, 0, 0, 8143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 1, 0, 0, 1, 0, 2, 0, 0, 7816, 0, 0, 0, 1, 279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3865, 0, 0, 0, 2887, 0, 0, 7, 0, 0, 2326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 5716, 0, 17, 0, 0, 0, 0, 6, 0, 2132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 22, 6513, 0, 0, 3, 0, 0, 1404, 390, 346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 2, 0, 0, 0, 0, 0, 1, 0, 411, 0, 0, 0, 0, 6790, 942, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8095, 7309, 8165, 8107, 9086, 7873, 8684, 8217] avg=8192.0000 max=9086 max_rank=4 min=7309 min_rank=1 imbalance_ratio=1.109131 max_load_reduction=56450 max_load_reduction_pct=0.861359 moved_to_echo=33860 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2, 1, 0, 3, 19, 620, 23, 0, 0, 7387, 0, 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 474, 1, 2, 2674, 12, 0, 0, 0, 2, 4771, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 1, 1, 0, 0, 6, 1, 0, 28, 3, 0, 0, 2, 1, 8144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 3, 3, 0, 0, 0, 0, 7835, 0, 0, 0, 3, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3629, 0, 2, 0, 2527, 1, 0, 3, 1, 0, 2759, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 6369, 0, 33, 0, 1, 2, 2, 2, 0, 2132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 2, 0, 0, 0, 40, 6029, 0, 0, 2, 1, 0, 1229, 388, 346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 1, 1, 0, 0, 0, 2, 356, 0, 0, 0, 0, 6294, 942, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8061, 7956, 8203, 8126, 8924, 8544, 8047, 7675] avg=8192.0000 max=8924 max_rank=4 min=7675 min_rank=7 imbalance_ratio=1.089355 max_load_reduction=56612 max_load_reduction_pct=0.863831 moved_to_echo=33190 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 30, 1066, 33, 0, 0, 7395, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 0, 0, 3269, 7, 2, 0, 0, 4, 4759, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4, 0, 1, 1, 0, 14, 0, 0, 0, 2, 4, 8144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 0, 7887, 0, 0, 0, 0, 282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3513, 0, 0, 2, 2411, 0, 0, 2, 2, 0, 1922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6324, 0, 1, 0, 0, 0, 2, 20, 0, 2137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1, 1, 0, 0, 0, 0, 0, 14, 5700, 0, 0, 4, 7, 0, 1177, 393, 347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 211, 0, 0, 0, 0, 0, 0, 0, 1, 0, 391, 0, 0, 0, 0, 6534, 947, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8532, 8533, 8176, 8176, 7852, 8485, 7651, 8131] avg=8192.0000 max=8533 max_rank=1 min=7651 min_rank=6 imbalance_ratio=1.041626 max_load_reduction=57003 max_load_reduction_pct=0.869797 moved_to_echo=32119 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True + [2026-08-26 15:24:12] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 446.8 | throughput per GPU (TFLOP/s/GPU): 404.7 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.354405E+01 | loss scale: 1.0 | grad norm: 121.424 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 42, 597, 51, 0, 0, 8010, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 465, 0, 0, 4153, 7, 1, 0, 8185, 4, 8184, 0, 0, 1, 3, 3, 0, 1, 0, 0, 0, 0, 2, 8, 0, 0, 0, 1, 0, 0, 5, 25, 0, 0, 0, 3, 0, 4, 5, 0, 8064, 0, 2, 1, 0, 2, 0, 0, 1, 3666, 0, 0, 0, 3756, 2, 6789, 7, 0, 3, 1778, 0, 0, 2, 0, 2, 11, 0, 1, 4260, 1, 10, 0, 0, 3, 0, 14, 0, 1, 38, 3, 0, 0, 0, 0, 0, 1, 4, 6788, 1, 0, 8, 3, 2, 0, 144, 0, 1, 0, 6, 1, 0, 0, 0, 0, 395, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 58, 835, 68, 0, 0, 8094, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 567, 0, 0, 2891, 32, 0, 0, 8192, 1, 8192, 0, 16, 0, 1, 7, 1, 1, 1, 0, 0, 2, 0, 60, 2, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5, 0, 0, 7908, 0, 0, 0, 2, 0, 0, 0, 0, 3893, 0, 0, 0, 4305, 0, 6770, 1, 0, 0, 2321, 0, 0, 0, 0, 0, 4, 0, 0, 4620, 0, 14, 0, 1, 4, 0, 8, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 18, 6063, 0, 0, 3, 4, 0, 0, 116, 0, 0, 0, 3, 7, 0, 0, 1, 0, 415, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 14, 94, 653, 7, 0, 0, 8132, 0, 0, 0, 2, 1, 5, 0, 1, 0, 1, 693, 0, 0, 2143, 13, 0, 0, 8191, 5, 8190, 1, 5, 1, 0, 6, 0, 1, 2, 0, 1, 2, 0, 11, 1, 0, 1, 0, 0, 0, 3, 4, 1, 0, 0, 1, 0, 2, 0, 0, 7891, 0, 0, 0, 2, 0, 0, 0, 0, 4086, 0, 0, 0, 4463, 0, 6824, 7, 0, 0, 2503, 0, 1, 0, 0, 0, 0, 0, 0, 4424, 0, 26, 0, 0, 0, 0, 6, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 21, 6642, 0, 0, 3, 0, 0, 0, 24, 0, 3, 0, 0, 0, 0, 0, 1, 0, 413, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 47, 815, 37, 0, 0, 8153, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 653, 0, 0, 3416, 2, 0, 0, 8191, 0, 8191, 3, 2, 0, 0, 12, 0, 0, 0, 0, 0, 10, 0, 20, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 11, 3, 0, 7990, 0, 0, 0, 2, 1, 0, 0, 0, 3993, 0, 0, 0, 3948, 0, 6959, 0, 0, 0, 2078, 0, 0, 0, 0, 0, 0, 0, 0, 4396, 0, 33, 0, 0, 6, 0, 43, 2, 0, 6, 0, 0, 0, 0, 0, 0, 0, 8, 6008, 0, 0, 6, 0, 0, 0, 70, 0, 1, 0, 1, 2, 0, 0, 0, 1, 410, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 25, 955, 18, 0, 0, 8106, 0, 0, 1, 0, 0, 0, 2, 1, 1, 2, 585, 0, 0, 2619, 7, 0, 0, 8190, 3, 8190, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 22, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 1, 0, 0, 0, 7957, 1, 0, 0, 0, 0, 1, 0, 0, 3098, 0, 0, 0, 4245, 0, 6690, 2, 0, 0, 2264, 0, 0, 0, 0, 0, 0, 0, 0, 5059, 0, 22, 0, 0, 6, 0, 3, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 4, 7004, 0, 0, 0, 0, 0, 1, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 334, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[2, 1, 0, 2, 32, 686, 24, 0, 0, 8095, 0, 0, 0, 1, 0, 5, 0, 0, 0, 1, 503, 1, 3, 2950, 10, 0, 0, 8192, 2, 8191, 1, 19, 0, 1, 12, 1, 1, 0, 0, 6, 1, 0, 32, 3, 0, 0, 2, 1, 0, 1, 3, 0, 0, 3, 3, 0, 0, 0, 0, 7918, 0, 0, 0, 3, 3, 0, 0, 0, 3776, 0, 2, 0, 3909, 1, 6410, 4, 1, 0, 2921, 0, 0, 2, 0, 0, 1, 0, 0, 5073, 0, 70, 0, 1, 2, 2, 1, 0, 0, 10, 0, 0, 0, 2, 0, 0, 0, 45, 6178, 0, 0, 3, 1, 0, 0, 42, 0, 0, 0, 1, 1, 0, 0, 0, 2, 360, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 43, 1165, 41, 0, 0, 8083, 0, 0, 0, 0, 2, 4, 0, 0, 0, 0, 516, 0, 0, 3591, 6, 2, 0, 8191, 3, 8190, 0, 4, 0, 0, 6, 0, 0, 4, 0, 1, 1, 0, 20, 0, 0, 0, 2, 4, 1, 0, 0, 0, 0, 1, 0, 0, 4, 3, 0, 7940, 0, 0, 0, 0, 0, 0, 0, 0, 3675, 0, 0, 2, 3702, 0, 6566, 3, 2, 0, 2025, 0, 0, 0, 0, 0, 1, 0, 0, 5160, 0, 1, 0, 0, 0, 2, 21, 0, 0, 8, 1, 0, 0, 0, 0, 0, 0, 14, 5894, 0, 0, 4, 7, 0, 0, 230, 0, 0, 0, 0, 1, 0, 0, 1, 0, 387, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 119, 669, 49, 0, 1, 8104, 0, 0, 0, 0, 3, 5, 0, 1, 0, 1, 609, 0, 0, 3802, 3, 1, 0, 8191, 13, 8190, 0, 2, 0, 0, 9, 0, 4, 1, 0, 3, 0, 0, 108, 3, 0, 0, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 8053, 0, 0, 0, 0, 0, 0, 0, 0, 3824, 0, 3, 0, 3342, 0, 7181, 9, 0, 0, 1957, 0, 0, 1, 0, 0, 3, 0, 0, 4462, 0, 97, 0, 0, 17, 0, 76, 0, 0, 54, 0, 0, 0, 1, 0, 0, 0, 23, 6087, 0, 0, 11, 0, 0, 0, 32, 0, 0, 0, 9, 0, 1, 0, 0, 0, 385, 3, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 42, 597, 51, 0, 0, 7214, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 0, 0, 4153, 7, 1, 0, 0, 4, 4397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 3, 0, 1, 0, 0, 0, 0, 2, 8, 0, 0, 0, 1, 0, 8135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 25, 0, 0, 0, 3, 0, 4, 5, 0, 8064, 0, 2, 1, 0, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 3666, 0, 0, 0, 2091, 2, 0, 7, 0, 3, 1778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 11, 0, 1, 4260, 1, 10, 0, 0, 3, 0, 14, 0, 2908, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 38, 3, 0, 0, 0, 0, 0, 1, 4, 6788, 1, 0, 8, 3, 2, 1159, 586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 1, 0, 6, 1, 0, 0, 0, 0, 395, 0, 0, 0, 0, 6789, 879, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7909, 9027, 8154, 8319, 7550, 7718, 8594, 8265] avg=8192.0000 max=9027 max_rank=1 min=7550 min_rank=4 imbalance_ratio=1.101929 max_load_reduction=56509 max_load_reduction_pct=0.862259 moved_to_echo=32127 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 58, 835, 68, 0, 0, 7290, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 567, 0, 0, 2891, 32, 0, 0, 0, 1, 4402, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7, 1, 1, 1, 0, 0, 2, 0, 60, 2, 0, 0, 1, 0, 8140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5, 0, 0, 7908, 0, 0, 0, 2, 212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3893, 0, 0, 0, 2397, 0, 0, 1, 0, 0, 2321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4620, 0, 14, 0, 1, 4, 0, 8, 0, 2911, 580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 18, 6063, 0, 0, 3, 4, 0, 1328, 592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 0, 0, 0, 3, 7, 0, 0, 1, 0, 415, 0, 0, 0, 0, 6770, 879, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8260, 7911, 8216, 8128, 8612, 8142, 8024, 8243] avg=8192.0000 max=8612 max_rank=4 min=7911 min_rank=1 imbalance_ratio=1.051270 max_load_reduction=56924 max_load_reduction_pct=0.868591 moved_to_echo=33021 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 25, 955, 18, 0, 0, 7301, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 2, 585, 0, 0, 2619, 7, 0, 0, 0, 3, 4401, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 22, 0, 0, 0, 0, 0, 8139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 1, 0, 0, 0, 7957, 1, 0, 0, 0, 212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3098, 0, 0, 0, 2363, 0, 0, 2, 0, 0, 2264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5059, 0, 22, 0, 0, 6, 0, 3, 0, 2910, 572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 4, 7004, 0, 0, 0, 0, 0, 1310, 593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 334, 0, 0, 0, 0, 6690, 879, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8300, 7622, 8162, 8174, 7728, 8572, 8927, 8051] avg=8192.0000 max=8927 max_rank=6 min=7622 min_rank=1 imbalance_ratio=1.089722 max_load_reduction=56609 max_load_reduction_pct=0.863785 moved_to_echo=33278 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 43, 1165, 41, 0, 0, 7273, 0, 0, 0, 0, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 0, 0, 3591, 6, 2, 0, 0, 3, 4389, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4, 0, 1, 1, 0, 20, 0, 0, 0, 2, 4, 8140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 4, 3, 0, 7940, 0, 0, 0, 0, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3675, 0, 0, 2, 2052, 0, 0, 3, 2, 0, 2025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5160, 0, 1, 0, 0, 0, 2, 21, 0, 2916, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 0, 0, 0, 0, 0, 0, 14, 5894, 0, 0, 4, 7, 0, 1146, 594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 230, 0, 0, 0, 0, 1, 0, 0, 1, 0, 387, 0, 0, 0, 0, 6566, 885, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8529, 8511, 8178, 8165, 7759, 8605, 7668, 8121] avg=8192.0000 max=8605 max_rank=5 min=7668 min_rank=6 imbalance_ratio=1.050415 max_load_reduction=56931 max_load_reduction_pct=0.868698 moved_to_echo=32153 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 47, 815, 37, 0, 0, 7343, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 653, 0, 0, 3416, 2, 0, 0, 0, 0, 4401, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 10, 0, 20, 0, 0, 0, 0, 0, 8140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 11, 3, 0, 7990, 0, 0, 0, 2, 214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3993, 0, 0, 0, 2198, 0, 0, 0, 0, 0, 2078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4396, 0, 33, 0, 0, 6, 0, 43, 2, 2911, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 8, 6008, 0, 0, 6, 0, 0, 1218, 596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 1, 0, 1, 2, 0, 0, 0, 1, 410, 0, 0, 0, 0, 6959, 879, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8246, 8478, 8182, 8221, 8270, 7923, 7842, 8374] avg=8192.0000 max=8478 max_rank=1 min=7842 min_rank=6 imbalance_ratio=1.034912 max_load_reduction=57058 max_load_reduction_pct=0.870636 moved_to_echo=32409 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 119, 669, 49, 0, 1, 7300, 0, 0, 0, 0, 3, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 609, 0, 0, 3802, 3, 1, 0, 0, 13, 4401, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4, 1, 0, 3, 0, 0, 108, 3, 0, 0, 8, 1, 8140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 8053, 0, 0, 0, 0, 212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3824, 0, 3, 0, 1860, 0, 0, 9, 0, 0, 1957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 4462, 0, 97, 0, 0, 17, 0, 76, 0, 2910, 451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 1, 0, 0, 0, 23, 6087, 0, 0, 11, 0, 0, 1031, 592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 9, 0, 1, 0, 0, 0, 385, 3, 0, 0, 0, 7181, 879, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8147, 8833, 8277, 8269, 7653, 8017, 7799, 8541] avg=8192.0000 max=8833 max_rank=1 min=7653 min_rank=4 imbalance_ratio=1.078247 max_load_reduction=56703 max_load_reduction_pct=0.865219 moved_to_echo=32010 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 14, 94, 653, 7, 0, 0, 7325, 0, 0, 0, 2, 1, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 693, 0, 0, 2143, 13, 0, 0, 0, 5, 4401, 1, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 0, 1, 2, 0, 1, 2, 0, 11, 1, 0, 1, 0, 0, 8140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 1, 0, 0, 1, 0, 2, 0, 0, 7891, 0, 0, 0, 2, 213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 2484, 0, 0, 7, 0, 0, 2503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4424, 0, 26, 0, 0, 0, 0, 6, 0, 2910, 602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 21, 6642, 0, 0, 3, 0, 0, 1377, 594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 3, 0, 0, 0, 0, 0, 1, 0, 413, 0, 0, 0, 0, 6824, 879, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8101, 7263, 8166, 8117, 9080, 7969, 8645, 8195] avg=8192.0000 max=9080 max_rank=4 min=7263 min_rank=1 imbalance_ratio=1.108398 max_load_reduction=56456 max_load_reduction_pct=0.861450 moved_to_echo=33889 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2, 1, 0, 2, 32, 686, 24, 0, 0, 7291, 0, 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 503, 1, 3, 2950, 10, 0, 0, 0, 2, 4401, 1, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 12, 1, 1, 0, 0, 6, 1, 0, 32, 3, 0, 0, 2, 1, 8141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 3, 3, 0, 0, 0, 0, 7918, 0, 0, 0, 3, 212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3776, 0, 2, 0, 2176, 1, 0, 4, 1, 0, 2921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 5073, 0, 70, 0, 1, 2, 2, 1, 0, 2911, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 2, 0, 0, 0, 45, 6178, 0, 0, 3, 1, 0, 1206, 592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 1, 1, 0, 0, 0, 2, 360, 0, 0, 0, 0, 6410, 879, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8044, 7891, 8201, 8143, 8884, 8590, 8037, 7746] avg=8192.0000 max=8884 max_rank=4 min=7746 min_rank=7 imbalance_ratio=1.084473 max_load_reduction=56652 max_load_reduction_pct=0.864441 moved_to_echo=33257 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 4224, 128, 128, 23040, 128, 128, 0, 0, 128, 38272, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 0, 128, 28672, 0, 128, 128, 20480, 128, 0, 128, 128, 128, 16640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 0, 128, 128, 0, 128, 48128, 128, 256, 0, 128, 128, 128, 256, 128, 17152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 256, 128, 128, 0, 128, 0, 0, 128, 256, 49408, 128, 0, 128, 128, 128, 9984, 3200, 2816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 768, 0, 128, 0, 128, 128, 128, 0, 128, 128, 3072, 128, 0, 0, 0, 53888, 7552, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 384, 5760, 256, 0, 128, 59264, 0, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[0, 128, 128, 128, 0, 128, 128, 128, 128, 128, 128, 63232, 128, 128, 128, 128, 2304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 128, 128, 0, 128, 128, 128, 256, 128, 0, 128, 128, 128, 65152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 0, 128, 128, 0, 128, 37504, 128, 384, 0, 128, 128, 128, 256, 128, 23296, 4352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 4608, 128, 128, 25600, 128, 128, 0, 0, 128, 35200, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 128, 30080, 0, 128, 128, 17664, 128, 0, 128, 128, 128, 17920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 256, 128, 0, 0, 128, 0, 0, 128, 256, 50688, 128, 0, 128, 128, 128, 9856, 4864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 768, 0, 128, 0, 128, 128, 128, 0, 128, 128, 3200, 128, 0, 0, 0, 54272, 7040, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 512, 6400, 384, 0, 128, 58368, 0, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=67328 observed_loss=-1792 tokens_per_expert=[128, 128, 128, 128, 0, 128, 128, 128, 128, 128, 128, 63744, 128, 128, 128, 128, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 128, 128, 128, 0, 128, 128, 128, 384, 128, 0, 128, 128, 128, 65152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[8185, 1, 0, 1, 0, 0, 3, 2, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 6, 2, 8186, 8184, 2, 8184, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 8186, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8184, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=67328 slack_slots=63744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8189, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[8191, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8191, 0, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[8191, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8189, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[8185, 1, 0, 1, 0, 0, 3, 2, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 5, 2, 8187, 8184, 2, 8184, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 8186, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 8184, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[8191, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8181, 1, 0, 1, 0, 0, 3, 2, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2, 0, 8184, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8186, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3, 8182, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 1, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8195, 8191, 8193, 8192, 8193, 8188, 8192] avg=8192.0000 max=8195 max_rank=1 min=8188 min_rank=6 imbalance_ratio=1.000366 max_load_reduction=57341 max_load_reduction_pct=0.874954 moved_to_echo=32765 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8188, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8192, 8187, 8192, 8191, 8191, 8192, 8193] avg=8192.0000 max=8198 max_rank=0 min=8187 min_rank=2 imbalance_ratio=1.000732 max_load_reduction=57338 max_load_reduction_pct=0.874908 moved_to_echo=32767 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8187, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8174, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8192, 8195, 8185, 8180, 8192, 8192, 8193] avg=8192.0000 max=8207 max_rank=0 min=8180 min_rank=4 imbalance_ratio=1.001831 max_load_reduction=57329 max_load_reduction_pct=0.874771 moved_to_echo=32757 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8187, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 8186, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8192, 8196, 8191, 8193, 8192, 8192, 8192] avg=8192.0000 max=8196 max_rank=2 min=8188 min_rank=0 imbalance_ratio=1.000488 max_load_reduction=57340 max_load_reduction_pct=0.874939 moved_to_echo=32769 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8188, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8187, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8189, 8194, 8192, 8193, 8192, 8192, 8191] avg=8192.0000 max=8194 max_rank=2 min=8189 min_rank=1 imbalance_ratio=1.000244 max_load_reduction=57342 max_load_reduction_pct=0.874969 moved_to_echo=32768 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8188, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8192, 8197, 8192, 8191, 8192, 8192, 8191] avg=8192.0000 max=8197 max_rank=2 min=8189 min_rank=0 imbalance_ratio=1.000610 max_load_reduction=57339 max_load_reduction_pct=0.874924 moved_to_echo=32766 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8175, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8191, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8192, 8188, 8199, 8200, 8192, 8196, 8191] avg=8192.0000 max=8200 max_rank=4 min=8178 min_rank=0 imbalance_ratio=1.000977 max_load_reduction=57336 max_load_reduction_pct=0.874878 moved_to_echo=32779 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8188, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 8184, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8192, 8188, 8192, 8196, 8192, 8192, 8193] avg=8192.0000 max=8196 max_rank=4 min=8188 min_rank=2 imbalance_ratio=1.000488 max_load_reduction=57340 max_load_reduction_pct=0.874939 moved_to_echo=32773 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8187, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 8185, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8193, 8187, 8193, 8197, 8192, 8192, 8192] avg=8192.0000 max=8197 max_rank=4 min=8187 min_rank=2 imbalance_ratio=1.000610 max_load_reduction=57339 max_load_reduction_pct=0.874924 moved_to_echo=32773 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8181, 1, 0, 1, 0, 0, 3, 2, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 0, 8184, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8186, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3, 8182, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 1, 0, 0, 8184, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8194, 8191, 8192, 8192, 8193, 8189, 8193] avg=8192.0000 max=8194 max_rank=1 min=8189 min_rank=6 imbalance_ratio=1.000244 max_load_reduction=57342 max_load_reduction_pct=0.874969 moved_to_echo=32767 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8187, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8188, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8189, 8192, 8193, 8194, 8192, 8192, 8191] avg=8192.0000 max=8194 max_rank=4 min=8189 min_rank=1 imbalance_ratio=1.000244 max_load_reduction=57342 max_load_reduction_pct=0.874969 moved_to_echo=32769 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8186, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 8187, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8192, 8195, 8192, 8193, 8192, 8192, 8193] avg=8192.0000 max=8195 max_rank=2 min=8187 min_rank=0 imbalance_ratio=1.000366 max_load_reduction=57341 max_load_reduction_pct=0.874954 moved_to_echo=32770 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8187, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8192, 8188, 8193, 8192, 8191, 8192, 8193] avg=8192.0000 max=8195 max_rank=0 min=8188 min_rank=2 imbalance_ratio=1.000366 max_load_reduction=57341 max_load_reduction_pct=0.874954 moved_to_echo=32768 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8187, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8192, 8196, 8193, 8192, 8192, 8192, 8191] avg=8192.0000 max=8196 max_rank=2 min=8188 min_rank=0 imbalance_ratio=1.000488 max_load_reduction=57340 max_load_reduction_pct=0.874939 moved_to_echo=32767 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8182, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8188, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8192, 8191, 8194, 8196, 8192, 8195, 8191] avg=8192.0000 max=8196 max_rank=4 min=8185 min_rank=0 imbalance_ratio=1.000488 max_load_reduction=57340 max_load_reduction_pct=0.874939 moved_to_echo=32774 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8186, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8174, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8192, 8196, 8186, 8180, 8192, 8192, 8192] avg=8192.0000 max=8206 max_rank=0 min=8180 min_rank=4 imbalance_ratio=1.001709 max_load_reduction=57330 max_load_reduction_pct=0.874786 moved_to_echo=32756 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 128, 0, 65536, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 128, 128, 128, 0, 0, 128, 65536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 0, 128, 0, 0, 128, 0, 0, 0, 0, 128, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 65536, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 65536, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[65536, 128, 128, 128, 0, 0, 128, 128, 128, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 128, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 65536, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 0, 0, 0, 128, 0, 128, 0, 0, 0, 128, 128, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 128, 128, 128, 0, 0, 128, 65536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 128, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 128, 128, 0, 65536, 128, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2127, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 0, 8190, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 3, 8181, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8121, 0, 5, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 5, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 853, 0, 0, 0, 5, 8182, 0, 0, 8, 0, 0, 0, 0, 0, 8140, 0, 8180, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 6, 0, 0, 0, 0, 0, 4, 0, 0, 0, 7363, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 6, 0, 0, 0, 0, 0, 0, 8183, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 2622, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 7952, 0, 8183, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5782, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 8176, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1023, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7214, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1250, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8150, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1632, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 7837, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2183, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 7985, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3591, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8190, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4587, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 8181, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8121, 0, 5, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 853, 0, 0, 0, 6260, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 8, 0, 0, 0, 0, 0, 8140, 0, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 6, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 7363, 0, 0, 0, 0, 0, 0, 0, 1800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 6, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8137, 7187, 8273, 8201, 9167, 8176, 8200] avg=8192.0000 max=9167 max_rank=5 min=7187 min_rank=2 imbalance_ratio=1.119019 max_load_reduction=56369 max_load_reduction_pct=0.860123 moved_to_echo=33744 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 2622, 0, 0, 0, 6262, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7952, 0, 120, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5782, 0, 9, 0, 0, 0, 0, 0, 1801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 8178, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8203, 8904, 8076, 8191, 7592, 8191, 8188] avg=8192.0000 max=8904 max_rank=2 min=7592 min_rank=5 imbalance_ratio=1.086914 max_load_reduction=56632 max_load_reduction_pct=0.864136 moved_to_echo=32162 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 1, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1023, 0, 0, 0, 6268, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 0, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7214, 0, 0, 0, 0, 1, 0, 0, 1802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8177, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8188, 7297, 8272, 8190, 9017, 8179, 8195] avg=8192.0000 max=9017 max_rank=5 min=7297 min_rank=2 imbalance_ratio=1.100708 max_load_reduction=56519 max_load_reduction_pct=0.862411 moved_to_echo=33581 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2127, 0, 0, 0, 6268, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 0, 120, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6091, 0, 0, 0, 0, 0, 0, 0, 1802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 8178, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8203, 8399, 8277, 8190, 7893, 8192, 8191] avg=8192.0000 max=8399 max_rank=2 min=7893 min_rank=5 imbalance_ratio=1.025269 max_load_reduction=57137 max_load_reduction_pct=0.871841 moved_to_echo=32466 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2183, 0, 0, 0, 6268, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7985, 0, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6168, 0, 0, 0, 0, 0, 0, 0, 1803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 8178, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8203, 8454, 8105, 8191, 7971, 8228, 8189] avg=8192.0000 max=8454 max_rank=2 min=7971 min_rank=5 imbalance_ratio=1.031982 max_load_reduction=57082 max_load_reduction_pct=0.871002 moved_to_echo=32579 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3591, 0, 0, 0, 6268, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 120, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4587, 0, 2, 0, 0, 0, 0, 0, 1802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 8178, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8203, 9862, 8314, 8191, 6391, 8191, 8189] avg=8192.0000 max=9862 max_rank=2 min=6391 min_rank=5 imbalance_ratio=1.203857 max_load_reduction=55674 max_load_reduction_pct=0.849518 moved_to_echo=30962 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1632, 0, 0, 0, 6268, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7837, 0, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6914, 0, 0, 0, 0, 0, 0, 0, 1803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 8178, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8203, 7904, 7957, 8191, 8717, 8183, 8193] avg=8192.0000 max=8717 max_rank=5 min=7904 min_rank=2 imbalance_ratio=1.064087 max_load_reduction=56819 max_load_reduction_pct=0.866989 moved_to_echo=33284 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1250, 0, 0, 0, 6273, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8150, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6981, 0, 0, 0, 0, 0, 0, 0, 1807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 8182, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8196, 7529, 8262, 8191, 8788, 8196, 8191] avg=8192.0000 max=8788 max_rank=5 min=7529 min_rank=2 imbalance_ratio=1.072754 max_load_reduction=56748 max_load_reduction_pct=0.865906 moved_to_echo=33366 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 3, 8179, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8120, 0, 5, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 5, 0, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 437, 0, 0, 0, 5, 8182, 0, 0, 8, 0, 0, 0, 0, 0, 8174, 0, 8180, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 5, 4, 0, 6, 0, 0, 0, 0, 0, 4, 0, 0, 0, 7746, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1, 6, 0, 0, 0, 0, 0, 0, 8183, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1250, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8095, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 715, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1099, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 8190, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2758, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8190, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5410, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1054, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8088, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 8176, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 478, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7728, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 1555, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8104, 0, 8183, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6695, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 8179, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8120, 0, 5, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 437, 0, 0, 0, 6997, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 8, 0, 0, 0, 0, 0, 8174, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 5, 4, 0, 6, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 7746, 0, 0, 0, 0, 0, 0, 0, 1143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1, 6, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8136, 7515, 8227, 8201, 8893, 8174, 8197] avg=8192.0000 max=8893 max_rank=5 min=7515 min_rank=2 imbalance_ratio=1.085571 max_load_reduction=56643 max_load_reduction_pct=0.864304 moved_to_echo=33465 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1099, 0, 0, 0, 7006, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 39, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7083, 0, 0, 0, 0, 0, 0, 0, 1145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 8175, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8203, 8112, 8230, 8190, 8228, 8191, 8191] avg=8192.0000 max=8230 max_rank=3 min=8112 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=57306 max_load_reduction_pct=0.874420 moved_to_echo=32800 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2758, 0, 0, 0, 7006, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 39, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5410, 0, 1, 0, 0, 0, 0, 0, 1145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 8175, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8203, 9771, 8233, 8191, 6556, 8198, 8189] avg=8192.0000 max=9771 max_rank=2 min=6556 min_rank=5 imbalance_ratio=1.192749 max_load_reduction=55765 max_load_reduction_pct=0.850906 moved_to_echo=31134 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1250, 0, 0, 0, 7007, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8095, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6979, 0, 0, 0, 0, 0, 0, 0, 1145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 8175, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8203, 8263, 8134, 8191, 8124, 8237, 8189] avg=8192.0000 max=8263 max_rank=2 min=8124 min_rank=5 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32741 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 715, 0, 0, 0, 7012, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7478, 0, 0, 0, 0, 0, 0, 0, 1149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 8177, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8197, 7737, 8216, 8191, 8627, 8191, 8192] avg=8192.0000 max=8627 max_rank=5 min=7737 min_rank=2 imbalance_ratio=1.053101 max_load_reduction=56909 max_load_reduction_pct=0.868362 moved_to_echo=33201 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1054, 0, 0, 0, 7007, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8088, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7241, 0, 0, 0, 0, 0, 0, 0, 1145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 8175, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8203, 8068, 8127, 8191, 8386, 8180, 8193] avg=8192.0000 max=8386 max_rank=5 min=8068 min_rank=2 imbalance_ratio=1.023682 max_load_reduction=57150 max_load_reduction_pct=0.872040 moved_to_echo=32950 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 1, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 478, 0, 0, 0, 7006, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7728, 0, 0, 0, 0, 1, 0, 0, 1145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8174, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8188, 7492, 8222, 8190, 8874, 8176, 8196] avg=8192.0000 max=8874 max_rank=5 min=7492 min_rank=2 imbalance_ratio=1.083252 max_load_reduction=56662 max_load_reduction_pct=0.864594 moved_to_echo=33436 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 1555, 0, 0, 0, 7000, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8104, 0, 39, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6695, 0, 9, 0, 0, 0, 0, 0, 1144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 8175, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8203, 8578, 8147, 8191, 7848, 8189, 8189] avg=8192.0000 max=8578 max_rank=2 min=7848 min_rank=5 imbalance_ratio=1.047119 max_load_reduction=56958 max_load_reduction_pct=0.869110 moved_to_echo=32417 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 128, 15360, 0, 0, 0, 50176, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 51200, 0, 128, 0, 0, 128, 0, 0, 14464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 64640, 0, 1024, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 128, 9472, 0, 0, 0, 56064, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 56448, 0, 128, 0, 0, 128, 0, 0, 9216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 65536, 0, 128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 65536, 0, 0, 0, 128, 0, 0, 128, 0, 128, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 128, 65536, 0, 0, 0, 128, 0, 0, 128, 0, 128, 0, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 65408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 65280, 0, 384, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 0, 128, 128, 128, 0, 0, 0, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 5, 0, 0, 0, 55, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 1, 0, 6, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 8182, 0, 0, 0, 0, 1, 7631, 0, 0, 0, 0, 0, 0, 0, 8164, 8177, 0, 0, 1, 0, 0, 0, 0, 6, 0, 0, 0, 8, 4, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 7, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 496, 0, 0, 0, 0, 0, 20, 0, 0, 0, 3, 8176, 8177, 0, 0, 0, 8, 1, 8182, 8172, 0, 0, 7] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6898, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 8192, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5635, 8192, 0, 0, 0, 0, 0, 8192, 8180, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8123, 8192, 0, 0, 0, 0, 0, 8192, 8190, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8025, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8113, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 33, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 1, 8146, 0, 0, 0, 0, 0, 0, 0, 8190, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8191, 8191, 0, 0, 0, 1, 0, 8191, 8191, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 5, 0, 8175, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8, 0, 0, 8106, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 8, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 76, 0, 0, 0, 0, 1, 7631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8177, 0, 0, 1, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 7, 0, 8157, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 496, 0, 0, 0, 0, 0, 20, 7583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8176, 0, 0, 0, 0, 8, 1, 0, 525, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8234, 8196, 7718, 8191, 8182, 8105, 8720] avg=8192.0000 max=8720 max_rank=7 min=7718 min_rank=3 imbalance_ratio=1.064453 max_load_reduction=56816 max_load_reduction_pct=0.866943 moved_to_echo=33198 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8116, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1294, 0, 0, 0, 0, 0, 0, 7602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6898, 0, 0, 0, 0, 0, 0, 0, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8180, 8192, 8268, 8198, 8188, 8896, 7424] avg=8192.0000 max=8896 max_rank=6 min=7424 min_rank=7 imbalance_ratio=1.085938 max_load_reduction=56640 max_load_reduction_pct=0.864258 moved_to_echo=32706 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8116, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 7602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8180, 8192, 8268, 8197, 8188, 7642, 8678] avg=8192.0000 max=8678 max_rank=7 min=7642 min_rank=6 imbalance_ratio=1.059326 max_load_reduction=56858 max_load_reduction_pct=0.867584 moved_to_echo=32705 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8116, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 7600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8123, 0, 0, 0, 0, 0, 0, 0, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8180, 8191, 8268, 8197, 8189, 7669, 8649] avg=8192.0000 max=8649 max_rank=7 min=7669 min_rank=6 imbalance_ratio=1.055786 max_load_reduction=56887 max_load_reduction_pct=0.868027 moved_to_echo=32704 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8122, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 167, 0, 0, 0, 0, 0, 0, 7607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8025, 0, 0, 0, 0, 0, 0, 0, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8187, 8191, 8262, 8194, 8192, 7774, 8545] avg=8192.0000 max=8545 max_rank=7 min=7774 min_rank=6 imbalance_ratio=1.043091 max_load_reduction=56991 max_load_reduction_pct=0.869614 moved_to_echo=32705 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8116, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 0, 0, 0, 0, 7602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8113, 0, 0, 0, 0, 0, 0, 0, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8180, 8191, 8268, 8199, 8187, 7681, 8639] avg=8192.0000 max=8639 max_rank=7 min=7681 min_rank=6 imbalance_ratio=1.054565 max_load_reduction=56897 max_load_reduction_pct=0.868179 moved_to_echo=32706 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8116, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2564, 0, 0, 0, 0, 0, 0, 7591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5635, 0, 0, 0, 0, 0, 0, 0, 525, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8181, 8191, 8260, 8162, 8224, 10155, 6163] avg=8192.0000 max=10155 max_rank=6 min=6163 min_rank=7 imbalance_ratio=1.239624 max_load_reduction=55381 max_load_reduction_pct=0.845047 moved_to_echo=32704 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 33, 0, 0, 0, 1, 0, 0, 8115, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 1, 1, 8146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 1, 7601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 1, 0, 0, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8214, 8192, 8224, 8198, 8186, 7614, 8718] avg=8192.0000 max=8718 max_rank=7 min=7614 min_rank=6 imbalance_ratio=1.064209 max_load_reduction=56818 max_load_reduction_pct=0.866974 moved_to_echo=32716 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 1, 8152, 0, 0, 0, 0, 0, 0, 0, 8190, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8191, 8191, 0, 0, 0, 1, 0, 8191, 8191, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8134, 8192, 0, 0, 0, 0, 0, 8192, 8190, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 5, 0, 0, 0, 58, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 1, 0, 6, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 8182, 0, 0, 0, 0, 1, 7701, 0, 0, 0, 0, 0, 0, 0, 8164, 8177, 0, 0, 1, 0, 0, 0, 0, 6, 0, 0, 0, 8, 4, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 7, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 423, 0, 0, 0, 0, 0, 20, 0, 0, 0, 2, 8176, 8177, 0, 0, 0, 8, 1, 8182, 8173, 0, 0, 7] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6199, 8192, 0, 0, 0, 0, 0, 8192, 8180, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7150, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8127, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 7738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8180, 8192, 8257, 8197, 8188, 7764, 8567] avg=8192.0000 max=8567 max_rank=7 min=7764 min_rank=6 imbalance_ratio=1.045776 max_load_reduction=56969 max_load_reduction_pct=0.869278 moved_to_echo=32716 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 5, 0, 8175, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8, 0, 0, 8117, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 8, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 65, 0, 0, 0, 0, 1, 7701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8177, 0, 0, 1, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 7, 0, 8157, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 423, 0, 0, 0, 0, 0, 20, 7720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8176, 0, 0, 0, 0, 8, 1, 0, 400, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8237, 8196, 7777, 8191, 8182, 8169, 8594] avg=8192.0000 max=8594 max_rank=7 min=7777 min_rank=3 imbalance_ratio=1.049072 max_load_reduction=56942 max_load_reduction_pct=0.868866 moved_to_echo=33136 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8127, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 7737, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8134, 0, 0, 0, 0, 0, 0, 0, 400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8180, 8191, 8257, 8197, 8189, 7795, 8534] avg=8192.0000 max=8534 max_rank=7 min=7795 min_rank=6 imbalance_ratio=1.041748 max_load_reduction=57002 max_load_reduction_pct=0.869781 moved_to_echo=32715 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 1, 0, 0, 8126, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 1, 1, 8152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 2, 7737, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 1, 0, 0, 401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8214, 8192, 8219, 8198, 8186, 7743, 8593] avg=8192.0000 max=8593 max_rank=7 min=7743 min_rank=6 imbalance_ratio=1.048950 max_load_reduction=56943 max_load_reduction_pct=0.868881 moved_to_echo=32720 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8127, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 0, 0, 0, 0, 0, 0, 7738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7150, 0, 0, 0, 0, 0, 0, 0, 401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8180, 8192, 8257, 8198, 8189, 8780, 7551] avg=8192.0000 max=8780 max_rank=6 min=7551 min_rank=7 imbalance_ratio=1.071777 max_load_reduction=56756 max_load_reduction_pct=0.866028 moved_to_echo=32718 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8129, 54, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 7744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 0, 0, 0, 0, 0, 394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8184, 8191, 8255, 8193, 8192, 7792, 8538] avg=8192.0000 max=8538 max_rank=7 min=7792 min_rank=6 imbalance_ratio=1.042236 max_load_reduction=56998 max_load_reduction_pct=0.869720 moved_to_echo=32715 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8127, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 7738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8180, 8191, 8257, 8199, 8187, 7774, 8557] avg=8192.0000 max=8557 max_rank=7 min=7774 min_rank=6 imbalance_ratio=1.044556 max_load_reduction=56979 max_load_reduction_pct=0.869431 moved_to_echo=32717 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8127, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1992, 0, 0, 0, 0, 0, 0, 7727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6199, 0, 0, 0, 0, 0, 0, 0, 400, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8181, 8191, 8257, 8163, 8223, 9719, 6602] avg=8192.0000 max=9719 max_rank=6 min=6602 min_rank=7 imbalance_ratio=1.186401 max_load_reduction=55817 max_load_reduction_pct=0.851700 moved_to_echo=32707 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 4736, 0, 0, 0, 0, 0, 128, 60800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 61440, 0, 0, 0, 0, 128, 128, 0, 4224, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 0, 0, 640, 0, 0, 0, 128, 128, 65024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 128, 0, 0, 128, 128, 0, 0, 128, 0, 0, 0, 128, 0, 0, 65024, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 3712, 0, 0, 0, 0, 0, 128, 61952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 62336, 0, 0, 0, 0, 128, 128, 0, 3200, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 65536, 0, 0, 128, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 128, 0, 0, 65024, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 128, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 0, 0, 640, 0, 0, 0, 128, 128, 65024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 70, 0, 0, 8108, 0, 0, 0, 9, 0, 0, 0, 0, 8179, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 13, 10, 0, 0, 7066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8124, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0, 6, 10, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8142, 7, 0, 5, 0, 25, 0, 9, 0, 0, 0, 0, 945, 0, 29, 8, 0, 4, 8161, 0, 5, 0, 0, 0, 0, 0, 0, 10] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4099, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 8173, 0, 0, 0, 358, 0, 0, 0, 0, 8191, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 16, 1, 1, 0, 0, 5780, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 10, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8178, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2051, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 1, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7457, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3448, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1680, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7101, 0, 1, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7776, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 735, 0, 0, 0, 0, 0, 0, 0, 0, 4324, 41, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7457, 0, 0, 0, 0, 0, 3868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8141, 8189, 5110, 8201, 8184, 8194, 8192, 11325] avg=8192.0000 max=11325 max_rank=7 min=5110 min_rank=2 imbalance_ratio=1.382446 max_load_reduction=54211 max_load_reduction_pct=0.827194 moved_to_echo=35895 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 70, 0, 0, 8068, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 13, 10, 0, 0, 7066, 0, 0, 0, 0, 0, 0, 0, 0, 4308, 40, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8116, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 10, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8142, 7, 0, 5, 0, 25, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 945, 0, 29, 8, 0, 4, 3853, 0, 5, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8147, 8181, 11517, 8196, 8255, 8198, 8188, 4854] avg=8192.0000 max=11517 max_rank=2 min=4854 min_rank=7 imbalance_ratio=1.405884 max_load_reduction=54019 max_load_reduction_pct=0.824265 moved_to_echo=29495 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 8133, 0, 0, 0, 358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 1, 1, 0, 0, 5780, 0, 0, 0, 1, 0, 0, 0, 0, 4322, 40, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8170, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2051, 0, 0, 0, 0, 0, 3866, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8495, 8194, 10171, 8198, 8178, 8191, 8190, 5919] avg=8192.0000 max=10171 max_rank=2 min=5919 min_rank=7 imbalance_ratio=1.241577 max_load_reduction=55365 max_load_reduction_pct=0.844803 moved_to_echo=30478 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6509, 0, 0, 0, 0, 0, 0, 0, 0, 4324, 41, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1680, 0, 0, 0, 0, 0, 3868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 8192, 10884, 8193, 8184, 8192, 8192, 5548] avg=8192.0000 max=10884 max_rank=2 min=5548 min_rank=7 imbalance_ratio=1.328613 max_load_reduction=54652 max_load_reduction_pct=0.833923 moved_to_echo=30116 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 416, 0, 0, 0, 0, 0, 0, 0, 0, 4324, 41, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7776, 0, 0, 0, 0, 0, 3868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 8194, 4791, 8190, 8184, 8190, 8192, 11644] avg=8192.0000 max=11644 max_rank=7 min=4791 min_rank=2 imbalance_ratio=1.421387 max_load_reduction=53892 max_load_reduction_pct=0.822327 moved_to_echo=36210 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4324, 41, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4099, 0, 0, 0, 0, 0, 3868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 8194, 8467, 8191, 8184, 8190, 8192, 7967] avg=8192.0000 max=8467 max_rank=2 min=7967 min_rank=7 imbalance_ratio=1.033569 max_load_reduction=57069 max_load_reduction_pct=0.870804 moved_to_echo=32533 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4740, 0, 0, 0, 0, 0, 0, 0, 0, 4324, 41, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8190, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3448, 0, 0, 0, 0, 0, 3868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 8192, 9114, 8196, 8184, 8191, 8192, 7316] avg=8192.0000 max=9114 max_rank=2 min=7316 min_rank=7 imbalance_ratio=1.112549 max_load_reduction=56422 max_load_reduction_pct=0.860931 moved_to_echo=31883 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1091, 0, 0, 0, 0, 0, 0, 0, 0, 4331, 43, 9, 2, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7101, 0, 1, 0, 0, 0, 3861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8149, 8200, 5482, 8171, 8183, 8190, 8198, 10963] avg=8192.0000 max=10963 max_rank=7 min=5482 min_rank=2 imbalance_ratio=1.338257 max_load_reduction=54573 max_load_reduction_pct=0.832718 moved_to_echo=35534 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 71, 0, 0, 8110, 0, 0, 0, 18, 0, 0, 0, 0, 8179, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 13, 10, 0, 0, 7183, 1, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8126, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 6, 11, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8141, 7, 0, 5, 0, 25, 0, 9, 0, 0, 0, 0, 819, 0, 35, 8, 0, 2, 8160, 0, 5, 0, 0, 0, 0, 0, 0, 10] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7494, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 2, 0, 0, 0, 0, 8192, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1400, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2573, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 8173, 0, 0, 0, 383, 0, 0, 0, 0, 8191, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 17, 1, 1, 0, 0, 5797, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 5, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8179, 0, 7, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2010, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 1, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7207, 0, 4, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2818, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8183, 0, 0, 0, 3, 0, 0, 0, 0, 8192, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7374, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 71, 0, 0, 8064, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 13, 10, 0, 0, 7183, 1, 0, 0, 0, 0, 0, 0, 0, 3952, 46, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8119, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 11, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8141, 7, 0, 5, 0, 25, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 819, 0, 35, 8, 0, 2, 4208, 0, 5, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8153, 8183, 11279, 8197, 8251, 8199, 8187, 5087] avg=8192.0000 max=11279 max_rank=2 min=5087 min_rank=7 imbalance_ratio=1.376831 max_load_reduction=54257 max_load_reduction_pct=0.827896 moved_to_echo=29724 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5618, 0, 0, 0, 0, 0, 0, 0, 0, 3967, 46, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8190, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2573, 0, 0, 0, 0, 0, 4225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8146, 8193, 9639, 8193, 8185, 8190, 8192, 6798] avg=8192.0000 max=9639 max_rank=2 min=6798 min_rank=7 imbalance_ratio=1.176636 max_load_reduction=55897 max_load_reduction_pct=0.852921 moved_to_echo=31365 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8146, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6788, 0, 0, 0, 0, 0, 0, 0, 0, 3967, 46, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1400, 0, 0, 0, 0, 0, 4225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8148, 8192, 10809, 8192, 8185, 8193, 8192, 5625] avg=8192.0000 max=10809 max_rank=2 min=5625 min_rank=7 imbalance_ratio=1.319458 max_load_reduction=54727 max_load_reduction_pct=0.835068 moved_to_echo=30195 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 985, 0, 0, 0, 0, 0, 0, 0, 0, 3973, 50, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7207, 0, 4, 0, 0, 0, 4219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8142, 8199, 5021, 8179, 8177, 8189, 8199, 11430] avg=8192.0000 max=11430 max_rank=7 min=5021 min_rank=2 imbalance_ratio=1.395264 max_load_reduction=54106 max_load_reduction_pct=0.825592 moved_to_echo=35995 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 8127, 0, 0, 0, 383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 1, 1, 0, 0, 5797, 0, 0, 0, 1, 0, 0, 0, 0, 3965, 46, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8172, 0, 7, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2010, 0, 0, 0, 0, 0, 4223, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8514, 8195, 9836, 8193, 8183, 8191, 8189, 6235] avg=8192.0000 max=9836 max_rank=2 min=6235 min_rank=7 imbalance_ratio=1.200684 max_load_reduction=55700 max_load_reduction_pct=0.849915 moved_to_echo=30798 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 698, 0, 0, 0, 0, 0, 0, 0, 0, 3967, 46, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7494, 0, 0, 0, 0, 0, 4225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8146, 8195, 4719, 8190, 8185, 8190, 8192, 11719] avg=8192.0000 max=11719 max_rank=7 min=4719 min_rank=2 imbalance_ratio=1.430542 max_load_reduction=53817 max_load_reduction_pct=0.821182 moved_to_echo=36286 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8146, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5372, 0, 0, 0, 0, 0, 0, 0, 0, 3967, 46, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2818, 0, 0, 0, 0, 0, 4225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8147, 8195, 9394, 8191, 8185, 8189, 8192, 7043] avg=8192.0000 max=9394 max_rank=2 min=7043 min_rank=7 imbalance_ratio=1.146729 max_load_reduction=56142 max_load_reduction_pct=0.856659 moved_to_echo=31609 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8137, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 817, 0, 0, 0, 0, 0, 0, 0, 0, 3967, 46, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7374, 0, 0, 0, 0, 0, 4225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8140, 8184, 4839, 8201, 8185, 8195, 8193, 11599] avg=8192.0000 max=11599 max_rank=7 min=4839 min_rank=2 imbalance_ratio=1.415894 max_load_reduction=53937 max_load_reduction_pct=0.823013 moved_to_echo=36172 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[34560, 0, 128, 128, 0, 128, 30976, 0, 128, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 128, 0, 0, 30464, 0, 0, 0, 128, 0, 0, 0, 0, 34688, 384, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[31744, 0, 128, 128, 0, 128, 33792, 0, 128, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 128, 128, 128, 0, 0, 33280, 128, 0, 0, 128, 0, 0, 0, 0, 31744, 384, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 128, 0, 0, 65152, 0, 0, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 65536, 128, 0, 128, 0, 128, 0, 128, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 65536, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 128, 65408, 0, 256, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 128, 0, 0, 65152, 0, 0, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 0, 65536, 128, 0, 128, 0, 128, 128, 128, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 65536, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 8190, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 200, 0, 0, 0, 0, 0, 0, 7992, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 8185, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 36, 8182, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 145, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 8006, 2, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 9, 0, 0, 0, 0, 8190, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 8187, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 8135, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 8172, 2, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 204, 8077, 0, 4, 0, 0, 0, 9, 11, 0, 9, 0, 0, 8, 0, 8157, 0, 0, 0, 0, 0, 0, 1462, 0, 0, 0, 0, 1, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 13, 6518, 0, 0, 0, 6, 0, 8179, 0, 0, 0, 1, 0, 4, 0, 0, 0, 8178, 0, 0, 0, 5, 0, 0, 0, 11, 0, 6, 0, 0, 10, 0, 11, 1, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 8176, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 11, 0, 0, 204, 0, 0, 0, 0, 0, 0, 7992, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 8179, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0, 8088, 2, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 0, 0, 0, 0, 8113, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 8182, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 8097, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 204, 8063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 9, 11, 0, 9, 0, 0, 8, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1462, 0, 0, 0, 0, 1, 0, 10, 0, 0, 0, 0, 0, 7858, 83, 14, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 13, 8086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6518, 0, 0, 0, 6, 0, 321, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 5, 0, 0, 0, 11, 0, 6, 0, 0, 10, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8269, 8198, 9440, 8122, 6850, 8218, 8260, 8179] avg=8192.0000 max=9440 max_rank=2 min=6850 min_rank=4 imbalance_ratio=1.152344 max_load_reduction=56096 max_load_reduction_pct=0.855957 moved_to_echo=31507 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7870, 84, 14, 9, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 200, 0, 0, 0, 0, 0, 0, 8104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7992, 0, 0, 0, 0, 0, 322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8191, 7981, 8306, 8314, 8189, 8183, 8194] avg=8192.0000 max=8314 max_rank=4 min=7981 min_rank=2 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32880 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7869, 85, 14, 9, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 11, 0, 0, 204, 0, 0, 0, 0, 0, 0, 8090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7992, 0, 0, 0, 0, 0, 322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8191, 7982, 8306, 8314, 8189, 8183, 8193] avg=8192.0000 max=8314 max_rank=4 min=7982 min_rank=2 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32879 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7870, 84, 14, 9, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 0, 0, 0, 0, 8106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8113, 0, 0, 0, 0, 0, 322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8190, 7982, 8185, 8435, 8189, 8183, 8194] avg=8192.0000 max=8435 max_rank=4 min=7982 min_rank=2 imbalance_ratio=1.029663 max_load_reduction=57101 max_load_reduction_pct=0.871292 moved_to_echo=33001 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7869, 84, 14, 9, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 8101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8135, 0, 0, 0, 0, 0, 322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8190, 7985, 8160, 8457, 8189, 8183, 8194] avg=8192.0000 max=8457 max_rank=4 min=7985 min_rank=2 imbalance_ratio=1.032349 max_load_reduction=57079 max_load_reduction_pct=0.870956 moved_to_echo=33023 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 7869, 84, 14, 9, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 8099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8006, 2, 0, 0, 0, 0, 322, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8195, 8127, 8103, 8339, 8190, 8183, 8195] avg=8192.0000 max=8339 max_rank=4 min=8103 min_rank=3 imbalance_ratio=1.017944 max_load_reduction=57197 max_load_reduction_pct=0.872757 moved_to_echo=32907 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7870, 83, 14, 9, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0, 8094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8088, 2, 0, 0, 0, 0, 322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8191, 8011, 8178, 8412, 8189, 8183, 8194] avg=8192.0000 max=8412 max_rank=4 min=8011 min_rank=2 imbalance_ratio=1.026855 max_load_reduction=57124 max_load_reduction_pct=0.871643 moved_to_echo=32978 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7874, 85, 18, 14, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 8096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8097, 0, 0, 0, 0, 0, 318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8173, 8190, 8028, 8176, 8415, 8183, 8178, 8193] avg=8192.0000 max=8415 max_rank=4 min=8028 min_rank=2 imbalance_ratio=1.027222 max_load_reduction=57121 max_load_reduction_pct=0.871597 moved_to_echo=32969 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 8172, 2, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 193, 8096, 0, 4, 0, 0, 0, 9, 4, 0, 8, 0, 0, 8, 0, 8157, 0, 0, 0, 0, 0, 0, 1517, 0, 0, 0, 0, 1, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 12, 6471, 0, 0, 0, 6, 0, 8179, 0, 0, 0, 1, 0, 4, 0, 0, 0, 8179, 0, 0, 0, 5, 0, 0, 0, 12, 0, 5, 0, 0, 10, 0, 11, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 8150, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 8186, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 8179, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 8137, 1, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 8182, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 8129, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 8175, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 11, 0, 0, 116, 0, 0, 0, 0, 0, 0, 8081, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 8190, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 127, 0, 0, 0, 0, 0, 0, 8037, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 8183, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 37, 8187, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 168, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 7986, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8, 0, 0, 0, 0, 8190, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 193, 8080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 9, 4, 0, 8, 0, 0, 8, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1517, 0, 0, 0, 0, 1, 0, 10, 0, 0, 0, 0, 0, 7884, 42, 16, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 12, 8128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6471, 0, 0, 0, 6, 0, 295, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 5, 0, 0, 0, 12, 0, 5, 0, 0, 10, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8275, 8190, 9481, 8163, 6777, 8219, 8252, 8179] avg=8192.0000 max=9481 max_rank=2 min=6777 min_rank=4 imbalance_ratio=1.157349 max_load_reduction=56055 max_load_reduction_pct=0.855331 moved_to_echo=31427 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7897, 42, 16, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 8147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8150, 0, 0, 0, 0, 0, 295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8191, 7967, 8189, 8445, 8189, 8184, 8195] avg=8192.0000 max=8445 max_rank=4 min=7967 min_rank=2 imbalance_ratio=1.030884 max_load_reduction=57091 max_load_reduction_pct=0.871140 moved_to_echo=33013 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7896, 42, 16, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 8141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8191, 7972, 8168, 8461, 8189, 8184, 8195] avg=8192.0000 max=8461 max_rank=4 min=7972 min_rank=2 imbalance_ratio=1.032837 max_load_reduction=57075 max_load_reduction_pct=0.870895 moved_to_echo=33029 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7898, 43, 19, 12, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 8138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8129, 0, 0, 0, 0, 0, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8173, 8194, 8012, 8179, 8423, 8182, 8180, 8193] avg=8192.0000 max=8423 max_rank=4 min=8012 min_rank=2 imbalance_ratio=1.028198 max_load_reduction=57113 max_load_reduction_pct=0.871475 moved_to_echo=32978 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7897, 42, 16, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 8136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 1, 0, 0, 0, 0, 295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8192, 8008, 8161, 8433, 8189, 8184, 8193] avg=8192.0000 max=8433 max_rank=4 min=8008 min_rank=2 imbalance_ratio=1.029419 max_load_reduction=57103 max_load_reduction_pct=0.871323 moved_to_echo=32999 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7897, 42, 16, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 127, 0, 0, 0, 0, 0, 0, 8147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8037, 0, 0, 0, 0, 0, 295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8192, 7994, 8276, 8332, 8189, 8184, 8193] avg=8192.0000 max=8332 max_rank=4 min=7994 min_rank=2 imbalance_ratio=1.017090 max_load_reduction=57204 max_load_reduction_pct=0.872864 moved_to_echo=32898 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 7896, 42, 16, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7986, 0, 0, 0, 0, 0, 295, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8194, 8135, 8141, 8289, 8190, 8184, 8195] avg=8192.0000 max=8289 max_rank=4 min=8135 min_rank=2 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32858 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7896, 43, 16, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 11, 0, 0, 116, 0, 0, 0, 0, 0, 0, 8131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8081, 0, 0, 0, 0, 0, 295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8192, 7967, 8259, 8376, 8189, 8184, 8193] avg=8192.0000 max=8376 max_rank=4 min=7967 min_rank=2 imbalance_ratio=1.022461 max_load_reduction=57160 max_load_reduction_pct=0.872192 moved_to_echo=32942 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[62976, 128, 0, 0, 128, 0, 2688, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 1792, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 128, 62976, 768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 0, 0, 128, 0, 128, 768, 128, 0, 0, 0, 0, 128, 64896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 128, 128, 0, 128, 0, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 1920, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 128, 63232, 384, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[63232, 128, 0, 0, 128, 0, 2432, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[65536, 0, 0, 0, 128, 0, 0, 0, 128, 0, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 65408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 128, 128, 0, 0, 128, 0, 128, 384, 128, 0, 0, 0, 0, 128, 65152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 0, 128, 128, 0, 128, 0, 0, 128, 0, 65536, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 25, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8173, 12, 0, 7570, 3519, 0, 0, 0, 0, 8171, 9, 0, 130, 0, 8051, 0, 9, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 10, 8033, 0, 0, 183, 0, 6, 0, 0, 0, 12, 3, 13, 12, 0, 924, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 8171, 0, 0, 9, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4159, 0, 0, 0, 0, 0, 10, 0, 0, 0, 8164] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8191, 1, 0, 8140, 524, 0, 0, 0, 0, 8191, 1, 0, 2, 0, 8181, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 153, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 70, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 12, 0, 7175, 3519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 9, 0, 130, 0, 8051, 0, 9, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 10, 7970, 0, 0, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 12, 3, 13, 12, 0, 924, 0, 0, 0, 0, 0, 0, 7535, 395, 63, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 8155, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4159, 0, 0, 0, 0, 0, 10, 0, 0, 0, 628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10708, 8205, 8170, 8979, 8184, 8313, 8180, 4797] avg=8192.0000 max=10708 max_rank=0 min=4797 min_rank=7 imbalance_ratio=1.307129 max_load_reduction=54828 max_load_reduction_pct=0.836609 moved_to_echo=29474 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8128, 0, 0, 25, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7560, 427, 64, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 8186, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7765, 8171, 8167, 8067, 8201, 8170, 8198, 8797] avg=8192.0000 max=8797 max_rank=7 min=7765 min_rank=0 imbalance_ratio=1.073853 max_load_reduction=56739 max_load_reduction_pct=0.865768 moved_to_echo=33366 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8128, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7561, 427, 64, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7765, 8196, 8233, 8068, 8188, 8176, 8193, 8717] avg=8192.0000 max=8717 max_rank=7 min=7765 min_rank=0 imbalance_ratio=1.064087 max_load_reduction=56819 max_load_reduction_pct=0.866989 moved_to_echo=33274 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8128, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7561, 427, 64, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7765, 8194, 8167, 8068, 8189, 8176, 8193, 8784] avg=8192.0000 max=8784 max_rank=7 min=7765 min_rank=0 imbalance_ratio=1.072266 max_load_reduction=56752 max_load_reduction_pct=0.865967 moved_to_echo=33342 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8128, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7561, 427, 64, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7765, 8193, 8153, 8068, 8190, 8174, 8193, 8800] avg=8192.0000 max=8800 max_rank=7 min=7765 min_rank=0 imbalance_ratio=1.074219 max_load_reduction=56736 max_load_reduction_pct=0.865723 moved_to_echo=33357 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 7716, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 2, 0, 8181, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8127, 0, 0, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 70, 0, 0, 1, 0, 0, 0, 7557, 424, 64, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8242, 8190, 8280, 8135, 8186, 8179, 8192, 8132] avg=8192.0000 max=8280 max_rank=2 min=8132 min_rank=7 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32689 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8128, 0, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7561, 427, 64, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7765, 8197, 8221, 8068, 8191, 8176, 8193, 8725] avg=8192.0000 max=8725 max_rank=7 min=7765 min_rank=0 imbalance_ratio=1.065063 max_load_reduction=56811 max_load_reduction_pct=0.866867 moved_to_echo=33285 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8126, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7566, 431, 66, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7761, 8190, 8145, 8083, 8207, 8172, 8194, 8784] avg=8192.0000 max=8784 max_rank=7 min=7761 min_rank=0 imbalance_ratio=1.072266 max_load_reduction=56752 max_load_reduction_pct=0.865967 moved_to_echo=33357 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8174, 12, 0, 7565, 3708, 0, 0, 0, 0, 8171, 8, 0, 129, 0, 8052, 0, 9, 0, 1, 1, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 10, 8025, 0, 0, 195, 0, 6, 0, 0, 0, 12, 3, 13, 11, 0, 855, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 8171, 0, 0, 9, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 0, 0, 0, 0, 11, 0, 0, 0, 8164] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 46, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8191, 1, 0, 8134, 606, 0, 0, 0, 0, 8191, 1, 0, 2, 0, 8180, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 202, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 74, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7373, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8187] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 12, 0, 7140, 3708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 8, 0, 129, 0, 8052, 0, 9, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 10, 7931, 0, 0, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 12, 3, 13, 11, 0, 855, 0, 0, 0, 0, 0, 0, 7478, 425, 94, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 8154, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 0, 0, 0, 0, 11, 0, 0, 0, 684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10863, 8205, 8143, 8914, 8183, 8321, 8179, 4728] avg=8192.0000 max=10863 max_rank=0 min=4728 min_rank=7 imbalance_ratio=1.326050 max_load_reduction=54673 max_load_reduction_pct=0.834244 moved_to_echo=29411 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7732, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8096, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7503, 460, 96, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7732, 8193, 8134, 8076, 8191, 8173, 8193, 8844] avg=8192.0000 max=8844 max_rank=7 min=7732 min_rank=0 imbalance_ratio=1.079590 max_load_reduction=56692 max_load_reduction_pct=0.865051 moved_to_echo=33401 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7732, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8096, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7503, 460, 96, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7732, 8194, 8148, 8076, 8190, 8175, 8193, 8828] avg=8192.0000 max=8828 max_rank=7 min=7732 min_rank=0 imbalance_ratio=1.077637 max_load_reduction=56708 max_load_reduction_pct=0.865295 moved_to_echo=33386 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8093, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7509, 464, 99, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7728, 8190, 8124, 8095, 8200, 8169, 8197, 8833] avg=8192.0000 max=8833 max_rank=7 min=7728 min_rank=0 imbalance_ratio=1.078247 max_load_reduction=56703 max_load_reduction_pct=0.865219 moved_to_echo=33399 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7732, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8096, 0, 0, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7503, 460, 96, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7732, 8197, 8303, 8076, 8191, 8175, 8193, 8669] avg=8192.0000 max=8669 max_rank=7 min=7732 min_rank=0 imbalance_ratio=1.058228 max_load_reduction=56867 max_load_reduction_pct=0.867722 moved_to_echo=33228 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7732, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8096, 0, 0, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7503, 460, 96, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7732, 8196, 8233, 8076, 8190, 8175, 8193, 8741] avg=8192.0000 max=8741 max_rank=7 min=7732 min_rank=0 imbalance_ratio=1.067017 max_load_reduction=56795 max_load_reduction_pct=0.866623 moved_to_echo=33299 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 7677, 606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 2, 0, 8180, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8094, 0, 0, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 74, 0, 0, 1, 0, 0, 0, 7499, 457, 96, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7373, 0, 0, 0, 1, 0, 0, 0, 0, 0, 686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8285, 8189, 8296, 8147, 8187, 8180, 8192, 8060] avg=8192.0000 max=8296 max_rank=2 min=8060 min_rank=7 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32619 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7732, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8096, 0, 0, 46, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7503, 460, 96, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 8186, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7732, 8172, 8155, 8076, 8204, 8168, 8196, 8833] avg=8192.0000 max=8833 max_rank=7 min=7732 min_rank=0 imbalance_ratio=1.078247 max_load_reduction=56703 max_load_reduction_pct=0.865219 moved_to_echo=33401 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 60544, 0, 0, 0, 0, 0, 128, 0, 0, 0, 5120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 0, 0, 128, 128, 128, 128, 0, 1024, 0, 0, 128, 0, 0, 0, 60544, 3456, 640, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 128, 0, 61568, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 128, 0, 256, 0, 65408, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 128, 64896, 0, 0, 640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 128, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 0, 0, 128, 128, 128, 128, 0, 1024, 0, 0, 128, 0, 0, 0, 60032, 3712, 896, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 60160, 0, 0, 0, 128, 0, 128, 0, 0, 0, 5504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 128, 0, 61312, 4352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 128, 64640, 0, 0, 1024, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 256, 65408, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 128, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 128, 128, 0, 256, 0, 65408, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 110, 3, 0, 0, 0, 0, 4, 7172, 0, 0, 0, 0, 1, 25, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 12, 19, 0, 0, 1543, 0, 0, 10, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 12, 0, 7484, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 8169, 0, 0, 0, 10, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 1, 0, 14, 0, 0, 0, 16, 0, 8169, 11, 0, 8172, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 8, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8093, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 8189, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 8191, 0, 0, 8191, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 8192, 0, 19, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 110, 3, 0, 0, 0, 0, 4, 7293, 0, 0, 0, 0, 1, 28, 0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 12, 19, 0, 0, 1560, 0, 0, 5, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 14, 0, 7352, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 8168, 0, 0, 0, 9, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 1, 0, 14, 0, 0, 0, 16, 0, 8169, 14, 0, 8173, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 8192, 0, 19, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 9, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8091, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 8189, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 8191, 0, 0, 8191, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 88, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7981, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8287, 8188, 8192, 7983, 8315, 8192, 8190, 8189] avg=8192.0000 max=8315 max_rank=4 min=7983 min_rank=3 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32886 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 89, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7982, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8285, 8194, 8192, 7984, 8310, 8192, 8190, 8189] avg=8192.0000 max=8310 max_rank=4 min=7984 min_rank=3 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32881 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 110, 3, 0, 0, 0, 0, 4, 7293, 0, 0, 0, 0, 1, 28, 0, 88, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 12, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 1560, 0, 0, 5, 14, 0, 0, 0, 0, 0, 0, 0, 0, 7945, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 14, 0, 7352, 0, 0, 0, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 9, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 14, 0, 0, 0, 16, 0, 8166, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7536, 8147, 8193, 9544, 7503, 8198, 8204, 8211] avg=8192.0000 max=9544 max_rank=3 min=7503 min_rank=4 imbalance_ratio=1.165039 max_load_reduction=55992 max_load_reduction_pct=0.854370 moved_to_echo=32116 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 90, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7981, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8272, 8213, 8192, 7983, 8305, 8192, 8190, 8189] avg=8192.0000 max=8305 max_rank=4 min=7983 min_rank=3 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32876 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 9, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 1, 2, 0, 88, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7980, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8091, 0, 0, 0, 0, 0, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8283, 8183, 8192, 8092, 8213, 8191, 8192, 8190] avg=8192.0000 max=8283 max_rank=0 min=8092 min_rank=3 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32786 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 88, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7981, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8284, 8196, 8191, 7983, 8308, 8192, 8191, 8191] avg=8192.0000 max=8308 max_rank=4 min=7983 min_rank=3 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32882 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 90, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7981, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8291, 8210, 8192, 7983, 8289, 8192, 8190, 8189] avg=8192.0000 max=8291 max_rank=0 min=7983 min_rank=3 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32860 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 90, 7, 4, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7981, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8298, 8205, 8192, 7984, 8293, 8187, 8189, 8188] avg=8192.0000 max=8298 max_rank=0 min=7984 min_rank=3 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32857 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 110, 3, 0, 0, 0, 0, 4, 7172, 0, 0, 0, 0, 1, 25, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 12, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 1543, 0, 0, 10, 14, 0, 0, 0, 0, 0, 0, 0, 0, 7949, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 12, 0, 7484, 0, 0, 0, 0, 0, 96, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 10, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 14, 0, 0, 0, 16, 0, 8167, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7429, 8147, 8192, 9537, 7615, 8200, 8207, 8209] avg=8192.0000 max=9537 max_rank=3 min=7429 min_rank=0 imbalance_ratio=1.164185 max_load_reduction=55999 max_load_reduction_pct=0.854477 moved_to_echo=32231 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7982, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 95, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8303, 8193, 8192, 7984, 8292, 8192, 8190, 8190] avg=8192.0000 max=8303 max_rank=0 min=7984 min_rank=3 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32864 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7981, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 96, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8301, 8196, 8192, 7983, 8290, 8192, 8190, 8192] avg=8192.0000 max=8301 max_rank=0 min=7983 min_rank=3 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32864 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 1, 3, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7978, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8093, 0, 0, 0, 0, 0, 95, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8295, 8183, 8192, 8095, 8197, 8191, 8192, 8191] avg=8192.0000 max=8295 max_rank=0 min=8095 min_rank=3 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32771 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7982, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 95, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8307, 8211, 8192, 7985, 8270, 8192, 8189, 8190] avg=8192.0000 max=8307 max_rank=0 min=7985 min_rank=3 imbalance_ratio=1.014038 max_load_reduction=57229 max_load_reduction_pct=0.873245 moved_to_echo=32841 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7981, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 95, 9, 8, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8308, 8204, 8192, 7984, 8290, 8185, 8189, 8184] avg=8192.0000 max=8308 max_rank=0 min=7984 min_rank=3 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32848 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7981, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 96, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8305, 8188, 8192, 7983, 8296, 8192, 8190, 8190] avg=8192.0000 max=8305 max_rank=0 min=7983 min_rank=3 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32868 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7982, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 95, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8288, 8214, 8192, 7985, 8286, 8192, 8189, 8190] avg=8192.0000 max=8288 max_rank=0 min=7985 min_rank=3 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32857 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 1792, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 63872, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 0, 0, 0, 0, 128, 64640, 0, 0, 0, 0, 128, 128, 0, 768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 64640, 0, 0, 0, 0, 0, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 65536, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 1664, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 63872, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 64768, 0, 0, 0, 0, 0, 768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 0, 0, 0, 128, 64512, 0, 0, 0, 0, 128, 128, 0, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 128, 0, 128, 0, 0, 0, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 65536, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 7811, 2, 0, 0, 8150, 0, 0, 4, 0, 0, 42, 0, 0, 0, 8169, 0, 0, 0, 0, 51, 0, 21, 82, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 6, 0, 1994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8017, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 15, 0, 246, 0, 0, 11, 0, 0, 0, 0, 4607, 1, 22, 0, 1722, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8152, 0, 0, 1, 13, 0, 0, 14, 0, 0, 5, 0, 0, 8163, 15, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 958, 0, 0, 0, 0, 0, 0, 0, 7229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 8191, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 8005, 1, 1, 0, 152, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 8104, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 227, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8234, 8176, 7946, 8193, 8213, 8640, 7944, 8190] avg=8192.0000 max=8640 max_rank=5 min=7944 min_rank=6 imbalance_ratio=1.054688 max_load_reduction=56896 max_load_reduction_pct=0.868164 moved_to_echo=32987 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 7811, 2, 0, 0, 43, 0, 0, 4, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 21, 82, 10, 0, 0, 0, 0, 0, 0, 0, 8107, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7920, 0, 6, 0, 1994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8017, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 246, 0, 0, 11, 0, 0, 0, 0, 4607, 1, 22, 0, 227, 221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1722, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 7904, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 14, 0, 0, 5, 0, 0, 8161, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7902, 8298, 9920, 8183, 8044, 5350, 9630, 8209] avg=8192.0000 max=9920 max_rank=2 min=5350 min_rank=5 imbalance_ratio=1.210938 max_load_reduction=55616 max_load_reduction_pct=0.848633 moved_to_echo=31233 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8191, 0, 0, 0, 42, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8147, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 8005, 1, 1, 0, 227, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8177, 7946, 8194, 8208, 8492, 8095, 8189] avg=8192.0000 max=8492 max_rank=5 min=7946 min_rank=2 imbalance_ratio=1.036621 max_load_reduction=57044 max_load_reduction_pct=0.870422 moved_to_echo=32984 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 8104, 0, 0, 0, 227, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8234, 8176, 7946, 8193, 8213, 8592, 7992, 8190] avg=8192.0000 max=8592 max_rank=5 min=7946 min_rank=2 imbalance_ratio=1.048828 max_load_reduction=56944 max_load_reduction_pct=0.868896 moved_to_echo=32987 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 0, 0, 0, 227, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8234, 8176, 7946, 8193, 8213, 8594, 7990, 8190] avg=8192.0000 max=8594 max_rank=5 min=7946 min_rank=2 imbalance_ratio=1.049072 max_load_reduction=56942 max_load_reduction_pct=0.868866 moved_to_echo=32987 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 227, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8234, 8176, 7946, 8193, 8213, 8626, 7958, 8190] avg=8192.0000 max=8626 max_rank=5 min=7946 min_rank=2 imbalance_ratio=1.052979 max_load_reduction=56910 max_load_reduction_pct=0.868378 moved_to_echo=32987 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 0, 234, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8229, 8181, 7935, 8194, 8219, 8606, 7984, 8188] avg=8192.0000 max=8606 max_rank=5 min=7935 min_rank=2 imbalance_ratio=1.050537 max_load_reduction=56930 max_load_reduction_pct=0.868683 moved_to_echo=32997 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7946, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 958, 0, 0, 0, 0, 0, 0, 0, 7229, 0, 0, 0, 227, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8234, 8176, 7951, 8193, 8213, 8636, 7943, 8190] avg=8192.0000 max=8636 max_rank=5 min=7943 min_rank=6 imbalance_ratio=1.054199 max_load_reduction=56900 max_load_reduction_pct=0.868225 moved_to_echo=32982 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8139, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 8106, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 8191, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 18, 0, 0, 0, 0, 0, 0, 0, 7965, 1, 1, 0, 205, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 7915, 2, 0, 0, 8150, 0, 0, 4, 0, 0, 41, 0, 0, 0, 8169, 0, 0, 0, 0, 51, 0, 24, 76, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 8, 0, 2213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8015, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 15, 0, 72, 0, 0, 12, 0, 0, 0, 0, 4727, 0, 21, 0, 1459, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8152, 0, 0, 0, 13, 0, 0, 14, 0, 0, 6, 0, 0, 8163, 15, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 648, 0, 0, 0, 0, 0, 0, 0, 7535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8139, 0, 0, 0, 261, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8176, 7917, 8193, 8213, 8606, 8020, 8190] avg=8192.0000 max=8606 max_rank=5 min=7917 min_rank=2 imbalance_ratio=1.050537 max_load_reduction=56930 max_load_reduction_pct=0.868683 moved_to_echo=33029 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 7915, 2, 0, 0, 29, 0, 0, 4, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 24, 76, 10, 0, 0, 0, 0, 0, 0, 0, 8120, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7892, 0, 8, 0, 2213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8015, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 72, 0, 0, 12, 0, 0, 0, 0, 4727, 0, 21, 0, 260, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1459, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 7929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 14, 0, 0, 6, 0, 0, 8161, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7991, 8295, 10113, 8184, 8042, 5311, 9391, 8209] avg=8192.0000 max=10113 max_rank=2 min=5311 min_rank=5 imbalance_ratio=1.234497 max_load_reduction=55423 max_load_reduction_pct=0.845688 moved_to_echo=30953 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 8106, 0, 0, 0, 261, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8176, 7917, 8193, 8213, 8605, 8021, 8190] avg=8192.0000 max=8605 max_rank=5 min=7917 min_rank=2 imbalance_ratio=1.050415 max_load_reduction=56931 max_load_reduction_pct=0.868698 moved_to_echo=33029 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 261, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8176, 7917, 8193, 8213, 8641, 7985, 8190] avg=8192.0000 max=8641 max_rank=5 min=7917 min_rank=2 imbalance_ratio=1.054810 max_load_reduction=56895 max_load_reduction_pct=0.868149 moved_to_echo=33029 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8191, 0, 0, 0, 29, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7916, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 18, 0, 0, 0, 0, 0, 0, 0, 7965, 1, 1, 0, 261, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 205, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8177, 7923, 8192, 8208, 8452, 8173, 8189] avg=8192.0000 max=8452 max_rank=5 min=7923 min_rank=2 imbalance_ratio=1.031738 max_load_reduction=57084 max_load_reduction_pct=0.871033 moved_to_echo=33022 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 0, 0, 0, 265, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8218, 8184, 7906, 8195, 8221, 8613, 8011, 8188] avg=8192.0000 max=8613 max_rank=5 min=7906 min_rank=2 imbalance_ratio=1.051392 max_load_reduction=56923 max_load_reduction_pct=0.868576 moved_to_echo=33033 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 261, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8176, 7917, 8193, 8213, 8658, 7968, 8190] avg=8192.0000 max=8658 max_rank=5 min=7917 min_rank=2 imbalance_ratio=1.056885 max_load_reduction=56878 max_load_reduction_pct=0.867889 moved_to_echo=33029 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7917, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 648, 0, 0, 0, 0, 0, 0, 0, 7535, 0, 0, 0, 261, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8176, 7926, 8193, 8213, 8650, 7967, 8190] avg=8192.0000 max=8650 max_rank=5 min=7926 min_rank=2 imbalance_ratio=1.055908 max_load_reduction=56886 max_load_reduction_pct=0.868011 moved_to_echo=33020 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 128, 0, 1408, 0, 0, 128, 0, 0, 0, 0, 60672, 128, 128, 0, 1920, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 63616, 0, 128, 0, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 65152, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[2048, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 63616, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 128, 0, 896, 0, 0, 128, 0, 0, 0, 0, 61056, 128, 128, 0, 2176, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 63360, 0, 128, 0, 2304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[1920, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 63744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 65280, 128, 0, 0, 256, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[65408, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 128, 0, 0, 128, 0, 0, 65536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 65280, 128, 0, 0, 384, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 128, 0, 0, 128, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 271, 8147, 0, 0, 0, 0, 0, 0, 0, 14, 0, 8164, 34, 1, 0, 84, 0, 0, 5, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8091, 0, 0, 0, 0, 0, 8, 0, 0, 8, 0, 18, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1259, 0, 0, 0, 7901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6923, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 12, 8168, 13, 0, 0, 20, 0, 2, 11, 0, 0, 0, 1, 0, 0, 0, 8176, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 70, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 9, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8191, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 120, 32, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8246, 8214, 8193, 8072, 8349, 8187, 8121] avg=8192.0000 max=8349 max_rank=5 min=8072 min_rank=4 imbalance_ratio=1.019165 max_load_reduction=57187 max_load_reduction_pct=0.872604 moved_to_echo=32729 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 271, 8110, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 1, 0, 84, 0, 0, 5, 13, 0, 0, 0, 0, 0, 0, 0, 0, 8131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8091, 0, 0, 0, 0, 0, 8, 0, 0, 8, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1259, 0, 0, 0, 7785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6923, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 33, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 12, 8164, 13, 0, 0, 20, 0, 2, 11, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8395, 8268, 8113, 8179, 9044, 7101, 8230, 8206] avg=8192.0000 max=9044 max_rank=4 min=7101 min_rank=5 imbalance_ratio=1.104004 max_load_reduction=56492 max_load_reduction_pct=0.862000 moved_to_echo=32581 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 33, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8179, 8200, 8195, 8073, 8362, 8187, 8186] avg=8192.0000 max=8362 max_rank=5 min=8073 min_rank=4 imbalance_ratio=1.020752 max_load_reduction=57174 max_load_reduction_pct=0.872406 moved_to_echo=32808 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 32, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8169, 8201, 8195, 8072, 8357, 8187, 8201] avg=8192.0000 max=8357 max_rank=5 min=8072 min_rank=4 imbalance_ratio=1.020142 max_load_reduction=57179 max_load_reduction_pct=0.872482 moved_to_echo=32817 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 70, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 9, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 33, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8186, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8223, 8171, 8200, 8188, 8066, 8290, 8192, 8206] avg=8192.0000 max=8290 max_rank=5 min=8066 min_rank=4 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32754 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 33, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8181, 8202, 8195, 8072, 8343, 8187, 8202] avg=8192.0000 max=8343 max_rank=5 min=8072 min_rank=4 imbalance_ratio=1.018433 max_load_reduction=57193 max_load_reduction_pct=0.872696 moved_to_echo=32804 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 32, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8155, 8161, 8199, 8195, 8072, 8361, 8187, 8206] avg=8192.0000 max=8361 max_rank=5 min=8072 min_rank=4 imbalance_ratio=1.020630 max_load_reduction=57175 max_load_reduction_pct=0.872421 moved_to_echo=32826 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 8146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 32, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8147, 8161, 8207, 8196, 8065, 8373, 8179, 8208] avg=8192.0000 max=8373 max_rank=5 min=8065 min_rank=4 imbalance_ratio=1.022095 max_load_reduction=57163 max_load_reduction_pct=0.872238 moved_to_echo=32825 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 65280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 259, 8148, 0, 0, 0, 0, 0, 0, 0, 13, 0, 8164, 33, 1, 0, 84, 0, 0, 5, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8091, 0, 0, 0, 0, 0, 8, 0, 0, 8, 0, 17, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1256, 0, 0, 0, 7914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6927, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 12, 8169, 13, 0, 0, 19, 0, 2, 11, 0, 0, 0, 1, 0, 0, 0, 8176, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 73, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 12, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8191, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 259, 8112, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 1, 0, 84, 0, 0, 5, 13, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8091, 0, 0, 0, 0, 0, 8, 0, 0, 8, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1256, 0, 0, 0, 7797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6927, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 28, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 12, 8164, 13, 0, 0, 19, 0, 2, 11, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8384, 8272, 8114, 8180, 9053, 7105, 8228, 8200] avg=8192.0000 max=9053 max_rank=4 min=7105 min_rank=5 imbalance_ratio=1.105103 max_load_reduction=56483 max_load_reduction_pct=0.861862 moved_to_echo=32586 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 28, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8155, 8183, 8202, 8195, 8070, 8345, 8187, 8199] avg=8192.0000 max=8345 max_rank=5 min=8070 min_rank=4 imbalance_ratio=1.018677 max_load_reduction=57191 max_load_reduction_pct=0.872665 moved_to_echo=32801 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 28, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8155, 8181, 8200, 8195, 8070, 8363, 8187, 8185] avg=8192.0000 max=8363 max_rank=5 min=8070 min_rank=4 imbalance_ratio=1.020874 max_load_reduction=57173 max_load_reduction_pct=0.872391 moved_to_echo=32805 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 73, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 12, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 121, 28, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8186, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8227, 8179, 8200, 8187, 8065, 8285, 8192, 8201] avg=8192.0000 max=8285 max_rank=5 min=8065 min_rank=4 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32743 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 28, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8155, 8170, 8201, 8195, 8070, 8359, 8187, 8199] avg=8192.0000 max=8359 max_rank=5 min=8070 min_rank=4 imbalance_ratio=1.020386 max_load_reduction=57177 max_load_reduction_pct=0.872452 moved_to_echo=32815 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 8148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 27, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8149, 8166, 8205, 8196, 8068, 8368, 8181, 8203] avg=8192.0000 max=8368 max_rank=5 min=8068 min_rank=4 imbalance_ratio=1.021484 max_load_reduction=57168 max_load_reduction_pct=0.872314 moved_to_echo=32820 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 27, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8156, 8165, 8199, 8195, 8070, 8362, 8187, 8202] avg=8192.0000 max=8362 max_rank=5 min=8070 min_rank=4 imbalance_ratio=1.020752 max_load_reduction=57174 max_load_reduction_pct=0.872406 moved_to_echo=32821 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 122, 27, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8155, 8220, 8215, 8193, 8070, 8349, 8187, 8147] avg=8192.0000 max=8349 max_rank=5 min=8070 min_rank=4 imbalance_ratio=1.019165 max_load_reduction=57187 max_load_reduction_pct=0.872604 moved_to_echo=32753 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 1280, 0, 0, 0, 64384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 65536, 128, 0, 0, 128, 0, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 0, 256, 128, 128, 0, 0, 0, 0, 0, 128, 0, 0, 65280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 65280, 0, 0, 0, 128, 0, 0, 0, 0, 0, 256, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[64256, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 1024, 384, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 384, 65280, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[64256, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 1024, 256, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 65280, 0, 0, 0, 128, 0, 0, 0, 0, 0, 256, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 128, 128, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 65536, 128, 0, 0, 128, 0, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 128, 0, 0, 65408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 7, 19, 0, 8009, 8166, 0, 0, 67, 0, 2690, 0, 0, 5, 0, 0, 0, 0, 0, 0, 28, 1, 0, 0, 7997, 0, 8034, 0, 0, 16, 0, 24, 0, 13, 0, 0, 8165, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 18, 5, 0, 0, 8, 0, 0, 2, 0, 0, 0, 0, 4, 0, 0, 0, 178, 0, 0, 18, 19, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 12, 0, 5612, 0, 0, 11, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 8183, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 8184, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 167, 0, 0, 0, 0, 0, 0, 8024, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 9, 18, 0, 8004, 8166, 0, 0, 66, 0, 2493, 0, 0, 4, 0, 0, 0, 0, 0, 0, 27, 1, 0, 0, 7986, 0, 8043, 0, 0, 17, 0, 17, 0, 13, 0, 0, 8165, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 19, 5, 0, 0, 8, 0, 0, 2, 0, 0, 0, 0, 3, 0, 0, 0, 181, 0, 0, 18, 18, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 12, 0, 5821, 0, 0, 11, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 4, 0, 8185, 8191, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 8188, 0, 0, 0, 0, 6, 0, 0, 0, 0, 8191, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8071, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 7, 19, 0, 8009, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 2690, 0, 0, 5, 0, 0, 0, 0, 0, 0, 28, 1, 0, 0, 7782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7997, 0, 40, 0, 0, 16, 0, 24, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 5, 0, 0, 8, 0, 0, 2, 0, 8153, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 178, 0, 0, 18, 19, 0, 0, 0, 0, 7983, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 334, 0, 0, 0, 0, 0, 0, 0, 12, 0, 5612, 0, 0, 11, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8102, 10573, 8090, 8197, 8245, 5978, 8176] avg=8192.0000 max=10573 max_rank=2 min=5978 min_rank=6 imbalance_ratio=1.290649 max_load_reduction=54963 max_load_reduction_pct=0.838669 moved_to_echo=30596 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 8184, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 167, 0, 0, 0, 0, 0, 0, 7814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8024, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8202, 7981, 8064, 8191, 8184, 8526, 8194] avg=8192.0000 max=8526 max_rank=6 min=7981 min_rank=2 imbalance_ratio=1.040771 max_load_reduction=57010 max_load_reduction_pct=0.869904 moved_to_echo=33095 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8201, 7815, 8231, 8191, 8184, 8526, 8195] avg=8192.0000 max=8526 max_rank=6 min=7815 min_rank=2 imbalance_ratio=1.040771 max_load_reduction=57010 max_load_reduction_pct=0.869904 moved_to_echo=33096 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 7814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8201, 7823, 8223, 8191, 8184, 8526, 8195] avg=8192.0000 max=8526 max_rank=6 min=7823 min_rank=2 imbalance_ratio=1.040771 max_load_reduction=57010 max_load_reduction_pct=0.869904 moved_to_echo=33096 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8188, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 4, 0, 8185, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 40, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8136, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 334, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8071, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8235, 7896, 8237, 8191, 8182, 8407, 8195] avg=8192.0000 max=8407 max_rank=6 min=7896 min_rank=2 imbalance_ratio=1.026245 max_load_reduction=57129 max_load_reduction_pct=0.871719 moved_to_echo=32975 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8191, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 7814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8201, 7816, 8232, 8191, 8184, 8526, 8193] avg=8192.0000 max=8526 max_rank=6 min=7816 min_rank=2 imbalance_ratio=1.040771 max_load_reduction=57010 max_load_reduction_pct=0.869904 moved_to_echo=33094 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8201, 7814, 8232, 8191, 8184, 8526, 8194] avg=8192.0000 max=8526 max_rank=6 min=7814 min_rank=2 imbalance_ratio=1.040771 max_load_reduction=57010 max_load_reduction_pct=0.869904 moved_to_echo=33095 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8193, 7818, 8227, 8193, 8189, 8521, 8194] avg=8192.0000 max=8521 max_rank=6 min=7818 min_rank=2 imbalance_ratio=1.040161 max_load_reduction=57015 max_load_reduction_pct=0.869980 moved_to_echo=33097 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 8184, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 4, 0, 8182, 8191, 0, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 8188, 0, 0, 0, 0, 8, 0, 0, 0, 0, 8191, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8053, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 8185, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 0, 0, 0, 0, 0, 0, 8012, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 9, 18, 0, 8004, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 2493, 0, 0, 4, 0, 0, 0, 0, 0, 0, 27, 1, 0, 0, 7804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7986, 0, 43, 0, 0, 17, 0, 17, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 5, 0, 0, 8, 0, 0, 2, 0, 8152, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 181, 0, 0, 18, 18, 0, 0, 0, 0, 7988, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 310, 0, 0, 0, 0, 0, 0, 0, 12, 0, 5821, 0, 0, 11, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8099, 10395, 8076, 8198, 8254, 6163, 8176] avg=8192.0000 max=10395 max_rank=2 min=6163 min_rank=6 imbalance_ratio=1.268921 max_load_reduction=55141 max_load_reduction_pct=0.841385 moved_to_echo=30791 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8138, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8192, 7838, 8232, 8196, 8186, 8498, 8194] avg=8192.0000 max=8498 max_rank=6 min=7838 min_rank=2 imbalance_ratio=1.037354 max_load_reduction=57038 max_load_reduction_pct=0.870331 moved_to_echo=33074 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8191, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 7835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8203, 7837, 8235, 8190, 8183, 8502, 8193] avg=8192.0000 max=8502 max_rank=6 min=7837 min_rank=2 imbalance_ratio=1.037842 max_load_reduction=57034 max_load_reduction_pct=0.870270 moved_to_echo=33068 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8203, 7836, 8234, 8190, 8183, 8502, 8195] avg=8192.0000 max=8502 max_rank=6 min=7836 min_rank=2 imbalance_ratio=1.037842 max_load_reduction=57034 max_load_reduction_pct=0.870270 moved_to_echo=33070 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 8188, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 4, 0, 8182, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 43, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 8132, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 310, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8053, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8231, 7938, 8242, 8190, 8181, 8365, 8195] avg=8192.0000 max=8365 max_rank=6 min=7938 min_rank=2 imbalance_ratio=1.021118 max_load_reduction=57171 max_load_reduction_pct=0.872360 moved_to_echo=32931 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 7835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8203, 7843, 8227, 8190, 8183, 8502, 8195] avg=8192.0000 max=8502 max_rank=6 min=7843 min_rank=2 imbalance_ratio=1.037842 max_load_reduction=57034 max_load_reduction_pct=0.870270 moved_to_echo=33070 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8203, 7835, 8235, 8190, 8183, 8502, 8194] avg=8192.0000 max=8502 max_rank=6 min=7835 min_rank=2 imbalance_ratio=1.037842 max_load_reduction=57034 max_load_reduction_pct=0.870270 moved_to_echo=33069 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 8185, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 0, 0, 0, 0, 0, 0, 7835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8012, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8202, 8014, 8055, 8192, 8183, 8502, 8194] avg=8192.0000 max=8502 max_rank=6 min=8014 min_rank=2 imbalance_ratio=1.037842 max_load_reduction=57034 max_load_reduction_pct=0.870270 moved_to_echo=33071 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[2688, 0, 0, 0, 0, 0, 0, 0, 128, 0, 62848, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 2816, 0, 0, 128, 0, 0, 0, 256, 0, 0, 128, 128, 0, 0, 62592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 0, 0, 128, 256, 0, 0, 128, 128, 0, 0, 0, 0, 65024, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[65280, 0, 384, 0, 0, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 2688, 0, 0, 128, 0, 0, 0, 256, 0, 0, 128, 128, 0, 0, 62720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[2560, 0, 0, 0, 0, 0, 0, 0, 128, 0, 63104, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[65152, 0, 384, 0, 0, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 128, 0, 0, 128, 0, 65408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 256, 0, 0, 128, 128, 0, 0, 0, 0, 65024, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 128, 0, 128, 128, 0, 65408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8191, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8190, 0, 8191, 0, 4263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8094, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 1, 0, 8173, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 96, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 9, 8014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8191, 0, 8124, 76, 0, 0, 0, 0, 0, 0, 0, 0, 156, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 25, 0, 0, 0, 8173, 0, 0, 8168, 0, 0, 0, 0, 0, 15, 0, 10, 5, 14, 0, 0, 8136, 0, 8173, 0, 1323, 0, 9, 0, 0, 0, 0, 0, 0, 0, 2189, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 7179, 0, 0, 0, 5072, 0, 0, 0, 7, 0, 6, 5, 0, 9, 0, 0, 0, 0, 7, 9, 0, 7652, 0, 8169, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 77, 0, 0, 0, 0, 0, 0, 467, 564, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 33, 8109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8190, 0, 4403, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3835, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 234, 8039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 6131, 8, 0, 0, 0, 0, 0, 0, 0, 0, 1971, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 14, 8125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 7568, 1, 0, 0, 0, 0, 0, 0, 0, 0, 671, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 64, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 5518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 55, 8082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 7611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 25, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 8168, 0, 0, 0, 0, 0, 15, 0, 10, 5, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 0, 0, 0, 1323, 0, 9, 0, 0, 0, 0, 0, 0, 0, 2189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 6735, 0, 0, 0, 5072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 6, 5, 0, 9, 0, 0, 0, 0, 7, 9, 0, 7652, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 77, 0, 8150, 12, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7569, 387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8235, 4076, 11831, 7752, 8190, 8256, 8987] avg=8192.0000 max=11831 max_rank=3 min=4076 min_rank=2 imbalance_ratio=1.444214 max_load_reduction=53705 max_load_reduction_pct=0.819473 moved_to_echo=33185 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 9, 8014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 0, 0, 0, 8124, 76, 0, 0, 0, 0, 0, 0, 0, 0, 156, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 12, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7621, 442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8046, 8930, 7685, 8257, 8192, 8174, 8063] avg=8192.0000 max=8930 max_rank=2 min=7685 min_rank=3 imbalance_ratio=1.090088 max_load_reduction=56606 max_load_reduction_pct=0.863739 moved_to_echo=32686 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 558, 0, 0, 0, 4263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7593, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 1, 0, 8173, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 8168, 12, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7620, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8215, 8733, 7621, 8242, 8195, 8185, 8154] avg=8192.0000 max=8733 max_rank=2 min=7621 min_rank=3 imbalance_ratio=1.066040 max_load_reduction=56803 max_load_reduction_pct=0.866745 moved_to_echo=32776 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 234, 8039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 0, 0, 0, 6131, 8, 0, 0, 0, 0, 0, 0, 0, 0, 1971, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7685, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 12, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7621, 442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8296, 8670, 7687, 8257, 8192, 8181, 8063] avg=8192.0000 max=8670 max_rank=2 min=7687 min_rank=3 imbalance_ratio=1.058350 max_load_reduction=56866 max_load_reduction_pct=0.867706 moved_to_echo=32693 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 14, 8125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 0, 0, 0, 7568, 1, 0, 0, 0, 0, 0, 0, 0, 0, 671, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 12, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7621, 442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8162, 8804, 7685, 8257, 8192, 8183, 8063] avg=8192.0000 max=8804 max_rank=2 min=7685 min_rank=3 imbalance_ratio=1.074707 max_load_reduction=56732 max_load_reduction_pct=0.865662 moved_to_echo=32695 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 33, 8109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 0, 0, 0, 4403, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3835, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 12, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7621, 442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8165, 8805, 7685, 8257, 8192, 8182, 8063] avg=8192.0000 max=8805 max_rank=2 min=7685 min_rank=3 imbalance_ratio=1.074829 max_load_reduction=56731 max_load_reduction_pct=0.865646 moved_to_echo=32694 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 64, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 0, 0, 0, 5518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 12, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7621, 442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8261, 8719, 7674, 8253, 8192, 8183, 8064] avg=8192.0000 max=8719 max_rank=2 min=7674 min_rank=3 imbalance_ratio=1.064331 max_load_reduction=56817 max_load_reduction_pct=0.866959 moved_to_echo=32692 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 55, 8082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 0, 0, 0, 7611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 69, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 17, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7627, 446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8156, 8799, 7668, 8261, 8191, 8192, 8079] avg=8192.0000 max=8799 max_rank=2 min=7668 min_rank=3 imbalance_ratio=1.074097 max_load_reduction=56737 max_load_reduction_pct=0.865738 moved_to_echo=32723 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 8, 8115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8190, 0, 3878, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4380, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 25, 0, 0, 0, 8174, 0, 0, 8169, 0, 0, 0, 0, 0, 14, 0, 10, 5, 14, 0, 0, 8135, 0, 8172, 0, 1056, 0, 9, 0, 0, 0, 0, 0, 0, 0, 3062, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 7126, 0, 0, 0, 4593, 0, 0, 0, 8, 0, 6, 5, 0, 9, 0, 0, 0, 0, 7, 8, 0, 7650, 0, 8169, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 484, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8190, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8190, 0, 8191, 0, 1677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8071, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 1, 0, 8170, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 118, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 88, 8087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 3560, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4643, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 6, 8082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 7379, 1, 0, 0, 0, 0, 0, 0, 0, 0, 912, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 42, 8071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 6671, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 7, 7993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8191, 0, 7989, 81, 0, 0, 0, 0, 0, 0, 0, 0, 307, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 42, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 2789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 25, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 8169, 0, 0, 0, 0, 0, 14, 0, 10, 5, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 449, 0, 0, 0, 1056, 0, 9, 0, 0, 0, 0, 0, 0, 0, 3062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 6747, 0, 0, 0, 4593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 6, 5, 0, 9, 0, 0, 0, 0, 7, 8, 0, 7650, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 71, 0, 8122, 41, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7645, 322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8264, 4576, 11363, 7750, 8190, 8250, 8934] avg=8192.0000 max=11363 max_rank=3 min=4576 min_rank=2 imbalance_ratio=1.387085 max_load_reduction=54173 max_load_reduction_pct=0.826614 moved_to_echo=33124 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 8, 8115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 452, 0, 0, 0, 3878, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4380, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8138, 42, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7698, 370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8175, 8717, 7756, 8258, 8192, 8183, 8068] avg=8192.0000 max=8717 max_rank=2 min=7756 min_rank=3 imbalance_ratio=1.064087 max_load_reduction=56819 max_load_reduction_pct=0.866989 moved_to_echo=32701 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 88, 8087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 452, 0, 0, 0, 3560, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4643, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7756, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8138, 42, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7698, 370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8227, 8662, 7757, 8258, 8192, 8182, 8068] avg=8192.0000 max=8662 max_rank=2 min=7757 min_rank=3 imbalance_ratio=1.057373 max_load_reduction=56874 max_load_reduction_pct=0.867828 moved_to_echo=32700 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 6, 8082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 452, 0, 0, 0, 7379, 1, 0, 0, 0, 0, 0, 0, 0, 0, 912, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 42, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7698, 370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8140, 8748, 7756, 8258, 8192, 8184, 8068] avg=8192.0000 max=8748 max_rank=2 min=7756 min_rank=3 imbalance_ratio=1.067871 max_load_reduction=56788 max_load_reduction_pct=0.866516 moved_to_echo=32702 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 451, 0, 0, 0, 1677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7642, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 1, 0, 8170, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 8138, 42, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7697, 364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8244, 8639, 7662, 8240, 8195, 8185, 8180] avg=8192.0000 max=8639 max_rank=2 min=7662 min_rank=3 imbalance_ratio=1.054565 max_load_reduction=56897 max_load_reduction_pct=0.868179 moved_to_echo=32800 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 42, 8071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 443, 0, 0, 0, 6671, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7741, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 69, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8147, 45, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7704, 373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8158, 8720, 7741, 8260, 8191, 8194, 8082] avg=8192.0000 max=8720 max_rank=2 min=7741 min_rank=3 imbalance_ratio=1.064453 max_load_reduction=56816 max_load_reduction_pct=0.866943 moved_to_echo=32727 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 42, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 452, 0, 0, 0, 2789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 42, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7698, 369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8276, 8627, 7745, 8254, 8192, 8184, 8068] avg=8192.0000 max=8627 max_rank=2 min=7745 min_rank=3 imbalance_ratio=1.053101 max_load_reduction=56909 max_load_reduction_pct=0.868362 moved_to_echo=32698 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 7, 7993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 452, 0, 0, 0, 7989, 81, 0, 0, 0, 0, 0, 0, 0, 0, 307, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8130, 42, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7698, 370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8052, 8847, 7756, 8258, 8192, 8174, 8068] avg=8192.0000 max=8847 max_rank=2 min=7756 min_rank=3 imbalance_ratio=1.079956 max_load_reduction=56689 max_load_reduction_pct=0.865005 moved_to_echo=32692 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 640, 640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 60928, 3584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[4480, 0, 0, 0, 45056, 128, 128, 0, 0, 0, 0, 0, 0, 0, 16128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 60416, 0, 0, 0, 5120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 65536, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[3712, 0, 0, 0, 35072, 128, 128, 0, 0, 0, 0, 0, 0, 0, 26880, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 640, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 61568, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 60928, 0, 0, 0, 4736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 0, 128, 0, 128, 128, 0, 128, 0, 0, 128, 0, 128, 128, 0, 65024, 640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[256, 0, 512, 65024, 0, 0, 0, 0, 0, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 65408, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 65536, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 128, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 65152, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[512, 0, 256, 64896, 0, 0, 0, 0, 0, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True - [2026-08-26 14:48:12] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 437.4 | throughput per GPU (TFLOP/s/GPU): 413.4 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 1.031592E+01 | loss scale: 1.0 | grad norm: 6.319 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 567, 0, 0, 0, 220, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 884, 0, 0, 2932, 0, 0, 0, 8192, 1, 8192, 4, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 2, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 7, 0, 0, 0, 4771, 0, 0, 0, 6507, 0, 7934, 5, 0, 0, 1243, 0, 0, 0, 0, 0, 14, 0, 0, 7828, 0, 2, 0, 0, 1, 0, 22, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8009, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 16, 483, 0, 0, 0, 463, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 870, 0, 1, 2717, 0, 0, 0, 8192, 0, 8192, 3, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8188, 0, 0, 0, 1, 0, 0, 0, 0, 4300, 0, 0, 0, 6795, 0, 7419, 3, 0, 0, 2055, 0, 0, 0, 0, 0, 0, 0, 0, 7696, 0, 1, 0, 0, 3, 0, 18, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 7906, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5, 756, 0, 0, 0, 149, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 776, 0, 0, 2343, 3, 0, 0, 8192, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 4979, 0, 0, 0, 6917, 0, 7863, 1, 0, 0, 1853, 0, 0, 0, 0, 0, 0, 0, 0, 7414, 0, 3, 0, 0, 1, 0, 10, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7875, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 465, 0, 0, 0, 121, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 914, 0, 0, 2785, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 4737, 0, 0, 0, 6703, 0, 7931, 4, 0, 0, 1512, 0, 0, 0, 0, 0, 0, 0, 0, 7663, 0, 1, 0, 0, 6, 0, 69, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 5, 7996, 0, 0, 2, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[0, 1, 0, 2, 4, 363, 0, 0, 0, 210, 0, 0, 0, 0, 0, 8, 0, 1, 0, 0, 669, 0, 1, 2766, 4, 0, 0, 8191, 1, 8191, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 4812, 0, 0, 0, 6874, 0, 7828, 7, 1, 0, 1659, 0, 0, 0, 0, 0, 0, 0, 0, 7653, 0, 0, 0, 0, 2, 0, 20, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8066, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 450, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 681, 0, 2, 1960, 1, 0, 0, 8192, 0, 8190, 1, 0, 0, 0, 0, 0, 1, 0, 0, 2, 4, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8182, 0, 1, 0, 0, 1, 0, 0, 0, 5352, 0, 1, 0, 6678, 0, 7767, 0, 4, 0, 2002, 0, 0, 1, 1, 3, 3, 0, 0, 7912, 0, 0, 0, 0, 5, 0, 31, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7992, 0, 0, 4, 1, 2, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 13, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 1021, 1, 0, 0, 349, 0, 2, 0, 2, 0, 14, 0, 1, 1, 0, 1032, 0, 0, 2564, 0, 0, 0, 8189, 0, 8188, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 4, 0, 0, 0, 23, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 1, 0, 0, 0, 0, 4266, 0, 0, 0, 6476, 0, 7490, 8, 0, 0, 2190, 0, 0, 2, 0, 0, 0, 0, 1, 7260, 0, 2, 0, 2, 7, 3, 40, 0, 0, 3, 1, 0, 1, 0, 0, 0, 4, 4, 8042, 0, 0, 5, 1, 0, 0, 3, 0, 0, 1, 1, 0, 0, 0, 0, 0, 166, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[0, 2, 1, 0, 7, 631, 1, 0, 0, 427, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1035, 0, 0, 2162, 1, 0, 0, 8191, 0, 8191, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 4, 0, 5, 4, 0, 2, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 3855, 0, 0, 0, 6865, 0, 7725, 31, 1, 0, 2270, 0, 0, 0, 0, 0, 0, 0, 0, 7913, 0, 0, 1, 2, 0, 1, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7976, 0, 0, 4, 0, 0, 0, 1, 0, 2, 1, 1, 4, 2, 0, 0, 0, 24, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 567, 0, 0, 0, 220, 0, 0, 1, 0, 0, 0, 4495, 2686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 884, 0, 0, 2932, 0, 0, 0, 0, 1, 4805, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 2, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4771, 0, 0, 0, 1646, 0, 0, 5, 0, 0, 1243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 7828, 0, 2, 0, 0, 1, 0, 22, 0, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8009, 0, 0, 2, 0, 0, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 7934, 366, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7970, 8626, 8188, 8193, 7672, 8354, 8215, 8318] avg=8192.0000 max=8626 max_rank=1 min=7672 min_rank=4 imbalance_ratio=1.052979 max_load_reduction=56910 max_load_reduction_pct=0.868378 moved_to_echo=32559 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 2, 4, 363, 0, 0, 0, 210, 0, 0, 0, 0, 0, 8, 4749, 2686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 669, 0, 1, 2766, 4, 0, 0, 0, 1, 4804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4812, 0, 0, 0, 1739, 0, 0, 7, 1, 0, 1659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7653, 0, 0, 0, 0, 2, 0, 20, 0, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8066, 0, 0, 1, 0, 0, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 7828, 386, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8023, 8246, 8186, 8196, 8218, 8162, 8270, 8235] avg=8192.0000 max=8270 max_rank=6 min=8023 min_rank=0 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32885 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 16, 483, 0, 0, 0, 463, 0, 0, 0, 0, 3, 2, 4694, 2686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 870, 0, 1, 2717, 0, 0, 0, 0, 0, 4805, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8188, 0, 0, 0, 1, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4300, 0, 0, 0, 1719, 0, 0, 3, 0, 0, 2055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7696, 0, 1, 0, 0, 3, 0, 18, 0, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 7906, 0, 0, 2, 2, 0, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 0, 0, 0, 0, 7419, 382, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8348, 8396, 8189, 8203, 8077, 8205, 8115, 8003] avg=8192.0000 max=8396 max_rank=1 min=8003 min_rank=7 imbalance_ratio=1.024902 max_load_reduction=57140 max_load_reduction_pct=0.871887 moved_to_echo=32400 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 465, 0, 0, 0, 121, 0, 0, 0, 0, 0, 25, 4631, 2686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 914, 0, 0, 2785, 0, 0, 0, 0, 0, 4805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4737, 0, 0, 0, 1695, 0, 0, 4, 0, 0, 1512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7663, 0, 1, 0, 0, 6, 0, 69, 0, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 5, 7996, 0, 0, 2, 0, 0, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 1, 0, 0, 0, 7931, 377, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7929, 8504, 8189, 8190, 7948, 8226, 8206, 8344] avg=8192.0000 max=8504 max_rank=1 min=7929 min_rank=0 imbalance_ratio=1.038086 max_load_reduction=57032 max_load_reduction_pct=0.870239 moved_to_echo=32724 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5, 756, 0, 0, 0, 149, 1, 0, 0, 0, 0, 0, 4779, 2686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 776, 0, 0, 2343, 3, 0, 0, 0, 0, 4805, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4979, 0, 0, 0, 1749, 0, 0, 1, 0, 0, 1853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7414, 0, 3, 0, 0, 1, 0, 10, 0, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7875, 0, 0, 0, 0, 0, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 7863, 389, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8376, 7928, 8182, 8201, 8582, 7915, 8078, 8274] avg=8192.0000 max=8582 max_rank=4 min=7915 min_rank=5 imbalance_ratio=1.047607 max_load_reduction=56954 max_load_reduction_pct=0.869049 moved_to_echo=32849 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 1021, 1, 0, 0, 349, 0, 2, 0, 2, 0, 14, 4474, 2685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1032, 0, 0, 2564, 0, 0, 0, 0, 0, 4802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 4, 0, 0, 0, 23, 2, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 1, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4266, 0, 0, 0, 1638, 0, 0, 8, 0, 0, 2190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 7260, 0, 2, 0, 2, 7, 3, 40, 0, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 1, 0, 0, 0, 4, 4, 8042, 0, 0, 5, 1, 0, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 1, 0, 0, 0, 0, 0, 166, 0, 0, 0, 0, 7490, 364, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8549, 8400, 8210, 8172, 8102, 7804, 8263, 8036] avg=8192.0000 max=8549 max_rank=0 min=7804 min_rank=5 imbalance_ratio=1.043579 max_load_reduction=56987 max_load_reduction_pct=0.869553 moved_to_echo=32205 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 450, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 4613, 2685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 681, 0, 2, 1960, 1, 0, 0, 0, 0, 4804, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 4, 0, 0, 5, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8182, 0, 1, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5352, 0, 1, 0, 1690, 0, 0, 0, 4, 0, 2002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 3, 3, 0, 0, 7912, 0, 0, 0, 0, 5, 0, 31, 1, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7992, 0, 0, 4, 1, 2, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 13, 0, 0, 0, 0, 7767, 375, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7832, 7450, 8194, 8196, 9050, 8444, 8202, 8168] avg=8192.0000 max=9050 max_rank=4 min=7450 min_rank=1 imbalance_ratio=1.104736 max_load_reduction=56486 max_load_reduction_pct=0.861908 moved_to_echo=33864 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 1, 0, 7, 631, 1, 0, 0, 427, 0, 0, 0, 0, 0, 3, 4747, 2690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 0, 0, 2162, 1, 0, 0, 0, 0, 4788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 4, 0, 5, 4, 0, 2, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3855, 0, 0, 0, 1730, 0, 0, 31, 1, 0, 2270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7913, 0, 0, 1, 2, 0, 1, 20, 0, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7976, 0, 0, 4, 0, 0, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 1, 1, 4, 2, 0, 0, 0, 24, 0, 0, 0, 0, 7725, 388, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8509, 7986, 8198, 8185, 7887, 8426, 8187, 8158] avg=8192.0000 max=8509 max_rank=0 min=7887 min_rank=4 imbalance_ratio=1.038696 max_load_reduction=57027 max_load_reduction_pct=0.870163 moved_to_echo=32658 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True + [2026-08-26 15:24:13] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 460.6 | throughput per GPU (TFLOP/s/GPU): 392.6 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 1.031625E+01 | loss scale: 1.0 | grad norm: 6.283 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 21, 558, 0, 0, 0, 431, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 1066, 0, 1, 3006, 0, 0, 0, 8192, 0, 8192, 3, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8188, 0, 0, 0, 1, 0, 0, 0, 0, 4431, 0, 0, 0, 6989, 0, 7530, 3, 0, 0, 2167, 0, 0, 0, 0, 0, 0, 0, 0, 6501, 0, 7, 0, 0, 3, 0, 20, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 7933, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 272, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 535, 0, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 803, 0, 2, 2360, 1, 0, 0, 8192, 0, 8190, 1, 1, 0, 0, 0, 0, 1, 0, 0, 2, 4, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8183, 0, 1, 0, 0, 1, 0, 0, 0, 5397, 0, 1, 0, 6901, 0, 7845, 0, 3, 0, 2222, 0, 0, 1, 1, 3, 4, 0, 0, 6677, 0, 0, 0, 0, 5, 0, 32, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8013, 0, 0, 4, 2, 2, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 66, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 1195, 0, 0, 0, 358, 0, 2, 0, 2, 0, 14, 0, 1, 1, 0, 1306, 0, 0, 2925, 0, 0, 0, 8189, 0, 8188, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 4, 0, 0, 0, 21, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 1, 0, 0, 0, 0, 4408, 0, 0, 0, 6682, 0, 7551, 9, 0, 0, 2428, 0, 0, 2, 0, 0, 0, 0, 2, 5717, 0, 2, 0, 2, 7, 3, 45, 0, 0, 3, 1, 0, 1, 0, 0, 0, 4, 4, 8073, 0, 0, 5, 1, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 207, 0, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 903, 0, 0, 0, 152, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1082, 0, 0, 2808, 4, 0, 0, 8192, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 5129, 0, 0, 0, 7097, 0, 7935, 2, 0, 0, 2115, 0, 0, 0, 0, 0, 0, 0, 0, 5768, 0, 3, 0, 0, 1, 0, 13, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7915, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[0, 2, 1, 1, 8, 719, 1, 0, 0, 363, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1213, 0, 0, 2444, 1, 0, 0, 8191, 0, 8191, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 4, 0, 5, 4, 0, 2, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 3930, 0, 0, 0, 7000, 0, 7780, 33, 1, 0, 2406, 0, 0, 0, 0, 0, 0, 0, 0, 6835, 0, 1, 1, 2, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7998, 0, 0, 3, 0, 0, 0, 1, 0, 2, 1, 1, 3, 2, 0, 0, 0, 182, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[0, 1, 0, 2, 5, 420, 0, 0, 0, 191, 0, 0, 0, 0, 0, 7, 0, 1, 0, 0, 923, 0, 1, 3219, 4, 0, 0, 8191, 1, 8191, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 4904, 0, 0, 0, 7083, 0, 7887, 8, 1, 0, 1860, 0, 0, 0, 0, 0, 0, 0, 0, 6306, 0, 0, 0, 0, 2, 0, 37, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8085, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 528, 0, 0, 0, 121, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 1171, 0, 0, 3231, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 4828, 0, 0, 0, 6952, 0, 7959, 6, 0, 0, 1621, 0, 0, 0, 0, 0, 0, 0, 0, 6387, 0, 1, 0, 0, 5, 0, 71, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 6, 8016, 0, 0, 2, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 654, 0, 0, 0, 209, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1122, 0, 0, 3419, 0, 0, 0, 8192, 1, 8192, 5, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 2, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 7, 0, 0, 0, 4839, 0, 0, 0, 6693, 0, 7987, 6, 0, 0, 1393, 0, 0, 0, 0, 0, 14, 0, 0, 6543, 0, 2, 0, 0, 1, 0, 24, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8019, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 21, 558, 0, 0, 0, 431, 0, 0, 0, 0, 3, 2, 5283, 2018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1066, 0, 1, 3006, 0, 0, 0, 0, 0, 4178, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8188, 0, 0, 0, 1, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4431, 0, 0, 0, 1436, 0, 0, 3, 0, 0, 2167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6501, 0, 7, 0, 0, 3, 0, 20, 0, 1807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 7933, 0, 0, 2, 2, 0, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 272, 0, 0, 0, 0, 7530, 270, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8317, 8254, 8189, 8202, 8037, 8338, 8117, 8082] avg=8192.0000 max=8338 max_rank=5 min=8037 min_rank=4 imbalance_ratio=1.017822 max_load_reduction=57198 max_load_reduction_pct=0.872772 moved_to_echo=32574 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 1195, 0, 0, 0, 358, 0, 2, 0, 2, 0, 14, 5051, 2017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1306, 0, 0, 2925, 0, 0, 0, 0, 0, 4176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 4, 0, 0, 0, 21, 2, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 1, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4408, 0, 0, 0, 1373, 0, 0, 9, 0, 0, 2428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 5717, 0, 2, 0, 2, 7, 3, 45, 0, 1806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 1, 0, 0, 0, 4, 4, 8073, 0, 0, 5, 1, 0, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 207, 0, 1, 0, 0, 7551, 258, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8640, 8409, 8208, 8174, 8218, 7586, 8270, 8031] avg=8192.0000 max=8640 max_rank=0 min=7586 min_rank=5 imbalance_ratio=1.054688 max_load_reduction=56896 max_load_reduction_pct=0.868164 moved_to_echo=32105 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 535, 0, 0, 0, 68, 0, 0, 0, 0, 0, 0, 5216, 2017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 803, 0, 2, 2360, 1, 0, 0, 0, 0, 4178, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 4, 0, 0, 4, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8183, 0, 1, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5397, 0, 1, 0, 1418, 0, 0, 0, 3, 0, 2222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 3, 4, 0, 0, 6677, 0, 0, 0, 0, 5, 0, 32, 1, 1806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8013, 0, 0, 4, 2, 2, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 66, 0, 0, 0, 0, 7845, 267, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7837, 7347, 8194, 8196, 9042, 8530, 8200, 8190] avg=8192.0000 max=9042 max_rank=4 min=7347 min_rank=1 imbalance_ratio=1.103760 max_load_reduction=56494 max_load_reduction_pct=0.862030 moved_to_echo=33962 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 1, 1, 8, 719, 1, 0, 0, 363, 0, 0, 0, 0, 0, 3, 5294, 2020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1213, 0, 0, 2444, 1, 0, 0, 0, 0, 4163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 4, 0, 5, 4, 0, 2, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3930, 0, 0, 0, 1433, 0, 0, 33, 1, 0, 2406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6835, 0, 1, 1, 2, 0, 0, 27, 0, 1810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7998, 0, 0, 3, 0, 0, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 1, 1, 3, 2, 0, 0, 0, 182, 0, 0, 0, 0, 7780, 273, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8412, 7821, 8199, 8185, 7803, 8676, 8186, 8254] avg=8192.0000 max=8676 max_rank=5 min=7803 min_rank=4 imbalance_ratio=1.059082 max_load_reduction=56860 max_load_reduction_pct=0.867615 moved_to_echo=32919 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 2, 5, 420, 0, 0, 0, 191, 0, 0, 0, 0, 0, 7, 5354, 2017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 923, 0, 1, 3219, 4, 0, 0, 0, 1, 4178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4904, 0, 0, 0, 1455, 0, 0, 8, 1, 0, 1860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6306, 0, 0, 0, 0, 2, 0, 37, 0, 1807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8085, 0, 0, 1, 0, 0, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 7887, 274, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7997, 8327, 8187, 8195, 8228, 8152, 8266, 8184] avg=8192.0000 max=8327 max_rank=1 min=7997 min_rank=0 imbalance_ratio=1.016479 max_load_reduction=57209 max_load_reduction_pct=0.872940 moved_to_echo=32830 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 903, 0, 0, 0, 152, 1, 0, 0, 0, 0, 0, 5364, 2018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1082, 0, 0, 2808, 4, 0, 0, 0, 0, 4178, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5129, 0, 0, 0, 1459, 0, 0, 2, 0, 0, 2115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5768, 0, 3, 0, 0, 1, 0, 13, 0, 1807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7915, 0, 0, 0, 0, 0, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 7935, 274, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8444, 8073, 8183, 8201, 8705, 7592, 8094, 8244] avg=8192.0000 max=8705 max_rank=4 min=7592 min_rank=5 imbalance_ratio=1.062622 max_load_reduction=56831 max_load_reduction_pct=0.867172 moved_to_echo=32635 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 654, 0, 0, 0, 209, 0, 0, 1, 0, 0, 0, 5059, 2018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1122, 0, 0, 3419, 0, 0, 0, 0, 1, 4178, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 2, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4839, 0, 0, 0, 1375, 0, 0, 6, 0, 0, 1393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 6543, 0, 2, 0, 0, 1, 0, 24, 0, 1807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8019, 0, 0, 1, 0, 0, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 7987, 259, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7942, 8725, 8188, 8193, 7620, 8391, 8200, 8277] avg=8192.0000 max=8725 max_rank=1 min=7620 min_rank=4 imbalance_ratio=1.065063 max_load_reduction=56811 max_load_reduction_pct=0.866867 moved_to_echo=32488 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 528, 0, 0, 0, 121, 0, 0, 0, 0, 0, 24, 5255, 2018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1171, 0, 0, 3231, 0, 0, 0, 0, 0, 4178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4828, 0, 0, 0, 1428, 0, 0, 6, 0, 0, 1621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6387, 0, 1, 0, 0, 5, 0, 71, 0, 1807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 6, 8016, 0, 0, 2, 0, 0, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 1, 0, 0, 0, 7959, 269, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7947, 8580, 8188, 8190, 7883, 8271, 8203, 8274] avg=8192.0000 max=8580 max_rank=1 min=7883 min_rank=4 imbalance_ratio=1.047363 max_load_reduction=56956 max_load_reduction_pct=0.869080 moved_to_echo=32631 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 128, 4736, 128, 0, 0, 2048, 128, 128, 128, 128, 128, 128, 37248, 21504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 0, 6912, 0, 128, 20352, 128, 0, 0, 0, 128, 38528, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 37120, 0, 128, 0, 13696, 0, 0, 128, 128, 0, 14848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 128, 128, 128, 128, 128, 128, 128, 128, 0, 0, 512, 128, 0, 0, 0, 62080, 3072, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 128, 128, 128, 128, 0, 128, 61440, 0, 128, 128, 128, 128, 128, 256, 128, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 128, 128, 0, 128, 0, 0, 128, 128, 128, 63872, 0, 0, 128, 128, 128, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 0, 128, 0, 128, 128, 0, 128, 128, 0, 128, 128, 0, 128, 128, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=67328 observed_loss=-1792 tokens_per_expert=[128, 128, 128, 128, 128, 5632, 128, 0, 0, 1920, 128, 128, 128, 128, 128, 128, 41984, 16256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 0, 8704, 0, 128, 23424, 128, 0, 0, 0, 128, 33408, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 37888, 0, 128, 0, 11392, 0, 0, 128, 128, 0, 16256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 128, 128, 128, 128, 0, 128, 50816, 0, 128, 128, 128, 128, 128, 384, 128, 14464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=67328 slack_slots=63744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 128, 128, 128, 128, 128, 128, 128, 128, 0, 0, 896, 128, 128, 0, 0, 62592, 2176, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 0, 128, 0, 0, 128, 128, 128, 64128, 0, 0, 128, 128, 128, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 0, 128, 0, 128, 128, 0, 128, 128, 0, 128, 128, 0, 128, 128, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 0, 0, 0, 0, 128, 0, 0, 128, 65536, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[8191, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8191, 8191, 1, 8166, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[8190, 0, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8190, 8189, 0, 8179, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 7969, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 241, 8188, 0, 0, 0, 0, 2, 2, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8175, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8188, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 8104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 8136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 21, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8127, 8221, 8225, 8199, 8191, 8192, 8191] avg=8192.0000 max=8225 max_rank=3 min=8127 min_rank=1 imbalance_ratio=1.004028 max_load_reduction=57311 max_load_reduction_pct=0.874496 moved_to_echo=32773 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 8136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8210, 8210, 8153, 8199, 8192, 8194, 8190] avg=8192.0000 max=8210 max_rank=1 min=8153 min_rank=3 imbalance_ratio=1.002197 max_load_reduction=57326 max_load_reduction_pct=0.874725 moved_to_echo=32775 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8188, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8208, 8222, 8143, 8182, 8192, 8194, 8192] avg=8192.0000 max=8222 max_rank=2 min=8143 min_rank=3 imbalance_ratio=1.003662 max_load_reduction=57314 max_load_reduction_pct=0.874542 moved_to_echo=32760 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8188, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8208, 8222, 8142, 8197, 8192, 8194, 8190] avg=8192.0000 max=8222 max_rank=2 min=8142 min_rank=3 imbalance_ratio=1.003662 max_load_reduction=57314 max_load_reduction_pct=0.874542 moved_to_echo=32773 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 8136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8181, 8222, 8172, 8197, 8194, 8192, 8190] avg=8192.0000 max=8222 max_rank=2 min=8172 min_rank=3 imbalance_ratio=1.003662 max_load_reduction=57314 max_load_reduction_pct=0.874542 moved_to_echo=32773 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8187, 0, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 23, 0, 8179, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7969, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 241, 8133, 0, 0, 0, 0, 2, 2, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 1, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8175, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8166, 21, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8206, 8000, 8382, 8180, 8192, 8178, 8193] avg=8192.0000 max=8382 max_rank=3 min=8000 min_rank=2 imbalance_ratio=1.023193 max_load_reduction=57154 max_load_reduction_pct=0.872101 moved_to_echo=32743 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 8136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8208, 8217, 8165, 8182, 8192, 8194, 8190] avg=8192.0000 max=8217 max_rank=2 min=8165 min_rank=3 imbalance_ratio=1.003052 max_load_reduction=57319 max_load_reduction_pct=0.874619 moved_to_echo=32758 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8181, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 19, 1, 8166, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 8129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8190, 0, 0, 0, 0, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 8172, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8188, 8222, 8154, 8200, 8191, 8198, 8200] avg=8192.0000 max=8222 max_rank=2 min=8154 min_rank=3 imbalance_ratio=1.003662 max_load_reduction=57314 max_load_reduction_pct=0.874542 moved_to_echo=32789 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8186, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3351, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4842, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[8191, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8191, 8191, 1, 8187, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1265, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6930, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[8190, 0, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8190, 8189, 0, 8186, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2826, 8188, 0, 0, 0, 0, 2, 2, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5396, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1856, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1646, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8190, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 703, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8187, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3351, 6608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4842, 1570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8200, 8197, 9959, 8183, 8192, 6412, 8192] avg=8192.0000 max=9959 max_rank=3 min=6412 min_rank=6 imbalance_ratio=1.215698 max_load_reduction=55577 max_load_reduction_pct=0.848038 moved_to_echo=30979 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8186, 0, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8, 0, 8186, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2826, 6606, 0, 0, 0, 0, 2, 2, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5396, 1569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8181, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8198, 8161, 9440, 8181, 8192, 6966, 8194] avg=8192.0000 max=9440 max_rank=3 min=6966 min_rank=6 imbalance_ratio=1.152344 max_load_reduction=56096 max_load_reduction_pct=0.855957 moved_to_echo=31533 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8187, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 8186, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 6608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7678, 1570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8196, 8197, 7125, 8196, 8192, 9248, 8190] avg=8192.0000 max=9248 max_rank=6 min=7125 min_rank=3 imbalance_ratio=1.128906 max_load_reduction=56288 max_load_reduction_pct=0.858887 moved_to_echo=33826 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8185, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 1, 8187, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1265, 6600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8190, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6930, 1571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 8189, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8192, 8201, 7866, 8199, 8191, 8501, 8199] avg=8192.0000 max=8501 max_rank=6 min=7866 min_rank=3 imbalance_ratio=1.037720 max_load_reduction=57035 max_load_reduction_pct=0.870285 moved_to_echo=33090 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8187, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 6608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7710, 1570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8171, 8197, 7117, 8198, 8194, 9280, 8190] avg=8192.0000 max=9280 max_rank=6 min=7117 min_rank=3 imbalance_ratio=1.132812 max_load_reduction=56256 max_load_reduction_pct=0.858398 moved_to_echo=33862 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8187, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1856, 6608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6350, 1570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8186, 8196, 8464, 8199, 8191, 7920, 8191] avg=8192.0000 max=8464 max_rank=3 min=7920 min_rank=6 imbalance_ratio=1.033203 max_load_reduction=57072 max_load_reduction_pct=0.870850 moved_to_echo=32501 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 8190, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 703, 6608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7493, 1570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8200, 8194, 7311, 8199, 8192, 9063, 8190] avg=8192.0000 max=9063 max_rank=6 min=7311 min_rank=3 imbalance_ratio=1.106323 max_load_reduction=56473 max_load_reduction_pct=0.861710 moved_to_echo=33644 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1646, 6608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6576, 1570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8193, 8193, 8254, 8181, 8192, 8146, 8190] avg=8192.0000 max=8254 max_rank=3 min=8146 min_rank=6 imbalance_ratio=1.007568 max_load_reduction=57282 max_load_reduction_pct=0.874054 moved_to_echo=32709 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 128, 65408, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 0, 0, 256, 128, 65408, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 65408, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[512, 65152, 0, 0, 0, 0, 128, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 65536, 0, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 52992, 12672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[12800, 52864, 0, 0, 0, 0, 128, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[65536, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 128, 65536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 65536, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 0, 0, 128, 128, 65536, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 65536, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 64, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 3, 8187, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 2, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 6861, 0, 1, 1301, 0, 8137, 0, 8188, 0, 3, 0, 0, 1, 0, 0, 0, 8188, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6626, 0, 0, 1566, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7951, 0, 0, 243, 0, 8190, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7386, 0, 0, 806, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 5, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 30, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7461, 0, 0, 700, 0, 8185, 0, 8191, 0, 1, 0, 0, 1, 0, 0, 0, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7979, 0, 0, 213, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7267, 0, 0, 924, 0, 8192, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8037, 0, 0, 155, 0, 8192, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7951, 0, 0, 731, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 0, 7459, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8193, 8182, 8192, 8190, 8191, 8694, 7702] avg=8192.0000 max=8694 max_rank=6 min=7702 min_rank=7 imbalance_ratio=1.061279 max_load_reduction=56842 max_load_reduction_pct=0.867340 moved_to_echo=32777 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6626, 0, 0, 731, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1566, 0, 7461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8192, 8182, 8192, 8192, 8191, 7370, 9027] avg=8192.0000 max=9027 max_rank=7 min=7370 min_rank=6 imbalance_ratio=1.101929 max_load_reduction=56509 max_load_reduction_pct=0.862259 moved_to_echo=32780 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7267, 0, 0, 731, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 924, 0, 7461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8195, 8182, 8192, 8192, 8189, 8011, 8385] avg=8192.0000 max=8385 max_rank=7 min=8011 min_rank=6 imbalance_ratio=1.023560 max_load_reduction=57151 max_load_reduction_pct=0.872055 moved_to_echo=32777 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7386, 0, 0, 731, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 806, 0, 7461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8192, 8182, 8192, 8191, 8191, 8130, 8267] avg=8192.0000 max=8267 max_rank=7 min=8130 min_rank=6 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32779 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 2, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8187, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 8185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 6861, 0, 1, 727, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1301, 0, 7410, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8189, 8237, 8194, 8195, 8203, 7610, 8715] avg=8192.0000 max=8715 max_rank=7 min=7610 min_rank=6 imbalance_ratio=1.063843 max_load_reduction=56821 max_load_reduction_pct=0.867020 moved_to_echo=32723 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7979, 0, 0, 731, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 213, 0, 7461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8192, 8182, 8192, 8192, 8191, 8723, 7674] avg=8192.0000 max=8723 max_rank=6 min=7674 min_rank=7 imbalance_ratio=1.064819 max_load_reduction=56813 max_load_reduction_pct=0.866898 moved_to_echo=32780 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8037, 0, 0, 731, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 0, 7461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8193, 8182, 8192, 8191, 8189, 8781, 7616] avg=8192.0000 max=8781 max_rank=6 min=7616 min_rank=7 imbalance_ratio=1.071899 max_load_reduction=56755 max_load_reduction_pct=0.866013 moved_to_echo=32777 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 5, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7461, 0, 0, 737, 12, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 700, 0, 7448, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8190, 8207, 8190, 8193, 8191, 8217, 8150] avg=8192.0000 max=8217 max_rank=6 min=8150 min_rank=7 imbalance_ratio=1.003052 max_load_reduction=57319 max_load_reduction_pct=0.874619 moved_to_echo=32751 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6793, 0, 0, 1398, 0, 8192, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7825, 0, 0, 367, 0, 8192, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 65, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 3, 8187, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 2, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 6515, 0, 1, 1647, 0, 8136, 0, 8188, 0, 3, 0, 0, 1, 0, 0, 0, 8188, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7130, 0, 0, 1062, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6085, 0, 0, 2107, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7873, 0, 0, 321, 0, 8190, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 27, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 6745, 0, 0, 1419, 0, 8185, 0, 8191, 0, 1, 0, 0, 1, 0, 0, 0, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7678, 0, 0, 514, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6793, 0, 0, 1098, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1398, 0, 7094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8195, 8182, 8192, 8192, 8189, 7904, 8492] avg=8192.0000 max=8492 max_rank=7 min=7904 min_rank=6 imbalance_ratio=1.036621 max_load_reduction=57044 max_load_reduction_pct=0.870422 moved_to_echo=32777 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 2, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8187, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 8185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 6515, 0, 1, 1090, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1647, 0, 7046, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8189, 8239, 8194, 8195, 8203, 7626, 8697] avg=8192.0000 max=8697 max_rank=7 min=7626 min_rank=6 imbalance_ratio=1.061646 max_load_reduction=56839 max_load_reduction_pct=0.867294 moved_to_echo=32721 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6085, 0, 0, 1098, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2107, 0, 7094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8192, 8182, 8192, 8192, 8191, 7196, 9201] avg=8192.0000 max=9201 max_rank=7 min=7196 min_rank=6 imbalance_ratio=1.123169 max_load_reduction=56335 max_load_reduction_pct=0.859604 moved_to_echo=32780 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7130, 0, 0, 1098, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 0, 7094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8182, 8192, 8191, 8190, 8241, 8156] avg=8192.0000 max=8241 max_rank=6 min=8156 min_rank=7 imbalance_ratio=1.005981 max_load_reduction=57295 max_load_reduction_pct=0.874252 moved_to_echo=32778 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7825, 0, 0, 1098, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 367, 0, 7094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8193, 8182, 8192, 8191, 8190, 8935, 7461] avg=8192.0000 max=8935 max_rank=6 min=7461 min_rank=7 imbalance_ratio=1.090698 max_load_reduction=56601 max_load_reduction_pct=0.863663 moved_to_echo=32777 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7873, 0, 0, 1097, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 321, 0, 7093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8193, 8182, 8192, 8190, 8191, 8982, 7414] avg=8192.0000 max=8982 max_rank=6 min=7414 min_rank=7 imbalance_ratio=1.096436 max_load_reduction=56554 max_load_reduction_pct=0.862946 moved_to_echo=32777 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7678, 0, 0, 1098, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 0, 7094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8192, 8182, 8192, 8192, 8191, 8789, 7608] avg=8192.0000 max=8789 max_rank=6 min=7608 min_rank=7 imbalance_ratio=1.072876 max_load_reduction=56747 max_load_reduction_pct=0.865891 moved_to_echo=32780 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 6745, 0, 0, 1099, 11, 2, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1419, 0, 7086, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8190, 8205, 8190, 8193, 8191, 7863, 8507] avg=8192.0000 max=8507 max_rank=7 min=7863 min_rank=6 imbalance_ratio=1.038452 max_load_reduction=57029 max_load_reduction_pct=0.870193 moved_to_echo=32754 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[6016, 0, 59648, 0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 59648, 0, 128, 5888, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8960, 0, 56704, 0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 56704, 0, 128, 8832, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 128, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 65536, 0, 128, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 20, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7314, 0, 7971, 0, 4, 0, 0, 0, 0, 18, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 7864, 0, 0, 0, 0, 0, 687, 0, 0, 2, 190, 8109, 8169, 0, 0, 0, 3, 0, 8186, 8168, 0, 0, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8190, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7522, 4, 7884, 0, 1, 0, 0, 0, 0, 33, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8039, 0, 0, 0, 0, 0, 499, 0, 0, 0, 227, 8190, 8190, 0, 0, 0, 1, 0, 8190, 8186, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8191, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7999, 89, 43, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8133, 8194, 8190, 8380, 8192, 8101, 8153] avg=8192.0000 max=8380 max_rank=4 min=8101 min_rank=6 imbalance_ratio=1.022949 max_load_reduction=57156 max_load_reduction_pct=0.872131 moved_to_echo=32826 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8001, 89, 43, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8135, 8194, 8190, 8380, 8193, 8103, 8149] avg=8192.0000 max=8380 max_rank=4 min=8103 min_rank=6 imbalance_ratio=1.022949 max_load_reduction=57156 max_load_reduction_pct=0.872131 moved_to_echo=32825 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 7785, 85, 42, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, 0, 1, 0, 0, 8169, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7314, 0, 184, 0, 4, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4, 0, 8167, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 7779, 0, 0, 0, 0, 0, 687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 190, 8067, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8518, 8179, 8206, 7520, 8187, 8469, 8266] avg=8192.0000 max=8518 max_rank=1 min=7520 min_rank=4 imbalance_ratio=1.039795 max_load_reduction=57018 max_load_reduction_pct=0.870026 moved_to_echo=32442 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7702, 92, 45, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7522, 4, 178, 0, 1, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 8184, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7947, 0, 0, 0, 0, 0, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 227, 8145, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8212, 8193, 8188, 7738, 8192, 8447, 8374] avg=8192.0000 max=8447 max_rank=6 min=7738 min_rank=4 imbalance_ratio=1.031128 max_load_reduction=57089 max_load_reduction_pct=0.871109 moved_to_echo=32751 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 7991, 89, 43, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8133, 8194, 8192, 8380, 8193, 8106, 8146] avg=8192.0000 max=8380 max_rank=4 min=8106 min_rank=6 imbalance_ratio=1.022949 max_load_reduction=57156 max_load_reduction_pct=0.872131 moved_to_echo=32825 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8001, 89, 43, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 188, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8135, 8194, 8190, 8379, 8193, 8103, 8150] avg=8192.0000 max=8379 max_rank=4 min=8103 min_rank=6 imbalance_ratio=1.022827 max_load_reduction=57157 max_load_reduction_pct=0.872147 moved_to_echo=32825 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8001, 89, 43, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 188, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8103, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8135, 8194, 8190, 8380, 8193, 8104, 8148] avg=8192.0000 max=8380 max_rank=4 min=8104 min_rank=6 imbalance_ratio=1.022949 max_load_reduction=57156 max_load_reduction_pct=0.872131 moved_to_echo=32825 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 7995, 89, 43, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8135, 8194, 8190, 8379, 8193, 8103, 8150] avg=8192.0000 max=8379 max_rank=4 min=8103 min_rank=6 imbalance_ratio=1.022827 max_load_reduction=57157 max_load_reduction_pct=0.872147 moved_to_echo=32825 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7532, 4, 7890, 0, 1, 0, 0, 0, 0, 10, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8043, 0, 0, 0, 0, 0, 511, 0, 0, 0, 224, 8190, 8190, 0, 0, 0, 1, 0, 8190, 8186, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 8190, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 8191, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 21, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7332, 0, 7964, 0, 4, 0, 0, 0, 0, 8, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 4, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 7862, 0, 0, 0, 0, 0, 714, 0, 0, 2, 194, 8100, 8171, 0, 0, 0, 3, 0, 8186, 8170, 0, 0, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 7778, 90, 41, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, 0, 1, 0, 0, 8171, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7332, 0, 184, 0, 4, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 4, 0, 8169, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 7772, 0, 0, 0, 0, 0, 714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 194, 8059, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8490, 8181, 8207, 7528, 8188, 8489, 8262] avg=8192.0000 max=8490 max_rank=1 min=7528 min_rank=4 imbalance_ratio=1.036377 max_load_reduction=57046 max_load_reduction_pct=0.870453 moved_to_echo=32467 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7998, 94, 42, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 190, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8098, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8137, 8193, 8190, 8382, 8193, 8099, 8150] avg=8192.0000 max=8382 max_rank=4 min=8099 min_rank=6 imbalance_ratio=1.023193 max_load_reduction=57154 max_load_reduction_pct=0.872101 moved_to_echo=32824 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7710, 98, 43, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7532, 4, 172, 0, 1, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 8184, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7945, 0, 0, 0, 0, 0, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 224, 8147, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8220, 8193, 8187, 7719, 8195, 8457, 8373] avg=8192.0000 max=8457 max_rank=6 min=7719 min_rank=4 imbalance_ratio=1.032349 max_load_reduction=57079 max_load_reduction_pct=0.870956 moved_to_echo=32744 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8000, 94, 42, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8138, 8194, 8190, 8382, 8192, 8098, 8150] avg=8192.0000 max=8382 max_rank=4 min=8098 min_rank=6 imbalance_ratio=1.023193 max_load_reduction=57154 max_load_reduction_pct=0.872101 moved_to_echo=32822 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8000, 94, 42, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 190, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8138, 8194, 8190, 8382, 8192, 8098, 8150] avg=8192.0000 max=8382 max_rank=4 min=8098 min_rank=6 imbalance_ratio=1.023193 max_load_reduction=57154 max_load_reduction_pct=0.872101 moved_to_echo=32822 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 7991, 94, 42, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8138, 8194, 8192, 8381, 8192, 8100, 8147] avg=8192.0000 max=8381 max_rank=4 min=8100 min_rank=6 imbalance_ratio=1.023071 max_load_reduction=57155 max_load_reduction_pct=0.872116 moved_to_echo=32820 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 7994, 94, 42, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8139, 8194, 8190, 8380, 8192, 8098, 8151] avg=8192.0000 max=8380 max_rank=4 min=8098 min_rank=6 imbalance_ratio=1.022949 max_load_reduction=57156 max_load_reduction_pct=0.872131 moved_to_echo=32821 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7998, 94, 42, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8136, 8193, 8190, 8382, 8192, 8097, 8153] avg=8192.0000 max=8382 max_rank=4 min=8097 min_rank=6 imbalance_ratio=1.023193 max_load_reduction=57154 max_load_reduction_pct=0.872101 moved_to_echo=32824 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 1024, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 63488, 768, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 64128, 128, 1536, 0, 128, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 64000, 128, 1536, 0, 128, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 512, 65152, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 128, 64384, 0, 0, 0, 0, 0, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 128, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 128, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 512, 65152, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 588, 0, 0, 0, 0, 0, 0, 7557, 0, 0, 0, 0, 8192, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 232, 8057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8067, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 296, 0, 0, 0, 0, 0, 0, 7806, 0, 0, 0, 1, 8192, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 219, 8124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7922, 0, 0, 0, 0, 0, 0, 0, 0, 166, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 120, 0, 0, 0, 0, 0, 0, 7739, 0, 0, 0, 0, 8192, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 355, 8113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 714, 0, 0, 0, 0, 0, 0, 7487, 0, 0, 0, 0, 8192, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 8062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 425, 0, 0, 0, 0, 0, 0, 7582, 0, 0, 0, 0, 8192, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 633, 8025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7810, 0, 0, 0, 0, 0, 0, 0, 0, 99, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 452, 0, 0, 0, 0, 0, 0, 7707, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8111, 0, 0, 0, 0, 0, 0, 5, 0, 43, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 7946, 0, 0, 0, 0, 0, 0, 3580, 0, 0, 0, 183, 8187, 0, 0, 53, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 4, 3, 0, 0, 0, 0, 8095, 0, 0, 0, 6, 0, 0, 0, 26, 0, 8162, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5499, 4411, 0, 0, 0, 0, 0, 0, 0, 3, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 3058, 0, 4, 0, 0, 0, 0, 0, 0, 88, 4, 0, 3, 7927, 0, 4, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 7033, 0, 0, 0, 0, 0, 0, 3236, 2, 0, 0, 95, 8190, 0, 0, 10, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 1, 1, 0, 0, 0, 0, 8097, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8189, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3695, 5256, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 5342, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8093, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 120, 0, 0, 0, 0, 0, 0, 7003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 355, 7616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 736, 497, 25, 14, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7123, 8232, 8191, 8192, 7982, 8191, 9447, 8178] avg=8192.0000 max=9447 max_rank=6 min=7123 min_rank=0 imbalance_ratio=1.153198 max_load_reduction=56089 max_load_reduction_pct=0.855850 moved_to_echo=33798 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 425, 0, 0, 0, 0, 0, 0, 6861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 633, 7533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7810, 0, 0, 0, 0, 0, 0, 721, 492, 24, 14, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7286, 8170, 8191, 8193, 8166, 8191, 9062, 8277] avg=8192.0000 max=9062 max_rank=6 min=7286 min_rank=0 imbalance_ratio=1.106201 max_load_reduction=56474 max_load_reduction_pct=0.861725 moved_to_echo=33696 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 588, 0, 0, 0, 0, 0, 0, 6839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 232, 7563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8067, 0, 0, 0, 0, 0, 0, 718, 494, 23, 14, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7427, 8188, 8191, 8192, 7795, 8191, 9317, 8235] avg=8192.0000 max=9317 max_rank=6 min=7427 min_rank=0 imbalance_ratio=1.137329 max_load_reduction=56219 max_load_reduction_pct=0.857834 moved_to_echo=33538 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 714, 0, 0, 0, 0, 0, 0, 6775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 7568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 712, 494, 25, 14, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7489, 8174, 8191, 8192, 7724, 8191, 9388, 8187] avg=8192.0000 max=9388 max_rank=6 min=7489 min_rank=0 imbalance_ratio=1.145996 max_load_reduction=56148 max_load_reduction_pct=0.856750 moved_to_echo=33490 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 7946, 0, 0, 0, 0, 0, 0, 3240, 0, 0, 0, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 4, 3, 0, 0, 0, 0, 8094, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 8162, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5499, 4141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 3058, 0, 4, 0, 0, 0, 0, 340, 270, 23, 14, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 4, 0, 3, 7913, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11369, 8213, 8199, 8190, 9640, 8198, 3714, 8013] avg=8192.0000 max=11369 max_rank=0 min=3714 min_rank=6 imbalance_ratio=1.387817 max_load_reduction=54167 max_load_reduction_pct=0.826523 moved_to_echo=29565 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 452, 0, 0, 0, 0, 0, 0, 6974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 7668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8111, 0, 0, 0, 0, 0, 0, 733, 500, 23, 14, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 43, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7426, 8170, 8191, 8192, 7758, 8191, 9382, 8226] avg=8192.0000 max=9382 max_rank=6 min=7426 min_rank=0 imbalance_ratio=1.145264 max_load_reduction=56154 max_load_reduction_pct=0.856842 moved_to_echo=33557 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 296, 0, 0, 0, 0, 0, 0, 7064, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 219, 7626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7922, 0, 0, 0, 0, 0, 0, 742, 498, 24, 14, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 166, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7361, 8211, 8191, 8192, 7845, 8191, 9201, 8344] avg=8192.0000 max=9201 max_rank=6 min=7361 min_rank=0 imbalance_ratio=1.123169 max_load_reduction=56335 max_load_reduction_pct=0.859604 moved_to_echo=33581 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 7033, 0, 0, 0, 0, 0, 0, 2925, 2, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8165, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 1, 1, 0, 0, 0, 0, 8095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3695, 4931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 5342, 0, 1, 0, 0, 0, 0, 311, 325, 23, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8074, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10055, 8178, 8191, 8193, 8626, 8192, 6025, 8076] avg=8192.0000 max=10055 max_rank=0 min=6025 min_rank=6 imbalance_ratio=1.227417 max_load_reduction=55481 max_load_reduction_pct=0.846573 moved_to_echo=30919 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 0, 0, 0, 0, 7800, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 433, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8141, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 217, 0, 0, 0, 0, 0, 0, 7837, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8123, 0, 0, 0, 0, 0, 0, 5, 0, 40, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 189, 0, 0, 0, 0, 0, 0, 7630, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 446, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8131, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 7588, 0, 0, 0, 0, 0, 0, 3372, 0, 0, 0, 160, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 4, 3, 0, 0, 0, 0, 8096, 0, 0, 0, 6, 0, 0, 0, 26, 0, 8161, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5501, 5163, 0, 0, 0, 0, 0, 0, 0, 3, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 2992, 0, 4, 0, 0, 0, 0, 0, 0, 39, 4, 0, 3, 7931, 0, 4, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 58, 0, 0, 0, 0, 0, 0, 7980, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 226, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7930, 0, 0, 0, 0, 0, 0, 0, 0, 195, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 6011, 0, 0, 0, 0, 0, 0, 3738, 2, 0, 0, 78, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 1, 1, 0, 0, 0, 0, 8102, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8189, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3772, 5935, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5123, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8100, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 277, 0, 0, 0, 0, 0, 0, 7525, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 932, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7601, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 144, 0, 0, 0, 0, 0, 0, 7748, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 344, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8104, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 7588, 0, 0, 0, 0, 0, 0, 3195, 0, 0, 0, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 4, 3, 0, 0, 0, 0, 8095, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 8161, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5501, 4607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 2992, 0, 4, 0, 0, 0, 0, 556, 177, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 4, 0, 3, 7925, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10943, 8191, 8198, 8189, 10108, 8191, 3740, 7976] avg=8192.0000 max=10943 max_rank=0 min=3740 min_rank=6 imbalance_ratio=1.335815 max_load_reduction=54593 max_load_reduction_pct=0.833023 moved_to_echo=30015 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 0, 0, 0, 0, 7390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 433, 7306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8141, 0, 0, 0, 0, 0, 0, 883, 410, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7393, 8192, 8191, 8193, 7748, 8191, 9441, 8187] avg=8192.0000 max=9441 max_rank=6 min=7393 min_rank=0 imbalance_ratio=1.152466 max_load_reduction=56095 max_load_reduction_pct=0.855942 moved_to_echo=33567 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 58, 0, 0, 0, 0, 0, 0, 7561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 226, 7304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7930, 0, 0, 0, 0, 0, 0, 883, 419, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 195, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7619, 8192, 8191, 8192, 7530, 8192, 9239, 8381] avg=8192.0000 max=9239 max_rank=6 min=7530 min_rank=4 imbalance_ratio=1.127808 max_load_reduction=56297 max_load_reduction_pct=0.859024 moved_to_echo=33342 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 277, 0, 0, 0, 0, 0, 0, 7130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 932, 7293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7601, 0, 0, 0, 0, 0, 0, 881, 395, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7407, 8192, 8191, 8193, 8225, 8191, 8884, 8253] avg=8192.0000 max=8884 max_rank=6 min=7407 min_rank=0 imbalance_ratio=1.084473 max_load_reduction=56652 max_load_reduction_pct=0.864441 moved_to_echo=33553 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 189, 0, 0, 0, 0, 0, 0, 7229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 446, 7296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8131, 0, 0, 0, 0, 0, 0, 881, 401, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7418, 8192, 8191, 8193, 7742, 8191, 9420, 8189] avg=8192.0000 max=9420 max_rank=6 min=7418 min_rank=0 imbalance_ratio=1.149902 max_load_reduction=56116 max_load_reduction_pct=0.856262 moved_to_echo=33542 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 217, 0, 0, 0, 0, 0, 0, 7425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 7307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8123, 0, 0, 0, 0, 0, 0, 883, 412, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 40, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7642, 8192, 8191, 8192, 7471, 8192, 9425, 8231] avg=8192.0000 max=9425 max_rank=6 min=7471 min_rank=4 imbalance_ratio=1.150513 max_load_reduction=56111 max_load_reduction_pct=0.856186 moved_to_echo=33319 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6011, 0, 0, 0, 0, 0, 0, 3538, 2, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 1, 1, 0, 0, 0, 0, 8098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3772, 5287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5123, 0, 1, 0, 0, 0, 0, 644, 200, 13, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8087, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9629, 8193, 8192, 8192, 9059, 8196, 5986, 8089] avg=8192.0000 max=9629 max_rank=0 min=5986 min_rank=6 imbalance_ratio=1.175415 max_load_reduction=55907 max_load_reduction_pct=0.853073 moved_to_echo=31330 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 144, 0, 0, 0, 0, 0, 0, 7341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 344, 7309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8104, 0, 0, 0, 0, 0, 0, 883, 407, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7485, 8192, 8191, 8192, 7653, 8192, 9401, 8230] avg=8192.0000 max=9401 max_rank=6 min=7485 min_rank=0 imbalance_ratio=1.147583 max_load_reduction=56135 max_load_reduction_pct=0.856552 moved_to_echo=33476 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 0, 0, 128, 0, 128, 0, 56704, 0, 128, 0, 0, 0, 0, 5120, 3584, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 17664, 0, 0, 0, 0, 0, 0, 47744, 128, 0, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 10880, 54656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 0, 128, 0, 0, 0, 56192, 0, 128, 0, 0, 0, 0, 6528, 2944, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 14592, 0, 0, 0, 0, 0, 0, 50816, 128, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 11904, 53760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 256, 128, 128, 0, 0, 0, 0, 65408, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 512, 128, 0, 128, 65152, 0, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 256, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 65408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8190, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 0, 8182, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8130, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 78, 0, 7964, 0, 3, 0, 0, 0, 0, 0, 257, 0, 0, 3, 0, 0, 0, 1, 0, 0, 2, 0, 0, 4, 0, 0, 0, 0, 0, 0, 3391, 0, 8108, 0, 0, 38, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1998, 0, 5, 0, 8167, 0, 0, 3, 0, 8187, 0, 0, 0, 4, 8185, 0, 0, 0, 0, 7943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 5, 1, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2798, 0, 0, 0, 173, 0, 0, 0, 0, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3, 0, 0, 8177, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 9, 0, 8145, 0, 0, 0, 0, 0, 0, 0, 191, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2495, 0, 8117, 0, 0, 20, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3841, 0, 1, 0, 8187, 0, 0, 1, 0, 8191, 0, 0, 0, 1, 8191, 0, 0, 0, 1, 7993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1855, 0, 0, 0, 64, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 7451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 0, 8182, 0, 0, 741, 558, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8160, 8192, 7457, 9515, 8192, 8245, 7612, 8163] avg=8192.0000 max=9515 max_rank=3 min=7457 min_rank=2 imbalance_ratio=1.161499 max_load_reduction=56021 max_load_reduction_pct=0.854813 moved_to_echo=32212 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 7451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8130, 0, 0, 741, 558, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8162, 8191, 7511, 9456, 8196, 8246, 7612, 8162] avg=8192.0000 max=9456 max_rank=3 min=7511 min_rank=2 imbalance_ratio=1.154297 max_load_reduction=56080 max_load_reduction_pct=0.855713 moved_to_echo=32216 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 7451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 741, 558, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8191, 7461, 9511, 8196, 8246, 7612, 8162] avg=8192.0000 max=9511 max_rank=3 min=7461 min_rank=2 imbalance_ratio=1.161011 max_load_reduction=56025 max_load_reduction_pct=0.854874 moved_to_echo=32216 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 78, 0, 7933, 0, 3, 0, 0, 0, 0, 0, 257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 2, 0, 0, 4, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3391, 0, 7375, 0, 0, 38, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1998, 0, 5, 733, 557, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 3, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 5, 31, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2798, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 173, 0, 0, 0, 0, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8271, 8196, 10808, 3339, 8178, 8004, 10406, 8334] avg=8192.0000 max=10808 max_rank=2 min=3339 min_rank=3 imbalance_ratio=1.319336 max_load_reduction=54728 max_load_reduction_pct=0.835083 moved_to_echo=34922 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 7451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3, 0, 0, 8177, 0, 0, 741, 558, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8159, 8191, 7456, 9514, 8192, 8246, 7612, 8166] avg=8192.0000 max=9514 max_rank=3 min=7456 min_rank=2 imbalance_ratio=1.161377 max_load_reduction=56022 max_load_reduction_pct=0.854828 moved_to_echo=32216 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8190, 0, 0, 741, 558, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8160, 8191, 7452, 9517, 8196, 8246, 7612, 8162] avg=8192.0000 max=9517 max_rank=3 min=7452 min_rank=2 imbalance_ratio=1.161743 max_load_reduction=56019 max_load_reduction_pct=0.854782 moved_to_echo=32216 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 7451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 741, 558, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8156, 8191, 7494, 9479, 8196, 8246, 7612, 8162] avg=8192.0000 max=9479 max_rank=3 min=7494 min_rank=2 imbalance_ratio=1.157104 max_load_reduction=56057 max_load_reduction_pct=0.855362 moved_to_echo=32216 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 9, 0, 8111, 0, 0, 0, 0, 0, 0, 0, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2495, 0, 7381, 0, 0, 20, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3841, 0, 1, 736, 559, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 34, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7603, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8311, 8193, 9897, 5205, 8190, 8057, 9458, 8225] avg=8192.0000 max=9897 max_rank=2 min=5205 min_rank=3 imbalance_ratio=1.208130 max_load_reduction=55639 max_load_reduction_pct=0.848984 moved_to_echo=33930 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8085, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8178, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 80, 0, 7997, 0, 3, 0, 0, 0, 0, 0, 256, 0, 0, 4, 0, 0, 0, 1, 0, 0, 2, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4086, 0, 8107, 0, 0, 9, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1541, 0, 5, 0, 8169, 0, 0, 3, 0, 8187, 0, 0, 0, 4, 8182, 0, 0, 0, 0, 7936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 5, 1, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2561, 0, 0, 0, 176, 0, 0, 0, 0, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 6, 0, 0, 8087, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 186, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2836, 0, 8119, 0, 0, 16, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3620, 0, 1, 0, 8188, 0, 0, 1, 0, 8191, 0, 0, 0, 1, 8191, 0, 0, 0, 1, 7997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1735, 0, 0, 0, 66, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8179, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 218, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7975, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 0, 7284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8085, 0, 0, 908, 518, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8156, 8191, 7390, 9539, 8195, 8247, 7656, 8162] avg=8192.0000 max=9539 max_rank=3 min=7390 min_rank=2 imbalance_ratio=1.164429 max_load_reduction=55997 max_load_reduction_pct=0.854446 moved_to_echo=32260 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8150, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 7284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 908, 518, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8152, 8191, 7296, 9637, 8195, 8247, 7656, 8162] avg=8192.0000 max=9637 max_rank=3 min=7296 min_rank=2 imbalance_ratio=1.176392 max_load_reduction=55899 max_load_reduction_pct=0.852951 moved_to_echo=32260 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 80, 0, 7961, 0, 3, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 2, 0, 0, 4, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 7208, 0, 0, 9, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1541, 0, 5, 899, 517, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 3, 0, 0, 0, 0, 0, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 5, 36, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7652, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 0, 0, 0, 0, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8300, 8197, 11307, 3008, 8179, 7997, 10214, 8334] avg=8192.0000 max=11307 max_rank=2 min=3008 min_rank=3 imbalance_ratio=1.380249 max_load_reduction=54229 max_load_reduction_pct=0.827469 moved_to_echo=34724 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 7284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8178, 0, 0, 908, 518, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8155, 8191, 7297, 9633, 8195, 8247, 7656, 8162] avg=8192.0000 max=9633 max_rank=3 min=7297 min_rank=2 imbalance_ratio=1.175903 max_load_reduction=55903 max_load_reduction_pct=0.853012 moved_to_echo=32260 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 0, 8116, 0, 0, 0, 0, 0, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2836, 0, 7215, 0, 0, 16, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3620, 0, 1, 904, 520, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 40, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8310, 8193, 10068, 5103, 8191, 8058, 9386, 8227] avg=8192.0000 max=10068 max_rank=2 min=5103 min_rank=3 imbalance_ratio=1.229004 max_load_reduction=55468 max_load_reduction_pct=0.846375 moved_to_echo=33862 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 7284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 6, 0, 0, 8087, 0, 0, 908, 518, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8191, 7380, 9551, 8194, 8247, 7656, 8163] avg=8192.0000 max=9551 max_rank=3 min=7380 min_rank=2 imbalance_ratio=1.165894 max_load_reduction=55985 max_load_reduction_pct=0.854263 moved_to_echo=32260 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 218, 0, 7284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7975, 0, 0, 908, 518, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8191, 7502, 9429, 8195, 8247, 7656, 8162] avg=8192.0000 max=9429 max_rank=3 min=7502 min_rank=2 imbalance_ratio=1.151001 max_load_reduction=56107 max_load_reduction_pct=0.856125 moved_to_echo=32260 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 7284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8179, 0, 0, 908, 518, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8155, 8191, 7296, 9636, 8192, 8246, 7656, 8164] avg=8192.0000 max=9636 max_rank=3 min=7296 min_rank=2 imbalance_ratio=1.176270 max_load_reduction=55900 max_load_reduction_pct=0.852966 moved_to_echo=32258 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 128, 128, 128, 0, 0, 54912, 0, 128, 6016, 4480, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 6016, 0, 59520, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 60928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 256, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 65408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 65536, 0, 0, 128, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 128, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 128, 128, 128, 0, 0, 53888, 0, 128, 7296, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 7424, 0, 58240, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 61312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 256, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 65280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[65152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 384, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 128, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 0, 65024, 0, 128, 0, 0, 0, 0, 0, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 65536, 0, 0, 128, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 967, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 8187, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2261, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4969] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 892, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2214, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5086] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 910, 0, 0, 1, 8192, 0, 0, 0, 0, 8192, 0, 0, 8191, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1540, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5743] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 671, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 8191, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 890, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6633] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 755, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7258] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7200, 2, 0, 0, 8187, 0, 0, 0, 0, 8186, 4, 0, 8187, 0, 8185, 0, 1, 0, 4, 0, 7854, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4, 0, 0, 3, 6531, 0, 1, 0, 0, 0, 5, 5, 5, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1883, 0, 0, 0, 4, 0, 0, 1, 0, 3, 2, 0, 0, 0, 0, 379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 707] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6985, 0, 0, 0, 8191, 0, 0, 0, 0, 8190, 1, 0, 8191, 0, 8189, 0, 0, 0, 1, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 7533, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 815] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 458, 0, 0, 3, 8192, 0, 0, 0, 0, 8192, 0, 0, 8180, 0, 8192, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2610, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5134] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 910, 0, 0, 1, 5837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1540, 0, 0, 7235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7775, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5743, 2355, 956, 369, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6748, 8239, 8775, 8191, 8194, 7826, 8139, 9424] avg=8192.0000 max=9424 max_rank=7 min=6748 min_rank=0 imbalance_ratio=1.150391 max_load_reduction=56112 max_load_reduction_pct=0.856201 moved_to_echo=33583 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 967, 0, 0, 0, 5837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2261, 0, 0, 7236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7771, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4969, 2355, 956, 368, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6804, 8240, 9497, 8191, 8192, 7822, 8141, 8649] avg=8192.0000 max=9497 max_rank=2 min=6804 min_rank=0 imbalance_ratio=1.159302 max_load_reduction=56039 max_load_reduction_pct=0.855087 moved_to_echo=32804 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 755, 0, 0, 0, 5837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 0, 0, 7236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7776, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7258, 2355, 956, 369, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6592, 8239, 7415, 8191, 8192, 7827, 8141, 10939] avg=8192.0000 max=10939 max_rank=7 min=6592 min_rank=0 imbalance_ratio=1.335327 max_load_reduction=54597 max_load_reduction_pct=0.833084 moved_to_echo=35099 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 892, 0, 0, 0, 5837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2214, 0, 0, 7236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7776, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5086, 2355, 956, 369, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6729, 8239, 9450, 8191, 8193, 7828, 8139, 8767] avg=8192.0000 max=9450 max_rank=2 min=6729 min_rank=0 imbalance_ratio=1.153564 max_load_reduction=56086 max_load_reduction_pct=0.855804 moved_to_echo=32927 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 671, 0, 0, 0, 5837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 890, 0, 0, 7235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7775, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6633, 2355, 956, 369, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6508, 8239, 8125, 8191, 8192, 7827, 8140, 10314] avg=8192.0000 max=10314 max_rank=7 min=6508 min_rank=0 imbalance_ratio=1.259033 max_load_reduction=55222 max_load_reduction_pct=0.842621 moved_to_echo=34473 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7200, 2, 0, 0, 5834, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 48, 0, 0, 0, 1, 0, 4, 0, 7854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4, 0, 0, 3, 5769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5, 5, 5, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1883, 0, 0, 0, 4, 0, 0, 1, 7771, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 379, 0, 0, 0, 0, 0, 0, 0, 0, 8134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 707, 2353, 762, 368, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13036, 7911, 5778, 8201, 8188, 9710, 8518, 4194] avg=8192.0000 max=13036 max_rank=0 min=4194 min_rank=7 imbalance_ratio=1.591309 max_load_reduction=52500 max_load_reduction_pct=0.801086 moved_to_echo=30610 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 458, 0, 0, 3, 5837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2610, 0, 0, 7236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7765, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5134, 2355, 956, 368, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6298, 8238, 9846, 8191, 8194, 7816, 8139, 8814] avg=8192.0000 max=9846 max_rank=2 min=6298 min_rank=0 imbalance_ratio=1.201904 max_load_reduction=55690 max_load_reduction_pct=0.849762 moved_to_echo=32963 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6985, 0, 0, 0, 5835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 38, 0, 0, 0, 0, 0, 1, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 6648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 0, 0, 0, 1, 0, 0, 0, 7781, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 815, 2356, 885, 372, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12821, 8191, 6650, 8189, 8191, 8880, 8179, 4435] avg=8192.0000 max=12821 max_rank=0 min=4435 min_rank=7 imbalance_ratio=1.565063 max_load_reduction=52715 max_load_reduction_pct=0.804367 moved_to_echo=29685 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8143, 0, 0, 0, 8191, 0, 0, 0, 0, 8190, 1, 0, 8191, 0, 8189, 0, 0, 0, 1, 0, 8107, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 7229, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 962, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 98] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 8191, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8029, 2, 0, 0, 8187, 0, 0, 0, 0, 8186, 4, 0, 8187, 0, 8185, 0, 1, 0, 4, 0, 7809, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4, 0, 0, 3, 6389, 0, 1, 0, 0, 0, 5, 5, 4, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1799, 0, 0, 0, 4, 0, 0, 1, 0, 3, 2, 0, 0, 0, 0, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 303] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 8180, 0, 8192, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1, 8192, 0, 0, 0, 0, 8192, 0, 0, 8170, 0, 8192, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 8190, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 8192, 0, 336, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7784, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8110, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8218, 8251, 8543, 8191, 8193, 7839, 8160, 8141] avg=8192.0000 max=8543 max_rank=2 min=7839 min_rank=5 imbalance_ratio=1.042847 max_load_reduction=56993 max_load_reduction_pct=0.869644 moved_to_echo=32333 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8029, 2, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 59, 0, 0, 0, 1, 0, 4, 0, 7809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4, 0, 0, 3, 6389, 0, 337, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5, 5, 4, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1799, 0, 0, 0, 4, 0, 0, 1, 7791, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 226, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 303, 8105, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8058, 7877, 6736, 8200, 8188, 9650, 8384, 8443] avg=8192.0000 max=9650 max_rank=5 min=6736 min_rank=2 imbalance_ratio=1.177979 max_load_reduction=55886 max_load_reduction_pct=0.852753 moved_to_echo=34665 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8143, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 51, 0, 0, 0, 0, 0, 1, 0, 8107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 7229, 0, 340, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 962, 0, 0, 0, 1, 0, 0, 0, 7800, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 98, 8113, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8160, 7575, 8190, 8191, 8821, 8191, 8244] avg=8192.0000 max=8821 max_rank=5 min=7575 min_rank=2 imbalance_ratio=1.076782 max_load_reduction=56715 max_load_reduction_pct=0.865402 moved_to_echo=33447 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8192, 0, 337, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7795, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8110, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 8250, 8532, 8191, 8192, 7850, 8161, 8141] avg=8192.0000 max=8532 max_rank=2 min=7850 min_rank=5 imbalance_ratio=1.041504 max_load_reduction=57004 max_load_reduction_pct=0.869812 moved_to_echo=32344 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8191, 0, 337, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7794, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8110, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 8252, 8530, 8191, 8193, 7849, 8160, 8142] avg=8192.0000 max=8530 max_rank=2 min=7849 min_rank=5 imbalance_ratio=1.041260 max_load_reduction=57006 max_load_reduction_pct=0.869843 moved_to_echo=32344 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 337, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7795, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8110, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 8251, 8530, 8191, 8193, 7850, 8160, 8142] avg=8192.0000 max=8530 max_rank=2 min=7850 min_rank=5 imbalance_ratio=1.041260 max_load_reduction=57006 max_load_reduction_pct=0.869843 moved_to_echo=32345 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 8192, 0, 336, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7774, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8110, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8220, 8243, 8559, 8191, 8193, 7829, 8160, 8141] avg=8192.0000 max=8559 max_rank=2 min=7829 min_rank=5 imbalance_ratio=1.044800 max_load_reduction=56977 max_load_reduction_pct=0.869400 moved_to_echo=32323 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8191, 0, 337, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7793, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8110, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 8252, 8531, 8191, 8193, 7848, 8160, 8142] avg=8192.0000 max=8531 max_rank=2 min=7848 min_rank=5 imbalance_ratio=1.041382 max_load_reduction=57005 max_load_reduction_pct=0.869827 moved_to_echo=32343 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 36352, 18944, 7424, 3072, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 18944, 128, 0, 128, 46720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 128, 9728, 0, 128, 55936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 2944, 0, 0, 0, 128, 0, 0, 128, 62208, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 2816, 0, 0, 0, 128, 0, 0, 128, 62336, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 128, 128, 0, 128, 62848, 0, 2816, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 65408, 128, 0, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 0, 512, 0, 0, 0, 128, 0, 128, 0, 65152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 128, 0, 0, 0, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 65152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 128, 0, 0, 0, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 65280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 128, 128, 128, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 0, 384, 0, 0, 0, 128, 0, 128, 0, 65280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 512, 64896, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7658, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7779, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7811, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 8186, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 52, 147, 0, 0, 8126, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 0, 5, 5, 0, 0, 8187, 0, 0, 4041, 6, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 1, 0, 8182, 0, 0, 0, 4, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4, 0, 0, 0, 6, 0, 3998, 54, 0, 8185, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 23, 52, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 2, 0, 0, 8191, 0, 0, 2256, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 8189, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 5878, 6, 0, 8191, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7923, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7592, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7548, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7013, 1144, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7047, 0, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7811, 0, 0, 1161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8203, 8190, 7431, 8198, 8192, 8193, 8972] avg=8192.0000 max=8972 max_rank=7 min=7431 min_rank=3 imbalance_ratio=1.095215 max_load_reduction=56564 max_load_reduction_pct=0.863098 moved_to_echo=33555 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7013, 1144, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7048, 0, 0, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7658, 0, 0, 1161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8159, 8203, 8190, 7582, 8198, 8192, 8193, 8819] avg=8192.0000 max=8819 max_rank=7 min=7582 min_rank=3 imbalance_ratio=1.076538 max_load_reduction=56717 max_load_reduction_pct=0.865433 moved_to_echo=33402 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7013, 1144, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7046, 0, 0, 415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7779, 0, 0, 1161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8159, 8203, 8190, 7461, 8198, 8192, 8193, 8940] avg=8192.0000 max=8940 max_rank=7 min=7461 min_rank=3 imbalance_ratio=1.091309 max_load_reduction=56596 max_load_reduction_pct=0.863586 moved_to_echo=33523 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7013, 1144, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7046, 0, 0, 271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7923, 0, 0, 1161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8158, 8203, 8190, 7317, 8198, 8193, 8193, 9084] avg=8192.0000 max=9084 max_rank=7 min=7317 min_rank=3 imbalance_ratio=1.108887 max_load_reduction=56452 max_load_reduction_pct=0.861389 moved_to_echo=33668 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7013, 1144, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7048, 0, 0, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7592, 0, 0, 1161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8159, 8203, 8190, 7648, 8198, 8192, 8193, 8753] avg=8192.0000 max=8753 max_rank=7 min=7648 min_rank=3 imbalance_ratio=1.068481 max_load_reduction=56783 max_load_reduction_pct=0.866440 moved_to_echo=33336 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 8153, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 52, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8126, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 0, 5, 7007, 1143, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 7044, 0, 0, 4041, 6, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8129, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8182, 0, 0, 0, 4, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4, 0, 0, 0, 6, 0, 3998, 54, 0, 1160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8354, 8138, 8194, 11099, 8152, 8192, 8183, 5224] avg=8192.0000 max=11099 max_rank=3 min=5224 min_rank=7 imbalance_ratio=1.354858 max_load_reduction=54437 max_load_reduction_pct=0.830643 moved_to_echo=29751 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7013, 1140, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7026, 0, 0, 670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7548, 0, 0, 1161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8203, 8186, 7696, 8198, 8194, 8193, 8709] avg=8192.0000 max=8709 max_rank=7 min=7696 min_rank=3 imbalance_ratio=1.063110 max_load_reduction=56827 max_load_reduction_pct=0.867111 moved_to_echo=33294 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 23, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 7020, 1148, 34, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 7043, 0, 0, 2256, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 5878, 6, 0, 1148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8233, 8180, 8206, 9302, 8196, 8189, 8195, 7035] avg=8192.0000 max=9302 max_rank=3 min=7035 min_rank=7 imbalance_ratio=1.135498 max_load_reduction=56234 max_load_reduction_pct=0.858063 moved_to_echo=31615 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8103, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 8186, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 47, 141, 0, 0, 8124, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 0, 5, 5, 0, 0, 8187, 0, 0, 2826, 6, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 1, 0, 8181, 0, 0, 0, 4, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 5, 0, 0, 0, 6, 0, 5222, 55, 0, 8185, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 21, 48, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 2, 0, 0, 8191, 0, 0, 1264, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 8189, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 6876, 6, 0, 8191, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8120, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7607, 552, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7638, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8159, 8203, 8190, 7683, 8198, 8192, 8193, 8718] avg=8192.0000 max=8718 max_rank=7 min=7683 min_rank=3 imbalance_ratio=1.064209 max_load_reduction=56818 max_load_reduction_pct=0.866974 moved_to_echo=33301 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 8155, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 47, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8124, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 0, 5, 7601, 552, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 7635, 0, 0, 2826, 6, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8131, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8181, 0, 0, 0, 4, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 5, 0, 0, 0, 6, 0, 5222, 55, 0, 566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8345, 8137, 8195, 10475, 8154, 8191, 8183, 5856] avg=8192.0000 max=10475 max_rank=3 min=5856 min_rank=7 imbalance_ratio=1.278687 max_load_reduction=55061 max_load_reduction_pct=0.840164 moved_to_echo=30384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7607, 552, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7640, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8103, 0, 0, 567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8203, 8190, 7729, 8198, 8192, 8193, 8670] avg=8192.0000 max=8670 max_rank=7 min=7729 min_rank=3 imbalance_ratio=1.058350 max_load_reduction=56866 max_load_reduction_pct=0.867706 moved_to_echo=33253 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7607, 552, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7640, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 0, 0, 567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8203, 8190, 7700, 8198, 8192, 8193, 8699] avg=8192.0000 max=8699 max_rank=7 min=7700 min_rank=3 imbalance_ratio=1.061890 max_load_reduction=56837 max_load_reduction_pct=0.867264 moved_to_echo=33282 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 8159, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 21, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 7611, 558, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 7633, 0, 0, 1264, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 6876, 6, 0, 558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 8181, 8203, 8900, 8194, 8190, 8195, 7443] avg=8192.0000 max=8900 max_rank=3 min=7443 min_rank=7 imbalance_ratio=1.086426 max_load_reduction=56636 max_load_reduction_pct=0.864197 moved_to_echo=32022 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7607, 552, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7638, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8120, 0, 0, 567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8203, 8190, 7712, 8198, 8192, 8193, 8687] avg=8192.0000 max=8687 max_rank=7 min=7712 min_rank=3 imbalance_ratio=1.060425 max_load_reduction=56849 max_load_reduction_pct=0.867447 moved_to_echo=33270 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7607, 550, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7611, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 0, 0, 567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8159, 8203, 8188, 7694, 8198, 8194, 8193, 8707] avg=8192.0000 max=8707 max_rank=7 min=7694 min_rank=3 imbalance_ratio=1.062866 max_load_reduction=56829 max_load_reduction_pct=0.867142 moved_to_echo=33292 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7607, 552, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7638, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8160, 8203, 8190, 7643, 8198, 8193, 8193, 8756] avg=8192.0000 max=8756 max_rank=7 min=7643 min_rank=3 imbalance_ratio=1.068848 max_load_reduction=56780 max_load_reduction_pct=0.866394 moved_to_echo=33340 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 128, 56192, 9216, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 56192, 128, 0, 9344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 56448, 0, 0, 9216, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 61056, 128, 0, 4608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 128, 60928, 4480, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 61184, 0, 0, 4480, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 65280, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 128, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 65536, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65408, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 0, 65536, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 65280, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 7211, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 6, 0, 0, 0, 8177, 0, 0, 0, 0, 7, 0, 12, 959, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 1, 0, 8186, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8090, 0, 0, 0, 1, 0, 1, 1, 0, 0, 120, 0, 0, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 6653, 0, 0, 0, 0, 1, 0, 8185, 0, 0, 23, 0, 0, 0, 8169, 0, 0, 0, 0, 36, 0, 0, 1533, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 6, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8165, 6, 0, 7977, 0, 1, 5, 0, 0, 0, 0, 0, 0, 8, 0, 8183, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8094, 0, 0, 0, 0, 0, 0, 6, 0, 0, 267, 0, 0, 0, 4, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 7705, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 6, 0, 0, 0, 8180, 0, 0, 0, 0, 6, 0, 3, 481, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 1, 0, 8187, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8047, 0, 0, 0, 1, 0, 1, 1, 0, 0, 152, 0, 0, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8443, 7937, 8192, 8193, 8194, 8217, 8223, 8137] avg=8192.0000 max=8443 max_rank=0 min=7937 min_rank=1 imbalance_ratio=1.030640 max_load_reduction=57093 max_load_reduction_pct=0.871170 moved_to_echo=32771 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8443, 7945, 8192, 8185, 8194, 8217, 8223, 8137] avg=8192.0000 max=8443 max_rank=0 min=7945 min_rank=1 imbalance_ratio=1.030640 max_load_reduction=57093 max_load_reduction_pct=0.871170 moved_to_echo=32771 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8443, 7936, 8192, 8194, 8194, 8217, 8223, 8137] avg=8192.0000 max=8443 max_rank=0 min=7936 min_rank=1 imbalance_ratio=1.030640 max_load_reduction=57093 max_load_reduction_pct=0.871170 moved_to_echo=32771 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8443, 7938, 8192, 8192, 8194, 8217, 8223, 8137] avg=8192.0000 max=8443 max_rank=0 min=7938 min_rank=1 imbalance_ratio=1.030640 max_load_reduction=57093 max_load_reduction_pct=0.871170 moved_to_echo=32771 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8443, 7936, 8192, 8194, 8194, 8217, 8223, 8137] avg=8192.0000 max=8443 max_rank=0 min=7936 min_rank=1 imbalance_ratio=1.030640 max_load_reduction=57093 max_load_reduction_pct=0.871170 moved_to_echo=32771 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 6653, 0, 0, 0, 0, 1, 0, 0, 0, 0, 23, 0, 0, 0, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 1533, 4, 0, 0, 0, 0, 0, 0, 0, 7913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 6, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8138, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 6, 0, 7977, 0, 1, 5, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 267, 0, 0, 0, 4, 0, 0, 0, 0, 0, 8123, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6928, 9486, 8192, 8197, 8179, 8022, 8127, 8405] avg=8192.0000 max=9486 max_rank=1 min=6928 min_rank=0 imbalance_ratio=1.157959 max_load_reduction=56050 max_load_reduction_pct=0.855255 moved_to_echo=32733 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8443, 7940, 8192, 8185, 8194, 8217, 8221, 8144] avg=8192.0000 max=8443 max_rank=0 min=7940 min_rank=1 imbalance_ratio=1.030640 max_load_reduction=57093 max_load_reduction_pct=0.871170 moved_to_echo=32776 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 7705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 3, 481, 1, 0, 0, 0, 0, 0, 0, 0, 7927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 1, 0, 8187, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 152, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8134, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7950, 8418, 8192, 8196, 8193, 8212, 8073, 8302] avg=8192.0000 max=8418 max_rank=1 min=7950 min_rank=0 imbalance_ratio=1.027588 max_load_reduction=57118 max_load_reduction_pct=0.871552 moved_to_echo=32780 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 5993, 0, 0, 0, 0, 1, 0, 8185, 0, 0, 25, 0, 0, 0, 8169, 0, 0, 0, 0, 37, 0, 17, 2176, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 3, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8159, 6, 0, 7964, 0, 1, 4, 0, 0, 0, 0, 0, 0, 7, 0, 8183, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8133, 0, 0, 0, 0, 0, 0, 6, 0, 0, 243, 0, 0, 0, 5, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8587, 7800, 8192, 8191, 8187, 8219, 8212, 8148] avg=8192.0000 max=8587 max_rank=0 min=7800 min_rank=1 imbalance_ratio=1.048218 max_load_reduction=56949 max_load_reduction_pct=0.868973 moved_to_echo=32766 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 7211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 12, 959, 1, 0, 0, 0, 0, 0, 0, 0, 7779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 1, 0, 8186, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 120, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8140, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7601, 8758, 8195, 8194, 8194, 8212, 8109, 8273] avg=8192.0000 max=8758 max_rank=1 min=7601 min_rank=0 imbalance_ratio=1.069092 max_load_reduction=56778 max_load_reduction_pct=0.866364 moved_to_echo=32788 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 5993, 0, 0, 0, 0, 1, 0, 0, 0, 0, 25, 0, 0, 0, 394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 17, 2176, 4, 0, 0, 0, 0, 0, 0, 0, 7771, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 3, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8185, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 6, 0, 7964, 0, 1, 4, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 243, 0, 0, 0, 5, 0, 0, 0, 0, 0, 8111, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6413, 10005, 8189, 8196, 8198, 8010, 8156, 8369] avg=8192.0000 max=10005 max_rank=1 min=6413 min_rank=0 imbalance_ratio=1.221313 max_load_reduction=55531 max_load_reduction_pct=0.847336 moved_to_echo=32733 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8587, 7793, 8192, 8191, 8194, 8219, 8212, 8148] avg=8192.0000 max=8587 max_rank=0 min=7793 min_rank=1 imbalance_ratio=1.048218 max_load_reduction=56949 max_load_reduction_pct=0.868973 moved_to_echo=32773 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8587, 7796, 8192, 8191, 8191, 8219, 8212, 8148] avg=8192.0000 max=8587 max_rank=0 min=7796 min_rank=1 imbalance_ratio=1.048218 max_load_reduction=56949 max_load_reduction_pct=0.868973 moved_to_echo=32770 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8587, 7797, 8192, 8191, 8185, 8219, 8211, 8154] avg=8192.0000 max=8587 max_rank=0 min=7797 min_rank=1 imbalance_ratio=1.048218 max_load_reduction=56949 max_load_reduction_pct=0.868973 moved_to_echo=32769 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8587, 7793, 8192, 8191, 8194, 8219, 8212, 8148] avg=8192.0000 max=8587 max_rank=0 min=7793 min_rank=1 imbalance_ratio=1.048218 max_load_reduction=56949 max_load_reduction_pct=0.868973 moved_to_echo=32773 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8587, 7794, 8192, 8191, 8193, 8219, 8212, 8148] avg=8192.0000 max=8587 max_rank=0 min=7794 min_rank=1 imbalance_ratio=1.048218 max_load_reduction=56949 max_load_reduction_pct=0.868973 moved_to_echo=32772 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 128, 2048, 128, 0, 0, 0, 0, 0, 0, 0, 63488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 63616, 0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 0, 0, 0, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 128, 128, 128, 0, 512, 0, 0, 0, 128, 0, 0, 0, 0, 0, 65152, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[256, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 65408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 65408, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 256, 128, 0, 65408, 0, 128, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 62464, 0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 0, 0, 0, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 128, 3200, 128, 0, 0, 0, 0, 0, 0, 0, 62336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65408, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 65536, 0, 128, 0, 128, 0, 0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 128, 128, 0, 384, 0, 0, 0, 128, 0, 0, 0, 0, 0, 65152, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 256, 128, 0, 65408, 0, 128, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[256, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 65408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1397, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 375, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7682, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1530, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 230, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 504, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3720, 8167, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8179, 20, 4, 0, 8185, 0, 8167, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1755, 0, 0, 0, 0, 0, 4, 0, 3, 0, 8165, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 1823, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8184, 0, 0, 0, 7, 0, 2, 2, 1, 0, 0, 4, 0, 0, 0, 1125, 0, 0, 0, 7, 3, 0, 0, 0, 0, 7917, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1200, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 793, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 4, 1, 0, 8190, 0, 8184, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2673, 0, 0, 1, 0, 0, 1, 0, 1, 0, 8148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 4404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 8190, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 404, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8134, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 230, 6977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 288, 0, 0, 0, 0, 0, 0, 0, 7518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 318, 151, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 897, 673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7207, 8192, 7806, 8476, 9432, 8193, 8189, 8041] avg=8192.0000 max=9432 max_rank=4 min=7207 min_rank=0 imbalance_ratio=1.151367 max_load_reduction=56104 max_load_reduction_pct=0.856079 moved_to_echo=33855 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1530, 6977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 7518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 318, 151, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 897, 673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8507, 8192, 7542, 8412, 8460, 8193, 8189, 8041] avg=8192.0000 max=8507 max_rank=0 min=7542 min_rank=2 imbalance_ratio=1.038452 max_load_reduction=57029 max_load_reduction_pct=0.870193 moved_to_echo=32883 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1397, 6977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 7518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 318, 151, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 897, 674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8374, 8192, 7583, 7590, 9375, 8192, 8189, 8041] avg=8192.0000 max=9375 max_rank=4 min=7583 min_rank=2 imbalance_ratio=1.144409 max_load_reduction=56161 max_load_reduction_pct=0.856949 moved_to_echo=33797 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 375, 6977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 202, 0, 0, 0, 0, 0, 0, 0, 7518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7682, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 318, 151, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 897, 674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7352, 8192, 7720, 8154, 9696, 8192, 8189, 8041] avg=8192.0000 max=9696 max_rank=4 min=7352 min_rank=0 imbalance_ratio=1.183594 max_load_reduction=55840 max_load_reduction_pct=0.852051 moved_to_echo=34118 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1200, 6977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 7519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 318, 151, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 897, 673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8192, 7614, 7647, 9484, 8192, 8189, 8041] avg=8192.0000 max=9484 max_rank=4 min=7614 min_rank=2 imbalance_ratio=1.157715 max_load_reduction=56052 max_load_reduction_pct=0.855286 moved_to_echo=33906 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3720, 6956, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 4, 0, 0, 0, 8167, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1755, 0, 0, 0, 0, 0, 4, 0, 7506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8165, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 317, 146, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 1823, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 894, 671, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8181, 0, 0, 0, 7, 0, 2, 2, 1, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1125, 0, 0, 0, 7, 3, 0, 0, 0, 0, 7771, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10681, 8196, 9265, 8637, 3449, 8191, 8210, 8907] avg=8192.0000 max=10681 max_rank=0 min=3449 min_rank=4 imbalance_ratio=1.303833 max_load_reduction=54855 max_load_reduction_pct=0.837021 moved_to_echo=28757 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 504, 6977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 293, 0, 0, 0, 0, 0, 0, 0, 7519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 318, 151, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 897, 673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7481, 8192, 7812, 7985, 9644, 8192, 8189, 8041] avg=8192.0000 max=9644 max_rank=4 min=7481 min_rank=0 imbalance_ratio=1.177246 max_load_reduction=55892 max_load_reduction_pct=0.852844 moved_to_echo=34066 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 793, 6964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 8182, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2673, 0, 0, 1, 0, 0, 1, 0, 7518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 323, 157, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 4404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 900, 672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 8186, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 404, 0, 0, 0, 2, 0, 0, 0, 0, 0, 7977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7757, 8188, 10194, 8635, 5996, 8191, 8192, 8383] avg=8192.0000 max=10194 max_rank=2 min=5996 min_rank=4 imbalance_ratio=1.244385 max_load_reduction=55342 max_load_reduction_pct=0.844452 moved_to_echo=30762 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3948, 8167, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8179, 20, 3, 0, 8185, 0, 8168, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 359, 0, 0, 0, 0, 0, 4, 0, 4, 0, 8042, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 3005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8184, 0, 0, 0, 7, 0, 3, 2, 0, 0, 0, 4, 0, 0, 0, 1200, 0, 0, 0, 7, 3, 0, 0, 0, 0, 7949, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2154, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1481, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6766, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4745, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1788, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 4, 1, 0, 8190, 0, 8184, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 967, 0, 0, 1, 0, 0, 1, 0, 1, 0, 7859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 5312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 8190, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 483, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8144, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1957, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4117, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5100, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1481, 5034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 8017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6766, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2223, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 935, 176, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6515, 8192, 8058, 9163, 9210, 8193, 8189, 8016] avg=8192.0000 max=9210 max_rank=4 min=6515 min_rank=0 imbalance_ratio=1.124268 max_load_reduction=56326 max_load_reduction_pct=0.859467 moved_to_echo=33608 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3948, 5018, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 3, 0, 0, 0, 8168, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 359, 0, 0, 0, 0, 0, 4, 0, 8005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8042, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2216, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 3005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 933, 171, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8181, 0, 0, 0, 7, 0, 3, 2, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1200, 0, 0, 0, 7, 3, 0, 0, 0, 0, 7778, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8971, 8196, 8368, 10438, 4173, 8190, 8210, 8990] avg=8192.0000 max=10438 max_rank=3 min=4173 min_rank=4 imbalance_ratio=1.274170 max_load_reduction=55098 max_load_reduction_pct=0.840729 moved_to_echo=29563 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2154, 5034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 8018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2223, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 935, 176, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7188, 8192, 8027, 8435, 9297, 8192, 8189, 8016] avg=8192.0000 max=9297 max_rank=4 min=7188 min_rank=0 imbalance_ratio=1.134888 max_load_reduction=56239 max_load_reduction_pct=0.858139 moved_to_echo=33694 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4745, 5034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2223, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 935, 176, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9779, 8192, 8017, 5856, 9295, 8192, 8189, 8016] avg=8192.0000 max=9779 max_rank=0 min=5856 min_rank=3 imbalance_ratio=1.193726 max_load_reduction=55757 max_load_reduction_pct=0.850784 moved_to_echo=33692 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1788, 5019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 8182, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 967, 0, 0, 1, 0, 0, 1, 0, 8017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2227, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 5312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 941, 179, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 8185, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 0, 0, 0, 2, 0, 0, 0, 0, 0, 7965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6807, 8188, 8987, 10260, 6461, 8191, 8192, 8450] avg=8192.0000 max=10260 max_rank=3 min=6461 min_rank=4 imbalance_ratio=1.252441 max_load_reduction=55276 max_load_reduction_pct=0.843445 moved_to_echo=31294 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1957, 5034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2223, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 935, 176, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6991, 8192, 8017, 9423, 8514, 8194, 8189, 8016] avg=8192.0000 max=9423 max_rank=3 min=6991 min_rank=0 imbalance_ratio=1.150269 max_load_reduction=56113 max_load_reduction_pct=0.856216 moved_to_echo=32913 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4117, 5034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 8019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2223, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 935, 176, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9151, 8192, 8038, 6452, 9306, 8192, 8189, 8016] avg=8192.0000 max=9306 max_rank=4 min=6452 min_rank=3 imbalance_ratio=1.135986 max_load_reduction=56230 max_load_reduction_pct=0.858002 moved_to_echo=33703 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5100, 5034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 8019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2223, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 935, 176, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10134, 8192, 8024, 5509, 9280, 8192, 8189, 8016] avg=8192.0000 max=10134 max_rank=0 min=5509 min_rank=3 imbalance_ratio=1.237061 max_load_reduction=55402 max_load_reduction_pct=0.845367 moved_to_echo=33677 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 0, 52992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7296, 5504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 9856, 55808, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 5504, 0, 0, 128, 0, 0, 128, 0, 60160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 61824, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 2560, 1280, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 1536, 0, 0, 0, 128, 128, 0, 0, 0, 0, 64000, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 128, 0, 0, 0, 65536, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 65536, 0, 0, 0, 128, 0, 128, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 25344, 40320, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 46464, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 17792, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 0, 56576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7552, 1408, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 1408, 0, 0, 128, 0, 0, 128, 0, 64256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 1792, 0, 0, 0, 128, 128, 0, 0, 0, 0, 63872, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 128, 128, 65536, 0, 0, 0, 128, 0, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 128, 0, 0, 0, 65536, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8191, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 4, 0, 4117, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 5, 0, 0, 8184, 0, 0, 0, 8177, 0, 0, 34, 0, 8181, 0, 0, 2, 0, 0, 0, 6, 0, 0, 0, 0, 7515, 0, 8153, 0, 0, 0, 2, 5, 0, 11, 0, 5, 0, 0, 8171, 0, 0, 1294, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4, 7826, 0, 0, 3574, 0, 0, 6, 4, 0, 0, 0, 0, 217, 0, 0, 0, 1, 7, 0, 0, 4, 0, 0, 0, 0, 5, 0, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 1, 0, 5814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 3, 0, 0, 8183, 0, 0, 4, 0, 8149, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7978, 0, 8188, 0, 0, 0, 1, 1, 0, 7, 0, 0, 0, 0, 8190, 0, 0, 1331, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 8177, 0, 0, 1256, 0, 0, 1, 1, 0, 0, 0, 0, 47, 0, 0, 0, 1, 4, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8084, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8998, 8195, 8298, 7866, 8191, 7632, 8162, 8194] avg=8192.0000 max=8998 max_rank=0 min=7632 min_rank=5 imbalance_ratio=1.098389 max_load_reduction=56538 max_load_reduction_pct=0.862701 moved_to_echo=32179 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8084, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8999, 8194, 8299, 7866, 8191, 7632, 8161, 8194] avg=8192.0000 max=8999 max_rank=0 min=7632 min_rank=5 imbalance_ratio=1.098511 max_load_reduction=56537 max_load_reduction_pct=0.862686 moved_to_echo=32178 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8084, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9001, 8192, 8298, 7866, 8192, 7631, 8160, 8196] avg=8192.0000 max=9001 max_rank=0 min=7631 min_rank=5 imbalance_ratio=1.098755 max_load_reduction=56535 max_load_reduction_pct=0.862656 moved_to_echo=32179 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8084, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8989, 8195, 8308, 7866, 8191, 7632, 8161, 8194] avg=8192.0000 max=8989 max_rank=0 min=7632 min_rank=5 imbalance_ratio=1.097290 max_load_reduction=56547 max_load_reduction_pct=0.862839 moved_to_echo=32178 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7632, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8084, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8997, 8195, 8297, 7866, 8191, 7633, 8163, 8194] avg=8192.0000 max=8997 max_rank=0 min=7633 min_rank=5 imbalance_ratio=1.098267 max_load_reduction=56539 max_load_reduction_pct=0.862717 moved_to_echo=32181 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 4117, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 463, 325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 3, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 104, 0, 0, 2, 0, 0, 0, 6, 0, 0, 0, 0, 7515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7828, 0, 0, 0, 2, 5, 0, 11, 0, 5, 0, 0, 0, 0, 0, 1294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4, 0, 0, 0, 0, 8179, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 7290, 0, 0, 3574, 0, 0, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 217, 0, 0, 0, 1, 7, 0, 0, 4, 0, 0, 0, 0, 5, 0, 5, 8073, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4913, 8185, 7661, 9145, 8193, 10878, 8385, 8176] avg=8192.0000 max=10878 max_rank=5 min=4913 min_rank=0 imbalance_ratio=1.327881 max_load_reduction=54658 max_load_reduction_pct=0.834015 moved_to_echo=35632 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8084, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9001, 8194, 8297, 7866, 8191, 7632, 8160, 8195] avg=8192.0000 max=9001 max_rank=0 min=7632 min_rank=5 imbalance_ratio=1.098755 max_load_reduction=56535 max_load_reduction_pct=0.862656 moved_to_echo=32178 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 5814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 489, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 95, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7855, 0, 0, 0, 1, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 1331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 8188, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7607, 0, 0, 1256, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 1, 4, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 8048, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6638, 8186, 8078, 9195, 8196, 8866, 8184, 8193] avg=8192.0000 max=9195 max_rank=3 min=6638 min_rank=0 imbalance_ratio=1.122437 max_load_reduction=56341 max_load_reduction_pct=0.859695 moved_to_echo=33439 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 4, 0, 4067, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 5, 0, 0, 8184, 0, 0, 0, 8178, 0, 0, 34, 0, 8181, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 7446, 0, 8153, 0, 0, 0, 3, 5, 0, 10, 0, 4, 0, 0, 8171, 0, 0, 1143, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4, 7806, 0, 0, 3689, 0, 0, 6, 4, 0, 0, 0, 0, 397, 0, 0, 0, 1, 7, 0, 0, 4, 0, 0, 0, 0, 5, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 7962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8191, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 8033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8190, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 7942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 1, 0, 5814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 3, 0, 0, 8183, 0, 0, 3, 0, 8140, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7967, 0, 8188, 0, 0, 0, 1, 1, 0, 7, 0, 0, 0, 0, 8190, 0, 0, 1247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 8177, 0, 0, 1335, 0, 0, 1, 1, 0, 0, 0, 0, 74, 0, 0, 0, 1, 4, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 4067, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 556, 296, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 3, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 116, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 7446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7857, 0, 0, 0, 3, 5, 0, 10, 0, 4, 0, 0, 0, 0, 0, 1143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4, 0, 0, 0, 0, 8179, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 7250, 0, 0, 3689, 0, 0, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 397, 0, 0, 0, 1, 7, 0, 0, 4, 0, 0, 0, 0, 5, 0, 5, 8036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4953, 8186, 7599, 9022, 8193, 10953, 8455, 8175] avg=8192.0000 max=10953 max_rank=5 min=4953 min_rank=0 imbalance_ratio=1.337036 max_load_reduction=54583 max_load_reduction_pct=0.832870 moved_to_echo=35776 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 7962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 583, 297, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8867, 8194, 8310, 7895, 8191, 7609, 8276, 8194] avg=8192.0000 max=8867 max_rank=0 min=7609 min_rank=5 imbalance_ratio=1.082397 max_load_reduction=56669 max_load_reduction_pct=0.864700 moved_to_echo=32270 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 583, 297, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7609, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9069, 8195, 8309, 7895, 8191, 7610, 8073, 8194] avg=8192.0000 max=9069 max_rank=0 min=7610 min_rank=5 imbalance_ratio=1.107056 max_load_reduction=56467 max_load_reduction_pct=0.861618 moved_to_echo=32068 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 7942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 583, 297, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8847, 8195, 8311, 7895, 8191, 7609, 8294, 8194] avg=8192.0000 max=8847 max_rank=0 min=7609 min_rank=5 imbalance_ratio=1.079956 max_load_reduction=56689 max_load_reduction_pct=0.865005 moved_to_echo=32288 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 583, 297, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9060, 8195, 8310, 7895, 8191, 7609, 8082, 8194] avg=8192.0000 max=9060 max_rank=0 min=7609 min_rank=5 imbalance_ratio=1.105957 max_load_reduction=56476 max_load_reduction_pct=0.861755 moved_to_echo=32076 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 583, 297, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8938, 8193, 8312, 7895, 8191, 7609, 8204, 8194] avg=8192.0000 max=8938 max_rank=0 min=7609 min_rank=5 imbalance_ratio=1.091064 max_load_reduction=56598 max_load_reduction_pct=0.863617 moved_to_echo=32198 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 5814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 586, 300, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 104, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7888, 0, 0, 0, 1, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 1247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8187, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7591, 0, 0, 1335, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 1, 4, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 7999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8190, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6730, 8186, 8075, 9144, 8197, 8929, 8081, 8194] avg=8192.0000 max=9144 max_rank=3 min=6730 min_rank=0 imbalance_ratio=1.116211 max_load_reduction=56392 max_load_reduction_pct=0.860474 moved_to_echo=33401 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 583, 297, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9072, 8192, 8310, 7895, 8191, 7608, 8071, 8197] avg=8192.0000 max=9072 max_rank=0 min=7608 min_rank=5 imbalance_ratio=1.107422 max_load_reduction=56464 max_load_reduction_pct=0.861572 moved_to_echo=32067 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 59136, 0, 128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 3968, 2688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 60800, 0, 0, 4864, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[62976, 0, 0, 0, 128, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 2688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 58368, 0, 128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 4736, 2432, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 60544, 0, 0, 5120, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[63232, 0, 0, 0, 128, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 1024, 0, 0, 128, 0, 128, 0, 128, 0, 0, 0, 0, 64640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 0, 0, 128, 128, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[384, 0, 0, 0, 128, 128, 0, 0, 128, 0, 0, 0, 0, 128, 0, 128, 64640, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 896, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 64768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[1280, 0, 0, 0, 128, 128, 0, 0, 128, 0, 0, 0, 0, 128, 0, 128, 64384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8070, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8041, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 5, 12, 27, 0, 0, 0, 8185, 0, 8088, 3, 0, 0, 0, 0, 0, 2, 0, 4, 3, 6, 0, 0, 8169, 0, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 5, 0, 0, 14, 0, 0, 0, 0, 0, 0, 10, 82, 0, 0, 0, 0, 0, 4, 5, 0, 3, 0, 0, 8042, 0, 4, 0, 5, 0, 0, 8113, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 1, 0, 10, 0, 0, 0, 8189, 0, 8105, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 8187, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 5, 0, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 8181, 0, 1, 0, 0, 0, 0, 8087, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 46, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8214, 8157, 8128, 8247, 8213, 8193, 8193] avg=8192.0000 max=8247 max_rank=4 min=8128 min_rank=3 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32846 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 8124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 46, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8214, 8157, 8160, 8215, 8213, 8193, 8193] avg=8192.0000 max=8215 max_rank=4 min=8157 min_rank=2 imbalance_ratio=1.002808 max_load_reduction=57321 max_load_reduction_pct=0.874649 moved_to_echo=32814 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 0, 8124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8070, 0, 0, 0, 0, 0, 46, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8214, 8157, 8246, 8129, 8213, 8193, 8193] avg=8192.0000 max=8246 max_rank=3 min=8129 min_rank=4 imbalance_ratio=1.006592 max_load_reduction=57290 max_load_reduction_pct=0.874176 moved_to_echo=32728 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 8124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 46, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8214, 8157, 8133, 8242, 8213, 8193, 8193] avg=8192.0000 max=8242 max_rank=4 min=8133 min_rank=3 imbalance_ratio=1.006104 max_load_reduction=57294 max_load_reduction_pct=0.874237 moved_to_echo=32841 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 0, 8124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8041, 0, 0, 0, 0, 0, 46, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8214, 8157, 8275, 8100, 8213, 8193, 8193] avg=8192.0000 max=8275 max_rank=3 min=8100 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32699 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 8124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 46, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8214, 8157, 8148, 8227, 8213, 8193, 8193] avg=8192.0000 max=8227 max_rank=4 min=8148 min_rank=3 imbalance_ratio=1.004272 max_load_reduction=57309 max_load_reduction_pct=0.874466 moved_to_echo=32826 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 5, 12, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 8088, 3, 0, 0, 0, 0, 0, 2, 0, 4, 3, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8134, 0, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 14, 0, 0, 0, 0, 0, 0, 10, 82, 0, 8117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5, 0, 3, 0, 0, 8042, 0, 4, 0, 5, 0, 46, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8113, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 8181, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8128, 8348, 8228, 8122, 8142, 8186, 8182] avg=8192.0000 max=8348 max_rank=2 min=8122 min_rank=4 imbalance_ratio=1.019043 max_load_reduction=57188 max_load_reduction_pct=0.872620 moved_to_echo=32632 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 1, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 8105, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 5, 0, 8124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 8181, 0, 1, 0, 0, 0, 49, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8087, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8188, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8124, 8246, 8218, 8254, 8116, 8192, 8196] avg=8192.0000 max=8254 max_rank=4 min=8116 min_rank=5 imbalance_ratio=1.007568 max_load_reduction=57282 max_load_reduction_pct=0.874054 moved_to_echo=32758 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 5, 14, 27, 0, 0, 0, 8185, 0, 8082, 3, 0, 0, 0, 0, 0, 2, 0, 4, 2, 6, 0, 0, 8170, 0, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 5, 0, 0, 14, 0, 0, 0, 0, 0, 0, 21, 89, 0, 0, 0, 0, 0, 4, 5, 0, 3, 0, 0, 8045, 0, 5, 1, 5, 0, 0, 8100, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 1, 0, 10, 0, 0, 0, 8189, 0, 8109, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 8187, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 5, 0, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 8184, 0, 1, 0, 0, 0, 0, 8089, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8023, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8139, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8027, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 5, 14, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 8082, 3, 0, 0, 0, 0, 0, 2, 0, 4, 2, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8135, 0, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 14, 0, 0, 0, 0, 0, 0, 21, 89, 0, 8103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5, 0, 3, 0, 0, 8045, 0, 5, 1, 5, 0, 59, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8100, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 8181, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8122, 8346, 8232, 8138, 8130, 8186, 8182] avg=8192.0000 max=8346 max_rank=2 min=8122 min_rank=1 imbalance_ratio=1.018799 max_load_reduction=57190 max_load_reduction_pct=0.872650 moved_to_echo=32636 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 0, 8110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8023, 0, 0, 0, 0, 0, 60, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8214, 8157, 8279, 8094, 8215, 8193, 8193] avg=8192.0000 max=8279 max_rank=3 min=8094 min_rank=4 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32695 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 8110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 60, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8214, 8157, 8124, 8249, 8215, 8193, 8193] avg=8192.0000 max=8249 max_rank=4 min=8124 min_rank=3 imbalance_ratio=1.006958 max_load_reduction=57287 max_load_reduction_pct=0.874130 moved_to_echo=32850 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 1, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 8109, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8146, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 5, 0, 8110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 8184, 0, 1, 0, 0, 0, 61, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8089, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8188, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8130, 8248, 8200, 8264, 8116, 8192, 8196] avg=8192.0000 max=8264 max_rank=4 min=8116 min_rank=5 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32768 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 8110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8139, 0, 0, 0, 0, 0, 60, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8214, 8157, 8163, 8210, 8215, 8193, 8193] avg=8192.0000 max=8215 max_rank=5 min=8157 min_rank=2 imbalance_ratio=1.002808 max_load_reduction=57321 max_load_reduction_pct=0.874649 moved_to_echo=32811 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 8110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 60, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8214, 8157, 8148, 8225, 8215, 8193, 8193] avg=8192.0000 max=8225 max_rank=4 min=8148 min_rank=3 imbalance_ratio=1.004028 max_load_reduction=57311 max_load_reduction_pct=0.874496 moved_to_echo=32826 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 165, 0, 8110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8027, 0, 0, 0, 0, 0, 60, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8214, 8157, 8275, 8098, 8215, 8193, 8193] avg=8192.0000 max=8275 max_rank=3 min=8098 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32699 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 60, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8214, 8157, 8115, 8258, 8215, 8193, 8193] avg=8192.0000 max=8258 max_rank=4 min=8115 min_rank=3 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32859 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 128, 512, 0, 65024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[256, 0, 65408, 128, 0, 0, 0, 0, 0, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[65280, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 65408, 128, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 0, 128, 128, 0, 128, 0, 0, 65152, 0, 128, 0, 128, 0, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 128, 640, 0, 64896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 65536, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 128, 0, 128, 128, 0, 128, 0, 0, 65024, 0, 128, 128, 128, 0, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[65280, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 65408, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[256, 0, 65408, 128, 0, 0, 0, 0, 0, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True - [2026-08-26 14:48:13] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 436.8 | throughput per GPU (TFLOP/s/GPU): 414.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.882549E+00 | loss scale: 1.0 | grad norm: 5.596 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[0, 3, 0, 2, 1, 759, 0, 0, 0, 23, 0, 0, 0, 0, 1, 0, 1, 3, 0, 0, 2067, 0, 0, 3175, 7, 0, 0, 8191, 0, 8189, 1, 4, 0, 3, 0, 0, 1, 0, 0, 1, 2, 0, 6, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 1, 0, 3895, 0, 0, 0, 5504, 0, 6796, 4, 0, 1, 2665, 0, 0, 1, 0, 0, 3, 0, 0, 8112, 0, 2, 0, 1, 0, 0, 127, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7729, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 5, 2, 0, 0, 0, 0, 57, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1034, 1, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2119, 0, 0, 2624, 0, 0, 0, 8192, 2, 8192, 2, 0, 1, 0, 0, 0, 3, 0, 0, 1, 0, 0, 1, 1, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 1, 0, 0, 0, 0, 0, 0, 3500, 0, 0, 0, 5950, 0, 7008, 24, 1, 0, 2603, 0, 0, 0, 0, 0, 6, 0, 0, 7939, 0, 5, 0, 0, 3, 0, 300, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7745, 0, 0, 2, 1, 0, 0, 2, 0, 2, 0, 6, 0, 0, 0, 0, 0, 68, 0, 1, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 1, 1126, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 0, 4, 0, 0, 1997, 0, 0, 2930, 0, 0, 0, 8191, 0, 8191, 1, 0, 1, 0, 6, 0, 2, 0, 0, 0, 1, 0, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 3840, 0, 0, 0, 5216, 0, 6986, 1, 0, 1, 2942, 0, 0, 0, 0, 2, 0, 0, 0, 7949, 0, 1, 0, 0, 0, 2, 135, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7769, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 51, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1017, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2360, 1, 0, 2245, 0, 1, 1, 8190, 0, 8190, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8189, 0, 1, 0, 0, 0, 1, 0, 2, 4068, 0, 0, 2, 5794, 0, 6691, 2, 0, 0, 2955, 0, 0, 0, 0, 0, 0, 0, 0, 7958, 0, 0, 0, 1, 2, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7721, 0, 0, 4, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 1588, 0, 0, 0, 7, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 2091, 0, 0, 2512, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 3820, 0, 0, 0, 5478, 0, 6516, 0, 0, 0, 3052, 0, 0, 0, 0, 0, 0, 0, 0, 8009, 0, 0, 0, 0, 0, 0, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7737, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 767, 0, 1, 0, 6, 0, 0, 0, 1, 2, 2, 0, 0, 0, 0, 2223, 0, 0, 2202, 0, 0, 0, 8191, 0, 8189, 0, 0, 1, 0, 4, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 5, 0, 0, 0, 3869, 0, 0, 0, 6006, 0, 6857, 19, 0, 0, 3183, 0, 0, 0, 1, 4, 1, 0, 0, 7760, 0, 0, 0, 0, 0, 0, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7803, 0, 0, 17, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 1, 0, 0, 40, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 1, 1455, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 1785, 0, 0, 2990, 0, 0, 0, 8192, 2, 8192, 1, 0, 0, 0, 13, 0, 0, 1, 0, 0, 5, 0, 0, 2, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 1, 0, 0, 0, 0, 3959, 0, 0, 0, 5422, 0, 6795, 4, 0, 1, 2672, 0, 0, 0, 0, 0, 0, 0, 0, 7970, 0, 0, 0, 0, 0, 0, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7683, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 2, 4, 0, 0, 0, 0, 22, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7, 668, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1892, 0, 0, 2920, 3, 0, 0, 8191, 0, 8191, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 3, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 3681, 0, 0, 0, 5494, 0, 6868, 3, 0, 0, 3496, 0, 0, 0, 0, 0, 0, 0, 0, 7885, 0, 0, 0, 0, 0, 0, 52, 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 7893, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 2, 1, 759, 0, 0, 0, 23, 0, 0, 0, 0, 1, 0, 3596, 3464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 2067, 0, 0, 3175, 7, 0, 0, 0, 0, 3419, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 1, 2, 0, 6, 0, 0, 1, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3895, 0, 0, 0, 1384, 0, 0, 4, 0, 1, 2665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 8112, 0, 2, 0, 1, 0, 0, 127, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7729, 0, 0, 1, 0, 1, 419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 0, 0, 0, 0, 57, 0, 0, 0, 0, 6796, 1306, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7849, 8677, 8195, 8195, 7950, 8343, 8151, 8176] avg=8192.0000 max=8677 max_rank=1 min=7849 min_rank=0 imbalance_ratio=1.059204 max_load_reduction=56859 max_load_reduction_pct=0.867599 moved_to_echo=32620 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 1, 1126, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 3408, 3464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1997, 0, 0, 2930, 0, 0, 0, 0, 0, 3421, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 0, 2, 0, 0, 0, 1, 0, 0, 1, 0, 0, 2, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8171, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3840, 0, 0, 0, 1311, 0, 0, 1, 0, 1, 2942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 7949, 0, 1, 0, 0, 0, 2, 135, 1, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7769, 0, 0, 0, 0, 6, 397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 51, 0, 0, 0, 0, 6986, 1306, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8004, 8353, 8193, 8180, 8095, 8182, 8173, 8356] avg=8192.0000 max=8356 max_rank=7 min=8004 min_rank=0 imbalance_ratio=1.020020 max_load_reduction=57180 max_load_reduction_pct=0.872498 moved_to_echo=32806 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1034, 1, 0, 0, 4, 0, 0, 0, 0, 0, 1, 3887, 3465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2119, 0, 0, 2624, 0, 0, 0, 0, 2, 3421, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 1, 0, 0, 1, 1, 0, 0, 8, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 1, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3500, 0, 0, 0, 1496, 0, 0, 24, 1, 0, 2603, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 7939, 0, 5, 0, 0, 3, 0, 300, 1, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7745, 0, 0, 2, 1, 0, 453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 6, 0, 0, 0, 0, 0, 68, 0, 1, 1, 0, 7008, 1306, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8392, 8168, 8197, 8191, 7624, 8359, 8201, 8404] avg=8192.0000 max=8404 max_rank=7 min=7624 min_rank=4 imbalance_ratio=1.025879 max_load_reduction=57132 max_load_reduction_pct=0.871765 moved_to_echo=32588 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 1588, 0, 0, 0, 7, 0, 0, 0, 0, 0, 1, 3579, 3465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2091, 0, 0, 2512, 0, 0, 0, 0, 0, 3421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3820, 0, 0, 0, 1377, 0, 0, 0, 0, 0, 3052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8009, 0, 0, 0, 0, 0, 0, 102, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7737, 0, 0, 0, 0, 0, 417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 6516, 1306, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8641, 8025, 8184, 8200, 8249, 8208, 8154, 7875] avg=8192.0000 max=8641 max_rank=0 min=7875 min_rank=7 imbalance_ratio=1.054810 max_load_reduction=56895 max_load_reduction_pct=0.868149 moved_to_echo=32486 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1017, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3785, 3464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2360, 1, 0, 2245, 0, 1, 1, 0, 0, 3420, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8189, 0, 1, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 4068, 0, 0, 2, 1458, 0, 0, 2, 0, 0, 2955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7958, 0, 0, 0, 1, 2, 0, 42, 0, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7721, 0, 0, 4, 0, 0, 441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 1, 6691, 1306, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8267, 8040, 8179, 8200, 8488, 8105, 8166, 8091] avg=8192.0000 max=8488 max_rank=4 min=8040 min_rank=1 imbalance_ratio=1.036133 max_load_reduction=57048 max_load_reduction_pct=0.870483 moved_to_echo=32850 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7, 668, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 3589, 3464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1892, 0, 0, 2920, 3, 0, 0, 0, 0, 3421, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 3, 0, 0, 0, 2, 1, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3681, 0, 0, 0, 1382, 0, 0, 3, 0, 0, 3496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7885, 0, 0, 0, 0, 0, 0, 52, 1, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 7893, 0, 0, 0, 0, 0, 418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 6868, 1306, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7732, 8238, 8189, 8195, 8562, 8035, 8313, 8272] avg=8192.0000 max=8562 max_rank=4 min=7732 min_rank=0 imbalance_ratio=1.045166 max_load_reduction=56974 max_load_reduction_pct=0.869354 moved_to_echo=33182 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 767, 0, 1, 0, 6, 0, 0, 0, 1, 2, 2, 3924, 3464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2223, 0, 0, 2202, 0, 0, 0, 0, 0, 3419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 5, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 3869, 0, 0, 0, 1510, 0, 0, 19, 0, 0, 3183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 1, 0, 0, 7760, 0, 0, 0, 0, 0, 0, 190, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7803, 0, 0, 17, 0, 0, 457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 1, 0, 0, 40, 0, 0, 0, 0, 6857, 1306, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8168, 7844, 8193, 8188, 8586, 8062, 8277, 8218] avg=8192.0000 max=8586 max_rank=4 min=7844 min_rank=1 imbalance_ratio=1.048096 max_load_reduction=56950 max_load_reduction_pct=0.868988 moved_to_echo=33143 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 1, 1455, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8, 3547, 3469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1785, 0, 0, 2990, 0, 0, 0, 0, 2, 3413, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 1, 0, 0, 5, 0, 0, 2, 0, 0, 2, 1, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 1, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3959, 0, 0, 0, 1346, 0, 0, 4, 0, 1, 2672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7970, 0, 0, 0, 0, 0, 0, 171, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7683, 0, 0, 1, 0, 0, 417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 4, 0, 0, 0, 0, 22, 0, 0, 0, 0, 6795, 1310, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8483, 8191, 8206, 8187, 7982, 8242, 8101, 8144] avg=8192.0000 max=8483 max_rank=0 min=7982 min_rank=4 imbalance_ratio=1.035522 max_load_reduction=57053 max_load_reduction_pct=0.870560 moved_to_echo=32469 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True + [2026-08-26 15:24:13] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 454.3 | throughput per GPU (TFLOP/s/GPU): 398.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.882246E+00 | loss scale: 1.0 | grad norm: 5.599 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1171, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2457, 1, 0, 2391, 0, 1, 1, 8190, 0, 8190, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 1, 0, 0, 0, 1, 0, 2, 4056, 0, 0, 1, 5805, 0, 6824, 2, 0, 0, 3135, 0, 0, 0, 0, 0, 0, 0, 0, 7120, 0, 0, 0, 1, 2, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7756, 0, 0, 5, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 157, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 2, 1337, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 4, 0, 0, 2159, 0, 0, 3136, 0, 0, 0, 8191, 0, 8191, 1, 0, 1, 0, 5, 0, 2, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 3827, 0, 0, 0, 5261, 0, 7160, 2, 0, 1, 3106, 0, 0, 0, 0, 1, 0, 0, 0, 6819, 0, 1, 0, 0, 0, 2, 182, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7793, 0, 0, 0, 0, 6, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 162, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 1189, 1, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2255, 0, 0, 2776, 0, 0, 0, 8192, 2, 8192, 2, 0, 1, 0, 0, 0, 3, 0, 0, 1, 0, 0, 1, 2, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 1, 0, 0, 0, 0, 0, 0, 3479, 0, 0, 0, 6023, 0, 7145, 25, 1, 0, 2757, 0, 0, 0, 0, 0, 8, 0, 0, 6891, 0, 11, 0, 0, 3, 0, 413, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7797, 0, 0, 2, 1, 0, 0, 2, 0, 3, 0, 6, 0, 0, 0, 0, 0, 154, 0, 1, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 11, 798, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 2071, 0, 0, 3124, 3, 0, 0, 8191, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5, 1, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 3694, 0, 0, 0, 5513, 0, 7035, 3, 0, 0, 3689, 0, 0, 0, 0, 0, 0, 0, 0, 6824, 0, 0, 0, 0, 0, 0, 63, 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 7930, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 925, 0, 1, 0, 3, 0, 0, 0, 1, 2, 2, 0, 0, 0, 0, 2399, 0, 0, 2374, 0, 0, 0, 8191, 0, 8189, 0, 0, 1, 0, 4, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 5, 0, 0, 0, 3902, 0, 0, 0, 6091, 0, 7047, 21, 0, 0, 3383, 0, 0, 0, 1, 3, 1, 0, 0, 6607, 0, 0, 0, 0, 0, 0, 257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7834, 0, 0, 17, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 1, 0, 0, 83, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 1846, 0, 0, 0, 7, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 2184, 0, 0, 2580, 1, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 3762, 0, 0, 0, 5501, 0, 6705, 0, 0, 0, 3293, 0, 0, 0, 0, 0, 0, 0, 0, 7038, 0, 4, 0, 0, 0, 0, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[0, 4, 0, 2, 1, 872, 0, 0, 0, 22, 0, 0, 0, 0, 1, 0, 1, 3, 0, 0, 2187, 0, 0, 3328, 6, 0, 0, 8191, 0, 8189, 1, 4, 0, 3, 0, 0, 1, 0, 0, 2, 2, 0, 6, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 1, 0, 3868, 0, 0, 0, 5531, 0, 6922, 4, 0, 1, 2780, 0, 0, 1, 0, 0, 3, 0, 0, 7348, 0, 5, 0, 1, 0, 0, 172, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7747, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 5, 2, 0, 0, 0, 0, 128, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 2, 1630, 0, 0, 0, 4, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 1913, 0, 0, 3104, 0, 0, 0, 8192, 2, 8192, 1, 0, 0, 0, 7, 0, 0, 1, 0, 0, 5, 0, 1, 1, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 1, 0, 0, 0, 0, 3959, 0, 0, 0, 5405, 0, 6941, 4, 0, 1, 2766, 0, 0, 0, 0, 0, 1, 0, 0, 7063, 0, 1, 0, 0, 0, 0, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7716, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 2, 4, 0, 0, 0, 0, 194, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 2, 1337, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 4001, 2758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 2159, 0, 0, 3136, 0, 0, 0, 0, 0, 3132, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 0, 2, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8171, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3827, 0, 0, 0, 1169, 0, 0, 2, 0, 1, 3106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 6819, 0, 1, 0, 0, 0, 2, 182, 1, 959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7793, 0, 0, 0, 0, 6, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 162, 0, 0, 0, 0, 7160, 1058, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8102, 8432, 8192, 8179, 8105, 7965, 8168, 8393] avg=8192.0000 max=8432 max_rank=1 min=7965 min_rank=5 imbalance_ratio=1.029297 max_load_reduction=57104 max_load_reduction_pct=0.871338 moved_to_echo=32631 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 1189, 1, 0, 0, 4, 0, 0, 0, 0, 0, 1, 4001, 3158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2255, 0, 0, 2776, 0, 0, 0, 0, 2, 3133, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 1, 0, 0, 1, 2, 0, 0, 7, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 1, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3479, 0, 0, 0, 1338, 0, 0, 25, 1, 0, 2757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 6891, 0, 11, 0, 0, 3, 0, 413, 1, 1098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7797, 0, 0, 2, 1, 0, 421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 0, 6, 0, 0, 0, 0, 0, 154, 0, 1, 1, 0, 7145, 1058, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8355, 8168, 8198, 8190, 7600, 8425, 8221, 8379] avg=8192.0000 max=8425 max_rank=5 min=7600 min_rank=4 imbalance_ratio=1.028442 max_load_reduction=57111 max_load_reduction_pct=0.871445 moved_to_echo=32625 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1171, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4001, 3043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2457, 1, 0, 2391, 0, 1, 1, 0, 0, 3131, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 1, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 4056, 0, 0, 1, 1289, 0, 0, 2, 0, 0, 3135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7120, 0, 0, 0, 1, 2, 0, 60, 0, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7756, 0, 0, 5, 1, 0, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 157, 0, 0, 0, 1, 6824, 1058, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 7993, 8182, 8199, 8486, 8242, 8168, 8050] avg=8192.0000 max=8486 max_rank=4 min=7993 min_rank=1 imbalance_ratio=1.035889 max_load_reduction=57050 max_load_reduction_pct=0.870514 moved_to_echo=32946 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 1846, 0, 0, 0, 7, 0, 0, 0, 0, 0, 1, 4001, 2884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2184, 0, 0, 2580, 1, 0, 0, 0, 0, 3133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3762, 0, 0, 0, 1221, 0, 0, 0, 0, 0, 3293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7038, 0, 4, 0, 0, 0, 0, 159, 0, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7764, 0, 0, 0, 0, 0, 385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 0, 0, 0, 6705, 1058, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8740, 7899, 8183, 8200, 8276, 8204, 8149, 7885] avg=8192.0000 max=8740 max_rank=0 min=7885 min_rank=7 imbalance_ratio=1.066895 max_load_reduction=56796 max_load_reduction_pct=0.866638 moved_to_echo=32514 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 11, 798, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 4001, 2890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2071, 0, 0, 3124, 3, 0, 0, 0, 0, 3132, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5, 1, 0, 0, 2, 1, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3694, 0, 0, 0, 1224, 0, 0, 3, 0, 0, 3689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6824, 0, 0, 0, 0, 0, 0, 63, 1, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 7930, 0, 0, 0, 0, 0, 386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 0, 0, 0, 0, 7035, 1058, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7704, 8332, 8191, 8195, 8610, 7893, 8318, 8293] avg=8192.0000 max=8610 max_rank=4 min=7704 min_rank=0 imbalance_ratio=1.051025 max_load_reduction=56926 max_load_reduction_pct=0.868622 moved_to_echo=33114 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 2, 1630, 0, 0, 0, 4, 0, 0, 0, 0, 0, 8, 4006, 2838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1913, 0, 0, 3104, 0, 0, 0, 0, 2, 3125, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 1, 0, 0, 5, 0, 1, 1, 0, 0, 2, 1, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 1, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3959, 0, 0, 0, 1185, 0, 0, 4, 0, 1, 2766, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7063, 0, 1, 0, 0, 0, 0, 233, 0, 990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7716, 0, 0, 1, 0, 0, 381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 4, 0, 0, 0, 0, 194, 0, 0, 0, 0, 6941, 1061, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8489, 8145, 8201, 8188, 7915, 8288, 8098, 8212] avg=8192.0000 max=8489 max_rank=0 min=7915 min_rank=4 imbalance_ratio=1.036255 max_load_reduction=57047 max_load_reduction_pct=0.870468 moved_to_echo=32513 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 925, 0, 1, 0, 3, 0, 0, 0, 1, 2, 2, 4000, 3193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2399, 0, 0, 2374, 0, 0, 0, 0, 0, 3132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 3902, 0, 0, 0, 1352, 0, 0, 21, 0, 0, 3383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 1, 0, 0, 6607, 0, 0, 0, 0, 0, 0, 257, 0, 1111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7834, 0, 0, 17, 0, 0, 426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 1, 0, 0, 83, 0, 0, 0, 0, 7047, 1057, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8128, 7905, 8192, 8190, 8663, 7980, 8277, 8201] avg=8192.0000 max=8663 max_rank=4 min=7905 min_rank=1 imbalance_ratio=1.057495 max_load_reduction=56873 max_load_reduction_pct=0.867813 moved_to_echo=33121 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 2, 1, 872, 0, 0, 0, 22, 0, 0, 0, 0, 1, 0, 4000, 2900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 2187, 0, 0, 3328, 6, 0, 0, 0, 0, 3132, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 2, 2, 0, 6, 0, 0, 1, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3868, 0, 0, 0, 1227, 0, 0, 4, 0, 1, 2780, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 7348, 0, 5, 0, 1, 0, 0, 172, 0, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7747, 0, 0, 1, 0, 1, 387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 0, 0, 0, 0, 128, 0, 0, 0, 0, 6922, 1057, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7802, 8662, 8197, 8195, 7881, 8539, 8137, 8123] avg=8192.0000 max=8662 max_rank=1 min=7802 min_rank=0 imbalance_ratio=1.057373 max_load_reduction=56874 max_load_reduction_pct=0.867828 moved_to_echo=32680 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 0, 128, 128, 8448, 128, 128, 0, 128, 0, 0, 128, 128, 128, 128, 29440, 27776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 0, 128, 16640, 128, 0, 21632, 128, 128, 128, 0, 128, 27392, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 128, 30720, 0, 0, 128, 11264, 0, 0, 128, 128, 128, 23680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 128, 128, 128, 0, 128, 128, 128, 128, 0, 128, 512, 0, 128, 128, 128, 54528, 10496, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 0, 128, 128, 9856, 128, 128, 0, 128, 0, 0, 128, 128, 128, 128, 32128, 23680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 0, 128, 17664, 128, 0, 22912, 128, 128, 128, 0, 128, 25088, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 128, 30592, 0, 0, 128, 10112, 0, 0, 128, 128, 128, 24960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 128, 128, 128, 128, 0, 0, 55808, 0, 128, 0, 128, 128, 128, 1664, 128, 8320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[0, 128, 128, 128, 0, 128, 128, 128, 128, 0, 128, 1280, 0, 128, 128, 128, 55808, 8576, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 128, 0, 0, 128, 0, 0, 0, 62080, 0, 0, 128, 128, 128, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 128, 128, 0, 0, 63616, 0, 128, 0, 128, 128, 128, 1152, 128, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 128, 0, 0, 128, 0, 0, 0, 62464, 0, 0, 128, 128, 128, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 128, 65536, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 0, 128, 128, 128, 128, 128, 0, 128, 128, 0, 128, 128, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[8190, 0, 1, 0, 0, 0, 8179, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8191, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 6, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[8190, 0, 5, 2, 0, 2, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8189, 8190, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[8191, 0, 7, 1, 0, 1, 8184, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 8191, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[8184, 0, 121, 0, 0, 0, 8071, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8183, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 38, 0, 12, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 15, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 77, 0, 0, 0, 8112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[8188, 0, 93, 0, 0, 0, 8102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[8191, 0, 0, 1, 0, 0, 8186, 0, 4, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 0, 8189, 8191, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 121, 0, 0, 0, 8071, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 38, 8178, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8191, 8194, 8186, 8157, 8195, 8220, 8197] avg=8192.0000 max=8220 max_rank=6 min=8157 min_rank=4 imbalance_ratio=1.003418 max_load_reduction=57316 max_load_reduction_pct=0.874573 moved_to_echo=32769 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 8179, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 8184, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 6, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8194, 8190, 8193, 8200, 8192, 8189, 8196] avg=8192.0000 max=8200 max_rank=4 min=8182 min_rank=0 imbalance_ratio=1.000977 max_load_reduction=57336 max_load_reduction_pct=0.874878 moved_to_echo=32777 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 7, 1, 0, 1, 8184, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8190, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8192, 8193, 8195, 8197, 8191, 8186, 8188] avg=8192.0000 max=8197 max_rank=4 min=8186 min_rank=6 imbalance_ratio=1.000610 max_load_reduction=57339 max_load_reduction_pct=0.874924 moved_to_echo=32762 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 15, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8186, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8200, 8193, 8192, 8183, 8193, 8194, 8192] avg=8192.0000 max=8200 max_rank=1 min=8183 min_rank=4 imbalance_ratio=1.000977 max_load_reduction=57336 max_load_reduction_pct=0.874878 moved_to_echo=32762 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 5, 2, 0, 2, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 8189, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8191, 8192, 8192, 8199, 8190, 8186, 8191] avg=8192.0000 max=8199 max_rank=4 min=8186 min_rank=6 imbalance_ratio=1.000854 max_load_reduction=57337 max_load_reduction_pct=0.874893 moved_to_echo=32766 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 93, 0, 0, 0, 8102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8193, 8193, 8194, 8196, 8191, 8182, 8190] avg=8192.0000 max=8197 max_rank=0 min=8182 min_rank=6 imbalance_ratio=1.000610 max_load_reduction=57339 max_load_reduction_pct=0.874924 moved_to_echo=32759 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 8186, 0, 4, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8188, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8191, 8186, 8192, 8199, 8192, 8191, 8191] avg=8192.0000 max=8199 max_rank=4 min=8186 min_rank=2 imbalance_ratio=1.000854 max_load_reduction=57337 max_load_reduction_pct=0.874893 moved_to_echo=32773 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 77, 0, 0, 0, 8112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8184, 8195, 8192, 8205, 8192, 8188, 8191] avg=8192.0000 max=8205 max_rank=4 min=8184 min_rank=1 imbalance_ratio=1.001587 max_load_reduction=57331 max_load_reduction_pct=0.874802 moved_to_echo=32776 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[8190, 0, 35, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 10, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[8191, 0, 4, 1, 0, 1, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 8191, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[8190, 0, 1, 0, 0, 0, 8182, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8191, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 5, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[8191, 0, 0, 1, 0, 0, 8188, 0, 4, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 8189, 8191, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[8190, 0, 1, 2, 0, 2, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8189, 8190, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 45, 0, 0, 0, 8146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[8188, 0, 98, 0, 0, 0, 8097, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8180, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 23, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 35, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8193, 8193, 8195, 8193, 8192, 8188, 8188] avg=8192.0000 max=8195 max_rank=3 min=8188 min_rank=6 imbalance_ratio=1.000366 max_load_reduction=57341 max_load_reduction_pct=0.874954 moved_to_echo=32761 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 8181, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 5, 0, 0, 0, 0, 8185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8192, 8189, 8194, 8198, 8192, 8192, 8194] avg=8192.0000 max=8198 max_rank=4 min=8185 min_rank=0 imbalance_ratio=1.000732 max_load_reduction=57338 max_load_reduction_pct=0.874908 moved_to_echo=32776 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 10, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8189, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8200, 8193, 8192, 8181, 8193, 8190, 8193] avg=8192.0000 max=8200 max_rank=1 min=8181 min_rank=4 imbalance_ratio=1.000977 max_load_reduction=57336 max_load_reduction_pct=0.874878 moved_to_echo=32757 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 1, 0, 1, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 3, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8190, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8192, 8193, 8196, 8195, 8191, 8189, 8187] avg=8192.0000 max=8196 max_rank=3 min=8187 min_rank=7 imbalance_ratio=1.000488 max_load_reduction=57340 max_load_reduction_pct=0.874939 moved_to_echo=32762 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 2, 0, 2, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 8190, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8191, 8192, 8193, 8198, 8190, 8189, 8190] avg=8192.0000 max=8198 max_rank=4 min=8189 min_rank=6 imbalance_ratio=1.000732 max_load_reduction=57338 max_load_reduction_pct=0.874908 moved_to_echo=32767 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 45, 0, 0, 0, 8140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 6, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8186, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8186, 8197, 8191, 8206, 8192, 8191, 8188] avg=8192.0000 max=8206 max_rank=4 min=8185 min_rank=0 imbalance_ratio=1.001709 max_load_reduction=57330 max_load_reduction_pct=0.874786 moved_to_echo=32777 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 8187, 0, 4, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8188, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8188, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8191, 8185, 8192, 8197, 8191, 8195, 8191] avg=8192.0000 max=8197 max_rank=4 min=8185 min_rank=2 imbalance_ratio=1.000610 max_load_reduction=57339 max_load_reduction_pct=0.874924 moved_to_echo=32774 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 98, 0, 0, 0, 8096, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8191, 8194, 8183, 8168, 8195, 8202, 8205] avg=8192.0000 max=8205 max_rank=7 min=8168 min_rank=4 imbalance_ratio=1.001587 max_load_reduction=57331 max_load_reduction_pct=0.874802 moved_to_echo=32770 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 384, 128, 0, 128, 65280, 0, 128, 0, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 0, 0, 128, 0, 128, 128, 128, 0, 0, 128, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 0, 65536, 0, 128, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 0, 128, 0, 0, 128, 0, 0, 0, 128, 128, 0, 0, 128, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[65536, 128, 0, 0, 128, 0, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 0, 128, 0, 0, 128, 0, 0, 0, 128, 128, 0, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 128, 0, 0, 128, 0, 0, 0, 65536, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[65536, 128, 0, 0, 128, 0, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 256, 128, 0, 128, 65408, 0, 128, 0, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 128, 128, 0, 0, 128, 0, 0, 0, 65536, 128, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 0, 65536, 0, 128, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 0, 0, 128, 0, 128, 128, 128, 0, 0, 128, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 8190, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 170, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 8021, 0, 8191, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 184, 0, 0, 0, 1, 8190, 0, 0, 0, 0, 1, 0, 8188, 2, 0, 0, 1, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 8006, 0, 8190, 0, 2, 0, 0, 1, 0, 0, 0, 8190, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 8191, 0, 8186, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 95, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 8096, 0, 8191, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 8182, 0, 8186, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 8190, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 173, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 8018, 0, 8191, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 188, 0, 0, 0, 1, 8190, 0, 0, 0, 0, 1, 0, 8189, 2, 0, 0, 1, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 8002, 0, 8190, 0, 2, 0, 0, 1, 0, 0, 0, 8190, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8123, 56, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8198, 8131, 8192, 8191, 8191, 8196, 8245] avg=8192.0000 max=8245 max_rank=7 min=8131 min_rank=2 imbalance_ratio=1.006470 max_load_reduction=57291 max_load_reduction_pct=0.874191 moved_to_echo=32823 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 0, 0, 8135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8130, 56, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 8096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8192, 8230, 8192, 8189, 8193, 8191, 8156] avg=8192.0000 max=8230 max_rank=2 min=8156 min_rank=7 imbalance_ratio=1.004639 max_load_reduction=57306 max_load_reduction_pct=0.874420 moved_to_echo=32729 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8135, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 56, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8193, 8136, 8194, 8190, 8192, 8191, 8250] avg=8192.0000 max=8250 max_rank=7 min=8136 min_rank=2 imbalance_ratio=1.007080 max_load_reduction=57286 max_load_reduction_pct=0.874115 moved_to_echo=32823 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 56, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8193, 8135, 8192, 8190, 8192, 8195, 8245] avg=8192.0000 max=8245 max_rank=7 min=8135 min_rank=2 imbalance_ratio=1.006470 max_load_reduction=57291 max_load_reduction_pct=0.874191 moved_to_echo=32822 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 56, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8192, 8136, 8192, 8190, 8191, 8193, 8251] avg=8192.0000 max=8251 max_rank=7 min=8136 min_rank=2 imbalance_ratio=1.007202 max_load_reduction=57285 max_load_reduction_pct=0.874100 moved_to_echo=32825 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 0, 0, 1, 8134, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8131, 56, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 8002, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8192, 8326, 8186, 8192, 8196, 8189, 8064] avg=8192.0000 max=8326 max_rank=2 min=8064 min_rank=7 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32641 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 173, 0, 0, 0, 0, 8135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 8132, 56, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 8018, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8192, 8308, 8192, 8193, 8190, 8190, 8078] avg=8192.0000 max=8308 max_rank=2 min=8078 min_rank=7 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32651 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 58, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8184, 8134, 8196, 8201, 8191, 8191, 8247] avg=8192.0000 max=8247 max_rank=7 min=8134 min_rank=2 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32830 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 8179, 0, 8186, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 55, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8193, 8137, 8193, 8191, 8192, 8191, 8249] avg=8192.0000 max=8249 max_rank=7 min=8137 min_rank=2 imbalance_ratio=1.006958 max_load_reduction=57287 max_load_reduction_pct=0.874130 moved_to_echo=32823 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8134, 55, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8192, 8137, 8192, 8190, 8192, 8192, 8250] avg=8192.0000 max=8250 max_rank=7 min=8137 min_rank=2 imbalance_ratio=1.007080 max_load_reduction=57286 max_load_reduction_pct=0.874115 moved_to_echo=32824 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 58, 2, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8184, 8134, 8195, 8202, 8191, 8191, 8247] avg=8192.0000 max=8247 max_rank=7 min=8134 min_rank=2 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32831 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 55, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8193, 8136, 8192, 8190, 8192, 8195, 8244] avg=8192.0000 max=8244 max_rank=7 min=8136 min_rank=2 imbalance_ratio=1.006348 max_load_reduction=57292 max_load_reduction_pct=0.874207 moved_to_echo=32821 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 0, 0, 8136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8131, 55, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 8096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8192, 8231, 8192, 8189, 8193, 8191, 8155] avg=8192.0000 max=8231 max_rank=2 min=8155 min_rank=7 imbalance_ratio=1.004761 max_load_reduction=57305 max_load_reduction_pct=0.874405 moved_to_echo=32728 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 170, 0, 0, 0, 0, 8136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 8133, 55, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 8021, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8192, 8306, 8192, 8192, 8190, 8190, 8080] avg=8192.0000 max=8306 max_rank=2 min=8080 min_rank=7 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32652 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 0, 0, 0, 1, 8135, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8132, 55, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8187, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 8006, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8192, 8323, 8187, 8192, 8195, 8190, 8067] avg=8192.0000 max=8323 max_rank=2 min=8067 min_rank=7 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32644 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8121, 55, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8198, 8132, 8193, 8190, 8191, 8196, 8244] avg=8192.0000 max=8244 max_rank=7 min=8132 min_rank=2 imbalance_ratio=1.006348 max_load_reduction=57292 max_load_reduction_pct=0.874207 moved_to_echo=32821 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 128, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[512, 0, 65152, 0, 0, 0, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 128, 0, 0, 128, 0, 65152, 512, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[512, 0, 0, 0, 128, 65152, 0, 0, 0, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 65536, 0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 704, 0, 1, 0, 0, 0, 0, 8191, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8029, 0, 0, 0, 8191, 5492, 8176, 0, 0, 0, 1, 0, 8190, 8176, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5452, 0, 0, 0, 0, 1, 0, 8190, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 4606, 0, 1, 0, 8191, 6321, 8188, 1, 0, 0, 0, 0, 8191, 8191, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3412, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5411, 0, 0, 0, 8192, 7552, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 1, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 8191, 6134, 8182, 0, 0, 0, 1, 0, 8189, 8189, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5763, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 11, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 29, 0, 0, 0, 0, 0, 4930, 0, 0, 0, 8192, 5603, 8187, 0, 0, 0, 0, 0, 8192, 8186, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5346, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4193, 0, 0, 0, 8192, 6838, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2953, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 5667, 0, 0, 0, 8192, 7757, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 2, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8190, 5546, 8168, 0, 0, 0, 2, 0, 8180, 8189, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6401, 923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5452, 0, 0, 0, 0, 1, 0, 5230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 4606, 2037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6321, 1787, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 7325, 8193, 8191, 10683, 8195, 6647, 8111] avg=8192.0000 max=10683 max_rank=4 min=6647 min_rank=6 imbalance_ratio=1.304077 max_load_reduction=54853 max_load_reduction_pct=0.836990 moved_to_echo=33636 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6392, 923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 704, 0, 1, 0, 0, 0, 0, 5230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8176, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8029, 2038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5492, 1784, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 9497, 8192, 8194, 5935, 8182, 10068, 7278] avg=8192.0000 max=10068 max_rank=6 min=5935 min_rank=4 imbalance_ratio=1.229004 max_load_reduction=55468 max_load_reduction_pct=0.846375 moved_to_echo=31463 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 6400, 921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5763, 0, 0, 0, 0, 0, 0, 5220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 29, 0, 0, 0, 0, 0, 4930, 2034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5603, 1787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 7380, 8198, 8192, 10984, 8199, 6995, 7390] avg=8192.0000 max=10984 max_rank=4 min=6995 min_rank=6 imbalance_ratio=1.340820 max_load_reduction=54552 max_load_reduction_pct=0.832397 moved_to_echo=33568 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6404, 923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3412, 0, 0, 0, 0, 0, 0, 5231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5411, 2038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7552, 1788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 7336, 8192, 8192, 8643, 8193, 7449, 9340] avg=8192.0000 max=9340 max_rank=7 min=7336 min_rank=1 imbalance_ratio=1.140137 max_load_reduction=56196 max_load_reduction_pct=0.857483 moved_to_echo=33625 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 6404, 923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2953, 0, 0, 0, 0, 0, 0, 5229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 5667, 2037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7757, 1788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 7331, 8194, 8192, 8183, 8192, 7707, 9545] avg=8192.0000 max=9545 max_rank=7 min=7331 min_rank=1 imbalance_ratio=1.165161 max_load_reduction=55991 max_load_reduction_pct=0.854355 moved_to_echo=33627 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 6409, 928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5346, 0, 0, 0, 0, 0, 0, 5223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4193, 2041, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6838, 1781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 7344, 8192, 8190, 10569, 8194, 6236, 8619] avg=8192.0000 max=10569 max_rank=4 min=6236 min_rank=6 imbalance_ratio=1.290161 max_load_reduction=54967 max_load_reduction_pct=0.838730 moved_to_echo=33618 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 6386, 923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 2, 0, 0, 0, 0, 5229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5546, 1782, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 9959, 8184, 8193, 5257, 8191, 10229, 7332] avg=8192.0000 max=10229 max_rank=6 min=5257 min_rank=4 imbalance_ratio=1.248657 max_load_reduction=55307 max_load_reduction_pct=0.843918 moved_to_echo=31009 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6397, 923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 1, 0, 0, 0, 0, 5228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 2037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6134, 1785, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 9364, 8191, 8192, 5282, 8190, 10205, 7921] avg=8192.0000 max=10205 max_rank=6 min=5282 min_rank=4 imbalance_ratio=1.245728 max_load_reduction=55331 max_load_reduction_pct=0.844284 moved_to_echo=31598 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3148, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5738, 0, 0, 0, 8192, 7481, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 2, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8190, 5225, 8170, 0, 0, 0, 2, 0, 8179, 8189, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1949, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 6738, 0, 0, 0, 8192, 7662, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4698, 0, 0, 0, 0, 1, 0, 8190, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 5460, 0, 0, 0, 8191, 6214, 8189, 0, 0, 0, 0, 0, 8192, 8191, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5371, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 10, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 28, 0, 0, 0, 0, 0, 4966, 0, 0, 0, 8192, 5913, 8187, 0, 0, 0, 0, 0, 8192, 8186, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 577, 0, 1, 0, 0, 0, 0, 8191, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8085, 0, 0, 0, 8191, 5034, 8178, 0, 0, 0, 1, 0, 8190, 8177, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 1, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 8191, 5487, 8181, 0, 0, 0, 1, 0, 8189, 8189, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4653, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5076, 0, 0, 0, 8192, 6642, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6204, 923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4698, 0, 0, 0, 0, 1, 0, 5634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 5460, 1633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6214, 1985, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 7127, 8193, 8191, 10339, 8196, 7098, 8200] avg=8192.0000 max=10339 max_rank=4 min=7098 min_rank=6 imbalance_ratio=1.262085 max_load_reduction=55197 max_load_reduction_pct=0.842239 moved_to_echo=33833 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6206, 923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3148, 0, 0, 0, 0, 0, 0, 5636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5738, 1633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7481, 1986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 7146, 8192, 8192, 8784, 8193, 7371, 9467] avg=8192.0000 max=9467 max_rank=7 min=7146 min_rank=1 imbalance_ratio=1.155640 max_load_reduction=56069 max_load_reduction_pct=0.855545 moved_to_echo=33815 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 6206, 922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1949, 0, 0, 0, 0, 0, 0, 5635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 6738, 1632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7662, 1986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 7160, 8194, 8192, 7585, 8192, 8372, 9648] avg=8192.0000 max=9648 max_rank=7 min=7160 min_rank=1 imbalance_ratio=1.177734 max_load_reduction=55888 max_load_reduction_pct=0.852783 moved_to_echo=33797 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 6189, 922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 2, 0, 0, 0, 0, 5635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5225, 1981, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 10084, 8183, 8193, 5661, 8192, 9824, 7210] avg=8192.0000 max=10084 max_rank=1 min=5661 min_rank=4 imbalance_ratio=1.230957 max_load_reduction=55452 max_load_reduction_pct=0.846130 moved_to_echo=30887 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6198, 922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 1, 0, 0, 0, 0, 5634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 1632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5487, 1983, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 9815, 8191, 8192, 5677, 8190, 9808, 7472] avg=8192.0000 max=9815 max_rank=1 min=5677 min_rank=4 imbalance_ratio=1.198120 max_load_reduction=55721 max_load_reduction_pct=0.850235 moved_to_echo=31147 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 6209, 927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4653, 0, 0, 0, 0, 0, 0, 5628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5076, 1637, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6642, 1980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 7149, 8192, 8188, 10281, 8195, 6717, 8622] avg=8192.0000 max=10281 max_rank=4 min=6717 min_rank=6 imbalance_ratio=1.255005 max_load_reduction=55255 max_load_reduction_pct=0.843124 moved_to_echo=33815 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6195, 923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 577, 0, 1, 0, 0, 0, 0, 5635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8177, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8085, 1633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5034, 1983, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 9826, 8192, 8196, 6213, 8181, 9719, 7019] avg=8192.0000 max=9826 max_rank=1 min=6213 min_rank=4 imbalance_ratio=1.199463 max_load_reduction=55710 max_load_reduction_pct=0.850067 moved_to_echo=31132 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 6202, 921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5371, 0, 0, 0, 0, 0, 0, 5624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 28, 0, 0, 0, 0, 0, 4966, 1630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5913, 1985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 7229, 8199, 8192, 10996, 8197, 6627, 7898] avg=8192.0000 max=10996 max_rank=4 min=6627 min_rank=6 imbalance_ratio=1.342285 max_load_reduction=54540 max_load_reduction_pct=0.832214 moved_to_echo=33718 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 7040, 0, 128, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 51200, 7424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 0, 23808, 0, 128, 0, 0, 128, 0, 41856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 0, 51328, 14336, 128, 0, 0, 128, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 49280, 16384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 8576, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 49664, 7424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 49664, 15872, 0, 0, 0, 128, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 0, 20480, 0, 128, 0, 0, 128, 0, 45184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 52480, 13184, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 128, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 128, 0, 65536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8184, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 8192, 8191, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8162, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8190, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8191, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 1, 1, 0, 0, 0, 0, 8107, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8189, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 107, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 46, 0, 1, 0, 0, 0, 0, 0, 0, 8059, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, 1, 1, 0, 0, 0, 0, 8090, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8186, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 111, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 38, 0, 1, 0, 0, 0, 0, 0, 0, 8019, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 8192, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 8167, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 8154, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8185, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8192, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8188, 8190, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 2, 2, 0, 0, 0, 0, 7880, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8186, 8134, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 112, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 103, 0, 2, 0, 0, 0, 0, 0, 0, 8076, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 8192, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 8107, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 8154, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8115, 8192, 8188, 8209, 8237, 8185, 8164, 8246] avg=8192.0000 max=8246 max_rank=7 min=8115 min_rank=0 imbalance_ratio=1.006592 max_load_reduction=57290 max_load_reduction_pct=0.874176 moved_to_echo=32832 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 1, 1, 0, 0, 0, 0, 8107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 13, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8131, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8176, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 46, 0, 1, 0, 0, 0, 0, 8155, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8059, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8214, 8192, 8217, 8160, 8238, 8191, 8215, 8109] avg=8192.0000 max=8238 max_rank=4 min=8109 min_rank=7 imbalance_ratio=1.005615 max_load_reduction=57298 max_load_reduction_pct=0.874298 moved_to_echo=32753 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8125, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8192, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8192, 8192, 8205, 8152, 8191, 8166, 8258] avg=8192.0000 max=8258 max_rank=7 min=8152 min_rank=4 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32767 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, 1, 1, 0, 0, 0, 0, 8090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 13, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8131, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8173, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 38, 0, 1, 0, 0, 0, 0, 8153, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8019, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8214, 8192, 8226, 8198, 8242, 8190, 8205, 8069] avg=8192.0000 max=8242 max_rank=4 min=8069 min_rank=7 imbalance_ratio=1.006104 max_load_reduction=57294 max_load_reduction_pct=0.874237 moved_to_echo=32706 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 2, 2, 0, 0, 0, 0, 7880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 13, 8134, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8130, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8173, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 103, 0, 2, 0, 0, 0, 0, 8152, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8076, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8214, 8193, 8139, 8151, 8242, 8199, 8271, 8127] avg=8192.0000 max=8271 max_rank=6 min=8127 min_rank=7 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32839 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 13, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8215, 8191, 8191, 8205, 8133, 8193, 8168, 8240] avg=8192.0000 max=8240 max_rank=7 min=8133 min_rank=4 imbalance_ratio=1.005859 max_load_reduction=57296 max_load_reduction_pct=0.874268 moved_to_echo=32734 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8122, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8192, 8194, 8204, 8156, 8194, 8179, 8242] avg=8192.0000 max=8242 max_rank=7 min=8156 min_rank=4 imbalance_ratio=1.006104 max_load_reduction=57294 max_load_reduction_pct=0.874237 moved_to_echo=32771 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8184, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8103, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8190, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8192, 8189, 8204, 8136, 8193, 8168, 8245] avg=8192.0000 max=8245 max_rank=7 min=8136 min_rank=4 imbalance_ratio=1.006470 max_load_reduction=57291 max_load_reduction_pct=0.874191 moved_to_echo=32742 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8107, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8184, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8188, 8190, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 227, 2, 2, 0, 0, 0, 0, 7783, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8187, 7760, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 612, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 103, 0, 2, 0, 0, 0, 0, 0, 0, 8074, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8151, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8184, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 8192, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 8164, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 8149, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 1, 1, 0, 0, 0, 0, 8094, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8189, 7569, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 719, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 47, 0, 1, 0, 0, 0, 0, 0, 0, 8040, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 1, 1, 0, 0, 0, 0, 8067, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8186, 8101, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 228, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 40, 0, 1, 0, 0, 0, 0, 0, 0, 8012, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8176, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7876, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8032, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8184, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8192, 8230, 8334, 7972, 8190, 8167, 8246] avg=8192.0000 max=8334 max_rank=3 min=7972 min_rank=4 imbalance_ratio=1.017334 max_load_reduction=57202 max_load_reduction_pct=0.872833 moved_to_echo=32575 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7959, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 17, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8033, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8211, 8191, 8235, 8334, 7962, 8191, 8170, 8242] avg=8192.0000 max=8334 max_rank=3 min=7962 min_rank=4 imbalance_ratio=1.017334 max_load_reduction=57202 max_load_reduction_pct=0.872833 moved_to_echo=32565 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8026, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 7919, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8032, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8184, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8192, 8070, 8334, 8150, 8187, 8168, 8256] avg=8192.0000 max=8334 max_rank=3 min=8070 min_rank=2 imbalance_ratio=1.017334 max_load_reduction=57202 max_load_reduction_pct=0.872833 moved_to_echo=32761 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 227, 2, 2, 0, 0, 0, 0, 7783, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 142, 7760, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7957, 612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8169, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 103, 0, 2, 0, 0, 0, 0, 8028, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8074, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8193, 8058, 7906, 8569, 8199, 8272, 8129] avg=8192.0000 max=8569 max_rank=4 min=7906 min_rank=3 imbalance_ratio=1.046021 max_load_reduction=56967 max_load_reduction_pct=0.869247 moved_to_echo=33169 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 1, 1, 0, 0, 0, 0, 8094, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 142, 7569, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7958, 719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8172, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 47, 0, 1, 0, 0, 0, 0, 8030, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8040, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8192, 8239, 7717, 8677, 8191, 8216, 8094] avg=8192.0000 max=8677 max_rank=4 min=7717 min_rank=3 imbalance_ratio=1.059204 max_load_reduction=56859 max_load_reduction_pct=0.867599 moved_to_echo=33178 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 1, 1, 0, 0, 0, 0, 8067, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 142, 8101, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7958, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8170, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 40, 0, 1, 0, 0, 0, 0, 8027, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8012, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8192, 8239, 8246, 8186, 8191, 8206, 8066] avg=8192.0000 max=8246 max_rank=3 min=8066 min_rank=7 imbalance_ratio=1.006592 max_load_reduction=57290 max_load_reduction_pct=0.874176 moved_to_echo=32649 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 8192, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 7932, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8026, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 8149, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8125, 8192, 8228, 8338, 8049, 8186, 8162, 8256] avg=8192.0000 max=8338 max_rank=3 min=8049 min_rank=4 imbalance_ratio=1.017822 max_load_reduction=57198 max_load_reduction_pct=0.872772 moved_to_echo=32653 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 7932, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8036, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8192, 8237, 8327, 7971, 8201, 8175, 8247] avg=8192.0000 max=8327 max_rank=3 min=7971 min_rank=4 imbalance_ratio=1.016479 max_load_reduction=57209 max_load_reduction_pct=0.872940 moved_to_echo=32594 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 128, 0, 0, 0, 256, 0, 128, 0, 0, 0, 0, 65280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 65408, 0, 0, 128, 0, 0, 0, 0, 128, 128, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 0, 128, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 65536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 512, 128, 128, 0, 0, 0, 0, 65152, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 65152, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 65024, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 63616, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 1152, 64512, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 512, 128, 128, 0, 0, 0, 0, 64768, 0, 0, 0, 0, 0, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 128, 0, 0, 0, 256, 0, 128, 0, 0, 0, 0, 64256, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 65024, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 65408, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 65408, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7145, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8185, 0, 39, 0, 0, 1173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 6987, 0, 0, 13, 0, 0, 0, 8191, 0, 0, 0, 1, 8186, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8191, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 52, 0, 0, 847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 7344, 0, 0, 18, 0, 0, 0, 8192, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 8139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 8162, 0, 105, 0, 0, 1655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 6453, 0, 0, 43, 0, 0, 0, 8192, 0, 0, 0, 7, 8185, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 8120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7529, 0, 130, 0, 0, 4672, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 2727, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 1, 0, 8191, 0, 0, 0, 1, 8119, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1294, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8191, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 80, 0, 0, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 7168, 0, 0, 6, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7338, 0, 80, 0, 0, 3984, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 2765, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 1, 0, 8191, 0, 0, 0, 1, 8117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2140, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8191, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 7111, 0, 98, 0, 0, 5133, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1311, 0, 0, 0, 0, 1, 0, 8170, 0, 0, 2, 0, 8190, 0, 0, 0, 2, 8062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2721, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8190, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5908, 0, 39, 0, 0, 1173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 6987, 0, 0, 13, 0, 0, 2277, 734, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8153, 8190, 7121, 10026, 8226, 8189, 7426, 8205] avg=8192.0000 max=10026 max_rank=3 min=7121 min_rank=2 imbalance_ratio=1.223877 max_load_reduction=55510 max_load_reduction_pct=0.847015 moved_to_echo=32046 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5891, 0, 105, 0, 0, 1655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 6453, 0, 0, 43, 0, 0, 2271, 734, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 7, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8156, 8193, 7653, 9547, 8229, 8186, 7420, 8152] avg=8192.0000 max=9547 max_rank=3 min=7420 min_rank=6 imbalance_ratio=1.165405 max_load_reduction=55989 max_load_reduction_pct=0.854324 moved_to_echo=31987 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5900, 0, 52, 0, 0, 847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 7344, 0, 0, 18, 0, 0, 2274, 734, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8150, 8192, 6799, 10388, 8226, 8193, 7417, 8171] avg=8192.0000 max=10388 max_rank=3 min=6799 min_rank=2 imbalance_ratio=1.268066 max_load_reduction=55148 max_load_reduction_pct=0.841492 moved_to_echo=32007 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5913, 0, 0, 0, 0, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7145, 0, 0, 0, 0, 0, 2279, 734, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 8191, 6983, 10165, 8226, 8194, 7426, 8200] avg=8192.0000 max=10165 max_rank=3 min=6983 min_rank=2 imbalance_ratio=1.240845 max_load_reduction=55371 max_load_reduction_pct=0.844894 moved_to_echo=32046 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 8141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5434, 0, 130, 0, 0, 4672, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 2727, 0, 0, 0, 0, 0, 2095, 734, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 1, 0, 0, 0, 0, 0, 1, 8119, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 8191, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8237, 8191, 10237, 5576, 8157, 8194, 8719, 8225] avg=8192.0000 max=10237 max_rank=2 min=5576 min_rank=3 imbalance_ratio=1.249634 max_load_reduction=55299 max_load_reduction_pct=0.843796 moved_to_echo=33295 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 0, 0, 8141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5297, 0, 80, 0, 0, 3984, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 2765, 0, 0, 0, 0, 0, 2041, 734, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 1, 0, 0, 0, 0, 0, 1, 8117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8256, 8193, 9362, 5589, 8154, 8192, 9565, 8225] avg=8192.0000 max=9565 max_rank=6 min=5589 min_rank=3 imbalance_ratio=1.167603 max_load_reduction=55971 max_load_reduction_pct=0.854050 moved_to_echo=34136 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 0, 8128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5133, 0, 98, 0, 0, 5133, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1311, 0, 0, 0, 0, 1, 1978, 734, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 2, 0, 0, 0, 0, 0, 2, 8062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8278, 8194, 10366, 4034, 8101, 8192, 10145, 8226] avg=8192.0000 max=10366 max_rank=2 min=4034 min_rank=3 imbalance_ratio=1.265381 max_load_reduction=55170 max_load_reduction_pct=0.841827 moved_to_echo=34664 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5898, 0, 80, 0, 0, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 7168, 0, 0, 6, 0, 0, 2279, 740, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8155, 8192, 7015, 10211, 8217, 8196, 7418, 8132] avg=8192.0000 max=10211 max_rank=3 min=7015 min_rank=2 imbalance_ratio=1.246460 max_load_reduction=55325 max_load_reduction_pct=0.844193 moved_to_echo=31963 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7509, 0, 1485, 0, 0, 3065, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 3255, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 1, 0, 8191, 0, 0, 0, 1, 8116, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1032, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8191, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 7063, 0, 1618, 0, 0, 3688, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1884, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 2, 0, 8190, 0, 0, 0, 2, 8060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2123, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8190, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 54, 0, 0, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7519, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8185, 0, 51, 0, 0, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8, 7708, 0, 0, 10, 0, 0, 0, 8192, 0, 0, 0, 1, 8184, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 75, 0, 0, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 7846, 0, 0, 22, 0, 0, 0, 8192, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 8130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 228, 0, 0, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 7622, 0, 0, 11, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7308, 0, 1479, 0, 0, 2596, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 3227, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 1, 0, 8191, 0, 0, 0, 1, 8118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1692, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8191, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 8163, 0, 677, 0, 0, 820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 6727, 0, 0, 52, 0, 0, 0, 8192, 0, 0, 0, 7, 8185, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 8113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6230, 0, 51, 0, 0, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8, 7708, 0, 0, 10, 0, 0, 1955, 561, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 8191, 6746, 10250, 8227, 8186, 7588, 8197] avg=8192.0000 max=10250 max_rank=3 min=6746 min_rank=2 imbalance_ratio=1.251221 max_load_reduction=55286 max_load_reduction_pct=0.843597 moved_to_echo=32198 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6235, 0, 54, 0, 0, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7519, 0, 0, 0, 0, 0, 1957, 561, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8150, 8191, 6933, 10044, 8227, 8194, 7589, 8208] avg=8192.0000 max=10044 max_rank=3 min=6933 min_rank=2 imbalance_ratio=1.226074 max_load_reduction=55492 max_load_reduction_pct=0.846741 moved_to_echo=32218 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 8140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5715, 0, 1485, 0, 0, 3065, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 3255, 0, 0, 0, 0, 0, 1794, 561, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 1, 0, 0, 0, 0, 0, 1, 8116, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 8191, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8243, 8191, 10266, 5632, 8154, 8195, 8620, 8235] avg=8192.0000 max=10266 max_rank=2 min=5632 min_rank=3 imbalance_ratio=1.253174 max_load_reduction=55270 max_load_reduction_pct=0.843353 moved_to_echo=33204 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6218, 0, 75, 0, 0, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 7846, 0, 0, 22, 0, 0, 1951, 560, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8150, 8192, 6626, 10391, 8227, 8194, 7584, 8172] avg=8192.0000 max=10391 max_rank=3 min=6626 min_rank=2 imbalance_ratio=1.268433 max_load_reduction=55145 max_load_reduction_pct=0.841446 moved_to_echo=32177 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 0, 8128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5376, 0, 1618, 0, 0, 3688, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1884, 0, 0, 0, 0, 0, 1687, 561, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 2, 0, 0, 0, 0, 0, 2, 8060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8190, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8278, 8193, 10684, 4142, 8099, 8194, 9710, 8236] avg=8192.0000 max=10684 max_rank=2 min=4142 min_rank=3 imbalance_ratio=1.304199 max_load_reduction=54852 max_load_reduction_pct=0.836975 moved_to_echo=34239 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0, 0, 8140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5562, 0, 1479, 0, 0, 2596, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 3227, 0, 0, 0, 0, 0, 1746, 561, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 1, 0, 0, 0, 0, 0, 1, 8118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8256, 8193, 9638, 5586, 8155, 8193, 9280, 8235] avg=8192.0000 max=9638 max_rank=2 min=5586 min_rank=3 imbalance_ratio=1.176514 max_load_reduction=55898 max_load_reduction_pct=0.852936 moved_to_echo=33863 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6220, 0, 228, 0, 0, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 7622, 0, 0, 11, 0, 0, 1958, 563, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8153, 8192, 6931, 10170, 8217, 8194, 7581, 8098] avg=8192.0000 max=10170 max_rank=3 min=6931 min_rank=2 imbalance_ratio=1.241455 max_load_reduction=55366 max_load_reduction_pct=0.844818 moved_to_echo=32090 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6213, 0, 677, 0, 0, 820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 6727, 0, 0, 52, 0, 0, 1950, 560, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 7, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8155, 8193, 7712, 9321, 8230, 8186, 7584, 8155] avg=8192.0000 max=9321 max_rank=3 min=7584 min_rank=6 imbalance_ratio=1.137817 max_load_reduction=56215 max_load_reduction_pct=0.857773 moved_to_echo=32155 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 41984, 0, 0, 128, 0, 128, 17536, 5888, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 45440, 0, 640, 0, 0, 19584, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[6272, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 59392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 65280, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 128, 0, 0, 0, 0, 0, 128, 45824, 0, 0, 128, 0, 0, 15104, 4608, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 47872, 0, 5760, 0, 0, 12160, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[4864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 384, 128, 0, 65280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 65536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 65536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 65280, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 384, 0, 0, 128, 128, 0, 0, 0, 0, 128, 65280, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 384, 128, 0, 65280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8185, 1, 0, 0, 0, 18, 0, 0, 8187, 0, 8191, 0, 0, 0, 0, 0, 8189, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 5, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 7, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8, 0, 0, 8186, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 6, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8150, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8186, 2, 0, 0, 0, 6, 0, 0, 8173, 0, 8191, 0, 0, 0, 0, 0, 8188, 0, 0, 6, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 23, 0, 0, 19, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8148, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 16, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 507, 2, 1, 8190, 0, 8186, 0, 0, 0, 2, 0, 8012, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7681, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 2, 2, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 178] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 332, 1, 0, 8191, 0, 8190, 0, 0, 0, 1, 0, 7956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8035, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 287, 1, 14, 8191, 0, 8185, 0, 0, 0, 1, 0, 8062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7915, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 108] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8185, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 5, 0, 0, 0, 1, 57, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 7, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 2, 8132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8129, 8304, 8192, 8194, 8190, 8192, 8148] avg=8192.0000 max=8304 max_rank=2 min=8129 min_rank=1 imbalance_ratio=1.013672 max_load_reduction=57232 max_load_reduction_pct=0.873291 moved_to_echo=32724 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8186, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 23, 0, 0, 19, 6, 57, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8148, 0, 0, 19, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 16, 0, 0, 0, 0, 0, 8118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8116, 8356, 8187, 8193, 8182, 8172, 8139] avg=8192.0000 max=8356 max_rank=2 min=8116 min_rank=1 imbalance_ratio=1.020020 max_load_reduction=57180 max_load_reduction_pct=0.872498 moved_to_echo=32686 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 6, 0, 0, 4, 0, 57, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8150, 0, 0, 6, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 8131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8122, 8304, 8192, 8194, 8171, 8188, 8173] avg=8192.0000 max=8304 max_rank=2 min=8122 min_rank=1 imbalance_ratio=1.013672 max_load_reduction=57232 max_load_reduction_pct=0.873291 moved_to_echo=32726 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 332, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8035, 0, 1, 0, 0, 0, 0, 0, 56, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 8137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8214, 8144, 8194, 8192, 8206, 8198, 8196] avg=8192.0000 max=8214 max_rank=1 min=8144 min_rank=2 imbalance_ratio=1.002686 max_load_reduction=57322 max_load_reduction_pct=0.874664 moved_to_echo=32792 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 2, 1, 0, 0, 0, 0, 0, 0, 2, 0, 7935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7681, 0, 2, 0, 0, 0, 0, 0, 56, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 2, 2, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 2, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 178, 8135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8447, 7794, 8197, 8188, 8207, 8198, 8315] avg=8192.0000 max=8447 max_rank=1 min=7794 min_rank=2 imbalance_ratio=1.031128 max_load_reduction=57089 max_load_reduction_pct=0.871109 moved_to_echo=32908 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 57, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8115, 8300, 8192, 8194, 8207, 8198, 8138] avg=8192.0000 max=8300 max_rank=2 min=8115 min_rank=1 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32737 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 287, 1, 14, 0, 0, 0, 0, 0, 0, 1, 0, 7984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7915, 0, 1, 0, 0, 0, 0, 0, 57, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 108, 8136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8287, 8026, 8194, 8187, 8207, 8198, 8245] avg=8192.0000 max=8287 max_rank=1 min=8026 min_rank=2 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32837 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 63, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 0, 0, 3, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 8137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8106, 8308, 8188, 8194, 8166, 8192, 8182] avg=8192.0000 max=8308 max_rank=2 min=8106 min_rank=1 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32734 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1, 0, 0, 8186, 0, 8192, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 6, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8147, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8186, 2, 0, 0, 0, 12, 0, 0, 8182, 0, 8191, 0, 0, 0, 0, 0, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 6, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 11] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 416, 2, 1, 8190, 0, 8186, 0, 0, 0, 2, 0, 7977, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7653, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 1, 2, 1, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 332] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 321, 1, 0, 8191, 0, 8190, 0, 0, 0, 1, 0, 7975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7942, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 5, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 268, 1, 5, 8191, 0, 8185, 0, 0, 0, 1, 0, 7997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7896, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 221] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8188, 2, 0, 0, 0, 10, 0, 0, 8172, 0, 8191, 0, 0, 0, 0, 0, 8188, 0, 0, 6, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 21, 0, 0, 21, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 22, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8186, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 6, 0, 0, 0, 1, 99, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 8, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 11, 8081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8152, 8320, 8191, 8193, 8192, 8192, 8107] avg=8192.0000 max=8320 max_rank=2 min=8107 min_rank=7 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32684 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 6, 0, 0, 8, 0, 100, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8147, 0, 0, 6, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 1, 8084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8141, 8332, 8192, 8194, 8170, 8186, 8129] avg=8192.0000 max=8332 max_rank=2 min=8129 min_rank=7 imbalance_ratio=1.017090 max_load_reduction=57204 max_load_reduction_pct=0.872864 moved_to_echo=32679 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 99, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8141, 8318, 8192, 8194, 8209, 8199, 8091] avg=8192.0000 max=8318 max_rank=2 min=8091 min_rank=7 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32693 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 416, 2, 1, 0, 0, 0, 0, 0, 0, 2, 0, 7928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7653, 0, 2, 0, 0, 0, 0, 0, 99, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 1, 2, 1, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 2, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 332, 8088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8349, 7782, 8197, 8189, 8208, 8199, 8422] avg=8192.0000 max=8422 max_rank=7 min=7782 min_rank=2 imbalance_ratio=1.028076 max_load_reduction=57114 max_load_reduction_pct=0.871490 moved_to_echo=33018 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 321, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7942, 0, 1, 0, 0, 0, 0, 0, 99, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 8090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8249, 8068, 8194, 8192, 8208, 8199, 8234] avg=8192.0000 max=8249 max_rank=1 min=8068 min_rank=2 imbalance_ratio=1.006958 max_load_reduction=57287 max_load_reduction_pct=0.874130 moved_to_echo=32833 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 268, 1, 5, 0, 0, 0, 0, 0, 0, 1, 0, 7947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7896, 0, 1, 0, 0, 0, 0, 0, 100, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 221, 8089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8222, 8023, 8194, 8187, 8209, 8199, 8311] avg=8192.0000 max=8311 max_rank=7 min=8023 min_rank=2 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32906 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 100, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 5, 8090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8134, 8321, 8189, 8194, 8161, 8193, 8147] avg=8192.0000 max=8321 max_rank=2 min=8134 min_rank=1 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32695 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8188, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 21, 0, 0, 21, 6, 100, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 18, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 22, 0, 0, 0, 0, 0, 8070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8148, 8372, 8187, 8193, 8179, 8169, 8095] avg=8192.0000 max=8372 max_rank=2 min=8095 min_rank=7 imbalance_ratio=1.021973 max_load_reduction=57164 max_load_reduction_pct=0.872253 moved_to_echo=32636 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 1280, 128, 128, 0, 0, 0, 0, 0, 0, 128, 0, 64384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 65536, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 128, 0, 0, 0, 384, 65152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 128, 64640, 0, 128, 128, 0, 0, 128, 128, 512, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 65408, 0, 0, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 128, 64512, 0, 128, 128, 0, 0, 128, 128, 896, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 65408, 0, 0, 128, 128, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 1152, 128, 128, 0, 0, 0, 0, 0, 0, 128, 0, 64512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 128, 128, 128, 0, 0, 65536, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 65536, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 0, 128, 128, 128, 128, 0, 65536, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 256, 128, 0, 0, 0, 768, 64768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7955, 0, 0, 8192, 8182, 0, 0, 1, 0, 0, 0, 0, 0, 0, 241, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 8191, 0, 0, 8160, 6290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 1901, 0, 0, 8172, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 27, 0, 8191, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7719, 0, 0, 8192, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7959, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7530, 0, 0, 8192, 8066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8187, 7, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 8189, 0, 0, 8076, 5878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 3, 0, 0, 2312, 0, 0, 8148, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 8189, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 46, 0, 8189, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7823, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 8186, 0, 0, 8159, 6301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 1890, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 20, 0, 8191, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 6290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 7961, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1901, 0, 0, 7418, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7412, 754, 12, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 27, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8444, 6808, 8193, 9322, 8180, 8191, 8190, 8208] avg=8192.0000 max=9322 max_rank=3 min=6808 min_rank=1 imbalance_ratio=1.137939 max_load_reduction=56214 max_load_reduction_pct=0.857758 moved_to_echo=32769 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 8182, 0, 0, 1, 0, 0, 0, 0, 0, 0, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7960, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7441, 756, 12, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8944, 8192, 7435, 8211, 8190, 8193, 8186] avg=8192.0000 max=8944 max_rank=1 min=7435 min_rank=3 imbalance_ratio=1.091797 max_load_reduction=56592 max_load_reduction_pct=0.863525 moved_to_echo=32780 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 8066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7955, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 7441, 756, 12, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 7, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7760, 9368, 8186, 7436, 8215, 8198, 8192, 8181] avg=8192.0000 max=9368 max_rank=1 min=7436 min_rank=3 imbalance_ratio=1.143555 max_load_reduction=56168 max_load_reduction_pct=0.857056 moved_to_echo=32786 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7959, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7441, 756, 12, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8945, 8190, 7436, 8210, 8194, 8192, 8180] avg=8192.0000 max=8945 max_rank=1 min=7436 min_rank=3 imbalance_ratio=1.091919 max_load_reduction=56591 max_load_reduction_pct=0.863510 moved_to_echo=32776 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7962, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7441, 756, 12, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7949, 9185, 8193, 7436, 8210, 8191, 8192, 8180] avg=8192.0000 max=9185 max_rank=1 min=7436 min_rank=3 imbalance_ratio=1.121216 max_load_reduction=56351 max_load_reduction_pct=0.859848 moved_to_echo=32773 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 5878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 7960, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 2312, 0, 0, 7396, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7335, 752, 12, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 46, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8529, 6391, 8192, 9715, 8102, 8192, 8189, 8226] avg=8192.0000 max=9715 max_rank=3 min=6391 min_rank=1 imbalance_ratio=1.185913 max_load_reduction=55821 max_load_reduction_pct=0.851761 moved_to_echo=32709 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 6301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 7961, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1890, 0, 0, 7433, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7411, 756, 12, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 20, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8430, 6819, 8193, 9326, 8181, 8191, 8194, 8202] avg=8192.0000 max=9326 max_rank=3 min=6819 min_rank=1 imbalance_ratio=1.138428 max_load_reduction=56210 max_load_reduction_pct=0.857697 moved_to_echo=32768 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7823, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7963, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7443, 762, 19, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8050, 9076, 8197, 7430, 8227, 8189, 8194, 8173] avg=8192.0000 max=9076 max_rank=1 min=7430 min_rank=3 imbalance_ratio=1.107910 max_load_reduction=56460 max_load_reduction_pct=0.861511 moved_to_echo=32783 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7696, 0, 0, 8192, 8188, 0, 0, 1, 0, 0, 0, 0, 0, 0, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7523, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 8189, 0, 0, 8066, 6394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 3, 0, 0, 1800, 0, 0, 8161, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 8189, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 39, 0, 8189, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 8191, 0, 0, 8149, 6646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 1545, 0, 0, 8174, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 32, 0, 8191, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6561, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 8185, 0, 0, 8149, 6568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 1625, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 24, 0, 8191, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7263, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6616, 0, 0, 8192, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8186, 7, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 1, 0, 0, 0, 0, 0, 0, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7518, 616, 44, 12, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8342, 8641, 8191, 7575, 8217, 8194, 8193, 8183] avg=8192.0000 max=8641 max_rank=1 min=7575 min_rank=3 imbalance_ratio=1.054810 max_load_reduction=56895 max_load_reduction_pct=0.868149 moved_to_echo=32787 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7518, 616, 44, 12, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8169, 8817, 8191, 7576, 8217, 8194, 8192, 8180] avg=8192.0000 max=8817 max_rank=1 min=7576 min_rank=3 imbalance_ratio=1.076294 max_load_reduction=56719 max_load_reduction_pct=0.865463 moved_to_echo=32783 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 7519, 614, 34, 12, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1800, 0, 0, 7547, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8065, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 39, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8944, 6360, 8184, 9354, 8093, 8192, 8190, 8219] avg=8192.0000 max=9354 max_rank=3 min=6360 min_rank=1 imbalance_ratio=1.141846 max_load_reduction=56182 max_load_reduction_pct=0.857269 moved_to_echo=32694 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7521, 616, 44, 12, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7206, 9779, 8194, 7576, 8217, 8191, 8193, 8180] avg=8192.0000 max=9779 max_rank=1 min=7206 min_rank=0 imbalance_ratio=1.193726 max_load_reduction=55757 max_load_reduction_pct=0.850784 moved_to_echo=32781 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 7520, 615, 35, 12, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1545, 0, 0, 7559, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8148, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 32, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8863, 6611, 8185, 9107, 8175, 8191, 8191, 8213] avg=8192.0000 max=9107 max_rank=3 min=6611 min_rank=1 imbalance_ratio=1.111694 max_load_reduction=56429 max_load_reduction_pct=0.861038 moved_to_echo=32770 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 638, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7525, 621, 47, 17, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7901, 9074, 8217, 7571, 8221, 8185, 8192, 8175] avg=8192.0000 max=9074 max_rank=1 min=7571 min_rank=3 imbalance_ratio=1.107666 max_load_reduction=56462 max_load_reduction_pct=0.861542 moved_to_echo=32773 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 7520, 616, 35, 12, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1625, 0, 0, 7573, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 24, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8850, 6533, 8186, 9201, 8176, 8191, 8193, 8206] avg=8192.0000 max=9201 max_rank=3 min=6533 min_rank=1 imbalance_ratio=1.123169 max_load_reduction=56335 max_load_reduction_pct=0.859604 moved_to_echo=32766 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7515, 616, 44, 12, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8192, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 7, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7261, 9721, 8188, 7576, 8220, 8198, 8192, 8180] avg=8192.0000 max=9721 max_rank=1 min=7261 min_rank=0 imbalance_ratio=1.186646 max_load_reduction=55815 max_load_reduction_pct=0.851669 moved_to_echo=32790 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 4224, 59392, 0, 0, 128, 0, 0, 0, 0, 0, 0, 2176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 6144, 0, 0, 59520, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 59392, 6144, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 63616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 63744, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 60160, 4992, 384, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 60288, 0, 0, 128, 0, 0, 0, 0, 0, 0, 5376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 5248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 60288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 4992, 0, 0, 60672, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 65408, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 0, 128, 0, 0, 0, 128, 0, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 65536, 128, 0, 0, 128, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 8051, 8051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 329, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 6828, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 7141, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 199, 1, 6, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5, 0, 0, 0, 1, 0, 1, 1, 7585, 0, 2757, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 7979, 8077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 313, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 5699, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 8029, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 0, 5, 0, 0, 0, 0, 0, 0, 198, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 2, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 3, 0, 8190, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 6519, 0, 3906, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 8191, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8135, 8103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 8147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 6722, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7577, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 0, 4, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 7185, 0, 2949, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 660, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8051, 275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 7181, 930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7776, 329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7249, 0, 329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8873, 8326, 8203, 8148, 8105, 8199, 8104, 7578] avg=8192.0000 max=8873 max_rank=0 min=7578 min_rank=7 imbalance_ratio=1.083130 max_load_reduction=56663 max_load_reduction_pct=0.864609 moved_to_echo=31986 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 6828, 0, 0, 0, 0, 0, 0, 661, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 7141, 280, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 199, 1, 6, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 7200, 861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7911, 330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 6712, 0, 2757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7494, 7432, 8190, 8307, 8243, 8189, 8209, 9472] avg=8192.0000 max=9472 max_rank=7 min=7432 min_rank=1 imbalance_ratio=1.156250 max_load_reduction=56064 max_load_reduction_pct=0.855469 moved_to_echo=34113 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 661, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7979, 276, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 1, 7200, 930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7801, 330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7248, 0, 313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8855, 8255, 8192, 8146, 8132, 8193, 8202, 7561] avg=8192.0000 max=8855 max_rank=0 min=7561 min_rank=7 imbalance_ratio=1.080933 max_load_reduction=56681 max_load_reduction_pct=0.864883 moved_to_echo=32088 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7201, 930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7878, 330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7248, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8853, 8469, 8192, 8131, 8208, 8192, 8205, 7286] avg=8192.0000 max=8853 max_rank=0 min=7286 min_rank=7 imbalance_ratio=1.080688 max_load_reduction=56683 max_load_reduction_pct=0.864914 moved_to_echo=31891 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8191, 0, 0, 0, 0, 0, 0, 661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8135, 277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 7200, 930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7826, 330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7249, 0, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8852, 8412, 8194, 8154, 8156, 8193, 8199, 7376] avg=8192.0000 max=8852 max_rank=0 min=7376 min_rank=7 imbalance_ratio=1.080566 max_load_reduction=56684 max_load_reduction_pct=0.864929 moved_to_echo=31924 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 5699, 0, 0, 0, 0, 0, 0, 661, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 8029, 280, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 0, 5, 0, 0, 0, 0, 0, 0, 198, 0, 0, 0, 0, 0, 7198, 740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7910, 330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 2, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 5769, 0, 3906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6373, 8326, 8188, 8333, 8244, 8191, 8202, 9679] avg=8192.0000 max=9679 max_rank=7 min=6373 min_rank=0 imbalance_ratio=1.181519 max_load_reduction=55857 max_load_reduction_pct=0.852310 moved_to_echo=34316 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 6722, 0, 0, 0, 0, 0, 0, 661, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7577, 280, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 0, 4, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 7200, 816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7911, 330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 6358, 0, 2949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7387, 7859, 8190, 8154, 8243, 8191, 8203, 9309] avg=8192.0000 max=9309 max_rank=7 min=7387 min_rank=0 imbalance_ratio=1.136353 max_load_reduction=56227 max_load_reduction_pct=0.857956 moved_to_echo=33946 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 7203, 934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7873, 332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7244, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8849, 8457, 8187, 8163, 8205, 8188, 8212, 7275] avg=8192.0000 max=8849 max_rank=0 min=7275 min_rank=7 imbalance_ratio=1.080200 max_load_reduction=56687 max_load_reduction_pct=0.864975 moved_to_echo=31880 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 7470, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 3828, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 8051, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 241, 1, 6, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 1, 1, 7903, 0, 4457, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 2211, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 8176, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 222, 0, 5, 0, 0, 0, 0, 0, 0, 266, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 2, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 3, 0, 8190, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 7628, 0, 6040, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 7002, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8169, 8050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1338, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 7557, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8189, 8105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 7446, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 8176, 8019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 968, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 7383, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 833, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 3818, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8143, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 0, 4, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 1, 7739, 0, 4692, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 7470, 0, 0, 0, 0, 0, 0, 2349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 8146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5755, 2348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5834, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9819, 8205, 8191, 8103, 8221, 8192, 8203, 6602] avg=8192.0000 max=9819 max_rank=0 min=6602 min_rank=7 imbalance_ratio=1.198608 max_load_reduction=55717 max_load_reduction_pct=0.850174 moved_to_echo=31218 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 7002, 0, 0, 0, 0, 0, 0, 2349, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 8050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 1, 5754, 2348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8110, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5834, 0, 1338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9353, 8109, 8191, 8117, 8199, 8193, 8202, 7172] avg=8192.0000 max=9353 max_rank=0 min=7172 min_rank=7 imbalance_ratio=1.141724 max_load_reduction=56183 max_load_reduction_pct=0.857285 moved_to_echo=31766 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3828, 0, 0, 0, 0, 0, 0, 2349, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 58, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 241, 1, 6, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 0, 5754, 2265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7993, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4, 0, 0, 0, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 5628, 0, 4457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6182, 8260, 8189, 8337, 8083, 8190, 8207, 10088] avg=8192.0000 max=10088 max_rank=7 min=6182 min_rank=0 imbalance_ratio=1.231445 max_load_reduction=55448 max_load_reduction_pct=0.846069 moved_to_echo=34568 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 7557, 0, 0, 0, 0, 0, 0, 2349, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 8105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 5754, 2348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8130, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5834, 0, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9907, 8164, 8193, 8133, 8218, 8192, 8197, 6532] avg=8192.0000 max=9907 max_rank=0 min=6532 min_rank=7 imbalance_ratio=1.209351 max_load_reduction=55629 max_load_reduction_pct=0.848831 moved_to_echo=31139 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2211, 0, 0, 0, 0, 0, 0, 2348, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 59, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 222, 0, 5, 0, 0, 0, 0, 0, 0, 266, 0, 0, 0, 0, 0, 5753, 2186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 2, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 5433, 0, 6040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4572, 8267, 8187, 8432, 8209, 8190, 8202, 11477] avg=8192.0000 max=11477 max_rank=7 min=4572 min_rank=0 imbalance_ratio=1.401001 max_load_reduction=54059 max_load_reduction_pct=0.824875 moved_to_echo=36078 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 7383, 0, 0, 0, 0, 0, 0, 2343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 8147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 5760, 2353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8138, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5825, 0, 833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9726, 8201, 8190, 8139, 8227, 8190, 8205, 6658] avg=8192.0000 max=9726 max_rank=0 min=6658 min_rank=7 imbalance_ratio=1.187256 max_load_reduction=55810 max_load_reduction_pct=0.851593 moved_to_echo=31280 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3818, 0, 0, 0, 0, 0, 0, 2349, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 58, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 0, 4, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 5754, 2218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8085, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 5511, 0, 4692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6171, 8252, 8189, 8148, 8175, 8191, 8205, 10205] avg=8192.0000 max=10205 max_rank=7 min=6171 min_rank=0 imbalance_ratio=1.245728 max_load_reduction=55331 max_load_reduction_pct=0.844284 moved_to_echo=34776 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 7446, 0, 0, 0, 0, 0, 0, 2344, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 8019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 1, 5739, 2348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5834, 0, 968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9806, 8078, 8206, 8127, 8204, 8198, 8115, 6802] avg=8192.0000 max=9806 max_rank=0 min=6802 min_rank=7 imbalance_ratio=1.197021 max_load_reduction=55730 max_load_reduction_pct=0.850372 moved_to_echo=31319 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 512, 128, 128, 0, 0, 0, 0, 0, 128, 256, 0, 0, 0, 0, 128, 57600, 7168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 60288, 0, 0, 0, 0, 0, 0, 5376, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 55168, 0, 10496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62976, 2688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 63360, 2304, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 640, 128, 128, 0, 0, 0, 0, 0, 128, 384, 0, 0, 0, 0, 128, 46080, 18432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 46720, 0, 0, 0, 0, 0, 0, 18816, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 45824, 0, 19840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 65536, 128, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 512, 65152, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64896, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[65536, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8186, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8189, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8189, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 2, 1, 0, 8191, 0, 8186, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 1, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 1, 8189, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 7, 0, 0, 0, 1, 1, 0, 0, 0, 0, 8162, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8181, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8191, 0, 8192, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8150, 8190, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8189, 3, 1, 0, 8191, 0, 8187, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 1, 8190, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 8168, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8180, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8191, 0, 8192, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8133, 8185, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8183, 7, 2, 0, 8190, 0, 8178, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8131, 0, 0, 1, 0, 0, 2, 0, 1, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, 0, 3, 2, 8189, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 8072, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8194, 8203, 8190, 8193, 8192, 8152, 8211] avg=8192.0000 max=8211 max_rank=7 min=8152 min_rank=6 imbalance_ratio=1.002319 max_load_reduction=57325 max_load_reduction_pct=0.874710 moved_to_echo=32748 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8200, 8203, 8190, 8185, 8191, 8160, 8211] avg=8192.0000 max=8211 max_rank=7 min=8160 min_rank=6 imbalance_ratio=1.002319 max_load_reduction=57325 max_load_reduction_pct=0.874710 moved_to_echo=32747 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 8185, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 1, 0, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 8169, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 1, 8155, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 1, 1, 0, 0, 0, 0, 8162, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8189, 8180, 8200, 8188, 8193, 8192, 8190] avg=8192.0000 max=8204 max_rank=0 min=8180 min_rank=2 imbalance_ratio=1.001465 max_load_reduction=57332 max_load_reduction_pct=0.874817 moved_to_echo=32763 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8191, 8203, 8189, 8184, 8193, 8158, 8211] avg=8192.0000 max=8211 max_rank=7 min=8158 min_rank=6 imbalance_ratio=1.002319 max_load_reduction=57325 max_load_reduction_pct=0.874710 moved_to_echo=32746 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8191, 8203, 8189, 8185, 8192, 8158, 8211] avg=8192.0000 max=8211 max_rank=7 min=8158 min_rank=6 imbalance_ratio=1.002319 max_load_reduction=57325 max_load_reduction_pct=0.874710 moved_to_echo=32746 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8150, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 8186, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 1, 0, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 1, 8156, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 8168, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8166, 8192, 8190, 8192, 8191, 8192, 8224, 8189] avg=8192.0000 max=8224 max_rank=6 min=8166 min_rank=0 imbalance_ratio=1.003906 max_load_reduction=57312 max_load_reduction_pct=0.874512 moved_to_echo=32796 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8133, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2, 0, 0, 0, 8177, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8131, 0, 0, 1, 0, 0, 2, 0, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 8168, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, 0, 3, 2, 8155, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 8072, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8149, 8190, 8145, 8196, 8224, 8191, 8346, 8095] avg=8192.0000 max=8346 max_rank=6 min=8095 min_rank=7 imbalance_ratio=1.018799 max_load_reduction=57190 max_load_reduction_pct=0.872650 moved_to_echo=32856 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 14, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8189, 8209, 8190, 8186, 8192, 8146, 8218] avg=8192.0000 max=8218 max_rank=7 min=8146 min_rank=6 imbalance_ratio=1.003174 max_load_reduction=57318 max_load_reduction_pct=0.874603 moved_to_echo=32742 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8126, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 2, 1, 0, 8191, 0, 8189, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 1, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 1, 8189, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 7, 0, 0, 0, 1, 1, 0, 0, 0, 0, 8165, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7873, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 4, 1, 0, 8191, 0, 8187, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 331, 0, 0, 1, 8190, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 8170, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7966, 8185, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8183, 7, 2, 0, 8190, 0, 8178, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8111, 0, 0, 1, 0, 0, 2, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 287, 0, 2, 2, 8189, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 8102, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 12, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8112, 67, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8260, 8200, 8205, 8189, 8184, 8191, 8100, 8207] avg=8192.0000 max=8260 max_rank=0 min=8100 min_rank=6 imbalance_ratio=1.008301 max_load_reduction=57276 max_load_reduction_pct=0.873962 moved_to_echo=32682 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 12, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8112, 67, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8272, 8191, 8205, 8189, 8181, 8193, 8098, 8207] avg=8192.0000 max=8272 max_rank=0 min=8098 min_rank=6 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32679 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 12, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8112, 67, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8272, 8191, 8205, 8190, 8181, 8192, 8098, 8207] avg=8192.0000 max=8272 max_rank=0 min=8098 min_rank=6 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32678 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 18, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 73, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8267, 8184, 8218, 8190, 8192, 8192, 8082, 8211] avg=8192.0000 max=8267 max_rank=0 min=8082 min_rank=6 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32677 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 8188, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 1, 0, 12, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 8105, 67, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 1, 8095, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 1, 1, 0, 0, 0, 0, 8165, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8192, 8180, 8199, 8187, 8193, 8191, 8189] avg=8192.0000 max=8205 max_rank=0 min=8180 min_rank=2 imbalance_ratio=1.001587 max_load_reduction=57331 max_load_reduction_pct=0.874802 moved_to_echo=32760 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7966, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2, 0, 0, 0, 8177, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8111, 0, 0, 1, 0, 0, 2, 0, 12, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 8104, 67, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 287, 0, 2, 2, 8095, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 8102, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8046, 8190, 8127, 8197, 8228, 8191, 8436, 8121] avg=8192.0000 max=8436 max_rank=6 min=8046 min_rank=0 imbalance_ratio=1.029785 max_load_reduction=57100 max_load_reduction_pct=0.871277 moved_to_echo=32976 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 8186, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 1, 0, 12, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8110, 67, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 331, 0, 0, 1, 8096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 8170, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7952, 8193, 8191, 8192, 8190, 8192, 8439, 8187] avg=8192.0000 max=8439 max_rank=6 min=7952 min_rank=0 imbalance_ratio=1.030151 max_load_reduction=57097 max_load_reduction_pct=0.871231 moved_to_echo=33008 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8182, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8190, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 12, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8112, 67, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8262, 8195, 8205, 8190, 8193, 8192, 8092, 8207] avg=8192.0000 max=8262 max_rank=0 min=8092 min_rank=6 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32684 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 65408, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 65536, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 65024, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 64896, 640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 0, 0, 65536, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 65408, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 128, 0, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[256, 0, 128, 128, 65280, 0, 0, 0, 128, 0, 0, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 65408, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[768, 0, 128, 128, 64768, 0, 0, 0, 128, 0, 0, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 45, 2, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8179, 0, 8192, 0, 0, 6, 0, 0, 0, 8153, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 2, 0, 0, 8, 0, 5370, 0, 0, 0, 0, 0, 0, 8096, 0, 0, 0, 0, 2718, 0, 8187, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8190, 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 8182, 0, 0, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 98, 0, 0, 0, 0, 9, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 98, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8099, 0, 8191, 0, 0, 1, 0, 0, 0, 8181, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8126, 0, 8192, 0, 0, 1, 0, 0, 0, 8184, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8167, 0, 8192, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8191, 3, 0, 0, 0, 0, 0, 0, 0, 4474, 0, 0, 0, 0, 0, 0, 8088, 0, 0, 0, 0, 3547, 0, 8190, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8191, 0, 0, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 8186, 0, 0, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 133, 0, 0, 0, 0, 15, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8189, 0, 0, 0, 13, 0, 0, 9, 0, 2154, 0, 0, 0, 0, 0, 0, 8016, 0, 0, 0, 0, 5762, 0, 8186, 0, 0, 0, 1, 1, 0, 8189, 0, 0, 0, 0, 8184, 0, 0, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 8181, 0, 0, 8189, 0, 0, 3, 1, 0, 0, 0, 0, 144, 0, 0, 0, 0, 29, 0, 0, 2, 0, 0, 0, 0, 2, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8148, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 2, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 8100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8126, 53, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8195, 8288, 8106, 8195, 8192, 8182, 8193] avg=8192.0000 max=8288 max_rank=2 min=8106 min_rank=3 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32762 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 1, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 8099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8126, 53, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8190, 8260, 8129, 8192, 8195, 8182, 8192] avg=8192.0000 max=8260 max_rank=2 min=8129 min_rank=3 imbalance_ratio=1.008301 max_load_reduction=57276 max_load_reduction_pct=0.873962 moved_to_echo=32761 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 5370, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 2718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 0, 0, 0, 0, 0, 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 8186, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8182, 0, 0, 4, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 0, 0, 9, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 8031, 54, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8190, 8157, 8231, 8189, 8188, 8198, 8192] avg=8192.0000 max=8231 max_rank=3 min=8157 min_rank=2 imbalance_ratio=1.004761 max_load_reduction=57305 max_load_reduction_pct=0.874405 moved_to_echo=32767 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 8126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8126, 54, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8192, 8260, 8131, 8192, 8195, 8183, 8193] avg=8192.0000 max=8260 max_rank=2 min=8131 min_rank=3 imbalance_ratio=1.008301 max_load_reduction=57276 max_load_reduction_pct=0.873962 moved_to_echo=32763 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8186, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 2154, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 5762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 8135, 0, 0, 0, 0, 0, 0, 0, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8185, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8181, 0, 0, 3, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 0, 0, 0, 29, 0, 0, 2, 0, 0, 0, 0, 2, 0, 2, 7951, 54, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8184, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8202, 7986, 8404, 8188, 8190, 8187, 8188] avg=8192.0000 max=8404 max_rank=3 min=7986 min_rank=2 imbalance_ratio=1.025879 max_load_reduction=57132 max_load_reduction_pct=0.871765 moved_to_echo=32753 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8188, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4474, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 3547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 0, 0, 0, 0, 0, 0, 0, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8189, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8186, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 133, 0, 0, 0, 0, 15, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 8023, 54, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8192, 8082, 8264, 8195, 8188, 8231, 8193] avg=8192.0000 max=8264 max_rank=3 min=8082 min_rank=2 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32807 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 8148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8126, 54, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8189, 8254, 8138, 8192, 8196, 8183, 8193] avg=8192.0000 max=8254 max_rank=2 min=8138 min_rank=3 imbalance_ratio=1.007568 max_load_reduction=57282 max_load_reduction_pct=0.874054 moved_to_echo=32764 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8128, 56, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8186, 8249, 8133, 8193, 8192, 8190, 8192] avg=8192.0000 max=8249 max_rank=2 min=8133 min_rank=3 imbalance_ratio=1.006958 max_load_reduction=57287 max_load_reduction_pct=0.874130 moved_to_echo=32767 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8178, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 24, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8168, 0, 8192, 0, 0, 1, 0, 0, 0, 8186, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8168, 0, 8192, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8191, 4, 0, 0, 0, 0, 0, 0, 0, 3372, 0, 0, 0, 0, 0, 0, 8081, 0, 0, 0, 0, 4632, 0, 8190, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8191, 0, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 8183, 0, 0, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 177, 0, 0, 0, 0, 12, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 364, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 7832, 0, 8192, 0, 0, 1, 0, 0, 0, 8185, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8189, 0, 0, 0, 1, 0, 0, 9, 0, 2048, 0, 0, 0, 0, 0, 0, 7994, 0, 0, 0, 0, 5845, 0, 8187, 0, 0, 0, 1, 1, 0, 8189, 0, 0, 0, 0, 8183, 0, 0, 296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 8181, 0, 0, 8189, 0, 0, 3, 1, 0, 0, 0, 0, 175, 0, 0, 0, 0, 27, 0, 0, 2, 0, 0, 0, 0, 2, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 39, 2, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8186, 0, 8192, 0, 0, 6, 0, 0, 0, 8151, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8, 0, 4951, 0, 0, 0, 0, 0, 0, 8085, 0, 0, 0, 0, 3116, 0, 8186, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8190, 0, 0, 99, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 8182, 0, 0, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 9, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 1, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 56, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8194, 8263, 8131, 8191, 8196, 8175, 8193] avg=8192.0000 max=8263 max_rank=2 min=8131 min_rank=3 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32755 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 56, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8190, 8262, 8136, 8192, 8195, 8175, 8194] avg=8192.0000 max=8262 max_rank=2 min=8136 min_rank=3 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32756 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8188, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 2048, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 0, 5845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 8133, 0, 0, 0, 0, 0, 0, 0, 296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8183, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8181, 0, 0, 4, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 175, 0, 0, 0, 0, 27, 0, 0, 2, 0, 0, 0, 0, 2, 0, 2, 7921, 56, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8191, 7970, 8431, 8186, 8191, 8186, 8190] avg=8192.0000 max=8431 max_rank=3 min=7970 min_rank=2 imbalance_ratio=1.029175 max_load_reduction=57105 max_load_reduction_pct=0.871353 moved_to_echo=32753 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 364, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 7832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 56, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8193, 8266, 8130, 8193, 8195, 8175, 8193] avg=8192.0000 max=8266 max_rank=2 min=8130 min_rank=3 imbalance_ratio=1.009033 max_load_reduction=57270 max_load_reduction_pct=0.873871 moved_to_echo=32756 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8119, 62, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8188, 8261, 8126, 8193, 8192, 8185, 8193] avg=8192.0000 max=8261 max_rank=2 min=8126 min_rank=3 imbalance_ratio=1.008423 max_load_reduction=57275 max_load_reduction_pct=0.873947 moved_to_echo=32763 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8189, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3372, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 4632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8135, 0, 0, 0, 0, 0, 0, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8183, 0, 0, 3, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 177, 0, 0, 0, 0, 12, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 8007, 56, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8194, 8073, 8247, 8193, 8188, 8257, 8192] avg=8192.0000 max=8257 max_rank=6 min=8073 min_rank=2 imbalance_ratio=1.007935 max_load_reduction=57279 max_load_reduction_pct=0.874008 moved_to_echo=32830 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4951, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 3116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8135, 0, 0, 0, 0, 0, 0, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 8189, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8182, 0, 0, 3, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 9, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 8011, 56, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8189, 8144, 8234, 8193, 8187, 8209, 8188] avg=8192.0000 max=8234 max_rank=3 min=8144 min_rank=2 imbalance_ratio=1.005127 max_load_reduction=57302 max_load_reduction_pct=0.874359 moved_to_echo=32777 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 2, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 8095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 56, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8197, 8297, 8101, 8195, 8192, 8174, 8193] avg=8192.0000 max=8297 max_rank=2 min=8101 min_rank=3 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32754 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[384, 0, 0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 128, 0, 128, 64640, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 12288, 128, 0, 0, 0, 0, 0, 512, 0, 0, 0, 0, 52864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 128, 128, 0, 65152, 0, 0, 0, 0, 0, 0, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 65536, 0, 0, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[512, 0, 0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 128, 0, 128, 64640, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 10880, 128, 0, 0, 0, 0, 0, 640, 0, 0, 0, 0, 54144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 128, 128, 0, 65024, 0, 0, 0, 0, 0, 0, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 128, 0, 65536, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8187, 0, 0, 46, 0, 0, 1, 0, 71, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 8156, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 0, 0, 0, 0, 0, 0, 8190, 7908, 0, 52, 66, 0, 0, 0, 1, 0, 1, 0, 0, 8189, 0, 1, 0, 1, 0, 0, 8145, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8186, 0, 0, 63, 0, 0, 1, 0, 72, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 8157, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 0, 0, 0, 0, 0, 0, 8190, 7943, 0, 21, 42, 0, 0, 0, 1, 0, 1, 0, 0, 8189, 0, 1, 0, 1, 0, 0, 8130, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8176, 0, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8177, 0, 1, 14, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8173, 0, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8186, 0, 0, 21, 0, 0, 2, 5, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 1, 2, 0, 0, 8165, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 0, 0, 0, 0, 0, 0, 8188, 7495, 0, 0, 477, 1, 0, 0, 2, 0, 2, 0, 0, 8188, 0, 2, 1, 0, 0, 0, 8093, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8167, 0, 11, 17, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8189, 0, 0, 105, 0, 0, 1, 1, 73, 0, 0, 0, 1, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 8190, 7742, 0, 0, 246, 0, 0, 0, 1, 0, 1, 0, 0, 8191, 0, 1, 0, 0, 0, 0, 8117, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8174, 0, 1, 17, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8161, 0, 15, 27, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8124, 0, 0, 46, 0, 0, 1, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 8041, 118, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 0, 0, 0, 0, 0, 0, 149, 7908, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 66, 0, 0, 0, 1, 0, 1, 0, 0, 8071, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8242, 8187, 8168, 8187, 8193, 8174, 8194, 8191] avg=8192.0000 max=8242 max_rank=0 min=8168 min_rank=2 imbalance_ratio=1.006104 max_load_reduction=57294 max_load_reduction_pct=0.874237 moved_to_echo=32752 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8043, 118, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8130, 8202, 8199, 8322, 8085, 8212, 8194, 8192] avg=8192.0000 max=8322 max_rank=3 min=8085 min_rank=4 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32683 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8043, 118, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 27, 0, 0, 0, 0, 0, 0, 0, 0, 8074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8131, 8185, 8199, 8310, 8116, 8220, 8183, 8192] avg=8192.0000 max=8310 max_rank=3 min=8116 min_rank=4 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32711 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8043, 118, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 17, 0, 0, 0, 0, 0, 0, 0, 0, 8074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8129, 8185, 8199, 8316, 8102, 8220, 8193, 8192] avg=8192.0000 max=8316 max_rank=3 min=8102 min_rank=4 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32707 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8123, 0, 0, 21, 0, 0, 2, 5, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 1, 2, 0, 0, 8039, 118, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 0, 0, 0, 0, 0, 0, 149, 7495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 1, 0, 0, 2, 0, 2, 0, 0, 8070, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8093, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8184, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8346, 8187, 8176, 7767, 8555, 8123, 8190, 8192] avg=8192.0000 max=8555 max_rank=4 min=7767 min_rank=3 imbalance_ratio=1.044312 max_load_reduction=56981 max_load_reduction_pct=0.869461 moved_to_echo=33060 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8043, 118, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8129, 8185, 8199, 8341, 8074, 8220, 8196, 8192] avg=8192.0000 max=8341 max_rank=3 min=8074 min_rank=4 imbalance_ratio=1.018188 max_load_reduction=57195 max_load_reduction_pct=0.872726 moved_to_echo=32682 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8126, 0, 0, 105, 0, 0, 1, 1, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 8041, 118, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 149, 7742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 0, 0, 0, 1, 0, 1, 0, 0, 8073, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8306, 8208, 8195, 7977, 8322, 8146, 8189, 8193] avg=8192.0000 max=8322 max_rank=4 min=7977 min_rank=3 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32850 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8050, 121, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 17, 0, 0, 0, 0, 0, 0, 0, 0, 8071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8123, 8197, 8201, 8316, 8089, 8221, 8197, 8192] avg=8192.0000 max=8316 max_rank=3 min=8089 min_rank=4 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32699 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8169, 0, 8, 18, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8157, 0, 14, 32, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8186, 0, 0, 22, 0, 0, 2, 7, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 1, 2, 0, 0, 8163, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 0, 0, 0, 0, 0, 0, 8188, 7633, 0, 0, 338, 1, 0, 0, 2, 0, 2, 0, 0, 8188, 0, 2, 1, 0, 0, 0, 8071, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8189, 0, 0, 112, 0, 0, 1, 1, 80, 0, 0, 0, 1, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 8190, 7866, 0, 0, 122, 0, 0, 0, 1, 0, 1, 0, 0, 8191, 0, 1, 0, 0, 0, 0, 8109, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8087, 78, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8127, 8202, 8199, 8281, 8123, 8219, 8193, 8192] avg=8192.0000 max=8281 max_rank=3 min=8123 min_rank=4 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32727 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8120, 0, 0, 63, 0, 0, 1, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 8085, 78, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 0, 0, 0, 0, 0, 0, 105, 7943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 42, 0, 0, 0, 1, 0, 1, 0, 0, 8111, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8130, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8256, 8189, 8170, 8194, 8178, 8164, 8194, 8191] avg=8192.0000 max=8256 max_rank=0 min=8164 min_rank=5 imbalance_ratio=1.007812 max_load_reduction=57280 max_load_reduction_pct=0.874023 moved_to_echo=32727 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8087, 78, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8126, 8187, 8199, 8297, 8114, 8225, 8196, 8192] avg=8192.0000 max=8297 max_rank=3 min=8114 min_rank=4 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32727 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8087, 78, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 18, 0, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8126, 8187, 8199, 8274, 8140, 8225, 8193, 8192] avg=8192.0000 max=8274 max_rank=3 min=8126 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32750 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8090, 80, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 14, 0, 0, 0, 0, 0, 0, 0, 0, 8112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8106, 8199, 8201, 8279, 8127, 8232, 8200, 8192] avg=8192.0000 max=8279 max_rank=3 min=8106 min_rank=0 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32751 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8120, 0, 0, 22, 0, 0, 2, 7, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 1, 2, 0, 0, 8083, 78, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 0, 0, 0, 0, 0, 0, 105, 7633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 338, 1, 0, 0, 2, 0, 2, 0, 0, 8110, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8071, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8183, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8350, 8189, 8174, 7880, 8456, 8106, 8189, 8192] avg=8192.0000 max=8456 max_rank=4 min=7880 min_rank=3 imbalance_ratio=1.032227 max_load_reduction=57080 max_load_reduction_pct=0.870972 moved_to_echo=32943 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8123, 0, 0, 112, 0, 0, 1, 1, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 8085, 78, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 105, 7866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 0, 0, 0, 1, 0, 1, 0, 0, 8113, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8109, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8317, 8196, 8192, 8069, 8238, 8143, 8188, 8193] avg=8192.0000 max=8317 max_rank=0 min=8069 min_rank=3 imbalance_ratio=1.015259 max_load_reduction=57219 max_load_reduction_pct=0.873093 moved_to_echo=32762 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8087, 78, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 32, 0, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8128, 8187, 8202, 8262, 8160, 8222, 8183, 8192] avg=8192.0000 max=8262 max_rank=3 min=8128 min_rank=0 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32757 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 128, 0, 0, 0, 0, 0, 0, 128, 0, 128, 128, 128, 0, 0, 64384, 1024, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 384, 0, 0, 0, 0, 0, 0, 1280, 64128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 65408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 896, 128, 0, 0, 128, 0, 128, 0, 0, 64640, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[65536, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 512, 0, 0, 0, 0, 0, 0, 896, 64384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 128, 0, 0, 0, 0, 0, 0, 128, 0, 128, 128, 128, 0, 0, 64768, 640, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 65024, 0, 0, 256, 0, 0, 128, 128, 384, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 65280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 640, 128, 0, 0, 128, 0, 128, 0, 0, 65024, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[65536, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 65536, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True - [2026-08-26 14:48:13] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 437.5 | throughput per GPU (TFLOP/s/GPU): 413.3 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.638084E+00 | loss scale: 1.0 | grad norm: 5.491 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 1777, 0, 0, 0, 3, 0, 0, 4, 0, 3, 1, 5, 0, 0, 0, 2820, 0, 0, 2688, 0, 0, 0, 8191, 0, 8188, 2, 0, 0, 1, 1, 2, 1, 0, 0, 0, 2, 0, 0, 1, 0, 1, 4, 1, 2, 1, 1, 2, 0, 0, 0, 0, 0, 2, 2, 8177, 0, 0, 0, 0, 1, 0, 0, 0, 2904, 0, 1, 0, 5322, 0, 5829, 9, 0, 0, 3316, 0, 0, 0, 0, 0, 0, 0, 0, 8029, 0, 0, 0, 1, 0, 0, 386, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7812, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 10, 1213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2802, 0, 5, 2794, 9, 1, 0, 8192, 0, 8190, 1, 0, 1, 0, 0, 0, 5, 2, 0, 0, 2, 1, 0, 3, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 3, 0, 0, 0, 0, 0, 0, 3965, 0, 0, 0, 4988, 0, 5496, 5, 0, 0, 3815, 0, 0, 0, 0, 0, 0, 1, 0, 7628, 0, 0, 0, 1, 0, 0, 439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7732, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 39, 0, 3, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 2, 2346, 0, 0, 0, 9, 0, 1, 0, 0, 0, 0, 0, 1, 1, 3, 2053, 0, 0, 3551, 8, 0, 0, 8186, 0, 8185, 0, 0, 1, 0, 1, 2, 0, 0, 0, 2, 2, 0, 1, 0, 1, 0, 5, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 8186, 0, 1, 0, 0, 0, 1, 0, 0, 3229, 0, 0, 0, 4689, 0, 5844, 8, 3, 3, 3020, 0, 1, 4, 0, 2, 0, 1, 0, 7852, 0, 0, 1, 0, 0, 0, 575, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7707, 1, 0, 3, 0, 3, 0, 1, 0, 1, 1, 8, 0, 0, 0, 0, 1, 24, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5, 1310, 0, 0, 0, 0, 0, 2, 0, 0, 6, 11, 1, 0, 0, 0, 3150, 0, 0, 3245, 3, 0, 0, 8190, 1, 8191, 5, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 2696, 1, 1, 0, 5078, 0, 6135, 0, 1, 0, 3385, 0, 0, 0, 0, 0, 1, 0, 0, 7880, 0, 2, 0, 0, 3, 0, 332, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7622, 0, 1, 11, 0, 1, 0, 1, 0, 0, 0, 2, 0, 0, 0, 1, 0, 40, 3, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1269, 1, 1, 3, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 2713, 1, 0, 2590, 0, 0, 0, 8189, 0, 8188, 2, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 3, 0, 0, 0, 0, 2977, 0, 1, 1, 5158, 0, 5953, 4, 0, 0, 4186, 0, 0, 0, 0, 0, 0, 0, 0, 8028, 0, 2, 1, 0, 0, 0, 178, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 7675, 0, 0, 12, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 187, 0, 3, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 1215, 0, 1, 1, 3, 0, 0, 0, 0, 0, 5, 0, 0, 1, 0, 2949, 1, 0, 3040, 0, 0, 0, 8190, 0, 8190, 4, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 3499, 0, 0, 0, 5153, 0, 5860, 12, 1, 0, 2964, 0, 0, 0, 0, 0, 0, 0, 0, 7872, 0, 3, 0, 0, 0, 0, 570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7712, 0, 0, 1, 0, 0, 0, 1, 0, 3, 0, 1, 0, 0, 0, 0, 0, 77, 0, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 1, 1195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 2694, 0, 0, 2212, 0, 0, 0, 8192, 0, 8190, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1, 8181, 0, 0, 0, 4, 0, 0, 0, 0, 2714, 0, 0, 0, 5746, 0, 6284, 3, 1, 1, 3877, 0, 0, 0, 0, 1, 1, 0, 0, 7971, 0, 1, 0, 0, 0, 0, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7590, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 166, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 9, 1048, 0, 0, 0, 9, 0, 0, 0, 0, 2, 5, 0, 0, 0, 0, 2554, 0, 0, 2599, 3, 0, 0, 8192, 0, 8192, 10, 0, 0, 0, 0, 0, 0, 1, 0, 1, 16, 0, 1, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 3, 0, 0, 0, 3491, 0, 0, 0, 5469, 0, 5970, 2, 0, 0, 3684, 0, 0, 0, 0, 0, 0, 0, 0, 7954, 0, 7, 0, 0, 2, 0, 270, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 7667, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 166, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 10, 1213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3405, 3214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2802, 0, 5, 2794, 9, 1, 0, 0, 0, 2627, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5, 2, 0, 0, 2, 1, 0, 3, 0, 0, 4, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 3, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3965, 0, 0, 0, 1410, 0, 0, 5, 0, 0, 3815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7515, 0, 0, 0, 1, 0, 0, 439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7732, 0, 0, 0, 0, 0, 364, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 39, 0, 3, 0, 0, 5496, 2158, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7843, 8239, 8195, 8188, 9195, 7956, 8208, 7712] avg=8192.0000 max=9195 max_rank=4 min=7712 min_rank=7 imbalance_ratio=1.122437 max_load_reduction=56341 max_load_reduction_pct=0.859695 moved_to_echo=33071 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 2, 2346, 0, 0, 0, 9, 0, 1, 0, 0, 0, 0, 3403, 3021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 3, 2053, 0, 0, 3551, 8, 0, 0, 0, 0, 2626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 2, 0, 0, 0, 2, 2, 0, 1, 0, 1, 0, 5, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 8186, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3229, 0, 0, 0, 1326, 0, 0, 8, 3, 3, 3020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 2, 0, 1, 0, 7736, 0, 0, 1, 0, 0, 0, 575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7707, 1, 0, 3, 0, 3, 342, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 8, 0, 0, 0, 0, 1, 24, 0, 0, 0, 0, 5844, 2156, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8783, 8243, 8186, 8192, 7590, 8320, 8171, 8051] avg=8192.0000 max=8783 max_rank=0 min=7590 min_rank=4 imbalance_ratio=1.072144 max_load_reduction=56753 max_load_reduction_pct=0.865982 moved_to_echo=32132 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 1, 1195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3405, 3703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2694, 0, 0, 2212, 0, 0, 0, 0, 0, 2627, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1, 8181, 0, 0, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2714, 0, 0, 0, 1623, 0, 0, 3, 1, 1, 3877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 7853, 0, 1, 0, 0, 0, 0, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7590, 0, 0, 2, 1, 0, 420, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 166, 1, 0, 0, 0, 6284, 2158, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8309, 7537, 8183, 8193, 8219, 8338, 8130, 8627] avg=8192.0000 max=8627 max_rank=7 min=7537 min_rank=1 imbalance_ratio=1.053101 max_load_reduction=56909 max_load_reduction_pct=0.868362 moved_to_echo=33314 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 1777, 0, 0, 0, 3, 0, 0, 4, 0, 3, 1, 3404, 3429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 2820, 0, 0, 2688, 0, 0, 0, 0, 0, 2627, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 1, 0, 0, 0, 2, 0, 0, 1, 0, 1, 4, 1, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 2, 0, 0, 0, 0, 0, 2, 2, 8177, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2904, 0, 1, 0, 1504, 0, 0, 9, 0, 0, 3316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7910, 0, 0, 0, 1, 0, 0, 386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7812, 0, 0, 0, 0, 0, 389, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 5829, 2157, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8623, 8142, 8189, 8189, 7735, 8297, 8319, 8042] avg=8192.0000 max=8623 max_rank=0 min=7735 min_rank=4 imbalance_ratio=1.052612 max_load_reduction=56913 max_load_reduction_pct=0.868423 moved_to_echo=32393 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1269, 1, 1, 3, 0, 0, 0, 1, 0, 0, 0, 3404, 3324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2713, 1, 0, 2590, 0, 0, 0, 0, 0, 2627, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 9, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2977, 0, 1, 1, 1457, 0, 0, 4, 0, 0, 4186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7909, 0, 2, 1, 0, 0, 0, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 7675, 0, 0, 12, 0, 0, 377, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 187, 0, 3, 0, 1, 5953, 2157, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8003, 7935, 8187, 8193, 8626, 8090, 8184, 8318] avg=8192.0000 max=8626 max_rank=4 min=7935 min_rank=1 imbalance_ratio=1.052979 max_load_reduction=56910 max_load_reduction_pct=0.868378 moved_to_echo=33218 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5, 1310, 0, 0, 0, 0, 0, 2, 0, 0, 6, 11, 3405, 3272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3150, 0, 0, 3245, 3, 0, 0, 0, 1, 2628, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 28, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8184, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2696, 1, 1, 0, 1435, 0, 0, 0, 1, 0, 3385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7763, 0, 2, 0, 0, 3, 0, 332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7622, 0, 1, 11, 0, 1, 371, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 1, 0, 40, 3, 0, 0, 0, 6135, 2158, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8011, 9033, 8206, 8189, 7519, 8101, 8122, 8355] avg=8192.0000 max=9033 max_rank=1 min=7519 min_rank=4 imbalance_ratio=1.102661 max_load_reduction=56503 max_load_reduction_pct=0.862167 moved_to_echo=32097 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 1215, 0, 1, 1, 3, 0, 0, 0, 0, 0, 5, 3410, 3324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2949, 1, 0, 3040, 0, 0, 0, 0, 0, 2618, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 2, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3499, 0, 0, 0, 1449, 0, 0, 12, 1, 0, 2964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7746, 0, 3, 0, 0, 0, 0, 570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7712, 0, 0, 1, 0, 0, 380, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 1, 0, 0, 0, 0, 0, 77, 0, 0, 0, 2, 5860, 2162, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7961, 8613, 8182, 8201, 7927, 8319, 8212, 8121] avg=8192.0000 max=8613 max_rank=1 min=7927 min_rank=4 imbalance_ratio=1.051392 max_load_reduction=56923 max_load_reduction_pct=0.868576 moved_to_echo=32579 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 9, 1048, 0, 0, 0, 9, 0, 0, 0, 0, 2, 5, 3406, 3524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2554, 0, 0, 2599, 3, 0, 0, 0, 0, 2628, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 16, 0, 1, 0, 0, 0, 12, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3491, 0, 0, 0, 1545, 0, 0, 2, 0, 0, 3684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7836, 0, 7, 0, 0, 2, 0, 270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 7667, 0, 0, 4, 0, 0, 400, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 166, 0, 0, 0, 0, 5970, 2158, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8003, 7794, 8208, 8191, 8725, 8115, 8190, 8310] avg=8192.0000 max=8725 max_rank=4 min=7794 min_rank=1 imbalance_ratio=1.065063 max_load_reduction=56811 max_load_reduction_pct=0.866867 moved_to_echo=33340 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True + [2026-08-26 15:24:14] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 448.7 | throughput per GPU (TFLOP/s/GPU): 403.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.637265E+00 | loss scale: 1.0 | grad norm: 5.494 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 18, 1387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3018, 0, 5, 2875, 8, 1, 0, 8192, 0, 8191, 1, 0, 1, 0, 0, 0, 5, 2, 0, 0, 2, 1, 0, 3, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 3, 0, 0, 0, 0, 0, 0, 4001, 0, 0, 0, 4997, 0, 5690, 8, 0, 0, 3934, 0, 0, 0, 0, 0, 0, 1, 0, 6492, 0, 0, 0, 1, 0, 0, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 166, 0, 3, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 1513, 0, 0, 0, 0, 0, 3, 0, 0, 5, 12, 1, 0, 0, 0, 3337, 0, 0, 3276, 3, 0, 0, 8190, 1, 8191, 5, 0, 1, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 2678, 1, 1, 0, 5047, 0, 6302, 1, 1, 0, 3490, 0, 0, 0, 0, 0, 1, 0, 0, 6958, 0, 5, 0, 0, 3, 0, 442, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7668, 0, 1, 11, 0, 1, 0, 1, 0, 0, 0, 2, 0, 0, 0, 1, 0, 154, 3, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 3, 2625, 0, 0, 0, 8, 0, 1, 0, 0, 0, 0, 0, 1, 1, 3, 2247, 0, 0, 3575, 9, 0, 0, 8186, 0, 8185, 0, 0, 1, 0, 0, 2, 0, 0, 0, 2, 2, 0, 1, 0, 1, 0, 2, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 8186, 0, 2, 0, 0, 0, 1, 0, 0, 3218, 0, 0, 0, 4648, 0, 5997, 8, 3, 3, 3140, 0, 1, 4, 0, 2, 0, 1, 0, 6857, 0, 3, 1, 0, 0, 0, 658, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7754, 1, 0, 3, 0, 3, 0, 1, 0, 1, 1, 8, 0, 0, 0, 0, 1, 170, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 1385, 0, 1, 1, 3, 0, 0, 0, 0, 0, 4, 0, 0, 1, 0, 3135, 1, 0, 3224, 0, 0, 0, 8190, 0, 8190, 4, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 3474, 0, 0, 0, 5116, 0, 5999, 13, 1, 0, 3045, 0, 0, 0, 0, 0, 0, 0, 0, 6775, 0, 5, 0, 0, 0, 0, 724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7776, 0, 0, 1, 0, 0, 0, 1, 0, 3, 0, 1, 0, 0, 0, 0, 0, 253, 0, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 1977, 0, 0, 0, 3, 0, 0, 3, 0, 3, 1, 5, 0, 0, 0, 3006, 0, 0, 2727, 0, 0, 0, 8191, 0, 8188, 2, 0, 0, 1, 1, 2, 1, 0, 0, 0, 2, 0, 0, 1, 0, 1, 2, 1, 2, 1, 1, 2, 0, 1, 0, 0, 0, 2, 2, 8177, 0, 0, 0, 0, 1, 0, 0, 0, 2839, 0, 1, 0, 5244, 0, 5959, 9, 0, 0, 3332, 0, 0, 0, 0, 0, 0, 1, 0, 7423, 0, 1, 0, 1, 0, 0, 483, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1387, 1, 1, 3, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 2857, 1, 0, 2705, 0, 0, 0, 8189, 0, 8188, 1, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 8185, 0, 0, 0, 3, 0, 0, 0, 0, 2974, 0, 1, 0, 5092, 0, 6095, 6, 0, 0, 4275, 0, 0, 0, 0, 0, 0, 0, 0, 7338, 0, 3, 1, 0, 0, 0, 251, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 7712, 0, 0, 12, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 225, 0, 3, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 15, 1187, 0, 0, 0, 9, 0, 0, 0, 0, 2, 5, 0, 0, 0, 0, 2743, 0, 0, 2754, 3, 0, 0, 8192, 0, 8192, 10, 0, 0, 0, 0, 0, 0, 1, 0, 1, 18, 0, 1, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 3, 0, 0, 0, 3504, 0, 0, 0, 5482, 0, 6072, 2, 0, 0, 3808, 0, 0, 0, 0, 0, 0, 0, 0, 7007, 0, 14, 0, 0, 2, 0, 345, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 7697, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 258, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 3, 1366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 2866, 0, 0, 2323, 0, 0, 0, 8192, 0, 8190, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1, 8181, 0, 0, 0, 4, 0, 0, 0, 0, 2751, 0, 0, 0, 5766, 0, 6407, 3, 1, 1, 3998, 0, 0, 0, 0, 1, 1, 0, 0, 6949, 0, 2, 0, 0, 0, 0, 611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7634, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 257, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 18, 1387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3929, 2553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3018, 0, 5, 2875, 8, 1, 0, 0, 0, 2351, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5, 2, 0, 0, 2, 1, 0, 3, 0, 0, 4, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 3, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4001, 0, 0, 0, 1329, 0, 0, 8, 0, 0, 3934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6492, 0, 0, 0, 1, 0, 0, 542, 0, 679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7797, 0, 0, 0, 0, 0, 434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 166, 0, 3, 0, 0, 5690, 1911, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7888, 8259, 8195, 8189, 9272, 7715, 8232, 7786] avg=8192.0000 max=9272 max_rank=4 min=7715 min_rank=5 imbalance_ratio=1.131836 max_load_reduction=56264 max_load_reduction_pct=0.858521 moved_to_echo=33005 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 1513, 0, 0, 0, 0, 0, 3, 0, 0, 5, 12, 3929, 2578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3337, 0, 0, 3276, 3, 0, 0, 0, 1, 2351, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 28, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8185, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2678, 1, 1, 0, 1344, 0, 0, 1, 1, 0, 3490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6958, 0, 5, 0, 0, 3, 0, 442, 0, 685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7668, 0, 1, 11, 0, 1, 438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 1, 0, 154, 3, 0, 0, 0, 6302, 1911, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8046, 8974, 8207, 8190, 7516, 8094, 8120, 8389] avg=8192.0000 max=8974 max_rank=1 min=7516 min_rank=4 imbalance_ratio=1.095459 max_load_reduction=56562 max_load_reduction_pct=0.863068 moved_to_echo=32119 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 3, 2625, 0, 0, 0, 8, 0, 1, 0, 0, 0, 0, 3926, 2374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 3, 2247, 0, 0, 3575, 9, 0, 0, 0, 0, 2349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 0, 2, 2, 0, 1, 0, 1, 0, 2, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 8186, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3218, 0, 0, 0, 1239, 0, 0, 8, 3, 3, 3140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 2, 0, 1, 0, 6857, 0, 3, 1, 0, 0, 0, 658, 0, 631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7754, 1, 0, 3, 0, 3, 403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 8, 0, 0, 0, 0, 1, 170, 0, 0, 0, 0, 5997, 1910, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8938, 8185, 8182, 8192, 7612, 8158, 8165, 8104] avg=8192.0000 max=8938 max_rank=0 min=7612 min_rank=4 imbalance_ratio=1.091064 max_load_reduction=56598 max_load_reduction_pct=0.863617 moved_to_echo=32039 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 1385, 0, 1, 1, 3, 0, 0, 0, 0, 0, 4, 3931, 2618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3135, 1, 0, 3224, 0, 0, 0, 0, 0, 2344, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 1, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3474, 0, 0, 0, 1347, 0, 0, 13, 1, 0, 3045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6775, 0, 5, 0, 0, 0, 0, 724, 0, 699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7776, 0, 0, 1, 0, 0, 448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 1, 0, 0, 0, 0, 0, 253, 0, 0, 0, 2, 5999, 1915, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7949, 8709, 8182, 8197, 7882, 8203, 8226, 8188] avg=8192.0000 max=8709 max_rank=1 min=7882 min_rank=4 imbalance_ratio=1.063110 max_load_reduction=56827 max_load_reduction_pct=0.867111 moved_to_echo=32499 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1387, 1, 1, 3, 0, 0, 0, 1, 0, 0, 0, 3928, 2601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 2857, 1, 0, 2705, 0, 0, 0, 0, 0, 2349, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 9, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 8185, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2974, 0, 1, 0, 1355, 0, 0, 6, 0, 0, 4275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7338, 0, 3, 1, 0, 0, 0, 251, 0, 692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 7712, 0, 0, 12, 0, 0, 442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 225, 0, 3, 0, 1, 6095, 1911, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7922, 7916, 8188, 8194, 8611, 8285, 8169, 8251] avg=8192.0000 max=8611 max_rank=4 min=7916 min_rank=1 imbalance_ratio=1.051147 max_load_reduction=56925 max_load_reduction_pct=0.868607 moved_to_echo=33316 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 1977, 0, 0, 0, 3, 0, 0, 3, 0, 3, 1, 3928, 2679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 3006, 0, 0, 2727, 0, 0, 0, 0, 0, 2349, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 1, 0, 0, 0, 2, 0, 0, 1, 0, 1, 2, 1, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 2, 0, 1, 0, 0, 0, 2, 2, 8177, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2839, 0, 1, 0, 1396, 0, 0, 9, 0, 0, 3332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7423, 0, 1, 0, 1, 0, 0, 483, 0, 712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7848, 0, 0, 0, 0, 0, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 5959, 1911, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8597, 8089, 8188, 8190, 7578, 8621, 8304, 7969] avg=8192.0000 max=8621 max_rank=5 min=7578 min_rank=4 imbalance_ratio=1.052368 max_load_reduction=56915 max_load_reduction_pct=0.868454 moved_to_echo=32472 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 15, 1187, 0, 0, 0, 9, 0, 0, 0, 0, 2, 5, 3930, 2800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2743, 0, 0, 2754, 3, 0, 0, 0, 0, 2351, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 18, 0, 1, 0, 0, 0, 11, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3504, 0, 0, 0, 1459, 0, 0, 2, 0, 0, 3808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7007, 0, 14, 0, 0, 2, 0, 345, 0, 745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 7697, 0, 0, 5, 0, 0, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 258, 0, 0, 0, 0, 6072, 1911, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7948, 7861, 8210, 8191, 8776, 8113, 8181, 8256] avg=8192.0000 max=8776 max_rank=4 min=7861 min_rank=1 imbalance_ratio=1.071289 max_load_reduction=56760 max_load_reduction_pct=0.866089 moved_to_echo=33326 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 3, 1366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3929, 2945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2866, 0, 0, 2323, 0, 0, 0, 0, 0, 2350, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1, 8181, 0, 0, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2751, 0, 0, 0, 1535, 0, 0, 3, 1, 1, 3998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 6949, 0, 2, 0, 0, 0, 0, 611, 0, 783, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7634, 0, 0, 2, 1, 0, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 257, 1, 0, 0, 0, 6407, 1911, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8248, 7543, 8184, 8193, 8289, 8347, 8139, 8593] avg=8192.0000 max=8593 max_rank=7 min=7543 min_rank=1 imbalance_ratio=1.048950 max_load_reduction=56943 max_load_reduction_pct=0.868881 moved_to_echo=33368 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 128, 0, 128, 128, 11392, 128, 128, 128, 128, 0, 128, 128, 0, 128, 128, 27264, 26880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 21760, 128, 128, 22784, 128, 128, 0, 0, 128, 21120, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 0, 25600, 128, 128, 128, 11776, 0, 0, 128, 128, 128, 28288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 128, 0, 128, 128, 128, 128, 0, 0, 128, 128, 768, 128, 128, 0, 128, 47488, 17280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 128, 0, 128, 128, 12928, 128, 128, 128, 128, 0, 128, 128, 0, 128, 128, 31488, 21248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 23296, 128, 128, 23552, 128, 128, 0, 0, 128, 18816, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 0, 25472, 128, 128, 0, 11008, 0, 0, 128, 128, 128, 29056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 128, 0, 128, 128, 128, 128, 0, 0, 128, 128, 1664, 128, 128, 0, 128, 48640, 15360, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 128, 128, 128, 0, 55808, 0, 128, 128, 128, 128, 0, 4096, 0, 5632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 0, 128, 0, 128, 0, 128, 0, 128, 128, 61568, 128, 128, 128, 128, 128, 3072, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 128, 128, 128, 0, 62336, 0, 128, 128, 128, 128, 0, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 128, 128, 0, 0, 0, 128, 128, 65536, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 0, 128, 0, 128, 0, 128, 0, 128, 128, 61952, 128, 128, 128, 128, 128, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=67328 observed_loss=-1792 tokens_per_expert=[128, 128, 128, 128, 128, 128, 0, 128, 128, 128, 128, 128, 128, 128, 128, 128, 65408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 128, 128, 0, 0, 128, 128, 128, 65536, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=67328 slack_slots=63744 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[8179, 0, 0, 0, 0, 0, 8189, 0, 5, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8189, 8191, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[8145, 0, 88, 0, 0, 0, 8134, 0, 7, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8190, 8192, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 16, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[8183, 4, 187, 2, 0, 0, 8001, 0, 0, 0, 0, 0, 0, 1, 0, 4, 1, 0, 1, 0, 8187, 8186, 0, 0, 0, 5, 0, 1, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 8184, 0, 0, 0, 0, 0, 5, 0, 4, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 4, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4, 3, 1, 0, 0, 2, 0, 11, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[8180, 0, 5, 2, 0, 2, 8186, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8189, 8190, 2, 0, 0, 9, 0, 2, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8190, 8190, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[8144, 0, 0, 0, 0, 0, 8187, 1, 123, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8089, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[8162, 0, 0, 0, 0, 0, 8188, 0, 6, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[8184, 0, 5, 1, 1, 0, 8187, 0, 2, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 8188, 8189, 0, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 8189, 1, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8187, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 88, 0, 0, 0, 8111, 0, 7, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 3, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 8145, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8189, 8201, 8191, 8198, 8192, 8194, 8164] avg=8192.0000 max=8207 max_rank=0 min=8164 min_rank=7 imbalance_ratio=1.001831 max_load_reduction=57329 max_load_reduction_pct=0.874771 moved_to_echo=32748 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 16, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 3, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8192, 8208, 8191, 8181, 8192, 8192, 8199] avg=8192.0000 max=8208 max_rank=2 min=8181 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=57328 max_load_reduction_pct=0.874756 moved_to_echo=32764 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8165, 0, 5, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 3, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1, 0, 0, 0, 8179, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 8192, 8198, 8191, 8200, 8191, 8192, 8201] avg=8192.0000 max=8201 max_rank=7 min=8171 min_rank=0 imbalance_ratio=1.001099 max_load_reduction=57335 max_load_reduction_pct=0.874863 moved_to_echo=32784 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 187, 2, 0, 0, 7978, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 8184, 0, 0, 0, 0, 5, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 5, 0, 4, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 3, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 4, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4, 3, 1, 0, 0, 2, 8183, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8183, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8192, 8203, 8195, 8179, 8193, 8195, 8203] avg=8192.0000 max=8203 max_rank=2 min=8176 min_rank=0 imbalance_ratio=1.001343 max_load_reduction=57333 max_load_reduction_pct=0.874832 moved_to_echo=32770 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 5, 1, 1, 0, 8163, 0, 2, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 1, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8187, 1, 0, 0, 0, 3, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 0, 0, 0, 8184, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8190, 8203, 8195, 8195, 8191, 8190, 8196] avg=8192.0000 max=8203 max_rank=2 min=8176 min_rank=0 imbalance_ratio=1.001343 max_load_reduction=57333 max_load_reduction_pct=0.874832 moved_to_echo=32772 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8163, 1, 123, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8089, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 3, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 8144, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8288, 8188, 8106, 8192, 8198, 8191, 8192, 8181] avg=8192.0000 max=8288 max_rank=0 min=8106 min_rank=2 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32762 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8164, 0, 6, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 3, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 8162, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 8194, 8208, 8191, 8192, 8193, 8191, 8196] avg=8192.0000 max=8208 max_rank=2 min=8171 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=57328 max_load_reduction_pct=0.874756 moved_to_echo=32772 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 5, 2, 0, 2, 8155, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8185, 0, 2, 0, 0, 9, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 4, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4, 0, 0, 0, 0, 8180, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8166, 8199, 8209, 8190, 8193, 8193, 8190, 8196] avg=8192.0000 max=8209 max_rank=2 min=8166 min_rank=0 imbalance_ratio=1.002075 max_load_reduction=57327 max_load_reduction_pct=0.874741 moved_to_echo=32772 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[8167, 0, 35, 0, 0, 0, 8167, 0, 6, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8190, 8192, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[8181, 0, 5, 2, 0, 2, 8187, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8190, 8190, 2, 0, 0, 9, 0, 2, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8190, 8190, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[8170, 0, 0, 0, 0, 0, 8189, 1, 119, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8075, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[8185, 0, 0, 0, 0, 0, 8189, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8189, 8191, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 9, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[8183, 4, 168, 2, 0, 0, 8021, 0, 0, 0, 0, 0, 0, 1, 0, 5, 1, 0, 2, 0, 8187, 8186, 0, 0, 0, 8, 0, 1, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 8184, 0, 0, 0, 0, 0, 5, 0, 4, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 4, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4, 3, 2, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[8186, 0, 3, 1, 1, 0, 8188, 0, 2, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 1, 8188, 8190, 0, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 8188, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 2, 1, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[8168, 0, 1, 0, 0, 0, 8190, 0, 3, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[8192, 0, 21, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 35, 0, 0, 0, 8145, 0, 6, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 8167, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8188, 8204, 8192, 8199, 8192, 8194, 8180] avg=8192.0000 max=8204 max_rank=2 min=8180 min_rank=7 imbalance_ratio=1.001465 max_load_reduction=57332 max_load_reduction_pct=0.874817 moved_to_echo=32765 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8167, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8186, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 9, 0, 0, 0, 0, 8185, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8193, 8199, 8192, 8198, 8191, 8194, 8197] avg=8192.0000 max=8199 max_rank=2 min=8172 min_rank=0 imbalance_ratio=1.000854 max_load_reduction=57337 max_load_reduction_pct=0.874893 moved_to_echo=32780 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 1, 1, 0, 8166, 0, 2, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8185, 0, 0, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 1, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 2, 1, 1, 0, 0, 0, 8186, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8191, 8204, 8194, 8194, 8191, 8190, 8195] avg=8192.0000 max=8204 max_rank=2 min=8177 min_rank=0 imbalance_ratio=1.001465 max_load_reduction=57332 max_load_reduction_pct=0.874817 moved_to_echo=32770 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 5, 2, 0, 2, 8161, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8183, 0, 2, 0, 0, 9, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 7, 4, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4, 0, 0, 0, 0, 8181, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8197, 8210, 8185, 8200, 8192, 8190, 8190] avg=8192.0000 max=8210 max_rank=2 min=8172 min_rank=0 imbalance_ratio=1.002197 max_load_reduction=57326 max_load_reduction_pct=0.874725 moved_to_echo=32772 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 168, 2, 0, 0, 8000, 0, 0, 0, 0, 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 8184, 0, 0, 0, 0, 8, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 5, 0, 4, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 4, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4, 3, 2, 0, 0, 0, 8183, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8183, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8196, 8205, 8196, 8176, 8192, 8194, 8197] avg=8192.0000 max=8205 max_rank=2 min=8176 min_rank=4 imbalance_ratio=1.001587 max_load_reduction=57331 max_load_reduction_pct=0.874802 moved_to_echo=32759 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8167, 1, 119, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8075, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 8170, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8288, 8187, 8094, 8193, 8199, 8191, 8192, 8192] avg=8192.0000 max=8288 max_rank=0 min=8094 min_rank=2 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32774 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 8168, 0, 3, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 8168, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8173, 8193, 8210, 8192, 8191, 8195, 8191, 8191] avg=8192.0000 max=8210 max_rank=2 min=8173 min_rank=0 imbalance_ratio=1.002197 max_load_reduction=57326 max_load_reduction_pct=0.874725 moved_to_echo=32768 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 21, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8191, 8210, 8192, 8179, 8192, 8191, 8194] avg=8192.0000 max=8210 max_rank=2 min=8179 min_rank=4 imbalance_ratio=1.002197 max_load_reduction=57326 max_load_reduction_pct=0.874725 moved_to_echo=32756 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 128, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 0, 128, 0, 0, 0, 128, 0, 128, 128, 128, 0, 0, 0, 65536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 65408, 128, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 128, 256, 128, 128, 128, 65152, 128, 256, 128, 0, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 128, 128, 65536, 0, 128, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 128, 128, 0, 0, 0, 128, 128, 128, 128, 128, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 65536, 128, 0, 128, 0, 0, 128, 128, 128, 0, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 509, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 1, 0, 8189, 0, 0, 0, 1, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 13, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 2, 1, 0, 1, 0, 0, 0, 0, 0, 0, 5969, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 1718, 0, 8190, 0, 2, 0, 0, 1, 0, 0, 0, 8190, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 171, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 8181, 1, 0, 0, 6, 8179, 0, 6, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4865, 0, 0, 0, 1, 0, 0, 8, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8153, 0, 3180, 0, 8184, 1, 7, 0, 0, 1, 0, 2, 0, 8185, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 69, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 3962, 0, 8191, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 113, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 8189, 0, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3878, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8191, 0, 4195, 0, 8190, 0, 0, 0, 1, 0, 0, 0, 0, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 8192, 0, 6312, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 5525, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 58, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 8191, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 6069, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8192, 0, 6331, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8170, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4149, 0, 4659, 112, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3532, 0, 3962, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8192, 8147, 8193, 8185, 8942, 8191, 7495] avg=8192.0000 max=8942 max_rank=5 min=7495 min_rank=7 imbalance_ratio=1.091553 max_load_reduction=56594 max_load_reduction_pct=0.863556 moved_to_echo=32813 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1824, 0, 4660, 112, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3532, 0, 6312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8080, 8192, 8222, 6617, 8197, 9844] avg=8192.0000 max=9844 max_rank=7 min=6617 min_rank=5 imbalance_ratio=1.201660 max_load_reduction=55692 max_load_reduction_pct=0.849792 moved_to_echo=32880 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 13, 0, 0, 0, 0, 0, 1, 0, 2, 0, 8171, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 1, 0, 0, 0, 0, 0, 0, 5969, 0, 4652, 115, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3519, 0, 1718, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8197, 8579, 8188, 8192, 10759, 8190, 5240] avg=8192.0000 max=10759 max_rank=5 min=5240 min_rank=7 imbalance_ratio=1.313354 max_load_reduction=54777 max_load_reduction_pct=0.835831 moved_to_echo=32381 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8069, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8179, 0, 6, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7, 0, 8164, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4865, 0, 4638, 112, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8183, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3515, 0, 3180, 0, 0, 1, 7, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8184, 8241, 8194, 8181, 9640, 8194, 6706] avg=8192.0000 max=9640 max_rank=5 min=6706 min_rank=7 imbalance_ratio=1.176758 max_load_reduction=55896 max_load_reduction_pct=0.852905 moved_to_echo=32721 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8077, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 1, 0, 8171, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3878, 0, 4659, 112, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8187, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3532, 0, 4195, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8196, 8191, 8194, 8177, 8670, 8190, 7728] avg=8192.0000 max=8670 max_rank=5 min=7728 min_rank=7 imbalance_ratio=1.058350 max_load_reduction=56866 max_load_reduction_pct=0.867706 moved_to_echo=32765 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2019, 0, 4659, 112, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3532, 0, 6069, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8191, 8138, 8191, 8220, 6811, 8191, 9603] avg=8192.0000 max=9603 max_rank=7 min=6811 min_rank=5 imbalance_ratio=1.172241 max_load_reduction=55933 max_load_reduction_pct=0.853470 moved_to_echo=32825 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2661, 0, 4660, 112, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3532, 0, 5525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8192, 8080, 8192, 8177, 7453, 8191, 9057] avg=8192.0000 max=9057 max_rank=7 min=7453 min_rank=5 imbalance_ratio=1.105591 max_load_reduction=56479 max_load_reduction_pct=0.861801 moved_to_echo=32878 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1852, 0, 4660, 112, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3532, 0, 6331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8192, 8080, 8192, 8182, 6644, 8192, 9863] avg=8192.0000 max=9863 max_rank=7 min=6644 min_rank=5 imbalance_ratio=1.203979 max_load_reduction=55673 max_load_reduction_pct=0.849503 moved_to_echo=32881 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 128, 0, 128, 128, 128, 0, 0, 0, 65408, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 0, 65536, 0, 128, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 128, 65536, 128, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 0, 0, 128, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 128, 128, 128, 128, 0, 0, 0, 128, 128, 128, 128, 128, 0, 0, 128, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 65536, 128, 0, 128, 128, 0, 128, 128, 128, 0, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 128, 384, 128, 128, 128, 65152, 128, 256, 128, 0, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 65408, 128, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 71, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 8191, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 6951, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 8192, 0, 7233, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 8183, 0, 199, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 8184, 1, 0, 0, 6, 8176, 0, 6, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 7, 0, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4387, 0, 0, 0, 1, 0, 0, 8, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8157, 0, 3629, 0, 8183, 1, 7, 0, 0, 1, 0, 1, 0, 8185, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8192, 0, 6796, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 61, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 5739, 0, 8191, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 118, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 8189, 0, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 6, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8191, 0, 5391, 0, 8190, 0, 1, 0, 1, 0, 0, 0, 0, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 5636, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 564, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 2, 0, 8189, 0, 0, 0, 1, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 34, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 2, 1, 0, 1, 0, 0, 0, 0, 0, 0, 5574, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 2035, 0, 8190, 0, 2, 0, 0, 1, 0, 0, 0, 8190, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 868, 0, 5426, 124, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2766, 0, 7233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8068, 8192, 8238, 6459, 8196, 9999] avg=8192.0000 max=9999 max_rank=7 min=6459 min_rank=5 imbalance_ratio=1.220581 max_load_reduction=55537 max_load_reduction_pct=0.847427 moved_to_echo=32892 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1343, 0, 5426, 124, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2766, 0, 6796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8068, 8192, 8204, 6934, 8192, 9562] avg=8192.0000 max=9562 max_rank=7 min=6934 min_rank=5 imbalance_ratio=1.167236 max_load_reduction=55974 max_load_reduction_pct=0.854095 moved_to_echo=32892 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 8183, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8060, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8176, 0, 6, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 7, 0, 8142, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4387, 0, 5402, 124, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2755, 0, 3629, 0, 0, 1, 7, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8185, 8260, 8190, 8159, 9959, 8194, 6394] avg=8192.0000 max=9959 max_rank=5 min=6394 min_rank=7 imbalance_ratio=1.215698 max_load_reduction=55577 max_load_reduction_pct=0.848038 moved_to_echo=32706 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8150, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1117, 0, 5425, 124, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2766, 0, 6951, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8191, 8139, 8191, 8206, 6708, 8191, 9719] avg=8192.0000 max=9719 max_rank=7 min=6708 min_rank=5 imbalance_ratio=1.186401 max_load_reduction=55817 max_load_reduction_pct=0.851700 moved_to_echo=32824 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8065, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1, 0, 0, 0, 0, 0, 0, 1, 0, 8149, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2673, 0, 5425, 124, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2766, 0, 5391, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8196, 8184, 8194, 8159, 8265, 8191, 8159] avg=8192.0000 max=8265 max_rank=5 min=8159 min_rank=4 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32774 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8149, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2332, 0, 5425, 124, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2766, 0, 5739, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8192, 8128, 8193, 8211, 7924, 8191, 8506] avg=8192.0000 max=8506 max_rank=7 min=7924 min_rank=5 imbalance_ratio=1.038330 max_load_reduction=57030 max_load_reduction_pct=0.870209 moved_to_echo=32832 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2535, 0, 5426, 124, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2766, 0, 5636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8192, 8068, 8192, 8170, 8126, 8191, 8402] avg=8192.0000 max=8402 max_rank=7 min=8068 min_rank=2 imbalance_ratio=1.025635 max_load_reduction=57134 max_load_reduction_pct=0.871796 moved_to_echo=32889 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 34, 0, 0, 0, 0, 0, 1, 0, 2, 0, 8149, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 1, 0, 0, 0, 0, 0, 0, 5574, 0, 5416, 126, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2757, 0, 2035, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8196, 8621, 8192, 8189, 11161, 8190, 4795] avg=8192.0000 max=11161 max_rank=5 min=4795 min_rank=7 imbalance_ratio=1.362427 max_load_reduction=54375 max_load_reduction_pct=0.829697 moved_to_echo=32335 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[22144, 0, 43520, 128, 0, 128, 128, 0, 128, 128, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 128, 0, 128, 0, 0, 0, 128, 20864, 0, 43392, 1024, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[28288, 0, 37376, 128, 0, 128, 128, 0, 128, 128, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 128, 0, 128, 0, 0, 0, 128, 27264, 0, 37248, 1024, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 128, 65536, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 0, 0, 128, 128, 0, 0, 128, 128, 0, 0, 128, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 65536, 0, 128, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1024, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 64640, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 0, 0, 128, 0, 0, 0, 128, 128, 0, 0, 128, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 65536, 0, 128, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 128, 128, 0, 128, 384, 128, 0, 0, 0, 0, 128, 0, 128, 0, 65280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 128, 128, 0, 128, 256, 128, 0, 0, 0, 0, 128, 0, 128, 0, 65408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 128, 65536, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 128, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 1, 0, 0, 0, 0, 8191, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5722, 0, 0, 0, 8191, 3574, 8095, 0, 0, 0, 1, 0, 8191, 8190, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6959, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 834, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 258, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1582, 0, 0, 0, 8192, 6728, 8190, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 1, 0, 0, 0, 0, 0, 0, 7663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 67, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 132, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 3, 0, 0, 0, 1, 0, 0, 3564, 1, 0, 0, 8190, 5099, 8042, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5820, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 867, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2162, 0, 0, 0, 8192, 6872, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 2, 1, 0, 0, 0, 8187, 0, 0, 0, 45, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6271, 0, 0, 0, 8190, 2442, 8087, 0, 0, 0, 2, 0, 8183, 8162, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 14, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2942, 0, 0, 0, 8191, 6842, 8192, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6786, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 399, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1431, 0, 0, 0, 8192, 6933, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 7042, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 61, 0, 7, 6, 0, 0, 0, 8169, 0, 0, 0, 522, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7622, 0, 0, 4, 8183, 1978, 7383, 0, 0, 0, 7, 0, 8182, 8078, 0, 0, 7] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6959, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 944, 286, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 834, 0, 0, 0, 0, 0, 0, 7905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 258, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1582, 4059, 290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6728, 3187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8218, 8193, 8184, 8739, 8161, 5932, 9915] avg=8192.0000 max=9915 max_rank=7 min=5932 min_rank=6 imbalance_ratio=1.210327 max_load_reduction=55621 max_load_reduction_pct=0.848709 moved_to_echo=32747 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 933, 286, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 1, 0, 0, 0, 0, 7905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7899, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5722, 4011, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3574, 3151, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8377, 8191, 8183, 7925, 7917, 10025, 6727] avg=8192.0000 max=10025 max_rank=6 min=6727 min_rank=7 imbalance_ratio=1.223755 max_load_reduction=55511 max_load_reduction_pct=0.847031 moved_to_echo=32594 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5820, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 945, 286, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 867, 0, 0, 0, 0, 0, 0, 7906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2162, 4060, 290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6872, 3187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 7063, 8190, 8184, 8773, 8559, 6514, 10059] avg=8192.0000 max=10059 max_rank=7 min=6514 min_rank=6 imbalance_ratio=1.227905 max_load_reduction=55477 max_load_reduction_pct=0.846512 moved_to_echo=33905 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 936, 290, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 2, 1, 0, 0, 0, 7897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6271, 4010, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2442, 3141, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8934, 8185, 8195, 7949, 7921, 10572, 5587] avg=8192.0000 max=10572 max_rank=6 min=5587 min_rank=7 imbalance_ratio=1.290527 max_load_reduction=54964 max_load_reduction_pct=0.838684 moved_to_echo=32029 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7042, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 851, 285, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 7, 6, 0, 0, 0, 7884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 1, 0, 7791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7622, 3659, 287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1978, 2873, 0, 0, 0, 7, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8194, 8196, 8237, 7958, 8325, 11572, 4869] avg=8192.0000 max=11572 max_rank=6 min=4869 min_rank=7 imbalance_ratio=1.412598 max_load_reduction=53964 max_load_reduction_pct=0.823425 moved_to_echo=32724 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6786, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 945, 286, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 399, 0, 0, 0, 0, 0, 0, 7906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1431, 4060, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6933, 3187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8065, 8195, 8184, 8305, 8693, 5782, 10120] avg=8192.0000 max=10120 max_rank=7 min=5782 min_rank=6 imbalance_ratio=1.235352 max_load_reduction=55416 max_load_reduction_pct=0.845581 moved_to_echo=32900 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 1, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 927, 286, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 67, 0, 0, 0, 0, 0, 0, 7899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 3, 0, 0, 0, 1, 0, 0, 3564, 3985, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5099, 3130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8885, 8193, 8184, 7967, 8036, 7846, 8230] avg=8192.0000 max=8885 max_rank=1 min=7846 min_rank=6 imbalance_ratio=1.084595 max_load_reduction=56651 max_load_reduction_pct=0.864426 moved_to_echo=32079 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 945, 286, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 14, 0, 0, 0, 0, 0, 0, 7905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2942, 4060, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6842, 3187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 7800, 8193, 8185, 7920, 7924, 7293, 10029] avg=8192.0000 max=10029 max_rank=7 min=7293 min_rank=6 imbalance_ratio=1.224243 max_load_reduction=55507 max_load_reduction_pct=0.846970 moved_to_echo=33166 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 1, 0, 0, 0, 0, 8190, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4878, 0, 0, 0, 8191, 3607, 8106, 0, 0, 0, 1, 0, 8191, 8191, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 7545, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 71, 0, 7, 6, 0, 0, 0, 8167, 0, 0, 0, 494, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7286, 0, 0, 4, 8183, 1802, 7418, 0, 0, 0, 7, 0, 8182, 8068, 0, 0, 7] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7660, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 258, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1100, 0, 0, 0, 8192, 7244, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 2, 1, 0, 0, 0, 8188, 0, 0, 0, 21, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6012, 0, 0, 0, 8190, 2333, 8115, 0, 0, 0, 2, 0, 8180, 8164, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 73, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2272, 0, 0, 0, 8191, 6749, 8192, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7009, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 980, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1148, 0, 0, 0, 8192, 6921, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 7957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 1, 0, 0, 89, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 130, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 1, 0, 0, 3047, 1, 0, 0, 8190, 5295, 8049, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7391, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1117, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 763, 0, 0, 0, 8192, 6976, 8191, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7660, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 330, 238, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 258, 0, 0, 0, 0, 0, 0, 7862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1100, 4828, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7244, 3126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8286, 8192, 8183, 8120, 8136, 6056, 10370] avg=8192.0000 max=10370 max_rank=7 min=6056 min_rank=6 imbalance_ratio=1.265869 max_load_reduction=55166 max_load_reduction_pct=0.841766 moved_to_echo=32682 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 330, 236, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 1, 0, 0, 0, 0, 7860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4878, 4778, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3607, 3092, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8524, 8192, 8182, 7886, 8073, 9786, 6701] avg=8192.0000 max=9786 max_rank=6 min=6701 min_rank=7 imbalance_ratio=1.194580 max_load_reduction=55750 max_load_reduction_pct=0.850677 moved_to_echo=32446 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8183, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7545, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 329, 216, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 7, 6, 0, 0, 0, 7838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 1, 0, 7942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7286, 4372, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1802, 2830, 0, 0, 0, 7, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8107, 8195, 8239, 7922, 8449, 11788, 4650] avg=8192.0000 max=11788 max_rank=6 min=4650 min_rank=7 imbalance_ratio=1.438965 max_load_reduction=53748 max_load_reduction_pct=0.820129 moved_to_echo=32809 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 334, 239, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 2, 1, 0, 0, 0, 7854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6012, 4788, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2333, 3088, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8641, 8184, 8198, 7907, 8059, 10929, 5425] avg=8192.0000 max=10929 max_rank=6 min=5425 min_rank=7 imbalance_ratio=1.334106 max_load_reduction=54607 max_load_reduction_pct=0.833237 moved_to_echo=32320 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 330, 238, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8182, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 73, 0, 0, 0, 0, 0, 0, 7861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2272, 4828, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6749, 3126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 7856, 8193, 8184, 7935, 8072, 7229, 9875] avg=8192.0000 max=9875 max_rank=7 min=7229 min_rank=6 imbalance_ratio=1.205444 max_load_reduction=55661 max_load_reduction_pct=0.849319 moved_to_echo=33111 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 330, 234, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 89, 0, 0, 0, 0, 0, 0, 7855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 1, 0, 0, 3047, 4744, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5295, 3071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8531, 8193, 8184, 7946, 8197, 7924, 8367] avg=8192.0000 max=8531 max_rank=1 min=7924 min_rank=6 imbalance_ratio=1.041382 max_load_reduction=57005 max_load_reduction_pct=0.869827 moved_to_echo=32434 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7009, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 330, 238, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 980, 0, 0, 0, 0, 0, 0, 7862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1148, 4828, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6921, 3126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 7597, 8192, 8183, 8842, 8378, 6104, 10047] avg=8192.0000 max=10047 max_rank=7 min=6104 min_rank=6 imbalance_ratio=1.226440 max_load_reduction=55489 max_load_reduction_pct=0.846695 moved_to_echo=33371 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7391, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 330, 238, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1117, 0, 0, 0, 0, 0, 0, 7861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 763, 4828, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6976, 3125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 7994, 8195, 8183, 8978, 8172, 5720, 10101] avg=8192.0000 max=10101 max_rank=7 min=5720 min_rank=6 imbalance_ratio=1.233032 max_load_reduction=55435 max_load_reduction_pct=0.845871 moved_to_echo=32971 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 40576, 25088, 0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 0, 128, 0, 0, 128, 0, 128, 0, 0, 0, 128, 0, 0, 31360, 32000, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 55680, 0, 128, 0, 0, 0, 0, 128, 0, 0, 128, 128, 128, 0, 7552, 2304, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 40960, 24704, 0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 0, 0, 128, 128, 0, 128, 0, 0, 0, 128, 0, 0, 26624, 38016, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 2560, 0, 128, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 128, 0, 63104, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 0, 128, 2432, 0, 128, 128, 0, 0, 0, 63232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 0, 0, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 0, 0, 0, 0, 0, 0, 128, 128, 0, 128, 0, 0, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 60928, 0, 128, 0, 0, 0, 0, 128, 0, 0, 128, 128, 128, 0, 2688, 1920, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 128, 0, 128, 2688, 0, 128, 128, 0, 0, 0, 62976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 1280, 0, 128, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 128, 0, 64384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 128, 0, 0, 128, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 128, 0, 0, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 8181, 8032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 160, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8185, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8183, 8184, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 28, 0, 0, 0, 483, 3, 0, 0, 0, 0, 0, 7637, 0, 0, 0, 4, 0, 0, 0, 41, 0, 7970, 2809, 0, 0, 0, 5, 0, 0, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 5823, 0, 0, 0, 0, 0, 0, 0, 6, 6, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 5, 1, 0, 167, 0, 7, 0, 0, 0, 0, 0, 0, 7694, 7, 0, 2, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8188, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 1, 1, 0, 0, 1, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 6879, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 5380, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 8173, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8077, 7695, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 607, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 8147, 8090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 1, 136, 0, 0, 0, 1, 1, 0, 8052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 2, 1, 8189, 7366, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 4, 0, 787, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 8190, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 34, 0, 0, 0, 568, 2, 1, 0, 0, 0, 0, 7600, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8166, 4602, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 5151, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 107, 0, 2, 0, 0, 0, 0, 0, 0, 6477, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1626, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 1037, 8032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7144, 1022, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 160, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8182, 8192, 9818, 9110, 8188, 8167, 5703] avg=8192.0000 max=9818 max_rank=3 min=5703 min_rank=7 imbalance_ratio=1.198486 max_load_reduction=55718 max_load_reduction_pct=0.850189 moved_to_echo=31168 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1627, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 1036, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7137, 1023, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8169, 8181, 8191, 9819, 9200, 8189, 8161, 5626] avg=8192.0000 max=9819 max_rank=3 min=5626 min_rank=7 imbalance_ratio=1.198608 max_load_reduction=55717 max_load_reduction_pct=0.850174 moved_to_echo=31176 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8185, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 28, 0, 8170, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 3, 0, 0, 0, 0, 0, 7637, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 1583, 2809, 0, 0, 0, 5, 0, 0, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 5823, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 6, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 5, 1, 0, 167, 0, 7, 0, 0, 0, 0, 7147, 995, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7694, 7, 0, 2, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 5392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8208, 8127, 4516, 6860, 8195, 8330, 13102] avg=8192.0000 max=13102 max_rank=7 min=4516 min_rank=3 imbalance_ratio=1.599365 max_load_reduction=52434 max_load_reduction_pct=0.800079 moved_to_echo=36487 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8175, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 1, 1, 0, 0, 1, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1625, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 6879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7153, 1022, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5380, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 5537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8179, 8271, 5675, 7917, 8192, 8179, 10919] avg=8192.0000 max=10919 max_rank=7 min=5675 min_rank=3 imbalance_ratio=1.332886 max_load_reduction=54617 max_load_reduction_pct=0.833389 moved_to_echo=35207 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1627, 8192, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 7695, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7053, 1023, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 607, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8180, 8195, 9821, 8720, 8188, 8077, 6150] avg=8192.0000 max=9821 max_rank=3 min=6150 min_rank=7 imbalance_ratio=1.198853 max_load_reduction=55715 max_load_reduction_pct=0.850143 moved_to_echo=31135 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 8176, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 136, 0, 0, 0, 1, 1, 0, 8052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1627, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 1, 1038, 7366, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 7151, 1022, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 787, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8203, 8191, 9819, 8409, 8191, 8184, 6339] avg=8192.0000 max=9819 max_rank=3 min=6339 min_rank=7 imbalance_ratio=1.198608 max_load_reduction=55717 max_load_reduction_pct=0.850174 moved_to_echo=31123 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1627, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 1033, 8090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7114, 1023, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8182, 8192, 9819, 9136, 8188, 8138, 5688] avg=8192.0000 max=9819 max_rank=3 min=5688 min_rank=7 imbalance_ratio=1.198608 max_load_reduction=55717 max_load_reduction_pct=0.850174 moved_to_echo=31150 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 34, 0, 8179, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 568, 2, 1, 0, 0, 0, 0, 7600, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1609, 4602, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 5151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 107, 0, 2, 0, 0, 0, 0, 7157, 1023, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6477, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 5528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8221, 8177, 6249, 6184, 8205, 8300, 12009] avg=8192.0000 max=12009 max_rank=7 min=6184 min_rank=4 imbalance_ratio=1.465942 max_load_reduction=53527 max_load_reduction_pct=0.816757 moved_to_echo=34698 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8188, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 1, 1, 0, 0, 0, 0, 8121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 2879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 7834, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 7, 0, 1, 0, 0, 0, 0, 0, 0, 5624, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 8143, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 8190, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 23, 0, 0, 0, 525, 2, 1, 0, 0, 0, 0, 7541, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8167, 3179, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 7142, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 128, 0, 2, 0, 0, 0, 0, 0, 0, 6001, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 8175, 8145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 54, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 7920, 8113, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 350, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 1, 153, 0, 0, 0, 1, 0, 0, 8011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 8191, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 8188, 7962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 4, 0, 222, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 8141, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8185, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8183, 8183, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 19, 0, 0, 0, 456, 2, 0, 0, 0, 0, 0, 7608, 0, 0, 0, 4, 0, 0, 0, 42, 0, 7912, 1557, 0, 0, 0, 5, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 7233, 0, 0, 0, 0, 0, 0, 0, 6, 9, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 6, 1, 0, 179, 0, 7, 0, 0, 0, 0, 0, 0, 7640, 7, 0, 2, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2111, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 337, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7806, 342, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8185, 8219, 10303, 8532, 8188, 8150, 5778] avg=8192.0000 max=10303 max_rank=3 min=5778 min_rank=7 imbalance_ratio=1.257690 max_load_reduction=55233 max_load_reduction_pct=0.842789 moved_to_echo=30648 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 23, 0, 8183, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 2, 1, 0, 0, 0, 0, 7541, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2093, 3179, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 335, 7142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 128, 0, 2, 0, 0, 0, 0, 7855, 343, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6001, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 5697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8214, 8103, 5307, 7477, 8207, 8335, 11702] avg=8192.0000 max=11702 max_rank=7 min=5307 min_rank=3 imbalance_ratio=1.428467 max_load_reduction=53834 max_load_reduction_pct=0.821442 moved_to_echo=35721 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 8179, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 153, 0, 0, 0, 1, 0, 0, 8011, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2110, 8191, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 339, 7962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 7849, 342, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 222, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8199, 8195, 10304, 8304, 8189, 8205, 5942] avg=8192.0000 max=10304 max_rank=3 min=5942 min_rank=7 imbalance_ratio=1.257812 max_load_reduction=55232 max_load_reduction_pct=0.842773 moved_to_echo=30640 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8178, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 1, 1, 0, 0, 0, 0, 8121, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2109, 2879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 339, 7834, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 7, 0, 1, 0, 0, 0, 0, 7852, 342, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5624, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 5706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8182, 8263, 4988, 8173, 8192, 8205, 11332] avg=8192.0000 max=11332 max_rank=7 min=4988 min_rank=3 imbalance_ratio=1.383301 max_load_reduction=54204 max_load_reduction_pct=0.827087 moved_to_echo=35902 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2111, 8192, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 328, 8113, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7592, 342, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 350, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5709, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8184, 8217, 10304, 8446, 8187, 7936, 6061] avg=8192.0000 max=10304 max_rank=3 min=6061 min_rank=7 imbalance_ratio=1.257812 max_load_reduction=55232 max_load_reduction_pct=0.842773 moved_to_echo=30630 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2111, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 337, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7804, 342, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8185, 8220, 10303, 8520, 8188, 8148, 5781] avg=8192.0000 max=10303 max_rank=3 min=5781 min_rank=7 imbalance_ratio=1.257690 max_load_reduction=55233 max_load_reduction_pct=0.842789 moved_to_echo=30637 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8185, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 19, 0, 8173, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 456, 2, 0, 0, 0, 0, 0, 7608, 0, 0, 0, 4, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 2039, 1557, 0, 0, 0, 5, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 339, 7233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 9, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 6, 1, 0, 179, 0, 7, 0, 0, 0, 0, 7845, 330, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7640, 7, 0, 2, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 5515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8202, 8098, 3725, 7572, 8197, 8376, 13171] avg=8192.0000 max=13171 max_rank=7 min=3725 min_rank=3 imbalance_ratio=1.607788 max_load_reduction=52365 max_load_reduction_pct=0.799026 moved_to_echo=37316 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2110, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 338, 8145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7837, 342, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 54, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8185, 8221, 10302, 8512, 8188, 8181, 5769] avg=8192.0000 max=10302 max_rank=3 min=5769 min_rank=7 imbalance_ratio=1.257568 max_load_reduction=55234 max_load_reduction_pct=0.842804 moved_to_echo=30650 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 21376, 128, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 44160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 13056, 52480, 0, 0, 0, 128, 0, 0, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 0, 0, 128, 128, 128, 0, 384, 0, 128, 0, 0, 0, 0, 57088, 8192, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 128, 128, 128, 8320, 57216, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 16896, 48640, 0, 0, 0, 128, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 20096, 128, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 45568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 0, 0, 128, 128, 128, 0, 384, 0, 128, 0, 0, 0, 0, 62464, 2816, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 128, 128, 128, 2816, 62848, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 0, 128, 128, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 1408, 128, 128, 0, 128, 128, 0, 64256, 0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 65536, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 1280, 128, 128, 0, 128, 0, 0, 64128, 0, 0, 0, 128, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 65536, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 158, 0, 0, 0, 6, 0, 0, 0, 0, 0, 179, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8, 0, 0, 0, 0, 0, 0, 6394, 0, 0, 0, 0, 3234, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 758, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4, 0, 8083, 0, 0, 7, 0, 8112, 0, 0, 0, 7, 5813, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 8, 8182, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 8184, 2, 0, 0, 0, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 28, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 37, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 0, 2, 1, 0, 0, 1, 0, 0, 161, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 7494, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 534, 0, 0, 0, 0, 0, 3, 2, 0, 0, 2, 0, 0, 0, 8191, 0, 0, 0, 2, 8186, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7, 0, 0, 0, 8192, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 7671, 0, 0, 0, 0, 1069, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 126, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 1, 0, 8180, 0, 0, 0, 1, 7517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 8191, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8165, 0, 14, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 39, 0, 0, 24, 0, 0, 0, 8192, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 8148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 77, 0, 0, 0, 1, 0, 0, 0, 0, 0, 247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 6764, 0, 0, 0, 0, 1458, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8138, 0, 0, 2, 0, 8185, 0, 0, 0, 2, 7841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8189, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8190, 1, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 22, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 52, 0, 0, 8, 0, 0, 0, 8192, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8178, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7991, 0, 14, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 39, 0, 0, 24, 0, 0, 7766, 160, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8097, 8188, 8009, 8088, 8616, 8191, 8185, 8162] avg=8192.0000 max=8616 max_rank=4 min=8009 min_rank=2 imbalance_ratio=1.051758 max_load_reduction=56920 max_load_reduction_pct=0.868530 moved_to_echo=33154 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 158, 0, 0, 0, 6, 0, 0, 0, 0, 0, 179, 0, 0, 8019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8, 0, 0, 0, 0, 0, 8174, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6258, 0, 0, 0, 0, 3234, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 758, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4, 7663, 125, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 420, 0, 0, 7, 0, 0, 0, 0, 0, 7, 5813, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 8, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 2, 0, 0, 0, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8362, 8194, 9498, 8636, 6253, 8195, 8192, 8206] avg=8192.0000 max=9498 max_rank=2 min=6253 min_rank=4 imbalance_ratio=1.159424 max_load_reduction=56038 max_load_reduction_pct=0.855072 moved_to_echo=30846 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 0, 2, 1, 0, 0, 1, 0, 0, 161, 3, 0, 8093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 8180, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7334, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 534, 0, 0, 0, 0, 0, 3, 2, 0, 0, 2, 0, 0, 7765, 147, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 426, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8264, 8191, 7336, 8540, 8617, 8192, 8194, 8202] avg=8192.0000 max=8617 max_rank=4 min=7336 min_rank=2 imbalance_ratio=1.051880 max_load_reduction=56919 max_load_reduction_pct=0.868515 moved_to_echo=33205 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7975, 0, 22, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 52, 0, 0, 8, 0, 0, 7766, 160, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8097, 8192, 8000, 8076, 8616, 8192, 8193, 8170] avg=8192.0000 max=8616 max_rank=4 min=8000 min_rank=2 imbalance_ratio=1.051758 max_load_reduction=56920 max_load_reduction_pct=0.868530 moved_to_echo=33171 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7994, 0, 28, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 37, 0, 0, 0, 0, 0, 7766, 160, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8100, 8190, 8023, 8059, 8618, 8192, 8178, 8176] avg=8192.0000 max=8618 max_rank=4 min=8023 min_rank=2 imbalance_ratio=1.052002 max_load_reduction=56918 max_load_reduction_pct=0.868500 moved_to_echo=33164 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 8097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8181, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7, 0, 0, 7766, 161, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 426, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8104, 8193, 8018, 8019, 8606, 8191, 8199, 8206] avg=8192.0000 max=8606 max_rank=4 min=8018 min_rank=2 imbalance_ratio=1.050537 max_load_reduction=56930 max_load_reduction_pct=0.868683 moved_to_echo=33202 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 8086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 8181, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7507, 0, 0, 0, 0, 1069, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 7762, 151, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 426, 0, 0, 1, 0, 0, 0, 0, 0, 1, 7517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8094, 8192, 8580, 8127, 7945, 8193, 8201, 8204] avg=8192.0000 max=8580 max_rank=2 min=7945 min_rank=4 imbalance_ratio=1.047363 max_load_reduction=56956 max_load_reduction_pct=0.869080 moved_to_echo=32543 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 77, 0, 0, 0, 1, 0, 0, 0, 0, 0, 247, 0, 0, 8093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8185, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6612, 0, 0, 0, 0, 1458, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7718, 137, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 420, 0, 0, 2, 0, 0, 0, 0, 0, 2, 7841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8418, 8196, 8072, 7991, 8265, 8190, 8194, 8210] avg=8192.0000 max=8418 max_rank=0 min=7991 min_rank=3 imbalance_ratio=1.027588 max_load_reduction=57118 max_load_reduction_pct=0.871552 moved_to_echo=32859 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8108, 0, 28, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 75, 0, 0, 17, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7378, 0, 0, 0, 0, 1221, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 151, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 8188, 0, 0, 1, 0, 8174, 0, 0, 0, 1, 7629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 8191, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8190, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8135, 0, 35, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 55, 0, 0, 10, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 78, 0, 0, 0, 1, 0, 0, 0, 0, 0, 249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 6084, 0, 0, 0, 0, 2097, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8142, 0, 0, 2, 0, 8181, 0, 0, 0, 2, 7869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8189, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8190, 1, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 8165, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 14, 0, 0, 0, 8192, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 167, 0, 0, 0, 6, 0, 0, 0, 0, 0, 183, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8, 0, 0, 0, 0, 0, 0, 5105, 0, 0, 0, 0, 4029, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 796, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4, 0, 8075, 0, 0, 7, 0, 8114, 0, 0, 0, 7, 6259, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6, 0, 8, 8181, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 8184, 2, 0, 0, 0, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 1, 0, 0, 178, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 7398, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 601, 0, 0, 0, 0, 0, 4, 3, 0, 0, 3, 0, 0, 0, 8191, 0, 0, 0, 2, 8186, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8138, 0, 20, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 40, 0, 0, 38, 0, 0, 0, 8192, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 8145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8042, 0, 28, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 75, 0, 0, 17, 0, 0, 7841, 87, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8095, 8191, 8080, 8074, 8541, 8192, 8190, 8173] avg=8192.0000 max=8541 max_rank=4 min=8074 min_rank=3 imbalance_ratio=1.042603 max_load_reduction=56995 max_load_reduction_pct=0.869675 moved_to_echo=33096 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 8077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8180, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7319, 0, 0, 0, 0, 1221, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 7837, 88, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 351, 0, 0, 1, 0, 0, 0, 0, 0, 1, 7629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8096, 8190, 8544, 8125, 7982, 8193, 8202, 8204] avg=8192.0000 max=8544 max_rank=2 min=7982 min_rank=4 imbalance_ratio=1.042969 max_load_reduction=56992 max_load_reduction_pct=0.869629 moved_to_echo=32581 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 78, 0, 0, 0, 1, 0, 0, 0, 0, 0, 249, 0, 0, 8086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8181, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6028, 0, 0, 0, 0, 2097, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7797, 86, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 345, 0, 0, 2, 0, 0, 0, 0, 0, 2, 7869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8414, 8193, 8127, 7987, 8218, 8190, 8198, 8209] avg=8192.0000 max=8414 max_rank=0 min=7987 min_rank=3 imbalance_ratio=1.027100 max_load_reduction=57122 max_load_reduction_pct=0.871613 moved_to_echo=32815 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 11, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8069, 0, 35, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 55, 0, 0, 10, 0, 0, 7841, 86, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8095, 8193, 8106, 8050, 8543, 8191, 8178, 8180] avg=8192.0000 max=8543 max_rank=4 min=8050 min_rank=3 imbalance_ratio=1.042847 max_load_reduction=56993 max_load_reduction_pct=0.869644 moved_to_echo=33092 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 8180, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8099, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 14, 0, 0, 7841, 86, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 351, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8096, 8193, 8113, 7999, 8539, 8191, 8199, 8206] avg=8192.0000 max=8539 max_rank=4 min=7999 min_rank=3 imbalance_ratio=1.042358 max_load_reduction=56997 max_load_reduction_pct=0.869705 moved_to_echo=33135 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 1, 0, 0, 178, 2, 0, 8089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 8179, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7338, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 601, 0, 0, 0, 0, 0, 4, 3, 0, 0, 3, 0, 0, 7840, 86, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 351, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8274, 8192, 7351, 8587, 8543, 8192, 8195, 8202] avg=8192.0000 max=8587 max_rank=3 min=7351 min_rank=2 imbalance_ratio=1.048218 max_load_reduction=56949 max_load_reduction_pct=0.868973 moved_to_echo=33132 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 167, 0, 0, 0, 6, 0, 0, 0, 0, 0, 183, 0, 0, 8018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8, 0, 0, 0, 0, 0, 8173, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5065, 0, 0, 0, 0, 4029, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 796, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4, 7729, 87, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 346, 0, 0, 7, 0, 0, 0, 0, 0, 7, 6259, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6, 0, 8, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 2, 0, 0, 0, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8374, 8195, 9100, 8649, 6627, 8196, 8192, 8203] avg=8192.0000 max=9100 max_rank=2 min=6627 min_rank=4 imbalance_ratio=1.110840 max_load_reduction=56436 max_load_reduction_pct=0.861145 moved_to_echo=31218 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8178, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8072, 0, 20, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 40, 0, 0, 38, 0, 0, 7841, 86, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8092, 8189, 8115, 8065, 8543, 8191, 8182, 8159] avg=8192.0000 max=8543 max_rank=4 min=8065 min_rank=3 imbalance_ratio=1.042847 max_load_reduction=56993 max_load_reduction_pct=0.869644 moved_to_echo=33075 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 3456, 0, 0, 128, 128, 0, 0, 0, 0, 128, 62208, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 128, 1536, 0, 0, 0, 0, 128, 128, 256, 0, 0, 128, 0, 128, 62080, 1280, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 2816, 0, 0, 128, 128, 0, 0, 0, 0, 128, 62720, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 128, 1664, 0, 0, 0, 0, 128, 128, 256, 0, 0, 128, 0, 128, 62592, 768, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 256, 0, 128, 128, 128, 0, 0, 128, 0, 0, 640, 128, 0, 64768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 59776, 0, 128, 0, 0, 5888, 0, 0, 0, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 58112, 0, 128, 0, 0, 7424, 0, 0, 0, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 0, 0, 0, 128, 0, 128, 128, 0, 128, 128, 0, 128, 0, 0, 65536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 65536, 128, 0, 0, 0, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 128, 128, 0, 128, 128, 0, 0, 0, 0, 65536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 128, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 8192, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8123, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 330, 7, 65, 8184, 1, 8180, 0, 0, 0, 8, 0, 4563, 0, 0, 0, 0, 572, 0, 0, 0, 0, 4093, 0, 7, 0, 0, 5, 0, 0, 2, 0, 0, 0, 8, 7, 8, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 708, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 8, 0, 0, 1, 0, 4, 4, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5959, 0, 0, 0, 0, 54] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8121, 4, 0, 0, 0, 50, 0, 1, 8188, 0, 8188, 0, 0, 0, 0, 0, 7831, 0, 0, 5, 0, 0, 0, 0, 0, 0, 356, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 29] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 22, 1, 6, 8191, 0, 8188, 0, 0, 0, 1, 0, 8114, 0, 0, 0, 0, 287, 0, 0, 0, 0, 2630, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5191, 3, 0, 0, 0, 31] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 8192, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 215, 2, 200, 8190, 0, 8186, 0, 0, 0, 2, 0, 7342, 0, 0, 0, 0, 5, 0, 0, 0, 1, 2523, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 2, 2, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1116, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4968, 2, 0, 0, 0, 11] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 1, 0, 0, 0, 7, 0, 0, 8156, 0, 8191, 0, 0, 0, 0, 0, 8186, 0, 0, 1, 0, 0, 0, 0, 0, 0, 13, 0, 0, 2, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8189, 0, 0, 0, 0, 32] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 330, 7, 65, 503, 1, 0, 0, 0, 0, 8, 0, 4563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 572, 0, 0, 0, 0, 4093, 0, 7, 0, 0, 5, 0, 0, 6623, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8, 7, 8, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 708, 7940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 8, 0, 0, 1, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5959, 0, 0, 0, 0, 54, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8215, 5477, 11540, 8206, 8650, 8204, 8193, 7051] avg=8192.0000 max=11540 max_rank=2 min=5477 min_rank=1 imbalance_ratio=1.408691 max_load_reduction=53996 max_load_reduction_pct=0.823914 moved_to_echo=32098 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 3, 0, 0, 0, 0, 6611, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 7951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8123, 0, 0, 26, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8692, 6919, 8190, 7954, 8159, 8193, 9228] avg=8192.0000 max=9228 max_rank=7 min=6919 min_rank=2 imbalance_ratio=1.126465 max_load_reduction=56308 max_load_reduction_pct=0.859192 moved_to_echo=33534 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 6, 0, 0, 0, 0, 6613, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 0, 0, 24, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8696, 6917, 8190, 7950, 8170, 8184, 9228] avg=8192.0000 max=9228 max_rank=7 min=6917 min_rank=2 imbalance_ratio=1.126465 max_load_reduction=56308 max_load_reduction_pct=0.859192 moved_to_echo=33532 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8121, 4, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 1, 504, 0, 0, 0, 0, 0, 0, 0, 7831, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 356, 0, 0, 4, 0, 0, 0, 1, 6626, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 4, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 29, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8137, 8386, 7233, 8189, 7947, 8198, 8189, 9257] avg=8192.0000 max=9257 max_rank=7 min=7233 min_rank=2 imbalance_ratio=1.130005 max_load_reduction=56279 max_load_reduction_pct=0.858749 moved_to_echo=33591 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 1, 6, 504, 0, 0, 0, 0, 0, 1, 0, 8114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 287, 0, 0, 0, 0, 2630, 0, 1, 0, 0, 0, 0, 0, 6629, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 7947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5191, 3, 0, 0, 0, 31, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8648, 9788, 8192, 8049, 8203, 8191, 6264] avg=8192.0000 max=9788 max_rank=2 min=6264 min_rank=7 imbalance_ratio=1.194824 max_load_reduction=55748 max_load_reduction_pct=0.850647 moved_to_echo=30707 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 6621, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 35, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8693, 6872, 8190, 7957, 8200, 8194, 9229] avg=8192.0000 max=9229 max_rank=7 min=6872 min_rank=2 imbalance_ratio=1.126587 max_load_reduction=56307 max_load_reduction_pct=0.859177 moved_to_echo=33580 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 1, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 503, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 13, 0, 0, 2, 0, 0, 4, 0, 6600, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 7950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8189, 0, 0, 0, 0, 32, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8173, 8696, 6861, 8190, 7969, 8196, 8195, 9256] avg=8192.0000 max=9256 max_rank=7 min=6861 min_rank=2 imbalance_ratio=1.129883 max_load_reduction=56280 max_load_reduction_pct=0.858765 moved_to_echo=33616 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 14, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 215, 2, 200, 487, 0, 0, 0, 0, 0, 2, 0, 7342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 1, 2523, 0, 2, 0, 0, 0, 0, 0, 6633, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 2, 2, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1116, 7944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 2, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4968, 2, 0, 0, 0, 11, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8248, 9406, 8189, 9060, 8206, 8197, 6023] avg=8192.0000 max=9406 max_rank=2 min=6023 min_rank=7 imbalance_ratio=1.148193 max_load_reduction=56130 max_load_reduction_pct=0.856476 moved_to_echo=31486 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 1, 0, 0, 0, 1, 0, 0, 8150, 0, 8191, 0, 0, 0, 0, 0, 8154, 0, 0, 1, 0, 0, 0, 0, 0, 0, 12, 0, 0, 4, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 88] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 15, 1, 1, 8191, 0, 8188, 0, 0, 0, 1, 0, 8004, 0, 0, 0, 0, 498, 0, 0, 0, 0, 2330, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5341, 3, 0, 0, 0, 170] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8104, 4, 0, 0, 0, 41, 0, 1, 8187, 0, 8188, 0, 0, 0, 0, 0, 7921, 0, 0, 5, 0, 0, 0, 0, 0, 0, 162, 0, 0, 5, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 163] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 187, 7, 0, 8184, 1, 8180, 0, 0, 0, 8, 0, 4740, 0, 0, 0, 0, 1139, 0, 0, 0, 0, 3086, 0, 8, 0, 0, 4, 0, 0, 2, 0, 0, 0, 8, 8, 8, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 295, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 8, 0, 0, 1, 0, 4, 4, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6180, 0, 0, 0, 0, 729] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 8192, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 123, 2, 73, 8190, 0, 8186, 0, 0, 0, 2, 0, 7302, 0, 0, 0, 0, 61, 0, 0, 0, 1, 2321, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 2, 2, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 965, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5021, 2, 0, 0, 0, 513] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 6, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8135, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 2, 0, 0, 0, 0, 6801, 158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 0, 0, 24, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 4, 794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8737, 7031, 8190, 8036, 8154, 8193, 8990] avg=8192.0000 max=8990 max_rank=7 min=7031 min_rank=2 imbalance_ratio=1.097412 max_load_reduction=56546 max_load_reduction_pct=0.862823 moved_to_echo=33373 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 1, 1, 551, 0, 0, 0, 0, 0, 1, 0, 8004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 0, 0, 0, 0, 2330, 0, 1, 0, 0, 0, 0, 0, 6818, 158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 8030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5341, 3, 0, 0, 0, 170, 796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8573, 9805, 8192, 8055, 8205, 8190, 6311] avg=8192.0000 max=9805 max_rank=2 min=6311 min_rank=7 imbalance_ratio=1.196899 max_load_reduction=55731 max_load_reduction_pct=0.850388 moved_to_echo=30761 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 1, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 548, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 12, 0, 0, 4, 0, 0, 9, 0, 6784, 158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 88, 792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8703, 6968, 8190, 8037, 8198, 8193, 9071] avg=8192.0000 max=9071 max_rank=7 min=6968 min_rank=2 imbalance_ratio=1.107300 max_load_reduction=56465 max_load_reduction_pct=0.861588 moved_to_echo=33499 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8104, 4, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 1, 550, 0, 0, 0, 0, 0, 0, 0, 7921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 162, 0, 0, 5, 0, 0, 0, 2, 6815, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 3, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 163, 796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8124, 8513, 7148, 8190, 8029, 8195, 8188, 9149] avg=8192.0000 max=9149 max_rank=7 min=7148 min_rank=2 imbalance_ratio=1.116821 max_load_reduction=56387 max_load_reduction_pct=0.860397 moved_to_echo=33561 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 2, 73, 533, 0, 0, 0, 0, 0, 2, 0, 7302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 1, 2321, 0, 2, 0, 0, 0, 0, 0, 6822, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 2, 2, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 965, 8027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 2, 0, 0, 0, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5021, 2, 0, 0, 0, 513, 800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8035, 9366, 8187, 8992, 8209, 8201, 6336] avg=8192.0000 max=9366 max_rank=2 min=6336 min_rank=7 imbalance_ratio=1.143311 max_load_reduction=56170 max_load_reduction_pct=0.857086 moved_to_echo=31738 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 6809, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 29, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 795, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8742, 6984, 8190, 8035, 8199, 8194, 8987] avg=8192.0000 max=8987 max_rank=7 min=6984 min_rank=2 imbalance_ratio=1.097046 max_load_reduction=56549 max_load_reduction_pct=0.862869 moved_to_echo=33415 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 187, 7, 0, 549, 1, 0, 0, 0, 0, 8, 0, 4740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1139, 0, 0, 0, 0, 3086, 0, 8, 0, 0, 4, 0, 0, 6813, 158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8, 8, 8, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 295, 8022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 8, 0, 0, 1, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6180, 0, 0, 0, 0, 729, 796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 5492, 11208, 8207, 8319, 8206, 8193, 7705] avg=8192.0000 max=11208 max_rank=2 min=5492 min_rank=1 imbalance_ratio=1.368164 max_load_reduction=54328 max_load_reduction_pct=0.828979 moved_to_echo=32423 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 6, 0, 0, 1, 0, 6803, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8135, 0, 0, 22, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 795, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8741, 7026, 8190, 8033, 8170, 8184, 8987] avg=8192.0000 max=8987 max_rank=7 min=7026 min_rank=2 imbalance_ratio=1.097046 max_load_reduction=56549 max_load_reduction_pct=0.862869 moved_to_echo=33374 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 128, 0, 896, 0, 0, 0, 128, 9856, 0, 128, 128, 0, 128, 128, 128, 52992, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 57088, 128, 0, 0, 0, 256, 8320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 640, 128, 384, 4096, 128, 0, 0, 0, 0, 128, 0, 60672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2048, 63616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 65536, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 1792, 0, 0, 0, 128, 8064, 0, 128, 128, 0, 128, 128, 128, 54528, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 57600, 128, 0, 0, 0, 1792, 6400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 384, 128, 128, 4480, 128, 0, 0, 0, 0, 128, 0, 60800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 1408, 64256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 65408, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 128, 128, 128, 0, 0, 65536, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 65408, 0, 0, 128, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 65408, 0, 0, 128, 128, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 128, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7358, 0, 0, 8192, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 175, 0, 0, 0, 0, 0, 0, 0, 663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8189, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 8188, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8015, 0, 0, 8192, 7817, 0, 1, 2, 0, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 353, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7861, 0, 0, 8192, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8191, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 8183, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 362, 8183, 0, 0, 7722, 2046, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3929, 0, 0, 0, 0, 0, 7, 7, 0, 0, 8, 5, 0, 0, 2292, 0, 0, 8028, 9, 0, 0, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 8180, 0, 0, 4, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 12, 0, 0, 0, 3, 0, 0, 171, 0, 8181, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 201, 8190, 0, 0, 7959, 5408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 0, 0, 0, 0, 0, 2, 2, 0, 0, 2, 1, 0, 0, 1759, 0, 0, 8104, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 8189, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 89, 0, 8190, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6928, 0, 0, 8192, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 1271, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 8188, 0, 0, 7999, 6597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 747, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 1, 0, 0, 993, 0, 0, 8183, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8187, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 1, 11, 0, 8190, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7893, 0, 0, 8191, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1343, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6555, 615, 35, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7900, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7648, 9700, 7869, 7577, 8194, 8200, 8191, 8157] avg=8192.0000 max=9700 max_rank=1 min=7577 min_rank=3 imbalance_ratio=1.184082 max_load_reduction=55836 max_load_reduction_pct=0.851990 moved_to_echo=32742 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1343, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6555, 615, 35, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7900, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 9587, 7479, 7577, 8194, 8198, 8193, 8157] avg=8192.0000 max=9587 max_rank=1 min=7479 min_rank=2 imbalance_ratio=1.170288 max_load_reduction=55949 max_load_reduction_pct=0.853714 moved_to_echo=32742 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 290, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 362, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1266, 2046, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3929, 0, 0, 0, 0, 0, 7, 7, 0, 0, 8, 6179, 602, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 2292, 0, 0, 7426, 9, 0, 0, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 7892, 277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 4, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 12, 0, 0, 0, 3, 0, 0, 171, 0, 8146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8839, 3314, 10767, 9740, 8180, 8173, 8189, 8334] avg=8192.0000 max=10767 max_rank=2 min=3314 min_rank=1 imbalance_ratio=1.314331 max_load_reduction=54769 max_load_reduction_pct=0.835709 moved_to_echo=32876 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 290, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1343, 7817, 0, 1, 2, 0, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 353, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 6555, 615, 35, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 7576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 7897, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8306, 9259, 7560, 7672, 8194, 8193, 8190, 8162] avg=8192.0000 max=9259 max_rank=1 min=7560 min_rank=2 imbalance_ratio=1.130249 max_load_reduction=56277 max_load_reduction_pct=0.858719 moved_to_echo=32739 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1343, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1271, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6555, 614, 35, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7899, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7218, 9534, 8477, 7571, 8193, 8192, 8193, 8158] avg=8192.0000 max=9534 max_rank=1 min=7218 min_rank=0 imbalance_ratio=1.163818 max_load_reduction=56002 max_load_reduction_pct=0.854523 moved_to_echo=32736 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 290, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1311, 6597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 747, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 6401, 614, 35, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 993, 0, 0, 7569, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7899, 287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 1, 11, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8518, 7908, 7801, 8565, 8187, 8194, 8193, 8170] avg=8192.0000 max=8565 max_rank=3 min=7801 min_rank=2 imbalance_ratio=1.045532 max_load_reduction=56971 max_load_reduction_pct=0.869308 moved_to_echo=32744 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1343, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6554, 615, 35, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7900, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 9534, 7498, 7577, 8194, 8199, 8194, 8157] avg=8192.0000 max=9534 max_rank=1 min=7498 min_rank=2 imbalance_ratio=1.163818 max_load_reduction=56002 max_load_reduction_pct=0.854523 moved_to_echo=32744 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 201, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1292, 5408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 0, 0, 0, 0, 0, 2, 2, 0, 0, 2, 6374, 611, 42, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1759, 0, 0, 7493, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7904, 293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 89, 0, 8148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8673, 6700, 8085, 9257, 8200, 8187, 8193, 8241] avg=8192.0000 max=9257 max_rank=3 min=6700 min_rank=1 imbalance_ratio=1.130005 max_load_reduction=56279 max_load_reduction_pct=0.858749 moved_to_echo=32821 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 8191, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 8188, 0, 0, 7979, 7129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 273, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 1, 0, 0, 962, 0, 0, 8184, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8188, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 1, 16, 0, 8191, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7530, 0, 0, 8192, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 0, 0, 0, 0, 0, 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8190, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 8188, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7570, 0, 0, 8192, 8113, 0, 0, 2, 0, 0, 0, 0, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7220, 0, 0, 8192, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 196, 0, 0, 0, 0, 0, 0, 0, 776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8189, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7613, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5836, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 2302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 165, 8190, 0, 0, 7910, 6571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 0, 0, 0, 0, 0, 2, 2, 0, 0, 2, 1, 0, 0, 1606, 0, 0, 8122, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 8189, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 87, 0, 8190, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 8183, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 352, 8183, 0, 0, 7720, 3894, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2153, 0, 0, 0, 0, 0, 7, 8, 0, 0, 8, 5, 0, 0, 2212, 0, 0, 8047, 9, 0, 0, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 8180, 0, 0, 4, 2, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 13, 0, 0, 0, 3, 0, 0, 172, 0, 8181, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 841, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6758, 575, 36, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7607, 593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7803, 9223, 8146, 7617, 8200, 8200, 8191, 8156] avg=8192.0000 max=9223 max_rank=1 min=7617 min_rank=3 imbalance_ratio=1.125854 max_load_reduction=56313 max_load_reduction_pct=0.859268 moved_to_echo=32747 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 583, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 819, 7129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 273, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 6583, 574, 36, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 962, 0, 0, 7610, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7606, 577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 1, 16, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8799, 7948, 7470, 8575, 8184, 8192, 8193, 8175] avg=8192.0000 max=8799 max_rank=0 min=7470 min_rank=2 imbalance_ratio=1.074097 max_load_reduction=56737 max_load_reduction_pct=0.865738 moved_to_echo=32744 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 583, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 841, 8113, 0, 0, 2, 0, 0, 0, 0, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6758, 575, 36, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 7616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 7603, 593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8188, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 9070, 7947, 7619, 8198, 8195, 8191, 8162] avg=8192.0000 max=9070 max_rank=1 min=7619 min_rank=3 imbalance_ratio=1.107178 max_load_reduction=56466 max_load_reduction_pct=0.861603 moved_to_echo=32746 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 165, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 804, 6571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 0, 0, 0, 0, 0, 2, 2, 0, 0, 2, 6531, 572, 42, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1606, 0, 0, 7550, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7609, 575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 87, 0, 8148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8931, 7375, 7262, 9161, 8187, 8187, 8194, 8239] avg=8192.0000 max=9161 max_rank=3 min=7262 min_rank=2 imbalance_ratio=1.118286 max_load_reduction=56375 max_load_reduction_pct=0.860214 moved_to_echo=32807 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 841, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6758, 575, 36, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7606, 593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6419, 9089, 9672, 7614, 8199, 8193, 8193, 8157] avg=8192.0000 max=9672 max_rank=2 min=6419 min_rank=0 imbalance_ratio=1.180664 max_load_reduction=55864 max_load_reduction_pct=0.852417 moved_to_echo=32742 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 840, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6758, 575, 36, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7607, 593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 9034, 7940, 7617, 8200, 8199, 8194, 8156] avg=8192.0000 max=9034 max_rank=1 min=7617 min_rank=3 imbalance_ratio=1.102783 max_load_reduction=56502 max_load_reduction_pct=0.862152 moved_to_echo=32749 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 582, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 352, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 793, 3894, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2153, 0, 0, 0, 0, 0, 7, 8, 0, 0, 8, 6369, 565, 36, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 2212, 0, 0, 7482, 9, 0, 0, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 7599, 558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 4, 2, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 13, 0, 0, 0, 3, 0, 0, 172, 0, 8145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9121, 4688, 9147, 9716, 8168, 8173, 8188, 8335] avg=8192.0000 max=9716 max_rank=3 min=4688 min_rank=1 imbalance_ratio=1.186035 max_load_reduction=55820 max_load_reduction_pct=0.851746 moved_to_echo=32864 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 841, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6758, 575, 36, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7607, 593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8113, 9109, 7952, 7617, 8200, 8197, 8192, 8156] avg=8192.0000 max=9109 max_rank=1 min=7617 min_rank=3 imbalance_ratio=1.111938 max_load_reduction=56427 max_load_reduction_pct=0.861008 moved_to_echo=32745 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 0, 0, 0, 0, 8704, 128, 0, 0, 128, 128, 128, 128, 0, 0, 128, 51840, 4992, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 10624, 54656, 0, 128, 128, 0, 0, 128, 0, 0, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 2432, 0, 0, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 640, 62720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 63232, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 5248, 0, 0, 60416, 128, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 0, 0, 7424, 0, 0, 0, 0, 128, 128, 128, 0, 0, 128, 53376, 4608, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 6656, 58496, 0, 0, 128, 0, 0, 128, 0, 0, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 4736, 0, 0, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 640, 60416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 60928, 4736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 4864, 0, 0, 60800, 128, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 65536, 0, 0, 0, 128, 0, 128, 128, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 0, 0, 128, 0, 0, 0, 128, 0, 128, 384, 0, 65280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 65536, 0, 0, 0, 128, 0, 128, 128, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3848, 3749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 8183, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 612, 7871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1097, 1, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 6813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2066, 1365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 8186, 0, 0, 0, 0, 0, 1, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5831, 2981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1474, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 6313, 0, 0, 0, 0, 2, 0, 8182, 0, 0, 36, 0, 0, 0, 0, 0, 2, 0, 0, 77, 0, 436, 7935, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 7, 0, 0, 0, 0, 1, 0, 0, 0, 1478, 0, 61, 0, 0, 0, 0, 0, 0, 2271, 0, 0, 0, 0, 0, 0, 0, 7, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8044, 8, 0, 0, 0, 1, 5, 2, 0, 0, 0, 0, 1, 7, 0, 8117, 0, 0, 0, 0, 0, 4, 0, 0, 0, 64, 1, 0, 0, 0, 0, 6, 0, 6, 9, 6190, 0, 8068, 0, 0, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2933, 2276, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 7869, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1370, 6927, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 381, 2, 8, 0, 0, 0, 0, 0, 0, 653, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 8141, 0, 0, 0, 0, 0, 1, 0, 0, 0, 15, 0, 0, 0, 0, 0, 1, 0, 0, 1, 7400, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 7108, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 12, 0, 0, 0, 0, 0, 1, 0, 0, 12, 0, 430, 7863, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 852, 0, 3, 0, 0, 0, 0, 0, 0, 1257, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 2, 0, 0, 0, 0, 0, 108, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 2, 7034, 0, 8117, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2066, 1365, 0, 0, 0, 0, 0, 0, 0, 0, 869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7492, 0, 0, 0, 0, 0, 0, 5132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7786, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1276, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8598, 4300, 8191, 12624, 8192, 8195, 6750, 8686] avg=8192.0000 max=12624 max_rank=3 min=4300 min_rank=1 imbalance_ratio=1.541016 max_load_reduction=52912 max_load_reduction_pct=0.807373 moved_to_echo=31823 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2933, 2276, 0, 0, 0, 0, 0, 0, 0, 0, 870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4153, 0, 0, 0, 0, 0, 0, 5137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7786, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8598, 6079, 8196, 9290, 8192, 8195, 8300, 8686] avg=8192.0000 max=9290 max_rank=3 min=6079 min_rank=1 imbalance_ratio=1.134033 max_load_reduction=56246 max_load_reduction_pct=0.858246 moved_to_echo=33373 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8183, 0, 0, 0, 0, 0, 0, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 612, 7871, 0, 0, 0, 0, 0, 0, 0, 0, 870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1097, 1, 0, 0, 0, 0, 4, 5138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7783, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8589, 9353, 8195, 6240, 8189, 8196, 8091, 8683] avg=8192.0000 max=9353 max_rank=1 min=6240 min_rank=3 imbalance_ratio=1.141724 max_load_reduction=56183 max_load_reduction_pct=0.857285 moved_to_echo=33159 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 6313, 0, 0, 0, 0, 2, 0, 406, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 77, 0, 436, 7935, 7, 0, 0, 0, 0, 0, 0, 0, 857, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 7, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1478, 0, 61, 0, 0, 0, 0, 0, 0, 2271, 0, 0, 0, 0, 0, 5062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7776, 400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8044, 8, 0, 0, 0, 1, 5, 2, 0, 0, 0, 0, 1, 7, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 0, 0, 0, 0, 0, 4, 0, 0, 0, 64, 1, 0, 0, 0, 0, 1259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 6, 9, 6190, 0, 488, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6757, 9314, 8186, 8872, 8192, 8070, 9445, 6700] avg=8192.0000 max=9445 max_rank=6 min=6700 min_rank=7 imbalance_ratio=1.152954 max_load_reduction=56091 max_load_reduction_pct=0.855881 moved_to_echo=32407 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3848, 3749, 0, 0, 0, 0, 0, 0, 0, 0, 869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3376, 0, 0, 0, 0, 0, 0, 5133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7786, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8598, 8466, 8191, 8509, 8192, 8195, 6699, 8686] avg=8192.0000 max=8686 max_rank=7 min=6699 min_rank=6 imbalance_ratio=1.060303 max_load_reduction=56850 max_load_reduction_pct=0.867462 moved_to_echo=31772 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8186, 0, 0, 0, 0, 0, 1, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5831, 2981, 0, 0, 0, 0, 0, 0, 0, 0, 869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1474, 0, 0, 0, 0, 0, 2, 5135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7784, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8593, 9681, 8201, 6611, 8191, 8196, 7376, 8687] avg=8192.0000 max=9681 max_rank=1 min=6611 min_rank=3 imbalance_ratio=1.181763 max_load_reduction=55855 max_load_reduction_pct=0.852280 moved_to_echo=32450 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 7869, 0, 0, 0, 0, 0, 0, 406, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1370, 6927, 1, 0, 0, 0, 0, 0, 0, 0, 870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 381, 2, 8, 0, 0, 0, 0, 0, 0, 653, 0, 0, 0, 0, 0, 5138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7785, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8141, 0, 0, 0, 0, 0, 1, 0, 0, 0, 15, 0, 0, 0, 0, 0, 1278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 7400, 0, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8278, 9169, 8189, 6182, 8193, 8193, 9435, 7897] avg=8192.0000 max=9435 max_rank=6 min=6182 min_rank=3 imbalance_ratio=1.151733 max_load_reduction=56101 max_load_reduction_pct=0.856033 moved_to_echo=33718 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 7108, 0, 0, 0, 0, 0, 0, 405, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 12, 0, 430, 7863, 2, 0, 0, 0, 0, 0, 0, 0, 866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 852, 0, 3, 0, 0, 0, 0, 0, 0, 1257, 0, 0, 0, 0, 0, 5096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7785, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 2, 0, 0, 0, 0, 0, 108, 0, 0, 0, 0, 0, 0, 0, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 2, 7034, 0, 472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7525, 9174, 8187, 7208, 8195, 8296, 9440, 7511] avg=8192.0000 max=9440 max_rank=6 min=7208 min_rank=3 imbalance_ratio=1.152344 max_load_reduction=56096 max_load_reduction_pct=0.855957 moved_to_echo=33442 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 7197, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3805, 7567, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 723, 2, 9, 0, 0, 0, 0, 0, 0, 760, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 1, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 1, 0, 1, 0, 0, 0, 13, 0, 0, 0, 0, 0, 1, 0, 0, 1, 7497, 0, 5181, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 3292, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 6494, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 12, 0, 0, 0, 0, 0, 1, 0, 0, 14, 0, 2523, 7954, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 907, 0, 3, 0, 0, 0, 0, 0, 0, 1403, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 2, 0, 0, 0, 0, 0, 220, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 2, 7144, 0, 6103, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 5045, 0, 0, 0, 0, 2, 0, 8182, 0, 0, 37, 0, 0, 0, 0, 0, 3, 0, 0, 84, 0, 2183, 8140, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 1605, 0, 61, 0, 0, 0, 0, 0, 0, 2449, 0, 0, 0, 0, 0, 0, 0, 7, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7966, 9, 0, 0, 0, 1, 4, 62, 0, 0, 0, 0, 1, 7, 0, 8177, 0, 0, 0, 0, 0, 4, 0, 0, 0, 68, 11, 1, 0, 0, 0, 5, 0, 6, 9, 6663, 0, 6541, 0, 0, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 8130, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2968, 6779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4378, 1, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 7361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 3139, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 8182, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6453, 2243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4818, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 7574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 3475, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6628, 2724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1740, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6637, 767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7732, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1569, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7942, 0, 0, 0, 0, 0, 0, 2886, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7459, 770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 159, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4536, 0, 3292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8925, 5486, 8188, 11028, 8229, 8183, 7669, 7828] avg=8192.0000 max=11028 max_rank=3 min=5486 min_rank=1 imbalance_ratio=1.346191 max_load_reduction=54508 max_load_reduction_pct=0.831726 moved_to_echo=31909 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 7197, 0, 0, 0, 0, 0, 0, 733, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3805, 6013, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 723, 2, 9, 0, 0, 0, 0, 0, 0, 760, 0, 0, 0, 0, 0, 2641, 865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7458, 704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 1, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 1, 0, 1, 0, 0, 0, 13, 0, 0, 0, 0, 0, 689, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 4152, 0, 5181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7933, 9820, 8187, 5000, 8164, 8190, 8907, 9335] avg=8192.0000 max=9820 max_rank=1 min=5000 min_rank=3 imbalance_ratio=1.198730 max_load_reduction=55716 max_load_reduction_pct=0.850159 moved_to_echo=34596 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8130, 0, 0, 0, 0, 0, 0, 732, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2968, 5387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4378, 1, 0, 0, 0, 0, 4, 2886, 775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7455, 770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 617, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4535, 0, 3139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8863, 8355, 8200, 8044, 8225, 8184, 7991, 7674] avg=8192.0000 max=8863 max_rank=0 min=7674 min_rank=7 imbalance_ratio=1.081909 max_load_reduction=56673 max_load_reduction_pct=0.864761 moved_to_echo=32074 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 6494, 0, 0, 0, 0, 0, 0, 728, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 14, 0, 2523, 6313, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 907, 0, 3, 0, 0, 0, 0, 0, 0, 1403, 0, 0, 0, 0, 0, 2522, 914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7462, 675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 2, 0, 0, 0, 0, 0, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 727, 10, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 2, 3947, 0, 6103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7234, 8853, 8183, 5749, 8141, 8386, 8935, 10055] avg=8192.0000 max=10055 max_rank=7 min=5749 min_rank=3 imbalance_ratio=1.227417 max_load_reduction=55481 max_load_reduction_pct=0.846573 moved_to_echo=35517 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8182, 0, 0, 0, 0, 0, 0, 733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6453, 1783, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4818, 0, 0, 0, 0, 0, 2, 2886, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7456, 770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 204, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4536, 0, 3475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8915, 8236, 8208, 7963, 8227, 8185, 7791, 8011] avg=8192.0000 max=8915 max_rank=0 min=7791 min_rank=6 imbalance_ratio=1.088257 max_load_reduction=56621 max_load_reduction_pct=0.863968 moved_to_echo=32214 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 5045, 0, 0, 0, 0, 2, 0, 732, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 84, 0, 2183, 6468, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1605, 0, 61, 0, 0, 0, 0, 0, 0, 2449, 0, 0, 0, 0, 0, 2347, 931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7450, 626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7958, 9, 0, 0, 0, 1, 4, 62, 0, 0, 0, 0, 1, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 4, 0, 0, 0, 68, 11, 1, 0, 0, 0, 741, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 6, 9, 3690, 0, 6541, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5816, 8745, 8182, 7393, 8092, 8042, 9014, 10252] avg=8192.0000 max=10252 max_rank=7 min=5816 min_rank=0 imbalance_ratio=1.251465 max_load_reduction=55284 max_load_reduction_pct=0.843567 moved_to_echo=35400 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6628, 2165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6457, 0, 0, 0, 0, 0, 0, 2886, 311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7459, 770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 248, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4536, 0, 1740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8925, 8793, 8188, 9654, 8229, 8183, 7288, 6276] avg=8192.0000 max=9654 max_rank=3 min=6276 min_rank=7 imbalance_ratio=1.178467 max_load_reduction=55882 max_load_reduction_pct=0.852692 moved_to_echo=29976 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6637, 611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7732, 0, 0, 0, 0, 0, 0, 2886, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7459, 770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4536, 0, 1569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8925, 7248, 8200, 10705, 8229, 8183, 7941, 6105] avg=8192.0000 max=10705 max_rank=3 min=6105 min_rank=7 imbalance_ratio=1.306763 max_load_reduction=54831 max_load_reduction_pct=0.836655 moved_to_echo=30458 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 2816, 128, 128, 0, 0, 0, 0, 0, 17664, 4224, 0, 0, 0, 0, 128, 41088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 128, 0, 17536, 41088, 128, 0, 0, 0, 0, 0, 0, 0, 7040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[55296, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 10240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 61696, 0, 3968, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 62336, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 62336, 0, 0, 0, 0, 128, 128, 3328, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 65408, 128, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 65536, 0, 0, 0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 3328, 128, 128, 0, 0, 0, 0, 0, 31360, 4736, 0, 0, 0, 0, 128, 22016, 4352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 128, 0, 35328, 30208, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 34560, 0, 31104, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 59776, 5888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 59648, 0, 0, 0, 0, 128, 0, 5888, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[61952, 0, 0, 0, 128, 0, 128, 0, 0, 0, 128, 128, 128, 0, 0, 0, 3456, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 65536, 0, 128, 0, 128, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 65280, 128, 0, 128, 0, 128, 128, 384, 0, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6772, 8165, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8146, 27, 7, 0, 8182, 0, 8143, 1, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7876, 0, 0, 0, 0, 0, 7, 0, 4, 0, 12, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 1, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1804, 0, 4, 10, 8179, 0, 0, 0, 10, 0, 2, 2, 10, 0, 0, 75, 0, 0, 0, 33, 0, 0, 0, 10, 6, 0, 0, 0, 0, 7899, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8184, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 8147, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8191, 0, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 48, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6664, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 2, 0, 0, 8191, 0, 8186, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7998, 0, 0, 0, 0, 0, 1, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1727, 0, 1, 1, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8154, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8186, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6421, 8185, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8180, 8, 2, 0, 8190, 0, 8182, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1821, 0, 0, 2, 8189, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 8073, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7616, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 8188, 0, 8192, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 573, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7522, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 7425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8854, 8192, 8266, 8192, 8166, 8192, 7433, 8241] avg=8192.0000 max=8854 max_rank=0 min=7433 min_rank=6 imbalance_ratio=1.080811 max_load_reduction=56682 max_load_reduction_pct=0.864899 moved_to_echo=32032 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6772, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 7, 0, 0, 0, 8143, 1, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7876, 0, 0, 0, 0, 0, 7, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 12, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 1, 0, 0, 41, 0, 0, 0, 0, 0, 0, 7480, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8182, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1804, 0, 4, 10, 7414, 0, 0, 0, 10, 0, 2, 2, 10, 0, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 10, 6, 0, 0, 0, 0, 7899, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7443, 8186, 7956, 8185, 8247, 8191, 9331, 7997] avg=8192.0000 max=9331 max_rank=6 min=7443 min_rank=0 imbalance_ratio=1.139038 max_load_reduction=56205 max_load_reduction_pct=0.857620 moved_to_echo=33766 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7522, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8861, 8192, 8266, 8191, 8166, 8193, 7426, 8241] avg=8192.0000 max=8861 max_rank=0 min=7426 min_rank=6 imbalance_ratio=1.081665 max_load_reduction=56675 max_load_reduction_pct=0.864792 moved_to_echo=32026 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 8147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7521, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 7424, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8818, 8192, 8260, 8193, 8165, 8194, 7473, 8241] avg=8192.0000 max=8818 max_rank=0 min=7473 min_rank=6 imbalance_ratio=1.076416 max_load_reduction=56718 max_load_reduction_pct=0.865448 moved_to_echo=32073 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7520, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 573, 0, 0, 0, 7419, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8285, 8200, 8264, 8198, 8164, 8189, 7995, 8241] avg=8192.0000 max=8285 max_rank=0 min=7995 min_rank=6 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32589 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8186, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7998, 0, 0, 0, 0, 0, 1, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 7516, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1727, 0, 1, 1, 7424, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8154, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7333, 8189, 8073, 8196, 8167, 8193, 9181, 8204] avg=8192.0000 max=9181 max_rank=6 min=7333 min_rank=0 imbalance_ratio=1.120728 max_load_reduction=56355 max_load_reduction_pct=0.859909 moved_to_echo=33745 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7522, 644, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 7425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8856, 8192, 8259, 8191, 8167, 8192, 7438, 8241] avg=8192.0000 max=8856 max_rank=0 min=7438 min_rank=6 imbalance_ratio=1.081055 max_load_reduction=56680 max_load_reduction_pct=0.864868 moved_to_echo=32038 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6421, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 2, 0, 0, 0, 8180, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 0, 0, 0, 0, 2, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 109, 0, 0, 0, 0, 0, 0, 7516, 646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1821, 0, 0, 2, 7416, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 8073, 0, 0, 51, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7086, 8193, 8192, 8190, 8294, 8192, 9259, 8130] avg=8192.0000 max=9259 max_rank=6 min=7086 min_rank=0 imbalance_ratio=1.130249 max_load_reduction=56277 max_load_reduction_pct=0.858719 moved_to_echo=33875 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5420, 8162, 0, 0, 0, 0, 0, 0, 0, 6, 0, 8146, 30, 7, 0, 8182, 0, 8141, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7804, 0, 0, 0, 0, 0, 7, 0, 4, 0, 18, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 1, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3140, 0, 4, 10, 8178, 0, 0, 0, 10, 0, 2, 2, 13, 0, 0, 52, 0, 0, 0, 35, 0, 0, 0, 10, 6, 0, 0, 0, 0, 7980, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8136, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4694, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 2, 1, 0, 8191, 0, 8187, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7966, 0, 0, 0, 0, 0, 1, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3723, 0, 1, 1, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8163, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5464, 8184, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8181, 8, 2, 0, 8190, 0, 8182, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8102, 0, 0, 0, 0, 0, 2, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2773, 0, 0, 2, 8189, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 8081, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8115, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7894, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8191, 0, 8191, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 377, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8158, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6753, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8188, 0, 8192, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1433, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6828, 1333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 6715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9500, 8192, 8300, 8191, 8161, 8193, 6771, 8228] avg=8192.0000 max=9500 max_rank=0 min=6771 min_rank=6 imbalance_ratio=1.159668 max_load_reduction=56036 max_load_reduction_pct=0.855042 moved_to_echo=31353 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 8187, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7966, 0, 0, 0, 0, 0, 1, 0, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 6822, 1333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3723, 0, 1, 1, 6714, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8163, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6057, 8191, 8075, 8195, 8161, 8193, 10464, 8200] avg=8192.0000 max=10464 max_rank=6 min=6057 min_rank=0 imbalance_ratio=1.277344 max_load_reduction=55072 max_load_reduction_pct=0.840332 moved_to_echo=35018 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5420, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 7, 0, 0, 0, 8141, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7804, 0, 0, 0, 0, 0, 7, 0, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 18, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 1, 0, 0, 62, 0, 0, 0, 0, 0, 0, 6790, 1331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8182, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3140, 0, 4, 10, 6705, 0, 0, 0, 10, 0, 2, 2, 13, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 10, 6, 0, 0, 0, 0, 7980, 1, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6782, 8186, 7918, 8188, 8266, 8191, 9938, 8067] avg=8192.0000 max=9938 max_rank=6 min=6782 min_rank=0 imbalance_ratio=1.213135 max_load_reduction=55598 max_load_reduction_pct=0.848358 moved_to_echo=34462 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5464, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 2, 0, 0, 0, 8175, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8102, 0, 0, 0, 0, 0, 2, 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 0, 0, 0, 0, 6825, 1339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2773, 0, 0, 2, 6702, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 8081, 0, 0, 38, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6821, 8188, 8214, 8190, 8304, 8192, 9497, 8130] avg=8192.0000 max=9497 max_rank=6 min=6821 min_rank=0 imbalance_ratio=1.159302 max_load_reduction=56039 max_load_reduction_pct=0.855087 moved_to_echo=34123 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6827, 1332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1433, 0, 0, 0, 6709, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8117, 8200, 8299, 8196, 8161, 8189, 8147, 8227] avg=8192.0000 max=8299 max_rank=2 min=8117 min_rank=0 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32724 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8109, 0, 0, 0, 0, 0, 0, 0, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6828, 1333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 377, 0, 0, 0, 6714, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9258, 8194, 8217, 8192, 8161, 8194, 7092, 8228] avg=8192.0000 max=9258 max_rank=0 min=7092 min_rank=6 imbalance_ratio=1.130127 max_load_reduction=56278 max_load_reduction_pct=0.858734 moved_to_echo=31675 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8109, 0, 0, 0, 0, 0, 0, 0, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6828, 1333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 0, 0, 0, 6715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9522, 8192, 8217, 8192, 8161, 8192, 6832, 8228] avg=8192.0000 max=9522 max_rank=0 min=6832 min_rank=6 imbalance_ratio=1.162354 max_load_reduction=56014 max_load_reduction_pct=0.854706 moved_to_echo=31413 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6828, 1333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 6715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9479, 8193, 8296, 8192, 8161, 8192, 6795, 8228] avg=8192.0000 max=9479 max_rank=0 min=6795 min_rank=6 imbalance_ratio=1.157104 max_load_reduction=56057 max_load_reduction_pct=0.855362 moved_to_echo=31376 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 60288, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 5376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 0, 0, 128, 128, 0, 0, 256, 0, 0, 128, 0, 0, 0, 60160, 5248, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[6016, 0, 128, 128, 59392, 0, 0, 0, 128, 0, 128, 128, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 65152, 0, 0, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 65024, 0, 0, 0, 0, 0, 128, 0, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 128, 128, 128, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 0, 128, 128, 0, 0, 256, 0, 0, 128, 0, 0, 0, 54656, 10752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 54656, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 11008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[11776, 0, 128, 128, 53760, 0, 0, 0, 128, 0, 128, 128, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 64768, 0, 0, 0, 0, 0, 128, 0, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 65280, 128, 0, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 0, 0, 65536, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8180, 0, 8192, 0, 0, 2, 0, 0, 0, 8188, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8186, 0, 8191, 0, 0, 4, 0, 0, 0, 8178, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 12, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8191, 3, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 8090, 0, 0, 0, 0, 7969, 0, 8190, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 8189, 0, 0, 218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 8189, 0, 0, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 90, 0, 0, 0, 0, 12, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 7, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 21, 0, 0, 8179, 0, 0, 0, 13, 0, 0, 96, 0, 429, 0, 0, 2, 0, 0, 0, 7940, 0, 0, 0, 0, 7262, 0, 8132, 0, 0, 0, 5, 5, 0, 8178, 0, 5, 0, 0, 8143, 0, 0, 185, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 14, 7, 0, 0, 0, 0, 0, 0, 0, 7, 8113, 0, 9, 8181, 0, 0, 5, 2, 2, 0, 0, 0, 470, 0, 0, 0, 3, 75, 0, 0, 7, 0, 0, 0, 0, 8, 0, 8, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8178, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 12, 0, 1, 0, 0, 0, 0, 0, 0, 8072, 0, 0, 0, 0, 7955, 0, 8183, 0, 0, 0, 1, 0, 0, 8189, 0, 0, 0, 0, 8187, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 35, 2, 0, 0, 0, 0, 0, 0, 0, 2, 8178, 0, 0, 8189, 0, 0, 2, 1, 0, 0, 0, 0, 209, 0, 0, 0, 0, 61, 0, 0, 2, 0, 0, 0, 0, 2, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 8191, 0, 8192, 0, 0, 1, 0, 0, 0, 8191, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 108, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8141, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8076, 51, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8190, 8302, 8143, 8187, 8196, 8131, 8191] avg=8192.0000 max=8302 max_rank=2 min=8131 min_rank=6 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32705 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1, 0, 0, 0, 0, 0, 108, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8076, 51, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8191, 8297, 8141, 8185, 8200, 8131, 8196] avg=8192.0000 max=8297 max_rank=2 min=8131 min_rank=6 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32712 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8139, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 8175, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 429, 0, 0, 2, 0, 0, 0, 105, 0, 0, 0, 0, 7262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8082, 0, 0, 0, 5, 5, 0, 0, 0, 5, 0, 0, 0, 0, 0, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 14, 7, 0, 0, 0, 0, 8166, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8113, 0, 9, 8, 0, 0, 5, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 0, 0, 0, 3, 75, 0, 0, 7, 0, 0, 0, 0, 8, 0, 8, 7827, 50, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8178, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8162, 8209, 7894, 8282, 8201, 8146, 8452, 8190] avg=8192.0000 max=8452 max_rank=6 min=7894 min_rank=2 imbalance_ratio=1.031738 max_load_reduction=57084 max_load_reduction_pct=0.871033 moved_to_echo=32989 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 108, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 12, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8076, 51, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8190, 8295, 8144, 8183, 8212, 8131, 8181] avg=8192.0000 max=8295 max_rank=2 min=8131 min_rank=6 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32707 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8187, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 107, 0, 0, 0, 0, 7969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8176, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8189, 0, 0, 8, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 0, 12, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 7975, 51, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8192, 8081, 8357, 8185, 8199, 8135, 8194] avg=8192.0000 max=8357 max_rank=3 min=8081 min_rank=2 imbalance_ratio=1.020142 max_load_reduction=57179 max_load_reduction_pct=0.872482 moved_to_echo=32713 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8141, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8074, 51, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8190, 8299, 8143, 8185, 8200, 8129, 8195] avg=8192.0000 max=8299 max_rank=2 min=8129 min_rank=6 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32709 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 108, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8076, 51, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8188, 8301, 8141, 8185, 8200, 8131, 8194] avg=8192.0000 max=8301 max_rank=2 min=8131 min_rank=6 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32710 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 1, 0, 0, 0, 0, 0, 0, 99, 0, 0, 0, 0, 7955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8131, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 35, 2, 0, 0, 0, 0, 8176, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8178, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 209, 0, 0, 0, 0, 61, 0, 0, 2, 0, 0, 0, 0, 2, 0, 2, 7962, 52, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8189, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8186, 8067, 8185, 8225, 8183, 8296, 8195] avg=8192.0000 max=8296 max_rank=6 min=8067 min_rank=2 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32899 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8181, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8191, 3, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8077, 0, 0, 0, 0, 7990, 0, 8190, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 8189, 0, 0, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 8189, 0, 0, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 164, 0, 0, 0, 0, 14, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 11, 0, 4, 0, 0, 0, 0, 0, 0, 8072, 0, 0, 0, 0, 7872, 0, 8184, 0, 0, 0, 1, 0, 0, 8189, 0, 0, 0, 0, 8185, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 11, 2, 0, 0, 0, 0, 0, 0, 0, 2, 8180, 0, 0, 8189, 0, 0, 2, 1, 0, 0, 0, 0, 311, 0, 0, 0, 0, 53, 0, 0, 2, 0, 0, 0, 0, 2, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8189, 0, 8192, 0, 0, 1, 0, 0, 0, 8190, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8185, 0, 8192, 0, 0, 4, 0, 0, 0, 8163, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 7, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 7, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 20, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 90, 0, 287, 0, 0, 2, 0, 0, 0, 7942, 0, 0, 0, 0, 7181, 0, 8134, 0, 0, 0, 5, 5, 0, 8178, 0, 5, 0, 0, 8136, 0, 0, 182, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 6, 7, 0, 0, 0, 0, 0, 0, 0, 7, 8119, 0, 10, 8181, 0, 0, 5, 2, 0, 0, 0, 0, 712, 0, 0, 0, 4, 82, 0, 0, 8, 0, 0, 0, 0, 9, 0, 8, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8180, 0, 8192, 0, 0, 1, 0, 0, 0, 8189, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 146, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8031, 42, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8193, 8334, 8150, 8189, 8200, 8076, 8198] avg=8192.0000 max=8334 max_rank=2 min=8076 min_rank=6 imbalance_ratio=1.017334 max_load_reduction=57202 max_load_reduction_pct=0.872833 moved_to_echo=32663 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8188, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 0, 0, 0, 7990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8173, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8189, 0, 0, 8, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 0, 0, 0, 0, 14, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 7918, 42, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8193, 8136, 8288, 8189, 8199, 8144, 8193] avg=8192.0000 max=8288 max_rank=3 min=8136 min_rank=2 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32725 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8150, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 7, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8031, 42, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8194, 8331, 8154, 8186, 8207, 8094, 8169] avg=8192.0000 max=8331 max_rank=2 min=8094 min_rank=6 imbalance_ratio=1.016968 max_load_reduction=57205 max_load_reduction_pct=0.872879 moved_to_echo=32656 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 4, 0, 0, 0, 0, 0, 0, 135, 0, 0, 0, 0, 7872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8138, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 11, 2, 0, 0, 0, 0, 8172, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8180, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 311, 0, 0, 0, 0, 53, 0, 0, 2, 0, 0, 0, 0, 2, 0, 2, 7918, 46, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8189, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8185, 8022, 8202, 8205, 8185, 8341, 8197] avg=8192.0000 max=8341 max_rank=6 min=8022 min_rank=2 imbalance_ratio=1.018188 max_load_reduction=57195 max_load_reduction_pct=0.872726 moved_to_echo=32928 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8150, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8030, 42, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8195, 8335, 8152, 8187, 8200, 8075, 8196] avg=8192.0000 max=8335 max_rank=2 min=8075 min_rank=6 imbalance_ratio=1.017456 max_load_reduction=57201 max_load_reduction_pct=0.872818 moved_to_echo=32658 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8031, 42, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8189, 8338, 8150, 8189, 8200, 8076, 8197] avg=8192.0000 max=8338 max_rank=2 min=8076 min_rank=6 imbalance_ratio=1.017822 max_load_reduction=57198 max_load_reduction_pct=0.872772 moved_to_echo=32662 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8130, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 287, 0, 0, 2, 0, 0, 0, 141, 0, 0, 0, 0, 7181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8092, 0, 0, 0, 5, 5, 0, 0, 0, 5, 0, 0, 0, 0, 0, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 6, 7, 0, 0, 0, 0, 8163, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8119, 0, 10, 8, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 712, 0, 0, 0, 4, 82, 0, 0, 8, 0, 0, 0, 0, 9, 0, 8, 7786, 42, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8178, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8156, 8196, 7701, 8289, 8197, 8151, 8654, 8192] avg=8192.0000 max=8654 max_rank=6 min=7701 min_rank=2 imbalance_ratio=1.056396 max_load_reduction=56882 max_load_reduction_pct=0.867950 moved_to_echo=33194 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 146, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8150, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8031, 42, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8191, 8339, 8151, 8194, 8194, 8076, 8194] avg=8192.0000 max=8339 max_rank=2 min=8076 min_rank=6 imbalance_ratio=1.017944 max_load_reduction=57197 max_load_reduction_pct=0.872757 moved_to_echo=32658 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 512, 128, 128, 128, 0, 0, 0, 896, 0, 0, 0, 0, 64128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[1280, 0, 0, 0, 128, 256, 0, 0, 128, 0, 0, 0, 0, 128, 0, 128, 63872, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 0, 384, 0, 0, 128, 0, 0, 0, 1152, 0, 0, 0, 0, 64000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[896, 0, 0, 0, 128, 256, 0, 0, 128, 0, 0, 0, 0, 128, 0, 128, 64256, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 0, 128, 0, 128, 128, 0, 128, 128, 0, 0, 0, 0, 65408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 65536, 0, 128, 128, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 128, 0, 65536, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[65152, 0, 0, 128, 128, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 0, 128, 0, 128, 128, 0, 128, 128, 0, 0, 0, 0, 65408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 65536, 0, 128, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 9, 0, 0, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1758, 6473, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 8148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 3119, 5079, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8191, 0, 0, 62, 0, 0, 1, 2, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 6657, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 819, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 2098, 6679, 0, 0, 0, 1, 0, 0, 0, 0, 8191, 0, 1, 0, 0, 0, 0, 8171, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8135, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 8145, 0, 0, 510, 0, 0, 9, 23, 481, 0, 0, 0, 11, 0, 0, 0, 0, 0, 3, 0, 0, 2, 0, 7, 0, 9, 0, 0, 5377, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 5, 0, 0, 2648, 0, 0, 0, 0, 0, 0, 8176, 116, 0, 993, 6872, 1, 0, 4, 8, 0, 1, 0, 0, 8166, 0, 8, 0, 5, 0, 0, 8049, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 0, 5, 5, 0, 0, 7638, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 2056, 6182, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8146, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 3615, 4583, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1616, 6626, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8186, 0, 0, 197, 0, 0, 2, 3, 398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 2, 0, 0, 7391, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 709, 0, 0, 0, 0, 0, 0, 8189, 16, 0, 536, 7351, 1, 0, 0, 2, 0, 2, 0, 0, 8188, 0, 2, 0, 0, 0, 0, 8033, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8116, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7970, 0, 0, 9, 0, 0, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 96, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1758, 6456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 28, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7984, 8188, 8825, 7656, 8214, 8195, 8281, 8193] avg=8192.0000 max=8825 max_rank=2 min=7656 min_rank=3 imbalance_ratio=1.077271 max_load_reduction=56711 max_load_reduction_pct=0.865341 moved_to_echo=32883 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 96, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1616, 6609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8142, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7970, 8191, 8825, 7656, 8225, 8235, 8240, 8194] avg=8192.0000 max=8825 max_rank=2 min=7656 min_rank=3 imbalance_ratio=1.077271 max_load_reduction=56711 max_load_reduction_pct=0.865341 moved_to_echo=32894 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 96, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2056, 6166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8146, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7970, 8191, 8825, 7656, 8222, 8234, 8244, 8194] avg=8192.0000 max=8825 max_rank=2 min=7656 min_rank=3 imbalance_ratio=1.077271 max_load_reduction=56711 max_load_reduction_pct=0.865341 moved_to_echo=32894 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 7924, 0, 0, 510, 0, 0, 9, 23, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 3, 0, 0, 2, 0, 7, 0, 9, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5377, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 535, 95, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 2648, 0, 0, 0, 0, 0, 0, 7641, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 993, 6854, 1, 0, 4, 8, 0, 1, 0, 0, 0, 0, 8, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8049, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 0, 5, 5, 0, 0, 7638, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8953, 8197, 6070, 10410, 7874, 8103, 7751, 8178] avg=8192.0000 max=10410 max_rank=3 min=6070 min_rank=2 imbalance_ratio=1.270752 max_load_reduction=55126 max_load_reduction_pct=0.841156 moved_to_echo=31906 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 536, 96, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3119, 5067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7971, 8193, 8826, 7656, 8186, 8230, 8282, 8192] avg=8192.0000 max=8826 max_rank=2 min=7656 min_rank=3 imbalance_ratio=1.077393 max_load_reduction=56710 max_load_reduction_pct=0.865326 moved_to_echo=32890 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7969, 0, 0, 62, 0, 0, 1, 2, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6657, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 536, 96, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 819, 0, 0, 0, 0, 0, 0, 7654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2098, 6662, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 28, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8135, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8193, 7292, 8473, 8762, 8222, 8233, 8194] avg=8192.0000 max=8762 max_rank=4 min=7292 min_rank=2 imbalance_ratio=1.069580 max_load_reduction=56774 max_load_reduction_pct=0.866302 moved_to_echo=33411 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 96, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3615, 4572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7970, 8191, 8825, 7656, 8187, 8230, 8284, 8193] avg=8192.0000 max=8825 max_rank=2 min=7656 min_rank=3 imbalance_ratio=1.077271 max_load_reduction=56711 max_load_reduction_pct=0.865341 moved_to_echo=32894 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7951, 0, 0, 197, 0, 0, 2, 3, 398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 2, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7391, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 541, 100, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 709, 0, 0, 0, 0, 0, 0, 7648, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 7323, 1, 0, 0, 2, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8033, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8116, 0, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8551, 8192, 8048, 8373, 7866, 8087, 8221, 8198] avg=8192.0000 max=8551 max_rank=0 min=7866 min_rank=4 imbalance_ratio=1.043823 max_load_reduction=56985 max_load_reduction_pct=0.869522 moved_to_echo=32372 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1166, 7069, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 8, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1199, 7024, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8186, 0, 0, 228, 0, 0, 2, 5, 408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 1, 2, 0, 0, 7128, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1107, 0, 0, 0, 0, 0, 0, 8189, 15, 0, 307, 7433, 1, 0, 0, 2, 0, 2, 0, 0, 8188, 0, 2, 0, 0, 0, 0, 8003, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8115, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8191, 0, 0, 78, 0, 0, 1, 8, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 6474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 877, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 1718, 7176, 0, 0, 0, 1, 0, 0, 0, 0, 8191, 0, 1, 0, 0, 0, 0, 8137, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1549, 6655, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 8139, 0, 0, 550, 0, 0, 9, 34, 499, 0, 0, 0, 12, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 7, 0, 9, 0, 0, 5169, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 5, 0, 0, 2855, 0, 0, 0, 0, 0, 0, 8176, 107, 0, 772, 7093, 1, 0, 4, 9, 0, 1, 0, 0, 8166, 0, 8, 0, 6, 0, 0, 8019, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 0, 5, 5, 0, 0, 7605, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1397, 6843, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 2694, 5503, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 618, 95, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1166, 7057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7954, 8191, 8906, 7574, 8223, 8245, 8249, 8194] avg=8192.0000 max=8906 max_rank=2 min=7574 min_rank=3 imbalance_ratio=1.087158 max_load_reduction=56630 max_load_reduction_pct=0.864105 moved_to_echo=32911 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7954, 0, 0, 8, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 618, 95, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1199, 7012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 43, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7965, 8190, 8906, 7574, 8211, 8214, 8283, 8193] avg=8192.0000 max=8906 max_rank=2 min=7574 min_rank=3 imbalance_ratio=1.087158 max_load_reduction=56630 max_load_reduction_pct=0.864105 moved_to_echo=32901 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7953, 0, 0, 78, 0, 0, 1, 8, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 618, 95, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 877, 0, 0, 0, 0, 0, 0, 7573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1718, 7164, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 43, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8192, 7190, 8450, 8884, 8197, 8251, 8194] avg=8192.0000 max=8884 max_rank=4 min=7190 min_rank=2 imbalance_ratio=1.084473 max_load_reduction=56652 max_load_reduction_pct=0.864441 moved_to_echo=33526 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7935, 0, 0, 228, 0, 0, 2, 5, 408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 1, 2, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7128, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 621, 96, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1107, 0, 0, 0, 0, 0, 0, 7568, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 307, 7413, 1, 0, 0, 2, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8003, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8115, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8578, 8193, 7861, 8690, 7727, 8068, 8222, 8197] avg=8192.0000 max=8690 max_rank=3 min=7727 min_rank=4 imbalance_ratio=1.060791 max_load_reduction=56846 max_load_reduction_pct=0.867401 moved_to_echo=32214 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 7902, 0, 0, 550, 0, 0, 9, 34, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 7, 0, 9, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5169, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 617, 94, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 2855, 0, 0, 0, 0, 0, 0, 7559, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 772, 7081, 1, 0, 4, 9, 0, 1, 0, 0, 0, 0, 8, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8019, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 0, 5, 5, 0, 0, 7605, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8998, 8197, 5953, 10526, 7882, 8082, 7720, 8178] avg=8192.0000 max=10526 max_rank=3 min=5953 min_rank=2 imbalance_ratio=1.284912 max_load_reduction=55010 max_load_reduction_pct=0.839386 moved_to_echo=31862 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 618, 95, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1549, 6644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 43, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7955, 8191, 8908, 7574, 8193, 8242, 8280, 8193] avg=8192.0000 max=8908 max_rank=2 min=7574 min_rank=3 imbalance_ratio=1.087402 max_load_reduction=56628 max_load_reduction_pct=0.864075 moved_to_echo=32908 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 618, 95, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2694, 5494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7954, 8191, 8906, 7574, 8188, 8243, 8287, 8193] avg=8192.0000 max=8906 max_rank=2 min=7574 min_rank=3 imbalance_ratio=1.087158 max_load_reduction=56630 max_load_reduction_pct=0.864105 moved_to_echo=32911 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 618, 95, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1397, 6831, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7954, 8191, 8906, 7574, 8228, 8245, 8244, 8194] avg=8192.0000 max=8906 max_rank=2 min=7574 min_rank=3 imbalance_ratio=1.087158 max_load_reduction=56630 max_load_reduction_pct=0.864105 moved_to_echo=32911 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[60416, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 4352, 896, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 0, 0, 4224, 0, 0, 0, 0, 0, 0, 61312, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[15872, 49792, 128, 0, 128, 128, 0, 128, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 0, 63744, 0, 0, 896, 0, 0, 128, 128, 1024, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 65280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 128, 0, 0, 0, 128, 0, 0, 128, 0, 128, 128, 128, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 64768, 0, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[59776, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 4992, 768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 0, 0, 4864, 0, 0, 0, 0, 0, 0, 60672, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 63616, 0, 0, 896, 0, 0, 128, 128, 1152, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[10880, 54784, 128, 0, 128, 128, 0, 128, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 65152, 128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 0, 0, 0, 128, 0, 0, 128, 0, 128, 128, 128, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True - [2026-08-26 14:48:14] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 436.6 | throughput per GPU (TFLOP/s/GPU): 414.1 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.544505E+00 | loss scale: 1.0 | grad norm: 5.368 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2678, 0, 0, 2141, 2, 0, 0, 8192, 0, 8192, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 2983, 0, 0, 0, 4428, 0, 5229, 6, 0, 0, 4496, 0, 0, 0, 0, 0, 1, 0, 0, 7988, 0, 2, 0, 0, 2, 0, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7791, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 259, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 1635, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2765, 0, 0, 2759, 2, 0, 0, 8192, 0, 8192, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 2757, 0, 0, 0, 4824, 0, 5152, 6, 1, 0, 4168, 0, 0, 0, 0, 0, 0, 0, 0, 7995, 0, 4, 1, 1, 1, 0, 975, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7758, 0, 0, 4, 1, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 135, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[0, 1, 0, 7, 6, 2788, 0, 0, 1, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 2646, 0, 0, 2629, 2, 1, 0, 8191, 0, 8192, 3, 4, 0, 3, 0, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 3331, 0, 0, 0, 4576, 0, 4315, 20, 0, 1, 3762, 0, 0, 0, 0, 0, 1, 0, 0, 7910, 0, 3, 0, 5, 1, 0, 890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7871, 0, 0, 4, 0, 1, 0, 19, 0, 2, 0, 9, 0, 0, 0, 0, 0, 136, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[2, 1, 0, 0, 1, 1789, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3042, 0, 1, 3730, 9, 0, 0, 8183, 0, 8184, 1, 0, 0, 2, 1, 1, 0, 2, 0, 3, 2, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 2454, 1, 1, 0, 4739, 0, 4282, 1, 0, 0, 3680, 2, 0, 0, 1, 0, 1, 0, 1, 8114, 1, 7, 0, 0, 0, 1, 1046, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 8016, 0, 0, 1, 0, 0, 0, 7, 0, 3, 0, 0, 2, 0, 0, 0, 1, 31, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 13, 2279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 3, 2394, 0, 1, 2568, 0, 0, 0, 8192, 0, 8190, 2, 0, 0, 0, 6, 0, 0, 0, 0, 0, 1, 0, 0, 4, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 8184, 0, 2, 0, 0, 0, 0, 0, 0, 3400, 0, 0, 0, 4248, 0, 4315, 3, 2, 0, 4779, 0, 0, 0, 0, 0, 1, 0, 0, 8063, 0, 1, 0, 1, 0, 0, 967, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 7777, 0, 0, 11, 0, 0, 0, 1, 0, 4, 0, 0, 4, 0, 0, 0, 0, 108, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 2156, 0, 0, 0, 2, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 2648, 0, 0, 2953, 2, 0, 0, 8192, 0, 8192, 12, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 1, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 2693, 0, 6, 0, 4404, 0, 5026, 3, 0, 0, 3699, 0, 0, 0, 0, 0, 0, 0, 0, 8064, 0, 15, 0, 0, 3, 0, 1105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7892, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 238, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 1, 2989, 0, 1, 1, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 2055, 0, 1, 2367, 0, 0, 0, 8191, 0, 8190, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 1, 0, 0, 0, 0, 0, 3686, 0, 0, 0, 4110, 0, 4315, 9, 0, 0, 4463, 1, 0, 0, 0, 0, 0, 0, 0, 7985, 0, 1, 0, 0, 0, 0, 945, 2, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 7884, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 1, 0, 132, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2665, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2687, 0, 0, 3001, 0, 0, 0, 8192, 0, 8192, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7, 0, 0, 0, 6, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 3145, 0, 0, 0, 4213, 0, 3824, 13, 0, 0, 4004, 0, 0, 0, 0, 0, 0, 0, 0, 7958, 0, 3, 0, 0, 0, 0, 1419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7748, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 252, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 13, 2279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3296, 1832, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3, 2394, 0, 1, 2568, 0, 0, 0, 0, 0, 2800, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 1, 0, 0, 4, 0, 0, 2, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 8184, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3400, 0, 0, 0, 952, 0, 0, 3, 2, 0, 4779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7189, 0, 1, 0, 1, 0, 0, 967, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 7777, 0, 0, 11, 0, 0, 345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 4, 0, 0, 0, 0, 108, 0, 0, 0, 0, 5390, 2483, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7950, 7770, 8196, 8190, 9136, 8160, 8135, 7999] avg=8192.0000 max=9136 max_rank=4 min=7770 min_rank=1 imbalance_ratio=1.115234 max_load_reduction=56400 max_load_reduction_pct=0.860596 moved_to_echo=33430 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 7, 6, 2788, 0, 0, 1, 2, 0, 0, 0, 0, 2, 0, 3551, 1831, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2646, 0, 0, 2629, 2, 1, 0, 0, 0, 2801, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3331, 0, 0, 0, 1025, 0, 0, 20, 0, 1, 3762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7052, 0, 3, 0, 5, 1, 0, 890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7871, 0, 0, 4, 0, 1, 339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 2, 0, 9, 0, 0, 0, 0, 0, 136, 0, 0, 0, 0, 5391, 2484, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8706, 8088, 8190, 8195, 8139, 7952, 8215, 8051] avg=8192.0000 max=8706 max_rank=0 min=7952 min_rank=5 imbalance_ratio=1.062744 max_load_reduction=56830 max_load_reduction_pct=0.867157 moved_to_echo=32357 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2, 1, 0, 0, 1, 1789, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3677, 1818, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3042, 0, 1, 3730, 9, 0, 0, 0, 0, 2798, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 0, 2, 0, 3, 2, 0, 0, 0, 1, 0, 0, 1, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 8179, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2454, 1, 1, 0, 1062, 0, 0, 1, 0, 0, 3680, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 7234, 1, 7, 0, 0, 0, 1, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 8016, 0, 0, 1, 0, 0, 347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 3, 0, 0, 2, 0, 0, 0, 1, 31, 0, 0, 0, 0, 5386, 2464, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7822, 9581, 8187, 8184, 7201, 8292, 8366, 7903] avg=8192.0000 max=9581 max_rank=1 min=7201 min_rank=4 imbalance_ratio=1.169556 max_load_reduction=55955 max_load_reduction_pct=0.853806 moved_to_echo=31762 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 1635, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3743, 2187, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2765, 0, 0, 2759, 2, 0, 0, 0, 0, 2801, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 3, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2757, 0, 0, 0, 1081, 0, 0, 6, 1, 0, 4168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7128, 0, 4, 1, 1, 1, 0, 975, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7758, 0, 0, 4, 1, 1, 342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 135, 0, 0, 0, 0, 5391, 2965, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8092, 8328, 8189, 8193, 8013, 8112, 8107, 8502] avg=8192.0000 max=8502 max_rank=7 min=8013 min_rank=4 imbalance_ratio=1.037842 max_load_reduction=57034 max_load_reduction_pct=0.870270 moved_to_echo=32734 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3436, 2219, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2678, 0, 0, 2141, 2, 0, 0, 0, 0, 2801, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2983, 0, 0, 0, 992, 0, 0, 6, 0, 0, 4496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7121, 0, 2, 0, 0, 2, 0, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7791, 0, 0, 4, 0, 0, 342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 259, 0, 0, 0, 0, 5391, 3010, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8098, 7637, 8188, 8191, 8477, 8138, 8137, 8670] avg=8192.0000 max=8670 max_rank=7 min=7637 min_rank=1 imbalance_ratio=1.058350 max_load_reduction=56866 max_load_reduction_pct=0.867706 moved_to_echo=33422 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 1, 2989, 0, 1, 1, 1, 0, 0, 1, 0, 0, 1, 3189, 1831, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2055, 0, 1, 2367, 0, 0, 0, 0, 0, 2800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3686, 0, 0, 0, 921, 0, 0, 9, 0, 0, 4463, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7119, 0, 1, 0, 0, 0, 0, 945, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 7884, 0, 0, 4, 0, 0, 342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 1, 0, 132, 0, 0, 0, 0, 5390, 2484, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8538, 7224, 8183, 8192, 9080, 8067, 8232, 8020] avg=8192.0000 max=9080 max_rank=4 min=7224 min_rank=1 imbalance_ratio=1.108398 max_load_reduction=56456 max_load_reduction_pct=0.861450 moved_to_echo=33399 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2665, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3269, 1624, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2687, 0, 0, 3001, 0, 0, 0, 0, 0, 2801, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7, 0, 0, 0, 6, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3145, 0, 0, 0, 944, 0, 0, 13, 0, 0, 4004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7094, 0, 3, 0, 0, 0, 0, 1419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7748, 0, 0, 3, 0, 0, 341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 252, 0, 0, 0, 0, 5391, 2200, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8080, 8498, 8197, 8195, 8106, 8516, 8092, 7852] avg=8192.0000 max=8516 max_rank=5 min=7852 min_rank=7 imbalance_ratio=1.039551 max_load_reduction=57020 max_load_reduction_pct=0.870056 moved_to_echo=32566 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 2156, 0, 0, 0, 2, 0, 0, 0, 1, 1, 0, 3421, 2134, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2648, 0, 0, 2953, 2, 0, 0, 0, 0, 2794, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 1, 0, 0, 0, 18, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2693, 0, 6, 0, 983, 0, 0, 3, 0, 0, 3699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7176, 0, 15, 0, 0, 3, 0, 1105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7892, 0, 0, 3, 0, 0, 350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 238, 0, 0, 0, 0, 5398, 2892, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8250, 8410, 8206, 8196, 7384, 8299, 8252, 8539] avg=8192.0000 max=8539 max_rank=7 min=7384 min_rank=4 imbalance_ratio=1.042358 max_load_reduction=56997 max_load_reduction_pct=0.869705 moved_to_echo=32474 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 64768, 0, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True + [2026-08-26 15:24:14] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 440.9 | throughput per GPU (TFLOP/s/GPU): 410.2 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.543559E+00 | loss scale: 1.0 | grad norm: 5.353 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 2411, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 2726, 0, 0, 2907, 2, 0, 0, 8192, 0, 8192, 13, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 1, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 2699, 0, 6, 0, 4281, 0, 5106, 3, 0, 0, 3924, 0, 0, 0, 0, 0, 0, 0, 0, 7354, 0, 28, 0, 0, 3, 0, 1167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7963, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 328, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 1, 3234, 0, 1, 1, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 2173, 0, 1, 2342, 0, 0, 0, 8191, 0, 8190, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 1, 0, 0, 0, 0, 0, 3721, 0, 0, 0, 4044, 0, 4450, 15, 0, 0, 4773, 1, 0, 0, 0, 0, 0, 0, 0, 7017, 0, 4, 0, 0, 0, 0, 1022, 2, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 7949, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 1, 0, 196, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2939, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2790, 0, 0, 2909, 0, 0, 0, 8192, 0, 8192, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7, 0, 0, 0, 5, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 3150, 0, 0, 0, 4157, 0, 3940, 15, 0, 0, 4257, 0, 0, 0, 0, 0, 0, 0, 0, 7117, 0, 9, 0, 0, 0, 0, 1511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7844, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 293, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 1827, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2867, 0, 0, 2766, 2, 0, 0, 8192, 0, 8192, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 2791, 0, 0, 0, 4769, 0, 5276, 9, 1, 0, 4498, 0, 0, 0, 0, 0, 0, 0, 0, 6949, 0, 9, 1, 1, 1, 0, 1091, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7859, 0, 0, 5, 1, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 221, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[2, 1, 0, 0, 2, 1952, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3137, 0, 1, 3528, 9, 0, 0, 8184, 0, 8184, 1, 0, 0, 2, 1, 1, 0, 2, 0, 2, 2, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 2478, 1, 1, 0, 4667, 0, 4372, 2, 0, 0, 3856, 2, 0, 0, 1, 0, 1, 0, 1, 7579, 1, 12, 0, 0, 0, 1, 1089, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 8064, 0, 0, 1, 0, 0, 0, 7, 0, 3, 0, 0, 2, 0, 0, 0, 1, 195, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[0, 1, 0, 6, 8, 3002, 0, 0, 1, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 2749, 0, 0, 2614, 3, 1, 0, 8191, 0, 8192, 2, 6, 0, 2, 0, 0, 2, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 3371, 0, 0, 0, 4503, 0, 4411, 22, 0, 1, 4046, 0, 0, 0, 0, 0, 1, 0, 0, 7017, 0, 4, 0, 5, 1, 0, 945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7948, 0, 0, 4, 0, 1, 0, 18, 0, 2, 0, 11, 0, 0, 0, 0, 0, 243, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 19, 2464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 0, 3, 2473, 0, 1, 2458, 0, 0, 0, 8192, 0, 8190, 1, 0, 0, 0, 3, 0, 0, 0, 0, 1, 1, 0, 0, 4, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 8184, 0, 2, 0, 0, 0, 0, 0, 0, 3421, 0, 0, 0, 4172, 0, 4404, 4, 1, 0, 5096, 0, 0, 0, 0, 0, 1, 0, 0, 7310, 0, 1, 0, 1, 0, 0, 1037, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 7885, 0, 0, 12, 0, 0, 0, 1, 0, 4, 0, 0, 4, 0, 0, 0, 0, 174, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2797, 0, 0, 2127, 2, 0, 0, 8192, 0, 8192, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 3015, 0, 0, 0, 4362, 0, 5334, 7, 0, 0, 4832, 0, 0, 0, 0, 0, 1, 0, 0, 7007, 0, 14, 0, 0, 3, 0, 1084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7889, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 331, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 1, 3234, 0, 1, 1, 1, 0, 0, 1, 0, 0, 1, 3816, 2063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2173, 0, 1, 2342, 0, 0, 0, 0, 0, 2762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3721, 0, 0, 0, 4044, 0, 0, 15, 0, 0, 791, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6911, 0, 4, 0, 0, 0, 0, 1022, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 7949, 0, 0, 4, 0, 0, 166, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 1, 0, 196, 0, 0, 0, 0, 5428, 2387, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9120, 7279, 8185, 8193, 8572, 7939, 8225, 8023] avg=8192.0000 max=9120 max_rank=0 min=7279 min_rank=1 imbalance_ratio=1.113281 max_load_reduction=56416 max_load_reduction_pct=0.860840 moved_to_echo=32759 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 2411, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 3141, 2368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2726, 0, 0, 2907, 2, 0, 0, 0, 0, 2758, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 1, 0, 0, 0, 16, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2699, 0, 6, 0, 4281, 0, 0, 3, 0, 0, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7237, 0, 28, 0, 0, 3, 0, 1167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7963, 0, 0, 4, 0, 0, 140, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 328, 0, 0, 0, 0, 5434, 2738, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7926, 8407, 8205, 8195, 7632, 8435, 8226, 8510] avg=8192.0000 max=8510 max_rank=7 min=7632 min_rank=4 imbalance_ratio=1.038818 max_load_reduction=57026 max_load_reduction_pct=0.870148 moved_to_echo=32803 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2, 1, 0, 0, 2, 1952, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3082, 2028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3137, 0, 1, 3528, 9, 0, 0, 0, 0, 2760, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 0, 2, 0, 2, 2, 0, 0, 0, 0, 1, 0, 1, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 8179, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2478, 1, 1, 0, 4667, 0, 0, 2, 0, 0, 640, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 7465, 1, 12, 0, 0, 0, 1, 1089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 8064, 0, 0, 1, 0, 0, 134, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 3, 0, 0, 2, 0, 0, 0, 1, 195, 0, 0, 0, 0, 5424, 2344, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7069, 9436, 8188, 8184, 7791, 8571, 8313, 7984] avg=8192.0000 max=9436 max_rank=1 min=7069 min_rank=0 imbalance_ratio=1.151855 max_load_reduction=56100 max_load_reduction_pct=0.856018 moved_to_echo=32659 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 1827, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3596, 2446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2867, 0, 0, 2766, 2, 0, 0, 0, 0, 2763, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 2, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2791, 0, 0, 0, 4769, 0, 0, 9, 1, 0, 746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6844, 0, 9, 1, 1, 1, 0, 1091, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7859, 0, 0, 5, 1, 1, 156, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 221, 0, 0, 0, 0, 5429, 2830, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7874, 8399, 8189, 8193, 8316, 7949, 8126, 8490] avg=8192.0000 max=8490 max_rank=7 min=7874 min_rank=0 imbalance_ratio=1.036377 max_load_reduction=57046 max_load_reduction_pct=0.870453 moved_to_echo=32881 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 6, 8, 3002, 0, 0, 1, 2, 0, 0, 0, 0, 2, 0, 3234, 2045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2749, 0, 0, 2614, 3, 1, 0, 0, 0, 2763, 2, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3371, 0, 0, 0, 4503, 0, 0, 22, 0, 1, 672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6911, 0, 4, 0, 5, 1, 0, 945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7948, 0, 0, 4, 0, 1, 140, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 2, 0, 11, 0, 0, 0, 0, 0, 243, 0, 0, 0, 0, 5429, 2366, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8301, 8140, 8190, 8195, 8569, 7867, 8197, 8077] avg=8192.0000 max=8569 max_rank=4 min=7867 min_rank=5 imbalance_ratio=1.046021 max_load_reduction=56967 max_load_reduction_pct=0.869247 moved_to_echo=32710 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2939, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3403, 1828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2790, 0, 0, 2909, 0, 0, 0, 0, 0, 2763, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7, 0, 0, 0, 5, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3150, 0, 0, 0, 4157, 0, 0, 15, 0, 0, 706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7010, 0, 9, 0, 0, 0, 0, 1511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7844, 0, 0, 4, 0, 0, 148, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 293, 0, 0, 0, 0, 5429, 2112, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8471, 8197, 8195, 8028, 8530, 8101, 7842] avg=8192.0000 max=8530 max_rank=5 min=7842 min_rank=7 imbalance_ratio=1.041260 max_load_reduction=57006 max_load_reduction_pct=0.869843 moved_to_echo=32501 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3863, 2473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2797, 0, 0, 2127, 2, 0, 0, 0, 0, 2763, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3015, 0, 0, 0, 4362, 0, 0, 7, 0, 0, 801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6902, 0, 14, 0, 0, 3, 0, 1084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7889, 0, 0, 4, 0, 0, 168, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 331, 0, 0, 0, 0, 5429, 2861, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8470, 7704, 8188, 8191, 8185, 8004, 8164, 8630] avg=8192.0000 max=8630 max_rank=7 min=7704 min_rank=1 imbalance_ratio=1.053467 max_load_reduction=56906 max_load_reduction_pct=0.868317 moved_to_echo=32983 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 19, 2464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4074, 2043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3, 2473, 0, 1, 2458, 0, 0, 0, 0, 0, 2762, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 1, 0, 0, 4, 0, 0, 1, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 8184, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3421, 0, 0, 0, 4172, 0, 0, 4, 1, 0, 845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7200, 0, 1, 0, 1, 0, 0, 1037, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 7885, 0, 0, 12, 0, 0, 177, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 4, 0, 0, 0, 0, 174, 0, 0, 0, 0, 5428, 2361, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8604, 7700, 8194, 8190, 8443, 8241, 8184, 7980] avg=8192.0000 max=8604 max_rank=0 min=7700 min_rank=1 imbalance_ratio=1.050293 max_load_reduction=56932 max_load_reduction_pct=0.868713 moved_to_echo=32848 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[128, 128, 0, 128, 128, 18304, 128, 128, 128, 128, 0, 0, 128, 128, 128, 128, 27648, 15488, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 128, 20992, 0, 128, 22272, 128, 128, 0, 0, 0, 22400, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 0, 0, 0, 24576, 128, 128, 0, 8064, 0, 0, 128, 128, 128, 33152, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 0, 128, 0, 128, 128, 0, 0, 128, 128, 1408, 0, 0, 0, 0, 43136, 20992, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 0, 128, 0, 128, 0, 128, 57216, 128, 128, 128, 128, 128, 128, 8448, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 128, 21760, 0, 128, 21760, 128, 128, 0, 0, 0, 22144, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 0, 128, 128, 19968, 128, 128, 128, 128, 0, 0, 128, 0, 128, 128, 28288, 17408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 0, 24704, 128, 128, 0, 35072, 0, 0, 128, 128, 128, 5888, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 128, 0, 128, 128, 0, 0, 128, 128, 2048, 0, 0, 0, 0, 43520, 20096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 0, 128, 0, 128, 0, 128, 56576, 128, 128, 128, 128, 128, 128, 8960, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 128, 0, 128, 128, 0, 0, 128, 0, 128, 62848, 0, 0, 128, 128, 128, 2816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 128, 128, 128, 128, 128, 0, 128, 128, 0, 128, 128, 128, 0, 128, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[0, 128, 0, 128, 128, 0, 0, 128, 0, 128, 63488, 0, 0, 128, 128, 128, 1280, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[0, 128, 128, 128, 128, 128, 0, 128, 128, 0, 128, 128, 0, 128, 128, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 0, 128, 128, 0, 128, 128, 128, 0, 65536, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 1, 1, 0, 0, 8090, 1, 6486, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 8183, 8190, 1, 0, 0, 355, 0, 0, 0, 0, 0, 0, 0, 1, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8191, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 5466, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 36, 0, 0, 0, 7840, 0, 5958, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8162, 8192, 0, 0, 0, 160, 0, 0, 0, 0, 0, 0, 0, 0, 3499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6986, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[2, 0, 213, 2, 0, 0, 7279, 0, 2741, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 8190, 8190, 0, 0, 0, 370, 0, 2, 0, 0, 1, 0, 0, 0, 7975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8188, 0, 0, 0, 0, 0, 2, 0, 2, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 1, 2, 0, 0, 5156, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 193, 0, 0, 0, 7965, 0, 5900, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 145, 0, 0, 0, 0, 0, 0, 0, 0, 6309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 3, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3831, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 7686, 0, 6018, 0, 0, 8, 0, 0, 3, 0, 0, 0, 0, 0, 8183, 8192, 0, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 5158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5480, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 24, 0, 0, 0, 7699, 0, 5997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 152, 0, 0, 0, 0, 0, 0, 0, 0, 5542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4566, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8163, 0, 7294, 0, 0, 16, 0, 0, 5, 0, 0, 0, 0, 0, 8059, 8192, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 1632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 671, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6871, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 6, 0, 0, 0, 8105, 0, 7545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7981, 8192, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 1, 0, 0, 2200, 1, 6486, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8058, 0, 1, 0, 0, 355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3568, 349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 8188, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 0, 0, 7841, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 5466, 0, 0, 1, 0, 2322, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8690, 8415, 7967, 8194, 8195, 7976, 8188, 7911] avg=8192.0000 max=8690 max_rank=0 min=7911 min_rank=7 imbalance_ratio=1.060791 max_load_reduction=56846 max_load_reduction_pct=0.867401 moved_to_echo=32270 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 2090, 0, 6018, 0, 0, 8, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8058, 0, 0, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3390, 350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 0, 0, 0, 7842, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5480, 0, 0, 0, 0, 2206, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8121, 8188, 8898, 8192, 8193, 7947, 8191, 7806] avg=8192.0000 max=8898 max_rank=2 min=7806 min_rank=7 imbalance_ratio=1.086182 max_load_reduction=56638 max_load_reduction_pct=0.864227 moved_to_echo=32137 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 193, 0, 0, 0, 2166, 0, 5900, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8067, 0, 0, 0, 0, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3513, 349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 200, 0, 0, 0, 7838, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 3, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3831, 0, 0, 0, 0, 2286, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8264, 8212, 10171, 8190, 8184, 8043, 8205, 6267] avg=8192.0000 max=10171 max_rank=2 min=6267 min_rank=7 imbalance_ratio=1.241577 max_load_reduction=55365 max_load_reduction_pct=0.844803 moved_to_echo=30699 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 213, 2, 0, 0, 1980, 0, 2741, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8065, 0, 0, 0, 0, 370, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3210, 349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 2, 0, 2, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 828, 0, 0, 0, 7841, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 1, 2, 0, 0, 5156, 0, 0, 0, 0, 2089, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4940, 8440, 11534, 8194, 8190, 8675, 8187, 7376] avg=8192.0000 max=11534 max_rank=2 min=4940 min_rank=0 imbalance_ratio=1.407959 max_load_reduction=54002 max_load_reduction_pct=0.824005 moved_to_echo=32428 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 36, 0, 0, 0, 2132, 0, 5958, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8037, 0, 0, 0, 0, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3458, 349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 0, 0, 0, 7842, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6986, 0, 0, 0, 0, 2250, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8127, 8197, 7306, 8193, 8194, 7974, 8189, 9356] avg=8192.0000 max=9356 max_rank=7 min=7306 min_rank=2 imbalance_ratio=1.142090 max_load_reduction=56180 max_load_reduction_pct=0.857239 moved_to_echo=33713 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 24, 0, 0, 0, 2093, 0, 5997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8067, 0, 0, 0, 0, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3396, 349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 596, 0, 0, 0, 7841, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 2, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4566, 0, 0, 0, 0, 2210, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8114, 8219, 9287, 8189, 8183, 8442, 8199, 6903] avg=8192.0000 max=9287 max_rank=2 min=6903 min_rank=7 imbalance_ratio=1.133667 max_load_reduction=56249 max_load_reduction_pct=0.858292 moved_to_echo=31727 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6, 0, 0, 0, 2204, 0, 7545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7859, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3575, 349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 0, 0, 0, 7843, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 0, 0, 0, 0, 2326, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9755, 7879, 4787, 8192, 8197, 7962, 8189, 10575] avg=8192.0000 max=10575 max_rank=7 min=4787 min_rank=2 imbalance_ratio=1.290894 max_load_reduction=54961 max_load_reduction_pct=0.838638 moved_to_echo=34923 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 2210, 0, 7294, 0, 0, 16, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7926, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3604, 350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 671, 0, 0, 0, 7842, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6871, 0, 0, 0, 0, 2349, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9525, 7986, 5586, 8192, 8200, 8517, 8188, 9342] avg=8192.0000 max=9525 max_rank=0 min=5586 min_rank=2 imbalance_ratio=1.162720 max_load_reduction=56011 max_load_reduction_pct=0.854660 moved_to_echo=34247 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 3, 1, 0, 0, 8172, 1, 6224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8189, 8190, 1, 0, 0, 750, 0, 0, 0, 0, 0, 0, 0, 1, 2347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8191, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 6965, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[6, 0, 29, 0, 0, 0, 8118, 0, 5710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 438, 0, 0, 0, 0, 0, 0, 0, 0, 4239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5302, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[16, 0, 234, 2, 0, 0, 7991, 0, 1563, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 8190, 8190, 0, 0, 0, 255, 0, 2, 0, 0, 1, 0, 0, 0, 7520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8188, 0, 0, 0, 0, 0, 2, 0, 2, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 755, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 1, 2, 0, 0, 6229, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 214, 0, 0, 0, 8101, 0, 5603, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 206, 0, 0, 0, 0, 0, 0, 0, 0, 4629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 1, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5490, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 38, 0, 0, 0, 8147, 0, 5489, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8185, 8192, 0, 0, 0, 282, 0, 0, 0, 0, 0, 0, 0, 0, 3067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7463, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 7, 0, 0, 0, 8158, 0, 7477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 8192, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8131, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8190, 0, 7193, 0, 0, 2, 0, 0, 6, 0, 0, 0, 0, 0, 8161, 8192, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 1291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7462, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 0, 8112, 0, 5336, 0, 0, 5, 0, 0, 3, 0, 0, 0, 0, 0, 8187, 8192, 0, 0, 0, 266, 0, 0, 0, 0, 0, 0, 0, 0, 3801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6910, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 1, 0, 0, 2562, 1, 6224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7912, 0, 1, 0, 0, 750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4435, 331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 8188, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 0, 0, 0, 7859, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 6965, 0, 0, 1, 0, 1175, 272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8791, 8664, 7114, 8194, 8195, 7974, 8189, 8415] avg=8192.0000 max=8791 max_rank=0 min=7114 min_rank=2 imbalance_ratio=1.073120 max_load_reduction=56745 max_load_reduction_pct=0.865860 moved_to_echo=32773 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[16, 0, 234, 2, 0, 0, 2505, 0, 1563, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 7912, 0, 0, 0, 0, 255, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4337, 331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 2, 0, 2, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 755, 0, 0, 0, 7859, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 1, 2, 0, 0, 6229, 0, 0, 0, 0, 1149, 273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4322, 8172, 12188, 8195, 8190, 8619, 8188, 7662] avg=8192.0000 max=12188 max_rank=2 min=4322 min_rank=0 imbalance_ratio=1.487793 max_load_reduction=53348 max_load_reduction_pct=0.814026 moved_to_echo=32659 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 38, 0, 0, 0, 2555, 0, 5489, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7908, 0, 0, 0, 0, 282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4421, 331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 0, 7860, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7463, 0, 0, 0, 0, 1171, 272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8083, 8190, 7819, 8193, 8195, 7960, 8190, 8906] avg=8192.0000 max=8906 max_rank=7 min=7819 min_rank=2 imbalance_ratio=1.087158 max_load_reduction=56630 max_load_reduction_pct=0.864105 moved_to_echo=33251 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 29, 0, 0, 0, 2545, 0, 5710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7914, 0, 0, 0, 0, 438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4406, 331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 739, 0, 0, 0, 7858, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5302, 0, 0, 0, 0, 1167, 273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8290, 8352, 8976, 8188, 8183, 8602, 8197, 6748] avg=8192.0000 max=8976 max_rank=2 min=6748 min_rank=7 imbalance_ratio=1.095703 max_load_reduction=56560 max_load_reduction_pct=0.863037 moved_to_echo=31730 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 214, 0, 0, 0, 2540, 0, 5603, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7914, 0, 0, 0, 0, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4396, 331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 306, 0, 0, 0, 7856, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 1, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5490, 0, 0, 0, 0, 1165, 273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8358, 8120, 9356, 8190, 8184, 8167, 8201, 6960] avg=8192.0000 max=9356 max_rank=2 min=6960 min_rank=7 imbalance_ratio=1.142090 max_load_reduction=56180 max_load_reduction_pct=0.857239 moved_to_echo=31512 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 2558, 0, 7193, 0, 0, 2, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7875, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4449, 332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 415, 0, 0, 0, 7860, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7462, 0, 0, 0, 0, 1183, 276, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9759, 7925, 6072, 8192, 8200, 8278, 8189, 8921] avg=8192.0000 max=9759 max_rank=0 min=6072 min_rank=2 imbalance_ratio=1.191284 max_load_reduction=55777 max_load_reduction_pct=0.851089 moved_to_echo=33588 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 7, 0, 0, 0, 2558, 0, 7477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7910, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4427, 331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 7861, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8131, 0, 0, 0, 0, 1173, 272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10042, 7937, 5476, 8192, 8197, 7926, 8190, 9576] avg=8192.0000 max=10042 max_rank=0 min=5476 min_rank=2 imbalance_ratio=1.225830 max_load_reduction=55494 max_load_reduction_pct=0.846771 moved_to_echo=33889 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 0, 2544, 0, 5336, 0, 0, 5, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7910, 0, 0, 0, 0, 266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4402, 332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 0, 0, 0, 7860, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6910, 0, 0, 0, 0, 1166, 272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7891, 8176, 8535, 8192, 8192, 8010, 8192, 8348] avg=8192.0000 max=8535 max_rank=2 min=7891 min_rank=0 imbalance_ratio=1.041870 max_load_reduction=57001 max_load_reduction_pct=0.869766 moved_to_echo=32742 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 35072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27776, 2816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 512, 128, 0, 0, 17152, 128, 48000, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 0, 0, 0, 128, 128, 0, 0, 46592, 0, 0, 128, 0, 18048, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 27648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35328, 2688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 640, 128, 0, 0, 20480, 128, 44672, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 0, 0, 0, 0, 128, 128, 0, 0, 54016, 0, 0, 128, 0, 9472, 2304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 0, 2816, 0, 0, 0, 62848, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 128, 128, 64256, 0, 128, 0, 0, 1408, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 0, 2688, 0, 0, 0, 62976, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 128, 128, 63360, 0, 128, 0, 0, 2304, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 128, 65536, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 0, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 65536, 128, 0, 0, 0, 0, 128, 0, 128, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 575, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 1, 0, 8190, 0, 0, 1, 1, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4577, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8191, 0, 1, 0, 0, 1, 0, 0, 0, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8184, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 2, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 221, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 3, 8017, 0, 3, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 2, 0, 0, 0, 0, 3, 0, 0, 5224, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2927, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 8189, 0, 2, 1, 0, 0, 0, 3, 0, 8189, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 3, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 5, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 575, 0, 0, 0, 0, 8093, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4577, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1178, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3045, 0, 7009, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 4, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8192, 8670, 8201, 5833, 10059, 8191, 8197] avg=8192.0000 max=10059 max_rank=5 min=5833 min_rank=4 imbalance_ratio=1.227905 max_load_reduction=55477 max_load_reduction_pct=0.846512 moved_to_echo=32280 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1179, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 224, 0, 7013, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8192, 8095, 8213, 9209, 7240, 8192, 8193] avg=8192.0000 max=9209 max_rank=4 min=7240 min_rank=5 imbalance_ratio=1.124146 max_load_reduction=56327 max_load_reduction_pct=0.859482 moved_to_echo=32834 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1178, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 293, 0, 7008, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8193, 8093, 8213, 9161, 7305, 8193, 8190] avg=8192.0000 max=9161 max_rank=4 min=7305 min_rank=5 imbalance_ratio=1.118286 max_load_reduction=56375 max_load_reduction_pct=0.860214 moved_to_echo=32849 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1178, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 0, 7009, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8193, 8094, 8213, 9350, 7115, 8192, 8190] avg=8192.0000 max=9350 max_rank=4 min=7115 min_rank=5 imbalance_ratio=1.141357 max_load_reduction=56186 max_load_reduction_pct=0.857330 moved_to_echo=32847 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 221, 0, 0, 0, 0, 8092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8017, 0, 3, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 2, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 5224, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1178, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2927, 0, 7008, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 3, 0, 2, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8189, 8313, 8050, 6483, 9940, 8191, 8183] avg=8192.0000 max=9940 max_rank=5 min=6483 min_rank=4 imbalance_ratio=1.213379 max_load_reduction=55596 max_load_reduction_pct=0.848328 moved_to_echo=32797 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1179, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 361, 0, 7010, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8192, 8095, 8213, 9083, 7374, 8193, 8198] avg=8192.0000 max=9083 max_rank=4 min=7374 min_rank=5 imbalance_ratio=1.108765 max_load_reduction=56453 max_load_reduction_pct=0.861404 moved_to_echo=32848 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1179, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1118, 0, 7012, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8192, 8095, 8213, 8329, 8133, 8192, 8193] avg=8192.0000 max=8329 max_rank=4 min=8095 min_rank=2 imbalance_ratio=1.016724 max_load_reduction=57207 max_load_reduction_pct=0.872910 moved_to_echo=32847 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1182, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1357, 0, 7010, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8193, 8081, 8220, 8088, 8370, 8192, 8192] avg=8192.0000 max=8370 max_rank=5 min=8081 min_rank=2 imbalance_ratio=1.021729 max_load_reduction=57166 max_load_reduction_pct=0.872284 moved_to_echo=32842 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 596, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 1, 1, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5991, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1604, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8191, 0, 1, 0, 0, 1, 0, 0, 0, 8191, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 3, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8184, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 2, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8188, 0, 234, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 3, 8165, 0, 3, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 2, 0, 0, 0, 0, 3, 0, 0, 5821, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2174, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 8189, 0, 2, 1, 0, 0, 0, 4, 0, 8189, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 3, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 3, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8088, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5991, 0, 0, 0, 0, 0, 0, 0, 1, 0, 592, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1604, 0, 7596, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 3, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8192, 8685, 8191, 6685, 9205, 8191, 8196] avg=8192.0000 max=9205 max_rank=5 min=6685 min_rank=4 imbalance_ratio=1.123657 max_load_reduction=56331 max_load_reduction_pct=0.859543 moved_to_echo=32277 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 592, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 0, 7597, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8192, 8090, 8194, 8770, 7711, 8193, 8198] avg=8192.0000 max=8770 max_rank=4 min=7711 min_rank=5 imbalance_ratio=1.070557 max_load_reduction=56766 max_load_reduction_pct=0.866180 moved_to_echo=32872 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 592, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 7596, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8193, 8090, 8194, 8861, 7629, 8192, 8188] avg=8192.0000 max=8861 max_rank=4 min=7629 min_rank=5 imbalance_ratio=1.081665 max_load_reduction=56675 max_load_reduction_pct=0.864792 moved_to_echo=32870 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8165, 0, 3, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 5821, 0, 0, 0, 0, 0, 0, 0, 2, 0, 591, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2174, 0, 7594, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 4, 0, 2, 1, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8190, 8318, 8179, 6517, 9772, 8190, 8180] avg=8192.0000 max=9772 max_rank=5 min=6517 min_rank=4 imbalance_ratio=1.192871 max_load_reduction=55764 max_load_reduction_pct=0.850891 moved_to_echo=32659 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 592, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 163, 0, 7595, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8193, 8090, 8194, 8729, 7761, 8194, 8189] avg=8192.0000 max=8729 max_rank=4 min=7761 min_rank=5 imbalance_ratio=1.065552 max_load_reduction=56807 max_load_reduction_pct=0.866806 moved_to_echo=32873 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 594, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 409, 0, 7598, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8192, 8083, 8196, 8471, 8010, 8192, 8192] avg=8192.0000 max=8471 max_rank=4 min=8010 min_rank=5 imbalance_ratio=1.034058 max_load_reduction=57065 max_load_reduction_pct=0.870743 moved_to_echo=32865 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 592, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 0, 7597, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8192, 8090, 8194, 8705, 7776, 8192, 8198] avg=8192.0000 max=8705 max_rank=4 min=7776 min_rank=5 imbalance_ratio=1.062622 max_load_reduction=56831 max_load_reduction_pct=0.867172 moved_to_echo=32871 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 592, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 7598, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8192, 8090, 8194, 8798, 7672, 8192, 8195] avg=8192.0000 max=8798 max_rank=4 min=7672 min_rank=5 imbalance_ratio=1.073975 max_load_reduction=56738 max_load_reduction_pct=0.865753 moved_to_echo=32857 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9472, 0, 56192, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 0, 0, 55552, 0, 0, 0, 0, 0, 0, 0, 128, 0, 9472, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[896, 0, 0, 0, 0, 64768, 0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 65408, 0, 128, 128, 0, 0, 0, 0, 0, 128, 128, 128, 0, 128, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[65536, 0, 128, 0, 128, 0, 128, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4864, 0, 60800, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 0, 0, 60032, 0, 0, 0, 0, 0, 0, 0, 128, 0, 4864, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64768, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[65536, 0, 128, 0, 128, 0, 128, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 65536, 0, 128, 128, 0, 0, 0, 0, 0, 128, 128, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2253, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 8192, 8169, 5605, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 3, 0, 0, 0, 0, 8173, 0, 0, 0, 7791, 0, 0, 0, 0, 2, 0, 0, 17, 1, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5069, 0, 0, 1, 8189, 4858, 19, 0, 0, 0, 3, 0, 8187, 8146, 0, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 6721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 8190, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 28, 0, 1, 0, 0, 0, 0, 8189, 0, 0, 0, 8110, 1, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2891, 0, 0, 0, 8190, 6281, 585, 0, 0, 0, 1, 0, 8141, 8181, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2148, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 358, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 8192, 8099, 5761, 0, 0, 0, 0, 0, 8190, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3308, 0, 0, 0, 0, 0, 0, 209, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 8188, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 109, 0, 0, 0, 8192, 8183, 4417, 0, 0, 0, 0, 0, 8192, 8190, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 125, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 8192, 8173, 7863, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 404, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 202, 0, 0, 0, 8192, 8156, 6540, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3732, 0, 0, 0, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 740, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 8188, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 156, 0, 0, 0, 8192, 8154, 3503, 0, 0, 0, 0, 0, 8192, 8191, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2253, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 3921, 1061, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 0, 0, 0, 0, 0, 0, 7851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 7127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4248, 5605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 7565, 8199, 8186, 8097, 8234, 7210, 9853] avg=8192.0000 max=9853 max_rank=7 min=7210 min_rank=6 imbalance_ratio=1.202759 max_load_reduction=55683 max_load_reduction_pct=0.849655 moved_to_echo=33394 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3308, 0, 0, 0, 0, 0, 0, 209, 0, 0, 0, 0, 0, 0, 3928, 1059, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 0, 0, 0, 0, 0, 0, 7846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 109, 7127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4255, 4417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8793, 8200, 8186, 7997, 8250, 7244, 8672] avg=8192.0000 max=8793 max_rank=1 min=7244 min_rank=6 imbalance_ratio=1.073364 max_load_reduction=56743 max_load_reduction_pct=0.865829 moved_to_echo=32163 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2148, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 3888, 1059, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 358, 0, 0, 0, 0, 0, 0, 7851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 7129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4211, 5761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 7399, 8195, 8186, 8209, 8250, 7134, 9972] avg=8192.0000 max=9972 max_rank=7 min=7134 min_rank=6 imbalance_ratio=1.217285 max_load_reduction=55564 max_load_reduction_pct=0.847839 moved_to_echo=33565 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 125, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 3926, 1061, 295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, 0, 0, 0, 0, 0, 0, 7840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 7127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4247, 7863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 5414, 8196, 8181, 8022, 8258, 7163, 12110] avg=8192.0000 max=12110 max_rank=7 min=5414 min_rank=1 imbalance_ratio=1.478271 max_load_reduction=53426 max_load_reduction_pct=0.815216 moved_to_echo=35553 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 404, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 3915, 1059, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 0, 0, 0, 0, 0, 0, 7851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 202, 7127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4241, 6540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 5692, 8197, 8186, 8910, 8249, 7329, 10781] avg=8192.0000 max=10781 max_rank=7 min=5692 min_rank=1 imbalance_ratio=1.316040 max_load_reduction=54755 max_load_reduction_pct=0.835495 moved_to_echo=35269 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3732, 0, 0, 0, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 3914, 1059, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 740, 0, 0, 0, 0, 0, 0, 7846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 156, 7126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4240, 3503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 9087, 8203, 8186, 8586, 8250, 7288, 7743] avg=8192.0000 max=9087 max_rank=1 min=7288 min_rank=6 imbalance_ratio=1.109253 max_load_reduction=56449 max_load_reduction_pct=0.861343 moved_to_echo=31867 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 3015, 1058, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8141, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 8184, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 28, 0, 1, 0, 0, 0, 0, 7848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8110, 1, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1, 0, 52, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2891, 7119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3266, 585, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 11085, 8147, 8197, 7878, 8174, 10011, 3853] avg=8192.0000 max=11085 max_rank=1 min=3853 min_rank=7 imbalance_ratio=1.353149 max_load_reduction=54451 max_load_reduction_pct=0.830856 moved_to_echo=29916 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 2332, 1053, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 0, 0, 0, 0, 7832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7791, 0, 0, 0, 0, 2, 0, 0, 17, 1, 0, 0, 0, 2, 0, 52, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5069, 7087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2526, 19, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 10501, 8199, 8228, 7837, 7871, 12157, 2552] avg=8192.0000 max=12157 max_rank=6 min=2552 min_rank=7 imbalance_ratio=1.484009 max_load_reduction=53379 max_load_reduction_pct=0.814499 moved_to_echo=30417 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 6854, 0, 0, 0, 0, 0, 0, 5, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 8190, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 34, 0, 1, 0, 0, 0, 0, 8189, 0, 0, 0, 8120, 1, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3082, 0, 0, 0, 8190, 6094, 433, 0, 0, 0, 1, 0, 8137, 8181, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 205, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 999, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 0, 0, 0, 8192, 8154, 6827, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 163, 0, 0, 0, 0, 0, 0, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 282, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 8192, 8145, 7492, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 47, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 3, 0, 0, 0, 0, 8175, 0, 0, 0, 7873, 0, 0, 0, 0, 2, 0, 0, 15, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5113, 0, 0, 1, 8189, 4548, 4, 0, 0, 0, 3, 0, 8187, 8142, 0, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1756, 0, 0, 0, 0, 0, 0, 621, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 627, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 8192, 8165, 5219, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3582, 0, 0, 0, 0, 0, 0, 471, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 269, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 8188, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 56, 0, 0, 0, 8192, 8169, 3832, 0, 0, 0, 0, 0, 8192, 8190, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1748, 0, 0, 0, 0, 0, 0, 399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 733, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 8192, 8067, 5424, 0, 0, 0, 0, 0, 8189, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3841, 0, 0, 0, 0, 0, 0, 398, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 8188, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 106, 0, 0, 0, 8192, 8147, 2872, 0, 0, 0, 0, 0, 8192, 8191, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6854, 0, 0, 0, 0, 0, 0, 5, 0, 0, 1, 1, 0, 0, 2670, 1062, 449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 8184, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 34, 0, 1, 0, 0, 0, 0, 7699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8120, 1, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1, 0, 41, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3082, 7114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3424, 433, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 11042, 8144, 8195, 7735, 8173, 10197, 3859] avg=8192.0000 max=11042 max_rank=1 min=3859 min_rank=7 imbalance_ratio=1.347900 max_load_reduction=54494 max_load_reduction_pct=0.831512 moved_to_echo=29964 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 163, 0, 0, 0, 0, 0, 0, 232, 0, 0, 0, 0, 0, 0, 3572, 1064, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 282, 0, 0, 0, 0, 0, 0, 7689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 7123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4573, 7492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 5486, 8197, 8179, 7971, 8252, 7194, 12065] avg=8192.0000 max=12065 max_rank=7 min=5486 min_rank=1 imbalance_ratio=1.472778 max_load_reduction=53471 max_load_reduction_pct=0.815903 moved_to_echo=35482 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1992, 1057, 449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 1, 0, 0, 8187, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 0, 0, 0, 0, 7685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7873, 0, 0, 0, 0, 2, 0, 0, 15, 0, 0, 0, 0, 2, 0, 41, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5113, 7079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2556, 4, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 10524, 8200, 8232, 7690, 7939, 12193, 2567] avg=8192.0000 max=12193 max_rank=6 min=2567 min_rank=7 imbalance_ratio=1.488403 max_load_reduction=53343 max_load_reduction_pct=0.813950 moved_to_echo=30389 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3582, 0, 0, 0, 0, 0, 0, 471, 0, 0, 0, 0, 0, 0, 3579, 1063, 449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 269, 0, 0, 0, 0, 0, 0, 7696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 56, 7122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4590, 3832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 9144, 8201, 8186, 7965, 8238, 7186, 8422] avg=8192.0000 max=9144 max_rank=1 min=7186 min_rank=6 imbalance_ratio=1.116211 max_load_reduction=56392 max_load_reduction_pct=0.860474 moved_to_echo=31811 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1748, 0, 0, 0, 0, 0, 0, 399, 0, 0, 0, 0, 0, 0, 3534, 1063, 449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 733, 0, 0, 0, 0, 0, 0, 7702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 7124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4533, 5424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 7193, 8195, 8186, 8435, 8239, 7140, 9957] avg=8192.0000 max=9957 max_rank=7 min=7140 min_rank=6 imbalance_ratio=1.215454 max_load_reduction=55579 max_load_reduction_pct=0.848068 moved_to_echo=33771 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 205, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 3572, 1064, 449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 999, 0, 0, 0, 0, 0, 0, 7702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 7123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4582, 6827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 5360, 8196, 8186, 8701, 8238, 7254, 11409] avg=8192.0000 max=11409 max_rank=7 min=5360 min_rank=1 imbalance_ratio=1.392700 max_load_reduction=54127 max_load_reduction_pct=0.825912 moved_to_echo=35602 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3841, 0, 0, 0, 0, 0, 0, 398, 0, 0, 0, 0, 0, 0, 3569, 1063, 449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 0, 0, 0, 0, 0, 0, 7696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 106, 7122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4578, 2872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 9320, 8203, 8186, 8710, 8239, 7235, 7450] avg=8192.0000 max=9320 max_rank=1 min=7235 min_rank=6 imbalance_ratio=1.137695 max_load_reduction=56216 max_load_reduction_pct=0.857788 moved_to_echo=31634 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1756, 0, 0, 0, 0, 0, 0, 621, 0, 0, 0, 0, 0, 0, 3577, 1064, 449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 627, 0, 0, 0, 0, 0, 0, 7702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 7123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4588, 5219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 7467, 8200, 8186, 8329, 8218, 7137, 9807] avg=8192.0000 max=9807 max_rank=7 min=7137 min_rank=6 imbalance_ratio=1.197144 max_load_reduction=55729 max_load_reduction_pct=0.850357 moved_to_echo=33491 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 25600, 0, 0, 0, 0, 0, 0, 512, 0, 0, 128, 128, 0, 0, 28928, 8576, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 31360, 34304, 0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 8576, 57088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 25216, 0, 0, 0, 0, 0, 0, 2304, 0, 0, 128, 128, 0, 0, 26112, 8576, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 33536, 32128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 8704, 56960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 128, 2816, 0, 128, 0, 0, 0, 0, 62848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 128, 3968, 0, 128, 0, 0, 0, 0, 61696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 65152, 128, 128, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 128, 0, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 0, 0, 65536, 0, 128, 0, 128, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 65152, 128, 128, 0, 0, 128, 0, 0, 128, 128, 0, 0, 0, 128, 0, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 128, 0, 0, 65536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8190, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 18, 0, 1, 0, 7890, 1, 0, 0, 0, 0, 0, 5287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 3208, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8121, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 21, 0, 1, 0, 0, 0, 0, 0, 0, 5, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7866, 0, 0, 0, 0, 0, 0, 1073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 121, 0, 0, 7294, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8060, 0, 0, 0, 0, 0, 0, 447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 0, 0, 7512, 8085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 5, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 138, 0, 0, 0, 7437, 4, 0, 0, 0, 24, 0, 6166, 0, 0, 0, 0, 0, 0, 0, 2, 0, 7637, 3146, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8189, 7878, 0, 0, 0, 0, 0, 0, 0, 3, 22, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 1, 0, 94, 0, 3, 0, 0, 0, 0, 0, 0, 126, 4, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7133, 0, 0, 0, 0, 0, 0, 2115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 7123, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7618, 0, 0, 0, 0, 4, 0, 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 193, 0, 0, 8087, 8092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7930, 0, 0, 0, 0, 7, 0, 399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 0, 0, 7965, 8132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7718, 0, 0, 0, 0, 1, 0, 706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 315, 0, 0, 7792, 8057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 18, 0, 7865, 330, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6235, 1, 0, 0, 0, 0, 0, 5287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1246, 3208, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 21, 0, 1, 0, 0, 0, 0, 8170, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 6585, 1655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8221, 11524, 4492, 8447, 8199, 8201, 8248] avg=8192.0000 max=11524 max_rank=2 min=4492 min_rank=3 imbalance_ratio=1.406738 max_load_reduction=54012 max_load_reduction_pct=0.824158 moved_to_echo=33095 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7830, 331, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6370, 0, 0, 0, 0, 0, 0, 447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1250, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 0, 0, 7512, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 5, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6604, 1690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8128, 8168, 6817, 9441, 8310, 8187, 8177, 8308] avg=8192.0000 max=9441 max_rank=3 min=6817 min_rank=2 imbalance_ratio=1.152466 max_load_reduction=56095 max_load_reduction_pct=0.855942 moved_to_echo=32982 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7875, 331, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6267, 0, 0, 0, 0, 7, 0, 399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1251, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 0, 0, 7965, 257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6610, 1663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8212, 6673, 9443, 8358, 8188, 8176, 8285] avg=8192.0000 max=9443 max_rank=3 min=6673 min_rank=2 imbalance_ratio=1.152710 max_load_reduction=56093 max_load_reduction_pct=0.855911 moved_to_echo=33007 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 138, 0, 7629, 309, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5877, 4, 0, 0, 0, 24, 0, 6166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1166, 3146, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8189, 249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 22, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 1, 0, 94, 0, 3, 0, 0, 0, 0, 8166, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 4, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1, 6162, 1560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8085, 12071, 4398, 8439, 8211, 8273, 7856] avg=8192.0000 max=12071 max_rank=2 min=4398 min_rank=3 imbalance_ratio=1.473511 max_load_reduction=53465 max_load_reduction_pct=0.815811 moved_to_echo=32779 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7837, 331, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6020, 0, 0, 0, 0, 4, 0, 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1251, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 193, 0, 0, 8087, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6610, 1598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8174, 6606, 9442, 8535, 8187, 8177, 8209] avg=8192.0000 max=9442 max_rank=3 min=6606 min_rank=2 imbalance_ratio=1.152588 max_load_reduction=56094 max_load_reduction_pct=0.855927 moved_to_echo=33108 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7803, 331, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6099, 0, 0, 0, 0, 1, 0, 706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1251, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 315, 0, 0, 7792, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6605, 1619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8141, 6806, 9442, 8361, 8187, 8177, 8233] avg=8192.0000 max=9442 max_rank=3 min=6806 min_rank=2 imbalance_ratio=1.152588 max_load_reduction=56094 max_load_reduction_pct=0.855927 moved_to_echo=32958 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7913, 331, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5637, 0, 0, 0, 0, 0, 0, 2115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1251, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 7123, 258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6608, 1496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8250, 7752, 9443, 7417, 8188, 8176, 8104] avg=8192.0000 max=9443 max_rank=3 min=7417 min_rank=4 imbalance_ratio=1.152710 max_load_reduction=56093 max_load_reduction_pct=0.855911 moved_to_echo=31885 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7938, 337, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6214, 0, 0, 0, 0, 0, 0, 1073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1243, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 121, 0, 0, 7294, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6612, 1652, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8285, 7287, 9435, 7669, 8189, 8179, 8293] avg=8192.0000 max=9435 max_rank=3 min=7287 min_rank=2 imbalance_ratio=1.151733 max_load_reduction=56101 max_load_reduction_pct=0.856033 moved_to_echo=32330 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7099, 0, 0, 0, 0, 0, 0, 3082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 6163, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7887, 0, 0, 0, 0, 0, 0, 1668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 0, 0, 6613, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7804, 0, 0, 0, 0, 0, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 316, 0, 0, 7525, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7962, 0, 0, 0, 0, 0, 0, 990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 751, 0, 0, 6753, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 6, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7184, 0, 0, 0, 0, 0, 0, 1834, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 353, 0, 0, 7031, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 8190, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 23, 0, 1, 0, 7904, 1, 0, 0, 0, 0, 0, 5062, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8157, 3367, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 30, 0, 1, 0, 0, 0, 0, 0, 0, 12, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 125, 0, 0, 0, 7599, 4, 0, 0, 0, 0, 0, 5971, 0, 0, 0, 0, 0, 0, 0, 2, 0, 7751, 2869, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8189, 8017, 0, 0, 0, 0, 0, 0, 0, 3, 24, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 1, 0, 127, 0, 3, 0, 0, 0, 0, 0, 0, 183, 4, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7844, 0, 0, 0, 0, 1, 0, 715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 0, 0, 7629, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5844, 0, 0, 0, 0, 0, 0, 1668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1257, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 0, 0, 6613, 700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7492, 708, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6227, 2043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8176, 7512, 9449, 7492, 8188, 8219, 8306] avg=8192.0000 max=9449 max_rank=3 min=7492 min_rank=4 imbalance_ratio=1.153442 max_load_reduction=56087 max_load_reduction_pct=0.855820 moved_to_echo=32205 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5787, 0, 0, 0, 0, 0, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1264, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 316, 0, 0, 7525, 702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7461, 706, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6222, 2017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8173, 6555, 9455, 8543, 8188, 8179, 8240] avg=8192.0000 max=9455 max_rank=3 min=6555 min_rank=2 imbalance_ratio=1.154175 max_load_reduction=56081 max_load_reduction_pct=0.855728 moved_to_echo=33150 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 23, 0, 8166, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5861, 1, 0, 0, 0, 0, 0, 5062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1258, 3367, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 30, 0, 1, 0, 0, 0, 0, 7456, 703, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 6196, 2043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8196, 10925, 4663, 8892, 8199, 8206, 8254] avg=8192.0000 max=10925 max_rank=2 min=4663 min_rank=3 imbalance_ratio=1.333618 max_load_reduction=54611 max_load_reduction_pct=0.833298 moved_to_echo=33551 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 125, 0, 8163, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5635, 4, 0, 0, 0, 0, 0, 5971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1196, 2869, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8189, 689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 24, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 1, 0, 127, 0, 3, 0, 0, 0, 0, 7328, 668, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 4, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1, 5887, 1964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8296, 11610, 4153, 8879, 8212, 8144, 8042] avg=8192.0000 max=11610 max_rank=2 min=4153 min_rank=3 imbalance_ratio=1.417236 max_load_reduction=53926 max_load_reduction_pct=0.822845 moved_to_echo=33277 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5904, 0, 0, 0, 0, 0, 0, 990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1261, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 751, 0, 0, 6753, 702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7471, 705, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 6, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6213, 2058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8146, 8173, 6894, 9453, 8206, 8188, 8188, 8288] avg=8192.0000 max=9453 max_rank=3 min=6894 min_rank=2 imbalance_ratio=1.153931 max_load_reduction=56083 max_load_reduction_pct=0.855759 moved_to_echo=32870 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5327, 0, 0, 0, 0, 0, 0, 1834, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1263, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 353, 0, 0, 7031, 703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7482, 705, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6218, 1857, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8174, 7161, 9454, 8087, 8187, 8199, 8086] avg=8192.0000 max=9454 max_rank=3 min=7161 min_rank=2 imbalance_ratio=1.154053 max_load_reduction=56082 max_load_reduction_pct=0.855743 moved_to_echo=32559 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5264, 0, 0, 0, 0, 0, 0, 3082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1263, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 6163, 704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7482, 706, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6222, 1835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8174, 8346, 9454, 6915, 8187, 8200, 8057] avg=8192.0000 max=9454 max_rank=3 min=6915 min_rank=4 imbalance_ratio=1.154053 max_load_reduction=56082 max_load_reduction_pct=0.855743 moved_to_echo=31359 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5817, 0, 0, 0, 0, 1, 0, 715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1264, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 0, 0, 7629, 704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7483, 706, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6222, 2027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8174, 6533, 9455, 8522, 8187, 8201, 8263] avg=8192.0000 max=9455 max_rank=3 min=6533 min_rank=2 imbalance_ratio=1.154175 max_load_reduction=56081 max_load_reduction_pct=0.855728 moved_to_echo=33173 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 256, 128, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 128, 52480, 13056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 0, 9984, 55552, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 48768, 128, 0, 0, 0, 128, 0, 16896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 256, 0, 62720, 2688, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 1408, 0, 0, 62208, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 0, 0, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 256, 128, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 128, 49408, 15872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 45440, 128, 0, 0, 0, 128, 0, 20096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 0, 10112, 55424, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 0, 0, 128, 128, 128, 128, 256, 0, 128, 0, 0, 0, 0, 59776, 5632, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 1920, 0, 0, 58112, 5632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 0, 0, 0, 128, 128, 128, 128, 128, 0, 128, 0, 0, 0, 0, 65408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 0, 0, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 256, 0, 65408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 766, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7451, 0, 0, 0, 0, 0, 6895, 0, 0, 0, 0, 0, 94, 12, 0, 0, 1108, 0, 0, 0, 8192, 0, 0, 0, 14, 8191, 0, 0, 0, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 50, 0, 1, 0, 0, 0, 0, 0, 0, 0, 180, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3860, 0, 0, 0, 0, 323, 0, 1, 0, 1, 0, 0, 4671, 0, 0, 0, 0, 0, 7295, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 1, 0, 8187, 0, 0, 0, 1, 11, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 8190, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 1, 8190, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 299, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7823, 0, 0, 0, 0, 0, 7739, 0, 0, 0, 0, 0, 0, 31, 0, 0, 438, 0, 0, 0, 8192, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4318, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8083, 0, 0, 0, 0, 0, 3757, 0, 0, 0, 0, 0, 0, 2, 0, 0, 169, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1551, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7521, 0, 0, 0, 0, 0, 6914, 0, 0, 0, 0, 0, 0, 5, 0, 0, 290, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1259, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7932, 0, 0, 0, 0, 0, 6956, 0, 0, 0, 0, 0, 17, 34, 0, 0, 175, 0, 0, 0, 8192, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 125, 0, 0, 0, 2, 0, 0, 0, 0, 0, 135, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 3, 0, 0, 0, 0, 0, 0, 3521, 0, 0, 0, 0, 1705, 0, 0, 0, 3, 0, 0, 2884, 0, 0, 0, 0, 0, 8120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8097, 0, 0, 2, 0, 8156, 0, 0, 0, 3, 14, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 3, 8188, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 8183, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7665, 0, 0, 0, 0, 0, 6977, 0, 0, 0, 0, 0, 16, 3, 0, 0, 1118, 0, 0, 0, 8192, 0, 0, 0, 7, 8192, 0, 0, 0, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 50, 0, 1, 0, 0, 0, 0, 0, 0, 0, 180, 0, 0, 8130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 8187, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3860, 0, 0, 0, 0, 323, 0, 1, 0, 1, 0, 0, 4099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7295, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 572, 53, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8126, 0, 0, 1, 0, 0, 0, 0, 0, 1, 11, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8190, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8361, 8195, 8284, 7960, 8143, 8192, 8193, 8208] avg=8192.0000 max=8361 max_rank=0 min=7960 min_rank=3 imbalance_ratio=1.020630 max_load_reduction=57175 max_load_reduction_pct=0.872421 moved_to_echo=32736 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 299, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7739, 0, 0, 0, 0, 0, 0, 31, 0, 0, 438, 0, 0, 958, 53, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8133, 8194, 7178, 9258, 8210, 8192, 8193, 8178] avg=8192.0000 max=9258 max_rank=3 min=7178 min_rank=2 imbalance_ratio=1.130127 max_load_reduction=56278 max_load_reduction_pct=0.858734 moved_to_echo=32773 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8187, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 766, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6895, 0, 0, 0, 0, 0, 94, 12, 0, 0, 1108, 0, 0, 913, 53, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8139, 0, 0, 0, 14, 0, 0, 0, 0, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8136, 8192, 7306, 9114, 8254, 8192, 8189, 8153] avg=8192.0000 max=9114 max_rank=3 min=7306 min_rank=2 imbalance_ratio=1.112549 max_load_reduction=56422 max_load_reduction_pct=0.860931 moved_to_echo=32788 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1551, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 6600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6914, 0, 0, 0, 0, 0, 0, 5, 0, 0, 290, 0, 0, 921, 55, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8135, 8189, 8152, 8224, 8245, 8192, 8195, 8204] avg=8192.0000 max=8245 max_rank=4 min=8135 min_rank=0 imbalance_ratio=1.006470 max_load_reduction=57291 max_load_reduction_pct=0.874191 moved_to_echo=32836 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 125, 0, 0, 0, 2, 0, 0, 0, 0, 0, 135, 0, 0, 8100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 3, 0, 0, 0, 0, 0, 8186, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3521, 0, 0, 0, 0, 1705, 0, 0, 0, 3, 0, 0, 2531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 353, 53, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8044, 0, 0, 2, 0, 0, 0, 0, 0, 3, 14, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 3, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8362, 8195, 7760, 8567, 8066, 8193, 8191, 8202] avg=8192.0000 max=8567 max_rank=3 min=7760 min_rank=2 imbalance_ratio=1.045776 max_load_reduction=56969 max_load_reduction_pct=0.869278 moved_to_echo=32652 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4318, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3757, 0, 0, 0, 0, 0, 0, 2, 0, 0, 169, 0, 0, 990, 53, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8137, 8190, 11412, 5010, 8199, 8192, 8195, 8201] avg=8192.0000 max=11412 max_rank=2 min=5010 min_rank=3 imbalance_ratio=1.393066 max_load_reduction=54124 max_load_reduction_pct=0.825867 moved_to_echo=32787 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6977, 0, 0, 0, 0, 0, 16, 3, 0, 0, 1118, 0, 0, 939, 53, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8139, 0, 0, 0, 7, 0, 0, 0, 0, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8136, 8187, 7225, 9145, 8275, 8192, 8188, 8188] avg=8192.0000 max=9145 max_rank=3 min=7225 min_rank=2 imbalance_ratio=1.116333 max_load_reduction=56391 max_load_reduction_pct=0.860458 moved_to_echo=32843 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1259, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6956, 0, 0, 0, 0, 0, 17, 34, 0, 0, 175, 0, 0, 977, 53, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8136, 8194, 8219, 8258, 8144, 8191, 8192, 8202] avg=8192.0000 max=8258 max_rank=3 min=8136 min_rank=0 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32729 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 7250, 0, 0, 0, 0, 0, 6194, 0, 0, 0, 0, 0, 0, 5, 0, 0, 2705, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 60, 0, 1, 0, 0, 0, 0, 0, 0, 0, 170, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1809, 0, 0, 0, 0, 594, 0, 1, 0, 1, 0, 0, 5891, 0, 0, 0, 0, 0, 7861, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 1, 0, 8187, 0, 0, 0, 1, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 8190, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 1, 8190, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 828, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8049, 0, 0, 0, 0, 0, 6265, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1195, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 133, 0, 0, 0, 2, 0, 0, 0, 0, 0, 133, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1259, 0, 0, 0, 0, 2782, 0, 0, 0, 3, 0, 0, 3968, 0, 0, 0, 0, 0, 8107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 3, 0, 8094, 0, 0, 2, 0, 8155, 0, 0, 0, 3, 66, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 3, 8188, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 8183, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 35, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7219, 0, 0, 0, 0, 0, 6000, 0, 0, 0, 0, 0, 61, 14, 0, 0, 2976, 0, 0, 0, 8192, 0, 0, 0, 14, 8191, 0, 0, 0, 0, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 427, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7977, 0, 0, 0, 0, 0, 6047, 0, 0, 0, 0, 0, 2, 30, 0, 0, 1851, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7320, 0, 0, 0, 0, 0, 5051, 0, 0, 0, 0, 0, 12, 3, 0, 0, 3906, 0, 0, 0, 8192, 0, 0, 0, 7, 8192, 0, 0, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7577, 0, 0, 0, 0, 0, 6874, 0, 0, 0, 0, 0, 0, 29, 0, 0, 1842, 0, 0, 0, 8192, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 60, 0, 1, 0, 0, 0, 0, 0, 0, 0, 170, 0, 0, 8129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 8187, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1809, 0, 0, 0, 0, 594, 0, 1, 0, 1, 0, 0, 5891, 0, 0, 233, 54, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7628, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8115, 0, 0, 1, 0, 0, 0, 0, 0, 1, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8190, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8360, 8195, 8634, 7629, 8125, 8192, 8193, 8208] avg=8192.0000 max=8634 max_rank=2 min=7629 min_rank=3 imbalance_ratio=1.053955 max_load_reduction=56902 max_load_reduction_pct=0.868256 moved_to_echo=32718 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 7250, 0, 0, 184, 56, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6010, 0, 0, 0, 0, 0, 0, 5, 0, 0, 2705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8134, 8190, 7650, 8720, 8253, 8192, 8195, 8202] avg=8192.0000 max=8720 max_rank=3 min=7650 min_rank=2 imbalance_ratio=1.064453 max_load_reduction=56816 max_load_reduction_pct=0.866943 moved_to_echo=32842 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 133, 0, 0, 0, 2, 0, 0, 0, 0, 0, 133, 0, 0, 8099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 3, 0, 0, 0, 0, 0, 8186, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1259, 0, 0, 0, 0, 2782, 0, 0, 0, 3, 0, 0, 3968, 0, 0, 241, 53, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8030, 0, 0, 2, 0, 0, 0, 0, 0, 3, 66, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 3, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8367, 8195, 8356, 7927, 8105, 8193, 8191, 8202] avg=8192.0000 max=8367 max_rank=0 min=7927 min_rank=3 imbalance_ratio=1.021362 max_load_reduction=57169 max_load_reduction_pct=0.872330 moved_to_echo=32691 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8187, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 35, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7219, 0, 0, 178, 54, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5822, 0, 0, 0, 0, 0, 61, 14, 0, 0, 2976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8127, 0, 0, 0, 14, 0, 0, 0, 0, 0, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8136, 8192, 7540, 8873, 8259, 8192, 8189, 8155] avg=8192.0000 max=8873 max_rank=3 min=7540 min_rank=2 imbalance_ratio=1.083130 max_load_reduction=56663 max_load_reduction_pct=0.864609 moved_to_echo=32795 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7320, 0, 0, 150, 54, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4901, 0, 0, 0, 0, 0, 12, 3, 0, 0, 3906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8127, 0, 0, 0, 7, 0, 0, 0, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8135, 8187, 7576, 8822, 8248, 8192, 8188, 8188] avg=8192.0000 max=8822 max_rank=3 min=7576 min_rank=2 imbalance_ratio=1.076904 max_load_reduction=56714 max_load_reduction_pct=0.865387 moved_to_echo=32816 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 427, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7977, 0, 0, 182, 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5865, 0, 0, 0, 0, 0, 2, 30, 0, 0, 1851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8136, 8193, 8708, 7748, 8167, 8191, 8192, 8201] avg=8192.0000 max=8708 max_rank=2 min=7748 min_rank=3 imbalance_ratio=1.062988 max_load_reduction=56828 max_load_reduction_pct=0.867126 moved_to_echo=32751 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 828, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8049, 0, 0, 186, 54, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6079, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8136, 8190, 9169, 7276, 8177, 8192, 8195, 8201] avg=8192.0000 max=9169 max_rank=2 min=7276 min_rank=3 imbalance_ratio=1.119263 max_load_reduction=56367 max_load_reduction_pct=0.860092 moved_to_echo=32765 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7577, 0, 0, 204, 54, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6670, 0, 0, 0, 0, 0, 0, 29, 0, 0, 1842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8132, 8194, 7903, 8541, 8202, 8192, 8193, 8179] avg=8192.0000 max=8541 max_rank=3 min=7903 min_rank=2 imbalance_ratio=1.042603 max_load_reduction=56995 max_load_reduction_pct=0.869675 moved_to_echo=32766 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 54656, 0, 0, 0, 0, 128, 128, 128, 0, 0, 3328, 0, 128, 6656, 512, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 16128, 0, 128, 0, 0, 2048, 0, 128, 0, 128, 0, 0, 47488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 4480, 0, 128, 0, 0, 3456, 0, 128, 0, 128, 0, 0, 55296, 0, 0, 1664, 512, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 50944, 0, 0, 0, 0, 128, 128, 128, 0, 0, 14592, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 256, 0, 128, 0, 128, 0, 0, 0, 0, 0, 384, 0, 0, 65152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 65536, 0, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 256, 0, 128, 0, 128, 0, 0, 0, 0, 0, 384, 128, 0, 65152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 128, 0, 0, 128, 128, 0, 0, 0, 128, 65536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 65024, 0, 0, 128, 128, 0, 0, 0, 0, 128, 640, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 128, 0, 0, 128, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 65024, 0, 0, 128, 128, 0, 0, 0, 0, 128, 512, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 128, 0, 0, 128, 128, 0, 0, 0, 128, 65536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 65536, 0, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8043, 0, 8192, 0, 0, 0, 0, 0, 649, 0, 0, 4, 0, 0, 0, 0, 0, 0, 37, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 7905, 0, 0, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7798, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 11, 1, 191, 8190, 0, 8185, 1, 0, 0, 2, 0, 2103, 0, 0, 0, 0, 3180, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 2, 2, 0, 8190, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 0, 0, 0, 0, 0, 0, 1, 0, 8190, 0, 0, 1481, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1232, 0, 0, 0, 0, 25] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 8141, 0, 8192, 0, 0, 0, 0, 0, 560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8100, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7722, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 54, 2, 117, 8188, 0, 8184, 0, 3, 0, 3, 0, 784, 0, 0, 0, 0, 3694, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 2, 0, 0, 0, 3, 3, 3, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8146, 0, 0, 0, 0, 0, 0, 1, 0, 8189, 0, 0, 1173, 3, 0, 0, 1, 0, 2, 1, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2400, 8, 0, 0, 0, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 8192, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 8192, 0, 0, 0, 0, 0, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7987, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 8192, 0, 0, 0, 0, 0, 525, 0, 0, 3, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 8124, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7725, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 8192, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8115, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7500, 0, 0, 0, 0, 0, 0, 0, 560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 1, 0, 0, 0, 0, 7324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7794, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7722, 0, 0, 0, 0, 0, 868, 641, 284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8060, 7333, 8191, 8203, 7863, 8194, 9515] avg=8192.0000 max=9515 max_rank=7 min=7333 min_rank=2 imbalance_ratio=1.161499 max_load_reduction=56021 max_load_reduction_pct=0.854813 moved_to_echo=33775 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1, 191, 7545, 0, 0, 1, 0, 0, 2, 0, 2103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3180, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 2, 2, 0, 8189, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7879, 0, 0, 1481, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1232, 0, 0, 0, 0, 25, 868, 645, 287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8211, 9854, 10498, 8197, 8163, 9362, 8193, 3058] avg=8192.0000 max=10498 max_rank=2 min=3058 min_rank=7 imbalance_ratio=1.281494 max_load_reduction=55038 max_load_reduction_pct=0.839813 moved_to_echo=28776 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 2, 117, 7543, 0, 0, 0, 3, 0, 3, 0, 784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3694, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 7316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 3, 3, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8146, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7878, 0, 0, 1173, 3, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 1, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2400, 8, 0, 0, 0, 5, 868, 645, 287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8506, 11014, 8199, 8157, 9056, 8191, 4214] avg=8192.0000 max=11014 max_rank=2 min=4214 min_rank=7 imbalance_ratio=1.344482 max_load_reduction=54522 max_load_reduction_pct=0.831940 moved_to_echo=29618 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7410, 0, 0, 0, 0, 0, 0, 0, 649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 37, 0, 0, 2, 0, 0, 0, 2, 7323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7606, 0, 0, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7798, 0, 0, 0, 0, 0, 869, 633, 277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8059, 7368, 8191, 8201, 7757, 8181, 9577] avg=8192.0000 max=9577 max_rank=7 min=7368 min_rank=2 imbalance_ratio=1.169067 max_load_reduction=55959 max_load_reduction_pct=0.853867 moved_to_echo=33716 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7515, 0, 0, 0, 0, 0, 0, 0, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 2, 7323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7815, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7725, 0, 0, 0, 0, 0, 869, 642, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8040, 7348, 8191, 8200, 7861, 8184, 9521] avg=8192.0000 max=9521 max_rank=7 min=7348 min_rank=2 imbalance_ratio=1.162231 max_load_reduction=56015 max_load_reduction_pct=0.854721 moved_to_echo=33766 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7543, 0, 0, 0, 0, 0, 0, 0, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7856, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7987, 0, 0, 0, 0, 0, 869, 645, 286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8173, 7794, 7324, 8191, 8203, 7869, 8195, 9787] avg=8192.0000 max=9787 max_rank=7 min=7324 min_rank=2 imbalance_ratio=1.194702 max_load_reduction=55749 max_load_reduction_pct=0.850662 moved_to_echo=34054 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7537, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 7323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7856, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 0, 0, 0, 0, 0, 869, 644, 286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 7618, 7327, 8191, 8203, 7887, 8195, 9916] avg=8192.0000 max=9916 max_rank=7 min=7327 min_rank=2 imbalance_ratio=1.210449 max_load_reduction=55620 max_load_reduction_pct=0.848694 moved_to_echo=34201 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7509, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7796, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 4, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 0, 0, 0, 0, 0, 868, 647, 288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 7605, 7324, 8185, 8206, 7881, 8203, 9948] avg=8192.0000 max=9948 max_rank=7 min=7324 min_rank=2 imbalance_ratio=1.214355 max_load_reduction=55588 max_load_reduction_pct=0.848206 moved_to_echo=34238 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 8069, 0, 8192, 0, 0, 0, 0, 0, 471, 0, 0, 4, 0, 0, 0, 0, 0, 0, 38, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 7829, 0, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8075, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8125, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 8192, 0, 0, 0, 0, 0, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8150, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 0, 0, 0, 0, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 8192, 0, 0, 0, 0, 0, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8142, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 8192, 0, 0, 0, 0, 0, 131, 0, 0, 3, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 8115, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 9, 1, 108, 8190, 0, 8184, 1, 0, 0, 2, 0, 2244, 0, 0, 0, 0, 4005, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 2, 2, 0, 8190, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8103, 0, 0, 0, 0, 0, 0, 1, 0, 8190, 0, 0, 54, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1731, 0, 0, 0, 0, 129] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 36, 2, 42, 8187, 0, 8181, 0, 3, 0, 3, 0, 483, 0, 0, 0, 0, 4622, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 2, 0, 0, 0, 3, 3, 4, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8134, 0, 0, 0, 0, 0, 0, 1, 0, 8188, 0, 0, 101, 3, 0, 0, 1, 0, 2, 1, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2833, 8, 0, 0, 0, 127] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 0, 0, 0, 0, 0, 0, 0, 8134, 0, 8192, 0, 0, 0, 0, 0, 218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8088, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8118, 0, 0, 0, 0, 6] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8148, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 8192, 0, 0, 0, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8118, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7596, 0, 0, 0, 0, 0, 0, 0, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 38, 0, 0, 1, 0, 0, 0, 2, 7104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7829, 0, 0, 115, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8075, 0, 0, 0, 0, 0, 1088, 395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8067, 7149, 8191, 8206, 7974, 8179, 9558] avg=8192.0000 max=9558 max_rank=7 min=7149 min_rank=2 imbalance_ratio=1.166748 max_load_reduction=55978 max_load_reduction_pct=0.854156 moved_to_echo=33917 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 108, 7710, 0, 0, 1, 0, 0, 2, 0, 2244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4005, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 2, 2, 0, 8189, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8103, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8190, 0, 0, 54, 1, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1731, 0, 0, 0, 0, 129, 1087, 401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 10075, 11103, 8197, 8122, 8276, 8193, 3349] avg=8192.0000 max=11103 max_rank=2 min=3349 min_rank=7 imbalance_ratio=1.355347 max_load_reduction=54433 max_load_reduction_pct=0.830582 moved_to_echo=27940 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8125, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7704, 0, 0, 0, 0, 0, 0, 0, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8150, 0, 0, 6, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 0, 0, 0, 0, 5, 1088, 401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8153, 7870, 7104, 8191, 8211, 8186, 8195, 9626] avg=8192.0000 max=9626 max_rank=7 min=7104 min_rank=2 imbalance_ratio=1.175049 max_load_reduction=55910 max_load_reduction_pct=0.853119 moved_to_echo=34218 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 2, 42, 7707, 0, 0, 0, 3, 0, 3, 0, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4622, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 7096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 3, 4, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8134, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8188, 0, 0, 101, 3, 0, 0, 1, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2833, 8, 0, 0, 0, 127, 1085, 401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8276, 11722, 8199, 8154, 8324, 8190, 4455] avg=8192.0000 max=11722 max_rank=2 min=4455 min_rank=7 imbalance_ratio=1.430908 max_load_reduction=53814 max_load_reduction_pct=0.821136 moved_to_echo=29123 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7680, 0, 0, 0, 0, 0, 0, 0, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 2, 7104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8115, 0, 0, 38, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 0, 0, 0, 0, 0, 1088, 399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 7811, 7132, 8191, 8208, 8183, 8183, 9632] avg=8192.0000 max=9632 max_rank=7 min=7132 min_rank=2 imbalance_ratio=1.175781 max_load_reduction=55904 max_load_reduction_pct=0.853027 moved_to_echo=34206 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8148, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7655, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8118, 0, 0, 69, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 2, 1087, 403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 7750, 7105, 8185, 8213, 8221, 8206, 9677] avg=8192.0000 max=9677 max_rank=7 min=7105 min_rank=2 imbalance_ratio=1.181274 max_load_reduction=55859 max_load_reduction_pct=0.852341 moved_to_echo=34317 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7705, 0, 0, 0, 0, 0, 0, 0, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 7105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 15, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8142, 0, 0, 0, 0, 0, 1087, 401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 7812, 7108, 8191, 8211, 8203, 8195, 9630] avg=8192.0000 max=9630 max_rank=7 min=7108 min_rank=2 imbalance_ratio=1.175537 max_load_reduction=55906 max_load_reduction_pct=0.853058 moved_to_echo=34239 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7657, 0, 0, 0, 0, 0, 0, 0, 218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 1, 0, 0, 0, 0, 7105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8088, 0, 0, 51, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8118, 0, 0, 0, 0, 6, 1087, 398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8173, 7875, 7113, 8191, 8211, 8169, 8195, 9609] avg=8192.0000 max=9609 max_rank=7 min=7113 min_rank=2 imbalance_ratio=1.172974 max_load_reduction=55927 max_load_reduction_pct=0.853378 moved_to_echo=34184 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 51200, 128, 0, 0, 0, 128, 7040, 5248, 2304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 0, 6912, 0, 0, 0, 0, 128, 0, 128, 128, 0, 128, 128, 128, 58624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 128, 384, 60160, 0, 0, 128, 128, 0, 128, 0, 5120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 0, 62592, 0, 0, 3072, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 53376, 128, 0, 0, 0, 384, 8704, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 8704, 0, 0, 0, 0, 128, 0, 128, 128, 0, 128, 0, 128, 56832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 128, 256, 61440, 0, 0, 128, 128, 0, 128, 0, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65408, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 0, 64896, 0, 0, 512, 128, 0, 0, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 0, 128, 128, 128, 128, 0, 65536, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65408, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 8166, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 8189, 9, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 8181, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8192, 0, 0, 8192, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 8187, 0, 0, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 180, 8183, 0, 0, 545, 1916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8035, 0, 0, 0, 0, 0, 3, 2, 0, 0, 3, 2, 0, 0, 1217, 0, 0, 8025, 4, 0, 0, 0, 2, 0, 3, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4510, 8121, 0, 0, 8188, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 24, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 9, 0, 0, 0, 4, 0, 1, 163, 0, 8187, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 8190, 0, 0, 453, 3356, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8095, 0, 0, 0, 0, 0, 2, 1, 0, 0, 1, 0, 0, 0, 879, 0, 0, 8046, 2, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3506, 8188, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 112, 0, 8190, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 8192, 55, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8189, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 6104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7185, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5856, 2088, 1007, 225, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6104, 10508, 8221, 7967, 8192, 7209, 9178, 8157] avg=8192.0000 max=10508 max_rank=1 min=6104 min_rank=0 imbalance_ratio=1.282715 max_load_reduction=55028 max_load_reduction_pct=0.839661 moved_to_echo=32736 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 6102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 453, 952, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8095, 0, 0, 0, 0, 0, 2, 1, 0, 0, 1, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 879, 0, 0, 7825, 2, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3506, 7181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2404, 2088, 1007, 221, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 112, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14415, 1406, 8132, 8709, 8192, 10688, 5726, 8268] avg=8192.0000 max=14415 max_rank=0 min=1406 min_rank=1 imbalance_ratio=1.759644 max_load_reduction=51121 max_load_reduction_pct=0.780045 moved_to_echo=32874 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 6102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7184, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5858, 2088, 1007, 225, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6102, 10526, 8216, 7967, 8192, 7195, 9181, 8157] avg=8192.0000 max=10526 max_rank=1 min=6102 min_rank=0 imbalance_ratio=1.284912 max_load_reduction=55010 max_load_reduction_pct=0.839386 moved_to_echo=32725 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 6102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7182, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5828, 2088, 1007, 225, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6102, 10510, 8215, 7967, 8195, 7239, 9151, 8157] avg=8192.0000 max=10510 max_rank=1 min=6102 min_rank=0 imbalance_ratio=1.282959 max_load_reduction=55026 max_load_reduction_pct=0.839630 moved_to_echo=32742 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 6100, 0, 0, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 180, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8035, 0, 0, 0, 0, 0, 3, 2, 0, 0, 3, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1217, 0, 0, 7804, 4, 0, 0, 0, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4510, 7123, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1373, 2087, 998, 221, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 9, 0, 0, 0, 4, 0, 1, 163, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14474, 1088, 8076, 9032, 8191, 11636, 4709, 8330] avg=8192.0000 max=14474 max_rank=0 min=1088 min_rank=1 imbalance_ratio=1.766846 max_load_reduction=51062 max_load_reduction_pct=0.779144 moved_to_echo=32866 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 6104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 2322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5869, 2088, 1007, 225, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6129, 10489, 8225, 7967, 8191, 7186, 9192, 8157] avg=8192.0000 max=10489 max_rank=1 min=6129 min_rank=0 imbalance_ratio=1.280396 max_load_reduction=55047 max_load_reduction_pct=0.839951 moved_to_echo=32726 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 6104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 2321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7185, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5869, 2088, 1007, 225, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6106, 10513, 8224, 7967, 8191, 7186, 9192, 8157] avg=8192.0000 max=10513 max_rank=1 min=6106 min_rank=0 imbalance_ratio=1.283325 max_load_reduction=55023 max_load_reduction_pct=0.839584 moved_to_echo=32726 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 6100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 2315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7181, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5868, 2092, 1011, 232, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6104, 10496, 8227, 7960, 8192, 7197, 9207, 8153] avg=8192.0000 max=10496 max_rank=1 min=6104 min_rank=0 imbalance_ratio=1.281250 max_load_reduction=55040 max_load_reduction_pct=0.839844 moved_to_echo=32749 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 8185, 0, 0, 35, 5280, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8092, 0, 0, 0, 0, 0, 2, 1, 0, 0, 1, 0, 0, 0, 834, 0, 0, 8098, 2, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2049, 8188, 0, 0, 8190, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 104, 0, 8190, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 8189, 9, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 8179, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 8192, 48, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 8187, 0, 0, 12, 4, 0, 0, 0, 0, 0, 0, 0, 0, 158, 8183, 0, 0, 100, 3578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8024, 0, 0, 0, 0, 0, 3, 2, 0, 0, 3, 3, 0, 0, 1189, 0, 0, 8080, 4, 0, 0, 0, 2, 0, 3, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3321, 8104, 0, 0, 8188, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 13, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 9, 0, 0, 0, 4, 0, 0, 164, 0, 8187, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8189, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 8173, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 8192, 0, 0, 8192, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 6112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 1483, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8092, 0, 0, 0, 0, 0, 2, 1, 0, 0, 1, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 834, 0, 0, 7871, 2, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2049, 7515, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3797, 2078, 673, 227, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 104, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14379, 1519, 8130, 8709, 8192, 9566, 6779, 8262] avg=8192.0000 max=14379 max_rank=0 min=1519 min_rank=1 imbalance_ratio=1.755249 max_load_reduction=51157 max_load_reduction_pct=0.780594 moved_to_echo=32799 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 6113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 2300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5892, 2079, 673, 229, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6124, 10479, 8226, 7963, 8191, 7521, 8874, 8158] avg=8192.0000 max=10479 max_rank=1 min=6124 min_rank=0 imbalance_ratio=1.279175 max_load_reduction=55057 max_load_reduction_pct=0.840103 moved_to_echo=32744 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 6112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7518, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5882, 2079, 673, 229, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6112, 10501, 8217, 7963, 8191, 7530, 8864, 8158] avg=8192.0000 max=10501 max_rank=1 min=6112 min_rank=0 imbalance_ratio=1.281860 max_load_reduction=55035 max_load_reduction_pct=0.839767 moved_to_echo=32743 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 6112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7516, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5855, 2078, 673, 229, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6112, 10487, 8220, 7963, 8194, 7566, 8836, 8158] avg=8192.0000 max=10487 max_rank=1 min=6112 min_rank=0 imbalance_ratio=1.280151 max_load_reduction=55049 max_load_reduction_pct=0.839981 moved_to_echo=32754 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 6109, 0, 0, 12, 4, 0, 0, 0, 0, 0, 0, 0, 0, 158, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8024, 0, 0, 0, 0, 0, 3, 2, 0, 0, 3, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1189, 0, 0, 7854, 4, 0, 0, 0, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3321, 7438, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2573, 2078, 666, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 9, 0, 0, 0, 4, 0, 0, 164, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14466, 1105, 8066, 9055, 8192, 10762, 5559, 8331] avg=8192.0000 max=14466 max_rank=0 min=1105 min_rank=1 imbalance_ratio=1.765869 max_load_reduction=51070 max_load_reduction_pct=0.779266 moved_to_echo=32844 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 6109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 2295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 7515, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5895, 2083, 677, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6115, 10468, 8232, 7957, 8192, 7530, 8889, 8153] avg=8192.0000 max=10468 max_rank=1 min=6115 min_rank=0 imbalance_ratio=1.277832 max_load_reduction=55068 max_load_reduction_pct=0.840271 moved_to_echo=32764 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 6113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 2300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5892, 2079, 673, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6115, 10491, 8225, 7963, 8192, 7519, 8873, 8158] avg=8192.0000 max=10491 max_rank=1 min=6115 min_rank=0 imbalance_ratio=1.280640 max_load_reduction=55045 max_load_reduction_pct=0.839920 moved_to_echo=32742 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 6113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7519, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5881, 2079, 673, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6113, 10486, 8220, 7963, 8192, 7542, 8862, 8158] avg=8192.0000 max=10486 max_rank=1 min=6113 min_rank=0 imbalance_ratio=1.280029 max_load_reduction=55050 max_load_reduction_pct=0.839996 moved_to_echo=32754 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 39040, 16768, 8064, 1920, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 50176, 15488, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 48896, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 384, 16512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 8064, 57472, 0, 0, 0, 128, 0, 0, 0, 0, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 2176, 0, 0, 63488, 128, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 41728, 16640, 5504, 1920, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 49280, 16384, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 48896, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 256, 16512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 5504, 60160, 0, 0, 0, 128, 0, 0, 128, 0, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 2048, 0, 0, 63616, 128, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 0, 65280, 0, 0, 0, 0, 0, 128, 128, 0, 0, 128, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 128, 384, 0, 65280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 65280, 0, 0, 0, 0, 0, 128, 128, 0, 0, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 384, 0, 65280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 6954, 1, 0, 0, 0, 1, 0, 8190, 0, 0, 9, 0, 0, 0, 0, 0, 1, 0, 0, 6, 0, 0, 270, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 0, 5, 3945, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7783, 2, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 1, 0, 0, 7872, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 2, 4529, 0, 1432, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 1676, 0, 0, 0, 0, 7954, 0, 0, 0, 0, 0, 0, 0, 7064, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 7870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 2056, 0, 0, 0, 0, 7836, 0, 0, 0, 0, 0, 0, 0, 6493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 6487, 1, 0, 0, 0, 1, 0, 8188, 0, 0, 12, 0, 0, 0, 0, 0, 2, 0, 0, 20, 0, 0, 507, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 7867, 1, 2, 0, 0, 0, 0, 0, 0, 6594, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7548, 4, 0, 0, 0, 0, 2, 8161, 0, 0, 0, 0, 0, 2, 0, 6475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 3, 0, 2, 4, 3012, 0, 2399, 0, 0, 0, 1, 2, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 1798, 0, 0, 0, 0, 7943, 0, 0, 0, 0, 0, 0, 0, 6642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 1196, 0, 0, 0, 0, 8099, 0, 0, 0, 0, 0, 0, 0, 7102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 2197, 0, 0, 0, 0, 7854, 0, 0, 0, 0, 0, 0, 0, 6509, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 8014, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 668, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 7567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 6954, 1, 0, 0, 0, 1, 0, 365, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6, 0, 0, 270, 1, 0, 0, 0, 0, 0, 0, 0, 7619, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6993, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1736, 0, 0, 0, 0, 0, 0, 0, 5, 3945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8057, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7783, 2, 0, 0, 0, 0, 0, 133, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7872, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5927, 576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 4529, 0, 1432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7330, 8376, 6996, 5686, 8266, 7919, 14377, 6586] avg=8192.0000 max=14377 max_rank=6 min=5686 min_rank=3 imbalance_ratio=1.755005 max_load_reduction=51159 max_load_reduction_pct=0.780624 moved_to_echo=37148 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7621, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6994, 0, 0, 0, 0, 0, 0, 0, 2056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1671, 0, 0, 0, 0, 0, 0, 0, 6493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8037, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5704, 576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8557, 8085, 9050, 8164, 8243, 8325, 6296, 8816] avg=8192.0000 max=9050 max_rank=2 min=6296 min_rank=6 imbalance_ratio=1.104736 max_load_reduction=56486 max_load_reduction_pct=0.861908 moved_to_echo=31680 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 365, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7616, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6994, 0, 0, 0, 0, 0, 0, 0, 1676, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1696, 0, 0, 0, 0, 0, 0, 0, 7064, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 7742, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5790, 576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8560, 8084, 8670, 8762, 7957, 8312, 6366, 8825] avg=8192.0000 max=8825 max_rank=7 min=6366 min_rank=6 imbalance_ratio=1.077271 max_load_reduction=56711 max_load_reduction_pct=0.865341 moved_to_echo=31460 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 6487, 1, 0, 0, 0, 1, 0, 365, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 20, 0, 0, 507, 3, 0, 0, 0, 0, 0, 0, 0, 7617, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6987, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1678, 1, 2, 0, 0, 0, 0, 0, 0, 6594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8028, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7548, 4, 0, 0, 0, 0, 2, 133, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 5726, 576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 4, 3012, 0, 2399, 0, 0, 0, 1, 2, 0, 0, 0, 0, 621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6866, 8612, 6992, 8275, 8240, 7689, 12818, 6044] avg=8192.0000 max=12818 max_rank=6 min=6044 min_rank=7 imbalance_ratio=1.564697 max_load_reduction=52718 max_load_reduction_pct=0.804413 moved_to_echo=34791 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 364, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7616, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6994, 0, 0, 0, 0, 0, 0, 0, 2197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1675, 0, 0, 0, 0, 0, 0, 0, 6509, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 7883, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5717, 576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8559, 8078, 9191, 8185, 8095, 8319, 6294, 8815] avg=8192.0000 max=9191 max_rank=2 min=6294 min_rank=6 imbalance_ratio=1.121948 max_load_reduction=56345 max_load_reduction_pct=0.859756 moved_to_echo=31523 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7621, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6994, 0, 0, 0, 0, 0, 0, 0, 1196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1727, 0, 0, 0, 0, 0, 0, 0, 7102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8030, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5895, 576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8557, 8099, 8190, 8829, 8236, 8325, 6483, 8817] avg=8192.0000 max=8829 max_rank=3 min=6483 min_rank=6 imbalance_ratio=1.077759 max_load_reduction=56707 max_load_reduction_pct=0.865280 moved_to_echo=31861 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7621, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6994, 0, 0, 0, 0, 0, 0, 0, 668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1743, 0, 0, 0, 0, 0, 0, 0, 7567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8027, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5948, 576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8557, 8102, 7662, 9310, 8233, 8325, 6530, 8817] avg=8192.0000 max=9310 max_rank=3 min=6530 min_rank=6 imbalance_ratio=1.136475 max_load_reduction=56226 max_load_reduction_pct=0.857941 moved_to_echo=31905 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7626, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6987, 0, 0, 0, 0, 0, 0, 0, 1798, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1683, 0, 0, 0, 0, 0, 0, 0, 6642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8058, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5786, 581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8550, 8100, 8785, 8325, 8266, 8322, 6372, 8816] avg=8192.0000 max=8816 max_rank=7 min=6372 min_rank=6 imbalance_ratio=1.076172 max_load_reduction=56720 max_load_reduction_pct=0.865479 moved_to_echo=31776 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 6297, 1, 0, 0, 0, 1, 0, 8188, 0, 0, 12, 0, 0, 0, 0, 0, 2, 0, 0, 24, 0, 0, 909, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 1, 0, 0, 0, 0, 0, 0, 6655, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6974, 4, 0, 0, 0, 0, 2, 8172, 0, 0, 0, 0, 0, 3, 0, 6562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 3, 0, 2, 4, 3118, 0, 2182, 0, 0, 0, 1, 2, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7974, 0, 0, 0, 0, 8114, 0, 0, 0, 0, 0, 0, 0, 364, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 8123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 6746, 1, 0, 0, 0, 1, 0, 8190, 0, 0, 9, 0, 0, 0, 0, 0, 1, 0, 0, 7, 0, 0, 447, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 3778, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7617, 2, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 1, 0, 0, 7849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 2, 4828, 0, 1390, 0, 21, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7893, 0, 0, 0, 0, 7994, 0, 0, 0, 0, 0, 0, 0, 625, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 8061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7908, 0, 0, 0, 0, 8134, 0, 0, 0, 0, 0, 0, 0, 295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7858, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7647, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7569, 0, 0, 0, 0, 8092, 0, 0, 0, 0, 0, 0, 0, 718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 6746, 1, 0, 0, 0, 1, 0, 415, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7, 0, 0, 447, 1, 0, 0, 0, 0, 0, 0, 0, 7524, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2326, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6573, 0, 0, 0, 0, 0, 0, 0, 0, 3778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7963, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7617, 2, 0, 0, 0, 0, 0, 227, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5371, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 2, 4828, 0, 1390, 0, 21, 0, 0, 0, 0, 0, 0, 0, 606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7172, 8473, 2404, 10351, 8217, 7847, 14222, 6850] avg=8192.0000 max=14222 max_rank=6 min=2404 min_rank=2 imbalance_ratio=1.736084 max_load_reduction=51314 max_load_reduction_pct=0.782990 moved_to_echo=37136 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 6297, 1, 0, 0, 0, 1, 0, 415, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 24, 0, 0, 909, 3, 0, 0, 0, 0, 0, 0, 0, 7522, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2325, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6571, 0, 1, 0, 0, 0, 0, 0, 0, 6655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7945, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6974, 4, 0, 0, 0, 0, 2, 227, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 5368, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 4, 3118, 0, 2182, 0, 0, 0, 1, 2, 0, 0, 0, 0, 606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6726, 8952, 2330, 13227, 8202, 7210, 12971, 5918] avg=8192.0000 max=13227 max_rank=3 min=2330 min_rank=2 imbalance_ratio=1.614624 max_load_reduction=52309 max_load_reduction_pct=0.798172 moved_to_echo=34301 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 415, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7521, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2327, 0, 0, 0, 0, 0, 0, 0, 7974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6519, 0, 0, 0, 0, 0, 0, 0, 364, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 7898, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5372, 994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8610, 8014, 10301, 6885, 8157, 8409, 6366, 8794] avg=8192.0000 max=10301 max_rank=2 min=6366 min_rank=6 imbalance_ratio=1.257446 max_load_reduction=55235 max_load_reduction_pct=0.842819 moved_to_echo=31726 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7526, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2327, 0, 0, 0, 0, 0, 0, 0, 7647, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6570, 0, 0, 0, 0, 0, 0, 0, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7965, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5372, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8606, 8019, 9974, 7114, 8217, 8419, 6389, 8798] avg=8192.0000 max=9974 max_rank=2 min=6389 min_rank=6 imbalance_ratio=1.217529 max_load_reduction=55562 max_load_reduction_pct=0.847809 moved_to_echo=31823 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 415, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7522, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2327, 0, 0, 0, 0, 0, 0, 0, 7893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6423, 0, 0, 0, 0, 0, 0, 0, 625, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 7837, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5372, 979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8610, 8015, 10220, 7051, 8106, 8398, 6351, 8785] avg=8192.0000 max=10220 max_rank=2 min=6351 min_rank=6 imbalance_ratio=1.247559 max_load_reduction=55316 max_load_reduction_pct=0.844055 moved_to_echo=31640 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7528, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2318, 0, 0, 0, 0, 0, 0, 0, 7908, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6527, 0, 0, 0, 0, 0, 0, 0, 295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7967, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5378, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8600, 8024, 10226, 6822, 8223, 8415, 6427, 8799] avg=8192.0000 max=10226 max_rank=2 min=6427 min_rank=6 imbalance_ratio=1.248291 max_load_reduction=55310 max_load_reduction_pct=0.843964 moved_to_echo=31864 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7526, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2327, 0, 0, 0, 0, 0, 0, 0, 7858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6577, 0, 0, 0, 0, 0, 0, 0, 290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7952, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5372, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8606, 8019, 10185, 6867, 8204, 8419, 6436, 8800] avg=8192.0000 max=10185 max_rank=2 min=6436 min_rank=6 imbalance_ratio=1.243286 max_load_reduction=55351 max_load_reduction_pct=0.844589 moved_to_echo=31859 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7526, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2327, 0, 0, 0, 0, 0, 0, 0, 7569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6501, 0, 0, 0, 0, 0, 0, 0, 718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7958, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5372, 991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8606, 8020, 9896, 7219, 8210, 8419, 6374, 8792] avg=8192.0000 max=9896 max_rank=2 min=6374 min_rank=6 imbalance_ratio=1.208008 max_load_reduction=55640 max_load_reduction_pct=0.848999 moved_to_echo=31795 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[14464, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 128, 0, 0, 0, 46592, 4736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 56064, 0, 128, 0, 128, 0, 0, 0, 9600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 13696, 128, 128, 0, 0, 0, 0, 0, 41472, 10624, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 0, 128, 0, 128, 896, 128, 0, 0, 0, 0, 0, 0, 0, 61056, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 128, 56704, 0, 3968, 0, 0, 0, 128, 128, 0, 0, 0, 0, 4992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 62720, 128, 0, 0, 0, 128, 0, 2944, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 64512, 128, 0, 0, 0, 0, 128, 1152, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 63872, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[14592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 43008, 8064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 18688, 0, 128, 0, 128, 0, 0, 0, 46976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 52352, 0, 128, 0, 0, 0, 0, 0, 2944, 10496, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 128, 0, 128, 1408, 128, 0, 0, 0, 0, 0, 0, 0, 60288, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 128, 128, 57216, 0, 3584, 0, 128, 0, 128, 128, 0, 0, 0, 0, 4864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 62208, 128, 0, 0, 0, 128, 0, 3328, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 63488, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 63744, 128, 0, 0, 0, 0, 128, 1920, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6799, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 5, 1, 0, 8190, 0, 8089, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1377, 0, 0, 0, 0, 0, 2, 0, 1, 0, 7, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 120, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 2, 8189, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 193, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 8010, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8113, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8064, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7535, 8183, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8187, 7, 2, 0, 8188, 0, 8163, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 661, 0, 0, 0, 0, 0, 3, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 135, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 8132, 0, 0, 4, 8175, 0, 0, 0, 4, 0, 2, 0, 4, 0, 0, 166, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 0, 0, 0, 7917, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8177, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8126, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8189, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8137, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 8177, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 7888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 32, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8467, 8225, 7913, 8192, 8157, 8189, 8145, 8248] avg=8192.0000 max=8467 max_rank=0 min=7913 min_rank=2 imbalance_ratio=1.033569 max_load_reduction=57069 max_load_reduction_pct=0.870804 moved_to_echo=32739 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6799, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 40, 0, 8089, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1377, 0, 0, 0, 0, 0, 2, 0, 7875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 120, 0, 0, 1, 0, 0, 0, 8150, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 8010, 0, 0, 31, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7074, 8137, 9254, 8194, 8278, 8202, 8330, 8067] avg=8192.0000 max=9254 max_rank=2 min=7074 min_rank=0 imbalance_ratio=1.129639 max_load_reduction=56282 max_load_reduction_pct=0.858795 moved_to_echo=32877 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7535, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 276, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2, 0, 40, 0, 8163, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 661, 0, 0, 0, 0, 0, 3, 0, 7883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 135, 0, 0, 1, 0, 0, 0, 8148, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 8175, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8101, 0, 0, 4, 0, 0, 0, 0, 4, 0, 2, 0, 4, 0, 0, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 0, 0, 0, 7917, 0, 0, 31, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7813, 8216, 8548, 8186, 8309, 8206, 8281, 7977] avg=8192.0000 max=8548 max_rank=2 min=7813 min_rank=0 imbalance_ratio=1.043457 max_load_reduction=56988 max_load_reduction_pct=0.869568 moved_to_echo=32773 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 8064, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 234, 0, 0, 0, 0, 0, 0, 0, 7888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 8159, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 32, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8388, 8107, 8122, 8193, 8157, 8169, 8152, 8248] avg=8192.0000 max=8388 max_rank=0 min=8107 min_rank=1 imbalance_ratio=1.023926 max_load_reduction=57148 max_load_reduction_pct=0.872009 moved_to_echo=32726 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 7888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 32, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8467, 8220, 7900, 8191, 8157, 8193, 8160, 8248] avg=8192.0000 max=8467 max_rank=0 min=7900 min_rank=2 imbalance_ratio=1.033569 max_load_reduction=57069 max_load_reduction_pct=0.870804 moved_to_echo=32758 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 8189, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 0, 0, 0, 0, 7888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8185, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 32, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8401, 8231, 7963, 8197, 8157, 8189, 8150, 8248] avg=8192.0000 max=8401 max_rank=0 min=7963 min_rank=2 imbalance_ratio=1.025513 max_load_reduction=57135 max_load_reduction_pct=0.871811 moved_to_echo=32744 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 7888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 32, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8467, 8229, 7891, 8191, 8157, 8193, 8160, 8248] avg=8192.0000 max=8467 max_rank=0 min=7891 min_rank=2 imbalance_ratio=1.033569 max_load_reduction=57069 max_load_reduction_pct=0.870804 moved_to_echo=32758 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 8137, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 7893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 34, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8459, 8171, 7945, 8192, 8164, 8195, 8158, 8252] avg=8192.0000 max=8459 max_rank=0 min=7945 min_rank=2 imbalance_ratio=1.032593 max_load_reduction=57077 max_load_reduction_pct=0.870926 moved_to_echo=32769 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8106, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8150, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7389, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 5, 1, 0, 8190, 0, 8126, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 762, 0, 0, 0, 0, 0, 2, 0, 1, 0, 8, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 126, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 2, 8189, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 134, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 8047, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7877, 8183, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8187, 7, 2, 0, 8188, 0, 8165, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 290, 0, 0, 0, 0, 0, 3, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 143, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 8115, 0, 0, 4, 8183, 0, 0, 0, 5, 0, 2, 0, 4, 0, 0, 113, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 0, 0, 0, 7991, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8133, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8189, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8154, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 8181, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 20, 0, 8126, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 762, 0, 0, 0, 0, 0, 2, 0, 7991, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 126, 0, 0, 1, 0, 0, 0, 8152, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 8187, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 8047, 0, 0, 29, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7547, 8154, 8773, 8193, 8283, 8202, 8293, 8091] avg=8192.0000 max=8773 max_rank=2 min=7547 min_rank=0 imbalance_ratio=1.070923 max_load_reduction=56763 max_load_reduction_pct=0.866135 moved_to_echo=32869 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 8004, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 29, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8351, 8210, 8025, 8190, 8155, 8192, 8180, 8233] avg=8192.0000 max=8351 max_rank=0 min=8025 min_rank=2 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32760 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7877, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2, 0, 20, 0, 8165, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 290, 0, 0, 0, 0, 0, 3, 0, 7999, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 143, 0, 0, 1, 0, 0, 0, 8150, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 8181, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8103, 0, 0, 4, 0, 0, 0, 0, 5, 0, 2, 0, 4, 0, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 0, 0, 0, 7991, 0, 0, 29, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8038, 8198, 8310, 8186, 8315, 8220, 8231, 8038] avg=8192.0000 max=8315 max_rank=4 min=8038 min_rank=0 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32804 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 8150, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 0, 0, 0, 0, 0, 0, 0, 8004, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 8154, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 29, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8265, 8173, 8203, 8195, 8155, 8166, 8146, 8233] avg=8192.0000 max=8265 max_rank=0 min=8146 min_rank=6 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32700 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 8189, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 8004, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8183, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 29, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8292, 8212, 8090, 8197, 8155, 8187, 8170, 8233] avg=8192.0000 max=8292 max_rank=0 min=8090 min_rank=2 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32745 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 8154, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 8010, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 33, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8341, 8168, 8067, 8191, 8162, 8190, 8175, 8242] avg=8192.0000 max=8341 max_rank=0 min=8067 min_rank=2 imbalance_ratio=1.018188 max_load_reduction=57195 max_load_reduction_pct=0.872726 moved_to_echo=32769 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8004, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 29, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8351, 8211, 8024, 8191, 8155, 8191, 8180, 8233] avg=8192.0000 max=8351 max_rank=0 min=8024 min_rank=2 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32759 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 8181, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 8004, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8153, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 29, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8351, 8210, 8044, 8193, 8156, 8188, 8161, 8233] avg=8192.0000 max=8351 max_rank=0 min=8044 min_rank=2 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32738 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 2560, 0, 0, 0, 0, 0, 128, 0, 63104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 63360, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 2304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 384, 0, 65280, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 1408, 0, 0, 0, 0, 0, 128, 0, 64128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 64384, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 384, 0, 0, 128, 0, 0, 0, 65280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[65280, 0, 0, 128, 0, 0, 0, 0, 128, 0, 128, 0, 128, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 65280, 0, 0, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 256, 0, 65408, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 128, 0, 0, 0, 65280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 65536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 65152, 0, 0, 256, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[65280, 0, 0, 128, 0, 0, 0, 0, 128, 0, 128, 0, 128, 0, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 6, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8182, 0, 8192, 0, 0, 24, 0, 0, 0, 8172, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8187, 6, 0, 0, 0, 0, 0, 4, 0, 73, 0, 0, 0, 0, 0, 9, 7899, 0, 0, 0, 0, 4233, 0, 8174, 2, 0, 0, 1, 1, 0, 8159, 0, 2, 0, 0, 8151, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3821, 3, 0, 0, 0, 0, 0, 0, 0, 3, 8183, 0, 17, 8187, 0, 0, 5, 2, 0, 0, 0, 0, 274, 0, 0, 0, 2, 81, 0, 0, 3, 0, 0, 0, 0, 4, 0, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 3, 0, 21, 0, 0, 0, 0, 0, 0, 7981, 0, 0, 0, 0, 5892, 0, 8186, 0, 0, 0, 1, 0, 0, 8189, 0, 1, 0, 0, 8168, 0, 0, 21, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2195, 1, 0, 0, 0, 0, 0, 0, 0, 1, 8187, 0, 2, 8190, 0, 0, 2, 0, 1, 0, 0, 0, 185, 0, 0, 0, 1, 106, 0, 0, 2, 0, 0, 0, 0, 2, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8192, 0, 0, 0, 0, 8188, 0, 8192, 0, 0, 3, 0, 0, 0, 8191, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 10, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8136, 0, 8192, 0, 0, 80, 0, 0, 0, 8158, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8174, 0, 8192, 0, 0, 16, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8192, 0, 0, 0, 0, 8137, 0, 8192, 0, 0, 55, 0, 0, 0, 8181, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8192, 0, 0, 0, 0, 8141, 0, 8192, 0, 0, 50, 0, 0, 0, 8190, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 21, 0, 0, 0, 0, 0, 0, 777, 0, 0, 0, 0, 5892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8166, 0, 1, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2195, 1, 0, 0, 0, 0, 7204, 72, 23, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8185, 0, 2, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, 0, 0, 0, 1, 106, 0, 0, 2, 0, 0, 0, 0, 2, 0, 2, 8094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8190, 6693, 8189, 9501, 8191, 8392, 8192] avg=8192.0000 max=9501 max_rank=4 min=6693 min_rank=2 imbalance_ratio=1.159790 max_load_reduction=56035 max_load_reduction_pct=0.855026 moved_to_echo=34276 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 797, 0, 0, 0, 0, 8137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7395, 72, 23, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8193, 8941, 8213, 7495, 8190, 8116, 8192] avg=8192.0000 max=8941 max_rank=2 min=7495 min_rank=4 imbalance_ratio=1.091431 max_load_reduction=56595 max_load_reduction_pct=0.863571 moved_to_echo=31993 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 8173, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8184, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 73, 0, 0, 0, 0, 0, 9, 769, 0, 0, 0, 0, 4233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 1, 0, 8136, 0, 2, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3821, 3, 0, 0, 0, 0, 7130, 72, 23, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8181, 0, 17, 0, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 274, 0, 0, 0, 2, 81, 0, 0, 3, 0, 0, 0, 0, 4, 0, 4, 8076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8187, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8193, 5088, 8175, 11054, 8208, 8444, 8193] avg=8192.0000 max=11054 max_rank=4 min=5088 min_rank=2 imbalance_ratio=1.349365 max_load_reduction=54482 max_load_reduction_pct=0.831329 moved_to_echo=35899 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 10, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 797, 0, 0, 0, 0, 8136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 8135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7395, 72, 23, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8199, 8935, 8215, 7495, 8190, 8118, 8191] avg=8192.0000 max=8935 max_rank=2 min=7495 min_rank=4 imbalance_ratio=1.090698 max_load_reduction=56601 max_load_reduction_pct=0.863663 moved_to_echo=31994 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 6, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 797, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7395, 74, 23, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8196, 8980, 8173, 7497, 8190, 8116, 8191] avg=8192.0000 max=8980 max_rank=2 min=7497 min_rank=4 imbalance_ratio=1.096191 max_load_reduction=56556 max_load_reduction_pct=0.862976 moved_to_echo=31994 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 797, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7395, 72, 23, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8189, 8987, 8171, 7495, 8190, 8118, 8193] avg=8192.0000 max=8987 max_rank=2 min=7495 min_rank=4 imbalance_ratio=1.097046 max_load_reduction=56549 max_load_reduction_pct=0.862869 moved_to_echo=31996 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 797, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7395, 72, 23, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8189, 8973, 8185, 7495, 8190, 8116, 8192] avg=8192.0000 max=8973 max_rank=2 min=7495 min_rank=4 imbalance_ratio=1.095337 max_load_reduction=56563 max_load_reduction_pct=0.863083 moved_to_echo=31993 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 795, 0, 0, 0, 0, 8141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7397, 72, 25, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8187, 8939, 8215, 7504, 8187, 8116, 8192] avg=8192.0000 max=8939 max_rank=2 min=7504 min_rank=4 imbalance_ratio=1.091187 max_load_reduction=56597 max_load_reduction_pct=0.863602 moved_to_echo=31999 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 7936, 0, 0, 0, 0, 5814, 0, 8186, 0, 0, 0, 1, 0, 0, 8187, 0, 1, 0, 0, 8165, 0, 0, 23, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2292, 1, 0, 0, 0, 0, 0, 0, 0, 1, 8188, 0, 1, 8190, 0, 0, 2, 0, 1, 0, 0, 0, 254, 0, 0, 0, 1, 85, 0, 0, 2, 0, 0, 0, 0, 2, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8192, 0, 0, 0, 0, 8146, 0, 8192, 0, 0, 46, 0, 0, 0, 8190, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 6, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8183, 0, 8192, 0, 0, 20, 0, 0, 0, 8176, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8192, 0, 0, 0, 0, 8188, 0, 8192, 0, 0, 5, 0, 0, 0, 8190, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 7, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8147, 0, 8192, 0, 0, 75, 0, 0, 0, 8157, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8187, 6, 0, 0, 0, 0, 0, 4, 0, 16, 0, 0, 0, 0, 0, 1, 7769, 0, 0, 0, 0, 3825, 0, 8173, 2, 0, 0, 1, 1, 0, 8138, 0, 2, 0, 0, 8147, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4301, 3, 0, 0, 0, 0, 0, 0, 0, 3, 8183, 0, 6, 8187, 0, 0, 5, 1, 1, 0, 0, 0, 417, 0, 0, 0, 2, 96, 0, 0, 3, 0, 0, 0, 0, 4, 0, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8172, 0, 8192, 0, 0, 16, 0, 0, 0, 8191, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8192, 0, 0, 0, 0, 8142, 0, 8192, 0, 0, 51, 0, 0, 0, 8182, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 842, 0, 0, 0, 0, 5814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8168, 0, 1, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2292, 1, 0, 0, 0, 0, 7094, 99, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8188, 0, 1, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 254, 0, 0, 0, 1, 85, 0, 0, 2, 0, 0, 0, 0, 2, 0, 2, 8064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8190, 6662, 8193, 9508, 8193, 8410, 8192] avg=8192.0000 max=9508 max_rank=4 min=6662 min_rank=2 imbalance_ratio=1.160645 max_load_reduction=56028 max_load_reduction_pct=0.854919 moved_to_echo=34303 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 869, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7323, 97, 19, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8190, 9058, 8176, 7442, 8191, 8093, 8193] avg=8192.0000 max=9058 max_rank=2 min=7442 min_rank=4 imbalance_ratio=1.105713 max_load_reduction=56478 max_load_reduction_pct=0.861786 moved_to_echo=31919 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 6, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 869, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7322, 99, 19, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8197, 9053, 8177, 7443, 8191, 8091, 8191] avg=8192.0000 max=9053 max_rank=2 min=7443 min_rank=4 imbalance_ratio=1.105103 max_load_reduction=56483 max_load_reduction_pct=0.861862 moved_to_echo=31916 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 867, 0, 0, 0, 0, 8146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7325, 97, 20, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8186, 9015, 8216, 7451, 8189, 8091, 8192] avg=8192.0000 max=9015 max_rank=2 min=7451 min_rank=4 imbalance_ratio=1.100464 max_load_reduction=56521 max_load_reduction_pct=0.862442 moved_to_echo=31923 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 8172, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8185, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 16, 0, 0, 0, 0, 0, 1, 825, 0, 0, 0, 0, 3825, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 1, 0, 8119, 0, 2, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4301, 3, 0, 0, 0, 0, 6944, 98, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8183, 0, 6, 0, 0, 0, 5, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 417, 0, 0, 0, 2, 96, 0, 0, 3, 0, 0, 0, 0, 4, 0, 4, 8046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8187, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8194, 4671, 8159, 11367, 8199, 8572, 8193] avg=8192.0000 max=11367 max_rank=4 min=4671 min_rank=2 imbalance_ratio=1.387573 max_load_reduction=54169 max_load_reduction_pct=0.826553 moved_to_echo=36331 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 7, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 869, 0, 0, 0, 0, 8147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 8138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7322, 97, 19, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8196, 9018, 8213, 7441, 8191, 8093, 8191] avg=8192.0000 max=9018 max_rank=2 min=7441 min_rank=4 imbalance_ratio=1.100830 max_load_reduction=56518 max_load_reduction_pct=0.862396 moved_to_echo=31916 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 869, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7323, 97, 19, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8190, 9042, 8188, 7442, 8191, 8095, 8192] avg=8192.0000 max=9042 max_rank=2 min=7442 min_rank=4 imbalance_ratio=1.103760 max_load_reduction=56494 max_load_reduction_pct=0.862030 moved_to_echo=31920 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 869, 0, 0, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7323, 97, 19, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8193, 9017, 8214, 7442, 8191, 8091, 8192] avg=8192.0000 max=9017 max_rank=2 min=7442 min_rank=4 imbalance_ratio=1.100708 max_load_reduction=56519 max_load_reduction_pct=0.862411 moved_to_echo=31916 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 128, 0, 0, 0, 0, 0, 128, 6400, 0, 0, 0, 0, 59136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6016, 128, 0, 0, 0, 0, 58752, 640, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[512, 0, 0, 0, 128, 256, 0, 0, 128, 0, 0, 0, 0, 128, 0, 128, 64896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 128, 128, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 65536, 0, 128, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 128, 0, 0, 0, 0, 0, 128, 6912, 0, 0, 0, 0, 58624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6656, 128, 0, 0, 0, 0, 57984, 896, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 128, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[768, 0, 0, 0, 128, 256, 0, 0, 128, 0, 0, 0, 0, 128, 0, 128, 64768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 256, 128, 128, 0, 65280, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 65536, 0, 128, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 128, 128, 128, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 256, 128, 128, 0, 65280, 0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 7498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 322, 0, 0, 0, 0, 0, 0, 0, 0, 7486, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 6923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 787, 0, 0, 0, 0, 0, 0, 0, 0, 7964, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8185, 0, 0, 6566, 0, 0, 2, 6, 1475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6298, 0, 0, 0, 0, 0, 0, 8189, 6, 0, 5919, 1, 0, 0, 0, 2, 0, 1, 0, 0, 8189, 0, 1, 0, 0, 0, 0, 8057, 2, 1, 0, 0, 1, 0, 0, 1, 0, 4375, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 7308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 845, 0, 0, 0, 0, 0, 0, 0, 0, 7435, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8185, 0, 0, 6803, 0, 0, 4, 0, 1609, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 3, 1, 4, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5083, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 5791, 1, 1, 1, 0, 3, 0, 2, 0, 0, 8186, 0, 3, 1, 0, 0, 0, 7994, 4, 0, 0, 0, 1, 0, 0, 5, 0, 5330, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 134, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 7449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 7385, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 5, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 7238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 375, 0, 1, 0, 0, 0, 0, 0, 0, 6758, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 10, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 7449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2584, 0, 0, 0, 0, 0, 0, 0, 0, 6138, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 652, 0, 0, 7498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7611, 579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 322, 0, 0, 5617, 1944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7486, 0, 0, 0, 0, 0, 0, 1374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6961, 1201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8150, 8190, 7887, 8860, 8773, 7322, 8162, 8192] avg=8192.0000 max=8860 max_rank=3 min=7322 min_rank=5 imbalance_ratio=1.081543 max_load_reduction=56676 max_load_reduction_pct=0.864807 moved_to_echo=32449 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 651, 0, 0, 6566, 0, 0, 2, 6, 1475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 7608, 579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 5615, 1912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6298, 0, 0, 0, 0, 0, 0, 1373, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5919, 1, 0, 0, 0, 2, 0, 1, 0, 0, 581, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6145, 2, 1, 0, 0, 1, 0, 0, 1, 0, 4375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 6955, 1201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8700, 8190, 7530, 7677, 6505, 10525, 8218, 8191] avg=8192.0000 max=10525 max_rank=5 min=6505 min_rank=4 imbalance_ratio=1.284790 max_load_reduction=55011 max_load_reduction_pct=0.839401 moved_to_echo=33439 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 651, 0, 0, 6803, 0, 0, 4, 0, 1609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 3, 1, 4, 0, 0, 7606, 579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 5613, 1897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5083, 0, 0, 0, 0, 0, 0, 1373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5791, 1, 1, 1, 0, 3, 0, 2, 0, 0, 580, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6097, 4, 0, 0, 0, 1, 0, 0, 5, 0, 5330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 134, 0, 6955, 1200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9067, 8197, 7515, 6456, 6383, 11437, 8292, 8189] avg=8192.0000 max=11437 max_rank=5 min=6383 min_rank=4 imbalance_ratio=1.396118 max_load_reduction=54099 max_load_reduction_pct=0.825485 moved_to_echo=34301 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 652, 0, 0, 6923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7611, 579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 787, 0, 0, 5617, 1944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7964, 0, 0, 0, 0, 0, 0, 1374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 6961, 1201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7575, 8190, 8388, 9338, 8773, 6916, 8164, 8192] avg=8192.0000 max=9338 max_rank=3 min=6916 min_rank=5 imbalance_ratio=1.139893 max_load_reduction=56198 max_load_reduction_pct=0.857513 moved_to_echo=32045 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 652, 0, 0, 7238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7611, 579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 375, 0, 1, 5617, 1944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6758, 0, 0, 0, 0, 0, 0, 1374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 10, 0, 0, 0, 0, 0, 0, 0, 0, 581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6961, 1201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7890, 8190, 7941, 8132, 8783, 8246, 8162, 8192] avg=8192.0000 max=8783 max_rank=4 min=7890 min_rank=0 imbalance_ratio=1.072144 max_load_reduction=56753 max_load_reduction_pct=0.865982 moved_to_echo=33383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 652, 0, 0, 7449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7611, 579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 5617, 1944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7385, 0, 0, 0, 0, 0, 0, 1374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 5, 0, 0, 0, 0, 0, 0, 0, 0, 581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6961, 1201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8101, 8190, 7667, 8759, 8778, 7687, 8162, 8192] avg=8192.0000 max=8778 max_rank=4 min=7667 min_rank=2 imbalance_ratio=1.071533 max_load_reduction=56758 max_load_reduction_pct=0.866058 moved_to_echo=32819 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 652, 0, 0, 7449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7611, 579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2584, 0, 0, 5617, 1944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6138, 0, 0, 0, 0, 0, 0, 1374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6961, 1201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8101, 8190, 10169, 7512, 8773, 6436, 8163, 8192] avg=8192.0000 max=10169 max_rank=2 min=6436 min_rank=5 imbalance_ratio=1.241333 max_load_reduction=55367 max_load_reduction_pct=0.844833 moved_to_echo=31564 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 644, 0, 0, 7308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7616, 583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 845, 0, 0, 5620, 1951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7435, 0, 0, 0, 0, 0, 0, 1367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 6965, 1205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7952, 8199, 8439, 8802, 8768, 6967, 8213, 8196] avg=8192.0000 max=8802 max_rank=3 min=6967 min_rank=5 imbalance_ratio=1.074463 max_load_reduction=56734 max_load_reduction_pct=0.865692 moved_to_echo=32144 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 6918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 0, 3, 0, 0, 0, 0, 0, 0, 6395, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 8, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2822, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8185, 0, 0, 5664, 0, 0, 2, 8, 1369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4518, 0, 0, 0, 0, 0, 0, 8189, 5, 0, 7646, 0, 0, 0, 0, 2, 0, 1, 0, 0, 8190, 0, 1, 0, 0, 0, 0, 8025, 2, 1, 0, 0, 1, 0, 0, 1, 0, 5510, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 6677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 576, 0, 0, 0, 0, 0, 0, 0, 0, 7232, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 5346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 230, 0, 0, 0, 0, 0, 0, 0, 0, 7980, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 5290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 163, 0, 1, 0, 0, 0, 0, 0, 0, 7102, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 5, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 7492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 839, 0, 0, 0, 0, 0, 0, 0, 0, 7365, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8185, 0, 0, 6098, 0, 0, 4, 0, 1545, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 3, 1, 4, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4001, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 7658, 0, 1, 1, 0, 3, 0, 2, 0, 0, 8186, 0, 3, 1, 0, 0, 0, 7937, 4, 0, 0, 0, 1, 0, 0, 6, 0, 5454, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 7500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 271, 0, 0, 0, 0, 0, 0, 0, 0, 7686, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1453, 0, 0, 5664, 0, 0, 2, 8, 1369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 8058, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 4963, 2891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4518, 0, 0, 0, 0, 0, 0, 1657, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7646, 0, 0, 0, 0, 2, 0, 1, 0, 0, 132, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5134, 2, 1, 0, 0, 1, 0, 0, 1, 0, 5510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 6602, 1569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8496, 8191, 7857, 6180, 7782, 10649, 8190, 8191] avg=8192.0000 max=10649 max_rank=5 min=6180 min_rank=3 imbalance_ratio=1.299927 max_load_reduction=54887 max_load_reduction_pct=0.837509 moved_to_echo=34812 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1454, 0, 0, 5346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8060, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 230, 0, 0, 4965, 2951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7980, 0, 0, 0, 0, 0, 0, 1657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 6608, 1570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6800, 8190, 8149, 9637, 8325, 8061, 8182, 8192] avg=8192.0000 max=9637 max_rank=3 min=6800 min_rank=0 imbalance_ratio=1.176392 max_load_reduction=55899 max_load_reduction_pct=0.852951 moved_to_echo=32760 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1447, 0, 0, 6677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8065, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 576, 0, 0, 4966, 2956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7232, 0, 0, 0, 0, 0, 0, 1654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 6611, 1572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8124, 8199, 8508, 8886, 8319, 7076, 8228, 8196] avg=8192.0000 max=8886 max_rank=3 min=7076 min_rank=5 imbalance_ratio=1.084717 max_load_reduction=56650 max_load_reduction_pct=0.864410 moved_to_echo=31819 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1454, 0, 0, 6918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8060, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 0, 3, 4965, 2951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6395, 0, 0, 0, 0, 0, 0, 1657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8, 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2822, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6608, 1570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8372, 8190, 8159, 8052, 8332, 8061, 8178, 8192] avg=8192.0000 max=8372 max_rank=0 min=8052 min_rank=3 imbalance_ratio=1.021973 max_load_reduction=57164 max_load_reduction_pct=0.872253 moved_to_echo=32763 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1454, 0, 0, 5290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8060, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 163, 0, 1, 4965, 2951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7102, 0, 0, 0, 0, 0, 0, 1657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 5, 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6608, 1570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6744, 8190, 8088, 8759, 8329, 9056, 8178, 8192] avg=8192.0000 max=9056 max_rank=5 min=6744 min_rank=0 imbalance_ratio=1.105469 max_load_reduction=56480 max_load_reduction_pct=0.861816 moved_to_echo=33755 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1453, 0, 0, 6098, 0, 0, 4, 0, 1545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 3, 1, 4, 0, 0, 8054, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4962, 2860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4001, 0, 0, 0, 0, 0, 0, 1656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7658, 0, 1, 1, 0, 3, 0, 2, 0, 0, 132, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5077, 4, 0, 0, 0, 1, 0, 0, 6, 0, 5454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 50, 0, 6602, 1569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9100, 8196, 7827, 5657, 7801, 10542, 8224, 8189] avg=8192.0000 max=10542 max_rank=5 min=5657 min_rank=3 imbalance_ratio=1.286865 max_load_reduction=54994 max_load_reduction_pct=0.839142 moved_to_echo=34756 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1454, 0, 0, 7492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8060, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 839, 0, 0, 4965, 2951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7365, 0, 0, 0, 0, 0, 0, 1657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6608, 1570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8946, 8190, 8759, 9022, 8324, 5925, 8178, 8192] avg=8192.0000 max=9022 max_rank=3 min=5925 min_rank=5 imbalance_ratio=1.101318 max_load_reduction=56514 max_load_reduction_pct=0.862335 moved_to_echo=30619 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1454, 0, 0, 7500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8060, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 271, 0, 0, 4965, 2951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7686, 0, 0, 0, 0, 0, 0, 1657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6608, 1570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8954, 8190, 8189, 9343, 8324, 6166, 8178, 8192] avg=8192.0000 max=9343 max_rank=3 min=6166 min_rank=5 imbalance_ratio=1.140503 max_load_reduction=56193 max_load_reduction_pct=0.857437 moved_to_echo=30860 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 4992, 0, 128, 45056, 15488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 49792, 128, 128, 0, 0, 128, 0, 0, 128, 0, 15872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 256, 0, 55680, 9728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 54656, 0, 0, 0, 0, 0, 0, 11008, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 5248, 0, 0, 57344, 0, 0, 128, 128, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 128, 128, 128, 0, 0, 60928, 4736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[60928, 128, 128, 128, 0, 128, 0, 128, 0, 0, 4736, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 2432, 0, 128, 39808, 23552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 41728, 128, 128, 0, 0, 128, 0, 0, 128, 0, 23936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 52864, 12672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 11648, 0, 0, 51072, 0, 0, 128, 128, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 52352, 0, 0, 0, 0, 0, 0, 13312, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 128, 128, 128, 0, 0, 64512, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[64512, 128, 128, 128, 0, 128, 0, 128, 0, 0, 1152, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True - [2026-08-26 14:48:14] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 437.3 | throughput per GPU (TFLOP/s/GPU): 413.5 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.419350E+00 | loss scale: 1.0 | grad norm: 5.083 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 2871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 1909, 0, 0, 3169, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 1, 0, 0, 1, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 2621, 0, 0, 0, 3294, 0, 3593, 35, 0, 0, 4484, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 2, 0, 0, 0, 0, 2391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8077, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 241, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[2, 0, 0, 1, 0, 3450, 0, 0, 0, 0, 0, 4, 0, 0, 1, 16, 0, 6, 0, 2, 2604, 0, 1, 2595, 0, 1, 2, 8191, 2, 8186, 1, 0, 0, 4, 0, 0, 2, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 2, 3, 8178, 2, 0, 0, 0, 0, 0, 0, 0, 2963, 2, 0, 2, 3337, 0, 3165, 35, 0, 0, 3765, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 6, 2, 0, 2, 0, 2413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8044, 0, 0, 26, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 320, 2, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 14, 3001, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3150, 0, 0, 2924, 17, 0, 0, 8192, 1, 8192, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 1, 0, 0, 0, 3129, 0, 2, 0, 3310, 0, 2968, 19, 0, 0, 4271, 0, 0, 0, 0, 0, 0, 0, 0, 8092, 0, 0, 0, 0, 1, 0, 1549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 8096, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 348, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[3, 0, 3, 0, 0, 3009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 2791, 3, 1, 2634, 0, 0, 0, 8186, 0, 8188, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 3, 0, 3, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 2, 0, 3486, 0, 2, 0, 3241, 1, 3116, 34, 0, 0, 3808, 0, 0, 0, 0, 0, 0, 1, 0, 8013, 0, 1, 1, 0, 0, 0, 2374, 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 0, 8074, 1, 1, 1, 0, 0, 1, 0, 4, 0, 1, 2, 0, 0, 0, 0, 0, 339, 0, 0, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 1, 3444, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2641, 0, 0, 2540, 7, 0, 0, 8192, 0, 8191, 17, 0, 0, 0, 1, 0, 5, 0, 0, 1, 3, 2, 1, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 3079, 0, 0, 1, 4126, 0, 3012, 35, 0, 1, 4032, 0, 0, 3, 0, 0, 2, 0, 0, 8043, 0, 19, 0, 1, 0, 0, 1632, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8011, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 262, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 3332, 0, 0, 0, 1, 1, 0, 0, 0, 0, 23, 2, 0, 0, 0, 2845, 0, 0, 3653, 0, 0, 0, 8191, 1, 8191, 13, 0, 0, 0, 5, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 1, 0, 0, 0, 0, 0, 3163, 0, 0, 0, 3246, 0, 2583, 34, 0, 0, 3802, 0, 0, 0, 0, 0, 1, 0, 0, 8048, 0, 0, 0, 0, 0, 0, 1828, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 0, 0, 17, 0, 1, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 219, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2721, 0, 0, 3083, 1, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 2647, 0, 0, 0, 3664, 0, 3080, 11, 0, 0, 3747, 0, 0, 0, 0, 0, 0, 0, 0, 8143, 0, 11, 0, 0, 0, 0, 2094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8108, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 350, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 7, 3519, 0, 0, 4, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2549, 0, 0, 3323, 0, 0, 0, 8191, 1, 8182, 0, 2, 0, 0, 2, 1, 1, 5, 0, 0, 7, 0, 0, 0, 0, 2, 14, 1, 0, 0, 6, 0, 0, 1, 1, 0, 0, 0, 1, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 3039, 0, 0, 0, 3324, 0, 2196, 11, 3, 0, 3824, 2, 0, 0, 3, 4, 0, 1, 0, 8086, 0, 4, 0, 0, 0, 0, 2531, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 8143, 0, 1, 4, 0, 0, 0, 2, 0, 4, 0, 0, 0, 0, 2, 0, 0, 340, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 2871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2030, 1992, 1117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1909, 0, 0, 3169, 0, 0, 0, 0, 0, 2539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 1, 0, 0, 1, 0, 0, 58, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8187, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2621, 0, 0, 0, 2090, 0, 3593, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6135, 0, 2, 0, 0, 0, 0, 2391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8077, 0, 0, 36, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 241, 0, 0, 0, 0, 5653, 2492, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8020, 7617, 8229, 8190, 8339, 8528, 8198, 8415] avg=8192.0000 max=8528 max_rank=5 min=7617 min_rank=1 imbalance_ratio=1.041016 max_load_reduction=57008 max_load_reduction_pct=0.869873 moved_to_echo=33480 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 14, 3001, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 2012, 1896, 1122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3150, 0, 0, 2924, 17, 0, 0, 0, 1, 2539, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 13, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3129, 0, 2, 0, 2101, 0, 2968, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6080, 0, 0, 0, 0, 1, 0, 1549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 8096, 0, 0, 10, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 348, 0, 0, 0, 0, 5653, 2375, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8048, 8652, 8187, 8191, 8220, 7630, 8201, 8407] avg=8192.0000 max=8652 max_rank=1 min=7630 min_rank=5 imbalance_ratio=1.056152 max_load_reduction=56884 max_load_reduction_pct=0.867981 moved_to_echo=32458 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 1, 0, 3450, 0, 0, 0, 0, 0, 4, 0, 0, 1, 16, 2034, 1671, 1132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2, 2604, 0, 1, 2595, 0, 1, 2, 0, 2, 2537, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 2, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4, 1, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 2, 3, 8178, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2963, 2, 0, 2, 2117, 0, 3165, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6145, 0, 6, 2, 0, 2, 0, 2413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8044, 0, 0, 26, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 320, 2, 0, 1, 0, 5649, 2094, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8311, 7751, 8181, 8189, 8284, 8568, 8156, 8096] avg=8192.0000 max=8568 max_rank=5 min=7751 min_rank=1 imbalance_ratio=1.045898 max_load_reduction=56968 max_load_reduction_pct=0.869263 moved_to_echo=33104 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 3332, 0, 0, 0, 1, 1, 0, 0, 0, 0, 23, 2001, 1688, 1101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2845, 0, 0, 3653, 0, 0, 0, 0, 1, 2539, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 4, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3163, 0, 0, 0, 2059, 0, 2583, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6047, 0, 0, 0, 0, 0, 0, 1828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 0, 0, 17, 0, 1, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 219, 0, 0, 0, 0, 5652, 2114, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8148, 9053, 8174, 8192, 7839, 7876, 8235, 8019] avg=8192.0000 max=9053 max_rank=1 min=7839 min_rank=4 imbalance_ratio=1.105103 max_load_reduction=56483 max_load_reduction_pct=0.861862 moved_to_echo=31969 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 3, 0, 0, 3009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1993, 1692, 1099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2791, 3, 1, 2634, 0, 0, 0, 0, 0, 2538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 3, 0, 3, 0, 6, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 8186, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3486, 0, 2, 0, 2056, 1, 3116, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6020, 0, 1, 1, 0, 0, 0, 2374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 0, 8074, 1, 1, 1, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 1, 2, 0, 0, 0, 0, 0, 339, 0, 0, 0, 3, 5650, 2116, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7802, 7967, 8171, 8192, 8697, 8397, 8165, 8145] avg=8192.0000 max=8697 max_rank=4 min=7802 min_rank=0 imbalance_ratio=1.061646 max_load_reduction=56839 max_load_reduction_pct=0.867294 moved_to_echo=33404 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 7, 3519, 0, 0, 4, 0, 0, 0, 0, 0, 0, 2, 2011, 1698, 1127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2549, 0, 0, 3323, 0, 0, 0, 0, 1, 2536, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 5, 0, 0, 7, 0, 0, 0, 0, 2, 14, 1, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 1, 1, 0, 0, 0, 1, 8185, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3039, 0, 0, 0, 2109, 0, 2196, 11, 3, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 1, 0, 6075, 0, 4, 0, 0, 0, 0, 2531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 8143, 0, 1, 4, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4, 0, 0, 0, 0, 2, 0, 0, 340, 0, 0, 0, 1, 5646, 2126, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8369, 8411, 8195, 8196, 7360, 8618, 8237, 8150] avg=8192.0000 max=8618 max_rank=5 min=7360 min_rank=4 imbalance_ratio=1.052002 max_load_reduction=56918 max_load_reduction_pct=0.868500 moved_to_echo=32365 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 1, 3444, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2004, 1791, 1403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2641, 0, 0, 2540, 7, 0, 0, 0, 0, 2535, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 0, 0, 1, 3, 2, 1, 0, 0, 0, 23, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3079, 0, 0, 1, 2606, 0, 3012, 35, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 2, 0, 0, 6039, 0, 19, 0, 1, 0, 0, 1632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8011, 0, 0, 17, 0, 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 262, 0, 0, 0, 0, 5656, 2241, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8646, 7740, 8200, 8192, 8734, 7696, 8139, 8189] avg=8192.0000 max=8734 max_rank=4 min=7696 min_rank=5 imbalance_ratio=1.066162 max_load_reduction=56802 max_load_reduction_pct=0.866730 moved_to_echo=32758 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2025, 1664, 1243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2721, 0, 0, 3083, 1, 0, 0, 0, 0, 2539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 34, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2647, 0, 0, 0, 2325, 0, 3080, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6118, 0, 11, 0, 0, 0, 0, 2094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8108, 0, 0, 3, 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 350, 0, 0, 0, 0, 5653, 2083, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8345, 8199, 8194, 8063, 8223, 8205, 8115] avg=8192.0000 max=8345 max_rank=1 min=8063 min_rank=4 imbalance_ratio=1.018677 max_load_reduction=57191 max_load_reduction_pct=0.872665 moved_to_echo=32606 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] + [2026-08-26 15:24:15] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 444.1 | throughput per GPU (TFLOP/s/GPU): 407.2 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.418392E+00 | loss scale: 1.0 | grad norm: 5.086 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[2, 0, 0, 1, 0, 3616, 0, 0, 0, 0, 0, 4, 0, 0, 1, 18, 0, 6, 0, 2, 2612, 0, 1, 2432, 0, 1, 2, 8191, 2, 8186, 1, 0, 0, 4, 0, 0, 2, 0, 0, 8, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 0, 0, 1, 2, 3, 8178, 2, 0, 0, 0, 0, 0, 0, 0, 2974, 2, 0, 2, 3255, 0, 3236, 38, 0, 0, 3929, 0, 0, 0, 0, 0, 0, 0, 0, 7869, 0, 9, 2, 0, 2, 0, 2505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8077, 0, 0, 24, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 325, 2, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 20, 3169, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3128, 0, 0, 2839, 14, 0, 0, 8192, 1, 8192, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 1, 0, 0, 0, 3176, 0, 2, 0, 3207, 0, 3032, 24, 0, 0, 4480, 0, 0, 0, 0, 0, 0, 0, 0, 7706, 0, 13, 0, 0, 1, 0, 1608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 8118, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 353, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 3525, 0, 0, 0, 1, 1, 0, 0, 0, 0, 27, 1, 0, 0, 0, 2897, 0, 0, 3639, 0, 0, 0, 8191, 1, 8191, 12, 0, 0, 0, 3, 0, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 1, 0, 0, 0, 0, 0, 3220, 0, 0, 0, 3177, 0, 2663, 38, 0, 0, 4073, 0, 0, 0, 0, 0, 1, 0, 0, 7206, 0, 1, 0, 0, 0, 0, 1945, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 17, 0, 1, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 353, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 13, 3683, 0, 0, 4, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2608, 0, 0, 3234, 0, 0, 0, 8191, 1, 8182, 0, 3, 0, 0, 2, 1, 1, 5, 0, 0, 7, 0, 0, 0, 0, 2, 5, 1, 0, 0, 6, 0, 0, 1, 1, 0, 0, 0, 1, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 3064, 0, 0, 0, 3261, 0, 2246, 16, 3, 0, 3996, 2, 0, 0, 3, 3, 0, 1, 0, 7618, 0, 24, 0, 0, 0, 0, 2626, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 8156, 0, 1, 3, 0, 0, 0, 2, 0, 4, 0, 0, 0, 0, 2, 0, 0, 360, 0, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[3, 0, 3, 0, 0, 3175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2858, 3, 1, 2605, 0, 0, 0, 8186, 0, 8188, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 4, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 2, 0, 3557, 0, 2, 0, 3133, 1, 3208, 46, 0, 0, 4002, 0, 0, 0, 0, 0, 0, 1, 0, 7375, 0, 1, 1, 0, 0, 0, 2475, 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 8109, 1, 1, 0, 0, 0, 1, 0, 4, 0, 1, 2, 0, 0, 0, 0, 0, 379, 0, 0, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 8, 3622, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2738, 0, 0, 2446, 5, 0, 0, 8192, 0, 8191, 16, 0, 0, 0, 0, 0, 4, 0, 0, 1, 4, 2, 2, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 1, 0, 0, 0, 3116, 0, 0, 1, 4044, 0, 3101, 43, 0, 1, 4268, 0, 0, 3, 0, 0, 2, 0, 0, 7385, 0, 46, 0, 1, 0, 0, 1706, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8040, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 323, 0, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 3379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2740, 0, 0, 2950, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 2668, 0, 0, 0, 3564, 0, 3141, 12, 0, 0, 3880, 0, 0, 0, 0, 0, 0, 0, 0, 7953, 0, 42, 0, 0, 0, 0, 2123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8128, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 350, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 10, 2966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 1897, 0, 0, 3029, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 4, 2, 0, 0, 2, 0, 0, 1, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 2670, 0, 0, 0, 3170, 0, 3628, 42, 0, 0, 4709, 0, 0, 0, 0, 0, 0, 0, 0, 7894, 0, 6, 0, 0, 0, 0, 2481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8091, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 277, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 3525, 0, 0, 0, 1, 1, 0, 0, 0, 0, 27, 1889, 1547, 1149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2897, 0, 0, 3639, 0, 0, 0, 0, 1, 2600, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3220, 0, 0, 0, 1963, 0, 2663, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5659, 0, 1, 0, 0, 0, 0, 1945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 17, 0, 1, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 353, 0, 0, 0, 0, 5591, 2184, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8140, 9150, 8176, 8192, 7884, 7606, 8233, 8155] avg=8192.0000 max=9150 max_rank=1 min=7606 min_rank=5 imbalance_ratio=1.116943 max_load_reduction=56386 max_load_reduction_pct=0.860382 moved_to_echo=31878 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 1, 0, 3616, 0, 0, 0, 0, 0, 4, 0, 0, 1, 18, 1822, 1689, 1177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2, 2612, 0, 1, 2432, 0, 1, 2, 0, 2, 2598, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 2, 0, 0, 8, 1, 0, 0, 0, 0, 0, 1, 1, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 2, 3, 8178, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2974, 2, 0, 2, 2012, 0, 3236, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6180, 0, 9, 2, 0, 2, 0, 2505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8077, 0, 0, 24, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 325, 2, 0, 1, 0, 5588, 2107, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8330, 7657, 8185, 8190, 8264, 8698, 8165, 8047] avg=8192.0000 max=8698 max_rank=5 min=7657 min_rank=1 imbalance_ratio=1.061768 max_load_reduction=56838 max_load_reduction_pct=0.867279 moved_to_echo=33174 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 20, 3169, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 2077, 1654, 1160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3128, 0, 0, 2839, 14, 0, 0, 0, 1, 2600, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 11, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3176, 0, 2, 0, 1982, 0, 3032, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6052, 0, 13, 0, 0, 1, 0, 1608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 8118, 0, 0, 10, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 353, 0, 0, 0, 0, 5592, 2403, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8084, 8604, 8193, 8191, 8217, 7674, 8201, 8372] avg=8192.0000 max=8604 max_rank=1 min=7674 min_rank=5 imbalance_ratio=1.050293 max_load_reduction=56932 max_load_reduction_pct=0.868713 moved_to_echo=32464 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 3379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1799, 1708, 1289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2740, 0, 0, 2950, 0, 0, 0, 0, 0, 2600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 22, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2668, 0, 0, 0, 2203, 0, 3141, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6245, 0, 42, 0, 0, 0, 0, 2123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8128, 0, 0, 2, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 350, 0, 0, 0, 0, 5592, 2081, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8291, 8194, 8194, 8024, 8410, 8200, 8045] avg=8192.0000 max=8410 max_rank=5 min=8024 min_rank=4 imbalance_ratio=1.026611 max_load_reduction=57126 max_load_reduction_pct=0.871674 moved_to_echo=32679 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 13, 3683, 0, 0, 4, 0, 0, 0, 0, 0, 0, 2, 1853, 1636, 1180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2608, 0, 0, 3234, 0, 0, 0, 0, 1, 2597, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 5, 0, 0, 7, 0, 0, 0, 0, 2, 5, 1, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 1, 1, 0, 0, 0, 1, 8185, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3064, 0, 0, 0, 2015, 0, 2246, 16, 3, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 1, 0, 5982, 0, 24, 0, 0, 0, 0, 2626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 8156, 0, 1, 3, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4, 0, 0, 0, 0, 2, 0, 0, 360, 0, 0, 0, 2, 5585, 2143, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8372, 8443, 8192, 8196, 7346, 8639, 8227, 8121] avg=8192.0000 max=8639 max_rank=5 min=7346 min_rank=4 imbalance_ratio=1.054565 max_load_reduction=56897 max_load_reduction_pct=0.868179 moved_to_echo=32333 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 8, 3622, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1980, 1590, 1467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2738, 0, 0, 2446, 5, 0, 0, 0, 0, 2594, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1, 4, 2, 2, 0, 0, 0, 16, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3116, 0, 0, 1, 2488, 0, 3101, 43, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 2, 0, 0, 5795, 0, 46, 0, 1, 0, 0, 1706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8040, 0, 0, 15, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 323, 0, 1, 0, 0, 5597, 2288, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8669, 7799, 8199, 8191, 8751, 7553, 8140, 8234] avg=8192.0000 max=8751 max_rank=4 min=7553 min_rank=5 imbalance_ratio=1.068237 max_load_reduction=56785 max_load_reduction_pct=0.866470 moved_to_echo=32678 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 3, 0, 0, 3175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1857, 1583, 1133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2858, 3, 1, 2605, 0, 0, 0, 0, 0, 2599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 4, 0, 3, 0, 4, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8186, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3557, 0, 2, 0, 1936, 1, 3208, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5792, 0, 1, 1, 0, 0, 0, 2475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 8109, 1, 1, 0, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 1, 2, 0, 0, 0, 0, 0, 379, 0, 0, 0, 3, 5589, 2145, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7756, 8066, 8177, 8191, 8752, 8270, 8178, 8146] avg=8192.0000 max=8752 max_rank=4 min=7756 min_rank=0 imbalance_ratio=1.068359 max_load_reduction=56784 max_load_reduction_pct=0.866455 moved_to_echo=33346 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 10, 2966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2185, 1695, 1147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1897, 0, 0, 3029, 0, 0, 0, 0, 0, 2600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 0, 2, 0, 0, 1, 0, 0, 41, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8188, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2670, 0, 0, 0, 1958, 0, 3628, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6199, 0, 6, 0, 0, 0, 0, 2481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8091, 0, 0, 38, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 277, 0, 0, 0, 0, 5592, 2524, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8007, 7526, 8220, 8191, 8298, 8686, 8192, 8416] avg=8192.0000 max=8686 max_rank=5 min=7526 min_rank=1 imbalance_ratio=1.060303 max_load_reduction=56850 max_load_reduction_pct=0.867462 moved_to_echo=33592 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 128, 25984, 0, 0, 128, 128, 128, 128, 0, 0, 128, 128, 16128, 14208, 9344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 0, 128, 21248, 128, 128, 23936, 128, 128, 128, 0, 128, 20352, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 0, 128, 0, 24192, 128, 128, 128, 17536, 128, 23808, 256, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 128, 128, 128, 0, 48768, 0, 128, 128, 128, 128, 0, 16896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 0, 128, 0, 0, 2432, 128, 0, 128, 128, 45312, 17664, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 0, 0, 128, 0, 128, 128, 128, 64768, 128, 128, 128, 0, 128, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=67328 observed_loss=-1792 tokens_per_expert=[0, 128, 128, 128, 128, 128, 0, 128, 128, 128, 128, 128, 128, 128, 256, 128, 65408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 128, 27136, 0, 0, 128, 128, 128, 128, 0, 0, 128, 128, 15488, 13184, 9728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 0, 128, 21504, 128, 128, 23296, 128, 128, 128, 0, 128, 20864, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[128, 0, 128, 0, 24448, 128, 128, 128, 16640, 128, 24320, 384, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 128, 128, 128, 0, 48000, 0, 256, 128, 128, 128, 0, 17536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=67328 observed_loss=-1792 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 0, 128, 0, 0, 2816, 128, 128, 128, 128, 44800, 17920, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[0, 0, 128, 128, 0, 128, 128, 128, 0, 128, 128, 65536, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 0, 128, 0, 128, 128, 128, 64896, 128, 128, 128, 0, 128, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[0, 0, 128, 128, 0, 128, 128, 128, 128, 128, 128, 65536, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[0, 128, 128, 128, 128, 128, 0, 128, 128, 128, 128, 128, 128, 128, 128, 128, 65408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=67328 slack_slots=63744 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[2, 0, 30, 0, 0, 0, 6695, 0, 3266, 0, 0, 8190, 0, 0, 19, 0, 0, 0, 0, 0, 776, 8192, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 147, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 4, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 0, 5, 0, 0, 7523, 6, 3297, 4, 0, 8186, 0, 0, 31, 0, 2, 0, 0, 0, 586, 8186, 6, 0, 0, 8183, 0, 2, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 8187, 0, 0, 1, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 8185, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 5, 0, 0, 29, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[2, 0, 12, 0, 0, 0, 8052, 0, 5021, 0, 0, 8184, 0, 0, 23, 0, 0, 0, 0, 0, 180, 8192, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 1, 0, 9, 0, 0, 0, 0, 0, 1, 0, 0, 0, 42, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 0, 8110, 0, 7624, 0, 0, 8188, 0, 0, 241, 0, 0, 0, 0, 0, 7, 8192, 0, 0, 0, 8132, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 0, 7980, 0, 7519, 0, 0, 8181, 0, 0, 585, 0, 0, 0, 0, 0, 33, 8192, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8038, 0, 6855, 0, 0, 8157, 0, 0, 347, 0, 0, 0, 0, 0, 85, 8192, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 66, 0, 0, 0, 7954, 0, 7305, 0, 0, 8183, 0, 0, 177, 0, 0, 0, 0, 0, 233, 8192, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 24, 0, 0, 0, 7920, 0, 2580, 0, 0, 8192, 0, 0, 14, 0, 0, 0, 0, 0, 293, 8192, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 0, 2624, 0, 7519, 0, 0, 0, 0, 0, 585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 7929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8087, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 220, 0, 0, 0, 5356, 247, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 7998, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10732, 7962, 8145, 8192, 8198, 6006, 8111, 8190] avg=8192.0000 max=10732 max_rank=0 min=6006 min_rank=5 imbalance_ratio=1.310059 max_load_reduction=54804 max_load_reduction_pct=0.836243 moved_to_echo=30505 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 2643, 0, 6855, 0, 0, 0, 0, 0, 347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 7912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8085, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 0, 0, 0, 5395, 246, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7976, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9845, 7997, 8194, 8194, 8189, 6831, 8103, 8183] avg=8192.0000 max=9845 max_rank=0 min=6831 min_rank=5 imbalance_ratio=1.201782 max_load_reduction=55691 max_load_reduction_pct=0.849777 moved_to_echo=31306 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 24, 0, 0, 0, 2604, 0, 2580, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 293, 0, 0, 0, 0, 7921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8085, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5308, 0, 0, 0, 5316, 247, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 0, 0, 0, 8010, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5222, 8214, 8269, 8192, 8176, 11054, 8257, 8152] avg=8192.0000 max=11054 max_rank=5 min=5222 min_rank=0 imbalance_ratio=1.349365 max_load_reduction=54482 max_load_reduction_pct=0.831329 moved_to_echo=35639 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 30, 0, 0, 0, 2201, 0, 3266, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 776, 0, 0, 0, 0, 7924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8083, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5142, 0, 0, 0, 4494, 247, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 314, 0, 0, 0, 8008, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 4, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5518, 8700, 8273, 8192, 8197, 10066, 8430, 8160] avg=8192.0000 max=10066 max_rank=5 min=5518 min_rank=0 imbalance_ratio=1.228760 max_load_reduction=55470 max_load_reduction_pct=0.846405 moved_to_echo=34853 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 12, 0, 0, 0, 2648, 0, 5021, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, 0, 0, 0, 0, 7924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8080, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3016, 0, 0, 0, 5404, 247, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 1, 8002, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 1, 0, 0, 0, 42, 0, 0, 0, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7706, 8104, 8155, 8190, 8186, 8850, 8141, 8204] avg=8192.0000 max=8850 max_rank=5 min=7706 min_rank=0 imbalance_ratio=1.080322 max_load_reduction=56686 max_load_reduction_pct=0.864960 moved_to_echo=33381 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 5, 0, 0, 2474, 6, 3297, 4, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 586, 0, 6, 0, 0, 7926, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8081, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 1, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 8185, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4789, 0, 0, 0, 5049, 247, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 0, 8003, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 5, 0, 0, 29, 0, 0, 0, 0, 8145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5818, 8522, 8147, 8192, 8201, 10269, 8204, 8183] avg=8192.0000 max=10269 max_rank=5 min=5818 min_rank=0 imbalance_ratio=1.253540 max_load_reduction=55267 max_load_reduction_pct=0.843307 moved_to_echo=34857 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 66, 0, 0, 0, 2615, 0, 7305, 0, 0, 0, 0, 0, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 233, 0, 0, 0, 0, 7929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8085, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 0, 0, 0, 5339, 247, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 8000, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10163, 8162, 8209, 8193, 8184, 6279, 8172, 8174] avg=8192.0000 max=10163 max_rank=0 min=6279 min_rank=5 imbalance_ratio=1.240601 max_load_reduction=55373 max_load_reduction_pct=0.844925 moved_to_echo=30809 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 0, 2663, 0, 7624, 0, 0, 0, 0, 0, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 7868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8087, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 302, 0, 0, 0, 5447, 249, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 8005, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10532, 7875, 8144, 8191, 8205, 6181, 8118, 8290] avg=8192.0000 max=10532 max_rank=0 min=6181 min_rank=5 imbalance_ratio=1.285645 max_load_reduction=55004 max_load_reduction_pct=0.839294 moved_to_echo=30794 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=67328 slack_slots=63744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 23, 0, 0, 0, 8040, 0, 3029, 0, 0, 8192, 0, 0, 17, 0, 0, 0, 0, 0, 1015, 8192, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 0, 8154, 0, 7588, 0, 0, 8183, 0, 0, 281, 0, 0, 0, 0, 0, 198, 8192, 0, 0, 0, 8130, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[2, 0, 12, 0, 0, 0, 8136, 0, 6086, 0, 0, 8183, 0, 0, 47, 0, 0, 0, 0, 0, 785, 8192, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 1, 0, 9, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[2, 0, 24, 0, 0, 0, 7276, 0, 3912, 0, 0, 8187, 0, 0, 19, 0, 0, 0, 0, 0, 1999, 8192, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 4, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 67, 0, 0, 0, 8053, 0, 7068, 0, 0, 8175, 0, 0, 59, 0, 0, 0, 0, 0, 801, 8192, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8150, 0, 6960, 0, 0, 8153, 0, 0, 272, 0, 0, 0, 0, 0, 550, 8192, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 0, 5, 0, 0, 7924, 6, 4225, 4, 0, 8186, 0, 0, 19, 0, 2, 0, 0, 0, 1325, 8186, 6, 0, 0, 8184, 0, 2, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 8187, 0, 0, 1, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 8184, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 5, 0, 0, 15, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 0, 8038, 0, 7436, 0, 0, 8180, 0, 0, 552, 0, 0, 0, 0, 0, 234, 8192, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 23, 0, 0, 0, 2246, 0, 3029, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 0, 0, 0, 0, 7331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8138, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4221, 0, 0, 0, 5794, 834, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5315, 8346, 8233, 8192, 8178, 10904, 8191, 8177] avg=8192.0000 max=10904 max_rank=5 min=5315 min_rank=0 imbalance_ratio=1.331055 max_load_reduction=54632 max_load_reduction_pct=0.833618 moved_to_echo=35450 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 12, 0, 0, 0, 2273, 0, 6086, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 785, 0, 0, 0, 0, 7329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8129, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1311, 0, 0, 0, 5863, 833, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 1, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7, 0, 0, 0, 0, 8159, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8420, 8114, 8178, 8188, 8186, 8062, 8199, 8189] avg=8192.0000 max=8420 max_rank=0 min=8062 min_rank=5 imbalance_ratio=1.027832 max_load_reduction=57116 max_load_reduction_pct=0.871521 moved_to_echo=32636 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 0, 2274, 0, 7588, 0, 0, 0, 0, 0, 281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 198, 0, 0, 0, 0, 7284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8130, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 165, 0, 0, 0, 5880, 833, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 8164, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10147, 7482, 8171, 8190, 8202, 6931, 8181, 8232] avg=8192.0000 max=10147 max_rank=0 min=6931 min_rank=5 imbalance_ratio=1.238647 max_load_reduction=55389 max_load_reduction_pct=0.845169 moved_to_echo=31546 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 24, 0, 0, 0, 2033, 0, 3912, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1999, 0, 0, 0, 0, 7334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3045, 0, 0, 0, 5243, 834, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 27, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 4, 0, 8162, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5990, 9333, 8259, 8191, 8193, 9177, 8210, 8183] avg=8192.0000 max=9333 max_rank=1 min=5990 min_rank=0 imbalance_ratio=1.139282 max_load_reduction=56203 max_load_reduction_pct=0.857590 moved_to_echo=33763 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 2277, 0, 6960, 0, 0, 0, 0, 0, 272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 0, 0, 0, 0, 7323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8099, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 0, 0, 0, 5873, 833, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 8164, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9509, 7873, 8156, 8195, 8188, 7227, 8199, 8189] avg=8192.0000 max=9509 max_rank=0 min=7227 min_rank=5 imbalance_ratio=1.160767 max_load_reduction=56027 max_load_reduction_pct=0.854904 moved_to_echo=31803 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 67, 0, 0, 0, 2250, 0, 7068, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 801, 0, 0, 0, 0, 7340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8121, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 295, 0, 0, 0, 5803, 834, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 8164, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9444, 8141, 8207, 8193, 8187, 6986, 8194, 8184] avg=8192.0000 max=9444 max_rank=0 min=6986 min_rank=5 imbalance_ratio=1.152832 max_load_reduction=56092 max_load_reduction_pct=0.855896 moved_to_echo=31551 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 5, 0, 0, 2214, 6, 4225, 4, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1325, 0, 6, 0, 0, 7338, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 3, 0, 0, 1, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 8184, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2853, 0, 0, 0, 5710, 834, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 5, 0, 0, 15, 0, 0, 0, 0, 8159, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6474, 8673, 8164, 8195, 8202, 9452, 8180, 8196] avg=8192.0000 max=9452 max_rank=5 min=6474 min_rank=0 imbalance_ratio=1.153809 max_load_reduction=56084 max_load_reduction_pct=0.855774 moved_to_echo=34030 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 0, 2246, 0, 7436, 0, 0, 0, 0, 0, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 234, 0, 0, 0, 0, 7340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8126, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 0, 0, 0, 5792, 834, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 8165, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10237, 7574, 8168, 8192, 8200, 6797, 8182, 8186] avg=8192.0000 max=10237 max_rank=0 min=6797 min_rank=5 imbalance_ratio=1.249634 max_load_reduction=55299 max_load_reduction_pct=0.843796 moved_to_echo=31365 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 20352, 0, 0, 0, 41856, 2048, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 256, 128, 0, 0, 20480, 128, 43520, 128, 0, 0, 0, 0, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 2304, 0, 128, 0, 0, 63360, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 768, 0, 0, 128, 64128, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 0, 0, 0, 128, 128, 128, 0, 0, 384, 0, 0, 128, 0, 65280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 640, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64768, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 12544, 0, 0, 0, 46080, 6784, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 256, 128, 0, 0, 17920, 128, 46336, 128, 0, 0, 0, 0, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 6912, 0, 128, 0, 0, 58624, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 0, 0, 0, 128, 128, 128, 0, 0, 128, 0, 0, 128, 0, 65408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 128, 65536, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[65536, 128, 0, 0, 128, 0, 0, 0, 0, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 0, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 128, 65536, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 384, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65024, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8188, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 8184, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 6, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 1, 6, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 1894, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 37, 0, 8185, 3, 0, 0, 0, 6551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 6, 0, 0, 0, 0, 0, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 7885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 8186, 0, 6, 0, 0, 4, 0, 0, 0, 8186, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8126, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 95, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8192, 7953, 8389, 8169, 8264, 8192, 8190] avg=8192.0000 max=8389 max_rank=3 min=7953 min_rank=2 imbalance_ratio=1.024048 max_load_reduction=57147 max_load_reduction_pct=0.871994 moved_to_echo=32815 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8122, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8090, 0, 95, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8191, 7954, 8398, 8239, 8186, 8190, 8191] avg=8192.0000 max=8398 max_rank=3 min=7954 min_rank=2 imbalance_ratio=1.025146 max_load_reduction=57138 max_load_reduction_pct=0.871857 moved_to_echo=32806 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8126, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8014, 0, 95, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8198, 7944, 8399, 8300, 8110, 8201, 8194] avg=8192.0000 max=8399 max_rank=3 min=7944 min_rank=2 imbalance_ratio=1.025269 max_load_reduction=57137 max_load_reduction_pct=0.871841 moved_to_echo=32805 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8127, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 95, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8211, 8191, 7954, 8399, 8134, 8263, 8191, 8193] avg=8192.0000 max=8399 max_rank=3 min=7954 min_rank=2 imbalance_ratio=1.025269 max_load_reduction=57137 max_load_reduction_pct=0.871841 moved_to_echo=32781 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 143, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8124, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8155, 0, 95, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8194, 7948, 8403, 8165, 8253, 8196, 8191] avg=8192.0000 max=8403 max_rank=3 min=7948 min_rank=2 imbalance_ratio=1.025757 max_load_reduction=57133 max_load_reduction_pct=0.871780 moved_to_echo=32805 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8128, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 95, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8191, 7954, 8389, 8180, 8252, 8191, 8192] avg=8192.0000 max=8389 max_rank=3 min=7954 min_rank=2 imbalance_ratio=1.024048 max_load_reduction=57147 max_load_reduction_pct=0.871994 moved_to_echo=32815 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 1, 6, 8178, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1894, 0, 0, 0, 0, 7948, 0, 0, 0, 0, 37, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 6, 0, 8122, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 7885, 0, 95, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 6, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8191, 9882, 6758, 8144, 7991, 8185, 8193] avg=8192.0000 max=9882 max_rank=2 min=6758 min_rank=3 imbalance_ratio=1.206299 max_load_reduction=55654 max_load_reduction_pct=0.849213 moved_to_echo=32513 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8126, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 99, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8188, 7947, 8401, 8205, 8217, 8190, 8192] avg=8192.0000 max=8401 max_rank=3 min=7947 min_rank=2 imbalance_ratio=1.025513 max_load_reduction=57135 max_load_reduction_pct=0.871811 moved_to_echo=32804 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 823, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 7365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 8184, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 7, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8188, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 1, 9, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 1828, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 40, 0, 8185, 3, 0, 0, 0, 7877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 6, 0, 0, 0, 0, 0, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 6619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 8186, 0, 6, 0, 0, 4, 0, 0, 0, 8186, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7341, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7102, 0, 850, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8192, 7961, 8230, 8441, 8145, 8192, 8189] avg=8192.0000 max=8441 max_rank=4 min=7961 min_rank=2 imbalance_ratio=1.030396 max_load_reduction=57095 max_load_reduction_pct=0.871201 moved_to_echo=32967 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 37, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 823, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7338, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 7365, 0, 849, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8194, 7955, 8234, 8169, 8409, 8196, 8193] avg=8192.0000 max=8409 max_rank=5 min=7955 min_rank=2 imbalance_ratio=1.026489 max_load_reduction=57127 max_load_reduction_pct=0.871689 moved_to_echo=32967 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7962, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7341, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7788, 0, 851, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8211, 8191, 7963, 8230, 7726, 8832, 8191, 8192] avg=8192.0000 max=8832 max_rank=5 min=7726 min_rank=4 imbalance_ratio=1.078125 max_load_reduction=56704 max_load_reduction_pct=0.865234 moved_to_echo=32941 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7341, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7379, 0, 851, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8191, 7962, 8229, 8163, 8423, 8191, 8192] avg=8192.0000 max=8423 max_rank=5 min=7962 min_rank=2 imbalance_ratio=1.028198 max_load_reduction=57113 max_load_reduction_pct=0.871475 moved_to_echo=32969 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7343, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7716, 0, 849, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8188, 7953, 8238, 7818, 8760, 8190, 8192] avg=8192.0000 max=8760 max_rank=5 min=7818 min_rank=4 imbalance_ratio=1.069336 max_load_reduction=56776 max_load_reduction_pct=0.866333 moved_to_echo=32960 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7341, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5377, 0, 849, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8198, 7953, 8230, 10153, 6418, 8201, 8194] avg=8192.0000 max=10153 max_rank=4 min=6418 min_rank=5 imbalance_ratio=1.239380 max_load_reduction=55383 max_load_reduction_pct=0.845078 moved_to_echo=32966 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 1, 9, 8178, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1828, 0, 0, 0, 0, 7956, 0, 0, 0, 0, 40, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 6, 0, 7337, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 6619, 0, 849, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 6, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8191, 9827, 7915, 7359, 7671, 8185, 8193] avg=8192.0000 max=9827 max_rank=2 min=7359 min_rank=4 imbalance_ratio=1.199585 max_load_reduction=55709 max_load_reduction_pct=0.850052 moved_to_echo=31408 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7337, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7836, 0, 849, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8191, 7962, 8230, 7707, 8878, 8190, 8191] avg=8192.0000 max=8878 max_rank=5 min=7707 min_rank=4 imbalance_ratio=1.083740 max_load_reduction=56658 max_load_reduction_pct=0.864532 moved_to_echo=32966 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[1920, 0, 0, 0, 0, 63616, 0, 0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 64000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 1152, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 128, 0, 0, 0, 57216, 0, 6912, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 0, 0, 6784, 0, 0, 0, 0, 0, 128, 0, 128, 0, 58752, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[1920, 0, 0, 0, 0, 63744, 0, 0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 128, 128, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 0, 0, 512, 0, 0, 0, 0, 0, 128, 0, 128, 0, 65024, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[65536, 0, 128, 0, 128, 0, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 65280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 128, 0, 0, 0, 64768, 0, 768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[65536, 0, 0, 0, 128, 0, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7566, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 8192, 8190, 2, 0, 0, 0, 0, 0, 562, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 302, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3900, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 8178, 0, 7, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 8182, 3, 0, 0, 0, 0, 0, 3977, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 127, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6489, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8191, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 8191, 8190, 0, 0, 0, 0, 0, 0, 1566, 8191, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7053, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 8183, 0, 9, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 8183, 8180, 5, 0, 0, 0, 0, 0, 1063, 8187, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7962, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 223, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7846, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 8192, 8191, 0, 0, 0, 0, 0, 0, 329, 8191, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7196, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8191, 66, 0, 0, 0, 0, 0, 903, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7240, 0, 0, 0, 0, 0, 0, 1462, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 6, 0, 0, 0, 0, 8182, 0, 0, 0, 8176, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 6, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7870, 0, 0, 0, 7986, 7998, 43, 0, 0, 0, 6, 0, 0, 8162, 0, 0, 6] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7962, 0, 0, 0, 0, 0, 0, 2189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5986, 1069, 149, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7123, 0, 0, 0, 0, 0, 0, 0, 223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8043, 8211, 8174, 10151, 8193, 7224, 7346] avg=8192.0000 max=10151 max_rank=4 min=7224 min_rank=6 imbalance_ratio=1.239136 max_load_reduction=55385 max_load_reduction_pct=0.845108 moved_to_echo=32914 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 8043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7196, 0, 0, 0, 0, 0, 0, 2189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5986, 1069, 149, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7123, 0, 66, 0, 0, 0, 0, 0, 903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8071, 8215, 8174, 9385, 8185, 7220, 8092] avg=8192.0000 max=9385 max_rank=4 min=7220 min_rank=6 imbalance_ratio=1.145630 max_load_reduction=56151 max_load_reduction_pct=0.856796 moved_to_echo=32882 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7846, 0, 0, 0, 0, 0, 0, 2180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 7, 5988, 1075, 150, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7117, 0, 0, 0, 0, 0, 0, 0, 329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8044, 8223, 8168, 10026, 8192, 7242, 7446] avg=8192.0000 max=10026 max_rank=4 min=7242 min_rank=6 imbalance_ratio=1.223877 max_load_reduction=55510 max_load_reduction_pct=0.847015 moved_to_echo=32906 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7240, 0, 0, 0, 0, 0, 0, 1436, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 7996, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 6, 0, 0, 0, 0, 2187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7870, 5979, 1042, 26, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6944, 0, 43, 0, 0, 0, 6, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8682, 8024, 8324, 2218, 8186, 14939, 6999] avg=8192.0000 max=14939 max_rank=6 min=2218 min_rank=4 imbalance_ratio=1.823608 max_load_reduction=50597 max_load_reduction_pct=0.772049 moved_to_echo=32342 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 0, 0, 0, 0, 0, 0, 8041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6489, 0, 0, 0, 0, 0, 0, 2189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 11, 5985, 1069, 149, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7122, 0, 0, 0, 0, 0, 0, 0, 1566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8168, 8208, 8174, 8678, 8195, 7231, 8688] avg=8192.0000 max=8688 max_rank=7 min=7231 min_rank=6 imbalance_ratio=1.060547 max_load_reduction=56848 max_load_reduction_pct=0.867432 moved_to_echo=32792 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 8187, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 8043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7053, 0, 0, 0, 0, 0, 0, 2185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 9, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 16, 5972, 1068, 149, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7115, 0, 5, 0, 0, 0, 0, 0, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8088, 8225, 8174, 9238, 8199, 7233, 8183] avg=8192.0000 max=9238 max_rank=4 min=7233 min_rank=6 imbalance_ratio=1.127686 max_load_reduction=56298 max_load_reduction_pct=0.859039 moved_to_echo=32853 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 8041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7566, 0, 0, 0, 0, 0, 0, 2189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 5986, 1069, 149, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7123, 0, 2, 0, 0, 0, 0, 0, 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8095, 8208, 8174, 9755, 8194, 7229, 7687] avg=8192.0000 max=9755 max_rank=4 min=7229 min_rank=6 imbalance_ratio=1.190796 max_load_reduction=55781 max_load_reduction_pct=0.851151 moved_to_echo=32865 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 302, 0, 0, 0, 0, 0, 0, 8043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3900, 0, 0, 0, 0, 0, 0, 2185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 7, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 5974, 1068, 149, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7115, 0, 3, 0, 0, 0, 0, 0, 3977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8345, 8222, 8174, 6085, 8192, 7218, 11095] avg=8192.0000 max=11095 max_rank=7 min=6085 min_rank=4 imbalance_ratio=1.354370 max_load_reduction=54441 max_load_reduction_pct=0.830704 moved_to_echo=32590 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7867, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8192, 8189, 1, 0, 0, 0, 0, 0, 314, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6624, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8191, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 8191, 8188, 0, 0, 0, 0, 0, 0, 1476, 8191, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7242, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 8183, 0, 9, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 8183, 8180, 4, 0, 0, 0, 0, 0, 867, 8187, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7694, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8192, 8191, 80, 0, 0, 0, 0, 0, 410, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5423, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 8177, 0, 5, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 8181, 5, 0, 0, 0, 0, 0, 2680, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7649, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 8192, 8190, 2, 0, 0, 0, 0, 0, 496, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6148, 0, 0, 0, 0, 0, 0, 2984, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 6, 0, 0, 0, 0, 8180, 0, 0, 0, 8171, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 6, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7921, 0, 0, 0, 7508, 7974, 49, 0, 0, 0, 6, 0, 0, 8161, 0, 0, 6] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7964, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 221, 8192, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 8187, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 8035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7242, 0, 0, 0, 0, 0, 0, 1877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 9, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 29, 6277, 746, 157, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7437, 0, 4, 0, 0, 0, 0, 0, 867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8072, 8231, 8173, 9119, 8200, 7237, 8308] avg=8192.0000 max=9119 max_rank=4 min=7237 min_rank=6 imbalance_ratio=1.113159 max_load_reduction=56417 max_load_reduction_pct=0.860855 moved_to_echo=32864 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 8033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7649, 0, 0, 0, 0, 0, 0, 1881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 6291, 747, 157, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7445, 0, 2, 0, 0, 0, 0, 0, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8069, 8210, 8173, 9530, 8195, 7222, 7943] avg=8192.0000 max=9530 max_rank=4 min=7222 min_rank=6 imbalance_ratio=1.163330 max_load_reduction=56006 max_load_reduction_pct=0.854584 moved_to_echo=32890 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 8035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7694, 0, 0, 0, 0, 0, 0, 1881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6291, 747, 157, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7445, 0, 80, 0, 0, 0, 0, 0, 410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8043, 8218, 8173, 9575, 8186, 7212, 7935] avg=8192.0000 max=9575 max_rank=4 min=7212 min_rank=6 imbalance_ratio=1.168823 max_load_reduction=55961 max_load_reduction_pct=0.853897 moved_to_echo=32908 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 0, 8035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5423, 0, 0, 0, 0, 0, 0, 1878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 5, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 6278, 746, 157, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7437, 0, 5, 0, 0, 0, 0, 0, 2680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8111, 8225, 8173, 7301, 8190, 7209, 10122] avg=8192.0000 max=10122 max_rank=7 min=7209 min_rank=6 imbalance_ratio=1.235596 max_load_reduction=55414 max_load_reduction_pct=0.845551 moved_to_echo=32822 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6148, 0, 0, 0, 0, 0, 0, 2927, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 7972, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 6, 0, 0, 0, 0, 1879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 6, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7921, 6282, 684, 57, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6824, 0, 49, 0, 0, 0, 6, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8163, 9081, 8003, 8333, 1923, 8182, 14966, 6885] avg=8192.0000 max=14966 max_rank=6 min=1923 min_rank=4 imbalance_ratio=1.826904 max_load_reduction=50570 max_load_reduction_pct=0.771637 moved_to_echo=31956 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7964, 0, 0, 0, 0, 0, 0, 1880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 6291, 747, 157, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7445, 0, 0, 0, 0, 0, 0, 0, 221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8035, 8216, 8173, 9844, 8194, 7214, 7666] avg=8192.0000 max=9844 max_rank=4 min=7214 min_rank=6 imbalance_ratio=1.201660 max_load_reduction=55692 max_load_reduction_pct=0.849792 moved_to_echo=32918 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8191, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 8035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6624, 0, 0, 0, 0, 0, 0, 1880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 38, 6291, 747, 157, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7444, 0, 0, 0, 0, 0, 0, 0, 1476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8090, 8209, 8173, 8504, 8196, 7250, 8920] avg=8192.0000 max=8920 max_rank=7 min=7250 min_rank=6 imbalance_ratio=1.088867 max_load_reduction=56616 max_load_reduction_pct=0.863892 moved_to_echo=32870 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7867, 0, 0, 0, 0, 0, 0, 1873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 2, 6292, 750, 158, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7442, 0, 1, 0, 0, 0, 0, 0, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8035, 8224, 8165, 9740, 8193, 7226, 7757] avg=8192.0000 max=9740 max_rank=4 min=7226 min_rank=6 imbalance_ratio=1.188965 max_load_reduction=55796 max_load_reduction_pct=0.851379 moved_to_echo=32916 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 7936, 47872, 8576, 1152, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 48128, 0, 128, 0, 0, 0, 0, 17536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 56832, 0, 128, 0, 0, 0, 128, 0, 8704, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 7808, 0, 0, 0, 0, 0, 0, 57856, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 8064, 50304, 6016, 1280, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 50560, 0, 128, 0, 0, 0, 0, 15104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 59008, 0, 256, 0, 0, 0, 128, 0, 6528, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 256, 0, 0, 0, 65408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 65280, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 65536, 0, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 65408, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 6400, 0, 0, 0, 0, 0, 0, 59264, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 256, 0, 0, 0, 65408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8142, 0, 0, 0, 1155, 0, 0, 0, 0, 6842, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1996, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 6551, 0, 0, 19, 8058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8143, 0, 0, 0, 1535, 0, 0, 0, 0, 6371, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1857, 8191, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 6665, 0, 0, 1, 8138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8180, 8186, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2, 0, 0, 0, 0, 0, 1055, 0, 0, 0, 7192, 7, 3, 0, 0, 7893, 0, 2533, 0, 0, 0, 0, 0, 0, 0, 30, 0, 22, 5509, 0, 0, 0, 0, 0, 0, 0, 0, 436, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8175, 7975, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 1, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 0, 0, 20, 0, 6, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6540, 0, 0, 0, 1203, 0, 0, 0, 0, 7886, 0, 566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6584, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1809, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7390, 0, 0, 0, 530, 0, 0, 0, 0, 7085, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3838, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5759, 0, 0, 5, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5846, 0, 0, 0, 803, 0, 0, 0, 0, 7916, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4457, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5439, 0, 0, 1, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6983, 0, 0, 0, 243, 0, 0, 0, 0, 8175, 0, 666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5547, 8192, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2961, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7781, 0, 0, 0, 459, 0, 0, 0, 0, 8008, 0, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1438, 8192, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 6793, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8142, 0, 1347, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1155, 0, 0, 0, 0, 5495, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1996, 5125, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6551, 0, 0, 19, 2647, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5411, 2832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 9723, 6658, 7122, 9217, 8244, 8188, 8188] avg=8192.0000 max=9723 max_rank=1 min=6658 min_rank=2 imbalance_ratio=1.186890 max_load_reduction=55813 max_load_reduction_pct=0.851639 moved_to_echo=33837 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6540, 0, 1552, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1203, 0, 0, 0, 0, 6334, 0, 566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6584, 5126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1809, 0, 0, 0, 2690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5496, 2832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8326, 8103, 11710, 4499, 8329, 8189, 8186] avg=8192.0000 max=11710 max_rank=3 min=4499 min_rank=4 imbalance_ratio=1.429443 max_load_reduction=53826 max_load_reduction_pct=0.821320 moved_to_echo=29203 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8143, 0, 1254, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1535, 0, 0, 0, 0, 5117, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1857, 5125, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6665, 0, 0, 1, 2674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 5464, 2832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 9631, 6695, 6994, 9340, 8316, 8188, 8196] avg=8192.0000 max=9631 max_rank=1 min=6695 min_rank=2 imbalance_ratio=1.175659 max_load_reduction=55905 max_load_reduction_pct=0.853043 moved_to_echo=34040 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7390, 0, 1394, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 0, 0, 0, 0, 5691, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3838, 5119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5759, 0, 0, 5, 2682, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5482, 2828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 9018, 6229, 8957, 8446, 8310, 8188, 8187] avg=8192.0000 max=9018 max_rank=1 min=6229 min_rank=2 imbalance_ratio=1.100830 max_load_reduction=56518 max_load_reduction_pct=0.862396 moved_to_echo=33131 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2, 0, 0, 0, 0, 0, 1055, 0, 1553, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7192, 7, 3, 0, 0, 6340, 0, 2533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 22, 3448, 0, 0, 0, 0, 0, 0, 0, 0, 436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8175, 2620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 1, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 0, 5355, 1904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 0, 0, 20, 0, 6, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 8171, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 2772, 16075, 3936, 10797, 7356, 8218, 8186] avg=8192.0000 max=16075 max_rank=2 min=2772 min_rank=1 imbalance_ratio=1.962280 max_load_reduction=49461 max_load_reduction_pct=0.754715 moved_to_echo=34557 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7781, 0, 1576, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 459, 0, 0, 0, 0, 6432, 0, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1438, 5126, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6793, 0, 0, 0, 2683, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 5484, 2832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 9591, 6998, 6572, 9476, 8326, 8188, 8207] avg=8192.0000 max=9591 max_rank=1 min=6572 min_rank=3 imbalance_ratio=1.170776 max_load_reduction=55945 max_load_reduction_pct=0.853653 moved_to_echo=34197 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5846, 0, 1558, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 803, 0, 0, 0, 0, 6358, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4457, 5126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5439, 0, 0, 1, 2677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 5472, 2832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 7638, 7307, 9583, 8117, 8315, 8188, 8187] avg=8192.0000 max=9583 max_rank=3 min=7307 min_rank=2 imbalance_ratio=1.169800 max_load_reduction=55953 max_load_reduction_pct=0.853775 moved_to_echo=32807 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6983, 0, 1613, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 0, 0, 0, 0, 6562, 0, 666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5547, 5114, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2961, 0, 0, 1, 2682, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5503, 2837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 3, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8837, 7471, 10662, 5644, 8340, 8189, 8199] avg=8192.0000 max=10662 max_rank=3 min=5644 min_rank=4 imbalance_ratio=1.301514 max_load_reduction=54874 max_load_reduction_pct=0.837311 moved_to_echo=30372 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 2936, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5543, 8192, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7834, 0, 0, 57, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 3052, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5420, 8192, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 7834, 0, 0, 7, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7872, 0, 0, 0, 782, 0, 0, 0, 0, 2, 0, 955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7813, 8192, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7149, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8106, 0, 0, 0, 2534, 0, 0, 0, 0, 1, 0, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5858, 8192, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 7824, 0, 0, 1, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7319, 0, 0, 0, 2629, 0, 0, 0, 0, 4, 0, 320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7124, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7151, 0, 0, 6, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8178, 8186, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2, 0, 0, 0, 0, 0, 3159, 0, 0, 0, 8020, 7, 3, 0, 0, 1160, 0, 3998, 0, 0, 0, 0, 0, 0, 0, 32, 0, 286, 7413, 0, 0, 0, 0, 0, 0, 0, 0, 486, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8186, 8057, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 1, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 0, 0, 42, 0, 6, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8102, 0, 0, 0, 2409, 0, 0, 0, 0, 2, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6455, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7556, 0, 0, 9, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7820, 0, 0, 0, 2503, 0, 0, 0, 0, 3, 0, 1531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7594, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5131, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2936, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5543, 2393, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7834, 0, 0, 57, 848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7308, 857, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 9042, 7077, 7937, 8739, 8170, 8185, 8188] avg=8192.0000 max=9042 max_rank=1 min=7077 min_rank=2 imbalance_ratio=1.103760 max_load_reduction=56494 max_load_reduction_pct=0.862030 moved_to_echo=33282 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8106, 0, 861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2534, 0, 0, 0, 0, 1, 0, 220, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5858, 2393, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7824, 0, 0, 1, 851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 7337, 857, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8967, 6836, 8259, 8676, 8222, 8185, 8211] avg=8192.0000 max=8967 max_rank=1 min=6836 min_rank=2 imbalance_ratio=1.094604 max_load_reduction=56569 max_load_reduction_pct=0.863174 moved_to_echo=33294 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3052, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5420, 2393, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7834, 0, 0, 7, 851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 7328, 857, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 9025, 7215, 7825, 8692, 8217, 8185, 8199] avg=8192.0000 max=9025 max_rank=1 min=7215 min_rank=2 imbalance_ratio=1.101685 max_load_reduction=56511 max_load_reduction_pct=0.862289 moved_to_echo=33293 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7872, 0, 862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 782, 0, 0, 0, 0, 2, 0, 955, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7813, 2381, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7149, 0, 0, 0, 847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7345, 861, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8734, 5827, 10195, 7996, 8213, 8186, 8196] avg=8192.0000 max=10195 max_rank=3 min=5827 min_rank=2 imbalance_ratio=1.244507 max_load_reduction=55341 max_load_reduction_pct=0.844437 moved_to_echo=32591 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8102, 0, 860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2409, 0, 0, 0, 0, 2, 0, 58, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6455, 2392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7556, 0, 0, 9, 851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7330, 857, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8962, 6548, 8847, 8416, 8191, 8186, 8186] avg=8192.0000 max=8962 max_rank=1 min=6548 min_rank=2 imbalance_ratio=1.093994 max_load_reduction=56574 max_load_reduction_pct=0.863251 moved_to_echo=32979 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7319, 0, 861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2629, 0, 0, 0, 0, 4, 0, 320, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7124, 2393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7151, 0, 0, 6, 852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 7340, 857, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8180, 7034, 9517, 8009, 8224, 8186, 8186] avg=8192.0000 max=9517 max_rank=3 min=7034 min_rank=2 imbalance_ratio=1.161743 max_load_reduction=56019 max_load_reduction_pct=0.854782 moved_to_echo=32605 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2, 0, 0, 0, 0, 0, 3159, 0, 779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8020, 7, 3, 0, 0, 1160, 0, 3998, 0, 0, 0, 0, 0, 0, 3693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 286, 2165, 0, 0, 0, 0, 0, 0, 0, 0, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8186, 838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 1, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 0, 7219, 776, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 0, 0, 42, 0, 6, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 8166, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 3945, 16881, 2969, 9025, 8100, 8237, 8184] avg=8192.0000 max=16881 max_rank=2 min=2969 min_rank=3 imbalance_ratio=2.060669 max_load_reduction=48655 max_load_reduction_pct=0.742416 moved_to_echo=33546 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7820, 0, 861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2503, 0, 0, 0, 0, 3, 0, 1531, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7594, 2393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5131, 0, 0, 0, 852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7337, 857, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8681, 8118, 9987, 5983, 8199, 8186, 8186] avg=8192.0000 max=9987 max_rank=3 min=5983 min_rank=4 imbalance_ratio=1.219116 max_load_reduction=55549 max_load_reduction_pct=0.847610 moved_to_echo=30554 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 128, 43776, 21760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 25856, 39424, 0, 0, 0, 0, 0, 0, 0, 128, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 36096, 0, 0, 8320, 21376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 51968, 0, 11904, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 13184, 128, 128, 0, 0, 48384, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 24960, 128, 128, 0, 0, 1280, 0, 7296, 0, 0, 0, 0, 0, 0, 32384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 46208, 18944, 0, 0, 0, 0, 0, 0, 0, 128, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 128, 58624, 6784, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 58752, 0, 6912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 50560, 0, 0, 8320, 6912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 128, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7843, 0, 0, 0, 0, 0, 344, 0, 0, 0, 0, 0, 14, 1, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 7, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 11, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7889, 0, 9, 0, 0, 0, 261, 0, 0, 0, 0, 0, 38, 18, 0, 0, 8188, 0, 0, 0, 8192, 0, 0, 0, 51, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8087, 0, 24, 0, 0, 0, 76, 0, 0, 0, 0, 0, 7, 27, 0, 0, 8189, 0, 0, 0, 8192, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 12, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 5, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8122, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 40, 7, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 26, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 688, 0, 0, 0, 2, 0, 0, 0, 0, 0, 727, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 0, 0, 0, 5, 0, 0, 6669, 0, 0, 0, 0, 0, 7671, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 1, 0, 8000, 0, 0, 6, 0, 7851, 0, 0, 0, 6, 0, 0, 0, 0, 90, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 6, 8184, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 8186, 1, 0, 0, 0, 5, 6, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8051, 0, 0, 0, 0, 0, 145, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 2, 8192, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 3, 0, 0, 0, 10, 0, 0, 0, 0, 0, 8, 7, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 15, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 11, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 120, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 12, 0, 0, 0, 8107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 5, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8037, 8191, 8351, 8150, 8220, 8191, 8199, 8197] avg=8192.0000 max=8351 max_rank=2 min=8037 min_rank=0 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32807 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8178, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7843, 0, 0, 120, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 344, 0, 0, 0, 0, 0, 14, 1, 0, 0, 8107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 7, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8038, 8190, 8037, 8466, 8222, 8192, 8193, 8198] avg=8192.0000 max=8466 max_rank=3 min=8037 min_rank=2 imbalance_ratio=1.033447 max_load_reduction=57070 max_load_reduction_pct=0.870819 moved_to_echo=32805 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8087, 0, 24, 120, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 7, 27, 0, 0, 8104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8037, 8190, 8304, 8214, 8211, 8192, 8184, 8204] avg=8192.0000 max=8304 max_rank=2 min=8037 min_rank=0 imbalance_ratio=1.013672 max_load_reduction=57232 max_load_reduction_pct=0.873291 moved_to_echo=32791 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 8169, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8122, 0, 0, 120, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 40, 7, 0, 0, 8107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 26, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8042, 8191, 8325, 8189, 8224, 8192, 8195, 8178] avg=8192.0000 max=8325 max_rank=2 min=8042 min_rank=0 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32789 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8051, 0, 0, 120, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 2, 8192, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8040, 8190, 8245, 8253, 8222, 8192, 8192, 8202] avg=8192.0000 max=8253 max_rank=3 min=8040 min_rank=0 imbalance_ratio=1.007446 max_load_reduction=57283 max_load_reduction_pct=0.874069 moved_to_echo=32808 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 688, 0, 0, 0, 2, 0, 0, 0, 0, 0, 727, 0, 0, 7848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6, 0, 0, 3, 0, 0, 8171, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 0, 0, 0, 5, 0, 0, 6669, 0, 0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7671, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 6, 0, 7851, 0, 0, 0, 6, 0, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 6, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 1, 0, 0, 0, 5, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9265, 8192, 7794, 7717, 7977, 8194, 8199, 8198] avg=8192.0000 max=9265 max_rank=0 min=7717 min_rank=3 imbalance_ratio=1.130981 max_load_reduction=56271 max_load_reduction_pct=0.858627 moved_to_echo=32568 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 8165, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7889, 0, 9, 120, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 261, 0, 0, 0, 0, 0, 38, 18, 0, 0, 8103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 51, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8037, 8195, 8103, 8420, 8247, 8192, 8176, 8166] avg=8192.0000 max=8420 max_rank=3 min=8037 min_rank=0 imbalance_ratio=1.027832 max_load_reduction=57116 max_load_reduction_pct=0.871521 moved_to_echo=32781 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8178, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 3, 126, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 8, 7, 0, 0, 8102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 15, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8040, 8197, 8377, 8127, 8213, 8191, 8198, 8193] avg=8192.0000 max=8377 max_rank=2 min=8040 min_rank=0 imbalance_ratio=1.022583 max_load_reduction=57159 max_load_reduction_pct=0.872177 moved_to_echo=32795 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8109, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 33, 11, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 28, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8003, 0, 27, 0, 0, 0, 158, 0, 0, 0, 0, 0, 0, 29, 0, 0, 8191, 0, 0, 0, 8192, 0, 0, 0, 1, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 11, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7895, 0, 17, 0, 0, 0, 246, 0, 0, 0, 0, 0, 18, 18, 0, 0, 8189, 0, 0, 0, 8192, 0, 0, 0, 57, 8170, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7751, 0, 0, 0, 0, 0, 439, 0, 0, 0, 0, 0, 3, 1, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 7, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8029, 0, 0, 0, 0, 0, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 2, 8192, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 800, 0, 0, 0, 1, 0, 0, 0, 0, 0, 726, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1810, 0, 0, 0, 6, 0, 0, 6070, 0, 0, 0, 0, 0, 7362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 1, 0, 7983, 0, 0, 5, 0, 7841, 0, 0, 0, 6, 0, 0, 0, 0, 88, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 6, 8184, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 8186, 1, 0, 0, 0, 5, 6, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 1, 0, 0, 0, 14, 0, 0, 0, 0, 0, 2, 8, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 17, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 6, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 6, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 8172, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8109, 0, 0, 135, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 33, 11, 0, 0, 8126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 28, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8025, 8193, 8309, 8222, 8225, 8192, 8192, 8178] avg=8192.0000 max=8309 max_rank=2 min=8025 min_rank=0 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32787 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8181, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7751, 0, 0, 135, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 439, 0, 0, 0, 0, 0, 3, 1, 0, 0, 8126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 7, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8028, 8189, 7942, 8569, 8223, 8192, 8196, 8197] avg=8192.0000 max=8569 max_rank=3 min=7942 min_rank=2 imbalance_ratio=1.046021 max_load_reduction=56967 max_load_reduction_pct=0.869247 moved_to_echo=32808 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8029, 0, 0, 135, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 2, 8192, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8029, 8189, 8220, 8290, 8223, 8192, 8192, 8201] avg=8192.0000 max=8290 max_rank=3 min=8029 min_rank=0 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32808 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8003, 0, 27, 135, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 0, 0, 0, 0, 0, 0, 29, 0, 0, 8125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 1, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8025, 8189, 8220, 8312, 8212, 8192, 8184, 8202] avg=8192.0000 max=8312 max_rank=3 min=8025 min_rank=0 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32790 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8185, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 1, 142, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 2, 8, 0, 0, 8120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 17, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8031, 8199, 8371, 8144, 8215, 8191, 8192, 8193] avg=8192.0000 max=8371 max_rank=2 min=8031 min_rank=0 imbalance_ratio=1.021851 max_load_reduction=57165 max_load_reduction_pct=0.872269 moved_to_echo=32791 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 8168, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7895, 0, 17, 135, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 0, 0, 0, 0, 0, 18, 18, 0, 0, 8123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 57, 8170, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8025, 8195, 8114, 8405, 8253, 8192, 8182, 8170] avg=8192.0000 max=8405 max_rank=3 min=8025 min_rank=0 imbalance_ratio=1.026001 max_load_reduction=57131 max_load_reduction_pct=0.871750 moved_to_echo=32797 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 800, 0, 0, 0, 1, 0, 0, 0, 0, 0, 726, 0, 0, 7821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6, 0, 0, 4, 0, 0, 8174, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1810, 0, 0, 0, 6, 0, 0, 6070, 0, 0, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 5, 0, 7841, 0, 0, 0, 6, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 6, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 1, 0, 0, 0, 5, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9348, 8192, 8018, 7427, 7963, 8194, 8196, 8198] avg=8192.0000 max=9348 max_rank=0 min=7427 min_rank=3 imbalance_ratio=1.141113 max_load_reduction=56188 max_load_reduction_pct=0.857361 moved_to_echo=32551 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 0, 0, 135, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 6, 0, 0, 0, 8126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 6, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8025, 8190, 8342, 8167, 8222, 8191, 8202, 8197] avg=8192.0000 max=8342 max_rank=2 min=8025 min_rank=0 imbalance_ratio=1.018311 max_load_reduction=57194 max_load_reduction_pct=0.872711 moved_to_echo=32812 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 128, 0, 0, 1024, 0, 0, 0, 128, 0, 0, 63104, 0, 128, 1024, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 8576, 0, 0, 0, 0, 0, 128, 128, 0, 0, 56832, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 768, 0, 0, 0, 128, 0, 0, 0, 0, 0, 768, 128, 0, 64128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 128, 0, 128, 0, 0, 65408, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 65536, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 896, 0, 0, 0, 128, 0, 0, 0, 0, 0, 768, 0, 0, 64128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 128, 0, 0, 1920, 0, 0, 0, 128, 0, 0, 62208, 0, 128, 1152, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 128, 0, 128, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 8576, 0, 0, 0, 0, 0, 128, 128, 0, 0, 56960, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 65536, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 256, 0, 0, 128, 128, 65152, 0, 0, 0, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 128, 128, 0, 128, 0, 0, 65536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7217, 14, 0, 0, 0, 0, 0, 0, 5293, 0, 8192, 0, 0, 0, 0, 0, 3422, 0, 0, 11, 0, 77, 0, 0, 0, 0, 31, 0, 0, 12, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 7901, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 595, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6997, 0, 0, 0, 0, 0, 0, 0, 6522, 0, 8192, 0, 0, 0, 0, 0, 2645, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8115, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 288, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7861, 0, 0, 0, 0, 0, 0, 0, 7530, 0, 8192, 0, 0, 0, 0, 0, 435, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 580, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7275, 9, 0, 0, 0, 0, 0, 0, 7429, 0, 8192, 0, 0, 0, 0, 0, 1314, 0, 0, 9, 0, 94, 0, 0, 0, 0, 33, 0, 0, 11, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 8102, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 294, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7060, 0, 0, 0, 0, 0, 0, 0, 6079, 0, 8192, 0, 0, 0, 0, 0, 2641, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8129, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 573, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6881, 1, 0, 0, 0, 0, 0, 0, 6823, 0, 8192, 0, 0, 0, 0, 0, 2565, 0, 0, 1, 0, 27, 0, 0, 0, 0, 6, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 8059, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 203, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6800, 1, 0, 0, 0, 0, 0, 0, 5911, 0, 8192, 0, 0, 0, 0, 0, 2748, 0, 0, 1, 0, 18, 0, 0, 0, 0, 9, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8130, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 951, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 263, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8020, 0, 0, 0, 0, 0, 6, 1364, 7073, 4, 8177, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 5811, 0, 0, 0, 4, 0, 0, 6, 0, 0, 0, 0, 0, 4, 0, 0, 0, 6, 2, 6, 5, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8105, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 2061, 6, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 40] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7217, 14, 428, 350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4864, 0, 0, 0, 0, 0, 0, 0, 3422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 77, 0, 0, 0, 0, 31, 0, 0, 12, 0, 0, 0, 9, 6803, 439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 748, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 595, 0, 0, 0, 0, 0, 7753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8009, 8286, 7382, 8173, 8203, 8940, 8194, 8349] avg=8192.0000 max=8940 max_rank=5 min=7382 min_rank=2 imbalance_ratio=1.091309 max_load_reduction=56596 max_load_reduction_pct=0.863586 moved_to_echo=33686 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7861, 0, 609, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6920, 0, 0, 0, 0, 0, 0, 0, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7016, 440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 772, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 580, 0, 0, 0, 0, 0, 7752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8831, 7355, 7476, 8183, 8203, 8964, 8192, 8332] avg=8192.0000 max=8964 max_rank=5 min=7355 min_rank=1 imbalance_ratio=1.094238 max_load_reduction=56572 max_load_reduction_pct=0.863220 moved_to_echo=33691 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6881, 1, 552, 357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6270, 0, 0, 0, 0, 0, 0, 0, 2565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 27, 0, 0, 0, 0, 6, 0, 0, 1, 0, 0, 0, 1, 6939, 440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 763, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 203, 0, 0, 0, 0, 0, 7752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7791, 8835, 7415, 8182, 8202, 8955, 8201, 7955] avg=8192.0000 max=8955 max_rank=5 min=7415 min_rank=2 imbalance_ratio=1.093140 max_load_reduction=56581 max_load_reduction_pct=0.863358 moved_to_echo=33313 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6997, 0, 527, 359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5994, 0, 0, 0, 0, 0, 0, 0, 2645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6987, 440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 769, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 288, 0, 0, 0, 0, 0, 7752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7883, 8639, 7429, 8182, 8203, 8961, 8199, 8040] avg=8192.0000 max=8961 max_rank=5 min=7429 min_rank=2 imbalance_ratio=1.093872 max_load_reduction=56575 max_load_reduction_pct=0.863266 moved_to_echo=33403 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7060, 0, 492, 360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5586, 0, 0, 0, 0, 0, 0, 0, 2641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6999, 440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 770, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 573, 0, 0, 0, 0, 0, 7752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7912, 8227, 7481, 8234, 8203, 8962, 8192, 8325] avg=8192.0000 max=8962 max_rank=5 min=7481 min_rank=2 imbalance_ratio=1.093994 max_load_reduction=56574 max_load_reduction_pct=0.863251 moved_to_echo=33682 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7275, 9, 601, 359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6827, 0, 0, 0, 0, 0, 0, 0, 1314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 94, 0, 0, 0, 0, 33, 0, 0, 11, 0, 0, 0, 8, 6976, 439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 767, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 294, 0, 0, 0, 0, 0, 7753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8244, 8141, 7570, 8193, 8196, 8959, 8181, 8052] avg=8192.0000 max=8959 max_rank=5 min=7570 min_rank=2 imbalance_ratio=1.093628 max_load_reduction=56577 max_load_reduction_pct=0.863297 moved_to_echo=33388 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 263, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8020, 0, 572, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1364, 6500, 4, 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5811, 0, 0, 0, 4, 0, 0, 6, 0, 0, 0, 0, 0, 7048, 440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 6, 2, 6, 5, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8105, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 775, 0, 0, 2061, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 40, 7737, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9219, 7881, 13309, 8198, 8117, 2842, 8190, 7780] avg=8192.0000 max=13309 max_rank=2 min=2842 min_rank=5 imbalance_ratio=1.624634 max_load_reduction=52227 max_load_reduction_pct=0.796921 moved_to_echo=26929 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6800, 1, 482, 364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5424, 0, 0, 0, 0, 0, 0, 0, 2748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 18, 0, 0, 0, 0, 9, 0, 0, 1, 0, 0, 0, 0, 7005, 440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 11, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 761, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 951, 0, 0, 0, 0, 0, 7752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7647, 8172, 7474, 8191, 8209, 8953, 8187, 8703] avg=8192.0000 max=8953 max_rank=5 min=7474 min_rank=2 imbalance_ratio=1.092896 max_load_reduction=56583 max_load_reduction_pct=0.863388 moved_to_echo=34052 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 5066, 0, 8192, 0, 0, 0, 0, 0, 7261, 0, 0, 0, 0, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8118, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3955, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 6270, 0, 8192, 0, 0, 0, 0, 0, 5144, 0, 0, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4690, 0, 0, 0, 0, 82] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 5472, 0, 8192, 0, 0, 0, 0, 0, 7397, 0, 0, 1, 0, 143, 0, 0, 0, 0, 6, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 8066, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3468, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5126, 0, 8192, 0, 0, 0, 0, 0, 7025, 0, 0, 0, 0, 358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8122, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3874, 0, 0, 0, 0, 21] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 13, 0, 0, 0, 0, 0, 0, 4207, 0, 8192, 0, 0, 0, 0, 0, 5842, 0, 0, 10, 0, 605, 0, 0, 0, 0, 37, 0, 0, 12, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 7836, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6002, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4631, 0, 8192, 0, 0, 0, 0, 0, 6361, 0, 0, 1, 0, 858, 0, 0, 0, 0, 15, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8130, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4442, 0, 0, 0, 0, 132] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8, 0, 0, 0, 0, 0, 0, 6191, 0, 8192, 0, 0, 0, 0, 0, 5544, 0, 0, 9, 0, 1760, 0, 0, 0, 0, 35, 0, 0, 10, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 8089, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2918, 0, 0, 0, 0, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 153, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7641, 0, 0, 0, 0, 0, 6, 1190, 7180, 5, 8176, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 6998, 0, 0, 0, 4, 0, 0, 6, 0, 0, 0, 0, 0, 4, 0, 0, 0, 6, 3, 6, 5, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8078, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 1271, 6, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 222] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 7214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5472, 0, 0, 0, 0, 0, 0, 0, 3351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 143, 0, 0, 0, 0, 6, 0, 0, 1, 0, 0, 0, 1, 5790, 978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 9, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 863, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3468, 0, 0, 0, 0, 0, 4045, 1413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7217, 8823, 6920, 8183, 8205, 9055, 8207, 8926] avg=8192.0000 max=9055 max_rank=5 min=6920 min_rank=2 imbalance_ratio=1.105347 max_load_reduction=56481 max_load_reduction_pct=0.861832 moved_to_echo=34393 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 7215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5066, 0, 0, 0, 0, 0, 0, 0, 3290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5827, 977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 9, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 869, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3955, 0, 0, 0, 0, 0, 3970, 1422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7222, 8356, 6958, 8183, 8206, 9061, 8203, 9347] avg=8192.0000 max=9347 max_rank=7 min=6958 min_rank=2 imbalance_ratio=1.140991 max_load_reduction=56189 max_load_reduction_pct=0.857376 moved_to_echo=34817 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8, 7214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6191, 0, 0, 0, 0, 0, 0, 0, 2512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 1760, 0, 0, 0, 0, 35, 0, 0, 10, 0, 0, 0, 8, 5806, 978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 9, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 866, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2918, 0, 0, 0, 0, 5, 3031, 1417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7224, 8703, 8606, 8193, 8199, 9058, 8177, 7376] avg=8192.0000 max=9058 max_rank=5 min=7224 min_rank=0 imbalance_ratio=1.105713 max_load_reduction=56478 max_load_reduction_pct=0.861786 moved_to_echo=32810 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5126, 0, 0, 0, 0, 0, 0, 0, 3183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5830, 978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 9, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 869, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3874, 0, 0, 0, 0, 21, 3841, 1423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7214, 8309, 7166, 8231, 8206, 9061, 8190, 9159] avg=8192.0000 max=9159 max_rank=7 min=7166 min_rank=2 imbalance_ratio=1.118042 max_load_reduction=56377 max_load_reduction_pct=0.860245 moved_to_echo=34616 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 13, 7214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4207, 0, 0, 0, 0, 0, 0, 0, 2647, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 605, 0, 0, 0, 0, 37, 0, 0, 12, 0, 0, 0, 6, 5625, 978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 9, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 838, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6002, 0, 0, 0, 0, 0, 3194, 1373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7230, 6854, 7273, 8175, 8206, 9030, 8198, 10570] avg=8192.0000 max=10570 max_rank=7 min=6854 min_rank=1 imbalance_ratio=1.290283 max_load_reduction=54966 max_load_reduction_pct=0.838715 moved_to_echo=36004 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4631, 0, 0, 0, 0, 0, 0, 0, 2872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 858, 0, 0, 0, 0, 15, 0, 0, 1, 0, 0, 0, 0, 5841, 977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 12, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 861, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4442, 0, 0, 0, 0, 132, 3483, 1428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7216, 7503, 7693, 8188, 8216, 9053, 8182, 9485] avg=8192.0000 max=9485 max_rank=7 min=7216 min_rank=0 imbalance_ratio=1.157837 max_load_reduction=56051 max_load_reduction_pct=0.855270 moved_to_echo=34936 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 153, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7641, 0, 7201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1190, 7180, 5, 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6998, 0, 0, 0, 4, 0, 0, 6, 0, 0, 0, 0, 0, 5876, 975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 6, 3, 6, 5, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8078, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 876, 0, 0, 1271, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 222, 0, 1434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14996, 8388, 13859, 8200, 8092, 2153, 8188, 1660] avg=8192.0000 max=14996 max_rank=0 min=1660 min_rank=7 imbalance_ratio=1.830566 max_load_reduction=50540 max_load_reduction_pct=0.771179 moved_to_echo=20093 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 7215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6270, 0, 0, 0, 0, 0, 0, 0, 2330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5853, 977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 9, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 873, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4690, 0, 0, 0, 0, 82, 2813, 1428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7217, 8600, 7061, 8183, 8206, 9065, 8191, 9013] avg=8192.0000 max=9065 max_rank=5 min=7061 min_rank=2 imbalance_ratio=1.106567 max_load_reduction=56471 max_load_reduction_pct=0.861679 moved_to_echo=34475 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 6144, 0, 0, 0, 128, 128, 0, 128, 128, 0, 0, 0, 128, 55808, 3584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 384, 0, 0, 0, 0, 0, 0, 128, 0, 0, 58112, 128, 4352, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 6144, 0, 0, 59520, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 0, 128, 1408, 48512, 128, 0, 0, 128, 0, 128, 0, 15872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 3584, 128, 0, 0, 0, 128, 62080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 0, 128, 128, 128, 128, 0, 65536, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 29440, 128, 0, 0, 0, 512, 24448, 11392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 128, 1280, 44160, 128, 0, 0, 128, 0, 128, 0, 20224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 11136, 0, 0, 0, 128, 128, 0, 128, 128, 0, 0, 0, 128, 46464, 7936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 128, 0, 0, 7680, 128, 57728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 7040, 0, 0, 58624, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 0, 128, 128, 128, 128, 0, 65536, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8192, 0, 0, 8192, 8018, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8190, 0, 0, 8192, 7633, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 8191, 0, 0, 8192, 8145, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8187, 8191, 0, 0, 8191, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 8172, 8177, 0, 0, 8185, 8052, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 8177, 0, 0, 8182, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8181, 8191, 0, 0, 8190, 8063, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 8185, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1429, 8185, 0, 0, 0, 136, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7689, 0, 0, 0, 0, 0, 6, 6, 0, 0, 6, 2, 760, 0, 1320, 0, 0, 6281, 7, 0, 0, 0, 3, 0, 5, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 7831, 5681, 0, 0, 8184, 169, 0, 0, 5, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 5, 0, 0, 1329, 0, 8181, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 6991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 6962, 706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1172, 0, 0, 0, 7633, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6991, 10209, 8252, 6821, 8244, 8809, 8194, 8016] avg=8192.0000 max=10209 max_rank=1 min=6821 min_rank=3 imbalance_ratio=1.246216 max_load_reduction=55327 max_load_reduction_pct=0.844223 moved_to_echo=33263 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 6990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 6962, 706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8184, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1173, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6990, 7899, 8251, 8588, 8241, 9356, 8194, 8017] avg=8192.0000 max=9356 max_rank=5 min=6990 min_rank=0 imbalance_ratio=1.142090 max_load_reduction=56180 max_load_reduction_pct=0.857239 moved_to_echo=33808 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 6991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 6963, 706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1173, 0, 0, 0, 8018, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6991, 8533, 8253, 8113, 8243, 9192, 8195, 8016] avg=8192.0000 max=9192 max_rank=5 min=6991 min_rank=0 imbalance_ratio=1.122070 max_load_reduction=56344 max_load_reduction_pct=0.859741 moved_to_echo=33646 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 6981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 6951, 705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1170, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6981, 7865, 8254, 8626, 8215, 9384, 8185, 8026] avg=8192.0000 max=9384 max_rank=5 min=6981 min_rank=0 imbalance_ratio=1.145508 max_load_reduction=56152 max_load_reduction_pct=0.856812 moved_to_echo=33810 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 6991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6962, 706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1173, 0, 0, 0, 8145, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6991, 8370, 8255, 8141, 8241, 9326, 8195, 8017] avg=8192.0000 max=9326 max_rank=5 min=6991 min_rank=0 imbalance_ratio=1.138428 max_load_reduction=56210 max_load_reduction_pct=0.857697 moved_to_echo=33779 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 6983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 6951, 706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 8169, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1170, 0, 0, 0, 8052, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6983, 8150, 8243, 8477, 8236, 9236, 8188, 8023] avg=8192.0000 max=9236 max_rank=5 min=6983 min_rank=0 imbalance_ratio=1.127441 max_load_reduction=56300 max_load_reduction_pct=0.859070 moved_to_echo=33683 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 6984, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1429, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4829, 706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7689, 0, 0, 0, 0, 0, 6, 6, 0, 0, 6, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 760, 0, 1320, 0, 0, 6281, 7, 0, 0, 0, 3, 0, 5, 0, 0, 495, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 7829, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 814, 0, 0, 0, 169, 0, 0, 5, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 5, 0, 0, 1329, 0, 8005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16631, 5672, 7771, 8986, 7877, 993, 8193, 9413] avg=8192.0000 max=16631 max_rank=0 min=993 min_rank=5 imbalance_ratio=2.030151 max_load_reduction=48905 max_load_reduction_pct=0.746231 moved_to_echo=26476 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 6978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 6966, 711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8179, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1167, 0, 0, 0, 8063, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6978, 8838, 8257, 7784, 8239, 9240, 8192, 8008] avg=8192.0000 max=9240 max_rank=5 min=6978 min_rank=0 imbalance_ratio=1.127930 max_load_reduction=56296 max_load_reduction_pct=0.859009 moved_to_echo=33679 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 732, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8189, 8191, 0, 0, 8191, 7464, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8191, 0, 0, 8192, 1561, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 8185, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 8185, 0, 0, 0, 40, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7644, 0, 0, 0, 0, 0, 6, 6, 0, 0, 6, 2, 672, 0, 1198, 0, 0, 7198, 7, 0, 0, 0, 3, 0, 5, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 7921, 5700, 0, 0, 8184, 0, 0, 0, 5, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 5, 0, 0, 1206, 0, 8181, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8182, 8192, 0, 0, 8190, 4569, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 2118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 8176, 8178, 0, 0, 8185, 6075, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 8191, 0, 0, 8192, 6938, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8192, 0, 0, 8192, 6604, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 732, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 8178, 0, 0, 8182, 7461, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 7036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 732, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4925, 1155, 160, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8186, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3223, 0, 0, 0, 7464, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7036, 7015, 8254, 8081, 8230, 10693, 8194, 8033] avg=8192.0000 max=10693 max_rank=5 min=7015 min_rank=1 imbalance_ratio=1.305298 max_load_reduction=54843 max_load_reduction_pct=0.836838 moved_to_echo=35150 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 7030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 2118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 4917, 1154, 160, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 8173, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3218, 0, 0, 0, 6075, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7030, 8414, 8250, 8080, 8228, 9307, 8188, 8039] avg=8192.0000 max=9307 max_rank=5 min=7030 min_rank=0 imbalance_ratio=1.136108 max_load_reduction=56229 max_load_reduction_pct=0.857986 moved_to_echo=33762 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 7032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4927, 1158, 165, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8180, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3220, 0, 0, 0, 4569, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7032, 9924, 8261, 8075, 8226, 7798, 8192, 8028] avg=8192.0000 max=9924 max_rank=1 min=7032 min_rank=0 imbalance_ratio=1.211426 max_load_reduction=55612 max_load_reduction_pct=0.848572 moved_to_echo=32244 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 7036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4925, 1156, 160, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3223, 0, 0, 0, 6938, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7036, 7536, 8263, 8080, 8229, 10165, 8195, 8032] avg=8192.0000 max=10165 max_rank=5 min=7036 min_rank=0 imbalance_ratio=1.240845 max_load_reduction=55371 max_load_reduction_pct=0.844894 moved_to_echo=34621 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 7030, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3427, 1155, 160, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7644, 0, 0, 0, 0, 0, 6, 6, 0, 0, 6, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 672, 0, 1198, 0, 0, 7101, 7, 0, 0, 0, 3, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 7918, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2243, 0, 0, 0, 0, 0, 0, 5, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 5, 0, 0, 1206, 0, 8021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16303, 4818, 7725, 8988, 7957, 2254, 8193, 9298] avg=8192.0000 max=16303 max_rank=0 min=2254 min_rank=5 imbalance_ratio=1.990112 max_load_reduction=49233 max_load_reduction_pct=0.751236 moved_to_echo=27702 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 7036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4925, 1156, 160, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3224, 0, 0, 0, 6604, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7036, 7873, 8261, 8080, 8230, 9829, 8195, 8032] avg=8192.0000 max=9829 max_rank=5 min=7036 min_rank=0 imbalance_ratio=1.199829 max_load_reduction=55707 max_load_reduction_pct=0.850021 moved_to_echo=34286 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 7027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 732, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 4917, 1154, 160, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3218, 0, 0, 0, 7461, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7027, 7031, 8262, 8081, 8205, 10703, 8185, 8042] avg=8192.0000 max=10703 max_rank=5 min=7027 min_rank=0 imbalance_ratio=1.306519 max_load_reduction=54833 max_load_reduction_pct=0.836685 moved_to_echo=35135 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 7036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4925, 1156, 160, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3223, 0, 0, 0, 1561, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7036, 12925, 8260, 8071, 8231, 4787, 8194, 8032] avg=8192.0000 max=12925 max_rank=1 min=4787 min_rank=5 imbalance_ratio=1.577759 max_load_reduction=52611 max_load_reduction_pct=0.802780 moved_to_echo=29244 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 6272, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 53632, 5760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 9088, 0, 0, 0, 56448, 0, 0, 128, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 55936, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1536, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 768, 0, 1408, 0, 0, 58624, 128, 0, 0, 0, 128, 0, 128, 0, 0, 3968, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 0, 128, 0, 0, 0, 128, 0, 0, 1408, 0, 64128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 0, 65024, 0, 0, 0, 0, 0, 128, 128, 0, 0, 128, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 65152, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 16768, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 37888, 9344, 1408, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 24832, 0, 0, 0, 40704, 0, 0, 128, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 56320, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1152, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 0, 0, 65024, 0, 0, 0, 0, 0, 128, 128, 0, 0, 128, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 768, 0, 1280, 0, 0, 63744, 128, 0, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 0, 128, 0, 0, 0, 128, 0, 0, 1280, 0, 64256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 65280, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 483, 0, 0, 0, 0, 0, 0, 0, 54, 7819, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 433, 0, 0, 0, 0, 0, 0, 0, 4, 7852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8093, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1377, 0, 0, 0, 0, 0, 0, 0, 47, 7273, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 341, 0, 0, 0, 0, 0, 0, 0, 28, 7906, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8129, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 2364, 0, 0, 0, 0, 0, 0, 0, 12, 7852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 139, 0, 0, 0, 0, 0, 0, 0, 18, 8143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8085, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 6542, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 66, 0, 0, 0, 0, 0, 2, 0, 0, 83, 0, 0, 179, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 2, 0, 0, 0, 0, 0, 616, 0, 0, 0, 0, 8097, 0, 18, 0, 0, 0, 0, 0, 0, 8121, 0, 0, 0, 0, 0, 0, 0, 6, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3110, 7, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 2, 2, 0, 4632, 0, 0, 0, 0, 0, 3, 0, 0, 0, 167, 0, 0, 0, 0, 0, 6, 0, 6, 7, 1, 0, 1297, 0, 8007, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 851, 0, 0, 0, 0, 0, 0, 0, 13, 8183, 0, 0, 0, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7324, 821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 948, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 0, 0, 0, 0, 0, 0, 0, 54, 6343, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 7975, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 0, 0, 0, 0, 0, 0, 650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6423, 1158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8390, 8153, 9140, 6901, 8187, 8786, 7581, 8398] avg=8192.0000 max=9140 max_rank=2 min=6901 min_rank=3 imbalance_ratio=1.115723 max_load_reduction=56396 max_load_reduction_pct=0.860535 moved_to_echo=32952 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7337, 821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 948, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 341, 0, 0, 0, 0, 0, 0, 0, 28, 6413, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 7991, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 652, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6423, 1171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8129, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8391, 8158, 9140, 6784, 8196, 8820, 7594, 8453] avg=8192.0000 max=9140 max_rank=2 min=6784 min_rank=3 imbalance_ratio=1.115723 max_load_reduction=56396 max_load_reduction_pct=0.860535 moved_to_echo=33063 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7338, 821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 948, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2364, 0, 0, 0, 0, 0, 0, 0, 12, 6369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 7992, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 652, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6423, 1163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8391, 8159, 9140, 8745, 8190, 8838, 7587, 6486] avg=8192.0000 max=9140 max_rank=2 min=6486 min_rank=7 imbalance_ratio=1.115723 max_load_reduction=56396 max_load_reduction_pct=0.860535 moved_to_echo=31101 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7345, 821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 948, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 433, 0, 0, 0, 0, 0, 0, 0, 4, 6369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 7992, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6423, 1163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8093, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8391, 8166, 9140, 6806, 8189, 8842, 7587, 8415] avg=8192.0000 max=9140 max_rank=2 min=6806 min_rank=3 imbalance_ratio=1.115723 max_load_reduction=56396 max_load_reduction_pct=0.860535 moved_to_echo=33033 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7339, 821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 948, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 0, 0, 0, 0, 0, 0, 0, 18, 6605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7993, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 652, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6423, 1206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8085, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8391, 8160, 9140, 6762, 8187, 8844, 7629, 8423] avg=8192.0000 max=9140 max_rank=2 min=6762 min_rank=3 imbalance_ratio=1.115723 max_load_reduction=56396 max_load_reduction_pct=0.860535 moved_to_echo=33083 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7324, 821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 948, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1377, 0, 0, 0, 0, 0, 0, 0, 47, 5900, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 7981, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 650, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6423, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8390, 8145, 9140, 7336, 8194, 8811, 7500, 8020] avg=8192.0000 max=9140 max_rank=2 min=7336 min_rank=3 imbalance_ratio=1.115723 max_load_reduction=56396 max_load_reduction_pct=0.860535 moved_to_echo=32525 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 6542, 0, 0, 0, 0, 0, 0, 198, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 83, 0, 0, 179, 6, 0, 0, 0, 0, 0, 0, 0, 7337, 819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 947, 0, 2, 0, 0, 0, 0, 0, 616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8097, 0, 18, 0, 0, 0, 0, 0, 0, 6587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7986, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3110, 7, 0, 0, 0, 0, 0, 652, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4632, 0, 0, 0, 0, 0, 3, 0, 0, 0, 167, 0, 0, 0, 0, 0, 6411, 1203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 6, 7, 1, 0, 1297, 0, 8007, 0, 0, 0, 0, 0, 0, 0, 331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6806, 8426, 1565, 14702, 8193, 3773, 12416, 9655] avg=8192.0000 max=14702 max_rank=3 min=1565 min_rank=2 imbalance_ratio=1.794678 max_load_reduction=50834 max_load_reduction_pct=0.775665 moved_to_echo=34037 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7344, 825, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 939, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 851, 0, 0, 0, 0, 0, 0, 0, 13, 6632, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 7994, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6428, 1214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8386, 8169, 9131, 7500, 8200, 8822, 7642, 7686] avg=8192.0000 max=9131 max_rank=2 min=7500 min_rank=3 imbalance_ratio=1.114624 max_load_reduction=56405 max_load_reduction_pct=0.860672 moved_to_echo=32350 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 2710, 0, 0, 0, 0, 6525, 0, 0, 0, 0, 0, 0, 0, 66, 8123, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 7132, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 3728, 0, 0, 0, 0, 6337, 0, 0, 0, 0, 0, 0, 0, 97, 8020, 0, 0, 0, 0, 22, 1, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 6376, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 3591, 0, 0, 0, 0, 5173, 0, 0, 0, 0, 0, 0, 0, 60, 7829, 0, 0, 0, 0, 13, 1, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 7938, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 2791, 0, 0, 0, 0, 5892, 0, 0, 0, 0, 0, 0, 0, 11, 7990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 7868, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 1, 0, 17, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 1749, 0, 0, 0, 0, 6478, 0, 0, 0, 0, 0, 0, 0, 42, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8097, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 6256, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 69, 0, 0, 0, 0, 0, 2, 0, 0, 85, 0, 0, 181, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8099, 0, 20, 0, 0, 0, 0, 0, 0, 8120, 0, 0, 0, 0, 0, 0, 0, 5, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1653, 7, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 3, 2, 0, 5984, 0, 0, 0, 0, 0, 3, 0, 0, 0, 686, 0, 0, 0, 0, 0, 6, 0, 6, 7, 555, 0, 1216, 0, 8011, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 325, 0, 0, 0, 0, 7698, 0, 0, 0, 0, 0, 0, 0, 17, 8189, 0, 0, 0, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 8146, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 200, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 611, 0, 0, 0, 0, 7690, 0, 0, 0, 0, 0, 0, 0, 19, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8097, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6682, 1486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6256, 0, 0, 0, 0, 0, 0, 0, 2710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6525, 0, 0, 0, 0, 0, 0, 0, 66, 1419, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 7956, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7132, 0, 0, 0, 0, 0, 0, 1278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5218, 1936, 266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7926, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8426, 8168, 8966, 8012, 8197, 8410, 7420, 7937] avg=8192.0000 max=8966 max_rank=2 min=7420 min_rank=6 imbalance_ratio=1.094482 max_load_reduction=56570 max_load_reduction_pct=0.863190 moved_to_echo=31964 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6665, 1433, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6256, 0, 0, 0, 0, 0, 0, 0, 3591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5173, 0, 0, 0, 0, 0, 0, 0, 60, 1367, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 7945, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7938, 0, 0, 0, 0, 0, 0, 1275, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5029, 1936, 265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7908, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8426, 8098, 9847, 6613, 8194, 9219, 7230, 7909] avg=8192.0000 max=9847 max_rank=2 min=6613 min_rank=3 imbalance_ratio=1.202026 max_load_reduction=55689 max_load_reduction_pct=0.849747 moved_to_echo=32552 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6685, 1462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6256, 0, 0, 0, 0, 0, 0, 0, 2791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5892, 0, 0, 0, 0, 0, 0, 0, 11, 1396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 7957, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7868, 0, 0, 0, 0, 0, 0, 1279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5132, 1936, 266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7923, 0, 1, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8426, 8147, 9047, 7299, 8194, 9147, 7335, 7941] avg=8192.0000 max=9147 max_rank=5 min=7299 min_rank=3 imbalance_ratio=1.116577 max_load_reduction=56389 max_load_reduction_pct=0.860428 moved_to_echo=32617 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6671, 1468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6256, 0, 0, 0, 0, 0, 0, 0, 3728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6337, 0, 0, 0, 0, 0, 0, 0, 97, 1400, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 7939, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6376, 0, 0, 0, 0, 0, 0, 1276, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5152, 1936, 266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7924, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8426, 8148, 9984, 7856, 8187, 7652, 7354, 7929] avg=8192.0000 max=9984 max_rank=2 min=7354 min_rank=6 imbalance_ratio=1.218750 max_load_reduction=55552 max_load_reduction_pct=0.847656 moved_to_echo=31122 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6683, 1492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6256, 0, 0, 0, 0, 0, 0, 0, 1749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6478, 0, 0, 0, 0, 0, 0, 0, 42, 1424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7958, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8097, 0, 0, 0, 0, 0, 0, 1279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5238, 1936, 266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7924, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8426, 8175, 8005, 7944, 8186, 9376, 7440, 7984] avg=8192.0000 max=9376 max_rank=5 min=7440 min_rank=6 imbalance_ratio=1.144531 max_load_reduction=56160 max_load_reduction_pct=0.856934 moved_to_echo=32986 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 6256, 0, 0, 0, 0, 0, 0, 234, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 85, 0, 0, 181, 6, 0, 0, 0, 0, 0, 0, 0, 6678, 1486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6244, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8099, 0, 20, 0, 0, 0, 0, 0, 0, 1418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7950, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1653, 7, 0, 0, 0, 0, 0, 1278, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5984, 0, 0, 0, 0, 0, 3, 0, 0, 0, 686, 0, 0, 0, 0, 0, 5216, 1933, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 6, 7, 537, 0, 1216, 0, 8011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6559, 8438, 6246, 9537, 8190, 2943, 13840, 9783] avg=8192.0000 max=13840 max_rank=6 min=2943 min_rank=5 imbalance_ratio=1.689453 max_load_reduction=51696 max_load_reduction_pct=0.788818 moved_to_echo=34756 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6685, 1502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6249, 0, 0, 0, 0, 0, 0, 0, 325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7698, 0, 0, 0, 0, 0, 0, 0, 17, 1423, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 7959, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8146, 0, 0, 0, 0, 0, 0, 1268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5264, 1943, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7924, 0, 0, 0, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8421, 8187, 6574, 9142, 8199, 9414, 7475, 8124] avg=8192.0000 max=9414 max_rank=5 min=6574 min_rank=2 imbalance_ratio=1.149170 max_load_reduction=56122 max_load_reduction_pct=0.856354 moved_to_echo=33212 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8192, 0, 0, 0, 0, 0, 0, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6682, 1493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6256, 0, 0, 0, 0, 0, 0, 0, 611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7690, 0, 0, 0, 0, 0, 0, 0, 19, 1424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 7957, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8097, 0, 0, 0, 0, 0, 0, 1278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5239, 1936, 266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7926, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8426, 8175, 6867, 9133, 8189, 9375, 7442, 7929] avg=8192.0000 max=9375 max_rank=5 min=6867 min_rank=2 imbalance_ratio=1.144409 max_load_reduction=56161 max_load_reduction_pct=0.856949 moved_to_echo=32935 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 14208, 0, 128, 0, 0, 0, 0, 0, 256, 51328, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 128, 0, 128, 256, 128, 0, 0, 0, 0, 0, 0, 0, 58752, 6656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[4736, 0, 0, 0, 0, 0, 128, 0, 0, 0, 256, 0, 0, 0, 0, 0, 51456, 9472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 7680, 0, 128, 0, 0, 0, 0, 0, 57984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 60416, 128, 0, 0, 0, 0, 0, 5248, 0, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 53632, 0, 1408, 0, 8064, 0, 0, 0, 0, 0, 0, 0, 2688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 64000, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 64000, 0, 0, 0, 0, 0, 0, 1664, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 128, 0, 128, 256, 128, 0, 0, 0, 0, 0, 0, 0, 53504, 11904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[6016, 0, 0, 0, 0, 0, 128, 0, 0, 0, 768, 0, 0, 0, 0, 0, 41600, 15616, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 54016, 0, 128, 0, 0, 0, 0, 0, 384, 11392, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 50048, 0, 128, 0, 0, 0, 0, 0, 15616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 55424, 128, 0, 0, 0, 0, 0, 10240, 0, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 63616, 0, 0, 0, 0, 0, 0, 1920, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 56064, 0, 1280, 0, 8320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 63744, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8190, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 1, 0, 0, 341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1693, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 4438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 8122, 0, 0, 0, 4174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 302, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 4729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2164, 0, 0, 0, 0, 0, 0, 8119, 0, 0, 0, 4353, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8171, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8188, 0, 0, 0, 0, 637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 4562, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 3935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 8152, 0, 0, 0, 3820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8004, 8170, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8043, 45, 5, 0, 8184, 0, 285, 8, 6, 1594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 6, 0, 2, 0, 20, 2498, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 2978, 0, 0, 1, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 313, 0, 0, 0, 0, 0, 0, 7120, 0, 0, 7, 2624, 2, 0, 0, 8, 0, 1, 0, 82, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 7, 6, 0, 0, 0, 0, 7303, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7467, 0, 1, 0, 0, 341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 616, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1439, 0, 0, 0, 0, 0, 0, 5139, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1693, 0, 0, 0, 0, 0, 0, 7631, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2946, 0, 0, 0, 4438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8217, 7809, 8191, 8901, 7112, 9327, 7678, 8301] avg=8192.0000 max=9327 max_rank=5 min=7112 min_rank=4 imbalance_ratio=1.138550 max_load_reduction=56209 max_load_reduction_pct=0.857681 moved_to_echo=32418 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7467, 0, 0, 0, 0, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 616, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3211, 0, 0, 0, 0, 0, 0, 5104, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 7631, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2926, 0, 0, 0, 4174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 7974, 8194, 8900, 8849, 7646, 7453, 8301] avg=8192.0000 max=8900 max_rank=3 min=7453 min_rank=6 imbalance_ratio=1.086426 max_load_reduction=56636 max_load_reduction_pct=0.864197 moved_to_echo=32249 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7467, 0, 0, 0, 0, 354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 616, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1193, 0, 0, 0, 0, 0, 0, 5102, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2164, 0, 0, 0, 0, 0, 0, 7631, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2925, 0, 0, 0, 4353, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8218, 7821, 8202, 8900, 6829, 9798, 7467, 8301] avg=8192.0000 max=9798 max_rank=5 min=6829 min_rank=4 imbalance_ratio=1.196045 max_load_reduction=55738 max_load_reduction_pct=0.850494 moved_to_echo=32395 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7467, 0, 0, 0, 0, 345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 616, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2666, 0, 0, 0, 0, 0, 0, 5132, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 302, 0, 0, 0, 0, 0, 0, 7631, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2941, 0, 0, 0, 4729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 7812, 8192, 8900, 8332, 7935, 7845, 8301] avg=8192.0000 max=8900 max_rank=3 min=7812 min_rank=1 imbalance_ratio=1.086426 max_load_reduction=56636 max_load_reduction_pct=0.864197 moved_to_echo=32413 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7463, 0, 0, 0, 0, 637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 616, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2909, 0, 0, 0, 0, 0, 0, 5127, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 7631, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2939, 0, 0, 0, 4562, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8100, 8199, 8901, 8570, 7651, 7616, 8301] avg=8192.0000 max=8901 max_rank=3 min=7616 min_rank=6 imbalance_ratio=1.086548 max_load_reduction=56635 max_load_reduction_pct=0.864182 moved_to_echo=32138 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7467, 0, 0, 0, 0, 800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 616, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3264, 0, 0, 0, 0, 0, 0, 5139, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 0, 0, 0, 0, 7631, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2945, 0, 0, 0, 3935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 8267, 8190, 8901, 8937, 7712, 7009, 8301] avg=8192.0000 max=8937 max_rank=4 min=7009 min_rank=6 imbalance_ratio=1.090942 max_load_reduction=56599 max_load_reduction_pct=0.863632 moved_to_echo=31959 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7457, 0, 0, 0, 0, 894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 622, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3251, 0, 0, 0, 0, 0, 0, 5128, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 7635, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2926, 0, 0, 0, 3820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8351, 8192, 8912, 8915, 7654, 6994, 8305] avg=8192.0000 max=8915 max_rank=4 min=6994 min_rank=6 imbalance_ratio=1.088257 max_load_reduction=56621 max_load_reduction_pct=0.863968 moved_to_echo=31868 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8004, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 5, 0, 7459, 0, 285, 8, 6, 1594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 6, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 20, 2498, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 616, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 2978, 0, 0, 1, 0, 0, 0, 4474, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 313, 0, 0, 0, 0, 0, 0, 7492, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2565, 0, 0, 7, 2624, 2, 0, 0, 8, 0, 1, 0, 82, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 6, 0, 0, 0, 0, 7303, 0, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8033, 9402, 8176, 3221, 7992, 7813, 13474, 7425] avg=8192.0000 max=13474 max_rank=6 min=3221 min_rank=3 imbalance_ratio=1.644775 max_load_reduction=52062 max_load_reduction_pct=0.794403 moved_to_echo=36704 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 1, 0, 0, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3757, 0, 0, 0, 0, 0, 0, 7874, 0, 0, 0, 3534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 0, 0, 0, 0, 0, 0, 7152, 0, 0, 0, 3907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 2, 0, 0, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 0, 0, 0, 0, 0, 0, 7808, 0, 0, 0, 4505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8172, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8188, 0, 1, 0, 0, 653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 175, 0, 0, 0, 0, 0, 0, 7823, 0, 0, 0, 2236, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8029, 8169, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8044, 45, 5, 0, 8184, 0, 1175, 8, 6, 1598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 6, 0, 2, 0, 22, 2920, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 3324, 0, 0, 2, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1049, 0, 0, 0, 0, 0, 0, 3939, 0, 0, 7, 3057, 2, 0, 0, 8, 0, 1, 0, 79, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 7, 6, 0, 0, 0, 0, 7629, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8190, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4663, 0, 0, 0, 0, 0, 0, 7533, 0, 0, 0, 3294, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 7504, 0, 0, 0, 3185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 799, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 5764, 0, 0, 0, 3211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7317, 0, 1, 0, 0, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 0, 0, 0, 0, 0, 0, 3780, 1313, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3757, 0, 0, 0, 0, 0, 0, 6855, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 3534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8215, 7450, 8192, 8847, 6300, 10614, 7658, 8260] avg=8192.0000 max=10614 max_rank=5 min=6300 min_rank=4 imbalance_ratio=1.295654 max_load_reduction=54922 max_load_reduction_pct=0.838043 moved_to_echo=32832 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7313, 0, 1, 0, 0, 653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5442, 0, 0, 0, 0, 0, 0, 3755, 1313, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 175, 0, 0, 0, 0, 0, 0, 6855, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 2236, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 7967, 8199, 8847, 10663, 7032, 6372, 8260] avg=8192.0000 max=10663 max_rank=4 min=6372 min_rank=6 imbalance_ratio=1.301636 max_load_reduction=54873 max_load_reduction_pct=0.837296 moved_to_echo=32327 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7317, 0, 2, 0, 0, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2930, 0, 0, 0, 0, 0, 0, 3748, 1313, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 0, 0, 0, 0, 0, 0, 6855, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 4505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 7799, 8191, 8846, 8143, 7374, 8707, 8260] avg=8192.0000 max=8846 max_rank=3 min=7374 min_rank=5 imbalance_ratio=1.079834 max_load_reduction=56690 max_load_reduction_pct=0.865021 moved_to_echo=32484 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7317, 0, 0, 0, 0, 813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3913, 0, 0, 0, 0, 0, 0, 3433, 1313, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 0, 0, 0, 0, 0, 0, 6855, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3719, 0, 0, 0, 3907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8130, 8191, 8847, 8811, 7394, 7687, 8260] avg=8192.0000 max=8847 max_rank=3 min=7394 min_rank=5 imbalance_ratio=1.079956 max_load_reduction=56689 max_load_reduction_pct=0.865005 moved_to_echo=32152 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7317, 0, 0, 0, 0, 391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 0, 0, 0, 0, 0, 0, 3616, 1313, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4663, 0, 0, 0, 0, 0, 0, 6855, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3917, 0, 0, 0, 3294, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8214, 7708, 8202, 8847, 5549, 11520, 7236, 8260] avg=8192.0000 max=11520 max_rank=5 min=5549 min_rank=4 imbalance_ratio=1.406250 max_load_reduction=54016 max_load_reduction_pct=0.824219 moved_to_echo=32565 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7303, 0, 0, 0, 0, 917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4388, 0, 0, 0, 0, 0, 0, 3605, 1317, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 6859, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3899, 0, 0, 0, 3185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8220, 8193, 8852, 9465, 6915, 7417, 8266] avg=8192.0000 max=9465 max_rank=4 min=6915 min_rank=5 imbalance_ratio=1.155396 max_load_reduction=56071 max_load_reduction_pct=0.855576 moved_to_echo=32063 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8029, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 5, 0, 7309, 0, 1175, 8, 6, 1598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 6, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 22, 2920, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 3324, 0, 0, 2, 0, 0, 0, 1890, 1289, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1049, 0, 0, 0, 0, 0, 0, 6731, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2049, 0, 0, 7, 3057, 2, 0, 0, 8, 0, 1, 0, 79, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 6, 0, 0, 0, 0, 7629, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8055, 10146, 8176, 3603, 6672, 7786, 13388, 7710] avg=8192.0000 max=13388 max_rank=6 min=3603 min_rank=3 imbalance_ratio=1.634277 max_load_reduction=52148 max_load_reduction_pct=0.795715 moved_to_echo=35556 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7317, 0, 0, 0, 0, 799, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5701, 0, 0, 0, 0, 0, 0, 2767, 1313, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 6855, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2997, 0, 0, 0, 3211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8116, 8192, 8847, 9933, 6901, 7071, 8260] avg=8192.0000 max=9933 max_rank=4 min=6901 min_rank=5 imbalance_ratio=1.212524 max_load_reduction=55603 max_load_reduction_pct=0.848434 moved_to_echo=32165 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[23168, 0, 0, 128, 32640, 128, 0, 0, 128, 0, 128, 0, 128, 128, 0, 9728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 20992, 0, 0, 128, 0, 0, 0, 40448, 4352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 59904, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 4992, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 128, 0, 0, 0, 27264, 0, 0, 128, 0, 0, 0, 26624, 10496, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[28928, 0, 0, 128, 27008, 128, 0, 0, 128, 0, 128, 0, 128, 128, 0, 9728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 0, 10880, 0, 0, 0, 0, 0, 0, 54784, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 59776, 0, 384, 128, 128, 5504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 0, 0, 4608, 0, 0, 0, 0, 0, 0, 60928, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 65408, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 64768, 0, 0, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 60288, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 5248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 58624, 0, 1280, 128, 128, 5888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 128, 0, 0, 128, 0, 65536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 65024, 0, 0, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 65408, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8188, 8, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 4, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 6, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 10, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 1, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 8180, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 28, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 16, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 10, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 1, 0, 0, 5261, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 6833, 5, 2, 0, 6883, 0, 1, 0, 0, 2641, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 7, 3, 0, 8105, 6, 0, 0, 0, 0, 0, 0, 0, 6, 8163, 0, 1579, 8183, 1, 0, 6, 3, 0, 0, 0, 0, 131, 0, 0, 0, 0, 1230, 0, 0, 6, 0, 0, 0, 0, 6, 0, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 29, 0, 8181, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8192, 77, 0, 0, 0, 0, 1, 0, 8192, 0, 0, 8192, 0, 0, 0, 1, 0, 0, 0, 0, 8115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8188, 8, 0, 0, 0, 0, 0, 0, 193, 163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 8, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8185, 8553, 8020, 8205, 7996, 8189, 8195] avg=8192.0000 max=8553 max_rank=2 min=7996 min_rank=5 imbalance_ratio=1.044067 max_load_reduction=56983 max_load_reduction_pct=0.869492 moved_to_echo=32585 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 6, 0, 0, 0, 0, 0, 0, 193, 163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 8, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8184, 8551, 8022, 8204, 7998, 8189, 8194] avg=8192.0000 max=8551 max_rank=2 min=7998 min_rank=5 imbalance_ratio=1.043823 max_load_reduction=56985 max_load_reduction_pct=0.869522 moved_to_echo=32585 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 8172, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 28, 0, 0, 0, 0, 0, 0, 192, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 8, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8209, 8575, 7991, 8203, 7982, 8188, 8195] avg=8192.0000 max=8575 max_rank=2 min=7982 min_rank=5 imbalance_ratio=1.046753 max_load_reduction=56961 max_load_reduction_pct=0.869156 moved_to_echo=32568 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 10, 0, 0, 0, 0, 0, 0, 193, 163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 8, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8196, 8558, 8019, 8188, 7998, 8189, 8194] avg=8192.0000 max=8558 max_rank=2 min=7998 min_rank=5 imbalance_ratio=1.044678 max_load_reduction=56978 max_load_reduction_pct=0.869415 moved_to_echo=32569 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 29, 0, 8173, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8192, 77, 0, 0, 0, 0, 1, 0, 192, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 8, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8210, 8626, 7949, 8187, 7988, 8188, 8195] avg=8192.0000 max=8626 max_rank=2 min=7949 min_rank=3 imbalance_ratio=1.052979 max_load_reduction=56910 max_load_reduction_pct=0.868378 moved_to_echo=32558 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8170, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 1, 0, 0, 5261, 0, 0, 0, 0, 0, 0, 0, 192, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 0, 6883, 0, 1, 0, 0, 2587, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3, 0, 8105, 6, 0, 0, 0, 0, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 1579, 7991, 1, 0, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 0, 0, 0, 0, 1230, 0, 0, 6, 0, 0, 0, 0, 6, 0, 7, 6830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5, 0, 0, 0, 8161, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8184, 5557, 9510, 8131, 9586, 8210, 8177] avg=8192.0000 max=9586 max_rank=5 min=5557 min_rank=2 imbalance_ratio=1.170166 max_load_reduction=55950 max_load_reduction_pct=0.853729 moved_to_echo=34104 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 16, 0, 0, 0, 0, 0, 0, 196, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 12, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8183, 8569, 8002, 8212, 7990, 8193, 8193] avg=8192.0000 max=8569 max_rank=2 min=7990 min_rank=5 imbalance_ratio=1.046021 max_load_reduction=56967 max_load_reduction_pct=0.869247 moved_to_echo=32588 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 193, 163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8185, 8547, 8023, 8206, 7998, 8190, 8193] avg=8192.0000 max=8547 max_rank=2 min=7998 min_rank=5 imbalance_ratio=1.043335 max_load_reduction=56989 max_load_reduction_pct=0.869583 moved_to_echo=32587 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8187, 15, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 5, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 31, 0, 8181, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8192, 423, 0, 0, 0, 0, 1, 0, 8192, 0, 0, 8192, 0, 0, 0, 111, 0, 0, 0, 0, 7659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 294, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 91, 0, 0, 0, 0, 7817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 640, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 116, 0, 0, 0, 0, 7436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 8181, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8192, 220, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 619, 0, 0, 0, 0, 7348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 9, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 1, 0, 0, 5279, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 6863, 5, 2, 0, 7108, 0, 1, 0, 0, 2140, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 7, 4, 0, 8105, 6, 0, 0, 0, 0, 0, 0, 0, 6, 8163, 0, 1655, 8183, 1, 0, 6, 3, 0, 0, 0, 0, 318, 0, 0, 0, 0, 1198, 0, 0, 6, 0, 0, 0, 0, 6, 0, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 89, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 157, 0, 0, 0, 0, 7947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 374, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 712, 0, 0, 0, 0, 7106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8187, 15, 0, 0, 0, 0, 0, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 0, 0, 0, 5, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8066, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8186, 8304, 8304, 8204, 7988, 8162, 8194] avg=8192.0000 max=8304 max_rank=2 min=7988 min_rank=5 imbalance_ratio=1.013672 max_load_reduction=57232 max_load_reduction_pct=0.873291 moved_to_echo=32548 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 31, 0, 8173, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8192, 423, 0, 0, 0, 0, 1, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 0, 0, 0, 111, 0, 0, 0, 0, 7659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8066, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8211, 8720, 7892, 8185, 7978, 8162, 8194] avg=8192.0000 max=8720 max_rank=2 min=7892 min_rank=3 imbalance_ratio=1.064453 max_load_reduction=56816 max_load_reduction_pct=0.866943 moved_to_echo=32519 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 294, 0, 0, 0, 0, 0, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 0, 0, 0, 91, 0, 0, 0, 0, 7817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8066, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8184, 8582, 8030, 8200, 7990, 8162, 8194] avg=8192.0000 max=8582 max_rank=2 min=7990 min_rank=5 imbalance_ratio=1.047607 max_load_reduction=56954 max_load_reduction_pct=0.869049 moved_to_echo=32546 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 640, 0, 0, 0, 0, 0, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 0, 0, 0, 116, 0, 0, 0, 0, 7436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8066, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8195, 8933, 7674, 8194, 7990, 8162, 8194] avg=8192.0000 max=8933 max_rank=2 min=7674 min_rank=3 imbalance_ratio=1.090454 max_load_reduction=56603 max_load_reduction_pct=0.863693 moved_to_echo=32540 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 89, 0, 0, 0, 0, 0, 0, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 0, 157, 0, 0, 0, 0, 7947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 13, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8072, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8182, 8383, 8219, 8211, 7977, 8175, 8194] avg=8192.0000 max=8383 max_rank=2 min=7977 min_rank=5 imbalance_ratio=1.023315 max_load_reduction=57153 max_load_reduction_pct=0.872086 moved_to_echo=32557 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 8173, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8192, 220, 0, 0, 0, 0, 0, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 0, 0, 0, 619, 0, 0, 0, 0, 7348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8066, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8209, 8515, 8089, 8202, 7971, 8162, 8194] avg=8192.0000 max=8515 max_rank=2 min=7971 min_rank=5 imbalance_ratio=1.039429 max_load_reduction=57021 max_load_reduction_pct=0.870071 moved_to_echo=32529 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8167, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 1, 0, 0, 5279, 0, 0, 0, 0, 0, 0, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 5, 2, 0, 7108, 0, 1, 0, 0, 2140, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4, 0, 8105, 6, 0, 0, 0, 0, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 1655, 7981, 1, 0, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 318, 0, 0, 0, 0, 1198, 0, 0, 6, 0, 0, 0, 0, 6, 0, 7, 6758, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5, 0, 0, 0, 8162, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8183, 5434, 9388, 8135, 9652, 8389, 8178] avg=8192.0000 max=9652 max_rank=5 min=5434 min_rank=2 imbalance_ratio=1.178223 max_load_reduction=55884 max_load_reduction_pct=0.852722 moved_to_echo=34354 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 374, 0, 0, 0, 0, 0, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 0, 0, 0, 712, 0, 0, 0, 0, 7106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8066, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8186, 8665, 7940, 8205, 7990, 8162, 8194] avg=8192.0000 max=8665 max_rank=2 min=7940 min_rank=3 imbalance_ratio=1.057739 max_load_reduction=56871 max_load_reduction_pct=0.867783 moved_to_echo=32551 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 0, 128, 0, 128, 0, 0, 62720, 256, 0, 0, 0, 0, 128, 0, 1664, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 6912, 0, 128, 0, 0, 58624, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 128, 128, 128, 0, 65536, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 1024, 128, 128, 0, 8960, 0, 128, 0, 0, 55680, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 128, 0, 128, 0, 0, 62720, 2176, 0, 0, 0, 0, 128, 0, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 0, 0, 1664, 63872, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 0, 0, 0, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 0, 0, 1664, 64000, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 65408, 128, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[256, 0, 0, 0, 0, 1280, 0, 0, 128, 0, 0, 0, 0, 128, 0, 128, 64256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[384, 0, 0, 0, 0, 1280, 0, 0, 128, 0, 0, 0, 0, 128, 0, 128, 63232, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 128, 128, 128, 0, 65536, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 128, 128, 0, 65536, 128, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 65536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7721, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 0, 0, 8041, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 5, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8046, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 8110, 0, 0, 119, 0, 0, 7, 43, 8186, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 6, 4, 7, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7808, 0, 0, 0, 0, 0, 1, 0, 0, 391, 0, 0, 0, 0, 0, 0, 8180, 10, 0, 0, 0, 2, 0, 2, 6, 0, 6, 0, 0, 8180, 0, 6, 1, 0, 0, 0, 7937, 7, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 140, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 103, 0, 0, 0, 0, 0, 0, 0, 7721, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8056, 123, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7832, 8191, 8237, 8144, 8539, 8218, 8183, 8192] avg=8192.0000 max=8539 max_rank=4 min=7832 min_rank=0 imbalance_ratio=1.042358 max_load_reduction=56997 max_load_reduction_pct=0.869705 moved_to_echo=33132 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 103, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 2, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8056, 123, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8262, 8191, 8238, 8144, 8100, 8226, 8183, 8192] avg=8192.0000 max=8262 max_rank=0 min=8100 min_rank=4 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32701 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 103, 0, 0, 0, 0, 0, 0, 0, 8046, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 2, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8056, 123, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8150, 8191, 8238, 8144, 8213, 8225, 8183, 8192] avg=8192.0000 max=8238 max_rank=2 min=8144 min_rank=3 imbalance_ratio=1.005615 max_load_reduction=57298 max_load_reduction_pct=0.874298 moved_to_echo=32813 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 103, 0, 0, 1, 0, 0, 0, 0, 8041, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 10, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8056, 122, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8148, 8191, 8246, 8144, 8217, 8216, 8182, 8192] avg=8192.0000 max=8246 max_rank=2 min=8144 min_rank=3 imbalance_ratio=1.006592 max_load_reduction=57290 max_load_reduction_pct=0.874176 moved_to_echo=32807 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 103, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8056, 123, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8261, 8191, 8236, 8144, 8103, 8226, 8183, 8192] avg=8192.0000 max=8261 max_rank=0 min=8103 min_rank=4 imbalance_ratio=1.008423 max_load_reduction=57275 max_load_reduction_pct=0.873947 moved_to_echo=32704 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 103, 0, 0, 0, 0, 0, 0, 0, 8096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 5, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8056, 123, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8191, 8241, 8145, 8159, 8226, 8183, 8192] avg=8192.0000 max=8241 max_rank=2 min=8145 min_rank=3 imbalance_ratio=1.005981 max_load_reduction=57295 max_load_reduction_pct=0.874252 moved_to_echo=32760 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 102, 0, 0, 119, 0, 0, 7, 43, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 6, 4, 7, 0, 0, 8134, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7808, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 391, 0, 0, 0, 0, 0, 0, 8132, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 6, 0, 6, 0, 0, 8058, 0, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7937, 7, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 7975, 122, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8459, 8189, 7859, 8534, 8081, 7978, 8245, 8191] avg=8192.0000 max=8534 max_rank=3 min=7859 min_rank=2 imbalance_ratio=1.041748 max_load_reduction=57002 max_load_reduction_pct=0.869781 moved_to_echo=32495 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 0, 8132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 3, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8061, 125, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 8201, 8241, 8137, 8124, 8221, 8194, 8193] avg=8192.0000 max=8241 max_rank=2 min=8124 min_rank=4 imbalance_ratio=1.005981 max_load_reduction=57295 max_load_reduction_pct=0.874252 moved_to_echo=32732 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7647, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7939, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7207, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8103, 0, 0, 202, 0, 0, 7, 65, 8186, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 6, 4, 7, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7851, 0, 0, 0, 0, 0, 1, 0, 0, 437, 0, 0, 0, 0, 0, 0, 8180, 10, 0, 0, 0, 1, 0, 2, 6, 0, 6, 0, 0, 8180, 0, 6, 1, 0, 0, 0, 7775, 7, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 270, 0, 0, 0, 0, 0, 0, 0, 7647, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8138, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 1, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7868, 304, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7918, 8192, 8232, 8138, 8433, 8245, 8186, 8192] avg=8192.0000 max=8433 max_rank=4 min=7918 min_rank=0 imbalance_ratio=1.029419 max_load_reduction=57103 max_load_reduction_pct=0.871323 moved_to_echo=33056 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 270, 0, 0, 0, 0, 0, 0, 0, 7939, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8138, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 10, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7868, 304, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8192, 8242, 8138, 8138, 8236, 8186, 8192] avg=8192.0000 max=8242 max_rank=2 min=8138 min_rank=3 imbalance_ratio=1.006104 max_load_reduction=57294 max_load_reduction_pct=0.874237 moved_to_echo=32752 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 270, 0, 0, 0, 0, 0, 0, 0, 7901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8138, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 2, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7868, 304, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 8192, 8233, 8139, 8177, 8246, 8186, 8192] avg=8192.0000 max=8246 max_rank=5 min=8139 min_rank=3 imbalance_ratio=1.006592 max_load_reduction=57290 max_load_reduction_pct=0.874176 moved_to_echo=32801 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 270, 0, 0, 0, 0, 0, 0, 0, 7207, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8138, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7868, 304, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7485, 8192, 8232, 8138, 8873, 8238, 8186, 8192] avg=8192.0000 max=8873 max_rank=4 min=7485 min_rank=0 imbalance_ratio=1.083130 max_load_reduction=56663 max_load_reduction_pct=0.864609 moved_to_echo=33489 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 270, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8138, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7868, 304, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8384, 8192, 8232, 8138, 7966, 8246, 8186, 8192] avg=8192.0000 max=8384 max_rank=0 min=7966 min_rank=4 imbalance_ratio=1.023438 max_load_reduction=57152 max_load_reduction_pct=0.872070 moved_to_echo=32590 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 267, 0, 0, 0, 0, 0, 0, 0, 8025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7869, 311, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8292, 8196, 8234, 8133, 8048, 8243, 8197, 8193] avg=8192.0000 max=8292 max_rank=0 min=8048 min_rank=4 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32681 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 268, 0, 0, 202, 0, 0, 7, 65, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 6, 4, 7, 0, 0, 8113, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7851, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 437, 0, 0, 0, 0, 0, 0, 8126, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 6, 0, 6, 0, 0, 7876, 0, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7775, 7, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 7782, 304, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8728, 8188, 7898, 8574, 7898, 7836, 8223, 8191] avg=8192.0000 max=8728 max_rank=0 min=7836 min_rank=5 imbalance_ratio=1.065430 max_load_reduction=56808 max_load_reduction_pct=0.866821 moved_to_echo=32148 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 270, 0, 0, 0, 0, 0, 0, 0, 8076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8138, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7868, 304, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8346, 8192, 8233, 8138, 8003, 8246, 8186, 8192] avg=8192.0000 max=8346 max_rank=0 min=8003 min_rank=4 imbalance_ratio=1.018799 max_load_reduction=57190 max_load_reduction_pct=0.872650 moved_to_echo=32627 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 896, 0, 0, 128, 0, 0, 128, 128, 64640, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 128, 128, 0, 0, 65280, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 62976, 2560, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 2176, 0, 0, 256, 0, 0, 128, 128, 63104, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 65280, 0, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[2432, 0, 128, 0, 128, 128, 0, 128, 0, 0, 63104, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 0, 0, 512, 0, 0, 0, 0, 0, 0, 65152, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 65152, 0, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 64384, 1024, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 65280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[1024, 0, 128, 0, 128, 128, 0, 128, 0, 0, 64640, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 65152, 128, 0, 0, 0, 0, 0, 0, 0, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 65536, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 128, 128, 0, 0, 65152, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True - [2026-08-26 14:48:14] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 436.0 | throughput per GPU (TFLOP/s/GPU): 414.8 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.297282E+00 | loss scale: 1.0 | grad norm: 4.765 | number of skipped iterations: 0 | number of nan iterations: 0 | -[after training is done] datetime: 2026-08-26 14:48:14 -[rank0]:[W826 14:48:16.064533303 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -W0826 14:48:20.020000 11720 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers -W0826 14:48:20.021000 11720 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 11816 closing signal SIGTERM -W0826 14:48:20.021000 11720 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 11820 closing signal SIGTERM -Traceback (most recent call last): - File "/usr/local/bin/torchrun", line 8, in - sys.exit(main()) - ^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper - return f(*args, **kwargs) - ^^^^^^^^^^^^^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main - run(args) - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run - elastic_launch( - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ - return launch_agent(self._config, self._entrypoint, list(args)) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent - result = agent.run() - ^^^^^^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper - result = f(*args, **kwargs) - ^^^^^^^^^^^^^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run - result = self._invoke_run(role) - ^^^^^^^^^^^^^^^^^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run - time.sleep(monitor_interval) - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler - raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 11720 got signal: 15 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True + [2026-08-26 15:24:15] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 441.2 | throughput per GPU (TFLOP/s/GPU): 409.9 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.296476E+00 | loss scale: 1.0 | grad norm: 4.768 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-26 15:24:15 +[rank0]:[W826 15:24:16.742019055 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) diff --git a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00__forcebal/rep1/mem_samples.csv b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00__forcebal/rep1/mem_samples.csv index ac8cf52ae033a7df86bfff5a7ff51f61eed98ac3..62e0f1440e11459e6efd9d98fb33a9ec33624f88 100644 --- a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00__forcebal/rep1/mem_samples.csv +++ b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00__forcebal/rep1/mem_samples.csv @@ -26,7 +26,7 @@ 1, 4 2, 4 3, 4 -4, 2 +4, 1 5, 1 6, 1 7, 1 @@ -38,22 +38,22 @@ 5, 4 6, 4 7, 4 -0, 725 -1, 725 -2, 725 -3, 725 -4, 725 -5, 725 -6, 725 -7, 725 -0, 1057 -1, 1061 -2, 1061 -3, 1059 -4, 1051 -5, 1059 -6, 1077 -7, 1077 +0, 4 +1, 4 +2, 4 +3, 629 +4, 4 +5, 4 +6, 4 +7, 4 +0, 821 +1, 821 +2, 821 +3, 821 +4, 821 +5, 821 +6, 821 +7, 821 0, 1975 1, 1975 2, 1975 @@ -79,9 +79,9 @@ 6, 49279 7, 49279 0, 49927 -1, 49927 +1, 49929 2, 49927 -3, 49929 +3, 49927 4, 49639 5, 49927 6, 49927 @@ -102,14 +102,14 @@ 5, 54105 6, 54105 7, 54105 -0, 54495 -1, 54493 -2, 54505 -3, 54493 -4, 54181 -5, 54499 -6, 54477 -7, 54473 +0, 54475 +1, 54495 +2, 54527 +3, 54473 +4, 54207 +5, 54495 +6, 54483 +7, 54489 0, 55543 1, 55543 2, 55543 @@ -126,14 +126,6 @@ 5, 56567 6, 56567 7, 56567 -0, 58377 -1, 58377 -2, 58377 -3, 58377 -4, 58089 -5, 58377 -6, 58377 -7, 58377 0, 59799 1, 59799 2, 59799 @@ -142,14 +134,22 @@ 5, 59799 6, 59799 7, 59799 -0, 105803 -1, 105849 -2, 105837 -3, 105865 -4, 105839 -5, 105865 -6, 105843 -7, 105857 +0, 59799 +1, 59799 +2, 59799 +3, 59799 +4, 59511 +5, 59799 +6, 59799 +7, 59799 +0, 105827 +1, 105877 +2, 105911 +3, 105909 +4, 105871 +5, 105869 +6, 105879 +7, 105893 0, 106307 1, 106359 2, 106359 @@ -182,59 +182,67 @@ 5, 106361 6, 106361 7, 106361 -0, 110361 -1, 110413 -2, 110413 -3, 110413 -4, 110381 -5, 110413 -6, 110413 -7, 109645 -0, 111009 -1, 111061 +0, 107673 +1, 107725 +2, 107725 +3, 107725 +4, 107693 +5, 107725 +6, 107725 +7, 107725 +0, 111393 +1, 111445 2, 111061 -3, 111061 -4, 111029 -5, 111061 +3, 111445 +4, 111413 +5, 111445 6, 111061 -7, 110293 -0, 111687 -1, 111727 -2, 111739 -3, 111745 -4, 111667 -5, 111697 -6, 111719 -7, 110961 -0, 141435 -1, 141555 +7, 111061 +0, 112041 +1, 112037 +2, 111703 +3, 112135 +4, 112067 +5, 112109 +6, 111685 +7, 111713 +0, 141819 +1, 141939 2, 141555 -3, 141555 -4, 141539 -5, 141555 +3, 141939 +4, 141923 +5, 141939 6, 141555 -7, 140787 -0, 141435 -1, 141555 +7, 141555 +0, 141819 +1, 141939 2, 141555 -3, 141555 -4, 141539 -5, 141555 +3, 141939 +4, 141923 +5, 141939 6, 141555 -7, 140787 -0, 141435 -1, 141555 +7, 141555 +0, 141819 +1, 141939 2, 141555 -3, 141555 -4, 141539 -5, 141555 +3, 141939 +4, 141923 +5, 141939 6, 141555 -7, 140787 -0, 141430 -1, 5566 +7, 141555 +0, 3963 +1, 3966 2, 3966 3, 3966 -4, 3966 -5, 3966 -6, 3966 -7, 4286 +4, 4286 +5, 141937 +6, 5566 +7, 3966 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 diff --git a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00__forcebal/rep1/train.log b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00__forcebal/rep1/train.log index a75a8f71d279797d1c082225c1373a9ab434e4b6..901b383f18b7fdfdcb9c185383b472533ccbef76 100644 --- a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00__forcebal/rep1/train.log +++ b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00__forcebal/rep1/train.log @@ -1,5 +1,5 @@ -[14:48:22] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) -[14:48:22] WARN: token-drop-analysis lacks --moe-router-fusion; QWEN3_235B runs unfused (timing only, routing identical) +[15:24:25] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +[15:24:25] WARN: token-drop-analysis lacks --moe-router-fusion; QWEN3_235B runs unfused (timing only, routing identical) ============================================================== QWEN3_235B / echo (slots/rank=16 policy=matched) E=128 K=8 H'=1536 m=16 @@ -8,13 +8,21 @@ memory_dump=none snapshots=/workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00__forcebal/rep1/memory_snapshots/snapshot.pickle repo=/workspace/Megatron-MoE-Echo @ b099fd84 ============================================================== -W0826 14:48:23.825000 16295 torch/distributed/run.py:766] -W0826 14:48:23.825000 16295 torch/distributed/run.py:766] ***************************************** -W0826 14:48:23.825000 16295 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -W0826 14:48:23.825000 16295 torch/distributed/run.py:766] ***************************************** +W0826 15:24:26.660000 83333 torch/distributed/run.py:766] +W0826 15:24:26.660000 83333 torch/distributed/run.py:766] ***************************************** +W0826 15:24:26.660000 83333 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 15:24:26.660000 83333 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer Number of virtual stages per pipeline stage: None @@ -569,59 +577,51 @@ WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -[Gloo] Rank [Gloo] Rank 2 is connected to 70 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. -Expected number of connected peer ranks is : 7 -[Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank 74513 is connected to 6 is connected to is connected to is connected to is connected to 7 is connected to 7777 peer ranks. 7 peer ranks. peer ranks. peer ranks. peer ranks. Expected number of connected peer ranks is : peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : 7Expected number of connected peer ranks is : 7777 - -7 - - - [Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank [Gloo] Rank 7 is connected to 76 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. -Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank 152476 is connected to is connected to is connected to is connected to is connected to is connected to 777777 peer ranks. peer ranks. peer ranks. peer ranks. peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : 7777 +77 + + + + [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 > initialized tensor model parallel with size 1 > initialized pipeline model parallel with size 1 > setting random seeds to 1234 ... -[Gloo] Rank 0[Gloo] Rank is connected to 00 is connected to peer ranks. 0Expected number of connected peer ranks is : peer ranks. 0Expected number of connected peer ranks is : -0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[rank2]:[W826 14:48:41.701129497 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank1]:[W826 14:48:41.701212638 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank7]:[W826 14:48:41.701622547 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank4]:[W826 14:48:41.701759906 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank5]:[W826 14:48:41.702074514 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank6]:[W826 14:48:41.703071803 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank3]:[W826 14:48:41.703447401 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank7]:[W826 15:24:44.779641390 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 15:24:44.779774013 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 15:24:44.781933711 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 15:24:44.781933070 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 15:24:44.782006252 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 15:24:44.782020237 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 15:24:44.790550204 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. > compiling dataset index builder ... make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' make: Nothing to be done for 'default'. make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' ->>> done with dataset index builder. Compilation time: 0.036 seconds +>>> done with dataset index builder. Compilation time: 0.033 seconds WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. > compiling and loading fused kernels ... -[rank0]:[W826 14:48:41.859250524 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. ->>> done with compiling and loading fused kernels. Compilation time: 3.455 seconds -time to initialize megatron (seconds): 7.213 -[after megatron is initialized] datetime: 2026-08-26 14:48:47 +[rank0]:[W826 15:24:44.935572071 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.820 seconds +time to initialize megatron (seconds): 7.185 +[after megatron is initialized] datetime: 2026-08-26 15:24:50 building GPT model ... /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. warnings.warn( @@ -645,46 +645,46 @@ INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradi Params for bucket 1 (206045184 elements, 206045184 padded size): module.output_layer.weight Params for bucket 2 (71840000 elements, 71840000 padded size): + module.decoder.final_layernorm.weight module.decoder.layers.11.pre_mlp_layernorm.weight module.decoder.layers.11.self_attention.k_layernorm.weight module.decoder.layers.11.self_attention.linear_proj.weight - module.decoder.final_layernorm.weight - module.decoder.layers.11.mlp.router.weight module.decoder.layers.11.self_attention.q_layernorm.weight + module.decoder.layers.11.mlp.router.weight module.decoder.layers.11.self_attention.linear_qkv.weight module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight Params for bucket 3 (71835904 elements, 71835904 padded size): + module.decoder.layers.10.self_attention.linear_proj.weight module.decoder.layers.10.mlp.router.weight - module.decoder.layers.10.pre_mlp_layernorm.weight module.decoder.layers.10.self_attention.linear_qkv.weight - module.decoder.layers.10.self_attention.linear_proj.weight - module.decoder.layers.10.self_attention.q_layernorm.weight module.decoder.layers.10.self_attention.k_layernorm.weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.q_layernorm.weight module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight Params for bucket 4 (71835904 elements, 71835904 padded size): - module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.mlp.router.weight module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.pre_mlp_layernorm.weight module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.9.self_attention.linear_proj.weight - module.decoder.layers.9.mlp.router.weight - module.decoder.layers.9.self_attention.q_layernorm.weight module.decoder.layers.9.self_attention.k_layernorm.weight + module.decoder.layers.9.self_attention.q_layernorm.weight Params for bucket 5 (71835904 elements, 71835904 padded size): + module.decoder.layers.8.mlp.router.weight module.decoder.layers.8.pre_mlp_layernorm.weight module.decoder.layers.8.self_attention.k_layernorm.weight module.decoder.layers.8.self_attention.linear_proj.weight - module.decoder.layers.8.mlp.router.weight module.decoder.layers.8.self_attention.q_layernorm.weight module.decoder.layers.8.self_attention.linear_qkv.weight module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight Params for bucket 6 (71835904 elements, 71835904 padded size): - module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.7.self_attention.linear_proj.weight module.decoder.layers.7.mlp.router.weight - module.decoder.layers.7.self_attention.k_layernorm.weight + module.decoder.layers.7.self_attention.q_layernorm.weight module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.k_layernorm.weight module.decoder.layers.7.pre_mlp_layernorm.weight - module.decoder.layers.7.self_attention.q_layernorm.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.self_attention.linear_proj.weight Params for bucket 7 (71835904 elements, 71835904 padded size): module.decoder.layers.6.pre_mlp_layernorm.weight module.decoder.layers.6.self_attention.linear_qkv.weight @@ -694,63 +694,63 @@ Params for bucket 7 (71835904 elements, 71835904 padded size): module.decoder.layers.6.self_attention.q_layernorm.weight module.decoder.layers.6.self_attention.k_layernorm.weight Params for bucket 8 (71835904 elements, 71835904 padded size): - module.decoder.layers.5.self_attention.q_layernorm.weight - module.decoder.layers.5.pre_mlp_layernorm.weight - module.decoder.layers.5.self_attention.k_layernorm.weight module.decoder.layers.5.mlp.router.weight module.decoder.layers.5.self_attention.linear_qkv.weight module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.k_layernorm.weight + module.decoder.layers.5.self_attention.q_layernorm.weight Params for bucket 9 (71835904 elements, 71835904 padded size): - module.decoder.layers.4.pre_mlp_layernorm.weight - module.decoder.layers.4.self_attention.q_layernorm.weight - module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.4.mlp.router.weight - module.decoder.layers.4.self_attention.k_layernorm.weight + module.decoder.layers.4.pre_mlp_layernorm.weight module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.4.self_attention.k_layernorm.weight + module.decoder.layers.4.self_attention.q_layernorm.weight Params for bucket 10 (71835904 elements, 71835904 padded size): - module.decoder.layers.3.mlp.router.weight module.decoder.layers.3.self_attention.linear_qkv.weight module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.3.mlp.router.weight module.decoder.layers.3.self_attention.q_layernorm.weight module.decoder.layers.3.self_attention.k_layernorm.weight module.decoder.layers.3.pre_mlp_layernorm.weight Params for bucket 11 (71835904 elements, 71835904 padded size): - module.decoder.layers.2.mlp.router.weight - module.decoder.layers.2.self_attention.q_layernorm.weight module.decoder.layers.2.pre_mlp_layernorm.weight module.decoder.layers.2.self_attention.k_layernorm.weight + module.decoder.layers.2.self_attention.q_layernorm.weight + module.decoder.layers.2.mlp.router.weight module.decoder.layers.2.self_attention.linear_qkv.weight module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.2.self_attention.linear_proj.weight Params for bucket 12 (71835904 elements, 71835904 padded size): + module.decoder.layers.1.self_attention.k_layernorm.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.q_layernorm.weight module.decoder.layers.1.self_attention.linear_qkv.weight module.decoder.layers.1.pre_mlp_layernorm.weight module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.1.self_attention.linear_proj.weight - module.decoder.layers.1.mlp.router.weight - module.decoder.layers.1.self_attention.q_layernorm.weight - module.decoder.layers.1.self_attention.k_layernorm.weight Params for bucket 13 (71835904 elements, 71835904 padded size): - module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.self_attention.q_layernorm.weight module.decoder.layers.0.pre_mlp_layernorm.weight module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.linear_qkv.weight module.decoder.layers.0.self_attention.k_layernorm.weight - module.decoder.layers.0.self_attention.q_layernorm.weight module.decoder.layers.0.self_attention.linear_proj.weight Params for bucket 14 (206045184 elements, 206045184 padded size): module.embedding.word_embeddings.weight INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 76 Params for bucket 1 (44040192 elements, 44040192 padded size): module.decoder.layers.11.mlp.experts.linear_fc2.weight15 - module.decoder.layers.11.mlp.experts.linear_fc2.weight13 module.decoder.layers.11.mlp.experts.linear_fc2.weight11 + module.decoder.layers.11.mlp.experts.linear_fc2.weight13 module.decoder.layers.11.mlp.experts.linear_fc2.weight9 - module.decoder.layers.11.mlp.experts.linear_fc2.weight12 module.decoder.layers.11.mlp.experts.linear_fc2.weight14 + module.decoder.layers.11.mlp.experts.linear_fc2.weight12 module.decoder.layers.11.mlp.experts.linear_fc2.weight10 Params for bucket 2 (44040192 elements, 44040192 padded size): module.decoder.layers.11.mlp.experts.linear_fc2.weight5 @@ -758,19 +758,19 @@ Params for bucket 2 (44040192 elements, 44040192 padded size): module.decoder.layers.11.mlp.experts.linear_fc2.weight3 module.decoder.layers.11.mlp.experts.linear_fc2.weight2 module.decoder.layers.11.mlp.experts.linear_fc2.weight8 - module.decoder.layers.11.mlp.experts.linear_fc2.weight4 module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 Params for bucket 3 (50331648 elements, 50331648 padded size): - module.decoder.layers.11.mlp.experts.linear_fc2.weight0 module.decoder.layers.11.mlp.experts.linear_fc1.weight14 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 module.decoder.layers.11.mlp.experts.linear_fc2.weight1 module.decoder.layers.11.mlp.experts.linear_fc1.weight15 module.decoder.layers.11.mlp.experts.linear_fc1.weight13 Params for bucket 4 (50331648 elements, 50331648 padded size): module.decoder.layers.11.mlp.experts.linear_fc1.weight12 + module.decoder.layers.11.mlp.experts.linear_fc1.weight11 module.decoder.layers.11.mlp.experts.linear_fc1.weight9 module.decoder.layers.11.mlp.experts.linear_fc1.weight10 - module.decoder.layers.11.mlp.experts.linear_fc1.weight11 Params for bucket 5 (50331648 elements, 50331648 padded size): module.decoder.layers.11.mlp.experts.linear_fc1.weight8 module.decoder.layers.11.mlp.experts.linear_fc1.weight5 @@ -778,20 +778,20 @@ Params for bucket 5 (50331648 elements, 50331648 padded size): module.decoder.layers.11.mlp.experts.linear_fc1.weight7 Params for bucket 6 (50331648 elements, 50331648 padded size): module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 module.decoder.layers.11.mlp.experts.linear_fc1.weight3 module.decoder.layers.11.mlp.experts.linear_fc1.weight1 - module.decoder.layers.11.mlp.experts.linear_fc1.weight2 Params for bucket 7 (44040192 elements, 44040192 padded size): module.decoder.layers.11.mlp.experts.linear_fc1.weight0 module.decoder.layers.10.mlp.experts.linear_fc2.weight14 module.decoder.layers.10.mlp.experts.linear_fc2.weight12 - module.decoder.layers.10.mlp.experts.linear_fc2.weight15 module.decoder.layers.10.mlp.experts.linear_fc2.weight13 + module.decoder.layers.10.mlp.experts.linear_fc2.weight15 module.decoder.layers.10.mlp.experts.linear_fc2.weight11 Params for bucket 8 (44040192 elements, 44040192 padded size): module.decoder.layers.10.mlp.experts.linear_fc2.weight10 - module.decoder.layers.10.mlp.experts.linear_fc2.weight8 module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight8 module.decoder.layers.10.mlp.experts.linear_fc2.weight4 module.decoder.layers.10.mlp.experts.linear_fc2.weight7 module.decoder.layers.10.mlp.experts.linear_fc2.weight9 @@ -806,13 +806,13 @@ Params for bucket 9 (50331648 elements, 50331648 padded size): Params for bucket 10 (50331648 elements, 50331648 padded size): module.decoder.layers.10.mlp.experts.linear_fc1.weight13 module.decoder.layers.10.mlp.experts.linear_fc1.weight10 - module.decoder.layers.10.mlp.experts.linear_fc1.weight11 module.decoder.layers.10.mlp.experts.linear_fc1.weight12 + module.decoder.layers.10.mlp.experts.linear_fc1.weight11 Params for bucket 11 (50331648 elements, 50331648 padded size): module.decoder.layers.10.mlp.experts.linear_fc1.weight9 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 module.decoder.layers.10.mlp.experts.linear_fc1.weight7 module.decoder.layers.10.mlp.experts.linear_fc1.weight8 - module.decoder.layers.10.mlp.experts.linear_fc1.weight6 Params for bucket 12 (50331648 elements, 50331648 padded size): module.decoder.layers.10.mlp.experts.linear_fc1.weight5 module.decoder.layers.10.mlp.experts.linear_fc1.weight4 @@ -820,41 +820,41 @@ Params for bucket 12 (50331648 elements, 50331648 padded size): module.decoder.layers.10.mlp.experts.linear_fc1.weight3 Params for bucket 13 (44040192 elements, 44040192 padded size): module.decoder.layers.10.mlp.experts.linear_fc1.weight1 - module.decoder.layers.9.mlp.experts.linear_fc2.weight15 module.decoder.layers.9.mlp.experts.linear_fc2.weight13 - module.decoder.layers.9.mlp.experts.linear_fc2.weight14 + module.decoder.layers.9.mlp.experts.linear_fc2.weight15 module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight14 Params for bucket 14 (44040192 elements, 44040192 padded size): module.decoder.layers.9.mlp.experts.linear_fc2.weight11 module.decoder.layers.9.mlp.experts.linear_fc2.weight9 module.decoder.layers.9.mlp.experts.linear_fc2.weight7 module.decoder.layers.9.mlp.experts.linear_fc2.weight12 - module.decoder.layers.9.mlp.experts.linear_fc2.weight8 module.decoder.layers.9.mlp.experts.linear_fc2.weight10 + module.decoder.layers.9.mlp.experts.linear_fc2.weight8 module.decoder.layers.9.mlp.experts.linear_fc2.weight6 Params for bucket 15 (50331648 elements, 50331648 padded size): module.decoder.layers.9.mlp.experts.linear_fc2.weight5 module.decoder.layers.9.mlp.experts.linear_fc2.weight3 module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 module.decoder.layers.9.mlp.experts.linear_fc2.weight1 module.decoder.layers.9.mlp.experts.linear_fc2.weight2 - module.decoder.layers.9.mlp.experts.linear_fc2.weight4 module.decoder.layers.9.mlp.experts.linear_fc1.weight15 Params for bucket 16 (50331648 elements, 50331648 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight14 + module.decoder.layers.9.mlp.experts.linear_fc1.weight11 module.decoder.layers.9.mlp.experts.linear_fc1.weight12 module.decoder.layers.9.mlp.experts.linear_fc1.weight13 - module.decoder.layers.9.mlp.experts.linear_fc1.weight11 Params for bucket 17 (50331648 elements, 50331648 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight10 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 module.decoder.layers.9.mlp.experts.linear_fc1.weight8 module.decoder.layers.9.mlp.experts.linear_fc1.weight9 - module.decoder.layers.9.mlp.experts.linear_fc1.weight7 Params for bucket 18 (50331648 elements, 50331648 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 module.decoder.layers.9.mlp.experts.linear_fc1.weight5 module.decoder.layers.9.mlp.experts.linear_fc1.weight3 - module.decoder.layers.9.mlp.experts.linear_fc1.weight4 Params for bucket 19 (44040192 elements, 44040192 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight2 module.decoder.layers.8.mlp.experts.linear_fc2.weight15 @@ -878,8 +878,8 @@ Params for bucket 21 (44040192 elements, 44040192 padded size): module.decoder.layers.8.mlp.experts.linear_fc2.weight2 Params for bucket 22 (44040192 elements, 44040192 padded size): module.decoder.layers.8.mlp.experts.linear_fc2.weight0 - module.decoder.layers.8.mlp.experts.linear_fc1.weight15 module.decoder.layers.8.mlp.experts.linear_fc1.weight13 + module.decoder.layers.8.mlp.experts.linear_fc1.weight15 module.decoder.layers.8.mlp.experts.linear_fc1.weight14 Params for bucket 23 (50331648 elements, 50331648 padded size): module.decoder.layers.8.mlp.experts.linear_fc1.weight12 @@ -889,24 +889,24 @@ Params for bucket 23 (50331648 elements, 50331648 padded size): Params for bucket 24 (50331648 elements, 50331648 padded size): module.decoder.layers.8.mlp.experts.linear_fc1.weight8 module.decoder.layers.8.mlp.experts.linear_fc1.weight6 - module.decoder.layers.8.mlp.experts.linear_fc1.weight5 module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 Params for bucket 25 (50331648 elements, 50331648 padded size): module.decoder.layers.8.mlp.experts.linear_fc1.weight4 module.decoder.layers.8.mlp.experts.linear_fc1.weight1 - module.decoder.layers.8.mlp.experts.linear_fc1.weight3 module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 Params for bucket 26 (44040192 elements, 44040192 padded size): - module.decoder.layers.7.mlp.experts.linear_fc2.weight12 module.decoder.layers.7.mlp.experts.linear_fc2.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight12 module.decoder.layers.8.mlp.experts.linear_fc1.weight0 module.decoder.layers.7.mlp.experts.linear_fc2.weight13 module.decoder.layers.7.mlp.experts.linear_fc2.weight15 module.decoder.layers.7.mlp.experts.linear_fc2.weight11 Params for bucket 27 (44040192 elements, 44040192 padded size): module.decoder.layers.7.mlp.experts.linear_fc2.weight10 - module.decoder.layers.7.mlp.experts.linear_fc2.weight6 module.decoder.layers.7.mlp.experts.linear_fc2.weight8 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 module.decoder.layers.7.mlp.experts.linear_fc2.weight4 module.decoder.layers.7.mlp.experts.linear_fc2.weight7 module.decoder.layers.7.mlp.experts.linear_fc2.weight9 @@ -920,13 +920,13 @@ Params for bucket 28 (50331648 elements, 50331648 padded size): module.decoder.layers.7.mlp.experts.linear_fc1.weight14 Params for bucket 29 (50331648 elements, 50331648 padded size): module.decoder.layers.7.mlp.experts.linear_fc1.weight13 - module.decoder.layers.7.mlp.experts.linear_fc1.weight12 module.decoder.layers.7.mlp.experts.linear_fc1.weight10 module.decoder.layers.7.mlp.experts.linear_fc1.weight11 + module.decoder.layers.7.mlp.experts.linear_fc1.weight12 Params for bucket 30 (50331648 elements, 50331648 padded size): module.decoder.layers.7.mlp.experts.linear_fc1.weight9 - module.decoder.layers.7.mlp.experts.linear_fc1.weight6 module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 module.decoder.layers.7.mlp.experts.linear_fc1.weight8 Params for bucket 31 (50331648 elements, 50331648 padded size): module.decoder.layers.7.mlp.experts.linear_fc1.weight5 @@ -944,27 +944,27 @@ Params for bucket 33 (44040192 elements, 44040192 padded size): module.decoder.layers.6.mlp.experts.linear_fc2.weight9 module.decoder.layers.6.mlp.experts.linear_fc2.weight7 module.decoder.layers.6.mlp.experts.linear_fc2.weight12 - module.decoder.layers.6.mlp.experts.linear_fc2.weight10 module.decoder.layers.6.mlp.experts.linear_fc2.weight8 + module.decoder.layers.6.mlp.experts.linear_fc2.weight10 module.decoder.layers.6.mlp.experts.linear_fc2.weight6 Params for bucket 34 (50331648 elements, 50331648 padded size): module.decoder.layers.6.mlp.experts.linear_fc2.weight5 module.decoder.layers.6.mlp.experts.linear_fc2.weight3 - module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight15 module.decoder.layers.6.mlp.experts.linear_fc2.weight4 - module.decoder.layers.6.mlp.experts.linear_fc2.weight1 module.decoder.layers.6.mlp.experts.linear_fc2.weight2 - module.decoder.layers.6.mlp.experts.linear_fc1.weight15 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 Params for bucket 35 (50331648 elements, 50331648 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight14 + module.decoder.layers.6.mlp.experts.linear_fc1.weight13 module.decoder.layers.6.mlp.experts.linear_fc1.weight11 module.decoder.layers.6.mlp.experts.linear_fc1.weight12 - module.decoder.layers.6.mlp.experts.linear_fc1.weight13 Params for bucket 36 (50331648 elements, 50331648 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight10 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 module.decoder.layers.6.mlp.experts.linear_fc1.weight8 module.decoder.layers.6.mlp.experts.linear_fc1.weight9 - module.decoder.layers.6.mlp.experts.linear_fc1.weight7 Params for bucket 37 (50331648 elements, 50331648 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight6 module.decoder.layers.6.mlp.experts.linear_fc1.weight4 @@ -972,50 +972,50 @@ Params for bucket 37 (50331648 elements, 50331648 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight3 Params for bucket 38 (44040192 elements, 44040192 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 module.decoder.layers.5.mlp.experts.linear_fc2.weight15 module.decoder.layers.6.mlp.experts.linear_fc1.weight1 - module.decoder.layers.6.mlp.experts.linear_fc1.weight0 Params for bucket 39 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight11 + module.decoder.layers.5.mlp.experts.linear_fc2.weight13 + module.decoder.layers.5.mlp.experts.linear_fc2.weight9 module.decoder.layers.5.mlp.experts.linear_fc2.weight14 module.decoder.layers.5.mlp.experts.linear_fc2.weight12 module.decoder.layers.5.mlp.experts.linear_fc2.weight10 module.decoder.layers.5.mlp.experts.linear_fc2.weight8 - module.decoder.layers.5.mlp.experts.linear_fc2.weight13 - module.decoder.layers.5.mlp.experts.linear_fc2.weight11 - module.decoder.layers.5.mlp.experts.linear_fc2.weight9 Params for bucket 40 (44040192 elements, 44040192 padded size): - module.decoder.layers.5.mlp.experts.linear_fc2.weight6 - module.decoder.layers.5.mlp.experts.linear_fc2.weight4 - module.decoder.layers.5.mlp.experts.linear_fc2.weight1 module.decoder.layers.5.mlp.experts.linear_fc2.weight7 - module.decoder.layers.5.mlp.experts.linear_fc2.weight3 module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 Params for bucket 41 (44040192 elements, 44040192 padded size): module.decoder.layers.5.mlp.experts.linear_fc2.weight0 - module.decoder.layers.5.mlp.experts.linear_fc1.weight13 - module.decoder.layers.5.mlp.experts.linear_fc1.weight15 module.decoder.layers.5.mlp.experts.linear_fc1.weight14 + module.decoder.layers.5.mlp.experts.linear_fc1.weight15 + module.decoder.layers.5.mlp.experts.linear_fc1.weight13 Params for bucket 42 (50331648 elements, 50331648 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight12 - module.decoder.layers.5.mlp.experts.linear_fc1.weight9 - module.decoder.layers.5.mlp.experts.linear_fc1.weight10 module.decoder.layers.5.mlp.experts.linear_fc1.weight11 + module.decoder.layers.5.mlp.experts.linear_fc1.weight10 + module.decoder.layers.5.mlp.experts.linear_fc1.weight9 Params for bucket 43 (50331648 elements, 50331648 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight8 - module.decoder.layers.5.mlp.experts.linear_fc1.weight6 - module.decoder.layers.5.mlp.experts.linear_fc1.weight7 module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 Params for bucket 44 (50331648 elements, 50331648 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight4 - module.decoder.layers.5.mlp.experts.linear_fc1.weight1 module.decoder.layers.5.mlp.experts.linear_fc1.weight2 module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 Params for bucket 45 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 module.decoder.layers.4.mlp.experts.linear_fc2.weight13 module.decoder.layers.4.mlp.experts.linear_fc2.weight15 module.decoder.layers.4.mlp.experts.linear_fc2.weight11 - module.decoder.layers.5.mlp.experts.linear_fc1.weight0 module.decoder.layers.4.mlp.experts.linear_fc2.weight14 module.decoder.layers.4.mlp.experts.linear_fc2.weight12 Params for bucket 46 (44040192 elements, 44040192 padded size): @@ -1023,98 +1023,98 @@ Params for bucket 46 (44040192 elements, 44040192 padded size): module.decoder.layers.4.mlp.experts.linear_fc2.weight9 module.decoder.layers.4.mlp.experts.linear_fc2.weight5 module.decoder.layers.4.mlp.experts.linear_fc2.weight10 - module.decoder.layers.4.mlp.experts.linear_fc2.weight8 module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight8 module.decoder.layers.4.mlp.experts.linear_fc2.weight4 Params for bucket 47 (50331648 elements, 50331648 padded size): module.decoder.layers.4.mlp.experts.linear_fc2.weight3 module.decoder.layers.4.mlp.experts.linear_fc2.weight0 module.decoder.layers.4.mlp.experts.linear_fc2.weight2 module.decoder.layers.4.mlp.experts.linear_fc1.weight14 - module.decoder.layers.4.mlp.experts.linear_fc1.weight15 module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight15 Params for bucket 48 (50331648 elements, 50331648 padded size): module.decoder.layers.4.mlp.experts.linear_fc1.weight13 - module.decoder.layers.4.mlp.experts.linear_fc1.weight12 module.decoder.layers.4.mlp.experts.linear_fc1.weight10 module.decoder.layers.4.mlp.experts.linear_fc1.weight11 + module.decoder.layers.4.mlp.experts.linear_fc1.weight12 Params for bucket 49 (50331648 elements, 50331648 padded size): module.decoder.layers.4.mlp.experts.linear_fc1.weight9 - module.decoder.layers.4.mlp.experts.linear_fc1.weight6 module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 module.decoder.layers.4.mlp.experts.linear_fc1.weight8 Params for bucket 50 (50331648 elements, 50331648 padded size): module.decoder.layers.4.mlp.experts.linear_fc1.weight5 module.decoder.layers.4.mlp.experts.linear_fc1.weight4 - module.decoder.layers.4.mlp.experts.linear_fc1.weight2 module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 Params for bucket 51 (44040192 elements, 44040192 padded size): module.decoder.layers.4.mlp.experts.linear_fc1.weight0 - module.decoder.layers.3.mlp.experts.linear_fc2.weight14 - module.decoder.layers.4.mlp.experts.linear_fc1.weight1 module.decoder.layers.3.mlp.experts.linear_fc2.weight13 module.decoder.layers.3.mlp.experts.linear_fc2.weight15 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight14 Params for bucket 52 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight9 + module.decoder.layers.3.mlp.experts.linear_fc2.weight11 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 module.decoder.layers.3.mlp.experts.linear_fc2.weight12 module.decoder.layers.3.mlp.experts.linear_fc2.weight10 module.decoder.layers.3.mlp.experts.linear_fc2.weight8 module.decoder.layers.3.mlp.experts.linear_fc2.weight6 - module.decoder.layers.3.mlp.experts.linear_fc2.weight11 - module.decoder.layers.3.mlp.experts.linear_fc2.weight9 - module.decoder.layers.3.mlp.experts.linear_fc2.weight7 Params for bucket 53 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight15 module.decoder.layers.3.mlp.experts.linear_fc2.weight4 module.decoder.layers.3.mlp.experts.linear_fc2.weight1 - module.decoder.layers.3.mlp.experts.linear_fc1.weight15 - module.decoder.layers.3.mlp.experts.linear_fc2.weight5 module.decoder.layers.3.mlp.experts.linear_fc2.weight2 - module.decoder.layers.3.mlp.experts.linear_fc2.weight3 module.decoder.layers.3.mlp.experts.linear_fc2.weight0 Params for bucket 54 (50331648 elements, 50331648 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight14 + module.decoder.layers.3.mlp.experts.linear_fc1.weight13 module.decoder.layers.3.mlp.experts.linear_fc1.weight11 module.decoder.layers.3.mlp.experts.linear_fc1.weight12 - module.decoder.layers.3.mlp.experts.linear_fc1.weight13 Params for bucket 55 (50331648 elements, 50331648 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight10 - module.decoder.layers.3.mlp.experts.linear_fc1.weight8 module.decoder.layers.3.mlp.experts.linear_fc1.weight9 module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.3.mlp.experts.linear_fc1.weight8 Params for bucket 56 (50331648 elements, 50331648 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight6 - module.decoder.layers.3.mlp.experts.linear_fc1.weight5 module.decoder.layers.3.mlp.experts.linear_fc1.weight3 module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 Params for bucket 57 (44040192 elements, 44040192 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight2 - module.decoder.layers.3.mlp.experts.linear_fc1.weight0 - module.decoder.layers.3.mlp.experts.linear_fc1.weight1 module.decoder.layers.2.mlp.experts.linear_fc2.weight15 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 Params for bucket 58 (44040192 elements, 44040192 padded size): - module.decoder.layers.2.mlp.experts.linear_fc2.weight13 - module.decoder.layers.2.mlp.experts.linear_fc2.weight11 - module.decoder.layers.2.mlp.experts.linear_fc2.weight9 module.decoder.layers.2.mlp.experts.linear_fc2.weight14 - module.decoder.layers.2.mlp.experts.linear_fc2.weight10 module.decoder.layers.2.mlp.experts.linear_fc2.weight12 + module.decoder.layers.2.mlp.experts.linear_fc2.weight10 module.decoder.layers.2.mlp.experts.linear_fc2.weight8 + module.decoder.layers.2.mlp.experts.linear_fc2.weight13 + module.decoder.layers.2.mlp.experts.linear_fc2.weight11 + module.decoder.layers.2.mlp.experts.linear_fc2.weight9 Params for bucket 59 (44040192 elements, 44040192 padded size): - module.decoder.layers.2.mlp.experts.linear_fc2.weight7 - module.decoder.layers.2.mlp.experts.linear_fc2.weight3 - module.decoder.layers.2.mlp.experts.linear_fc2.weight5 module.decoder.layers.2.mlp.experts.linear_fc2.weight6 module.decoder.layers.2.mlp.experts.linear_fc2.weight4 module.decoder.layers.2.mlp.experts.linear_fc2.weight2 module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 Params for bucket 60 (44040192 elements, 44040192 padded size): module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight14 module.decoder.layers.2.mlp.experts.linear_fc1.weight15 module.decoder.layers.2.mlp.experts.linear_fc1.weight13 - module.decoder.layers.2.mlp.experts.linear_fc1.weight14 Params for bucket 61 (50331648 elements, 50331648 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight12 - module.decoder.layers.2.mlp.experts.linear_fc1.weight9 module.decoder.layers.2.mlp.experts.linear_fc1.weight11 + module.decoder.layers.2.mlp.experts.linear_fc1.weight9 module.decoder.layers.2.mlp.experts.linear_fc1.weight10 Params for bucket 62 (50331648 elements, 50331648 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight8 @@ -1127,27 +1127,27 @@ Params for bucket 63 (50331648 elements, 50331648 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight1 module.decoder.layers.2.mlp.experts.linear_fc1.weight2 Params for bucket 64 (44040192 elements, 44040192 padded size): - module.decoder.layers.1.mlp.experts.linear_fc2.weight12 - module.decoder.layers.1.mlp.experts.linear_fc2.weight14 module.decoder.layers.2.mlp.experts.linear_fc1.weight0 - module.decoder.layers.1.mlp.experts.linear_fc2.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight14 + module.decoder.layers.1.mlp.experts.linear_fc2.weight12 module.decoder.layers.1.mlp.experts.linear_fc2.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight13 module.decoder.layers.1.mlp.experts.linear_fc2.weight11 Params for bucket 65 (44040192 elements, 44040192 padded size): module.decoder.layers.1.mlp.experts.linear_fc2.weight10 - module.decoder.layers.1.mlp.experts.linear_fc2.weight6 module.decoder.layers.1.mlp.experts.linear_fc2.weight8 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 module.decoder.layers.1.mlp.experts.linear_fc2.weight4 module.decoder.layers.1.mlp.experts.linear_fc2.weight9 module.decoder.layers.1.mlp.experts.linear_fc2.weight7 module.decoder.layers.1.mlp.experts.linear_fc2.weight5 Params for bucket 66 (50331648 elements, 50331648 padded size): - module.decoder.layers.1.mlp.experts.linear_fc1.weight15 module.decoder.layers.1.mlp.experts.linear_fc2.weight1 - module.decoder.layers.1.mlp.experts.linear_fc2.weight3 module.decoder.layers.1.mlp.experts.linear_fc2.weight2 - module.decoder.layers.1.mlp.experts.linear_fc2.weight0 module.decoder.layers.1.mlp.experts.linear_fc1.weight14 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight15 Params for bucket 67 (50331648 elements, 50331648 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight13 module.decoder.layers.1.mlp.experts.linear_fc1.weight12 @@ -1160,53 +1160,53 @@ Params for bucket 68 (50331648 elements, 50331648 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight8 Params for bucket 69 (50331648 elements, 50331648 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight5 - module.decoder.layers.1.mlp.experts.linear_fc1.weight3 module.decoder.layers.1.mlp.experts.linear_fc1.weight4 module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 Params for bucket 70 (44040192 elements, 44040192 padded size): - module.decoder.layers.1.mlp.experts.linear_fc1.weight1 - module.decoder.layers.0.mlp.experts.linear_fc2.weight14 - module.decoder.layers.0.mlp.experts.linear_fc2.weight15 module.decoder.layers.1.mlp.experts.linear_fc1.weight0 module.decoder.layers.0.mlp.experts.linear_fc2.weight13 + module.decoder.layers.0.mlp.experts.linear_fc2.weight15 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight14 Params for bucket 71 (44040192 elements, 44040192 padded size): - module.decoder.layers.0.mlp.experts.linear_fc2.weight12 - module.decoder.layers.0.mlp.experts.linear_fc2.weight10 - module.decoder.layers.0.mlp.experts.linear_fc2.weight8 - module.decoder.layers.0.mlp.experts.linear_fc2.weight6 module.decoder.layers.0.mlp.experts.linear_fc2.weight9 module.decoder.layers.0.mlp.experts.linear_fc2.weight11 module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight12 + module.decoder.layers.0.mlp.experts.linear_fc2.weight8 + module.decoder.layers.0.mlp.experts.linear_fc2.weight10 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 Params for bucket 72 (50331648 elements, 50331648 padded size): - module.decoder.layers.0.mlp.experts.linear_fc2.weight1 - module.decoder.layers.0.mlp.experts.linear_fc2.weight2 - module.decoder.layers.0.mlp.experts.linear_fc2.weight4 module.decoder.layers.0.mlp.experts.linear_fc2.weight5 module.decoder.layers.0.mlp.experts.linear_fc2.weight3 module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 module.decoder.layers.0.mlp.experts.linear_fc1.weight15 Params for bucket 73 (50331648 elements, 50331648 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight14 - module.decoder.layers.0.mlp.experts.linear_fc1.weight11 - module.decoder.layers.0.mlp.experts.linear_fc1.weight13 module.decoder.layers.0.mlp.experts.linear_fc1.weight12 + module.decoder.layers.0.mlp.experts.linear_fc1.weight13 + module.decoder.layers.0.mlp.experts.linear_fc1.weight11 Params for bucket 74 (50331648 elements, 50331648 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight10 - module.decoder.layers.0.mlp.experts.linear_fc1.weight7 module.decoder.layers.0.mlp.experts.linear_fc1.weight9 module.decoder.layers.0.mlp.experts.linear_fc1.weight8 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 Params for bucket 75 (50331648 elements, 50331648 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight6 - module.decoder.layers.0.mlp.experts.linear_fc1.weight4 - module.decoder.layers.0.mlp.experts.linear_fc1.weight3 module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 Params for bucket 76 (37748736 elements, 37748736 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight2 module.decoder.layers.0.mlp.experts.linear_fc1.weight0 module.decoder.layers.0.mlp.experts.linear_fc1.weight1 -INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine -[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 14:48:47 +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 15:24:51 > building train, validation, and test datasets ... > datasets target sizes (minimum size): train: 96 @@ -1214,7 +1214,7 @@ INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine test: 0 INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] > building train, validation, and test datasets for GPT ... -INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers @@ -1227,30 +1227,30 @@ INFO:megatron.core.datasets.gpt_dataset: Load the sample index from eadc3756447e INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-shuffle_index.npy INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536 > finished creating GPT datasets ... -[after dataloaders are built] datetime: 2026-08-26 14:48:47 +[after dataloaders are built] datetime: 2026-08-26 15:24:51 done with setup ... -training ... (min, max) time across ranks (ms): - model-and-optimizer-setup ......................: (660.54, 675.23) - train/valid/test-data-iterators-setup ..........: (62.67, 67.48) + model-and-optimizer-setup ......................: (662.23, 673.46) + train/valid/test-data-iterators-setup ..........: (60.00, 67.04) +training ... Overwriting rerun_state_machine.current_iteration from -1 to 0... -[before the start of training step] datetime: 2026-08-26 14:48:48 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[473, 498, 483, 518, 506, 509, 534, 532, 487, 539, 526, 520, 508, 538, 511, 497, 494, 446, 523, 508, 528, 473, 503, 480, 503, 529, 505, 555, 527, 503, 513, 539, 526, 522, 530, 480, 558, 526, 510, 518, 544, 551, 522, 539, 554, 515, 514, 536, 449, 521, 476, 517, 501, 512, 468, 533, 533, 467, 520, 498, 500, 515, 507, 491, 502, 511, 549, 516, 514, 486, 525, 518, 526, 530, 522, 513, 523, 499, 503, 512, 498, 549, 516, 453, 509, 510, 487, 514, 485, 497, 538, 518, 539, 496, 519, 495, 485, 489, 533, 503, 520, 512, 520, 538, 502, 523, 508, 522, 516, 501, 488, 514, 473, 497, 521, 514, 566, 512, 539, 504, 514, 506, 509, 517, 508, 513, 515, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[507, 535, 534, 494, 526, 480, 505, 507, 513, 490, 467, 529, 510, 495, 509, 548, 461, 490, 531, 509, 533, 535, 501, 518, 516, 477, 541, 516, 520, 534, 505, 519, 538, 509, 512, 473, 514, 494, 482, 466, 557, 523, 535, 464, 539, 536, 501, 486, 519, 530, 517, 460, 496, 544, 512, 529, 534, 483, 493, 480, 522, 511, 509, 540, 513, 529, 503, 536, 483, 509, 484, 516, 484, 503, 559, 539, 513, 519, 506, 504, 483, 469, 522, 472, 514, 543, 496, 497, 533, 508, 516, 487, 552, 506, 501, 507, 561, 489, 509, 534, 500, 533, 528, 521, 527, 522, 503, 540, 506, 483, 554, 498, 549, 518, 533, 488, 556, 474, 510, 493, 499, 521, 522, 490, 519, 528, 523, 536] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +[before the start of training step] datetime: 2026-08-26 15:24:51 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[494, 464, 509, 533, 538, 500, 492, 503, 530, 529, 512, 568, 517, 484, 506, 503, 523, 518, 484, 493, 516, 519, 489, 517, 484, 541, 504, 534, 466, 525, 569, 491, 534, 555, 502, 525, 517, 518, 520, 537, 512, 527, 509, 537, 504, 483, 527, 511, 533, 538, 512, 472, 510, 513, 525, 491, 514, 479, 510, 537, 497, 516, 475, 508, 514, 500, 507, 516, 473, 513, 490, 516, 523, 532, 506, 517, 553, 482, 492, 528, 544, 509, 521, 516, 482, 504, 540, 564, 487, 526, 545, 527, 492, 530, 531, 518, 454, 511, 481, 472, 491, 501, 500, 468, 554, 482, 480, 501, 561, 513, 534, 495, 501, 503, 518, 533, 525, 497, 532, 569, 525, 503, 497, 488, 517, 537, 475, 517] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[492, 513, 547, 511, 525, 519, 499, 511, 486, 527, 506, 511, 518, 481, 484, 548, 478, 518, 525, 538, 495, 506, 540, 488, 530, 503, 524, 528, 480, 519, 464, 531, 500, 503, 519, 531, 516, 511, 477, 476, 498, 523, 507, 508, 508, 512, 518, 539, 492, 487, 534, 511, 537, 521, 501, 539, 547, 489, 512, 477, 521, 505, 515, 539, 562, 478, 513, 519, 501, 481, 518, 467, 509, 529, 510, 504, 519, 544, 527, 469, 524, 499, 510, 541, 529, 502, 589, 488, 475, 493, 530, 528, 483, 508, 514, 478, 521, 516, 488, 489, 473, 512, 517, 524, 497, 540, 505, 514, 523, 512, 533, 522, 504, 537, 513, 488, 546, 519, 498, 500, 552, 509, 513, 526, 499, 507, 563, 517] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[500, 467, 517, 489, 503, 534, 522, 497, 512, 493, 530, 489, 524, 465, 562, 535, 552, 517, 480, 492, 488, 509, 520, 535, 502, 546, 524, 529, 537, 550, 528, 503, 513, 522, 477, 501, 541, 555, 515, 455, 507, 481, 512, 541, 529, 486, 560, 550, 486, 525, 500, 503, 474, 533, 561, 518, 494, 502, 535, 501, 542, 518, 508, 512, 510, 487, 495, 530, 493, 500, 522, 534, 481, 543, 532, 542, 538, 496, 492, 544, 516, 482, 502, 497, 508, 517, 500, 490, 493, 518, 476, 518, 497, 482, 510, 526, 555, 485, 504, 502, 499, 509, 507, 479, 533, 490, 519, 512, 529, 483, 533, 531, 530, 528, 512, 504, 512, 507, 476, 502, 502, 513, 525, 522, 513, 529, 494, 518] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[494, 464, 509, 533, 538, 500, 492, 503, 530, 529, 512, 568, 517, 484, 506, 503, 523, 518, 484, 493, 516, 519, 489, 517, 484, 541, 504, 534, 466, 525, 569, 491, 534, 555, 502, 525, 517, 518, 520, 537, 512, 527, 509, 537, 504, 483, 527, 511, 533, 538, 512, 472, 510, 513, 525, 491, 514, 479, 510, 537, 497, 516, 475, 508, 514, 500, 507, 516, 473, 513, 490, 516, 523, 532, 506, 517, 553, 482, 492, 528, 544, 509, 521, 516, 482, 504, 540, 564, 487, 526, 545, 527, 492, 530, 531, 518, 454, 511, 481, 472, 491, 501, 500, 468, 554, 482, 480, 501, 561, 513, 534, 495, 501, 503, 518, 533, 525, 497, 532, 569, 525, 503, 497, 488, 517, 537, 475, 517] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[532, 472, 529, 506, 506, 502, 527, 532, 546, 504, 526, 493, 514, 534, 524, 489, 540, 495, 531, 473, 519, 462, 523, 505, 519, 489, 536, 529, 509, 526, 542, 477, 536, 503, 521, 484, 552, 494, 504, 514, 483, 497, 531, 504, 519, 531, 502, 520, 498, 539, 516, 497, 512, 501, 519, 524, 501, 483, 522, 501, 491, 486, 517, 495, 526, 517, 514, 522, 540, 532, 508, 532, 517, 554, 503, 526, 503, 478, 521, 541, 527, 504, 488, 545, 473, 515, 493, 535, 490, 490, 535, 523, 513, 491, 516, 491, 540, 515, 484, 522, 555, 529, 481, 498, 523, 488, 539, 528, 521, 513, 497, 479, 502, 511, 551, 536, 487, 496, 532, 517, 514, 467, 495, 500, 521, 480, 520, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[547, 561, 479, 501, 514, 493, 555, 514, 519, 489, 506, 510, 497, 510, 491, 540, 529, 486, 517, 494, 543, 463, 497, 544, 497, 491, 526, 522, 500, 505, 544, 502, 504, 538, 482, 481, 542, 518, 498, 510, 495, 520, 487, 534, 569, 520, 543, 501, 526, 519, 510, 500, 495, 507, 518, 492, 537, 458, 523, 522, 530, 512, 491, 530, 474, 496, 534, 508, 509, 562, 531, 512, 540, 508, 513, 498, 483, 493, 535, 553, 493, 505, 539, 540, 538, 533, 495, 508, 515, 505, 528, 490, 496, 517, 536, 499, 512, 503, 482, 483, 532, 499, 547, 567, 505, 462, 502, 542, 491, 501, 493, 497, 525, 526, 495, 494, 484, 515, 497, 501, 471, 519, 515, 539, 506, 530, 537, 480] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[473, 498, 483, 518, 506, 509, 534, 532, 487, 539, 526, 520, 508, 538, 511, 497, 494, 446, 523, 508, 528, 473, 503, 480, 503, 529, 505, 555, 527, 503, 513, 539, 526, 522, 530, 480, 558, 526, 510, 518, 544, 551, 522, 539, 554, 515, 514, 536, 449, 521, 476, 517, 501, 512, 468, 533, 533, 467, 520, 498, 500, 515, 507, 491, 502, 511, 549, 516, 514, 486, 525, 518, 526, 530, 522, 513, 523, 499, 503, 512, 498, 549, 516, 453, 509, 510, 487, 514, 485, 497, 538, 518, 539, 496, 519, 495, 485, 489, 533, 503, 520, 512, 520, 538, 502, 523, 508, 522, 516, 501, 488, 514, 473, 497, 521, 514, 566, 512, 539, 504, 514, 506, 509, 517, 508, 513, 515, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[507, 535, 534, 494, 526, 480, 505, 507, 513, 490, 467, 529, 510, 495, 509, 548, 461, 490, 531, 509, 533, 535, 501, 518, 516, 477, 541, 516, 520, 534, 505, 519, 538, 509, 512, 473, 514, 494, 482, 466, 557, 523, 535, 464, 539, 536, 501, 486, 519, 530, 517, 460, 496, 544, 512, 529, 534, 483, 493, 480, 522, 511, 509, 540, 513, 529, 503, 536, 483, 509, 484, 516, 484, 503, 559, 539, 513, 519, 506, 504, 483, 469, 522, 472, 514, 543, 496, 497, 533, 508, 516, 487, 552, 506, 501, 507, 561, 489, 509, 534, 500, 533, 528, 521, 527, 522, 503, 540, 506, 483, 554, 498, 549, 518, 533, 488, 556, 474, 510, 493, 499, 521, 522, 490, 519, 528, 523, 536] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[506, 484, 511, 472, 493, 545, 499, 527, 521, 500, 483, 556, 525, 501, 507, 536, 542, 525, 534, 525, 527, 498, 510, 502, 492, 516, 530, 510, 465, 483, 487, 498, 523, 479, 508, 518, 516, 529, 505, 509, 489, 500, 534, 540, 513, 483, 533, 496, 540, 499, 513, 552, 527, 504, 528, 490, 540, 543, 519, 528, 547, 494, 521, 490, 492, 531, 487, 528, 500, 499, 502, 488, 538, 541, 528, 491, 474, 477, 483, 540, 495, 510, 523, 514, 492, 500, 515, 492, 529, 535, 554, 565, 535, 489, 521, 522, 486, 521, 500, 481, 454, 490, 553, 543, 507, 554, 517, 465, 536, 478, 460, 517, 537, 485, 511, 497, 547, 490, 503, 542, 519, 505, 516, 551, 525, 525, 494, 517] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[473, 498, 483, 518, 506, 509, 534, 532, 487, 539, 526, 520, 508, 538, 511, 497, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 446, 523, 508, 528, 473, 503, 480, 503, 529, 505, 555, 527, 503, 513, 539, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 522, 530, 480, 512, 526, 510, 518, 544, 551, 522, 539, 554, 515, 514, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 449, 521, 476, 517, 501, 512, 468, 533, 533, 467, 520, 498, 500, 515, 507, 491, 16, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 511, 549, 516, 514, 486, 525, 518, 526, 513, 522, 513, 523, 499, 503, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 549, 516, 453, 509, 510, 487, 514, 485, 497, 538, 518, 539, 496, 519, 495, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 489, 533, 503, 520, 512, 520, 538, 502, 523, 508, 522, 516, 501, 488, 514, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 497, 521, 514, 538, 512, 539, 504, 514, 506, 509, 517, 508, 513, 515, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8137, 8399, 8030, 8232, 8135, 8214, 8201] avg=8192.0000 max=8399 max_rank=2 min=8030 min_rank=3 imbalance_ratio=1.025269 max_load_reduction=57137 max_load_reduction_pct=0.871841 moved_to_echo=32782 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[500, 467, 517, 489, 503, 534, 522, 497, 512, 493, 530, 489, 524, 465, 562, 535, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 517, 480, 492, 488, 509, 520, 535, 502, 546, 524, 529, 537, 550, 528, 503, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 522, 477, 501, 496, 555, 515, 455, 507, 481, 512, 541, 529, 486, 560, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 525, 500, 503, 474, 533, 561, 518, 494, 502, 535, 501, 542, 518, 508, 512, 14, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 487, 495, 530, 493, 500, 522, 534, 481, 526, 532, 542, 538, 496, 492, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 482, 502, 497, 508, 517, 500, 490, 493, 518, 476, 518, 497, 482, 510, 526, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 485, 504, 502, 499, 509, 507, 479, 533, 490, 519, 512, 529, 483, 533, 531, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 528, 512, 504, 487, 507, 476, 502, 502, 513, 525, 522, 513, 529, 494, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8148, 8320, 8200, 8232, 8222, 8043, 8209, 8162] avg=8192.0000 max=8320 max_rank=1 min=8043 min_rank=5 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32636 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[492, 513, 547, 511, 525, 519, 499, 511, 486, 527, 506, 511, 518, 481, 484, 548, 13, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 518, 525, 538, 495, 506, 540, 488, 530, 503, 524, 528, 480, 519, 464, 531, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 503, 519, 531, 467, 511, 477, 476, 498, 523, 507, 508, 508, 512, 518, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 487, 534, 511, 537, 521, 501, 539, 547, 489, 512, 477, 521, 505, 515, 539, 19, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 562, 478, 513, 519, 501, 481, 518, 467, 509, 503, 510, 504, 519, 544, 527, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 499, 510, 541, 529, 502, 589, 488, 475, 493, 530, 528, 483, 508, 514, 478, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 516, 488, 489, 473, 512, 517, 524, 497, 540, 505, 514, 523, 512, 533, 522, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 537, 513, 488, 507, 519, 498, 500, 552, 509, 513, 526, 499, 507, 563, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8180, 8097, 8255, 8124, 8206, 8226, 8252] avg=8192.0000 max=8255 max_rank=3 min=8097 min_rank=2 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32808 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[507, 535, 534, 494, 526, 480, 505, 507, 513, 490, 467, 529, 510, 495, 509, 548, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 490, 531, 509, 533, 535, 501, 518, 516, 477, 541, 516, 520, 534, 505, 519, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 509, 512, 473, 471, 494, 482, 466, 557, 523, 535, 464, 539, 536, 501, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 530, 517, 460, 496, 544, 512, 529, 534, 483, 493, 480, 522, 511, 509, 540, 16, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 529, 503, 536, 483, 509, 484, 516, 484, 486, 559, 539, 513, 519, 506, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 469, 522, 472, 514, 543, 496, 497, 533, 508, 516, 487, 552, 506, 501, 507, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 489, 509, 534, 500, 533, 528, 521, 527, 522, 503, 540, 506, 483, 554, 498, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 518, 533, 488, 529, 474, 510, 493, 499, 521, 522, 490, 519, 528, 523, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8158, 8214, 8086, 8201, 8183, 8117, 8345, 8232] avg=8192.0000 max=8345 max_rank=6 min=8086 min_rank=2 imbalance_ratio=1.018677 max_load_reduction=57191 max_load_reduction_pct=0.872665 moved_to_echo=32877 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[547, 561, 479, 501, 514, 493, 555, 514, 519, 489, 506, 510, 497, 510, 491, 540, 529, 486, 517, 494, 543, 463, 497, 544, 497, 491, 526, 522, 500, 505, 544, 502, 504, 538, 482, 481, 542, 518, 498, 510, 495, 520, 487, 534, 569, 520, 543, 501, 526, 519, 510, 500, 495, 507, 518, 492, 537, 458, 523, 522, 530, 512, 491, 530, 474, 496, 534, 508, 509, 562, 531, 512, 540, 508, 513, 498, 483, 493, 535, 553, 493, 505, 539, 540, 538, 533, 495, 508, 515, 505, 528, 490, 496, 517, 536, 499, 512, 503, 482, 483, 532, 499, 547, 567, 505, 462, 502, 542, 491, 501, 493, 497, 525, 526, 495, 494, 484, 515, 497, 501, 471, 519, 515, 539, 506, 530, 537, 480] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[494, 464, 509, 533, 538, 500, 492, 503, 530, 529, 512, 568, 517, 484, 506, 503, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 518, 484, 493, 516, 519, 489, 517, 484, 541, 504, 534, 466, 525, 569, 491, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 555, 502, 525, 474, 518, 520, 537, 512, 527, 509, 537, 504, 483, 527, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 538, 512, 472, 510, 513, 525, 491, 514, 479, 510, 537, 497, 516, 475, 508, 15, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 500, 507, 516, 473, 513, 490, 516, 523, 515, 506, 517, 553, 482, 492, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 509, 521, 516, 482, 504, 540, 564, 487, 526, 545, 527, 492, 530, 531, 518, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 454, 511, 481, 472, 491, 501, 500, 468, 554, 482, 480, 501, 561, 513, 534, 495, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 503, 518, 533, 499, 497, 532, 569, 525, 503, 497, 488, 517, 537, 475, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8181, 8275, 8151, 8145, 8347, 8035, 8211] avg=8192.0000 max=8347 max_rank=5 min=8035 min_rank=6 imbalance_ratio=1.018921 max_load_reduction=57189 max_load_reduction_pct=0.872635 moved_to_echo=32738 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[492, 513, 547, 511, 525, 519, 499, 511, 486, 527, 506, 511, 518, 481, 484, 548, 13, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 518, 525, 538, 495, 506, 540, 488, 530, 503, 524, 528, 480, 519, 464, 531, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 503, 519, 531, 467, 511, 477, 476, 498, 523, 507, 508, 508, 512, 518, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 487, 534, 511, 537, 521, 501, 539, 547, 489, 512, 477, 521, 505, 515, 539, 19, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 562, 478, 513, 519, 501, 481, 518, 467, 509, 503, 510, 504, 519, 544, 527, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 499, 510, 541, 529, 502, 589, 488, 475, 493, 530, 528, 483, 508, 514, 478, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 516, 488, 489, 473, 512, 517, 524, 497, 540, 505, 514, 523, 512, 533, 522, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 537, 513, 488, 507, 519, 498, 500, 552, 509, 513, 526, 499, 507, 563, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8180, 8097, 8255, 8124, 8206, 8226, 8252] avg=8192.0000 max=8255 max_rank=3 min=8097 min_rank=2 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32808 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[532, 472, 529, 506, 506, 502, 527, 532, 546, 504, 526, 493, 514, 534, 524, 489, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 495, 531, 473, 519, 462, 523, 505, 519, 489, 536, 529, 509, 526, 542, 477, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 503, 521, 484, 507, 494, 504, 514, 483, 497, 531, 504, 519, 531, 502, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 539, 516, 497, 512, 501, 519, 524, 501, 483, 522, 501, 491, 486, 517, 495, 14, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 517, 514, 522, 540, 532, 508, 532, 517, 536, 503, 526, 503, 478, 521, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 504, 488, 545, 473, 515, 493, 535, 490, 490, 535, 523, 513, 491, 516, 491, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 515, 484, 522, 555, 529, 481, 498, 523, 488, 539, 528, 521, 513, 497, 479, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 511, 551, 536, 463, 496, 532, 517, 514, 467, 495, 500, 521, 480, 520, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 8184, 8150, 8122, 8316, 8139, 8251, 8129] avg=8192.0000 max=8316 max_rank=4 min=8122 min_rank=3 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32835 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[547, 561, 479, 501, 514, 493, 555, 514, 519, 489, 506, 510, 497, 510, 491, 540, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 486, 517, 494, 543, 463, 497, 544, 497, 491, 526, 522, 500, 505, 544, 502, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 538, 482, 481, 497, 518, 498, 510, 495, 520, 487, 534, 569, 520, 543, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 519, 510, 500, 495, 507, 518, 492, 537, 458, 523, 522, 530, 512, 491, 530, 13, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 496, 534, 508, 509, 562, 531, 512, 540, 491, 513, 498, 483, 493, 535, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 505, 539, 540, 538, 533, 495, 508, 515, 505, 528, 490, 496, 517, 536, 499, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 503, 482, 483, 532, 499, 547, 567, 505, 462, 502, 542, 491, 501, 493, 497, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 526, 495, 494, 461, 515, 497, 501, 471, 519, 515, 539, 506, 530, 537, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8168, 8197, 8189, 8232, 8247, 8157, 8111] avg=8192.0000 max=8247 max_rank=5 min=8111 min_rank=7 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32747 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[506, 484, 511, 472, 493, 545, 499, 527, 521, 500, 483, 556, 525, 501, 507, 536, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 525, 534, 525, 527, 498, 510, 502, 492, 516, 530, 510, 465, 483, 487, 498, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 479, 508, 518, 473, 529, 505, 509, 489, 500, 534, 540, 513, 483, 533, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 499, 513, 552, 527, 504, 528, 490, 540, 543, 519, 528, 547, 494, 521, 490, 15, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 531, 487, 528, 500, 499, 502, 488, 538, 524, 528, 491, 474, 477, 483, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 510, 523, 514, 492, 500, 515, 492, 529, 535, 554, 565, 535, 489, 521, 522, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 521, 500, 481, 454, 490, 553, 543, 507, 554, 517, 465, 536, 478, 460, 517, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 485, 511, 497, 521, 490, 503, 542, 519, 505, 516, 551, 525, 525, 494, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8152, 8132, 8356, 8082, 8302, 8099, 8238] avg=8192.0000 max=8356 max_rank=3 min=8082 min_rank=4 imbalance_ratio=1.020020 max_load_reduction=57180 max_load_reduction_pct=0.872498 moved_to_echo=32721 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[500, 467, 517, 489, 503, 534, 522, 497, 512, 493, 530, 489, 524, 465, 562, 535, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 517, 480, 492, 488, 509, 520, 535, 502, 546, 524, 529, 537, 550, 528, 503, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 522, 477, 501, 496, 555, 515, 455, 507, 481, 512, 541, 529, 486, 560, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 525, 500, 503, 474, 533, 561, 518, 494, 502, 535, 501, 542, 518, 508, 512, 14, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 487, 495, 530, 493, 500, 522, 534, 481, 526, 532, 542, 538, 496, 492, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 482, 502, 497, 508, 517, 500, 490, 493, 518, 476, 518, 497, 482, 510, 526, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 485, 504, 502, 499, 509, 507, 479, 533, 490, 519, 512, 529, 483, 533, 531, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 528, 512, 504, 487, 507, 476, 502, 502, 513, 525, 522, 513, 529, 494, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8148, 8320, 8200, 8232, 8222, 8043, 8209, 8162] avg=8192.0000 max=8320 max_rank=1 min=8043 min_rank=5 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32636 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[507, 535, 534, 494, 526, 480, 505, 507, 513, 490, 467, 529, 510, 495, 509, 548, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 490, 531, 509, 533, 535, 501, 518, 516, 477, 541, 516, 520, 534, 505, 519, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 509, 512, 473, 471, 494, 482, 466, 557, 523, 535, 464, 539, 536, 501, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 530, 517, 460, 496, 544, 512, 529, 534, 483, 493, 480, 522, 511, 509, 540, 16, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 529, 503, 536, 483, 509, 484, 516, 484, 486, 559, 539, 513, 519, 506, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 469, 522, 472, 514, 543, 496, 497, 533, 508, 516, 487, 552, 506, 501, 507, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 489, 509, 534, 500, 533, 528, 521, 527, 522, 503, 540, 506, 483, 554, 498, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 518, 533, 488, 529, 474, 510, 493, 499, 521, 522, 490, 519, 528, 523, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8158, 8214, 8086, 8201, 8183, 8117, 8345, 8232] avg=8192.0000 max=8345 max_rank=6 min=8086 min_rank=2 imbalance_ratio=1.018677 max_load_reduction=57191 max_load_reduction_pct=0.872665 moved_to_echo=32877 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[547, 561, 479, 501, 514, 493, 555, 514, 519, 489, 506, 510, 497, 510, 491, 540, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 486, 517, 494, 543, 463, 497, 544, 497, 491, 526, 522, 500, 505, 544, 502, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 538, 482, 481, 497, 518, 498, 510, 495, 520, 487, 534, 569, 520, 543, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 519, 510, 500, 495, 507, 518, 492, 537, 458, 523, 522, 530, 512, 491, 530, 13, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 496, 534, 508, 509, 562, 531, 512, 540, 491, 513, 498, 483, 493, 535, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 505, 539, 540, 538, 533, 495, 508, 515, 505, 528, 490, 496, 517, 536, 499, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 503, 482, 483, 532, 499, 547, 567, 505, 462, 502, 542, 491, 501, 493, 497, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 526, 495, 494, 461, 515, 497, 501, 471, 519, 515, 539, 506, 530, 537, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8168, 8197, 8189, 8232, 8247, 8157, 8111] avg=8192.0000 max=8247 max_rank=5 min=8111 min_rank=7 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32747 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[473, 498, 483, 518, 506, 509, 534, 532, 487, 539, 526, 520, 508, 538, 511, 497, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 446, 523, 508, 528, 473, 503, 480, 503, 529, 505, 555, 527, 503, 513, 539, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 522, 530, 480, 512, 526, 510, 518, 544, 551, 522, 539, 554, 515, 514, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 449, 521, 476, 517, 501, 512, 468, 533, 533, 467, 520, 498, 500, 515, 507, 491, 16, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 511, 549, 516, 514, 486, 525, 518, 526, 513, 522, 513, 523, 499, 503, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 549, 516, 453, 509, 510, 487, 514, 485, 497, 538, 518, 539, 496, 519, 495, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 489, 533, 503, 520, 512, 520, 538, 502, 523, 508, 522, 516, 501, 488, 514, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 497, 521, 514, 538, 512, 539, 504, 514, 506, 509, 517, 508, 513, 515, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8137, 8399, 8030, 8232, 8135, 8214, 8201] avg=8192.0000 max=8399 max_rank=2 min=8030 min_rank=3 imbalance_ratio=1.025269 max_load_reduction=57137 max_load_reduction_pct=0.871841 moved_to_echo=32782 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. @@ -1269,10 +1269,10 @@ WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] glob warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. @@ -1283,30 +1283,30 @@ WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] glob warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 3968, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 3968, 4224, 4096, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4096, 3968, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4352, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 3968, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4096, 3968, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4352, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 3968, 4224, 4096, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. @@ -1323,30 +1323,30 @@ WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] glob warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[521, 509, 523, 501, 487, 485, 538, 523, 530, 486, 477, 506, 484, 575, 509, 508, 510, 528, 523, 498, 540, 510, 495, 518, 519, 508, 503, 473, 505, 524, 529, 532, 556, 528, 522, 538, 531, 517, 509, 535, 513, 528, 536, 515, 514, 523, 461, 507, 475, 517, 517, 490, 498, 527, 506, 516, 522, 529, 520, 519, 541, 504, 506, 498, 472, 508, 500, 541, 530, 515, 532, 537, 528, 513, 482, 517, 548, 522, 536, 558, 503, 491, 497, 512, 502, 479, 524, 489, 510, 491, 520, 484, 494, 521, 518, 522, 558, 526, 539, 503, 536, 503, 517, 478, 479, 480, 517, 490, 485, 493, 535, 521, 520, 506, 504, 517, 544, 509, 469, 500, 505, 482, 524, 485, 487, 529, 518, 486] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[507, 517, 525, 523, 531, 511, 533, 495, 531, 509, 536, 493, 557, 522, 519, 547, 509, 527, 492, 507, 505, 508, 465, 512, 501, 502, 543, 477, 507, 518, 496, 515, 482, 483, 504, 477, 526, 515, 490, 499, 547, 499, 488, 505, 562, 520, 523, 499, 517, 511, 499, 526, 535, 526, 515, 507, 539, 547, 500, 476, 559, 492, 520, 489, 516, 530, 555, 505, 504, 470, 558, 486, 508, 515, 555, 482, 520, 507, 512, 508, 529, 510, 516, 478, 504, 496, 503, 533, 505, 534, 503, 511, 506, 518, 507, 507, 505, 530, 521, 512, 512, 496, 518, 547, 500, 503, 459, 523, 524, 513, 562, 470, 518, 497, 507, 526, 530, 504, 522, 499, 497, 512, 480, 460, 498, 554, 552, 477] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[571, 517, 539, 519, 506, 474, 503, 518, 539, 527, 535, 516, 517, 525, 500, 463, 469, 488, 502, 494, 571, 495, 477, 502, 516, 521, 475, 509, 560, 501, 518, 533, 529, 525, 512, 524, 501, 530, 491, 534, 487, 524, 503, 527, 513, 529, 492, 547, 505, 520, 489, 529, 505, 500, 511, 532, 523, 510, 550, 511, 520, 514, 518, 477, 524, 503, 520, 547, 511, 493, 518, 497, 542, 510, 490, 498, 491, 500, 483, 503, 483, 525, 523, 491, 475, 512, 528, 529, 481, 531, 511, 526, 467, 546, 550, 505, 535, 501, 523, 510, 480, 522, 517, 557, 519, 508, 491, 516, 529, 515, 478, 517, 517, 519, 529, 512, 478, 523, 516, 461, 486, 488, 511, 524, 565, 490, 493, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[536, 543, 547, 541, 479, 550, 512, 528, 511, 468, 504, 526, 524, 521, 515, 510, 469, 550, 511, 561, 528, 499, 529, 562, 484, 491, 525, 512, 515, 512, 472, 530, 492, 540, 484, 489, 481, 515, 502, 511, 527, 538, 501, 533, 513, 509, 489, 518, 543, 508, 514, 512, 510, 489, 511, 451, 531, 514, 514, 542, 495, 532, 497, 520, 491, 521, 513, 505, 524, 506, 503, 512, 515, 567, 521, 471, 490, 520, 489, 538, 495, 505, 520, 496, 522, 499, 487, 523, 537, 526, 520, 495, 542, 506, 520, 551, 490, 521, 514, 485, 519, 505, 488, 520, 493, 517, 513, 480, 509, 465, 489, 516, 490, 505, 510, 501, 538, 492, 527, 545, 510, 543, 514, 517, 511, 451, 525, 513] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[483, 491, 551, 490, 497, 521, 523, 534, 501, 524, 500, 480, 538, 478, 518, 537, 538, 534, 520, 539, 524, 498, 504, 549, 525, 492, 531, 531, 526, 527, 455, 532, 505, 511, 530, 481, 489, 564, 531, 473, 515, 508, 505, 558, 506, 514, 534, 465, 554, 508, 500, 541, 471, 524, 511, 493, 508, 518, 498, 526, 521, 515, 512, 527, 506, 526, 520, 521, 514, 479, 520, 517, 520, 505, 536, 499, 494, 514, 509, 502, 535, 508, 493, 487, 521, 501, 519, 502, 477, 540, 488, 476, 566, 495, 505, 527, 524, 508, 493, 514, 483, 510, 493, 512, 515, 518, 526, 529, 531, 493, 516, 525, 476, 531, 514, 511, 566, 506, 534, 513, 470, 501, 522, 506, 493, 457, 509, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[494, 480, 506, 487, 504, 542, 511, 536, 512, 523, 487, 497, 543, 471, 487, 515, 505, 560, 505, 530, 517, 508, 539, 523, 489, 485, 492, 482, 571, 534, 532, 509, 522, 523, 479, 526, 493, 492, 535, 472, 510, 522, 501, 545, 520, 469, 525, 543, 538, 522, 507, 486, 517, 508, 510, 503, 482, 509, 498, 547, 580, 513, 495, 525, 509, 525, 482, 511, 518, 530, 521, 479, 497, 519, 536, 480, 497, 475, 535, 475, 528, 508, 510, 501, 506, 499, 497, 509, 533, 504, 532, 522, 517, 566, 498, 537, 483, 518, 495, 552, 532, 481, 520, 529, 509, 506, 539, 529, 526, 516, 490, 514, 500, 502, 525, 534, 529, 507, 508, 491, 501, 492, 524, 549, 495, 510, 451, 530] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[540, 532, 508, 492, 543, 482, 546, 512, 520, 492, 533, 500, 505, 524, 534, 525, 541, 490, 500, 475, 506, 526, 512, 467, 509, 480, 499, 516, 530, 527, 512, 528, 522, 509, 509, 500, 521, 504, 540, 518, 511, 536, 528, 517, 496, 517, 459, 482, 536, 559, 521, 536, 495, 518, 498, 553, 489, 462, 519, 538, 513, 529, 481, 537, 508, 531, 527, 533, 522, 461, 509, 492, 535, 566, 489, 473, 520, 493, 497, 517, 531, 548, 513, 502, 515, 503, 507, 527, 522, 544, 510, 474, 527, 529, 506, 500, 518, 515, 493, 497, 489, 540, 514, 517, 474, 507, 503, 497, 530, 522, 509, 526, 533, 493, 487, 492, 494, 545, 536, 501, 498, 489, 507, 512, 485, 496, 502, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[521, 509, 523, 501, 487, 485, 538, 523, 530, 486, 477, 506, 484, 575, 509, 508, 510, 528, 523, 498, 540, 510, 495, 518, 519, 508, 503, 473, 505, 524, 529, 532, 556, 528, 522, 538, 531, 517, 509, 535, 513, 528, 536, 515, 514, 523, 461, 507, 475, 517, 517, 490, 498, 527, 506, 516, 522, 529, 520, 519, 541, 504, 506, 498, 472, 508, 500, 541, 530, 515, 532, 537, 528, 513, 482, 517, 548, 522, 536, 558, 503, 491, 497, 512, 502, 479, 524, 489, 510, 491, 520, 484, 494, 521, 518, 522, 558, 526, 539, 503, 536, 503, 517, 478, 479, 480, 517, 490, 485, 493, 535, 521, 520, 506, 504, 517, 544, 509, 469, 500, 505, 482, 524, 485, 487, 529, 518, 486] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[514, 509, 507, 506, 503, 502, 491, 518, 502, 501, 506, 500, 505, 549, 499, 549, 498, 530, 505, 493, 513, 543, 477, 500, 491, 527, 520, 544, 523, 509, 484, 519, 461, 498, 554, 502, 515, 497, 490, 536, 520, 501, 502, 507, 494, 504, 541, 506, 544, 498, 488, 506, 512, 503, 504, 500, 515, 533, 509, 537, 537, 546, 482, 523, 506, 514, 543, 501, 485, 529, 495, 498, 510, 507, 517, 532, 533, 545, 500, 500, 508, 498, 462, 534, 558, 515, 466, 546, 544, 479, 519, 555, 492, 506, 493, 507, 517, 532, 547, 522, 521, 523, 483, 516, 489, 486, 535, 512, 534, 530, 491, 501, 521, 483, 545, 515, 511, 518, 513, 505, 469, 543, 519, 514, 510, 510, 505, 517] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[483, 491, 551, 490, 497, 521, 523, 534, 501, 524, 500, 480, 538, 478, 518, 537, 538, 534, 520, 539, 524, 498, 504, 549, 525, 492, 531, 531, 526, 527, 455, 532, 505, 511, 530, 481, 489, 564, 531, 473, 515, 508, 505, 558, 506, 514, 534, 465, 554, 508, 500, 541, 471, 524, 511, 493, 508, 518, 498, 526, 521, 515, 512, 527, 506, 526, 520, 521, 514, 479, 520, 517, 520, 505, 536, 499, 494, 514, 509, 502, 535, 508, 493, 487, 521, 501, 519, 502, 477, 540, 488, 476, 566, 495, 505, 527, 524, 508, 493, 514, 483, 510, 493, 512, 515, 518, 526, 529, 531, 493, 516, 525, 476, 531, 514, 511, 566, 506, 534, 513, 470, 501, 522, 506, 493, 457, 509, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[571, 517, 539, 519, 506, 474, 503, 518, 539, 527, 535, 516, 517, 525, 500, 463, 469, 488, 502, 494, 571, 495, 477, 502, 516, 521, 475, 509, 560, 501, 518, 533, 529, 525, 512, 524, 501, 530, 491, 534, 487, 524, 503, 527, 513, 529, 492, 547, 505, 520, 489, 529, 505, 500, 511, 532, 523, 510, 550, 511, 520, 514, 518, 477, 524, 503, 520, 547, 511, 493, 518, 497, 542, 510, 490, 498, 491, 500, 483, 503, 483, 525, 523, 491, 475, 512, 528, 529, 481, 531, 511, 526, 467, 546, 550, 505, 535, 501, 523, 510, 480, 522, 517, 557, 519, 508, 491, 516, 529, 515, 478, 517, 517, 519, 529, 512, 478, 523, 516, 461, 486, 488, 511, 524, 565, 490, 493, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[507, 517, 492, 523, 531, 511, 533, 495, 531, 509, 536, 493, 557, 522, 519, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 527, 492, 507, 505, 508, 465, 512, 501, 502, 543, 477, 502, 518, 496, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 483, 504, 477, 526, 515, 490, 499, 547, 499, 488, 505, 562, 520, 523, 499, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 511, 499, 526, 535, 526, 515, 507, 539, 547, 500, 476, 521, 492, 520, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 530, 555, 505, 504, 470, 558, 486, 508, 514, 555, 482, 520, 507, 512, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 510, 516, 478, 504, 496, 503, 533, 505, 534, 503, 511, 506, 518, 507, 507, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 530, 521, 512, 512, 496, 518, 547, 500, 503, 459, 523, 524, 513, 562, 470, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 497, 507, 526, 530, 504, 522, 499, 497, 512, 480, 460, 498, 554, 552, 477, 38, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8323, 8079, 8120, 8220, 8230, 8165, 8209, 8190] avg=8192.0000 max=8323 max_rank=0 min=8079 min_rank=1 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32794 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[540, 532, 465, 492, 543, 482, 546, 512, 520, 492, 533, 500, 505, 524, 534, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 490, 500, 475, 506, 526, 512, 467, 509, 480, 499, 516, 521, 527, 512, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 509, 509, 500, 521, 504, 540, 518, 511, 536, 528, 517, 496, 517, 459, 482, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 559, 521, 536, 495, 518, 498, 553, 489, 462, 519, 538, 474, 529, 481, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 531, 527, 533, 522, 461, 509, 492, 535, 564, 489, 473, 520, 493, 497, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 548, 513, 502, 515, 503, 507, 527, 522, 544, 510, 474, 527, 529, 506, 500, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 515, 493, 497, 489, 540, 514, 517, 474, 507, 503, 497, 530, 522, 509, 526, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 493, 487, 492, 494, 545, 536, 501, 498, 489, 507, 512, 485, 496, 502, 525, 39, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 8109, 8173, 8245, 8171, 8268, 8170, 8155] avg=8192.0000 max=8268 max_rank=5 min=8109 min_rank=1 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32764 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[494, 480, 474, 487, 504, 542, 511, 536, 512, 523, 487, 497, 543, 471, 487, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 560, 505, 530, 517, 508, 539, 523, 489, 485, 492, 482, 566, 534, 532, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 523, 479, 526, 493, 492, 535, 472, 510, 522, 501, 545, 520, 469, 525, 543, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 522, 507, 486, 517, 508, 510, 503, 482, 509, 498, 547, 541, 513, 495, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 525, 482, 511, 518, 530, 521, 479, 497, 518, 536, 480, 497, 475, 535, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 508, 510, 501, 506, 499, 497, 509, 533, 504, 532, 522, 517, 566, 498, 537, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 518, 495, 552, 532, 481, 520, 529, 509, 506, 539, 529, 526, 516, 490, 514, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 502, 525, 534, 529, 507, 508, 491, 501, 492, 524, 549, 495, 510, 451, 530, 39, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8063, 8276, 8178, 8201, 8088, 8272, 8253, 8205] avg=8192.0000 max=8276 max_rank=1 min=8063 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32818 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[536, 543, 513, 541, 479, 550, 512, 528, 511, 468, 504, 526, 524, 521, 515, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 550, 511, 561, 528, 499, 529, 562, 484, 491, 525, 512, 510, 512, 472, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 540, 484, 489, 481, 515, 502, 511, 527, 538, 501, 533, 513, 509, 489, 518, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 508, 514, 512, 510, 489, 511, 451, 531, 514, 514, 542, 462, 532, 497, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 521, 513, 505, 524, 506, 503, 512, 515, 566, 521, 471, 490, 520, 489, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 505, 520, 496, 522, 499, 487, 523, 537, 526, 520, 495, 542, 506, 520, 551, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 521, 514, 485, 519, 505, 488, 520, 493, 517, 513, 480, 509, 465, 489, 516, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 505, 510, 501, 538, 492, 527, 545, 510, 543, 514, 517, 511, 451, 525, 513, 33, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8281, 8245, 8143, 8150, 8185, 8249, 8039, 8244] avg=8192.0000 max=8281 max_rank=0 min=8039 min_rank=6 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32717 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[571, 517, 505, 519, 506, 474, 503, 518, 539, 527, 535, 516, 517, 525, 500, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 488, 502, 494, 571, 495, 477, 502, 516, 521, 475, 509, 555, 501, 518, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 525, 512, 524, 501, 530, 491, 534, 487, 524, 503, 527, 513, 529, 492, 547, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 520, 489, 529, 505, 500, 511, 532, 523, 510, 550, 511, 485, 514, 518, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 503, 520, 547, 511, 493, 518, 497, 542, 509, 490, 498, 491, 500, 483, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 525, 523, 491, 475, 512, 528, 529, 481, 531, 511, 526, 467, 546, 550, 505, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 501, 523, 510, 480, 522, 517, 557, 519, 508, 491, 516, 529, 515, 478, 517, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 519, 529, 512, 478, 523, 516, 461, 486, 488, 511, 524, 565, 490, 493, 511, 35, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8126, 8269, 8179, 8129, 8188, 8233, 8177] avg=8192.0000 max=8269 max_rank=2 min=8126 min_rank=1 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32727 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[514, 509, 475, 506, 503, 502, 491, 518, 502, 501, 506, 500, 505, 549, 499, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 530, 505, 493, 513, 543, 477, 500, 491, 527, 520, 544, 518, 509, 484, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 498, 554, 502, 515, 497, 490, 536, 520, 501, 502, 507, 494, 504, 541, 506, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 498, 488, 506, 512, 503, 504, 500, 515, 533, 509, 537, 501, 546, 482, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 514, 543, 501, 485, 529, 495, 498, 510, 506, 517, 532, 533, 545, 500, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 498, 462, 534, 558, 515, 466, 546, 544, 479, 519, 555, 492, 506, 493, 507, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 532, 547, 522, 521, 523, 483, 516, 489, 486, 535, 512, 534, 530, 491, 501, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 483, 545, 515, 511, 518, 513, 505, 469, 543, 519, 514, 510, 510, 505, 517, 36, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8129, 8171, 8129, 8201, 8214, 8187, 8253, 8252] avg=8192.0000 max=8253 max_rank=6 min=8129 min_rank=0 imbalance_ratio=1.007446 max_load_reduction=57283 max_load_reduction_pct=0.874069 moved_to_echo=32906 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[483, 491, 516, 490, 497, 521, 523, 534, 501, 524, 500, 480, 538, 478, 518, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 534, 520, 539, 524, 498, 504, 549, 525, 492, 531, 531, 521, 527, 455, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 511, 530, 481, 489, 564, 531, 473, 515, 508, 505, 558, 506, 514, 534, 465, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 508, 500, 541, 471, 524, 511, 493, 508, 518, 498, 526, 486, 515, 512, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 526, 520, 521, 514, 479, 520, 517, 520, 504, 536, 499, 494, 514, 509, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 508, 493, 487, 521, 501, 519, 502, 477, 540, 488, 476, 566, 495, 505, 527, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 508, 493, 514, 483, 510, 493, 512, 515, 518, 526, 529, 531, 493, 516, 525, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 531, 514, 511, 566, 506, 534, 513, 470, 501, 522, 506, 493, 457, 509, 508, 35, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8131, 8320, 8190, 8192, 8181, 8145, 8205, 8172] avg=8192.0000 max=8320 max_rank=1 min=8131 min_rank=0 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32703 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[494, 480, 474, 487, 504, 542, 511, 536, 512, 523, 487, 497, 543, 471, 487, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 560, 505, 530, 517, 508, 539, 523, 489, 485, 492, 482, 566, 534, 532, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 523, 479, 526, 493, 492, 535, 472, 510, 522, 501, 545, 520, 469, 525, 543, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 522, 507, 486, 517, 508, 510, 503, 482, 509, 498, 547, 541, 513, 495, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 525, 482, 511, 518, 530, 521, 479, 497, 518, 536, 480, 497, 475, 535, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 508, 510, 501, 506, 499, 497, 509, 533, 504, 532, 522, 517, 566, 498, 537, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 518, 495, 552, 532, 481, 520, 529, 509, 506, 539, 529, 526, 516, 490, 514, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 502, 525, 534, 529, 507, 508, 491, 501, 492, 524, 549, 495, 510, 451, 530, 39, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8063, 8276, 8178, 8201, 8088, 8272, 8253, 8205] avg=8192.0000 max=8276 max_rank=1 min=8063 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32818 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[521, 509, 490, 501, 487, 485, 538, 523, 530, 486, 477, 506, 484, 575, 509, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 528, 523, 498, 540, 510, 495, 518, 519, 508, 503, 473, 500, 524, 529, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 528, 522, 538, 531, 517, 509, 535, 513, 528, 536, 515, 514, 523, 461, 507, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 517, 517, 490, 498, 527, 506, 516, 522, 529, 520, 519, 504, 504, 506, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 508, 500, 541, 530, 515, 532, 537, 528, 512, 482, 517, 548, 522, 536, 558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 491, 497, 512, 502, 479, 524, 489, 510, 491, 520, 484, 494, 521, 518, 522, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 558, 526, 539, 503, 536, 503, 517, 478, 479, 480, 517, 490, 485, 493, 535, 521, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 506, 504, 517, 544, 509, 469, 500, 505, 482, 524, 485, 487, 529, 518, 486, 37, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8129, 8210, 8334, 8148, 8338, 8062, 8174, 8141] avg=8192.0000 max=8338 max_rank=4 min=8062 min_rank=5 imbalance_ratio=1.017822 max_load_reduction=57198 max_load_reduction_pct=0.872772 moved_to_echo=32715 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[483, 491, 516, 490, 497, 521, 523, 534, 501, 524, 500, 480, 538, 478, 518, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 534, 520, 539, 524, 498, 504, 549, 525, 492, 531, 531, 521, 527, 455, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 511, 530, 481, 489, 564, 531, 473, 515, 508, 505, 558, 506, 514, 534, 465, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 508, 500, 541, 471, 524, 511, 493, 508, 518, 498, 526, 486, 515, 512, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 526, 520, 521, 514, 479, 520, 517, 520, 504, 536, 499, 494, 514, 509, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 508, 493, 487, 521, 501, 519, 502, 477, 540, 488, 476, 566, 495, 505, 527, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 508, 493, 514, 483, 510, 493, 512, 515, 518, 526, 529, 531, 493, 516, 525, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 531, 514, 511, 566, 506, 534, 513, 470, 501, 522, 506, 493, 457, 509, 508, 35, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8131, 8320, 8190, 8192, 8181, 8145, 8205, 8172] avg=8192.0000 max=8320 max_rank=1 min=8131 min_rank=0 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32703 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[514, 509, 475, 506, 503, 502, 491, 518, 502, 501, 506, 500, 505, 549, 499, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 530, 505, 493, 513, 543, 477, 500, 491, 527, 520, 544, 518, 509, 484, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 498, 554, 502, 515, 497, 490, 536, 520, 501, 502, 507, 494, 504, 541, 506, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 498, 488, 506, 512, 503, 504, 500, 515, 533, 509, 537, 501, 546, 482, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 514, 543, 501, 485, 529, 495, 498, 510, 506, 517, 532, 533, 545, 500, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 498, 462, 534, 558, 515, 466, 546, 544, 479, 519, 555, 492, 506, 493, 507, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 532, 547, 522, 521, 523, 483, 516, 489, 486, 535, 512, 534, 530, 491, 501, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 483, 545, 515, 511, 518, 513, 505, 469, 543, 519, 514, 510, 510, 505, 517, 36, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8129, 8171, 8129, 8201, 8214, 8187, 8253, 8252] avg=8192.0000 max=8253 max_rank=6 min=8129 min_rank=0 imbalance_ratio=1.007446 max_load_reduction=57283 max_load_reduction_pct=0.874069 moved_to_echo=32906 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[536, 543, 513, 541, 479, 550, 512, 528, 511, 468, 504, 526, 524, 521, 515, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 550, 511, 561, 528, 499, 529, 562, 484, 491, 525, 512, 510, 512, 472, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 540, 484, 489, 481, 515, 502, 511, 527, 538, 501, 533, 513, 509, 489, 518, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 508, 514, 512, 510, 489, 511, 451, 531, 514, 514, 542, 462, 532, 497, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 521, 513, 505, 524, 506, 503, 512, 515, 566, 521, 471, 490, 520, 489, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 505, 520, 496, 522, 499, 487, 523, 537, 526, 520, 495, 542, 506, 520, 551, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 521, 514, 485, 519, 505, 488, 520, 493, 517, 513, 480, 509, 465, 489, 516, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 505, 510, 501, 538, 492, 527, 545, 510, 543, 514, 517, 511, 451, 525, 513, 33, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8281, 8245, 8143, 8150, 8185, 8249, 8039, 8244] avg=8192.0000 max=8281 max_rank=0 min=8039 min_rank=6 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32717 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. @@ -1371,429 +1371,429 @@ WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] glob warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 3968, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 384, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 3968, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 3968, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 3968, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 384, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[506, 511, 504, 525, 523, 482, 522, 515, 533, 536, 518, 530, 489, 507, 527, 532, 515, 523, 533, 542, 533, 498, 542, 535, 477, 486, 515, 509, 512, 472, 529, 529, 502, 511, 537, 524, 499, 503, 511, 499, 522, 511, 497, 498, 506, 535, 516, 539, 524, 548, 520, 536, 501, 476, 550, 477, 496, 507, 551, 474, 512, 492, 499, 530, 539, 527, 502, 528, 521, 508, 534, 519, 488, 482, 531, 509, 521, 492, 513, 492, 529, 529, 561, 515, 483, 489, 504, 487, 479, 489, 535, 498, 512, 521, 538, 503, 522, 495, 504, 507, 497, 520, 492, 494, 505, 514, 480, 524, 512, 501, 528, 485, 509, 509, 498, 558, 485, 475, 481, 525, 532, 499, 496, 499, 514, 505, 538, 542] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[515, 487, 509, 499, 489, 476, 482, 507, 524, 502, 511, 497, 531, 511, 490, 492, 530, 479, 535, 499, 463, 499, 514, 524, 521, 516, 488, 493, 542, 517, 480, 507, 505, 495, 542, 506, 537, 540, 517, 522, 518, 513, 494, 550, 496, 516, 553, 498, 549, 516, 479, 520, 525, 525, 505, 512, 502, 496, 513, 519, 554, 533, 544, 464, 521, 531, 579, 532, 485, 492, 537, 535, 512, 494, 541, 498, 457, 528, 536, 531, 475, 509, 497, 494, 506, 485, 488, 539, 513, 479, 526, 499, 483, 532, 524, 548, 529, 505, 506, 521, 514, 515, 506, 514, 455, 517, 495, 525, 551, 517, 510, 518, 491, 521, 490, 546, 497, 520, 484, 523, 544, 550, 529, 510, 519, 525, 492, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[501, 502, 500, 506, 487, 506, 508, 486, 508, 503, 489, 517, 494, 544, 513, 523, 488, 514, 531, 551, 504, 498, 523, 518, 523, 529, 536, 521, 511, 512, 570, 527, 524, 533, 535, 509, 527, 509, 498, 504, 538, 502, 464, 512, 512, 496, 516, 482, 488, 482, 543, 502, 496, 477, 490, 501, 489, 489, 550, 533, 522, 538, 526, 523, 519, 560, 553, 504, 528, 511, 528, 493, 490, 542, 511, 506, 541, 491, 525, 504, 476, 533, 484, 474, 562, 508, 504, 485, 508, 530, 510, 513, 483, 526, 513, 491, 550, 524, 519, 502, 541, 510, 507, 520, 527, 498, 512, 487, 512, 534, 483, 491, 535, 538, 552, 540, 489, 507, 483, 500, 512, 499, 502, 503, 488, 525, 515, 472] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[532, 508, 513, 471, 540, 523, 543, 523, 535, 487, 516, 495, 521, 514, 496, 536, 537, 525, 497, 534, 496, 524, 499, 534, 479, 541, 513, 525, 491, 497, 534, 509, 471, 529, 447, 489, 522, 505, 474, 523, 513, 519, 556, 495, 577, 514, 494, 523, 523, 498, 504, 545, 499, 514, 522, 502, 512, 572, 491, 519, 534, 515, 465, 502, 511, 545, 519, 519, 549, 495, 462, 517, 515, 491, 489, 504, 534, 489, 523, 520, 547, 536, 534, 504, 512, 515, 470, 533, 515, 490, 505, 507, 546, 543, 497, 517, 506, 540, 478, 516, 479, 477, 529, 476, 545, 484, 538, 503, 469, 470, 498, 527, 527, 493, 500, 492, 513, 545, 520, 514, 496, 530, 469, 535, 547, 511, 474, 526] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[494, 533, 502, 542, 532, 553, 501, 511, 495, 487, 498, 501, 547, 522, 527, 525, 478, 499, 532, 535, 575, 495, 493, 488, 500, 551, 494, 512, 484, 489, 522, 521, 530, 521, 512, 504, 505, 501, 554, 485, 496, 504, 503, 562, 516, 519, 539, 516, 488, 519, 462, 507, 499, 509, 524, 568, 561, 528, 543, 537, 518, 490, 507, 499, 510, 488, 503, 482, 545, 507, 534, 517, 516, 516, 560, 494, 495, 484, 520, 516, 509, 497, 522, 505, 488, 518, 487, 519, 518, 507, 499, 469, 491, 494, 482, 524, 511, 525, 507, 477, 502, 503, 508, 520, 546, 521, 516, 519, 523, 503, 503, 526, 516, 515, 490, 463, 530, 520, 540, 496, 510, 510, 495, 501, 527, 499, 515, 519] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[501, 502, 500, 506, 487, 506, 508, 486, 508, 503, 489, 517, 494, 544, 513, 523, 488, 514, 531, 551, 504, 498, 523, 518, 523, 529, 536, 521, 511, 512, 570, 527, 524, 533, 535, 509, 527, 509, 498, 504, 538, 502, 464, 512, 512, 496, 516, 482, 488, 482, 543, 502, 496, 477, 490, 501, 489, 489, 550, 533, 522, 538, 526, 523, 519, 560, 553, 504, 528, 511, 528, 493, 490, 542, 511, 506, 541, 491, 525, 504, 476, 533, 484, 474, 562, 508, 504, 485, 508, 530, 510, 513, 483, 526, 513, 491, 550, 524, 519, 502, 541, 510, 507, 520, 527, 498, 512, 487, 512, 534, 483, 491, 535, 538, 552, 540, 489, 507, 483, 500, 512, 499, 502, 503, 488, 525, 515, 472] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[546, 513, 525, 506, 501, 518, 510, 545, 501, 524, 463, 556, 506, 532, 521, 499, 522, 486, 541, 487, 512, 524, 560, 514, 515, 493, 493, 463, 521, 469, 490, 500, 517, 501, 476, 497, 544, 550, 489, 544, 528, 549, 512, 533, 512, 500, 541, 487, 486, 462, 532, 545, 470, 484, 474, 508, 514, 514, 494, 524, 504, 551, 523, 535, 504, 501, 566, 516, 503, 492, 506, 540, 549, 530, 496, 501, 487, 518, 534, 496, 473, 542, 543, 530, 490, 543, 477, 536, 460, 536, 505, 470, 500, 527, 487, 543, 513, 501, 509, 527, 532, 479, 497, 497, 507, 514, 488, 518, 552, 494, 514, 510, 527, 529, 478, 516, 492, 477, 491, 523, 539, 535, 534, 512, 530, 502, 514, 528] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[513, 489, 515, 483, 561, 514, 534, 468, 531, 502, 480, 513, 489, 500, 520, 514, 526, 527, 557, 472, 497, 479, 480, 527, 523, 501, 498, 496, 501, 486, 508, 507, 547, 517, 494, 525, 480, 523, 482, 494, 500, 538, 505, 504, 498, 495, 530, 519, 513, 500, 488, 523, 506, 514, 488, 523, 497, 549, 531, 471, 510, 553, 532, 533, 538, 508, 539, 505, 524, 516, 526, 533, 519, 528, 538, 507, 489, 516, 508, 518, 510, 479, 492, 513, 510, 483, 500, 490, 515, 548, 495, 518, 497, 525, 525, 521, 499, 532, 491, 517, 496, 550, 544, 532, 533, 541, 534, 515, 515, 509, 510, 547, 520, 504, 537, 484, 522, 536, 505, 504, 518, 530, 523, 496, 486, 508, 496, 476] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[506, 511, 504, 525, 523, 482, 522, 515, 533, 536, 518, 530, 489, 507, 527, 532, 515, 523, 533, 542, 533, 498, 542, 535, 477, 486, 515, 509, 512, 472, 529, 529, 502, 511, 537, 524, 499, 503, 511, 499, 522, 511, 497, 498, 506, 535, 516, 539, 524, 548, 520, 536, 501, 476, 550, 477, 496, 507, 551, 474, 512, 492, 499, 530, 539, 527, 502, 528, 521, 508, 534, 519, 488, 482, 531, 509, 521, 492, 513, 492, 529, 529, 561, 515, 483, 489, 504, 487, 479, 489, 535, 498, 512, 521, 538, 503, 522, 495, 504, 507, 497, 520, 492, 494, 505, 514, 480, 524, 512, 501, 528, 485, 509, 509, 498, 558, 485, 475, 481, 525, 532, 499, 496, 499, 514, 505, 538, 542] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[505, 522, 513, 526, 509, 518, 508, 513, 547, 495, 486, 526, 526, 535, 547, 502, 455, 481, 558, 494, 516, 548, 517, 490, 511, 475, 515, 531, 525, 508, 491, 497, 495, 522, 515, 468, 480, 499, 532, 521, 514, 546, 502, 494, 513, 523, 511, 525, 524, 527, 488, 519, 489, 498, 517, 519, 495, 492, 510, 519, 496, 488, 521, 549, 566, 512, 490, 529, 496, 494, 494, 524, 529, 490, 549, 523, 525, 479, 517, 512, 542, 518, 514, 486, 527, 528, 525, 529, 492, 534, 501, 521, 517, 529, 515, 519, 506, 565, 539, 471, 535, 518, 484, 525, 501, 482, 500, 520, 533, 524, 524, 505, 521, 455, 477, 531, 521, 527, 511, 473, 502, 504, 528, 494, 516, 495, 515, 507] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[494, 533, 502, 542, 532, 553, 501, 511, 495, 487, 498, 501, 547, 522, 527, 525, 478, 499, 532, 535, 575, 495, 493, 488, 500, 551, 494, 512, 484, 489, 522, 521, 530, 521, 512, 504, 505, 501, 554, 485, 496, 504, 503, 562, 516, 519, 539, 516, 488, 519, 462, 507, 499, 509, 524, 568, 561, 528, 543, 537, 518, 490, 507, 499, 510, 488, 503, 482, 545, 507, 534, 517, 516, 516, 560, 494, 495, 484, 520, 516, 509, 497, 522, 505, 488, 518, 487, 519, 518, 507, 499, 469, 491, 494, 482, 524, 511, 525, 507, 477, 502, 503, 508, 520, 546, 521, 516, 519, 523, 503, 503, 526, 516, 515, 490, 463, 530, 520, 540, 496, 510, 510, 495, 501, 527, 499, 515, 519] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[506, 511, 504, 525, 523, 482, 522, 515, 530, 536, 518, 530, 489, 507, 527, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 523, 533, 542, 533, 498, 542, 535, 477, 486, 515, 509, 512, 472, 529, 529, 14, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 511, 537, 524, 499, 503, 511, 499, 522, 511, 497, 498, 506, 535, 499, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 548, 520, 532, 501, 476, 550, 477, 496, 507, 551, 474, 512, 492, 499, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 527, 452, 528, 521, 508, 534, 519, 488, 482, 531, 509, 521, 492, 513, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 529, 561, 515, 483, 489, 504, 487, 479, 489, 535, 498, 512, 521, 538, 503, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 495, 504, 507, 497, 520, 492, 494, 505, 514, 480, 524, 512, 501, 528, 485, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 509, 498, 558, 485, 475, 481, 525, 532, 499, 496, 499, 514, 505, 538, 542, 17, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8257, 8266, 8193, 8189, 8156, 8205, 8085, 8185] avg=8192.0000 max=8266 max_rank=1 min=8085 min_rank=6 imbalance_ratio=1.009033 max_load_reduction=57270 max_load_reduction_pct=0.873871 moved_to_echo=32631 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[505, 522, 513, 526, 509, 518, 508, 513, 544, 495, 486, 526, 526, 535, 547, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 455, 481, 558, 494, 516, 548, 517, 490, 511, 475, 515, 531, 525, 508, 491, 497, 13, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 522, 515, 468, 480, 499, 532, 521, 514, 546, 502, 494, 513, 523, 495, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 527, 488, 515, 489, 498, 517, 519, 495, 492, 510, 519, 496, 488, 521, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 566, 512, 441, 529, 496, 494, 494, 524, 529, 490, 549, 523, 525, 479, 517, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 518, 514, 486, 527, 528, 525, 529, 492, 534, 501, 521, 517, 529, 515, 519, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 565, 539, 471, 535, 518, 484, 525, 501, 482, 500, 520, 533, 524, 524, 505, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 455, 477, 531, 521, 527, 511, 473, 502, 504, 528, 494, 516, 495, 515, 507, 16, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8275, 8127, 8144, 8147, 8180, 8330, 8237, 8096] avg=8192.0000 max=8330 max_rank=5 min=8096 min_rank=7 imbalance_ratio=1.016846 max_load_reduction=57206 max_load_reduction_pct=0.872894 moved_to_echo=32843 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[494, 533, 502, 542, 532, 553, 501, 511, 492, 487, 498, 501, 547, 522, 527, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 499, 532, 535, 575, 495, 493, 488, 500, 551, 494, 512, 484, 489, 522, 521, 14, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 521, 512, 504, 505, 501, 554, 485, 496, 504, 503, 562, 516, 519, 521, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 519, 462, 503, 499, 509, 524, 568, 561, 528, 543, 537, 518, 490, 507, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 488, 453, 482, 545, 507, 534, 517, 516, 516, 560, 494, 495, 484, 520, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 497, 522, 505, 488, 518, 487, 519, 518, 507, 499, 469, 491, 494, 482, 524, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 525, 507, 477, 502, 503, 508, 520, 546, 521, 516, 519, 523, 503, 503, 526, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 515, 490, 463, 530, 520, 540, 496, 510, 510, 495, 501, 527, 499, 515, 519, 17, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8267, 8184, 8249, 8255, 8137, 8062, 8215, 8167] avg=8192.0000 max=8267 max_rank=0 min=8062 min_rank=5 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32581 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[501, 502, 500, 506, 487, 506, 508, 486, 503, 503, 489, 517, 494, 544, 513, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 514, 531, 551, 504, 498, 523, 518, 523, 529, 536, 521, 511, 512, 570, 527, 17, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 533, 535, 509, 527, 509, 498, 504, 538, 502, 464, 512, 512, 496, 492, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 482, 543, 490, 496, 477, 490, 501, 489, 489, 550, 533, 522, 538, 526, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 560, 490, 504, 528, 511, 528, 493, 490, 542, 511, 506, 541, 491, 525, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 533, 484, 474, 562, 508, 504, 485, 508, 530, 510, 513, 483, 526, 513, 491, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 524, 519, 502, 541, 510, 507, 520, 527, 498, 512, 487, 512, 534, 483, 491, 5, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 538, 552, 540, 489, 507, 483, 500, 512, 499, 502, 503, 488, 525, 515, 472, 23, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8082, 8378, 8137, 8137, 8243, 8140, 8229, 8190] avg=8192.0000 max=8378 max_rank=1 min=8082 min_rank=0 imbalance_ratio=1.022705 max_load_reduction=57158 max_load_reduction_pct=0.872162 moved_to_echo=32802 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[515, 487, 509, 499, 489, 476, 482, 507, 521, 502, 511, 497, 531, 511, 490, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 479, 535, 499, 463, 499, 514, 524, 521, 516, 488, 493, 542, 517, 480, 507, 15, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 495, 542, 506, 537, 540, 517, 522, 518, 513, 494, 550, 496, 516, 534, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 516, 479, 516, 525, 525, 505, 512, 502, 496, 513, 519, 554, 533, 544, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 531, 520, 532, 485, 492, 537, 535, 512, 494, 541, 498, 457, 528, 536, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 509, 497, 494, 506, 485, 488, 539, 513, 479, 526, 499, 483, 532, 524, 548, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 505, 506, 521, 514, 515, 506, 514, 455, 517, 495, 525, 551, 517, 510, 518, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 521, 490, 546, 497, 520, 484, 523, 544, 550, 529, 510, 519, 525, 492, 504, 20, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8019, 8124, 8283, 8252, 8250, 8136, 8203, 8269] avg=8192.0000 max=8283 max_rank=2 min=8019 min_rank=0 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32858 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[501, 502, 500, 506, 487, 506, 508, 486, 503, 503, 489, 517, 494, 544, 513, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 514, 531, 551, 504, 498, 523, 518, 523, 529, 536, 521, 511, 512, 570, 527, 17, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 533, 535, 509, 527, 509, 498, 504, 538, 502, 464, 512, 512, 496, 492, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 482, 543, 490, 496, 477, 490, 501, 489, 489, 550, 533, 522, 538, 526, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 560, 490, 504, 528, 511, 528, 493, 490, 542, 511, 506, 541, 491, 525, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 533, 484, 474, 562, 508, 504, 485, 508, 530, 510, 513, 483, 526, 513, 491, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 524, 519, 502, 541, 510, 507, 520, 527, 498, 512, 487, 512, 534, 483, 491, 5, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 538, 552, 540, 489, 507, 483, 500, 512, 499, 502, 503, 488, 525, 515, 472, 23, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8082, 8378, 8137, 8137, 8243, 8140, 8229, 8190] avg=8192.0000 max=8378 max_rank=1 min=8082 min_rank=0 imbalance_ratio=1.022705 max_load_reduction=57158 max_load_reduction_pct=0.872162 moved_to_echo=32802 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[532, 508, 513, 471, 540, 523, 543, 523, 532, 487, 516, 495, 521, 514, 496, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 525, 497, 534, 496, 524, 499, 534, 479, 541, 513, 525, 491, 497, 534, 509, 13, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 529, 447, 489, 522, 505, 474, 523, 513, 519, 556, 495, 577, 514, 478, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 498, 504, 541, 499, 514, 522, 502, 512, 572, 491, 519, 534, 515, 465, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 545, 467, 519, 549, 495, 462, 517, 515, 491, 489, 504, 534, 489, 523, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 536, 534, 504, 512, 515, 470, 533, 515, 490, 505, 507, 546, 543, 497, 517, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 540, 478, 516, 479, 477, 529, 476, 545, 484, 538, 503, 469, 470, 498, 527, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 493, 500, 492, 513, 545, 520, 514, 496, 530, 469, 535, 547, 511, 474, 526, 17, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8250, 8250, 8135, 8213, 8130, 8306, 8040, 8212] avg=8192.0000 max=8306 max_rank=5 min=8040 min_rank=6 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32688 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[513, 489, 515, 483, 561, 514, 534, 468, 528, 502, 480, 513, 489, 500, 520, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 527, 557, 472, 497, 479, 480, 527, 523, 501, 498, 496, 501, 486, 508, 507, 14, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 517, 494, 525, 480, 523, 482, 494, 500, 538, 505, 504, 498, 495, 512, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 500, 488, 519, 506, 514, 488, 523, 497, 549, 531, 471, 510, 553, 532, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 508, 485, 505, 524, 516, 526, 533, 519, 528, 538, 507, 489, 516, 508, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 479, 492, 513, 510, 483, 500, 490, 515, 548, 495, 518, 497, 525, 525, 521, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 532, 491, 517, 496, 550, 544, 532, 533, 541, 534, 515, 515, 509, 510, 547, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 504, 537, 484, 522, 536, 505, 504, 518, 530, 523, 496, 486, 508, 496, 476, 18, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8123, 8101, 8133, 8227, 8258, 8157, 8370, 8167] avg=8192.0000 max=8370 max_rank=6 min=8101 min_rank=1 imbalance_ratio=1.021729 max_load_reduction=57166 max_load_reduction_pct=0.872284 moved_to_echo=32952 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[546, 513, 525, 506, 501, 518, 510, 545, 498, 524, 463, 556, 506, 532, 521, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 486, 541, 487, 512, 524, 560, 514, 515, 493, 493, 463, 521, 469, 490, 500, 14, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 501, 476, 497, 544, 550, 489, 544, 528, 549, 512, 533, 512, 500, 523, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 462, 532, 541, 470, 484, 474, 508, 514, 514, 494, 524, 504, 551, 523, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 501, 509, 516, 503, 492, 506, 540, 549, 530, 496, 501, 487, 518, 534, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 542, 543, 530, 490, 543, 477, 536, 460, 536, 505, 470, 500, 527, 487, 543, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 501, 509, 527, 532, 479, 497, 497, 507, 514, 488, 518, 552, 494, 514, 510, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 529, 478, 516, 492, 477, 491, 523, 539, 535, 534, 512, 530, 502, 514, 528, 19, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8263, 8106, 8262, 8116, 8182, 8200, 8157, 8250] avg=8192.0000 max=8263 max_rank=0 min=8106 min_rank=1 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32789 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[505, 522, 513, 526, 509, 518, 508, 513, 544, 495, 486, 526, 526, 535, 547, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 455, 481, 558, 494, 516, 548, 517, 490, 511, 475, 515, 531, 525, 508, 491, 497, 13, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 522, 515, 468, 480, 499, 532, 521, 514, 546, 502, 494, 513, 523, 495, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 527, 488, 515, 489, 498, 517, 519, 495, 492, 510, 519, 496, 488, 521, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 566, 512, 441, 529, 496, 494, 494, 524, 529, 490, 549, 523, 525, 479, 517, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 518, 514, 486, 527, 528, 525, 529, 492, 534, 501, 521, 517, 529, 515, 519, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 565, 539, 471, 535, 518, 484, 525, 501, 482, 500, 520, 533, 524, 524, 505, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 455, 477, 531, 521, 527, 511, 473, 502, 504, 528, 494, 516, 495, 515, 507, 16, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8275, 8127, 8144, 8147, 8180, 8330, 8237, 8096] avg=8192.0000 max=8330 max_rank=5 min=8096 min_rank=7 imbalance_ratio=1.016846 max_load_reduction=57206 max_load_reduction_pct=0.872894 moved_to_echo=32843 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[494, 533, 502, 542, 532, 553, 501, 511, 492, 487, 498, 501, 547, 522, 527, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 499, 532, 535, 575, 495, 493, 488, 500, 551, 494, 512, 484, 489, 522, 521, 14, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 521, 512, 504, 505, 501, 554, 485, 496, 504, 503, 562, 516, 519, 521, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 519, 462, 503, 499, 509, 524, 568, 561, 528, 543, 537, 518, 490, 507, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 488, 453, 482, 545, 507, 534, 517, 516, 516, 560, 494, 495, 484, 520, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 497, 522, 505, 488, 518, 487, 519, 518, 507, 499, 469, 491, 494, 482, 524, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 525, 507, 477, 502, 503, 508, 520, 546, 521, 516, 519, 523, 503, 503, 526, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 515, 490, 463, 530, 520, 540, 496, 510, 510, 495, 501, 527, 499, 515, 519, 17, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8267, 8184, 8249, 8255, 8137, 8062, 8215, 8167] avg=8192.0000 max=8267 max_rank=0 min=8062 min_rank=5 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32581 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 3968, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4352, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 3968, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 3968, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 3840, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4352, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 3968, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 3840, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 3968, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[509, 542, 543, 496, 476, 540, 498, 500, 526, 530, 515, 494, 515, 545, 525, 511, 531, 459, 524, 513, 511, 511, 495, 508, 509, 505, 516, 490, 538, 519, 507, 519, 487, 485, 526, 492, 530, 576, 518, 498, 492, 514, 536, 480, 519, 495, 532, 517, 510, 512, 517, 505, 527, 527, 508, 487, 452, 527, 483, 513, 503, 508, 514, 492, 511, 553, 473, 494, 521, 501, 510, 512, 503, 531, 508, 542, 517, 591, 527, 515, 520, 518, 532, 480, 493, 510, 483, 525, 542, 516, 521, 504, 526, 508, 566, 519, 527, 464, 521, 552, 525, 483, 492, 485, 506, 499, 529, 478, 519, 496, 511, 480, 528, 541, 532, 486, 505, 465, 497, 487, 513, 492, 543, 549, 481, 530, 525, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[515, 516, 478, 498, 504, 490, 529, 513, 553, 522, 530, 496, 497, 495, 496, 539, 547, 542, 509, 517, 498, 485, 506, 509, 488, 503, 506, 490, 530, 570, 526, 489, 493, 521, 515, 498, 460, 495, 493, 501, 506, 522, 519, 520, 479, 504, 530, 509, 506, 527, 518, 526, 524, 502, 514, 472, 530, 494, 510, 524, 481, 551, 477, 521, 543, 550, 496, 512, 524, 544, 553, 533, 547, 515, 516, 546, 464, 512, 515, 519, 496, 523, 498, 514, 505, 518, 527, 557, 490, 535, 489, 502, 484, 521, 492, 508, 534, 508, 513, 509, 511, 505, 467, 518, 523, 481, 511, 495, 529, 515, 536, 499, 536, 525, 536, 501, 499, 505, 464, 543, 519, 537, 537, 524, 492, 522, 477, 489] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[496, 521, 520, 485, 495, 503, 473, 538, 491, 452, 497, 504, 514, 456, 520, 540, 501, 504, 485, 511, 498, 531, 524, 506, 530, 510, 497, 487, 515, 516, 502, 530, 504, 497, 497, 550, 491, 490, 517, 494, 511, 528, 533, 499, 552, 500, 493, 509, 508, 485, 544, 543, 518, 498, 520, 488, 477, 525, 556, 469, 530, 536, 528, 524, 504, 517, 483, 487, 509, 538, 539, 510, 511, 551, 500, 538, 493, 528, 510, 513, 494, 557, 512, 533, 514, 515, 531, 536, 521, 522, 515, 502, 527, 497, 508, 570, 501, 528, 508, 523, 496, 464, 526, 508, 519, 510, 507, 519, 549, 491, 506, 540, 510, 489, 516, 514, 517, 488, 532, 513, 491, 502, 483, 546, 532, 529, 524, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[515, 448, 549, 537, 508, 503, 531, 504, 497, 492, 525, 523, 506, 523, 532, 517, 502, 466, 510, 518, 468, 498, 533, 503, 484, 549, 554, 511, 504, 494, 523, 482, 505, 513, 533, 530, 528, 535, 481, 498, 498, 540, 505, 529, 502, 511, 512, 501, 510, 511, 515, 504, 521, 510, 511, 522, 532, 534, 487, 526, 516, 481, 510, 488, 496, 506, 501, 497, 517, 533, 548, 551, 522, 532, 546, 501, 501, 510, 499, 500, 477, 548, 527, 542, 519, 509, 531, 528, 513, 526, 501, 496, 498, 512, 498, 515, 496, 526, 478, 516, 486, 491, 533, 495, 501, 533, 550, 481, 524, 556, 523, 527, 521, 521, 505, 472, 508, 524, 515, 511, 486, 498, 511, 490, 517, 503, 535, 495] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[509, 542, 543, 496, 476, 540, 498, 500, 526, 530, 515, 494, 515, 545, 525, 511, 531, 459, 524, 513, 511, 511, 495, 508, 509, 505, 516, 490, 538, 519, 507, 519, 487, 485, 526, 492, 530, 576, 518, 498, 492, 514, 536, 480, 519, 495, 532, 517, 510, 512, 517, 505, 527, 527, 508, 487, 452, 527, 483, 513, 503, 508, 514, 492, 511, 553, 473, 494, 521, 501, 510, 512, 503, 531, 508, 542, 517, 591, 527, 515, 520, 518, 532, 480, 493, 510, 483, 525, 542, 516, 521, 504, 526, 508, 566, 519, 527, 464, 521, 552, 525, 483, 492, 485, 506, 499, 529, 478, 519, 496, 511, 480, 528, 541, 532, 486, 505, 465, 497, 487, 513, 492, 543, 549, 481, 530, 525, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[500, 486, 505, 515, 534, 551, 484, 503, 531, 517, 527, 490, 503, 506, 503, 497, 520, 495, 508, 499, 526, 520, 534, 521, 531, 547, 510, 498, 491, 554, 538, 515, 505, 511, 557, 487, 522, 488, 515, 526, 505, 477, 542, 538, 529, 499, 553, 514, 551, 552, 550, 524, 513, 511, 473, 473, 492, 541, 497, 511, 517, 518, 523, 493, 536, 492, 521, 487, 494, 512, 507, 538, 524, 477, 495, 509, 502, 506, 521, 496, 499, 538, 530, 530, 512, 532, 525, 514, 476, 528, 486, 526, 501, 517, 515, 528, 530, 503, 471, 504, 491, 460, 536, 534, 511, 497, 489, 490, 507, 508, 502, 468, 534, 490, 509, 500, 518, 519, 508, 523, 543, 509, 500, 495, 514, 527, 514, 492] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[514, 551, 503, 511, 497, 504, 506, 471, 504, 527, 512, 531, 530, 489, 489, 527, 533, 529, 499, 527, 484, 498, 541, 511, 526, 525, 509, 507, 511, 508, 535, 531, 537, 498, 491, 501, 492, 508, 505, 487, 525, 513, 509, 506, 500, 506, 508, 520, 537, 496, 517, 529, 503, 517, 510, 522, 519, 507, 520, 518, 491, 546, 470, 476, 490, 532, 505, 540, 512, 526, 516, 510, 505, 510, 525, 483, 506, 513, 535, 484, 521, 565, 500, 495, 506, 489, 534, 524, 515, 511, 509, 474, 525, 548, 477, 512, 492, 543, 530, 536, 516, 538, 501, 472, 486, 529, 482, 517, 519, 506, 519, 508, 510, 499, 528, 519, 522, 512, 521, 509, 532, 505, 480, 539, 517, 497, 539, 492] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[530, 553, 442, 535, 523, 522, 534, 538, 462, 537, 504, 529, 517, 531, 522, 507, 515, 542, 540, 522, 537, 512, 485, 512, 524, 565, 512, 509, 502, 487, 538, 515, 527, 543, 514, 490, 486, 484, 496, 494, 526, 490, 453, 533, 505, 547, 544, 498, 492, 499, 512, 535, 502, 548, 516, 491, 532, 512, 506, 515, 501, 511, 495, 479, 493, 557, 534, 550, 512, 464, 536, 498, 491, 516, 544, 485, 504, 539, 542, 482, 528, 500, 509, 509, 555, 494, 491, 490, 512, 502, 543, 493, 539, 490, 504, 517, 514, 498, 514, 508, 493, 496, 518, 542, 503, 511, 523, 501, 518, 514, 486, 495, 485, 489, 504, 530, 507, 525, 508, 511, 488, 501, 471, 518, 502, 512, 511, 538] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[496, 521, 520, 485, 495, 503, 473, 538, 491, 452, 497, 504, 514, 456, 520, 540, 501, 504, 485, 511, 498, 531, 524, 506, 530, 510, 497, 487, 515, 516, 502, 530, 504, 497, 497, 550, 491, 490, 517, 494, 511, 528, 533, 499, 552, 500, 493, 509, 508, 485, 544, 543, 518, 498, 520, 488, 477, 525, 556, 469, 530, 536, 528, 524, 504, 517, 483, 487, 509, 538, 539, 510, 511, 551, 500, 538, 493, 528, 510, 513, 494, 557, 512, 533, 514, 515, 531, 536, 521, 522, 515, 502, 527, 497, 508, 570, 501, 528, 508, 523, 496, 464, 526, 508, 519, 510, 507, 519, 549, 491, 506, 540, 510, 489, 516, 514, 517, 488, 532, 513, 491, 502, 483, 546, 532, 529, 524, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[536, 511, 487, 525, 483, 516, 503, 490, 483, 531, 508, 521, 533, 519, 479, 503, 538, 504, 496, 516, 470, 489, 484, 487, 530, 499, 548, 471, 525, 511, 486, 572, 544, 564, 559, 478, 546, 508, 502, 514, 527, 532, 540, 499, 505, 506, 515, 518, 511, 519, 529, 494, 503, 507, 556, 532, 494, 524, 509, 487, 504, 501, 493, 501, 547, 518, 542, 492, 518, 511, 536, 507, 533, 509, 503, 526, 514, 527, 533, 521, 512, 517, 500, 534, 506, 511, 531, 506, 506, 508, 502, 512, 502, 534, 568, 526, 540, 498, 512, 496, 479, 499, 508, 495, 535, 499, 496, 536, 495, 502, 506, 472, 456, 519, 536, 501, 529, 488, 504, 529, 490, 493, 505, 496, 551, 499, 473, 512] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[500, 486, 505, 515, 534, 551, 484, 503, 531, 517, 527, 490, 503, 506, 503, 497, 520, 495, 508, 499, 526, 520, 534, 521, 531, 547, 510, 498, 491, 554, 538, 515, 505, 511, 557, 487, 522, 488, 515, 526, 505, 477, 542, 538, 529, 499, 553, 514, 551, 552, 550, 524, 513, 511, 473, 473, 492, 541, 497, 511, 517, 518, 523, 493, 536, 492, 521, 487, 494, 512, 507, 538, 524, 477, 495, 509, 502, 506, 521, 496, 499, 538, 530, 530, 512, 532, 525, 514, 476, 528, 486, 526, 501, 517, 515, 528, 530, 503, 471, 504, 491, 460, 536, 534, 511, 497, 489, 490, 507, 508, 502, 468, 534, 490, 509, 500, 518, 519, 508, 523, 543, 509, 500, 495, 514, 527, 514, 492] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[515, 516, 478, 498, 504, 490, 529, 513, 553, 522, 530, 496, 497, 495, 496, 539, 547, 542, 509, 517, 498, 485, 506, 509, 488, 503, 506, 490, 530, 570, 526, 489, 493, 521, 515, 498, 460, 495, 493, 501, 506, 522, 519, 520, 479, 504, 530, 509, 506, 527, 518, 526, 524, 502, 514, 472, 530, 494, 510, 524, 481, 551, 477, 521, 543, 550, 496, 512, 524, 544, 553, 533, 547, 515, 516, 546, 464, 512, 515, 519, 496, 523, 498, 514, 505, 518, 527, 557, 490, 535, 489, 502, 484, 521, 492, 508, 534, 508, 513, 509, 511, 505, 467, 518, 523, 481, 511, 495, 529, 515, 536, 499, 536, 525, 536, 501, 499, 505, 464, 543, 519, 537, 537, 524, 492, 522, 477, 489] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[530, 553, 442, 535, 523, 522, 534, 538, 462, 537, 504, 529, 517, 531, 522, 507, 515, 542, 540, 522, 537, 512, 485, 512, 524, 565, 512, 509, 502, 487, 538, 515, 527, 543, 514, 490, 486, 484, 496, 494, 526, 490, 453, 533, 505, 547, 544, 498, 492, 499, 512, 535, 502, 548, 516, 491, 532, 512, 506, 515, 501, 511, 495, 479, 493, 557, 534, 550, 512, 464, 536, 498, 491, 516, 544, 485, 504, 539, 542, 482, 528, 500, 509, 509, 555, 494, 491, 490, 512, 502, 543, 493, 539, 490, 504, 517, 514, 498, 514, 508, 493, 496, 518, 542, 503, 511, 523, 501, 518, 514, 486, 495, 485, 489, 504, 530, 507, 525, 508, 511, 488, 501, 471, 518, 502, 512, 511, 538] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[515, 448, 549, 537, 508, 503, 531, 504, 497, 492, 525, 523, 506, 523, 532, 517, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 466, 510, 518, 468, 498, 533, 503, 484, 536, 554, 511, 504, 494, 523, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 513, 533, 530, 528, 535, 481, 498, 498, 540, 505, 529, 502, 511, 512, 501, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 511, 515, 504, 521, 510, 511, 522, 532, 534, 487, 526, 516, 481, 510, 488, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 506, 501, 497, 517, 533, 478, 551, 522, 532, 546, 501, 501, 510, 499, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 499, 527, 542, 519, 509, 531, 528, 513, 526, 501, 496, 498, 512, 498, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 526, 478, 516, 486, 491, 533, 495, 501, 533, 550, 481, 524, 556, 523, 527, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 521, 505, 472, 508, 524, 515, 511, 486, 498, 511, 490, 517, 503, 535, 495, 25, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8229, 8086, 8224, 8193, 8190, 8191, 8281, 8142] avg=8192.0000 max=8281 max_rank=6 min=8086 min_rank=1 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32804 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[509, 542, 543, 496, 476, 540, 498, 500, 526, 530, 515, 494, 515, 545, 525, 511, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 459, 524, 513, 511, 511, 495, 508, 509, 493, 516, 490, 538, 519, 507, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 485, 526, 492, 530, 576, 518, 498, 492, 514, 536, 480, 519, 495, 532, 517, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 512, 517, 505, 527, 527, 508, 487, 452, 527, 483, 513, 503, 508, 514, 492, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 553, 473, 494, 521, 501, 446, 512, 503, 531, 508, 542, 517, 591, 527, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 472, 532, 480, 493, 510, 483, 525, 542, 516, 521, 504, 526, 508, 566, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 464, 521, 552, 525, 483, 492, 485, 506, 499, 529, 478, 519, 496, 511, 480, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 541, 532, 486, 505, 465, 497, 487, 513, 492, 543, 549, 481, 530, 525, 521, 23, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8283, 8143, 8200, 8099, 8245, 8217, 8127, 8222] avg=8192.0000 max=8283 max_rank=0 min=8099 min_rank=3 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32811 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[515, 516, 478, 498, 504, 490, 529, 513, 553, 522, 530, 496, 497, 495, 496, 539, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 542, 509, 517, 498, 485, 506, 509, 488, 491, 506, 490, 530, 570, 526, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 521, 515, 498, 460, 495, 493, 501, 506, 522, 519, 520, 479, 504, 530, 509, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 527, 518, 526, 524, 502, 514, 472, 530, 494, 510, 524, 481, 551, 477, 521, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 550, 496, 512, 524, 544, 482, 533, 547, 515, 516, 546, 464, 512, 515, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 476, 498, 514, 505, 518, 527, 557, 490, 535, 489, 502, 484, 521, 492, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 508, 513, 509, 511, 505, 467, 518, 523, 481, 511, 495, 529, 515, 536, 499, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 525, 536, 501, 499, 505, 464, 543, 519, 537, 537, 524, 492, 522, 477, 489, 24, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8203, 8068, 8191, 8318, 8112, 8220, 8235] avg=8192.0000 max=8318 max_rank=4 min=8068 min_rank=2 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32885 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[536, 511, 487, 525, 483, 516, 503, 490, 483, 531, 508, 521, 533, 519, 479, 503, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 504, 496, 516, 470, 489, 484, 487, 530, 482, 548, 471, 525, 511, 486, 572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 564, 559, 478, 546, 508, 502, 514, 527, 532, 540, 499, 505, 506, 515, 518, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 519, 529, 494, 503, 507, 556, 532, 494, 524, 509, 487, 504, 501, 493, 501, 11, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 518, 542, 492, 518, 511, 460, 507, 533, 509, 503, 526, 514, 527, 533, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 459, 500, 534, 506, 511, 531, 506, 506, 508, 502, 512, 502, 534, 568, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 498, 512, 496, 479, 499, 508, 495, 535, 499, 496, 536, 495, 502, 506, 472, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 456, 519, 536, 501, 529, 488, 504, 529, 490, 493, 505, 496, 551, 499, 473, 512, 27, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 8109, 8363, 8183, 8261, 8217, 8135, 8117] avg=8192.0000 max=8363 max_rank=2 min=8109 min_rank=1 imbalance_ratio=1.020874 max_load_reduction=57173 max_load_reduction_pct=0.872391 moved_to_echo=32730 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[496, 521, 520, 485, 495, 503, 473, 538, 491, 452, 497, 504, 514, 456, 520, 540, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 504, 485, 511, 498, 531, 524, 506, 530, 498, 497, 487, 515, 516, 502, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 497, 497, 550, 491, 490, 517, 494, 511, 528, 533, 499, 552, 500, 493, 509, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 485, 544, 543, 518, 498, 520, 488, 477, 525, 556, 469, 530, 536, 528, 524, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 517, 483, 487, 509, 538, 471, 510, 511, 551, 500, 538, 493, 528, 510, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 508, 512, 533, 514, 515, 531, 536, 521, 522, 515, 502, 527, 497, 508, 570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 528, 508, 523, 496, 464, 526, 508, 519, 510, 507, 519, 549, 491, 506, 540, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 489, 516, 514, 517, 488, 532, 513, 491, 502, 483, 546, 532, 529, 524, 504, 25, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8024, 8135, 8168, 8263, 8163, 8305, 8259, 8219] avg=8192.0000 max=8305 max_rank=5 min=8024 min_rank=0 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32946 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[514, 551, 503, 511, 497, 504, 506, 471, 504, 527, 512, 531, 530, 489, 489, 527, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 529, 499, 527, 484, 498, 541, 511, 526, 513, 509, 507, 511, 508, 535, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 498, 491, 501, 492, 508, 505, 487, 525, 513, 509, 506, 500, 506, 508, 520, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 496, 517, 529, 503, 517, 510, 522, 519, 507, 520, 518, 491, 546, 470, 476, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 532, 505, 540, 512, 526, 451, 510, 505, 510, 525, 483, 506, 513, 535, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 514, 500, 495, 506, 489, 534, 524, 515, 511, 509, 474, 525, 548, 477, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 543, 530, 536, 516, 538, 501, 472, 486, 529, 482, 517, 519, 506, 519, 508, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 499, 528, 519, 522, 512, 521, 509, 532, 505, 480, 539, 517, 497, 539, 492, 26, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8262, 8109, 8192, 8127, 8154, 8255, 8251] avg=8192.0000 max=8262 max_rank=1 min=8109 min_rank=2 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32787 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[530, 553, 442, 535, 523, 522, 534, 538, 462, 537, 504, 529, 517, 531, 522, 507, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 542, 540, 522, 537, 512, 485, 512, 524, 552, 512, 509, 502, 487, 538, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 543, 514, 490, 486, 484, 496, 494, 526, 490, 453, 533, 505, 547, 544, 498, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 499, 512, 535, 502, 548, 516, 491, 532, 512, 506, 515, 501, 511, 495, 479, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 557, 534, 550, 512, 464, 468, 498, 491, 516, 544, 485, 504, 539, 542, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 455, 509, 509, 555, 494, 491, 490, 512, 502, 543, 493, 539, 490, 504, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 498, 514, 508, 493, 496, 518, 542, 503, 511, 523, 501, 518, 514, 486, 495, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 489, 504, 530, 507, 525, 508, 511, 488, 501, 471, 518, 502, 512, 511, 538, 23, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8303, 8304, 8133, 8161, 8179, 8131, 8198, 8127] avg=8192.0000 max=8304 max_rank=1 min=8127 min_rank=7 imbalance_ratio=1.013672 max_load_reduction=57232 max_load_reduction_pct=0.873291 moved_to_echo=32635 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[515, 448, 549, 537, 508, 503, 531, 504, 497, 492, 525, 523, 506, 523, 532, 517, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 466, 510, 518, 468, 498, 533, 503, 484, 536, 554, 511, 504, 494, 523, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 513, 533, 530, 528, 535, 481, 498, 498, 540, 505, 529, 502, 511, 512, 501, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 511, 515, 504, 521, 510, 511, 522, 532, 534, 487, 526, 516, 481, 510, 488, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 506, 501, 497, 517, 533, 478, 551, 522, 532, 546, 501, 501, 510, 499, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 499, 527, 542, 519, 509, 531, 528, 513, 526, 501, 496, 498, 512, 498, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 526, 478, 516, 486, 491, 533, 495, 501, 533, 550, 481, 524, 556, 523, 527, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 521, 505, 472, 508, 524, 515, 511, 486, 498, 511, 490, 517, 503, 535, 495, 25, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8229, 8086, 8224, 8193, 8190, 8191, 8281, 8142] avg=8192.0000 max=8281 max_rank=6 min=8086 min_rank=1 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32804 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[496, 521, 520, 485, 495, 503, 473, 538, 491, 452, 497, 504, 514, 456, 520, 540, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 504, 485, 511, 498, 531, 524, 506, 530, 498, 497, 487, 515, 516, 502, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 497, 497, 550, 491, 490, 517, 494, 511, 528, 533, 499, 552, 500, 493, 509, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 485, 544, 543, 518, 498, 520, 488, 477, 525, 556, 469, 530, 536, 528, 524, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 517, 483, 487, 509, 538, 471, 510, 511, 551, 500, 538, 493, 528, 510, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 508, 512, 533, 514, 515, 531, 536, 521, 522, 515, 502, 527, 497, 508, 570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 528, 508, 523, 496, 464, 526, 508, 519, 510, 507, 519, 549, 491, 506, 540, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 489, 516, 514, 517, 488, 532, 513, 491, 502, 483, 546, 532, 529, 524, 504, 25, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8024, 8135, 8168, 8263, 8163, 8305, 8259, 8219] avg=8192.0000 max=8305 max_rank=5 min=8024 min_rank=0 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32946 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[536, 511, 487, 525, 483, 516, 503, 490, 483, 531, 508, 521, 533, 519, 479, 503, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 504, 496, 516, 470, 489, 484, 487, 530, 482, 548, 471, 525, 511, 486, 572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 564, 559, 478, 546, 508, 502, 514, 527, 532, 540, 499, 505, 506, 515, 518, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 519, 529, 494, 503, 507, 556, 532, 494, 524, 509, 487, 504, 501, 493, 501, 11, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 518, 542, 492, 518, 511, 460, 507, 533, 509, 503, 526, 514, 527, 533, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 459, 500, 534, 506, 511, 531, 506, 506, 508, 502, 512, 502, 534, 568, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 498, 512, 496, 479, 499, 508, 495, 535, 499, 496, 536, 495, 502, 506, 472, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 456, 519, 536, 501, 529, 488, 504, 529, 490, 493, 505, 496, 551, 499, 473, 512, 27, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 8109, 8363, 8183, 8261, 8217, 8135, 8117] avg=8192.0000 max=8363 max_rank=2 min=8109 min_rank=1 imbalance_ratio=1.020874 max_load_reduction=57173 max_load_reduction_pct=0.872391 moved_to_echo=32730 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[500, 486, 505, 515, 534, 551, 484, 503, 531, 517, 527, 490, 503, 506, 503, 497, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 495, 508, 499, 526, 520, 534, 521, 531, 534, 510, 498, 491, 554, 538, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 511, 557, 487, 522, 488, 515, 526, 505, 477, 542, 538, 529, 499, 553, 514, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 552, 550, 524, 513, 511, 473, 473, 492, 541, 497, 511, 517, 518, 523, 493, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 492, 521, 487, 494, 512, 443, 538, 524, 477, 495, 509, 502, 506, 521, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 490, 530, 530, 512, 532, 525, 514, 476, 528, 486, 526, 501, 517, 515, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 503, 471, 504, 491, 460, 536, 534, 511, 497, 489, 490, 507, 508, 502, 468, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 490, 509, 500, 518, 519, 508, 523, 543, 509, 500, 495, 514, 527, 514, 492, 24, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 8294, 8271, 8254, 8053, 8209, 8061, 8223] avg=8192.0000 max=8294 max_rank=1 min=8053 min_rank=4 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32546 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[515, 516, 478, 498, 504, 490, 529, 513, 553, 522, 530, 496, 497, 495, 496, 539, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 542, 509, 517, 498, 485, 506, 509, 488, 491, 506, 490, 530, 570, 526, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 521, 515, 498, 460, 495, 493, 501, 506, 522, 519, 520, 479, 504, 530, 509, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 527, 518, 526, 524, 502, 514, 472, 530, 494, 510, 524, 481, 551, 477, 521, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 550, 496, 512, 524, 544, 482, 533, 547, 515, 516, 546, 464, 512, 515, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 476, 498, 514, 505, 518, 527, 557, 490, 535, 489, 502, 484, 521, 492, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 508, 513, 509, 511, 505, 467, 518, 523, 481, 511, 495, 529, 515, 536, 499, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 525, 536, 501, 499, 505, 464, 543, 519, 537, 537, 524, 492, 522, 477, 489, 24, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8203, 8068, 8191, 8318, 8112, 8220, 8235] avg=8192.0000 max=8318 max_rank=4 min=8068 min_rank=2 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32885 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 3968, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 3968, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 3968, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4352, 4096, 4096, 4224, 4224, 3712, 4224, 4224, 4224, 4224, 4224, 4096, 4352, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 3968, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[514, 498, 532, 483, 498, 511, 552, 546, 491, 510, 535, 475, 526, 525, 482, 510, 521, 508, 464, 533, 533, 524, 530, 518, 495, 517, 541, 573, 505, 518, 494, 519, 478, 520, 516, 490, 512, 497, 508, 505, 521, 511, 458, 487, 526, 521, 519, 477, 486, 501, 490, 467, 499, 509, 519, 511, 527, 524, 516, 514, 512, 521, 538, 501, 512, 546, 498, 554, 519, 573, 503, 515, 516, 509, 485, 525, 537, 505, 470, 484, 478, 499, 510, 531, 530, 475, 562, 530, 493, 487, 498, 489, 508, 527, 523, 518, 536, 516, 482, 524, 512, 514, 506, 509, 518, 499, 546, 524, 492, 503, 503, 474, 515, 500, 516, 536, 506, 555, 511, 464, 521, 524, 542, 529, 539, 511, 509, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[518, 520, 530, 553, 510, 506, 506, 498, 502, 512, 474, 497, 521, 512, 485, 483, 535, 566, 510, 500, 515, 534, 539, 490, 533, 483, 478, 520, 549, 501, 498, 501, 500, 496, 516, 548, 506, 512, 526, 475, 509, 485, 557, 462, 504, 518, 501, 514, 526, 543, 545, 469, 520, 525, 505, 531, 540, 517, 504, 519, 529, 503, 516, 530, 489, 489, 494, 507, 517, 510, 513, 496, 514, 492, 549, 547, 498, 529, 506, 502, 470, 472, 509, 484, 492, 510, 517, 513, 529, 502, 518, 472, 516, 519, 532, 500, 485, 502, 493, 508, 505, 530, 531, 522, 533, 545, 516, 503, 588, 516, 484, 512, 490, 545, 518, 521, 510, 487, 515, 509, 545, 542, 479, 515, 482, 524, 492, 552] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[478, 531, 508, 491, 495, 486, 532, 506, 499, 484, 519, 519, 531, 540, 558, 515, 513, 514, 472, 505, 505, 496, 518, 518, 493, 521, 521, 514, 478, 518, 501, 530, 526, 537, 515, 519, 497, 491, 532, 496, 497, 534, 512, 517, 543, 501, 538, 521, 523, 545, 532, 549, 534, 524, 518, 505, 523, 494, 545, 494, 529, 513, 481, 484, 510, 524, 513, 500, 500, 526, 505, 515, 523, 508, 530, 537, 507, 514, 482, 482, 489, 493, 493, 500, 531, 519, 509, 525, 560, 520, 493, 491, 511, 479, 526, 519, 502, 505, 521, 526, 505, 508, 557, 516, 488, 534, 498, 493, 516, 515, 512, 510, 488, 541, 493, 503, 505, 522, 494, 494, 538, 490, 505, 510, 490, 520, 490, 535] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[525, 489, 506, 501, 491, 511, 502, 496, 546, 535, 490, 492, 521, 517, 533, 527, 499, 513, 518, 502, 534, 511, 521, 504, 502, 508, 516, 538, 503, 541, 499, 528, 535, 492, 479, 499, 497, 486, 474, 501, 508, 496, 524, 502, 503, 525, 501, 472, 465, 483, 505, 527, 480, 510, 500, 501, 513, 525, 503, 487, 559, 508, 525, 508, 508, 527, 504, 541, 570, 496, 500, 525, 524, 514, 466, 542, 532, 521, 502, 473, 508, 511, 488, 512, 496, 504, 538, 534, 532, 508, 528, 507, 506, 566, 469, 542, 505, 531, 550, 499, 529, 507, 487, 497, 527, 506, 527, 505, 464, 490, 503, 525, 534, 548, 507, 491, 554, 541, 512, 515, 542, 516, 499, 500, 532, 541, 558, 488] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[533, 508, 568, 520, 553, 516, 511, 472, 514, 488, 507, 516, 523, 513, 483, 546, 508, 517, 485, 498, 500, 509, 502, 526, 493, 532, 547, 477, 459, 493, 524, 512, 490, 501, 481, 521, 516, 487, 495, 515, 509, 539, 500, 569, 519, 533, 538, 519, 529, 502, 535, 515, 468, 508, 552, 499, 523, 516, 511, 472, 505, 499, 481, 486, 539, 537, 501, 516, 528, 514, 502, 515, 509, 510, 508, 539, 524, 480, 514, 526, 476, 520, 525, 512, 494, 488, 512, 497, 512, 517, 549, 454, 539, 502, 527, 546, 497, 510, 527, 520, 495, 529, 467, 515, 513, 475, 508, 537, 501, 532, 507, 523, 535, 528, 483, 522, 514, 503, 535, 490, 506, 513, 512, 546, 526, 541, 490, 518] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[478, 531, 508, 491, 495, 486, 532, 506, 499, 484, 519, 519, 531, 540, 558, 515, 513, 514, 472, 505, 505, 496, 518, 518, 493, 521, 521, 514, 478, 518, 501, 530, 526, 537, 515, 519, 497, 491, 532, 496, 497, 534, 512, 517, 543, 501, 538, 521, 523, 545, 532, 549, 534, 524, 518, 505, 523, 494, 545, 494, 529, 513, 481, 484, 510, 524, 513, 500, 500, 526, 505, 515, 523, 508, 530, 537, 507, 514, 482, 482, 489, 493, 493, 500, 531, 519, 509, 525, 560, 520, 493, 491, 511, 479, 526, 519, 502, 505, 521, 526, 505, 508, 557, 516, 488, 534, 498, 493, 516, 515, 512, 510, 488, 541, 493, 503, 505, 522, 494, 494, 538, 490, 505, 510, 490, 520, 490, 535] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[484, 485, 479, 518, 496, 477, 496, 516, 474, 501, 495, 529, 489, 508, 526, 516, 525, 524, 507, 523, 488, 592, 498, 553, 507, 502, 540, 525, 547, 501, 499, 450, 555, 497, 486, 509, 500, 498, 493, 502, 526, 492, 510, 496, 516, 510, 537, 564, 545, 510, 501, 537, 527, 541, 492, 508, 510, 468, 514, 507, 510, 461, 540, 534, 471, 500, 521, 517, 480, 506, 491, 496, 499, 515, 564, 557, 532, 496, 529, 484, 528, 527, 512, 508, 504, 514, 496, 570, 502, 508, 540, 501, 550, 511, 501, 526, 530, 528, 475, 540, 525, 501, 513, 520, 518, 511, 511, 520, 522, 476, 547, 484, 517, 518, 517, 534, 519, 525, 475, 514, 495, 505, 480, 526, 517, 525, 486, 540] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[514, 498, 532, 483, 498, 511, 552, 546, 491, 510, 535, 475, 526, 525, 482, 510, 521, 508, 464, 533, 533, 524, 530, 518, 495, 517, 541, 573, 505, 518, 494, 519, 478, 520, 516, 490, 512, 497, 508, 505, 521, 511, 458, 487, 526, 521, 519, 477, 486, 501, 490, 467, 499, 509, 519, 511, 527, 524, 516, 514, 512, 521, 538, 501, 512, 546, 498, 554, 519, 573, 503, 515, 516, 509, 485, 525, 537, 505, 470, 484, 478, 499, 510, 531, 530, 475, 562, 530, 493, 487, 498, 489, 508, 527, 523, 518, 536, 516, 482, 524, 512, 514, 506, 509, 518, 499, 546, 524, 492, 503, 503, 474, 515, 500, 516, 536, 506, 555, 511, 464, 521, 524, 542, 529, 539, 511, 509, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[497, 516, 498, 534, 523, 486, 498, 531, 496, 512, 481, 469, 491, 475, 529, 537, 534, 488, 501, 520, 506, 529, 539, 498, 540, 536, 523, 556, 505, 498, 501, 499, 487, 527, 470, 504, 499, 473, 513, 526, 512, 491, 491, 493, 521, 521, 569, 489, 515, 530, 470, 480, 509, 513, 497, 532, 502, 535, 527, 524, 512, 497, 503, 523, 527, 503, 526, 506, 481, 493, 509, 558, 519, 542, 490, 498, 554, 516, 529, 536, 511, 520, 562, 499, 473, 497, 504, 500, 528, 551, 491, 516, 557, 510, 513, 534, 514, 500, 541, 504, 516, 513, 536, 506, 509, 492, 518, 527, 484, 488, 506, 509, 561, 526, 511, 460, 508, 493, 518, 520, 551, 524, 522, 476, 504, 478, 514, 553] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[518, 520, 530, 553, 510, 506, 506, 498, 502, 512, 474, 497, 521, 512, 485, 483, 535, 566, 510, 500, 515, 534, 539, 490, 533, 483, 478, 520, 549, 501, 498, 501, 500, 496, 516, 548, 506, 512, 526, 475, 509, 485, 557, 462, 504, 518, 501, 514, 526, 543, 545, 469, 520, 525, 505, 531, 540, 517, 504, 519, 529, 503, 516, 530, 489, 489, 494, 507, 517, 510, 513, 496, 514, 492, 549, 547, 498, 529, 506, 502, 470, 472, 509, 484, 492, 510, 517, 513, 529, 502, 518, 472, 516, 519, 532, 500, 485, 502, 493, 508, 505, 530, 531, 522, 533, 545, 516, 503, 588, 516, 484, 512, 490, 545, 518, 521, 510, 487, 515, 509, 545, 542, 479, 515, 482, 524, 492, 552] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[493, 499, 504, 521, 459, 507, 530, 511, 495, 563, 513, 552, 490, 506, 539, 545, 518, 516, 476, 496, 507, 515, 498, 481, 494, 547, 492, 525, 485, 476, 492, 528, 536, 500, 506, 510, 554, 516, 485, 526, 532, 495, 544, 495, 515, 478, 498, 500, 541, 495, 491, 513, 552, 491, 507, 509, 487, 503, 523, 489, 513, 525, 498, 477, 486, 484, 514, 484, 535, 512, 508, 499, 529, 519, 507, 504, 491, 516, 480, 504, 525, 505, 529, 536, 522, 507, 546, 488, 535, 500, 499, 494, 520, 528, 549, 529, 497, 490, 532, 537, 555, 525, 496, 502, 529, 534, 526, 542, 501, 557, 476, 500, 542, 491, 513, 483, 497, 538, 541, 545, 496, 545, 497, 493, 526, 547, 514, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[525, 489, 506, 501, 491, 511, 502, 496, 546, 535, 490, 492, 521, 517, 533, 527, 16, 11, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 513, 518, 502, 534, 511, 521, 504, 502, 508, 516, 533, 503, 541, 499, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 492, 479, 499, 497, 486, 474, 501, 508, 496, 524, 502, 503, 525, 501, 472, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 483, 505, 527, 480, 510, 500, 501, 513, 525, 503, 487, 559, 508, 525, 508, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 527, 504, 541, 570, 496, 500, 525, 524, 514, 466, 535, 532, 521, 502, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 511, 488, 512, 496, 504, 538, 534, 532, 508, 528, 507, 506, 566, 469, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 531, 550, 488, 529, 507, 487, 497, 527, 506, 527, 505, 464, 490, 503, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 548, 507, 491, 554, 541, 512, 515, 542, 516, 499, 500, 532, 541, 558, 438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8215, 8232, 8039, 8110, 8238, 8233, 8141, 8328] avg=8192.0000 max=8328 max_rank=7 min=8039 min_rank=2 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=32940 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[497, 516, 498, 534, 523, 486, 498, 531, 496, 512, 481, 469, 491, 475, 529, 537, 534, 488, 501, 520, 506, 529, 539, 498, 540, 536, 523, 556, 505, 498, 501, 499, 487, 527, 470, 504, 499, 473, 513, 526, 512, 491, 491, 493, 521, 521, 569, 489, 515, 530, 470, 480, 509, 513, 497, 532, 502, 535, 527, 524, 512, 497, 503, 523, 527, 503, 526, 506, 481, 493, 509, 558, 519, 542, 490, 498, 554, 516, 529, 536, 511, 520, 562, 499, 473, 497, 504, 500, 528, 551, 491, 516, 557, 510, 513, 534, 514, 500, 541, 504, 516, 513, 536, 506, 509, 492, 518, 527, 484, 488, 506, 509, 561, 526, 511, 460, 508, 493, 518, 520, 551, 524, 522, 476, 504, 478, 514, 553] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[514, 498, 532, 483, 498, 511, 552, 546, 491, 510, 535, 475, 526, 525, 482, 510, 15, 11, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 508, 464, 533, 533, 524, 530, 518, 495, 517, 541, 567, 505, 518, 494, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 520, 516, 490, 512, 497, 508, 505, 521, 511, 458, 487, 526, 521, 519, 477, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 501, 490, 467, 499, 509, 519, 511, 527, 524, 516, 514, 512, 521, 538, 501, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 546, 498, 554, 519, 573, 503, 515, 516, 509, 485, 518, 537, 505, 470, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 499, 510, 531, 530, 475, 562, 530, 493, 487, 498, 489, 508, 527, 523, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 516, 482, 513, 512, 514, 506, 509, 518, 499, 546, 524, 492, 503, 503, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 500, 516, 536, 506, 555, 511, 464, 521, 524, 542, 529, 539, 511, 509, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8287, 8095, 8147, 8244, 8143, 8147, 8252] avg=8192.0000 max=8287 max_rank=1 min=8095 min_rank=2 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32786 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[478, 531, 508, 491, 495, 486, 532, 506, 499, 484, 519, 519, 531, 540, 558, 515, 16, 11, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 514, 472, 505, 505, 496, 518, 518, 493, 521, 521, 509, 478, 518, 501, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 537, 515, 519, 497, 491, 532, 496, 497, 534, 512, 517, 543, 501, 538, 521, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 545, 532, 549, 534, 524, 518, 505, 523, 494, 545, 494, 529, 513, 481, 484, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 524, 513, 500, 500, 526, 505, 515, 523, 508, 530, 530, 507, 514, 482, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 493, 493, 500, 531, 519, 509, 525, 560, 520, 493, 491, 511, 479, 526, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 505, 521, 515, 505, 508, 557, 516, 488, 534, 498, 493, 516, 515, 512, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 541, 493, 503, 505, 522, 494, 494, 538, 490, 505, 510, 490, 520, 490, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 8112, 8325, 8304, 8169, 8142, 8195, 8063] avg=8192.0000 max=8325 max_rank=2 min=8063 min_rank=7 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32569 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[518, 520, 530, 553, 510, 506, 506, 498, 502, 512, 474, 497, 521, 512, 485, 483, 15, 11, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 566, 510, 500, 515, 534, 539, 490, 533, 483, 478, 515, 549, 501, 498, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 496, 516, 548, 506, 512, 526, 475, 509, 485, 557, 462, 504, 518, 501, 514, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 543, 545, 469, 520, 525, 505, 531, 540, 517, 504, 519, 529, 503, 516, 530, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 489, 494, 507, 517, 510, 513, 496, 514, 492, 549, 540, 498, 529, 506, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 472, 509, 484, 492, 510, 517, 513, 529, 502, 518, 472, 516, 519, 532, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 502, 493, 497, 505, 530, 531, 522, 533, 545, 516, 503, 588, 516, 484, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 545, 518, 521, 510, 487, 515, 509, 545, 542, 479, 515, 482, 524, 492, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8160, 8247, 8180, 8333, 8145, 8040, 8262, 8169] avg=8192.0000 max=8333 max_rank=3 min=8040 min_rank=5 imbalance_ratio=1.017212 max_load_reduction=57203 max_load_reduction_pct=0.872849 moved_to_echo=32616 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[525, 489, 506, 501, 491, 511, 502, 496, 546, 535, 490, 492, 521, 517, 533, 527, 16, 11, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 513, 518, 502, 534, 511, 521, 504, 502, 508, 516, 533, 503, 541, 499, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 492, 479, 499, 497, 486, 474, 501, 508, 496, 524, 502, 503, 525, 501, 472, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 483, 505, 527, 480, 510, 500, 501, 513, 525, 503, 487, 559, 508, 525, 508, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 527, 504, 541, 570, 496, 500, 525, 524, 514, 466, 535, 532, 521, 502, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 511, 488, 512, 496, 504, 538, 534, 532, 508, 528, 507, 506, 566, 469, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 531, 550, 488, 529, 507, 487, 497, 527, 506, 527, 505, 464, 490, 503, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 548, 507, 491, 554, 541, 512, 515, 542, 516, 499, 500, 532, 541, 558, 438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8215, 8232, 8039, 8110, 8238, 8233, 8141, 8328] avg=8192.0000 max=8328 max_rank=7 min=8039 min_rank=2 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=32940 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[478, 531, 508, 491, 495, 486, 532, 506, 499, 484, 519, 519, 531, 540, 558, 515, 16, 11, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 514, 472, 505, 505, 496, 518, 518, 493, 521, 521, 509, 478, 518, 501, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 537, 515, 519, 497, 491, 532, 496, 497, 534, 512, 517, 543, 501, 538, 521, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 545, 532, 549, 534, 524, 518, 505, 523, 494, 545, 494, 529, 513, 481, 484, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 524, 513, 500, 500, 526, 505, 515, 523, 508, 530, 530, 507, 514, 482, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 493, 493, 500, 531, 519, 509, 525, 560, 520, 493, 491, 511, 479, 526, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 505, 521, 515, 505, 508, 557, 516, 488, 534, 498, 493, 516, 515, 512, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 541, 493, 503, 505, 522, 494, 494, 538, 490, 505, 510, 490, 520, 490, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 8112, 8325, 8304, 8169, 8142, 8195, 8063] avg=8192.0000 max=8325 max_rank=2 min=8063 min_rank=7 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32569 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[493, 499, 504, 521, 459, 507, 530, 511, 495, 563, 513, 552, 490, 506, 539, 545, 16, 11, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 516, 476, 496, 507, 515, 498, 481, 494, 547, 492, 520, 485, 476, 492, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 500, 506, 510, 554, 516, 485, 526, 532, 495, 544, 495, 515, 478, 498, 500, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 495, 491, 513, 552, 491, 507, 509, 487, 503, 523, 489, 513, 525, 498, 477, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 484, 514, 484, 535, 512, 508, 499, 529, 519, 507, 497, 491, 516, 480, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 505, 529, 536, 522, 507, 546, 488, 535, 500, 499, 494, 520, 528, 549, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 490, 532, 526, 555, 525, 496, 502, 529, 534, 526, 542, 501, 557, 476, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 491, 513, 483, 497, 538, 541, 545, 496, 545, 497, 493, 526, 547, 514, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8261, 8041, 8237, 8125, 8065, 8296, 8288, 8223] avg=8192.0000 max=8296 max_rank=5 min=8041 min_rank=1 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32872 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[497, 516, 498, 534, 523, 486, 498, 531, 496, 512, 481, 469, 491, 475, 529, 537, 16, 11, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 488, 501, 520, 506, 529, 539, 498, 540, 536, 523, 551, 505, 498, 501, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 527, 470, 504, 499, 473, 513, 526, 512, 491, 491, 493, 521, 521, 569, 489, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 530, 470, 480, 509, 513, 497, 532, 502, 535, 527, 524, 512, 497, 503, 523, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 503, 526, 506, 481, 493, 509, 558, 519, 542, 490, 491, 554, 516, 529, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 520, 562, 499, 473, 497, 504, 500, 528, 551, 491, 516, 557, 510, 513, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 500, 541, 493, 516, 513, 536, 506, 509, 492, 518, 527, 484, 488, 506, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 526, 511, 460, 508, 493, 518, 520, 551, 524, 522, 476, 504, 478, 514, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8107, 8268, 8137, 8180, 8280, 8250, 8152, 8162] avg=8192.0000 max=8280 max_rank=4 min=8107 min_rank=0 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32844 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[533, 508, 568, 520, 553, 516, 511, 472, 514, 488, 507, 516, 523, 513, 483, 546, 16, 11, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 517, 485, 498, 500, 509, 502, 526, 493, 532, 547, 472, 459, 493, 524, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 501, 481, 521, 516, 487, 495, 515, 509, 539, 500, 569, 519, 533, 538, 519, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 502, 535, 515, 468, 508, 552, 499, 523, 516, 511, 472, 505, 499, 481, 486, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 537, 501, 516, 528, 514, 502, 515, 509, 510, 508, 532, 524, 480, 514, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 520, 525, 512, 494, 488, 512, 497, 512, 517, 549, 454, 539, 502, 527, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 510, 527, 509, 495, 529, 467, 515, 513, 475, 508, 537, 501, 532, 507, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 528, 483, 522, 514, 503, 535, 490, 506, 513, 512, 546, 526, 541, 490, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8305, 8077, 8280, 8112, 8255, 8154, 8145, 8208] avg=8192.0000 max=8305 max_rank=0 min=8077 min_rank=1 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32762 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[484, 485, 479, 518, 496, 477, 496, 516, 474, 501, 495, 529, 489, 508, 526, 516, 20, 15, 10, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 524, 507, 523, 488, 592, 498, 553, 507, 502, 540, 516, 547, 501, 499, 450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 497, 486, 509, 500, 498, 493, 502, 526, 492, 510, 496, 516, 510, 537, 564, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 510, 501, 537, 527, 541, 492, 508, 510, 468, 514, 507, 510, 461, 540, 534, 11, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 500, 521, 517, 480, 506, 491, 496, 499, 515, 564, 539, 532, 496, 529, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 527, 512, 508, 504, 514, 496, 570, 502, 508, 540, 501, 550, 511, 501, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 528, 475, 525, 525, 501, 513, 520, 518, 511, 511, 520, 522, 476, 547, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 518, 517, 534, 519, 525, 475, 514, 495, 505, 480, 526, 517, 525, 486, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8041, 8272, 8243, 8225, 8140, 8278, 8206, 8131] avg=8192.0000 max=8278 max_rank=5 min=8041 min_rank=0 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32755 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[533, 508, 568, 520, 553, 516, 511, 472, 514, 488, 507, 516, 523, 513, 483, 546, 16, 11, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 517, 485, 498, 500, 509, 502, 526, 493, 532, 547, 472, 459, 493, 524, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 501, 481, 521, 516, 487, 495, 515, 509, 539, 500, 569, 519, 533, 538, 519, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 502, 535, 515, 468, 508, 552, 499, 523, 516, 511, 472, 505, 499, 481, 486, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 537, 501, 516, 528, 514, 502, 515, 509, 510, 508, 532, 524, 480, 514, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 520, 525, 512, 494, 488, 512, 497, 512, 517, 549, 454, 539, 502, 527, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 510, 527, 509, 495, 529, 467, 515, 513, 475, 508, 537, 501, 532, 507, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 528, 483, 522, 514, 503, 535, 490, 506, 513, 512, 546, 526, 541, 490, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8305, 8077, 8280, 8112, 8255, 8154, 8145, 8208] avg=8192.0000 max=8305 max_rank=0 min=8077 min_rank=1 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32762 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 256, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4224, 3968, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 3968, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 3968, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 3968, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4224, 3968, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 3968, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[524, 497, 550, 517, 531, 558, 503, 536, 519, 568, 512, 498, 514, 503, 487, 475, 490, 505, 494, 503, 560, 502, 537, 547, 497, 553, 549, 521, 554, 538, 528, 547, 519, 502, 482, 519, 513, 535, 517, 542, 478, 507, 537, 540, 497, 517, 510, 524, 537, 476, 502, 512, 494, 476, 480, 534, 541, 477, 504, 528, 489, 515, 533, 527, 487, 539, 483, 504, 488, 537, 498, 500, 486, 497, 485, 536, 539, 507, 492, 506, 516, 497, 556, 485, 512, 528, 507, 502, 523, 505, 509, 507, 480, 523, 516, 485, 532, 467, 466, 533, 502, 465, 496, 498, 474, 507, 482, 519, 502, 512, 498, 522, 462, 503, 534, 542, 502, 533, 524, 511, 542, 509, 474, 494, 541, 497, 539, 538] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[540, 491, 509, 547, 487, 467, 498, 527, 479, 496, 515, 444, 481, 529, 539, 541, 537, 530, 476, 513, 518, 498, 496, 508, 471, 537, 502, 554, 546, 505, 489, 503, 525, 518, 514, 512, 527, 516, 536, 522, 481, 515, 549, 480, 534, 492, 500, 486, 476, 529, 518, 514, 541, 535, 520, 541, 485, 513, 506, 554, 511, 529, 526, 508, 560, 499, 510, 521, 489, 496, 513, 509, 469, 532, 517, 502, 463, 512, 522, 511, 486, 518, 494, 518, 477, 500, 490, 555, 496, 487, 524, 518, 530, 521, 485, 513, 493, 507, 543, 506, 540, 561, 511, 534, 521, 507, 510, 481, 524, 494, 474, 493, 508, 491, 541, 499, 499, 529, 524, 528, 467, 546, 511, 536, 553, 533, 525, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[506, 516, 491, 488, 484, 538, 539, 513, 537, 495, 529, 529, 547, 527, 524, 501, 499, 544, 519, 572, 501, 514, 511, 483, 501, 483, 513, 493, 539, 497, 544, 529, 515, 495, 506, 513, 536, 569, 499, 519, 514, 487, 509, 512, 462, 523, 477, 458, 492, 527, 499, 540, 511, 486, 500, 488, 500, 482, 513, 538, 501, 485, 495, 529, 538, 495, 513, 532, 527, 517, 465, 528, 538, 533, 540, 501, 503, 508, 504, 537, 506, 520, 504, 545, 525, 514, 530, 533, 484, 517, 495, 510, 524, 513, 497, 498, 515, 538, 502, 516, 510, 522, 518, 524, 534, 525, 501, 525, 501, 510, 547, 482, 498, 499, 542, 506, 513, 527, 523, 469, 509, 497, 473, 518, 489, 489, 514, 520] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[480, 499, 493, 522, 499, 503, 565, 507, 504, 497, 536, 499, 526, 487, 487, 503, 480, 517, 545, 492, 524, 505, 537, 509, 521, 516, 461, 509, 495, 478, 528, 498, 503, 504, 490, 527, 510, 501, 490, 500, 529, 482, 484, 498, 526, 504, 548, 563, 497, 518, 520, 501, 517, 517, 539, 488, 497, 519, 541, 536, 528, 510, 505, 511, 534, 517, 505, 507, 494, 535, 539, 552, 490, 531, 527, 573, 509, 495, 472, 535, 524, 448, 490, 548, 522, 490, 499, 482, 459, 521, 506, 562, 486, 523, 541, 508, 483, 517, 533, 504, 535, 488, 486, 536, 540, 522, 487, 507, 523, 523, 538, 488, 515, 500, 527, 489, 515, 528, 509, 530, 559, 542, 518, 527, 473, 508, 507, 530] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[507, 482, 523, 571, 526, 539, 527, 512, 518, 520, 494, 546, 539, 532, 518, 480, 516, 523, 504, 540, 529, 512, 500, 546, 524, 509, 507, 538, 515, 478, 489, 484, 520, 523, 523, 532, 476, 548, 499, 476, 513, 522, 540, 543, 556, 504, 505, 521, 550, 515, 524, 547, 524, 505, 543, 528, 508, 505, 507, 482, 499, 542, 491, 464, 503, 524, 487, 522, 492, 530, 497, 482, 533, 477, 552, 518, 480, 528, 502, 512, 486, 501, 543, 509, 500, 530, 517, 505, 518, 500, 531, 466, 490, 489, 487, 522, 493, 498, 540, 532, 470, 522, 502, 501, 490, 508, 489, 492, 513, 543, 512, 519, 457, 526, 500, 530, 509, 550, 490, 479, 523, 491, 505, 529, 524, 478, 496, 509] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[506, 516, 491, 488, 484, 538, 539, 513, 537, 495, 529, 529, 547, 527, 524, 501, 499, 544, 519, 572, 501, 514, 511, 483, 501, 483, 513, 493, 539, 497, 544, 529, 515, 495, 506, 513, 536, 569, 499, 519, 514, 487, 509, 512, 462, 523, 477, 458, 492, 527, 499, 540, 511, 486, 500, 488, 500, 482, 513, 538, 501, 485, 495, 529, 538, 495, 513, 532, 527, 517, 465, 528, 538, 533, 540, 501, 503, 508, 504, 537, 506, 520, 504, 545, 525, 514, 530, 533, 484, 517, 495, 510, 524, 513, 497, 498, 515, 538, 502, 516, 510, 522, 518, 524, 534, 525, 501, 525, 501, 510, 547, 482, 498, 499, 542, 506, 513, 527, 523, 469, 509, 497, 473, 518, 489, 489, 514, 520] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[517, 541, 524, 523, 527, 496, 561, 528, 543, 514, 529, 489, 493, 521, 517, 490, 522, 538, 509, 512, 523, 519, 504, 519, 493, 498, 479, 521, 519, 509, 516, 493, 508, 481, 468, 509, 522, 503, 523, 520, 528, 513, 503, 541, 521, 464, 468, 521, 507, 514, 510, 502, 534, 509, 511, 476, 511, 486, 512, 504, 487, 519, 537, 547, 527, 523, 506, 531, 487, 514, 520, 487, 539, 527, 533, 502, 533, 512, 535, 477, 499, 500, 526, 472, 465, 490, 508, 526, 464, 483, 519, 506, 518, 531, 496, 487, 518, 518, 550, 536, 503, 547, 510, 531, 535, 532, 519, 529, 479, 483, 499, 497, 476, 520, 504, 516, 530, 560, 508, 497, 551, 557, 511, 544, 507, 497, 467, 516] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[540, 500, 523, 500, 520, 521, 532, 513, 473, 498, 484, 503, 463, 510, 515, 547, 528, 553, 475, 486, 468, 556, 496, 523, 502, 507, 496, 501, 529, 502, 470, 527, 557, 524, 489, 518, 497, 576, 509, 493, 520, 504, 485, 490, 534, 550, 527, 503, 530, 534, 505, 524, 517, 488, 532, 503, 515, 512, 492, 504, 529, 517, 453, 525, 502, 495, 496, 521, 487, 486, 565, 501, 500, 539, 498, 542, 508, 464, 505, 501, 529, 538, 510, 493, 484, 506, 496, 529, 510, 529, 531, 481, 504, 535, 500, 530, 487, 519, 509, 547, 512, 535, 534, 466, 520, 505, 509, 497, 489, 532, 528, 517, 559, 553, 511, 544, 515, 510, 556, 469, 494, 507, 527, 529, 507, 481, 517, 519] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[540, 491, 509, 547, 487, 467, 498, 527, 479, 496, 515, 444, 481, 529, 539, 541, 537, 530, 476, 513, 518, 498, 496, 508, 471, 537, 502, 554, 546, 505, 489, 503, 525, 518, 514, 512, 527, 516, 536, 522, 481, 515, 549, 480, 534, 492, 500, 486, 476, 529, 518, 514, 541, 535, 520, 541, 485, 513, 506, 554, 511, 529, 526, 508, 560, 499, 510, 521, 489, 496, 513, 509, 469, 532, 517, 502, 463, 512, 522, 511, 486, 518, 494, 518, 477, 500, 490, 555, 496, 487, 524, 518, 530, 521, 485, 513, 493, 507, 543, 506, 540, 561, 511, 534, 521, 507, 510, 481, 524, 494, 474, 493, 508, 491, 541, 499, 499, 529, 524, 528, 467, 546, 511, 536, 553, 533, 525, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[480, 499, 493, 522, 499, 503, 565, 507, 504, 497, 536, 499, 526, 487, 487, 503, 480, 517, 545, 492, 524, 505, 537, 509, 521, 516, 461, 509, 495, 478, 528, 498, 503, 504, 490, 527, 510, 501, 490, 500, 529, 482, 484, 498, 526, 504, 548, 563, 497, 518, 520, 501, 517, 517, 539, 488, 497, 519, 541, 536, 528, 510, 505, 511, 534, 517, 505, 507, 494, 535, 539, 552, 490, 531, 527, 573, 509, 495, 472, 535, 524, 448, 490, 548, 522, 490, 499, 482, 459, 521, 506, 562, 486, 523, 541, 508, 483, 517, 533, 504, 535, 488, 486, 536, 540, 522, 487, 507, 523, 523, 538, 488, 515, 500, 527, 489, 515, 528, 509, 530, 559, 542, 518, 527, 473, 508, 507, 530] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[524, 497, 550, 517, 531, 558, 503, 536, 519, 568, 512, 498, 514, 503, 487, 475, 490, 505, 494, 503, 560, 502, 537, 547, 497, 553, 549, 521, 554, 538, 528, 547, 519, 502, 482, 519, 513, 535, 517, 542, 478, 507, 537, 540, 497, 517, 510, 524, 537, 476, 502, 512, 494, 476, 480, 534, 541, 477, 504, 528, 489, 515, 533, 527, 487, 539, 483, 504, 488, 537, 498, 500, 486, 497, 485, 536, 539, 507, 492, 506, 516, 497, 556, 485, 512, 528, 507, 502, 523, 505, 509, 507, 480, 523, 516, 485, 532, 467, 466, 533, 502, 465, 496, 498, 474, 507, 482, 519, 502, 512, 498, 522, 462, 503, 534, 542, 502, 533, 524, 511, 542, 509, 474, 494, 541, 497, 539, 538] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[557, 516, 544, 531, 496, 521, 526, 526, 490, 527, 545, 490, 501, 492, 525, 477, 543, 508, 475, 524, 517, 508, 504, 526, 471, 491, 471, 496, 561, 493, 523, 533, 522, 524, 527, 484, 516, 480, 483, 515, 528, 479, 522, 507, 492, 534, 483, 477, 547, 486, 498, 487, 545, 538, 529, 550, 498, 510, 531, 501, 521, 524, 564, 513, 502, 517, 485, 546, 519, 511, 488, 520, 517, 503, 509, 532, 496, 499, 526, 483, 468, 511, 488, 537, 528, 525, 504, 524, 535, 478, 526, 504, 512, 500, 513, 522, 483, 509, 571, 497, 505, 562, 528, 533, 498, 486, 587, 485, 505, 569, 496, 533, 510, 509, 542, 525, 521, 481, 427, 568, 502, 499, 486, 502, 495, 511, 468, 492] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[540, 491, 509, 547, 487, 467, 467, 527, 479, 496, 515, 444, 481, 529, 539, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 530, 476, 513, 518, 498, 496, 508, 471, 537, 502, 554, 536, 505, 489, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 518, 514, 512, 527, 516, 536, 522, 481, 515, 549, 480, 534, 492, 500, 486, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 529, 518, 514, 522, 535, 520, 541, 485, 513, 506, 554, 511, 529, 526, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 560, 499, 510, 521, 489, 496, 513, 509, 469, 532, 517, 502, 463, 512, 522, 511, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 518, 494, 518, 477, 500, 490, 555, 496, 487, 524, 518, 530, 521, 485, 513, 31, 19, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 507, 534, 506, 540, 561, 511, 534, 521, 507, 510, 481, 524, 494, 474, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 491, 541, 499, 499, 520, 524, 528, 467, 546, 511, 536, 553, 533, 525, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8059, 8173, 8218, 8287, 8134, 8170, 8190, 8305] avg=8192.0000 max=8305 max_rank=7 min=8059 min_rank=0 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32799 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[507, 482, 523, 571, 526, 539, 494, 512, 518, 520, 494, 546, 539, 532, 518, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 523, 504, 540, 529, 512, 500, 546, 524, 509, 507, 538, 506, 478, 489, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 523, 523, 532, 476, 548, 499, 476, 513, 522, 540, 543, 556, 504, 505, 521, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 515, 524, 547, 506, 505, 543, 528, 508, 505, 507, 482, 499, 542, 491, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 524, 487, 522, 492, 530, 497, 482, 533, 477, 552, 518, 480, 528, 502, 512, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 501, 543, 509, 500, 530, 517, 505, 518, 500, 531, 466, 490, 489, 487, 522, 33, 18, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 498, 531, 532, 470, 522, 502, 501, 490, 508, 489, 492, 513, 543, 512, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 457, 526, 500, 530, 509, 540, 490, 479, 523, 491, 505, 529, 524, 478, 496, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8301, 8205, 8311, 8216, 8149, 8153, 8115, 8086] avg=8192.0000 max=8311 max_rank=2 min=8086 min_rank=7 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32503 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[506, 516, 491, 488, 484, 538, 505, 513, 537, 495, 529, 529, 547, 527, 524, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 544, 519, 572, 501, 514, 511, 483, 501, 483, 513, 493, 529, 497, 544, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 495, 506, 513, 536, 569, 499, 519, 514, 487, 509, 512, 462, 523, 477, 458, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 527, 499, 540, 494, 486, 500, 488, 500, 482, 513, 538, 501, 485, 495, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 495, 513, 532, 527, 517, 465, 528, 538, 533, 540, 501, 503, 508, 504, 537, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 520, 504, 545, 525, 514, 530, 533, 484, 517, 495, 510, 524, 513, 497, 498, 34, 17, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 538, 493, 516, 510, 522, 518, 524, 534, 525, 501, 525, 501, 510, 547, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 499, 542, 506, 513, 518, 523, 469, 509, 497, 473, 518, 489, 489, 514, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 8232, 8105, 8069, 8288, 8274, 8261, 8077] avg=8192.0000 max=8288 max_rank=4 min=8069 min_rank=3 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32900 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[480, 499, 493, 522, 499, 503, 525, 507, 504, 497, 536, 499, 526, 487, 487, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 517, 545, 492, 524, 505, 537, 509, 521, 516, 461, 509, 482, 478, 528, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 504, 490, 527, 510, 501, 490, 500, 529, 482, 484, 498, 526, 504, 548, 563, 13, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 518, 520, 501, 496, 517, 539, 488, 497, 519, 541, 536, 528, 510, 505, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 517, 505, 507, 494, 535, 539, 552, 490, 531, 527, 573, 509, 495, 472, 535, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 448, 490, 548, 522, 490, 499, 482, 459, 521, 506, 562, 486, 523, 541, 508, 40, 21, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 517, 515, 504, 535, 488, 486, 536, 540, 522, 487, 507, 523, 523, 538, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 500, 527, 489, 515, 513, 509, 530, 559, 542, 518, 527, 473, 508, 507, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8067, 8102, 8179, 8223, 8329, 8182, 8192, 8262] avg=8192.0000 max=8329 max_rank=4 min=8067 min_rank=0 imbalance_ratio=1.016724 max_load_reduction=57207 max_load_reduction_pct=0.872910 moved_to_echo=32965 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[524, 497, 550, 517, 531, 558, 472, 536, 519, 568, 512, 498, 514, 503, 487, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 505, 494, 503, 560, 502, 537, 547, 497, 553, 549, 521, 544, 538, 528, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 502, 482, 519, 513, 535, 517, 542, 478, 507, 537, 540, 497, 517, 510, 524, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 476, 502, 512, 477, 476, 480, 534, 541, 477, 504, 528, 489, 515, 533, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 539, 483, 504, 488, 537, 498, 500, 486, 497, 485, 536, 539, 507, 492, 506, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 497, 556, 485, 512, 528, 507, 502, 523, 505, 509, 507, 480, 523, 516, 485, 31, 17, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 467, 458, 533, 502, 465, 496, 498, 474, 507, 482, 519, 502, 512, 498, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 503, 534, 542, 502, 524, 524, 511, 542, 509, 474, 494, 541, 497, 539, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8261, 8415, 8250, 8108, 8093, 8206, 7967, 8236] avg=8192.0000 max=8415 max_rank=1 min=7967 min_rank=6 imbalance_ratio=1.027222 max_load_reduction=57121 max_load_reduction_pct=0.871597 moved_to_echo=32502 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[517, 541, 524, 523, 527, 496, 526, 528, 543, 514, 529, 489, 493, 521, 517, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 538, 509, 512, 523, 519, 504, 519, 493, 498, 479, 521, 510, 509, 516, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 481, 468, 509, 522, 503, 523, 520, 528, 513, 503, 541, 521, 464, 468, 521, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 514, 510, 502, 516, 509, 511, 476, 511, 486, 512, 504, 487, 519, 537, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 523, 506, 531, 487, 514, 520, 487, 539, 527, 533, 502, 533, 512, 535, 477, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 500, 526, 472, 465, 490, 508, 526, 464, 483, 519, 506, 518, 531, 496, 487, 35, 18, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 518, 541, 536, 503, 547, 510, 531, 535, 532, 519, 529, 479, 483, 499, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 520, 504, 516, 530, 550, 508, 497, 551, 557, 511, 544, 507, 497, 467, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8278, 8165, 8103, 8148, 8263, 8051, 8277, 8251] avg=8192.0000 max=8278 max_rank=0 min=8051 min_rank=5 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32842 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[540, 491, 509, 547, 487, 467, 467, 527, 479, 496, 515, 444, 481, 529, 539, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 530, 476, 513, 518, 498, 496, 508, 471, 537, 502, 554, 536, 505, 489, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 518, 514, 512, 527, 516, 536, 522, 481, 515, 549, 480, 534, 492, 500, 486, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 529, 518, 514, 522, 535, 520, 541, 485, 513, 506, 554, 511, 529, 526, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 560, 499, 510, 521, 489, 496, 513, 509, 469, 532, 517, 502, 463, 512, 522, 511, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 518, 494, 518, 477, 500, 490, 555, 496, 487, 524, 518, 530, 521, 485, 513, 31, 19, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 507, 534, 506, 540, 561, 511, 534, 521, 507, 510, 481, 524, 494, 474, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 491, 541, 499, 499, 520, 524, 528, 467, 546, 511, 536, 553, 533, 525, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8059, 8173, 8218, 8287, 8134, 8170, 8190, 8305] avg=8192.0000 max=8305 max_rank=7 min=8059 min_rank=0 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32799 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[540, 500, 523, 500, 520, 521, 499, 513, 473, 498, 484, 503, 463, 510, 515, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 553, 475, 486, 468, 556, 496, 523, 502, 507, 496, 501, 520, 502, 470, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 557, 524, 489, 518, 497, 576, 509, 493, 520, 504, 485, 490, 534, 550, 527, 503, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 534, 505, 524, 499, 488, 532, 503, 515, 512, 492, 504, 529, 517, 453, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 495, 496, 521, 487, 486, 565, 501, 500, 539, 498, 542, 508, 464, 505, 501, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 538, 510, 493, 484, 506, 496, 529, 510, 529, 531, 481, 504, 535, 500, 530, 33, 18, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 519, 500, 547, 512, 535, 534, 466, 520, 505, 509, 497, 489, 532, 528, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 553, 511, 544, 515, 501, 556, 469, 494, 507, 527, 529, 507, 481, 517, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8109, 8110, 8286, 8162, 8119, 8264, 8197, 8289] avg=8192.0000 max=8289 max_rank=7 min=8109 min_rank=0 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32869 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[507, 482, 523, 571, 526, 539, 494, 512, 518, 520, 494, 546, 539, 532, 518, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 523, 504, 540, 529, 512, 500, 546, 524, 509, 507, 538, 506, 478, 489, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 523, 523, 532, 476, 548, 499, 476, 513, 522, 540, 543, 556, 504, 505, 521, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 515, 524, 547, 506, 505, 543, 528, 508, 505, 507, 482, 499, 542, 491, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 524, 487, 522, 492, 530, 497, 482, 533, 477, 552, 518, 480, 528, 502, 512, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 501, 543, 509, 500, 530, 517, 505, 518, 500, 531, 466, 490, 489, 487, 522, 33, 18, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 498, 531, 532, 470, 522, 502, 501, 490, 508, 489, 492, 513, 543, 512, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 457, 526, 500, 530, 509, 540, 490, 479, 523, 491, 505, 529, 524, 478, 496, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8301, 8205, 8311, 8216, 8149, 8153, 8115, 8086] avg=8192.0000 max=8311 max_rank=2 min=8086 min_rank=7 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32503 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[517, 541, 524, 523, 527, 496, 526, 528, 543, 514, 529, 489, 493, 521, 517, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 538, 509, 512, 523, 519, 504, 519, 493, 498, 479, 521, 510, 509, 516, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 481, 468, 509, 522, 503, 523, 520, 528, 513, 503, 541, 521, 464, 468, 521, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 514, 510, 502, 516, 509, 511, 476, 511, 486, 512, 504, 487, 519, 537, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 523, 506, 531, 487, 514, 520, 487, 539, 527, 533, 502, 533, 512, 535, 477, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 500, 526, 472, 465, 490, 508, 526, 464, 483, 519, 506, 518, 531, 496, 487, 35, 18, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 518, 541, 536, 503, 547, 510, 531, 535, 532, 519, 529, 479, 483, 499, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 520, 504, 516, 530, 550, 508, 497, 551, 557, 511, 544, 507, 497, 467, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8278, 8165, 8103, 8148, 8263, 8051, 8277, 8251] avg=8192.0000 max=8278 max_rank=0 min=8051 min_rank=5 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32842 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[524, 497, 550, 517, 531, 558, 472, 536, 519, 568, 512, 498, 514, 503, 487, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 505, 494, 503, 560, 502, 537, 547, 497, 553, 549, 521, 544, 538, 528, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 502, 482, 519, 513, 535, 517, 542, 478, 507, 537, 540, 497, 517, 510, 524, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 476, 502, 512, 477, 476, 480, 534, 541, 477, 504, 528, 489, 515, 533, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 539, 483, 504, 488, 537, 498, 500, 486, 497, 485, 536, 539, 507, 492, 506, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 497, 556, 485, 512, 528, 507, 502, 523, 505, 509, 507, 480, 523, 516, 485, 31, 17, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 467, 458, 533, 502, 465, 496, 498, 474, 507, 482, 519, 502, 512, 498, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 503, 534, 542, 502, 524, 524, 511, 542, 509, 474, 494, 541, 497, 539, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8261, 8415, 8250, 8108, 8093, 8206, 7967, 8236] avg=8192.0000 max=8415 max_rank=1 min=7967 min_rank=6 imbalance_ratio=1.027222 max_load_reduction=57121 max_load_reduction_pct=0.871597 moved_to_echo=32502 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[557, 516, 544, 531, 496, 521, 493, 526, 490, 527, 545, 490, 501, 492, 525, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 508, 475, 524, 517, 508, 504, 526, 471, 491, 471, 496, 551, 493, 523, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 524, 527, 484, 516, 480, 483, 515, 528, 479, 522, 507, 492, 534, 483, 477, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 486, 498, 487, 526, 538, 529, 550, 498, 510, 531, 501, 521, 524, 564, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 517, 485, 546, 519, 511, 488, 520, 517, 503, 509, 532, 496, 499, 526, 483, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 511, 488, 537, 528, 525, 504, 524, 535, 478, 526, 504, 512, 500, 513, 522, 33, 19, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 509, 561, 497, 505, 562, 528, 533, 498, 486, 587, 485, 505, 569, 496, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 509, 542, 525, 521, 473, 427, 568, 502, 499, 486, 502, 495, 511, 468, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8231, 8134, 8084, 8323, 8161, 8236, 8337, 8030] avg=8192.0000 max=8337 max_rank=6 min=8030 min_rank=7 imbalance_ratio=1.017700 max_load_reduction=57199 max_load_reduction_pct=0.872787 moved_to_echo=32764 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4352, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 384, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4352, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[495, 520, 522, 507, 475, 495, 478, 511, 505, 487, 515, 515, 479, 504, 519, 512, 518, 508, 530, 511, 502, 501, 554, 474, 482, 528, 519, 530, 488, 512, 525, 511, 470, 531, 502, 522, 513, 514, 502, 521, 485, 518, 506, 540, 529, 479, 494, 505, 517, 515, 522, 499, 526, 513, 505, 533, 472, 515, 514, 522, 526, 509, 522, 510, 547, 512, 566, 494, 543, 523, 502, 481, 516, 494, 500, 495, 514, 479, 508, 546, 512, 517, 515, 539, 525, 530, 528, 524, 473, 545, 540, 480, 504, 514, 517, 486, 526, 501, 534, 478, 527, 530, 521, 508, 485, 498, 491, 500, 527, 511, 510, 571, 539, 533, 499, 496, 502, 526, 478, 554, 512, 521, 558, 497, 490, 510, 512, 539] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[484, 513, 491, 507, 515, 525, 528, 532, 507, 512, 515, 491, 513, 506, 501, 524, 563, 523, 499, 513, 528, 497, 507, 524, 542, 545, 476, 527, 523, 502, 495, 512, 468, 488, 555, 510, 516, 553, 500, 531, 464, 473, 523, 440, 502, 525, 534, 520, 515, 499, 538, 521, 508, 535, 475, 502, 549, 520, 481, 494, 508, 473, 534, 508, 521, 515, 480, 514, 502, 529, 521, 493, 505, 540, 501, 526, 525, 529, 472, 511, 545, 522, 456, 498, 530, 510, 533, 520, 481, 506, 528, 523, 503, 499, 542, 461, 524, 518, 510, 525, 528, 493, 523, 513, 506, 539, 511, 503, 510, 536, 506, 501, 482, 500, 564, 540, 501, 525, 514, 521, 449, 544, 504, 483, 518, 529, 524, 549] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[529, 500, 478, 535, 509, 537, 489, 507, 503, 489, 527, 572, 501, 546, 516, 506, 547, 530, 513, 533, 462, 528, 477, 495, 506, 500, 538, 499, 513, 479, 554, 477, 485, 495, 539, 522, 517, 510, 494, 497, 492, 492, 513, 518, 510, 521, 528, 517, 486, 513, 503, 510, 526, 507, 522, 507, 469, 519, 529, 499, 508, 480, 558, 546, 493, 497, 500, 503, 510, 548, 551, 543, 523, 479, 506, 507, 554, 510, 535, 546, 515, 499, 528, 478, 531, 503, 530, 524, 507, 489, 509, 504, 507, 529, 482, 516, 536, 509, 500, 509, 489, 534, 532, 499, 487, 515, 526, 537, 477, 537, 498, 530, 483, 487, 523, 525, 530, 478, 517, 520, 528, 546, 505, 495, 508, 506, 494, 493] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[522, 533, 495, 528, 517, 514, 502, 477, 477, 486, 506, 531, 500, 541, 505, 511, 563, 519, 509, 481, 501, 514, 541, 504, 507, 504, 531, 494, 483, 504, 492, 494, 521, 524, 559, 547, 501, 539, 510, 522, 517, 533, 479, 519, 508, 482, 501, 534, 523, 508, 528, 519, 523, 511, 515, 538, 480, 510, 529, 540, 546, 517, 469, 519, 504, 531, 496, 504, 522, 530, 574, 528, 532, 513, 490, 467, 507, 561, 477, 541, 510, 511, 504, 486, 533, 514, 490, 511, 529, 506, 466, 511, 517, 516, 502, 524, 484, 575, 499, 490, 522, 483, 513, 457, 467, 531, 477, 539, 547, 490, 521, 502, 538, 494, 514, 507, 518, 498, 537, 499, 512, 519, 496, 476, 557, 504, 477, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[499, 542, 521, 507, 530, 473, 516, 493, 554, 485, 579, 476, 494, 493, 485, 496, 521, 508, 482, 501, 551, 535, 499, 508, 494, 497, 487, 488, 535, 458, 512, 525, 492, 499, 534, 508, 545, 500, 526, 550, 546, 517, 490, 493, 520, 492, 525, 499, 493, 517, 497, 474, 521, 499, 538, 534, 544, 544, 517, 542, 457, 525, 496, 495, 489, 538, 496, 520, 534, 507, 520, 475, 533, 524, 534, 517, 504, 481, 497, 520, 542, 528, 503, 483, 523, 557, 518, 510, 478, 526, 507, 537, 503, 460, 492, 531, 534, 537, 510, 509, 500, 523, 562, 520, 476, 531, 534, 482, 525, 471, 528, 570, 546, 492, 511, 502, 503, 511, 542, 494, 517, 516, 468, 535, 499, 524, 510, 494] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[497, 491, 545, 503, 511, 515, 467, 506, 538, 533, 543, 503, 509, 512, 507, 522, 496, 513, 539, 560, 475, 518, 511, 498, 496, 567, 534, 465, 490, 563, 523, 518, 501, 512, 488, 545, 509, 479, 516, 494, 507, 518, 506, 473, 510, 484, 456, 516, 539, 520, 519, 509, 555, 496, 527, 519, 507, 541, 504, 512, 496, 560, 529, 534, 505, 487, 520, 548, 501, 490, 493, 538, 511, 498, 487, 523, 485, 510, 526, 511, 501, 517, 509, 501, 485, 501, 510, 545, 474, 495, 523, 485, 525, 493, 492, 498, 515, 535, 536, 518, 509, 506, 482, 543, 558, 494, 540, 510, 508, 479, 510, 521, 537, 507, 487, 497, 560, 536, 522, 518, 489, 550, 468, 532, 504, 506, 475, 548] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[513, 519, 465, 534, 549, 504, 506, 522, 492, 497, 502, 510, 508, 531, 512, 527, 504, 504, 542, 483, 514, 498, 486, 520, 490, 538, 459, 491, 468, 524, 537, 490, 504, 501, 523, 521, 494, 530, 529, 520, 550, 534, 522, 490, 530, 501, 495, 500, 521, 541, 526, 506, 509, 542, 518, 523, 487, 577, 512, 539, 509, 515, 524, 452, 526, 488, 531, 510, 497, 514, 504, 524, 524, 505, 491, 478, 497, 543, 512, 555, 536, 539, 512, 500, 526, 547, 476, 525, 507, 514, 465, 475, 520, 535, 516, 500, 508, 509, 499, 520, 497, 509, 533, 532, 530, 484, 530, 507, 539, 500, 498, 509, 527, 535, 493, 471, 495, 502, 496, 519, 519, 524, 547, 499, 491, 542, 472, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[529, 500, 478, 535, 509, 537, 489, 507, 503, 489, 527, 572, 501, 546, 516, 506, 547, 530, 513, 533, 462, 528, 477, 495, 506, 500, 538, 499, 513, 479, 554, 477, 485, 495, 539, 522, 517, 510, 494, 497, 492, 492, 513, 518, 510, 521, 528, 517, 486, 513, 503, 510, 526, 507, 522, 507, 469, 519, 529, 499, 508, 480, 558, 546, 493, 497, 500, 503, 510, 548, 551, 543, 523, 479, 506, 507, 554, 510, 535, 546, 515, 499, 528, 478, 531, 503, 530, 524, 507, 489, 509, 504, 507, 529, 482, 516, 536, 509, 500, 509, 489, 534, 532, 499, 487, 515, 526, 537, 477, 537, 498, 530, 483, 487, 523, 525, 530, 478, 517, 520, 528, 546, 505, 495, 508, 506, 494, 493] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[480, 495, 480, 485, 523, 516, 487, 524, 524, 521, 513, 511, 535, 511, 519, 559, 498, 510, 529, 529, 496, 478, 478, 511, 553, 512, 511, 507, 461, 483, 528, 555, 505, 505, 522, 499, 473, 530, 509, 495, 543, 523, 524, 538, 514, 528, 529, 529, 516, 526, 491, 536, 499, 513, 557, 539, 526, 487, 500, 495, 535, 507, 474, 486, 497, 499, 494, 498, 528, 531, 497, 541, 543, 544, 526, 499, 520, 517, 496, 531, 515, 494, 514, 531, 536, 497, 516, 540, 486, 492, 522, 485, 540, 515, 523, 501, 541, 475, 510, 477, 536, 518, 548, 467, 502, 506, 503, 490, 510, 506, 517, 544, 505, 504, 481, 520, 518, 472, 519, 508, 493, 509, 519, 538, 520, 523, 459, 555] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[499, 542, 521, 507, 530, 473, 516, 493, 554, 485, 579, 476, 494, 493, 485, 496, 521, 508, 482, 501, 551, 535, 499, 508, 494, 497, 487, 488, 535, 458, 512, 525, 492, 499, 534, 508, 545, 500, 526, 550, 546, 517, 490, 493, 520, 492, 525, 499, 493, 517, 497, 474, 521, 499, 538, 534, 544, 544, 517, 542, 457, 525, 496, 495, 489, 538, 496, 520, 534, 507, 520, 475, 533, 524, 534, 517, 504, 481, 497, 520, 542, 528, 503, 483, 523, 557, 518, 510, 478, 526, 507, 537, 503, 460, 492, 531, 534, 537, 510, 509, 500, 523, 562, 520, 476, 531, 534, 482, 525, 471, 528, 570, 546, 492, 511, 502, 503, 511, 542, 494, 517, 516, 468, 535, 499, 524, 510, 494] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[495, 520, 522, 507, 475, 495, 478, 511, 505, 487, 515, 515, 479, 504, 519, 512, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 508, 530, 511, 502, 501, 554, 474, 482, 528, 519, 530, 488, 512, 525, 511, 15, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 531, 502, 522, 513, 514, 502, 521, 485, 518, 506, 540, 529, 479, 494, 505, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 515, 522, 499, 526, 513, 505, 533, 472, 473, 514, 522, 526, 509, 522, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 512, 566, 494, 543, 523, 502, 481, 516, 494, 500, 495, 514, 479, 508, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 517, 515, 539, 525, 530, 528, 524, 473, 545, 540, 480, 504, 514, 517, 486, 17, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 501, 534, 478, 527, 530, 521, 508, 485, 498, 491, 500, 527, 511, 510, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 533, 499, 496, 502, 526, 478, 554, 512, 521, 558, 497, 490, 510, 512, 539, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8066, 8220, 8144, 8178, 8191, 8274, 8196, 8267] avg=8192.0000 max=8274 max_rank=5 min=8066 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32928 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[513, 519, 465, 534, 549, 504, 506, 522, 492, 497, 502, 510, 508, 531, 512, 527, 504, 504, 542, 483, 514, 498, 486, 520, 490, 538, 459, 491, 468, 524, 537, 490, 504, 501, 523, 521, 494, 530, 529, 520, 550, 534, 522, 490, 530, 501, 495, 500, 521, 541, 526, 506, 509, 542, 518, 523, 487, 577, 512, 539, 509, 515, 524, 452, 526, 488, 531, 510, 497, 514, 504, 524, 524, 505, 491, 478, 497, 543, 512, 555, 536, 539, 512, 500, 526, 547, 476, 525, 507, 514, 465, 475, 520, 535, 516, 500, 508, 509, 499, 520, 497, 509, 533, 532, 530, 484, 530, 507, 539, 500, 498, 509, 527, 535, 493, 471, 495, 502, 496, 519, 519, 524, 547, 499, 491, 542, 472, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[522, 533, 495, 528, 517, 514, 502, 477, 477, 486, 506, 531, 500, 541, 505, 511, 563, 519, 509, 481, 501, 514, 541, 504, 507, 504, 531, 494, 483, 504, 492, 494, 521, 524, 559, 547, 501, 539, 510, 522, 517, 533, 479, 519, 508, 482, 501, 534, 523, 508, 528, 519, 523, 511, 515, 538, 480, 510, 529, 540, 546, 517, 469, 519, 504, 531, 496, 504, 522, 530, 574, 528, 532, 513, 490, 467, 507, 561, 477, 541, 510, 511, 504, 486, 533, 514, 490, 511, 529, 506, 466, 511, 517, 516, 502, 524, 484, 575, 499, 490, 522, 483, 513, 457, 467, 531, 477, 539, 547, 490, 521, 502, 538, 494, 514, 507, 518, 498, 537, 499, 512, 519, 496, 476, 557, 504, 477, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[495, 520, 522, 507, 475, 495, 478, 511, 505, 487, 515, 515, 479, 504, 519, 512, 518, 508, 530, 511, 502, 501, 554, 474, 482, 528, 519, 530, 488, 512, 525, 511, 470, 531, 502, 522, 513, 514, 502, 521, 485, 518, 506, 540, 529, 479, 494, 505, 517, 515, 522, 499, 526, 513, 505, 533, 472, 515, 514, 522, 526, 509, 522, 510, 547, 512, 566, 494, 543, 523, 502, 481, 516, 494, 500, 495, 514, 479, 508, 546, 512, 517, 515, 539, 525, 530, 528, 524, 473, 545, 540, 480, 504, 514, 517, 486, 526, 501, 534, 478, 527, 530, 521, 508, 485, 498, 491, 500, 527, 511, 510, 571, 539, 533, 499, 496, 502, 526, 478, 554, 512, 521, 558, 497, 490, 510, 512, 539] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[484, 513, 491, 507, 515, 525, 528, 532, 507, 512, 515, 491, 513, 506, 501, 524, 563, 523, 499, 513, 528, 497, 507, 524, 542, 545, 476, 527, 523, 502, 495, 512, 468, 488, 555, 510, 516, 553, 500, 531, 464, 473, 523, 440, 502, 525, 534, 520, 515, 499, 538, 521, 508, 535, 475, 502, 549, 520, 481, 494, 508, 473, 534, 508, 521, 515, 480, 514, 502, 529, 521, 493, 505, 540, 501, 526, 525, 529, 472, 511, 545, 522, 456, 498, 530, 510, 533, 520, 481, 506, 528, 523, 503, 499, 542, 461, 524, 518, 510, 525, 528, 493, 523, 513, 506, 539, 511, 503, 510, 536, 506, 501, 482, 500, 564, 540, 501, 525, 514, 521, 449, 544, 504, 483, 518, 529, 524, 549] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[497, 491, 545, 503, 511, 515, 467, 506, 538, 533, 543, 503, 509, 512, 507, 522, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 513, 539, 560, 475, 518, 511, 498, 496, 567, 534, 465, 490, 563, 523, 518, 15, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 512, 488, 545, 509, 479, 516, 494, 507, 518, 506, 473, 510, 484, 456, 516, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 520, 519, 509, 555, 496, 527, 519, 507, 498, 504, 512, 496, 560, 529, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 487, 520, 548, 501, 490, 493, 538, 511, 498, 487, 523, 485, 510, 526, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 517, 509, 501, 485, 501, 510, 545, 474, 495, 523, 485, 525, 493, 492, 498, 16, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 535, 536, 518, 509, 506, 482, 543, 558, 494, 540, 510, 508, 479, 510, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 507, 487, 497, 560, 536, 522, 518, 489, 550, 468, 532, 504, 506, 475, 548, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 8292, 8025, 8324, 8106, 8077, 8245, 8237] avg=8192.0000 max=8324 max_rank=3 min=8025 min_rank=2 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32665 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[513, 519, 465, 534, 549, 504, 506, 522, 492, 497, 502, 510, 508, 531, 512, 527, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 504, 542, 483, 514, 498, 486, 520, 490, 538, 459, 491, 468, 524, 537, 490, 19, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 501, 523, 521, 494, 530, 529, 520, 550, 534, 522, 490, 530, 501, 495, 500, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 541, 526, 506, 509, 542, 518, 523, 487, 524, 512, 539, 509, 515, 524, 452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 488, 531, 510, 497, 514, 504, 524, 524, 505, 491, 478, 497, 543, 512, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 539, 512, 500, 526, 547, 476, 525, 507, 514, 465, 475, 520, 535, 516, 500, 20, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 509, 499, 520, 497, 509, 533, 532, 530, 484, 530, 507, 539, 500, 498, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 535, 493, 471, 495, 502, 496, 519, 519, 524, 547, 499, 491, 542, 472, 524, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 8083, 8258, 8248, 8163, 8224, 8175, 8160] avg=8192.0000 max=8258 max_rank=2 min=8083 min_rank=1 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32722 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[499, 542, 521, 507, 530, 473, 516, 493, 554, 485, 579, 476, 494, 493, 485, 496, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 508, 482, 501, 551, 535, 499, 508, 494, 497, 487, 488, 535, 458, 512, 525, 16, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 499, 534, 508, 545, 500, 526, 550, 546, 517, 490, 493, 520, 492, 525, 499, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 517, 497, 474, 521, 499, 538, 534, 544, 499, 517, 542, 457, 525, 496, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 538, 496, 520, 534, 507, 520, 475, 533, 524, 534, 517, 504, 481, 497, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 528, 503, 483, 523, 557, 518, 510, 478, 526, 507, 537, 503, 460, 492, 531, 16, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 537, 510, 509, 500, 523, 562, 520, 476, 531, 534, 482, 525, 471, 528, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 492, 511, 502, 503, 511, 542, 494, 517, 516, 468, 535, 499, 524, 510, 494, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8128, 8249, 8148, 8162, 8222, 8290, 8165] avg=8192.0000 max=8290 max_rank=6 min=8128 min_rank=1 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32839 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[522, 533, 495, 528, 517, 514, 502, 477, 477, 486, 506, 531, 500, 541, 505, 511, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 563, 519, 509, 481, 501, 514, 541, 504, 507, 504, 531, 494, 483, 504, 492, 494, 15, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 524, 559, 547, 501, 539, 510, 522, 517, 533, 479, 519, 508, 482, 501, 534, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 508, 528, 519, 523, 511, 515, 538, 480, 468, 529, 540, 546, 517, 469, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 531, 496, 504, 522, 530, 574, 528, 532, 513, 490, 467, 507, 561, 477, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 511, 504, 486, 533, 514, 490, 511, 529, 506, 466, 511, 517, 516, 502, 524, 17, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 575, 499, 490, 522, 483, 513, 457, 467, 531, 477, 539, 547, 490, 521, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 494, 514, 507, 518, 498, 537, 499, 512, 519, 496, 476, 557, 504, 477, 529, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8168, 8307, 8233, 8248, 8154, 8078, 8176] avg=8192.0000 max=8307 max_rank=2 min=8078 min_rank=6 imbalance_ratio=1.014038 max_load_reduction=57229 max_load_reduction_pct=0.873245 moved_to_echo=32656 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[529, 500, 478, 535, 509, 537, 489, 507, 503, 489, 527, 572, 501, 546, 516, 506, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 530, 513, 533, 462, 528, 477, 495, 506, 500, 538, 499, 513, 479, 554, 477, 15, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 495, 539, 522, 517, 510, 494, 497, 492, 492, 513, 518, 510, 521, 528, 517, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 513, 503, 510, 526, 507, 522, 507, 469, 477, 529, 499, 508, 480, 558, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 497, 500, 503, 510, 548, 551, 543, 523, 479, 506, 507, 554, 510, 535, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 499, 528, 478, 531, 503, 530, 524, 507, 489, 509, 504, 507, 529, 482, 516, 17, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 509, 500, 509, 489, 534, 532, 499, 487, 515, 526, 537, 477, 537, 498, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 487, 523, 525, 530, 478, 517, 520, 528, 546, 505, 495, 508, 506, 494, 493, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8271, 8178, 8162, 8140, 8276, 8175, 8195, 8139] avg=8192.0000 max=8276 max_rank=4 min=8139 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32785 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[480, 495, 480, 485, 523, 516, 487, 524, 524, 521, 513, 511, 535, 511, 519, 559, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 510, 529, 529, 496, 478, 478, 511, 553, 512, 511, 507, 461, 483, 528, 555, 14, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 505, 522, 499, 473, 530, 509, 495, 543, 523, 524, 538, 514, 528, 529, 529, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 526, 491, 536, 499, 513, 557, 539, 526, 447, 500, 495, 535, 507, 474, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 499, 494, 498, 528, 531, 497, 541, 543, 544, 526, 499, 520, 517, 496, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 494, 514, 531, 536, 497, 516, 540, 486, 492, 522, 485, 540, 515, 523, 501, 16, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 475, 510, 477, 536, 518, 548, 467, 502, 506, 503, 490, 510, 506, 517, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 504, 481, 520, 518, 472, 519, 508, 493, 509, 519, 538, 520, 523, 459, 555, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8165, 8278, 8147, 8233, 8230, 8130, 8144] avg=8192.0000 max=8278 max_rank=2 min=8130 min_rank=6 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32737 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[495, 520, 522, 507, 475, 495, 478, 511, 505, 487, 515, 515, 479, 504, 519, 512, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 508, 530, 511, 502, 501, 554, 474, 482, 528, 519, 530, 488, 512, 525, 511, 15, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 531, 502, 522, 513, 514, 502, 521, 485, 518, 506, 540, 529, 479, 494, 505, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 515, 522, 499, 526, 513, 505, 533, 472, 473, 514, 522, 526, 509, 522, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 512, 566, 494, 543, 523, 502, 481, 516, 494, 500, 495, 514, 479, 508, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 517, 515, 539, 525, 530, 528, 524, 473, 545, 540, 480, 504, 514, 517, 486, 17, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 501, 534, 478, 527, 530, 521, 508, 485, 498, 491, 500, 527, 511, 510, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 533, 499, 496, 502, 526, 478, 554, 512, 521, 558, 497, 490, 510, 512, 539, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8066, 8220, 8144, 8178, 8191, 8274, 8196, 8267] avg=8192.0000 max=8274 max_rank=5 min=8066 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32928 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[484, 513, 491, 507, 515, 525, 528, 532, 507, 512, 515, 491, 513, 506, 501, 524, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 563, 523, 499, 513, 528, 497, 507, 524, 542, 545, 476, 527, 523, 502, 495, 512, 15, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 488, 555, 510, 516, 553, 500, 531, 464, 473, 523, 440, 502, 525, 534, 520, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 499, 538, 521, 508, 535, 475, 502, 549, 478, 481, 494, 508, 473, 534, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 515, 480, 514, 502, 529, 521, 493, 505, 540, 501, 526, 525, 529, 472, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 522, 456, 498, 530, 510, 533, 520, 481, 506, 528, 523, 503, 499, 542, 461, 16, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 518, 510, 525, 528, 493, 523, 513, 506, 539, 511, 503, 510, 536, 506, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 500, 564, 540, 501, 525, 514, 521, 449, 544, 504, 483, 518, 529, 524, 549, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8302, 8113, 8118, 8157, 8180, 8227, 8248] avg=8192.0000 max=8302 max_rank=1 min=8113 min_rank=2 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32812 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[522, 533, 495, 528, 517, 514, 502, 477, 477, 486, 506, 531, 500, 541, 505, 511, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 563, 519, 509, 481, 501, 514, 541, 504, 507, 504, 531, 494, 483, 504, 492, 494, 15, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 524, 559, 547, 501, 539, 510, 522, 517, 533, 479, 519, 508, 482, 501, 534, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 508, 528, 519, 523, 511, 515, 538, 480, 468, 529, 540, 546, 517, 469, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 531, 496, 504, 522, 530, 574, 528, 532, 513, 490, 467, 507, 561, 477, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 511, 504, 486, 533, 514, 490, 511, 529, 506, 466, 511, 517, 516, 502, 524, 17, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 575, 499, 490, 522, 483, 513, 457, 467, 531, 477, 539, 547, 490, 521, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 494, 514, 507, 518, 498, 537, 499, 512, 519, 496, 476, 557, 504, 477, 529, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8168, 8307, 8233, 8248, 8154, 8078, 8176] avg=8192.0000 max=8307 max_rank=2 min=8078 min_rank=6 imbalance_ratio=1.014038 max_load_reduction=57229 max_load_reduction_pct=0.873245 moved_to_echo=32656 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[499, 542, 521, 507, 530, 473, 516, 493, 554, 485, 579, 476, 494, 493, 485, 496, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 508, 482, 501, 551, 535, 499, 508, 494, 497, 487, 488, 535, 458, 512, 525, 16, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 499, 534, 508, 545, 500, 526, 550, 546, 517, 490, 493, 520, 492, 525, 499, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 517, 497, 474, 521, 499, 538, 534, 544, 499, 517, 542, 457, 525, 496, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 538, 496, 520, 534, 507, 520, 475, 533, 524, 534, 517, 504, 481, 497, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 528, 503, 483, 523, 557, 518, 510, 478, 526, 507, 537, 503, 460, 492, 531, 16, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 537, 510, 509, 500, 523, 562, 520, 476, 531, 534, 482, 525, 471, 528, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 492, 511, 502, 503, 511, 542, 494, 517, 516, 468, 535, 499, 524, 510, 494, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8128, 8249, 8148, 8162, 8222, 8290, 8165] avg=8192.0000 max=8290 max_rank=6 min=8128 min_rank=1 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32839 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[513, 519, 465, 534, 549, 504, 506, 522, 492, 497, 502, 510, 508, 531, 512, 527, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 504, 542, 483, 514, 498, 486, 520, 490, 538, 459, 491, 468, 524, 537, 490, 19, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 501, 523, 521, 494, 530, 529, 520, 550, 534, 522, 490, 530, 501, 495, 500, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 541, 526, 506, 509, 542, 518, 523, 487, 524, 512, 539, 509, 515, 524, 452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 488, 531, 510, 497, 514, 504, 524, 524, 505, 491, 478, 497, 543, 512, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 539, 512, 500, 526, 547, 476, 525, 507, 514, 465, 475, 520, 535, 516, 500, 20, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 509, 499, 520, 497, 509, 533, 532, 530, 484, 530, 507, 539, 500, 498, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 535, 493, 471, 495, 502, 496, 519, 519, 524, 547, 499, 491, 542, 472, 524, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 8083, 8258, 8248, 8163, 8224, 8175, 8160] avg=8192.0000 max=8258 max_rank=2 min=8083 min_rank=1 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32722 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[480, 495, 480, 485, 523, 516, 487, 524, 524, 521, 513, 511, 535, 511, 519, 559, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 510, 529, 529, 496, 478, 478, 511, 553, 512, 511, 507, 461, 483, 528, 555, 14, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 505, 522, 499, 473, 530, 509, 495, 543, 523, 524, 538, 514, 528, 529, 529, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 526, 491, 536, 499, 513, 557, 539, 526, 447, 500, 495, 535, 507, 474, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 499, 494, 498, 528, 531, 497, 541, 543, 544, 526, 499, 520, 517, 496, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 494, 514, 531, 536, 497, 516, 540, 486, 492, 522, 485, 540, 515, 523, 501, 16, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 475, 510, 477, 536, 518, 548, 467, 502, 506, 503, 490, 510, 506, 517, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 504, 481, 520, 518, 472, 519, 508, 493, 509, 519, 538, 520, 523, 459, 555, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8165, 8278, 8147, 8233, 8230, 8130, 8144] avg=8192.0000 max=8278 max_rank=2 min=8130 min_rank=6 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32737 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 3968, 4096, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[3968, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4352, 4096, 4096, 4096, 4224, 3968, 4352, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 3968, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 3968, 4096, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 3968, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 3968, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4352, 4096, 4096, 4096, 4224, 3968, 4352, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[499, 536, 535, 522, 499, 501, 539, 504, 511, 496, 477, 500, 521, 531, 533, 555, 534, 515, 511, 528, 527, 514, 542, 490, 556, 523, 453, 523, 511, 542, 511, 500, 503, 508, 510, 476, 500, 540, 499, 504, 490, 492, 509, 513, 492, 484, 534, 512, 535, 478, 518, 524, 511, 512, 532, 533, 535, 506, 469, 499, 516, 559, 499, 517, 505, 468, 512, 545, 496, 530, 499, 493, 517, 546, 504, 514, 491, 508, 524, 529, 518, 508, 541, 519, 531, 498, 488, 520, 553, 541, 506, 533, 487, 479, 494, 510, 497, 508, 535, 483, 506, 466, 506, 542, 494, 485, 481, 513, 523, 533, 491, 509, 537, 501, 464, 499, 517, 537, 557, 494, 504, 511, 527, 528, 530, 495, 505, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[499, 499, 532, 530, 518, 531, 516, 501, 479, 475, 509, 519, 547, 518, 495, 543, 544, 475, 481, 527, 481, 500, 505, 548, 502, 483, 488, 542, 500, 505, 529, 530, 523, 542, 549, 511, 512, 533, 501, 518, 516, 503, 483, 495, 513, 495, 498, 530, 496, 526, 512, 539, 540, 521, 536, 493, 451, 530, 557, 514, 514, 545, 526, 510, 477, 491, 469, 488, 528, 520, 529, 495, 517, 518, 510, 539, 538, 490, 524, 504, 511, 484, 510, 492, 508, 508, 515, 550, 511, 524, 473, 538, 521, 493, 509, 511, 510, 562, 531, 499, 491, 507, 550, 509, 498, 485, 529, 488, 494, 525, 497, 524, 506, 525, 530, 521, 505, 529, 478, 493, 522, 505, 487, 504, 515, 491, 495, 553] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[476, 499, 494, 499, 523, 488, 500, 527, 520, 543, 528, 485, 526, 540, 530, 533, 533, 537, 545, 513, 496, 543, 526, 479, 519, 515, 517, 526, 527, 511, 514, 517, 504, 537, 581, 499, 499, 487, 522, 521, 536, 504, 501, 511, 513, 498, 517, 524, 520, 514, 528, 518, 499, 548, 517, 535, 544, 514, 496, 520, 502, 532, 529, 489, 498, 504, 528, 539, 504, 543, 526, 531, 527, 466, 516, 537, 490, 500, 524, 528, 530, 549, 495, 516, 475, 498, 506, 499, 511, 491, 482, 490, 510, 463, 537, 484, 515, 493, 519, 508, 548, 504, 492, 518, 513, 525, 493, 489, 513, 514, 526, 504, 504, 498, 492, 469, 490, 554, 496, 510, 478, 499, 494, 511, 485, 526, 509, 462] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[490, 499, 518, 531, 513, 531, 542, 460, 491, 489, 499, 472, 523, 499, 506, 498, 515, 478, 514, 500, 501, 529, 518, 514, 516, 512, 519, 455, 514, 509, 478, 512, 488, 485, 487, 512, 535, 549, 509, 521, 475, 502, 521, 501, 524, 514, 538, 527, 486, 548, 474, 495, 513, 495, 505, 512, 508, 517, 481, 538, 526, 498, 536, 494, 539, 494, 455, 532, 510, 496, 504, 523, 513, 507, 532, 539, 539, 531, 550, 548, 521, 494, 513, 555, 534, 506, 539, 489, 521, 537, 515, 518, 485, 511, 526, 493, 514, 513, 486, 532, 512, 521, 506, 526, 482, 568, 523, 512, 539, 531, 509, 506, 516, 531, 522, 510, 515, 514, 484, 499, 524, 504, 566, 506, 538, 505, 485, 509] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[496, 506, 524, 506, 495, 506, 508, 479, 499, 524, 529, 538, 508, 509, 514, 524, 547, 543, 497, 554, 490, 539, 548, 482, 498, 503, 509, 498, 482, 505, 520, 517, 536, 511, 524, 485, 498, 516, 484, 523, 523, 493, 481, 523, 531, 525, 502, 501, 527, 511, 528, 500, 500, 514, 502, 527, 524, 521, 504, 513, 541, 541, 462, 520, 544, 522, 517, 544, 506, 484, 533, 536, 490, 546, 498, 506, 508, 495, 551, 506, 483, 490, 526, 502, 492, 562, 520, 522, 490, 488, 516, 516, 519, 547, 515, 552, 509, 513, 519, 508, 515, 519, 531, 488, 481, 508, 509, 518, 469, 499, 519, 525, 496, 529, 478, 515, 487, 534, 488, 530, 480, 541, 510, 519, 466, 506, 502, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[490, 544, 548, 474, 484, 519, 502, 521, 508, 492, 518, 481, 514, 513, 525, 493, 555, 503, 519, 494, 534, 498, 553, 497, 525, 536, 477, 510, 481, 488, 521, 494, 518, 548, 508, 534, 479, 490, 524, 552, 519, 489, 528, 488, 514, 506, 454, 482, 522, 525, 510, 532, 514, 457, 495, 537, 515, 527, 516, 482, 467, 525, 509, 482, 507, 549, 520, 527, 497, 496, 544, 540, 534, 525, 523, 558, 498, 509, 550, 508, 497, 494, 518, 526, 558, 442, 482, 513, 509, 513, 489, 518, 540, 488, 519, 514, 481, 546, 536, 505, 486, 495, 530, 528, 530, 504, 487, 539, 507, 477, 523, 501, 503, 554, 503, 549, 570, 517, 519, 524, 502, 531, 508, 490, 496, 507, 489, 535] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[512, 543, 487, 515, 498, 518, 560, 496, 489, 547, 518, 543, 521, 502, 509, 506, 518, 535, 540, 525, 512, 509, 532, 521, 485, 486, 511, 542, 551, 517, 491, 501, 479, 513, 535, 503, 500, 519, 497, 502, 523, 555, 525, 507, 513, 507, 530, 501, 483, 501, 521, 500, 536, 527, 547, 512, 528, 505, 517, 514, 511, 511, 531, 511, 498, 510, 496, 545, 513, 512, 516, 519, 524, 514, 494, 553, 496, 505, 512, 508, 531, 477, 515, 484, 504, 530, 511, 565, 509, 495, 526, 463, 515, 516, 485, 467, 506, 552, 453, 502, 533, 493, 501, 517, 519, 522, 520, 486, 522, 508, 559, 541, 498, 477, 479, 460, 494, 483, 558, 524, 494, 449, 513, 506, 552, 504, 505, 494] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[496, 506, 524, 506, 495, 506, 508, 479, 499, 524, 529, 538, 508, 509, 514, 524, 547, 543, 497, 554, 490, 539, 548, 482, 498, 503, 509, 498, 482, 505, 520, 517, 536, 511, 524, 485, 498, 516, 484, 523, 523, 493, 481, 523, 531, 525, 502, 501, 527, 511, 528, 500, 500, 514, 502, 527, 524, 521, 504, 513, 541, 541, 462, 520, 544, 522, 517, 544, 506, 484, 533, 536, 490, 546, 498, 506, 508, 495, 551, 506, 483, 490, 526, 502, 492, 562, 520, 522, 490, 488, 516, 516, 519, 547, 515, 552, 509, 513, 519, 508, 515, 519, 531, 488, 481, 508, 509, 518, 469, 499, 519, 525, 496, 529, 478, 515, 487, 534, 488, 530, 480, 541, 510, 519, 466, 506, 502, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[499, 499, 532, 530, 518, 531, 516, 501, 479, 475, 509, 519, 547, 518, 495, 543, 544, 475, 481, 527, 481, 500, 505, 548, 502, 483, 488, 542, 500, 505, 529, 530, 523, 542, 549, 511, 512, 533, 501, 518, 516, 503, 483, 495, 513, 495, 498, 530, 496, 526, 512, 539, 540, 521, 536, 493, 451, 530, 557, 514, 514, 545, 526, 510, 477, 491, 469, 488, 528, 520, 529, 495, 517, 518, 510, 539, 538, 490, 524, 504, 511, 484, 510, 492, 508, 508, 515, 550, 511, 524, 473, 538, 521, 493, 509, 511, 510, 562, 531, 499, 491, 507, 550, 509, 498, 485, 529, 488, 494, 525, 497, 524, 506, 525, 530, 521, 505, 529, 478, 493, 522, 505, 487, 504, 515, 491, 495, 553] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[522, 546, 514, 523, 490, 481, 555, 505, 500, 497, 489, 516, 480, 499, 513, 531, 516, 514, 487, 522, 459, 490, 532, 539, 513, 558, 524, 475, 473, 510, 517, 537, 495, 518, 526, 502, 551, 546, 513, 479, 480, 531, 508, 532, 504, 524, 518, 520, 491, 488, 529, 513, 512, 494, 502, 508, 563, 508, 544, 505, 525, 509, 474, 505, 494, 515, 474, 489, 504, 524, 551, 467, 533, 499, 522, 494, 540, 515, 467, 439, 501, 494, 499, 517, 499, 520, 525, 521, 517, 500, 512, 521, 514, 515, 546, 511, 520, 516, 477, 520, 498, 551, 513, 526, 494, 524, 478, 493, 542, 512, 552, 516, 555, 502, 490, 556, 521, 513, 564, 502, 550, 535, 556, 452, 476, 523, 503, 523] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[499, 536, 535, 522, 499, 501, 539, 504, 511, 496, 477, 500, 521, 531, 533, 555, 534, 515, 511, 528, 527, 514, 542, 490, 556, 523, 453, 523, 511, 542, 511, 500, 503, 508, 510, 476, 500, 540, 499, 504, 490, 492, 509, 513, 492, 484, 534, 512, 535, 478, 518, 524, 511, 512, 532, 533, 535, 506, 469, 499, 516, 559, 499, 517, 505, 468, 512, 545, 496, 530, 499, 493, 517, 546, 504, 514, 491, 508, 524, 529, 518, 508, 541, 519, 531, 498, 488, 520, 553, 541, 506, 533, 487, 479, 494, 510, 497, 508, 535, 483, 506, 466, 506, 542, 494, 485, 481, 513, 523, 533, 491, 509, 537, 501, 464, 499, 517, 537, 557, 494, 504, 511, 527, 528, 530, 495, 505, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[490, 499, 518, 531, 513, 531, 542, 460, 491, 489, 499, 472, 523, 499, 506, 498, 515, 478, 514, 500, 501, 529, 518, 514, 516, 512, 519, 455, 514, 509, 478, 512, 488, 485, 487, 512, 535, 549, 509, 521, 475, 502, 521, 501, 524, 514, 538, 527, 486, 548, 474, 495, 513, 495, 505, 512, 508, 517, 481, 538, 526, 498, 536, 494, 539, 494, 455, 532, 510, 496, 504, 523, 513, 507, 532, 539, 539, 531, 550, 548, 521, 494, 513, 555, 534, 506, 539, 489, 521, 537, 515, 518, 485, 511, 526, 493, 514, 513, 486, 532, 512, 521, 506, 526, 482, 568, 523, 512, 539, 531, 509, 506, 516, 531, 522, 510, 515, 514, 484, 499, 524, 504, 566, 506, 538, 505, 485, 509] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[499, 499, 532, 530, 518, 531, 516, 501, 479, 475, 509, 519, 547, 518, 495, 543, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 475, 481, 527, 481, 500, 505, 548, 502, 483, 488, 542, 500, 505, 529, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 542, 549, 511, 512, 533, 501, 518, 516, 503, 483, 495, 513, 495, 498, 530, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 526, 512, 539, 540, 521, 536, 493, 451, 530, 557, 514, 514, 517, 526, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 491, 469, 488, 528, 520, 529, 495, 517, 518, 510, 506, 538, 490, 524, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 484, 510, 492, 508, 508, 515, 550, 511, 524, 473, 538, 521, 493, 509, 511, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 562, 531, 499, 491, 507, 550, 509, 498, 485, 529, 488, 494, 525, 497, 524, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 525, 530, 521, 505, 529, 478, 493, 522, 505, 487, 504, 515, 491, 495, 553, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8220, 8123, 8229, 8282, 8104, 8182, 8204, 8192] avg=8192.0000 max=8282 max_rank=3 min=8104 min_rank=4 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32682 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[476, 499, 494, 499, 523, 488, 500, 527, 520, 543, 528, 485, 526, 540, 530, 533, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 537, 545, 513, 496, 543, 526, 479, 519, 515, 517, 526, 527, 511, 514, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 537, 581, 499, 499, 487, 522, 521, 536, 504, 501, 511, 513, 498, 517, 524, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 514, 528, 518, 499, 548, 517, 535, 544, 514, 496, 520, 502, 505, 529, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 504, 528, 539, 504, 543, 526, 531, 527, 466, 516, 504, 490, 500, 524, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 549, 495, 516, 475, 498, 506, 499, 511, 491, 482, 490, 510, 463, 537, 484, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 493, 519, 508, 548, 504, 492, 518, 513, 525, 493, 489, 513, 514, 526, 504, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 498, 492, 469, 490, 554, 496, 510, 478, 499, 494, 511, 485, 526, 509, 462, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8220, 8301, 8261, 8278, 8228, 8059, 8179, 8010] avg=8192.0000 max=8301 max_rank=1 min=8010 min_rank=7 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32476 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[499, 536, 535, 522, 499, 501, 539, 504, 511, 496, 477, 500, 521, 531, 533, 555, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 515, 511, 528, 527, 514, 542, 490, 556, 523, 453, 523, 511, 542, 511, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 508, 510, 476, 500, 540, 499, 504, 490, 492, 509, 513, 492, 484, 534, 512, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 478, 518, 524, 511, 512, 532, 533, 535, 506, 469, 499, 516, 530, 499, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 468, 512, 545, 496, 530, 499, 493, 517, 546, 504, 483, 491, 508, 524, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 508, 541, 519, 531, 498, 488, 520, 553, 541, 506, 533, 487, 479, 494, 510, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 508, 535, 483, 506, 466, 506, 542, 494, 485, 481, 513, 523, 533, 491, 509, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 501, 464, 499, 517, 537, 557, 494, 504, 511, 527, 528, 530, 495, 505, 503, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8268, 8263, 8073, 8214, 8150, 8251, 8077, 8240] avg=8192.0000 max=8268 max_rank=0 min=8073 min_rank=2 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32718 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[490, 544, 548, 474, 484, 519, 502, 521, 508, 492, 518, 481, 514, 513, 525, 493, 9, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 503, 519, 494, 534, 498, 553, 497, 525, 536, 477, 510, 481, 488, 521, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 548, 508, 534, 479, 490, 524, 552, 519, 489, 528, 488, 514, 506, 454, 482, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 525, 510, 532, 514, 457, 495, 537, 515, 527, 516, 482, 467, 492, 509, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 549, 520, 527, 497, 496, 544, 540, 534, 525, 523, 515, 498, 509, 550, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 494, 518, 526, 558, 442, 482, 513, 509, 513, 489, 518, 540, 488, 519, 514, 27, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 546, 536, 505, 486, 495, 530, 528, 530, 504, 487, 539, 507, 477, 523, 501, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 554, 503, 549, 570, 517, 519, 524, 502, 531, 508, 490, 496, 507, 489, 535, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8141, 8156, 8145, 8082, 8342, 8152, 8183, 8335] avg=8192.0000 max=8342 max_rank=4 min=8082 min_rank=3 imbalance_ratio=1.018311 max_load_reduction=57194 max_load_reduction_pct=0.872711 moved_to_echo=33012 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[512, 543, 487, 515, 498, 518, 560, 496, 489, 547, 518, 543, 521, 502, 509, 506, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 535, 540, 525, 512, 509, 532, 521, 485, 486, 511, 542, 551, 517, 491, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 513, 535, 503, 500, 519, 497, 502, 523, 555, 525, 507, 513, 507, 530, 501, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 501, 521, 500, 536, 527, 547, 512, 528, 505, 517, 514, 511, 485, 531, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 510, 496, 545, 513, 512, 516, 519, 524, 514, 494, 519, 496, 505, 512, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 477, 515, 484, 504, 530, 511, 565, 509, 495, 526, 463, 515, 516, 485, 467, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 552, 453, 502, 533, 493, 501, 517, 519, 522, 520, 486, 522, 508, 559, 541, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 477, 479, 460, 494, 483, 558, 524, 494, 449, 513, 506, 552, 504, 505, 494, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8273, 8260, 8216, 8229, 8181, 8115, 8238, 8024] avg=8192.0000 max=8273 max_rank=0 min=8024 min_rank=7 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32558 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[490, 544, 548, 474, 484, 519, 502, 521, 508, 492, 518, 481, 514, 513, 525, 493, 9, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 503, 519, 494, 534, 498, 553, 497, 525, 536, 477, 510, 481, 488, 521, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 548, 508, 534, 479, 490, 524, 552, 519, 489, 528, 488, 514, 506, 454, 482, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 525, 510, 532, 514, 457, 495, 537, 515, 527, 516, 482, 467, 492, 509, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 549, 520, 527, 497, 496, 544, 540, 534, 525, 523, 515, 498, 509, 550, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 494, 518, 526, 558, 442, 482, 513, 509, 513, 489, 518, 540, 488, 519, 514, 27, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 546, 536, 505, 486, 495, 530, 528, 530, 504, 487, 539, 507, 477, 523, 501, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 554, 503, 549, 570, 517, 519, 524, 502, 531, 508, 490, 496, 507, 489, 535, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8141, 8156, 8145, 8082, 8342, 8152, 8183, 8335] avg=8192.0000 max=8342 max_rank=4 min=8082 min_rank=3 imbalance_ratio=1.018311 max_load_reduction=57194 max_load_reduction_pct=0.872711 moved_to_echo=33012 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[522, 546, 514, 523, 490, 481, 555, 505, 500, 497, 489, 516, 480, 499, 513, 531, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 514, 487, 522, 459, 490, 532, 539, 513, 558, 524, 475, 473, 510, 517, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 518, 526, 502, 551, 546, 513, 479, 480, 531, 508, 532, 504, 524, 518, 520, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 488, 529, 513, 512, 494, 502, 508, 563, 508, 544, 505, 525, 483, 474, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 515, 474, 489, 504, 524, 551, 467, 533, 499, 522, 464, 540, 515, 467, 439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 494, 499, 517, 499, 520, 525, 521, 517, 500, 512, 521, 514, 515, 546, 511, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 516, 477, 520, 498, 551, 513, 526, 494, 524, 478, 493, 542, 512, 552, 516, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 502, 490, 556, 521, 513, 564, 502, 550, 535, 556, 452, 476, 523, 503, 523, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8170, 8150, 8254, 8144, 7997, 8234, 8236, 8351] avg=8192.0000 max=8351 max_rank=7 min=7997 min_rank=4 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32818 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[490, 499, 518, 531, 513, 531, 542, 460, 491, 489, 499, 472, 523, 499, 506, 498, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 478, 514, 500, 501, 529, 518, 514, 516, 512, 519, 455, 514, 509, 478, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 485, 487, 512, 535, 549, 509, 521, 475, 502, 521, 501, 524, 514, 538, 527, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 548, 474, 495, 513, 495, 505, 512, 508, 517, 481, 538, 526, 472, 536, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 494, 455, 532, 510, 496, 504, 523, 513, 507, 532, 506, 539, 531, 550, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 494, 513, 555, 534, 506, 539, 489, 521, 537, 515, 518, 485, 511, 526, 493, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 513, 486, 532, 512, 521, 506, 526, 482, 568, 523, 512, 539, 531, 509, 506, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 531, 522, 510, 515, 514, 484, 499, 524, 504, 566, 506, 538, 505, 485, 509, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8070, 8068, 8195, 8100, 8279, 8279, 8284, 8261] avg=8192.0000 max=8284 max_rank=6 min=8068 min_rank=1 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=33103 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[496, 506, 524, 506, 495, 506, 508, 479, 499, 524, 529, 538, 508, 509, 514, 524, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 543, 497, 554, 490, 539, 548, 482, 498, 503, 509, 498, 482, 505, 520, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 511, 524, 485, 498, 516, 484, 523, 523, 493, 481, 523, 531, 525, 502, 501, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 511, 528, 500, 500, 514, 502, 527, 524, 521, 504, 513, 541, 513, 462, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 522, 517, 544, 506, 484, 533, 536, 490, 546, 498, 475, 508, 495, 551, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 490, 526, 502, 492, 562, 520, 522, 490, 488, 516, 516, 519, 547, 515, 552, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 513, 519, 508, 515, 519, 531, 488, 481, 508, 509, 518, 469, 499, 519, 525, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 529, 478, 515, 487, 534, 488, 530, 480, 541, 510, 519, 466, 506, 502, 511, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 8215, 8163, 8207, 8255, 8264, 8135, 8123] avg=8192.0000 max=8264 max_rank=5 min=8123 min_rank=7 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32777 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[522, 546, 514, 523, 490, 481, 555, 505, 500, 497, 489, 516, 480, 499, 513, 531, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 514, 487, 522, 459, 490, 532, 539, 513, 558, 524, 475, 473, 510, 517, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 518, 526, 502, 551, 546, 513, 479, 480, 531, 508, 532, 504, 524, 518, 520, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 488, 529, 513, 512, 494, 502, 508, 563, 508, 544, 505, 525, 483, 474, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 515, 474, 489, 504, 524, 551, 467, 533, 499, 522, 464, 540, 515, 467, 439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 494, 499, 517, 499, 520, 525, 521, 517, 500, 512, 521, 514, 515, 546, 511, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 516, 477, 520, 498, 551, 513, 526, 494, 524, 478, 493, 542, 512, 552, 516, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 502, 490, 556, 521, 513, 564, 502, 550, 535, 556, 452, 476, 523, 503, 523, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8170, 8150, 8254, 8144, 7997, 8234, 8236, 8351] avg=8192.0000 max=8351 max_rank=7 min=7997 min_rank=4 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32818 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[499, 536, 535, 522, 499, 501, 539, 504, 511, 496, 477, 500, 521, 531, 533, 555, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 515, 511, 528, 527, 514, 542, 490, 556, 523, 453, 523, 511, 542, 511, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 508, 510, 476, 500, 540, 499, 504, 490, 492, 509, 513, 492, 484, 534, 512, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 478, 518, 524, 511, 512, 532, 533, 535, 506, 469, 499, 516, 530, 499, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 468, 512, 545, 496, 530, 499, 493, 517, 546, 504, 483, 491, 508, 524, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 508, 541, 519, 531, 498, 488, 520, 553, 541, 506, 533, 487, 479, 494, 510, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 508, 535, 483, 506, 466, 506, 542, 494, 485, 481, 513, 523, 533, 491, 509, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 501, 464, 499, 517, 537, 557, 494, 504, 511, 527, 528, 530, 495, 505, 503, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8268, 8263, 8073, 8214, 8150, 8251, 8077, 8240] avg=8192.0000 max=8268 max_rank=0 min=8073 min_rank=2 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32718 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4352, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 3968, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4352, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 3968, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[501, 527, 524, 521, 516, 499, 493, 480, 511, 517, 487, 494, 522, 495, 546, 487, 514, 524, 523, 484, 517, 502, 519, 500, 538, 495, 549, 484, 506, 571, 506, 496, 479, 546, 482, 505, 524, 496, 480, 556, 475, 505, 537, 494, 542, 531, 517, 530, 510, 554, 507, 478, 510, 514, 538, 521, 462, 519, 498, 522, 530, 579, 548, 490, 545, 512, 516, 529, 489, 505, 484, 544, 522, 522, 542, 511, 492, 529, 501, 494, 501, 497, 513, 494, 490, 528, 509, 511, 545, 495, 571, 504, 493, 525, 534, 517, 479, 509, 526, 509, 521, 484, 476, 500, 505, 498, 499, 496, 483, 533, 520, 550, 503, 504, 534, 506, 508, 505, 521, 515, 502, 574, 497, 494, 490, 492, 486, 526] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[533, 510, 500, 542, 508, 531, 540, 551, 523, 500, 538, 541, 531, 513, 490, 488, 501, 537, 515, 526, 484, 496, 493, 513, 502, 545, 513, 493, 489, 538, 532, 490, 526, 510, 521, 514, 483, 470, 518, 541, 527, 492, 518, 453, 520, 516, 517, 485, 496, 493, 500, 503, 509, 529, 466, 498, 468, 501, 504, 487, 508, 516, 520, 542, 512, 554, 495, 488, 501, 474, 505, 501, 528, 531, 523, 564, 507, 510, 517, 487, 539, 503, 554, 510, 565, 502, 527, 463, 502, 508, 520, 492, 497, 521, 475, 515, 552, 517, 514, 504, 500, 505, 462, 546, 491, 515, 518, 509, 488, 497, 506, 482, 542, 480, 499, 566, 529, 538, 516, 554, 519, 507, 570, 528, 503, 505, 518, 509] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[525, 501, 497, 536, 455, 502, 530, 527, 508, 517, 513, 474, 521, 501, 498, 532, 488, 520, 504, 475, 509, 537, 513, 528, 501, 541, 508, 500, 492, 524, 504, 494, 487, 564, 504, 485, 517, 540, 507, 514, 522, 560, 484, 534, 539, 492, 514, 501, 523, 531, 542, 554, 516, 506, 548, 514, 516, 506, 496, 520, 494, 484, 523, 482, 518, 550, 484, 531, 539, 503, 500, 492, 529, 516, 530, 545, 498, 491, 536, 517, 543, 519, 504, 496, 489, 502, 517, 507, 560, 505, 491, 508, 490, 497, 482, 467, 504, 499, 535, 508, 503, 512, 497, 480, 549, 537, 530, 560, 524, 515, 516, 500, 493, 491, 543, 488, 535, 537, 512, 545, 541, 513, 481, 480, 497, 460, 507, 494] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[517, 501, 483, 555, 495, 540, 527, 525, 485, 487, 506, 510, 502, 519, 553, 471, 452, 495, 530, 498, 536, 468, 511, 512, 537, 530, 474, 544, 470, 512, 500, 491, 501, 495, 474, 496, 518, 490, 539, 533, 513, 529, 496, 485, 534, 524, 507, 497, 511, 514, 487, 536, 511, 529, 528, 493, 511, 511, 508, 494, 507, 520, 504, 552, 522, 481, 527, 541, 492, 528, 503, 526, 548, 538, 494, 502, 486, 496, 491, 522, 515, 501, 503, 550, 505, 523, 523, 511, 465, 501, 509, 477, 555, 481, 541, 526, 601, 521, 508, 508, 509, 522, 505, 504, 509, 515, 518, 498, 519, 511, 515, 480, 497, 524, 569, 541, 510, 542, 531, 523, 560, 525, 527, 504, 464, 512, 515, 483] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[470, 487, 526, 500, 500, 525, 522, 550, 510, 538, 480, 546, 527, 487, 517, 470, 485, 499, 515, 539, 500, 505, 534, 493, 572, 542, 500, 482, 528, 524, 513, 486, 498, 504, 483, 514, 517, 506, 539, 515, 552, 500, 554, 516, 554, 503, 533, 523, 516, 523, 569, 506, 520, 521, 528, 480, 518, 488, 521, 500, 526, 511, 509, 521, 452, 512, 501, 519, 551, 473, 526, 528, 532, 539, 503, 468, 549, 490, 481, 543, 488, 514, 516, 530, 556, 556, 531, 519, 535, 495, 477, 483, 523, 488, 521, 507, 537, 527, 494, 510, 506, 481, 541, 470, 525, 523, 487, 505, 484, 513, 498, 508, 474, 471, 529, 516, 485, 480, 528, 484, 505, 533, 519, 517, 532, 485, 527, 496] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[533, 510, 500, 542, 508, 531, 540, 551, 523, 500, 538, 541, 531, 513, 490, 488, 501, 537, 515, 526, 484, 496, 493, 513, 502, 545, 513, 493, 489, 538, 532, 490, 526, 510, 521, 514, 483, 470, 518, 541, 527, 492, 518, 453, 520, 516, 517, 485, 496, 493, 500, 503, 509, 529, 466, 498, 468, 501, 504, 487, 508, 516, 520, 542, 512, 554, 495, 488, 501, 474, 505, 501, 528, 531, 523, 564, 507, 510, 517, 487, 539, 503, 554, 510, 565, 502, 527, 463, 502, 508, 520, 492, 497, 521, 475, 515, 552, 517, 514, 504, 500, 505, 462, 546, 491, 515, 518, 509, 488, 497, 506, 482, 542, 480, 499, 566, 529, 538, 516, 554, 519, 507, 570, 528, 503, 505, 518, 509] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[501, 527, 524, 521, 516, 499, 493, 480, 511, 517, 487, 494, 522, 495, 546, 487, 514, 524, 523, 484, 517, 502, 519, 500, 538, 495, 549, 484, 506, 571, 506, 496, 479, 546, 482, 505, 524, 496, 480, 556, 475, 505, 537, 494, 542, 531, 517, 530, 510, 554, 507, 478, 510, 514, 538, 521, 462, 519, 498, 522, 530, 579, 548, 490, 545, 512, 516, 529, 489, 505, 484, 544, 522, 522, 542, 511, 492, 529, 501, 494, 501, 497, 513, 494, 490, 528, 509, 511, 545, 495, 571, 504, 493, 525, 534, 517, 479, 509, 526, 509, 521, 484, 476, 500, 505, 498, 499, 496, 483, 533, 520, 550, 503, 504, 534, 506, 508, 505, 521, 515, 502, 574, 497, 494, 490, 492, 486, 526] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[534, 538, 540, 499, 553, 491, 527, 507, 521, 535, 539, 505, 502, 555, 480, 485, 544, 523, 511, 531, 528, 535, 519, 483, 545, 517, 525, 508, 536, 508, 509, 510, 500, 517, 497, 508, 467, 546, 543, 539, 535, 505, 507, 561, 467, 518, 546, 537, 502, 515, 519, 524, 518, 495, 510, 505, 528, 550, 502, 546, 447, 512, 529, 532, 505, 519, 525, 494, 525, 493, 462, 492, 506, 509, 515, 524, 468, 503, 494, 505, 520, 501, 503, 517, 523, 509, 472, 484, 535, 529, 499, 469, 506, 500, 504, 521, 505, 498, 526, 504, 505, 530, 509, 491, 538, 492, 518, 534, 487, 510, 542, 494, 493, 485, 495, 499, 482, 541, 508, 512, 526, 489, 510, 487, 512, 492, 483, 538] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[537, 494, 512, 457, 553, 527, 511, 502, 525, 522, 490, 534, 501, 494, 533, 473, 528, 501, 546, 489, 503, 530, 499, 530, 481, 464, 484, 545, 494, 518, 525, 502, 507, 518, 529, 499, 519, 487, 498, 515, 511, 515, 537, 521, 494, 511, 497, 534, 511, 535, 508, 471, 492, 495, 502, 515, 528, 554, 476, 469, 513, 491, 494, 558, 508, 501, 515, 490, 524, 463, 465, 490, 503, 527, 532, 532, 539, 529, 521, 533, 546, 495, 525, 519, 525, 527, 494, 511, 549, 514, 544, 511, 483, 537, 514, 498, 534, 507, 562, 525, 508, 494, 529, 454, 498, 496, 518, 521, 552, 505, 516, 521, 528, 482, 544, 508, 495, 504, 504, 510, 516, 520, 541, 529, 500, 503, 501, 539] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[536, 535, 548, 501, 501, 506, 534, 521, 514, 477, 482, 523, 468, 479, 526, 491, 492, 539, 502, 512, 528, 538, 553, 516, 516, 529, 485, 517, 523, 548, 463, 499, 479, 534, 530, 488, 545, 515, 477, 544, 504, 484, 531, 521, 526, 501, 513, 513, 541, 507, 517, 532, 495, 491, 524, 519, 514, 490, 494, 519, 506, 509, 515, 492, 518, 486, 504, 484, 543, 517, 561, 508, 528, 508, 493, 558, 551, 483, 526, 501, 499, 506, 505, 497, 514, 531, 487, 527, 515, 513, 493, 499, 480, 502, 552, 527, 528, 483, 468, 496, 543, 517, 491, 530, 470, 517, 480, 543, 484, 527, 513, 505, 487, 492, 541, 484, 495, 528, 529, 523, 518, 491, 553, 563, 515, 497, 501, 536] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[534, 538, 540, 499, 553, 491, 527, 507, 521, 535, 539, 505, 502, 555, 480, 485, 544, 523, 511, 531, 528, 535, 519, 483, 545, 517, 525, 508, 536, 508, 509, 510, 500, 517, 497, 508, 467, 546, 543, 539, 535, 505, 507, 561, 467, 518, 546, 537, 502, 515, 519, 524, 518, 495, 510, 505, 528, 550, 502, 546, 447, 512, 529, 532, 505, 519, 525, 494, 525, 493, 462, 492, 506, 509, 515, 524, 468, 503, 494, 505, 520, 501, 503, 517, 523, 509, 472, 484, 535, 529, 499, 469, 506, 500, 504, 521, 505, 498, 526, 504, 505, 530, 509, 491, 538, 492, 518, 534, 487, 510, 542, 494, 493, 485, 495, 499, 482, 541, 508, 512, 526, 489, 510, 487, 512, 492, 483, 538] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[470, 487, 526, 500, 500, 525, 521, 550, 510, 538, 480, 546, 527, 487, 517, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 499, 515, 539, 500, 505, 534, 493, 572, 542, 500, 482, 528, 524, 513, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 504, 483, 514, 517, 506, 539, 495, 552, 500, 554, 516, 554, 503, 533, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 521, 569, 506, 520, 521, 528, 480, 518, 488, 521, 500, 526, 511, 509, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 452, 512, 501, 519, 551, 473, 526, 528, 532, 539, 503, 466, 549, 490, 481, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 514, 516, 530, 556, 556, 531, 519, 535, 495, 477, 483, 523, 488, 521, 507, 3, 2, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 527, 494, 510, 506, 481, 541, 470, 525, 523, 487, 505, 484, 513, 498, 508, 20, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 471, 522, 516, 485, 480, 528, 484, 505, 533, 519, 517, 532, 485, 527, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8217, 8291, 8255, 8165, 8247, 8133, 8074] avg=8192.0000 max=8291 max_rank=2 min=8074 min_rank=7 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32619 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[537, 494, 512, 457, 553, 527, 510, 502, 525, 522, 490, 534, 501, 494, 533, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 501, 546, 489, 503, 530, 499, 530, 481, 464, 484, 545, 494, 518, 525, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 518, 529, 499, 519, 487, 498, 495, 511, 515, 537, 521, 494, 511, 497, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 533, 508, 471, 492, 495, 502, 515, 528, 554, 476, 469, 513, 491, 494, 558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 501, 515, 490, 524, 463, 465, 490, 503, 527, 532, 530, 539, 529, 521, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 495, 525, 519, 525, 527, 494, 511, 549, 514, 544, 511, 483, 537, 514, 498, 3, 2, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 507, 562, 525, 508, 494, 529, 454, 498, 496, 518, 521, 552, 505, 516, 521, 20, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 482, 537, 508, 495, 504, 504, 510, 516, 520, 541, 529, 500, 503, 501, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8139, 8172, 8110, 8170, 8300, 8264, 8217] avg=8192.0000 max=8300 max_rank=5 min=8110 min_rank=3 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32951 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[525, 501, 497, 536, 455, 502, 528, 527, 508, 517, 513, 474, 521, 501, 498, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 520, 504, 475, 509, 537, 513, 528, 501, 541, 508, 500, 492, 519, 504, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 564, 504, 485, 517, 540, 507, 490, 522, 560, 484, 534, 539, 492, 514, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 523, 542, 554, 516, 506, 548, 514, 516, 506, 496, 520, 494, 484, 523, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 550, 484, 531, 539, 503, 500, 492, 529, 516, 530, 538, 498, 491, 536, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 519, 504, 496, 489, 502, 517, 507, 560, 505, 491, 508, 490, 497, 482, 467, 6, 8, 7, 2, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 499, 535, 508, 503, 512, 497, 480, 549, 537, 530, 560, 524, 515, 516, 500, 24, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 491, 530, 488, 535, 537, 512, 545, 541, 513, 481, 480, 497, 460, 507, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8135, 8133, 8240, 8247, 8272, 8105, 8300, 8104] avg=8192.0000 max=8300 max_rank=6 min=8104 min_rank=7 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32781 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[470, 487, 526, 500, 500, 525, 522, 550, 510, 538, 480, 546, 527, 487, 517, 470, 485, 499, 515, 539, 500, 505, 534, 493, 572, 542, 500, 482, 528, 524, 513, 486, 498, 504, 483, 514, 517, 506, 539, 515, 552, 500, 554, 516, 554, 503, 533, 523, 516, 523, 569, 506, 520, 521, 528, 480, 518, 488, 521, 500, 526, 511, 509, 521, 452, 512, 501, 519, 551, 473, 526, 528, 532, 539, 503, 468, 549, 490, 481, 543, 488, 514, 516, 530, 556, 556, 531, 519, 535, 495, 477, 483, 523, 488, 521, 507, 537, 527, 494, 510, 506, 481, 541, 470, 525, 523, 487, 505, 484, 513, 498, 508, 474, 471, 529, 516, 485, 480, 528, 484, 505, 533, 519, 517, 532, 485, 527, 496] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[501, 527, 524, 521, 516, 499, 492, 480, 511, 517, 487, 494, 522, 495, 546, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 524, 523, 484, 517, 502, 519, 500, 538, 495, 549, 484, 506, 571, 506, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 546, 482, 505, 524, 496, 480, 534, 475, 505, 537, 494, 542, 531, 517, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 551, 507, 478, 510, 514, 538, 521, 462, 519, 498, 522, 530, 579, 548, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 512, 516, 529, 489, 505, 484, 544, 522, 522, 542, 509, 492, 529, 501, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 497, 513, 494, 490, 528, 509, 511, 545, 495, 571, 504, 493, 525, 534, 517, 3, 3, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 509, 526, 509, 521, 484, 476, 500, 505, 498, 499, 496, 483, 533, 520, 550, 22, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 504, 527, 506, 508, 505, 521, 515, 502, 574, 497, 494, 490, 492, 486, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8119, 8228, 8177, 8277, 8235, 8236, 8114, 8150] avg=8192.0000 max=8277 max_rank=3 min=8114 min_rank=6 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32735 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[517, 501, 483, 555, 495, 540, 526, 525, 485, 487, 506, 510, 502, 519, 553, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 452, 495, 530, 498, 536, 468, 511, 512, 537, 530, 474, 544, 470, 512, 500, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 495, 474, 496, 518, 490, 539, 512, 513, 529, 496, 485, 534, 524, 507, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 512, 487, 536, 511, 529, 528, 493, 511, 511, 508, 494, 507, 520, 504, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 481, 527, 541, 492, 528, 503, 526, 548, 538, 494, 500, 486, 496, 491, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 501, 503, 550, 505, 523, 523, 511, 465, 501, 509, 477, 555, 481, 541, 526, 3, 2, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 601, 521, 508, 508, 509, 522, 505, 504, 509, 515, 518, 498, 519, 511, 515, 480, 21, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 524, 562, 541, 510, 542, 531, 523, 560, 525, 527, 504, 464, 512, 515, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8060, 8110, 8214, 8195, 8194, 8268, 8320] avg=8192.0000 max=8320 max_rank=7 min=8060 min_rank=1 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32977 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[533, 510, 500, 542, 508, 531, 539, 551, 523, 500, 538, 541, 531, 513, 490, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 537, 515, 526, 484, 496, 493, 513, 502, 545, 513, 493, 489, 538, 532, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 510, 521, 514, 483, 470, 518, 520, 527, 492, 518, 453, 520, 516, 517, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 491, 500, 503, 509, 529, 466, 498, 468, 501, 504, 487, 508, 516, 520, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 554, 495, 488, 501, 474, 505, 501, 528, 531, 523, 562, 507, 510, 517, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 503, 554, 510, 565, 502, 527, 463, 502, 508, 520, 492, 497, 521, 475, 515, 2, 2, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 517, 514, 504, 500, 505, 462, 546, 491, 515, 518, 509, 488, 497, 506, 482, 21, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 480, 494, 566, 529, 538, 516, 554, 519, 507, 570, 528, 503, 505, 518, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8338, 8167, 8090, 8038, 8195, 8200, 8130, 8378] avg=8192.0000 max=8378 max_rank=7 min=8038 min_rank=3 imbalance_ratio=1.022705 max_load_reduction=57158 max_load_reduction_pct=0.872162 moved_to_echo=32903 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[536, 535, 548, 501, 501, 506, 533, 521, 514, 477, 482, 523, 468, 479, 526, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 539, 502, 512, 528, 538, 553, 516, 516, 529, 485, 517, 523, 548, 463, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 534, 530, 488, 545, 515, 477, 523, 504, 484, 531, 521, 526, 501, 513, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 505, 517, 532, 495, 491, 524, 519, 514, 490, 494, 519, 506, 509, 515, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 486, 504, 484, 543, 517, 561, 508, 528, 508, 493, 556, 551, 483, 526, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 506, 505, 497, 514, 531, 487, 527, 515, 513, 493, 499, 480, 502, 552, 527, 3, 2, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 483, 468, 496, 543, 517, 491, 530, 470, 517, 480, 543, 484, 527, 513, 505, 21, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 492, 534, 484, 495, 528, 529, 523, 518, 491, 553, 563, 515, 497, 501, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8141, 8260, 8184, 8163, 8267, 8155, 8120, 8246] avg=8192.0000 max=8267 max_rank=4 min=8120 min_rank=6 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32788 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[537, 494, 512, 457, 553, 527, 510, 502, 525, 522, 490, 534, 501, 494, 533, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 501, 546, 489, 503, 530, 499, 530, 481, 464, 484, 545, 494, 518, 525, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 518, 529, 499, 519, 487, 498, 495, 511, 515, 537, 521, 494, 511, 497, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 533, 508, 471, 492, 495, 502, 515, 528, 554, 476, 469, 513, 491, 494, 558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 501, 515, 490, 524, 463, 465, 490, 503, 527, 532, 530, 539, 529, 521, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 495, 525, 519, 525, 527, 494, 511, 549, 514, 544, 511, 483, 537, 514, 498, 3, 2, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 507, 562, 525, 508, 494, 529, 454, 498, 496, 518, 521, 552, 505, 516, 521, 20, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 482, 537, 508, 495, 504, 504, 510, 516, 520, 541, 529, 500, 503, 501, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8139, 8172, 8110, 8170, 8300, 8264, 8217] avg=8192.0000 max=8300 max_rank=5 min=8110 min_rank=3 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32951 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[534, 538, 540, 499, 553, 491, 526, 507, 521, 535, 539, 505, 502, 555, 480, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 523, 511, 531, 528, 535, 519, 483, 545, 517, 525, 508, 536, 508, 509, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 517, 497, 508, 467, 546, 543, 518, 535, 505, 507, 561, 467, 518, 546, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 513, 519, 524, 518, 495, 510, 505, 528, 550, 502, 546, 447, 512, 529, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 519, 525, 494, 525, 493, 462, 492, 506, 509, 515, 522, 468, 503, 494, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 501, 503, 517, 523, 509, 472, 484, 535, 529, 499, 469, 506, 500, 504, 521, 2, 2, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 498, 526, 504, 505, 530, 509, 491, 538, 492, 518, 534, 487, 510, 542, 494, 21, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 485, 490, 499, 482, 541, 508, 512, 526, 489, 510, 487, 512, 492, 483, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8310, 8332, 8272, 8232, 8037, 8099, 8207, 8047] avg=8192.0000 max=8332 max_rank=1 min=8037 min_rank=4 imbalance_ratio=1.017090 max_load_reduction=57204 max_load_reduction_pct=0.872864 moved_to_echo=32390 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[533, 510, 500, 542, 508, 531, 539, 551, 523, 500, 538, 541, 531, 513, 490, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 537, 515, 526, 484, 496, 493, 513, 502, 545, 513, 493, 489, 538, 532, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 510, 521, 514, 483, 470, 518, 520, 527, 492, 518, 453, 520, 516, 517, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 491, 500, 503, 509, 529, 466, 498, 468, 501, 504, 487, 508, 516, 520, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 554, 495, 488, 501, 474, 505, 501, 528, 531, 523, 562, 507, 510, 517, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 503, 554, 510, 565, 502, 527, 463, 502, 508, 520, 492, 497, 521, 475, 515, 2, 2, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 517, 514, 504, 500, 505, 462, 546, 491, 515, 518, 509, 488, 497, 506, 482, 21, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 480, 494, 566, 529, 538, 516, 554, 519, 507, 570, 528, 503, 505, 518, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8338, 8167, 8090, 8038, 8195, 8200, 8130, 8378] avg=8192.0000 max=8378 max_rank=7 min=8038 min_rank=3 imbalance_ratio=1.022705 max_load_reduction=57158 max_load_reduction_pct=0.872162 moved_to_echo=32903 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[525, 501, 497, 536, 455, 502, 528, 527, 508, 517, 513, 474, 521, 501, 498, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 520, 504, 475, 509, 537, 513, 528, 501, 541, 508, 500, 492, 519, 504, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 564, 504, 485, 517, 540, 507, 490, 522, 560, 484, 534, 539, 492, 514, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 523, 542, 554, 516, 506, 548, 514, 516, 506, 496, 520, 494, 484, 523, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 550, 484, 531, 539, 503, 500, 492, 529, 516, 530, 538, 498, 491, 536, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 519, 504, 496, 489, 502, 517, 507, 560, 505, 491, 508, 490, 497, 482, 467, 6, 8, 7, 2, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 499, 535, 508, 503, 512, 497, 480, 549, 537, 530, 560, 524, 515, 516, 500, 24, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 491, 530, 488, 535, 537, 512, 545, 541, 513, 481, 480, 497, 460, 507, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8135, 8133, 8240, 8247, 8272, 8105, 8300, 8104] avg=8192.0000 max=8300 max_rank=6 min=8104 min_rank=7 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32781 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[517, 501, 483, 555, 495, 540, 526, 525, 485, 487, 506, 510, 502, 519, 553, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 452, 495, 530, 498, 536, 468, 511, 512, 537, 530, 474, 544, 470, 512, 500, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 495, 474, 496, 518, 490, 539, 512, 513, 529, 496, 485, 534, 524, 507, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 512, 487, 536, 511, 529, 528, 493, 511, 511, 508, 494, 507, 520, 504, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 481, 527, 541, 492, 528, 503, 526, 548, 538, 494, 500, 486, 496, 491, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 501, 503, 550, 505, 523, 523, 511, 465, 501, 509, 477, 555, 481, 541, 526, 3, 2, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 601, 521, 508, 508, 509, 522, 505, 504, 509, 515, 518, 498, 519, 511, 515, 480, 21, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 524, 562, 541, 510, 542, 531, 523, 560, 525, 527, 504, 464, 512, 515, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8060, 8110, 8214, 8195, 8194, 8268, 8320] avg=8192.0000 max=8320 max_rank=7 min=8060 min_rank=1 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32977 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[470, 487, 526, 500, 500, 525, 521, 550, 510, 538, 480, 546, 527, 487, 517, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 499, 515, 539, 500, 505, 534, 493, 572, 542, 500, 482, 528, 524, 513, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 504, 483, 514, 517, 506, 539, 495, 552, 500, 554, 516, 554, 503, 533, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 521, 569, 506, 520, 521, 528, 480, 518, 488, 521, 500, 526, 511, 509, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 452, 512, 501, 519, 551, 473, 526, 528, 532, 539, 503, 466, 549, 490, 481, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 514, 516, 530, 556, 556, 531, 519, 535, 495, 477, 483, 523, 488, 521, 507, 3, 2, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 527, 494, 510, 506, 481, 541, 470, 525, 523, 487, 505, 484, 513, 498, 508, 20, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 471, 522, 516, 485, 480, 528, 484, 505, 533, 519, 517, 532, 485, 527, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8217, 8291, 8255, 8165, 8247, 8133, 8074] avg=8192.0000 max=8291 max_rank=2 min=8074 min_rank=7 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32619 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4352, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4352, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 3968, 4096, 4096, 4224, 4096, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 3968, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4352, 4096, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 3968, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 3968, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 3968, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 3968, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[519, 544, 519, 495, 533, 532, 541, 504, 487, 462, 476, 502, 511, 533, 532, 496, 536, 513, 515, 543, 530, 503, 506, 518, 476, 524, 501, 525, 529, 495, 522, 526, 521, 473, 524, 490, 501, 525, 484, 457, 490, 535, 510, 530, 543, 506, 526, 497, 526, 515, 491, 497, 511, 503, 514, 504, 551, 519, 482, 502, 545, 503, 495, 546, 516, 542, 498, 524, 497, 521, 479, 486, 521, 501, 543, 518, 486, 550, 476, 501, 512, 483, 532, 519, 513, 491, 513, 503, 535, 530, 542, 519, 530, 451, 504, 512, 506, 520, 522, 508, 536, 522, 525, 527, 542, 499, 471, 486, 515, 515, 544, 535, 495, 481, 528, 533, 484, 522, 544, 546, 519, 520, 483, 529, 522, 492, 469, 484] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[501, 493, 506, 492, 498, 554, 517, 549, 534, 526, 490, 508, 529, 494, 509, 474, 500, 526, 523, 494, 534, 475, 535, 519, 557, 502, 513, 496, 462, 527, 508, 504, 522, 512, 536, 542, 482, 497, 481, 526, 528, 497, 500, 507, 500, 506, 536, 503, 533, 505, 503, 502, 529, 498, 499, 487, 519, 506, 473, 501, 590, 494, 508, 497, 542, 541, 515, 504, 520, 531, 519, 513, 507, 514, 532, 524, 481, 529, 501, 521, 526, 515, 535, 493, 537, 521, 524, 520, 488, 496, 534, 518, 503, 540, 507, 517, 503, 433, 537, 522, 494, 532, 524, 491, 519, 500, 505, 504, 539, 480, 506, 518, 501, 541, 488, 555, 502, 537, 518, 523, 507, 501, 498, 521, 459, 560, 486, 496] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[529, 528, 535, 498, 515, 507, 516, 513, 506, 501, 496, 527, 530, 558, 499, 516, 491, 501, 494, 524, 486, 527, 562, 539, 514, 492, 488, 531, 492, 496, 508, 492, 508, 508, 512, 515, 477, 539, 520, 528, 524, 525, 538, 520, 545, 472, 502, 516, 535, 481, 522, 486, 498, 526, 509, 517, 498, 457, 502, 513, 515, 522, 509, 488, 524, 457, 512, 534, 536, 535, 507, 544, 513, 499, 530, 490, 484, 495, 489, 519, 495, 501, 456, 535, 541, 498, 508, 484, 522, 524, 505, 526, 518, 496, 523, 520, 502, 537, 556, 513, 518, 514, 529, 464, 489, 531, 479, 513, 524, 544, 483, 505, 511, 508, 483, 548, 476, 545, 504, 518, 481, 552, 516, 533, 536, 522, 494, 550] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[503, 533, 480, 490, 546, 501, 536, 508, 513, 487, 483, 498, 503, 519, 493, 482, 499, 488, 543, 515, 503, 543, 510, 496, 485, 518, 494, 523, 525, 486, 530, 513, 493, 528, 516, 529, 513, 551, 512, 528, 530, 509, 501, 480, 473, 528, 529, 510, 497, 559, 523, 476, 530, 498, 532, 496, 507, 496, 505, 507, 508, 508, 537, 526, 518, 462, 513, 507, 519, 494, 533, 493, 511, 506, 512, 544, 511, 513, 509, 514, 517, 513, 522, 517, 506, 539, 516, 496, 486, 485, 498, 521, 534, 503, 519, 499, 506, 475, 531, 493, 489, 500, 516, 526, 495, 523, 506, 471, 546, 502, 524, 510, 544, 532, 516, 502, 521, 526, 565, 561, 519, 515, 532, 487, 533, 508, 553, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[505, 512, 483, 520, 532, 505, 475, 519, 526, 491, 514, 521, 543, 507, 485, 494, 513, 532, 515, 514, 540, 513, 524, 529, 527, 527, 470, 519, 529, 509, 497, 527, 512, 518, 506, 526, 531, 502, 533, 536, 545, 542, 521, 534, 524, 526, 492, 525, 464, 491, 481, 500, 517, 523, 507, 488, 543, 486, 518, 514, 496, 549, 497, 491, 493, 477, 495, 523, 474, 507, 490, 533, 517, 528, 502, 488, 518, 509, 486, 480, 472, 515, 518, 523, 529, 499, 485, 532, 507, 508, 500, 546, 506, 506, 525, 522, 477, 512, 498, 520, 525, 533, 528, 488, 508, 515, 554, 504, 499, 541, 496, 520, 525, 535, 489, 545, 506, 487, 521, 517, 495, 493, 531, 526, 500, 522, 540, 518] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[484, 496, 512, 513, 520, 529, 520, 512, 496, 545, 515, 520, 507, 515, 554, 495, 540, 521, 515, 525, 512, 513, 513, 506, 527, 523, 527, 507, 513, 519, 481, 506, 534, 532, 511, 497, 539, 498, 523, 496, 505, 526, 552, 471, 511, 522, 533, 530, 530, 506, 524, 521, 485, 513, 547, 534, 516, 511, 487, 520, 486, 522, 477, 523, 498, 495, 510, 528, 498, 512, 493, 492, 515, 522, 505, 498, 541, 484, 510, 493, 546, 504, 477, 529, 510, 548, 495, 492, 496, 538, 504, 494, 492, 510, 505, 530, 505, 490, 493, 524, 458, 539, 513, 516, 482, 511, 493, 507, 478, 507, 554, 482, 524, 502, 521, 508, 536, 542, 534, 510, 508, 517, 513, 515, 507, 492, 501, 527] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[529, 508, 545, 535, 547, 518, 497, 526, 528, 530, 501, 547, 502, 487, 509, 505, 530, 539, 511, 477, 508, 492, 552, 523, 508, 499, 543, 512, 544, 511, 502, 547, 498, 522, 537, 484, 546, 492, 519, 491, 518, 510, 509, 485, 488, 532, 491, 502, 495, 514, 510, 523, 506, 545, 514, 524, 498, 487, 491, 523, 492, 522, 524, 516, 507, 497, 491, 520, 509, 545, 487, 489, 464, 487, 510, 497, 520, 541, 524, 462, 532, 533, 524, 529, 547, 504, 544, 536, 501, 505, 501, 501, 502, 563, 519, 516, 489, 494, 463, 506, 543, 486, 488, 463, 491, 502, 492, 506, 517, 504, 516, 470, 514, 493, 514, 508, 506, 528, 488, 526, 525, 520, 536, 498, 546, 527, 491, 559] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[515, 513, 569, 545, 501, 512, 508, 539, 538, 523, 498, 506, 486, 484, 523, 478, 506, 520, 541, 490, 524, 505, 486, 523, 488, 481, 517, 530, 508, 522, 501, 491, 556, 469, 536, 525, 511, 529, 537, 494, 509, 471, 501, 481, 504, 515, 552, 483, 489, 527, 538, 490, 544, 508, 546, 540, 519, 521, 489, 509, 487, 497, 512, 518, 541, 531, 484, 473, 516, 501, 521, 498, 549, 534, 538, 475, 497, 518, 515, 512, 528, 485, 506, 492, 493, 565, 535, 546, 472, 498, 480, 518, 486, 533, 545, 520, 493, 507, 509, 466, 483, 489, 514, 490, 504, 506, 498, 524, 518, 534, 487, 561, 528, 541, 522, 540, 528, 517, 517, 511, 516, 549, 511, 539, 502, 480, 481, 488] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[505, 512, 483, 520, 532, 505, 475, 519, 526, 491, 514, 521, 543, 507, 485, 494, 513, 532, 515, 514, 540, 513, 524, 529, 527, 527, 470, 519, 529, 509, 497, 527, 512, 518, 506, 526, 531, 502, 533, 536, 545, 542, 521, 534, 524, 526, 492, 525, 464, 491, 481, 500, 517, 523, 507, 488, 543, 486, 518, 514, 496, 549, 497, 491, 493, 477, 495, 523, 474, 507, 490, 533, 517, 528, 502, 488, 518, 509, 486, 480, 472, 515, 518, 523, 529, 499, 485, 532, 507, 508, 500, 546, 506, 506, 525, 522, 477, 512, 498, 520, 525, 533, 528, 488, 508, 515, 554, 504, 499, 541, 496, 520, 525, 535, 489, 545, 506, 487, 521, 517, 495, 493, 531, 526, 500, 522, 540, 518] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[503, 533, 480, 490, 546, 501, 536, 508, 513, 487, 483, 498, 503, 519, 493, 482, 499, 488, 543, 515, 503, 543, 510, 496, 485, 518, 494, 523, 525, 486, 530, 513, 493, 528, 516, 529, 513, 551, 512, 528, 530, 509, 501, 480, 473, 528, 529, 510, 497, 559, 523, 476, 530, 498, 532, 496, 507, 496, 505, 507, 508, 508, 537, 526, 518, 462, 513, 507, 519, 494, 533, 493, 511, 506, 512, 544, 511, 513, 509, 514, 517, 513, 522, 517, 506, 539, 516, 496, 486, 485, 498, 521, 534, 503, 519, 499, 506, 475, 531, 493, 489, 500, 516, 526, 495, 523, 506, 471, 546, 502, 524, 510, 544, 532, 516, 502, 521, 526, 565, 561, 519, 515, 532, 487, 533, 508, 553, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[519, 544, 519, 495, 522, 532, 541, 504, 487, 462, 476, 502, 511, 533, 532, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 513, 515, 543, 530, 503, 486, 518, 476, 524, 501, 525, 529, 495, 522, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 473, 503, 490, 501, 525, 484, 457, 490, 535, 510, 530, 543, 506, 526, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 515, 491, 497, 511, 503, 514, 504, 551, 519, 482, 502, 545, 503, 495, 546, 15, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 542, 498, 524, 497, 521, 479, 486, 521, 501, 543, 518, 486, 550, 476, 501, 21, 20, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 483, 532, 519, 493, 491, 513, 503, 535, 530, 542, 519, 530, 451, 504, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 520, 522, 508, 536, 522, 525, 527, 542, 499, 471, 486, 515, 515, 544, 535, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 481, 528, 464, 484, 522, 544, 546, 519, 520, 483, 529, 522, 492, 469, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8242, 8091, 8230, 8205, 8169, 8342, 8082] avg=8192.0000 max=8342 max_rank=6 min=8082 min_rank=7 imbalance_ratio=1.018311 max_load_reduction=57194 max_load_reduction_pct=0.872711 moved_to_echo=32798 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[501, 493, 506, 492, 498, 554, 517, 549, 534, 526, 490, 508, 529, 494, 509, 474, 500, 526, 523, 494, 534, 475, 535, 519, 557, 502, 513, 496, 462, 527, 508, 504, 522, 512, 536, 542, 482, 497, 481, 526, 528, 497, 500, 507, 500, 506, 536, 503, 533, 505, 503, 502, 529, 498, 499, 487, 519, 506, 473, 501, 590, 494, 508, 497, 542, 541, 515, 504, 520, 531, 519, 513, 507, 514, 532, 524, 481, 529, 501, 521, 526, 515, 535, 493, 537, 521, 524, 520, 488, 496, 534, 518, 503, 540, 507, 517, 503, 433, 537, 522, 494, 532, 524, 491, 519, 500, 505, 504, 539, 480, 506, 518, 501, 541, 488, 555, 502, 537, 518, 523, 507, 501, 498, 521, 459, 560, 486, 496] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[529, 508, 545, 535, 547, 518, 497, 526, 528, 530, 501, 547, 502, 487, 509, 505, 530, 539, 511, 477, 508, 492, 552, 523, 508, 499, 543, 512, 544, 511, 502, 547, 498, 522, 537, 484, 546, 492, 519, 491, 518, 510, 509, 485, 488, 532, 491, 502, 495, 514, 510, 523, 506, 545, 514, 524, 498, 487, 491, 523, 492, 522, 524, 516, 507, 497, 491, 520, 509, 545, 487, 489, 464, 487, 510, 497, 520, 541, 524, 462, 532, 533, 524, 529, 547, 504, 544, 536, 501, 505, 501, 501, 502, 563, 519, 516, 489, 494, 463, 506, 543, 486, 488, 463, 491, 502, 492, 506, 517, 504, 516, 470, 514, 493, 514, 508, 506, 528, 488, 526, 525, 520, 536, 498, 546, 527, 491, 559] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[519, 544, 519, 495, 533, 532, 541, 504, 487, 462, 476, 502, 511, 533, 532, 496, 536, 513, 515, 543, 530, 503, 506, 518, 476, 524, 501, 525, 529, 495, 522, 526, 521, 473, 524, 490, 501, 525, 484, 457, 490, 535, 510, 530, 543, 506, 526, 497, 526, 515, 491, 497, 511, 503, 514, 504, 551, 519, 482, 502, 545, 503, 495, 546, 516, 542, 498, 524, 497, 521, 479, 486, 521, 501, 543, 518, 486, 550, 476, 501, 512, 483, 532, 519, 513, 491, 513, 503, 535, 530, 542, 519, 530, 451, 504, 512, 506, 520, 522, 508, 536, 522, 525, 527, 542, 499, 471, 486, 515, 515, 544, 535, 495, 481, 528, 533, 484, 522, 544, 546, 519, 520, 483, 529, 522, 492, 469, 484] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[529, 528, 535, 498, 504, 507, 516, 513, 506, 501, 496, 527, 530, 558, 499, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 501, 494, 524, 486, 527, 539, 539, 514, 492, 488, 531, 492, 496, 508, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 508, 491, 515, 477, 539, 520, 528, 524, 525, 538, 520, 545, 472, 502, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 481, 522, 486, 498, 526, 509, 517, 498, 457, 502, 513, 515, 522, 509, 488, 16, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 457, 512, 534, 536, 535, 507, 544, 513, 499, 530, 490, 484, 495, 489, 519, 21, 23, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 501, 456, 535, 520, 498, 508, 484, 522, 524, 505, 526, 518, 496, 523, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 537, 556, 513, 518, 514, 529, 464, 489, 531, 479, 513, 524, 544, 483, 505, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 508, 483, 477, 476, 545, 504, 518, 481, 552, 516, 533, 536, 522, 494, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8263, 8114, 8228, 8105, 8217, 8131, 8272, 8206] avg=8192.0000 max=8272 max_rank=6 min=8105 min_rank=3 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32826 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[501, 493, 506, 492, 488, 554, 517, 549, 534, 526, 490, 508, 529, 494, 509, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 526, 523, 494, 534, 475, 513, 519, 557, 502, 513, 496, 462, 527, 508, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 512, 514, 542, 482, 497, 481, 526, 528, 497, 500, 507, 500, 506, 536, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 505, 503, 502, 529, 498, 499, 487, 519, 506, 473, 501, 590, 494, 508, 497, 16, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 541, 515, 504, 520, 531, 519, 513, 507, 514, 532, 524, 481, 529, 501, 521, 22, 22, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 515, 535, 493, 516, 521, 524, 520, 488, 496, 534, 518, 503, 540, 507, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 433, 537, 522, 494, 532, 524, 491, 519, 500, 505, 504, 539, 480, 506, 518, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 541, 488, 483, 502, 537, 518, 523, 507, 501, 498, 521, 459, 560, 486, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8153, 8153, 8170, 8343, 8253, 8179, 8121] avg=8192.0000 max=8343 max_rank=4 min=8121 min_rank=7 imbalance_ratio=1.018433 max_load_reduction=57193 max_load_reduction_pct=0.872696 moved_to_echo=32896 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[519, 544, 519, 495, 522, 532, 541, 504, 487, 462, 476, 502, 511, 533, 532, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 513, 515, 543, 530, 503, 486, 518, 476, 524, 501, 525, 529, 495, 522, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 473, 503, 490, 501, 525, 484, 457, 490, 535, 510, 530, 543, 506, 526, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 515, 491, 497, 511, 503, 514, 504, 551, 519, 482, 502, 545, 503, 495, 546, 15, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 542, 498, 524, 497, 521, 479, 486, 521, 501, 543, 518, 486, 550, 476, 501, 21, 20, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 483, 532, 519, 493, 491, 513, 503, 535, 530, 542, 519, 530, 451, 504, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 520, 522, 508, 536, 522, 525, 527, 542, 499, 471, 486, 515, 515, 544, 535, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 481, 528, 464, 484, 522, 544, 546, 519, 520, 483, 529, 522, 492, 469, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8242, 8091, 8230, 8205, 8169, 8342, 8082] avg=8192.0000 max=8342 max_rank=6 min=8082 min_rank=7 imbalance_ratio=1.018311 max_load_reduction=57194 max_load_reduction_pct=0.872711 moved_to_echo=32798 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[503, 533, 480, 490, 531, 501, 536, 508, 513, 487, 483, 498, 503, 519, 493, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 488, 543, 515, 503, 543, 486, 496, 485, 518, 494, 523, 525, 486, 530, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 528, 486, 529, 513, 551, 512, 528, 530, 509, 501, 480, 473, 528, 529, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 559, 523, 476, 530, 498, 532, 496, 507, 496, 505, 507, 508, 508, 537, 526, 20, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 462, 513, 507, 519, 494, 533, 493, 511, 506, 512, 544, 511, 513, 509, 514, 24, 24, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 513, 522, 517, 478, 539, 516, 496, 486, 485, 498, 521, 534, 503, 519, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 475, 531, 493, 489, 500, 516, 526, 495, 523, 506, 471, 546, 502, 524, 510, 68, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 532, 516, 434, 521, 526, 565, 561, 519, 515, 532, 487, 533, 508, 553, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8060, 8147, 8200, 8240, 8215, 8143, 8187, 8344] avg=8192.0000 max=8344 max_rank=7 min=8060 min_rank=0 imbalance_ratio=1.018555 max_load_reduction=57192 max_load_reduction_pct=0.872681 moved_to_echo=32889 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[484, 496, 512, 513, 509, 529, 520, 512, 496, 545, 515, 520, 507, 515, 554, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 521, 515, 525, 512, 513, 492, 506, 527, 523, 527, 507, 513, 519, 481, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 532, 490, 497, 539, 498, 523, 496, 505, 526, 552, 471, 511, 522, 533, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 506, 524, 521, 485, 513, 547, 534, 516, 511, 487, 520, 486, 522, 477, 523, 15, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 495, 510, 528, 498, 512, 493, 492, 515, 522, 505, 498, 541, 484, 510, 493, 21, 21, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 504, 477, 529, 490, 548, 495, 492, 496, 538, 504, 494, 492, 510, 505, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 490, 493, 524, 458, 539, 513, 516, 482, 511, 493, 507, 478, 507, 554, 482, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 502, 521, 442, 536, 542, 534, 510, 508, 517, 513, 515, 507, 492, 501, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8227, 8259, 8228, 8141, 8150, 8118, 8191] avg=8192.0000 max=8259 max_rank=2 min=8118 min_rank=6 imbalance_ratio=1.008179 max_load_reduction=57277 max_load_reduction_pct=0.873978 moved_to_echo=32600 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[505, 512, 483, 520, 521, 505, 475, 519, 526, 491, 514, 521, 543, 507, 485, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 532, 515, 514, 540, 513, 503, 529, 527, 527, 470, 519, 529, 509, 497, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 518, 485, 526, 531, 502, 533, 536, 545, 542, 521, 534, 524, 526, 492, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 464, 491, 481, 500, 517, 523, 507, 488, 543, 486, 518, 514, 496, 549, 497, 491, 16, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 477, 495, 523, 474, 507, 490, 533, 517, 528, 502, 488, 518, 509, 486, 480, 21, 21, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 515, 518, 523, 508, 499, 485, 532, 507, 508, 500, 546, 506, 506, 525, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 512, 498, 520, 525, 533, 528, 488, 508, 515, 554, 504, 499, 541, 496, 520, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 535, 489, 474, 506, 487, 521, 517, 495, 493, 531, 526, 500, 522, 540, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8121, 8264, 8352, 8092, 8067, 8172, 8289, 8179] avg=8192.0000 max=8352 max_rank=2 min=8067 min_rank=4 imbalance_ratio=1.019531 max_load_reduction=57184 max_load_reduction_pct=0.872559 moved_to_echo=32707 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[515, 513, 569, 545, 491, 512, 508, 539, 538, 523, 498, 506, 486, 484, 523, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 520, 541, 490, 524, 505, 466, 523, 488, 481, 517, 530, 508, 522, 501, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 469, 514, 525, 511, 529, 537, 494, 509, 471, 501, 481, 504, 515, 552, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 527, 538, 490, 544, 508, 546, 540, 519, 521, 489, 509, 487, 497, 512, 518, 15, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 531, 484, 473, 516, 501, 521, 498, 549, 534, 538, 475, 497, 518, 515, 512, 22, 20, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 485, 506, 492, 474, 565, 535, 546, 472, 498, 480, 518, 486, 533, 545, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 507, 509, 466, 483, 489, 514, 490, 504, 506, 498, 524, 518, 534, 487, 561, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 541, 522, 470, 528, 517, 517, 511, 516, 549, 511, 539, 502, 480, 481, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8228, 8113, 8151, 8259, 8249, 8183, 8153, 8200] avg=8192.0000 max=8259 max_rank=3 min=8113 min_rank=1 imbalance_ratio=1.008179 max_load_reduction=57277 max_load_reduction_pct=0.873978 moved_to_echo=32785 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[501, 493, 506, 492, 488, 554, 517, 549, 534, 526, 490, 508, 529, 494, 509, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 526, 523, 494, 534, 475, 513, 519, 557, 502, 513, 496, 462, 527, 508, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 512, 514, 542, 482, 497, 481, 526, 528, 497, 500, 507, 500, 506, 536, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 505, 503, 502, 529, 498, 499, 487, 519, 506, 473, 501, 590, 494, 508, 497, 16, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 541, 515, 504, 520, 531, 519, 513, 507, 514, 532, 524, 481, 529, 501, 521, 22, 22, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 515, 535, 493, 516, 521, 524, 520, 488, 496, 534, 518, 503, 540, 507, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 433, 537, 522, 494, 532, 524, 491, 519, 500, 505, 504, 539, 480, 506, 518, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 541, 488, 483, 502, 537, 518, 523, 507, 501, 498, 521, 459, 560, 486, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8153, 8153, 8170, 8343, 8253, 8179, 8121] avg=8192.0000 max=8343 max_rank=4 min=8121 min_rank=7 imbalance_ratio=1.018433 max_load_reduction=57193 max_load_reduction_pct=0.872696 moved_to_echo=32896 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[529, 508, 545, 535, 536, 518, 497, 526, 528, 530, 501, 547, 502, 487, 509, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 539, 511, 477, 508, 492, 530, 523, 508, 499, 543, 512, 544, 511, 502, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 522, 515, 484, 546, 492, 519, 491, 518, 510, 509, 485, 488, 532, 491, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 514, 510, 523, 506, 545, 514, 524, 498, 487, 491, 523, 492, 522, 524, 516, 17, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 497, 491, 520, 509, 545, 487, 489, 464, 487, 510, 497, 520, 541, 524, 462, 22, 22, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 533, 524, 529, 525, 504, 544, 536, 501, 505, 501, 501, 502, 563, 519, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 494, 463, 506, 543, 486, 488, 463, 491, 502, 492, 506, 517, 504, 516, 470, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 493, 514, 442, 506, 528, 488, 526, 525, 520, 536, 498, 546, 527, 491, 559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8303, 8276, 8102, 8212, 8099, 8335, 7996, 8213] avg=8192.0000 max=8335 max_rank=5 min=7996 min_rank=6 imbalance_ratio=1.017456 max_load_reduction=57201 max_load_reduction_pct=0.872818 moved_to_echo=32643 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[505, 512, 483, 520, 521, 505, 475, 519, 526, 491, 514, 521, 543, 507, 485, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 532, 515, 514, 540, 513, 503, 529, 527, 527, 470, 519, 529, 509, 497, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 518, 485, 526, 531, 502, 533, 536, 545, 542, 521, 534, 524, 526, 492, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 464, 491, 481, 500, 517, 523, 507, 488, 543, 486, 518, 514, 496, 549, 497, 491, 16, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 477, 495, 523, 474, 507, 490, 533, 517, 528, 502, 488, 518, 509, 486, 480, 21, 21, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 515, 518, 523, 508, 499, 485, 532, 507, 508, 500, 546, 506, 506, 525, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 512, 498, 520, 525, 533, 528, 488, 508, 515, 554, 504, 499, 541, 496, 520, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 535, 489, 474, 506, 487, 521, 517, 495, 493, 531, 526, 500, 522, 540, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8121, 8264, 8352, 8092, 8067, 8172, 8289, 8179] avg=8192.0000 max=8352 max_rank=2 min=8067 min_rank=4 imbalance_ratio=1.019531 max_load_reduction=57184 max_load_reduction_pct=0.872559 moved_to_echo=32707 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[503, 533, 480, 490, 531, 501, 536, 508, 513, 487, 483, 498, 503, 519, 493, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 488, 543, 515, 503, 543, 486, 496, 485, 518, 494, 523, 525, 486, 530, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 528, 486, 529, 513, 551, 512, 528, 530, 509, 501, 480, 473, 528, 529, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 559, 523, 476, 530, 498, 532, 496, 507, 496, 505, 507, 508, 508, 537, 526, 20, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 462, 513, 507, 519, 494, 533, 493, 511, 506, 512, 544, 511, 513, 509, 514, 24, 24, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 513, 522, 517, 478, 539, 516, 496, 486, 485, 498, 521, 534, 503, 519, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 475, 531, 493, 489, 500, 516, 526, 495, 523, 506, 471, 546, 502, 524, 510, 68, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 532, 516, 434, 521, 526, 565, 561, 519, 515, 532, 487, 533, 508, 553, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8060, 8147, 8200, 8240, 8215, 8143, 8187, 8344] avg=8192.0000 max=8344 max_rank=7 min=8060 min_rank=0 imbalance_ratio=1.018555 max_load_reduction=57192 max_load_reduction_pct=0.872681 moved_to_echo=32889 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 3968, 4224, 4096, 4096, 4224, 4224, 3968, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 3712, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 3968, 4096, 4224, 4224, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 3968, 4224, 4096, 4096, 4224, 4224, 3968, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 256, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[571, 501, 521, 495, 508, 531, 506, 500, 550, 487, 536, 502, 528, 508, 552, 510, 505, 538, 507, 511, 499, 484, 529, 502, 496, 520, 488, 515, 504, 558, 508, 520, 505, 504, 582, 497, 501, 521, 531, 497, 542, 529, 491, 508, 523, 512, 507, 510, 505, 529, 555, 516, 526, 483, 507, 503, 501, 530, 469, 495, 515, 522, 489, 489, 492, 516, 503, 491, 504, 520, 500, 561, 531, 497, 470, 539, 529, 513, 517, 500, 519, 538, 496, 551, 470, 570, 497, 503, 508, 528, 525, 498, 472, 517, 511, 519, 495, 466, 499, 478, 536, 548, 508, 512, 502, 508, 483, 498, 496, 502, 510, 515, 521, 505, 471, 520, 533, 525, 487, 487, 537, 526, 489, 531, 509, 517, 526, 507] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[491, 496, 493, 501, 550, 560, 511, 532, 472, 520, 515, 525, 484, 544, 510, 542, 524, 498, 475, 505, 510, 486, 520, 499, 515, 497, 521, 519, 529, 506, 495, 522, 516, 520, 489, 495, 511, 523, 496, 559, 512, 513, 491, 498, 537, 498, 505, 490, 503, 521, 525, 502, 499, 548, 534, 543, 486, 495, 487, 532, 500, 507, 521, 509, 520, 502, 524, 542, 540, 495, 502, 494, 499, 550, 509, 488, 488, 511, 497, 487, 512, 510, 526, 508, 524, 511, 508, 532, 471, 554, 520, 509, 517, 522, 536, 483, 547, 485, 532, 519, 522, 512, 517, 499, 551, 551, 481, 540, 499, 510, 516, 495, 509, 533, 491, 490, 501, 489, 504, 539, 499, 495, 522, 525, 510, 504, 492, 534] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[498, 537, 491, 501, 532, 499, 526, 533, 513, 512, 494, 525, 515, 477, 529, 510, 529, 530, 498, 470, 528, 542, 468, 535, 491, 526, 478, 543, 493, 475, 497, 532, 491, 498, 522, 530, 486, 522, 539, 557, 489, 540, 495, 498, 532, 503, 503, 482, 503, 510, 535, 541, 515, 497, 485, 535, 519, 527, 500, 504, 499, 486, 491, 524, 520, 546, 541, 508, 492, 528, 510, 518, 520, 553, 517, 533, 510, 498, 520, 527, 485, 554, 488, 514, 492, 547, 504, 508, 530, 501, 511, 489, 510, 499, 556, 512, 518, 514, 491, 520, 506, 502, 493, 515, 498, 506, 492, 503, 503, 503, 511, 496, 506, 542, 486, 530, 511, 479, 507, 552, 500, 521, 502, 529, 547, 518, 537, 472] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[487, 499, 479, 524, 502, 558, 527, 463, 481, 521, 507, 512, 527, 517, 557, 556, 522, 521, 495, 587, 453, 494, 491, 487, 514, 490, 493, 532, 481, 499, 503, 518, 527, 518, 497, 460, 536, 476, 511, 519, 479, 526, 508, 545, 502, 527, 524, 505, 529, 541, 489, 516, 482, 519, 505, 537, 553, 488, 568, 524, 512, 562, 506, 548, 499, 473, 493, 538, 539, 564, 502, 532, 497, 554, 527, 495, 480, 508, 480, 479, 491, 538, 524, 514, 532, 519, 483, 504, 515, 455, 532, 506, 520, 532, 504, 456, 513, 510, 520, 540, 489, 523, 481, 532, 528, 511, 483, 542, 497, 527, 518, 515, 522, 497, 517, 520, 512, 537, 513, 540, 490, 473, 529, 518, 481, 528, 518, 491] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[500, 558, 534, 528, 544, 504, 546, 500, 506, 556, 511, 517, 525, 519, 485, 565, 513, 486, 530, 521, 549, 503, 484, 488, 525, 537, 524, 478, 490, 494, 504, 524, 540, 477, 514, 478, 482, 526, 498, 502, 536, 523, 509, 500, 505, 492, 476, 510, 520, 488, 546, 474, 483, 519, 488, 534, 540, 529, 532, 508, 551, 488, 511, 523, 490, 497, 536, 523, 501, 492, 506, 479, 480, 534, 518, 474, 499, 553, 532, 478, 518, 518, 526, 502, 550, 502, 508, 514, 518, 553, 544, 511, 498, 546, 500, 531, 528, 510, 517, 526, 525, 519, 537, 497, 491, 505, 487, 490, 505, 516, 490, 501, 493, 534, 507, 520, 510, 533, 487, 509, 496, 495, 514, 497, 549, 484, 507, 476] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[505, 566, 542, 530, 533, 546, 512, 533, 497, 478, 527, 509, 480, 504, 497, 512, 511, 493, 516, 510, 512, 530, 503, 480, 502, 470, 492, 544, 512, 532, 504, 533, 530, 505, 512, 514, 501, 502, 494, 489, 484, 476, 486, 509, 536, 498, 522, 559, 518, 487, 504, 507, 520, 489, 550, 523, 513, 513, 494, 490, 500, 523, 536, 489, 479, 527, 507, 501, 527, 486, 501, 544, 497, 533, 532, 526, 510, 524, 516, 525, 515, 502, 520, 511, 543, 514, 541, 508, 480, 523, 500, 546, 528, 530, 490, 470, 517, 488, 501, 484, 511, 492, 521, 561, 539, 495, 471, 536, 523, 520, 489, 495, 507, 487, 542, 512, 534, 500, 497, 500, 545, 503, 483, 559, 497, 517, 561, 505] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[549, 542, 512, 495, 519, 530, 494, 534, 547, 581, 542, 510, 492, 526, 538, 525, 470, 483, 461, 499, 535, 528, 508, 493, 499, 523, 539, 509, 488, 513, 515, 486, 516, 500, 497, 514, 500, 509, 506, 530, 489, 526, 540, 490, 493, 506, 486, 551, 550, 488, 486, 517, 488, 534, 500, 517, 529, 474, 517, 508, 458, 516, 531, 489, 512, 524, 494, 506, 528, 528, 495, 508, 519, 485, 488, 508, 496, 517, 536, 491, 490, 509, 508, 537, 501, 541, 538, 498, 527, 533, 537, 529, 529, 549, 516, 524, 492, 517, 472, 468, 515, 504, 516, 506, 506, 515, 509, 478, 493, 521, 557, 487, 523, 513, 525, 554, 506, 517, 516, 540, 490, 516, 540, 474, 532, 501, 494, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[474, 512, 489, 514, 500, 514, 477, 493, 518, 554, 495, 522, 507, 512, 486, 492, 478, 544, 532, 519, 498, 487, 527, 504, 512, 545, 488, 511, 510, 565, 524, 510, 505, 524, 525, 547, 548, 515, 543, 485, 495, 479, 505, 502, 496, 523, 503, 496, 504, 510, 522, 476, 495, 491, 480, 454, 522, 561, 497, 533, 504, 487, 536, 552, 519, 544, 470, 538, 539, 547, 511, 493, 499, 481, 509, 498, 507, 509, 498, 489, 520, 542, 502, 503, 466, 526, 530, 530, 456, 509, 535, 511, 503, 534, 508, 509, 454, 489, 491, 529, 528, 529, 510, 510, 512, 512, 515, 547, 480, 506, 513, 533, 521, 527, 509, 550, 505, 547, 514, 551, 505, 516, 526, 521, 528, 545, 525, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[571, 501, 521, 495, 508, 531, 506, 500, 550, 487, 536, 502, 528, 508, 552, 510, 505, 538, 507, 511, 499, 484, 529, 502, 496, 520, 488, 515, 504, 558, 508, 520, 505, 504, 582, 497, 501, 521, 531, 497, 542, 529, 491, 508, 523, 512, 507, 510, 505, 529, 555, 516, 526, 483, 507, 503, 501, 530, 469, 495, 515, 522, 489, 489, 492, 516, 503, 491, 504, 520, 500, 561, 531, 497, 470, 539, 529, 513, 517, 500, 519, 538, 496, 551, 470, 570, 497, 503, 508, 528, 525, 498, 472, 517, 511, 519, 495, 466, 499, 478, 536, 548, 508, 512, 502, 508, 483, 498, 496, 502, 510, 515, 521, 505, 471, 520, 533, 525, 487, 487, 537, 526, 489, 531, 509, 517, 526, 507] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[491, 496, 493, 501, 550, 560, 511, 532, 472, 520, 515, 525, 484, 544, 510, 542, 524, 498, 475, 505, 510, 486, 520, 499, 515, 497, 521, 519, 529, 506, 495, 522, 516, 520, 489, 495, 511, 523, 496, 559, 512, 513, 491, 498, 537, 498, 505, 490, 503, 521, 525, 502, 499, 548, 534, 543, 486, 495, 487, 532, 500, 507, 521, 509, 520, 502, 524, 542, 540, 495, 502, 494, 499, 550, 509, 488, 488, 511, 497, 487, 512, 510, 526, 508, 524, 511, 508, 532, 471, 554, 520, 509, 517, 522, 536, 483, 547, 485, 532, 519, 522, 512, 517, 499, 551, 551, 481, 540, 499, 510, 516, 495, 509, 533, 491, 490, 501, 489, 504, 539, 499, 495, 522, 525, 510, 504, 492, 534] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[505, 566, 542, 530, 533, 472, 512, 533, 497, 478, 527, 509, 480, 504, 497, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 493, 516, 510, 512, 530, 503, 480, 502, 470, 492, 544, 512, 532, 504, 533, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 505, 512, 514, 501, 502, 494, 489, 484, 476, 486, 509, 536, 498, 522, 559, 15, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 487, 504, 507, 520, 489, 550, 523, 513, 513, 494, 490, 500, 523, 536, 489, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 527, 507, 501, 527, 486, 501, 544, 497, 533, 532, 526, 510, 524, 516, 525, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 502, 520, 511, 543, 471, 541, 508, 480, 523, 500, 546, 528, 530, 490, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 488, 501, 484, 511, 492, 521, 561, 539, 495, 471, 536, 523, 520, 489, 495, 28, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 487, 542, 512, 534, 500, 497, 475, 545, 503, 483, 559, 497, 517, 561, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8197, 8145, 8158, 8245, 8178, 8192, 8224] avg=8192.0000 max=8245 max_rank=4 min=8145 min_rank=2 imbalance_ratio=1.006470 max_load_reduction=57291 max_load_reduction_pct=0.874191 moved_to_echo=32839 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[487, 499, 479, 524, 502, 558, 527, 463, 481, 521, 507, 512, 527, 517, 557, 556, 522, 521, 495, 587, 453, 494, 491, 487, 514, 490, 493, 532, 481, 499, 503, 518, 527, 518, 497, 460, 536, 476, 511, 519, 479, 526, 508, 545, 502, 527, 524, 505, 529, 541, 489, 516, 482, 519, 505, 537, 553, 488, 568, 524, 512, 562, 506, 548, 499, 473, 493, 538, 539, 564, 502, 532, 497, 554, 527, 495, 480, 508, 480, 479, 491, 538, 524, 514, 532, 519, 483, 504, 515, 455, 532, 506, 520, 532, 504, 456, 513, 510, 520, 540, 489, 523, 481, 532, 528, 511, 483, 542, 497, 527, 518, 515, 522, 497, 517, 520, 512, 537, 513, 540, 490, 473, 529, 518, 481, 528, 518, 491] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[505, 566, 542, 530, 533, 546, 512, 533, 497, 478, 527, 509, 480, 504, 497, 512, 511, 493, 516, 510, 512, 530, 503, 480, 502, 470, 492, 544, 512, 532, 504, 533, 530, 505, 512, 514, 501, 502, 494, 489, 484, 476, 486, 509, 536, 498, 522, 559, 518, 487, 504, 507, 520, 489, 550, 523, 513, 513, 494, 490, 500, 523, 536, 489, 479, 527, 507, 501, 527, 486, 501, 544, 497, 533, 532, 526, 510, 524, 516, 525, 515, 502, 520, 511, 543, 514, 541, 508, 480, 523, 500, 546, 528, 530, 490, 470, 517, 488, 501, 484, 511, 492, 521, 561, 539, 495, 471, 536, 523, 520, 489, 495, 507, 487, 542, 512, 534, 500, 497, 500, 545, 503, 483, 559, 497, 517, 561, 505] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[498, 537, 491, 501, 532, 431, 526, 533, 513, 512, 494, 525, 515, 477, 529, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 530, 498, 470, 528, 542, 468, 535, 491, 526, 478, 543, 493, 475, 497, 532, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 498, 522, 530, 486, 522, 539, 557, 489, 540, 495, 498, 532, 503, 503, 482, 16, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 510, 535, 541, 515, 497, 485, 535, 519, 527, 500, 504, 499, 486, 491, 524, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 546, 541, 508, 492, 528, 510, 518, 520, 553, 517, 533, 510, 498, 520, 527, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 554, 488, 514, 492, 501, 504, 508, 530, 501, 511, 489, 510, 499, 556, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 514, 491, 520, 506, 502, 493, 515, 498, 506, 492, 503, 503, 503, 511, 496, 30, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 542, 486, 530, 511, 479, 507, 523, 500, 521, 502, 529, 547, 518, 537, 472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8124, 8184, 8218, 8174, 8352, 8154, 8120, 8210] avg=8192.0000 max=8352 max_rank=4 min=8120 min_rank=6 imbalance_ratio=1.019531 max_load_reduction=57184 max_load_reduction_pct=0.872559 moved_to_echo=32836 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[487, 499, 479, 524, 502, 481, 527, 463, 481, 521, 507, 512, 527, 517, 557, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 521, 495, 587, 453, 494, 491, 487, 514, 490, 493, 532, 481, 499, 503, 518, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 518, 497, 460, 536, 476, 511, 519, 479, 526, 508, 545, 502, 527, 524, 505, 16, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 541, 489, 516, 482, 519, 505, 537, 553, 488, 568, 524, 512, 562, 506, 548, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 473, 493, 538, 539, 564, 502, 532, 497, 554, 527, 495, 480, 508, 480, 479, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 538, 524, 514, 532, 475, 483, 504, 515, 455, 532, 506, 520, 532, 504, 456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 510, 520, 540, 489, 523, 481, 532, 528, 511, 483, 542, 497, 527, 518, 515, 28, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 497, 517, 520, 512, 537, 513, 512, 490, 473, 529, 518, 481, 528, 518, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8140, 8135, 8190, 8382, 8171, 8081, 8279, 8158] avg=8192.0000 max=8382 max_rank=3 min=8081 min_rank=5 imbalance_ratio=1.023193 max_load_reduction=57154 max_load_reduction_pct=0.872101 moved_to_echo=32689 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[474, 512, 489, 514, 500, 444, 477, 493, 518, 554, 495, 522, 507, 512, 486, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 544, 532, 519, 498, 487, 527, 504, 512, 545, 488, 511, 510, 565, 524, 510, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 524, 525, 547, 548, 515, 543, 485, 495, 479, 505, 502, 496, 523, 503, 496, 16, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 510, 522, 476, 495, 491, 480, 454, 522, 561, 497, 533, 504, 487, 536, 552, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 544, 470, 538, 539, 547, 511, 493, 499, 481, 509, 498, 507, 509, 498, 489, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 542, 502, 503, 466, 482, 530, 530, 456, 509, 535, 511, 503, 534, 508, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 454, 489, 491, 529, 528, 529, 510, 510, 512, 512, 515, 547, 480, 506, 513, 533, 28, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 527, 509, 550, 505, 547, 514, 522, 505, 516, 526, 521, 528, 545, 525, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7989, 8304, 8222, 8127, 8162, 8140, 8206, 8386] avg=8192.0000 max=8386 max_rank=7 min=7989 min_rank=0 imbalance_ratio=1.023682 max_load_reduction=57150 max_load_reduction_pct=0.872040 moved_to_echo=32894 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[549, 542, 512, 495, 519, 457, 494, 534, 547, 581, 542, 510, 492, 526, 538, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 483, 461, 499, 535, 528, 508, 493, 499, 523, 539, 509, 488, 513, 515, 486, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 500, 497, 514, 500, 509, 506, 530, 489, 526, 540, 490, 493, 506, 486, 551, 16, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 488, 486, 517, 488, 534, 500, 517, 529, 474, 517, 508, 458, 516, 531, 489, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 524, 494, 506, 528, 528, 495, 508, 519, 485, 488, 508, 496, 517, 536, 491, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 509, 508, 537, 501, 496, 538, 498, 527, 533, 537, 529, 529, 549, 516, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 517, 472, 468, 515, 504, 516, 506, 506, 515, 509, 478, 493, 521, 557, 487, 29, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 513, 525, 554, 506, 517, 516, 512, 490, 516, 540, 474, 532, 501, 494, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8363, 8101, 8183, 8105, 8146, 8321, 8106, 8211] avg=8192.0000 max=8363 max_rank=0 min=8101 min_rank=1 imbalance_ratio=1.020874 max_load_reduction=57173 max_load_reduction_pct=0.872391 moved_to_echo=32784 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[505, 566, 542, 530, 533, 472, 512, 533, 497, 478, 527, 509, 480, 504, 497, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 493, 516, 510, 512, 530, 503, 480, 502, 470, 492, 544, 512, 532, 504, 533, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 505, 512, 514, 501, 502, 494, 489, 484, 476, 486, 509, 536, 498, 522, 559, 15, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 487, 504, 507, 520, 489, 550, 523, 513, 513, 494, 490, 500, 523, 536, 489, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 527, 507, 501, 527, 486, 501, 544, 497, 533, 532, 526, 510, 524, 516, 525, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 502, 520, 511, 543, 471, 541, 508, 480, 523, 500, 546, 528, 530, 490, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 488, 501, 484, 511, 492, 521, 561, 539, 495, 471, 536, 523, 520, 489, 495, 28, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 487, 542, 512, 534, 500, 497, 475, 545, 503, 483, 559, 497, 517, 561, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8197, 8145, 8158, 8245, 8178, 8192, 8224] avg=8192.0000 max=8245 max_rank=4 min=8145 min_rank=2 imbalance_ratio=1.006470 max_load_reduction=57291 max_load_reduction_pct=0.874191 moved_to_echo=32839 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[500, 558, 534, 528, 544, 435, 546, 500, 506, 556, 511, 517, 525, 519, 485, 565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 486, 530, 521, 549, 503, 484, 488, 525, 537, 524, 478, 490, 494, 504, 524, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 477, 514, 478, 482, 526, 498, 502, 536, 523, 509, 500, 505, 492, 476, 510, 15, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 488, 546, 474, 483, 519, 488, 534, 540, 529, 532, 508, 551, 488, 511, 523, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 497, 536, 523, 501, 492, 506, 479, 480, 534, 518, 474, 499, 553, 532, 478, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 518, 526, 502, 550, 460, 508, 514, 518, 553, 544, 511, 498, 546, 500, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 510, 517, 526, 525, 519, 537, 497, 491, 505, 487, 490, 505, 516, 490, 501, 27, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 534, 507, 520, 510, 533, 487, 484, 496, 495, 514, 497, 549, 484, 507, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8329, 8199, 8096, 8236, 8102, 8297, 8191, 8086] avg=8192.0000 max=8329 max_rank=0 min=8086 min_rank=7 imbalance_ratio=1.016724 max_load_reduction=57207 max_load_reduction_pct=0.872910 moved_to_echo=32676 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[571, 501, 521, 495, 508, 450, 506, 500, 550, 487, 536, 502, 528, 508, 552, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 538, 507, 511, 499, 484, 529, 502, 496, 520, 488, 515, 504, 558, 508, 520, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 504, 582, 497, 501, 521, 531, 497, 542, 529, 491, 508, 523, 512, 507, 510, 22, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 529, 555, 516, 526, 483, 507, 503, 501, 530, 469, 495, 515, 522, 489, 489, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 516, 503, 491, 504, 520, 500, 561, 531, 497, 470, 539, 529, 513, 517, 500, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 538, 496, 551, 470, 513, 497, 503, 508, 528, 525, 498, 472, 517, 511, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 466, 499, 478, 536, 548, 508, 512, 502, 508, 483, 498, 496, 502, 510, 515, 35, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 505, 471, 520, 533, 525, 487, 448, 537, 526, 489, 531, 509, 517, 526, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 8240, 8300, 8139, 8199, 8165, 8116, 8152] avg=8192.0000 max=8300 max_rank=2 min=8116 min_rank=6 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32632 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[491, 496, 493, 501, 550, 483, 511, 532, 472, 520, 515, 525, 484, 544, 510, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 498, 475, 505, 510, 486, 520, 499, 515, 497, 521, 519, 529, 506, 495, 522, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 520, 489, 495, 511, 523, 496, 559, 512, 513, 491, 498, 537, 498, 505, 490, 15, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 521, 525, 502, 499, 548, 534, 543, 486, 495, 487, 532, 500, 507, 521, 509, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 502, 524, 542, 540, 495, 502, 494, 499, 550, 509, 488, 488, 511, 497, 487, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 510, 526, 508, 524, 468, 508, 532, 471, 554, 520, 509, 517, 522, 536, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 485, 532, 519, 522, 512, 517, 499, 551, 551, 481, 540, 499, 510, 516, 495, 28, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 533, 491, 490, 501, 489, 504, 511, 499, 495, 522, 525, 510, 504, 492, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8169, 8176, 8182, 8215, 8159, 8200, 8326, 8109] avg=8192.0000 max=8326 max_rank=6 min=8109 min_rank=7 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32794 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[474, 512, 489, 514, 500, 444, 477, 493, 518, 554, 495, 522, 507, 512, 486, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 544, 532, 519, 498, 487, 527, 504, 512, 545, 488, 511, 510, 565, 524, 510, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 524, 525, 547, 548, 515, 543, 485, 495, 479, 505, 502, 496, 523, 503, 496, 16, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 510, 522, 476, 495, 491, 480, 454, 522, 561, 497, 533, 504, 487, 536, 552, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 544, 470, 538, 539, 547, 511, 493, 499, 481, 509, 498, 507, 509, 498, 489, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 542, 502, 503, 466, 482, 530, 530, 456, 509, 535, 511, 503, 534, 508, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 454, 489, 491, 529, 528, 529, 510, 510, 512, 512, 515, 547, 480, 506, 513, 533, 28, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 527, 509, 550, 505, 547, 514, 522, 505, 516, 526, 521, 528, 545, 525, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7989, 8304, 8222, 8127, 8162, 8140, 8206, 8386] avg=8192.0000 max=8386 max_rank=7 min=7989 min_rank=0 imbalance_ratio=1.023682 max_load_reduction=57150 max_load_reduction_pct=0.872040 moved_to_echo=32894 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[487, 499, 479, 524, 502, 481, 527, 463, 481, 521, 507, 512, 527, 517, 557, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 521, 495, 587, 453, 494, 491, 487, 514, 490, 493, 532, 481, 499, 503, 518, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 518, 497, 460, 536, 476, 511, 519, 479, 526, 508, 545, 502, 527, 524, 505, 16, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 541, 489, 516, 482, 519, 505, 537, 553, 488, 568, 524, 512, 562, 506, 548, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 473, 493, 538, 539, 564, 502, 532, 497, 554, 527, 495, 480, 508, 480, 479, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 538, 524, 514, 532, 475, 483, 504, 515, 455, 532, 506, 520, 532, 504, 456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 510, 520, 540, 489, 523, 481, 532, 528, 511, 483, 542, 497, 527, 518, 515, 28, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 497, 517, 520, 512, 537, 513, 512, 490, 473, 529, 518, 481, 528, 518, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8140, 8135, 8190, 8382, 8171, 8081, 8279, 8158] avg=8192.0000 max=8382 max_rank=3 min=8081 min_rank=5 imbalance_ratio=1.023193 max_load_reduction=57154 max_load_reduction_pct=0.872101 moved_to_echo=32689 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[549, 542, 512, 495, 519, 457, 494, 534, 547, 581, 542, 510, 492, 526, 538, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 483, 461, 499, 535, 528, 508, 493, 499, 523, 539, 509, 488, 513, 515, 486, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 500, 497, 514, 500, 509, 506, 530, 489, 526, 540, 490, 493, 506, 486, 551, 16, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 488, 486, 517, 488, 534, 500, 517, 529, 474, 517, 508, 458, 516, 531, 489, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 524, 494, 506, 528, 528, 495, 508, 519, 485, 488, 508, 496, 517, 536, 491, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 509, 508, 537, 501, 496, 538, 498, 527, 533, 537, 529, 529, 549, 516, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 517, 472, 468, 515, 504, 516, 506, 506, 515, 509, 478, 493, 521, 557, 487, 29, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 513, 525, 554, 506, 517, 516, 512, 490, 516, 540, 474, 532, 501, 494, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8363, 8101, 8183, 8105, 8146, 8321, 8106, 8211] avg=8192.0000 max=8363 max_rank=0 min=8101 min_rank=1 imbalance_ratio=1.020874 max_load_reduction=57173 max_load_reduction_pct=0.872391 moved_to_echo=32784 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 3968, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4352, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 3968, 4224, 4096, 4224, 4224, 4096, 256, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 3712, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[536, 513, 535, 487, 490, 496, 482, 548, 523, 535, 522, 501, 527, 475, 518, 563, 519, 551, 517, 514, 520, 488, 527, 511, 506, 559, 517, 516, 520, 505, 503, 548, 519, 463, 504, 510, 475, 506, 482, 513, 510, 536, 531, 497, 521, 510, 504, 520, 515, 530, 507, 498, 483, 531, 488, 535, 519, 538, 520, 499, 478, 529, 506, 531, 506, 462, 490, 525, 492, 539, 499, 516, 488, 503, 538, 516, 497, 497, 506, 474, 527, 530, 496, 493, 492, 513, 507, 519, 471, 528, 533, 563, 465, 496, 535, 543, 503, 524, 514, 513, 494, 498, 526, 498, 512, 506, 487, 572, 502, 492, 528, 537, 499, 497, 492, 519, 540, 516, 558, 491, 515, 509, 501, 521, 498, 553, 512, 470] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[549, 560, 479, 482, 522, 496, 535, 537, 518, 548, 517, 541, 527, 494, 506, 511, 490, 518, 528, 509, 492, 515, 481, 511, 501, 486, 518, 530, 526, 512, 474, 497, 508, 533, 517, 530, 497, 478, 508, 492, 505, 499, 530, 541, 517, 466, 536, 532, 514, 528, 515, 520, 526, 479, 535, 501, 540, 515, 541, 527, 523, 485, 490, 505, 484, 548, 499, 530, 469, 538, 540, 516, 511, 489, 504, 481, 492, 508, 509, 495, 512, 536, 503, 530, 516, 505, 488, 471, 515, 524, 546, 528, 514, 479, 473, 541, 503, 560, 514, 485, 465, 531, 513, 532, 528, 553, 521, 509, 534, 527, 496, 503, 483, 518, 502, 488, 517, 473, 515, 478, 504, 555, 520, 474, 532, 555, 517, 494] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[537, 505, 467, 478, 500, 525, 535, 471, 474, 540, 479, 484, 531, 511, 509, 501, 548, 520, 510, 488, 509, 520, 527, 509, 486, 525, 459, 495, 535, 546, 525, 556, 503, 503, 550, 480, 540, 496, 504, 502, 534, 511, 509, 532, 528, 491, 522, 522, 526, 498, 483, 511, 502, 497, 536, 501, 522, 486, 518, 476, 481, 449, 486, 525, 520, 531, 482, 505, 504, 541, 528, 504, 494, 534, 532, 523, 535, 504, 486, 500, 490, 492, 518, 568, 512, 527, 493, 481, 484, 514, 506, 507, 547, 536, 507, 502, 517, 505, 536, 515, 500, 499, 496, 499, 501, 507, 517, 479, 508, 502, 491, 513, 549, 575, 539, 521, 519, 524, 527, 511, 535, 520, 533, 499, 541, 537, 569, 516] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[502, 523, 492, 516, 516, 502, 497, 530, 536, 482, 485, 513, 491, 543, 483, 517, 517, 535, 531, 473, 494, 557, 537, 544, 486, 535, 473, 496, 524, 490, 535, 565, 492, 516, 491, 546, 503, 535, 514, 557, 515, 511, 523, 526, 507, 492, 503, 495, 513, 508, 517, 525, 515, 535, 494, 509, 521, 516, 503, 568, 538, 488, 521, 532, 494, 506, 503, 507, 508, 526, 495, 496, 517, 504, 523, 551, 518, 470, 500, 534, 509, 479, 509, 526, 492, 490, 491, 522, 520, 530, 455, 542, 542, 500, 515, 493, 509, 502, 511, 480, 546, 479, 506, 532, 534, 532, 483, 488, 477, 558, 496, 501, 555, 483, 504, 511, 505, 513, 534, 520, 510, 517, 522, 505, 475, 483, 524, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[510, 536, 515, 528, 545, 554, 517, 501, 492, 500, 526, 515, 530, 532, 485, 496, 489, 504, 503, 506, 536, 517, 526, 471, 509, 504, 506, 482, 536, 496, 496, 507, 512, 518, 516, 503, 525, 505, 502, 563, 459, 533, 504, 497, 520, 542, 519, 505, 524, 485, 491, 502, 505, 522, 523, 564, 548, 494, 487, 496, 539, 480, 530, 543, 538, 505, 534, 533, 536, 489, 523, 487, 511, 544, 539, 486, 510, 531, 527, 528, 501, 465, 539, 502, 496, 487, 490, 503, 524, 507, 498, 476, 509, 487, 482, 524, 539, 505, 550, 510, 490, 515, 532, 502, 497, 497, 498, 536, 533, 547, 550, 464, 520, 525, 511, 558, 525, 499, 469, 509, 493, 506, 483, 554, 519, 509, 485, 469] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[525, 555, 511, 485, 486, 485, 485, 501, 549, 490, 519, 497, 554, 505, 497, 510, 491, 480, 521, 540, 509, 517, 509, 527, 535, 523, 485, 531, 545, 513, 538, 490, 527, 437, 506, 498, 502, 504, 526, 491, 478, 539, 491, 487, 517, 528, 492, 490, 496, 502, 459, 538, 512, 500, 540, 503, 485, 511, 532, 538, 536, 519, 514, 503, 523, 528, 523, 504, 529, 497, 558, 507, 552, 520, 527, 513, 501, 549, 526, 507, 535, 483, 521, 513, 536, 555, 518, 534, 453, 506, 512, 524, 466, 528, 498, 517, 503, 562, 514, 518, 537, 513, 513, 519, 503, 469, 519, 495, 476, 519, 492, 518, 484, 495, 550, 498, 523, 505, 506, 450, 574, 513, 526, 525, 530, 517, 497, 501] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[529, 524, 558, 508, 548, 521, 501, 505, 509, 544, 482, 501, 519, 538, 517, 511, 497, 473, 513, 508, 543, 505, 510, 483, 490, 553, 480, 543, 513, 521, 509, 518, 518, 501, 529, 533, 488, 495, 510, 527, 544, 514, 557, 536, 507, 544, 558, 527, 527, 524, 544, 532, 492, 512, 521, 504, 532, 532, 502, 523, 497, 540, 531, 493, 516, 519, 500, 475, 497, 493, 532, 476, 473, 499, 504, 513, 476, 529, 471, 520, 509, 494, 494, 516, 517, 510, 498, 510, 459, 521, 497, 521, 498, 508, 521, 530, 491, 454, 515, 507, 490, 524, 479, 525, 515, 532, 538, 524, 537, 523, 507, 491, 523, 527, 518, 477, 497, 527, 502, 499, 524, 466, 469, 530, 486, 527, 526, 522] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[536, 513, 535, 487, 490, 496, 482, 548, 523, 535, 522, 501, 527, 475, 518, 563, 519, 551, 517, 514, 520, 488, 527, 511, 506, 559, 517, 516, 520, 505, 503, 548, 519, 463, 504, 510, 475, 506, 482, 513, 510, 536, 531, 497, 521, 510, 504, 520, 515, 530, 507, 498, 483, 531, 488, 535, 519, 538, 520, 499, 478, 529, 506, 531, 506, 462, 490, 525, 492, 539, 499, 516, 488, 503, 538, 516, 497, 497, 506, 474, 527, 530, 496, 493, 492, 513, 507, 519, 471, 528, 533, 563, 465, 496, 535, 543, 503, 524, 514, 513, 494, 498, 526, 498, 512, 506, 487, 572, 502, 492, 528, 537, 499, 497, 492, 519, 540, 516, 558, 491, 515, 509, 501, 521, 498, 553, 512, 470] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[472, 534, 519, 546, 555, 501, 512, 538, 499, 528, 516, 523, 515, 506, 464, 549, 527, 517, 463, 534, 531, 522, 513, 528, 499, 511, 537, 513, 496, 547, 504, 494, 485, 523, 537, 511, 495, 510, 492, 547, 500, 523, 530, 548, 523, 522, 517, 526, 510, 499, 513, 510, 539, 504, 493, 502, 502, 508, 540, 522, 506, 491, 506, 461, 510, 457, 520, 494, 506, 492, 565, 524, 500, 500, 515, 469, 519, 511, 509, 527, 489, 499, 521, 535, 469, 514, 469, 509, 496, 505, 504, 518, 550, 523, 551, 504, 484, 527, 482, 497, 529, 505, 533, 486, 489, 513, 517, 525, 494, 513, 524, 480, 521, 531, 531, 501, 509, 525, 513, 538, 508, 484, 489, 511, 514, 495, 544, 542] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[536, 485, 535, 487, 490, 496, 482, 548, 523, 535, 522, 501, 527, 475, 518, 563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 551, 517, 514, 520, 488, 527, 511, 506, 538, 517, 516, 520, 505, 503, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 463, 504, 510, 475, 506, 482, 500, 510, 536, 531, 497, 521, 510, 504, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 530, 507, 498, 483, 531, 488, 535, 514, 538, 520, 499, 478, 529, 506, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 462, 490, 525, 492, 539, 499, 516, 488, 503, 538, 516, 497, 497, 506, 474, 21, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 530, 496, 493, 492, 513, 507, 519, 471, 528, 533, 563, 465, 496, 535, 543, 28, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 524, 514, 513, 494, 498, 526, 498, 512, 506, 487, 572, 502, 492, 528, 537, 12, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 497, 492, 519, 540, 516, 558, 491, 515, 509, 501, 521, 498, 530, 512, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8223, 8300, 8088, 8202, 8073, 8259, 8223, 8168] avg=8192.0000 max=8300 max_rank=1 min=8073 min_rank=4 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32723 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[549, 529, 479, 482, 522, 496, 535, 537, 518, 548, 517, 541, 527, 494, 506, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 518, 528, 509, 492, 515, 481, 511, 501, 468, 518, 530, 526, 512, 474, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 533, 517, 530, 497, 478, 508, 479, 505, 499, 530, 541, 517, 466, 536, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 528, 515, 520, 526, 479, 535, 501, 534, 515, 541, 527, 523, 485, 490, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 548, 499, 530, 469, 538, 540, 516, 511, 489, 504, 481, 492, 508, 509, 495, 18, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 536, 503, 530, 516, 505, 488, 471, 515, 524, 546, 528, 514, 479, 473, 541, 31, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 560, 514, 485, 465, 531, 513, 532, 528, 553, 521, 509, 534, 527, 496, 503, 12, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 518, 502, 488, 517, 473, 515, 478, 504, 555, 520, 474, 532, 532, 517, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8291, 8070, 8176, 8238, 8135, 8232, 8292, 8102] avg=8192.0000 max=8292 max_rank=6 min=8070 min_rank=1 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32761 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[537, 477, 467, 478, 500, 525, 535, 471, 474, 540, 479, 484, 531, 511, 509, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 520, 510, 488, 509, 520, 527, 509, 486, 505, 459, 495, 535, 546, 525, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 503, 550, 480, 540, 496, 504, 489, 534, 511, 509, 532, 528, 491, 522, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 498, 483, 511, 502, 497, 536, 501, 516, 486, 518, 476, 481, 449, 486, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 531, 482, 505, 504, 541, 528, 504, 494, 534, 532, 523, 535, 504, 486, 500, 20, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 492, 518, 568, 512, 527, 493, 481, 484, 514, 506, 507, 547, 536, 507, 502, 28, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 505, 536, 515, 500, 499, 496, 499, 501, 507, 517, 479, 508, 502, 491, 513, 12, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 575, 539, 521, 519, 524, 527, 511, 535, 520, 533, 499, 541, 514, 569, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8019, 8238, 8214, 7991, 8247, 8232, 8103, 8492] avg=8192.0000 max=8492 max_rank=7 min=7991 min_rank=3 imbalance_ratio=1.036621 max_load_reduction=57044 max_load_reduction_pct=0.870422 moved_to_echo=33074 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[502, 494, 492, 516, 516, 502, 497, 530, 536, 482, 485, 513, 491, 543, 483, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 535, 531, 473, 494, 557, 537, 544, 486, 515, 473, 496, 524, 490, 535, 565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 516, 491, 546, 503, 535, 514, 542, 515, 511, 523, 526, 507, 492, 503, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 508, 517, 525, 515, 535, 494, 509, 516, 516, 503, 568, 538, 488, 521, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 506, 503, 507, 508, 526, 495, 496, 517, 504, 523, 551, 518, 470, 500, 534, 20, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 479, 509, 526, 492, 490, 491, 522, 520, 530, 455, 542, 542, 500, 515, 493, 29, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 502, 511, 480, 546, 479, 506, 532, 534, 532, 483, 488, 477, 558, 496, 501, 13, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 483, 504, 511, 505, 513, 534, 520, 510, 517, 522, 505, 475, 462, 524, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8099, 8272, 8211, 8298, 8177, 8162, 8152, 8165] avg=8192.0000 max=8298 max_rank=3 min=8099 min_rank=0 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32656 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[525, 555, 511, 485, 486, 485, 485, 501, 549, 490, 519, 497, 554, 505, 497, 510, 491, 480, 521, 540, 509, 517, 509, 527, 535, 523, 485, 531, 545, 513, 538, 490, 527, 437, 506, 498, 502, 504, 526, 491, 478, 539, 491, 487, 517, 528, 492, 490, 496, 502, 459, 538, 512, 500, 540, 503, 485, 511, 532, 538, 536, 519, 514, 503, 523, 528, 523, 504, 529, 497, 558, 507, 552, 520, 527, 513, 501, 549, 526, 507, 535, 483, 521, 513, 536, 555, 518, 534, 453, 506, 512, 524, 466, 528, 498, 517, 503, 562, 514, 518, 537, 513, 513, 519, 503, 469, 519, 495, 476, 519, 492, 518, 484, 495, 550, 498, 523, 505, 506, 450, 574, 513, 526, 525, 530, 517, 497, 501] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[537, 505, 467, 478, 500, 525, 535, 471, 474, 540, 479, 484, 531, 511, 509, 501, 548, 520, 510, 488, 509, 520, 527, 509, 486, 525, 459, 495, 535, 546, 525, 556, 503, 503, 550, 480, 540, 496, 504, 502, 534, 511, 509, 532, 528, 491, 522, 522, 526, 498, 483, 511, 502, 497, 536, 501, 522, 486, 518, 476, 481, 449, 486, 525, 520, 531, 482, 505, 504, 541, 528, 504, 494, 534, 532, 523, 535, 504, 486, 500, 490, 492, 518, 568, 512, 527, 493, 481, 484, 514, 506, 507, 547, 536, 507, 502, 517, 505, 536, 515, 500, 499, 496, 499, 501, 507, 517, 479, 508, 502, 491, 513, 549, 575, 539, 521, 519, 524, 527, 511, 535, 520, 533, 499, 541, 537, 569, 516] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[502, 523, 492, 516, 516, 502, 497, 530, 536, 482, 485, 513, 491, 543, 483, 517, 517, 535, 531, 473, 494, 557, 537, 544, 486, 535, 473, 496, 524, 490, 535, 565, 492, 516, 491, 546, 503, 535, 514, 557, 515, 511, 523, 526, 507, 492, 503, 495, 513, 508, 517, 525, 515, 535, 494, 509, 521, 516, 503, 568, 538, 488, 521, 532, 494, 506, 503, 507, 508, 526, 495, 496, 517, 504, 523, 551, 518, 470, 500, 534, 509, 479, 509, 526, 492, 490, 491, 522, 520, 530, 455, 542, 542, 500, 515, 493, 509, 502, 511, 480, 546, 479, 506, 532, 534, 532, 483, 488, 477, 558, 496, 501, 555, 483, 504, 511, 505, 513, 534, 520, 510, 517, 522, 505, 475, 483, 524, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[549, 560, 479, 482, 522, 496, 535, 537, 518, 548, 517, 541, 527, 494, 506, 511, 490, 518, 528, 509, 492, 515, 481, 511, 501, 486, 518, 530, 526, 512, 474, 497, 508, 533, 517, 530, 497, 478, 508, 492, 505, 499, 530, 541, 517, 466, 536, 532, 514, 528, 515, 520, 526, 479, 535, 501, 540, 515, 541, 527, 523, 485, 490, 505, 484, 548, 499, 530, 469, 538, 540, 516, 511, 489, 504, 481, 492, 508, 509, 495, 512, 536, 503, 530, 516, 505, 488, 471, 515, 524, 546, 528, 514, 479, 473, 541, 503, 560, 514, 485, 465, 531, 513, 532, 528, 553, 521, 509, 534, 527, 496, 503, 483, 518, 502, 488, 517, 473, 515, 478, 504, 555, 520, 474, 532, 555, 517, 494] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[510, 506, 515, 528, 545, 554, 517, 501, 492, 500, 526, 515, 530, 532, 485, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 504, 503, 506, 536, 517, 526, 471, 509, 485, 506, 482, 536, 496, 496, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 518, 516, 503, 525, 505, 502, 548, 459, 533, 504, 497, 520, 542, 519, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 485, 491, 502, 505, 522, 523, 564, 542, 494, 487, 496, 539, 480, 530, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 505, 534, 533, 536, 489, 523, 487, 511, 544, 539, 486, 510, 531, 527, 528, 19, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 465, 539, 502, 496, 487, 490, 503, 524, 507, 498, 476, 509, 487, 482, 524, 30, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 505, 550, 510, 490, 515, 532, 502, 497, 497, 498, 536, 533, 547, 550, 464, 13, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 525, 511, 558, 525, 499, 469, 509, 493, 506, 483, 554, 519, 487, 485, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8252, 8069, 8208, 8227, 8345, 8039, 8284, 8112] avg=8192.0000 max=8345 max_rank=4 min=8039 min_rank=5 imbalance_ratio=1.018677 max_load_reduction=57191 max_load_reduction_pct=0.872665 moved_to_echo=32780 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[525, 520, 511, 485, 486, 485, 485, 501, 549, 490, 519, 497, 554, 505, 497, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 480, 521, 540, 509, 517, 509, 527, 535, 501, 485, 531, 545, 513, 538, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 437, 506, 498, 502, 504, 526, 470, 478, 539, 491, 487, 517, 528, 492, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 502, 459, 538, 512, 500, 540, 503, 476, 511, 532, 538, 536, 519, 514, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 528, 523, 504, 529, 497, 558, 507, 552, 520, 527, 513, 501, 549, 526, 507, 22, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 483, 521, 513, 536, 555, 518, 534, 453, 506, 512, 524, 466, 528, 498, 517, 35, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 562, 514, 518, 537, 513, 513, 519, 503, 469, 519, 495, 476, 519, 492, 518, 16, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 495, 550, 498, 523, 505, 506, 450, 574, 513, 526, 525, 530, 487, 497, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8119, 8232, 7992, 8179, 8398, 8257, 8195, 8164] avg=8192.0000 max=8398 max_rank=4 min=7992 min_rank=2 imbalance_ratio=1.025146 max_load_reduction=57138 max_load_reduction_pct=0.871857 moved_to_echo=33014 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[529, 495, 558, 508, 548, 521, 501, 505, 509, 544, 482, 501, 519, 538, 517, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 473, 513, 508, 543, 505, 510, 483, 490, 532, 480, 543, 513, 521, 509, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 501, 529, 533, 488, 495, 510, 512, 544, 514, 557, 536, 507, 544, 558, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 524, 544, 532, 492, 512, 521, 504, 526, 532, 502, 523, 497, 540, 531, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 519, 500, 475, 497, 493, 532, 476, 473, 499, 504, 513, 476, 529, 471, 520, 21, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 494, 494, 516, 517, 510, 498, 510, 459, 521, 497, 521, 498, 508, 521, 530, 29, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 454, 515, 507, 490, 524, 479, 525, 515, 532, 538, 524, 537, 523, 507, 491, 13, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 527, 518, 477, 497, 527, 502, 499, 524, 466, 469, 530, 486, 505, 526, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8286, 8138, 8373, 8300, 8019, 8151, 8171, 8098] avg=8192.0000 max=8373 max_rank=2 min=8019 min_rank=4 imbalance_ratio=1.022095 max_load_reduction=57163 max_load_reduction_pct=0.872238 moved_to_echo=32439 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[472, 504, 519, 546, 555, 501, 512, 538, 499, 528, 516, 523, 515, 506, 464, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 517, 463, 534, 531, 522, 513, 528, 499, 492, 537, 513, 496, 547, 504, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 523, 537, 511, 495, 510, 492, 532, 500, 523, 530, 548, 523, 522, 517, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 499, 513, 510, 539, 504, 493, 502, 497, 508, 540, 522, 506, 491, 506, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 457, 520, 494, 506, 492, 565, 524, 500, 500, 515, 469, 519, 511, 509, 527, 19, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 499, 521, 535, 469, 514, 469, 509, 496, 505, 504, 518, 550, 523, 551, 504, 30, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 527, 482, 497, 529, 505, 533, 486, 489, 513, 517, 525, 494, 513, 524, 480, 13, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 531, 531, 501, 509, 525, 513, 538, 508, 484, 489, 511, 514, 474, 544, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8247, 8217, 8274, 8101, 8142, 8204, 8116, 8235] avg=8192.0000 max=8274 max_rank=2 min=8101 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32697 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[549, 529, 479, 482, 522, 496, 535, 537, 518, 548, 517, 541, 527, 494, 506, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 518, 528, 509, 492, 515, 481, 511, 501, 468, 518, 530, 526, 512, 474, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 533, 517, 530, 497, 478, 508, 479, 505, 499, 530, 541, 517, 466, 536, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 528, 515, 520, 526, 479, 535, 501, 534, 515, 541, 527, 523, 485, 490, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 548, 499, 530, 469, 538, 540, 516, 511, 489, 504, 481, 492, 508, 509, 495, 18, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 536, 503, 530, 516, 505, 488, 471, 515, 524, 546, 528, 514, 479, 473, 541, 31, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 560, 514, 485, 465, 531, 513, 532, 528, 553, 521, 509, 534, 527, 496, 503, 12, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 518, 502, 488, 517, 473, 515, 478, 504, 555, 520, 474, 532, 532, 517, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8291, 8070, 8176, 8238, 8135, 8232, 8292, 8102] avg=8192.0000 max=8292 max_rank=6 min=8070 min_rank=1 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32761 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[536, 485, 535, 487, 490, 496, 482, 548, 523, 535, 522, 501, 527, 475, 518, 563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 551, 517, 514, 520, 488, 527, 511, 506, 538, 517, 516, 520, 505, 503, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 463, 504, 510, 475, 506, 482, 500, 510, 536, 531, 497, 521, 510, 504, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 530, 507, 498, 483, 531, 488, 535, 514, 538, 520, 499, 478, 529, 506, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 462, 490, 525, 492, 539, 499, 516, 488, 503, 538, 516, 497, 497, 506, 474, 21, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 530, 496, 493, 492, 513, 507, 519, 471, 528, 533, 563, 465, 496, 535, 543, 28, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 524, 514, 513, 494, 498, 526, 498, 512, 506, 487, 572, 502, 492, 528, 537, 12, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 497, 492, 519, 540, 516, 558, 491, 515, 509, 501, 521, 498, 530, 512, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8223, 8300, 8088, 8202, 8073, 8259, 8223, 8168] avg=8192.0000 max=8300 max_rank=1 min=8073 min_rank=4 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32723 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[525, 520, 511, 485, 486, 485, 485, 501, 549, 490, 519, 497, 554, 505, 497, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 480, 521, 540, 509, 517, 509, 527, 535, 501, 485, 531, 545, 513, 538, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 437, 506, 498, 502, 504, 526, 470, 478, 539, 491, 487, 517, 528, 492, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 502, 459, 538, 512, 500, 540, 503, 476, 511, 532, 538, 536, 519, 514, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 528, 523, 504, 529, 497, 558, 507, 552, 520, 527, 513, 501, 549, 526, 507, 22, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 483, 521, 513, 536, 555, 518, 534, 453, 506, 512, 524, 466, 528, 498, 517, 35, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 562, 514, 518, 537, 513, 513, 519, 503, 469, 519, 495, 476, 519, 492, 518, 16, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 495, 550, 498, 523, 505, 506, 450, 574, 513, 526, 525, 530, 487, 497, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8119, 8232, 7992, 8179, 8398, 8257, 8195, 8164] avg=8192.0000 max=8398 max_rank=4 min=7992 min_rank=2 imbalance_ratio=1.025146 max_load_reduction=57138 max_load_reduction_pct=0.871857 moved_to_echo=33014 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[537, 477, 467, 478, 500, 525, 535, 471, 474, 540, 479, 484, 531, 511, 509, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 520, 510, 488, 509, 520, 527, 509, 486, 505, 459, 495, 535, 546, 525, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 503, 550, 480, 540, 496, 504, 489, 534, 511, 509, 532, 528, 491, 522, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 498, 483, 511, 502, 497, 536, 501, 516, 486, 518, 476, 481, 449, 486, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 531, 482, 505, 504, 541, 528, 504, 494, 534, 532, 523, 535, 504, 486, 500, 20, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 492, 518, 568, 512, 527, 493, 481, 484, 514, 506, 507, 547, 536, 507, 502, 28, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 505, 536, 515, 500, 499, 496, 499, 501, 507, 517, 479, 508, 502, 491, 513, 12, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 575, 539, 521, 519, 524, 527, 511, 535, 520, 533, 499, 541, 514, 569, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8019, 8238, 8214, 7991, 8247, 8232, 8103, 8492] avg=8192.0000 max=8492 max_rank=7 min=7991 min_rank=3 imbalance_ratio=1.036621 max_load_reduction=57044 max_load_reduction_pct=0.870422 moved_to_echo=33074 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[502, 494, 492, 516, 516, 502, 497, 530, 536, 482, 485, 513, 491, 543, 483, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 535, 531, 473, 494, 557, 537, 544, 486, 515, 473, 496, 524, 490, 535, 565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 516, 491, 546, 503, 535, 514, 542, 515, 511, 523, 526, 507, 492, 503, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 508, 517, 525, 515, 535, 494, 509, 516, 516, 503, 568, 538, 488, 521, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 506, 503, 507, 508, 526, 495, 496, 517, 504, 523, 551, 518, 470, 500, 534, 20, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 479, 509, 526, 492, 490, 491, 522, 520, 530, 455, 542, 542, 500, 515, 493, 29, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 502, 511, 480, 546, 479, 506, 532, 534, 532, 483, 488, 477, 558, 496, 501, 13, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 483, 504, 511, 505, 513, 534, 520, 510, 517, 522, 505, 475, 462, 524, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8099, 8272, 8211, 8298, 8177, 8162, 8152, 8165] avg=8192.0000 max=8298 max_rank=3 min=8099 min_rank=0 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32656 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 3968, 4096, 3968, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 256, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 3968, 4096, 3968, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 256, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4352, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.rerun_state_machine:Result validation enabled /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. @@ -1828,29 +1828,29 @@ WARNING:megatron.core.rerun_state_machine:Result validation enabled warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") - [2026-08-26 14:49:26] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 38118.1 | throughput per GPU (TFLOP/s/GPU): 4.7 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.115716E+01 | loss scale: 1.0 | grad norm: 44.807 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 15:25:29] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 38031.9 | throughput per GPU (TFLOP/s/GPU): 4.8 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.115716E+01 | loss scale: 1.0 | grad norm: 44.806 | number of skipped iterations: 0 | number of nan iterations: 0 | Number of parameters in transformer block in billions: 29.85 Number of parameters in embedding layers in billions: 0.41 Total number of parameters in billions: 30.26 Number of parameters in most loaded shard in billions: 30.2590 Theoretical memory footprints: weight and optimizer=216428.96 MB -[Rank 0] (after 1 iterations) memory (MB) | allocated: 81277.55859375 | max allocated: 96153.076171875 | reserved: 101740.0 | max reserved: 101740.0 +[Rank 0] (after 1 iterations) memory (MB) | allocated: 81277.55859375 | max allocated: 96153.076171875 | reserved: 102124.0 | max reserved: 102124.0 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[530, 505, 525, 523, 538, 470, 511, 505, 513, 536, 573, 502, 480, 530, 499, 538, 534, 499, 508, 515, 512, 499, 512, 492, 532, 524, 537, 518, 527, 542, 558, 512, 487, 509, 524, 491, 465, 513, 456, 518, 497, 495, 482, 491, 508, 513, 512, 515, 480, 521, 470, 526, 542, 526, 487, 506, 528, 559, 531, 523, 523, 477, 564, 497, 514, 510, 485, 471, 467, 539, 545, 506, 517, 502, 519, 545, 504, 520, 477, 515, 526, 468, 500, 508, 485, 511, 503, 475, 482, 474, 548, 497, 493, 539, 500, 465, 497, 525, 530, 496, 554, 524, 528, 572, 514, 500, 492, 578, 502, 522, 519, 477, 518, 505, 475, 509, 524, 530, 518, 498, 497, 513, 501, 543, 511, 539, 528, 552] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[543, 533, 488, 503, 524, 526, 513, 511, 497, 492, 516, 539, 490, 489, 528, 518, 529, 524, 529, 488, 534, 519, 499, 518, 470, 532, 505, 512, 527, 517, 498, 504, 504, 503, 465, 534, 531, 549, 532, 495, 526, 488, 506, 558, 533, 478, 538, 506, 532, 497, 537, 526, 506, 501, 512, 521, 446, 506, 520, 506, 498, 504, 465, 493, 519, 494, 490, 504, 519, 490, 551, 515, 477, 533, 537, 539, 512, 504, 530, 483, 508, 488, 460, 517, 544, 533, 513, 525, 522, 485, 511, 490, 543, 562, 531, 520, 502, 550, 527, 483, 539, 494, 505, 499, 501, 482, 553, 454, 566, 511, 484, 550, 459, 501, 488, 486, 545, 506, 557, 508, 522, 514, 518, 496, 512, 512, 535, 497] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[491, 468, 507, 527, 515, 504, 523, 528, 519, 528, 490, 510, 514, 551, 490, 492, 506, 515, 550, 548, 539, 526, 521, 545, 525, 518, 525, 495, 523, 497, 488, 534, 550, 458, 543, 550, 524, 539, 485, 482, 517, 515, 513, 486, 517, 504, 522, 452, 521, 524, 515, 520, 507, 478, 497, 532, 478, 530, 537, 470, 512, 518, 556, 499, 494, 463, 525, 508, 527, 537, 524, 499, 545, 489, 514, 522, 533, 536, 455, 515, 502, 501, 514, 513, 498, 505, 560, 515, 476, 499, 496, 501, 516, 506, 510, 507, 552, 481, 513, 543, 492, 507, 485, 470, 500, 507, 509, 543, 498, 530, 537, 498, 532, 485, 494, 546, 512, 495, 526, 501, 532, 534, 519, 504, 513, 497, 517, 496] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[487, 521, 510, 481, 488, 496, 506, 536, 536, 508, 516, 521, 530, 507, 512, 527, 478, 517, 537, 522, 534, 502, 523, 509, 510, 499, 504, 490, 525, 525, 519, 511, 513, 532, 511, 476, 517, 509, 532, 491, 525, 518, 523, 527, 501, 456, 500, 508, 527, 516, 523, 494, 533, 485, 507, 484, 517, 509, 520, 533, 530, 480, 514, 471, 497, 494, 526, 516, 534, 527, 511, 554, 503, 516, 534, 540, 479, 512, 519, 496, 482, 449, 509, 545, 532, 527, 496, 555, 480, 528, 484, 514, 477, 522, 513, 534, 496, 507, 481, 474, 538, 530, 506, 514, 471, 497, 563, 482, 516, 554, 503, 535, 522, 513, 533, 491, 535, 485, 514, 518, 530, 526, 505, 505, 512, 519, 545, 542] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[549, 470, 557, 513, 481, 507, 532, 499, 508, 516, 491, 565, 505, 470, 502, 554, 510, 476, 532, 531, 521, 496, 533, 510, 494, 498, 524, 545, 488, 519, 514, 525, 486, 531, 494, 558, 490, 540, 489, 505, 532, 533, 547, 497, 496, 523, 492, 526, 486, 518, 516, 527, 524, 524, 492, 522, 488, 507, 536, 524, 496, 517, 476, 498, 508, 527, 494, 516, 522, 501, 525, 499, 488, 550, 514, 530, 518, 535, 519, 503, 510, 484, 485, 509, 482, 550, 533, 486, 519, 492, 532, 527, 479, 481, 505, 496, 487, 517, 522, 522, 517, 527, 553, 478, 483, 525, 530, 511, 467, 515, 525, 539, 490, 520, 536, 488, 481, 525, 481, 496, 531, 501, 536, 508, 503, 537, 519, 522] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[483, 490, 480, 478, 540, 483, 490, 516, 526, 529, 510, 514, 500, 560, 466, 541, 517, 486, 517, 537, 500, 524, 548, 506, 529, 506, 516, 534, 492, 467, 541, 521, 517, 480, 532, 541, 501, 480, 467, 542, 515, 476, 487, 504, 514, 500, 511, 506, 517, 484, 527, 503, 543, 520, 497, 488, 523, 519, 509, 497, 550, 489, 514, 560, 509, 530, 532, 502, 500, 507, 511, 497, 533, 510, 537, 579, 528, 531, 530, 526, 540, 477, 515, 551, 525, 516, 526, 519, 513, 519, 498, 532, 529, 494, 522, 494, 538, 500, 499, 514, 504, 506, 536, 526, 496, 537, 504, 486, 497, 520, 499, 510, 491, 482, 498, 507, 501, 509, 475, 492, 527, 500, 530, 519, 506, 532, 542, 461] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[488, 501, 516, 473, 525, 511, 492, 523, 504, 512, 509, 467, 549, 527, 494, 500, 530, 528, 514, 502, 454, 516, 508, 517, 535, 505, 526, 518, 496, 490, 491, 543, 462, 546, 504, 524, 509, 498, 532, 495, 550, 528, 530, 502, 577, 530, 491, 520, 525, 504, 487, 513, 512, 523, 537, 475, 520, 537, 523, 529, 550, 544, 523, 502, 496, 496, 518, 491, 509, 479, 539, 527, 473, 553, 509, 522, 530, 515, 516, 499, 498, 488, 511, 518, 553, 521, 480, 547, 522, 508, 480, 474, 510, 519, 527, 484, 501, 474, 481, 490, 521, 520, 527, 494, 509, 520, 510, 521, 500, 484, 515, 520, 511, 510, 528, 546, 511, 501, 480, 473, 529, 522, 545, 527, 533, 505, 525, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[487, 521, 510, 481, 488, 496, 506, 536, 536, 508, 516, 521, 530, 507, 512, 527, 478, 517, 537, 522, 534, 502, 523, 509, 510, 499, 504, 490, 525, 525, 519, 511, 513, 532, 511, 476, 517, 509, 532, 491, 525, 518, 523, 527, 501, 456, 500, 508, 527, 516, 523, 494, 533, 485, 507, 484, 517, 509, 520, 533, 530, 480, 514, 471, 497, 494, 526, 516, 534, 527, 511, 554, 503, 516, 534, 540, 479, 512, 519, 496, 482, 449, 509, 545, 532, 527, 496, 555, 480, 528, 484, 514, 477, 522, 513, 534, 496, 507, 481, 474, 538, 530, 506, 514, 471, 497, 563, 482, 516, 554, 503, 535, 522, 513, 533, 491, 535, 485, 514, 518, 530, 526, 505, 505, 512, 519, 545, 542] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[483, 490, 480, 478, 540, 483, 490, 516, 526, 529, 510, 514, 500, 560, 466, 541, 517, 486, 517, 537, 500, 524, 548, 506, 529, 506, 516, 534, 492, 467, 541, 521, 517, 480, 532, 541, 501, 480, 467, 542, 515, 476, 487, 504, 514, 500, 511, 506, 517, 484, 527, 503, 543, 520, 497, 488, 523, 519, 509, 497, 550, 489, 514, 560, 509, 530, 532, 502, 500, 507, 511, 497, 533, 510, 537, 579, 528, 531, 530, 526, 540, 477, 515, 551, 525, 516, 526, 519, 513, 519, 498, 532, 529, 494, 522, 494, 538, 500, 499, 514, 504, 506, 536, 526, 496, 537, 504, 486, 497, 520, 499, 510, 491, 482, 498, 507, 501, 509, 475, 492, 527, 500, 530, 519, 506, 532, 542, 461] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[530, 505, 525, 523, 538, 470, 511, 505, 513, 536, 573, 502, 480, 530, 499, 538, 534, 499, 508, 515, 512, 499, 512, 492, 532, 524, 537, 518, 527, 542, 558, 512, 487, 509, 524, 491, 465, 513, 456, 518, 497, 495, 482, 491, 508, 513, 512, 515, 480, 521, 470, 526, 542, 526, 487, 506, 528, 559, 531, 523, 523, 477, 564, 497, 514, 510, 485, 471, 467, 539, 545, 506, 517, 502, 519, 545, 504, 520, 477, 515, 526, 468, 500, 508, 485, 511, 503, 475, 482, 474, 548, 497, 493, 539, 500, 465, 497, 525, 530, 496, 554, 524, 528, 572, 514, 500, 492, 578, 502, 522, 519, 477, 518, 505, 475, 509, 524, 530, 518, 498, 497, 513, 501, 543, 511, 539, 528, 552] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[491, 468, 507, 527, 515, 504, 523, 528, 519, 528, 490, 510, 514, 551, 490, 492, 506, 515, 550, 548, 539, 526, 521, 545, 525, 518, 525, 495, 523, 497, 488, 534, 550, 458, 543, 550, 524, 539, 485, 482, 517, 515, 513, 486, 517, 504, 522, 452, 521, 524, 515, 520, 507, 478, 497, 532, 478, 530, 537, 470, 512, 518, 556, 499, 494, 463, 525, 508, 527, 537, 524, 499, 545, 489, 514, 522, 533, 536, 455, 515, 502, 501, 514, 513, 498, 505, 560, 515, 476, 499, 496, 501, 516, 506, 510, 507, 552, 481, 513, 543, 492, 507, 485, 470, 500, 507, 509, 543, 498, 530, 537, 498, 532, 485, 494, 546, 512, 495, 526, 501, 532, 534, 519, 504, 513, 497, 517, 496] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[518, 546, 500, 531, 510, 480, 528, 526, 486, 493, 500, 498, 483, 537, 491, 483, 512, 496, 549, 529, 543, 514, 545, 541, 502, 491, 488, 509, 516, 534, 496, 465, 488, 509, 478, 486, 540, 496, 522, 521, 508, 520, 503, 503, 546, 536, 513, 520, 499, 521, 535, 512, 514, 528, 522, 541, 523, 493, 461, 524, 510, 491, 508, 500, 493, 534, 514, 501, 488, 442, 489, 476, 523, 498, 541, 515, 497, 515, 538, 491, 543, 498, 510, 528, 501, 518, 525, 524, 565, 505, 477, 539, 556, 515, 492, 540, 511, 503, 535, 482, 536, 543, 515, 511, 503, 514, 547, 551, 524, 480, 531, 503, 511, 507, 508, 503, 511, 510, 471, 520, 500, 511, 521, 506, 572, 510, 489, 495] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[488, 501, 516, 473, 525, 511, 492, 523, 504, 512, 509, 467, 549, 527, 494, 500, 9, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 528, 465, 502, 454, 516, 508, 517, 535, 505, 526, 518, 496, 490, 491, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 546, 504, 524, 509, 498, 532, 495, 550, 528, 530, 502, 577, 530, 491, 520, 22, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 504, 487, 513, 512, 523, 537, 475, 520, 537, 523, 529, 550, 544, 523, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 496, 518, 491, 509, 479, 539, 527, 473, 553, 509, 513, 530, 515, 516, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 488, 511, 518, 553, 521, 480, 547, 522, 508, 480, 474, 510, 519, 527, 484, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 474, 481, 490, 521, 520, 527, 494, 509, 520, 500, 521, 500, 484, 515, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 510, 528, 546, 511, 501, 480, 473, 529, 522, 545, 527, 533, 505, 520, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8111, 8124, 8324, 8304, 8163, 8167, 8077, 8266] avg=8192.0000 max=8324 max_rank=2 min=8077 min_rank=6 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32673 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[483, 490, 480, 478, 540, 483, 490, 516, 526, 529, 510, 514, 500, 560, 466, 541, 10, 6, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 486, 468, 537, 500, 524, 548, 506, 529, 506, 516, 534, 492, 467, 541, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 480, 532, 541, 501, 480, 467, 542, 515, 476, 487, 504, 514, 500, 511, 506, 22, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 484, 527, 503, 542, 520, 497, 488, 523, 519, 509, 497, 550, 489, 514, 560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 530, 532, 502, 500, 507, 511, 497, 533, 510, 537, 569, 528, 531, 530, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 477, 515, 551, 525, 516, 526, 519, 513, 519, 498, 532, 529, 494, 522, 494, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 500, 499, 514, 504, 506, 536, 526, 496, 537, 494, 486, 497, 520, 499, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 482, 498, 507, 501, 509, 475, 492, 527, 500, 530, 519, 506, 532, 537, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8128, 8192, 8099, 8239, 8352, 8297, 8162, 8067] avg=8192.0000 max=8352 max_rank=4 min=8067 min_rank=7 imbalance_ratio=1.019531 max_load_reduction=57184 max_load_reduction_pct=0.872559 moved_to_echo=32878 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[530, 505, 525, 523, 538, 470, 511, 505, 513, 536, 573, 502, 480, 530, 499, 538, 15, 10, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 499, 451, 515, 512, 499, 512, 492, 532, 524, 537, 518, 527, 542, 558, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 509, 524, 491, 465, 513, 456, 518, 497, 495, 482, 491, 508, 513, 512, 515, 25, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 521, 470, 526, 537, 526, 487, 506, 528, 559, 531, 523, 523, 477, 564, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 510, 485, 471, 467, 539, 545, 506, 517, 502, 519, 530, 504, 520, 477, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 468, 500, 508, 485, 511, 503, 475, 482, 474, 548, 497, 493, 539, 500, 465, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 525, 530, 496, 554, 524, 528, 572, 514, 500, 473, 578, 502, 522, 519, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 505, 475, 509, 524, 530, 518, 498, 497, 513, 501, 543, 511, 539, 521, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8315, 8264, 8010, 8255, 8121, 8006, 8311, 8254] avg=8192.0000 max=8315 max_rank=0 min=8006 min_rank=5 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32692 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[543, 533, 488, 503, 524, 526, 513, 511, 497, 492, 516, 539, 490, 489, 528, 518, 9, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 524, 479, 488, 534, 519, 499, 518, 470, 532, 505, 512, 527, 517, 498, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 503, 465, 534, 531, 549, 532, 495, 526, 488, 506, 558, 533, 478, 538, 506, 22, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 497, 537, 526, 506, 501, 512, 521, 446, 506, 520, 506, 498, 504, 465, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 494, 490, 504, 519, 490, 551, 515, 477, 533, 537, 530, 512, 504, 530, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 488, 460, 517, 544, 533, 513, 525, 522, 485, 511, 490, 543, 562, 531, 520, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 550, 527, 483, 539, 494, 505, 499, 501, 482, 542, 454, 566, 511, 484, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 459, 501, 488, 486, 545, 506, 557, 508, 522, 514, 518, 496, 512, 512, 530, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8231, 8155, 8272, 8070, 8188, 8280, 8189, 8151] avg=8192.0000 max=8280 max_rank=5 min=8070 min_rank=3 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32808 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[491, 468, 507, 527, 515, 504, 523, 528, 519, 528, 490, 510, 514, 551, 490, 492, 9, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 515, 498, 548, 539, 526, 521, 545, 525, 518, 525, 495, 523, 497, 488, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 458, 543, 550, 524, 539, 485, 482, 517, 515, 513, 486, 517, 504, 522, 452, 23, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 524, 515, 520, 507, 478, 497, 532, 478, 530, 537, 470, 512, 518, 556, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 463, 525, 508, 527, 537, 524, 499, 545, 489, 514, 513, 533, 536, 455, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 501, 514, 513, 498, 505, 560, 515, 476, 499, 496, 501, 516, 506, 510, 507, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 481, 513, 543, 492, 507, 485, 470, 500, 507, 499, 543, 498, 530, 537, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 485, 494, 546, 512, 495, 526, 501, 532, 534, 519, 504, 513, 497, 512, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8303, 8184, 8194, 8177, 8148, 8155, 8198] avg=8192.0000 max=8303 max_rank=1 min=8148 min_rank=5 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32678 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[518, 546, 500, 531, 510, 480, 528, 526, 486, 493, 500, 498, 483, 537, 491, 483, 9, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 496, 497, 529, 543, 514, 545, 541, 502, 491, 488, 509, 516, 534, 496, 465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 509, 478, 486, 540, 496, 522, 521, 508, 520, 503, 503, 546, 536, 513, 520, 23, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 521, 535, 512, 514, 528, 522, 541, 523, 493, 461, 524, 510, 491, 508, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 534, 514, 501, 488, 442, 489, 476, 523, 498, 541, 506, 497, 515, 538, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 498, 510, 528, 501, 518, 525, 524, 565, 505, 477, 539, 556, 515, 492, 540, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 503, 535, 482, 536, 543, 515, 511, 503, 514, 536, 551, 524, 480, 531, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 507, 508, 503, 511, 510, 471, 520, 500, 511, 521, 506, 572, 510, 485, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8130, 8178, 8216, 8182, 8046, 8365, 8278, 8141] avg=8192.0000 max=8365 max_rank=5 min=8046 min_rank=4 imbalance_ratio=1.021118 max_load_reduction=57171 max_load_reduction_pct=0.872360 moved_to_echo=32830 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[530, 505, 525, 523, 538, 470, 511, 505, 513, 536, 573, 502, 480, 530, 499, 538, 15, 10, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 499, 451, 515, 512, 499, 512, 492, 532, 524, 537, 518, 527, 542, 558, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 509, 524, 491, 465, 513, 456, 518, 497, 495, 482, 491, 508, 513, 512, 515, 25, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 521, 470, 526, 537, 526, 487, 506, 528, 559, 531, 523, 523, 477, 564, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 510, 485, 471, 467, 539, 545, 506, 517, 502, 519, 530, 504, 520, 477, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 468, 500, 508, 485, 511, 503, 475, 482, 474, 548, 497, 493, 539, 500, 465, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 525, 530, 496, 554, 524, 528, 572, 514, 500, 473, 578, 502, 522, 519, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 505, 475, 509, 524, 530, 518, 498, 497, 513, 501, 543, 511, 539, 521, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8315, 8264, 8010, 8255, 8121, 8006, 8311, 8254] avg=8192.0000 max=8315 max_rank=0 min=8006 min_rank=5 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32692 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[549, 470, 557, 513, 481, 507, 532, 499, 508, 516, 491, 565, 505, 470, 502, 554, 9, 6, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 476, 482, 531, 521, 496, 533, 510, 494, 498, 524, 545, 488, 519, 514, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 531, 494, 558, 490, 540, 489, 505, 532, 533, 547, 497, 496, 523, 492, 526, 22, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 518, 516, 527, 523, 524, 492, 522, 488, 507, 536, 524, 496, 517, 476, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 527, 494, 516, 522, 501, 525, 499, 488, 550, 514, 521, 518, 535, 519, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 484, 485, 509, 482, 550, 533, 486, 519, 492, 532, 527, 479, 481, 505, 496, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 517, 522, 522, 517, 527, 553, 478, 483, 525, 520, 511, 467, 515, 525, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 520, 536, 488, 481, 525, 481, 496, 531, 501, 536, 508, 503, 537, 514, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8240, 8166, 8265, 8150, 8240, 8098, 8208, 8169] avg=8192.0000 max=8265 max_rank=2 min=8098 min_rank=5 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32715 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[487, 521, 510, 481, 488, 496, 506, 536, 536, 508, 516, 521, 530, 507, 512, 527, 9, 7, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 517, 486, 522, 534, 502, 523, 509, 510, 499, 504, 490, 525, 525, 519, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 532, 511, 476, 517, 509, 532, 491, 525, 518, 523, 527, 501, 456, 500, 508, 23, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 516, 523, 494, 532, 485, 507, 484, 517, 509, 520, 533, 530, 480, 514, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 494, 526, 516, 534, 527, 511, 554, 503, 516, 534, 531, 479, 512, 519, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 449, 509, 545, 532, 527, 496, 555, 480, 528, 484, 514, 477, 522, 513, 534, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 507, 481, 474, 538, 530, 506, 514, 471, 497, 552, 482, 516, 554, 503, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 513, 533, 491, 535, 485, 514, 518, 530, 526, 505, 505, 512, 519, 540, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8154, 8166, 8142, 8249, 8175, 8156, 8290] avg=8192.0000 max=8290 max_rank=7 min=8142 min_rank=3 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32870 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[491, 468, 507, 527, 515, 504, 523, 528, 519, 528, 490, 510, 514, 551, 490, 492, 9, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 515, 498, 548, 539, 526, 521, 545, 525, 518, 525, 495, 523, 497, 488, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 458, 543, 550, 524, 539, 485, 482, 517, 515, 513, 486, 517, 504, 522, 452, 23, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 524, 515, 520, 507, 478, 497, 532, 478, 530, 537, 470, 512, 518, 556, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 463, 525, 508, 527, 537, 524, 499, 545, 489, 514, 513, 533, 536, 455, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 501, 514, 513, 498, 505, 560, 515, 476, 499, 496, 501, 516, 506, 510, 507, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 481, 513, 543, 492, 507, 485, 470, 500, 507, 499, 543, 498, 530, 537, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 485, 494, 546, 512, 495, 526, 501, 532, 534, 519, 504, 513, 497, 512, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8303, 8184, 8194, 8177, 8148, 8155, 8198] avg=8192.0000 max=8303 max_rank=1 min=8148 min_rank=5 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32678 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[488, 501, 516, 473, 525, 511, 492, 523, 504, 512, 509, 467, 549, 527, 494, 500, 9, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 528, 465, 502, 454, 516, 508, 517, 535, 505, 526, 518, 496, 490, 491, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 546, 504, 524, 509, 498, 532, 495, 550, 528, 530, 502, 577, 530, 491, 520, 22, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 504, 487, 513, 512, 523, 537, 475, 520, 537, 523, 529, 550, 544, 523, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 496, 518, 491, 509, 479, 539, 527, 473, 553, 509, 513, 530, 515, 516, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 488, 511, 518, 553, 521, 480, 547, 522, 508, 480, 474, 510, 519, 527, 484, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 474, 481, 490, 521, 520, 527, 494, 509, 520, 500, 521, 500, 484, 515, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 510, 528, 546, 511, 501, 480, 473, 529, 522, 545, 527, 533, 505, 520, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8111, 8124, 8324, 8304, 8163, 8167, 8077, 8266] avg=8192.0000 max=8324 max_rank=2 min=8077 min_rank=6 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32673 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[549, 470, 557, 513, 481, 507, 532, 499, 508, 516, 491, 565, 505, 470, 502, 554, 9, 6, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 476, 482, 531, 521, 496, 533, 510, 494, 498, 524, 545, 488, 519, 514, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 531, 494, 558, 490, 540, 489, 505, 532, 533, 547, 497, 496, 523, 492, 526, 22, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 518, 516, 527, 523, 524, 492, 522, 488, 507, 536, 524, 496, 517, 476, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 527, 494, 516, 522, 501, 525, 499, 488, 550, 514, 521, 518, 535, 519, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 484, 485, 509, 482, 550, 533, 486, 519, 492, 532, 527, 479, 481, 505, 496, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 517, 522, 522, 517, 527, 553, 478, 483, 525, 520, 511, 467, 515, 525, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 520, 536, 488, 481, 525, 481, 496, 531, 501, 536, 508, 503, 537, 514, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8240, 8166, 8265, 8150, 8240, 8098, 8208, 8169] avg=8192.0000 max=8265 max_rank=2 min=8098 min_rank=5 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32715 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[483, 490, 480, 478, 540, 483, 490, 516, 526, 529, 510, 514, 500, 560, 466, 541, 10, 6, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 486, 468, 537, 500, 524, 548, 506, 529, 506, 516, 534, 492, 467, 541, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 480, 532, 541, 501, 480, 467, 542, 515, 476, 487, 504, 514, 500, 511, 506, 22, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 484, 527, 503, 542, 520, 497, 488, 523, 519, 509, 497, 550, 489, 514, 560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 530, 532, 502, 500, 507, 511, 497, 533, 510, 537, 569, 528, 531, 530, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 477, 515, 551, 525, 516, 526, 519, 513, 519, 498, 532, 529, 494, 522, 494, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 500, 499, 514, 504, 506, 536, 526, 496, 537, 494, 486, 497, 520, 499, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 482, 498, 507, 501, 509, 475, 492, 527, 500, 530, 519, 506, 532, 537, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8128, 8192, 8099, 8239, 8352, 8297, 8162, 8067] avg=8192.0000 max=8352 max_rank=4 min=8067 min_rank=7 imbalance_ratio=1.019531 max_load_reduction=57184 max_load_reduction_pct=0.872559 moved_to_echo=32878 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. @@ -1867,5291 +1867,5291 @@ WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] glob warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 3968, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 3840, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 3968, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[575, 511, 528, 525, 498, 513, 486, 499, 496, 507, 499, 499, 489, 532, 495, 500, 574, 501, 545, 514, 496, 481, 512, 525, 539, 513, 531, 523, 509, 493, 517, 545, 508, 549, 498, 456, 529, 547, 507, 548, 535, 542, 525, 504, 493, 543, 509, 524, 490, 476, 496, 510, 518, 469, 500, 510, 495, 489, 493, 491, 517, 525, 519, 501, 506, 462, 560, 531, 515, 508, 533, 483, 467, 533, 507, 557, 521, 517, 511, 522, 525, 515, 521, 494, 527, 513, 506, 523, 533, 489, 491, 540, 552, 496, 478, 513, 542, 508, 499, 528, 524, 527, 533, 529, 526, 475, 496, 506, 492, 450, 483, 513, 542, 476, 512, 531, 520, 502, 508, 496, 491, 509, 481, 506, 533, 533, 544, 486] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[497, 558, 517, 486, 479, 498, 513, 522, 505, 500, 556, 479, 493, 514, 491, 466, 532, 500, 482, 505, 495, 480, 511, 524, 523, 497, 479, 510, 548, 502, 530, 512, 533, 518, 530, 509, 526, 517, 495, 536, 478, 539, 530, 481, 507, 494, 532, 524, 538, 524, 533, 522, 508, 532, 528, 496, 558, 516, 545, 527, 507, 472, 520, 508, 486, 532, 488, 472, 524, 484, 535, 535, 492, 546, 507, 527, 493, 546, 524, 507, 489, 482, 521, 500, 491, 474, 524, 525, 525, 520, 560, 529, 518, 547, 491, 538, 519, 506, 474, 511, 536, 513, 485, 509, 490, 555, 526, 530, 529, 539, 484, 561, 501, 516, 499, 499, 528, 527, 502, 491, 490, 516, 523, 468, 491, 537, 484, 478] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[526, 521, 519, 497, 533, 519, 503, 545, 549, 509, 456, 534, 495, 472, 545, 517, 529, 536, 530, 568, 484, 541, 522, 500, 515, 495, 522, 489, 524, 494, 483, 520, 524, 500, 493, 519, 498, 516, 539, 464, 497, 536, 490, 503, 531, 485, 501, 529, 521, 524, 550, 532, 511, 506, 527, 512, 517, 518, 549, 522, 502, 525, 541, 511, 530, 545, 524, 518, 509, 506, 514, 524, 518, 493, 469, 497, 491, 514, 551, 502, 463, 517, 480, 507, 483, 529, 494, 499, 510, 479, 526, 542, 529, 530, 518, 495, 528, 517, 484, 500, 488, 517, 505, 476, 521, 530, 524, 526, 510, 522, 497, 479, 502, 556, 512, 526, 509, 504, 476, 508, 447, 548, 508, 489, 515, 526, 488, 507] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[502, 493, 528, 527, 476, 487, 528, 501, 490, 517, 517, 491, 531, 473, 539, 509, 551, 523, 509, 512, 495, 513, 537, 492, 499, 487, 493, 471, 544, 487, 497, 519, 528, 516, 511, 534, 509, 507, 507, 505, 530, 475, 530, 485, 512, 525, 530, 499, 521, 536, 564, 489, 508, 520, 518, 498, 525, 489, 489, 495, 543, 506, 497, 493, 493, 561, 521, 493, 525, 500, 535, 520, 531, 478, 502, 490, 511, 514, 539, 537, 533, 534, 499, 525, 508, 518, 534, 488, 500, 493, 535, 526, 489, 518, 481, 523, 536, 509, 540, 492, 518, 508, 508, 545, 517, 501, 528, 524, 501, 497, 538, 534, 493, 483, 489, 552, 517, 539, 490, 493, 507, 512, 535, 479, 520, 540, 507, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[547, 516, 506, 578, 525, 496, 516, 529, 483, 504, 507, 491, 470, 503, 482, 509, 548, 510, 511, 497, 504, 468, 508, 515, 488, 527, 527, 534, 528, 530, 532, 494, 543, 470, 520, 570, 542, 520, 549, 486, 502, 520, 521, 497, 519, 455, 523, 516, 516, 490, 493, 450, 518, 519, 497, 530, 459, 551, 531, 492, 473, 545, 508, 525, 509, 494, 516, 508, 492, 562, 538, 513, 512, 510, 531, 521, 537, 484, 510, 520, 492, 517, 497, 541, 473, 538, 489, 527, 516, 508, 486, 490, 485, 507, 516, 522, 514, 549, 517, 504, 549, 506, 502, 490, 525, 486, 495, 523, 510, 501, 532, 510, 508, 507, 530, 534, 499, 531, 504, 524, 511, 492, 500, 491, 532, 548, 494, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[496, 537, 520, 527, 515, 539, 498, 512, 475, 550, 550, 504, 516, 517, 529, 467, 548, 500, 522, 508, 521, 511, 502, 504, 532, 534, 502, 521, 526, 541, 499, 517, 493, 474, 517, 532, 528, 457, 515, 494, 535, 507, 495, 497, 553, 503, 553, 473, 544, 479, 496, 492, 501, 501, 530, 516, 540, 535, 502, 516, 505, 527, 513, 546, 520, 501, 513, 474, 490, 513, 539, 527, 486, 491, 554, 514, 530, 524, 476, 508, 504, 522, 562, 506, 501, 534, 522, 556, 523, 484, 519, 547, 470, 527, 518, 473, 529, 503, 511, 495, 520, 534, 525, 540, 476, 498, 550, 502, 509, 515, 513, 490, 506, 498, 468, 496, 492, 535, 477, 526, 504, 490, 495, 494, 515, 485, 493, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[502, 493, 528, 527, 476, 487, 528, 501, 490, 517, 517, 491, 531, 473, 539, 509, 551, 523, 509, 512, 495, 513, 537, 492, 499, 487, 493, 471, 544, 487, 497, 519, 528, 516, 511, 534, 509, 507, 507, 505, 530, 475, 530, 485, 512, 525, 530, 499, 521, 536, 564, 489, 508, 520, 518, 498, 525, 489, 489, 495, 543, 506, 497, 493, 493, 561, 521, 493, 525, 500, 535, 520, 531, 478, 502, 490, 511, 514, 539, 537, 533, 534, 499, 525, 508, 518, 534, 488, 500, 493, 535, 526, 489, 518, 481, 523, 536, 509, 540, 492, 518, 508, 508, 545, 517, 501, 528, 524, 501, 497, 538, 534, 493, 483, 489, 552, 517, 539, 490, 493, 507, 512, 535, 479, 520, 540, 507, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[526, 521, 519, 497, 533, 519, 503, 545, 549, 509, 456, 534, 495, 472, 545, 517, 529, 536, 530, 568, 484, 541, 522, 500, 515, 495, 522, 489, 524, 494, 483, 520, 524, 500, 493, 519, 498, 516, 539, 464, 497, 536, 490, 503, 531, 485, 501, 529, 521, 524, 550, 532, 511, 506, 527, 512, 517, 518, 549, 522, 502, 525, 541, 511, 530, 545, 524, 518, 509, 506, 514, 524, 518, 493, 469, 497, 491, 514, 551, 502, 463, 517, 480, 507, 483, 529, 494, 499, 510, 479, 526, 542, 529, 530, 518, 495, 528, 517, 484, 500, 488, 517, 505, 476, 521, 530, 524, 526, 510, 522, 497, 479, 502, 556, 512, 526, 509, 504, 476, 508, 447, 548, 508, 489, 515, 526, 488, 507] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[493, 498, 522, 500, 515, 502, 526, 524, 543, 525, 515, 567, 549, 533, 516, 513, 512, 550, 521, 551, 568, 517, 506, 539, 509, 504, 465, 543, 515, 521, 453, 532, 507, 502, 474, 493, 500, 486, 487, 490, 463, 505, 506, 504, 530, 508, 524, 520, 539, 519, 507, 524, 477, 492, 511, 526, 521, 494, 510, 476, 508, 502, 500, 487, 508, 528, 509, 509, 522, 528, 503, 502, 519, 534, 492, 532, 538, 490, 540, 515, 547, 510, 509, 547, 538, 505, 506, 520, 506, 480, 498, 485, 501, 529, 470, 522, 515, 519, 470, 530, 477, 548, 500, 522, 518, 525, 485, 516, 510, 519, 492, 511, 505, 566, 510, 487, 490, 518, 456, 555, 504, 522, 507, 525, 497, 520, 494, 542] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[497, 558, 517, 486, 479, 498, 513, 522, 505, 500, 556, 479, 493, 514, 491, 466, 532, 500, 482, 505, 495, 480, 511, 524, 523, 497, 479, 510, 548, 502, 530, 512, 533, 518, 530, 509, 526, 517, 495, 536, 478, 539, 530, 481, 507, 494, 532, 524, 538, 524, 533, 522, 508, 532, 528, 496, 558, 516, 545, 527, 507, 472, 520, 508, 486, 532, 488, 472, 524, 484, 535, 535, 492, 546, 507, 527, 493, 546, 524, 507, 489, 482, 521, 500, 491, 474, 524, 525, 525, 520, 560, 529, 518, 547, 491, 538, 519, 506, 474, 511, 536, 513, 485, 509, 490, 555, 526, 530, 529, 539, 484, 561, 501, 516, 499, 499, 528, 527, 502, 491, 490, 516, 523, 468, 491, 537, 484, 478] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[523, 540, 516, 558, 550, 490, 500, 504, 493, 504, 524, 498, 519, 489, 538, 517, 501, 514, 523, 490, 477, 495, 500, 540, 512, 500, 482, 486, 483, 507, 529, 515, 504, 514, 491, 488, 542, 499, 516, 523, 515, 478, 515, 498, 509, 522, 512, 502, 543, 512, 499, 530, 566, 533, 516, 553, 537, 517, 476, 513, 498, 469, 501, 512, 557, 549, 468, 500, 524, 530, 484, 474, 524, 547, 508, 486, 525, 515, 491, 517, 488, 518, 516, 512, 521, 556, 524, 515, 526, 504, 527, 538, 494, 519, 471, 538, 495, 523, 490, 529, 516, 491, 523, 455, 505, 567, 537, 508, 501, 488, 505, 523, 514, 517, 523, 530, 468, 554, 472, 500, 536, 503, 503, 502, 531, 526, 485, 530] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[526, 521, 519, 491, 533, 519, 503, 545, 549, 509, 456, 534, 495, 472, 545, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 536, 530, 568, 484, 541, 522, 500, 515, 495, 522, 489, 524, 494, 483, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 500, 493, 519, 498, 516, 539, 464, 497, 536, 490, 503, 531, 485, 501, 529, 6, 4, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 524, 550, 532, 511, 506, 527, 512, 517, 518, 549, 522, 502, 525, 541, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 545, 524, 518, 509, 506, 486, 524, 518, 493, 469, 497, 491, 514, 551, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 463, 517, 480, 507, 483, 529, 494, 499, 510, 479, 526, 538, 529, 530, 518, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 517, 484, 500, 488, 517, 505, 476, 521, 530, 524, 526, 510, 522, 497, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 556, 512, 526, 509, 504, 476, 508, 447, 548, 508, 489, 515, 526, 488, 507, 28, 19, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8234, 8233, 8140, 8361, 8177, 8097, 8122, 8172] avg=8192.0000 max=8361 max_rank=3 min=8097 min_rank=5 imbalance_ratio=1.020630 max_load_reduction=57175 max_load_reduction_pct=0.872421 moved_to_echo=32568 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[496, 537, 520, 527, 515, 539, 498, 512, 475, 550, 550, 504, 516, 517, 529, 467, 548, 500, 522, 508, 521, 511, 502, 504, 532, 534, 502, 521, 526, 541, 499, 517, 493, 474, 517, 532, 528, 457, 515, 494, 535, 507, 495, 497, 553, 503, 553, 473, 544, 479, 496, 492, 501, 501, 530, 516, 540, 535, 502, 516, 505, 527, 513, 546, 520, 501, 513, 474, 490, 513, 539, 527, 486, 491, 554, 514, 530, 524, 476, 508, 504, 522, 562, 506, 501, 534, 522, 556, 523, 484, 519, 547, 470, 527, 518, 473, 529, 503, 511, 495, 520, 534, 525, 540, 476, 498, 550, 502, 509, 515, 513, 490, 506, 498, 468, 496, 492, 535, 477, 526, 504, 490, 495, 494, 515, 485, 493, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[547, 516, 506, 571, 525, 496, 516, 529, 483, 504, 507, 491, 470, 503, 482, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 510, 511, 497, 504, 468, 508, 515, 488, 527, 527, 534, 528, 530, 532, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 470, 520, 570, 542, 520, 549, 486, 502, 520, 521, 497, 519, 455, 523, 516, 7, 3, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 490, 493, 450, 518, 519, 497, 530, 459, 551, 531, 492, 473, 545, 508, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 494, 516, 508, 492, 562, 508, 513, 512, 510, 531, 521, 537, 484, 510, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 517, 497, 541, 473, 538, 489, 527, 516, 508, 486, 487, 485, 507, 516, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 549, 517, 504, 549, 506, 502, 490, 525, 486, 495, 523, 510, 501, 532, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 507, 530, 534, 499, 531, 504, 524, 511, 492, 500, 491, 532, 548, 494, 524, 30, 20, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8155, 8201, 8268, 8090, 8227, 8101, 8211, 8283] avg=8192.0000 max=8283 max_rank=7 min=8090 min_rank=3 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32822 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[497, 558, 517, 476, 479, 498, 513, 522, 505, 500, 556, 479, 493, 514, 491, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 500, 482, 505, 495, 480, 511, 524, 523, 497, 479, 510, 548, 502, 530, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 518, 530, 509, 526, 517, 495, 536, 478, 539, 530, 481, 507, 494, 532, 524, 10, 8, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 524, 533, 522, 508, 532, 528, 496, 558, 516, 545, 527, 507, 472, 520, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 532, 488, 472, 524, 484, 502, 535, 492, 546, 507, 527, 493, 546, 524, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 482, 521, 500, 491, 474, 524, 525, 525, 520, 560, 521, 518, 547, 491, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 506, 474, 511, 536, 513, 485, 509, 490, 555, 526, 530, 529, 539, 484, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 516, 499, 499, 528, 527, 502, 491, 490, 516, 523, 468, 491, 537, 484, 478, 33, 25, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8064, 8105, 8280, 8318, 8165, 8226, 8263, 8115] avg=8192.0000 max=8318 max_rank=3 min=8064 min_rank=0 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32769 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[575, 511, 528, 518, 498, 513, 486, 499, 496, 507, 499, 499, 489, 532, 495, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 501, 545, 514, 496, 481, 512, 525, 539, 513, 531, 523, 509, 493, 517, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 549, 498, 456, 529, 547, 507, 548, 535, 542, 525, 504, 493, 543, 509, 524, 7, 3, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 476, 496, 510, 518, 469, 500, 510, 495, 489, 493, 491, 517, 525, 519, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 462, 560, 531, 515, 508, 504, 483, 467, 533, 507, 557, 521, 517, 511, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 515, 521, 494, 527, 513, 506, 523, 533, 489, 491, 537, 552, 496, 478, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 508, 499, 528, 524, 527, 533, 529, 526, 475, 496, 506, 492, 450, 483, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 476, 512, 531, 520, 502, 508, 496, 491, 509, 481, 506, 533, 533, 544, 486, 29, 21, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8145, 8297, 8332, 7993, 8204, 8213, 8129, 8223] avg=8192.0000 max=8332 max_rank=2 min=7993 min_rank=3 imbalance_ratio=1.017090 max_load_reduction=57204 max_load_reduction_pct=0.872864 moved_to_echo=32769 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[502, 493, 528, 520, 476, 487, 528, 501, 490, 517, 517, 491, 531, 473, 539, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 523, 509, 512, 495, 513, 537, 492, 499, 487, 493, 471, 544, 487, 497, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 516, 511, 534, 509, 507, 507, 505, 530, 475, 530, 485, 512, 525, 530, 499, 7, 3, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 536, 564, 489, 508, 520, 518, 498, 525, 489, 489, 495, 543, 506, 497, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 561, 521, 493, 525, 500, 505, 520, 531, 478, 502, 490, 511, 514, 539, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 534, 499, 525, 508, 518, 534, 488, 500, 493, 535, 523, 489, 518, 481, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 509, 540, 492, 518, 508, 508, 545, 517, 501, 528, 524, 501, 497, 538, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 483, 489, 552, 517, 539, 490, 493, 507, 512, 535, 479, 520, 540, 507, 498, 30, 20, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8102, 8109, 8218, 8184, 8220, 8201, 8294, 8208] avg=8192.0000 max=8294 max_rank=6 min=8102 min_rank=0 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32923 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[496, 537, 520, 520, 515, 539, 498, 512, 475, 550, 550, 504, 516, 517, 529, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 500, 522, 508, 521, 511, 502, 504, 532, 534, 502, 521, 526, 541, 499, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 474, 517, 532, 528, 457, 515, 494, 535, 507, 495, 497, 553, 503, 553, 473, 7, 4, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 479, 496, 492, 501, 501, 530, 516, 540, 535, 502, 516, 505, 527, 513, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 501, 513, 474, 490, 513, 509, 527, 486, 491, 554, 514, 530, 524, 476, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 522, 562, 506, 501, 534, 522, 556, 523, 484, 519, 543, 470, 527, 518, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 503, 511, 495, 520, 534, 525, 540, 476, 498, 550, 502, 509, 515, 513, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 498, 468, 496, 492, 535, 477, 526, 504, 490, 495, 494, 515, 485, 493, 515, 30, 20, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 8268, 8142, 8236, 8130, 8264, 8208, 8043] avg=8192.0000 max=8268 max_rank=1 min=8043 min_rank=7 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32645 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[523, 540, 516, 551, 550, 490, 500, 504, 493, 504, 524, 498, 519, 489, 538, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 514, 523, 490, 477, 495, 500, 540, 512, 500, 482, 486, 483, 507, 529, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 514, 491, 488, 542, 499, 516, 523, 515, 478, 515, 498, 509, 522, 512, 502, 7, 3, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 512, 499, 530, 566, 533, 516, 553, 537, 517, 476, 513, 498, 469, 501, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 557, 549, 468, 500, 524, 530, 457, 474, 524, 547, 508, 486, 525, 515, 491, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 518, 516, 512, 521, 556, 524, 515, 526, 504, 527, 535, 494, 519, 471, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 523, 490, 529, 516, 491, 523, 455, 505, 567, 537, 508, 501, 488, 505, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 517, 523, 530, 468, 554, 472, 500, 536, 503, 503, 502, 531, 526, 485, 530, 27, 18, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8256, 8036, 8143, 8268, 8172, 8264, 8154, 8243] avg=8192.0000 max=8268 max_rank=3 min=8036 min_rank=1 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32833 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[493, 498, 522, 494, 515, 502, 526, 524, 543, 525, 515, 567, 549, 533, 516, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 550, 521, 551, 568, 517, 506, 539, 509, 504, 465, 543, 515, 521, 453, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 502, 474, 493, 500, 486, 487, 490, 463, 505, 506, 504, 530, 508, 524, 520, 6, 3, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 519, 507, 524, 477, 492, 511, 526, 521, 494, 510, 476, 508, 502, 500, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 528, 509, 509, 522, 528, 475, 502, 519, 534, 492, 532, 538, 490, 540, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 510, 509, 547, 538, 505, 506, 520, 506, 480, 498, 482, 501, 529, 470, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 519, 470, 530, 477, 548, 500, 522, 518, 525, 485, 516, 510, 519, 492, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 566, 510, 487, 490, 518, 456, 555, 504, 522, 507, 525, 497, 520, 494, 542, 28, 19, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8335, 8287, 8013, 8086, 8241, 8170, 8155, 8249] avg=8192.0000 max=8335 max_rank=0 min=8013 min_rank=2 imbalance_ratio=1.017456 max_load_reduction=57201 max_load_reduction_pct=0.872818 moved_to_echo=32815 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[497, 558, 517, 476, 479, 498, 513, 522, 505, 500, 556, 479, 493, 514, 491, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 500, 482, 505, 495, 480, 511, 524, 523, 497, 479, 510, 548, 502, 530, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 518, 530, 509, 526, 517, 495, 536, 478, 539, 530, 481, 507, 494, 532, 524, 10, 8, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 524, 533, 522, 508, 532, 528, 496, 558, 516, 545, 527, 507, 472, 520, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 532, 488, 472, 524, 484, 502, 535, 492, 546, 507, 527, 493, 546, 524, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 482, 521, 500, 491, 474, 524, 525, 525, 520, 560, 521, 518, 547, 491, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 506, 474, 511, 536, 513, 485, 509, 490, 555, 526, 530, 529, 539, 484, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 516, 499, 499, 528, 527, 502, 491, 490, 516, 523, 468, 491, 537, 484, 478, 33, 25, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8064, 8105, 8280, 8318, 8165, 8226, 8263, 8115] avg=8192.0000 max=8318 max_rank=3 min=8064 min_rank=0 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32769 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[526, 521, 519, 491, 533, 519, 503, 545, 549, 509, 456, 534, 495, 472, 545, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 536, 530, 568, 484, 541, 522, 500, 515, 495, 522, 489, 524, 494, 483, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 500, 493, 519, 498, 516, 539, 464, 497, 536, 490, 503, 531, 485, 501, 529, 6, 4, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 524, 550, 532, 511, 506, 527, 512, 517, 518, 549, 522, 502, 525, 541, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 545, 524, 518, 509, 506, 486, 524, 518, 493, 469, 497, 491, 514, 551, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 463, 517, 480, 507, 483, 529, 494, 499, 510, 479, 526, 538, 529, 530, 518, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 517, 484, 500, 488, 517, 505, 476, 521, 530, 524, 526, 510, 522, 497, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 556, 512, 526, 509, 504, 476, 508, 447, 548, 508, 489, 515, 526, 488, 507, 28, 19, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8234, 8233, 8140, 8361, 8177, 8097, 8122, 8172] avg=8192.0000 max=8361 max_rank=3 min=8097 min_rank=5 imbalance_ratio=1.020630 max_load_reduction=57175 max_load_reduction_pct=0.872421 moved_to_echo=32568 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[523, 540, 516, 551, 550, 490, 500, 504, 493, 504, 524, 498, 519, 489, 538, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 514, 523, 490, 477, 495, 500, 540, 512, 500, 482, 486, 483, 507, 529, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 514, 491, 488, 542, 499, 516, 523, 515, 478, 515, 498, 509, 522, 512, 502, 7, 3, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 512, 499, 530, 566, 533, 516, 553, 537, 517, 476, 513, 498, 469, 501, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 557, 549, 468, 500, 524, 530, 457, 474, 524, 547, 508, 486, 525, 515, 491, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 518, 516, 512, 521, 556, 524, 515, 526, 504, 527, 535, 494, 519, 471, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 523, 490, 529, 516, 491, 523, 455, 505, 567, 537, 508, 501, 488, 505, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 517, 523, 530, 468, 554, 472, 500, 536, 503, 503, 502, 531, 526, 485, 530, 27, 18, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8256, 8036, 8143, 8268, 8172, 8264, 8154, 8243] avg=8192.0000 max=8268 max_rank=3 min=8036 min_rank=1 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32833 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[496, 537, 520, 520, 515, 539, 498, 512, 475, 550, 550, 504, 516, 517, 529, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 500, 522, 508, 521, 511, 502, 504, 532, 534, 502, 521, 526, 541, 499, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 474, 517, 532, 528, 457, 515, 494, 535, 507, 495, 497, 553, 503, 553, 473, 7, 4, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 479, 496, 492, 501, 501, 530, 516, 540, 535, 502, 516, 505, 527, 513, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 501, 513, 474, 490, 513, 509, 527, 486, 491, 554, 514, 530, 524, 476, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 522, 562, 506, 501, 534, 522, 556, 523, 484, 519, 543, 470, 527, 518, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 503, 511, 495, 520, 534, 525, 540, 476, 498, 550, 502, 509, 515, 513, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 498, 468, 496, 492, 535, 477, 526, 504, 490, 495, 494, 515, 485, 493, 515, 30, 20, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 8268, 8142, 8236, 8130, 8264, 8208, 8043] avg=8192.0000 max=8268 max_rank=1 min=8043 min_rank=7 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32645 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 3968, 4096, 4096, 4096, 4096, 3968, 4224, 4224, 4096, 4096, 256, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4224, 3968, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 3968, 4224, 4224, 4096, 4224, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 3968, 4096, 4096, 4096, 4096, 3968, 4224, 4224, 4096, 4096, 256, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 3968, 4224, 4224, 4096, 4224, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[570, 524, 550, 520, 529, 495, 493, 482, 520, 507, 498, 516, 494, 463, 549, 487, 533, 527, 498, 509, 508, 543, 498, 494, 473, 498, 469, 507, 502, 488, 495, 485, 501, 536, 477, 476, 510, 494, 481, 532, 507, 499, 538, 504, 495, 513, 513, 505, 528, 524, 522, 493, 544, 503, 535, 512, 491, 493, 516, 493, 503, 542, 524, 540, 516, 539, 520, 530, 523, 505, 473, 518, 528, 500, 526, 510, 520, 534, 469, 493, 516, 510, 495, 514, 483, 511, 562, 513, 528, 530, 486, 520, 536, 532, 517, 502, 498, 519, 533, 527, 491, 471, 540, 485, 523, 534, 518, 492, 515, 527, 519, 522, 512, 519, 503, 566, 518, 522, 521, 483, 542, 507, 517, 532, 546, 494, 521, 492] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[492, 490, 499, 535, 485, 491, 485, 574, 500, 520, 537, 488, 538, 525, 542, 533, 499, 508, 536, 508, 534, 560, 518, 488, 510, 501, 509, 531, 462, 483, 541, 536, 510, 531, 495, 489, 516, 514, 501, 520, 514, 510, 508, 523, 548, 527, 520, 497, 519, 543, 484, 484, 504, 496, 513, 537, 507, 526, 518, 540, 516, 508, 539, 497, 524, 513, 550, 527, 498, 525, 521, 493, 501, 515, 512, 500, 517, 499, 518, 509, 512, 516, 495, 525, 541, 557, 507, 505, 507, 506, 506, 525, 484, 499, 522, 509, 513, 533, 522, 524, 472, 474, 489, 487, 457, 528, 520, 474, 542, 503, 515, 502, 548, 496, 474, 509, 518, 537, 503, 495, 495, 490, 484, 497, 504, 521, 545, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[488, 522, 518, 478, 508, 520, 515, 510, 524, 499, 595, 506, 510, 523, 507, 513, 542, 536, 552, 526, 545, 521, 502, 524, 519, 491, 501, 481, 494, 504, 484, 506, 507, 530, 533, 545, 526, 553, 509, 528, 516, 545, 493, 489, 496, 494, 531, 540, 498, 499, 493, 533, 494, 512, 511, 484, 497, 518, 524, 495, 513, 531, 497, 490, 486, 519, 514, 523, 441, 473, 540, 537, 533, 505, 521, 551, 490, 544, 521, 501, 469, 496, 506, 510, 524, 525, 508, 529, 488, 534, 498, 494, 551, 505, 525, 525, 551, 562, 492, 525, 495, 478, 492, 502, 493, 515, 517, 547, 489, 520, 482, 501, 545, 509, 487, 492, 527, 517, 538, 505, 539, 499, 497, 466, 482, 482, 510, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[527, 504, 486, 515, 517, 513, 505, 493, 544, 494, 502, 488, 490, 476, 503, 551, 491, 515, 507, 505, 511, 507, 520, 539, 541, 543, 553, 524, 510, 538, 507, 534, 518, 499, 496, 503, 519, 471, 516, 496, 498, 516, 518, 486, 493, 513, 518, 500, 457, 505, 526, 493, 552, 508, 518, 494, 514, 470, 548, 522, 494, 476, 531, 546, 519, 536, 494, 490, 470, 483, 514, 529, 518, 556, 479, 529, 474, 428, 571, 496, 505, 528, 472, 485, 508, 512, 527, 527, 518, 526, 519, 473, 530, 542, 503, 549, 499, 530, 512, 518, 475, 517, 512, 499, 549, 493, 514, 510, 542, 517, 528, 543, 556, 538, 505, 523, 523, 512, 523, 524, 555, 460, 474, 547, 525, 498, 520, 518] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[527, 472, 513, 513, 558, 485, 540, 487, 526, 475, 480, 505, 515, 531, 495, 474, 510, 509, 525, 491, 503, 540, 498, 503, 502, 479, 536, 530, 509, 506, 512, 515, 478, 548, 500, 506, 487, 497, 477, 498, 533, 488, 544, 550, 535, 503, 534, 525, 503, 545, 531, 523, 515, 535, 485, 497, 529, 505, 529, 522, 555, 525, 542, 557, 511, 547, 566, 489, 552, 535, 485, 486, 513, 491, 533, 527, 516, 512, 482, 508, 526, 532, 518, 503, 470, 522, 515, 523, 462, 488, 522, 536, 485, 489, 487, 531, 487, 481, 519, 532, 526, 485, 516, 485, 512, 533, 513, 513, 505, 513, 530, 478, 522, 517, 517, 539, 500, 511, 506, 477, 498, 540, 484, 519, 514, 528, 532, 477] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[514, 498, 550, 544, 524, 546, 524, 521, 520, 517, 501, 531, 472, 529, 514, 491, 521, 542, 469, 501, 510, 522, 460, 532, 536, 517, 491, 520, 521, 479, 513, 517, 486, 502, 534, 501, 536, 459, 521, 494, 521, 556, 501, 500, 469, 528, 511, 519, 494, 511, 474, 484, 530, 564, 459, 526, 523, 481, 524, 517, 548, 482, 535, 504, 557, 501, 495, 527, 521, 460, 495, 527, 496, 519, 531, 526, 494, 558, 493, 513, 549, 510, 524, 518, 482, 501, 564, 515, 538, 541, 509, 486, 477, 520, 506, 462, 479, 540, 513, 546, 537, 498, 478, 493, 487, 537, 533, 524, 514, 574, 495, 546, 513, 538, 526, 504, 494, 492, 522, 539, 491, 488, 489, 459, 523, 497, 483, 528] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[539, 558, 523, 540, 524, 514, 476, 467, 479, 509, 524, 525, 470, 530, 520, 533, 508, 569, 495, 510, 514, 552, 473, 516, 511, 489, 535, 487, 491, 484, 509, 499, 518, 500, 535, 513, 526, 510, 513, 490, 489, 503, 539, 518, 523, 536, 503, 515, 520, 498, 517, 518, 529, 469, 536, 503, 504, 495, 519, 478, 495, 542, 538, 550, 500, 548, 495, 494, 486, 491, 522, 539, 544, 526, 494, 499, 494, 487, 528, 558, 490, 504, 520, 482, 488, 488, 489, 553, 520, 518, 532, 465, 558, 479, 512, 522, 482, 492, 519, 507, 509, 518, 490, 526, 520, 489, 498, 565, 545, 502, 495, 512, 518, 524, 507, 493, 511, 498, 501, 537, 525, 498, 526, 506, 517, 530, 525, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[527, 504, 486, 515, 517, 513, 505, 493, 544, 494, 502, 488, 490, 476, 503, 551, 491, 515, 507, 505, 511, 507, 520, 539, 541, 543, 553, 524, 510, 538, 507, 534, 518, 499, 496, 503, 519, 471, 516, 496, 498, 516, 518, 486, 493, 513, 518, 500, 457, 505, 526, 493, 552, 508, 518, 494, 514, 470, 548, 522, 494, 476, 531, 546, 519, 536, 494, 490, 470, 483, 514, 529, 518, 556, 479, 529, 474, 428, 571, 496, 505, 528, 472, 485, 508, 512, 527, 527, 518, 526, 519, 473, 530, 542, 503, 549, 499, 530, 512, 518, 475, 517, 512, 499, 549, 493, 514, 510, 542, 517, 528, 543, 556, 538, 505, 523, 523, 512, 523, 524, 555, 460, 474, 547, 525, 498, 520, 518] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[474, 491, 478, 528, 500, 529, 524, 484, 510, 506, 503, 495, 505, 508, 507, 523, 503, 507, 549, 515, 516, 510, 510, 535, 513, 515, 525, 500, 513, 482, 513, 513, 512, 516, 468, 504, 564, 494, 528, 491, 499, 496, 492, 495, 491, 513, 494, 510, 510, 520, 528, 524, 532, 514, 536, 543, 533, 491, 515, 549, 517, 492, 554, 512, 490, 501, 513, 493, 544, 504, 506, 512, 468, 510, 498, 514, 509, 508, 505, 502, 536, 535, 555, 537, 514, 494, 456, 544, 517, 542, 566, 535, 511, 527, 508, 535, 480, 491, 520, 534, 529, 491, 491, 524, 524, 514, 525, 479, 523, 500, 518, 479, 521, 514, 519, 515, 502, 528, 511, 498, 521, 498, 508, 526, 528, 494, 505, 516] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[570, 524, 550, 520, 529, 495, 493, 482, 520, 507, 498, 516, 494, 463, 549, 487, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 527, 498, 509, 508, 543, 498, 494, 473, 498, 469, 507, 502, 488, 495, 485, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 536, 477, 476, 510, 494, 481, 532, 507, 499, 538, 504, 495, 513, 513, 505, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 524, 522, 493, 544, 503, 535, 512, 491, 493, 516, 493, 503, 542, 484, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 539, 520, 530, 523, 505, 473, 518, 528, 500, 526, 510, 520, 534, 469, 493, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 510, 495, 514, 483, 511, 562, 492, 528, 530, 486, 520, 536, 532, 517, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 519, 533, 527, 491, 471, 540, 485, 523, 534, 518, 492, 515, 527, 519, 522, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 519, 503, 566, 518, 522, 521, 483, 542, 507, 517, 532, 546, 494, 521, 492, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8030, 8105, 8223, 8213, 8234, 8230, 8296] avg=8192.0000 max=8296 max_rank=7 min=8030 min_rank=1 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32973 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[488, 522, 518, 478, 508, 520, 515, 510, 524, 499, 595, 506, 510, 523, 507, 513, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 536, 552, 526, 545, 521, 502, 524, 519, 491, 501, 481, 494, 504, 484, 506, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 530, 533, 545, 526, 553, 509, 528, 516, 545, 493, 489, 496, 494, 531, 540, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 499, 493, 533, 494, 512, 511, 484, 497, 518, 524, 495, 513, 531, 459, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 519, 514, 523, 441, 473, 540, 537, 533, 505, 521, 551, 490, 544, 521, 501, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 496, 506, 510, 524, 525, 508, 506, 488, 534, 498, 494, 551, 505, 525, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 562, 492, 525, 495, 478, 492, 502, 493, 515, 517, 547, 489, 520, 482, 501, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 509, 487, 492, 527, 517, 538, 505, 539, 499, 497, 466, 482, 482, 510, 506, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 8232, 8358, 8051, 8208, 8164, 8176, 8102] avg=8192.0000 max=8358 max_rank=2 min=8051 min_rank=3 imbalance_ratio=1.020264 max_load_reduction=57178 max_load_reduction_pct=0.872467 moved_to_echo=32650 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[514, 498, 550, 544, 524, 546, 524, 521, 520, 517, 501, 531, 472, 529, 514, 491, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 542, 469, 501, 510, 522, 460, 532, 536, 517, 491, 520, 521, 479, 513, 517, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 502, 534, 501, 536, 459, 521, 494, 521, 556, 501, 500, 469, 528, 511, 519, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 511, 474, 484, 530, 564, 459, 526, 523, 481, 524, 517, 548, 482, 495, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 557, 501, 495, 527, 521, 460, 495, 527, 496, 519, 531, 526, 494, 558, 493, 513, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 510, 524, 518, 482, 501, 564, 494, 538, 541, 509, 486, 477, 520, 506, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 540, 513, 546, 537, 498, 478, 493, 487, 537, 533, 524, 514, 574, 495, 546, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 538, 526, 504, 494, 492, 522, 539, 491, 488, 489, 459, 523, 497, 483, 528, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8304, 8154, 8162, 8116, 8222, 8181, 8310, 8087] avg=8192.0000 max=8310 max_rank=6 min=8087 min_rank=7 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32800 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[492, 490, 499, 535, 485, 491, 485, 574, 500, 520, 537, 488, 538, 525, 542, 533, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 508, 536, 508, 534, 560, 518, 488, 510, 501, 509, 531, 462, 483, 541, 536, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 531, 495, 489, 516, 514, 501, 520, 514, 510, 508, 523, 548, 527, 520, 497, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 543, 484, 484, 504, 496, 513, 537, 507, 526, 518, 540, 516, 508, 497, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 513, 550, 527, 498, 525, 521, 493, 501, 515, 512, 500, 517, 499, 518, 509, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 516, 495, 525, 541, 557, 507, 485, 507, 506, 506, 525, 484, 499, 522, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 533, 522, 524, 472, 474, 489, 487, 457, 528, 520, 474, 542, 503, 515, 502, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 496, 474, 509, 518, 537, 503, 495, 495, 490, 484, 497, 504, 521, 545, 515, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8242, 8227, 8248, 8189, 8230, 8196, 8072, 8132] avg=8192.0000 max=8248 max_rank=2 min=8072 min_rank=6 imbalance_ratio=1.006836 max_load_reduction=57288 max_load_reduction_pct=0.874146 moved_to_echo=32630 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[539, 558, 523, 540, 524, 514, 476, 467, 479, 509, 524, 525, 470, 530, 520, 533, 508, 569, 495, 510, 514, 552, 473, 516, 511, 489, 535, 487, 491, 484, 509, 499, 518, 500, 535, 513, 526, 510, 513, 490, 489, 503, 539, 518, 523, 536, 503, 515, 520, 498, 517, 518, 529, 469, 536, 503, 504, 495, 519, 478, 495, 542, 538, 550, 500, 548, 495, 494, 486, 491, 522, 539, 544, 526, 494, 499, 494, 487, 528, 558, 490, 504, 520, 482, 488, 488, 489, 553, 520, 518, 532, 465, 558, 479, 512, 522, 482, 492, 519, 507, 509, 518, 490, 526, 520, 489, 498, 565, 545, 502, 495, 512, 518, 524, 507, 493, 511, 498, 501, 537, 525, 498, 526, 506, 517, 530, 525, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[570, 524, 550, 520, 529, 495, 493, 482, 520, 507, 498, 516, 494, 463, 549, 487, 533, 527, 498, 509, 508, 543, 498, 494, 473, 498, 469, 507, 502, 488, 495, 485, 501, 536, 477, 476, 510, 494, 481, 532, 507, 499, 538, 504, 495, 513, 513, 505, 528, 524, 522, 493, 544, 503, 535, 512, 491, 493, 516, 493, 503, 542, 524, 540, 516, 539, 520, 530, 523, 505, 473, 518, 528, 500, 526, 510, 520, 534, 469, 493, 516, 510, 495, 514, 483, 511, 562, 513, 528, 530, 486, 520, 536, 532, 517, 502, 498, 519, 533, 527, 491, 471, 540, 485, 523, 534, 518, 492, 515, 527, 519, 522, 512, 519, 503, 566, 518, 522, 521, 483, 542, 507, 517, 532, 546, 494, 521, 492] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[488, 522, 518, 478, 508, 520, 515, 510, 524, 499, 595, 506, 510, 523, 507, 513, 542, 536, 552, 526, 545, 521, 502, 524, 519, 491, 501, 481, 494, 504, 484, 506, 507, 530, 533, 545, 526, 553, 509, 528, 516, 545, 493, 489, 496, 494, 531, 540, 498, 499, 493, 533, 494, 512, 511, 484, 497, 518, 524, 495, 513, 531, 497, 490, 486, 519, 514, 523, 441, 473, 540, 537, 533, 505, 521, 551, 490, 544, 521, 501, 469, 496, 506, 510, 524, 525, 508, 529, 488, 534, 498, 494, 551, 505, 525, 525, 551, 562, 492, 525, 495, 478, 492, 502, 493, 515, 517, 547, 489, 520, 482, 501, 545, 509, 487, 492, 527, 517, 538, 505, 539, 499, 497, 466, 482, 482, 510, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[492, 490, 499, 535, 485, 491, 485, 574, 500, 520, 537, 488, 538, 525, 542, 533, 499, 508, 536, 508, 534, 560, 518, 488, 510, 501, 509, 531, 462, 483, 541, 536, 510, 531, 495, 489, 516, 514, 501, 520, 514, 510, 508, 523, 548, 527, 520, 497, 519, 543, 484, 484, 504, 496, 513, 537, 507, 526, 518, 540, 516, 508, 539, 497, 524, 513, 550, 527, 498, 525, 521, 493, 501, 515, 512, 500, 517, 499, 518, 509, 512, 516, 495, 525, 541, 557, 507, 505, 507, 506, 506, 525, 484, 499, 522, 509, 513, 533, 522, 524, 472, 474, 489, 487, 457, 528, 520, 474, 542, 503, 515, 502, 548, 496, 474, 509, 518, 537, 503, 495, 495, 490, 484, 497, 504, 521, 545, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[527, 504, 486, 515, 517, 513, 505, 493, 544, 494, 502, 488, 490, 476, 503, 551, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 515, 507, 505, 511, 507, 520, 539, 541, 543, 553, 524, 510, 538, 507, 534, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 499, 496, 503, 519, 471, 516, 496, 498, 516, 518, 486, 493, 513, 518, 500, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 457, 505, 526, 493, 552, 508, 518, 494, 514, 470, 548, 522, 494, 476, 481, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 536, 494, 490, 470, 483, 514, 529, 518, 556, 479, 529, 474, 428, 571, 496, 12, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 528, 472, 485, 508, 512, 527, 491, 518, 526, 519, 473, 530, 542, 503, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 530, 512, 518, 475, 517, 512, 499, 549, 493, 514, 510, 542, 517, 528, 543, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 538, 505, 523, 523, 512, 523, 524, 555, 460, 474, 547, 525, 498, 520, 518, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8121, 8353, 8087, 8104, 8101, 8188, 8278, 8304] avg=8192.0000 max=8353 max_rank=1 min=8087 min_rank=2 imbalance_ratio=1.019653 max_load_reduction=57183 max_load_reduction_pct=0.872543 moved_to_echo=32871 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[527, 472, 513, 513, 558, 485, 540, 487, 526, 475, 480, 505, 515, 531, 495, 474, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 509, 525, 491, 503, 540, 498, 503, 502, 479, 536, 530, 509, 506, 512, 515, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 548, 500, 506, 487, 497, 477, 498, 533, 488, 544, 550, 535, 503, 534, 525, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 545, 531, 523, 515, 535, 485, 497, 529, 505, 529, 522, 555, 525, 500, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 547, 566, 489, 552, 535, 485, 486, 513, 491, 533, 527, 516, 512, 482, 508, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 532, 518, 503, 470, 522, 515, 501, 462, 488, 522, 536, 485, 489, 487, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 481, 519, 532, 526, 485, 516, 485, 512, 533, 513, 513, 505, 513, 530, 478, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 517, 517, 539, 500, 511, 506, 477, 498, 540, 484, 519, 514, 528, 532, 477, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8105, 8171, 8228, 8356, 8262, 8087, 8145, 8182] avg=8192.0000 max=8356 max_rank=3 min=8087 min_rank=5 imbalance_ratio=1.020020 max_load_reduction=57180 max_load_reduction_pct=0.872498 moved_to_echo=32676 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[514, 498, 550, 544, 524, 546, 524, 521, 520, 517, 501, 531, 472, 529, 514, 491, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 542, 469, 501, 510, 522, 460, 532, 536, 517, 491, 520, 521, 479, 513, 517, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 502, 534, 501, 536, 459, 521, 494, 521, 556, 501, 500, 469, 528, 511, 519, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 511, 474, 484, 530, 564, 459, 526, 523, 481, 524, 517, 548, 482, 495, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 557, 501, 495, 527, 521, 460, 495, 527, 496, 519, 531, 526, 494, 558, 493, 513, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 510, 524, 518, 482, 501, 564, 494, 538, 541, 509, 486, 477, 520, 506, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 540, 513, 546, 537, 498, 478, 493, 487, 537, 533, 524, 514, 574, 495, 546, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 538, 526, 504, 494, 492, 522, 539, 491, 488, 489, 459, 523, 497, 483, 528, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8304, 8154, 8162, 8116, 8222, 8181, 8310, 8087] avg=8192.0000 max=8310 max_rank=6 min=8087 min_rank=7 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32800 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[570, 524, 550, 520, 529, 495, 493, 482, 520, 507, 498, 516, 494, 463, 549, 487, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 527, 498, 509, 508, 543, 498, 494, 473, 498, 469, 507, 502, 488, 495, 485, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 536, 477, 476, 510, 494, 481, 532, 507, 499, 538, 504, 495, 513, 513, 505, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 524, 522, 493, 544, 503, 535, 512, 491, 493, 516, 493, 503, 542, 484, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 539, 520, 530, 523, 505, 473, 518, 528, 500, 526, 510, 520, 534, 469, 493, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 510, 495, 514, 483, 511, 562, 492, 528, 530, 486, 520, 536, 532, 517, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 519, 533, 527, 491, 471, 540, 485, 523, 534, 518, 492, 515, 527, 519, 522, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 519, 503, 566, 518, 522, 521, 483, 542, 507, 517, 532, 546, 494, 521, 492, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8030, 8105, 8223, 8213, 8234, 8230, 8296] avg=8192.0000 max=8296 max_rank=7 min=8030 min_rank=1 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32973 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[474, 491, 478, 528, 500, 529, 524, 484, 510, 506, 503, 495, 505, 508, 507, 523, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 507, 549, 515, 516, 510, 510, 535, 513, 515, 525, 500, 513, 482, 513, 513, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 516, 468, 504, 564, 494, 528, 491, 499, 496, 492, 495, 491, 513, 494, 510, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 520, 528, 524, 532, 514, 536, 543, 533, 491, 515, 549, 517, 492, 512, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 501, 513, 493, 544, 504, 506, 512, 468, 510, 498, 514, 509, 508, 505, 502, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 535, 555, 537, 514, 494, 456, 521, 517, 542, 566, 535, 511, 527, 508, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 491, 520, 534, 529, 491, 491, 524, 524, 514, 525, 479, 523, 500, 518, 479, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 514, 519, 515, 502, 528, 511, 498, 521, 498, 508, 526, 528, 494, 505, 516, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8074, 8223, 8092, 8328, 8086, 8389, 8139, 8205] avg=8192.0000 max=8389 max_rank=5 min=8074 min_rank=0 imbalance_ratio=1.024048 max_load_reduction=57147 max_load_reduction_pct=0.871994 moved_to_echo=32819 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[527, 504, 486, 515, 517, 513, 505, 493, 544, 494, 502, 488, 490, 476, 503, 551, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 515, 507, 505, 511, 507, 520, 539, 541, 543, 553, 524, 510, 538, 507, 534, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 499, 496, 503, 519, 471, 516, 496, 498, 516, 518, 486, 493, 513, 518, 500, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 457, 505, 526, 493, 552, 508, 518, 494, 514, 470, 548, 522, 494, 476, 481, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 536, 494, 490, 470, 483, 514, 529, 518, 556, 479, 529, 474, 428, 571, 496, 12, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 528, 472, 485, 508, 512, 527, 491, 518, 526, 519, 473, 530, 542, 503, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 530, 512, 518, 475, 517, 512, 499, 549, 493, 514, 510, 542, 517, 528, 543, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 538, 505, 523, 523, 512, 523, 524, 555, 460, 474, 547, 525, 498, 520, 518, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8121, 8353, 8087, 8104, 8101, 8188, 8278, 8304] avg=8192.0000 max=8353 max_rank=1 min=8087 min_rank=2 imbalance_ratio=1.019653 max_load_reduction=57183 max_load_reduction_pct=0.872543 moved_to_echo=32871 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[539, 558, 523, 540, 524, 514, 476, 467, 479, 509, 524, 525, 470, 530, 520, 533, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 569, 495, 510, 514, 552, 473, 516, 511, 489, 535, 487, 491, 484, 509, 499, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 500, 535, 513, 526, 510, 513, 490, 489, 503, 539, 518, 523, 536, 503, 515, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 498, 517, 518, 529, 469, 536, 503, 504, 495, 519, 478, 495, 542, 496, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 548, 495, 494, 486, 491, 522, 539, 544, 526, 494, 499, 494, 487, 528, 558, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 504, 520, 482, 488, 488, 489, 530, 520, 518, 532, 465, 558, 479, 512, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 492, 519, 507, 509, 518, 490, 526, 520, 489, 498, 565, 545, 502, 495, 512, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 524, 507, 493, 511, 498, 501, 537, 525, 498, 526, 506, 517, 530, 525, 511, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8240, 8146, 8256, 8169, 8214, 8097, 8186, 8228] avg=8192.0000 max=8256 max_rank=2 min=8097 min_rank=5 imbalance_ratio=1.007812 max_load_reduction=57280 max_load_reduction_pct=0.874023 moved_to_echo=32725 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[488, 522, 518, 478, 508, 520, 515, 510, 524, 499, 595, 506, 510, 523, 507, 513, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 536, 552, 526, 545, 521, 502, 524, 519, 491, 501, 481, 494, 504, 484, 506, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 530, 533, 545, 526, 553, 509, 528, 516, 545, 493, 489, 496, 494, 531, 540, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 499, 493, 533, 494, 512, 511, 484, 497, 518, 524, 495, 513, 531, 459, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 519, 514, 523, 441, 473, 540, 537, 533, 505, 521, 551, 490, 544, 521, 501, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 496, 506, 510, 524, 525, 508, 506, 488, 534, 498, 494, 551, 505, 525, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 562, 492, 525, 495, 478, 492, 502, 493, 515, 517, 547, 489, 520, 482, 501, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 509, 487, 492, 527, 517, 538, 505, 539, 499, 497, 466, 482, 482, 510, 506, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 8232, 8358, 8051, 8208, 8164, 8176, 8102] avg=8192.0000 max=8358 max_rank=2 min=8051 min_rank=3 imbalance_ratio=1.020264 max_load_reduction=57178 max_load_reduction_pct=0.872467 moved_to_echo=32650 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[492, 490, 499, 535, 485, 491, 485, 574, 500, 520, 537, 488, 538, 525, 542, 533, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 508, 536, 508, 534, 560, 518, 488, 510, 501, 509, 531, 462, 483, 541, 536, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 531, 495, 489, 516, 514, 501, 520, 514, 510, 508, 523, 548, 527, 520, 497, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 543, 484, 484, 504, 496, 513, 537, 507, 526, 518, 540, 516, 508, 497, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 513, 550, 527, 498, 525, 521, 493, 501, 515, 512, 500, 517, 499, 518, 509, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 516, 495, 525, 541, 557, 507, 485, 507, 506, 506, 525, 484, 499, 522, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 533, 522, 524, 472, 474, 489, 487, 457, 528, 520, 474, 542, 503, 515, 502, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 496, 474, 509, 518, 537, 503, 495, 495, 490, 484, 497, 504, 521, 545, 515, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8242, 8227, 8248, 8189, 8230, 8196, 8072, 8132] avg=8192.0000 max=8248 max_rank=2 min=8072 min_rank=6 imbalance_ratio=1.006836 max_load_reduction=57288 max_load_reduction_pct=0.874146 moved_to_echo=32630 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4352, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 3968, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 3968, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4352, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 3968, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4352, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 3968, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[494, 507, 520, 502, 481, 536, 531, 463, 490, 511, 535, 513, 485, 484, 518, 484, 500, 518, 518, 489, 551, 547, 520, 511, 502, 530, 527, 532, 561, 543, 538, 529, 514, 486, 501, 527, 475, 505, 506, 504, 511, 479, 508, 526, 522, 471, 529, 489, 504, 466, 524, 522, 568, 501, 479, 503, 513, 515, 468, 544, 513, 510, 489, 558, 510, 501, 486, 518, 512, 486, 552, 508, 537, 545, 482, 494, 510, 525, 526, 482, 504, 565, 543, 530, 564, 533, 541, 496, 526, 449, 546, 485, 543, 551, 468, 482, 508, 502, 491, 548, 519, 535, 504, 509, 471, 548, 519, 490, 582, 515, 524, 522, 514, 513, 507, 527, 508, 469, 511, 502, 505, 471, 532, 479, 537, 486, 482, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[509, 501, 497, 519, 478, 497, 531, 524, 499, 510, 519, 501, 526, 514, 488, 496, 489, 469, 536, 501, 523, 548, 519, 523, 503, 546, 513, 513, 534, 533, 524, 503, 493, 516, 518, 519, 467, 522, 501, 503, 516, 530, 493, 502, 503, 478, 544, 503, 521, 532, 512, 525, 502, 493, 509, 494, 489, 511, 477, 535, 518, 551, 519, 483, 534, 485, 507, 495, 486, 520, 522, 528, 509, 481, 540, 507, 520, 519, 493, 491, 484, 524, 528, 489, 506, 515, 511, 495, 540, 543, 513, 556, 540, 534, 478, 480, 518, 531, 504, 527, 507, 531, 520, 532, 566, 485, 535, 482, 491, 495, 493, 545, 528, 492, 501, 480, 531, 495, 501, 526, 522, 518, 505, 521, 568, 515, 504, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[493, 509, 545, 493, 527, 515, 504, 519, 527, 492, 493, 546, 487, 508, 495, 540, 513, 526, 520, 495, 520, 480, 480, 520, 528, 532, 462, 517, 507, 564, 499, 476, 516, 534, 530, 505, 509, 512, 518, 500, 517, 495, 526, 506, 503, 519, 519, 501, 527, 488, 511, 531, 496, 489, 522, 509, 539, 552, 527, 536, 547, 490, 505, 538, 508, 521, 505, 507, 523, 543, 493, 548, 515, 495, 527, 532, 493, 482, 531, 498, 470, 530, 509, 534, 510, 493, 502, 552, 492, 523, 494, 514, 526, 537, 503, 539, 506, 484, 504, 513, 505, 525, 485, 531, 490, 526, 479, 516, 502, 527, 548, 496, 519, 521, 534, 560, 516, 492, 538, 459, 529, 487, 528, 469, 506, 504, 473, 466] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[525, 518, 502, 499, 501, 472, 545, 531, 516, 466, 525, 514, 550, 480, 533, 504, 493, 509, 519, 519, 491, 497, 522, 467, 466, 504, 545, 493, 488, 515, 520, 513, 517, 473, 504, 505, 485, 538, 447, 541, 494, 506, 504, 530, 508, 508, 522, 513, 465, 516, 504, 489, 536, 521, 491, 511, 533, 507, 562, 536, 537, 481, 541, 515, 530, 523, 502, 514, 560, 521, 497, 524, 491, 528, 523, 555, 531, 515, 531, 545, 500, 517, 503, 552, 495, 485, 519, 521, 525, 500, 535, 514, 545, 480, 536, 494, 491, 501, 502, 511, 516, 539, 540, 503, 509, 511, 529, 528, 504, 502, 517, 507, 518, 504, 515, 494, 471, 490, 517, 495, 507, 468, 545, 519, 535, 524, 510, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[494, 511, 491, 502, 471, 517, 501, 520, 531, 497, 504, 500, 530, 517, 548, 488, 514, 476, 496, 518, 520, 564, 534, 497, 498, 533, 492, 570, 477, 482, 510, 518, 498, 499, 522, 516, 499, 513, 517, 523, 540, 487, 524, 530, 532, 543, 547, 475, 553, 472, 485, 481, 555, 521, 495, 504, 544, 554, 553, 529, 515, 531, 495, 552, 499, 494, 521, 489, 486, 530, 525, 546, 491, 509, 490, 523, 498, 511, 523, 511, 520, 475, 495, 505, 505, 478, 498, 496, 496, 478, 502, 478, 508, 511, 517, 509, 492, 516, 555, 570, 516, 491, 529, 474, 500, 519, 506, 547, 514, 500, 532, 512, 541, 512, 520, 496, 521, 511, 496, 507, 522, 491, 552, 497, 543, 518, 491, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[494, 507, 520, 502, 481, 536, 531, 463, 490, 511, 535, 513, 485, 484, 518, 484, 500, 518, 518, 489, 551, 547, 520, 511, 502, 530, 527, 532, 561, 543, 538, 529, 514, 486, 501, 527, 475, 505, 506, 504, 511, 479, 508, 526, 522, 471, 529, 489, 504, 466, 524, 522, 568, 501, 479, 503, 513, 515, 468, 544, 513, 510, 489, 558, 510, 501, 486, 518, 512, 486, 552, 508, 537, 545, 482, 494, 510, 525, 526, 482, 504, 565, 543, 530, 564, 533, 541, 496, 526, 449, 546, 485, 543, 551, 468, 482, 508, 502, 491, 548, 519, 535, 504, 509, 471, 548, 519, 490, 582, 515, 524, 522, 514, 513, 507, 527, 508, 469, 511, 502, 505, 471, 532, 479, 537, 486, 482, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[545, 508, 506, 516, 527, 492, 465, 525, 512, 482, 542, 547, 517, 508, 529, 513, 481, 517, 524, 530, 481, 510, 516, 517, 529, 515, 507, 532, 531, 533, 512, 494, 565, 525, 462, 497, 511, 492, 527, 488, 522, 497, 513, 501, 514, 457, 509, 541, 544, 505, 515, 482, 521, 545, 512, 549, 511, 469, 467, 501, 512, 497, 533, 547, 555, 452, 542, 502, 497, 484, 519, 500, 502, 484, 492, 487, 537, 476, 544, 535, 478, 497, 536, 569, 526, 507, 482, 490, 475, 531, 489, 518, 453, 515, 506, 541, 551, 502, 507, 526, 496, 519, 542, 532, 487, 501, 517, 510, 530, 481, 502, 509, 517, 522, 509, 566, 539, 486, 496, 530, 556, 497, 505, 506, 541, 552, 506, 481] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[525, 518, 502, 499, 501, 472, 545, 531, 516, 466, 525, 514, 550, 480, 533, 504, 493, 509, 519, 519, 491, 497, 522, 467, 466, 504, 545, 493, 488, 515, 520, 513, 517, 473, 504, 505, 485, 538, 447, 541, 494, 506, 504, 530, 508, 508, 522, 513, 465, 516, 504, 489, 536, 521, 491, 511, 533, 507, 562, 536, 537, 481, 541, 515, 530, 523, 502, 514, 560, 521, 497, 524, 491, 528, 523, 555, 531, 515, 531, 545, 500, 517, 503, 552, 495, 485, 519, 521, 525, 500, 535, 514, 545, 480, 536, 494, 491, 501, 502, 511, 516, 539, 540, 503, 509, 511, 529, 528, 504, 502, 517, 507, 518, 504, 515, 494, 471, 490, 517, 495, 507, 468, 545, 519, 535, 524, 510, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[514, 537, 544, 476, 521, 500, 537, 497, 523, 526, 489, 551, 494, 527, 526, 483, 516, 474, 479, 530, 548, 516, 529, 509, 508, 528, 528, 507, 517, 561, 512, 549, 504, 528, 527, 474, 479, 526, 478, 545, 520, 479, 461, 502, 532, 517, 514, 498, 544, 530, 501, 539, 536, 538, 533, 525, 504, 508, 512, 501, 479, 495, 527, 503, 517, 505, 505, 502, 541, 528, 498, 523, 527, 506, 523, 520, 537, 512, 511, 526, 491, 461, 488, 492, 477, 523, 490, 475, 494, 514, 452, 503, 497, 541, 492, 490, 564, 521, 483, 517, 480, 517, 486, 517, 524, 540, 498, 526, 533, 488, 494, 491, 511, 523, 523, 533, 512, 477, 536, 550, 529, 512, 521, 524, 485, 544, 509, 492] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[536, 515, 505, 526, 550, 494, 524, 535, 508, 532, 507, 511, 513, 529, 509, 475, 485, 556, 521, 474, 551, 517, 503, 521, 493, 516, 435, 554, 531, 567, 509, 529, 502, 499, 549, 528, 506, 539, 486, 505, 500, 497, 524, 542, 515, 497, 487, 489, 515, 504, 502, 516, 539, 503, 533, 508, 519, 546, 460, 485, 519, 535, 513, 535, 510, 475, 492, 475, 501, 529, 533, 499, 557, 528, 518, 489, 524, 500, 493, 504, 497, 503, 477, 483, 524, 524, 480, 514, 523, 560, 486, 532, 522, 534, 490, 542, 470, 521, 501, 519, 517, 497, 536, 488, 497, 496, 530, 487, 505, 497, 491, 507, 524, 519, 482, 495, 529, 539, 531, 520, 471, 520, 528, 537, 506, 519, 530, 481] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[493, 509, 545, 493, 527, 515, 504, 519, 527, 492, 493, 546, 487, 508, 495, 540, 513, 526, 520, 495, 520, 480, 480, 520, 528, 532, 462, 517, 507, 564, 499, 476, 516, 534, 530, 505, 509, 512, 518, 500, 517, 495, 526, 506, 503, 519, 519, 501, 527, 488, 511, 531, 496, 489, 522, 509, 539, 552, 527, 536, 547, 490, 505, 538, 508, 521, 505, 507, 523, 543, 493, 548, 515, 495, 527, 532, 493, 482, 531, 498, 470, 530, 509, 534, 510, 493, 502, 552, 492, 523, 494, 514, 526, 537, 503, 539, 506, 484, 504, 513, 505, 525, 485, 531, 490, 526, 479, 516, 502, 527, 548, 496, 519, 521, 534, 560, 516, 492, 538, 459, 529, 487, 528, 469, 506, 504, 473, 466] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[514, 537, 544, 476, 521, 500, 537, 497, 523, 526, 489, 551, 494, 527, 526, 483, 516, 474, 479, 530, 548, 516, 529, 509, 508, 528, 528, 507, 517, 561, 512, 549, 504, 528, 527, 474, 479, 526, 478, 545, 520, 479, 461, 502, 532, 517, 514, 498, 544, 530, 501, 539, 536, 538, 533, 525, 504, 508, 512, 501, 479, 495, 527, 503, 517, 505, 505, 502, 541, 528, 498, 523, 527, 506, 523, 520, 537, 512, 511, 526, 491, 461, 488, 492, 477, 523, 490, 475, 494, 514, 452, 503, 497, 541, 492, 490, 564, 521, 483, 517, 480, 517, 486, 517, 524, 540, 498, 526, 533, 488, 494, 491, 511, 523, 523, 533, 512, 477, 536, 550, 529, 512, 521, 524, 485, 544, 509, 492] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[525, 518, 502, 499, 501, 472, 545, 531, 516, 466, 525, 514, 550, 480, 533, 504, 12, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 509, 519, 519, 491, 497, 522, 467, 466, 504, 545, 493, 488, 463, 520, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 473, 504, 505, 485, 538, 447, 541, 494, 506, 504, 530, 508, 508, 522, 513, 57, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 516, 504, 489, 479, 521, 491, 511, 533, 507, 562, 536, 537, 481, 541, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 523, 502, 514, 560, 521, 497, 515, 491, 528, 523, 555, 531, 515, 531, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 517, 503, 552, 495, 485, 519, 521, 525, 500, 535, 514, 545, 480, 536, 494, 44, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 501, 502, 493, 516, 539, 540, 503, 509, 511, 529, 528, 504, 502, 517, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 504, 515, 494, 471, 490, 517, 495, 507, 468, 545, 519, 528, 524, 510, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8009, 8160, 8188, 8381, 8271, 8192, 8126] avg=8192.0000 max=8381 max_rank=4 min=8009 min_rank=1 imbalance_ratio=1.023071 max_load_reduction=57155 max_load_reduction_pct=0.872116 moved_to_echo=32970 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[509, 501, 497, 519, 478, 497, 531, 524, 499, 510, 519, 501, 526, 514, 488, 496, 6, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 469, 536, 501, 523, 548, 519, 523, 503, 546, 513, 513, 534, 490, 524, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 516, 518, 519, 467, 522, 501, 503, 516, 530, 493, 502, 503, 478, 544, 503, 49, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 532, 512, 525, 453, 493, 509, 494, 489, 511, 477, 535, 518, 551, 519, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 485, 507, 495, 486, 520, 522, 522, 509, 481, 540, 507, 520, 519, 493, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 524, 528, 489, 506, 515, 511, 495, 540, 543, 513, 556, 540, 534, 478, 480, 42, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 531, 504, 518, 507, 531, 520, 532, 566, 485, 535, 482, 491, 495, 493, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 492, 501, 480, 531, 495, 501, 526, 522, 518, 505, 521, 565, 515, 504, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8124, 8234, 8158, 8122, 8131, 8281, 8253, 8233] avg=8192.0000 max=8281 max_rank=5 min=8122 min_rank=3 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32898 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[494, 511, 491, 502, 471, 517, 501, 520, 531, 497, 504, 500, 530, 517, 548, 488, 7, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 476, 496, 518, 520, 564, 534, 497, 498, 533, 492, 570, 477, 443, 510, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 499, 522, 516, 499, 513, 517, 523, 540, 487, 524, 530, 532, 543, 547, 475, 54, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 472, 485, 481, 501, 521, 495, 504, 544, 554, 553, 529, 515, 531, 495, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 494, 521, 489, 486, 530, 525, 540, 491, 509, 490, 523, 498, 511, 523, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 475, 495, 505, 505, 478, 498, 496, 496, 478, 502, 478, 508, 511, 517, 509, 38, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 516, 555, 560, 516, 491, 529, 474, 500, 519, 506, 547, 514, 500, 532, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 512, 520, 496, 521, 511, 496, 507, 522, 491, 552, 497, 540, 518, 491, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8138, 8160, 8320, 8285, 8140, 8012, 8263, 8218] avg=8192.0000 max=8320 max_rank=2 min=8012 min_rank=5 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32633 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[514, 537, 544, 476, 521, 500, 537, 497, 523, 526, 489, 551, 494, 527, 526, 483, 6, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 474, 479, 530, 548, 516, 529, 509, 508, 528, 528, 507, 517, 516, 512, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 528, 527, 474, 479, 526, 478, 545, 520, 479, 461, 502, 532, 517, 514, 498, 52, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 530, 501, 539, 484, 538, 533, 525, 504, 508, 512, 501, 479, 495, 527, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 505, 505, 502, 541, 528, 498, 517, 527, 506, 523, 520, 537, 512, 511, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 461, 488, 492, 477, 523, 490, 475, 494, 514, 452, 503, 497, 541, 492, 490, 44, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 564, 521, 483, 508, 480, 517, 486, 517, 524, 540, 498, 526, 533, 488, 494, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 523, 523, 533, 512, 477, 536, 550, 529, 512, 521, 524, 483, 544, 509, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8259, 8266, 8137, 8223, 8275, 7927, 8170, 8279] avg=8192.0000 max=8279 max_rank=7 min=7927 min_rank=5 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32651 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[493, 509, 545, 493, 527, 515, 504, 519, 527, 492, 493, 546, 487, 508, 495, 540, 6, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 526, 520, 495, 520, 480, 480, 520, 528, 532, 462, 517, 507, 518, 499, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 534, 530, 505, 509, 512, 518, 500, 517, 495, 526, 506, 503, 519, 519, 501, 48, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 488, 511, 531, 448, 489, 522, 509, 539, 552, 527, 536, 547, 490, 505, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 521, 505, 507, 523, 543, 493, 542, 515, 495, 527, 532, 493, 482, 531, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 530, 509, 534, 510, 493, 502, 552, 492, 523, 494, 514, 526, 537, 503, 539, 45, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 484, 504, 504, 505, 525, 485, 531, 490, 526, 479, 516, 502, 527, 548, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 521, 534, 560, 516, 492, 538, 459, 529, 487, 528, 469, 504, 504, 473, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8093, 8259, 8259, 8215, 8276, 8128, 8099] avg=8192.0000 max=8276 max_rank=5 min=8093 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32718 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[494, 507, 520, 502, 481, 536, 531, 463, 490, 511, 535, 513, 485, 484, 518, 484, 7, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 518, 518, 489, 551, 547, 520, 511, 502, 530, 527, 532, 561, 499, 538, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 486, 501, 527, 475, 505, 506, 504, 511, 479, 508, 526, 522, 471, 529, 489, 56, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 466, 524, 522, 512, 501, 479, 503, 513, 515, 468, 544, 513, 510, 489, 558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 501, 486, 518, 512, 486, 552, 503, 537, 545, 482, 494, 510, 525, 526, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 565, 543, 530, 564, 533, 541, 496, 526, 449, 546, 485, 543, 551, 468, 482, 43, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 502, 491, 538, 519, 535, 504, 509, 471, 548, 519, 490, 582, 515, 524, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 513, 507, 527, 508, 469, 511, 502, 505, 471, 532, 479, 534, 486, 482, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8069, 8372, 8110, 8121, 8169, 8372, 8277, 8046] avg=8192.0000 max=8372 max_rank=1 min=8046 min_rank=7 imbalance_ratio=1.021973 max_load_reduction=57164 max_load_reduction_pct=0.872253 moved_to_echo=32864 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[525, 518, 502, 499, 501, 472, 545, 531, 516, 466, 525, 514, 550, 480, 533, 504, 12, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 509, 519, 519, 491, 497, 522, 467, 466, 504, 545, 493, 488, 463, 520, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 473, 504, 505, 485, 538, 447, 541, 494, 506, 504, 530, 508, 508, 522, 513, 57, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 516, 504, 489, 479, 521, 491, 511, 533, 507, 562, 536, 537, 481, 541, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 523, 502, 514, 560, 521, 497, 515, 491, 528, 523, 555, 531, 515, 531, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 517, 503, 552, 495, 485, 519, 521, 525, 500, 535, 514, 545, 480, 536, 494, 44, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 501, 502, 493, 516, 539, 540, 503, 509, 511, 529, 528, 504, 502, 517, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 504, 515, 494, 471, 490, 517, 495, 507, 468, 545, 519, 528, 524, 510, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8009, 8160, 8188, 8381, 8271, 8192, 8126] avg=8192.0000 max=8381 max_rank=4 min=8009 min_rank=1 imbalance_ratio=1.023071 max_load_reduction=57155 max_load_reduction_pct=0.872116 moved_to_echo=32970 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[545, 508, 506, 516, 527, 492, 465, 525, 512, 482, 542, 547, 517, 508, 529, 513, 6, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 517, 524, 530, 481, 510, 516, 517, 529, 515, 507, 532, 531, 490, 512, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 565, 525, 462, 497, 511, 492, 527, 488, 522, 497, 513, 501, 514, 457, 509, 541, 51, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 505, 515, 482, 470, 545, 512, 549, 511, 469, 467, 501, 512, 497, 533, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 452, 542, 502, 497, 484, 519, 495, 502, 484, 492, 487, 537, 476, 544, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 497, 536, 569, 526, 507, 482, 490, 475, 531, 489, 518, 453, 515, 506, 541, 42, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 502, 507, 517, 496, 519, 542, 532, 487, 501, 517, 510, 530, 481, 502, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 522, 509, 566, 539, 486, 496, 530, 556, 497, 505, 506, 538, 552, 506, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8248, 8186, 8173, 8159, 8103, 8158, 8203, 8306] avg=8192.0000 max=8306 max_rank=7 min=8103 min_rank=4 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32770 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[536, 515, 505, 526, 550, 494, 524, 535, 508, 532, 507, 511, 513, 529, 509, 475, 6, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 556, 521, 474, 551, 517, 503, 521, 493, 516, 435, 554, 531, 521, 509, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 499, 549, 528, 506, 539, 486, 505, 500, 497, 524, 542, 515, 497, 487, 489, 53, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 504, 502, 516, 486, 503, 533, 508, 519, 546, 460, 485, 519, 535, 513, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 475, 492, 475, 501, 529, 533, 494, 557, 528, 518, 489, 524, 500, 493, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 503, 477, 483, 524, 524, 480, 514, 523, 560, 486, 532, 522, 534, 490, 542, 45, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 521, 501, 510, 517, 497, 536, 488, 497, 496, 530, 487, 505, 497, 491, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 519, 482, 495, 529, 539, 531, 520, 471, 520, 528, 537, 504, 519, 530, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8282, 8216, 8219, 8179, 8122, 8239, 8050, 8229] avg=8192.0000 max=8282 max_rank=0 min=8050 min_rank=6 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32640 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[493, 509, 545, 493, 527, 515, 504, 519, 527, 492, 493, 546, 487, 508, 495, 540, 6, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 526, 520, 495, 520, 480, 480, 520, 528, 532, 462, 517, 507, 518, 499, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 534, 530, 505, 509, 512, 518, 500, 517, 495, 526, 506, 503, 519, 519, 501, 48, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 488, 511, 531, 448, 489, 522, 509, 539, 552, 527, 536, 547, 490, 505, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 521, 505, 507, 523, 543, 493, 542, 515, 495, 527, 532, 493, 482, 531, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 530, 509, 534, 510, 493, 502, 552, 492, 523, 494, 514, 526, 537, 503, 539, 45, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 484, 504, 504, 505, 525, 485, 531, 490, 526, 479, 516, 502, 527, 548, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 521, 534, 560, 516, 492, 538, 459, 529, 487, 528, 469, 504, 504, 473, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8093, 8259, 8259, 8215, 8276, 8128, 8099] avg=8192.0000 max=8276 max_rank=5 min=8093 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32718 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[545, 508, 506, 516, 527, 492, 465, 525, 512, 482, 542, 547, 517, 508, 529, 513, 6, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 517, 524, 530, 481, 510, 516, 517, 529, 515, 507, 532, 531, 490, 512, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 565, 525, 462, 497, 511, 492, 527, 488, 522, 497, 513, 501, 514, 457, 509, 541, 51, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 505, 515, 482, 470, 545, 512, 549, 511, 469, 467, 501, 512, 497, 533, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 452, 542, 502, 497, 484, 519, 495, 502, 484, 492, 487, 537, 476, 544, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 497, 536, 569, 526, 507, 482, 490, 475, 531, 489, 518, 453, 515, 506, 541, 42, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 502, 507, 517, 496, 519, 542, 532, 487, 501, 517, 510, 530, 481, 502, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 522, 509, 566, 539, 486, 496, 530, 556, 497, 505, 506, 538, 552, 506, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8248, 8186, 8173, 8159, 8103, 8158, 8203, 8306] avg=8192.0000 max=8306 max_rank=7 min=8103 min_rank=4 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32770 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[514, 537, 544, 476, 521, 500, 537, 497, 523, 526, 489, 551, 494, 527, 526, 483, 6, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 474, 479, 530, 548, 516, 529, 509, 508, 528, 528, 507, 517, 516, 512, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 528, 527, 474, 479, 526, 478, 545, 520, 479, 461, 502, 532, 517, 514, 498, 52, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 530, 501, 539, 484, 538, 533, 525, 504, 508, 512, 501, 479, 495, 527, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 505, 505, 502, 541, 528, 498, 517, 527, 506, 523, 520, 537, 512, 511, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 461, 488, 492, 477, 523, 490, 475, 494, 514, 452, 503, 497, 541, 492, 490, 44, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 564, 521, 483, 508, 480, 517, 486, 517, 524, 540, 498, 526, 533, 488, 494, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 523, 523, 533, 512, 477, 536, 550, 529, 512, 521, 524, 483, 544, 509, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8259, 8266, 8137, 8223, 8275, 7927, 8170, 8279] avg=8192.0000 max=8279 max_rank=7 min=7927 min_rank=5 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32651 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 3968, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[4224, 4096, 4224, 4096, 3968, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 3968, 4224, 4096, 4224, 3968, 4224, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4096, 3840, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 3968, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 3968, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[3968, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[537, 492, 480, 515, 503, 509, 510, 498, 527, 509, 532, 475, 471, 509, 502, 518, 518, 479, 531, 511, 501, 525, 496, 519, 568, 536, 529, 492, 508, 513, 497, 529, 518, 482, 492, 522, 517, 509, 542, 555, 526, 541, 531, 512, 541, 524, 529, 528, 494, 472, 521, 482, 495, 499, 543, 532, 508, 490, 539, 554, 525, 525, 502, 522, 506, 512, 480, 490, 486, 508, 514, 531, 490, 505, 504, 534, 522, 512, 528, 535, 539, 490, 484, 551, 515, 531, 485, 488, 499, 518, 484, 545, 511, 492, 517, 524, 465, 514, 495, 511, 540, 510, 480, 490, 509, 479, 480, 516, 515, 523, 530, 500, 507, 534, 531, 546, 520, 494, 542, 523, 538, 496, 484, 513, 522, 471, 526, 491] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[483, 547, 505, 497, 435, 476, 513, 505, 536, 494, 513, 523, 510, 552, 524, 486, 492, 542, 518, 466, 514, 537, 546, 544, 525, 537, 529, 511, 515, 534, 496, 537, 507, 518, 525, 536, 504, 498, 498, 514, 510, 519, 557, 555, 522, 525, 497, 463, 539, 493, 500, 514, 531, 491, 523, 490, 485, 508, 519, 517, 541, 507, 467, 485, 526, 484, 494, 531, 532, 501, 509, 528, 496, 506, 485, 529, 520, 451, 487, 536, 503, 512, 503, 513, 528, 480, 493, 505, 535, 514, 486, 511, 502, 534, 502, 516, 519, 515, 521, 524, 498, 479, 523, 515, 526, 491, 503, 483, 556, 536, 509, 544, 541, 505, 501, 507, 513, 509, 513, 552, 489, 518, 508, 502, 532, 536, 512, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[502, 527, 496, 487, 511, 487, 506, 503, 509, 521, 484, 502, 493, 554, 512, 548, 526, 477, 506, 464, 513, 550, 494, 497, 484, 551, 523, 471, 553, 513, 513, 543, 535, 496, 499, 528, 524, 541, 498, 546, 509, 532, 517, 497, 485, 519, 479, 568, 485, 491, 465, 507, 499, 517, 505, 504, 520, 476, 518, 484, 504, 476, 536, 553, 522, 506, 506, 490, 512, 537, 501, 527, 517, 499, 513, 492, 519, 530, 495, 496, 496, 523, 489, 480, 502, 486, 498, 507, 550, 535, 507, 486, 513, 516, 494, 545, 490, 501, 494, 556, 532, 490, 507, 523, 498, 529, 501, 546, 504, 548, 547, 477, 524, 518, 527, 529, 539, 502, 507, 498, 526, 542, 489, 512, 545, 532, 556, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[505, 510, 504, 545, 541, 518, 547, 470, 529, 506, 472, 534, 506, 530, 487, 481, 520, 493, 520, 467, 482, 551, 491, 540, 512, 515, 466, 507, 512, 542, 537, 510, 511, 494, 512, 527, 547, 506, 509, 498, 502, 515, 533, 497, 524, 509, 499, 469, 523, 516, 517, 531, 477, 493, 497, 540, 525, 486, 534, 518, 535, 528, 553, 503, 488, 559, 519, 537, 509, 480, 501, 544, 529, 524, 518, 531, 509, 491, 526, 518, 511, 517, 500, 528, 489, 477, 494, 526, 516, 491, 492, 540, 524, 523, 494, 503, 499, 490, 505, 523, 509, 517, 520, 516, 513, 512, 491, 532, 502, 534, 488, 535, 512, 509, 490, 491, 513, 532, 536, 475, 521, 487, 519, 545, 520, 496, 525, 493] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[522, 503, 504, 514, 456, 538, 525, 499, 499, 487, 521, 487, 506, 504, 520, 512, 547, 521, 477, 490, 502, 528, 522, 494, 511, 527, 520, 551, 509, 539, 486, 508, 502, 537, 512, 515, 523, 514, 485, 510, 535, 490, 496, 516, 513, 494, 485, 493, 504, 510, 513, 503, 517, 512, 492, 478, 527, 502, 505, 487, 522, 524, 504, 526, 496, 478, 543, 521, 541, 504, 546, 526, 526, 512, 526, 542, 492, 530, 503, 522, 525, 508, 502, 498, 533, 513, 489, 524, 533, 488, 547, 503, 545, 504, 509, 483, 542, 538, 527, 501, 502, 480, 524, 500, 521, 532, 570, 525, 497, 494, 545, 502, 529, 514, 491, 496, 521, 536, 522, 509, 533, 497, 480, 538, 466, 510, 511, 496] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[517, 502, 478, 484, 511, 539, 491, 503, 519, 521, 486, 474, 561, 502, 541, 492, 481, 560, 543, 508, 522, 478, 517, 516, 528, 529, 497, 470, 500, 500, 467, 516, 509, 535, 510, 537, 520, 502, 538, 519, 511, 513, 525, 535, 515, 507, 534, 493, 516, 526, 493, 527, 486, 509, 513, 490, 527, 509, 487, 532, 512, 512, 516, 490, 497, 517, 501, 477, 489, 522, 519, 478, 519, 512, 522, 505, 529, 481, 497, 520, 521, 491, 498, 522, 535, 505, 541, 533, 490, 505, 505, 542, 512, 535, 531, 515, 478, 487, 548, 547, 519, 537, 506, 556, 548, 542, 492, 527, 489, 538, 503, 500, 534, 518, 504, 485, 531, 480, 528, 537, 507, 531, 519, 536, 441, 492, 489, 520] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[474, 484, 485, 550, 502, 504, 491, 514, 474, 534, 541, 501, 489, 498, 515, 546, 514, 506, 541, 545, 503, 498, 528, 484, 559, 558, 509, 504, 506, 486, 537, 544, 470, 507, 524, 498, 504, 497, 464, 532, 525, 509, 513, 504, 504, 484, 486, 560, 480, 513, 506, 495, 496, 535, 508, 561, 559, 505, 515, 509, 520, 488, 481, 481, 512, 488, 525, 528, 513, 545, 487, 502, 490, 488, 509, 485, 533, 487, 541, 492, 513, 517, 488, 515, 487, 505, 548, 553, 498, 542, 512, 524, 490, 533, 561, 482, 534, 472, 515, 547, 530, 518, 561, 517, 491, 524, 516, 547, 498, 513, 534, 536, 497, 509, 487, 521, 533, 493, 516, 483, 489, 501, 510, 525, 503, 515, 529, 522] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[502, 527, 496, 487, 511, 487, 506, 503, 509, 521, 484, 502, 493, 554, 512, 548, 526, 477, 506, 464, 513, 550, 494, 497, 484, 551, 523, 471, 553, 513, 513, 543, 535, 496, 499, 528, 524, 541, 498, 546, 509, 532, 517, 497, 485, 519, 479, 568, 485, 491, 465, 507, 499, 517, 505, 504, 520, 476, 518, 484, 504, 476, 536, 553, 522, 506, 506, 490, 512, 537, 501, 527, 517, 499, 513, 492, 519, 530, 495, 496, 496, 523, 489, 480, 502, 486, 498, 507, 550, 535, 507, 486, 513, 516, 494, 545, 490, 501, 494, 556, 532, 490, 507, 523, 498, 529, 501, 546, 504, 548, 547, 477, 524, 518, 527, 529, 539, 502, 507, 498, 526, 542, 489, 512, 545, 532, 556, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[483, 547, 505, 497, 435, 476, 513, 505, 536, 494, 513, 523, 510, 552, 524, 486, 492, 542, 518, 466, 514, 537, 546, 544, 525, 537, 529, 511, 515, 534, 496, 537, 507, 518, 525, 536, 504, 498, 498, 514, 510, 519, 557, 555, 522, 525, 497, 463, 539, 493, 500, 514, 531, 491, 523, 490, 485, 508, 519, 517, 541, 507, 467, 485, 526, 484, 494, 531, 532, 501, 509, 528, 496, 506, 485, 529, 520, 451, 487, 536, 503, 512, 503, 513, 528, 480, 493, 505, 535, 514, 486, 511, 502, 534, 502, 516, 519, 515, 521, 524, 498, 479, 523, 515, 526, 491, 503, 483, 556, 536, 509, 544, 541, 505, 501, 507, 513, 509, 513, 552, 489, 518, 508, 502, 532, 536, 512, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[541, 516, 541, 496, 527, 538, 500, 501, 527, 484, 513, 484, 493, 502, 522, 533, 530, 527, 509, 527, 518, 521, 497, 509, 543, 526, 491, 485, 531, 525, 476, 542, 518, 484, 522, 503, 520, 495, 504, 538, 515, 552, 525, 521, 471, 505, 533, 541, 518, 523, 504, 518, 525, 534, 525, 487, 523, 498, 527, 471, 510, 487, 507, 496, 531, 529, 562, 482, 519, 537, 483, 507, 496, 499, 509, 507, 526, 520, 516, 551, 518, 503, 547, 499, 470, 539, 501, 478, 508, 510, 526, 499, 507, 482, 514, 513, 517, 510, 528, 492, 482, 494, 485, 484, 488, 528, 517, 523, 505, 508, 531, 539, 517, 570, 464, 535, 477, 529, 533, 509, 498, 510, 506, 517, 496, 468, 512, 501] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[505, 510, 504, 545, 541, 518, 547, 470, 529, 506, 472, 534, 506, 530, 487, 481, 520, 493, 520, 467, 482, 551, 491, 540, 512, 515, 466, 507, 512, 542, 537, 510, 511, 494, 512, 527, 547, 506, 509, 498, 502, 515, 533, 497, 524, 509, 499, 469, 523, 516, 517, 531, 477, 493, 497, 540, 525, 486, 534, 518, 535, 528, 553, 503, 488, 559, 519, 537, 509, 480, 501, 544, 529, 524, 518, 531, 509, 491, 526, 518, 511, 517, 500, 528, 489, 477, 494, 526, 516, 491, 492, 540, 524, 523, 494, 503, 499, 490, 505, 523, 509, 517, 520, 516, 513, 512, 491, 532, 502, 534, 488, 535, 512, 509, 490, 491, 513, 532, 536, 475, 521, 487, 519, 545, 520, 496, 525, 493] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[537, 492, 480, 515, 503, 509, 510, 498, 527, 509, 532, 475, 471, 509, 502, 518, 43, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 479, 531, 511, 501, 525, 496, 519, 568, 493, 529, 492, 508, 513, 497, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 482, 492, 522, 517, 509, 542, 522, 526, 541, 531, 512, 541, 524, 529, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 472, 521, 482, 495, 499, 543, 532, 508, 490, 539, 554, 525, 525, 502, 522, 23, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 512, 480, 490, 486, 508, 514, 531, 490, 505, 504, 534, 522, 512, 528, 535, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 490, 484, 551, 515, 531, 485, 488, 499, 518, 484, 545, 511, 492, 517, 524, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 514, 495, 476, 540, 510, 480, 490, 509, 479, 480, 516, 515, 523, 530, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 534, 531, 546, 520, 494, 536, 523, 538, 496, 484, 513, 522, 471, 526, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8147, 8209, 8336, 8244, 8160, 8186, 8022, 8232] avg=8192.0000 max=8336 max_rank=2 min=8022 min_rank=6 imbalance_ratio=1.017578 max_load_reduction=57200 max_load_reduction_pct=0.872803 moved_to_echo=32600 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[517, 502, 478, 484, 511, 539, 491, 503, 519, 521, 486, 474, 561, 502, 541, 492, 481, 560, 543, 508, 522, 478, 517, 516, 528, 529, 497, 470, 500, 500, 467, 516, 509, 535, 510, 537, 520, 502, 538, 519, 511, 513, 525, 535, 515, 507, 534, 493, 516, 526, 493, 527, 486, 509, 513, 490, 527, 509, 487, 532, 512, 512, 516, 490, 497, 517, 501, 477, 489, 522, 519, 478, 519, 512, 522, 505, 529, 481, 497, 520, 521, 491, 498, 522, 535, 505, 541, 533, 490, 505, 505, 542, 512, 535, 531, 515, 478, 487, 548, 547, 519, 537, 506, 556, 548, 542, 492, 527, 489, 538, 503, 500, 534, 518, 504, 485, 531, 480, 528, 537, 507, 531, 519, 536, 441, 492, 489, 520] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[537, 492, 480, 515, 503, 509, 510, 498, 527, 509, 532, 475, 471, 509, 502, 518, 518, 479, 531, 511, 501, 525, 496, 519, 568, 536, 529, 492, 508, 513, 497, 529, 518, 482, 492, 522, 517, 509, 542, 555, 526, 541, 531, 512, 541, 524, 529, 528, 494, 472, 521, 482, 495, 499, 543, 532, 508, 490, 539, 554, 525, 525, 502, 522, 506, 512, 480, 490, 486, 508, 514, 531, 490, 505, 504, 534, 522, 512, 528, 535, 539, 490, 484, 551, 515, 531, 485, 488, 499, 518, 484, 545, 511, 492, 517, 524, 465, 514, 495, 511, 540, 510, 480, 490, 509, 479, 480, 516, 515, 523, 530, 500, 507, 534, 531, 546, 520, 494, 542, 523, 538, 496, 484, 513, 522, 471, 526, 491] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[502, 527, 496, 487, 511, 487, 506, 503, 509, 521, 484, 502, 493, 554, 512, 548, 44, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 477, 506, 464, 513, 550, 494, 497, 484, 507, 523, 471, 553, 513, 513, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 496, 499, 528, 524, 541, 498, 513, 509, 532, 517, 497, 485, 519, 479, 568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 491, 465, 507, 499, 517, 505, 504, 520, 476, 518, 484, 504, 476, 536, 553, 23, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 506, 506, 490, 512, 537, 501, 527, 517, 499, 513, 492, 519, 530, 495, 496, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 523, 489, 480, 502, 486, 498, 507, 550, 535, 507, 486, 513, 516, 494, 545, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 501, 494, 518, 532, 490, 507, 523, 498, 529, 501, 546, 504, 548, 547, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 518, 527, 529, 539, 502, 501, 498, 526, 542, 489, 512, 545, 532, 556, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8134, 8240, 8083, 8165, 8140, 8205, 8365] avg=8192.0000 max=8365 max_rank=7 min=8083 min_rank=3 imbalance_ratio=1.021118 max_load_reduction=57171 max_load_reduction_pct=0.872360 moved_to_echo=32875 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[522, 503, 504, 514, 456, 538, 525, 499, 499, 487, 521, 487, 506, 504, 520, 512, 46, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 521, 477, 490, 502, 528, 522, 494, 511, 481, 520, 551, 509, 539, 486, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 537, 512, 515, 523, 514, 485, 472, 535, 490, 496, 516, 513, 494, 485, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 510, 513, 503, 517, 512, 492, 478, 527, 502, 505, 487, 522, 524, 504, 526, 25, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 478, 543, 521, 541, 504, 546, 526, 526, 512, 526, 542, 492, 530, 503, 522, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 508, 502, 498, 533, 513, 489, 524, 533, 488, 547, 503, 545, 504, 509, 483, 13, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 538, 527, 460, 502, 480, 524, 500, 521, 532, 570, 525, 497, 494, 545, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 514, 491, 496, 521, 536, 509, 509, 533, 497, 480, 538, 466, 510, 511, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8162, 8186, 8082, 8173, 8314, 8224, 8259, 8136] avg=8192.0000 max=8314 max_rank=4 min=8082 min_rank=2 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32933 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[517, 502, 478, 484, 511, 539, 491, 503, 519, 521, 486, 474, 561, 502, 541, 492, 42, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 560, 543, 508, 522, 478, 517, 516, 528, 487, 497, 470, 500, 500, 467, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 535, 510, 537, 520, 502, 538, 489, 511, 513, 525, 535, 515, 507, 534, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 526, 493, 527, 486, 509, 513, 490, 527, 509, 487, 532, 512, 512, 516, 490, 21, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 517, 501, 477, 489, 522, 519, 478, 519, 512, 522, 505, 529, 481, 497, 520, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 491, 498, 522, 535, 505, 541, 533, 490, 505, 505, 542, 512, 535, 531, 515, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 487, 548, 510, 519, 537, 506, 556, 548, 542, 492, 527, 489, 538, 503, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 518, 504, 485, 531, 480, 522, 537, 507, 531, 519, 536, 441, 492, 489, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8090, 8273, 8185, 8088, 8293, 8280, 8146] avg=8192.0000 max=8293 max_rank=5 min=8088 min_rank=4 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32807 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[483, 547, 505, 497, 435, 476, 513, 505, 536, 494, 513, 523, 510, 552, 524, 486, 43, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 542, 518, 466, 514, 537, 546, 544, 525, 494, 529, 511, 515, 534, 496, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 518, 525, 536, 504, 498, 498, 484, 510, 519, 557, 555, 522, 525, 497, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 493, 500, 514, 531, 491, 523, 490, 485, 508, 519, 517, 541, 507, 467, 485, 21, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 484, 494, 531, 532, 501, 509, 528, 496, 506, 485, 529, 520, 451, 487, 536, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 512, 503, 513, 528, 480, 493, 505, 535, 514, 486, 511, 502, 534, 502, 516, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 515, 521, 488, 498, 479, 523, 515, 526, 491, 503, 483, 556, 536, 509, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 505, 501, 507, 513, 509, 507, 552, 489, 518, 508, 502, 532, 536, 512, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8159, 8300, 8218, 8150, 8118, 8149, 8206, 8236] avg=8192.0000 max=8300 max_rank=1 min=8118 min_rank=4 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32709 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[474, 484, 485, 550, 502, 504, 491, 514, 474, 534, 541, 501, 489, 498, 515, 546, 44, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 506, 541, 545, 503, 498, 528, 484, 559, 514, 509, 504, 506, 486, 537, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 507, 524, 498, 504, 497, 464, 501, 525, 509, 513, 504, 504, 484, 486, 560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 513, 506, 495, 496, 535, 508, 561, 559, 505, 515, 509, 520, 488, 481, 481, 22, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 488, 525, 528, 513, 545, 487, 502, 490, 488, 509, 485, 533, 487, 541, 492, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 517, 488, 515, 487, 505, 548, 553, 498, 542, 512, 524, 490, 533, 561, 482, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 472, 515, 510, 530, 518, 561, 517, 491, 524, 516, 547, 498, 513, 534, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 509, 487, 521, 533, 493, 510, 483, 489, 501, 510, 525, 503, 515, 529, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8278, 8050, 8193, 8128, 8280, 8316, 8127] avg=8192.0000 max=8316 max_rank=6 min=8050 min_rank=2 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32851 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[505, 510, 504, 545, 541, 518, 547, 470, 529, 506, 472, 534, 506, 530, 487, 481, 41, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 493, 520, 467, 482, 551, 491, 540, 512, 474, 466, 507, 512, 542, 537, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 494, 512, 527, 547, 506, 509, 468, 502, 515, 533, 497, 524, 509, 499, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 516, 517, 531, 477, 493, 497, 540, 525, 486, 534, 518, 535, 528, 553, 503, 21, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 559, 519, 537, 509, 480, 501, 544, 529, 524, 518, 531, 509, 491, 526, 518, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 517, 500, 528, 489, 477, 494, 526, 516, 491, 492, 540, 524, 523, 494, 503, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 490, 505, 487, 509, 517, 520, 516, 513, 512, 491, 532, 502, 534, 488, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 509, 490, 491, 513, 532, 530, 475, 521, 487, 519, 545, 520, 496, 525, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8243, 8124, 8122, 8316, 8286, 8137, 8150, 8158] avg=8192.0000 max=8316 max_rank=3 min=8122 min_rank=2 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32731 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[502, 527, 496, 487, 511, 487, 506, 503, 509, 521, 484, 502, 493, 554, 512, 548, 44, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 477, 506, 464, 513, 550, 494, 497, 484, 507, 523, 471, 553, 513, 513, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 496, 499, 528, 524, 541, 498, 513, 509, 532, 517, 497, 485, 519, 479, 568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 491, 465, 507, 499, 517, 505, 504, 520, 476, 518, 484, 504, 476, 536, 553, 23, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 506, 506, 490, 512, 537, 501, 527, 517, 499, 513, 492, 519, 530, 495, 496, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 523, 489, 480, 502, 486, 498, 507, 550, 535, 507, 486, 513, 516, 494, 545, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 501, 494, 518, 532, 490, 507, 523, 498, 529, 501, 546, 504, 548, 547, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 518, 527, 529, 539, 502, 501, 498, 526, 542, 489, 512, 545, 532, 556, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8134, 8240, 8083, 8165, 8140, 8205, 8365] avg=8192.0000 max=8365 max_rank=7 min=8083 min_rank=3 imbalance_ratio=1.021118 max_load_reduction=57171 max_load_reduction_pct=0.872360 moved_to_echo=32875 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[483, 547, 505, 497, 435, 476, 513, 505, 536, 494, 513, 523, 510, 552, 524, 486, 43, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 542, 518, 466, 514, 537, 546, 544, 525, 494, 529, 511, 515, 534, 496, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 518, 525, 536, 504, 498, 498, 484, 510, 519, 557, 555, 522, 525, 497, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 493, 500, 514, 531, 491, 523, 490, 485, 508, 519, 517, 541, 507, 467, 485, 21, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 484, 494, 531, 532, 501, 509, 528, 496, 506, 485, 529, 520, 451, 487, 536, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 512, 503, 513, 528, 480, 493, 505, 535, 514, 486, 511, 502, 534, 502, 516, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 515, 521, 488, 498, 479, 523, 515, 526, 491, 503, 483, 556, 536, 509, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 505, 501, 507, 513, 509, 507, 552, 489, 518, 508, 502, 532, 536, 512, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8159, 8300, 8218, 8150, 8118, 8149, 8206, 8236] avg=8192.0000 max=8300 max_rank=1 min=8118 min_rank=4 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32709 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[517, 502, 478, 484, 511, 539, 491, 503, 519, 521, 486, 474, 561, 502, 541, 492, 42, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 560, 543, 508, 522, 478, 517, 516, 528, 487, 497, 470, 500, 500, 467, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 535, 510, 537, 520, 502, 538, 489, 511, 513, 525, 535, 515, 507, 534, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 526, 493, 527, 486, 509, 513, 490, 527, 509, 487, 532, 512, 512, 516, 490, 21, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 517, 501, 477, 489, 522, 519, 478, 519, 512, 522, 505, 529, 481, 497, 520, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 491, 498, 522, 535, 505, 541, 533, 490, 505, 505, 542, 512, 535, 531, 515, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 487, 548, 510, 519, 537, 506, 556, 548, 542, 492, 527, 489, 538, 503, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 518, 504, 485, 531, 480, 522, 537, 507, 531, 519, 536, 441, 492, 489, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8090, 8273, 8185, 8088, 8293, 8280, 8146] avg=8192.0000 max=8293 max_rank=5 min=8088 min_rank=4 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32807 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[541, 516, 541, 496, 527, 538, 500, 501, 527, 484, 513, 484, 493, 502, 522, 533, 42, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 527, 509, 527, 518, 521, 497, 509, 543, 484, 491, 485, 531, 525, 476, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 484, 522, 503, 520, 495, 504, 506, 515, 552, 525, 521, 471, 505, 533, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 523, 504, 518, 525, 534, 525, 487, 523, 498, 527, 471, 510, 487, 507, 496, 22, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 529, 562, 482, 519, 537, 483, 507, 496, 499, 509, 507, 526, 520, 516, 551, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 503, 547, 499, 470, 539, 501, 478, 508, 510, 526, 499, 507, 482, 514, 513, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 510, 528, 459, 482, 494, 485, 484, 488, 528, 517, 523, 505, 508, 531, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 570, 464, 535, 477, 529, 527, 509, 498, 510, 506, 517, 496, 468, 512, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8276, 8215, 8215, 8192, 8277, 8127, 8098, 8136] avg=8192.0000 max=8277 max_rank=4 min=8098 min_rank=6 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32638 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[474, 484, 485, 550, 502, 504, 491, 514, 474, 534, 541, 501, 489, 498, 515, 546, 44, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 506, 541, 545, 503, 498, 528, 484, 559, 514, 509, 504, 506, 486, 537, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 507, 524, 498, 504, 497, 464, 501, 525, 509, 513, 504, 504, 484, 486, 560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 513, 506, 495, 496, 535, 508, 561, 559, 505, 515, 509, 520, 488, 481, 481, 22, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 488, 525, 528, 513, 545, 487, 502, 490, 488, 509, 485, 533, 487, 541, 492, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 517, 488, 515, 487, 505, 548, 553, 498, 542, 512, 524, 490, 533, 561, 482, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 472, 515, 510, 530, 518, 561, 517, 491, 524, 516, 547, 498, 513, 534, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 509, 487, 521, 533, 493, 510, 483, 489, 501, 510, 525, 503, 515, 529, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8278, 8050, 8193, 8128, 8280, 8316, 8127] avg=8192.0000 max=8316 max_rank=6 min=8050 min_rank=2 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32851 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[537, 492, 480, 515, 503, 509, 510, 498, 527, 509, 532, 475, 471, 509, 502, 518, 43, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 479, 531, 511, 501, 525, 496, 519, 568, 493, 529, 492, 508, 513, 497, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 482, 492, 522, 517, 509, 542, 522, 526, 541, 531, 512, 541, 524, 529, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 472, 521, 482, 495, 499, 543, 532, 508, 490, 539, 554, 525, 525, 502, 522, 23, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 512, 480, 490, 486, 508, 514, 531, 490, 505, 504, 534, 522, 512, 528, 535, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 490, 484, 551, 515, 531, 485, 488, 499, 518, 484, 545, 511, 492, 517, 524, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 514, 495, 476, 540, 510, 480, 490, 509, 479, 480, 516, 515, 523, 530, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 534, 531, 546, 520, 494, 536, 523, 538, 496, 484, 513, 522, 471, 526, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8147, 8209, 8336, 8244, 8160, 8186, 8022, 8232] avg=8192.0000 max=8336 max_rank=2 min=8022 min_rank=6 imbalance_ratio=1.017578 max_load_reduction=57200 max_load_reduction_pct=0.872803 moved_to_echo=32600 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4352, 3968, 4096, 4096, 4224, 4224, 4096, 4352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 3968, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 256, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 384, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 3968, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4352, 3968, 4096, 4096, 4224, 4224, 4096, 4352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4096, 3968, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 256, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[498, 536, 522, 552, 497, 547, 528, 473, 515, 511, 524, 495, 495, 540, 513, 530, 505, 538, 518, 485, 556, 494, 432, 529, 501, 564, 520, 537, 502, 525, 488, 481, 495, 497, 522, 513, 540, 514, 525, 500, 480, 522, 520, 540, 514, 552, 517, 529, 508, 517, 522, 505, 504, 481, 543, 510, 487, 538, 487, 481, 503, 538, 533, 528, 539, 548, 548, 491, 553, 506, 529, 498, 510, 485, 484, 554, 502, 503, 462, 528, 479, 550, 512, 485, 525, 500, 494, 505, 505, 517, 521, 511, 515, 493, 517, 493, 526, 543, 486, 511, 529, 526, 520, 493, 501, 501, 469, 549, 511, 495, 494, 480, 524, 549, 526, 484, 522, 509, 553, 458, 505, 488, 488, 513, 511, 501, 489, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[501, 560, 522, 482, 497, 543, 529, 488, 535, 516, 498, 522, 523, 497, 526, 507, 516, 502, 524, 524, 533, 489, 498, 480, 503, 486, 526, 527, 539, 521, 513, 506, 540, 495, 513, 476, 500, 480, 529, 505, 489, 528, 510, 512, 482, 488, 528, 525, 559, 520, 546, 530, 508, 533, 517, 568, 506, 459, 509, 512, 511, 504, 514, 540, 506, 553, 500, 513, 517, 533, 509, 482, 524, 513, 501, 488, 538, 507, 494, 525, 485, 498, 523, 518, 495, 534, 534, 531, 556, 481, 514, 519, 503, 517, 497, 507, 515, 502, 496, 516, 536, 511, 483, 484, 500, 481, 564, 503, 542, 482, 515, 509, 527, 492, 520, 502, 511, 494, 488, 512, 549, 508, 487, 508, 489, 516, 505, 505] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[510, 478, 544, 529, 505, 494, 516, 529, 492, 508, 474, 498, 553, 498, 509, 517, 506, 503, 550, 515, 519, 489, 482, 511, 523, 510, 523, 515, 549, 512, 504, 526, 500, 540, 538, 501, 491, 474, 491, 488, 520, 522, 488, 538, 511, 545, 480, 519, 499, 522, 489, 495, 489, 509, 477, 510, 523, 544, 488, 477, 514, 554, 489, 506, 517, 532, 526, 554, 499, 498, 532, 489, 519, 485, 492, 449, 518, 560, 495, 491, 533, 477, 520, 519, 502, 524, 482, 491, 498, 544, 519, 532, 543, 523, 513, 520, 525, 529, 535, 510, 533, 536, 474, 534, 538, 550, 467, 524, 501, 497, 540, 543, 486, 493, 512, 546, 512, 507, 555, 519, 500, 486, 485, 550, 511, 497, 510, 513] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[489, 482, 556, 516, 508, 513, 472, 502, 509, 499, 535, 547, 503, 488, 534, 524, 535, 514, 497, 528, 535, 506, 510, 511, 488, 477, 504, 525, 510, 546, 539, 542, 484, 533, 486, 514, 498, 474, 502, 521, 512, 487, 533, 533, 532, 528, 510, 503, 516, 540, 520, 489, 532, 562, 533, 506, 519, 489, 507, 503, 489, 500, 495, 518, 520, 497, 553, 552, 504, 527, 503, 562, 539, 529, 480, 493, 500, 537, 514, 466, 510, 506, 499, 488, 511, 535, 500, 482, 487, 498, 523, 506, 523, 498, 527, 506, 504, 503, 493, 490, 534, 510, 477, 486, 532, 481, 528, 498, 514, 463, 521, 520, 528, 545, 488, 495, 511, 505, 521, 536, 476, 550, 491, 529, 522, 539, 533, 526] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[522, 526, 513, 517, 486, 511, 503, 536, 516, 543, 502, 492, 498, 557, 480, 519, 498, 486, 517, 553, 516, 517, 521, 503, 480, 488, 506, 555, 549, 487, 453, 492, 549, 553, 499, 513, 456, 513, 538, 483, 512, 523, 517, 518, 531, 506, 508, 505, 488, 465, 523, 532, 525, 492, 524, 514, 520, 499, 483, 519, 528, 516, 518, 495, 491, 532, 507, 510, 561, 533, 506, 504, 484, 534, 533, 511, 484, 512, 516, 555, 526, 540, 491, 523, 498, 546, 536, 492, 501, 511, 500, 518, 533, 504, 490, 534, 523, 512, 498, 501, 515, 504, 473, 527, 523, 496, 543, 510, 510, 512, 496, 489, 498, 530, 492, 525, 491, 478, 524, 523, 492, 540, 531, 504, 506, 515, 538, 494] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[515, 510, 501, 477, 496, 524, 529, 514, 537, 543, 480, 496, 465, 512, 527, 490, 517, 567, 498, 534, 532, 519, 501, 509, 519, 507, 502, 513, 503, 514, 513, 489, 489, 508, 504, 478, 531, 505, 502, 562, 546, 510, 466, 512, 529, 529, 482, 494, 507, 513, 489, 449, 558, 515, 522, 502, 507, 502, 473, 494, 504, 525, 555, 524, 514, 520, 506, 555, 518, 522, 501, 533, 509, 542, 495, 530, 538, 516, 523, 526, 542, 503, 488, 497, 514, 481, 529, 512, 546, 528, 519, 505, 487, 564, 484, 532, 478, 503, 518, 515, 523, 508, 550, 507, 503, 512, 479, 528, 513, 503, 487, 514, 515, 507, 484, 536, 489, 529, 501, 513, 539, 509, 498, 519, 546, 517, 499, 476] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[510, 478, 544, 529, 505, 494, 516, 529, 492, 508, 474, 498, 553, 498, 509, 517, 506, 503, 550, 515, 519, 489, 482, 511, 523, 510, 523, 515, 549, 512, 504, 526, 500, 540, 538, 501, 491, 474, 491, 488, 520, 522, 488, 538, 511, 545, 480, 519, 499, 522, 489, 495, 489, 509, 477, 510, 523, 544, 488, 477, 514, 554, 489, 506, 517, 532, 526, 554, 499, 498, 532, 489, 519, 485, 492, 449, 518, 560, 495, 491, 533, 477, 520, 519, 502, 524, 482, 491, 498, 544, 519, 532, 543, 523, 513, 520, 525, 529, 535, 510, 533, 536, 474, 534, 538, 550, 467, 524, 501, 497, 540, 543, 486, 493, 512, 546, 512, 507, 555, 519, 500, 486, 485, 550, 511, 497, 510, 513] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[539, 483, 501, 538, 510, 534, 506, 454, 539, 536, 520, 529, 526, 532, 527, 508, 555, 507, 520, 525, 507, 534, 530, 536, 477, 497, 494, 534, 493, 508, 525, 505, 529, 507, 533, 525, 507, 525, 533, 554, 497, 506, 486, 537, 480, 512, 489, 469, 505, 470, 532, 531, 510, 500, 482, 536, 523, 490, 500, 532, 535, 509, 491, 531, 549, 541, 514, 500, 528, 502, 502, 535, 471, 537, 550, 496, 521, 528, 442, 512, 521, 489, 516, 528, 559, 503, 465, 494, 528, 509, 480, 561, 479, 516, 500, 522, 489, 524, 512, 509, 498, 489, 523, 522, 556, 506, 502, 544, 496, 484, 508, 511, 479, 509, 507, 534, 535, 516, 513, 483, 526, 508, 533, 520, 473, 450, 491, 493] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[522, 526, 513, 517, 486, 511, 503, 536, 516, 543, 502, 492, 498, 557, 480, 519, 498, 486, 517, 553, 516, 517, 521, 503, 480, 488, 506, 555, 549, 487, 453, 492, 549, 553, 499, 513, 456, 513, 538, 483, 512, 523, 517, 518, 531, 506, 508, 505, 488, 465, 523, 532, 525, 492, 524, 514, 520, 499, 483, 519, 528, 516, 518, 495, 491, 532, 507, 510, 561, 533, 506, 504, 484, 534, 533, 511, 484, 512, 516, 555, 526, 540, 491, 523, 498, 546, 536, 492, 501, 511, 500, 518, 533, 504, 490, 534, 523, 512, 498, 501, 515, 504, 473, 527, 523, 496, 543, 510, 510, 512, 496, 489, 498, 530, 492, 525, 491, 478, 524, 523, 492, 540, 531, 504, 506, 515, 538, 494] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[510, 482, 493, 490, 541, 508, 518, 563, 512, 516, 531, 518, 514, 505, 522, 516, 510, 486, 499, 523, 521, 520, 546, 504, 502, 482, 516, 535, 505, 522, 501, 515, 554, 487, 521, 547, 512, 518, 490, 537, 523, 519, 503, 503, 488, 450, 483, 556, 506, 526, 521, 523, 516, 509, 506, 517, 506, 492, 511, 491, 551, 495, 501, 494, 488, 520, 547, 508, 515, 541, 539, 543, 505, 553, 502, 510, 519, 495, 533, 499, 494, 505, 496, 540, 518, 474, 504, 504, 508, 484, 509, 519, 528, 538, 495, 477, 490, 535, 481, 488, 512, 497, 522, 504, 533, 510, 499, 521, 491, 550, 517, 481, 515, 535, 500, 461, 526, 499, 517, 541, 521, 516, 512, 526, 481, 485, 564, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[498, 536, 522, 552, 497, 525, 528, 473, 515, 511, 524, 495, 495, 540, 513, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 538, 518, 485, 556, 494, 432, 529, 501, 564, 520, 522, 502, 525, 488, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 497, 522, 513, 540, 514, 525, 500, 480, 522, 520, 540, 514, 552, 517, 529, 10, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 517, 522, 505, 504, 481, 543, 510, 487, 538, 487, 481, 503, 538, 533, 528, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 484, 548, 491, 553, 506, 529, 498, 510, 485, 484, 554, 502, 503, 462, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 550, 512, 485, 525, 500, 494, 505, 505, 517, 521, 511, 515, 493, 517, 493, 12, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 543, 486, 511, 529, 526, 520, 493, 501, 501, 469, 549, 511, 495, 494, 480, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 549, 526, 484, 522, 509, 553, 458, 505, 488, 488, 513, 511, 501, 489, 504, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8254, 8160, 8294, 8196, 8176, 8137, 8172, 8147] avg=8192.0000 max=8294 max_rank=2 min=8137 min_rank=5 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32632 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[498, 536, 522, 552, 497, 547, 528, 473, 515, 511, 524, 495, 495, 540, 513, 530, 505, 538, 518, 485, 556, 494, 432, 529, 501, 564, 520, 537, 502, 525, 488, 481, 495, 497, 522, 513, 540, 514, 525, 500, 480, 522, 520, 540, 514, 552, 517, 529, 508, 517, 522, 505, 504, 481, 543, 510, 487, 538, 487, 481, 503, 538, 533, 528, 539, 548, 548, 491, 553, 506, 529, 498, 510, 485, 484, 554, 502, 503, 462, 528, 479, 550, 512, 485, 525, 500, 494, 505, 505, 517, 521, 511, 515, 493, 517, 493, 526, 543, 486, 511, 529, 526, 520, 493, 501, 501, 469, 549, 511, 495, 494, 480, 524, 549, 526, 484, 522, 509, 553, 458, 505, 488, 488, 513, 511, 501, 489, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[539, 483, 501, 538, 510, 534, 506, 454, 539, 536, 520, 529, 526, 532, 527, 508, 555, 507, 520, 525, 507, 534, 530, 536, 477, 497, 494, 534, 493, 508, 525, 505, 529, 507, 533, 525, 507, 525, 533, 554, 497, 506, 486, 537, 480, 512, 489, 469, 505, 470, 532, 531, 510, 500, 482, 536, 523, 490, 500, 532, 535, 509, 491, 531, 549, 541, 514, 500, 528, 502, 502, 535, 471, 537, 550, 496, 521, 528, 442, 512, 521, 489, 516, 528, 559, 503, 465, 494, 528, 509, 480, 561, 479, 516, 500, 522, 489, 524, 512, 509, 498, 489, 523, 522, 556, 506, 502, 544, 496, 484, 508, 511, 479, 509, 507, 534, 535, 516, 513, 483, 526, 508, 533, 520, 473, 450, 491, 493] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[510, 478, 544, 529, 505, 474, 516, 529, 492, 508, 474, 498, 553, 498, 509, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 503, 550, 515, 519, 489, 482, 511, 523, 510, 523, 502, 549, 512, 504, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 540, 538, 501, 491, 474, 491, 488, 520, 522, 488, 538, 511, 545, 480, 519, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 522, 489, 495, 489, 509, 477, 510, 523, 544, 488, 477, 514, 554, 489, 506, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 470, 526, 554, 499, 498, 532, 489, 519, 485, 492, 449, 518, 560, 495, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 477, 520, 519, 502, 524, 482, 491, 498, 544, 519, 532, 543, 523, 513, 520, 11, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 529, 535, 510, 533, 536, 474, 534, 538, 550, 467, 524, 501, 497, 540, 543, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 493, 512, 546, 512, 507, 555, 519, 500, 486, 485, 550, 511, 497, 510, 513, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8134, 8224, 8158, 8095, 8094, 8254, 8373, 8204] avg=8192.0000 max=8373 max_rank=6 min=8094 min_rank=4 imbalance_ratio=1.022095 max_load_reduction=57163 max_load_reduction_pct=0.872238 moved_to_echo=32925 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[501, 560, 522, 482, 497, 521, 529, 488, 535, 516, 498, 522, 523, 497, 526, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 502, 524, 524, 533, 489, 498, 480, 503, 486, 526, 513, 539, 521, 513, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 495, 513, 476, 500, 480, 529, 505, 489, 528, 510, 512, 482, 488, 528, 525, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 520, 546, 530, 508, 533, 517, 568, 506, 459, 509, 512, 511, 504, 514, 540, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 489, 500, 513, 517, 533, 509, 482, 524, 513, 501, 488, 538, 507, 494, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 498, 523, 518, 495, 534, 534, 531, 556, 481, 514, 519, 503, 517, 497, 507, 12, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 502, 496, 516, 536, 511, 483, 484, 500, 481, 564, 503, 542, 482, 515, 509, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 492, 520, 502, 511, 494, 488, 512, 549, 508, 487, 508, 489, 516, 505, 505, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8224, 8173, 8113, 8347, 8139, 8227, 8177, 8136] avg=8192.0000 max=8347 max_rank=3 min=8113 min_rank=2 imbalance_ratio=1.018921 max_load_reduction=57189 max_load_reduction_pct=0.872635 moved_to_echo=32679 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[489, 482, 556, 516, 508, 492, 472, 502, 509, 499, 535, 547, 503, 488, 534, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 514, 497, 528, 535, 506, 510, 511, 488, 477, 504, 511, 510, 546, 539, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 533, 486, 514, 498, 474, 502, 521, 512, 487, 533, 533, 532, 528, 510, 503, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 540, 520, 489, 532, 562, 533, 506, 519, 489, 507, 503, 489, 500, 495, 518, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 439, 553, 552, 504, 527, 503, 562, 539, 529, 480, 493, 500, 537, 514, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 506, 499, 488, 511, 535, 500, 482, 487, 498, 523, 506, 523, 498, 527, 506, 12, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 503, 493, 490, 534, 510, 477, 486, 532, 481, 528, 498, 514, 463, 521, 520, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 545, 488, 495, 511, 505, 521, 536, 476, 550, 491, 529, 522, 539, 533, 526, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8156, 8253, 8162, 8229, 8218, 8114, 8088, 8316] avg=8192.0000 max=8316 max_rank=7 min=8088 min_rank=6 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32736 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[539, 483, 501, 538, 510, 505, 506, 454, 539, 536, 520, 529, 526, 532, 527, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 507, 520, 525, 507, 534, 530, 536, 477, 497, 494, 511, 493, 508, 525, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 507, 533, 525, 507, 525, 533, 554, 497, 506, 486, 537, 480, 512, 489, 469, 12, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 470, 532, 531, 510, 500, 482, 536, 523, 490, 500, 532, 535, 509, 491, 531, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 468, 514, 500, 528, 502, 502, 535, 471, 537, 550, 496, 521, 528, 442, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 489, 516, 528, 559, 503, 465, 494, 528, 509, 480, 561, 479, 516, 500, 522, 17, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 524, 512, 509, 498, 489, 523, 522, 556, 506, 502, 544, 496, 484, 508, 511, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 509, 507, 534, 535, 516, 513, 483, 526, 508, 533, 520, 473, 450, 491, 493, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8253, 8224, 8209, 8192, 8155, 8194, 8213, 8096] avg=8192.0000 max=8253 max_rank=0 min=8096 min_rank=7 imbalance_ratio=1.007446 max_load_reduction=57283 max_load_reduction_pct=0.874069 moved_to_echo=32658 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[522, 526, 513, 517, 486, 491, 503, 536, 516, 543, 502, 492, 498, 557, 480, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 486, 517, 553, 516, 517, 521, 503, 480, 488, 506, 540, 549, 487, 453, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 553, 499, 513, 456, 513, 538, 483, 512, 523, 517, 518, 531, 506, 508, 505, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 465, 523, 532, 525, 492, 524, 514, 520, 499, 483, 519, 528, 516, 518, 495, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 470, 507, 510, 561, 533, 506, 504, 484, 534, 533, 511, 484, 512, 516, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 540, 491, 523, 498, 546, 536, 492, 501, 511, 500, 518, 533, 504, 490, 534, 11, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 512, 498, 501, 515, 504, 473, 527, 523, 496, 543, 510, 510, 512, 496, 489, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 530, 492, 525, 491, 478, 524, 523, 492, 540, 531, 504, 506, 515, 538, 494, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8106, 8237, 8152, 8211, 8257, 8169, 8203] avg=8192.0000 max=8257 max_rank=5 min=8106 min_rank=1 imbalance_ratio=1.007935 max_load_reduction=57279 max_load_reduction_pct=0.874008 moved_to_echo=32840 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[515, 510, 501, 477, 496, 503, 529, 514, 537, 543, 480, 496, 465, 512, 527, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 567, 498, 534, 532, 519, 501, 509, 519, 507, 502, 500, 503, 514, 513, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 508, 504, 478, 531, 505, 502, 562, 546, 510, 466, 512, 529, 529, 482, 494, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 513, 489, 449, 558, 515, 522, 502, 507, 502, 473, 494, 504, 525, 555, 524, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 459, 506, 555, 518, 522, 501, 533, 509, 542, 495, 530, 538, 516, 523, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 503, 488, 497, 514, 481, 529, 512, 546, 528, 519, 505, 487, 564, 484, 532, 12, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 503, 518, 515, 523, 508, 550, 507, 503, 512, 479, 528, 513, 503, 487, 514, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 507, 484, 536, 489, 529, 501, 513, 539, 509, 498, 519, 546, 517, 499, 476, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8095, 8224, 8159, 8149, 8287, 8246, 8177, 8199] avg=8192.0000 max=8287 max_rank=4 min=8095 min_rank=0 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32909 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[539, 483, 501, 538, 510, 505, 506, 454, 539, 536, 520, 529, 526, 532, 527, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 507, 520, 525, 507, 534, 530, 536, 477, 497, 494, 511, 493, 508, 525, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 507, 533, 525, 507, 525, 533, 554, 497, 506, 486, 537, 480, 512, 489, 469, 12, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 470, 532, 531, 510, 500, 482, 536, 523, 490, 500, 532, 535, 509, 491, 531, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 468, 514, 500, 528, 502, 502, 535, 471, 537, 550, 496, 521, 528, 442, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 489, 516, 528, 559, 503, 465, 494, 528, 509, 480, 561, 479, 516, 500, 522, 17, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 524, 512, 509, 498, 489, 523, 522, 556, 506, 502, 544, 496, 484, 508, 511, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 509, 507, 534, 535, 516, 513, 483, 526, 508, 533, 520, 473, 450, 491, 493, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8253, 8224, 8209, 8192, 8155, 8194, 8213, 8096] avg=8192.0000 max=8253 max_rank=0 min=8096 min_rank=7 imbalance_ratio=1.007446 max_load_reduction=57283 max_load_reduction_pct=0.874069 moved_to_echo=32658 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[510, 482, 493, 490, 541, 488, 518, 563, 512, 516, 531, 518, 514, 505, 522, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 486, 499, 523, 521, 520, 546, 504, 502, 482, 516, 520, 505, 522, 501, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 487, 521, 547, 512, 518, 490, 537, 523, 519, 503, 503, 488, 450, 483, 556, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 526, 521, 523, 516, 509, 506, 517, 506, 492, 511, 491, 551, 495, 501, 494, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 459, 547, 508, 515, 541, 539, 543, 505, 553, 502, 510, 519, 495, 533, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 505, 496, 540, 518, 474, 504, 504, 508, 484, 509, 519, 528, 538, 495, 477, 11, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 535, 481, 488, 512, 497, 522, 504, 533, 510, 499, 521, 491, 550, 517, 481, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 535, 500, 461, 526, 499, 517, 541, 521, 516, 512, 526, 481, 485, 564, 514, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 8172, 8204, 8176, 8256, 8107, 8167, 8235] avg=8192.0000 max=8256 max_rank=4 min=8107 min_rank=5 imbalance_ratio=1.007812 max_load_reduction=57280 max_load_reduction_pct=0.874023 moved_to_echo=32765 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[522, 526, 513, 517, 486, 491, 503, 536, 516, 543, 502, 492, 498, 557, 480, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 486, 517, 553, 516, 517, 521, 503, 480, 488, 506, 540, 549, 487, 453, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 553, 499, 513, 456, 513, 538, 483, 512, 523, 517, 518, 531, 506, 508, 505, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 465, 523, 532, 525, 492, 524, 514, 520, 499, 483, 519, 528, 516, 518, 495, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 470, 507, 510, 561, 533, 506, 504, 484, 534, 533, 511, 484, 512, 516, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 540, 491, 523, 498, 546, 536, 492, 501, 511, 500, 518, 533, 504, 490, 534, 11, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 512, 498, 501, 515, 504, 473, 527, 523, 496, 543, 510, 510, 512, 496, 489, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 530, 492, 525, 491, 478, 524, 523, 492, 540, 531, 504, 506, 515, 538, 494, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8106, 8237, 8152, 8211, 8257, 8169, 8203] avg=8192.0000 max=8257 max_rank=5 min=8106 min_rank=1 imbalance_ratio=1.007935 max_load_reduction=57279 max_load_reduction_pct=0.874008 moved_to_echo=32840 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[498, 536, 522, 552, 497, 525, 528, 473, 515, 511, 524, 495, 495, 540, 513, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 538, 518, 485, 556, 494, 432, 529, 501, 564, 520, 522, 502, 525, 488, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 497, 522, 513, 540, 514, 525, 500, 480, 522, 520, 540, 514, 552, 517, 529, 10, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 517, 522, 505, 504, 481, 543, 510, 487, 538, 487, 481, 503, 538, 533, 528, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 484, 548, 491, 553, 506, 529, 498, 510, 485, 484, 554, 502, 503, 462, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 550, 512, 485, 525, 500, 494, 505, 505, 517, 521, 511, 515, 493, 517, 493, 12, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 543, 486, 511, 529, 526, 520, 493, 501, 501, 469, 549, 511, 495, 494, 480, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 549, 526, 484, 522, 509, 553, 458, 505, 488, 488, 513, 511, 501, 489, 504, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8254, 8160, 8294, 8196, 8176, 8137, 8172, 8147] avg=8192.0000 max=8294 max_rank=2 min=8137 min_rank=5 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32632 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 3968, 4096, 4224, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 3968, 4096, 4224, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 3840, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[520, 563, 485, 485, 555, 511, 491, 483, 489, 522, 507, 523, 521, 516, 489, 498, 503, 509, 525, 545, 523, 492, 533, 431, 484, 494, 483, 548, 537, 492, 562, 481, 522, 473, 507, 515, 517, 525, 519, 487, 558, 493, 549, 526, 498, 564, 528, 473, 508, 496, 534, 518, 489, 557, 480, 517, 510, 524, 517, 514, 514, 501, 546, 491, 524, 494, 461, 525, 474, 529, 469, 514, 536, 532, 514, 505, 539, 523, 512, 514, 489, 505, 473, 525, 473, 504, 480, 524, 557, 542, 525, 510, 508, 506, 486, 513, 496, 510, 526, 490, 506, 532, 478, 502, 528, 491, 505, 506, 499, 530, 519, 537, 494, 544, 531, 535, 519, 496, 492, 539, 527, 546, 512, 503, 513, 532, 517, 526] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[522, 513, 470, 479, 502, 502, 504, 494, 502, 513, 498, 523, 519, 524, 494, 476, 488, 503, 546, 516, 513, 494, 549, 531, 528, 513, 477, 557, 569, 499, 502, 481, 507, 557, 513, 484, 526, 459, 521, 522, 534, 505, 498, 482, 543, 490, 517, 536, 502, 516, 560, 502, 511, 556, 547, 515, 511, 528, 483, 499, 548, 483, 532, 518, 496, 495, 526, 542, 507, 513, 477, 499, 490, 563, 539, 502, 565, 514, 496, 551, 499, 473, 526, 485, 506, 504, 510, 472, 528, 524, 491, 505, 523, 524, 544, 513, 503, 487, 519, 475, 498, 503, 539, 503, 512, 513, 498, 533, 514, 503, 520, 529, 490, 561, 516, 528, 476, 513, 494, 526, 479, 522, 497, 530, 516, 530, 473, 528] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[548, 508, 529, 480, 549, 520, 516, 516, 534, 485, 497, 493, 503, 519, 521, 520, 473, 482, 516, 510, 477, 516, 529, 516, 501, 492, 522, 541, 535, 550, 517, 503, 463, 516, 531, 529, 535, 494, 528, 476, 509, 520, 501, 528, 504, 526, 505, 519, 513, 512, 511, 536, 493, 497, 531, 549, 477, 474, 534, 512, 498, 515, 492, 484, 461, 505, 461, 508, 546, 480, 551, 499, 509, 505, 560, 478, 480, 477, 506, 534, 508, 517, 494, 537, 522, 512, 546, 514, 485, 517, 522, 503, 494, 469, 513, 475, 534, 519, 480, 509, 587, 481, 545, 488, 550, 512, 501, 516, 520, 511, 488, 523, 530, 570, 496, 494, 508, 503, 521, 503, 504, 489, 577, 501, 557, 549, 519, 533] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[530, 466, 487, 474, 499, 529, 498, 479, 520, 530, 502, 498, 555, 514, 495, 522, 498, 494, 483, 492, 505, 498, 516, 509, 504, 511, 496, 543, 548, 540, 505, 541, 456, 517, 517, 527, 494, 520, 520, 514, 547, 530, 478, 521, 511, 503, 478, 504, 490, 519, 517, 515, 543, 510, 510, 512, 485, 501, 547, 516, 511, 542, 535, 495, 528, 513, 527, 498, 462, 531, 529, 514, 516, 486, 510, 535, 537, 539, 528, 526, 523, 526, 504, 499, 527, 557, 544, 550, 522, 500, 470, 508, 523, 458, 541, 529, 539, 509, 496, 576, 493, 523, 528, 503, 516, 515, 493, 517, 544, 530, 521, 507, 519, 543, 513, 482, 484, 501, 493, 472, 510, 507, 526, 486, 512, 445, 509, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[521, 490, 562, 497, 534, 477, 507, 476, 532, 505, 498, 520, 528, 518, 546, 504, 500, 504, 495, 527, 579, 475, 467, 502, 541, 494, 540, 530, 481, 532, 479, 526, 535, 544, 489, 524, 489, 535, 513, 491, 525, 540, 512, 489, 492, 492, 496, 519, 542, 497, 554, 519, 504, 555, 498, 525, 515, 502, 515, 516, 545, 493, 503, 531, 500, 507, 499, 471, 538, 544, 547, 506, 445, 495, 526, 482, 554, 476, 549, 522, 473, 505, 512, 496, 492, 523, 496, 528, 524, 511, 536, 508, 475, 507, 548, 489, 517, 468, 518, 507, 478, 472, 552, 494, 542, 497, 482, 495, 507, 548, 508, 497, 520, 507, 567, 520, 542, 512, 509, 526, 529, 502, 499, 510, 488, 501, 517, 535] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[518, 493, 503, 519, 517, 539, 510, 496, 500, 515, 522, 495, 554, 520, 482, 488, 491, 505, 486, 553, 551, 466, 546, 493, 493, 530, 485, 512, 512, 546, 521, 495, 528, 513, 474, 525, 517, 537, 517, 525, 520, 550, 506, 525, 478, 497, 488, 554, 531, 469, 510, 485, 513, 523, 503, 529, 499, 536, 532, 517, 492, 524, 526, 507, 499, 537, 495, 531, 532, 533, 524, 541, 510, 517, 469, 498, 530, 482, 516, 497, 546, 498, 485, 516, 490, 496, 536, 474, 538, 535, 495, 454, 496, 518, 475, 527, 568, 509, 506, 537, 510, 515, 502, 513, 494, 504, 514, 501, 480, 504, 553, 516, 501, 520, 510, 521, 500, 507, 521, 515, 520, 538, 566, 502, 475, 504, 493, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[516, 503, 496, 498, 533, 484, 528, 547, 519, 489, 516, 537, 500, 528, 528, 497, 515, 493, 524, 524, 519, 560, 516, 485, 497, 500, 515, 525, 510, 498, 479, 493, 560, 514, 514, 517, 517, 469, 528, 468, 515, 505, 539, 532, 512, 541, 536, 454, 512, 520, 505, 532, 510, 475, 519, 477, 472, 520, 512, 546, 516, 549, 507, 527, 462, 495, 555, 558, 493, 512, 512, 475, 502, 501, 560, 486, 533, 529, 515, 507, 511, 513, 525, 524, 483, 502, 509, 503, 544, 528, 502, 508, 491, 494, 521, 538, 509, 527, 498, 525, 485, 489, 532, 507, 545, 515, 478, 541, 521, 476, 538, 514, 507, 485, 541, 520, 496, 509, 497, 496, 501, 527, 511, 515, 498, 534, 516, 500] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[477, 513, 510, 525, 482, 487, 550, 528, 502, 487, 487, 534, 471, 567, 512, 496, 525, 495, 539, 488, 493, 516, 527, 469, 500, 496, 528, 520, 508, 496, 521, 493, 496, 494, 545, 509, 551, 528, 526, 516, 504, 454, 524, 485, 543, 496, 501, 504, 548, 520, 567, 542, 487, 502, 534, 508, 487, 530, 492, 530, 506, 516, 510, 474, 529, 491, 534, 512, 515, 521, 552, 549, 518, 487, 481, 534, 521, 517, 495, 502, 519, 555, 499, 506, 517, 527, 540, 518, 499, 515, 521, 489, 466, 534, 529, 481, 513, 546, 529, 494, 488, 529, 472, 502, 509, 503, 501, 491, 527, 522, 538, 505, 506, 518, 512, 533, 517, 475, 524, 504, 507, 547, 523, 490, 531, 509, 488, 539] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[520, 563, 485, 485, 555, 511, 491, 483, 489, 522, 507, 523, 521, 516, 489, 498, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 509, 525, 545, 523, 492, 533, 431, 484, 494, 483, 548, 537, 492, 562, 481, 8, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 473, 507, 515, 517, 525, 519, 487, 550, 493, 549, 526, 498, 564, 528, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 496, 494, 518, 489, 557, 480, 517, 510, 524, 517, 514, 514, 501, 546, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 494, 461, 525, 474, 529, 469, 514, 536, 532, 514, 505, 531, 523, 512, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 505, 473, 525, 473, 504, 480, 524, 557, 542, 525, 510, 508, 506, 486, 513, 25, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 510, 526, 490, 506, 532, 478, 502, 526, 491, 505, 506, 499, 530, 519, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 519, 531, 535, 519, 496, 492, 539, 527, 546, 512, 503, 513, 532, 517, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8159, 8246, 8176, 8157, 8152, 8153, 8301] avg=8192.0000 max=8301 max_rank=7 min=8152 min_rank=5 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32763 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[522, 513, 470, 479, 502, 502, 504, 494, 502, 513, 498, 523, 519, 524, 494, 476, 488, 503, 546, 516, 513, 494, 549, 531, 528, 513, 477, 557, 569, 499, 502, 481, 507, 557, 513, 484, 526, 459, 521, 522, 534, 505, 498, 482, 543, 490, 517, 536, 502, 516, 560, 502, 511, 556, 547, 515, 511, 528, 483, 499, 548, 483, 532, 518, 496, 495, 526, 542, 507, 513, 477, 499, 490, 563, 539, 502, 565, 514, 496, 551, 499, 473, 526, 485, 506, 504, 510, 472, 528, 524, 491, 505, 523, 524, 544, 513, 503, 487, 519, 475, 498, 503, 539, 503, 512, 513, 498, 533, 514, 503, 520, 529, 490, 561, 516, 528, 476, 513, 494, 526, 479, 522, 497, 530, 516, 530, 473, 528] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[530, 466, 487, 474, 499, 529, 498, 479, 520, 530, 502, 498, 555, 514, 495, 522, 498, 494, 483, 492, 505, 498, 516, 509, 504, 511, 496, 543, 548, 540, 505, 541, 456, 517, 517, 527, 494, 520, 520, 514, 547, 530, 478, 521, 511, 503, 478, 504, 490, 519, 517, 515, 543, 510, 510, 512, 485, 501, 547, 516, 511, 542, 535, 495, 528, 513, 527, 498, 462, 531, 529, 514, 516, 486, 510, 535, 537, 539, 528, 526, 523, 526, 504, 499, 527, 557, 544, 550, 522, 500, 470, 508, 523, 458, 541, 529, 539, 509, 496, 576, 493, 523, 528, 503, 516, 515, 493, 517, 544, 530, 521, 507, 519, 543, 513, 482, 484, 501, 493, 472, 510, 507, 526, 486, 512, 445, 509, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[520, 563, 485, 485, 555, 511, 491, 483, 489, 522, 507, 523, 521, 516, 489, 498, 503, 509, 525, 545, 523, 492, 533, 431, 484, 494, 483, 548, 537, 492, 562, 481, 522, 473, 507, 515, 517, 525, 519, 487, 558, 493, 549, 526, 498, 564, 528, 473, 508, 496, 534, 518, 489, 557, 480, 517, 510, 524, 517, 514, 514, 501, 546, 491, 524, 494, 461, 525, 474, 529, 469, 514, 536, 532, 514, 505, 539, 523, 512, 514, 489, 505, 473, 525, 473, 504, 480, 524, 557, 542, 525, 510, 508, 506, 486, 513, 496, 510, 526, 490, 506, 532, 478, 502, 528, 491, 505, 506, 499, 530, 519, 537, 494, 544, 531, 535, 519, 496, 492, 539, 527, 546, 512, 503, 513, 532, 517, 526] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[548, 508, 529, 480, 549, 520, 516, 516, 534, 485, 497, 493, 503, 519, 521, 520, 473, 482, 516, 510, 477, 516, 529, 516, 501, 492, 522, 541, 535, 550, 517, 503, 463, 516, 531, 529, 535, 494, 528, 476, 509, 520, 501, 528, 504, 526, 505, 519, 513, 512, 511, 536, 493, 497, 531, 549, 477, 474, 534, 512, 498, 515, 492, 484, 461, 505, 461, 508, 546, 480, 551, 499, 509, 505, 560, 478, 480, 477, 506, 534, 508, 517, 494, 537, 522, 512, 546, 514, 485, 517, 522, 503, 494, 469, 513, 475, 534, 519, 480, 509, 587, 481, 545, 488, 550, 512, 501, 516, 520, 511, 488, 523, 530, 570, 496, 494, 508, 503, 521, 503, 504, 489, 577, 501, 557, 549, 519, 533] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[518, 493, 503, 519, 517, 539, 510, 496, 500, 515, 522, 495, 554, 520, 482, 488, 491, 505, 486, 553, 551, 466, 546, 493, 493, 530, 485, 512, 512, 546, 521, 495, 528, 513, 474, 525, 517, 537, 517, 525, 520, 550, 506, 525, 478, 497, 488, 554, 531, 469, 510, 485, 513, 523, 503, 529, 499, 536, 532, 517, 492, 524, 526, 507, 499, 537, 495, 531, 532, 533, 524, 541, 510, 517, 469, 498, 530, 482, 516, 497, 546, 498, 485, 516, 490, 496, 536, 474, 538, 535, 495, 454, 496, 518, 475, 527, 568, 509, 506, 537, 510, 515, 502, 513, 494, 504, 514, 501, 480, 504, 553, 516, 501, 520, 510, 521, 500, 507, 521, 515, 520, 538, 566, 502, 475, 504, 493, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[521, 490, 562, 497, 534, 477, 507, 476, 532, 505, 498, 520, 528, 518, 546, 504, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 504, 495, 527, 579, 475, 467, 502, 541, 494, 540, 530, 481, 532, 479, 526, 8, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 544, 489, 524, 489, 535, 513, 491, 518, 540, 512, 489, 492, 492, 496, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 497, 512, 519, 504, 555, 498, 525, 515, 502, 515, 516, 545, 493, 503, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 507, 499, 471, 538, 544, 547, 506, 445, 495, 526, 482, 546, 476, 549, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 505, 512, 496, 492, 523, 496, 528, 524, 511, 536, 508, 475, 507, 548, 489, 23, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 468, 518, 507, 478, 472, 552, 494, 540, 497, 482, 495, 507, 548, 508, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 484, 567, 520, 542, 512, 509, 526, 529, 502, 499, 510, 488, 501, 517, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8250, 8188, 8178, 8272, 8153, 8154, 8080, 8261] avg=8192.0000 max=8272 max_rank=3 min=8080 min_rank=6 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32648 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[530, 466, 487, 474, 499, 529, 498, 479, 520, 530, 502, 498, 555, 514, 495, 522, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 494, 483, 492, 505, 498, 516, 509, 504, 511, 496, 543, 548, 540, 505, 541, 8, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 456, 517, 517, 527, 494, 520, 520, 514, 539, 530, 478, 521, 511, 503, 478, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 519, 478, 515, 543, 510, 510, 512, 485, 501, 547, 516, 511, 542, 535, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 513, 527, 498, 462, 531, 529, 514, 516, 486, 510, 535, 529, 539, 528, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 526, 504, 499, 527, 557, 544, 550, 522, 500, 470, 508, 523, 458, 541, 529, 25, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 509, 496, 576, 493, 523, 528, 503, 514, 515, 493, 517, 544, 530, 521, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 518, 513, 482, 484, 501, 493, 472, 510, 507, 526, 486, 512, 445, 509, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8131, 8200, 8129, 8209, 8271, 8313, 8308, 7975] avg=8192.0000 max=8313 max_rank=5 min=7975 min_rank=7 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32867 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[522, 513, 470, 479, 502, 502, 504, 494, 502, 513, 498, 523, 519, 524, 494, 476, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 503, 546, 516, 513, 494, 549, 531, 528, 513, 477, 557, 569, 499, 502, 481, 13, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 557, 513, 484, 526, 459, 521, 522, 516, 505, 498, 482, 543, 490, 517, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 516, 509, 502, 511, 556, 547, 515, 511, 528, 483, 499, 548, 483, 532, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 495, 526, 542, 507, 513, 477, 499, 490, 563, 539, 502, 552, 514, 496, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 473, 526, 485, 506, 504, 510, 472, 528, 524, 491, 505, 523, 524, 544, 513, 28, 11, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 487, 519, 475, 498, 503, 539, 503, 507, 513, 498, 533, 514, 503, 520, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 533, 516, 528, 476, 513, 494, 526, 479, 522, 497, 530, 516, 530, 473, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8075, 8294, 8176, 8260, 8262, 8174, 8144, 8151] avg=8192.0000 max=8294 max_rank=1 min=8075 min_rank=0 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32731 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[518, 493, 503, 519, 517, 539, 510, 496, 500, 515, 522, 495, 554, 520, 482, 488, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 505, 486, 553, 551, 466, 546, 493, 493, 530, 485, 512, 512, 546, 521, 495, 8, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 513, 474, 525, 517, 537, 517, 525, 513, 550, 506, 525, 478, 497, 488, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 469, 472, 485, 513, 523, 503, 529, 499, 536, 532, 517, 492, 524, 526, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 537, 495, 531, 532, 533, 524, 541, 510, 517, 469, 498, 522, 482, 516, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 498, 485, 516, 490, 496, 536, 474, 538, 535, 495, 454, 496, 518, 475, 527, 24, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 568, 509, 506, 537, 510, 515, 502, 513, 492, 504, 514, 501, 480, 504, 553, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 496, 510, 521, 500, 507, 521, 515, 520, 538, 566, 502, 475, 504, 493, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8201, 8247, 8158, 8203, 8110, 8224, 8190] avg=8192.0000 max=8247 max_rank=2 min=8110 min_rank=5 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32727 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[548, 508, 529, 480, 549, 520, 516, 516, 534, 485, 497, 493, 503, 519, 521, 520, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 482, 516, 510, 477, 516, 529, 516, 501, 492, 522, 541, 535, 550, 517, 503, 7, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 463, 516, 531, 529, 535, 494, 528, 476, 502, 520, 501, 528, 504, 526, 505, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 512, 473, 536, 493, 497, 531, 549, 477, 474, 534, 512, 498, 515, 492, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 505, 461, 508, 546, 480, 551, 499, 509, 505, 560, 478, 473, 477, 506, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 517, 494, 537, 522, 512, 546, 514, 485, 517, 522, 503, 494, 469, 513, 475, 26, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 519, 480, 509, 587, 481, 545, 488, 548, 512, 501, 516, 520, 511, 488, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 544, 496, 494, 508, 503, 521, 503, 504, 489, 577, 501, 557, 549, 519, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8270, 8195, 8177, 8090, 8053, 8161, 8262, 8328] avg=8192.0000 max=8328 max_rank=7 min=8053 min_rank=4 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=32804 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[521, 490, 562, 497, 534, 477, 507, 476, 532, 505, 498, 520, 528, 518, 546, 504, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 504, 495, 527, 579, 475, 467, 502, 541, 494, 540, 530, 481, 532, 479, 526, 8, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 544, 489, 524, 489, 535, 513, 491, 518, 540, 512, 489, 492, 492, 496, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 497, 512, 519, 504, 555, 498, 525, 515, 502, 515, 516, 545, 493, 503, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 507, 499, 471, 538, 544, 547, 506, 445, 495, 526, 482, 546, 476, 549, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 505, 512, 496, 492, 523, 496, 528, 524, 511, 536, 508, 475, 507, 548, 489, 23, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 468, 518, 507, 478, 472, 552, 494, 540, 497, 482, 495, 507, 548, 508, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 484, 567, 520, 542, 512, 509, 526, 529, 502, 499, 510, 488, 501, 517, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8250, 8188, 8178, 8272, 8153, 8154, 8080, 8261] avg=8192.0000 max=8272 max_rank=3 min=8080 min_rank=6 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32648 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[516, 503, 496, 498, 533, 484, 528, 547, 519, 489, 516, 537, 500, 528, 528, 497, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 493, 524, 524, 519, 560, 516, 485, 497, 500, 515, 525, 510, 498, 479, 493, 8, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 560, 514, 514, 517, 517, 469, 528, 468, 508, 505, 539, 532, 512, 541, 536, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 520, 467, 532, 510, 475, 519, 477, 472, 520, 512, 546, 516, 549, 507, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 495, 555, 558, 493, 512, 512, 475, 502, 501, 560, 486, 525, 529, 515, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 513, 525, 524, 483, 502, 509, 503, 544, 528, 502, 508, 491, 494, 521, 538, 22, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 527, 498, 525, 485, 489, 532, 507, 543, 515, 478, 541, 521, 476, 538, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 463, 541, 520, 496, 509, 497, 496, 501, 527, 511, 515, 498, 534, 516, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8251, 8169, 8214, 8161, 8187, 8225, 8198, 8131] avg=8192.0000 max=8251 max_rank=0 min=8131 min_rank=7 imbalance_ratio=1.007202 max_load_reduction=57285 max_load_reduction_pct=0.874100 moved_to_echo=32741 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[518, 493, 503, 519, 517, 539, 510, 496, 500, 515, 522, 495, 554, 520, 482, 488, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 505, 486, 553, 551, 466, 546, 493, 493, 530, 485, 512, 512, 546, 521, 495, 8, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 513, 474, 525, 517, 537, 517, 525, 513, 550, 506, 525, 478, 497, 488, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 469, 472, 485, 513, 523, 503, 529, 499, 536, 532, 517, 492, 524, 526, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 537, 495, 531, 532, 533, 524, 541, 510, 517, 469, 498, 522, 482, 516, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 498, 485, 516, 490, 496, 536, 474, 538, 535, 495, 454, 496, 518, 475, 527, 24, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 568, 509, 506, 537, 510, 515, 502, 513, 492, 504, 514, 501, 480, 504, 553, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 496, 510, 521, 500, 507, 521, 515, 520, 538, 566, 502, 475, 504, 493, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8201, 8247, 8158, 8203, 8110, 8224, 8190] avg=8192.0000 max=8247 max_rank=2 min=8110 min_rank=5 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32727 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[477, 513, 510, 525, 482, 487, 550, 528, 502, 487, 487, 534, 471, 567, 512, 496, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 495, 539, 488, 493, 516, 527, 469, 500, 496, 528, 520, 508, 496, 521, 493, 8, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 494, 545, 509, 551, 528, 526, 516, 497, 454, 524, 485, 543, 496, 501, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 520, 524, 542, 487, 502, 534, 508, 487, 530, 492, 530, 506, 516, 510, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 491, 534, 512, 515, 521, 552, 549, 518, 487, 481, 534, 513, 517, 495, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 555, 499, 506, 517, 527, 540, 518, 499, 515, 521, 489, 466, 534, 529, 481, 24, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 546, 529, 494, 488, 529, 472, 502, 507, 503, 501, 491, 527, 522, 538, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 494, 512, 533, 517, 475, 524, 504, 507, 547, 523, 490, 531, 509, 488, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8130, 8169, 8210, 8250, 8247, 8167, 8199] avg=8192.0000 max=8250 max_rank=4 min=8130 min_rank=1 imbalance_ratio=1.007080 max_load_reduction=57286 max_load_reduction_pct=0.874115 moved_to_echo=32863 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[520, 563, 485, 485, 555, 511, 491, 483, 489, 522, 507, 523, 521, 516, 489, 498, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 509, 525, 545, 523, 492, 533, 431, 484, 494, 483, 548, 537, 492, 562, 481, 8, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 473, 507, 515, 517, 525, 519, 487, 550, 493, 549, 526, 498, 564, 528, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 496, 494, 518, 489, 557, 480, 517, 510, 524, 517, 514, 514, 501, 546, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 494, 461, 525, 474, 529, 469, 514, 536, 532, 514, 505, 531, 523, 512, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 505, 473, 525, 473, 504, 480, 524, 557, 542, 525, 510, 508, 506, 486, 513, 25, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 510, 526, 490, 506, 532, 478, 502, 526, 491, 505, 506, 499, 530, 519, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 519, 531, 535, 519, 496, 492, 539, 527, 546, 512, 503, 513, 532, 517, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8159, 8246, 8176, 8157, 8152, 8153, 8301] avg=8192.0000 max=8301 max_rank=7 min=8152 min_rank=5 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32763 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[548, 508, 529, 480, 549, 520, 516, 516, 534, 485, 497, 493, 503, 519, 521, 520, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 482, 516, 510, 477, 516, 529, 516, 501, 492, 522, 541, 535, 550, 517, 503, 7, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 463, 516, 531, 529, 535, 494, 528, 476, 502, 520, 501, 528, 504, 526, 505, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 512, 473, 536, 493, 497, 531, 549, 477, 474, 534, 512, 498, 515, 492, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 505, 461, 508, 546, 480, 551, 499, 509, 505, 560, 478, 473, 477, 506, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 517, 494, 537, 522, 512, 546, 514, 485, 517, 522, 503, 494, 469, 513, 475, 26, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 519, 480, 509, 587, 481, 545, 488, 548, 512, 501, 516, 520, 511, 488, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 544, 496, 494, 508, 503, 521, 503, 504, 489, 577, 501, 557, 549, 519, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8270, 8195, 8177, 8090, 8053, 8161, 8262, 8328] avg=8192.0000 max=8328 max_rank=7 min=8053 min_rank=4 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=32804 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 3968, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 3968, 4224, 4096, 4224, 4224, 4224, 3968, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4224, 3968, 4096, 4096, 4096, 4352, 4224, 4224, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 3968, 4224, 4096, 4224, 4224, 4224, 3968, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4224, 3968, 4096, 4096, 4096, 4352, 4224, 4224, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[474, 532, 539, 552, 544, 503, 506, 532, 498, 554, 533, 479, 498, 536, 556, 490, 497, 540, 468, 456, 520, 505, 489, 518, 520, 468, 524, 501, 495, 511, 492, 519, 523, 534, 508, 479, 486, 495, 506, 491, 485, 471, 490, 523, 515, 508, 527, 519, 516, 493, 531, 515, 546, 502, 509, 502, 492, 531, 571, 512, 519, 489, 514, 485, 536, 489, 490, 493, 542, 534, 523, 502, 501, 513, 535, 511, 535, 488, 473, 496, 514, 492, 482, 517, 514, 512, 550, 520, 539, 541, 511, 548, 500, 480, 533, 526, 528, 521, 481, 521, 522, 493, 512, 517, 523, 504, 540, 515, 483, 524, 524, 498, 526, 501, 531, 527, 521, 484, 504, 537, 529, 511, 503, 523, 507, 526, 520, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[533, 549, 535, 516, 499, 517, 508, 513, 542, 530, 521, 513, 516, 483, 555, 498, 504, 495, 500, 505, 524, 517, 540, 525, 533, 538, 516, 519, 532, 514, 496, 494, 534, 510, 515, 513, 511, 529, 545, 541, 510, 477, 494, 526, 504, 510, 531, 490, 547, 527, 538, 487, 490, 495, 513, 514, 562, 498, 494, 509, 538, 486, 499, 490, 581, 523, 464, 500, 471, 509, 502, 522, 523, 533, 491, 468, 492, 532, 508, 476, 504, 532, 505, 478, 496, 541, 482, 456, 449, 512, 483, 490, 537, 513, 484, 503, 479, 513, 461, 531, 522, 543, 521, 543, 549, 488, 485, 506, 539, 545, 531, 516, 532, 519, 529, 508, 490, 520, 517, 493, 551, 517, 526, 499, 497, 501, 476, 522] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[474, 532, 539, 552, 544, 503, 506, 532, 498, 554, 533, 479, 498, 536, 556, 490, 497, 540, 468, 456, 520, 505, 489, 518, 520, 468, 524, 501, 495, 511, 492, 519, 523, 534, 508, 479, 486, 495, 506, 491, 485, 471, 490, 523, 515, 508, 527, 519, 516, 493, 531, 515, 546, 502, 509, 502, 492, 531, 571, 512, 519, 489, 514, 485, 536, 489, 490, 493, 542, 534, 523, 502, 501, 513, 535, 511, 535, 488, 473, 496, 514, 492, 482, 517, 514, 512, 550, 520, 539, 541, 511, 548, 500, 480, 533, 526, 528, 521, 481, 521, 522, 493, 512, 517, 523, 504, 540, 515, 483, 524, 524, 498, 526, 501, 531, 527, 521, 484, 504, 537, 529, 511, 503, 523, 507, 526, 520, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[540, 510, 474, 457, 463, 532, 522, 456, 516, 515, 516, 518, 551, 487, 525, 515, 483, 543, 505, 573, 523, 511, 517, 508, 527, 533, 527, 515, 484, 518, 462, 534, 484, 495, 524, 481, 517, 540, 523, 502, 518, 502, 522, 509, 520, 533, 508, 506, 521, 544, 505, 519, 555, 543, 553, 519, 514, 499, 503, 547, 518, 493, 547, 481, 511, 515, 533, 543, 501, 540, 488, 445, 499, 534, 465, 492, 500, 489, 474, 494, 491, 552, 515, 530, 541, 497, 511, 484, 489, 505, 519, 501, 509, 486, 520, 534, 511, 526, 497, 495, 519, 544, 507, 529, 482, 504, 522, 495, 535, 496, 531, 502, 545, 503, 520, 503, 480, 500, 523, 539, 521, 523, 532, 477, 566, 496, 487, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[523, 496, 473, 481, 483, 485, 528, 473, 507, 510, 485, 511, 512, 488, 518, 446, 515, 511, 524, 493, 521, 497, 509, 466, 485, 540, 498, 521, 508, 551, 531, 488, 507, 534, 469, 522, 519, 497, 493, 536, 537, 528, 512, 511, 515, 511, 527, 489, 532, 525, 523, 511, 546, 509, 518, 497, 503, 522, 495, 517, 503, 535, 525, 507, 491, 513, 518, 490, 526, 517, 503, 498, 501, 520, 530, 528, 538, 539, 557, 463, 535, 498, 505, 523, 506, 517, 507, 521, 458, 479, 561, 510, 532, 522, 477, 506, 522, 496, 526, 519, 509, 532, 509, 521, 504, 517, 522, 542, 514, 512, 494, 529, 502, 532, 520, 525, 530, 515, 532, 530, 505, 554, 568, 505, 460, 488, 554, 507] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[538, 513, 515, 502, 533, 530, 526, 489, 509, 496, 538, 495, 498, 485, 505, 513, 538, 521, 500, 556, 470, 482, 517, 461, 497, 491, 541, 540, 532, 534, 526, 525, 502, 511, 522, 530, 502, 488, 473, 533, 528, 486, 516, 494, 510, 475, 505, 484, 531, 495, 505, 508, 499, 518, 517, 509, 517, 498, 567, 480, 532, 496, 523, 513, 548, 565, 502, 512, 491, 540, 514, 548, 491, 538, 493, 472, 498, 497, 554, 512, 526, 493, 461, 535, 520, 541, 546, 509, 547, 519, 530, 479, 507, 511, 523, 486, 495, 510, 504, 522, 535, 503, 521, 558, 509, 514, 546, 501, 480, 516, 487, 469, 495, 532, 488, 542, 545, 505, 511, 494, 512, 493, 503, 499, 525, 512, 495, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[522, 502, 498, 546, 531, 490, 487, 484, 524, 530, 482, 518, 510, 471, 494, 487, 462, 508, 548, 507, 533, 521, 528, 505, 482, 494, 502, 530, 515, 539, 491, 511, 511, 528, 484, 522, 541, 488, 533, 513, 503, 536, 513, 517, 509, 472, 524, 526, 511, 523, 550, 514, 511, 523, 526, 514, 512, 533, 500, 553, 478, 521, 474, 490, 480, 523, 514, 492, 524, 489, 473, 567, 534, 500, 493, 539, 529, 523, 487, 516, 496, 513, 527, 533, 508, 513, 528, 527, 475, 496, 528, 515, 500, 539, 496, 507, 520, 519, 480, 509, 567, 494, 481, 494, 521, 542, 524, 490, 482, 514, 507, 508, 502, 579, 508, 504, 539, 526, 551, 519, 482, 513, 541, 539, 519, 455, 533, 485] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[517, 513, 516, 509, 556, 507, 489, 488, 515, 463, 488, 529, 512, 493, 515, 517, 511, 505, 477, 515, 504, 531, 514, 528, 468, 525, 538, 499, 508, 499, 513, 502, 511, 524, 491, 515, 509, 480, 492, 529, 497, 500, 478, 493, 505, 507, 490, 516, 527, 467, 512, 467, 503, 517, 569, 496, 494, 478, 519, 512, 496, 518, 520, 554, 490, 483, 533, 538, 502, 544, 511, 520, 548, 537, 518, 520, 533, 506, 546, 479, 507, 476, 498, 522, 534, 561, 497, 517, 522, 455, 473, 514, 498, 554, 507, 507, 505, 513, 534, 521, 512, 537, 535, 534, 522, 528, 518, 470, 503, 532, 552, 529, 548, 532, 491, 531, 496, 554, 505, 519, 515, 514, 499, 495, 519, 489, 534, 550] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[521, 495, 511, 461, 525, 535, 486, 526, 525, 518, 474, 489, 481, 533, 522, 552, 539, 523, 518, 496, 508, 513, 510, 481, 545, 522, 512, 495, 513, 537, 524, 513, 517, 506, 500, 486, 479, 500, 521, 523, 496, 471, 509, 538, 488, 495, 521, 477, 522, 520, 506, 532, 488, 531, 492, 493, 502, 535, 473, 577, 474, 510, 526, 545, 508, 495, 520, 528, 475, 520, 500, 493, 532, 530, 560, 513, 528, 519, 525, 511, 511, 558, 523, 505, 526, 543, 499, 495, 511, 493, 544, 550, 504, 515, 521, 534, 510, 471, 555, 490, 489, 500, 510, 499, 490, 521, 521, 523, 468, 500, 479, 539, 506, 499, 542, 542, 514, 525, 481, 542, 504, 544, 502, 507, 501, 533, 476, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[523, 496, 473, 481, 483, 485, 528, 473, 507, 510, 485, 511, 512, 488, 518, 446, 515, 511, 524, 493, 521, 497, 509, 466, 485, 540, 498, 521, 508, 551, 531, 488, 507, 534, 469, 522, 519, 497, 493, 536, 537, 528, 512, 511, 515, 511, 527, 489, 532, 525, 523, 511, 546, 509, 518, 497, 503, 522, 495, 517, 503, 535, 525, 507, 491, 513, 518, 490, 526, 517, 503, 498, 501, 520, 530, 528, 538, 539, 557, 463, 535, 498, 505, 523, 506, 517, 507, 521, 458, 479, 561, 510, 532, 522, 477, 506, 522, 496, 526, 519, 509, 532, 509, 521, 504, 517, 522, 542, 514, 512, 494, 529, 502, 532, 520, 525, 530, 515, 532, 530, 505, 554, 568, 505, 460, 488, 554, 507] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[522, 502, 498, 546, 531, 490, 487, 484, 524, 530, 482, 518, 510, 471, 494, 487, 462, 508, 548, 507, 533, 521, 528, 505, 482, 494, 502, 530, 515, 539, 491, 511, 511, 528, 484, 522, 541, 488, 533, 513, 503, 536, 513, 517, 509, 472, 524, 526, 511, 523, 550, 514, 511, 523, 526, 514, 512, 533, 500, 553, 478, 521, 474, 490, 480, 523, 514, 492, 524, 489, 473, 567, 534, 500, 493, 539, 529, 523, 487, 516, 496, 513, 527, 533, 508, 513, 528, 527, 475, 496, 528, 515, 500, 539, 496, 507, 520, 519, 480, 509, 567, 494, 481, 494, 521, 542, 524, 490, 482, 514, 507, 508, 502, 579, 508, 504, 539, 526, 551, 519, 482, 513, 541, 539, 519, 455, 533, 485] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[540, 510, 474, 457, 463, 532, 522, 456, 516, 515, 516, 518, 551, 487, 525, 515, 483, 543, 505, 573, 523, 511, 517, 508, 527, 533, 527, 515, 484, 518, 462, 534, 484, 495, 524, 481, 517, 540, 523, 502, 518, 502, 522, 509, 520, 533, 508, 506, 521, 544, 505, 519, 555, 543, 553, 519, 514, 499, 503, 547, 518, 493, 547, 481, 511, 515, 533, 543, 501, 540, 488, 445, 499, 534, 465, 492, 500, 489, 474, 494, 491, 552, 515, 530, 541, 497, 511, 484, 489, 505, 519, 501, 509, 486, 520, 534, 511, 526, 497, 495, 519, 544, 507, 529, 482, 504, 522, 495, 535, 496, 531, 502, 545, 503, 520, 503, 480, 500, 523, 539, 521, 523, 532, 477, 566, 496, 487, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[517, 513, 516, 509, 556, 507, 489, 488, 515, 463, 488, 529, 512, 493, 515, 517, 34, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 505, 477, 515, 504, 531, 514, 528, 468, 525, 538, 499, 508, 499, 513, 502, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 524, 491, 515, 509, 480, 492, 529, 497, 500, 478, 493, 505, 507, 490, 516, 58, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 467, 512, 467, 503, 517, 569, 496, 494, 478, 519, 512, 496, 518, 520, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 483, 533, 538, 502, 544, 511, 520, 548, 537, 518, 520, 533, 506, 546, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 476, 498, 522, 534, 561, 497, 517, 522, 455, 473, 514, 498, 554, 507, 507, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 513, 534, 521, 512, 537, 535, 519, 522, 528, 518, 470, 503, 532, 552, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 474, 491, 531, 496, 554, 505, 519, 515, 514, 499, 495, 519, 489, 534, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8166, 8142, 8100, 8110, 8308, 8147, 8330, 8233] avg=8192.0000 max=8330 max_rank=6 min=8100 min_rank=2 imbalance_ratio=1.016846 max_load_reduction=57206 max_load_reduction_pct=0.872894 moved_to_echo=33018 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[474, 532, 539, 552, 544, 503, 506, 532, 498, 554, 533, 479, 498, 536, 556, 490, 34, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 540, 468, 456, 520, 505, 489, 518, 520, 468, 524, 501, 495, 511, 492, 519, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 534, 508, 479, 486, 495, 506, 491, 485, 471, 490, 523, 515, 508, 527, 519, 54, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 493, 531, 515, 546, 502, 509, 502, 492, 531, 571, 512, 519, 489, 514, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 489, 490, 493, 542, 534, 523, 502, 501, 513, 535, 511, 535, 488, 473, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 492, 482, 517, 514, 512, 550, 520, 539, 541, 511, 548, 500, 480, 533, 526, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 521, 481, 521, 522, 493, 512, 502, 523, 504, 540, 515, 483, 524, 524, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 447, 531, 527, 521, 484, 504, 537, 529, 511, 503, 523, 507, 526, 520, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8365, 8028, 8119, 8188, 8161, 8284, 8191, 8200] avg=8192.0000 max=8365 max_rank=0 min=8028 min_rank=1 imbalance_ratio=1.021118 max_load_reduction=57171 max_load_reduction_pct=0.872360 moved_to_echo=32836 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[533, 549, 535, 516, 499, 517, 508, 513, 542, 530, 521, 513, 516, 483, 555, 498, 36, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 495, 500, 505, 524, 517, 540, 525, 533, 538, 516, 519, 532, 514, 496, 494, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 510, 515, 513, 511, 529, 545, 541, 510, 477, 494, 526, 504, 510, 531, 490, 56, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 527, 538, 487, 490, 495, 513, 514, 562, 498, 494, 509, 538, 486, 499, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 581, 523, 464, 500, 471, 509, 502, 522, 523, 533, 491, 468, 492, 532, 508, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 532, 505, 478, 496, 541, 482, 456, 449, 512, 483, 490, 537, 513, 484, 503, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 513, 461, 531, 522, 543, 521, 527, 549, 488, 485, 506, 539, 545, 531, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 463, 529, 508, 490, 520, 517, 493, 551, 517, 526, 499, 497, 501, 476, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8369, 8258, 8301, 8146, 8095, 7970, 8256, 8141] avg=8192.0000 max=8369 max_rank=0 min=7970 min_rank=5 imbalance_ratio=1.021606 max_load_reduction=57167 max_load_reduction_pct=0.872299 moved_to_echo=32462 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[523, 496, 473, 481, 483, 485, 528, 473, 507, 510, 485, 511, 512, 488, 518, 446, 39, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 511, 524, 493, 521, 497, 509, 466, 485, 540, 498, 521, 508, 551, 531, 488, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 534, 469, 522, 519, 497, 493, 536, 537, 528, 512, 511, 515, 511, 527, 489, 61, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 525, 523, 511, 546, 509, 518, 497, 503, 522, 495, 517, 503, 535, 525, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 513, 518, 490, 526, 517, 503, 498, 501, 520, 530, 528, 538, 539, 557, 463, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 498, 505, 523, 506, 517, 507, 521, 458, 479, 561, 510, 532, 522, 477, 506, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 496, 526, 519, 509, 532, 509, 491, 504, 517, 522, 542, 514, 512, 494, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 471, 520, 525, 530, 515, 532, 530, 505, 554, 568, 505, 460, 488, 554, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7968, 8168, 8277, 8220, 8234, 8165, 8238, 8266] avg=8192.0000 max=8277 max_rank=2 min=7968 min_rank=0 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32903 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[474, 532, 539, 552, 544, 503, 506, 532, 498, 554, 533, 479, 498, 536, 556, 490, 34, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 540, 468, 456, 520, 505, 489, 518, 520, 468, 524, 501, 495, 511, 492, 519, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 534, 508, 479, 486, 495, 506, 491, 485, 471, 490, 523, 515, 508, 527, 519, 54, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 493, 531, 515, 546, 502, 509, 502, 492, 531, 571, 512, 519, 489, 514, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 489, 490, 493, 542, 534, 523, 502, 501, 513, 535, 511, 535, 488, 473, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 492, 482, 517, 514, 512, 550, 520, 539, 541, 511, 548, 500, 480, 533, 526, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 521, 481, 521, 522, 493, 512, 502, 523, 504, 540, 515, 483, 524, 524, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 447, 531, 527, 521, 484, 504, 537, 529, 511, 503, 523, 507, 526, 520, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8365, 8028, 8119, 8188, 8161, 8284, 8191, 8200] avg=8192.0000 max=8365 max_rank=0 min=8028 min_rank=1 imbalance_ratio=1.021118 max_load_reduction=57171 max_load_reduction_pct=0.872360 moved_to_echo=32836 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[540, 510, 474, 457, 463, 532, 522, 456, 516, 515, 516, 518, 551, 487, 525, 515, 34, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 543, 505, 573, 523, 511, 517, 508, 527, 533, 527, 515, 484, 518, 462, 534, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 495, 524, 481, 517, 540, 523, 502, 518, 502, 522, 509, 520, 533, 508, 506, 54, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 544, 505, 519, 555, 543, 553, 519, 514, 499, 503, 547, 518, 493, 547, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 515, 533, 543, 501, 540, 488, 445, 499, 534, 465, 492, 500, 489, 474, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 552, 515, 530, 541, 497, 511, 484, 489, 505, 519, 501, 509, 486, 520, 534, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 526, 497, 495, 519, 544, 507, 514, 482, 504, 522, 495, 535, 496, 531, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 449, 520, 503, 480, 500, 523, 539, 521, 523, 532, 477, 566, 496, 487, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8136, 8268, 8243, 8322, 8023, 8189, 8180, 8175] avg=8192.0000 max=8322 max_rank=3 min=8023 min_rank=4 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32567 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[521, 495, 511, 461, 525, 535, 486, 526, 525, 518, 474, 489, 481, 533, 522, 552, 34, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 523, 518, 496, 508, 513, 510, 481, 545, 522, 512, 495, 513, 537, 524, 513, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 506, 500, 486, 479, 500, 521, 523, 496, 471, 509, 538, 488, 495, 521, 477, 54, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 520, 506, 532, 488, 531, 492, 493, 502, 535, 473, 577, 474, 510, 526, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 495, 520, 528, 475, 520, 500, 493, 532, 530, 560, 513, 528, 519, 525, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 558, 523, 505, 526, 543, 499, 495, 511, 493, 544, 550, 504, 515, 521, 534, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 471, 555, 490, 489, 500, 510, 484, 490, 521, 521, 523, 468, 500, 479, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 445, 542, 542, 514, 525, 481, 542, 504, 544, 502, 507, 501, 533, 476, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8254, 8086, 8187, 8257, 8337, 8050, 8172] avg=8192.0000 max=8337 max_rank=5 min=8050 min_rank=6 imbalance_ratio=1.017700 max_load_reduction=57199 max_load_reduction_pct=0.872787 moved_to_echo=32816 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[538, 513, 515, 502, 533, 530, 526, 489, 509, 496, 538, 495, 498, 485, 505, 513, 35, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 521, 500, 556, 470, 482, 517, 461, 497, 491, 541, 540, 532, 534, 526, 525, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 511, 522, 530, 502, 488, 473, 533, 528, 486, 516, 494, 510, 475, 505, 484, 58, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 495, 505, 508, 499, 518, 517, 509, 517, 498, 567, 480, 532, 496, 523, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 565, 502, 512, 491, 540, 514, 548, 491, 538, 493, 472, 498, 497, 554, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 493, 461, 535, 520, 541, 546, 509, 547, 519, 530, 479, 507, 511, 523, 486, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 510, 504, 522, 535, 503, 521, 542, 509, 514, 546, 501, 480, 516, 487, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 474, 488, 542, 545, 505, 511, 494, 512, 493, 503, 499, 525, 512, 495, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 8237, 8122, 8168, 8275, 8238, 8154, 8117] avg=8192.0000 max=8275 max_rank=4 min=8117 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32784 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[517, 513, 516, 509, 556, 507, 489, 488, 515, 463, 488, 529, 512, 493, 515, 517, 34, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 505, 477, 515, 504, 531, 514, 528, 468, 525, 538, 499, 508, 499, 513, 502, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 524, 491, 515, 509, 480, 492, 529, 497, 500, 478, 493, 505, 507, 490, 516, 58, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 467, 512, 467, 503, 517, 569, 496, 494, 478, 519, 512, 496, 518, 520, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 483, 533, 538, 502, 544, 511, 520, 548, 537, 518, 520, 533, 506, 546, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 476, 498, 522, 534, 561, 497, 517, 522, 455, 473, 514, 498, 554, 507, 507, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 513, 534, 521, 512, 537, 535, 519, 522, 528, 518, 470, 503, 532, 552, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 474, 491, 531, 496, 554, 505, 519, 515, 514, 499, 495, 519, 489, 534, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8166, 8142, 8100, 8110, 8308, 8147, 8330, 8233] avg=8192.0000 max=8330 max_rank=6 min=8100 min_rank=2 imbalance_ratio=1.016846 max_load_reduction=57206 max_load_reduction_pct=0.872894 moved_to_echo=33018 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[522, 502, 498, 546, 531, 490, 487, 484, 524, 530, 482, 518, 510, 471, 494, 487, 33, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 508, 548, 507, 533, 521, 528, 505, 482, 494, 502, 530, 515, 539, 491, 511, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 528, 484, 522, 541, 488, 533, 513, 503, 536, 513, 517, 509, 472, 524, 526, 63, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 523, 550, 514, 511, 523, 526, 514, 512, 533, 500, 553, 478, 521, 474, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 523, 514, 492, 524, 489, 473, 567, 534, 500, 493, 539, 529, 523, 487, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 513, 527, 533, 508, 513, 528, 527, 475, 496, 528, 515, 500, 539, 496, 507, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 519, 480, 509, 567, 494, 481, 479, 521, 542, 524, 490, 482, 514, 507, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 516, 508, 504, 539, 526, 551, 519, 482, 513, 541, 539, 519, 455, 533, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8114, 8181, 8288, 8195, 8183, 8206, 8137, 8232] avg=8192.0000 max=8288 max_rank=2 min=8114 min_rank=0 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32758 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[540, 510, 474, 457, 463, 532, 522, 456, 516, 515, 516, 518, 551, 487, 525, 515, 34, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 543, 505, 573, 523, 511, 517, 508, 527, 533, 527, 515, 484, 518, 462, 534, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 495, 524, 481, 517, 540, 523, 502, 518, 502, 522, 509, 520, 533, 508, 506, 54, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 544, 505, 519, 555, 543, 553, 519, 514, 499, 503, 547, 518, 493, 547, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 515, 533, 543, 501, 540, 488, 445, 499, 534, 465, 492, 500, 489, 474, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 552, 515, 530, 541, 497, 511, 484, 489, 505, 519, 501, 509, 486, 520, 534, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 526, 497, 495, 519, 544, 507, 514, 482, 504, 522, 495, 535, 496, 531, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 449, 520, 503, 480, 500, 523, 539, 521, 523, 532, 477, 566, 496, 487, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8136, 8268, 8243, 8322, 8023, 8189, 8180, 8175] avg=8192.0000 max=8322 max_rank=3 min=8023 min_rank=4 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32567 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[3968, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4352, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 3840, 4224, 4224, 4224, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4096, 3968, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[3968, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 3968, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4096, 3968, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 3840, 4224, 4224, 4224, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[484, 520, 534, 520, 537, 492, 555, 546, 545, 503, 480, 507, 544, 511, 492, 539, 491, 450, 489, 547, 516, 569, 516, 525, 513, 498, 511, 551, 505, 499, 500, 493, 519, 546, 531, 482, 522, 518, 508, 536, 537, 538, 491, 525, 497, 494, 522, 547, 485, 496, 532, 530, 492, 481, 532, 461, 509, 528, 507, 534, 472, 480, 541, 497, 555, 514, 493, 520, 537, 531, 512, 520, 504, 491, 477, 530, 502, 501, 494, 495, 457, 522, 494, 494, 501, 480, 529, 487, 497, 493, 491, 520, 505, 561, 519, 501, 503, 560, 484, 495, 485, 514, 495, 534, 493, 521, 547, 503, 503, 509, 510, 490, 496, 524, 517, 498, 489, 516, 517, 542, 573, 522, 545, 491, 538, 524, 505, 494] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[493, 520, 506, 516, 462, 513, 500, 536, 491, 502, 503, 570, 498, 472, 525, 498, 477, 509, 517, 481, 513, 506, 493, 492, 543, 519, 492, 529, 474, 498, 540, 523, 573, 500, 493, 497, 520, 522, 485, 549, 518, 561, 522, 550, 506, 533, 488, 540, 456, 486, 520, 528, 532, 540, 527, 486, 522, 517, 528, 499, 503, 508, 509, 538, 524, 568, 478, 541, 525, 488, 506, 493, 535, 528, 494, 526, 514, 522, 491, 506, 522, 506, 508, 529, 521, 485, 532, 490, 484, 503, 492, 492, 528, 516, 516, 558, 508, 464, 525, 520, 522, 506, 514, 533, 546, 477, 516, 530, 544, 475, 527, 480, 504, 522, 463, 502, 494, 507, 538, 529, 523, 524, 516, 508, 486, 523, 517, 505] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[525, 475, 518, 515, 499, 549, 505, 516, 487, 480, 499, 501, 507, 508, 504, 514, 495, 509, 503, 495, 514, 501, 551, 523, 477, 511, 520, 482, 511, 533, 501, 472, 533, 504, 527, 548, 509, 493, 531, 484, 545, 530, 506, 509, 505, 543, 540, 510, 516, 461, 475, 500, 494, 524, 526, 508, 536, 538, 526, 525, 512, 485, 533, 546, 526, 505, 513, 549, 501, 552, 491, 512, 544, 520, 533, 487, 526, 492, 516, 476, 510, 520, 473, 510, 531, 488, 515, 501, 457, 527, 555, 523, 512, 488, 518, 533, 542, 483, 518, 513, 526, 501, 484, 522, 506, 494, 506, 492, 528, 504, 476, 518, 525, 522, 526, 522, 497, 523, 503, 534, 522, 556, 548, 487, 490, 527, 509, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[512, 539, 508, 505, 535, 464, 510, 524, 542, 511, 498, 511, 500, 488, 510, 529, 506, 493, 486, 548, 531, 485, 494, 520, 531, 501, 518, 494, 550, 502, 467, 517, 499, 535, 544, 528, 473, 505, 528, 517, 520, 546, 517, 515, 529, 511, 517, 519, 477, 553, 509, 522, 505, 513, 485, 524, 512, 493, 522, 533, 504, 524, 527, 503, 517, 503, 508, 573, 486, 549, 543, 550, 513, 517, 505, 491, 492, 505, 505, 544, 516, 524, 507, 488, 533, 511, 497, 526, 506, 509, 494, 492, 494, 510, 507, 480, 497, 494, 537, 512, 523, 492, 520, 499, 494, 535, 560, 478, 476, 506, 545, 503, 524, 507, 509, 511, 513, 479, 516, 477, 523, 533, 526, 526, 483, 482, 501, 522] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[506, 562, 530, 532, 497, 533, 489, 531, 493, 504, 497, 511, 506, 490, 519, 515, 499, 491, 507, 524, 508, 506, 534, 539, 486, 487, 548, 517, 525, 492, 505, 491, 503, 496, 494, 502, 526, 494, 519, 519, 509, 532, 487, 485, 505, 507, 541, 498, 485, 500, 518, 514, 488, 516, 490, 531, 511, 507, 551, 521, 509, 535, 528, 521, 520, 506, 491, 541, 496, 479, 501, 553, 499, 531, 522, 480, 507, 525, 538, 483, 486, 484, 516, 538, 484, 487, 541, 521, 517, 481, 511, 505, 537, 535, 535, 553, 526, 539, 547, 496, 506, 472, 525, 493, 495, 530, 499, 510, 498, 504, 557, 514, 515, 527, 518, 498, 515, 523, 499, 500, 505, 562, 498, 493, 556, 507, 481, 509] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[518, 468, 505, 507, 502, 512, 507, 493, 503, 543, 533, 524, 522, 502, 508, 511, 474, 507, 506, 505, 510, 528, 540, 534, 516, 527, 492, 512, 470, 529, 531, 498, 534, 500, 526, 480, 473, 555, 521, 487, 513, 523, 486, 529, 537, 506, 520, 489, 509, 498, 508, 520, 517, 496, 513, 492, 501, 502, 556, 522, 500, 524, 594, 491, 524, 533, 496, 512, 497, 502, 524, 511, 485, 518, 473, 485, 495, 513, 515, 522, 518, 521, 538, 506, 514, 490, 493, 489, 523, 516, 520, 537, 490, 511, 528, 535, 531, 475, 541, 504, 510, 494, 523, 522, 498, 479, 519, 528, 524, 480, 534, 509, 531, 538, 476, 529, 545, 506, 505, 538, 489, 507, 472, 537, 529, 521, 567, 482] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[493, 520, 506, 516, 462, 513, 500, 536, 491, 502, 503, 570, 498, 472, 525, 498, 477, 509, 517, 481, 513, 506, 493, 492, 543, 519, 492, 529, 474, 498, 540, 523, 573, 500, 493, 497, 520, 522, 485, 549, 518, 561, 522, 550, 506, 533, 488, 540, 456, 486, 520, 528, 532, 540, 527, 486, 522, 517, 528, 499, 503, 508, 509, 538, 524, 568, 478, 541, 525, 488, 506, 493, 535, 528, 494, 526, 514, 522, 491, 506, 522, 506, 508, 529, 521, 485, 532, 490, 484, 503, 492, 492, 528, 516, 516, 558, 508, 464, 525, 520, 522, 506, 514, 533, 546, 477, 516, 530, 544, 475, 527, 480, 504, 522, 463, 502, 494, 507, 538, 529, 523, 524, 516, 508, 486, 523, 517, 505] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[522, 519, 487, 507, 492, 490, 508, 478, 517, 492, 540, 497, 521, 560, 522, 559, 532, 501, 513, 494, 484, 465, 522, 478, 501, 518, 506, 507, 525, 540, 483, 557, 539, 495, 516, 508, 506, 524, 516, 527, 518, 498, 511, 514, 532, 520, 501, 516, 479, 521, 545, 484, 529, 528, 503, 549, 530, 529, 480, 530, 542, 496, 541, 515, 501, 515, 506, 522, 510, 512, 505, 490, 533, 487, 482, 486, 530, 503, 493, 488, 500, 511, 488, 497, 507, 524, 525, 498, 523, 534, 504, 536, 529, 515, 535, 533, 497, 499, 496, 489, 468, 518, 513, 525, 529, 527, 496, 545, 495, 492, 555, 485, 545, 526, 521, 517, 551, 490, 493, 522, 496, 515, 515, 490, 496, 468, 543, 518] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[501, 496, 526, 508, 515, 488, 523, 499, 491, 517, 497, 537, 517, 505, 506, 502, 567, 504, 496, 492, 468, 495, 545, 538, 540, 525, 499, 546, 531, 496, 529, 525, 515, 508, 531, 493, 553, 492, 528, 478, 471, 522, 503, 520, 521, 542, 529, 470, 518, 512, 531, 513, 514, 530, 514, 496, 500, 531, 489, 530, 531, 496, 517, 498, 496, 515, 527, 483, 509, 541, 488, 506, 477, 509, 471, 542, 503, 506, 537, 452, 532, 530, 482, 535, 499, 498, 496, 539, 445, 492, 568, 529, 482, 504, 511, 536, 469, 504, 506, 511, 555, 517, 504, 503, 553, 481, 505, 483, 526, 519, 488, 503, 495, 483, 475, 518, 544, 493, 543, 494, 511, 551, 536, 543, 543, 543, 532, 545] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[493, 520, 506, 516, 462, 513, 500, 536, 491, 502, 503, 570, 498, 472, 525, 498, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 509, 517, 481, 513, 506, 493, 492, 543, 519, 492, 529, 474, 498, 540, 523, 24, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 573, 500, 493, 497, 520, 522, 485, 549, 518, 500, 522, 550, 506, 533, 488, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 456, 486, 520, 528, 532, 540, 527, 486, 522, 517, 528, 499, 503, 508, 493, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 568, 478, 541, 525, 488, 506, 493, 535, 528, 494, 526, 514, 522, 491, 506, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 506, 508, 529, 521, 485, 532, 490, 484, 503, 492, 492, 528, 516, 516, 558, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 464, 525, 520, 522, 506, 514, 533, 546, 477, 516, 530, 544, 475, 527, 480, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 522, 463, 502, 494, 507, 538, 529, 523, 479, 516, 508, 486, 523, 517, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8119, 8138, 8296, 8183, 8260, 8200, 8224, 8116] avg=8192.0000 max=8296 max_rank=2 min=8116 min_rank=7 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32800 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[512, 539, 508, 505, 535, 464, 510, 524, 542, 511, 498, 511, 500, 488, 510, 529, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 493, 486, 548, 531, 485, 494, 520, 531, 501, 518, 494, 550, 502, 467, 517, 23, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 535, 544, 528, 473, 505, 528, 517, 520, 487, 517, 515, 529, 511, 517, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 553, 509, 522, 505, 513, 485, 524, 512, 493, 522, 533, 504, 524, 511, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 503, 508, 573, 486, 549, 543, 550, 513, 517, 505, 491, 492, 505, 505, 544, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 524, 507, 488, 533, 511, 497, 526, 506, 509, 494, 492, 494, 510, 507, 480, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 494, 537, 512, 523, 492, 520, 499, 494, 535, 560, 478, 476, 506, 545, 503, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 507, 509, 511, 513, 479, 516, 477, 523, 488, 526, 526, 483, 482, 501, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8174, 8244, 8190, 8322, 8112, 8207, 8087] avg=8192.0000 max=8322 max_rank=4 min=8087 min_rank=7 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32728 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[484, 520, 534, 520, 537, 492, 555, 546, 545, 503, 480, 507, 544, 511, 492, 539, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 450, 489, 547, 516, 569, 516, 525, 513, 498, 511, 551, 505, 499, 500, 493, 23, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 546, 531, 482, 522, 518, 508, 536, 537, 480, 491, 525, 497, 494, 522, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 496, 532, 530, 492, 481, 532, 461, 509, 528, 507, 534, 472, 480, 524, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 514, 493, 520, 537, 531, 512, 520, 504, 491, 477, 530, 502, 501, 494, 495, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 457, 522, 494, 494, 501, 480, 529, 487, 497, 493, 491, 520, 505, 561, 519, 501, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 560, 484, 495, 485, 514, 495, 534, 493, 521, 547, 503, 503, 509, 510, 490, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 524, 517, 498, 489, 516, 517, 542, 573, 477, 545, 491, 538, 524, 505, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8324, 8204, 8255, 8060, 8197, 8069, 8181, 8246] avg=8192.0000 max=8324 max_rank=0 min=8060 min_rank=3 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32693 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[525, 475, 518, 515, 499, 549, 505, 516, 487, 480, 499, 501, 507, 508, 504, 514, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 509, 503, 495, 514, 501, 551, 523, 477, 511, 520, 482, 511, 533, 501, 472, 28, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 504, 527, 548, 509, 493, 531, 484, 545, 464, 506, 509, 505, 543, 540, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 461, 475, 500, 494, 524, 526, 508, 536, 538, 526, 525, 512, 485, 510, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 505, 513, 549, 501, 552, 491, 512, 544, 520, 533, 487, 526, 492, 516, 476, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 520, 473, 510, 531, 488, 515, 501, 457, 527, 555, 523, 512, 488, 518, 533, 20, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 483, 518, 513, 526, 501, 484, 522, 506, 494, 506, 492, 528, 504, 476, 518, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 522, 526, 522, 497, 523, 503, 534, 522, 498, 548, 487, 490, 527, 509, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8121, 8138, 8251, 8182, 8269, 8185, 8151, 8239] avg=8192.0000 max=8269 max_rank=4 min=8121 min_rank=0 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32844 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[525, 475, 518, 515, 499, 549, 505, 516, 487, 480, 499, 501, 507, 508, 504, 514, 495, 509, 503, 495, 514, 501, 551, 523, 477, 511, 520, 482, 511, 533, 501, 472, 533, 504, 527, 548, 509, 493, 531, 484, 545, 530, 506, 509, 505, 543, 540, 510, 516, 461, 475, 500, 494, 524, 526, 508, 536, 538, 526, 525, 512, 485, 533, 546, 526, 505, 513, 549, 501, 552, 491, 512, 544, 520, 533, 487, 526, 492, 516, 476, 510, 520, 473, 510, 531, 488, 515, 501, 457, 527, 555, 523, 512, 488, 518, 533, 542, 483, 518, 513, 526, 501, 484, 522, 506, 494, 506, 492, 528, 504, 476, 518, 525, 522, 526, 522, 497, 523, 503, 534, 522, 556, 548, 487, 490, 527, 509, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[484, 520, 534, 520, 537, 492, 555, 546, 545, 503, 480, 507, 544, 511, 492, 539, 491, 450, 489, 547, 516, 569, 516, 525, 513, 498, 511, 551, 505, 499, 500, 493, 519, 546, 531, 482, 522, 518, 508, 536, 537, 538, 491, 525, 497, 494, 522, 547, 485, 496, 532, 530, 492, 481, 532, 461, 509, 528, 507, 534, 472, 480, 541, 497, 555, 514, 493, 520, 537, 531, 512, 520, 504, 491, 477, 530, 502, 501, 494, 495, 457, 522, 494, 494, 501, 480, 529, 487, 497, 493, 491, 520, 505, 561, 519, 501, 503, 560, 484, 495, 485, 514, 495, 534, 493, 521, 547, 503, 503, 509, 510, 490, 496, 524, 517, 498, 489, 516, 517, 542, 573, 522, 545, 491, 538, 524, 505, 494] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[518, 468, 505, 507, 502, 512, 507, 493, 503, 543, 533, 524, 522, 502, 508, 511, 474, 507, 506, 505, 510, 528, 540, 534, 516, 527, 492, 512, 470, 529, 531, 498, 534, 500, 526, 480, 473, 555, 521, 487, 513, 523, 486, 529, 537, 506, 520, 489, 509, 498, 508, 520, 517, 496, 513, 492, 501, 502, 556, 522, 500, 524, 594, 491, 524, 533, 496, 512, 497, 502, 524, 511, 485, 518, 473, 485, 495, 513, 515, 522, 518, 521, 538, 506, 514, 490, 493, 489, 523, 516, 520, 537, 490, 511, 528, 535, 531, 475, 541, 504, 510, 494, 523, 522, 498, 479, 519, 528, 524, 480, 534, 509, 531, 538, 476, 529, 545, 506, 505, 538, 489, 507, 472, 537, 529, 521, 567, 482] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[512, 539, 508, 505, 535, 464, 510, 524, 542, 511, 498, 511, 500, 488, 510, 529, 506, 493, 486, 548, 531, 485, 494, 520, 531, 501, 518, 494, 550, 502, 467, 517, 499, 535, 544, 528, 473, 505, 528, 517, 520, 546, 517, 515, 529, 511, 517, 519, 477, 553, 509, 522, 505, 513, 485, 524, 512, 493, 522, 533, 504, 524, 527, 503, 517, 503, 508, 573, 486, 549, 543, 550, 513, 517, 505, 491, 492, 505, 505, 544, 516, 524, 507, 488, 533, 511, 497, 526, 506, 509, 494, 492, 494, 510, 507, 480, 497, 494, 537, 512, 523, 492, 520, 499, 494, 535, 560, 478, 476, 506, 545, 503, 524, 507, 509, 511, 513, 479, 516, 477, 523, 533, 526, 526, 483, 482, 501, 522] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[506, 562, 530, 532, 497, 533, 489, 531, 493, 504, 497, 511, 506, 490, 519, 515, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 491, 507, 524, 508, 506, 534, 539, 486, 487, 548, 517, 525, 492, 505, 491, 23, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 496, 494, 502, 526, 494, 519, 519, 509, 474, 487, 485, 505, 507, 541, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 500, 518, 514, 488, 516, 490, 531, 511, 507, 551, 521, 509, 535, 511, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 506, 491, 541, 496, 479, 501, 553, 499, 531, 522, 480, 507, 525, 538, 483, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 484, 516, 538, 484, 487, 541, 521, 517, 481, 511, 505, 537, 535, 535, 553, 17, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 539, 547, 496, 506, 472, 525, 493, 495, 530, 499, 510, 498, 504, 557, 514, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 527, 518, 498, 515, 523, 499, 500, 505, 513, 498, 493, 556, 507, 481, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 8191, 8059, 8208, 8195, 8250, 8246, 8157] avg=8192.0000 max=8250 max_rank=5 min=8059 min_rank=2 imbalance_ratio=1.007080 max_load_reduction=57286 max_load_reduction_pct=0.874115 moved_to_echo=32848 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[525, 475, 518, 515, 499, 549, 505, 516, 487, 480, 499, 501, 507, 508, 504, 514, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 509, 503, 495, 514, 501, 551, 523, 477, 511, 520, 482, 511, 533, 501, 472, 28, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 504, 527, 548, 509, 493, 531, 484, 545, 464, 506, 509, 505, 543, 540, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 461, 475, 500, 494, 524, 526, 508, 536, 538, 526, 525, 512, 485, 510, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 505, 513, 549, 501, 552, 491, 512, 544, 520, 533, 487, 526, 492, 516, 476, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 520, 473, 510, 531, 488, 515, 501, 457, 527, 555, 523, 512, 488, 518, 533, 20, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 483, 518, 513, 526, 501, 484, 522, 506, 494, 506, 492, 528, 504, 476, 518, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 522, 526, 522, 497, 523, 503, 534, 522, 498, 548, 487, 490, 527, 509, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8121, 8138, 8251, 8182, 8269, 8185, 8151, 8239] avg=8192.0000 max=8269 max_rank=4 min=8121 min_rank=0 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32844 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[484, 520, 534, 520, 537, 492, 555, 546, 545, 503, 480, 507, 544, 511, 492, 539, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 450, 489, 547, 516, 569, 516, 525, 513, 498, 511, 551, 505, 499, 500, 493, 23, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 546, 531, 482, 522, 518, 508, 536, 537, 480, 491, 525, 497, 494, 522, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 496, 532, 530, 492, 481, 532, 461, 509, 528, 507, 534, 472, 480, 524, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 514, 493, 520, 537, 531, 512, 520, 504, 491, 477, 530, 502, 501, 494, 495, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 457, 522, 494, 494, 501, 480, 529, 487, 497, 493, 491, 520, 505, 561, 519, 501, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 560, 484, 495, 485, 514, 495, 534, 493, 521, 547, 503, 503, 509, 510, 490, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 524, 517, 498, 489, 516, 517, 542, 573, 477, 545, 491, 538, 524, 505, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8324, 8204, 8255, 8060, 8197, 8069, 8181, 8246] avg=8192.0000 max=8324 max_rank=0 min=8060 min_rank=3 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32693 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[522, 519, 487, 507, 492, 490, 508, 478, 517, 492, 540, 497, 521, 560, 522, 559, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 501, 513, 494, 484, 465, 522, 478, 501, 518, 506, 507, 525, 540, 483, 557, 21, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 495, 516, 508, 506, 524, 516, 527, 518, 445, 511, 514, 532, 520, 501, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 521, 545, 484, 529, 528, 503, 549, 530, 529, 480, 530, 542, 496, 524, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 515, 506, 522, 510, 512, 505, 490, 533, 487, 482, 486, 530, 503, 493, 488, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 511, 488, 497, 507, 524, 525, 498, 523, 534, 504, 536, 529, 515, 535, 533, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 499, 496, 489, 468, 518, 513, 525, 529, 527, 496, 545, 495, 492, 555, 485, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 526, 521, 517, 551, 490, 493, 522, 496, 470, 515, 490, 496, 468, 543, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 8155, 8188, 8284, 8084, 8277, 8161, 8161] avg=8192.0000 max=8284 max_rank=3 min=8084 min_rank=4 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32683 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[518, 468, 505, 507, 502, 512, 507, 493, 503, 543, 533, 524, 522, 502, 508, 511, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 507, 506, 505, 510, 528, 540, 534, 516, 527, 492, 512, 470, 529, 531, 498, 22, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 500, 526, 480, 473, 555, 521, 487, 513, 467, 486, 529, 537, 506, 520, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 498, 508, 520, 517, 496, 513, 492, 501, 502, 556, 522, 500, 524, 576, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 533, 496, 512, 497, 502, 524, 511, 485, 518, 473, 485, 495, 513, 515, 522, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 521, 538, 506, 514, 490, 493, 489, 523, 516, 520, 537, 490, 511, 528, 535, 15, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 475, 541, 504, 510, 494, 523, 522, 498, 479, 519, 528, 524, 480, 534, 509, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 538, 476, 529, 545, 506, 505, 538, 489, 464, 472, 537, 529, 521, 567, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 8209, 8123, 8225, 8125, 8246, 8205, 8229] avg=8192.0000 max=8246 max_rank=5 min=8123 min_rank=2 imbalance_ratio=1.006592 max_load_reduction=57290 max_load_reduction_pct=0.874176 moved_to_echo=32805 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[493, 520, 506, 516, 462, 513, 500, 536, 491, 502, 503, 570, 498, 472, 525, 498, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 509, 517, 481, 513, 506, 493, 492, 543, 519, 492, 529, 474, 498, 540, 523, 24, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 573, 500, 493, 497, 520, 522, 485, 549, 518, 500, 522, 550, 506, 533, 488, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 456, 486, 520, 528, 532, 540, 527, 486, 522, 517, 528, 499, 503, 508, 493, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 568, 478, 541, 525, 488, 506, 493, 535, 528, 494, 526, 514, 522, 491, 506, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 506, 508, 529, 521, 485, 532, 490, 484, 503, 492, 492, 528, 516, 516, 558, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 464, 525, 520, 522, 506, 514, 533, 546, 477, 516, 530, 544, 475, 527, 480, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 522, 463, 502, 494, 507, 538, 529, 523, 479, 516, 508, 486, 523, 517, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8119, 8138, 8296, 8183, 8260, 8200, 8224, 8116] avg=8192.0000 max=8296 max_rank=2 min=8116 min_rank=7 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32800 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[512, 539, 508, 505, 535, 464, 510, 524, 542, 511, 498, 511, 500, 488, 510, 529, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 493, 486, 548, 531, 485, 494, 520, 531, 501, 518, 494, 550, 502, 467, 517, 23, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 535, 544, 528, 473, 505, 528, 517, 520, 487, 517, 515, 529, 511, 517, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 553, 509, 522, 505, 513, 485, 524, 512, 493, 522, 533, 504, 524, 511, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 503, 508, 573, 486, 549, 543, 550, 513, 517, 505, 491, 492, 505, 505, 544, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 524, 507, 488, 533, 511, 497, 526, 506, 509, 494, 492, 494, 510, 507, 480, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 494, 537, 512, 523, 492, 520, 499, 494, 535, 560, 478, 476, 506, 545, 503, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 507, 509, 511, 513, 479, 516, 477, 523, 488, 526, 526, 483, 482, 501, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8174, 8244, 8190, 8322, 8112, 8207, 8087] avg=8192.0000 max=8322 max_rank=4 min=8087 min_rank=7 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32728 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[501, 496, 526, 508, 515, 488, 523, 499, 491, 517, 497, 537, 517, 505, 506, 502, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 567, 504, 496, 492, 468, 495, 545, 538, 540, 525, 499, 546, 531, 496, 529, 525, 22, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 508, 531, 493, 553, 492, 528, 478, 471, 466, 503, 520, 521, 542, 529, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 512, 531, 513, 514, 530, 514, 496, 500, 531, 489, 530, 531, 496, 501, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 515, 527, 483, 509, 541, 488, 506, 477, 509, 471, 542, 503, 506, 537, 452, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 530, 482, 535, 499, 498, 496, 539, 445, 492, 568, 529, 482, 504, 511, 536, 17, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 504, 506, 511, 555, 517, 504, 503, 553, 481, 505, 483, 526, 519, 488, 503, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 483, 475, 518, 544, 493, 543, 494, 511, 503, 536, 543, 543, 543, 532, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8142, 8327, 8120, 8204, 8084, 8197, 8161, 8301] avg=8192.0000 max=8327 max_rank=1 min=8084 min_rank=4 imbalance_ratio=1.016479 max_load_reduction=57209 max_load_reduction_pct=0.872940 moved_to_echo=32743 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[518, 468, 505, 507, 502, 512, 507, 493, 503, 543, 533, 524, 522, 502, 508, 511, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 507, 506, 505, 510, 528, 540, 534, 516, 527, 492, 512, 470, 529, 531, 498, 22, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 500, 526, 480, 473, 555, 521, 487, 513, 467, 486, 529, 537, 506, 520, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 498, 508, 520, 517, 496, 513, 492, 501, 502, 556, 522, 500, 524, 576, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 533, 496, 512, 497, 502, 524, 511, 485, 518, 473, 485, 495, 513, 515, 522, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 521, 538, 506, 514, 490, 493, 489, 523, 516, 520, 537, 490, 511, 528, 535, 15, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 475, 541, 504, 510, 494, 523, 522, 498, 479, 519, 528, 524, 480, 534, 509, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 538, 476, 529, 545, 506, 505, 538, 489, 464, 472, 537, 529, 521, 567, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 8209, 8123, 8225, 8125, 8246, 8205, 8229] avg=8192.0000 max=8246 max_rank=5 min=8123 min_rank=2 imbalance_ratio=1.006592 max_load_reduction=57290 max_load_reduction_pct=0.874176 moved_to_echo=32805 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 3968, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[3968, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 3968, 4224, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 3968, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 3968, 4224, 4096, 3968, 4096, 4224, 4224, 4096, 4224, 4224, 4352, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4352, 4096, 4224, 4096, 4224, 4096, 4224, 3968, 4096, 4096, 4096, 4096, 3968, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[3968, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 3840, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 3968, 4224, 4096, 3968, 4096, 4224, 4224, 4096, 4224, 4224, 4352, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[519, 512, 523, 543, 511, 488, 486, 534, 509, 515, 548, 519, 481, 516, 528, 513, 494, 511, 514, 501, 504, 502, 493, 517, 530, 509, 500, 525, 498, 518, 540, 516, 467, 556, 489, 533, 487, 492, 490, 511, 519, 492, 544, 512, 528, 500, 516, 517, 523, 492, 513, 482, 501, 466, 519, 564, 532, 507, 517, 481, 524, 501, 518, 518, 517, 548, 488, 512, 558, 490, 566, 520, 523, 528, 504, 485, 523, 522, 522, 502, 487, 512, 535, 523, 503, 496, 527, 553, 494, 508, 530, 509, 530, 521, 519, 554, 496, 480, 561, 489, 487, 510, 487, 534, 490, 529, 479, 497, 520, 496, 514, 501, 533, 520, 513, 529, 484, 538, 499, 511, 494, 475, 512, 516, 501, 495, 506, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[509, 508, 524, 488, 505, 531, 543, 499, 505, 518, 534, 523, 505, 509, 488, 561, 531, 556, 496, 531, 544, 556, 537, 499, 522, 549, 525, 510, 521, 486, 484, 517, 514, 500, 541, 517, 465, 507, 538, 465, 532, 498, 538, 500, 487, 511, 535, 484, 512, 474, 534, 482, 486, 480, 539, 506, 517, 502, 503, 468, 502, 506, 498, 519, 539, 510, 497, 545, 516, 508, 504, 487, 488, 498, 491, 512, 532, 554, 500, 490, 506, 495, 478, 472, 513, 532, 537, 529, 521, 538, 513, 487, 493, 551, 513, 517, 556, 497, 510, 497, 533, 506, 520, 532, 562, 534, 501, 480, 516, 498, 542, 521, 491, 503, 490, 456, 516, 520, 490, 507, 536, 536, 518, 514, 499, 489, 522, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[525, 527, 536, 502, 496, 479, 467, 473, 522, 472, 520, 516, 525, 521, 538, 487, 519, 510, 532, 505, 512, 495, 536, 503, 496, 526, 519, 490, 546, 522, 530, 553, 497, 530, 486, 488, 511, 559, 541, 531, 516, 516, 531, 515, 509, 515, 495, 508, 522, 484, 504, 526, 485, 532, 510, 479, 526, 485, 491, 515, 515, 532, 481, 502, 508, 546, 533, 525, 514, 509, 541, 476, 544, 514, 526, 510, 501, 505, 548, 524, 490, 526, 519, 486, 517, 498, 487, 496, 519, 525, 545, 470, 515, 512, 520, 501, 512, 489, 516, 504, 499, 481, 528, 519, 503, 491, 531, 522, 487, 516, 496, 498, 519, 548, 518, 531, 561, 493, 477, 487, 485, 531, 516, 531, 505, 516, 510, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[547, 511, 536, 491, 540, 519, 541, 504, 521, 519, 499, 498, 500, 529, 504, 541, 484, 488, 539, 502, 509, 511, 488, 538, 516, 516, 507, 520, 513, 516, 491, 505, 494, 535, 523, 528, 545, 515, 532, 541, 504, 525, 529, 506, 500, 487, 493, 534, 470, 515, 543, 504, 519, 525, 492, 514, 479, 492, 512, 548, 553, 486, 498, 467, 525, 516, 497, 478, 496, 523, 474, 473, 530, 496, 547, 516, 513, 512, 504, 526, 498, 471, 513, 521, 538, 510, 500, 523, 530, 514, 516, 485, 468, 547, 530, 480, 518, 522, 492, 516, 506, 495, 525, 499, 515, 525, 501, 508, 510, 511, 525, 541, 489, 545, 500, 541, 543, 479, 521, 515, 517, 503, 517, 483, 499, 540, 516, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[446, 540, 474, 518, 513, 497, 506, 529, 507, 495, 515, 528, 463, 502, 501, 521, 529, 523, 509, 469, 532, 478, 521, 492, 516, 528, 503, 528, 518, 501, 516, 491, 501, 506, 533, 499, 517, 526, 501, 522, 565, 454, 534, 497, 498, 549, 473, 485, 549, 508, 578, 507, 482, 549, 540, 527, 547, 538, 497, 512, 483, 526, 540, 528, 504, 477, 537, 535, 505, 501, 523, 530, 517, 521, 517, 494, 480, 551, 536, 472, 530, 483, 540, 518, 532, 523, 477, 507, 492, 516, 512, 490, 494, 493, 522, 509, 518, 496, 496, 503, 505, 494, 496, 510, 538, 509, 536, 508, 496, 513, 504, 492, 545, 475, 507, 511, 520, 541, 537, 531, 475, 493, 556, 530, 513, 539, 500, 531] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[524, 502, 473, 523, 537, 518, 508, 511, 493, 547, 495, 498, 501, 512, 502, 526, 518, 508, 509, 534, 501, 456, 523, 512, 517, 486, 547, 539, 520, 500, 530, 520, 532, 456, 484, 512, 528, 507, 471, 515, 529, 531, 518, 511, 510, 466, 509, 505, 519, 513, 499, 521, 507, 515, 523, 506, 565, 509, 514, 491, 527, 551, 558, 538, 543, 503, 477, 489, 533, 512, 514, 481, 510, 463, 500, 487, 493, 512, 510, 528, 538, 503, 505, 498, 478, 534, 521, 533, 524, 508, 550, 548, 504, 506, 536, 539, 479, 530, 528, 511, 487, 509, 521, 516, 500, 529, 495, 520, 522, 513, 537, 476, 492, 522, 486, 511, 553, 492, 495, 504, 498, 507, 545, 522, 494, 499, 532, 501] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[506, 529, 488, 512, 524, 521, 538, 515, 521, 540, 556, 513, 561, 524, 479, 478, 497, 512, 494, 518, 488, 501, 541, 498, 499, 507, 503, 465, 492, 475, 512, 478, 482, 485, 553, 494, 514, 522, 506, 558, 555, 503, 518, 517, 495, 490, 521, 552, 450, 535, 543, 524, 510, 510, 555, 534, 545, 504, 548, 488, 502, 526, 529, 524, 497, 517, 526, 522, 522, 551, 515, 499, 496, 511, 504, 512, 490, 512, 505, 525, 530, 525, 503, 503, 526, 475, 510, 533, 522, 513, 493, 508, 495, 517, 523, 535, 496, 494, 481, 495, 528, 511, 512, 501, 485, 556, 507, 487, 504, 481, 491, 529, 496, 498, 496, 492, 489, 528, 527, 514, 567, 504, 529, 509, 472, 503, 529, 533] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[525, 527, 536, 502, 496, 479, 467, 473, 522, 472, 520, 516, 525, 521, 538, 487, 519, 510, 532, 505, 512, 495, 536, 503, 496, 526, 519, 490, 546, 522, 530, 553, 497, 530, 486, 488, 511, 559, 541, 531, 516, 516, 531, 515, 509, 515, 495, 508, 522, 484, 504, 526, 485, 532, 510, 479, 526, 485, 491, 515, 515, 532, 481, 502, 508, 546, 533, 525, 514, 509, 541, 476, 544, 514, 526, 510, 501, 505, 548, 524, 490, 526, 519, 486, 517, 498, 487, 496, 519, 525, 545, 470, 515, 512, 520, 501, 512, 489, 516, 504, 499, 481, 528, 519, 503, 491, 531, 522, 487, 516, 496, 498, 519, 548, 518, 531, 561, 493, 477, 487, 485, 531, 516, 531, 505, 516, 510, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[474, 533, 502, 534, 553, 480, 505, 507, 492, 531, 465, 540, 522, 464, 503, 518, 540, 485, 545, 492, 480, 478, 529, 530, 553, 519, 535, 524, 523, 478, 546, 517, 524, 494, 493, 510, 536, 497, 493, 524, 508, 553, 530, 520, 529, 510, 485, 504, 503, 489, 532, 511, 544, 467, 498, 502, 477, 504, 520, 514, 533, 536, 545, 495, 499, 506, 530, 512, 521, 496, 495, 529, 530, 477, 548, 523, 479, 463, 492, 504, 505, 517, 525, 543, 519, 505, 519, 503, 524, 560, 496, 519, 529, 505, 526, 507, 529, 500, 553, 462, 506, 496, 510, 522, 485, 539, 509, 489, 495, 487, 504, 510, 490, 501, 546, 519, 530, 503, 488, 522, 511, 490, 520, 502, 552, 513, 518, 552] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[509, 508, 524, 488, 503, 531, 543, 499, 505, 518, 534, 523, 505, 509, 488, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 556, 496, 531, 544, 556, 529, 499, 522, 549, 525, 510, 521, 486, 484, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 500, 541, 517, 465, 507, 538, 465, 532, 498, 538, 500, 487, 511, 535, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 474, 519, 482, 486, 480, 539, 506, 517, 502, 503, 468, 502, 506, 498, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 510, 497, 545, 516, 508, 504, 487, 488, 498, 491, 512, 532, 554, 500, 490, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 495, 478, 472, 513, 532, 537, 529, 521, 538, 513, 487, 493, 551, 488, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 497, 510, 497, 533, 506, 520, 532, 562, 534, 501, 480, 516, 498, 542, 521, 25, 15, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 503, 490, 456, 516, 520, 490, 507, 536, 536, 513, 514, 499, 489, 522, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8248, 8356, 8132, 8013, 8175, 8170, 8356, 8086] avg=8192.0000 max=8356 max_rank=1 min=8013 min_rank=3 imbalance_ratio=1.020020 max_load_reduction=57180 max_load_reduction_pct=0.872498 moved_to_echo=32787 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[519, 512, 523, 543, 509, 488, 486, 534, 509, 515, 548, 519, 481, 516, 528, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 511, 514, 501, 504, 502, 486, 517, 530, 509, 500, 525, 498, 518, 540, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 556, 489, 533, 487, 492, 490, 511, 519, 492, 544, 512, 528, 500, 516, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 492, 499, 482, 501, 466, 519, 564, 532, 507, 517, 481, 524, 501, 518, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 548, 488, 512, 558, 490, 566, 520, 523, 528, 504, 485, 523, 522, 522, 502, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 512, 535, 523, 503, 496, 527, 553, 494, 508, 530, 509, 530, 521, 494, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 480, 561, 489, 487, 510, 487, 534, 490, 529, 479, 497, 520, 496, 514, 501, 25, 14, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 520, 513, 529, 484, 538, 499, 511, 494, 475, 507, 516, 501, 495, 506, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8243, 8165, 8153, 8144, 8312, 8276, 8119, 8124] avg=8192.0000 max=8312 max_rank=4 min=8119 min_rank=6 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32831 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[519, 512, 523, 543, 511, 488, 486, 534, 509, 515, 548, 519, 481, 516, 528, 513, 494, 511, 514, 501, 504, 502, 493, 517, 530, 509, 500, 525, 498, 518, 540, 516, 467, 556, 489, 533, 487, 492, 490, 511, 519, 492, 544, 512, 528, 500, 516, 517, 523, 492, 513, 482, 501, 466, 519, 564, 532, 507, 517, 481, 524, 501, 518, 518, 517, 548, 488, 512, 558, 490, 566, 520, 523, 528, 504, 485, 523, 522, 522, 502, 487, 512, 535, 523, 503, 496, 527, 553, 494, 508, 530, 509, 530, 521, 519, 554, 496, 480, 561, 489, 487, 510, 487, 534, 490, 529, 479, 497, 520, 496, 514, 501, 533, 520, 513, 529, 484, 538, 499, 511, 494, 475, 512, 516, 501, 495, 506, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[524, 502, 473, 523, 537, 518, 508, 511, 493, 547, 495, 498, 501, 512, 502, 526, 518, 508, 509, 534, 501, 456, 523, 512, 517, 486, 547, 539, 520, 500, 530, 520, 532, 456, 484, 512, 528, 507, 471, 515, 529, 531, 518, 511, 510, 466, 509, 505, 519, 513, 499, 521, 507, 515, 523, 506, 565, 509, 514, 491, 527, 551, 558, 538, 543, 503, 477, 489, 533, 512, 514, 481, 510, 463, 500, 487, 493, 512, 510, 528, 538, 503, 505, 498, 478, 534, 521, 533, 524, 508, 550, 548, 504, 506, 536, 539, 479, 530, 528, 511, 487, 509, 521, 516, 500, 529, 495, 520, 522, 513, 537, 476, 492, 522, 486, 511, 553, 492, 495, 504, 498, 507, 545, 522, 494, 499, 532, 501] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[509, 508, 524, 488, 505, 531, 543, 499, 505, 518, 534, 523, 505, 509, 488, 561, 531, 556, 496, 531, 544, 556, 537, 499, 522, 549, 525, 510, 521, 486, 484, 517, 514, 500, 541, 517, 465, 507, 538, 465, 532, 498, 538, 500, 487, 511, 535, 484, 512, 474, 534, 482, 486, 480, 539, 506, 517, 502, 503, 468, 502, 506, 498, 519, 539, 510, 497, 545, 516, 508, 504, 487, 488, 498, 491, 512, 532, 554, 500, 490, 506, 495, 478, 472, 513, 532, 537, 529, 521, 538, 513, 487, 493, 551, 513, 517, 556, 497, 510, 497, 533, 506, 520, 532, 562, 534, 501, 480, 516, 498, 542, 521, 491, 503, 490, 456, 516, 520, 490, 507, 536, 536, 518, 514, 499, 489, 522, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[547, 511, 536, 491, 540, 519, 541, 504, 521, 519, 499, 498, 500, 529, 504, 541, 484, 488, 539, 502, 509, 511, 488, 538, 516, 516, 507, 520, 513, 516, 491, 505, 494, 535, 523, 528, 545, 515, 532, 541, 504, 525, 529, 506, 500, 487, 493, 534, 470, 515, 543, 504, 519, 525, 492, 514, 479, 492, 512, 548, 553, 486, 498, 467, 525, 516, 497, 478, 496, 523, 474, 473, 530, 496, 547, 516, 513, 512, 504, 526, 498, 471, 513, 521, 538, 510, 500, 523, 530, 514, 516, 485, 468, 547, 530, 480, 518, 522, 492, 516, 506, 495, 525, 499, 515, 525, 501, 508, 510, 511, 525, 541, 489, 545, 500, 541, 543, 479, 521, 515, 517, 503, 517, 483, 499, 540, 516, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[446, 540, 474, 518, 509, 497, 506, 529, 507, 495, 515, 528, 463, 502, 501, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 523, 509, 469, 532, 478, 510, 492, 516, 528, 503, 528, 518, 501, 516, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 506, 533, 499, 517, 526, 501, 522, 565, 454, 527, 497, 498, 549, 473, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 508, 560, 507, 482, 549, 540, 527, 547, 538, 497, 512, 483, 526, 540, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 477, 537, 535, 505, 501, 523, 530, 517, 521, 517, 494, 480, 551, 536, 472, 5, 4, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 483, 540, 518, 532, 523, 477, 507, 492, 516, 512, 490, 494, 493, 493, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 496, 496, 503, 505, 494, 496, 510, 538, 509, 536, 508, 496, 513, 504, 492, 29, 18, 11, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 475, 507, 511, 520, 541, 537, 531, 475, 493, 544, 530, 513, 539, 500, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8051, 8143, 8153, 8393, 8216, 8109, 8179, 8292] avg=8192.0000 max=8393 max_rank=3 min=8051 min_rank=0 imbalance_ratio=1.024536 max_load_reduction=57143 max_load_reduction_pct=0.871933 moved_to_echo=32796 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[525, 527, 536, 502, 494, 479, 467, 473, 522, 472, 520, 516, 525, 521, 538, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 510, 532, 505, 512, 495, 528, 503, 496, 526, 519, 490, 546, 522, 530, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 530, 486, 488, 511, 559, 541, 531, 516, 516, 531, 515, 509, 515, 495, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 484, 490, 526, 485, 532, 510, 479, 526, 485, 491, 515, 515, 532, 481, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 546, 533, 525, 514, 509, 541, 476, 544, 514, 526, 510, 501, 505, 548, 524, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 526, 519, 486, 517, 498, 487, 496, 519, 525, 545, 470, 515, 512, 495, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 489, 516, 504, 499, 481, 528, 519, 503, 491, 531, 522, 487, 516, 496, 498, 25, 14, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 548, 518, 531, 561, 493, 477, 487, 485, 531, 511, 531, 505, 516, 510, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8104, 8286, 8248, 8075, 8328, 8101, 8142, 8252] avg=8192.0000 max=8328 max_rank=4 min=8075 min_rank=3 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=32823 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[524, 502, 473, 523, 535, 518, 508, 511, 493, 547, 495, 498, 501, 512, 502, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 508, 509, 534, 501, 456, 515, 512, 517, 486, 547, 539, 520, 500, 530, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 456, 484, 512, 528, 507, 471, 515, 529, 531, 518, 511, 510, 466, 509, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 513, 485, 521, 507, 515, 523, 506, 565, 509, 514, 491, 527, 551, 558, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 503, 477, 489, 533, 512, 514, 481, 510, 463, 500, 487, 493, 512, 510, 528, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 503, 505, 498, 478, 534, 521, 533, 524, 508, 550, 548, 504, 506, 510, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 530, 528, 511, 487, 509, 521, 516, 500, 529, 495, 520, 522, 513, 537, 476, 26, 14, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 522, 486, 511, 553, 492, 495, 504, 498, 507, 540, 522, 494, 499, 532, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8168, 8212, 8084, 8342, 8059, 8299, 8224, 8148] avg=8192.0000 max=8342 max_rank=3 min=8059 min_rank=4 imbalance_ratio=1.018311 max_load_reduction=57194 max_load_reduction_pct=0.872711 moved_to_echo=32730 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[547, 511, 536, 491, 538, 519, 541, 504, 521, 519, 499, 498, 500, 529, 504, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 488, 539, 502, 509, 511, 481, 538, 516, 516, 507, 520, 513, 516, 491, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 535, 523, 528, 545, 515, 532, 541, 504, 525, 529, 506, 500, 487, 493, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 515, 528, 504, 519, 525, 492, 514, 479, 492, 512, 548, 553, 486, 498, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 516, 497, 478, 496, 523, 474, 473, 530, 496, 547, 516, 513, 512, 504, 526, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 471, 513, 521, 538, 510, 500, 523, 530, 514, 516, 485, 468, 547, 504, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 522, 492, 516, 506, 495, 525, 499, 515, 525, 501, 508, 510, 511, 525, 541, 26, 15, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 545, 500, 541, 543, 479, 521, 515, 517, 503, 512, 483, 499, 540, 516, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8298, 8136, 8291, 8102, 8130, 8118, 8260, 8201] avg=8192.0000 max=8298 max_rank=0 min=8102 min_rank=3 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32709 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[506, 529, 488, 512, 522, 521, 538, 515, 521, 540, 556, 513, 561, 524, 479, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 512, 494, 518, 488, 501, 533, 498, 499, 507, 503, 465, 492, 475, 512, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 485, 553, 494, 514, 522, 506, 558, 555, 503, 518, 517, 495, 490, 521, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 450, 535, 528, 524, 510, 510, 555, 534, 545, 504, 548, 488, 502, 526, 529, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 517, 526, 522, 522, 551, 515, 499, 496, 511, 504, 512, 490, 512, 505, 525, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 525, 503, 503, 526, 475, 510, 533, 522, 513, 493, 508, 495, 517, 498, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 494, 481, 495, 528, 511, 512, 501, 485, 556, 507, 487, 504, 481, 491, 529, 25, 15, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 498, 496, 492, 489, 528, 527, 514, 567, 504, 524, 509, 472, 503, 529, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8303, 7972, 8265, 8312, 8208, 8186, 8109, 8181] avg=8192.0000 max=8312 max_rank=3 min=7972 min_rank=1 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32684 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[519, 512, 523, 543, 509, 488, 486, 534, 509, 515, 548, 519, 481, 516, 528, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 511, 514, 501, 504, 502, 486, 517, 530, 509, 500, 525, 498, 518, 540, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 556, 489, 533, 487, 492, 490, 511, 519, 492, 544, 512, 528, 500, 516, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 492, 499, 482, 501, 466, 519, 564, 532, 507, 517, 481, 524, 501, 518, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 548, 488, 512, 558, 490, 566, 520, 523, 528, 504, 485, 523, 522, 522, 502, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 512, 535, 523, 503, 496, 527, 553, 494, 508, 530, 509, 530, 521, 494, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 480, 561, 489, 487, 510, 487, 534, 490, 529, 479, 497, 520, 496, 514, 501, 25, 14, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 520, 513, 529, 484, 538, 499, 511, 494, 475, 507, 516, 501, 495, 506, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8243, 8165, 8153, 8144, 8312, 8276, 8119, 8124] avg=8192.0000 max=8312 max_rank=4 min=8119 min_rank=6 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32831 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[474, 533, 502, 534, 551, 480, 505, 507, 492, 531, 465, 540, 522, 464, 503, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 485, 545, 492, 480, 478, 521, 530, 553, 519, 535, 524, 523, 478, 546, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 494, 493, 510, 536, 497, 493, 524, 508, 553, 530, 520, 529, 510, 485, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 489, 517, 511, 544, 467, 498, 502, 477, 504, 520, 514, 533, 536, 545, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 506, 530, 512, 521, 496, 495, 529, 530, 477, 548, 523, 479, 463, 492, 504, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 517, 525, 543, 519, 505, 519, 503, 524, 560, 496, 519, 529, 505, 501, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 500, 553, 462, 506, 496, 510, 522, 485, 539, 509, 489, 495, 487, 504, 510, 25, 15, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 501, 546, 519, 530, 503, 488, 522, 511, 490, 515, 502, 552, 513, 518, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8121, 8266, 8210, 8155, 8108, 8277, 8147, 8252] avg=8192.0000 max=8277 max_rank=5 min=8108 min_rank=4 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32784 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[506, 529, 488, 512, 522, 521, 538, 515, 521, 540, 556, 513, 561, 524, 479, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 512, 494, 518, 488, 501, 533, 498, 499, 507, 503, 465, 492, 475, 512, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 485, 553, 494, 514, 522, 506, 558, 555, 503, 518, 517, 495, 490, 521, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 450, 535, 528, 524, 510, 510, 555, 534, 545, 504, 548, 488, 502, 526, 529, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 517, 526, 522, 522, 551, 515, 499, 496, 511, 504, 512, 490, 512, 505, 525, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 525, 503, 503, 526, 475, 510, 533, 522, 513, 493, 508, 495, 517, 498, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 494, 481, 495, 528, 511, 512, 501, 485, 556, 507, 487, 504, 481, 491, 529, 25, 15, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 498, 496, 492, 489, 528, 527, 514, 567, 504, 524, 509, 472, 503, 529, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8303, 7972, 8265, 8312, 8208, 8186, 8109, 8181] avg=8192.0000 max=8312 max_rank=3 min=7972 min_rank=1 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32684 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[509, 508, 524, 488, 503, 531, 543, 499, 505, 518, 534, 523, 505, 509, 488, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 556, 496, 531, 544, 556, 529, 499, 522, 549, 525, 510, 521, 486, 484, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 500, 541, 517, 465, 507, 538, 465, 532, 498, 538, 500, 487, 511, 535, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 474, 519, 482, 486, 480, 539, 506, 517, 502, 503, 468, 502, 506, 498, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 510, 497, 545, 516, 508, 504, 487, 488, 498, 491, 512, 532, 554, 500, 490, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 495, 478, 472, 513, 532, 537, 529, 521, 538, 513, 487, 493, 551, 488, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 497, 510, 497, 533, 506, 520, 532, 562, 534, 501, 480, 516, 498, 542, 521, 25, 15, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 503, 490, 456, 516, 520, 490, 507, 536, 536, 513, 514, 499, 489, 522, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8248, 8356, 8132, 8013, 8175, 8170, 8356, 8086] avg=8192.0000 max=8356 max_rank=1 min=8013 min_rank=3 imbalance_ratio=1.020020 max_load_reduction=57180 max_load_reduction_pct=0.872498 moved_to_echo=32787 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[547, 511, 536, 491, 538, 519, 541, 504, 521, 519, 499, 498, 500, 529, 504, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 488, 539, 502, 509, 511, 481, 538, 516, 516, 507, 520, 513, 516, 491, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 535, 523, 528, 545, 515, 532, 541, 504, 525, 529, 506, 500, 487, 493, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 515, 528, 504, 519, 525, 492, 514, 479, 492, 512, 548, 553, 486, 498, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 516, 497, 478, 496, 523, 474, 473, 530, 496, 547, 516, 513, 512, 504, 526, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 471, 513, 521, 538, 510, 500, 523, 530, 514, 516, 485, 468, 547, 504, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 522, 492, 516, 506, 495, 525, 499, 515, 525, 501, 508, 510, 511, 525, 541, 26, 15, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 545, 500, 541, 543, 479, 521, 515, 517, 503, 512, 483, 499, 540, 516, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8298, 8136, 8291, 8102, 8130, 8118, 8260, 8201] avg=8192.0000 max=8298 max_rank=0 min=8102 min_rank=3 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32709 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 256, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4352, 4096, 4352, 4096, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 256, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4352, 4096, 4352, 4096, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[518, 493, 512, 527, 534, 495, 484, 543, 513, 498, 548, 521, 512, 481, 483, 513, 503, 500, 489, 509, 548, 548, 526, 528, 479, 489, 515, 503, 494, 541, 511, 522, 464, 512, 532, 525, 587, 501, 522, 511, 487, 557, 541, 475, 476, 487, 498, 527, 527, 535, 525, 544, 510, 476, 508, 501, 535, 496, 507, 509, 479, 477, 484, 519, 520, 512, 490, 506, 492, 483, 534, 533, 491, 520, 502, 535, 504, 522, 517, 530, 523, 535, 502, 550, 503, 521, 531, 488, 505, 506, 501, 515, 509, 530, 525, 524, 507, 518, 508, 507, 522, 470, 486, 513, 506, 497, 505, 502, 496, 534, 554, 550, 504, 550, 493, 480, 493, 510, 517, 505, 508, 492, 511, 500, 536, 519, 529, 541] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[480, 511, 520, 507, 524, 485, 510, 499, 525, 485, 497, 506, 532, 502, 454, 535, 512, 532, 521, 547, 469, 522, 526, 483, 542, 480, 539, 466, 513, 509, 468, 516, 520, 520, 498, 536, 514, 513, 511, 530, 530, 474, 508, 458, 493, 555, 525, 507, 506, 541, 525, 513, 551, 513, 521, 557, 453, 504, 529, 504, 496, 498, 509, 550, 517, 507, 543, 495, 538, 496, 495, 501, 469, 532, 514, 549, 545, 478, 506, 488, 484, 522, 510, 520, 517, 497, 521, 525, 486, 543, 505, 505, 524, 542, 505, 536, 505, 494, 522, 531, 546, 487, 543, 518, 482, 516, 528, 498, 474, 511, 522, 500, 493, 492, 567, 526, 487, 503, 577, 511, 508, 512, 515, 517, 521, 515, 523, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[507, 541, 520, 503, 531, 501, 567, 476, 514, 504, 519, 537, 512, 535, 466, 515, 484, 505, 540, 492, 480, 516, 500, 514, 547, 503, 519, 523, 460, 477, 488, 516, 517, 544, 515, 543, 479, 506, 507, 506, 549, 514, 529, 553, 489, 505, 448, 514, 509, 506, 521, 542, 491, 483, 521, 532, 520, 499, 507, 545, 528, 536, 527, 513, 489, 525, 542, 507, 473, 489, 489, 498, 530, 534, 552, 535, 520, 489, 523, 544, 516, 503, 497, 512, 531, 471, 522, 535, 493, 530, 509, 511, 529, 489, 462, 468, 523, 511, 507, 470, 486, 479, 519, 547, 484, 522, 524, 522, 536, 524, 525, 552, 521, 517, 465, 539, 473, 509, 521, 518, 512, 494, 557, 514, 518, 471, 519, 530] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[482, 548, 500, 521, 545, 482, 501, 493, 543, 541, 502, 474, 502, 512, 468, 494, 529, 517, 488, 489, 512, 508, 494, 493, 486, 492, 505, 564, 523, 523, 468, 502, 525, 505, 510, 532, 519, 538, 521, 508, 522, 493, 488, 495, 550, 509, 498, 501, 518, 522, 545, 537, 462, 508, 531, 498, 505, 536, 517, 500, 493, 526, 559, 519, 545, 523, 469, 549, 508, 533, 493, 523, 517, 522, 504, 497, 518, 523, 498, 538, 524, 520, 483, 513, 493, 512, 532, 536, 510, 497, 492, 481, 528, 511, 498, 511, 509, 513, 511, 507, 507, 530, 488, 526, 524, 497, 525, 503, 537, 524, 517, 521, 517, 495, 496, 496, 485, 521, 513, 540, 507, 509, 532, 527, 516, 559, 500, 492] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[513, 480, 501, 497, 485, 527, 497, 512, 571, 499, 501, 530, 498, 518, 480, 462, 477, 505, 510, 533, 514, 522, 489, 550, 534, 544, 517, 491, 524, 483, 586, 512, 512, 512, 483, 538, 531, 497, 525, 470, 524, 491, 486, 545, 541, 547, 472, 499, 532, 514, 533, 501, 526, 494, 567, 520, 511, 524, 506, 536, 535, 520, 518, 504, 507, 529, 503, 528, 529, 520, 527, 534, 530, 507, 517, 486, 502, 523, 506, 516, 554, 504, 472, 509, 516, 515, 531, 480, 487, 475, 489, 499, 497, 473, 508, 502, 509, 513, 537, 488, 520, 525, 504, 500, 510, 500, 502, 528, 521, 506, 475, 537, 484, 497, 572, 531, 524, 505, 525, 526, 490, 531, 522, 526, 478, 525, 506, 468] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[506, 542, 534, 521, 498, 526, 501, 486, 494, 552, 539, 542, 504, 498, 545, 491, 479, 518, 482, 514, 492, 507, 476, 513, 509, 505, 512, 484, 497, 520, 518, 484, 468, 497, 507, 530, 476, 520, 506, 522, 563, 498, 501, 500, 517, 504, 541, 475, 550, 511, 525, 529, 480, 521, 515, 541, 517, 499, 510, 543, 480, 554, 541, 494, 501, 544, 528, 521, 480, 531, 491, 505, 527, 533, 488, 476, 462, 510, 524, 514, 528, 523, 493, 515, 498, 483, 559, 506, 497, 527, 492, 533, 517, 515, 511, 523, 538, 494, 519, 480, 517, 498, 527, 521, 539, 521, 501, 494, 481, 504, 488, 484, 490, 478, 546, 477, 554, 558, 493, 544, 485, 520, 519, 540, 545, 487, 564, 551] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[507, 528, 548, 489, 465, 520, 487, 542, 537, 495, 480, 518, 480, 497, 499, 506, 502, 524, 519, 490, 495, 515, 526, 533, 469, 518, 507, 519, 538, 506, 497, 521, 525, 533, 502, 503, 581, 462, 538, 520, 477, 510, 509, 502, 529, 531, 511, 500, 498, 535, 538, 540, 527, 484, 513, 516, 546, 491, 500, 534, 534, 525, 491, 524, 502, 519, 484, 518, 526, 508, 486, 512, 499, 523, 534, 501, 555, 502, 514, 520, 503, 509, 483, 509, 508, 539, 527, 498, 550, 525, 520, 522, 517, 533, 470, 485, 532, 525, 492, 537, 517, 474, 476, 499, 533, 513, 530, 521, 498, 534, 487, 512, 528, 537, 517, 508, 503, 489, 502, 540, 484, 476, 504, 507, 506, 515, 506, 527] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[518, 493, 512, 527, 534, 495, 484, 543, 513, 498, 548, 521, 512, 481, 483, 513, 503, 500, 489, 509, 548, 548, 526, 528, 479, 489, 515, 503, 494, 541, 511, 522, 464, 512, 532, 525, 587, 501, 522, 511, 487, 557, 541, 475, 476, 487, 498, 527, 527, 535, 525, 544, 510, 476, 508, 501, 535, 496, 507, 509, 479, 477, 484, 519, 520, 512, 490, 506, 492, 483, 534, 533, 491, 520, 502, 535, 504, 522, 517, 530, 523, 535, 502, 550, 503, 521, 531, 488, 505, 506, 501, 515, 509, 530, 525, 524, 507, 518, 508, 507, 522, 470, 486, 513, 506, 497, 505, 502, 496, 534, 554, 550, 504, 550, 493, 480, 493, 510, 517, 505, 508, 492, 511, 500, 536, 519, 529, 541] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[542, 506, 511, 487, 575, 524, 498, 523, 485, 514, 530, 507, 501, 520, 504, 510, 513, 524, 473, 502, 508, 501, 530, 508, 494, 497, 462, 523, 510, 536, 538, 509, 533, 513, 494, 517, 494, 552, 484, 493, 481, 512, 514, 538, 560, 525, 526, 515, 508, 486, 514, 496, 483, 533, 489, 542, 517, 520, 522, 449, 532, 466, 494, 509, 500, 525, 539, 521, 522, 492, 513, 496, 495, 470, 516, 580, 545, 510, 542, 514, 528, 526, 501, 547, 507, 494, 525, 507, 479, 503, 521, 512, 540, 550, 526, 536, 506, 510, 523, 514, 504, 487, 500, 521, 509, 560, 502, 527, 493, 519, 497, 478, 524, 503, 504, 499, 484, 510, 523, 522, 502, 525, 499, 551, 515, 488, 492, 487] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[480, 511, 520, 507, 524, 485, 510, 499, 525, 485, 497, 506, 532, 502, 454, 535, 25, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 532, 521, 547, 469, 522, 526, 483, 542, 480, 539, 466, 513, 509, 468, 516, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 520, 498, 527, 514, 513, 511, 530, 530, 474, 508, 458, 493, 555, 525, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 541, 473, 513, 551, 513, 521, 557, 453, 504, 529, 504, 496, 498, 509, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 482, 543, 495, 538, 496, 495, 501, 469, 532, 514, 549, 545, 478, 506, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 522, 510, 520, 517, 497, 521, 525, 486, 543, 505, 505, 524, 542, 505, 536, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 494, 522, 531, 546, 487, 543, 518, 482, 516, 528, 498, 474, 511, 522, 500, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 492, 567, 526, 487, 503, 577, 496, 508, 512, 515, 517, 521, 515, 523, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8100, 8197, 8183, 8218, 8148, 8251, 8189, 8250] avg=8192.0000 max=8251 max_rank=5 min=8100 min_rank=0 imbalance_ratio=1.007202 max_load_reduction=57285 max_load_reduction_pct=0.874100 moved_to_echo=32838 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[518, 493, 512, 527, 534, 495, 484, 543, 513, 498, 548, 521, 512, 481, 483, 513, 25, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 500, 489, 509, 548, 548, 526, 528, 479, 489, 515, 503, 494, 541, 511, 522, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 464, 512, 532, 517, 587, 501, 522, 511, 487, 557, 541, 475, 476, 487, 498, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 535, 473, 544, 510, 476, 508, 501, 535, 496, 507, 509, 479, 477, 484, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 487, 490, 506, 492, 483, 534, 533, 491, 520, 502, 535, 504, 522, 517, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 535, 502, 550, 503, 521, 531, 488, 505, 506, 501, 515, 509, 530, 525, 524, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 518, 508, 507, 522, 470, 486, 513, 506, 497, 505, 502, 496, 534, 554, 550, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 550, 493, 480, 493, 510, 517, 490, 508, 492, 511, 500, 536, 519, 529, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8257, 8194, 8080, 8166, 8276, 8187, 8173] avg=8192.0000 max=8276 max_rank=5 min=8080 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32802 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[507, 541, 520, 503, 531, 501, 567, 476, 514, 504, 519, 537, 512, 535, 466, 515, 26, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 505, 540, 492, 480, 516, 500, 514, 547, 503, 519, 523, 460, 477, 488, 516, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 544, 515, 534, 479, 506, 507, 506, 549, 514, 529, 553, 489, 505, 448, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 506, 470, 542, 491, 483, 521, 532, 520, 499, 507, 545, 528, 536, 527, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 499, 542, 507, 473, 489, 489, 498, 530, 534, 552, 535, 520, 489, 523, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 503, 497, 512, 531, 471, 522, 535, 493, 530, 509, 511, 529, 489, 462, 468, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 511, 507, 470, 486, 479, 519, 547, 484, 522, 524, 522, 536, 524, 525, 552, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 517, 465, 539, 473, 509, 521, 503, 512, 494, 557, 514, 518, 471, 519, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8277, 8115, 8209, 8229, 8213, 8087, 8243, 8163] avg=8192.0000 max=8277 max_rank=0 min=8087 min_rank=5 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32706 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[507, 528, 548, 489, 465, 520, 487, 542, 537, 495, 480, 518, 480, 497, 499, 506, 502, 524, 519, 490, 495, 515, 526, 533, 469, 518, 507, 519, 538, 506, 497, 521, 525, 533, 502, 503, 581, 462, 538, 520, 477, 510, 509, 502, 529, 531, 511, 500, 498, 535, 538, 540, 527, 484, 513, 516, 546, 491, 500, 534, 534, 525, 491, 524, 502, 519, 484, 518, 526, 508, 486, 512, 499, 523, 534, 501, 555, 502, 514, 520, 503, 509, 483, 509, 508, 539, 527, 498, 550, 525, 520, 522, 517, 533, 470, 485, 532, 525, 492, 537, 517, 474, 476, 499, 533, 513, 530, 521, 498, 534, 487, 512, 528, 537, 517, 508, 503, 489, 502, 540, 484, 476, 504, 507, 506, 515, 506, 527] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[506, 542, 534, 521, 498, 526, 501, 486, 494, 552, 539, 542, 504, 498, 545, 491, 479, 518, 482, 514, 492, 507, 476, 513, 509, 505, 512, 484, 497, 520, 518, 484, 468, 497, 507, 530, 476, 520, 506, 522, 563, 498, 501, 500, 517, 504, 541, 475, 550, 511, 525, 529, 480, 521, 515, 541, 517, 499, 510, 543, 480, 554, 541, 494, 501, 544, 528, 521, 480, 531, 491, 505, 527, 533, 488, 476, 462, 510, 524, 514, 528, 523, 493, 515, 498, 483, 559, 506, 497, 527, 492, 533, 517, 515, 511, 523, 538, 494, 519, 480, 517, 498, 527, 521, 539, 521, 501, 494, 481, 504, 488, 484, 490, 478, 546, 477, 554, 558, 493, 544, 485, 520, 519, 540, 545, 487, 564, 551] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[482, 548, 500, 521, 545, 482, 501, 493, 543, 541, 502, 474, 502, 512, 468, 494, 529, 517, 488, 489, 512, 508, 494, 493, 486, 492, 505, 564, 523, 523, 468, 502, 525, 505, 510, 532, 519, 538, 521, 508, 522, 493, 488, 495, 550, 509, 498, 501, 518, 522, 545, 537, 462, 508, 531, 498, 505, 536, 517, 500, 493, 526, 559, 519, 545, 523, 469, 549, 508, 533, 493, 523, 517, 522, 504, 497, 518, 523, 498, 538, 524, 520, 483, 513, 493, 512, 532, 536, 510, 497, 492, 481, 528, 511, 498, 511, 509, 513, 511, 507, 507, 530, 488, 526, 524, 497, 525, 503, 537, 524, 517, 521, 517, 495, 496, 496, 485, 521, 513, 540, 507, 509, 532, 527, 516, 559, 500, 492] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[513, 480, 501, 497, 485, 527, 497, 512, 571, 499, 501, 530, 498, 518, 480, 462, 29, 9, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 505, 510, 533, 514, 522, 489, 550, 534, 544, 517, 491, 524, 483, 586, 512, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 512, 483, 526, 531, 497, 525, 470, 524, 491, 486, 545, 541, 547, 472, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 514, 471, 501, 526, 494, 567, 520, 511, 524, 506, 536, 535, 520, 518, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 500, 503, 528, 529, 520, 527, 534, 530, 507, 517, 486, 502, 523, 506, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 504, 472, 509, 516, 515, 531, 480, 487, 475, 489, 499, 497, 473, 508, 502, 12, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 513, 537, 488, 520, 525, 504, 500, 510, 500, 502, 528, 521, 506, 475, 537, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 497, 572, 531, 524, 505, 525, 497, 490, 531, 522, 526, 478, 525, 506, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8115, 8347, 8161, 8279, 8235, 8027, 8191, 8181] avg=8192.0000 max=8347 max_rank=1 min=8027 min_rank=5 imbalance_ratio=1.018921 max_load_reduction=57189 max_load_reduction_pct=0.872635 moved_to_echo=32634 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[507, 541, 520, 503, 531, 501, 567, 476, 514, 504, 519, 537, 512, 535, 466, 515, 26, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 505, 540, 492, 480, 516, 500, 514, 547, 503, 519, 523, 460, 477, 488, 516, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 544, 515, 534, 479, 506, 507, 506, 549, 514, 529, 553, 489, 505, 448, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 506, 470, 542, 491, 483, 521, 532, 520, 499, 507, 545, 528, 536, 527, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 499, 542, 507, 473, 489, 489, 498, 530, 534, 552, 535, 520, 489, 523, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 503, 497, 512, 531, 471, 522, 535, 493, 530, 509, 511, 529, 489, 462, 468, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 511, 507, 470, 486, 479, 519, 547, 484, 522, 524, 522, 536, 524, 525, 552, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 517, 465, 539, 473, 509, 521, 503, 512, 494, 557, 514, 518, 471, 519, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8277, 8115, 8209, 8229, 8213, 8087, 8243, 8163] avg=8192.0000 max=8277 max_rank=0 min=8087 min_rank=5 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32706 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[518, 493, 512, 527, 534, 495, 484, 543, 513, 498, 548, 521, 512, 481, 483, 513, 25, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 500, 489, 509, 548, 548, 526, 528, 479, 489, 515, 503, 494, 541, 511, 522, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 464, 512, 532, 517, 587, 501, 522, 511, 487, 557, 541, 475, 476, 487, 498, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 535, 473, 544, 510, 476, 508, 501, 535, 496, 507, 509, 479, 477, 484, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 487, 490, 506, 492, 483, 534, 533, 491, 520, 502, 535, 504, 522, 517, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 535, 502, 550, 503, 521, 531, 488, 505, 506, 501, 515, 509, 530, 525, 524, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 518, 508, 507, 522, 470, 486, 513, 506, 497, 505, 502, 496, 534, 554, 550, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 550, 493, 480, 493, 510, 517, 490, 508, 492, 511, 500, 536, 519, 529, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8257, 8194, 8080, 8166, 8276, 8187, 8173] avg=8192.0000 max=8276 max_rank=5 min=8080 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32802 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[480, 511, 520, 507, 524, 485, 510, 499, 525, 485, 497, 506, 532, 502, 454, 535, 25, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 532, 521, 547, 469, 522, 526, 483, 542, 480, 539, 466, 513, 509, 468, 516, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 520, 498, 527, 514, 513, 511, 530, 530, 474, 508, 458, 493, 555, 525, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 541, 473, 513, 551, 513, 521, 557, 453, 504, 529, 504, 496, 498, 509, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 482, 543, 495, 538, 496, 495, 501, 469, 532, 514, 549, 545, 478, 506, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 522, 510, 520, 517, 497, 521, 525, 486, 543, 505, 505, 524, 542, 505, 536, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 494, 522, 531, 546, 487, 543, 518, 482, 516, 528, 498, 474, 511, 522, 500, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 492, 567, 526, 487, 503, 577, 496, 508, 512, 515, 517, 521, 515, 523, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8100, 8197, 8183, 8218, 8148, 8251, 8189, 8250] avg=8192.0000 max=8251 max_rank=5 min=8100 min_rank=0 imbalance_ratio=1.007202 max_load_reduction=57285 max_load_reduction_pct=0.874100 moved_to_echo=32838 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[542, 506, 511, 487, 575, 524, 498, 523, 485, 514, 530, 507, 501, 520, 504, 510, 26, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 524, 473, 502, 508, 501, 530, 508, 494, 497, 462, 523, 510, 536, 538, 509, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 513, 494, 509, 494, 552, 484, 493, 481, 512, 514, 538, 560, 525, 526, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 486, 463, 496, 483, 533, 489, 542, 517, 520, 522, 449, 532, 466, 494, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 499, 539, 521, 522, 492, 513, 496, 495, 470, 516, 580, 545, 510, 542, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 526, 501, 547, 507, 494, 525, 507, 479, 503, 521, 512, 540, 550, 526, 536, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 510, 523, 514, 504, 487, 500, 521, 509, 560, 502, 527, 493, 519, 497, 478, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 503, 504, 499, 484, 510, 523, 507, 502, 525, 499, 551, 515, 488, 492, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8266, 8179, 8243, 8009, 8254, 8310, 8162, 8113] avg=8192.0000 max=8310 max_rank=5 min=8009 min_rank=3 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32839 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[507, 528, 548, 489, 465, 520, 487, 542, 537, 495, 480, 518, 480, 497, 499, 506, 25, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 524, 519, 490, 495, 515, 526, 533, 469, 518, 507, 519, 538, 506, 497, 521, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 533, 502, 495, 581, 462, 538, 520, 477, 510, 509, 502, 529, 531, 511, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 535, 484, 540, 527, 484, 513, 516, 546, 491, 500, 534, 534, 525, 491, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 494, 484, 518, 526, 508, 486, 512, 499, 523, 534, 501, 555, 502, 514, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 509, 483, 509, 508, 539, 527, 498, 550, 525, 520, 522, 517, 533, 470, 485, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 525, 492, 537, 517, 474, 476, 499, 533, 513, 530, 521, 498, 534, 487, 512, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 537, 517, 508, 503, 489, 502, 524, 484, 476, 504, 507, 506, 515, 506, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8127, 8232, 8225, 8242, 8178, 8206, 8193, 8133] avg=8192.0000 max=8242 max_rank=3 min=8127 min_rank=0 imbalance_ratio=1.006104 max_load_reduction=57294 max_load_reduction_pct=0.874237 moved_to_echo=32710 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[482, 548, 500, 521, 545, 482, 501, 493, 543, 541, 502, 474, 502, 512, 468, 494, 26, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 517, 488, 489, 512, 508, 494, 493, 486, 492, 505, 564, 523, 523, 468, 502, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 505, 510, 523, 519, 538, 521, 508, 522, 493, 488, 495, 550, 509, 498, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 522, 490, 537, 462, 508, 531, 498, 505, 536, 517, 500, 493, 526, 559, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 497, 469, 549, 508, 533, 493, 523, 517, 522, 504, 497, 518, 523, 498, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 520, 483, 513, 493, 512, 532, 536, 510, 497, 492, 481, 528, 511, 498, 511, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 513, 511, 507, 507, 530, 488, 526, 524, 497, 525, 503, 537, 524, 517, 521, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 495, 496, 496, 485, 521, 513, 524, 507, 509, 532, 527, 516, 559, 500, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8138, 8147, 8205, 8221, 8234, 8150, 8252, 8189] avg=8192.0000 max=8252 max_rank=6 min=8138 min_rank=0 imbalance_ratio=1.007324 max_load_reduction=57284 max_load_reduction_pct=0.874084 moved_to_echo=32825 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[506, 542, 534, 521, 498, 526, 501, 486, 494, 552, 539, 542, 504, 498, 545, 491, 27, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 518, 482, 514, 492, 507, 476, 513, 509, 505, 512, 484, 497, 520, 518, 484, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 497, 507, 521, 476, 520, 506, 522, 563, 498, 501, 500, 517, 504, 541, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 511, 473, 529, 480, 521, 515, 541, 517, 499, 510, 543, 480, 554, 541, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 517, 528, 521, 480, 531, 491, 505, 527, 533, 488, 476, 462, 510, 524, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 523, 493, 515, 498, 483, 559, 506, 497, 527, 492, 533, 517, 515, 511, 523, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 494, 519, 480, 517, 498, 527, 521, 539, 521, 501, 494, 481, 504, 488, 484, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 478, 546, 477, 554, 558, 493, 527, 485, 520, 519, 540, 545, 487, 564, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8310, 8062, 8116, 8258, 8108, 8229, 8119, 8334] avg=8192.0000 max=8334 max_rank=7 min=8062 min_rank=1 imbalance_ratio=1.017334 max_load_reduction=57202 max_load_reduction_pct=0.872833 moved_to_echo=32790 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[507, 528, 548, 489, 465, 520, 487, 542, 537, 495, 480, 518, 480, 497, 499, 506, 25, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 524, 519, 490, 495, 515, 526, 533, 469, 518, 507, 519, 538, 506, 497, 521, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 533, 502, 495, 581, 462, 538, 520, 477, 510, 509, 502, 529, 531, 511, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 535, 484, 540, 527, 484, 513, 516, 546, 491, 500, 534, 534, 525, 491, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 494, 484, 518, 526, 508, 486, 512, 499, 523, 534, 501, 555, 502, 514, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 509, 483, 509, 508, 539, 527, 498, 550, 525, 520, 522, 517, 533, 470, 485, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 525, 492, 537, 517, 474, 476, 499, 533, 513, 530, 521, 498, 534, 487, 512, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 537, 517, 508, 503, 489, 502, 524, 484, 476, 504, 507, 506, 515, 506, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8127, 8232, 8225, 8242, 8178, 8206, 8193, 8133] avg=8192.0000 max=8242 max_rank=3 min=8127 min_rank=0 imbalance_ratio=1.006104 max_load_reduction=57294 max_load_reduction_pct=0.874237 moved_to_echo=32710 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[542, 506, 511, 487, 575, 524, 498, 523, 485, 514, 530, 507, 501, 520, 504, 510, 26, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 524, 473, 502, 508, 501, 530, 508, 494, 497, 462, 523, 510, 536, 538, 509, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 513, 494, 509, 494, 552, 484, 493, 481, 512, 514, 538, 560, 525, 526, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 486, 463, 496, 483, 533, 489, 542, 517, 520, 522, 449, 532, 466, 494, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 499, 539, 521, 522, 492, 513, 496, 495, 470, 516, 580, 545, 510, 542, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 526, 501, 547, 507, 494, 525, 507, 479, 503, 521, 512, 540, 550, 526, 536, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 510, 523, 514, 504, 487, 500, 521, 509, 560, 502, 527, 493, 519, 497, 478, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 503, 504, 499, 484, 510, 523, 507, 502, 525, 499, 551, 515, 488, 492, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8266, 8179, 8243, 8009, 8254, 8310, 8162, 8113] avg=8192.0000 max=8310 max_rank=5 min=8009 min_rank=3 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32839 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 3968, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 3968, 4096, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 3840, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 3968, 4224, 4096, 4096, 4352, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 3840, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 3968, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[516, 569, 534, 447, 521, 513, 552, 516, 477, 518, 479, 505, 498, 462, 542, 545, 497, 480, 513, 550, 507, 536, 530, 496, 530, 498, 534, 522, 531, 516, 500, 528, 526, 504, 545, 528, 508, 544, 496, 538, 515, 479, 512, 516, 498, 492, 493, 513, 512, 503, 513, 502, 479, 514, 515, 528, 529, 518, 496, 500, 511, 482, 539, 520, 530, 494, 501, 494, 506, 501, 505, 498, 523, 531, 488, 494, 534, 510, 511, 540, 492, 498, 534, 494, 448, 497, 519, 488, 527, 530, 533, 534, 508, 528, 522, 479, 524, 496, 528, 457, 530, 493, 524, 524, 537, 502, 507, 546, 525, 504, 505, 502, 459, 536, 508, 514, 534, 533, 504, 466, 498, 511, 545, 534, 547, 470, 545, 507] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[486, 519, 518, 500, 472, 508, 528, 507, 502, 511, 478, 510, 535, 487, 512, 524, 513, 527, 517, 494, 511, 473, 503, 529, 535, 493, 550, 497, 537, 493, 529, 500, 553, 531, 480, 517, 513, 539, 516, 531, 536, 521, 499, 508, 499, 495, 502, 499, 501, 514, 486, 541, 530, 475, 533, 508, 523, 556, 479, 526, 504, 511, 503, 524, 500, 551, 508, 529, 487, 500, 508, 505, 503, 491, 504, 491, 537, 544, 512, 491, 490, 521, 499, 542, 540, 515, 466, 495, 508, 521, 524, 477, 568, 489, 511, 526, 482, 527, 523, 492, 517, 534, 494, 531, 496, 518, 487, 521, 546, 531, 515, 533, 512, 527, 492, 486, 535, 520, 554, 497, 493, 546, 471, 533, 531, 476, 491, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[479, 519, 536, 487, 501, 501, 484, 521, 542, 531, 543, 536, 524, 519, 513, 558, 513, 471, 497, 530, 534, 513, 571, 516, 525, 483, 535, 528, 509, 516, 526, 498, 508, 484, 514, 506, 560, 533, 477, 495, 502, 499, 533, 546, 517, 524, 509, 526, 521, 525, 532, 535, 495, 493, 509, 524, 510, 502, 478, 508, 512, 520, 524, 504, 506, 488, 515, 499, 492, 513, 495, 504, 502, 478, 553, 524, 520, 518, 535, 544, 515, 511, 524, 476, 520, 517, 508, 502, 513, 491, 484, 524, 471, 510, 469, 516, 542, 474, 516, 501, 485, 481, 502, 537, 528, 528, 502, 523, 511, 533, 514, 497, 519, 522, 467, 511, 541, 529, 554, 479, 438, 501, 507, 546, 539, 471, 523, 494] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[474, 497, 524, 488, 533, 506, 485, 479, 514, 507, 509, 492, 512, 549, 518, 465, 504, 497, 532, 485, 562, 517, 498, 529, 490, 503, 506, 516, 540, 486, 516, 489, 520, 545, 531, 533, 519, 489, 504, 493, 546, 493, 526, 521, 520, 520, 523, 546, 475, 509, 520, 482, 507, 504, 483, 514, 504, 479, 552, 509, 508, 527, 497, 502, 529, 507, 526, 525, 512, 522, 526, 497, 515, 562, 502, 495, 539, 478, 468, 541, 504, 514, 517, 525, 525, 485, 529, 518, 510, 506, 505, 492, 520, 512, 500, 483, 495, 532, 514, 499, 519, 504, 530, 531, 499, 539, 556, 564, 518, 509, 528, 520, 516, 528, 490, 529, 502, 507, 525, 470, 509, 525, 504, 468, 513, 573, 479, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[506, 533, 511, 539, 516, 543, 501, 513, 501, 493, 498, 532, 520, 523, 504, 530, 506, 526, 543, 514, 554, 476, 529, 474, 520, 539, 538, 517, 503, 543, 472, 560, 522, 490, 515, 489, 545, 496, 507, 509, 500, 510, 510, 490, 493, 497, 493, 507, 534, 482, 549, 520, 536, 510, 503, 521, 523, 506, 500, 517, 495, 537, 517, 516, 532, 504, 486, 528, 566, 501, 514, 512, 474, 535, 489, 517, 498, 536, 513, 486, 559, 515, 519, 493, 471, 536, 510, 514, 485, 502, 499, 489, 533, 507, 497, 489, 463, 500, 502, 525, 508, 524, 517, 514, 523, 485, 505, 519, 518, 504, 545, 516, 488, 465, 522, 514, 509, 493, 530, 477, 516, 553, 509, 524, 503, 545, 464, 531] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[516, 569, 534, 447, 521, 513, 552, 516, 477, 518, 479, 505, 498, 462, 542, 545, 497, 480, 513, 550, 507, 536, 530, 496, 530, 498, 534, 522, 531, 516, 500, 528, 526, 504, 545, 528, 508, 544, 496, 538, 515, 479, 512, 516, 498, 492, 493, 513, 512, 503, 513, 502, 479, 514, 515, 528, 529, 518, 496, 500, 511, 482, 539, 520, 530, 494, 501, 494, 506, 501, 505, 498, 523, 531, 488, 494, 534, 510, 511, 540, 492, 498, 534, 494, 448, 497, 519, 488, 527, 530, 533, 534, 508, 528, 522, 479, 524, 496, 528, 457, 530, 493, 524, 524, 537, 502, 507, 546, 525, 504, 505, 502, 459, 536, 508, 514, 534, 533, 504, 466, 498, 511, 545, 534, 547, 470, 545, 507] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[517, 485, 535, 509, 522, 471, 525, 537, 527, 499, 491, 522, 513, 482, 511, 505, 525, 485, 517, 512, 561, 481, 510, 529, 548, 502, 523, 515, 450, 500, 506, 515, 511, 498, 541, 511, 545, 510, 501, 542, 494, 512, 525, 484, 531, 506, 476, 539, 517, 484, 472, 500, 516, 496, 512, 476, 513, 533, 514, 504, 484, 535, 522, 478, 521, 521, 522, 472, 529, 489, 513, 508, 537, 515, 509, 541, 522, 488, 518, 522, 546, 539, 513, 496, 533, 472, 529, 540, 519, 514, 535, 522, 476, 504, 535, 507, 479, 524, 539, 498, 511, 489, 523, 506, 515, 483, 526, 517, 512, 553, 499, 541, 524, 507, 552, 512, 524, 527, 520, 480, 472, 486, 505, 509, 526, 496, 535, 527] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[512, 534, 515, 527, 566, 493, 521, 522, 491, 524, 486, 512, 506, 529, 519, 488, 543, 527, 518, 506, 507, 506, 482, 569, 472, 529, 504, 498, 518, 501, 528, 541, 515, 537, 516, 507, 525, 545, 496, 527, 495, 498, 531, 520, 470, 498, 495, 558, 484, 485, 507, 531, 526, 515, 526, 506, 508, 517, 499, 554, 479, 528, 524, 506, 502, 524, 515, 483, 496, 463, 504, 492, 511, 513, 534, 499, 512, 504, 499, 517, 476, 504, 532, 502, 519, 516, 497, 518, 530, 514, 528, 520, 506, 486, 558, 509, 494, 498, 483, 532, 510, 533, 500, 494, 547, 499, 540, 489, 543, 489, 515, 523, 483, 539, 549, 501, 513, 494, 542, 490, 485, 515, 508, 487, 535, 499, 475, 527] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[474, 497, 524, 488, 533, 506, 485, 479, 514, 507, 509, 492, 512, 549, 518, 465, 504, 497, 532, 485, 562, 517, 498, 529, 490, 503, 506, 516, 540, 486, 516, 489, 520, 545, 531, 533, 519, 489, 504, 493, 546, 493, 526, 521, 520, 520, 523, 546, 475, 509, 520, 482, 507, 504, 483, 514, 504, 479, 552, 509, 508, 527, 497, 502, 529, 507, 526, 525, 512, 522, 526, 497, 515, 562, 502, 495, 539, 478, 468, 541, 504, 514, 517, 525, 525, 485, 529, 518, 510, 506, 505, 492, 520, 512, 500, 483, 495, 532, 514, 499, 519, 504, 530, 531, 499, 539, 556, 564, 518, 509, 528, 520, 516, 528, 490, 529, 502, 507, 525, 470, 509, 525, 504, 468, 513, 573, 479, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[479, 519, 536, 487, 501, 501, 484, 521, 542, 531, 543, 536, 524, 519, 513, 558, 513, 471, 497, 530, 534, 513, 571, 516, 525, 483, 535, 528, 509, 516, 526, 498, 508, 484, 514, 506, 560, 533, 477, 495, 502, 499, 533, 546, 517, 524, 509, 526, 521, 525, 532, 535, 495, 493, 509, 524, 510, 502, 478, 508, 512, 520, 524, 504, 506, 488, 515, 499, 492, 513, 495, 504, 502, 478, 553, 524, 520, 518, 535, 544, 515, 511, 524, 476, 520, 517, 508, 502, 513, 491, 484, 524, 471, 510, 469, 516, 542, 474, 516, 501, 485, 481, 502, 537, 528, 528, 502, 523, 511, 533, 514, 497, 519, 522, 467, 511, 541, 529, 554, 479, 438, 501, 507, 546, 539, 471, 523, 494] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[486, 519, 518, 500, 472, 508, 528, 507, 502, 511, 478, 510, 535, 487, 512, 524, 513, 527, 517, 494, 511, 473, 503, 529, 535, 493, 550, 497, 537, 493, 529, 500, 553, 531, 480, 517, 513, 539, 516, 531, 536, 521, 499, 508, 499, 495, 502, 499, 501, 514, 486, 541, 530, 475, 533, 508, 523, 556, 479, 526, 504, 511, 503, 524, 500, 551, 508, 529, 487, 500, 508, 505, 503, 491, 504, 491, 537, 544, 512, 491, 490, 521, 499, 542, 540, 515, 466, 495, 508, 521, 524, 477, 568, 489, 511, 526, 482, 527, 523, 492, 517, 534, 494, 531, 496, 518, 487, 521, 546, 531, 515, 533, 512, 527, 492, 486, 535, 520, 554, 497, 493, 546, 471, 533, 531, 476, 491, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[506, 540, 519, 533, 499, 531, 490, 531, 503, 499, 490, 510, 534, 501, 529, 485, 533, 476, 500, 516, 525, 534, 550, 492, 492, 511, 515, 475, 512, 480, 493, 475, 513, 537, 525, 518, 507, 524, 476, 512, 489, 500, 499, 529, 505, 534, 489, 515, 534, 529, 510, 500, 508, 511, 519, 520, 504, 510, 503, 544, 498, 515, 521, 524, 482, 518, 486, 521, 529, 512, 499, 483, 497, 503, 491, 497, 478, 514, 561, 555, 498, 521, 541, 534, 490, 481, 525, 526, 525, 518, 515, 515, 514, 510, 539, 478, 519, 481, 501, 498, 504, 511, 517, 519, 518, 519, 508, 490, 517, 471, 538, 530, 520, 499, 499, 518, 524, 565, 526, 498, 537, 518, 485, 520, 543, 538, 538, 510] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[517, 485, 535, 509, 522, 471, 525, 537, 527, 499, 491, 522, 513, 482, 511, 505, 525, 485, 517, 512, 561, 481, 510, 529, 548, 502, 523, 515, 450, 500, 506, 515, 511, 498, 541, 511, 545, 510, 501, 542, 494, 512, 525, 484, 531, 506, 476, 539, 517, 484, 472, 500, 516, 496, 512, 476, 513, 533, 514, 504, 484, 535, 522, 478, 521, 521, 522, 472, 529, 489, 513, 508, 537, 515, 509, 541, 522, 488, 518, 522, 546, 539, 513, 496, 533, 472, 529, 540, 519, 514, 535, 522, 476, 504, 535, 507, 479, 524, 539, 498, 511, 489, 523, 506, 515, 483, 526, 517, 512, 553, 499, 541, 524, 507, 552, 512, 524, 527, 520, 480, 472, 486, 505, 509, 526, 496, 535, 527] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[516, 569, 534, 447, 521, 513, 552, 516, 477, 518, 479, 505, 498, 462, 542, 545, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 480, 513, 550, 486, 536, 530, 496, 530, 498, 534, 522, 531, 516, 500, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 504, 545, 528, 487, 544, 496, 538, 515, 479, 512, 516, 498, 492, 493, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 503, 513, 502, 479, 514, 515, 528, 529, 518, 496, 500, 511, 482, 539, 520, 14, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 494, 501, 494, 506, 501, 505, 498, 523, 531, 488, 494, 534, 510, 511, 540, 19, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 498, 534, 494, 448, 497, 519, 488, 527, 530, 533, 534, 508, 528, 522, 479, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 496, 528, 457, 530, 493, 524, 524, 537, 502, 507, 546, 506, 504, 505, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 459, 536, 508, 514, 534, 533, 504, 466, 498, 511, 545, 534, 547, 470, 545, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8247, 8186, 8177, 8180, 8151, 8185, 8211] avg=8192.0000 max=8247 max_rank=1 min=8151 min_rank=5 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32727 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[486, 519, 518, 500, 472, 508, 528, 507, 502, 511, 478, 510, 535, 487, 512, 524, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 527, 517, 494, 490, 473, 503, 529, 535, 493, 550, 497, 537, 493, 529, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 531, 480, 517, 492, 539, 516, 531, 536, 521, 499, 508, 499, 495, 502, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 514, 486, 541, 530, 475, 533, 508, 523, 556, 479, 526, 504, 511, 503, 524, 14, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 551, 508, 529, 487, 500, 508, 505, 503, 491, 504, 491, 537, 544, 512, 491, 19, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 521, 499, 542, 540, 515, 466, 495, 508, 521, 524, 477, 568, 489, 511, 526, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 527, 523, 492, 517, 534, 494, 531, 496, 518, 487, 521, 527, 531, 515, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 527, 492, 486, 535, 520, 554, 497, 493, 546, 471, 533, 531, 476, 491, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8102, 8180, 8218, 8230, 8181, 8212, 8228, 8185] avg=8192.0000 max=8230 max_rank=3 min=8102 min_rank=0 imbalance_ratio=1.004639 max_load_reduction=57306 max_load_reduction_pct=0.874420 moved_to_echo=32806 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[512, 534, 515, 527, 566, 493, 521, 522, 491, 524, 486, 512, 506, 529, 519, 488, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 527, 518, 506, 486, 506, 482, 569, 472, 529, 504, 498, 518, 501, 528, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 537, 516, 507, 504, 545, 496, 527, 495, 498, 531, 520, 470, 498, 495, 558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 485, 507, 531, 526, 515, 526, 506, 508, 517, 499, 554, 479, 528, 524, 506, 14, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 524, 515, 483, 496, 463, 504, 492, 511, 513, 534, 499, 512, 504, 499, 517, 19, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 504, 532, 502, 519, 516, 497, 518, 530, 514, 528, 520, 506, 486, 558, 509, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 498, 483, 532, 510, 533, 500, 494, 547, 499, 540, 489, 524, 489, 515, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 539, 549, 501, 513, 494, 542, 490, 485, 515, 508, 487, 535, 499, 475, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8250, 8228, 8212, 8211, 8088, 8235, 8170, 8142] avg=8192.0000 max=8250 max_rank=0 min=8088 min_rank=4 imbalance_ratio=1.007080 max_load_reduction=57286 max_load_reduction_pct=0.874115 moved_to_echo=32635 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[479, 519, 536, 487, 501, 501, 484, 521, 542, 531, 543, 536, 524, 519, 513, 558, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 471, 497, 530, 499, 513, 571, 516, 525, 483, 535, 528, 509, 516, 526, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 484, 514, 506, 532, 533, 477, 495, 502, 499, 533, 546, 517, 524, 509, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 525, 532, 535, 495, 493, 509, 524, 510, 502, 478, 508, 512, 520, 524, 504, 18, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 488, 515, 499, 492, 513, 495, 504, 502, 478, 553, 524, 520, 518, 535, 544, 24, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 511, 524, 476, 520, 517, 508, 502, 513, 491, 484, 524, 471, 510, 469, 516, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 474, 516, 501, 485, 481, 502, 537, 528, 528, 502, 523, 477, 533, 514, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 522, 467, 511, 541, 529, 554, 479, 438, 501, 507, 546, 539, 471, 523, 494, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8303, 8230, 8205, 8214, 8218, 8078, 8140, 8148] avg=8192.0000 max=8303 max_rank=0 min=8078 min_rank=5 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32584 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[474, 497, 524, 488, 533, 506, 485, 479, 514, 507, 509, 492, 512, 549, 518, 465, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 497, 532, 485, 539, 517, 498, 529, 490, 503, 506, 516, 540, 486, 516, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 545, 531, 533, 498, 489, 504, 493, 546, 493, 526, 521, 520, 520, 523, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 509, 520, 482, 507, 504, 483, 514, 504, 479, 552, 509, 508, 527, 497, 502, 13, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 507, 526, 525, 512, 522, 526, 497, 515, 562, 502, 495, 539, 478, 468, 541, 19, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 514, 517, 525, 525, 485, 529, 518, 510, 506, 505, 492, 520, 512, 500, 483, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 532, 514, 499, 519, 504, 530, 531, 499, 539, 556, 564, 501, 509, 528, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 528, 490, 529, 502, 507, 525, 470, 509, 525, 504, 468, 513, 573, 479, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8056, 8147, 8308, 8087, 8264, 8167, 8340, 8167] avg=8192.0000 max=8340 max_rank=6 min=8056 min_rank=0 imbalance_ratio=1.018066 max_load_reduction=57196 max_load_reduction_pct=0.872742 moved_to_echo=32938 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[512, 534, 515, 527, 566, 493, 521, 522, 491, 524, 486, 512, 506, 529, 519, 488, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 527, 518, 506, 486, 506, 482, 569, 472, 529, 504, 498, 518, 501, 528, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 537, 516, 507, 504, 545, 496, 527, 495, 498, 531, 520, 470, 498, 495, 558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 485, 507, 531, 526, 515, 526, 506, 508, 517, 499, 554, 479, 528, 524, 506, 14, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 524, 515, 483, 496, 463, 504, 492, 511, 513, 534, 499, 512, 504, 499, 517, 19, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 504, 532, 502, 519, 516, 497, 518, 530, 514, 528, 520, 506, 486, 558, 509, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 498, 483, 532, 510, 533, 500, 494, 547, 499, 540, 489, 524, 489, 515, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 539, 549, 501, 513, 494, 542, 490, 485, 515, 508, 487, 535, 499, 475, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8250, 8228, 8212, 8211, 8088, 8235, 8170, 8142] avg=8192.0000 max=8250 max_rank=0 min=8088 min_rank=4 imbalance_ratio=1.007080 max_load_reduction=57286 max_load_reduction_pct=0.874115 moved_to_echo=32635 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[506, 533, 511, 539, 516, 543, 501, 513, 501, 493, 498, 532, 520, 523, 504, 530, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 526, 543, 514, 531, 476, 529, 474, 520, 539, 538, 517, 503, 543, 472, 560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 490, 515, 489, 523, 496, 507, 509, 500, 510, 510, 490, 493, 497, 493, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 482, 549, 520, 536, 510, 503, 521, 523, 506, 500, 517, 495, 537, 517, 516, 13, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 504, 486, 528, 566, 501, 514, 512, 474, 535, 489, 517, 498, 536, 513, 486, 20, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 515, 519, 493, 471, 536, 510, 514, 485, 502, 499, 489, 533, 507, 497, 489, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 463, 500, 502, 525, 508, 524, 517, 514, 523, 485, 505, 519, 501, 504, 545, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 465, 522, 514, 509, 493, 530, 477, 516, 553, 509, 524, 503, 545, 464, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8267, 8291, 8051, 8281, 8212, 8140, 8151, 8143] avg=8192.0000 max=8291 max_rank=1 min=8051 min_rank=2 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32646 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[517, 485, 535, 509, 522, 471, 525, 537, 527, 499, 491, 522, 513, 482, 511, 505, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 485, 517, 512, 538, 481, 510, 529, 548, 502, 523, 515, 450, 500, 506, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 498, 541, 511, 523, 510, 501, 542, 494, 512, 525, 484, 531, 506, 476, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 484, 472, 500, 516, 496, 512, 476, 513, 533, 514, 504, 484, 535, 522, 478, 13, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 521, 522, 472, 529, 489, 513, 508, 537, 515, 509, 541, 522, 488, 518, 522, 20, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 539, 513, 496, 533, 472, 529, 540, 519, 514, 535, 522, 476, 504, 535, 507, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 524, 539, 498, 511, 489, 523, 506, 515, 483, 526, 517, 495, 553, 499, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 507, 552, 512, 524, 527, 520, 480, 472, 486, 505, 509, 526, 496, 535, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8155, 8156, 8204, 8071, 8248, 8302, 8198, 8202] avg=8192.0000 max=8302 max_rank=5 min=8071 min_rank=3 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32950 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[506, 533, 511, 539, 516, 543, 501, 513, 501, 493, 498, 532, 520, 523, 504, 530, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 526, 543, 514, 531, 476, 529, 474, 520, 539, 538, 517, 503, 543, 472, 560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 490, 515, 489, 523, 496, 507, 509, 500, 510, 510, 490, 493, 497, 493, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 482, 549, 520, 536, 510, 503, 521, 523, 506, 500, 517, 495, 537, 517, 516, 13, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 504, 486, 528, 566, 501, 514, 512, 474, 535, 489, 517, 498, 536, 513, 486, 20, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 515, 519, 493, 471, 536, 510, 514, 485, 502, 499, 489, 533, 507, 497, 489, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 463, 500, 502, 525, 508, 524, 517, 514, 523, 485, 505, 519, 501, 504, 545, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 465, 522, 514, 509, 493, 530, 477, 516, 553, 509, 524, 503, 545, 464, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8267, 8291, 8051, 8281, 8212, 8140, 8151, 8143] avg=8192.0000 max=8291 max_rank=1 min=8051 min_rank=2 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32646 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[486, 519, 518, 500, 472, 508, 528, 507, 502, 511, 478, 510, 535, 487, 512, 524, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 527, 517, 494, 490, 473, 503, 529, 535, 493, 550, 497, 537, 493, 529, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 531, 480, 517, 492, 539, 516, 531, 536, 521, 499, 508, 499, 495, 502, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 514, 486, 541, 530, 475, 533, 508, 523, 556, 479, 526, 504, 511, 503, 524, 14, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 551, 508, 529, 487, 500, 508, 505, 503, 491, 504, 491, 537, 544, 512, 491, 19, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 521, 499, 542, 540, 515, 466, 495, 508, 521, 524, 477, 568, 489, 511, 526, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 527, 523, 492, 517, 534, 494, 531, 496, 518, 487, 521, 527, 531, 515, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 527, 492, 486, 535, 520, 554, 497, 493, 546, 471, 533, 531, 476, 491, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8102, 8180, 8218, 8230, 8181, 8212, 8228, 8185] avg=8192.0000 max=8230 max_rank=3 min=8102 min_rank=0 imbalance_ratio=1.004639 max_load_reduction=57306 max_load_reduction_pct=0.874420 moved_to_echo=32806 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[506, 540, 519, 533, 499, 531, 490, 531, 503, 499, 490, 510, 534, 501, 529, 485, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 476, 500, 516, 503, 534, 550, 492, 492, 511, 515, 475, 512, 480, 493, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 537, 525, 518, 487, 524, 476, 512, 489, 500, 499, 529, 505, 534, 489, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 529, 510, 500, 508, 511, 519, 520, 504, 510, 503, 544, 498, 515, 521, 524, 13, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 518, 486, 521, 529, 512, 499, 483, 497, 503, 491, 497, 478, 514, 561, 555, 18, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 521, 541, 534, 490, 481, 525, 526, 525, 518, 515, 515, 514, 510, 539, 478, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 481, 501, 498, 504, 511, 517, 519, 518, 519, 508, 490, 500, 471, 538, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 499, 499, 518, 524, 565, 526, 498, 537, 518, 485, 520, 543, 538, 538, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8057, 8152, 8265, 8145, 8251, 8124, 8338] avg=8192.0000 max=8338 max_rank=7 min=8057 min_rank=1 imbalance_ratio=1.017822 max_load_reduction=57198 max_load_reduction_pct=0.872772 moved_to_echo=32858 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4352, 3968, 3968, 4224, 4096, 4224, 4352, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True - [2026-08-26 14:49:26] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 685.1 | throughput per GPU (TFLOP/s/GPU): 263.9 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.117269E+01 | loss scale: 1.0 | grad norm: 44.201 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True + [2026-08-26 15:25:30] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 704.0 | throughput per GPU (TFLOP/s/GPU): 256.8 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.117269E+01 | loss scale: 1.0 | grad norm: 44.201 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[518, 489, 504, 511, 495, 507, 533, 542, 522, 533, 505, 523, 493, 516, 524, 503, 506, 531, 493, 488, 511, 535, 526, 510, 465, 505, 528, 554, 517, 506, 490, 541, 468, 503, 519, 497, 552, 525, 491, 524, 515, 526, 513, 513, 511, 499, 483, 505, 495, 524, 490, 530, 498, 515, 502, 490, 538, 512, 498, 527, 541, 507, 507, 498, 473, 466, 528, 539, 544, 510, 546, 487, 531, 549, 534, 502, 510, 498, 522, 508, 520, 499, 526, 519, 520, 529, 506, 542, 500, 473, 508, 558, 553, 460, 543, 564, 484, 496, 501, 525, 494, 509, 529, 489, 475, 506, 495, 525, 519, 506, 488, 475, 511, 516, 513, 491, 543, 540, 497, 509, 474, 502, 502, 525, 507, 517, 559, 507] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[483, 477, 491, 507, 521, 473, 536, 467, 493, 497, 467, 475, 513, 537, 469, 482, 520, 509, 524, 515, 474, 515, 532, 506, 462, 533, 524, 572, 497, 528, 544, 473, 505, 528, 509, 481, 532, 474, 521, 507, 511, 525, 546, 511, 514, 546, 458, 495, 501, 471, 491, 487, 456, 497, 516, 514, 569, 501, 510, 515, 512, 521, 524, 516, 542, 544, 557, 526, 530, 498, 525, 494, 506, 532, 509, 470, 499, 519, 524, 505, 527, 498, 517, 566, 519, 505, 524, 559, 485, 521, 534, 531, 493, 484, 489, 517, 553, 493, 502, 490, 509, 538, 521, 555, 522, 522, 520, 548, 503, 515, 466, 502, 493, 513, 492, 528, 521, 509, 551, 586, 543, 517, 554, 523, 500, 524, 486, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[504, 564, 487, 453, 516, 524, 529, 499, 497, 498, 508, 518, 506, 508, 485, 501, 519, 466, 498, 529, 466, 478, 477, 542, 545, 502, 519, 516, 518, 528, 488, 528, 464, 500, 520, 549, 526, 533, 522, 548, 500, 505, 501, 511, 539, 481, 516, 515, 537, 496, 491, 502, 504, 547, 519, 511, 488, 508, 519, 490, 529, 485, 531, 527, 497, 498, 488, 497, 524, 496, 510, 504, 553, 506, 475, 521, 512, 485, 512, 504, 557, 491, 473, 514, 487, 494, 504, 566, 524, 530, 522, 512, 500, 505, 534, 509, 498, 524, 505, 524, 521, 554, 523, 523, 503, 531, 543, 532, 510, 472, 515, 509, 497, 557, 532, 516, 530, 517, 486, 525, 533, 525, 527, 497, 526, 521, 494, 532] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[501, 500, 526, 483, 534, 511, 500, 528, 515, 514, 515, 552, 513, 517, 480, 521, 494, 514, 509, 513, 501, 474, 519, 520, 490, 526, 556, 474, 523, 511, 549, 495, 518, 525, 508, 524, 524, 493, 508, 497, 509, 528, 511, 493, 505, 498, 468, 492, 506, 523, 526, 512, 516, 549, 489, 497, 514, 512, 488, 509, 515, 489, 478, 531, 485, 507, 487, 501, 514, 512, 517, 486, 484, 525, 532, 490, 500, 509, 501, 510, 548, 519, 491, 539, 525, 525, 554, 493, 522, 503, 519, 532, 528, 506, 554, 540, 536, 547, 534, 523, 535, 527, 502, 467, 507, 513, 486, 483, 508, 500, 532, 542, 508, 490, 548, 512, 513, 508, 514, 514, 503, 513, 482, 540, 544, 535, 494, 485] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[519, 524, 499, 544, 511, 504, 509, 498, 519, 495, 515, 518, 522, 489, 502, 499, 522, 493, 538, 508, 544, 517, 528, 473, 494, 523, 487, 515, 512, 531, 473, 506, 512, 483, 511, 518, 517, 539, 500, 508, 542, 519, 527, 548, 495, 495, 496, 522, 519, 507, 523, 483, 488, 500, 484, 506, 535, 541, 513, 501, 465, 523, 527, 497, 513, 516, 494, 497, 530, 496, 484, 501, 504, 512, 515, 532, 484, 510, 475, 509, 537, 507, 516, 495, 478, 503, 474, 514, 540, 577, 458, 537, 542, 480, 533, 506, 543, 547, 544, 489, 487, 477, 525, 543, 520, 492, 522, 527, 549, 569, 482, 530, 552, 498, 534, 512, 545, 506, 492, 493, 496, 517, 509, 518, 508, 527, 536, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[483, 477, 491, 507, 521, 473, 536, 467, 493, 497, 467, 475, 513, 537, 469, 482, 520, 509, 524, 515, 474, 515, 532, 506, 462, 533, 524, 572, 497, 528, 544, 473, 505, 528, 509, 481, 532, 474, 521, 507, 511, 525, 546, 511, 514, 546, 458, 495, 501, 471, 491, 487, 456, 497, 516, 514, 569, 501, 510, 515, 512, 521, 524, 516, 542, 544, 557, 526, 530, 498, 525, 494, 506, 532, 509, 470, 499, 519, 524, 505, 527, 498, 517, 566, 519, 505, 524, 559, 485, 521, 534, 531, 493, 484, 489, 517, 553, 493, 502, 490, 509, 538, 521, 555, 522, 522, 520, 548, 503, 515, 466, 502, 493, 513, 492, 528, 521, 509, 551, 586, 543, 517, 554, 523, 500, 524, 486, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[479, 489, 535, 482, 514, 497, 522, 507, 512, 520, 532, 485, 500, 515, 486, 501, 526, 496, 492, 505, 495, 507, 565, 525, 491, 528, 476, 526, 513, 559, 526, 509, 512, 515, 511, 540, 531, 538, 506, 489, 493, 503, 519, 538, 516, 537, 510, 528, 512, 518, 514, 492, 507, 515, 479, 534, 504, 538, 453, 478, 514, 528, 547, 499, 512, 581, 505, 532, 475, 487, 490, 555, 501, 520, 528, 455, 500, 568, 457, 527, 495, 497, 479, 548, 517, 507, 545, 499, 540, 529, 496, 520, 522, 492, 514, 476, 510, 473, 507, 552, 475, 487, 525, 557, 514, 513, 530, 532, 536, 518, 501, 505, 515, 543, 487, 512, 517, 518, 533, 485, 516, 506, 483, 477, 559, 510, 514, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[518, 489, 504, 511, 495, 507, 533, 542, 522, 533, 505, 523, 493, 516, 524, 503, 506, 531, 493, 488, 511, 535, 526, 510, 465, 505, 528, 554, 517, 506, 490, 541, 468, 503, 519, 497, 552, 525, 491, 524, 515, 526, 513, 513, 511, 499, 483, 505, 495, 524, 490, 530, 498, 515, 502, 490, 538, 512, 498, 527, 541, 507, 507, 498, 473, 466, 528, 539, 544, 510, 546, 487, 531, 549, 534, 502, 510, 498, 522, 508, 520, 499, 526, 519, 520, 529, 506, 542, 500, 473, 508, 558, 553, 460, 543, 564, 484, 496, 501, 525, 494, 509, 529, 489, 475, 506, 495, 525, 519, 506, 488, 475, 511, 516, 513, 491, 543, 540, 497, 509, 474, 502, 502, 525, 507, 517, 559, 507] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[529, 547, 520, 516, 495, 505, 543, 495, 553, 524, 513, 485, 471, 520, 515, 507, 531, 516, 541, 502, 529, 522, 530, 497, 500, 505, 523, 558, 494, 533, 527, 460, 493, 528, 497, 473, 546, 523, 557, 535, 509, 501, 504, 511, 556, 567, 487, 540, 501, 495, 522, 483, 502, 509, 530, 487, 511, 472, 539, 465, 516, 547, 491, 486, 507, 522, 530, 530, 558, 504, 504, 524, 498, 516, 540, 501, 511, 499, 502, 478, 506, 512, 510, 484, 475, 512, 532, 473, 514, 510, 535, 565, 488, 488, 541, 534, 480, 531, 509, 527, 465, 521, 484, 484, 502, 508, 509, 538, 519, 511, 490, 521, 514, 495, 501, 514, 476, 505, 541, 510, 510, 508, 551, 494, 495, 489, 537, 505] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[526, 549, 535, 507, 521, 472, 554, 472, 516, 501, 515, 518, 539, 516, 527, 512, 495, 521, 484, 529, 501, 487, 525, 529, 515, 515, 523, 520, 506, 502, 486, 508, 486, 545, 510, 480, 502, 497, 534, 486, 498, 525, 493, 532, 479, 477, 523, 509, 531, 472, 537, 506, 497, 493, 521, 511, 531, 471, 561, 554, 481, 485, 478, 542, 499, 509, 501, 537, 502, 488, 544, 493, 533, 539, 476, 495, 522, 522, 509, 493, 539, 538, 506, 515, 530, 492, 528, 475, 493, 491, 517, 516, 490, 516, 515, 517, 528, 551, 498, 501, 498, 498, 479, 529, 503, 504, 517, 519, 521, 533, 513, 506, 517, 506, 510, 524, 548, 482, 551, 541, 536, 520, 512, 499, 496, 498, 538, 547] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[501, 500, 526, 483, 534, 511, 500, 528, 515, 514, 515, 552, 513, 517, 480, 521, 494, 514, 509, 513, 501, 474, 519, 520, 490, 526, 556, 474, 523, 511, 549, 495, 518, 525, 508, 524, 524, 493, 508, 497, 509, 528, 511, 493, 505, 498, 468, 492, 506, 523, 526, 512, 516, 549, 489, 497, 514, 512, 488, 509, 515, 489, 478, 531, 485, 507, 487, 501, 514, 512, 517, 486, 484, 525, 532, 490, 500, 509, 501, 510, 548, 519, 491, 539, 525, 525, 554, 493, 522, 503, 519, 532, 528, 506, 554, 540, 536, 547, 534, 523, 535, 527, 502, 467, 507, 513, 486, 483, 508, 500, 532, 542, 508, 490, 548, 512, 513, 508, 514, 514, 503, 513, 482, 540, 544, 535, 494, 485] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[518, 489, 504, 511, 495, 507, 533, 542, 522, 533, 505, 523, 493, 516, 524, 503, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 531, 493, 488, 511, 535, 526, 510, 465, 505, 528, 554, 517, 506, 490, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 503, 519, 497, 549, 525, 491, 524, 515, 526, 513, 513, 511, 499, 483, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 524, 490, 530, 498, 515, 502, 490, 538, 512, 498, 527, 541, 507, 507, 498, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 466, 528, 539, 544, 510, 546, 487, 531, 549, 534, 502, 510, 498, 522, 508, 18, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 499, 526, 519, 520, 529, 506, 542, 500, 473, 508, 507, 553, 460, 543, 564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 496, 501, 525, 494, 509, 529, 489, 475, 506, 495, 507, 519, 506, 488, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 516, 513, 491, 485, 540, 497, 509, 474, 502, 502, 525, 507, 517, 559, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8264, 8206, 8141, 8230, 8273, 8269, 7998, 8155] avg=8192.0000 max=8273 max_rank=4 min=7998 min_rank=6 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32695 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[504, 564, 487, 453, 516, 524, 529, 499, 497, 498, 508, 518, 506, 508, 485, 501, 46, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 466, 498, 529, 466, 478, 477, 542, 545, 502, 519, 514, 518, 528, 488, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 464, 500, 520, 549, 518, 533, 522, 548, 500, 505, 501, 511, 539, 481, 516, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 496, 491, 502, 504, 547, 519, 511, 488, 508, 519, 490, 529, 485, 531, 527, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 498, 488, 497, 524, 496, 510, 504, 553, 506, 475, 521, 512, 485, 512, 504, 21, 10, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 557, 491, 473, 514, 487, 494, 504, 566, 524, 530, 522, 456, 500, 505, 534, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 524, 505, 524, 521, 554, 523, 523, 503, 531, 543, 511, 510, 472, 515, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 557, 532, 516, 465, 517, 486, 525, 533, 525, 527, 497, 526, 521, 494, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8147, 8117, 8222, 8245, 8123, 8166, 8266, 8250] avg=8192.0000 max=8266 max_rank=6 min=8117 min_rank=1 imbalance_ratio=1.009033 max_load_reduction=57270 max_load_reduction_pct=0.873871 moved_to_echo=32805 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[501, 500, 526, 483, 534, 511, 500, 528, 515, 514, 515, 552, 513, 517, 480, 521, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 514, 509, 513, 501, 474, 519, 520, 490, 526, 556, 474, 523, 511, 549, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 525, 508, 524, 522, 493, 508, 497, 509, 528, 511, 493, 505, 498, 468, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 523, 526, 512, 516, 549, 489, 497, 514, 512, 488, 509, 515, 489, 478, 531, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 507, 487, 501, 514, 512, 517, 486, 484, 525, 532, 490, 500, 509, 501, 510, 16, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 519, 491, 539, 525, 525, 554, 493, 522, 503, 519, 483, 528, 506, 554, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 547, 534, 523, 535, 527, 502, 467, 507, 513, 486, 467, 508, 500, 532, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 490, 548, 512, 458, 508, 514, 514, 503, 513, 482, 540, 544, 535, 494, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8254, 8168, 8099, 8209, 8083, 8349, 8226, 8148] avg=8192.0000 max=8349 max_rank=5 min=8083 min_rank=4 imbalance_ratio=1.019165 max_load_reduction=57187 max_load_reduction_pct=0.872604 moved_to_echo=32806 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[479, 489, 535, 482, 514, 497, 522, 507, 512, 520, 532, 485, 500, 515, 486, 501, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 496, 492, 505, 495, 507, 565, 525, 491, 528, 476, 526, 513, 559, 526, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 515, 511, 540, 529, 538, 506, 489, 493, 503, 519, 538, 516, 537, 510, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 518, 514, 492, 507, 515, 479, 534, 504, 538, 453, 478, 514, 528, 547, 499, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 581, 505, 532, 475, 487, 490, 555, 501, 520, 528, 455, 500, 568, 457, 527, 18, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 497, 479, 548, 517, 507, 545, 499, 540, 529, 496, 472, 522, 492, 514, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 473, 507, 552, 475, 487, 525, 557, 514, 513, 530, 514, 536, 518, 501, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 543, 487, 512, 462, 518, 533, 485, 516, 506, 483, 477, 559, 510, 514, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8119, 8239, 8284, 8187, 8218, 8128, 8217, 8144] avg=8192.0000 max=8284 max_rank=2 min=8119 min_rank=0 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32707 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[519, 524, 499, 544, 511, 504, 509, 498, 519, 495, 515, 518, 522, 489, 502, 499, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 493, 538, 508, 544, 517, 528, 473, 494, 523, 487, 515, 512, 531, 473, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 483, 511, 518, 515, 539, 500, 508, 542, 519, 527, 548, 495, 495, 496, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 507, 523, 483, 488, 500, 484, 506, 535, 541, 513, 501, 465, 523, 527, 497, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 516, 494, 497, 530, 496, 484, 501, 504, 512, 515, 532, 484, 510, 475, 509, 18, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 507, 516, 495, 478, 503, 474, 514, 540, 577, 458, 487, 542, 480, 533, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 547, 544, 489, 487, 477, 525, 543, 520, 492, 522, 509, 549, 569, 482, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 498, 534, 512, 486, 506, 492, 493, 496, 517, 509, 518, 508, 527, 536, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8164, 8230, 8171, 8097, 8147, 8328, 8187] avg=8192.0000 max=8328 max_rank=6 min=8097 min_rank=4 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=32759 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[529, 547, 520, 516, 495, 505, 543, 495, 553, 524, 513, 485, 471, 520, 515, 507, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 516, 541, 502, 529, 522, 530, 497, 500, 505, 523, 558, 494, 533, 527, 460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 528, 497, 473, 544, 523, 557, 535, 509, 501, 504, 511, 556, 567, 487, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 495, 522, 483, 502, 509, 530, 487, 511, 472, 539, 465, 516, 547, 491, 486, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 522, 530, 530, 558, 504, 504, 524, 498, 516, 540, 501, 511, 499, 502, 478, 18, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 512, 510, 484, 475, 512, 532, 473, 514, 510, 535, 513, 488, 488, 541, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 531, 509, 527, 465, 521, 484, 484, 502, 508, 509, 520, 519, 511, 490, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 495, 501, 514, 425, 505, 541, 510, 510, 508, 551, 494, 495, 489, 537, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8285, 8268, 8325, 8107, 8249, 8127, 8081, 8094] avg=8192.0000 max=8325 max_rank=2 min=8081 min_rank=6 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32551 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[483, 477, 491, 507, 521, 473, 536, 467, 493, 497, 467, 475, 513, 537, 469, 482, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 509, 524, 515, 474, 515, 532, 506, 462, 533, 524, 572, 497, 528, 544, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 528, 509, 481, 530, 474, 521, 507, 511, 525, 546, 511, 514, 546, 458, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 471, 491, 487, 456, 497, 516, 514, 569, 501, 510, 515, 512, 521, 524, 516, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 544, 557, 526, 530, 498, 525, 494, 506, 532, 509, 470, 499, 519, 524, 505, 19, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 498, 517, 566, 519, 505, 524, 559, 485, 521, 534, 482, 493, 484, 489, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 493, 502, 490, 509, 538, 521, 555, 522, 522, 520, 529, 503, 515, 466, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 513, 492, 528, 465, 509, 551, 586, 543, 517, 554, 523, 500, 524, 486, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7932, 8228, 8161, 8157, 8306, 8220, 8240, 8292] avg=8192.0000 max=8306 max_rank=4 min=7932 min_rank=0 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=33058 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[479, 489, 535, 482, 514, 497, 522, 507, 512, 520, 532, 485, 500, 515, 486, 501, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 496, 492, 505, 495, 507, 565, 525, 491, 528, 476, 526, 513, 559, 526, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 515, 511, 540, 529, 538, 506, 489, 493, 503, 519, 538, 516, 537, 510, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 518, 514, 492, 507, 515, 479, 534, 504, 538, 453, 478, 514, 528, 547, 499, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 581, 505, 532, 475, 487, 490, 555, 501, 520, 528, 455, 500, 568, 457, 527, 18, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 497, 479, 548, 517, 507, 545, 499, 540, 529, 496, 472, 522, 492, 514, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 473, 507, 552, 475, 487, 525, 557, 514, 513, 530, 514, 536, 518, 501, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 543, 487, 512, 462, 518, 533, 485, 516, 506, 483, 477, 559, 510, 514, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8119, 8239, 8284, 8187, 8218, 8128, 8217, 8144] avg=8192.0000 max=8284 max_rank=2 min=8119 min_rank=0 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32707 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[518, 489, 504, 511, 495, 507, 533, 542, 522, 533, 505, 523, 493, 516, 524, 503, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 531, 493, 488, 511, 535, 526, 510, 465, 505, 528, 554, 517, 506, 490, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 503, 519, 497, 549, 525, 491, 524, 515, 526, 513, 513, 511, 499, 483, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 524, 490, 530, 498, 515, 502, 490, 538, 512, 498, 527, 541, 507, 507, 498, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 466, 528, 539, 544, 510, 546, 487, 531, 549, 534, 502, 510, 498, 522, 508, 18, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 499, 526, 519, 520, 529, 506, 542, 500, 473, 508, 507, 553, 460, 543, 564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 496, 501, 525, 494, 509, 529, 489, 475, 506, 495, 507, 519, 506, 488, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 516, 513, 491, 485, 540, 497, 509, 474, 502, 502, 525, 507, 517, 559, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8264, 8206, 8141, 8230, 8273, 8269, 7998, 8155] avg=8192.0000 max=8273 max_rank=4 min=7998 min_rank=6 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32695 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[526, 549, 535, 507, 521, 472, 554, 472, 516, 501, 515, 518, 539, 516, 527, 512, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 521, 484, 529, 501, 487, 525, 529, 515, 515, 523, 520, 506, 502, 486, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 545, 510, 480, 500, 497, 534, 486, 498, 525, 493, 532, 479, 477, 523, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 472, 537, 506, 497, 493, 521, 511, 531, 471, 561, 554, 481, 485, 478, 542, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 509, 501, 537, 502, 488, 544, 493, 533, 539, 476, 495, 522, 522, 509, 493, 18, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 538, 506, 515, 530, 492, 528, 475, 493, 491, 517, 468, 490, 516, 515, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 551, 498, 501, 498, 498, 479, 529, 503, 504, 517, 501, 521, 533, 513, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 506, 510, 524, 489, 482, 551, 541, 536, 520, 512, 499, 496, 498, 538, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8323, 8146, 8074, 8230, 8187, 8130, 8180, 8266] avg=8192.0000 max=8323 max_rank=0 min=8074 min_rank=2 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32763 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[529, 547, 520, 516, 495, 505, 543, 495, 553, 524, 513, 485, 471, 520, 515, 507, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 516, 541, 502, 529, 522, 530, 497, 500, 505, 523, 558, 494, 533, 527, 460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 528, 497, 473, 544, 523, 557, 535, 509, 501, 504, 511, 556, 567, 487, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 495, 522, 483, 502, 509, 530, 487, 511, 472, 539, 465, 516, 547, 491, 486, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 522, 530, 530, 558, 504, 504, 524, 498, 516, 540, 501, 511, 499, 502, 478, 18, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 512, 510, 484, 475, 512, 532, 473, 514, 510, 535, 513, 488, 488, 541, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 531, 509, 527, 465, 521, 484, 484, 502, 508, 509, 520, 519, 511, 490, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 495, 501, 514, 425, 505, 541, 510, 510, 508, 551, 494, 495, 489, 537, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8285, 8268, 8325, 8107, 8249, 8127, 8081, 8094] avg=8192.0000 max=8325 max_rank=2 min=8081 min_rank=6 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32551 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[501, 500, 526, 483, 534, 511, 500, 528, 515, 514, 515, 552, 513, 517, 480, 521, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 514, 509, 513, 501, 474, 519, 520, 490, 526, 556, 474, 523, 511, 549, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 525, 508, 524, 522, 493, 508, 497, 509, 528, 511, 493, 505, 498, 468, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 523, 526, 512, 516, 549, 489, 497, 514, 512, 488, 509, 515, 489, 478, 531, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 507, 487, 501, 514, 512, 517, 486, 484, 525, 532, 490, 500, 509, 501, 510, 16, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 519, 491, 539, 525, 525, 554, 493, 522, 503, 519, 483, 528, 506, 554, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 547, 534, 523, 535, 527, 502, 467, 507, 513, 486, 467, 508, 500, 532, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 490, 548, 512, 458, 508, 514, 514, 503, 513, 482, 540, 544, 535, 494, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8254, 8168, 8099, 8209, 8083, 8349, 8226, 8148] avg=8192.0000 max=8349 max_rank=5 min=8083 min_rank=4 imbalance_ratio=1.019165 max_load_reduction=57187 max_load_reduction_pct=0.872604 moved_to_echo=32806 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4352, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4096, 3968, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 3968, 4224, 4224, 4352, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[3968, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4224, 4224, 4224, 3840, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 3968, 4096, 4224, 4096, 4096, 256, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 3968, 4224, 4224, 4352, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[3968, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4096, 3968, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4352, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 3968, 4224, 3968, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[499, 508, 510, 539, 486, 519, 511, 574, 507, 498, 536, 518, 466, 469, 464, 480, 537, 530, 507, 475, 558, 520, 479, 509, 498, 502, 549, 501, 515, 543, 560, 490, 496, 490, 514, 476, 516, 527, 503, 487, 506, 540, 487, 460, 564, 483, 523, 519, 507, 495, 502, 548, 546, 501, 534, 502, 510, 539, 506, 529, 479, 517, 510, 481, 527, 497, 546, 507, 541, 540, 500, 506, 496, 501, 497, 526, 545, 516, 530, 526, 516, 544, 495, 495, 505, 494, 508, 504, 474, 477, 520, 510, 519, 508, 501, 510, 522, 513, 532, 490, 509, 507, 485, 522, 531, 508, 463, 554, 528, 520, 519, 474, 517, 538, 516, 556, 490, 507, 525, 530, 532, 497, 505, 497, 491, 562, 543, 518] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[514, 508, 514, 472, 490, 520, 543, 475, 521, 500, 510, 483, 511, 475, 503, 505, 522, 525, 558, 499, 559, 504, 565, 487, 516, 527, 492, 516, 514, 519, 516, 510, 525, 487, 503, 490, 511, 492, 521, 544, 527, 538, 513, 531, 499, 466, 516, 552, 491, 497, 521, 508, 543, 464, 481, 527, 488, 502, 499, 528, 505, 523, 514, 500, 511, 538, 470, 510, 475, 495, 535, 516, 549, 482, 522, 512, 506, 536, 535, 500, 506, 509, 510, 527, 524, 509, 484, 476, 510, 526, 507, 488, 562, 516, 487, 506, 494, 518, 500, 512, 555, 498, 528, 496, 531, 525, 527, 504, 490, 491, 532, 516, 534, 511, 509, 516, 505, 531, 527, 531, 467, 538, 519, 527, 552, 553, 477, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[530, 531, 518, 510, 526, 520, 496, 514, 542, 477, 527, 516, 474, 495, 505, 528, 540, 504, 487, 473, 506, 498, 548, 513, 500, 512, 553, 503, 526, 512, 528, 499, 515, 550, 541, 490, 535, 477, 487, 520, 524, 488, 504, 504, 510, 522, 536, 541, 471, 511, 511, 533, 538, 509, 492, 508, 554, 514, 536, 511, 520, 487, 497, 505, 511, 484, 523, 528, 529, 484, 513, 489, 501, 484, 532, 532, 521, 519, 488, 498, 520, 499, 545, 486, 520, 533, 482, 502, 515, 492, 491, 530, 508, 509, 511, 496, 485, 532, 484, 497, 551, 491, 508, 472, 542, 499, 521, 538, 512, 496, 519, 505, 502, 515, 522, 518, 492, 541, 543, 500, 488, 545, 504, 526, 507, 500, 553, 501] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[495, 517, 514, 514, 511, 495, 498, 497, 526, 521, 499, 519, 523, 505, 482, 480, 532, 521, 527, 514, 522, 506, 492, 506, 513, 519, 507, 519, 555, 497, 550, 535, 527, 496, 508, 541, 519, 473, 543, 529, 516, 527, 484, 533, 502, 532, 535, 503, 497, 537, 506, 540, 549, 502, 515, 485, 527, 496, 495, 502, 541, 504, 488, 523, 521, 563, 479, 522, 484, 547, 466, 528, 503, 539, 527, 531, 511, 529, 519, 498, 496, 500, 525, 529, 521, 524, 540, 525, 507, 525, 506, 510, 487, 483, 504, 515, 537, 516, 514, 475, 494, 512, 506, 551, 516, 526, 505, 526, 532, 515, 509, 514, 479, 476, 526, 480, 530, 489, 492, 507, 450, 492, 531, 529, 498, 511, 464, 484] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[469, 516, 544, 539, 500, 524, 514, 465, 501, 497, 495, 527, 531, 520, 534, 485, 522, 517, 518, 486, 509, 485, 503, 486, 503, 545, 512, 522, 500, 513, 542, 490, 558, 531, 542, 518, 537, 504, 529, 524, 518, 517, 511, 514, 530, 518, 507, 516, 504, 526, 540, 492, 532, 505, 516, 504, 541, 524, 487, 571, 546, 517, 502, 513, 505, 499, 494, 496, 521, 544, 482, 487, 488, 485, 467, 501, 512, 530, 490, 528, 492, 504, 496, 529, 520, 471, 550, 541, 489, 514, 524, 533, 524, 520, 483, 524, 483, 510, 513, 518, 514, 474, 493, 497, 527, 504, 498, 515, 535, 502, 511, 498, 504, 472, 494, 522, 537, 519, 567, 525, 478, 546, 461, 530, 500, 525, 509, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[472, 539, 549, 535, 519, 506, 500, 519, 506, 527, 478, 520, 504, 521, 474, 492, 497, 501, 481, 538, 518, 507, 546, 485, 515, 536, 486, 518, 506, 451, 506, 504, 520, 480, 547, 563, 486, 531, 502, 516, 474, 505, 533, 502, 519, 480, 520, 540, 485, 475, 552, 504, 487, 518, 499, 523, 518, 490, 505, 484, 500, 526, 519, 543, 547, 522, 488, 496, 532, 519, 503, 526, 546, 488, 520, 548, 506, 506, 457, 489, 512, 507, 503, 529, 494, 523, 499, 516, 520, 553, 549, 492, 543, 514, 507, 521, 505, 482, 535, 526, 540, 495, 500, 527, 523, 510, 507, 513, 523, 524, 508, 541, 530, 514, 500, 506, 532, 505, 497, 505, 525, 516, 507, 524, 511, 483, 501, 544] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[499, 508, 510, 539, 486, 519, 511, 574, 507, 498, 536, 518, 466, 469, 464, 480, 537, 530, 507, 475, 558, 520, 479, 509, 498, 502, 549, 501, 515, 543, 560, 490, 496, 490, 514, 476, 516, 527, 503, 487, 506, 540, 487, 460, 564, 483, 523, 519, 507, 495, 502, 548, 546, 501, 534, 502, 510, 539, 506, 529, 479, 517, 510, 481, 527, 497, 546, 507, 541, 540, 500, 506, 496, 501, 497, 526, 545, 516, 530, 526, 516, 544, 495, 495, 505, 494, 508, 504, 474, 477, 520, 510, 519, 508, 501, 510, 522, 513, 532, 490, 509, 507, 485, 522, 531, 508, 463, 554, 528, 520, 519, 474, 517, 538, 516, 556, 490, 507, 525, 530, 532, 497, 505, 497, 491, 562, 543, 518] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[489, 503, 516, 494, 505, 523, 530, 528, 565, 472, 492, 508, 566, 538, 529, 521, 491, 527, 506, 528, 500, 532, 537, 534, 492, 490, 522, 483, 535, 515, 492, 531, 474, 496, 516, 500, 523, 502, 554, 498, 544, 526, 510, 533, 535, 469, 514, 480, 491, 504, 462, 504, 495, 475, 508, 475, 512, 498, 506, 506, 516, 502, 499, 508, 502, 490, 524, 520, 516, 481, 472, 549, 494, 481, 501, 537, 519, 533, 552, 516, 519, 522, 499, 533, 493, 519, 537, 525, 549, 543, 516, 504, 515, 516, 525, 510, 520, 493, 512, 501, 485, 499, 523, 511, 495, 537, 546, 478, 508, 526, 472, 555, 471, 502, 564, 524, 522, 507, 514, 495, 519, 529, 518, 513, 517, 519, 503, 517] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[490, 478, 523, 502, 527, 491, 518, 512, 521, 455, 510, 477, 526, 498, 479, 505, 527, 495, 504, 517, 473, 500, 514, 516, 528, 556, 486, 521, 504, 503, 532, 498, 512, 490, 484, 526, 544, 550, 515, 539, 494, 537, 500, 539, 509, 500, 518, 510, 534, 503, 476, 505, 525, 507, 503, 544, 493, 541, 515, 507, 526, 487, 502, 517, 516, 501, 517, 486, 527, 521, 524, 555, 542, 510, 503, 551, 516, 519, 518, 551, 503, 492, 528, 526, 523, 522, 505, 544, 528, 501, 530, 544, 505, 500, 481, 528, 491, 481, 521, 501, 485, 510, 495, 482, 518, 491, 518, 512, 504, 506, 501, 531, 526, 495, 522, 513, 491, 526, 520, 499, 518, 503, 518, 528, 481, 544, 499, 551] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[499, 508, 510, 539, 486, 519, 511, 574, 507, 498, 536, 518, 466, 469, 464, 480, 45, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 530, 507, 475, 558, 520, 479, 509, 498, 502, 549, 501, 515, 543, 523, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 490, 514, 476, 516, 527, 503, 487, 506, 495, 487, 460, 564, 483, 523, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 495, 502, 548, 546, 501, 534, 502, 510, 539, 506, 529, 479, 517, 510, 481, 23, 9, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 497, 546, 507, 541, 540, 500, 506, 496, 501, 497, 504, 545, 516, 530, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 544, 495, 495, 505, 494, 508, 504, 474, 477, 520, 510, 503, 508, 501, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 513, 532, 490, 509, 507, 485, 522, 531, 508, 463, 554, 528, 520, 519, 474, 16, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 538, 516, 556, 490, 507, 525, 530, 532, 497, 505, 497, 491, 539, 543, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8156, 8236, 8046, 8248, 8279, 8064, 8206, 8301] avg=8192.0000 max=8301 max_rank=7 min=8046 min_rank=2 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32850 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[472, 539, 549, 535, 519, 506, 500, 519, 506, 527, 478, 520, 504, 521, 474, 492, 38, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 501, 481, 538, 518, 507, 546, 485, 515, 536, 486, 518, 506, 451, 481, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 480, 547, 563, 486, 531, 502, 516, 474, 467, 533, 502, 519, 480, 520, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 475, 552, 504, 487, 518, 499, 523, 518, 490, 505, 484, 500, 526, 519, 543, 16, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 522, 488, 496, 532, 519, 503, 526, 546, 488, 520, 533, 506, 506, 457, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 507, 503, 529, 494, 523, 499, 516, 520, 553, 549, 492, 529, 514, 507, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 482, 535, 526, 540, 495, 500, 527, 523, 510, 507, 513, 523, 524, 508, 541, 14, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 514, 500, 506, 532, 505, 497, 505, 525, 516, 507, 524, 511, 467, 501, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 8070, 8180, 8155, 8178, 8268, 8282, 8184] avg=8192.0000 max=8282 max_rank=6 min=8070 min_rank=1 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32912 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[514, 508, 514, 472, 490, 520, 543, 475, 521, 500, 510, 483, 511, 475, 503, 505, 40, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 525, 558, 499, 559, 504, 565, 487, 516, 527, 492, 516, 514, 519, 490, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 487, 503, 490, 511, 492, 521, 544, 527, 498, 513, 531, 499, 466, 516, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 497, 521, 508, 543, 464, 481, 527, 488, 502, 499, 528, 505, 523, 514, 500, 19, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 538, 470, 510, 475, 495, 535, 516, 549, 482, 522, 497, 506, 536, 535, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 509, 510, 527, 524, 509, 484, 476, 510, 526, 507, 488, 548, 516, 487, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 518, 500, 512, 555, 498, 528, 496, 531, 525, 527, 504, 490, 491, 532, 516, 14, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 511, 509, 516, 505, 531, 527, 531, 467, 538, 519, 527, 552, 534, 477, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8105, 8303, 8175, 8121, 8177, 8133, 8240, 8282] avg=8192.0000 max=8303 max_rank=1 min=8105 min_rank=0 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32832 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[514, 508, 514, 472, 490, 520, 543, 475, 521, 500, 510, 483, 511, 475, 503, 505, 522, 525, 558, 499, 559, 504, 565, 487, 516, 527, 492, 516, 514, 519, 516, 510, 525, 487, 503, 490, 511, 492, 521, 544, 527, 538, 513, 531, 499, 466, 516, 552, 491, 497, 521, 508, 543, 464, 481, 527, 488, 502, 499, 528, 505, 523, 514, 500, 511, 538, 470, 510, 475, 495, 535, 516, 549, 482, 522, 512, 506, 536, 535, 500, 506, 509, 510, 527, 524, 509, 484, 476, 510, 526, 507, 488, 562, 516, 487, 506, 494, 518, 500, 512, 555, 498, 528, 496, 531, 525, 527, 504, 490, 491, 532, 516, 534, 511, 509, 516, 505, 531, 527, 531, 467, 538, 519, 527, 552, 553, 477, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[469, 516, 544, 539, 500, 524, 514, 465, 501, 497, 495, 527, 531, 520, 534, 485, 522, 517, 518, 486, 509, 485, 503, 486, 503, 545, 512, 522, 500, 513, 542, 490, 558, 531, 542, 518, 537, 504, 529, 524, 518, 517, 511, 514, 530, 518, 507, 516, 504, 526, 540, 492, 532, 505, 516, 504, 541, 524, 487, 571, 546, 517, 502, 513, 505, 499, 494, 496, 521, 544, 482, 487, 488, 485, 467, 501, 512, 530, 490, 528, 492, 504, 496, 529, 520, 471, 550, 541, 489, 514, 524, 533, 524, 520, 483, 524, 483, 510, 513, 518, 514, 474, 493, 497, 527, 504, 498, 515, 535, 502, 511, 498, 504, 472, 494, 522, 537, 519, 567, 525, 478, 546, 461, 530, 500, 525, 509, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[472, 539, 549, 535, 519, 506, 500, 519, 506, 527, 478, 520, 504, 521, 474, 492, 497, 501, 481, 538, 518, 507, 546, 485, 515, 536, 486, 518, 506, 451, 506, 504, 520, 480, 547, 563, 486, 531, 502, 516, 474, 505, 533, 502, 519, 480, 520, 540, 485, 475, 552, 504, 487, 518, 499, 523, 518, 490, 505, 484, 500, 526, 519, 543, 547, 522, 488, 496, 532, 519, 503, 526, 546, 488, 520, 548, 506, 506, 457, 489, 512, 507, 503, 529, 494, 523, 499, 516, 520, 553, 549, 492, 543, 514, 507, 521, 505, 482, 535, 526, 540, 495, 500, 527, 523, 510, 507, 513, 523, 524, 508, 541, 530, 514, 500, 506, 532, 505, 497, 505, 525, 516, 507, 524, 511, 483, 501, 544] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[530, 531, 518, 510, 526, 520, 496, 514, 542, 477, 527, 516, 474, 495, 505, 528, 36, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 504, 487, 473, 506, 498, 548, 513, 500, 512, 553, 503, 526, 512, 502, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 550, 541, 490, 535, 477, 487, 520, 524, 452, 504, 504, 510, 522, 536, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 511, 511, 533, 538, 509, 492, 508, 554, 514, 536, 511, 520, 487, 497, 505, 17, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 484, 523, 528, 529, 484, 513, 489, 501, 484, 532, 517, 521, 519, 488, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 499, 545, 486, 520, 533, 482, 502, 515, 492, 491, 530, 495, 509, 511, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 532, 484, 497, 551, 491, 508, 472, 542, 499, 521, 538, 512, 496, 519, 505, 13, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 515, 522, 518, 492, 541, 543, 500, 488, 545, 504, 526, 507, 483, 553, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8266, 8176, 8208, 8225, 8121, 8126, 8174, 8240] avg=8192.0000 max=8266 max_rank=0 min=8121 min_rank=4 imbalance_ratio=1.009033 max_load_reduction=57270 max_load_reduction_pct=0.873871 moved_to_echo=32661 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[495, 517, 514, 514, 511, 495, 498, 497, 526, 521, 499, 519, 523, 505, 482, 480, 39, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 521, 527, 514, 522, 506, 492, 506, 513, 519, 507, 519, 555, 497, 523, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 496, 508, 541, 519, 473, 543, 529, 516, 488, 484, 533, 502, 532, 535, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 537, 506, 540, 549, 502, 515, 485, 527, 496, 495, 502, 541, 504, 488, 523, 17, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 563, 479, 522, 484, 547, 466, 528, 503, 539, 527, 516, 511, 529, 519, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 500, 525, 529, 521, 524, 540, 525, 507, 525, 506, 510, 475, 483, 504, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 516, 514, 475, 494, 512, 506, 551, 516, 526, 505, 526, 532, 515, 509, 514, 12, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 476, 526, 480, 530, 489, 492, 507, 450, 492, 531, 529, 498, 494, 464, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8288, 8229, 8235, 8252, 8185, 8269, 7921] avg=8192.0000 max=8288 max_rank=1 min=7921 min_rank=7 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32627 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[469, 516, 544, 539, 500, 524, 514, 465, 501, 497, 495, 527, 531, 520, 534, 485, 38, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 517, 518, 486, 509, 485, 503, 486, 503, 545, 512, 522, 500, 513, 515, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 558, 531, 542, 518, 537, 504, 529, 524, 518, 479, 511, 514, 530, 518, 507, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 526, 540, 492, 532, 505, 516, 504, 541, 524, 487, 571, 546, 517, 502, 513, 18, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 499, 494, 496, 521, 544, 482, 487, 488, 485, 467, 487, 512, 530, 490, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 504, 496, 529, 520, 471, 550, 541, 489, 514, 524, 533, 511, 520, 483, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 510, 513, 518, 514, 474, 493, 497, 527, 504, 498, 515, 535, 502, 511, 498, 13, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 472, 494, 522, 537, 519, 567, 525, 478, 546, 461, 530, 500, 507, 509, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8126, 8336, 8349, 8015, 8201, 8113, 8175] avg=8192.0000 max=8349 max_rank=3 min=8015 min_rank=4 imbalance_ratio=1.019165 max_load_reduction=57187 max_load_reduction_pct=0.872604 moved_to_echo=32504 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[499, 508, 510, 539, 486, 519, 511, 574, 507, 498, 536, 518, 466, 469, 464, 480, 45, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 530, 507, 475, 558, 520, 479, 509, 498, 502, 549, 501, 515, 543, 523, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 490, 514, 476, 516, 527, 503, 487, 506, 495, 487, 460, 564, 483, 523, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 495, 502, 548, 546, 501, 534, 502, 510, 539, 506, 529, 479, 517, 510, 481, 23, 9, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 497, 546, 507, 541, 540, 500, 506, 496, 501, 497, 504, 545, 516, 530, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 544, 495, 495, 505, 494, 508, 504, 474, 477, 520, 510, 503, 508, 501, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 513, 532, 490, 509, 507, 485, 522, 531, 508, 463, 554, 528, 520, 519, 474, 16, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 538, 516, 556, 490, 507, 525, 530, 532, 497, 505, 497, 491, 539, 543, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8156, 8236, 8046, 8248, 8279, 8064, 8206, 8301] avg=8192.0000 max=8301 max_rank=7 min=8046 min_rank=2 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32850 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[489, 503, 516, 494, 505, 523, 530, 528, 565, 472, 492, 508, 566, 538, 529, 521, 39, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 527, 506, 528, 500, 532, 537, 534, 492, 490, 522, 483, 535, 515, 467, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 496, 516, 500, 523, 502, 554, 498, 544, 487, 510, 533, 535, 469, 514, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 504, 462, 504, 495, 475, 508, 475, 512, 498, 506, 506, 516, 502, 499, 508, 17, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 490, 524, 520, 516, 481, 472, 549, 494, 481, 501, 522, 519, 533, 552, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 522, 499, 533, 493, 519, 537, 525, 549, 543, 516, 504, 502, 516, 525, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 493, 512, 501, 485, 499, 523, 511, 495, 537, 546, 478, 508, 526, 472, 555, 13, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 502, 564, 524, 522, 507, 514, 495, 519, 529, 518, 513, 517, 502, 503, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8338, 8190, 8135, 7989, 8172, 8312, 8183, 8217] avg=8192.0000 max=8338 max_rank=0 min=7989 min_rank=3 imbalance_ratio=1.017822 max_load_reduction=57198 max_load_reduction_pct=0.872772 moved_to_echo=32884 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[514, 508, 514, 472, 490, 520, 543, 475, 521, 500, 510, 483, 511, 475, 503, 505, 40, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 525, 558, 499, 559, 504, 565, 487, 516, 527, 492, 516, 514, 519, 490, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 487, 503, 490, 511, 492, 521, 544, 527, 498, 513, 531, 499, 466, 516, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 497, 521, 508, 543, 464, 481, 527, 488, 502, 499, 528, 505, 523, 514, 500, 19, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 538, 470, 510, 475, 495, 535, 516, 549, 482, 522, 497, 506, 536, 535, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 509, 510, 527, 524, 509, 484, 476, 510, 526, 507, 488, 548, 516, 487, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 518, 500, 512, 555, 498, 528, 496, 531, 525, 527, 504, 490, 491, 532, 516, 14, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 511, 509, 516, 505, 531, 527, 531, 467, 538, 519, 527, 552, 534, 477, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8105, 8303, 8175, 8121, 8177, 8133, 8240, 8282] avg=8192.0000 max=8303 max_rank=1 min=8105 min_rank=0 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32832 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[490, 478, 523, 502, 527, 491, 518, 512, 521, 455, 510, 477, 526, 498, 479, 505, 40, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 495, 504, 517, 473, 500, 514, 516, 528, 556, 486, 521, 504, 503, 505, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 490, 484, 526, 544, 550, 515, 539, 494, 497, 500, 539, 509, 500, 518, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 503, 476, 505, 525, 507, 503, 544, 493, 541, 515, 507, 526, 487, 502, 517, 18, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 501, 517, 486, 527, 521, 524, 555, 542, 510, 503, 536, 516, 519, 518, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 492, 528, 526, 523, 522, 505, 544, 528, 501, 530, 544, 492, 500, 481, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 481, 521, 501, 485, 510, 495, 482, 518, 491, 518, 512, 504, 506, 501, 531, 13, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 495, 522, 513, 491, 526, 520, 499, 518, 503, 518, 528, 481, 526, 499, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8074, 8147, 8227, 8214, 8342, 8247, 8069, 8216] avg=8192.0000 max=8342 max_rank=4 min=8069 min_rank=6 imbalance_ratio=1.018311 max_load_reduction=57194 max_load_reduction_pct=0.872711 moved_to_echo=32874 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[472, 539, 549, 535, 519, 506, 500, 519, 506, 527, 478, 520, 504, 521, 474, 492, 38, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 501, 481, 538, 518, 507, 546, 485, 515, 536, 486, 518, 506, 451, 481, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 480, 547, 563, 486, 531, 502, 516, 474, 467, 533, 502, 519, 480, 520, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 475, 552, 504, 487, 518, 499, 523, 518, 490, 505, 484, 500, 526, 519, 543, 16, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 522, 488, 496, 532, 519, 503, 526, 546, 488, 520, 533, 506, 506, 457, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 507, 503, 529, 494, 523, 499, 516, 520, 553, 549, 492, 529, 514, 507, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 482, 535, 526, 540, 495, 500, 527, 523, 510, 507, 513, 523, 524, 508, 541, 14, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 514, 500, 506, 532, 505, 497, 505, 525, 516, 507, 524, 511, 467, 501, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 8070, 8180, 8155, 8178, 8268, 8282, 8184] avg=8192.0000 max=8282 max_rank=6 min=8070 min_rank=1 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32912 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[469, 516, 544, 539, 500, 524, 514, 465, 501, 497, 495, 527, 531, 520, 534, 485, 38, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 517, 518, 486, 509, 485, 503, 486, 503, 545, 512, 522, 500, 513, 515, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 558, 531, 542, 518, 537, 504, 529, 524, 518, 479, 511, 514, 530, 518, 507, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 526, 540, 492, 532, 505, 516, 504, 541, 524, 487, 571, 546, 517, 502, 513, 18, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 499, 494, 496, 521, 544, 482, 487, 488, 485, 467, 487, 512, 530, 490, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 504, 496, 529, 520, 471, 550, 541, 489, 514, 524, 533, 511, 520, 483, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 510, 513, 518, 514, 474, 493, 497, 527, 504, 498, 515, 535, 502, 511, 498, 13, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 472, 494, 522, 537, 519, 567, 525, 478, 546, 461, 530, 500, 507, 509, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8126, 8336, 8349, 8015, 8201, 8113, 8175] avg=8192.0000 max=8349 max_rank=3 min=8015 min_rank=4 imbalance_ratio=1.019165 max_load_reduction=57187 max_load_reduction_pct=0.872604 moved_to_echo=32504 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[3968, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 3968, 4096, 4096, 4224, 4096, 4096, 4096, 384, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 3968, 4096, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[511, 507, 506, 535, 521, 553, 509, 552, 549, 477, 482, 515, 493, 496, 498, 512, 494, 509, 475, 489, 517, 506, 540, 497, 518, 543, 522, 480, 534, 507, 527, 525, 536, 514, 543, 489, 519, 510, 529, 518, 507, 503, 491, 512, 501, 522, 500, 487, 512, 490, 502, 497, 526, 523, 502, 491, 499, 499, 458, 526, 533, 517, 508, 509, 494, 487, 513, 508, 511, 515, 482, 494, 525, 524, 489, 476, 501, 521, 494, 523, 489, 538, 511, 539, 493, 473, 546, 511, 479, 510, 518, 495, 508, 512, 549, 521, 536, 484, 504, 547, 502, 528, 502, 531, 482, 506, 526, 511, 511, 551, 543, 524, 528, 557, 514, 522, 522, 556, 507, 499, 483, 511, 495, 535, 512, 537, 508, 541] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[495, 486, 516, 538, 474, 523, 474, 513, 537, 474, 494, 528, 516, 509, 525, 542, 531, 504, 516, 540, 484, 483, 511, 499, 507, 526, 484, 500, 516, 501, 513, 501, 482, 487, 490, 479, 479, 535, 485, 541, 472, 525, 502, 517, 510, 531, 487, 507, 485, 524, 512, 555, 489, 479, 506, 551, 495, 508, 501, 542, 492, 528, 538, 529, 490, 559, 517, 538, 505, 501, 529, 499, 494, 508, 533, 538, 550, 527, 534, 531, 485, 518, 536, 466, 487, 522, 498, 500, 494, 531, 531, 530, 526, 542, 516, 541, 517, 521, 522, 520, 493, 513, 547, 505, 509, 480, 540, 515, 491, 509, 504, 512, 535, 480, 549, 532, 532, 492, 499, 482, 499, 506, 549, 522, 500, 516, 534, 512] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[540, 501, 535, 498, 504, 491, 479, 460, 452, 558, 524, 501, 472, 511, 557, 539, 513, 527, 518, 503, 501, 503, 548, 512, 529, 475, 508, 543, 487, 487, 486, 542, 499, 478, 496, 534, 549, 502, 512, 503, 542, 527, 529, 546, 491, 528, 485, 532, 512, 517, 493, 505, 517, 490, 513, 520, 483, 537, 509, 508, 489, 513, 530, 462, 514, 513, 505, 519, 472, 524, 522, 528, 521, 516, 524, 486, 490, 503, 541, 527, 520, 487, 530, 512, 513, 511, 494, 515, 492, 522, 526, 517, 481, 507, 506, 522, 535, 519, 551, 537, 531, 493, 531, 504, 509, 520, 532, 514, 556, 516, 533, 507, 524, 495, 507, 504, 514, 480, 487, 522, 538, 466, 514, 526, 548, 505, 505, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[523, 480, 487, 514, 510, 534, 518, 521, 531, 519, 499, 549, 496, 527, 508, 533, 529, 528, 553, 520, 507, 503, 467, 542, 521, 484, 521, 542, 513, 528, 518, 515, 534, 531, 498, 533, 537, 500, 503, 489, 523, 486, 510, 514, 519, 493, 498, 510, 553, 520, 531, 526, 486, 519, 511, 483, 502, 487, 505, 497, 500, 503, 475, 499, 526, 520, 516, 499, 522, 528, 496, 514, 509, 484, 525, 525, 523, 523, 540, 514, 509, 503, 518, 488, 486, 517, 507, 512, 503, 560, 516, 488, 474, 497, 540, 522, 497, 495, 538, 501, 482, 542, 517, 489, 490, 530, 529, 504, 520, 530, 500, 544, 488, 485, 504, 510, 474, 526, 473, 521, 517, 525, 525, 523, 536, 510, 500, 492] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[512, 519, 521, 531, 561, 495, 514, 506, 498, 507, 532, 550, 501, 530, 538, 474, 496, 535, 502, 521, 459, 524, 521, 519, 494, 516, 482, 522, 494, 511, 499, 544, 538, 538, 501, 518, 483, 497, 529, 497, 513, 538, 535, 483, 535, 505, 462, 533, 498, 511, 476, 572, 529, 488, 458, 513, 517, 458, 516, 509, 505, 536, 491, 506, 537, 522, 495, 512, 515, 470, 507, 518, 506, 530, 526, 505, 536, 490, 510, 529, 544, 533, 480, 504, 519, 500, 513, 508, 515, 490, 504, 526, 491, 500, 562, 487, 529, 499, 535, 524, 517, 506, 487, 484, 511, 527, 521, 516, 522, 520, 516, 465, 527, 491, 508, 486, 515, 521, 520, 531, 513, 527, 515, 502, 552, 534, 506, 509] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[504, 505, 541, 520, 538, 514, 502, 488, 508, 510, 500, 545, 544, 511, 518, 542, 526, 496, 507, 515, 510, 480, 514, 481, 521, 501, 519, 506, 503, 521, 505, 510, 520, 505, 465, 532, 478, 538, 536, 492, 525, 511, 502, 529, 546, 492, 577, 543, 495, 503, 532, 496, 486, 536, 545, 510, 510, 492, 486, 490, 473, 498, 538, 474, 492, 506, 489, 526, 514, 470, 554, 474, 487, 498, 465, 511, 498, 520, 514, 471, 522, 503, 470, 490, 532, 533, 552, 492, 502, 504, 528, 532, 549, 544, 496, 531, 543, 482, 506, 514, 515, 528, 500, 499, 522, 514, 540, 525, 539, 534, 511, 508, 509, 510, 514, 512, 507, 516, 526, 465, 491, 529, 533, 516, 493, 543, 531, 532] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[540, 501, 535, 498, 504, 491, 479, 460, 452, 558, 524, 501, 472, 511, 557, 539, 513, 527, 518, 503, 501, 503, 548, 512, 529, 475, 508, 543, 487, 487, 486, 542, 499, 478, 496, 534, 549, 502, 512, 503, 542, 527, 529, 546, 491, 528, 485, 532, 512, 517, 493, 505, 517, 490, 513, 520, 483, 537, 509, 508, 489, 513, 530, 462, 514, 513, 505, 519, 472, 524, 522, 528, 521, 516, 524, 486, 490, 503, 541, 527, 520, 487, 530, 512, 513, 511, 494, 515, 492, 522, 526, 517, 481, 507, 506, 522, 535, 519, 551, 537, 531, 493, 531, 504, 509, 520, 532, 514, 556, 516, 533, 507, 524, 495, 507, 504, 514, 480, 487, 522, 538, 466, 514, 526, 548, 505, 505, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[495, 486, 516, 538, 474, 523, 474, 513, 537, 474, 494, 528, 516, 509, 525, 542, 531, 504, 516, 540, 484, 483, 511, 499, 507, 526, 484, 500, 516, 501, 513, 501, 482, 487, 490, 479, 479, 535, 485, 541, 472, 525, 502, 517, 510, 531, 487, 507, 485, 524, 512, 555, 489, 479, 506, 551, 495, 508, 501, 542, 492, 528, 538, 529, 490, 559, 517, 538, 505, 501, 529, 499, 494, 508, 533, 538, 550, 527, 534, 531, 485, 518, 536, 466, 487, 522, 498, 500, 494, 531, 531, 530, 526, 542, 516, 541, 517, 521, 522, 520, 493, 513, 547, 505, 509, 480, 540, 515, 491, 509, 504, 512, 535, 480, 549, 532, 532, 492, 499, 482, 499, 506, 549, 522, 500, 516, 534, 512] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[500, 501, 538, 496, 512, 520, 505, 515, 540, 482, 498, 504, 509, 507, 545, 480, 499, 479, 486, 530, 554, 523, 477, 502, 499, 539, 518, 523, 504, 516, 484, 514, 523, 492, 491, 494, 501, 542, 558, 522, 567, 484, 491, 478, 525, 525, 492, 537, 531, 509, 504, 542, 514, 518, 479, 531, 511, 522, 520, 509, 518, 478, 501, 472, 494, 568, 533, 475, 485, 567, 489, 502, 514, 519, 560, 526, 540, 492, 510, 503, 553, 520, 512, 547, 517, 496, 521, 452, 502, 491, 506, 514, 499, 460, 523, 522, 520, 498, 473, 519, 539, 525, 485, 534, 524, 508, 503, 538, 508, 512, 524, 511, 518, 482, 507, 534, 552, 494, 547, 460, 519, 528, 536, 518, 506, 506, 521, 495] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[512, 519, 521, 531, 561, 495, 514, 506, 498, 507, 532, 550, 501, 530, 538, 474, 496, 535, 502, 521, 459, 524, 521, 519, 494, 516, 482, 522, 494, 511, 499, 544, 538, 538, 501, 518, 483, 497, 529, 497, 513, 538, 535, 483, 535, 505, 462, 533, 498, 511, 476, 572, 529, 488, 458, 513, 517, 458, 516, 509, 505, 536, 491, 506, 537, 522, 495, 512, 515, 470, 507, 518, 506, 530, 526, 505, 536, 490, 510, 529, 544, 533, 480, 504, 519, 500, 513, 508, 515, 490, 504, 526, 491, 500, 562, 487, 529, 499, 535, 524, 517, 506, 487, 484, 511, 527, 521, 516, 522, 520, 516, 465, 527, 491, 508, 486, 515, 521, 520, 531, 513, 527, 515, 502, 552, 534, 506, 509] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[511, 507, 506, 535, 521, 553, 509, 552, 549, 477, 482, 515, 493, 496, 498, 512, 494, 509, 475, 489, 517, 506, 540, 497, 518, 543, 522, 480, 534, 507, 527, 525, 536, 514, 543, 489, 519, 510, 529, 518, 507, 503, 491, 512, 501, 522, 500, 487, 512, 490, 502, 497, 526, 523, 502, 491, 499, 499, 458, 526, 533, 517, 508, 509, 494, 487, 513, 508, 511, 515, 482, 494, 525, 524, 489, 476, 501, 521, 494, 523, 489, 538, 511, 539, 493, 473, 546, 511, 479, 510, 518, 495, 508, 512, 549, 521, 536, 484, 504, 547, 502, 528, 502, 531, 482, 506, 526, 511, 511, 551, 543, 524, 528, 557, 514, 522, 522, 556, 507, 499, 483, 511, 495, 535, 512, 537, 508, 541] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[536, 535, 534, 499, 555, 512, 553, 538, 504, 509, 520, 529, 506, 531, 490, 493, 504, 498, 526, 530, 517, 508, 513, 505, 525, 534, 505, 508, 522, 547, 585, 524, 495, 492, 519, 492, 523, 503, 486, 530, 500, 510, 491, 499, 540, 498, 522, 491, 507, 477, 535, 507, 521, 525, 499, 560, 542, 531, 503, 508, 509, 475, 518, 510, 533, 536, 489, 524, 500, 562, 492, 526, 531, 493, 549, 477, 511, 519, 506, 517, 515, 545, 474, 508, 480, 513, 475, 475, 547, 506, 486, 516, 512, 490, 538, 502, 467, 474, 515, 514, 527, 534, 533, 512, 510, 480, 486, 458, 485, 513, 482, 499, 509, 496, 529, 534, 460, 554, 503, 510, 480, 546, 495, 536, 504, 520, 526, 485] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[523, 480, 487, 514, 510, 534, 518, 521, 531, 519, 499, 549, 496, 527, 508, 533, 529, 528, 553, 520, 507, 503, 467, 542, 521, 484, 521, 542, 513, 528, 518, 515, 534, 531, 498, 533, 537, 500, 503, 489, 523, 486, 510, 514, 519, 493, 498, 510, 553, 520, 531, 526, 486, 519, 511, 483, 502, 487, 505, 497, 500, 503, 475, 499, 526, 520, 516, 499, 522, 528, 496, 514, 509, 484, 525, 525, 523, 523, 540, 514, 509, 503, 518, 488, 486, 517, 507, 512, 503, 560, 516, 488, 474, 497, 540, 522, 497, 495, 538, 501, 482, 542, 517, 489, 490, 530, 529, 504, 520, 530, 500, 544, 488, 485, 504, 510, 474, 526, 473, 521, 517, 525, 525, 523, 536, 510, 500, 492] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[511, 507, 506, 535, 521, 553, 509, 552, 549, 477, 482, 483, 493, 496, 498, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 509, 475, 489, 517, 506, 540, 497, 518, 543, 522, 480, 534, 507, 527, 525, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 514, 543, 489, 519, 510, 529, 518, 507, 503, 491, 512, 501, 522, 500, 487, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 490, 502, 497, 526, 523, 502, 491, 499, 499, 458, 526, 533, 517, 508, 509, 32, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 478, 513, 508, 511, 515, 482, 494, 525, 524, 489, 476, 501, 521, 494, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 538, 511, 539, 493, 473, 546, 511, 479, 510, 518, 495, 508, 512, 549, 521, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 484, 504, 547, 502, 528, 502, 531, 482, 506, 526, 511, 511, 524, 543, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 557, 514, 522, 522, 556, 507, 499, 483, 511, 495, 514, 512, 537, 508, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8184, 8191, 8151, 8048, 8211, 8261, 8306] avg=8192.0000 max=8306 max_rank=7 min=8048 min_rank=4 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32826 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[495, 486, 516, 538, 474, 523, 474, 513, 537, 474, 494, 491, 516, 509, 525, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 504, 516, 540, 484, 483, 511, 499, 507, 526, 484, 500, 516, 501, 513, 501, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 487, 490, 479, 479, 535, 485, 541, 472, 525, 502, 517, 510, 531, 487, 507, 13, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 524, 512, 555, 489, 479, 506, 551, 495, 508, 501, 542, 492, 528, 538, 529, 37, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 541, 517, 538, 505, 501, 529, 499, 494, 508, 533, 538, 550, 527, 534, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 518, 536, 466, 487, 522, 498, 500, 494, 531, 531, 530, 526, 542, 516, 541, 22, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 521, 522, 520, 493, 513, 547, 505, 509, 480, 540, 515, 491, 476, 504, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 480, 549, 532, 532, 492, 499, 482, 499, 506, 549, 495, 500, 516, 534, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8107, 8121, 8047, 8301, 8335, 8248, 8165, 8212] avg=8192.0000 max=8335 max_rank=4 min=8047 min_rank=2 imbalance_ratio=1.017456 max_load_reduction=57201 max_load_reduction_pct=0.872818 moved_to_echo=32960 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[500, 501, 538, 496, 512, 520, 505, 515, 540, 482, 498, 472, 509, 507, 545, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 479, 486, 530, 554, 523, 477, 502, 499, 539, 518, 523, 504, 516, 484, 514, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 492, 491, 494, 501, 542, 558, 522, 567, 484, 491, 478, 525, 525, 492, 537, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 509, 504, 542, 514, 518, 479, 531, 511, 522, 520, 509, 518, 478, 501, 472, 32, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 558, 533, 475, 485, 567, 489, 502, 514, 519, 560, 526, 540, 492, 510, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 520, 512, 547, 517, 496, 521, 452, 502, 491, 506, 514, 499, 460, 523, 522, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 498, 473, 519, 539, 525, 485, 534, 524, 508, 503, 538, 508, 487, 524, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 482, 507, 534, 552, 494, 547, 460, 519, 528, 536, 499, 506, 506, 521, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8120, 8148, 8232, 8216, 8267, 8153, 8196, 8204] avg=8192.0000 max=8267 max_rank=4 min=8120 min_rank=0 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32820 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[504, 505, 541, 520, 538, 514, 502, 488, 508, 510, 500, 511, 544, 511, 518, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 496, 507, 515, 510, 480, 514, 481, 521, 501, 519, 506, 503, 521, 505, 510, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 505, 465, 532, 478, 538, 536, 492, 525, 511, 502, 529, 546, 492, 577, 543, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 503, 532, 496, 486, 536, 545, 510, 510, 492, 486, 490, 473, 498, 538, 474, 34, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 497, 489, 526, 514, 470, 554, 474, 487, 498, 465, 511, 498, 520, 514, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 503, 470, 490, 532, 533, 552, 492, 502, 504, 528, 532, 549, 544, 496, 531, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 482, 506, 514, 515, 528, 500, 499, 522, 514, 540, 525, 539, 507, 511, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 510, 514, 512, 507, 516, 526, 465, 491, 529, 533, 497, 493, 543, 531, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8256, 8116, 8300, 8125, 7980, 8298, 8253, 8208] avg=8192.0000 max=8300 max_rank=2 min=7980 min_rank=4 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32739 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[540, 501, 535, 498, 504, 491, 479, 460, 452, 558, 524, 469, 472, 511, 557, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 527, 518, 503, 501, 503, 548, 512, 529, 475, 508, 543, 487, 487, 486, 542, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 478, 496, 534, 549, 502, 512, 503, 542, 527, 529, 546, 491, 528, 485, 532, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 517, 493, 505, 517, 490, 513, 520, 483, 537, 509, 508, 489, 513, 530, 462, 32, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 504, 505, 519, 472, 524, 522, 528, 521, 516, 524, 486, 490, 503, 541, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 487, 530, 512, 513, 511, 494, 515, 492, 522, 526, 517, 481, 507, 506, 522, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 519, 551, 537, 531, 493, 531, 504, 509, 520, 532, 514, 556, 490, 533, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 495, 507, 504, 514, 480, 487, 522, 538, 466, 514, 506, 548, 505, 505, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8090, 8183, 8263, 8156, 8196, 8173, 8362, 8113] avg=8192.0000 max=8362 max_rank=6 min=8090 min_rank=0 imbalance_ratio=1.020752 max_load_reduction=57174 max_load_reduction_pct=0.872406 moved_to_echo=32844 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[523, 480, 487, 514, 510, 534, 518, 521, 531, 519, 499, 514, 496, 527, 508, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 528, 553, 520, 507, 503, 467, 542, 521, 484, 521, 542, 513, 528, 518, 515, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 531, 498, 533, 537, 500, 503, 489, 523, 486, 510, 514, 519, 493, 498, 510, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 520, 531, 526, 486, 519, 511, 483, 502, 487, 505, 497, 500, 503, 475, 499, 35, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 511, 516, 499, 522, 528, 496, 514, 509, 484, 525, 525, 523, 523, 540, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 503, 518, 488, 486, 517, 507, 512, 503, 560, 516, 488, 474, 497, 540, 522, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 495, 538, 501, 482, 542, 517, 489, 490, 530, 529, 504, 520, 504, 500, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 485, 504, 510, 474, 526, 473, 521, 517, 525, 525, 503, 536, 510, 500, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8214, 8292, 8188, 8158, 8255, 8158, 8182, 8089] avg=8192.0000 max=8292 max_rank=1 min=8089 min_rank=7 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32684 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[512, 519, 521, 531, 561, 495, 514, 506, 498, 507, 532, 515, 501, 530, 538, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 535, 502, 521, 459, 524, 521, 519, 494, 516, 482, 522, 494, 511, 499, 544, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 538, 501, 518, 483, 497, 529, 497, 513, 538, 535, 483, 535, 505, 462, 533, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 511, 476, 572, 529, 488, 458, 513, 517, 458, 516, 509, 505, 536, 491, 506, 35, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 513, 495, 512, 515, 470, 507, 518, 506, 530, 526, 505, 536, 490, 510, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 533, 480, 504, 519, 500, 513, 508, 515, 490, 504, 526, 491, 500, 562, 487, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 499, 535, 524, 517, 506, 487, 484, 511, 527, 521, 516, 522, 494, 516, 465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 491, 508, 486, 515, 521, 520, 531, 513, 527, 515, 483, 552, 534, 506, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8254, 8140, 8214, 8144, 8199, 8194, 8153, 8238] avg=8192.0000 max=8254 max_rank=0 min=8140 min_rank=1 imbalance_ratio=1.007568 max_load_reduction=57282 max_load_reduction_pct=0.874054 moved_to_echo=32784 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[536, 535, 534, 499, 555, 512, 553, 538, 504, 509, 520, 496, 506, 531, 490, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 498, 526, 530, 517, 508, 513, 505, 525, 534, 505, 508, 522, 547, 585, 524, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 492, 519, 492, 523, 503, 486, 530, 500, 510, 491, 499, 540, 498, 522, 491, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 477, 535, 507, 521, 525, 499, 560, 542, 531, 503, 508, 509, 475, 518, 510, 33, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 527, 489, 524, 500, 562, 492, 526, 531, 493, 549, 477, 511, 519, 506, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 545, 474, 508, 480, 513, 475, 475, 547, 506, 486, 516, 512, 490, 538, 502, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 474, 515, 514, 527, 534, 533, 512, 510, 480, 486, 458, 485, 488, 482, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 496, 529, 534, 460, 554, 503, 510, 480, 546, 495, 515, 504, 520, 526, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8311, 8352, 8101, 8285, 8256, 8101, 7964, 8166] avg=8192.0000 max=8352 max_rank=1 min=7964 min_rank=6 imbalance_ratio=1.019531 max_load_reduction=57184 max_load_reduction_pct=0.872559 moved_to_echo=32487 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[504, 505, 541, 520, 538, 514, 502, 488, 508, 510, 500, 511, 544, 511, 518, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 496, 507, 515, 510, 480, 514, 481, 521, 501, 519, 506, 503, 521, 505, 510, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 505, 465, 532, 478, 538, 536, 492, 525, 511, 502, 529, 546, 492, 577, 543, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 503, 532, 496, 486, 536, 545, 510, 510, 492, 486, 490, 473, 498, 538, 474, 34, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 497, 489, 526, 514, 470, 554, 474, 487, 498, 465, 511, 498, 520, 514, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 503, 470, 490, 532, 533, 552, 492, 502, 504, 528, 532, 549, 544, 496, 531, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 482, 506, 514, 515, 528, 500, 499, 522, 514, 540, 525, 539, 507, 511, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 510, 514, 512, 507, 516, 526, 465, 491, 529, 533, 497, 493, 543, 531, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8256, 8116, 8300, 8125, 7980, 8298, 8253, 8208] avg=8192.0000 max=8300 max_rank=2 min=7980 min_rank=4 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32739 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[523, 480, 487, 514, 510, 534, 518, 521, 531, 519, 499, 514, 496, 527, 508, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 528, 553, 520, 507, 503, 467, 542, 521, 484, 521, 542, 513, 528, 518, 515, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 531, 498, 533, 537, 500, 503, 489, 523, 486, 510, 514, 519, 493, 498, 510, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 520, 531, 526, 486, 519, 511, 483, 502, 487, 505, 497, 500, 503, 475, 499, 35, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 511, 516, 499, 522, 528, 496, 514, 509, 484, 525, 525, 523, 523, 540, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 503, 518, 488, 486, 517, 507, 512, 503, 560, 516, 488, 474, 497, 540, 522, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 495, 538, 501, 482, 542, 517, 489, 490, 530, 529, 504, 520, 504, 500, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 485, 504, 510, 474, 526, 473, 521, 517, 525, 525, 503, 536, 510, 500, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8214, 8292, 8188, 8158, 8255, 8158, 8182, 8089] avg=8192.0000 max=8292 max_rank=1 min=8089 min_rank=7 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32684 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[540, 501, 535, 498, 504, 491, 479, 460, 452, 558, 524, 469, 472, 511, 557, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 527, 518, 503, 501, 503, 548, 512, 529, 475, 508, 543, 487, 487, 486, 542, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 478, 496, 534, 549, 502, 512, 503, 542, 527, 529, 546, 491, 528, 485, 532, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 517, 493, 505, 517, 490, 513, 520, 483, 537, 509, 508, 489, 513, 530, 462, 32, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 504, 505, 519, 472, 524, 522, 528, 521, 516, 524, 486, 490, 503, 541, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 487, 530, 512, 513, 511, 494, 515, 492, 522, 526, 517, 481, 507, 506, 522, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 519, 551, 537, 531, 493, 531, 504, 509, 520, 532, 514, 556, 490, 533, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 495, 507, 504, 514, 480, 487, 522, 538, 466, 514, 506, 548, 505, 505, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8090, 8183, 8263, 8156, 8196, 8173, 8362, 8113] avg=8192.0000 max=8362 max_rank=6 min=8090 min_rank=0 imbalance_ratio=1.020752 max_load_reduction=57174 max_load_reduction_pct=0.872406 moved_to_echo=32844 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[500, 501, 538, 496, 512, 520, 505, 515, 540, 482, 498, 472, 509, 507, 545, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 479, 486, 530, 554, 523, 477, 502, 499, 539, 518, 523, 504, 516, 484, 514, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 492, 491, 494, 501, 542, 558, 522, 567, 484, 491, 478, 525, 525, 492, 537, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 509, 504, 542, 514, 518, 479, 531, 511, 522, 520, 509, 518, 478, 501, 472, 32, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 558, 533, 475, 485, 567, 489, 502, 514, 519, 560, 526, 540, 492, 510, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 520, 512, 547, 517, 496, 521, 452, 502, 491, 506, 514, 499, 460, 523, 522, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 498, 473, 519, 539, 525, 485, 534, 524, 508, 503, 538, 508, 487, 524, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 482, 507, 534, 552, 494, 547, 460, 519, 528, 536, 499, 506, 506, 521, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8120, 8148, 8232, 8216, 8267, 8153, 8196, 8204] avg=8192.0000 max=8267 max_rank=4 min=8120 min_rank=0 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32820 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[511, 507, 506, 535, 521, 553, 509, 552, 549, 477, 482, 483, 493, 496, 498, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 509, 475, 489, 517, 506, 540, 497, 518, 543, 522, 480, 534, 507, 527, 525, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 514, 543, 489, 519, 510, 529, 518, 507, 503, 491, 512, 501, 522, 500, 487, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 490, 502, 497, 526, 523, 502, 491, 499, 499, 458, 526, 533, 517, 508, 509, 32, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 478, 513, 508, 511, 515, 482, 494, 525, 524, 489, 476, 501, 521, 494, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 538, 511, 539, 493, 473, 546, 511, 479, 510, 518, 495, 508, 512, 549, 521, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 484, 504, 547, 502, 528, 502, 531, 482, 506, 526, 511, 511, 524, 543, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 557, 514, 522, 522, 556, 507, 499, 483, 511, 495, 514, 512, 537, 508, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8184, 8191, 8151, 8048, 8211, 8261, 8306] avg=8192.0000 max=8306 max_rank=7 min=8048 min_rank=4 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32826 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 3968, 4096, 4224, 4224, 4224, 4096, 4096, 4352, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 3968, 384, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 3968, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 3968, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 3968, 4096, 4224, 4224, 4224, 4096, 4096, 4352, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 3968, 384, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[492, 520, 509, 535, 541, 526, 523, 516, 500, 514, 475, 537, 523, 533, 525, 495, 465, 482, 553, 509, 504, 503, 489, 546, 531, 561, 513, 528, 496, 495, 468, 529, 550, 529, 535, 464, 497, 548, 531, 509, 490, 547, 549, 529, 539, 513, 525, 512, 504, 499, 501, 558, 470, 520, 504, 464, 487, 503, 473, 582, 516, 559, 531, 490, 520, 560, 497, 499, 496, 516, 537, 554, 516, 501, 495, 534, 513, 491, 476, 521, 466, 468, 516, 461, 488, 513, 539, 506, 476, 523, 509, 528, 507, 557, 491, 478, 493, 504, 545, 522, 521, 523, 507, 483, 498, 512, 491, 531, 519, 485, 519, 491, 502, 507, 528, 561, 518, 493, 508, 498, 441, 528, 495, 526, 527, 506, 508, 530] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[540, 511, 505, 523, 502, 501, 519, 514, 549, 496, 485, 491, 539, 500, 526, 487, 522, 520, 497, 549, 544, 508, 473, 468, 491, 504, 501, 504, 534, 526, 495, 511, 493, 505, 513, 521, 486, 533, 487, 500, 530, 530, 524, 493, 477, 505, 517, 538, 512, 518, 492, 509, 560, 509, 481, 485, 514, 491, 489, 547, 512, 513, 510, 557, 531, 503, 519, 513, 512, 565, 549, 552, 513, 495, 534, 514, 517, 509, 505, 521, 532, 506, 531, 499, 483, 546, 523, 481, 495, 537, 482, 535, 520, 516, 492, 496, 554, 494, 551, 483, 494, 524, 507, 514, 558, 493, 512, 482, 491, 508, 501, 571, 503, 501, 516, 531, 538, 509, 509, 500, 483, 503, 513, 464, 466, 545, 486, 520] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[518, 521, 485, 506, 512, 510, 494, 523, 528, 515, 505, 509, 514, 521, 488, 526, 540, 478, 517, 506, 497, 516, 494, 558, 501, 552, 512, 520, 507, 499, 544, 511, 554, 519, 509, 544, 502, 554, 557, 495, 505, 504, 507, 534, 517, 499, 535, 514, 525, 524, 510, 528, 537, 503, 516, 467, 502, 552, 517, 490, 556, 495, 506, 529, 510, 504, 535, 486, 514, 526, 474, 486, 533, 500, 517, 509, 499, 511, 485, 522, 485, 521, 510, 531, 536, 516, 509, 516, 507, 531, 498, 502, 495, 518, 489, 534, 539, 536, 561, 499, 500, 509, 487, 524, 508, 464, 498, 511, 482, 510, 519, 514, 495, 475, 502, 511, 524, 538, 475, 510, 528, 499, 501, 518, 516, 483, 510, 448] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[514, 514, 533, 561, 526, 540, 530, 482, 549, 515, 521, 504, 507, 466, 518, 492, 511, 494, 490, 533, 497, 513, 518, 499, 555, 515, 493, 509, 531, 523, 537, 495, 543, 547, 520, 509, 534, 526, 477, 551, 516, 539, 490, 486, 514, 494, 532, 528, 488, 544, 533, 503, 506, 492, 484, 502, 539, 531, 489, 536, 521, 508, 510, 488, 510, 507, 487, 537, 530, 519, 485, 485, 512, 495, 498, 504, 520, 549, 494, 511, 493, 557, 511, 532, 474, 533, 507, 495, 519, 548, 497, 531, 512, 499, 509, 506, 522, 469, 486, 533, 483, 503, 514, 530, 536, 491, 487, 513, 513, 531, 518, 503, 535, 478, 495, 528, 525, 489, 482, 519, 517, 517, 494, 490, 503, 525, 473, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[522, 509, 498, 517, 564, 548, 505, 513, 519, 507, 508, 491, 494, 498, 515, 529, 458, 544, 505, 492, 487, 488, 518, 537, 496, 494, 511, 497, 473, 476, 505, 499, 537, 532, 506, 527, 476, 485, 464, 546, 559, 511, 557, 522, 515, 506, 488, 546, 528, 518, 494, 515, 499, 515, 487, 524, 533, 521, 495, 500, 527, 484, 499, 513, 508, 516, 525, 462, 510, 486, 513, 489, 520, 518, 534, 521, 499, 538, 492, 512, 485, 491, 547, 514, 535, 468, 510, 525, 489, 500, 466, 530, 508, 528, 527, 508, 513, 522, 513, 506, 519, 545, 532, 508, 509, 508, 535, 544, 537, 514, 547, 571, 516, 508, 495, 526, 511, 539, 506, 473, 522, 491, 492, 528, 512, 533, 493, 548] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[518, 521, 485, 506, 512, 510, 494, 523, 528, 515, 505, 509, 514, 521, 488, 526, 540, 478, 517, 506, 497, 516, 494, 558, 501, 552, 512, 520, 507, 499, 544, 511, 554, 519, 509, 544, 502, 554, 557, 495, 505, 504, 507, 534, 517, 499, 535, 514, 525, 524, 510, 528, 537, 503, 516, 467, 502, 552, 517, 490, 556, 495, 506, 529, 510, 504, 535, 486, 514, 526, 474, 486, 533, 500, 517, 509, 499, 511, 485, 522, 485, 521, 510, 531, 536, 516, 509, 516, 507, 531, 498, 502, 495, 518, 489, 534, 539, 536, 561, 499, 500, 509, 487, 524, 508, 464, 498, 511, 482, 510, 519, 514, 495, 475, 502, 511, 524, 538, 475, 510, 528, 499, 501, 518, 516, 483, 510, 448] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[501, 518, 531, 516, 508, 553, 480, 487, 531, 505, 549, 500, 467, 559, 555, 536, 494, 536, 535, 512, 505, 503, 506, 483, 504, 469, 531, 480, 474, 502, 510, 508, 543, 526, 507, 525, 505, 570, 499, 491, 448, 500, 503, 530, 537, 517, 502, 522, 512, 488, 549, 488, 513, 482, 550, 514, 491, 501, 517, 518, 530, 553, 490, 518, 471, 562, 527, 516, 549, 503, 520, 516, 518, 499, 507, 522, 473, 519, 486, 480, 475, 511, 517, 556, 529, 531, 520, 499, 526, 508, 448, 545, 517, 487, 530, 496, 524, 512, 542, 558, 505, 535, 507, 506, 466, 527, 530, 543, 492, 546, 502, 513, 448, 509, 465, 494, 513, 500, 517, 480, 511, 532, 519, 492, 517, 523, 534, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[520, 529, 519, 494, 515, 525, 514, 497, 494, 553, 508, 569, 504, 526, 506, 505, 505, 485, 520, 462, 509, 489, 513, 516, 507, 542, 512, 508, 529, 507, 521, 498, 495, 550, 492, 493, 536, 506, 496, 506, 513, 496, 537, 500, 504, 544, 502, 470, 506, 503, 502, 526, 491, 520, 528, 513, 518, 507, 495, 532, 501, 511, 543, 530, 488, 502, 519, 529, 527, 542, 512, 493, 486, 502, 526, 523, 512, 513, 490, 522, 501, 503, 533, 534, 510, 494, 526, 496, 501, 508, 549, 522, 513, 514, 528, 488, 519, 496, 502, 525, 547, 515, 494, 482, 509, 498, 499, 533, 526, 490, 492, 494, 524, 512, 488, 463, 526, 527, 487, 544, 539, 548, 496, 561, 513, 483, 468, 563] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[522, 509, 498, 517, 564, 548, 505, 513, 519, 507, 508, 491, 494, 498, 515, 529, 458, 544, 505, 492, 487, 488, 518, 537, 496, 494, 511, 497, 473, 476, 505, 499, 537, 532, 506, 527, 476, 485, 464, 546, 559, 511, 557, 522, 515, 506, 488, 546, 528, 518, 494, 515, 499, 515, 487, 524, 533, 521, 495, 500, 527, 484, 499, 513, 508, 516, 525, 462, 510, 486, 513, 489, 520, 518, 534, 521, 499, 538, 492, 512, 485, 491, 547, 514, 535, 468, 510, 525, 489, 500, 466, 530, 508, 528, 527, 508, 513, 522, 513, 506, 519, 545, 532, 508, 509, 508, 535, 544, 537, 514, 547, 571, 516, 508, 495, 526, 511, 539, 506, 473, 522, 491, 492, 528, 512, 533, 493, 548] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[457, 505, 501, 484, 499, 496, 527, 484, 516, 532, 541, 517, 525, 531, 485, 553, 505, 496, 504, 508, 538, 483, 476, 500, 539, 514, 476, 508, 480, 553, 523, 513, 507, 491, 504, 508, 545, 510, 502, 517, 517, 523, 523, 490, 482, 543, 574, 511, 551, 504, 477, 542, 555, 492, 516, 501, 505, 514, 527, 536, 499, 556, 500, 503, 500, 510, 536, 527, 519, 533, 523, 514, 516, 488, 515, 527, 509, 508, 481, 493, 540, 503, 511, 518, 505, 528, 496, 500, 502, 483, 509, 498, 509, 479, 534, 502, 531, 512, 507, 468, 513, 521, 532, 498, 538, 483, 484, 535, 499, 509, 561, 513, 498, 532, 510, 501, 531, 518, 522, 554, 544, 471, 508, 484, 510, 537, 516, 486] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[501, 518, 531, 516, 508, 553, 480, 487, 531, 505, 549, 500, 467, 559, 555, 536, 494, 536, 535, 512, 505, 503, 506, 483, 504, 469, 531, 480, 474, 502, 510, 508, 543, 526, 507, 525, 505, 570, 499, 491, 448, 500, 503, 530, 537, 517, 502, 522, 512, 488, 549, 488, 513, 482, 550, 514, 491, 501, 517, 518, 530, 553, 490, 518, 471, 562, 527, 516, 549, 503, 520, 516, 518, 499, 507, 522, 473, 519, 486, 480, 475, 511, 517, 556, 529, 531, 520, 499, 526, 508, 448, 545, 517, 487, 530, 496, 524, 512, 542, 558, 505, 535, 507, 506, 466, 527, 530, 543, 492, 546, 502, 513, 448, 509, 465, 494, 513, 500, 517, 480, 511, 532, 519, 492, 517, 523, 534, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[520, 529, 519, 494, 515, 485, 514, 497, 494, 553, 508, 569, 504, 526, 506, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 485, 520, 462, 509, 489, 513, 516, 507, 542, 512, 508, 529, 507, 521, 498, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 550, 492, 493, 536, 444, 496, 506, 513, 496, 537, 500, 504, 544, 502, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 503, 502, 526, 491, 520, 528, 513, 518, 507, 495, 517, 501, 511, 543, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 502, 519, 529, 527, 542, 512, 493, 486, 502, 526, 523, 512, 513, 490, 522, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 503, 533, 534, 510, 494, 526, 496, 501, 508, 549, 522, 513, 514, 528, 488, 16, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 496, 479, 525, 547, 515, 494, 482, 509, 498, 499, 533, 526, 490, 492, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 512, 488, 463, 526, 527, 487, 544, 539, 548, 496, 561, 513, 483, 468, 563, 40, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8180, 8078, 8211, 8187, 8250, 8098, 8294] avg=8192.0000 max=8294 max_rank=7 min=8078 min_rank=2 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32829 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[518, 521, 485, 506, 512, 471, 494, 523, 528, 515, 505, 509, 514, 521, 488, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 478, 517, 506, 497, 516, 494, 558, 501, 552, 512, 520, 507, 499, 544, 511, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 519, 509, 544, 502, 486, 557, 495, 505, 504, 507, 534, 517, 499, 535, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 524, 510, 528, 537, 503, 516, 467, 502, 552, 517, 477, 556, 495, 506, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 504, 535, 486, 514, 526, 474, 486, 533, 500, 517, 509, 499, 511, 485, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 521, 510, 531, 536, 516, 509, 516, 507, 531, 498, 502, 495, 518, 489, 534, 18, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 536, 536, 499, 500, 509, 487, 524, 508, 464, 498, 511, 482, 510, 519, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 475, 502, 511, 524, 538, 475, 510, 528, 499, 501, 518, 516, 483, 510, 448, 39, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8136, 8314, 8281, 8244, 8111, 8229, 8136, 8085] avg=8192.0000 max=8314 max_rank=1 min=8085 min_rank=7 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32561 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[492, 520, 509, 535, 541, 526, 523, 516, 500, 514, 475, 537, 523, 533, 525, 495, 465, 482, 553, 509, 504, 503, 489, 546, 531, 561, 513, 528, 496, 495, 468, 529, 550, 529, 535, 464, 497, 548, 531, 509, 490, 547, 549, 529, 539, 513, 525, 512, 504, 499, 501, 558, 470, 520, 504, 464, 487, 503, 473, 582, 516, 559, 531, 490, 520, 560, 497, 499, 496, 516, 537, 554, 516, 501, 495, 534, 513, 491, 476, 521, 466, 468, 516, 461, 488, 513, 539, 506, 476, 523, 509, 528, 507, 557, 491, 478, 493, 504, 545, 522, 521, 523, 507, 483, 498, 512, 491, 531, 519, 485, 519, 491, 502, 507, 528, 561, 518, 493, 508, 498, 441, 528, 495, 526, 527, 506, 508, 530] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[501, 518, 531, 516, 508, 510, 480, 487, 531, 505, 549, 500, 467, 559, 555, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 536, 535, 512, 505, 503, 506, 483, 504, 469, 531, 480, 474, 502, 510, 508, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 526, 507, 525, 505, 500, 499, 491, 448, 500, 503, 530, 537, 517, 502, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 488, 549, 488, 513, 482, 550, 514, 491, 501, 517, 504, 530, 553, 490, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 562, 527, 516, 549, 503, 520, 516, 518, 499, 507, 522, 473, 519, 486, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 511, 517, 556, 529, 531, 520, 499, 526, 508, 448, 545, 517, 487, 530, 496, 17, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 512, 518, 558, 505, 535, 507, 506, 466, 527, 530, 543, 492, 546, 502, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 448, 509, 465, 494, 513, 500, 517, 480, 511, 532, 519, 492, 517, 523, 534, 524, 43, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8253, 8116, 8155, 8200, 8168, 8226, 8284, 8134] avg=8192.0000 max=8284 max_rank=6 min=8116 min_rank=1 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32812 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[492, 520, 509, 535, 541, 482, 523, 516, 500, 514, 475, 537, 523, 533, 525, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 482, 553, 509, 504, 503, 489, 546, 531, 561, 513, 528, 496, 495, 468, 529, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 529, 535, 464, 497, 472, 531, 509, 490, 547, 549, 529, 539, 513, 525, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 499, 501, 558, 470, 520, 504, 464, 487, 503, 473, 558, 516, 559, 531, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 560, 497, 499, 496, 516, 537, 554, 516, 501, 495, 534, 513, 491, 476, 521, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 466, 468, 516, 461, 488, 513, 539, 506, 476, 523, 509, 528, 507, 557, 491, 478, 20, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 504, 514, 522, 521, 523, 507, 483, 498, 512, 491, 531, 519, 485, 519, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 507, 528, 561, 518, 493, 508, 498, 441, 528, 495, 526, 527, 506, 508, 530, 44, 11, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8220, 8239, 8291, 8137, 8230, 8066, 8113, 8240] avg=8192.0000 max=8291 max_rank=2 min=8066 min_rank=5 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32649 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[514, 514, 533, 561, 526, 498, 530, 482, 549, 515, 521, 504, 507, 466, 518, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 494, 490, 533, 497, 513, 518, 499, 555, 515, 493, 509, 531, 523, 537, 495, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 547, 520, 509, 534, 462, 477, 551, 516, 539, 490, 486, 514, 494, 532, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 544, 533, 503, 506, 492, 484, 502, 539, 531, 489, 521, 521, 508, 510, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 507, 487, 537, 530, 519, 485, 485, 512, 495, 498, 504, 520, 549, 494, 511, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 557, 511, 532, 474, 533, 507, 495, 519, 548, 497, 531, 512, 499, 509, 506, 15, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 469, 465, 533, 483, 503, 514, 530, 536, 491, 487, 513, 513, 531, 518, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 478, 495, 528, 525, 489, 482, 519, 517, 517, 494, 490, 503, 525, 473, 503, 42, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 8272, 8242, 8159, 8144, 8252, 8111, 8126] avg=8192.0000 max=8272 max_rank=1 min=8111 min_rank=6 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32633 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[457, 505, 501, 484, 499, 458, 527, 484, 516, 532, 541, 517, 525, 531, 485, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 496, 504, 508, 538, 483, 476, 500, 539, 514, 476, 508, 480, 553, 523, 513, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 491, 504, 508, 545, 448, 502, 517, 517, 523, 523, 490, 482, 543, 574, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 504, 477, 542, 555, 492, 516, 501, 505, 514, 527, 521, 499, 556, 500, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 510, 536, 527, 519, 533, 523, 514, 516, 488, 515, 527, 509, 508, 481, 493, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 503, 511, 518, 505, 528, 496, 500, 502, 483, 509, 498, 509, 479, 534, 502, 16, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 512, 484, 468, 513, 521, 532, 498, 538, 483, 484, 535, 499, 509, 561, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 532, 510, 501, 531, 518, 522, 554, 544, 471, 508, 484, 510, 537, 516, 486, 38, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8115, 8173, 8185, 8263, 8200, 8147, 8181, 8272] avg=8192.0000 max=8272 max_rank=7 min=8115 min_rank=0 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32800 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[540, 511, 505, 523, 502, 462, 519, 514, 549, 496, 485, 491, 539, 500, 526, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 520, 497, 549, 544, 508, 473, 468, 491, 504, 501, 504, 534, 526, 495, 511, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 505, 513, 521, 486, 468, 487, 500, 530, 530, 524, 493, 477, 505, 517, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 518, 492, 509, 560, 509, 481, 485, 514, 491, 489, 531, 512, 513, 510, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 503, 519, 513, 512, 565, 549, 552, 513, 495, 534, 514, 517, 509, 505, 521, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 506, 531, 499, 483, 546, 523, 481, 495, 537, 482, 535, 520, 516, 492, 496, 17, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 494, 527, 483, 494, 524, 507, 514, 558, 493, 512, 482, 491, 508, 501, 571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 501, 516, 531, 538, 509, 509, 500, 483, 503, 513, 464, 466, 545, 486, 520, 39, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8149, 8207, 8087, 8183, 8353, 8206, 8213, 8138] avg=8192.0000 max=8353 max_rank=4 min=8087 min_rank=2 imbalance_ratio=1.019653 max_load_reduction=57183 max_load_reduction_pct=0.872543 moved_to_echo=32910 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[518, 521, 485, 506, 512, 471, 494, 523, 528, 515, 505, 509, 514, 521, 488, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 478, 517, 506, 497, 516, 494, 558, 501, 552, 512, 520, 507, 499, 544, 511, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 519, 509, 544, 502, 486, 557, 495, 505, 504, 507, 534, 517, 499, 535, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 524, 510, 528, 537, 503, 516, 467, 502, 552, 517, 477, 556, 495, 506, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 504, 535, 486, 514, 526, 474, 486, 533, 500, 517, 509, 499, 511, 485, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 521, 510, 531, 536, 516, 509, 516, 507, 531, 498, 502, 495, 518, 489, 534, 18, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 536, 536, 499, 500, 509, 487, 524, 508, 464, 498, 511, 482, 510, 519, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 475, 502, 511, 524, 538, 475, 510, 528, 499, 501, 518, 516, 483, 510, 448, 39, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8136, 8314, 8281, 8244, 8111, 8229, 8136, 8085] avg=8192.0000 max=8314 max_rank=1 min=8085 min_rank=7 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32561 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[522, 509, 498, 517, 564, 506, 505, 513, 519, 507, 508, 491, 494, 498, 515, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 458, 544, 505, 492, 487, 488, 518, 537, 496, 494, 511, 497, 473, 476, 505, 499, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 532, 506, 527, 476, 425, 464, 546, 559, 511, 557, 522, 515, 506, 488, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 518, 494, 515, 499, 515, 487, 524, 533, 521, 495, 487, 527, 484, 499, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 516, 525, 462, 510, 486, 513, 489, 520, 518, 534, 521, 499, 538, 492, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 491, 547, 514, 535, 468, 510, 525, 489, 500, 466, 530, 508, 528, 527, 508, 16, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 522, 490, 506, 519, 545, 532, 508, 509, 508, 535, 544, 537, 514, 547, 571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 508, 495, 526, 511, 539, 506, 473, 522, 491, 492, 528, 512, 533, 493, 548, 42, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8035, 8217, 8139, 8143, 8160, 8400, 8247] avg=8192.0000 max=8400 max_rank=6 min=8035 min_rank=1 imbalance_ratio=1.025391 max_load_reduction=57136 max_load_reduction_pct=0.871826 moved_to_echo=32950 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[540, 511, 505, 523, 502, 462, 519, 514, 549, 496, 485, 491, 539, 500, 526, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 520, 497, 549, 544, 508, 473, 468, 491, 504, 501, 504, 534, 526, 495, 511, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 505, 513, 521, 486, 468, 487, 500, 530, 530, 524, 493, 477, 505, 517, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 518, 492, 509, 560, 509, 481, 485, 514, 491, 489, 531, 512, 513, 510, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 503, 519, 513, 512, 565, 549, 552, 513, 495, 534, 514, 517, 509, 505, 521, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 506, 531, 499, 483, 546, 523, 481, 495, 537, 482, 535, 520, 516, 492, 496, 17, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 494, 527, 483, 494, 524, 507, 514, 558, 493, 512, 482, 491, 508, 501, 571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 501, 516, 531, 538, 509, 509, 500, 483, 503, 513, 464, 466, 545, 486, 520, 39, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8149, 8207, 8087, 8183, 8353, 8206, 8213, 8138] avg=8192.0000 max=8353 max_rank=4 min=8087 min_rank=2 imbalance_ratio=1.019653 max_load_reduction=57183 max_load_reduction_pct=0.872543 moved_to_echo=32910 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[514, 514, 533, 561, 526, 498, 530, 482, 549, 515, 521, 504, 507, 466, 518, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 494, 490, 533, 497, 513, 518, 499, 555, 515, 493, 509, 531, 523, 537, 495, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 547, 520, 509, 534, 462, 477, 551, 516, 539, 490, 486, 514, 494, 532, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 544, 533, 503, 506, 492, 484, 502, 539, 531, 489, 521, 521, 508, 510, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 507, 487, 537, 530, 519, 485, 485, 512, 495, 498, 504, 520, 549, 494, 511, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 557, 511, 532, 474, 533, 507, 495, 519, 548, 497, 531, 512, 499, 509, 506, 15, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 469, 465, 533, 483, 503, 514, 530, 536, 491, 487, 513, 513, 531, 518, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 478, 495, 528, 525, 489, 482, 519, 517, 517, 494, 490, 503, 525, 473, 503, 42, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 8272, 8242, 8159, 8144, 8252, 8111, 8126] avg=8192.0000 max=8272 max_rank=1 min=8111 min_rank=6 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32633 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[520, 529, 519, 494, 515, 485, 514, 497, 494, 553, 508, 569, 504, 526, 506, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 485, 520, 462, 509, 489, 513, 516, 507, 542, 512, 508, 529, 507, 521, 498, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 550, 492, 493, 536, 444, 496, 506, 513, 496, 537, 500, 504, 544, 502, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 503, 502, 526, 491, 520, 528, 513, 518, 507, 495, 517, 501, 511, 543, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 502, 519, 529, 527, 542, 512, 493, 486, 502, 526, 523, 512, 513, 490, 522, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 503, 533, 534, 510, 494, 526, 496, 501, 508, 549, 522, 513, 514, 528, 488, 16, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 496, 479, 525, 547, 515, 494, 482, 509, 498, 499, 533, 526, 490, 492, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 512, 488, 463, 526, 527, 487, 544, 539, 548, 496, 561, 513, 483, 468, 563, 40, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8180, 8078, 8211, 8187, 8250, 8098, 8294] avg=8192.0000 max=8294 max_rank=7 min=8078 min_rank=2 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32829 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 3968, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 3968, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 3712, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 384, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 3968, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 3968, 4224, 4096, 4224, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 384, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[523, 486, 558, 504, 500, 504, 476, 529, 553, 518, 518, 483, 479, 467, 483, 485, 515, 527, 480, 524, 530, 487, 457, 481, 520, 502, 505, 523, 508, 499, 493, 536, 526, 542, 524, 511, 511, 464, 514, 492, 527, 525, 526, 481, 535, 496, 495, 502, 515, 512, 517, 538, 542, 521, 514, 539, 556, 486, 533, 514, 486, 476, 522, 574, 503, 492, 536, 527, 521, 523, 532, 490, 527, 486, 472, 507, 507, 526, 511, 554, 502, 478, 515, 489, 504, 514, 576, 487, 486, 522, 499, 501, 509, 502, 535, 512, 498, 503, 476, 515, 524, 514, 537, 532, 500, 528, 512, 521, 504, 485, 529, 516, 498, 522, 513, 484, 494, 516, 527, 516, 565, 537, 522, 515, 534, 522, 557, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[499, 509, 485, 469, 547, 506, 512, 483, 507, 472, 461, 481, 516, 525, 518, 513, 527, 551, 533, 532, 542, 499, 518, 519, 552, 496, 480, 521, 545, 487, 529, 489, 520, 521, 482, 524, 535, 497, 555, 525, 472, 484, 484, 550, 523, 508, 494, 529, 551, 543, 503, 526, 519, 502, 482, 555, 509, 501, 471, 481, 520, 480, 517, 497, 496, 519, 514, 500, 524, 508, 510, 463, 529, 525, 504, 532, 477, 500, 532, 480, 501, 481, 507, 472, 518, 521, 502, 540, 488, 521, 504, 502, 524, 475, 533, 522, 520, 513, 526, 501, 584, 516, 545, 527, 533, 531, 497, 532, 483, 520, 552, 534, 496, 481, 541, 530, 529, 477, 512, 501, 522, 485, 535, 534, 510, 524, 542, 496] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[502, 513, 531, 485, 490, 519, 535, 503, 550, 534, 506, 527, 525, 497, 506, 511, 497, 509, 511, 484, 534, 506, 464, 529, 539, 517, 547, 487, 522, 496, 523, 533, 519, 518, 587, 505, 505, 502, 529, 491, 497, 503, 523, 477, 504, 502, 482, 511, 516, 561, 506, 529, 494, 498, 495, 508, 494, 541, 526, 482, 550, 536, 516, 520, 513, 518, 532, 446, 525, 522, 517, 499, 484, 507, 521, 504, 520, 508, 524, 529, 481, 504, 533, 489, 525, 514, 506, 516, 494, 525, 544, 489, 461, 546, 499, 517, 542, 544, 542, 521, 528, 552, 476, 517, 534, 504, 503, 503, 533, 485, 487, 561, 517, 509, 479, 487, 505, 526, 492, 470, 535, 476, 523, 499, 490, 507, 538, 480] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[508, 534, 505, 488, 523, 498, 515, 483, 504, 487, 575, 487, 498, 483, 538, 533, 512, 499, 528, 538, 493, 485, 523, 501, 512, 490, 506, 525, 526, 529, 503, 517, 517, 522, 523, 520, 504, 473, 556, 524, 526, 511, 528, 495, 523, 564, 513, 502, 506, 548, 543, 574, 526, 533, 532, 530, 501, 515, 500, 553, 516, 527, 494, 509, 508, 514, 483, 480, 537, 486, 511, 529, 521, 513, 486, 506, 499, 491, 539, 528, 501, 496, 508, 534, 501, 519, 477, 509, 549, 511, 499, 497, 529, 530, 496, 476, 492, 504, 497, 489, 507, 492, 542, 489, 518, 483, 538, 492, 522, 524, 514, 521, 514, 481, 509, 495, 481, 480, 513, 482, 488, 529, 513, 501, 519, 545, 529, 516] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[508, 534, 508, 512, 505, 526, 528, 469, 501, 509, 551, 522, 500, 491, 526, 518, 484, 527, 484, 505, 520, 477, 503, 482, 532, 554, 512, 468, 519, 525, 492, 491, 523, 491, 537, 514, 502, 490, 525, 500, 505, 534, 526, 528, 489, 519, 533, 494, 493, 539, 518, 532, 503, 516, 491, 504, 545, 482, 543, 504, 522, 493, 532, 522, 496, 510, 504, 507, 544, 510, 543, 461, 520, 488, 518, 561, 497, 515, 500, 539, 523, 528, 517, 517, 476, 509, 539, 499, 517, 505, 472, 502, 474, 500, 545, 502, 508, 504, 519, 532, 510, 522, 490, 531, 488, 502, 498, 554, 525, 508, 477, 505, 542, 479, 513, 512, 495, 497, 519, 546, 538, 546, 524, 543, 466, 532, 531, 510] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[514, 526, 501, 486, 509, 529, 540, 501, 478, 504, 521, 491, 507, 533, 540, 478, 542, 512, 530, 506, 545, 518, 461, 490, 505, 516, 491, 508, 510, 498, 540, 522, 537, 542, 525, 550, 556, 485, 491, 533, 504, 500, 513, 501, 525, 518, 559, 507, 474, 519, 496, 510, 519, 492, 529, 508, 528, 500, 508, 509, 533, 531, 531, 497, 521, 535, 539, 504, 476, 469, 492, 509, 525, 496, 494, 501, 521, 475, 544, 517, 529, 525, 509, 517, 545, 534, 518, 534, 520, 506, 518, 498, 507, 512, 489, 521, 521, 527, 468, 518, 502, 461, 516, 539, 508, 475, 537, 521, 522, 507, 466, 505, 544, 511, 509, 498, 494, 497, 484, 512, 515, 545, 518, 490, 500, 538, 512, 494] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[523, 486, 558, 504, 500, 504, 476, 529, 553, 518, 518, 483, 479, 467, 483, 485, 515, 527, 480, 524, 530, 487, 457, 481, 520, 502, 505, 523, 508, 499, 493, 536, 526, 542, 524, 511, 511, 464, 514, 492, 527, 525, 526, 481, 535, 496, 495, 502, 515, 512, 517, 538, 542, 521, 514, 539, 556, 486, 533, 514, 486, 476, 522, 574, 503, 492, 536, 527, 521, 523, 532, 490, 527, 486, 472, 507, 507, 526, 511, 554, 502, 478, 515, 489, 504, 514, 576, 487, 486, 522, 499, 501, 509, 502, 535, 512, 498, 503, 476, 515, 524, 514, 537, 532, 500, 528, 512, 521, 504, 485, 529, 516, 498, 522, 513, 484, 494, 516, 527, 516, 565, 537, 522, 515, 534, 522, 557, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[559, 515, 574, 507, 494, 516, 537, 470, 524, 498, 496, 487, 507, 466, 550, 519, 512, 504, 502, 501, 542, 492, 493, 480, 505, 510, 526, 550, 506, 499, 507, 506, 542, 543, 512, 563, 481, 535, 517, 508, 512, 541, 481, 534, 536, 535, 490, 510, 521, 519, 475, 510, 539, 513, 557, 470, 528, 520, 504, 545, 530, 488, 476, 530, 481, 513, 523, 537, 527, 504, 494, 516, 532, 477, 505, 516, 507, 565, 476, 506, 502, 500, 496, 524, 478, 511, 532, 522, 501, 505, 533, 490, 497, 517, 535, 492, 507, 537, 511, 523, 495, 538, 506, 500, 544, 510, 502, 507, 510, 522, 500, 509, 524, 479, 487, 514, 467, 517, 515, 522, 522, 501, 502, 483, 493, 525, 532, 499] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[508, 534, 505, 488, 523, 498, 515, 483, 504, 487, 575, 487, 498, 483, 538, 533, 512, 499, 528, 538, 493, 485, 523, 501, 512, 490, 506, 525, 526, 529, 503, 517, 517, 522, 523, 520, 504, 473, 556, 524, 526, 511, 528, 495, 523, 564, 513, 502, 506, 548, 543, 574, 526, 533, 532, 530, 501, 515, 500, 553, 516, 527, 494, 509, 508, 514, 483, 480, 537, 486, 511, 529, 521, 513, 486, 506, 499, 491, 539, 528, 501, 496, 508, 534, 501, 519, 477, 509, 549, 511, 499, 497, 529, 530, 496, 476, 492, 504, 497, 489, 507, 492, 542, 489, 518, 483, 538, 492, 522, 524, 514, 521, 514, 481, 509, 495, 481, 480, 513, 482, 488, 529, 513, 501, 519, 545, 529, 516] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[534, 488, 518, 502, 495, 537, 509, 551, 529, 512, 518, 501, 479, 512, 476, 515, 535, 500, 530, 518, 499, 530, 532, 516, 491, 505, 526, 519, 527, 532, 494, 489, 512, 488, 530, 525, 512, 504, 527, 531, 498, 533, 473, 466, 528, 525, 503, 517, 515, 471, 467, 494, 517, 515, 533, 519, 544, 501, 510, 498, 525, 538, 504, 480, 496, 521, 509, 475, 486, 500, 500, 492, 532, 473, 511, 507, 533, 468, 541, 511, 499, 517, 506, 471, 505, 517, 506, 480, 525, 479, 535, 541, 547, 504, 510, 533, 551, 501, 501, 505, 508, 510, 519, 470, 526, 534, 502, 534, 516, 550, 524, 520, 535, 573, 486, 525, 492, 452, 590, 516, 532, 515, 534, 546, 523, 505, 471, 518] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[499, 509, 485, 469, 547, 506, 512, 483, 507, 472, 461, 481, 516, 525, 518, 513, 527, 551, 533, 532, 542, 499, 518, 519, 552, 496, 480, 521, 545, 487, 529, 489, 520, 521, 482, 524, 535, 497, 555, 525, 472, 484, 484, 550, 523, 508, 494, 529, 551, 543, 503, 526, 519, 502, 482, 555, 509, 501, 471, 481, 520, 480, 517, 497, 496, 519, 514, 500, 524, 508, 510, 463, 529, 525, 504, 532, 477, 500, 532, 480, 501, 481, 507, 472, 518, 521, 502, 540, 488, 521, 504, 502, 524, 475, 533, 522, 520, 513, 526, 501, 584, 516, 545, 527, 533, 531, 497, 532, 483, 520, 552, 534, 496, 481, 541, 530, 529, 477, 512, 501, 522, 485, 535, 534, 510, 524, 542, 496] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[502, 513, 531, 485, 490, 519, 535, 503, 550, 534, 506, 527, 525, 497, 506, 511, 32, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 509, 511, 484, 534, 506, 464, 529, 539, 517, 547, 487, 522, 496, 523, 533, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 518, 538, 505, 505, 502, 529, 491, 497, 503, 523, 477, 504, 502, 482, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 561, 506, 476, 494, 498, 495, 508, 494, 541, 526, 482, 550, 536, 516, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 518, 532, 446, 525, 522, 517, 499, 484, 507, 521, 504, 520, 508, 524, 529, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 504, 533, 489, 525, 514, 506, 516, 494, 525, 544, 489, 461, 546, 499, 517, 23, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 544, 542, 521, 528, 552, 476, 517, 534, 504, 503, 503, 533, 485, 487, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 509, 479, 487, 505, 526, 492, 470, 535, 476, 523, 499, 490, 507, 538, 480, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8276, 8211, 8106, 8219, 8212, 8183, 8294, 8035] avg=8192.0000 max=8294 max_rank=6 min=8035 min_rank=7 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32724 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[508, 534, 508, 512, 505, 526, 528, 469, 501, 509, 551, 522, 500, 491, 526, 518, 29, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 527, 484, 505, 520, 477, 503, 482, 532, 554, 512, 468, 519, 525, 492, 491, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 491, 492, 514, 502, 490, 525, 500, 505, 534, 526, 528, 489, 519, 533, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 539, 518, 479, 503, 516, 491, 504, 545, 482, 543, 504, 522, 493, 532, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 510, 504, 507, 544, 510, 543, 461, 520, 488, 518, 561, 497, 515, 500, 539, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 528, 517, 517, 476, 509, 539, 499, 517, 505, 472, 502, 474, 500, 545, 502, 20, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 504, 519, 532, 510, 522, 490, 531, 488, 502, 498, 554, 525, 508, 477, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 479, 513, 512, 495, 497, 519, 546, 538, 546, 524, 543, 466, 532, 531, 510, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8247, 8086, 8165, 8186, 8256, 8161, 8141, 8294] avg=8192.0000 max=8294 max_rank=7 min=8086 min_rank=1 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32852 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[508, 534, 505, 488, 523, 498, 515, 483, 504, 487, 575, 487, 498, 483, 538, 533, 28, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 499, 528, 538, 493, 485, 523, 501, 512, 490, 506, 525, 526, 529, 503, 517, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 522, 480, 520, 504, 473, 556, 524, 526, 511, 528, 495, 523, 564, 513, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 548, 543, 518, 526, 533, 532, 530, 501, 515, 500, 553, 516, 527, 494, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 514, 483, 480, 537, 486, 511, 529, 521, 513, 486, 506, 499, 491, 539, 528, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 496, 508, 534, 501, 519, 477, 509, 549, 511, 499, 497, 529, 530, 496, 476, 21, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 504, 497, 489, 507, 492, 542, 489, 518, 483, 538, 492, 522, 524, 514, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 481, 509, 495, 481, 480, 513, 482, 488, 529, 513, 501, 519, 545, 529, 516, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8199, 8258, 8351, 8177, 8168, 8090, 8096] avg=8192.0000 max=8351 max_rank=3 min=8090 min_rank=6 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32531 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[514, 526, 501, 486, 509, 529, 540, 501, 478, 504, 521, 491, 507, 533, 540, 478, 28, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 512, 530, 506, 545, 518, 461, 490, 505, 516, 491, 508, 510, 498, 540, 522, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 542, 481, 550, 556, 485, 491, 533, 504, 500, 513, 501, 525, 518, 559, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 519, 496, 460, 519, 492, 529, 508, 528, 500, 508, 509, 533, 531, 531, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 535, 539, 504, 476, 469, 492, 509, 525, 496, 494, 501, 521, 475, 544, 517, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 525, 509, 517, 545, 534, 518, 534, 520, 506, 518, 498, 507, 512, 489, 521, 20, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 527, 468, 518, 502, 461, 516, 539, 508, 475, 537, 521, 522, 507, 466, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 511, 509, 498, 494, 497, 484, 512, 515, 545, 518, 490, 500, 538, 512, 494, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8205, 8302, 8134, 8159, 8318, 8061, 8162] avg=8192.0000 max=8318 max_rank=5 min=8061 min_rank=6 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32700 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[559, 515, 574, 507, 494, 516, 537, 470, 524, 498, 496, 487, 507, 466, 550, 519, 512, 504, 502, 501, 542, 492, 493, 480, 505, 510, 526, 550, 506, 499, 507, 506, 542, 543, 512, 563, 481, 535, 517, 508, 512, 541, 481, 534, 536, 535, 490, 510, 521, 519, 475, 510, 539, 513, 557, 470, 528, 520, 504, 545, 530, 488, 476, 530, 481, 513, 523, 537, 527, 504, 494, 516, 532, 477, 505, 516, 507, 565, 476, 506, 502, 500, 496, 524, 478, 511, 532, 522, 501, 505, 533, 490, 497, 517, 535, 492, 507, 537, 511, 523, 495, 538, 506, 500, 544, 510, 502, 507, 510, 522, 500, 509, 524, 479, 487, 514, 467, 517, 515, 522, 522, 501, 502, 483, 493, 525, 532, 499] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[523, 486, 558, 504, 500, 504, 476, 529, 553, 518, 518, 483, 479, 467, 483, 485, 28, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 527, 480, 524, 530, 487, 457, 481, 520, 502, 505, 523, 508, 499, 493, 536, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 542, 480, 511, 511, 464, 514, 492, 527, 525, 526, 481, 535, 496, 495, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 512, 517, 485, 542, 521, 514, 539, 556, 486, 533, 514, 486, 476, 522, 574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 492, 536, 527, 521, 523, 532, 490, 527, 486, 472, 507, 507, 526, 511, 554, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 478, 515, 489, 504, 514, 576, 487, 486, 522, 499, 501, 509, 502, 535, 512, 21, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 503, 476, 515, 524, 514, 537, 532, 500, 528, 512, 521, 504, 485, 529, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 522, 513, 484, 494, 516, 527, 516, 565, 537, 522, 515, 534, 522, 557, 506, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8104, 8099, 8127, 8292, 8257, 8168, 8160, 8329] avg=8192.0000 max=8329 max_rank=7 min=8099 min_rank=1 imbalance_ratio=1.016724 max_load_reduction=57207 max_load_reduction_pct=0.872910 moved_to_echo=32914 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[559, 515, 574, 507, 494, 516, 537, 470, 524, 498, 496, 487, 507, 466, 550, 519, 28, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 504, 502, 501, 542, 492, 493, 480, 505, 510, 526, 550, 506, 499, 507, 506, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 543, 469, 563, 481, 535, 517, 508, 512, 541, 481, 534, 536, 535, 490, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 519, 475, 460, 539, 513, 557, 470, 528, 520, 504, 545, 530, 488, 476, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 513, 523, 537, 527, 504, 494, 516, 532, 477, 505, 516, 507, 565, 476, 506, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 500, 496, 524, 478, 511, 532, 522, 501, 505, 533, 490, 497, 517, 535, 492, 21, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 537, 511, 523, 495, 538, 506, 500, 544, 510, 502, 507, 510, 522, 500, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 479, 487, 514, 467, 517, 515, 522, 522, 501, 502, 483, 493, 525, 532, 499, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8256, 8146, 8297, 8175, 8220, 8171, 8188, 8083] avg=8192.0000 max=8297 max_rank=2 min=8083 min_rank=7 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32662 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[508, 534, 508, 512, 505, 526, 528, 469, 501, 509, 551, 522, 500, 491, 526, 518, 29, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 527, 484, 505, 520, 477, 503, 482, 532, 554, 512, 468, 519, 525, 492, 491, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 491, 492, 514, 502, 490, 525, 500, 505, 534, 526, 528, 489, 519, 533, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 539, 518, 479, 503, 516, 491, 504, 545, 482, 543, 504, 522, 493, 532, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 510, 504, 507, 544, 510, 543, 461, 520, 488, 518, 561, 497, 515, 500, 539, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 528, 517, 517, 476, 509, 539, 499, 517, 505, 472, 502, 474, 500, 545, 502, 20, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 504, 519, 532, 510, 522, 490, 531, 488, 502, 498, 554, 525, 508, 477, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 479, 513, 512, 495, 497, 519, 546, 538, 546, 524, 543, 466, 532, 531, 510, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8247, 8086, 8165, 8186, 8256, 8161, 8141, 8294] avg=8192.0000 max=8294 max_rank=7 min=8086 min_rank=1 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32852 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[502, 513, 531, 485, 490, 519, 535, 503, 550, 534, 506, 527, 525, 497, 506, 511, 32, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 509, 511, 484, 534, 506, 464, 529, 539, 517, 547, 487, 522, 496, 523, 533, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 518, 538, 505, 505, 502, 529, 491, 497, 503, 523, 477, 504, 502, 482, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 561, 506, 476, 494, 498, 495, 508, 494, 541, 526, 482, 550, 536, 516, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 518, 532, 446, 525, 522, 517, 499, 484, 507, 521, 504, 520, 508, 524, 529, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 504, 533, 489, 525, 514, 506, 516, 494, 525, 544, 489, 461, 546, 499, 517, 23, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 544, 542, 521, 528, 552, 476, 517, 534, 504, 503, 503, 533, 485, 487, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 509, 479, 487, 505, 526, 492, 470, 535, 476, 523, 499, 490, 507, 538, 480, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8276, 8211, 8106, 8219, 8212, 8183, 8294, 8035] avg=8192.0000 max=8294 max_rank=6 min=8035 min_rank=7 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32724 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[499, 509, 485, 469, 547, 506, 512, 483, 507, 472, 461, 481, 516, 525, 518, 513, 26, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 551, 533, 532, 542, 499, 518, 519, 552, 496, 480, 521, 545, 487, 529, 489, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 521, 442, 524, 535, 497, 555, 525, 472, 484, 484, 550, 523, 508, 494, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 543, 503, 475, 519, 502, 482, 555, 509, 501, 471, 481, 520, 480, 517, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 519, 514, 500, 524, 508, 510, 463, 529, 525, 504, 532, 477, 500, 532, 480, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 481, 507, 472, 518, 521, 502, 540, 488, 521, 504, 502, 524, 475, 533, 522, 22, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 513, 526, 501, 584, 516, 545, 527, 533, 531, 497, 532, 483, 520, 552, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 481, 541, 530, 529, 477, 512, 501, 522, 485, 535, 534, 510, 524, 542, 496, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8038, 8332, 8163, 8106, 8155, 8147, 8378, 8217] avg=8192.0000 max=8378 max_rank=6 min=8038 min_rank=0 imbalance_ratio=1.022705 max_load_reduction=57158 max_load_reduction_pct=0.872162 moved_to_echo=32897 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[514, 526, 501, 486, 509, 529, 540, 501, 478, 504, 521, 491, 507, 533, 540, 478, 28, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 512, 530, 506, 545, 518, 461, 490, 505, 516, 491, 508, 510, 498, 540, 522, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 542, 481, 550, 556, 485, 491, 533, 504, 500, 513, 501, 525, 518, 559, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 519, 496, 460, 519, 492, 529, 508, 528, 500, 508, 509, 533, 531, 531, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 535, 539, 504, 476, 469, 492, 509, 525, 496, 494, 501, 521, 475, 544, 517, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 525, 509, 517, 545, 534, 518, 534, 520, 506, 518, 498, 507, 512, 489, 521, 20, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 527, 468, 518, 502, 461, 516, 539, 508, 475, 537, 521, 522, 507, 466, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 511, 509, 498, 494, 497, 484, 512, 515, 545, 518, 490, 500, 538, 512, 494, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8205, 8302, 8134, 8159, 8318, 8061, 8162] avg=8192.0000 max=8318 max_rank=5 min=8061 min_rank=6 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32700 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[508, 534, 505, 488, 523, 498, 515, 483, 504, 487, 575, 487, 498, 483, 538, 533, 28, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 499, 528, 538, 493, 485, 523, 501, 512, 490, 506, 525, 526, 529, 503, 517, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 522, 480, 520, 504, 473, 556, 524, 526, 511, 528, 495, 523, 564, 513, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 548, 543, 518, 526, 533, 532, 530, 501, 515, 500, 553, 516, 527, 494, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 514, 483, 480, 537, 486, 511, 529, 521, 513, 486, 506, 499, 491, 539, 528, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 496, 508, 534, 501, 519, 477, 509, 549, 511, 499, 497, 529, 530, 496, 476, 21, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 504, 497, 489, 507, 492, 542, 489, 518, 483, 538, 492, 522, 524, 514, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 481, 509, 495, 481, 480, 513, 482, 488, 529, 513, 501, 519, 545, 529, 516, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8199, 8258, 8351, 8177, 8168, 8090, 8096] avg=8192.0000 max=8351 max_rank=3 min=8090 min_rank=6 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32531 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[534, 488, 518, 502, 495, 537, 509, 551, 529, 512, 518, 501, 479, 512, 476, 515, 34, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 500, 530, 518, 499, 530, 532, 516, 491, 505, 526, 519, 527, 532, 494, 489, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 488, 476, 525, 512, 504, 527, 531, 498, 533, 473, 466, 528, 525, 503, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 471, 467, 436, 517, 515, 533, 519, 544, 501, 510, 498, 525, 538, 504, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 521, 509, 475, 486, 500, 500, 492, 532, 473, 511, 507, 533, 468, 541, 511, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 517, 506, 471, 505, 517, 506, 480, 525, 479, 535, 541, 547, 504, 510, 533, 25, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 501, 501, 505, 508, 510, 519, 470, 526, 534, 502, 534, 516, 550, 524, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 573, 486, 525, 492, 452, 590, 516, 532, 515, 534, 546, 523, 505, 471, 518, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8223, 8258, 8118, 8073, 8100, 8220, 8224, 8320] avg=8192.0000 max=8320 max_rank=7 min=8073 min_rank=3 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32864 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[559, 515, 574, 507, 494, 516, 537, 470, 524, 498, 496, 487, 507, 466, 550, 519, 28, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 504, 502, 501, 542, 492, 493, 480, 505, 510, 526, 550, 506, 499, 507, 506, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 543, 469, 563, 481, 535, 517, 508, 512, 541, 481, 534, 536, 535, 490, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 519, 475, 460, 539, 513, 557, 470, 528, 520, 504, 545, 530, 488, 476, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 513, 523, 537, 527, 504, 494, 516, 532, 477, 505, 516, 507, 565, 476, 506, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 500, 496, 524, 478, 511, 532, 522, 501, 505, 533, 490, 497, 517, 535, 492, 21, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 537, 511, 523, 495, 538, 506, 500, 544, 510, 502, 507, 510, 522, 500, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 479, 487, 514, 467, 517, 515, 522, 522, 501, 502, 483, 493, 525, 532, 499, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8256, 8146, 8297, 8175, 8220, 8171, 8188, 8083] avg=8192.0000 max=8297 max_rank=2 min=8083 min_rank=7 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32662 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 3968, 4224, 3968, 4096, 4224, 4096, 4096, 4224, 4224, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 3968, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4224, 3968, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4096, 3968, 3968, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 3840, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4224, 3968, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 256, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 3968, 4224, 4224, 3968, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 3968, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 3968, 4224, 3968, 4096, 4224, 4096, 4096, 4224, 4224, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4096, 3968, 3968, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[516, 505, 523, 542, 488, 509, 493, 474, 520, 517, 521, 504, 525, 513, 507, 492, 519, 494, 538, 486, 527, 518, 530, 506, 538, 521, 516, 496, 524, 521, 555, 497, 532, 463, 492, 529, 519, 555, 534, 469, 482, 465, 501, 499, 495, 468, 494, 522, 538, 478, 486, 527, 524, 540, 526, 532, 527, 489, 551, 504, 536, 513, 527, 495, 504, 505, 514, 543, 493, 537, 506, 514, 515, 520, 495, 524, 495, 505, 507, 518, 533, 490, 527, 523, 532, 489, 469, 505, 537, 510, 495, 486, 490, 534, 455, 508, 492, 504, 510, 519, 520, 524, 488, 478, 536, 532, 528, 507, 548, 541, 504, 539, 520, 505, 497, 499, 518, 485, 489, 487, 539, 498, 578, 502, 522, 532, 548, 522] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[492, 503, 521, 531, 492, 486, 473, 509, 513, 499, 501, 501, 550, 515, 493, 548, 488, 501, 535, 504, 481, 513, 526, 488, 506, 483, 514, 557, 522, 493, 515, 523, 532, 502, 531, 524, 521, 515, 523, 517, 483, 540, 516, 519, 535, 520, 544, 511, 505, 508, 522, 486, 500, 513, 515, 552, 534, 505, 495, 514, 497, 513, 504, 517, 505, 509, 517, 510, 519, 509, 537, 500, 481, 498, 491, 512, 556, 496, 507, 498, 512, 505, 497, 527, 479, 509, 493, 521, 497, 484, 526, 512, 524, 496, 533, 538, 498, 526, 481, 513, 547, 543, 502, 516, 521, 519, 546, 533, 501, 485, 507, 487, 515, 531, 493, 544, 492, 524, 497, 469, 509, 542, 521, 545, 510, 512, 477, 543] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[467, 495, 497, 524, 526, 525, 524, 486, 495, 486, 527, 504, 525, 515, 515, 478, 535, 503, 540, 506, 491, 492, 521, 483, 475, 524, 521, 521, 495, 510, 547, 516, 541, 525, 511, 481, 501, 495, 510, 511, 509, 524, 496, 504, 476, 513, 510, 513, 479, 501, 471, 545, 493, 532, 534, 514, 494, 481, 509, 530, 541, 529, 546, 515, 514, 500, 476, 505, 512, 535, 554, 483, 496, 504, 511, 500, 494, 472, 496, 550, 488, 541, 459, 505, 510, 546, 508, 538, 499, 508, 540, 504, 509, 513, 516, 525, 503, 472, 500, 520, 549, 524, 510, 510, 504, 516, 502, 519, 542, 537, 542, 557, 551, 524, 516, 505, 512, 510, 521, 514, 514, 522, 511, 554, 492, 512, 532, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[502, 485, 504, 546, 532, 537, 483, 482, 560, 497, 488, 531, 499, 544, 490, 507, 478, 493, 495, 524, 524, 520, 479, 509, 505, 486, 471, 493, 519, 517, 541, 529, 541, 469, 488, 572, 487, 538, 506, 589, 491, 495, 520, 533, 503, 506, 557, 541, 520, 475, 511, 519, 512, 508, 485, 568, 509, 518, 491, 488, 505, 545, 544, 487, 540, 537, 547, 500, 534, 500, 493, 508, 529, 515, 514, 530, 502, 503, 491, 497, 516, 514, 547, 505, 503, 503, 520, 502, 505, 523, 530, 546, 500, 472, 520, 525, 519, 527, 451, 567, 526, 497, 521, 526, 469, 500, 534, 520, 507, 509, 520, 503, 492, 497, 524, 507, 500, 496, 513, 471, 488, 490, 497, 495, 592, 510, 503, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[467, 495, 497, 524, 526, 525, 524, 486, 495, 486, 527, 504, 525, 515, 515, 478, 535, 503, 540, 506, 491, 492, 521, 483, 475, 524, 521, 521, 495, 510, 547, 516, 541, 525, 511, 481, 501, 495, 510, 511, 509, 524, 496, 504, 476, 513, 510, 513, 479, 501, 471, 545, 493, 532, 534, 514, 494, 481, 509, 530, 541, 529, 546, 515, 514, 500, 476, 505, 512, 535, 554, 483, 496, 504, 511, 500, 494, 472, 496, 550, 488, 541, 459, 505, 510, 546, 508, 538, 499, 508, 540, 504, 509, 513, 516, 525, 503, 472, 500, 520, 549, 524, 510, 510, 504, 516, 502, 519, 542, 537, 542, 557, 551, 524, 516, 505, 512, 510, 521, 514, 514, 522, 511, 554, 492, 512, 532, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[516, 505, 523, 542, 488, 509, 493, 474, 520, 517, 521, 504, 525, 513, 507, 492, 519, 494, 538, 486, 527, 518, 530, 506, 538, 521, 516, 496, 524, 521, 555, 497, 532, 463, 492, 529, 519, 555, 534, 469, 482, 465, 501, 499, 495, 468, 494, 522, 538, 478, 486, 527, 524, 540, 526, 532, 527, 489, 551, 504, 536, 513, 527, 495, 504, 505, 514, 543, 493, 537, 506, 514, 515, 520, 495, 524, 495, 505, 507, 518, 533, 490, 527, 523, 532, 489, 469, 505, 537, 510, 495, 486, 490, 534, 455, 508, 492, 504, 510, 519, 520, 524, 488, 478, 536, 532, 528, 507, 548, 541, 504, 539, 520, 505, 497, 499, 518, 485, 489, 487, 539, 498, 578, 502, 522, 532, 548, 522] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[467, 497, 515, 509, 502, 529, 510, 510, 544, 525, 528, 481, 520, 533, 491, 505, 508, 521, 536, 497, 514, 505, 515, 513, 521, 526, 528, 553, 537, 552, 492, 503, 511, 518, 505, 527, 518, 557, 502, 501, 498, 517, 543, 486, 532, 485, 484, 520, 504, 542, 537, 508, 487, 511, 498, 536, 558, 498, 518, 512, 536, 516, 502, 504, 477, 511, 485, 518, 526, 511, 530, 481, 485, 501, 522, 522, 487, 513, 563, 497, 516, 482, 486, 494, 521, 524, 456, 517, 521, 532, 484, 522, 541, 511, 495, 513, 522, 528, 489, 526, 480, 520, 517, 484, 477, 485, 485, 505, 520, 513, 508, 551, 519, 493, 491, 511, 541, 458, 472, 534, 512, 520, 512, 554, 540, 538, 501, 528] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[492, 503, 521, 531, 492, 486, 473, 509, 513, 499, 501, 501, 550, 515, 493, 548, 488, 501, 535, 504, 481, 513, 526, 488, 506, 483, 514, 557, 522, 493, 515, 523, 532, 502, 531, 524, 521, 515, 523, 517, 483, 540, 516, 519, 535, 520, 544, 511, 505, 508, 522, 486, 500, 513, 515, 552, 534, 505, 495, 514, 497, 513, 504, 517, 505, 509, 517, 510, 519, 509, 537, 500, 481, 498, 491, 512, 556, 496, 507, 498, 512, 505, 497, 527, 479, 509, 493, 521, 497, 484, 526, 512, 524, 496, 533, 538, 498, 526, 481, 513, 547, 543, 502, 516, 521, 519, 546, 533, 501, 485, 507, 487, 515, 531, 493, 544, 492, 524, 497, 469, 509, 542, 521, 545, 510, 512, 477, 543] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[533, 497, 466, 473, 558, 565, 485, 523, 531, 558, 518, 494, 533, 527, 484, 525, 498, 525, 490, 535, 524, 548, 494, 521, 523, 517, 477, 515, 499, 501, 529, 510, 497, 511, 495, 519, 516, 539, 521, 535, 522, 513, 497, 491, 494, 534, 474, 532, 507, 512, 478, 560, 507, 504, 493, 528, 494, 523, 507, 523, 511, 493, 520, 511, 563, 519, 512, 494, 512, 494, 504, 542, 497, 477, 512, 496, 555, 511, 532, 539, 551, 511, 487, 511, 480, 518, 491, 522, 469, 543, 541, 484, 550, 492, 512, 483, 489, 507, 514, 497, 493, 515, 523, 534, 518, 506, 525, 507, 523, 520, 511, 504, 499, 518, 481, 475, 482, 527, 465, 523, 520, 506, 494, 497, 544, 508, 517, 553] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[508, 520, 558, 543, 556, 551, 502, 518, 522, 530, 575, 527, 529, 517, 520, 523, 488, 494, 501, 526, 491, 500, 460, 517, 518, 506, 525, 541, 506, 518, 494, 506, 524, 473, 537, 512, 496, 497, 507, 528, 527, 490, 548, 497, 521, 478, 498, 518, 502, 527, 531, 499, 527, 529, 537, 517, 509, 509, 499, 475, 545, 540, 506, 520, 486, 531, 517, 490, 512, 500, 467, 486, 502, 526, 532, 491, 510, 501, 456, 496, 503, 503, 469, 532, 529, 461, 512, 513, 536, 524, 507, 519, 498, 513, 536, 521, 525, 518, 509, 522, 533, 537, 516, 476, 520, 494, 486, 526, 517, 520, 526, 538, 489, 495, 489, 516, 493, 528, 518, 541, 543, 484, 485, 521, 476, 515, 495, 493] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[485, 512, 505, 543, 525, 512, 521, 523, 481, 529, 568, 506, 517, 519, 477, 493, 472, 504, 499, 500, 520, 530, 491, 512, 552, 483, 489, 493, 554, 542, 498, 562, 546, 542, 517, 492, 539, 514, 499, 553, 514, 505, 514, 481, 516, 504, 534, 535, 505, 534, 493, 496, 568, 490, 499, 530, 509, 496, 541, 509, 505, 511, 525, 494, 479, 501, 531, 505, 490, 498, 539, 544, 501, 499, 537, 496, 509, 507, 494, 494, 496, 517, 495, 509, 503, 506, 526, 463, 466, 485, 506, 503, 561, 533, 518, 525, 486, 503, 495, 507, 477, 514, 551, 504, 499, 514, 541, 491, 512, 501, 488, 541, 530, 503, 518, 519, 508, 517, 540, 504, 558, 517, 485, 534, 495, 514, 487, 520] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[516, 505, 523, 542, 488, 509, 493, 474, 520, 517, 502, 504, 525, 513, 507, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 494, 538, 486, 527, 518, 530, 506, 538, 521, 516, 496, 524, 521, 555, 497, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 463, 492, 529, 519, 555, 534, 469, 482, 465, 501, 499, 495, 468, 494, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 478, 486, 527, 524, 540, 526, 501, 527, 489, 551, 504, 536, 513, 527, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 505, 514, 543, 493, 537, 506, 514, 515, 520, 495, 524, 495, 505, 507, 518, 31, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 490, 527, 523, 532, 489, 469, 505, 537, 510, 495, 486, 490, 534, 455, 508, 18, 11, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 504, 510, 519, 520, 524, 488, 478, 536, 532, 528, 507, 548, 541, 504, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 505, 497, 499, 518, 485, 489, 487, 539, 498, 578, 502, 522, 532, 548, 522, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8130, 8288, 8005, 8262, 8236, 8121, 8252, 8242] avg=8192.0000 max=8288 max_rank=1 min=8005 min_rank=2 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32851 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[502, 485, 504, 546, 532, 537, 483, 482, 560, 497, 469, 531, 499, 544, 490, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 493, 495, 524, 524, 520, 479, 509, 505, 486, 471, 493, 519, 517, 541, 529, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 469, 488, 572, 487, 538, 506, 589, 491, 495, 520, 533, 503, 506, 557, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 475, 511, 519, 512, 508, 485, 535, 509, 518, 491, 488, 505, 545, 544, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 537, 547, 500, 534, 500, 493, 508, 529, 515, 514, 530, 502, 503, 491, 497, 33, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 514, 547, 505, 503, 503, 520, 502, 505, 523, 530, 546, 500, 472, 520, 525, 17, 11, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 527, 451, 567, 526, 497, 521, 526, 469, 500, 534, 520, 507, 509, 520, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 497, 524, 507, 500, 496, 513, 471, 488, 490, 497, 495, 592, 510, 503, 503, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8168, 8085, 8322, 8152, 8283, 8268, 8179, 8079] avg=8192.0000 max=8322 max_rank=2 min=8079 min_rank=7 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32809 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[467, 495, 497, 524, 526, 525, 524, 486, 495, 486, 508, 504, 525, 515, 515, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 503, 540, 506, 491, 492, 521, 483, 475, 524, 521, 521, 495, 510, 547, 516, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 525, 511, 481, 501, 495, 510, 511, 509, 524, 496, 504, 476, 513, 510, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 501, 471, 545, 493, 532, 534, 484, 494, 481, 509, 530, 541, 529, 546, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 500, 476, 505, 512, 535, 554, 483, 496, 504, 511, 500, 494, 472, 496, 550, 30, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 541, 459, 505, 510, 546, 508, 538, 499, 508, 540, 504, 509, 513, 516, 525, 19, 11, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 472, 500, 520, 549, 524, 510, 510, 504, 516, 502, 519, 542, 537, 542, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 524, 516, 505, 512, 510, 521, 514, 514, 522, 511, 554, 492, 512, 532, 525, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8070, 8182, 8106, 8184, 8142, 8248, 8288, 8316] avg=8192.0000 max=8316 max_rank=7 min=8070 min_rank=0 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32994 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[492, 503, 521, 531, 492, 486, 473, 509, 513, 499, 482, 501, 550, 515, 493, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 501, 535, 504, 481, 513, 526, 488, 506, 483, 514, 557, 522, 493, 515, 523, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 502, 531, 524, 521, 515, 523, 517, 483, 540, 516, 519, 535, 520, 544, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 508, 522, 486, 500, 513, 515, 520, 534, 505, 495, 514, 497, 513, 504, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 509, 517, 510, 519, 509, 537, 500, 481, 498, 491, 512, 556, 496, 507, 498, 32, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 505, 497, 527, 479, 509, 493, 521, 497, 484, 526, 512, 524, 496, 533, 538, 16, 11, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 526, 481, 513, 547, 543, 502, 516, 521, 519, 546, 533, 501, 485, 507, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 531, 493, 544, 492, 524, 497, 469, 509, 542, 521, 545, 510, 512, 477, 543, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8108, 8151, 8319, 8148, 8187, 8189, 8209, 8225] avg=8192.0000 max=8319 max_rank=2 min=8108 min_rank=0 imbalance_ratio=1.015503 max_load_reduction=57217 max_load_reduction_pct=0.873062 moved_to_echo=32810 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[467, 497, 515, 509, 502, 529, 510, 510, 544, 525, 509, 481, 520, 533, 491, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 521, 536, 497, 514, 505, 515, 513, 521, 526, 528, 553, 537, 552, 492, 503, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 518, 505, 527, 518, 557, 502, 501, 498, 517, 543, 486, 532, 485, 484, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 542, 537, 508, 487, 511, 498, 505, 558, 498, 518, 512, 536, 516, 502, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 511, 485, 518, 526, 511, 530, 481, 485, 501, 522, 522, 487, 513, 563, 497, 31, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 482, 486, 494, 521, 524, 456, 517, 521, 532, 484, 522, 541, 511, 495, 513, 18, 10, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 528, 489, 526, 480, 520, 517, 484, 477, 485, 485, 505, 520, 513, 508, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 493, 491, 511, 541, 458, 472, 534, 512, 520, 512, 554, 540, 538, 501, 528, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8147, 8323, 8191, 8236, 8170, 8152, 8092, 8225] avg=8192.0000 max=8323 max_rank=1 min=8092 min_rank=6 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32639 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[533, 497, 466, 473, 558, 565, 485, 523, 531, 558, 499, 494, 533, 527, 484, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 525, 490, 535, 524, 548, 494, 521, 523, 517, 477, 515, 499, 501, 529, 510, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 511, 495, 519, 516, 539, 521, 535, 522, 513, 497, 491, 494, 534, 474, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 512, 478, 560, 507, 504, 493, 498, 494, 523, 507, 523, 511, 493, 520, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 563, 519, 512, 494, 512, 494, 504, 542, 497, 477, 512, 496, 555, 511, 532, 539, 30, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 511, 487, 511, 480, 518, 491, 522, 469, 543, 541, 484, 550, 492, 512, 483, 17, 10, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 507, 514, 497, 493, 515, 523, 534, 518, 506, 525, 507, 523, 520, 511, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 518, 481, 475, 482, 527, 465, 523, 520, 506, 494, 497, 544, 508, 517, 553, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8251, 8208, 8177, 8141, 8299, 8181, 8169, 8110] avg=8192.0000 max=8299 max_rank=4 min=8110 min_rank=7 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32759 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[467, 497, 515, 509, 502, 529, 510, 510, 544, 525, 509, 481, 520, 533, 491, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 521, 536, 497, 514, 505, 515, 513, 521, 526, 528, 553, 537, 552, 492, 503, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 518, 505, 527, 518, 557, 502, 501, 498, 517, 543, 486, 532, 485, 484, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 542, 537, 508, 487, 511, 498, 505, 558, 498, 518, 512, 536, 516, 502, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 511, 485, 518, 526, 511, 530, 481, 485, 501, 522, 522, 487, 513, 563, 497, 31, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 482, 486, 494, 521, 524, 456, 517, 521, 532, 484, 522, 541, 511, 495, 513, 18, 10, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 528, 489, 526, 480, 520, 517, 484, 477, 485, 485, 505, 520, 513, 508, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 493, 491, 511, 541, 458, 472, 534, 512, 520, 512, 554, 540, 538, 501, 528, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8147, 8323, 8191, 8236, 8170, 8152, 8092, 8225] avg=8192.0000 max=8323 max_rank=1 min=8092 min_rank=6 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32639 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[492, 503, 521, 531, 492, 486, 473, 509, 513, 499, 482, 501, 550, 515, 493, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 501, 535, 504, 481, 513, 526, 488, 506, 483, 514, 557, 522, 493, 515, 523, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 502, 531, 524, 521, 515, 523, 517, 483, 540, 516, 519, 535, 520, 544, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 508, 522, 486, 500, 513, 515, 520, 534, 505, 495, 514, 497, 513, 504, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 509, 517, 510, 519, 509, 537, 500, 481, 498, 491, 512, 556, 496, 507, 498, 32, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 505, 497, 527, 479, 509, 493, 521, 497, 484, 526, 512, 524, 496, 533, 538, 16, 11, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 526, 481, 513, 547, 543, 502, 516, 521, 519, 546, 533, 501, 485, 507, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 531, 493, 544, 492, 524, 497, 469, 509, 542, 521, 545, 510, 512, 477, 543, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8108, 8151, 8319, 8148, 8187, 8189, 8209, 8225] avg=8192.0000 max=8319 max_rank=2 min=8108 min_rank=0 imbalance_ratio=1.015503 max_load_reduction=57217 max_load_reduction_pct=0.873062 moved_to_echo=32810 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[508, 520, 558, 543, 556, 551, 502, 518, 522, 530, 554, 527, 529, 517, 520, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 494, 501, 526, 491, 500, 460, 517, 518, 506, 525, 541, 506, 518, 494, 506, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 473, 537, 512, 496, 497, 507, 528, 527, 490, 548, 497, 521, 478, 498, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 527, 531, 499, 527, 529, 537, 487, 509, 509, 499, 475, 545, 540, 506, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 531, 517, 490, 512, 500, 467, 486, 502, 526, 532, 491, 510, 501, 456, 496, 30, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 503, 469, 532, 529, 461, 512, 513, 536, 524, 507, 519, 498, 513, 536, 521, 18, 10, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 518, 509, 522, 533, 537, 516, 476, 520, 494, 486, 526, 517, 520, 526, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 495, 489, 516, 493, 528, 518, 541, 543, 484, 485, 521, 476, 515, 495, 493, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8478, 8093, 8138, 8242, 8044, 8214, 8245, 8082] avg=8192.0000 max=8478 max_rank=0 min=8044 min_rank=4 imbalance_ratio=1.034912 max_load_reduction=57058 max_load_reduction_pct=0.870636 moved_to_echo=32585 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[502, 485, 504, 546, 532, 537, 483, 482, 560, 497, 469, 531, 499, 544, 490, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 493, 495, 524, 524, 520, 479, 509, 505, 486, 471, 493, 519, 517, 541, 529, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 469, 488, 572, 487, 538, 506, 589, 491, 495, 520, 533, 503, 506, 557, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 475, 511, 519, 512, 508, 485, 535, 509, 518, 491, 488, 505, 545, 544, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 537, 547, 500, 534, 500, 493, 508, 529, 515, 514, 530, 502, 503, 491, 497, 33, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 514, 547, 505, 503, 503, 520, 502, 505, 523, 530, 546, 500, 472, 520, 525, 17, 11, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 527, 451, 567, 526, 497, 521, 526, 469, 500, 534, 520, 507, 509, 520, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 497, 524, 507, 500, 496, 513, 471, 488, 490, 497, 495, 592, 510, 503, 503, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8168, 8085, 8322, 8152, 8283, 8268, 8179, 8079] avg=8192.0000 max=8322 max_rank=2 min=8079 min_rank=7 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32809 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[516, 505, 523, 542, 488, 509, 493, 474, 520, 517, 502, 504, 525, 513, 507, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 494, 538, 486, 527, 518, 530, 506, 538, 521, 516, 496, 524, 521, 555, 497, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 463, 492, 529, 519, 555, 534, 469, 482, 465, 501, 499, 495, 468, 494, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 478, 486, 527, 524, 540, 526, 501, 527, 489, 551, 504, 536, 513, 527, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 505, 514, 543, 493, 537, 506, 514, 515, 520, 495, 524, 495, 505, 507, 518, 31, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 490, 527, 523, 532, 489, 469, 505, 537, 510, 495, 486, 490, 534, 455, 508, 18, 11, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 504, 510, 519, 520, 524, 488, 478, 536, 532, 528, 507, 548, 541, 504, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 505, 497, 499, 518, 485, 489, 487, 539, 498, 578, 502, 522, 532, 548, 522, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8130, 8288, 8005, 8262, 8236, 8121, 8252, 8242] avg=8192.0000 max=8288 max_rank=1 min=8005 min_rank=2 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32851 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[485, 512, 505, 543, 525, 512, 521, 523, 481, 529, 536, 506, 517, 519, 477, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 504, 499, 500, 520, 530, 491, 512, 552, 483, 489, 493, 554, 542, 498, 562, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 542, 517, 492, 539, 514, 499, 553, 514, 505, 514, 481, 516, 504, 534, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 534, 493, 496, 568, 490, 499, 496, 509, 496, 541, 509, 505, 511, 525, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 501, 531, 505, 490, 498, 539, 544, 501, 499, 537, 496, 509, 507, 494, 494, 34, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 517, 495, 509, 503, 506, 526, 463, 466, 485, 506, 503, 561, 533, 518, 525, 22, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 503, 495, 507, 477, 514, 551, 504, 499, 514, 541, 491, 512, 501, 488, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 503, 518, 519, 508, 517, 540, 504, 558, 517, 485, 534, 495, 514, 487, 520, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8206, 8278, 8171, 8175, 8163, 8102, 8257] avg=8192.0000 max=8278 max_rank=2 min=8102 min_rank=6 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32697 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 3968, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 3968, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 3968, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 3968, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[540, 516, 504, 501, 510, 526, 511, 451, 512, 503, 497, 513, 490, 488, 518, 504, 533, 518, 501, 537, 498, 517, 506, 490, 477, 523, 453, 519, 477, 509, 537, 478, 487, 504, 505, 530, 534, 542, 503, 527, 509, 529, 530, 524, 495, 494, 513, 495, 518, 509, 494, 519, 515, 513, 503, 508, 503, 477, 512, 508, 495, 553, 523, 493, 502, 508, 519, 508, 508, 523, 502, 534, 503, 519, 526, 541, 472, 474, 543, 515, 486, 508, 510, 510, 496, 527, 523, 525, 500, 507, 543, 504, 516, 514, 483, 518, 554, 491, 504, 528, 496, 552, 527, 531, 545, 509, 524, 494, 519, 514, 513, 498, 549, 530, 484, 528, 541, 526, 521, 527, 534, 558, 513, 519, 505, 485, 524, 505] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[543, 473, 503, 470, 510, 497, 480, 485, 502, 509, 509, 570, 496, 515, 530, 514, 508, 521, 505, 478, 475, 495, 514, 522, 521, 515, 530, 511, 512, 493, 517, 531, 531, 513, 517, 514, 551, 517, 514, 514, 492, 549, 509, 499, 503, 515, 484, 526, 477, 521, 492, 522, 528, 518, 508, 563, 548, 504, 546, 525, 504, 538, 512, 510, 529, 493, 532, 500, 532, 553, 502, 575, 501, 504, 491, 503, 527, 533, 538, 513, 466, 543, 495, 509, 532, 476, 516, 527, 476, 512, 494, 536, 498, 522, 499, 507, 510, 505, 500, 524, 521, 500, 505, 512, 487, 514, 548, 521, 516, 496, 487, 528, 522, 507, 486, 487, 525, 507, 493, 507, 485, 501, 504, 516, 518, 511, 533, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[513, 528, 534, 495, 505, 517, 491, 475, 472, 521, 536, 486, 517, 532, 474, 507, 501, 488, 545, 520, 531, 454, 510, 535, 532, 483, 502, 536, 484, 508, 504, 515, 549, 487, 495, 500, 510, 504, 504, 522, 545, 535, 531, 517, 515, 519, 500, 518, 523, 475, 554, 502, 522, 485, 533, 533, 494, 471, 494, 500, 502, 501, 498, 493, 531, 498, 521, 565, 522, 549, 533, 488, 489, 549, 519, 516, 514, 544, 511, 482, 484, 508, 518, 541, 529, 512, 517, 509, 505, 492, 503, 532, 501, 548, 518, 539, 485, 508, 506, 475, 521, 499, 530, 523, 516, 512, 527, 510, 496, 527, 514, 490, 541, 506, 552, 489, 462, 507, 553, 523, 507, 509, 487, 520, 511, 491, 529, 541] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[516, 492, 504, 495, 519, 496, 504, 500, 500, 478, 539, 529, 524, 498, 498, 533, 530, 510, 534, 532, 522, 507, 525, 527, 535, 504, 524, 543, 474, 478, 511, 497, 458, 507, 476, 526, 494, 503, 527, 534, 474, 501, 518, 556, 546, 501, 490, 484, 510, 515, 478, 534, 516, 502, 531, 496, 492, 525, 511, 492, 518, 529, 532, 496, 516, 519, 493, 527, 529, 514, 522, 537, 496, 479, 506, 548, 549, 505, 541, 516, 537, 489, 497, 535, 489, 522, 526, 516, 504, 510, 506, 519, 519, 491, 518, 514, 497, 597, 537, 497, 539, 493, 516, 500, 539, 526, 534, 515, 537, 518, 530, 528, 481, 499, 465, 515, 497, 525, 502, 510, 512, 490, 520, 492, 508, 499, 529, 450] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[540, 516, 504, 501, 510, 526, 511, 451, 512, 503, 497, 513, 490, 488, 518, 504, 533, 518, 501, 537, 498, 517, 506, 490, 477, 523, 453, 519, 477, 509, 537, 478, 487, 504, 505, 530, 534, 542, 503, 527, 509, 529, 530, 524, 495, 494, 513, 495, 518, 509, 494, 519, 515, 513, 503, 508, 503, 477, 512, 508, 495, 553, 523, 493, 502, 508, 519, 508, 508, 523, 502, 534, 503, 519, 526, 541, 472, 474, 543, 515, 486, 508, 510, 510, 496, 527, 523, 525, 500, 507, 543, 504, 516, 514, 483, 518, 554, 491, 504, 528, 496, 552, 527, 531, 545, 509, 524, 494, 519, 514, 513, 498, 549, 530, 484, 528, 541, 526, 521, 527, 534, 558, 513, 519, 505, 485, 524, 505] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[511, 498, 517, 507, 469, 510, 499, 547, 484, 539, 496, 518, 553, 521, 540, 527, 519, 543, 545, 523, 512, 521, 503, 519, 554, 508, 519, 530, 510, 500, 492, 471, 512, 521, 499, 533, 502, 514, 526, 520, 478, 538, 494, 511, 500, 511, 490, 528, 519, 529, 506, 453, 513, 520, 505, 492, 514, 551, 519, 524, 494, 542, 528, 506, 528, 500, 489, 482, 508, 518, 537, 522, 498, 515, 505, 517, 492, 534, 533, 561, 460, 497, 509, 508, 557, 538, 515, 525, 515, 542, 520, 512, 515, 504, 535, 494, 502, 473, 498, 496, 525, 488, 546, 518, 513, 486, 503, 495, 511, 472, 517, 511, 509, 508, 531, 513, 503, 504, 473, 475, 514, 519, 539, 500, 488, 526, 510, 488] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[501, 540, 534, 527, 516, 509, 529, 475, 479, 496, 503, 503, 508, 524, 527, 545, 458, 521, 491, 533, 516, 528, 513, 555, 536, 515, 528, 501, 503, 524, 501, 532, 514, 494, 492, 545, 504, 527, 510, 508, 549, 514, 510, 504, 542, 473, 500, 520, 549, 495, 533, 488, 501, 517, 533, 517, 566, 497, 494, 496, 526, 539, 542, 531, 499, 492, 505, 465, 536, 502, 484, 490, 508, 528, 480, 511, 504, 525, 506, 502, 531, 483, 516, 488, 505, 518, 498, 523, 534, 497, 502, 540, 494, 476, 529, 554, 532, 482, 494, 522, 480, 484, 497, 512, 536, 540, 513, 513, 535, 528, 514, 514, 491, 501, 495, 498, 502, 510, 536, 501, 490, 479, 502, 545, 528, 524, 485, 527] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[542, 508, 495, 496, 501, 505, 552, 509, 484, 517, 511, 515, 531, 488, 505, 547, 506, 513, 514, 544, 484, 455, 473, 515, 464, 521, 507, 491, 495, 492, 507, 537, 493, 494, 529, 527, 521, 518, 517, 502, 495, 521, 504, 533, 455, 509, 514, 534, 523, 520, 508, 512, 482, 509, 538, 532, 451, 473, 511, 529, 532, 499, 536, 525, 541, 542, 544, 500, 502, 507, 503, 507, 491, 472, 500, 529, 521, 489, 519, 503, 527, 519, 559, 497, 545, 526, 488, 486, 545, 526, 522, 529, 525, 505, 505, 522, 539, 533, 498, 489, 471, 493, 546, 525, 488, 533, 517, 534, 507, 526, 518, 521, 495, 524, 521, 509, 527, 474, 506, 507, 541, 539, 500, 520, 509, 537, 500, 523] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[516, 492, 504, 495, 519, 496, 504, 500, 500, 478, 539, 529, 524, 498, 498, 533, 530, 510, 534, 532, 522, 507, 525, 527, 535, 504, 524, 543, 474, 478, 511, 497, 458, 507, 476, 526, 494, 503, 527, 534, 474, 501, 518, 556, 546, 501, 490, 484, 510, 515, 478, 534, 516, 502, 531, 496, 492, 525, 511, 492, 518, 529, 532, 496, 516, 519, 493, 527, 529, 514, 522, 537, 496, 479, 506, 548, 549, 505, 541, 516, 537, 489, 497, 535, 489, 522, 526, 516, 504, 510, 506, 519, 519, 491, 518, 514, 497, 597, 537, 497, 539, 493, 516, 500, 539, 526, 534, 515, 537, 518, 530, 528, 481, 499, 465, 515, 497, 525, 502, 510, 512, 490, 520, 492, 508, 499, 529, 450] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[510, 569, 542, 530, 516, 481, 525, 493, 489, 540, 530, 505, 478, 522, 543, 534, 516, 513, 533, 510, 510, 509, 519, 487, 516, 523, 519, 489, 521, 508, 524, 520, 536, 520, 513, 544, 527, 515, 490, 482, 511, 522, 499, 489, 498, 498, 483, 521, 483, 528, 477, 494, 530, 478, 557, 541, 500, 522, 481, 473, 514, 517, 503, 522, 501, 515, 523, 493, 505, 522, 510, 515, 520, 469, 483, 492, 514, 522, 515, 546, 478, 496, 564, 495, 537, 477, 497, 474, 565, 501, 519, 542, 513, 507, 533, 516, 509, 539, 510, 514, 503, 495, 530, 522, 525, 513, 493, 498, 529, 535, 501, 517, 488, 522, 511, 491, 531, 494, 499, 507, 546, 477, 536, 530, 490, 535, 500, 495] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[540, 516, 504, 501, 510, 526, 511, 451, 512, 503, 497, 513, 490, 488, 518, 504, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 518, 501, 537, 498, 517, 506, 490, 477, 523, 453, 519, 477, 509, 537, 478, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 504, 505, 530, 534, 542, 503, 527, 509, 529, 530, 524, 495, 494, 513, 495, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 509, 494, 519, 515, 513, 503, 508, 503, 477, 512, 508, 495, 551, 523, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 508, 519, 508, 508, 523, 502, 534, 503, 519, 526, 541, 472, 474, 517, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 508, 510, 510, 496, 527, 523, 525, 500, 507, 543, 486, 516, 514, 483, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 491, 504, 528, 496, 552, 503, 531, 545, 509, 524, 494, 519, 514, 513, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 530, 484, 528, 541, 526, 521, 527, 534, 558, 513, 519, 505, 485, 524, 505, 26, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8102, 8091, 8223, 8141, 8171, 8152, 8275, 8381] avg=8192.0000 max=8381 max_rank=7 min=8091 min_rank=1 imbalance_ratio=1.023071 max_load_reduction=57155 max_load_reduction_pct=0.872116 moved_to_echo=32979 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[516, 492, 504, 495, 519, 496, 504, 500, 500, 478, 539, 529, 524, 498, 498, 533, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 510, 534, 532, 522, 507, 525, 527, 535, 504, 524, 543, 474, 478, 511, 497, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 458, 507, 476, 526, 494, 503, 527, 534, 474, 501, 518, 556, 546, 501, 490, 484, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 515, 478, 534, 516, 502, 531, 496, 492, 525, 511, 492, 518, 527, 532, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 519, 493, 527, 529, 514, 522, 537, 496, 479, 506, 548, 549, 505, 515, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 489, 497, 535, 489, 522, 526, 516, 504, 510, 506, 501, 519, 491, 518, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 597, 537, 497, 539, 493, 492, 500, 539, 526, 534, 515, 537, 518, 530, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 499, 465, 515, 497, 525, 502, 510, 512, 490, 520, 492, 508, 499, 529, 450, 26, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8143, 8271, 8097, 8175, 8271, 8174, 8379, 8026] avg=8192.0000 max=8379 max_rank=6 min=8026 min_rank=7 imbalance_ratio=1.022827 max_load_reduction=57157 max_load_reduction_pct=0.872147 moved_to_echo=32850 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[511, 498, 517, 507, 469, 510, 499, 547, 484, 539, 496, 518, 553, 521, 540, 527, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 543, 545, 523, 512, 521, 503, 519, 554, 508, 519, 530, 510, 500, 492, 471, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 521, 499, 533, 502, 514, 526, 520, 478, 538, 494, 511, 500, 511, 490, 528, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 529, 506, 453, 513, 520, 505, 492, 514, 551, 519, 524, 494, 540, 528, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 500, 489, 482, 508, 518, 537, 522, 498, 515, 505, 517, 492, 534, 507, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 460, 497, 509, 508, 557, 538, 515, 525, 515, 542, 520, 494, 515, 504, 535, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 473, 498, 496, 525, 488, 521, 518, 513, 486, 503, 495, 511, 472, 517, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 508, 531, 513, 503, 504, 473, 475, 514, 519, 539, 500, 488, 526, 510, 488, 26, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8254, 8288, 8179, 8213, 8213, 8228, 8029, 8132] avg=8192.0000 max=8288 max_rank=1 min=8029 min_rank=6 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32602 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[513, 528, 534, 495, 505, 517, 491, 475, 472, 521, 536, 486, 517, 532, 474, 507, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 488, 545, 520, 531, 454, 510, 535, 532, 483, 502, 536, 484, 508, 504, 515, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 487, 495, 500, 510, 504, 504, 522, 545, 535, 531, 517, 515, 519, 500, 518, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 475, 554, 502, 522, 485, 533, 533, 494, 471, 494, 500, 502, 499, 498, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 498, 521, 565, 522, 549, 533, 488, 489, 549, 519, 516, 514, 544, 486, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 508, 518, 541, 529, 512, 517, 509, 505, 492, 503, 513, 501, 548, 518, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 508, 506, 475, 521, 499, 506, 523, 516, 512, 527, 510, 496, 527, 514, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 506, 552, 489, 462, 507, 553, 523, 507, 509, 487, 520, 511, 491, 529, 541, 25, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8122, 8166, 8253, 8078, 8306, 8237, 8115, 8259] avg=8192.0000 max=8306 max_rank=4 min=8078 min_rank=3 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32917 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[543, 473, 503, 470, 510, 497, 480, 485, 502, 509, 509, 570, 496, 515, 530, 514, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 521, 505, 478, 475, 495, 514, 522, 521, 515, 530, 511, 512, 493, 517, 531, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 513, 517, 514, 551, 517, 514, 514, 492, 549, 509, 499, 503, 515, 484, 526, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 521, 492, 522, 528, 518, 508, 563, 548, 504, 546, 525, 504, 536, 512, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 493, 532, 500, 532, 553, 502, 575, 501, 504, 491, 503, 527, 533, 512, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 466, 543, 495, 509, 532, 476, 516, 527, 476, 512, 494, 517, 498, 522, 499, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 505, 500, 524, 521, 500, 482, 512, 487, 514, 548, 521, 516, 496, 487, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 507, 486, 487, 525, 507, 493, 507, 485, 501, 504, 516, 518, 511, 533, 508, 26, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8125, 8165, 8250, 8314, 8300, 8089, 8151, 8142] avg=8192.0000 max=8314 max_rank=3 min=8089 min_rank=5 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32682 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[516, 492, 504, 495, 519, 496, 504, 500, 500, 478, 539, 529, 524, 498, 498, 533, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 510, 534, 532, 522, 507, 525, 527, 535, 504, 524, 543, 474, 478, 511, 497, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 458, 507, 476, 526, 494, 503, 527, 534, 474, 501, 518, 556, 546, 501, 490, 484, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 515, 478, 534, 516, 502, 531, 496, 492, 525, 511, 492, 518, 527, 532, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 519, 493, 527, 529, 514, 522, 537, 496, 479, 506, 548, 549, 505, 515, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 489, 497, 535, 489, 522, 526, 516, 504, 510, 506, 501, 519, 491, 518, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 597, 537, 497, 539, 493, 492, 500, 539, 526, 534, 515, 537, 518, 530, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 499, 465, 515, 497, 525, 502, 510, 512, 490, 520, 492, 508, 499, 529, 450, 26, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8143, 8271, 8097, 8175, 8271, 8174, 8379, 8026] avg=8192.0000 max=8379 max_rank=6 min=8026 min_rank=7 imbalance_ratio=1.022827 max_load_reduction=57157 max_load_reduction_pct=0.872147 moved_to_echo=32850 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[510, 569, 542, 530, 516, 481, 525, 493, 489, 540, 530, 505, 478, 522, 543, 534, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 513, 533, 510, 510, 509, 519, 487, 516, 523, 519, 489, 521, 508, 524, 520, 23, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 520, 513, 544, 527, 515, 490, 482, 511, 522, 499, 489, 498, 498, 483, 521, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 528, 477, 494, 530, 478, 557, 541, 500, 522, 481, 473, 514, 512, 503, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 515, 523, 493, 505, 522, 510, 515, 520, 469, 483, 492, 514, 522, 487, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 496, 564, 495, 537, 477, 497, 474, 565, 501, 519, 508, 513, 507, 533, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 539, 510, 514, 503, 495, 498, 522, 525, 513, 493, 498, 529, 535, 501, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 522, 511, 491, 531, 494, 499, 507, 546, 477, 536, 530, 490, 535, 500, 495, 28, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8330, 8246, 8158, 8115, 8117, 8180, 8201, 8189] avg=8192.0000 max=8330 max_rank=0 min=8115 min_rank=3 imbalance_ratio=1.016846 max_load_reduction=57206 max_load_reduction_pct=0.872894 moved_to_echo=32687 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[542, 508, 495, 496, 501, 505, 552, 509, 484, 517, 511, 515, 531, 488, 505, 547, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 513, 514, 544, 484, 455, 473, 515, 464, 521, 507, 491, 495, 492, 507, 537, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 494, 529, 527, 521, 518, 517, 502, 495, 521, 504, 533, 455, 509, 514, 534, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 520, 508, 512, 482, 509, 538, 532, 451, 473, 511, 529, 532, 497, 536, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 542, 544, 500, 502, 507, 503, 507, 491, 472, 500, 529, 521, 489, 494, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 519, 559, 497, 545, 526, 488, 486, 545, 526, 522, 510, 525, 505, 505, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 533, 498, 489, 471, 493, 521, 525, 488, 533, 517, 534, 507, 526, 518, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 524, 521, 509, 527, 474, 506, 507, 541, 539, 500, 520, 509, 537, 500, 523, 25, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 8037, 8168, 8178, 8145, 8307, 8213, 8263] avg=8192.0000 max=8307 max_rank=5 min=8037 min_rank=1 imbalance_ratio=1.014038 max_load_reduction=57229 max_load_reduction_pct=0.873245 moved_to_echo=32928 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[540, 516, 504, 501, 510, 526, 511, 451, 512, 503, 497, 513, 490, 488, 518, 504, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 518, 501, 537, 498, 517, 506, 490, 477, 523, 453, 519, 477, 509, 537, 478, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 504, 505, 530, 534, 542, 503, 527, 509, 529, 530, 524, 495, 494, 513, 495, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 509, 494, 519, 515, 513, 503, 508, 503, 477, 512, 508, 495, 551, 523, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 508, 519, 508, 508, 523, 502, 534, 503, 519, 526, 541, 472, 474, 517, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 508, 510, 510, 496, 527, 523, 525, 500, 507, 543, 486, 516, 514, 483, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 491, 504, 528, 496, 552, 503, 531, 545, 509, 524, 494, 519, 514, 513, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 530, 484, 528, 541, 526, 521, 527, 534, 558, 513, 519, 505, 485, 524, 505, 26, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8102, 8091, 8223, 8141, 8171, 8152, 8275, 8381] avg=8192.0000 max=8381 max_rank=7 min=8091 min_rank=1 imbalance_ratio=1.023071 max_load_reduction=57155 max_load_reduction_pct=0.872116 moved_to_echo=32979 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[501, 540, 534, 527, 516, 509, 529, 475, 479, 496, 503, 503, 508, 524, 527, 545, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 458, 521, 491, 533, 516, 528, 513, 555, 536, 515, 528, 501, 503, 524, 501, 532, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 494, 492, 545, 504, 527, 510, 508, 549, 514, 510, 504, 542, 473, 500, 520, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 495, 533, 488, 501, 517, 533, 517, 566, 497, 494, 496, 526, 537, 542, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 492, 505, 465, 536, 502, 484, 490, 508, 528, 480, 511, 504, 525, 482, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 483, 516, 488, 505, 518, 498, 523, 534, 497, 502, 521, 494, 476, 529, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 482, 494, 522, 480, 484, 474, 512, 536, 540, 513, 513, 535, 528, 514, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 501, 495, 498, 502, 510, 536, 501, 490, 479, 502, 545, 528, 524, 485, 527, 24, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8272, 8208, 8322, 8013, 8169, 8173, 8144] avg=8192.0000 max=8322 max_rank=3 min=8013 min_rank=4 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32499 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[542, 508, 495, 496, 501, 505, 552, 509, 484, 517, 511, 515, 531, 488, 505, 547, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 513, 514, 544, 484, 455, 473, 515, 464, 521, 507, 491, 495, 492, 507, 537, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 494, 529, 527, 521, 518, 517, 502, 495, 521, 504, 533, 455, 509, 514, 534, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 520, 508, 512, 482, 509, 538, 532, 451, 473, 511, 529, 532, 497, 536, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 542, 544, 500, 502, 507, 503, 507, 491, 472, 500, 529, 521, 489, 494, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 519, 559, 497, 545, 526, 488, 486, 545, 526, 522, 510, 525, 505, 505, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 533, 498, 489, 471, 493, 521, 525, 488, 533, 517, 534, 507, 526, 518, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 524, 521, 509, 527, 474, 506, 507, 541, 539, 500, 520, 509, 537, 500, 523, 25, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 8037, 8168, 8178, 8145, 8307, 8213, 8263] avg=8192.0000 max=8307 max_rank=5 min=8037 min_rank=1 imbalance_ratio=1.014038 max_load_reduction=57229 max_load_reduction_pct=0.873245 moved_to_echo=32928 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[510, 569, 542, 530, 516, 481, 525, 493, 489, 540, 530, 505, 478, 522, 543, 534, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 513, 533, 510, 510, 509, 519, 487, 516, 523, 519, 489, 521, 508, 524, 520, 23, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 520, 513, 544, 527, 515, 490, 482, 511, 522, 499, 489, 498, 498, 483, 521, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 528, 477, 494, 530, 478, 557, 541, 500, 522, 481, 473, 514, 512, 503, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 515, 523, 493, 505, 522, 510, 515, 520, 469, 483, 492, 514, 522, 487, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 496, 564, 495, 537, 477, 497, 474, 565, 501, 519, 508, 513, 507, 533, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 539, 510, 514, 503, 495, 498, 522, 525, 513, 493, 498, 529, 535, 501, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 522, 511, 491, 531, 494, 499, 507, 546, 477, 536, 530, 490, 535, 500, 495, 28, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8330, 8246, 8158, 8115, 8117, 8180, 8201, 8189] avg=8192.0000 max=8330 max_rank=0 min=8115 min_rank=3 imbalance_ratio=1.016846 max_load_reduction=57206 max_load_reduction_pct=0.872894 moved_to_echo=32687 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 3968, 3968, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 3968, 3968, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[496, 521, 504, 516, 511, 519, 548, 486, 520, 541, 496, 548, 481, 533, 538, 500, 502, 463, 499, 503, 508, 534, 501, 539, 518, 484, 505, 503, 478, 549, 508, 494, 542, 550, 544, 544, 488, 518, 517, 515, 515, 481, 478, 508, 498, 502, 537, 495, 528, 507, 472, 480, 507, 498, 560, 474, 544, 473, 494, 519, 504, 536, 472, 503, 551, 459, 540, 495, 515, 471, 505, 475, 510, 527, 516, 504, 515, 470, 506, 522, 546, 522, 521, 536, 556, 509, 513, 519, 508, 498, 504, 489, 492, 493, 495, 519, 523, 513, 520, 554, 496, 521, 546, 498, 515, 483, 533, 525, 526, 495, 514, 527, 499, 517, 504, 506, 528, 543, 538, 513, 482, 543, 520, 489, 527, 533, 545, 510] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[523, 470, 532, 480, 503, 530, 528, 500, 489, 512, 530, 496, 516, 503, 506, 524, 470, 497, 476, 500, 542, 504, 495, 503, 484, 534, 514, 529, 485, 522, 526, 525, 507, 508, 519, 499, 571, 517, 508, 518, 496, 488, 509, 524, 515, 511, 521, 544, 487, 493, 540, 515, 539, 548, 509, 505, 484, 501, 496, 520, 470, 482, 480, 509, 518, 516, 496, 548, 458, 541, 532, 492, 487, 552, 497, 531, 497, 536, 521, 506, 493, 538, 562, 511, 522, 530, 526, 497, 516, 524, 499, 520, 480, 537, 500, 549, 469, 484, 522, 492, 485, 549, 540, 476, 530, 497, 519, 499, 486, 529, 498, 559, 542, 521, 484, 513, 532, 502, 507, 518, 543, 523, 509, 540, 503, 484, 519, 549] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[517, 555, 528, 516, 560, 563, 523, 507, 518, 504, 487, 518, 479, 492, 531, 526, 552, 474, 566, 524, 533, 523, 489, 507, 522, 519, 538, 536, 499, 514, 525, 497, 531, 510, 530, 491, 487, 511, 526, 497, 508, 510, 513, 551, 487, 501, 485, 484, 480, 458, 510, 491, 535, 506, 514, 503, 526, 471, 550, 555, 527, 482, 522, 489, 533, 529, 500, 510, 546, 511, 552, 510, 500, 516, 546, 515, 525, 471, 514, 521, 505, 527, 508, 489, 507, 495, 521, 539, 532, 515, 519, 490, 483, 496, 547, 509, 539, 488, 489, 545, 507, 507, 500, 505, 510, 516, 508, 475, 474, 486, 497, 545, 540, 492, 511, 511, 526, 490, 498, 497, 516, 509, 521, 521, 494, 465, 505, 485] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[533, 515, 527, 533, 504, 491, 512, 487, 490, 456, 498, 523, 525, 494, 523, 536, 519, 455, 533, 524, 519, 497, 523, 500, 514, 558, 469, 498, 551, 514, 502, 512, 528, 475, 546, 520, 523, 505, 542, 534, 513, 532, 534, 513, 524, 474, 522, 499, 520, 474, 522, 516, 504, 515, 513, 552, 476, 549, 502, 489, 479, 490, 518, 532, 475, 550, 488, 517, 506, 468, 491, 475, 524, 527, 498, 533, 533, 533, 508, 504, 558, 508, 508, 551, 521, 504, 517, 517, 553, 554, 511, 515, 509, 515, 514, 506, 476, 525, 483, 531, 496, 457, 510, 503, 511, 494, 520, 510, 490, 541, 545, 522, 510, 518, 522, 481, 504, 507, 537, 496, 500, 511, 522, 471, 543, 518, 537, 484] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[496, 521, 504, 516, 511, 519, 548, 486, 520, 541, 496, 548, 481, 533, 538, 500, 502, 463, 499, 503, 508, 534, 501, 539, 518, 484, 505, 503, 478, 549, 508, 494, 542, 550, 544, 544, 488, 518, 517, 515, 515, 481, 478, 508, 498, 502, 537, 495, 528, 507, 472, 480, 507, 498, 560, 474, 544, 473, 494, 519, 504, 536, 472, 503, 551, 459, 540, 495, 515, 471, 505, 475, 510, 527, 516, 504, 515, 470, 506, 522, 546, 522, 521, 536, 556, 509, 513, 519, 508, 498, 504, 489, 492, 493, 495, 519, 523, 513, 520, 554, 496, 521, 546, 498, 515, 483, 533, 525, 526, 495, 514, 527, 499, 517, 504, 506, 528, 543, 538, 513, 482, 543, 520, 489, 527, 533, 545, 510] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[526, 519, 491, 512, 516, 490, 508, 524, 518, 518, 468, 541, 523, 530, 511, 514, 493, 520, 526, 509, 514, 496, 493, 523, 520, 465, 500, 502, 549, 549, 505, 536, 521, 491, 481, 514, 495, 450, 507, 469, 521, 530, 489, 524, 507, 495, 522, 526, 543, 490, 524, 491, 511, 495, 546, 537, 483, 514, 500, 493, 554, 557, 547, 561, 526, 541, 522, 513, 502, 501, 497, 516, 500, 518, 493, 534, 477, 501, 483, 505, 534, 477, 487, 520, 548, 487, 513, 487, 481, 506, 506, 575, 507, 553, 472, 529, 479, 500, 522, 466, 518, 522, 517, 531, 513, 522, 538, 559, 527, 494, 516, 513, 499, 460, 514, 523, 491, 525, 520, 495, 467, 548, 583, 524, 513, 535, 486, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[487, 489, 517, 481, 494, 472, 514, 481, 492, 515, 490, 523, 501, 538, 496, 508, 446, 535, 507, 481, 512, 543, 517, 516, 518, 515, 516, 511, 509, 516, 531, 489, 493, 489, 534, 550, 499, 481, 503, 505, 530, 534, 482, 537, 501, 483, 496, 512, 502, 506, 482, 513, 502, 523, 516, 477, 484, 468, 496, 516, 505, 552, 505, 516, 508, 523, 489, 552, 526, 517, 531, 542, 503, 572, 505, 534, 521, 500, 546, 528, 525, 516, 537, 542, 501, 559, 505, 494, 498, 508, 510, 502, 508, 556, 496, 506, 516, 517, 490, 547, 553, 498, 502, 515, 506, 494, 487, 556, 530, 531, 536, 494, 540, 522, 525, 534, 499, 521, 499, 514, 500, 505, 455, 524, 523, 510, 540, 541] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[482, 522, 492, 527, 542, 528, 530, 533, 500, 501, 478, 548, 515, 494, 506, 553, 546, 526, 517, 525, 514, 497, 524, 472, 509, 530, 523, 569, 549, 506, 529, 502, 532, 525, 484, 493, 522, 540, 554, 519, 481, 574, 506, 471, 527, 484, 513, 523, 501, 532, 451, 498, 522, 536, 484, 495, 499, 523, 568, 502, 483, 514, 511, 510, 539, 504, 480, 485, 502, 474, 488, 528, 490, 480, 512, 525, 462, 507, 512, 501, 477, 529, 533, 484, 541, 515, 545, 503, 540, 481, 489, 506, 502, 504, 517, 510, 515, 528, 505, 531, 505, 535, 498, 503, 555, 533, 528, 498, 532, 476, 519, 510, 520, 525, 502, 518, 506, 558, 514, 491, 486, 478, 482, 489, 540, 513, 498, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[517, 555, 528, 516, 560, 563, 523, 507, 518, 504, 487, 518, 479, 492, 531, 526, 552, 474, 566, 524, 533, 523, 489, 507, 522, 519, 538, 536, 499, 514, 525, 497, 531, 510, 530, 491, 487, 511, 526, 497, 508, 510, 513, 551, 487, 501, 485, 484, 480, 458, 510, 491, 535, 506, 514, 503, 526, 471, 550, 555, 527, 482, 522, 489, 533, 529, 500, 510, 546, 511, 552, 510, 500, 516, 546, 515, 525, 471, 514, 521, 505, 527, 508, 489, 507, 495, 521, 539, 532, 515, 519, 490, 483, 496, 547, 509, 539, 488, 489, 545, 507, 507, 500, 505, 510, 516, 508, 475, 474, 486, 497, 545, 540, 492, 511, 511, 526, 490, 498, 497, 516, 509, 521, 521, 494, 465, 505, 485] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[530, 502, 509, 529, 520, 536, 538, 505, 506, 500, 510, 508, 534, 549, 501, 488, 483, 529, 504, 521, 522, 507, 527, 498, 512, 508, 489, 491, 508, 530, 535, 513, 534, 510, 518, 521, 518, 471, 522, 497, 534, 480, 559, 489, 503, 519, 485, 500, 511, 545, 519, 511, 501, 504, 490, 527, 483, 525, 509, 510, 523, 474, 541, 507, 495, 487, 513, 468, 496, 510, 500, 571, 502, 513, 541, 501, 536, 500, 555, 529, 505, 506, 460, 502, 503, 535, 517, 517, 524, 497, 485, 489, 522, 514, 515, 502, 540, 508, 534, 501, 491, 508, 515, 512, 512, 527, 537, 518, 507, 546, 504, 476, 525, 510, 515, 541, 549, 519, 523, 447, 509, 514, 530, 515, 461, 546, 513, 491] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[523, 470, 532, 480, 503, 530, 528, 500, 489, 512, 530, 490, 516, 503, 506, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 497, 476, 500, 542, 504, 495, 503, 484, 534, 514, 529, 485, 517, 526, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 508, 519, 499, 571, 517, 508, 518, 496, 488, 509, 524, 515, 511, 521, 544, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 493, 540, 515, 539, 548, 509, 505, 484, 501, 496, 520, 470, 482, 480, 509, 30, 13, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 516, 496, 548, 458, 541, 532, 492, 487, 552, 497, 531, 497, 536, 521, 506, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 538, 562, 511, 492, 530, 526, 497, 516, 524, 499, 520, 480, 537, 500, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 484, 522, 492, 485, 549, 540, 476, 530, 497, 506, 499, 486, 529, 498, 559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 521, 484, 513, 532, 502, 507, 518, 543, 523, 509, 540, 503, 484, 519, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8136, 8101, 8262, 8126, 8236, 8274, 8121, 8280] avg=8192.0000 max=8280 max_rank=7 min=8101 min_rank=1 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32911 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[523, 470, 532, 480, 503, 530, 528, 500, 489, 512, 530, 496, 516, 503, 506, 524, 470, 497, 476, 500, 542, 504, 495, 503, 484, 534, 514, 529, 485, 522, 526, 525, 507, 508, 519, 499, 571, 517, 508, 518, 496, 488, 509, 524, 515, 511, 521, 544, 487, 493, 540, 515, 539, 548, 509, 505, 484, 501, 496, 520, 470, 482, 480, 509, 518, 516, 496, 548, 458, 541, 532, 492, 487, 552, 497, 531, 497, 536, 521, 506, 493, 538, 562, 511, 522, 530, 526, 497, 516, 524, 499, 520, 480, 537, 500, 549, 469, 484, 522, 492, 485, 549, 540, 476, 530, 497, 519, 499, 486, 529, 498, 559, 542, 521, 484, 513, 532, 502, 507, 518, 543, 523, 509, 540, 503, 484, 519, 549] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[533, 515, 527, 533, 504, 491, 512, 487, 490, 456, 498, 517, 525, 494, 523, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 455, 533, 524, 519, 497, 523, 500, 514, 558, 469, 498, 551, 509, 502, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 475, 546, 520, 523, 505, 542, 534, 513, 532, 534, 513, 524, 474, 522, 499, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 474, 522, 516, 504, 515, 513, 552, 476, 549, 502, 489, 479, 490, 518, 532, 30, 13, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 550, 488, 517, 506, 468, 491, 475, 524, 527, 498, 533, 533, 533, 508, 504, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 558, 508, 508, 551, 491, 504, 517, 517, 553, 554, 511, 515, 509, 515, 514, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 525, 483, 531, 496, 457, 510, 503, 511, 494, 507, 510, 490, 541, 545, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 518, 522, 481, 504, 507, 537, 496, 500, 511, 522, 471, 543, 518, 537, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8141, 8183, 8291, 8199, 8137, 8331, 8101, 8153] avg=8192.0000 max=8331 max_rank=5 min=8101 min_rank=6 imbalance_ratio=1.016968 max_load_reduction=57205 max_load_reduction_pct=0.872879 moved_to_echo=32722 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[496, 521, 504, 516, 511, 519, 548, 486, 520, 541, 496, 542, 481, 533, 538, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 463, 499, 503, 508, 534, 501, 539, 518, 484, 505, 503, 478, 544, 508, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 550, 544, 544, 488, 518, 517, 515, 515, 481, 478, 508, 498, 502, 537, 495, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 507, 472, 480, 507, 498, 560, 474, 544, 473, 494, 519, 504, 536, 472, 503, 32, 13, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 459, 540, 495, 515, 471, 505, 475, 510, 527, 516, 504, 515, 470, 506, 522, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 522, 521, 536, 524, 509, 513, 519, 508, 498, 504, 489, 492, 493, 495, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 513, 520, 554, 496, 521, 546, 498, 515, 483, 520, 525, 526, 495, 514, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 517, 504, 506, 528, 543, 538, 513, 482, 543, 520, 489, 527, 533, 545, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8252, 8083, 8239, 8121, 8088, 8188, 8276, 8289] avg=8192.0000 max=8289 max_rank=7 min=8083 min_rank=1 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32841 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[517, 555, 528, 516, 560, 563, 523, 507, 518, 504, 487, 502, 479, 492, 531, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 474, 566, 524, 533, 523, 489, 507, 522, 519, 538, 536, 499, 507, 525, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 510, 530, 491, 487, 511, 526, 497, 508, 510, 513, 551, 487, 501, 485, 484, 7, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 458, 510, 491, 535, 506, 514, 503, 526, 471, 550, 555, 527, 482, 522, 489, 33, 18, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 529, 500, 510, 546, 511, 552, 510, 500, 516, 546, 515, 525, 471, 514, 521, 7, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 527, 508, 489, 474, 495, 521, 539, 532, 515, 519, 490, 483, 496, 547, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 488, 489, 545, 507, 507, 500, 505, 510, 516, 490, 475, 474, 486, 497, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 492, 511, 511, 526, 490, 498, 497, 516, 509, 521, 521, 494, 465, 505, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8308, 8311, 8138, 8181, 8314, 8149, 8073, 8062] avg=8192.0000 max=8314 max_rank=4 min=8062 min_rank=7 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32598 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[487, 489, 517, 481, 494, 472, 514, 481, 492, 515, 490, 517, 501, 538, 496, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 446, 535, 507, 481, 512, 543, 517, 516, 518, 515, 516, 511, 509, 511, 531, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 489, 534, 550, 499, 481, 503, 505, 530, 534, 482, 537, 501, 483, 496, 512, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 506, 482, 513, 502, 523, 516, 477, 484, 468, 496, 516, 505, 552, 505, 516, 29, 12, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 523, 489, 552, 526, 517, 531, 542, 503, 572, 505, 534, 521, 500, 546, 528, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 516, 537, 542, 472, 559, 505, 494, 498, 508, 510, 502, 508, 556, 496, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 517, 490, 547, 553, 498, 502, 515, 506, 494, 475, 556, 530, 531, 536, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 522, 525, 534, 499, 521, 499, 514, 500, 505, 455, 524, 523, 510, 540, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7992, 8157, 8136, 8109, 8405, 8234, 8260, 8243] avg=8192.0000 max=8405 max_rank=4 min=7992 min_rank=0 imbalance_ratio=1.026001 max_load_reduction=57131 max_load_reduction_pct=0.871750 moved_to_echo=33142 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[526, 519, 491, 512, 516, 490, 508, 524, 518, 518, 468, 535, 523, 530, 511, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 520, 526, 509, 514, 496, 493, 523, 520, 465, 500, 502, 549, 544, 505, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 491, 481, 514, 495, 450, 507, 469, 521, 530, 489, 524, 507, 495, 522, 526, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 490, 524, 491, 511, 495, 546, 537, 483, 514, 500, 493, 554, 557, 547, 561, 31, 13, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 541, 522, 513, 502, 501, 497, 516, 500, 518, 493, 534, 477, 501, 483, 505, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 477, 487, 520, 517, 487, 513, 487, 481, 506, 506, 575, 507, 553, 472, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 500, 522, 466, 518, 522, 517, 531, 513, 522, 525, 559, 527, 494, 516, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 460, 514, 523, 491, 525, 520, 495, 467, 548, 583, 524, 513, 535, 486, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8195, 8049, 8395, 8136, 8151, 8224, 8183] avg=8192.0000 max=8395 max_rank=3 min=8049 min_rank=2 imbalance_ratio=1.024780 max_load_reduction=57141 max_load_reduction_pct=0.871902 moved_to_echo=32694 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[530, 502, 509, 529, 520, 536, 538, 505, 506, 500, 510, 502, 534, 549, 501, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 529, 504, 521, 522, 507, 527, 498, 512, 508, 489, 491, 508, 525, 535, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 510, 518, 521, 518, 471, 522, 497, 534, 480, 559, 489, 503, 519, 485, 500, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 545, 519, 511, 501, 504, 490, 527, 483, 525, 509, 510, 523, 474, 541, 507, 29, 13, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 487, 513, 468, 496, 510, 500, 571, 502, 513, 541, 501, 536, 500, 555, 529, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 506, 460, 502, 474, 535, 517, 517, 524, 497, 485, 489, 522, 514, 515, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 508, 534, 501, 491, 508, 515, 512, 512, 527, 524, 518, 507, 546, 504, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 510, 515, 541, 549, 519, 523, 447, 509, 514, 530, 515, 461, 546, 513, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8259, 8172, 8167, 8227, 8224, 8064, 8223, 8200] avg=8192.0000 max=8259 max_rank=0 min=8064 min_rank=5 imbalance_ratio=1.008179 max_load_reduction=57277 max_load_reduction_pct=0.873978 moved_to_echo=32711 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[526, 519, 491, 512, 516, 490, 508, 524, 518, 518, 468, 535, 523, 530, 511, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 520, 526, 509, 514, 496, 493, 523, 520, 465, 500, 502, 549, 544, 505, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 491, 481, 514, 495, 450, 507, 469, 521, 530, 489, 524, 507, 495, 522, 526, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 490, 524, 491, 511, 495, 546, 537, 483, 514, 500, 493, 554, 557, 547, 561, 31, 13, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 541, 522, 513, 502, 501, 497, 516, 500, 518, 493, 534, 477, 501, 483, 505, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 477, 487, 520, 517, 487, 513, 487, 481, 506, 506, 575, 507, 553, 472, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 500, 522, 466, 518, 522, 517, 531, 513, 522, 525, 559, 527, 494, 516, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 460, 514, 523, 491, 525, 520, 495, 467, 548, 583, 524, 513, 535, 486, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8195, 8049, 8395, 8136, 8151, 8224, 8183] avg=8192.0000 max=8395 max_rank=3 min=8049 min_rank=2 imbalance_ratio=1.024780 max_load_reduction=57141 max_load_reduction_pct=0.871902 moved_to_echo=32694 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[487, 489, 517, 481, 494, 472, 514, 481, 492, 515, 490, 517, 501, 538, 496, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 446, 535, 507, 481, 512, 543, 517, 516, 518, 515, 516, 511, 509, 511, 531, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 489, 534, 550, 499, 481, 503, 505, 530, 534, 482, 537, 501, 483, 496, 512, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 506, 482, 513, 502, 523, 516, 477, 484, 468, 496, 516, 505, 552, 505, 516, 29, 12, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 523, 489, 552, 526, 517, 531, 542, 503, 572, 505, 534, 521, 500, 546, 528, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 516, 537, 542, 472, 559, 505, 494, 498, 508, 510, 502, 508, 556, 496, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 517, 490, 547, 553, 498, 502, 515, 506, 494, 475, 556, 530, 531, 536, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 522, 525, 534, 499, 521, 499, 514, 500, 505, 455, 524, 523, 510, 540, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7992, 8157, 8136, 8109, 8405, 8234, 8260, 8243] avg=8192.0000 max=8405 max_rank=4 min=7992 min_rank=0 imbalance_ratio=1.026001 max_load_reduction=57131 max_load_reduction_pct=0.871750 moved_to_echo=33142 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[496, 521, 504, 516, 511, 519, 548, 486, 520, 541, 496, 542, 481, 533, 538, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 463, 499, 503, 508, 534, 501, 539, 518, 484, 505, 503, 478, 544, 508, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 550, 544, 544, 488, 518, 517, 515, 515, 481, 478, 508, 498, 502, 537, 495, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 507, 472, 480, 507, 498, 560, 474, 544, 473, 494, 519, 504, 536, 472, 503, 32, 13, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 459, 540, 495, 515, 471, 505, 475, 510, 527, 516, 504, 515, 470, 506, 522, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 522, 521, 536, 524, 509, 513, 519, 508, 498, 504, 489, 492, 493, 495, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 513, 520, 554, 496, 521, 546, 498, 515, 483, 520, 525, 526, 495, 514, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 517, 504, 506, 528, 543, 538, 513, 482, 543, 520, 489, 527, 533, 545, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8252, 8083, 8239, 8121, 8088, 8188, 8276, 8289] avg=8192.0000 max=8289 max_rank=7 min=8083 min_rank=1 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32841 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[482, 522, 492, 527, 542, 528, 530, 533, 500, 501, 478, 542, 515, 494, 506, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 526, 517, 525, 514, 497, 524, 472, 509, 530, 523, 569, 549, 502, 529, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 525, 484, 493, 522, 540, 554, 519, 481, 574, 506, 471, 527, 484, 513, 523, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 532, 451, 498, 522, 536, 484, 495, 499, 523, 568, 502, 483, 514, 511, 510, 31, 13, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 504, 480, 485, 502, 474, 488, 528, 490, 480, 512, 525, 462, 507, 512, 501, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 529, 533, 484, 510, 515, 545, 503, 540, 481, 489, 506, 502, 504, 517, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 528, 505, 531, 505, 535, 498, 503, 555, 533, 515, 498, 532, 476, 519, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 525, 502, 518, 506, 558, 514, 491, 486, 478, 482, 489, 540, 513, 498, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 8334, 8254, 8178, 7996, 8145, 8258, 8126] avg=8192.0000 max=8334 max_rank=1 min=7996 min_rank=4 imbalance_ratio=1.017334 max_load_reduction=57202 max_load_reduction_pct=0.872833 moved_to_echo=32525 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[523, 470, 532, 480, 503, 530, 528, 500, 489, 512, 530, 490, 516, 503, 506, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 497, 476, 500, 542, 504, 495, 503, 484, 534, 514, 529, 485, 517, 526, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 508, 519, 499, 571, 517, 508, 518, 496, 488, 509, 524, 515, 511, 521, 544, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 493, 540, 515, 539, 548, 509, 505, 484, 501, 496, 520, 470, 482, 480, 509, 30, 13, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 516, 496, 548, 458, 541, 532, 492, 487, 552, 497, 531, 497, 536, 521, 506, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 538, 562, 511, 492, 530, 526, 497, 516, 524, 499, 520, 480, 537, 500, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 484, 522, 492, 485, 549, 540, 476, 530, 497, 506, 499, 486, 529, 498, 559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 521, 484, 513, 532, 502, 507, 518, 543, 523, 509, 540, 503, 484, 519, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8136, 8101, 8262, 8126, 8236, 8274, 8121, 8280] avg=8192.0000 max=8280 max_rank=7 min=8101 min_rank=1 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32911 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 3968, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4224, 3968, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[482, 512, 513, 538, 489, 523, 496, 545, 499, 535, 532, 589, 508, 531, 516, 524, 537, 541, 528, 475, 526, 538, 498, 475, 487, 490, 514, 521, 519, 481, 544, 519, 497, 454, 559, 518, 512, 522, 478, 530, 489, 479, 478, 482, 550, 557, 542, 487, 504, 493, 523, 523, 513, 491, 538, 526, 508, 496, 529, 490, 520, 510, 473, 521, 529, 519, 503, 512, 505, 514, 500, 493, 493, 524, 538, 518, 515, 525, 514, 511, 526, 509, 529, 523, 517, 511, 521, 511, 553, 524, 521, 483, 514, 485, 510, 522, 496, 509, 511, 461, 529, 508, 558, 499, 510, 502, 506, 521, 479, 495, 521, 510, 491, 501, 478, 519, 501, 507, 507, 493, 538, 492, 503, 517, 545, 509, 512, 519] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[556, 535, 501, 532, 554, 505, 490, 536, 517, 506, 528, 481, 527, 519, 501, 525, 479, 483, 519, 497, 497, 478, 542, 483, 526, 501, 483, 521, 457, 507, 497, 530, 516, 527, 574, 523, 502, 511, 515, 481, 488, 481, 484, 495, 528, 502, 494, 516, 544, 487, 524, 528, 539, 501, 454, 511, 552, 480, 525, 534, 493, 511, 493, 521, 522, 515, 491, 540, 502, 544, 501, 528, 507, 532, 511, 533, 520, 529, 483, 506, 493, 533, 529, 489, 528, 529, 519, 497, 545, 521, 496, 537, 516, 544, 503, 517, 509, 490, 498, 513, 562, 527, 491, 514, 507, 530, 510, 536, 492, 523, 481, 509, 502, 493, 545, 486, 519, 489, 502, 523, 520, 486, 513, 515, 500, 508, 525, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[506, 497, 506, 546, 510, 489, 503, 493, 497, 510, 528, 496, 484, 450, 501, 522, 496, 476, 515, 512, 509, 475, 510, 515, 511, 510, 535, 510, 492, 529, 503, 539, 554, 540, 494, 487, 508, 515, 500, 531, 487, 521, 498, 503, 549, 500, 522, 513, 518, 498, 501, 527, 481, 469, 525, 548, 536, 522, 545, 503, 514, 479, 528, 514, 524, 495, 553, 507, 510, 556, 520, 523, 537, 516, 497, 514, 509, 504, 518, 539, 532, 538, 519, 514, 515, 492, 475, 480, 499, 554, 539, 504, 517, 513, 516, 491, 505, 511, 487, 508, 510, 504, 503, 552, 509, 449, 509, 563, 483, 522, 519, 550, 510, 491, 520, 517, 515, 512, 510, 523, 522, 530, 511, 481, 512, 553, 506, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[540, 514, 491, 526, 497, 506, 520, 494, 483, 485, 544, 513, 535, 489, 512, 562, 502, 536, 468, 502, 519, 558, 488, 511, 498, 515, 527, 533, 497, 512, 527, 475, 538, 542, 512, 518, 495, 511, 512, 499, 534, 509, 508, 562, 555, 524, 538, 515, 501, 516, 520, 554, 495, 497, 508, 491, 490, 513, 450, 499, 481, 510, 490, 513, 489, 495, 467, 514, 515, 497, 527, 479, 497, 543, 503, 507, 503, 483, 489, 496, 530, 511, 501, 537, 533, 514, 498, 513, 504, 523, 490, 494, 553, 543, 514, 534, 527, 525, 518, 505, 528, 492, 537, 529, 526, 470, 543, 481, 487, 511, 470, 562, 509, 530, 525, 514, 503, 540, 508, 541, 496, 540, 532, 509, 497, 508, 504, 494] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[526, 546, 513, 504, 500, 535, 526, 483, 506, 530, 492, 496, 543, 511, 487, 490, 509, 542, 514, 541, 510, 473, 519, 480, 501, 539, 493, 561, 519, 516, 481, 560, 525, 491, 508, 531, 455, 482, 485, 542, 518, 502, 533, 490, 473, 554, 519, 534, 548, 468, 512, 488, 509, 502, 482, 537, 496, 493, 533, 495, 495, 505, 535, 536, 516, 521, 512, 512, 544, 542, 524, 549, 516, 486, 543, 503, 488, 484, 517, 545, 515, 526, 533, 527, 460, 514, 507, 545, 536, 536, 528, 495, 528, 493, 549, 508, 489, 476, 498, 516, 490, 513, 473, 515, 497, 479, 484, 519, 522, 504, 526, 545, 475, 519, 520, 517, 502, 530, 545, 503, 472, 489, 514, 535, 489, 498, 531, 527] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[515, 511, 548, 538, 495, 532, 516, 541, 520, 517, 502, 512, 516, 531, 533, 519, 473, 453, 518, 482, 519, 537, 538, 518, 529, 522, 525, 533, 502, 474, 528, 526, 504, 498, 494, 503, 516, 502, 524, 527, 501, 521, 480, 489, 503, 514, 533, 507, 537, 536, 483, 519, 509, 511, 513, 504, 513, 518, 516, 477, 499, 509, 498, 500, 484, 518, 490, 485, 509, 506, 475, 575, 505, 509, 508, 479, 519, 503, 540, 514, 530, 527, 485, 513, 550, 510, 531, 488, 540, 503, 503, 503, 509, 520, 531, 516, 515, 513, 544, 493, 520, 530, 534, 536, 468, 531, 526, 515, 499, 533, 431, 495, 507, 524, 495, 522, 512, 481, 537, 539, 510, 531, 489, 502, 521, 521, 483, 520] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[543, 502, 547, 539, 498, 494, 499, 542, 525, 548, 496, 486, 544, 544, 532, 511, 481, 520, 506, 507, 539, 515, 465, 539, 516, 479, 540, 519, 528, 443, 514, 540, 527, 515, 515, 519, 566, 501, 500, 525, 528, 497, 503, 505, 515, 522, 510, 476, 510, 490, 477, 472, 485, 494, 519, 541, 535, 501, 504, 469, 487, 502, 504, 520, 492, 503, 522, 521, 522, 524, 522, 541, 513, 527, 526, 563, 540, 499, 542, 568, 481, 480, 525, 495, 513, 509, 508, 507, 497, 519, 527, 488, 518, 507, 541, 500, 467, 520, 471, 474, 511, 527, 511, 510, 456, 509, 486, 506, 559, 479, 537, 514, 513, 497, 522, 508, 527, 538, 495, 501, 520, 485, 570, 489, 520, 548, 500, 491] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[551, 552, 519, 494, 488, 525, 503, 535, 479, 474, 527, 527, 514, 523, 518, 494, 529, 539, 512, 543, 531, 547, 547, 508, 532, 566, 507, 498, 501, 484, 518, 543, 458, 501, 554, 533, 495, 510, 526, 475, 545, 522, 560, 507, 488, 508, 537, 536, 529, 521, 483, 475, 524, 531, 539, 489, 501, 523, 517, 515, 489, 472, 503, 457, 521, 503, 514, 509, 509, 543, 480, 495, 509, 533, 507, 516, 543, 518, 504, 490, 544, 480, 513, 482, 505, 525, 522, 516, 521, 508, 496, 535, 512, 506, 555, 525, 530, 455, 469, 547, 499, 490, 537, 476, 473, 483, 495, 510, 552, 473, 500, 526, 540, 527, 547, 530, 468, 493, 513, 498, 524, 522, 481, 496, 500, 485, 486, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[515, 511, 548, 538, 495, 532, 516, 541, 520, 517, 502, 512, 516, 531, 533, 519, 473, 453, 518, 482, 519, 537, 538, 518, 529, 522, 525, 533, 502, 474, 528, 526, 504, 498, 494, 503, 516, 502, 524, 527, 501, 521, 480, 489, 503, 514, 533, 507, 537, 536, 483, 519, 509, 511, 513, 504, 513, 518, 516, 477, 499, 509, 498, 500, 484, 518, 490, 485, 509, 506, 475, 575, 505, 509, 508, 479, 519, 503, 540, 514, 530, 527, 485, 513, 550, 510, 531, 488, 540, 503, 503, 503, 509, 520, 531, 516, 515, 513, 544, 493, 520, 530, 534, 536, 468, 531, 526, 515, 499, 533, 431, 495, 507, 524, 495, 522, 512, 481, 537, 539, 510, 531, 489, 502, 521, 521, 483, 520] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[526, 546, 513, 504, 500, 535, 526, 483, 506, 530, 492, 496, 543, 511, 487, 490, 509, 542, 514, 541, 510, 473, 519, 480, 501, 539, 493, 561, 519, 516, 481, 560, 525, 491, 508, 531, 455, 482, 485, 542, 518, 502, 533, 490, 473, 554, 519, 534, 548, 468, 512, 488, 509, 502, 482, 537, 496, 493, 533, 495, 495, 505, 535, 536, 516, 521, 512, 512, 544, 542, 524, 549, 516, 486, 543, 503, 488, 484, 517, 545, 515, 526, 533, 527, 460, 514, 507, 545, 536, 536, 528, 495, 528, 493, 549, 508, 489, 476, 498, 516, 490, 513, 473, 515, 497, 479, 484, 519, 522, 504, 526, 545, 475, 519, 520, 517, 502, 530, 545, 503, 472, 489, 514, 535, 489, 498, 531, 527] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[556, 535, 501, 532, 554, 505, 490, 536, 517, 506, 528, 481, 527, 519, 501, 525, 479, 483, 519, 497, 497, 478, 542, 483, 526, 501, 483, 521, 457, 507, 497, 530, 516, 527, 574, 523, 502, 511, 515, 481, 488, 481, 484, 495, 528, 502, 494, 516, 544, 487, 524, 528, 539, 501, 454, 511, 552, 480, 525, 534, 493, 511, 493, 521, 522, 515, 491, 540, 502, 544, 501, 528, 507, 532, 511, 533, 520, 529, 483, 506, 493, 533, 529, 489, 528, 529, 519, 497, 545, 521, 496, 537, 516, 544, 503, 517, 509, 490, 498, 513, 562, 527, 491, 514, 507, 530, 510, 536, 492, 523, 481, 509, 502, 493, 545, 486, 519, 489, 502, 523, 520, 486, 513, 515, 500, 508, 525, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[482, 512, 513, 538, 489, 523, 496, 545, 499, 535, 532, 589, 508, 531, 516, 524, 537, 541, 528, 475, 526, 538, 498, 475, 487, 490, 514, 521, 519, 481, 544, 519, 497, 454, 559, 518, 512, 522, 478, 530, 489, 479, 478, 482, 550, 557, 542, 487, 504, 493, 523, 523, 513, 491, 538, 526, 508, 496, 529, 490, 520, 510, 473, 521, 529, 519, 503, 512, 505, 514, 500, 493, 493, 524, 538, 518, 515, 525, 514, 511, 526, 509, 529, 523, 517, 511, 521, 511, 553, 524, 521, 483, 514, 485, 510, 522, 496, 509, 511, 461, 529, 508, 558, 499, 510, 502, 506, 521, 479, 495, 521, 510, 491, 501, 478, 519, 501, 507, 507, 493, 538, 492, 503, 517, 545, 509, 512, 519] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[451, 497, 506, 546, 510, 489, 503, 493, 497, 510, 528, 496, 484, 450, 501, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 476, 515, 512, 509, 475, 510, 515, 511, 510, 535, 510, 492, 529, 503, 539, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 540, 487, 487, 508, 515, 500, 531, 487, 521, 498, 503, 549, 500, 522, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 498, 501, 527, 481, 469, 525, 548, 536, 522, 545, 503, 514, 479, 528, 514, 55, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 495, 553, 507, 510, 516, 520, 523, 537, 516, 497, 514, 509, 504, 518, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 538, 519, 514, 515, 492, 475, 480, 499, 554, 539, 504, 517, 513, 465, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 511, 487, 508, 510, 504, 503, 552, 509, 449, 509, 563, 483, 522, 519, 550, 36, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 491, 520, 517, 515, 512, 510, 523, 522, 530, 511, 481, 512, 553, 506, 514, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7983, 8142, 8215, 8278, 8282, 8147, 8253, 8236] avg=8192.0000 max=8282 max_rank=4 min=7983 min_rank=0 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32918 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[481, 514, 491, 526, 497, 506, 520, 494, 483, 485, 544, 513, 535, 489, 512, 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 536, 468, 502, 519, 558, 488, 511, 498, 515, 527, 533, 497, 512, 527, 475, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 542, 505, 518, 495, 511, 512, 499, 534, 509, 508, 562, 555, 524, 538, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 516, 520, 554, 495, 497, 508, 491, 490, 513, 450, 499, 481, 510, 490, 513, 59, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 495, 467, 514, 515, 462, 527, 479, 497, 543, 503, 507, 503, 483, 489, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 511, 501, 537, 533, 514, 498, 513, 504, 523, 490, 494, 553, 543, 463, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 525, 518, 505, 528, 492, 537, 529, 526, 470, 543, 481, 487, 511, 470, 562, 36, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 530, 525, 514, 503, 540, 508, 541, 496, 540, 532, 509, 497, 508, 504, 494, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8152, 8173, 8365, 8102, 7969, 8241, 8276, 8258] avg=8192.0000 max=8365 max_rank=2 min=7969 min_rank=4 imbalance_ratio=1.021118 max_load_reduction=57171 max_load_reduction_pct=0.872360 moved_to_echo=32744 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[491, 552, 519, 494, 488, 525, 503, 535, 479, 474, 527, 527, 514, 523, 518, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 539, 512, 543, 531, 547, 547, 508, 532, 566, 507, 498, 501, 484, 518, 543, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 458, 501, 546, 533, 495, 510, 526, 475, 545, 522, 560, 507, 488, 508, 537, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 521, 483, 475, 524, 531, 539, 489, 501, 523, 517, 515, 489, 472, 503, 457, 60, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 503, 514, 509, 509, 504, 480, 495, 509, 533, 507, 516, 543, 518, 504, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 480, 513, 482, 505, 525, 522, 516, 521, 508, 496, 535, 512, 506, 500, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 455, 469, 547, 499, 490, 537, 476, 473, 483, 495, 510, 552, 473, 500, 526, 39, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 527, 547, 530, 468, 493, 513, 498, 524, 522, 481, 496, 500, 485, 486, 521, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8163, 8411, 8247, 8144, 8155, 8190, 8086, 8140] avg=8192.0000 max=8411 max_rank=1 min=8086 min_rank=6 imbalance_ratio=1.026733 max_load_reduction=57125 max_load_reduction_pct=0.871658 moved_to_echo=32571 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[484, 502, 547, 539, 498, 494, 499, 542, 525, 548, 496, 486, 544, 544, 532, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 520, 506, 507, 539, 515, 465, 539, 516, 479, 540, 519, 528, 443, 514, 540, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 515, 508, 519, 566, 501, 500, 525, 528, 497, 503, 505, 515, 522, 510, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 490, 477, 472, 485, 494, 519, 541, 535, 501, 504, 469, 487, 502, 504, 520, 59, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 503, 522, 521, 522, 487, 522, 541, 513, 527, 526, 563, 540, 499, 542, 568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 480, 525, 495, 513, 509, 508, 507, 497, 519, 527, 488, 518, 507, 487, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 520, 471, 474, 511, 527, 511, 510, 456, 509, 486, 506, 559, 479, 537, 514, 38, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 497, 522, 508, 527, 538, 495, 501, 520, 485, 570, 489, 520, 548, 500, 491, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8291, 8156, 8217, 8085, 8388, 8061, 8106, 8232] avg=8192.0000 max=8388 max_rank=4 min=8061 min_rank=5 imbalance_ratio=1.023926 max_load_reduction=57148 max_load_reduction_pct=0.872009 moved_to_echo=32787 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[459, 511, 548, 538, 495, 532, 516, 541, 520, 517, 502, 512, 516, 531, 533, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 453, 518, 482, 519, 537, 538, 518, 529, 522, 525, 533, 502, 474, 528, 526, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 498, 487, 503, 516, 502, 524, 527, 501, 521, 480, 489, 503, 514, 533, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 536, 483, 519, 509, 511, 513, 504, 513, 518, 516, 477, 499, 509, 498, 500, 56, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 518, 490, 485, 509, 470, 475, 575, 505, 509, 508, 479, 519, 503, 540, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 527, 485, 513, 550, 510, 531, 488, 540, 503, 503, 503, 509, 520, 478, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 513, 544, 493, 520, 530, 534, 536, 468, 531, 526, 515, 499, 533, 431, 495, 38, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 524, 495, 522, 512, 481, 537, 539, 510, 531, 489, 502, 521, 521, 483, 520, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8290, 8182, 8109, 8213, 8083, 8206, 8251, 8202] avg=8192.0000 max=8290 max_rank=0 min=8083 min_rank=4 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32742 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[495, 535, 501, 532, 554, 505, 490, 536, 517, 506, 528, 481, 527, 519, 501, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 483, 519, 497, 497, 478, 542, 483, 526, 501, 483, 521, 457, 507, 497, 530, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 527, 566, 523, 502, 511, 515, 481, 488, 481, 484, 495, 528, 502, 494, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 487, 524, 528, 539, 501, 454, 511, 552, 480, 525, 534, 493, 511, 493, 521, 61, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 515, 491, 540, 502, 505, 501, 528, 507, 532, 511, 533, 520, 529, 483, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 533, 529, 489, 528, 529, 519, 497, 545, 521, 496, 537, 516, 544, 452, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 490, 498, 513, 562, 527, 491, 514, 507, 530, 510, 536, 492, 523, 481, 509, 36, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 493, 545, 486, 519, 489, 502, 523, 520, 486, 513, 515, 500, 508, 525, 511, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8252, 8006, 8129, 8273, 8225, 8245, 8260, 8146] avg=8192.0000 max=8273 max_rank=3 min=8006 min_rank=1 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32876 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[464, 546, 513, 504, 500, 535, 526, 483, 506, 530, 492, 496, 543, 511, 487, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 542, 514, 541, 510, 473, 519, 480, 501, 539, 493, 561, 519, 516, 481, 560, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 491, 494, 531, 455, 482, 485, 542, 518, 502, 533, 490, 473, 554, 519, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 468, 512, 488, 509, 502, 482, 537, 496, 493, 533, 495, 495, 505, 535, 536, 62, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 521, 512, 512, 544, 497, 524, 549, 516, 486, 543, 503, 488, 484, 517, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 526, 533, 527, 460, 514, 507, 545, 536, 536, 528, 495, 528, 493, 487, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 476, 498, 516, 490, 513, 473, 515, 497, 479, 484, 519, 522, 504, 526, 545, 43, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 519, 520, 517, 502, 530, 545, 503, 472, 489, 514, 535, 489, 498, 531, 527, 11, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8126, 8267, 8128, 8215, 8257, 8238, 8123, 8182] avg=8192.0000 max=8267 max_rank=1 min=8123 min_rank=6 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32800 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[451, 497, 506, 546, 510, 489, 503, 493, 497, 510, 528, 496, 484, 450, 501, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 476, 515, 512, 509, 475, 510, 515, 511, 510, 535, 510, 492, 529, 503, 539, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 540, 487, 487, 508, 515, 500, 531, 487, 521, 498, 503, 549, 500, 522, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 498, 501, 527, 481, 469, 525, 548, 536, 522, 545, 503, 514, 479, 528, 514, 55, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 495, 553, 507, 510, 516, 520, 523, 537, 516, 497, 514, 509, 504, 518, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 538, 519, 514, 515, 492, 475, 480, 499, 554, 539, 504, 517, 513, 465, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 511, 487, 508, 510, 504, 503, 552, 509, 449, 509, 563, 483, 522, 519, 550, 36, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 491, 520, 517, 515, 512, 510, 523, 522, 530, 511, 481, 512, 553, 506, 514, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7983, 8142, 8215, 8278, 8282, 8147, 8253, 8236] avg=8192.0000 max=8282 max_rank=4 min=7983 min_rank=0 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32918 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[429, 512, 513, 538, 489, 523, 496, 545, 499, 535, 532, 589, 508, 531, 516, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 541, 528, 475, 526, 538, 498, 475, 487, 490, 514, 521, 519, 481, 544, 519, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 454, 551, 518, 512, 522, 478, 530, 489, 479, 478, 482, 550, 557, 542, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 493, 523, 523, 513, 491, 538, 526, 508, 496, 529, 490, 520, 510, 473, 521, 53, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 519, 503, 512, 505, 478, 500, 493, 493, 524, 538, 518, 515, 525, 514, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 509, 529, 523, 517, 511, 521, 511, 553, 524, 521, 483, 514, 485, 459, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 509, 511, 461, 529, 508, 558, 499, 510, 502, 506, 521, 479, 495, 521, 510, 36, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 501, 478, 519, 501, 507, 507, 493, 538, 492, 503, 517, 545, 509, 512, 519, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8279, 8199, 8126, 8226, 8177, 8208, 8181, 8140] avg=8192.0000 max=8279 max_rank=0 min=8126 min_rank=2 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32706 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[481, 514, 491, 526, 497, 506, 520, 494, 483, 485, 544, 513, 535, 489, 512, 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 536, 468, 502, 519, 558, 488, 511, 498, 515, 527, 533, 497, 512, 527, 475, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 542, 505, 518, 495, 511, 512, 499, 534, 509, 508, 562, 555, 524, 538, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 516, 520, 554, 495, 497, 508, 491, 490, 513, 450, 499, 481, 510, 490, 513, 59, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 495, 467, 514, 515, 462, 527, 479, 497, 543, 503, 507, 503, 483, 489, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 511, 501, 537, 533, 514, 498, 513, 504, 523, 490, 494, 553, 543, 463, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 525, 518, 505, 528, 492, 537, 529, 526, 470, 543, 481, 487, 511, 470, 562, 36, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 530, 525, 514, 503, 540, 508, 541, 496, 540, 532, 509, 497, 508, 504, 494, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8152, 8173, 8365, 8102, 7969, 8241, 8276, 8258] avg=8192.0000 max=8365 max_rank=2 min=7969 min_rank=4 imbalance_ratio=1.021118 max_load_reduction=57171 max_load_reduction_pct=0.872360 moved_to_echo=32744 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[459, 511, 548, 538, 495, 532, 516, 541, 520, 517, 502, 512, 516, 531, 533, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 453, 518, 482, 519, 537, 538, 518, 529, 522, 525, 533, 502, 474, 528, 526, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 498, 487, 503, 516, 502, 524, 527, 501, 521, 480, 489, 503, 514, 533, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 536, 483, 519, 509, 511, 513, 504, 513, 518, 516, 477, 499, 509, 498, 500, 56, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 518, 490, 485, 509, 470, 475, 575, 505, 509, 508, 479, 519, 503, 540, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 527, 485, 513, 550, 510, 531, 488, 540, 503, 503, 503, 509, 520, 478, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 513, 544, 493, 520, 530, 534, 536, 468, 531, 526, 515, 499, 533, 431, 495, 38, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 524, 495, 522, 512, 481, 537, 539, 510, 531, 489, 502, 521, 521, 483, 520, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8290, 8182, 8109, 8213, 8083, 8206, 8251, 8202] avg=8192.0000 max=8290 max_rank=0 min=8083 min_rank=4 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32742 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[491, 552, 519, 494, 488, 525, 503, 535, 479, 474, 527, 527, 514, 523, 518, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 539, 512, 543, 531, 547, 547, 508, 532, 566, 507, 498, 501, 484, 518, 543, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 458, 501, 546, 533, 495, 510, 526, 475, 545, 522, 560, 507, 488, 508, 537, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 521, 483, 475, 524, 531, 539, 489, 501, 523, 517, 515, 489, 472, 503, 457, 60, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 503, 514, 509, 509, 504, 480, 495, 509, 533, 507, 516, 543, 518, 504, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 480, 513, 482, 505, 525, 522, 516, 521, 508, 496, 535, 512, 506, 500, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 455, 469, 547, 499, 490, 537, 476, 473, 483, 495, 510, 552, 473, 500, 526, 39, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 527, 547, 530, 468, 493, 513, 498, 524, 522, 481, 496, 500, 485, 486, 521, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8163, 8411, 8247, 8144, 8155, 8190, 8086, 8140] avg=8192.0000 max=8411 max_rank=1 min=8086 min_rank=6 imbalance_ratio=1.026733 max_load_reduction=57125 max_load_reduction_pct=0.871658 moved_to_echo=32571 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[3840, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 3968, 3968, 4096, 4224, 4096, 4096, 4096, 4224, 384, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 3968, 4224, 4352, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[3840, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 3840, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 3968, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[524, 504, 541, 489, 514, 545, 492, 478, 477, 520, 549, 496, 542, 508, 554, 530, 493, 493, 483, 467, 514, 510, 491, 503, 531, 502, 523, 513, 527, 522, 523, 468, 492, 516, 504, 504, 531, 530, 513, 535, 515, 548, 493, 522, 481, 544, 523, 546, 493, 490, 520, 484, 531, 487, 492, 527, 530, 489, 479, 537, 526, 523, 530, 532, 524, 482, 529, 493, 531, 516, 558, 565, 479, 508, 507, 548, 527, 547, 497, 521, 499, 480, 504, 525, 517, 526, 494, 507, 498, 518, 498, 526, 531, 548, 558, 509, 508, 514, 487, 488, 513, 494, 502, 531, 516, 463, 493, 510, 475, 548, 470, 506, 491, 552, 497, 512, 501, 496, 527, 548, 529, 489, 498, 487, 510, 496, 526, 496] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[524, 504, 537, 506, 539, 499, 568, 549, 521, 500, 524, 528, 490, 507, 515, 516, 481, 528, 548, 547, 577, 552, 502, 481, 502, 553, 542, 510, 504, 536, 532, 542, 507, 510, 552, 510, 521, 484, 482, 538, 476, 491, 484, 501, 486, 526, 476, 462, 492, 510, 511, 541, 506, 476, 496, 482, 521, 547, 505, 499, 508, 501, 512, 514, 520, 488, 499, 536, 502, 485, 519, 491, 516, 523, 527, 502, 486, 515, 501, 555, 506, 474, 542, 528, 517, 519, 502, 514, 484, 486, 493, 509, 498, 460, 514, 540, 537, 495, 534, 515, 527, 490, 490, 509, 544, 480, 500, 504, 505, 524, 561, 522, 486, 526, 521, 541, 529, 500, 499, 524, 507, 493, 525, 484, 482, 500, 545, 495] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[545, 544, 527, 510, 524, 512, 473, 515, 522, 539, 509, 513, 461, 547, 538, 510, 537, 502, 550, 508, 499, 497, 491, 515, 546, 511, 483, 516, 481, 480, 486, 515, 471, 472, 515, 530, 501, 530, 549, 490, 477, 523, 499, 522, 530, 475, 514, 507, 559, 566, 521, 537, 510, 484, 537, 492, 536, 516, 495, 524, 518, 515, 541, 529, 530, 520, 517, 522, 521, 499, 491, 500, 535, 510, 547, 518, 504, 470, 505, 507, 497, 517, 523, 491, 531, 514, 531, 560, 508, 509, 533, 522, 532, 450, 516, 502, 462, 515, 519, 477, 513, 472, 530, 512, 522, 455, 546, 504, 520, 516, 521, 520, 476, 478, 525, 517, 510, 519, 486, 494, 483, 523, 551, 498, 533, 510, 505, 501] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[540, 527, 511, 495, 507, 526, 542, 534, 507, 544, 487, 479, 502, 507, 496, 520, 502, 477, 454, 501, 489, 526, 503, 532, 512, 477, 520, 559, 536, 537, 513, 495, 512, 512, 518, 517, 493, 517, 506, 511, 520, 487, 547, 471, 554, 520, 459, 502, 528, 495, 515, 505, 523, 489, 515, 511, 508, 497, 530, 547, 507, 535, 506, 519, 508, 524, 501, 539, 490, 502, 534, 528, 523, 495, 530, 512, 517, 479, 562, 517, 459, 503, 509, 503, 500, 484, 533, 508, 490, 498, 560, 490, 505, 522, 481, 485, 523, 486, 502, 533, 516, 520, 525, 532, 520, 467, 511, 512, 485, 514, 501, 548, 475, 526, 521, 508, 538, 562, 525, 518, 517, 514, 508, 487, 566, 510, 527, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[522, 530, 553, 505, 513, 507, 523, 492, 505, 493, 532, 496, 518, 521, 515, 517, 525, 549, 481, 501, 505, 520, 507, 552, 489, 484, 499, 483, 513, 483, 561, 518, 502, 495, 559, 523, 514, 501, 575, 508, 559, 499, 528, 501, 511, 500, 476, 490, 497, 495, 527, 524, 513, 511, 525, 496, 480, 517, 498, 499, 496, 535, 513, 527, 507, 476, 500, 499, 490, 499, 526, 511, 518, 516, 555, 490, 516, 481, 531, 469, 489, 520, 531, 518, 561, 500, 506, 520, 568, 478, 522, 515, 465, 509, 547, 512, 488, 511, 510, 512, 493, 543, 485, 514, 529, 506, 567, 502, 494, 526, 542, 511, 530, 497, 527, 517, 523, 489, 527, 535, 489, 533, 455, 540, 515, 498, 505, 472] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[524, 504, 537, 506, 539, 499, 568, 549, 521, 500, 524, 528, 490, 507, 515, 516, 481, 528, 548, 547, 577, 552, 502, 481, 502, 553, 542, 510, 504, 536, 532, 542, 507, 510, 552, 510, 521, 484, 482, 538, 476, 491, 484, 501, 486, 526, 476, 462, 492, 510, 511, 541, 506, 476, 496, 482, 521, 547, 505, 499, 508, 501, 512, 514, 520, 488, 499, 536, 502, 485, 519, 491, 516, 523, 527, 502, 486, 515, 501, 555, 506, 474, 542, 528, 517, 519, 502, 514, 484, 486, 493, 509, 498, 460, 514, 540, 537, 495, 534, 515, 527, 490, 490, 509, 544, 480, 500, 504, 505, 524, 561, 522, 486, 526, 521, 541, 529, 500, 499, 524, 507, 493, 525, 484, 482, 500, 545, 495] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[524, 504, 541, 489, 514, 545, 492, 478, 477, 520, 549, 496, 542, 508, 554, 530, 493, 493, 483, 467, 514, 510, 491, 503, 531, 502, 523, 513, 527, 522, 523, 468, 492, 516, 504, 504, 531, 530, 513, 535, 515, 548, 493, 522, 481, 544, 523, 546, 493, 490, 520, 484, 531, 487, 492, 527, 530, 489, 479, 537, 526, 523, 530, 532, 524, 482, 529, 493, 531, 516, 558, 565, 479, 508, 507, 548, 527, 547, 497, 521, 499, 480, 504, 525, 517, 526, 494, 507, 498, 518, 498, 526, 531, 548, 558, 509, 508, 514, 487, 488, 513, 494, 502, 531, 516, 463, 493, 510, 475, 548, 470, 506, 491, 552, 497, 512, 501, 496, 527, 548, 529, 489, 498, 487, 510, 496, 526, 496] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[544, 545, 542, 507, 503, 536, 530, 517, 476, 531, 471, 515, 490, 511, 500, 501, 530, 491, 491, 525, 501, 524, 534, 502, 480, 504, 547, 463, 482, 510, 502, 531, 535, 500, 523, 504, 514, 526, 521, 489, 487, 523, 499, 500, 512, 514, 499, 538, 532, 552, 485, 495, 518, 510, 548, 535, 524, 538, 544, 477, 535, 530, 483, 563, 561, 473, 493, 477, 503, 464, 542, 499, 512, 524, 507, 532, 501, 487, 541, 544, 540, 518, 539, 532, 493, 493, 487, 482, 517, 546, 495, 483, 501, 521, 512, 460, 499, 488, 519, 537, 496, 503, 465, 489, 538, 528, 497, 558, 507, 514, 505, 489, 528, 543, 509, 461, 508, 550, 501, 514, 512, 483, 496, 540, 545, 483, 534, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[499, 514, 489, 544, 469, 514, 545, 525, 483, 507, 522, 499, 509, 500, 521, 509, 498, 504, 499, 517, 485, 525, 479, 516, 510, 563, 543, 520, 484, 508, 507, 495, 548, 509, 557, 538, 509, 525, 515, 501, 540, 498, 505, 515, 503, 525, 497, 503, 529, 519, 502, 541, 498, 527, 523, 531, 494, 507, 539, 526, 504, 568, 492, 519, 508, 527, 488, 499, 527, 469, 480, 536, 481, 491, 505, 531, 505, 497, 546, 493, 483, 514, 535, 510, 566, 522, 533, 511, 539, 545, 487, 514, 548, 475, 492, 527, 475, 519, 497, 522, 500, 504, 505, 479, 551, 428, 540, 511, 503, 491, 546, 479, 540, 593, 469, 549, 511, 481, 519, 494, 481, 526, 536, 497, 489, 464, 516, 528] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[530, 532, 483, 503, 530, 517, 517, 495, 497, 478, 559, 488, 485, 534, 514, 508, 523, 498, 481, 531, 518, 540, 519, 515, 533, 489, 485, 500, 502, 518, 516, 533, 504, 487, 506, 507, 525, 518, 519, 547, 529, 522, 505, 532, 516, 512, 478, 535, 526, 496, 524, 510, 503, 519, 514, 532, 589, 471, 510, 512, 507, 512, 510, 506, 492, 530, 517, 474, 492, 543, 545, 535, 505, 531, 474, 518, 543, 555, 495, 508, 493, 479, 494, 520, 506, 496, 547, 499, 481, 521, 498, 502, 502, 504, 520, 516, 486, 527, 505, 540, 503, 539, 502, 506, 500, 497, 465, 558, 506, 493, 495, 515, 487, 523, 493, 523, 509, 558, 532, 506, 507, 482, 508, 526, 520, 513, 527, 496] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[483, 504, 541, 489, 514, 545, 492, 478, 477, 520, 549, 496, 542, 508, 554, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 493, 483, 467, 514, 510, 491, 503, 531, 502, 523, 513, 527, 522, 523, 468, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 516, 504, 504, 531, 530, 513, 535, 515, 548, 493, 522, 481, 544, 523, 546, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 490, 520, 484, 531, 487, 492, 527, 530, 489, 479, 537, 526, 468, 530, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 482, 529, 493, 531, 516, 558, 565, 479, 508, 507, 548, 527, 547, 497, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 480, 504, 525, 517, 526, 494, 507, 498, 518, 498, 526, 531, 548, 558, 509, 21, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 514, 487, 488, 513, 494, 502, 531, 516, 463, 493, 510, 475, 548, 470, 506, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 552, 497, 512, 501, 496, 527, 548, 529, 489, 498, 487, 510, 496, 526, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8080, 8300, 8115, 8332, 8261, 8071, 8155] avg=8192.0000 max=8332 max_rank=4 min=8071 min_rank=6 imbalance_ratio=1.017090 max_load_reduction=57204 max_load_reduction_pct=0.872864 moved_to_echo=32819 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[540, 527, 511, 495, 507, 526, 542, 534, 507, 544, 487, 479, 502, 507, 496, 520, 502, 477, 454, 501, 489, 526, 503, 532, 512, 477, 520, 559, 536, 537, 513, 495, 512, 512, 518, 517, 493, 517, 506, 511, 520, 487, 547, 471, 554, 520, 459, 502, 528, 495, 515, 505, 523, 489, 515, 511, 508, 497, 530, 547, 507, 535, 506, 519, 508, 524, 501, 539, 490, 502, 534, 528, 523, 495, 530, 512, 517, 479, 562, 517, 459, 503, 509, 503, 500, 484, 533, 508, 490, 498, 560, 490, 505, 522, 481, 485, 523, 486, 502, 533, 516, 520, 525, 532, 520, 467, 511, 512, 485, 514, 501, 548, 475, 526, 521, 508, 538, 562, 525, 518, 517, 514, 508, 487, 566, 510, 527, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[545, 544, 527, 510, 524, 512, 473, 515, 522, 539, 509, 513, 461, 547, 538, 510, 537, 502, 550, 508, 499, 497, 491, 515, 546, 511, 483, 516, 481, 480, 486, 515, 471, 472, 515, 530, 501, 530, 549, 490, 477, 523, 499, 522, 530, 475, 514, 507, 559, 566, 521, 537, 510, 484, 537, 492, 536, 516, 495, 524, 518, 515, 541, 529, 530, 520, 517, 522, 521, 499, 491, 500, 535, 510, 547, 518, 504, 470, 505, 507, 497, 517, 523, 491, 531, 514, 531, 560, 508, 509, 533, 522, 532, 450, 516, 502, 462, 515, 519, 477, 513, 472, 530, 512, 522, 455, 546, 504, 520, 516, 521, 520, 476, 478, 525, 517, 510, 519, 486, 494, 483, 523, 551, 498, 533, 510, 505, 501] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[499, 514, 489, 544, 469, 514, 545, 525, 483, 507, 522, 499, 509, 500, 521, 509, 498, 504, 499, 517, 485, 525, 479, 516, 510, 563, 543, 520, 484, 508, 507, 495, 548, 509, 557, 538, 509, 525, 515, 501, 540, 498, 505, 515, 503, 525, 497, 503, 529, 519, 502, 541, 498, 527, 523, 531, 494, 507, 539, 526, 504, 568, 492, 519, 508, 527, 488, 499, 527, 469, 480, 536, 481, 491, 505, 531, 505, 497, 546, 493, 483, 514, 535, 510, 566, 522, 533, 511, 539, 545, 487, 514, 548, 475, 492, 527, 475, 519, 497, 522, 500, 504, 505, 479, 551, 428, 540, 511, 503, 491, 546, 479, 540, 593, 469, 549, 511, 481, 519, 494, 481, 526, 536, 497, 489, 464, 516, 528] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[465, 530, 553, 505, 513, 507, 523, 492, 505, 493, 532, 496, 518, 521, 515, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 549, 481, 501, 505, 520, 507, 552, 489, 484, 499, 483, 513, 483, 561, 518, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 495, 559, 523, 514, 501, 575, 508, 559, 499, 528, 501, 511, 500, 476, 490, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 495, 527, 524, 513, 511, 525, 496, 480, 517, 498, 499, 496, 475, 513, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 476, 500, 499, 490, 499, 524, 511, 518, 516, 555, 490, 516, 481, 531, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 520, 531, 518, 561, 500, 506, 520, 568, 478, 522, 515, 465, 509, 547, 512, 24, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 511, 510, 512, 493, 543, 485, 514, 529, 506, 567, 502, 494, 526, 542, 511, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 497, 527, 517, 523, 489, 527, 535, 489, 533, 455, 540, 515, 498, 505, 472, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8192, 8247, 8093, 8082, 8288, 8290, 8159] avg=8192.0000 max=8290 max_rank=6 min=8082 min_rank=4 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32819 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[503, 544, 527, 510, 524, 512, 473, 515, 522, 539, 509, 513, 461, 547, 538, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 502, 550, 508, 499, 497, 491, 515, 546, 511, 483, 516, 481, 480, 486, 515, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 472, 515, 530, 501, 530, 549, 490, 477, 523, 499, 522, 530, 475, 514, 507, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 566, 521, 537, 510, 484, 537, 492, 536, 516, 495, 524, 518, 461, 541, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 520, 517, 522, 521, 499, 491, 500, 535, 510, 547, 518, 504, 470, 505, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 517, 523, 491, 531, 514, 531, 560, 508, 509, 533, 522, 532, 450, 516, 502, 21, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 515, 519, 477, 513, 472, 530, 512, 522, 455, 546, 504, 520, 516, 521, 520, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 478, 525, 517, 510, 519, 486, 494, 483, 523, 551, 498, 533, 510, 505, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8247, 8135, 8108, 8326, 8196, 8259, 8156, 8109] avg=8192.0000 max=8326 max_rank=3 min=8108 min_rank=2 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32720 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[502, 545, 542, 507, 503, 536, 530, 517, 476, 531, 471, 515, 490, 511, 500, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 491, 491, 525, 501, 524, 534, 502, 480, 504, 547, 463, 482, 510, 502, 531, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 500, 523, 504, 514, 526, 521, 489, 487, 523, 499, 500, 512, 514, 499, 538, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 552, 485, 495, 518, 510, 548, 535, 524, 538, 544, 477, 535, 474, 483, 563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 473, 493, 477, 503, 464, 542, 499, 512, 524, 507, 532, 501, 487, 541, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 518, 539, 532, 493, 493, 487, 482, 517, 546, 495, 483, 501, 521, 512, 460, 21, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 488, 519, 537, 496, 503, 465, 489, 538, 528, 497, 558, 507, 514, 505, 489, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 543, 509, 461, 508, 550, 501, 514, 512, 483, 496, 540, 545, 483, 534, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8135, 8187, 8313, 8160, 8142, 8186, 8236] avg=8192.0000 max=8313 max_rank=3 min=8135 min_rank=1 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32724 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[483, 504, 537, 506, 539, 499, 568, 549, 521, 500, 524, 528, 490, 507, 515, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 528, 548, 547, 577, 552, 502, 481, 502, 553, 542, 510, 504, 536, 532, 542, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 510, 552, 510, 521, 484, 482, 538, 476, 491, 484, 501, 486, 526, 476, 462, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 510, 511, 541, 506, 476, 496, 482, 521, 547, 505, 499, 508, 448, 512, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 488, 499, 536, 502, 485, 519, 491, 516, 523, 527, 502, 486, 515, 501, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 474, 542, 528, 517, 519, 502, 514, 484, 486, 493, 509, 498, 460, 514, 540, 21, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 495, 534, 515, 527, 490, 490, 509, 544, 480, 500, 504, 505, 524, 561, 522, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 526, 521, 541, 529, 500, 499, 524, 507, 493, 525, 484, 482, 500, 545, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8286, 8454, 8009, 8068, 8165, 8109, 8288, 8157] avg=8192.0000 max=8454 max_rank=1 min=8009 min_rank=2 imbalance_ratio=1.031982 max_load_reduction=57082 max_load_reduction_pct=0.871002 moved_to_echo=32719 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[498, 527, 511, 495, 507, 526, 542, 534, 507, 544, 487, 479, 502, 507, 496, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 477, 454, 501, 489, 526, 503, 532, 512, 477, 520, 559, 536, 537, 513, 495, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 512, 518, 517, 493, 517, 506, 511, 520, 487, 547, 471, 554, 520, 459, 502, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 495, 515, 505, 523, 489, 515, 511, 508, 497, 530, 547, 507, 479, 506, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 524, 501, 539, 490, 502, 534, 528, 523, 495, 530, 512, 517, 479, 562, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 459, 503, 509, 503, 500, 484, 533, 508, 490, 498, 560, 490, 505, 522, 481, 485, 21, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 486, 502, 533, 516, 520, 525, 532, 520, 467, 511, 512, 485, 514, 501, 548, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 526, 521, 508, 538, 562, 525, 518, 517, 514, 508, 487, 566, 510, 527, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8151, 8149, 8174, 8261, 8053, 8249, 8317] avg=8192.0000 max=8317 max_rank=7 min=8053 min_rank=5 imbalance_ratio=1.015259 max_load_reduction=57219 max_load_reduction_pct=0.873093 moved_to_echo=32880 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[459, 514, 489, 544, 469, 514, 545, 525, 483, 507, 522, 499, 509, 500, 521, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 504, 499, 517, 485, 525, 479, 516, 510, 563, 543, 520, 484, 508, 507, 495, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 509, 557, 538, 509, 525, 515, 501, 540, 498, 505, 515, 503, 525, 497, 503, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 519, 502, 541, 498, 527, 523, 531, 494, 507, 539, 526, 504, 509, 492, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 527, 488, 499, 527, 469, 480, 536, 481, 491, 505, 531, 505, 497, 546, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 514, 535, 510, 566, 522, 533, 511, 539, 545, 487, 514, 548, 475, 492, 527, 20, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 519, 497, 522, 500, 504, 505, 479, 551, 428, 540, 511, 503, 491, 546, 479, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 593, 469, 549, 511, 481, 519, 494, 481, 526, 536, 497, 489, 464, 516, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8109, 8170, 8291, 8260, 8083, 8323, 8107, 8193] avg=8192.0000 max=8323 max_rank=5 min=8083 min_rank=4 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32706 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[483, 504, 541, 489, 514, 545, 492, 478, 477, 520, 549, 496, 542, 508, 554, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 493, 483, 467, 514, 510, 491, 503, 531, 502, 523, 513, 527, 522, 523, 468, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 516, 504, 504, 531, 530, 513, 535, 515, 548, 493, 522, 481, 544, 523, 546, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 490, 520, 484, 531, 487, 492, 527, 530, 489, 479, 537, 526, 468, 530, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 482, 529, 493, 531, 516, 558, 565, 479, 508, 507, 548, 527, 547, 497, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 480, 504, 525, 517, 526, 494, 507, 498, 518, 498, 526, 531, 548, 558, 509, 21, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 514, 487, 488, 513, 494, 502, 531, 516, 463, 493, 510, 475, 548, 470, 506, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 552, 497, 512, 501, 496, 527, 548, 529, 489, 498, 487, 510, 496, 526, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8080, 8300, 8115, 8332, 8261, 8071, 8155] avg=8192.0000 max=8332 max_rank=4 min=8071 min_rank=6 imbalance_ratio=1.017090 max_load_reduction=57204 max_load_reduction_pct=0.872864 moved_to_echo=32819 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[498, 527, 511, 495, 507, 526, 542, 534, 507, 544, 487, 479, 502, 507, 496, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 477, 454, 501, 489, 526, 503, 532, 512, 477, 520, 559, 536, 537, 513, 495, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 512, 518, 517, 493, 517, 506, 511, 520, 487, 547, 471, 554, 520, 459, 502, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 495, 515, 505, 523, 489, 515, 511, 508, 497, 530, 547, 507, 479, 506, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 524, 501, 539, 490, 502, 534, 528, 523, 495, 530, 512, 517, 479, 562, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 459, 503, 509, 503, 500, 484, 533, 508, 490, 498, 560, 490, 505, 522, 481, 485, 21, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 486, 502, 533, 516, 520, 525, 532, 520, 467, 511, 512, 485, 514, 501, 548, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 526, 521, 508, 538, 562, 525, 518, 517, 514, 508, 487, 566, 510, 527, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8151, 8149, 8174, 8261, 8053, 8249, 8317] avg=8192.0000 max=8317 max_rank=7 min=8053 min_rank=5 imbalance_ratio=1.015259 max_load_reduction=57219 max_load_reduction_pct=0.873093 moved_to_echo=32880 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[483, 504, 537, 506, 539, 499, 568, 549, 521, 500, 524, 528, 490, 507, 515, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 528, 548, 547, 577, 552, 502, 481, 502, 553, 542, 510, 504, 536, 532, 542, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 510, 552, 510, 521, 484, 482, 538, 476, 491, 484, 501, 486, 526, 476, 462, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 510, 511, 541, 506, 476, 496, 482, 521, 547, 505, 499, 508, 448, 512, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 488, 499, 536, 502, 485, 519, 491, 516, 523, 527, 502, 486, 515, 501, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 474, 542, 528, 517, 519, 502, 514, 484, 486, 493, 509, 498, 460, 514, 540, 21, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 495, 534, 515, 527, 490, 490, 509, 544, 480, 500, 504, 505, 524, 561, 522, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 526, 521, 541, 529, 500, 499, 524, 507, 493, 525, 484, 482, 500, 545, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8286, 8454, 8009, 8068, 8165, 8109, 8288, 8157] avg=8192.0000 max=8454 max_rank=1 min=8009 min_rank=2 imbalance_ratio=1.031982 max_load_reduction=57082 max_load_reduction_pct=0.871002 moved_to_echo=32719 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[488, 532, 483, 503, 530, 517, 517, 495, 497, 478, 559, 488, 485, 534, 514, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 498, 481, 531, 518, 540, 519, 515, 533, 489, 485, 500, 502, 518, 516, 533, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 487, 506, 507, 525, 518, 519, 547, 529, 522, 505, 532, 516, 512, 478, 535, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 496, 524, 510, 503, 519, 514, 532, 589, 471, 510, 512, 507, 458, 510, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 530, 517, 474, 492, 543, 545, 535, 505, 531, 474, 518, 543, 555, 495, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 479, 494, 520, 506, 496, 547, 499, 481, 521, 498, 502, 502, 504, 520, 516, 21, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 527, 505, 540, 503, 539, 502, 506, 500, 497, 465, 558, 506, 493, 495, 515, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 523, 493, 523, 509, 558, 532, 506, 507, 482, 508, 526, 520, 513, 527, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8128, 8219, 8245, 8187, 8257, 8101, 8189, 8210] avg=8192.0000 max=8257 max_rank=4 min=8101 min_rank=5 imbalance_ratio=1.007935 max_load_reduction=57279 max_load_reduction_pct=0.874008 moved_to_echo=32757 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[502, 545, 542, 507, 503, 536, 530, 517, 476, 531, 471, 515, 490, 511, 500, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 491, 491, 525, 501, 524, 534, 502, 480, 504, 547, 463, 482, 510, 502, 531, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 500, 523, 504, 514, 526, 521, 489, 487, 523, 499, 500, 512, 514, 499, 538, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 552, 485, 495, 518, 510, 548, 535, 524, 538, 544, 477, 535, 474, 483, 563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 473, 493, 477, 503, 464, 542, 499, 512, 524, 507, 532, 501, 487, 541, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 518, 539, 532, 493, 493, 487, 482, 517, 546, 495, 483, 501, 521, 512, 460, 21, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 488, 519, 537, 496, 503, 465, 489, 538, 528, 497, 558, 507, 514, 505, 489, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 543, 509, 461, 508, 550, 501, 514, 512, 483, 496, 540, 545, 483, 534, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8135, 8187, 8313, 8160, 8142, 8186, 8236] avg=8192.0000 max=8313 max_rank=3 min=8135 min_rank=1 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32724 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[503, 544, 527, 510, 524, 512, 473, 515, 522, 539, 509, 513, 461, 547, 538, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 502, 550, 508, 499, 497, 491, 515, 546, 511, 483, 516, 481, 480, 486, 515, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 472, 515, 530, 501, 530, 549, 490, 477, 523, 499, 522, 530, 475, 514, 507, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 566, 521, 537, 510, 484, 537, 492, 536, 516, 495, 524, 518, 461, 541, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 520, 517, 522, 521, 499, 491, 500, 535, 510, 547, 518, 504, 470, 505, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 517, 523, 491, 531, 514, 531, 560, 508, 509, 533, 522, 532, 450, 516, 502, 21, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 515, 519, 477, 513, 472, 530, 512, 522, 455, 546, 504, 520, 516, 521, 520, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 478, 525, 517, 510, 519, 486, 494, 483, 523, 551, 498, 533, 510, 505, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8247, 8135, 8108, 8326, 8196, 8259, 8156, 8109] avg=8192.0000 max=8326 max_rank=3 min=8108 min_rank=2 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32720 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 3840, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[3968, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 3840, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[3968, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[3968, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4352, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 3968, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4352, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 3840, 4224, 4224, 4096, 4224, 4224, 4096, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4352, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[523, 515, 491, 521, 501, 536, 491, 496, 522, 493, 497, 492, 545, 493, 508, 504, 491, 506, 518, 540, 505, 529, 525, 525, 517, 521, 525, 491, 474, 505, 529, 490, 503, 523, 532, 563, 523, 559, 488, 531, 556, 499, 518, 509, 476, 470, 498, 483, 514, 506, 508, 540, 511, 531, 550, 483, 484, 541, 506, 467, 530, 496, 524, 474, 519, 501, 518, 518, 499, 513, 515, 522, 517, 528, 521, 557, 509, 500, 466, 527, 501, 508, 513, 548, 498, 513, 476, 513, 525, 513, 537, 510, 495, 516, 492, 530, 549, 529, 496, 475, 509, 502, 479, 568, 494, 498, 547, 509, 509, 487, 517, 477, 499, 509, 543, 550, 481, 528, 515, 516, 485, 519, 521, 540, 493, 529, 515, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[530, 481, 515, 505, 500, 492, 515, 499, 495, 531, 501, 474, 526, 512, 516, 559, 476, 519, 514, 530, 523, 497, 510, 503, 524, 501, 511, 509, 498, 498, 531, 521, 530, 503, 553, 509, 520, 482, 534, 528, 526, 506, 504, 479, 522, 470, 499, 527, 533, 533, 544, 527, 495, 490, 514, 505, 539, 545, 514, 534, 550, 527, 509, 473, 485, 506, 537, 527, 497, 482, 506, 537, 500, 534, 535, 533, 465, 522, 491, 491, 502, 494, 532, 533, 489, 481, 499, 479, 514, 532, 538, 508, 468, 535, 486, 509, 531, 511, 474, 545, 551, 560, 535, 566, 511, 527, 491, 493, 505, 513, 504, 500, 466, 516, 501, 490, 515, 503, 501, 527, 505, 524, 516, 508, 504, 484, 525, 547] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[484, 496, 525, 514, 490, 510, 490, 495, 505, 491, 533, 546, 535, 518, 503, 509, 519, 500, 525, 523, 471, 507, 520, 532, 522, 521, 502, 521, 546, 495, 487, 541, 496, 498, 509, 473, 517, 531, 501, 494, 519, 513, 453, 501, 538, 506, 516, 520, 505, 510, 483, 503, 490, 543, 505, 529, 533, 508, 524, 476, 486, 500, 504, 541, 519, 532, 534, 471, 515, 492, 497, 490, 491, 504, 525, 495, 499, 524, 496, 502, 533, 546, 510, 488, 500, 560, 510, 501, 544, 563, 519, 509, 563, 530, 481, 526, 522, 529, 504, 501, 548, 540, 545, 554, 518, 512, 493, 496, 500, 505, 514, 487, 487, 530, 552, 502, 491, 514, 519, 496, 504, 527, 535, 502, 514, 530, 490, 505] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[536, 509, 485, 486, 492, 537, 494, 475, 549, 540, 516, 504, 453, 525, 527, 511, 466, 497, 518, 497, 508, 542, 520, 515, 537, 481, 557, 511, 514, 494, 496, 552, 509, 531, 517, 524, 522, 504, 521, 506, 515, 493, 509, 516, 521, 506, 539, 498, 491, 514, 520, 504, 489, 491, 516, 530, 534, 514, 513, 489, 514, 511, 508, 543, 516, 527, 517, 505, 512, 522, 506, 508, 541, 515, 569, 548, 485, 517, 513, 519, 503, 493, 496, 516, 531, 526, 458, 515, 492, 572, 500, 517, 505, 530, 554, 514, 532, 495, 514, 494, 491, 487, 510, 517, 492, 499, 514, 527, 516, 530, 491, 504, 507, 503, 524, 503, 539, 467, 514, 513, 501, 509, 483, 493, 491, 519, 497, 562] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[523, 515, 491, 521, 501, 536, 491, 496, 522, 493, 497, 492, 545, 493, 508, 504, 491, 506, 518, 540, 505, 529, 525, 525, 517, 521, 525, 491, 474, 505, 529, 490, 503, 523, 532, 563, 523, 559, 488, 531, 556, 499, 518, 509, 476, 470, 498, 483, 514, 506, 508, 540, 511, 531, 550, 483, 484, 541, 506, 467, 530, 496, 524, 474, 519, 501, 518, 518, 499, 513, 515, 522, 517, 528, 521, 557, 509, 500, 466, 527, 501, 508, 513, 548, 498, 513, 476, 513, 525, 513, 537, 510, 495, 516, 492, 530, 549, 529, 496, 475, 509, 502, 479, 568, 494, 498, 547, 509, 509, 487, 517, 477, 499, 509, 543, 550, 481, 528, 515, 516, 485, 519, 521, 540, 493, 529, 515, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[491, 521, 555, 501, 480, 492, 533, 477, 503, 527, 506, 532, 526, 534, 488, 536, 498, 472, 509, 487, 543, 530, 514, 534, 506, 483, 536, 489, 492, 543, 502, 512, 474, 507, 529, 534, 523, 501, 504, 473, 522, 512, 515, 508, 491, 528, 513, 504, 535, 494, 492, 524, 508, 482, 524, 498, 510, 537, 508, 544, 557, 555, 533, 533, 510, 526, 500, 549, 510, 510, 488, 510, 517, 503, 504, 525, 525, 531, 495, 532, 498, 506, 517, 491, 464, 525, 509, 498, 525, 516, 524, 509, 494, 494, 505, 501, 519, 454, 545, 561, 526, 517, 543, 480, 505, 533, 519, 480, 509, 536, 510, 536, 533, 520, 494, 537, 480, 504, 528, 501, 502, 517, 506, 501, 490, 517, 495, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[505, 499, 571, 515, 535, 505, 535, 532, 478, 465, 497, 519, 512, 499, 517, 521, 482, 505, 553, 503, 515, 516, 506, 492, 494, 520, 504, 481, 518, 503, 513, 532, 540, 519, 509, 503, 515, 491, 531, 525, 509, 487, 513, 528, 492, 510, 553, 554, 523, 501, 484, 486, 493, 530, 512, 473, 507, 547, 502, 499, 526, 499, 549, 480, 518, 489, 508, 487, 475, 490, 510, 541, 527, 502, 516, 526, 506, 495, 503, 460, 488, 518, 500, 513, 560, 516, 527, 506, 533, 492, 533, 500, 532, 504, 526, 523, 498, 494, 529, 494, 548, 554, 444, 518, 517, 520, 464, 528, 525, 525, 511, 509, 530, 545, 553, 507, 548, 485, 515, 497, 528, 512, 527, 525, 523, 504, 490, 513] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[484, 496, 525, 514, 490, 510, 490, 495, 505, 491, 533, 546, 535, 518, 503, 509, 519, 500, 525, 523, 471, 507, 520, 532, 522, 521, 502, 521, 546, 495, 487, 541, 496, 498, 509, 473, 517, 531, 501, 494, 519, 513, 453, 501, 538, 506, 516, 520, 505, 510, 483, 503, 490, 543, 505, 529, 533, 508, 524, 476, 486, 500, 504, 541, 519, 532, 534, 471, 515, 492, 497, 490, 491, 504, 525, 495, 499, 524, 496, 502, 533, 546, 510, 488, 500, 560, 510, 501, 544, 563, 519, 509, 563, 530, 481, 526, 522, 529, 504, 501, 548, 540, 545, 554, 518, 512, 493, 496, 500, 505, 514, 487, 487, 530, 552, 502, 491, 514, 519, 496, 504, 527, 535, 502, 514, 530, 490, 505] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[475, 504, 496, 483, 520, 523, 490, 547, 537, 484, 512, 498, 530, 453, 482, 463, 511, 506, 488, 553, 528, 475, 539, 499, 477, 526, 528, 518, 531, 520, 548, 503, 502, 508, 493, 520, 502, 522, 517, 558, 542, 526, 466, 515, 470, 562, 540, 538, 506, 562, 530, 499, 558, 528, 524, 537, 519, 495, 523, 486, 498, 509, 504, 505, 503, 540, 496, 521, 539, 519, 502, 504, 518, 486, 541, 537, 503, 546, 510, 477, 463, 528, 496, 524, 536, 524, 501, 518, 504, 533, 529, 527, 530, 495, 513, 489, 531, 512, 546, 499, 511, 505, 484, 535, 513, 499, 484, 484, 522, 522, 495, 489, 533, 523, 528, 502, 480, 520, 526, 517, 527, 491, 480, 509, 496, 488, 530, 492] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[491, 521, 555, 501, 480, 492, 533, 477, 503, 527, 506, 532, 526, 534, 488, 536, 498, 472, 509, 487, 543, 530, 514, 534, 506, 483, 536, 489, 492, 543, 502, 512, 474, 507, 529, 534, 523, 501, 504, 473, 522, 512, 515, 508, 491, 528, 513, 504, 535, 494, 492, 524, 508, 482, 524, 498, 510, 537, 508, 544, 557, 555, 533, 533, 510, 526, 500, 549, 510, 510, 488, 510, 517, 503, 504, 525, 525, 531, 495, 532, 498, 506, 517, 491, 464, 525, 509, 498, 525, 516, 524, 509, 494, 494, 505, 501, 519, 454, 545, 561, 526, 517, 543, 480, 505, 533, 519, 480, 509, 536, 510, 536, 533, 520, 494, 537, 480, 504, 528, 501, 502, 517, 506, 501, 490, 517, 495, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[517, 521, 517, 524, 538, 521, 518, 453, 483, 540, 528, 527, 501, 544, 486, 492, 489, 513, 510, 473, 514, 538, 503, 500, 498, 496, 525, 520, 506, 541, 504, 500, 472, 469, 521, 544, 522, 474, 529, 489, 531, 474, 522, 489, 511, 573, 520, 541, 513, 539, 568, 545, 506, 533, 516, 510, 541, 522, 473, 514, 503, 518, 495, 501, 475, 500, 507, 552, 507, 509, 462, 526, 502, 506, 526, 475, 519, 502, 550, 493, 537, 504, 519, 509, 483, 489, 506, 510, 533, 510, 492, 489, 478, 510, 510, 502, 475, 522, 561, 531, 529, 576, 466, 512, 514, 529, 510, 529, 503, 495, 497, 530, 514, 514, 501, 495, 531, 515, 501, 517, 520, 540, 552, 520, 508, 506, 503, 510] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[530, 481, 515, 505, 500, 492, 515, 499, 495, 531, 501, 474, 526, 512, 516, 559, 39, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 519, 514, 530, 523, 497, 510, 503, 524, 501, 511, 509, 498, 498, 531, 521, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 503, 553, 509, 520, 482, 534, 528, 518, 506, 504, 479, 522, 470, 499, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 533, 544, 527, 495, 490, 514, 505, 539, 506, 514, 534, 550, 527, 509, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 506, 537, 527, 497, 482, 506, 537, 500, 534, 535, 533, 465, 522, 491, 491, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 494, 532, 533, 489, 481, 499, 479, 514, 532, 538, 508, 468, 535, 486, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 511, 474, 545, 551, 560, 535, 545, 511, 527, 491, 493, 505, 513, 504, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 466, 516, 501, 490, 515, 503, 501, 527, 505, 524, 516, 508, 504, 484, 525, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8173, 8184, 8293, 8162, 8099, 8296, 8132] avg=8192.0000 max=8296 max_rank=6 min=8099 min_rank=5 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32689 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[523, 515, 491, 521, 501, 536, 491, 496, 522, 493, 497, 492, 545, 493, 508, 504, 39, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 506, 518, 540, 505, 529, 525, 525, 517, 521, 525, 491, 474, 505, 529, 490, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 523, 532, 563, 523, 559, 488, 531, 547, 499, 518, 509, 476, 470, 498, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 506, 508, 540, 511, 531, 550, 483, 484, 502, 506, 467, 530, 496, 524, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 501, 518, 518, 499, 513, 515, 522, 517, 528, 521, 557, 509, 500, 466, 527, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 508, 513, 548, 498, 513, 476, 513, 525, 513, 537, 510, 495, 516, 492, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 529, 496, 475, 509, 502, 479, 547, 494, 498, 547, 509, 509, 487, 517, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 509, 543, 550, 481, 528, 515, 516, 485, 519, 521, 540, 493, 529, 515, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 8200, 8222, 8126, 8244, 8188, 8124, 8258] avg=8192.0000 max=8258 max_rank=7 min=8124 min_rank=6 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32814 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[530, 481, 515, 505, 500, 492, 515, 499, 495, 531, 501, 474, 526, 512, 516, 559, 476, 519, 514, 530, 523, 497, 510, 503, 524, 501, 511, 509, 498, 498, 531, 521, 530, 503, 553, 509, 520, 482, 534, 528, 526, 506, 504, 479, 522, 470, 499, 527, 533, 533, 544, 527, 495, 490, 514, 505, 539, 545, 514, 534, 550, 527, 509, 473, 485, 506, 537, 527, 497, 482, 506, 537, 500, 534, 535, 533, 465, 522, 491, 491, 502, 494, 532, 533, 489, 481, 499, 479, 514, 532, 538, 508, 468, 535, 486, 509, 531, 511, 474, 545, 551, 560, 535, 566, 511, 527, 491, 493, 505, 513, 504, 500, 466, 516, 501, 490, 515, 503, 501, 527, 505, 524, 516, 508, 504, 484, 525, 547] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[536, 509, 485, 486, 492, 537, 494, 475, 549, 540, 516, 504, 453, 525, 527, 511, 40, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 466, 497, 518, 497, 508, 542, 520, 515, 537, 481, 557, 511, 514, 494, 496, 552, 13, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 531, 517, 524, 522, 504, 521, 506, 492, 493, 509, 516, 521, 506, 539, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 514, 520, 504, 489, 491, 516, 530, 534, 474, 513, 489, 514, 511, 508, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 527, 517, 505, 512, 522, 506, 508, 541, 515, 569, 548, 485, 517, 513, 519, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 493, 496, 516, 531, 526, 458, 515, 492, 572, 500, 517, 505, 530, 554, 514, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 495, 514, 494, 491, 487, 510, 485, 492, 499, 514, 527, 516, 530, 491, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 503, 524, 503, 539, 467, 514, 513, 501, 509, 483, 493, 491, 519, 497, 562, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8222, 8208, 8141, 8338, 8228, 8081, 8129] avg=8192.0000 max=8338 max_rank=4 min=8081 min_rank=6 imbalance_ratio=1.017822 max_load_reduction=57198 max_load_reduction_pct=0.872772 moved_to_echo=32776 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[523, 515, 491, 521, 501, 536, 491, 496, 522, 493, 497, 492, 545, 493, 508, 504, 39, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 506, 518, 540, 505, 529, 525, 525, 517, 521, 525, 491, 474, 505, 529, 490, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 523, 532, 563, 523, 559, 488, 531, 547, 499, 518, 509, 476, 470, 498, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 506, 508, 540, 511, 531, 550, 483, 484, 502, 506, 467, 530, 496, 524, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 501, 518, 518, 499, 513, 515, 522, 517, 528, 521, 557, 509, 500, 466, 527, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 508, 513, 548, 498, 513, 476, 513, 525, 513, 537, 510, 495, 516, 492, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 529, 496, 475, 509, 502, 479, 547, 494, 498, 547, 509, 509, 487, 517, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 509, 543, 550, 481, 528, 515, 516, 485, 519, 521, 540, 493, 529, 515, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 8200, 8222, 8126, 8244, 8188, 8124, 8258] avg=8192.0000 max=8258 max_rank=7 min=8124 min_rank=6 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32814 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[505, 499, 571, 515, 535, 505, 535, 532, 478, 465, 497, 519, 512, 499, 517, 521, 39, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 505, 553, 503, 515, 516, 506, 492, 494, 520, 504, 481, 518, 503, 513, 532, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 519, 509, 503, 515, 491, 531, 525, 501, 487, 513, 528, 492, 510, 553, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 501, 484, 486, 493, 530, 512, 473, 507, 508, 502, 499, 526, 499, 549, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 489, 508, 487, 475, 490, 510, 541, 527, 502, 516, 526, 506, 495, 503, 460, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 518, 500, 513, 560, 516, 527, 506, 533, 492, 533, 500, 532, 504, 526, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 494, 529, 494, 548, 554, 444, 499, 517, 520, 464, 528, 525, 525, 511, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 545, 553, 507, 548, 485, 515, 497, 528, 512, 527, 525, 523, 504, 490, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8250, 8145, 8271, 8072, 8066, 8271, 8159, 8302] avg=8192.0000 max=8302 max_rank=7 min=8066 min_rank=4 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32798 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[484, 496, 525, 514, 490, 510, 490, 495, 505, 491, 533, 546, 535, 518, 503, 509, 37, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 500, 525, 523, 471, 507, 520, 532, 522, 521, 502, 521, 546, 495, 487, 541, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 498, 509, 473, 517, 531, 501, 494, 511, 513, 453, 501, 538, 506, 516, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 510, 483, 503, 490, 543, 505, 529, 533, 471, 524, 476, 486, 500, 504, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 532, 534, 471, 515, 492, 497, 490, 491, 504, 525, 495, 499, 524, 496, 502, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 546, 510, 488, 500, 560, 510, 501, 544, 563, 519, 509, 563, 530, 481, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 529, 504, 501, 548, 540, 545, 534, 518, 512, 493, 496, 500, 505, 514, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 530, 552, 502, 491, 514, 519, 496, 504, 527, 535, 502, 514, 530, 490, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8240, 8077, 8103, 8100, 8383, 8248, 8198] avg=8192.0000 max=8383 max_rank=5 min=8077 min_rank=2 imbalance_ratio=1.023315 max_load_reduction=57153 max_load_reduction_pct=0.872086 moved_to_echo=32929 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[475, 504, 496, 483, 520, 523, 490, 547, 537, 484, 512, 498, 530, 453, 482, 463, 36, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 506, 488, 553, 528, 475, 539, 499, 477, 526, 528, 518, 531, 520, 548, 503, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 508, 493, 520, 502, 522, 517, 558, 533, 526, 466, 515, 470, 562, 540, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 562, 530, 499, 558, 528, 524, 537, 519, 459, 523, 486, 498, 509, 504, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 540, 496, 521, 539, 519, 502, 504, 518, 486, 541, 537, 503, 546, 510, 477, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 463, 528, 496, 524, 536, 524, 501, 518, 504, 533, 529, 527, 530, 495, 513, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 512, 546, 499, 511, 505, 484, 516, 513, 499, 484, 484, 522, 522, 495, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 523, 528, 502, 480, 520, 526, 517, 527, 491, 480, 509, 496, 488, 530, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8039, 8259, 8272, 8247, 8255, 8210, 8112, 8142] avg=8192.0000 max=8272 max_rank=2 min=8039 min_rank=0 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32719 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[491, 521, 555, 501, 480, 492, 533, 477, 503, 527, 506, 532, 526, 534, 488, 536, 39, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 472, 509, 487, 543, 530, 514, 534, 506, 483, 536, 489, 492, 543, 502, 512, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 507, 529, 534, 523, 501, 504, 473, 514, 512, 515, 508, 491, 528, 513, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 494, 492, 524, 508, 482, 524, 498, 510, 498, 508, 544, 557, 555, 533, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 526, 500, 549, 510, 510, 488, 510, 517, 503, 504, 525, 525, 531, 495, 532, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 506, 517, 491, 464, 525, 509, 498, 525, 516, 524, 509, 494, 494, 505, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 454, 545, 561, 526, 517, 543, 463, 505, 533, 519, 480, 509, 536, 510, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 520, 494, 537, 480, 504, 528, 501, 502, 517, 506, 501, 490, 517, 495, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8246, 8158, 8130, 8295, 8247, 8076, 8256, 8128] avg=8192.0000 max=8295 max_rank=3 min=8076 min_rank=5 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32707 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[517, 521, 517, 524, 538, 521, 518, 453, 483, 540, 528, 527, 501, 544, 486, 492, 38, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 513, 510, 473, 514, 538, 503, 500, 498, 496, 525, 520, 506, 541, 504, 500, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 469, 521, 544, 522, 474, 529, 489, 522, 474, 522, 489, 511, 573, 520, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 539, 568, 545, 506, 533, 516, 510, 541, 484, 473, 514, 503, 518, 495, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 500, 507, 552, 507, 509, 462, 526, 502, 506, 526, 475, 519, 502, 550, 493, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 504, 519, 509, 483, 489, 506, 510, 533, 510, 492, 489, 478, 510, 510, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 522, 561, 531, 529, 576, 466, 493, 514, 529, 510, 529, 503, 495, 497, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 514, 501, 495, 531, 515, 501, 517, 520, 540, 552, 520, 508, 506, 503, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8254, 8139, 8172, 8259, 8124, 8081, 8260, 8247] avg=8192.0000 max=8260 max_rank=6 min=8081 min_rank=5 imbalance_ratio=1.008301 max_load_reduction=57276 max_load_reduction_pct=0.873962 moved_to_echo=32712 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[530, 481, 515, 505, 500, 492, 515, 499, 495, 531, 501, 474, 526, 512, 516, 559, 39, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 519, 514, 530, 523, 497, 510, 503, 524, 501, 511, 509, 498, 498, 531, 521, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 503, 553, 509, 520, 482, 534, 528, 518, 506, 504, 479, 522, 470, 499, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 533, 544, 527, 495, 490, 514, 505, 539, 506, 514, 534, 550, 527, 509, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 506, 537, 527, 497, 482, 506, 537, 500, 534, 535, 533, 465, 522, 491, 491, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 494, 532, 533, 489, 481, 499, 479, 514, 532, 538, 508, 468, 535, 486, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 511, 474, 545, 551, 560, 535, 545, 511, 527, 491, 493, 505, 513, 504, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 466, 516, 501, 490, 515, 503, 501, 527, 505, 524, 516, 508, 504, 484, 525, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8173, 8184, 8293, 8162, 8099, 8296, 8132] avg=8192.0000 max=8296 max_rank=6 min=8099 min_rank=5 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32689 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[475, 504, 496, 483, 520, 523, 490, 547, 537, 484, 512, 498, 530, 453, 482, 463, 36, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 506, 488, 553, 528, 475, 539, 499, 477, 526, 528, 518, 531, 520, 548, 503, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 508, 493, 520, 502, 522, 517, 558, 533, 526, 466, 515, 470, 562, 540, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 562, 530, 499, 558, 528, 524, 537, 519, 459, 523, 486, 498, 509, 504, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 540, 496, 521, 539, 519, 502, 504, 518, 486, 541, 537, 503, 546, 510, 477, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 463, 528, 496, 524, 536, 524, 501, 518, 504, 533, 529, 527, 530, 495, 513, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 512, 546, 499, 511, 505, 484, 516, 513, 499, 484, 484, 522, 522, 495, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 523, 528, 502, 480, 520, 526, 517, 527, 491, 480, 509, 496, 488, 530, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8039, 8259, 8272, 8247, 8255, 8210, 8112, 8142] avg=8192.0000 max=8272 max_rank=2 min=8039 min_rank=0 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32719 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4352, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4352, 4224, 4096, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4352, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4352, 4224, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 3968, 4096, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4352, 4224, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[3968, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4352, 4224, 4096, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[537, 497, 550, 504, 502, 540, 524, 510, 519, 543, 502, 476, 490, 542, 507, 555, 504, 514, 511, 480, 496, 532, 498, 518, 525, 528, 519, 535, 500, 534, 502, 545, 479, 497, 546, 530, 499, 554, 508, 491, 503, 537, 507, 514, 474, 480, 491, 518, 521, 519, 519, 515, 528, 518, 496, 529, 495, 555, 513, 511, 498, 504, 481, 507, 493, 486, 515, 502, 537, 512, 509, 506, 514, 498, 509, 545, 528, 527, 501, 501, 503, 504, 492, 495, 528, 490, 546, 483, 496, 516, 537, 518, 480, 514, 492, 557, 510, 489, 520, 518, 496, 498, 520, 549, 491, 514, 564, 548, 493, 541, 516, 511, 503, 528, 485, 533, 486, 505, 486, 496, 492, 517, 513, 520, 473, 516, 514, 481] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[525, 532, 505, 484, 561, 538, 537, 501, 516, 540, 531, 477, 499, 530, 526, 514, 521, 490, 464, 524, 467, 510, 499, 499, 506, 493, 535, 554, 464, 505, 504, 489, 540, 508, 460, 536, 509, 478, 497, 494, 524, 515, 518, 521, 531, 556, 513, 517, 513, 557, 517, 488, 521, 537, 525, 494, 514, 558, 528, 538, 514, 519, 499, 504, 536, 498, 516, 485, 456, 508, 531, 527, 548, 522, 547, 520, 540, 490, 521, 500, 496, 510, 492, 487, 505, 543, 502, 545, 515, 493, 473, 504, 534, 506, 505, 526, 483, 529, 523, 534, 488, 527, 538, 524, 479, 491, 530, 499, 522, 505, 483, 516, 505, 530, 536, 512, 502, 494, 488, 497, 499, 500, 505, 514, 502, 486, 533, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[512, 495, 499, 502, 510, 500, 508, 499, 533, 563, 513, 519, 502, 520, 490, 508, 508, 517, 473, 506, 483, 544, 504, 518, 478, 558, 510, 491, 498, 511, 499, 552, 497, 522, 511, 495, 523, 509, 522, 507, 511, 452, 504, 512, 547, 488, 546, 509, 526, 507, 535, 515, 496, 544, 539, 523, 517, 523, 520, 544, 517, 508, 475, 524, 499, 504, 511, 473, 480, 552, 539, 501, 537, 490, 554, 521, 536, 471, 512, 498, 477, 519, 476, 513, 531, 499, 533, 496, 504, 545, 512, 517, 511, 510, 501, 537, 530, 499, 500, 515, 487, 522, 528, 522, 527, 482, 514, 533, 512, 506, 519, 497, 493, 512, 494, 516, 506, 510, 504, 519, 524, 539, 515, 536, 530, 498, 498, 499] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[537, 516, 494, 477, 528, 540, 514, 532, 474, 487, 514, 517, 506, 502, 497, 495, 564, 486, 513, 534, 537, 504, 486, 500, 553, 535, 479, 514, 492, 475, 516, 510, 504, 498, 523, 515, 512, 519, 520, 508, 491, 502, 496, 492, 530, 536, 507, 533, 525, 494, 518, 509, 527, 510, 495, 503, 492, 502, 506, 526, 525, 534, 497, 523, 526, 509, 515, 513, 547, 493, 501, 523, 484, 524, 511, 507, 525, 522, 532, 493, 491, 535, 528, 501, 501, 535, 514, 516, 523, 497, 501, 483, 464, 529, 500, 484, 528, 482, 485, 513, 551, 494, 541, 525, 506, 472, 528, 519, 507, 518, 554, 501, 515, 518, 497, 490, 569, 519, 557, 527, 510, 502, 484, 528, 531, 501, 526, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[503, 507, 484, 497, 530, 496, 507, 497, 538, 483, 486, 499, 528, 468, 497, 526, 534, 490, 513, 479, 496, 517, 544, 512, 505, 521, 484, 526, 496, 496, 523, 543, 519, 530, 530, 502, 525, 491, 526, 524, 522, 505, 540, 526, 524, 494, 500, 544, 498, 533, 555, 545, 480, 519, 538, 488, 544, 482, 530, 492, 498, 493, 493, 557, 504, 530, 521, 512, 523, 500, 508, 511, 554, 547, 511, 521, 545, 530, 514, 479, 532, 464, 518, 493, 494, 475, 508, 501, 519, 493, 556, 508, 548, 535, 531, 515, 470, 506, 512, 507, 508, 502, 528, 509, 531, 524, 538, 489, 514, 550, 499, 514, 505, 495, 493, 496, 467, 493, 521, 484, 507, 490, 505, 526, 548, 505, 510, 518] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[538, 586, 523, 485, 511, 480, 524, 534, 508, 540, 540, 474, 496, 473, 499, 502, 484, 501, 525, 528, 543, 532, 473, 523, 490, 529, 516, 506, 484, 532, 541, 521, 555, 499, 529, 534, 522, 502, 522, 541, 514, 516, 468, 513, 555, 543, 463, 481, 483, 526, 506, 523, 535, 500, 479, 511, 520, 514, 530, 508, 530, 518, 511, 518, 524, 522, 552, 497, 490, 507, 509, 503, 500, 491, 500, 554, 536, 507, 501, 524, 498, 482, 525, 541, 525, 515, 450, 488, 507, 540, 529, 520, 541, 515, 557, 486, 521, 498, 495, 510, 508, 500, 495, 527, 519, 510, 486, 485, 515, 481, 527, 448, 491, 514, 535, 544, 490, 530, 520, 515, 493, 504, 518, 510, 518, 482, 518, 483] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[512, 495, 499, 502, 510, 500, 508, 499, 533, 563, 513, 519, 502, 520, 490, 508, 508, 517, 473, 506, 483, 544, 504, 518, 478, 558, 510, 491, 498, 511, 499, 552, 497, 522, 511, 495, 523, 509, 522, 507, 511, 452, 504, 512, 547, 488, 546, 509, 526, 507, 535, 515, 496, 544, 539, 523, 517, 523, 520, 544, 517, 508, 475, 524, 499, 504, 511, 473, 480, 552, 539, 501, 537, 490, 554, 521, 536, 471, 512, 498, 477, 519, 476, 513, 531, 499, 533, 496, 504, 545, 512, 517, 511, 510, 501, 537, 530, 499, 500, 515, 487, 522, 528, 522, 527, 482, 514, 533, 512, 506, 519, 497, 493, 512, 494, 516, 506, 510, 504, 519, 524, 539, 515, 536, 530, 498, 498, 499] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[503, 507, 484, 497, 530, 496, 507, 497, 538, 483, 486, 499, 528, 468, 497, 526, 534, 490, 513, 479, 496, 517, 544, 512, 505, 521, 484, 526, 496, 496, 523, 543, 519, 530, 530, 502, 525, 491, 526, 524, 522, 505, 540, 526, 524, 494, 500, 544, 498, 533, 555, 545, 480, 519, 538, 488, 544, 482, 530, 492, 498, 493, 493, 557, 504, 530, 521, 512, 523, 500, 508, 511, 554, 547, 511, 521, 545, 530, 514, 479, 532, 464, 518, 493, 494, 475, 508, 501, 519, 493, 556, 508, 548, 535, 531, 515, 470, 506, 512, 507, 508, 502, 528, 509, 531, 524, 538, 489, 514, 550, 499, 514, 505, 495, 493, 496, 467, 493, 521, 484, 507, 490, 505, 526, 548, 505, 510, 518] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[514, 489, 500, 492, 510, 519, 538, 504, 489, 504, 509, 517, 503, 513, 492, 552, 503, 506, 511, 496, 542, 525, 495, 537, 514, 510, 489, 534, 466, 511, 527, 509, 520, 517, 503, 501, 504, 545, 532, 506, 470, 492, 507, 496, 476, 470, 493, 541, 533, 481, 494, 520, 466, 488, 538, 511, 477, 522, 488, 489, 484, 497, 542, 551, 523, 534, 537, 536, 489, 499, 529, 502, 508, 507, 511, 509, 475, 528, 513, 522, 533, 536, 504, 501, 485, 545, 525, 511, 569, 538, 513, 523, 527, 549, 469, 512, 476, 487, 502, 509, 529, 487, 521, 515, 542, 548, 500, 483, 488, 539, 521, 526, 500, 512, 501, 549, 528, 516, 503, 547, 526, 536, 531, 504, 542, 503, 509, 520] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[530, 509, 497, 500, 538, 494, 549, 489, 504, 522, 500, 519, 554, 493, 536, 497, 531, 516, 494, 508, 506, 535, 538, 532, 523, 468, 515, 511, 555, 560, 515, 509, 502, 499, 511, 547, 462, 523, 519, 497, 560, 506, 502, 530, 526, 538, 541, 502, 513, 499, 534, 492, 503, 526, 512, 521, 522, 563, 499, 480, 494, 488, 489, 555, 522, 500, 514, 514, 504, 497, 514, 539, 515, 505, 484, 510, 521, 507, 506, 524, 535, 498, 499, 507, 497, 495, 497, 459, 517, 520, 520, 505, 483, 505, 474, 502, 525, 499, 522, 470, 490, 531, 504, 543, 548, 484, 522, 499, 539, 522, 493, 482, 504, 493, 490, 493, 526, 490, 527, 546, 554, 535, 501, 471, 519, 512, 503, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[537, 497, 550, 504, 502, 540, 524, 510, 519, 543, 502, 476, 490, 542, 507, 555, 504, 514, 511, 480, 496, 532, 498, 518, 525, 528, 519, 535, 500, 534, 502, 545, 479, 497, 546, 530, 499, 554, 508, 491, 503, 537, 507, 514, 474, 480, 491, 518, 521, 519, 519, 515, 528, 518, 496, 529, 495, 555, 513, 511, 498, 504, 481, 507, 493, 486, 515, 502, 537, 512, 509, 506, 514, 498, 509, 545, 528, 527, 501, 501, 503, 504, 492, 495, 528, 490, 546, 483, 496, 516, 537, 518, 480, 514, 492, 557, 510, 489, 520, 518, 496, 498, 520, 549, 491, 514, 564, 548, 493, 541, 516, 511, 503, 528, 485, 533, 486, 505, 486, 496, 492, 517, 513, 520, 473, 516, 514, 481] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[525, 532, 505, 484, 561, 538, 530, 501, 516, 540, 531, 477, 499, 530, 526, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 490, 464, 524, 467, 510, 499, 499, 506, 493, 535, 554, 464, 505, 504, 489, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 508, 460, 536, 509, 478, 497, 494, 524, 515, 518, 521, 520, 556, 513, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 557, 517, 488, 521, 537, 525, 494, 514, 558, 528, 538, 514, 519, 499, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 498, 516, 485, 456, 508, 531, 527, 548, 522, 547, 520, 504, 490, 521, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 510, 492, 487, 505, 543, 502, 545, 515, 493, 473, 504, 534, 506, 505, 526, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 529, 523, 534, 488, 527, 538, 524, 479, 491, 530, 499, 522, 505, 483, 516, 12, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 530, 536, 512, 502, 494, 488, 497, 499, 500, 505, 514, 502, 486, 533, 498, 24, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8309, 8033, 8206, 8290, 8209, 8164, 8192, 8133] avg=8192.0000 max=8309 max_rank=0 min=8033 min_rank=1 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32698 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[514, 489, 500, 492, 510, 519, 536, 504, 489, 504, 509, 517, 503, 513, 492, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 506, 511, 496, 542, 525, 495, 537, 514, 510, 489, 534, 466, 511, 527, 509, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 517, 503, 501, 504, 545, 532, 506, 470, 492, 507, 496, 471, 470, 493, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 481, 494, 520, 466, 488, 538, 511, 477, 522, 488, 489, 484, 497, 542, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 534, 537, 536, 489, 499, 529, 502, 508, 507, 511, 509, 451, 528, 513, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 536, 504, 501, 485, 545, 525, 511, 569, 538, 513, 523, 527, 549, 469, 512, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 487, 502, 509, 529, 487, 521, 515, 542, 548, 500, 483, 488, 539, 521, 526, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 512, 501, 549, 528, 516, 503, 547, 526, 536, 531, 504, 542, 503, 509, 520, 18, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8143, 8178, 8068, 8053, 8198, 8366, 8183, 8347] avg=8192.0000 max=8366 max_rank=5 min=8053 min_rank=3 imbalance_ratio=1.021240 max_load_reduction=57170 max_load_reduction_pct=0.872345 moved_to_echo=33094 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[537, 497, 550, 504, 502, 540, 523, 510, 519, 543, 502, 476, 490, 542, 507, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 514, 511, 480, 496, 532, 498, 518, 525, 528, 519, 535, 500, 534, 502, 545, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 497, 546, 530, 499, 554, 508, 491, 503, 537, 507, 514, 469, 480, 491, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 519, 519, 515, 528, 518, 496, 529, 495, 555, 513, 511, 498, 504, 481, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 486, 515, 502, 537, 512, 509, 506, 514, 498, 509, 545, 501, 527, 501, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 504, 492, 495, 528, 490, 546, 483, 496, 516, 537, 518, 480, 514, 492, 557, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 489, 520, 518, 496, 498, 520, 549, 491, 514, 564, 548, 493, 541, 516, 511, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 528, 485, 533, 486, 505, 486, 496, 492, 517, 513, 520, 473, 516, 514, 481, 20, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8297, 8243, 8123, 8183, 8156, 8175, 8289, 8070] avg=8192.0000 max=8297 max_rank=0 min=8070 min_rank=7 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32690 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[537, 516, 494, 477, 528, 540, 513, 532, 474, 487, 514, 517, 506, 502, 497, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 564, 486, 513, 534, 537, 504, 486, 500, 553, 535, 479, 514, 492, 475, 516, 510, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 498, 523, 515, 512, 519, 520, 508, 491, 502, 496, 492, 525, 536, 507, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 494, 518, 509, 527, 510, 495, 503, 492, 502, 506, 526, 525, 534, 497, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 509, 515, 513, 547, 493, 501, 523, 484, 524, 511, 507, 499, 522, 532, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 535, 528, 501, 501, 535, 514, 516, 523, 497, 501, 483, 464, 529, 500, 484, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 482, 485, 513, 551, 494, 541, 525, 506, 472, 528, 519, 507, 518, 554, 501, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 518, 497, 490, 569, 519, 557, 527, 510, 502, 484, 528, 531, 501, 526, 511, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8129, 8200, 8181, 8159, 8199, 8127, 8235, 8306] avg=8192.0000 max=8306 max_rank=7 min=8127 min_rank=5 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32867 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[512, 495, 499, 502, 510, 500, 507, 499, 533, 563, 513, 519, 502, 520, 490, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 517, 473, 506, 483, 544, 504, 518, 478, 558, 510, 491, 498, 511, 499, 552, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 522, 511, 495, 523, 509, 522, 507, 511, 452, 504, 512, 542, 488, 546, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 507, 535, 515, 496, 544, 539, 523, 517, 523, 520, 544, 517, 508, 475, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 504, 511, 473, 480, 552, 539, 501, 537, 490, 554, 521, 509, 471, 512, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 519, 476, 513, 531, 499, 533, 496, 504, 545, 512, 517, 511, 510, 501, 537, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 499, 500, 515, 487, 522, 528, 522, 527, 482, 514, 533, 512, 506, 519, 497, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 512, 494, 516, 506, 510, 504, 519, 524, 539, 515, 536, 530, 498, 498, 499, 20, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8152, 8150, 8286, 8151, 8206, 8204, 8215] avg=8192.0000 max=8286 max_rank=3 min=8150 min_rank=2 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32776 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[503, 507, 484, 497, 530, 496, 506, 497, 538, 483, 486, 499, 528, 468, 497, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 490, 513, 479, 496, 517, 544, 512, 505, 521, 484, 526, 496, 496, 523, 543, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 530, 530, 502, 525, 491, 526, 524, 522, 505, 540, 526, 519, 494, 500, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 533, 555, 545, 480, 519, 538, 488, 544, 482, 530, 492, 498, 493, 493, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 530, 521, 512, 523, 500, 508, 511, 554, 547, 511, 521, 518, 530, 514, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 464, 518, 493, 494, 475, 508, 501, 519, 493, 556, 508, 548, 535, 531, 515, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 506, 512, 507, 508, 502, 528, 509, 531, 524, 538, 489, 514, 550, 499, 514, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 495, 493, 496, 467, 493, 521, 484, 507, 490, 505, 526, 548, 505, 510, 518, 20, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8045, 8181, 8297, 8217, 8283, 8216, 8212, 8085] avg=8192.0000 max=8297 max_rank=2 min=8045 min_rank=0 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32796 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[525, 532, 505, 484, 561, 538, 530, 501, 516, 540, 531, 477, 499, 530, 526, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 490, 464, 524, 467, 510, 499, 499, 506, 493, 535, 554, 464, 505, 504, 489, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 508, 460, 536, 509, 478, 497, 494, 524, 515, 518, 521, 520, 556, 513, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 557, 517, 488, 521, 537, 525, 494, 514, 558, 528, 538, 514, 519, 499, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 498, 516, 485, 456, 508, 531, 527, 548, 522, 547, 520, 504, 490, 521, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 510, 492, 487, 505, 543, 502, 545, 515, 493, 473, 504, 534, 506, 505, 526, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 529, 523, 534, 488, 527, 538, 524, 479, 491, 530, 499, 522, 505, 483, 516, 12, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 530, 536, 512, 502, 494, 488, 497, 499, 500, 505, 514, 502, 486, 533, 498, 24, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8309, 8033, 8206, 8290, 8209, 8164, 8192, 8133] avg=8192.0000 max=8309 max_rank=0 min=8033 min_rank=1 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32698 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[538, 586, 523, 485, 511, 480, 523, 534, 508, 540, 540, 474, 496, 473, 499, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 501, 525, 528, 543, 532, 473, 523, 490, 529, 516, 506, 484, 532, 541, 521, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 499, 529, 534, 522, 502, 522, 541, 514, 516, 468, 513, 549, 543, 463, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 526, 506, 523, 535, 500, 479, 511, 520, 514, 530, 508, 530, 518, 511, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 522, 552, 497, 490, 507, 509, 503, 500, 491, 500, 554, 509, 507, 501, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 482, 525, 541, 525, 515, 450, 488, 507, 540, 529, 520, 541, 515, 557, 486, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 498, 495, 510, 508, 500, 495, 527, 519, 510, 486, 485, 515, 481, 527, 448, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 514, 535, 544, 490, 530, 520, 515, 493, 504, 518, 510, 518, 482, 518, 483, 20, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8230, 8251, 8186, 8190, 8243, 8037, 8187] avg=8192.0000 max=8251 max_rank=2 min=8037 min_rank=6 imbalance_ratio=1.007202 max_load_reduction=57285 max_load_reduction_pct=0.874100 moved_to_echo=32657 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[537, 497, 550, 504, 502, 540, 523, 510, 519, 543, 502, 476, 490, 542, 507, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 514, 511, 480, 496, 532, 498, 518, 525, 528, 519, 535, 500, 534, 502, 545, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 497, 546, 530, 499, 554, 508, 491, 503, 537, 507, 514, 469, 480, 491, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 519, 519, 515, 528, 518, 496, 529, 495, 555, 513, 511, 498, 504, 481, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 486, 515, 502, 537, 512, 509, 506, 514, 498, 509, 545, 501, 527, 501, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 504, 492, 495, 528, 490, 546, 483, 496, 516, 537, 518, 480, 514, 492, 557, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 489, 520, 518, 496, 498, 520, 549, 491, 514, 564, 548, 493, 541, 516, 511, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 528, 485, 533, 486, 505, 486, 496, 492, 517, 513, 520, 473, 516, 514, 481, 20, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8297, 8243, 8123, 8183, 8156, 8175, 8289, 8070] avg=8192.0000 max=8297 max_rank=0 min=8070 min_rank=7 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32690 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[530, 509, 497, 500, 538, 494, 547, 489, 504, 522, 500, 519, 554, 493, 536, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 516, 494, 508, 506, 535, 538, 532, 523, 468, 515, 511, 555, 560, 515, 509, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 499, 511, 547, 462, 523, 519, 497, 560, 506, 502, 530, 521, 538, 541, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 499, 534, 492, 503, 526, 512, 521, 522, 563, 499, 480, 494, 488, 489, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 500, 514, 514, 504, 497, 514, 539, 515, 505, 484, 510, 495, 507, 506, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 498, 499, 507, 497, 495, 497, 459, 517, 520, 520, 505, 483, 505, 474, 502, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 499, 522, 470, 490, 531, 504, 543, 548, 484, 522, 499, 539, 522, 493, 482, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 493, 490, 493, 526, 490, 527, 546, 554, 535, 501, 471, 519, 512, 503, 508, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8229, 8319, 8260, 8162, 8150, 8039, 8184, 8193] avg=8192.0000 max=8319 max_rank=1 min=8039 min_rank=5 imbalance_ratio=1.015503 max_load_reduction=57217 max_load_reduction_pct=0.873062 moved_to_echo=32566 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[512, 495, 499, 502, 510, 500, 507, 499, 533, 563, 513, 519, 502, 520, 490, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 517, 473, 506, 483, 544, 504, 518, 478, 558, 510, 491, 498, 511, 499, 552, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 522, 511, 495, 523, 509, 522, 507, 511, 452, 504, 512, 542, 488, 546, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 507, 535, 515, 496, 544, 539, 523, 517, 523, 520, 544, 517, 508, 475, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 504, 511, 473, 480, 552, 539, 501, 537, 490, 554, 521, 509, 471, 512, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 519, 476, 513, 531, 499, 533, 496, 504, 545, 512, 517, 511, 510, 501, 537, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 499, 500, 515, 487, 522, 528, 522, 527, 482, 514, 533, 512, 506, 519, 497, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 512, 494, 516, 506, 510, 504, 519, 524, 539, 515, 536, 530, 498, 498, 499, 20, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8152, 8150, 8286, 8151, 8206, 8204, 8215] avg=8192.0000 max=8286 max_rank=3 min=8150 min_rank=2 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32776 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[514, 489, 500, 492, 510, 519, 536, 504, 489, 504, 509, 517, 503, 513, 492, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 506, 511, 496, 542, 525, 495, 537, 514, 510, 489, 534, 466, 511, 527, 509, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 517, 503, 501, 504, 545, 532, 506, 470, 492, 507, 496, 471, 470, 493, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 481, 494, 520, 466, 488, 538, 511, 477, 522, 488, 489, 484, 497, 542, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 534, 537, 536, 489, 499, 529, 502, 508, 507, 511, 509, 451, 528, 513, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 536, 504, 501, 485, 545, 525, 511, 569, 538, 513, 523, 527, 549, 469, 512, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 487, 502, 509, 529, 487, 521, 515, 542, 548, 500, 483, 488, 539, 521, 526, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 512, 501, 549, 528, 516, 503, 547, 526, 536, 531, 504, 542, 503, 509, 520, 18, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8143, 8178, 8068, 8053, 8198, 8366, 8183, 8347] avg=8192.0000 max=8366 max_rank=5 min=8053 min_rank=3 imbalance_ratio=1.021240 max_load_reduction=57170 max_load_reduction_pct=0.872345 moved_to_echo=33094 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[503, 507, 484, 497, 530, 496, 506, 497, 538, 483, 486, 499, 528, 468, 497, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 490, 513, 479, 496, 517, 544, 512, 505, 521, 484, 526, 496, 496, 523, 543, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 530, 530, 502, 525, 491, 526, 524, 522, 505, 540, 526, 519, 494, 500, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 533, 555, 545, 480, 519, 538, 488, 544, 482, 530, 492, 498, 493, 493, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 530, 521, 512, 523, 500, 508, 511, 554, 547, 511, 521, 518, 530, 514, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 464, 518, 493, 494, 475, 508, 501, 519, 493, 556, 508, 548, 535, 531, 515, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 506, 512, 507, 508, 502, 528, 509, 531, 524, 538, 489, 514, 550, 499, 514, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 495, 493, 496, 467, 493, 521, 484, 507, 490, 505, 526, 548, 505, 510, 518, 20, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8045, 8181, 8297, 8217, 8283, 8216, 8212, 8085] avg=8192.0000 max=8297 max_rank=2 min=8045 min_rank=0 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32796 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4224, 4096, 3968, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 3968, 4224, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True - [2026-08-26 14:49:27] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 937.3 | throughput per GPU (TFLOP/s/GPU): 192.9 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.700232E+00 | loss scale: 1.0 | grad norm: 69.521 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[499, 509, 493, 501, 560, 508, 505, 478, 560, 542, 527, 522, 513, 532, 535, 517, 498, 491, 520, 530, 513, 551, 535, 516, 512, 513, 548, 523, 518, 485, 511, 509, 529, 517, 495, 501, 479, 513, 488, 529, 545, 470, 557, 493, 476, 504, 564, 499, 565, 553, 480, 457, 509, 543, 512, 509, 492, 509, 507, 524, 485, 552, 472, 484, 550, 497, 497, 514, 508, 505, 517, 486, 526, 518, 486, 481, 549, 502, 511, 498, 487, 550, 512, 545, 509, 485, 483, 499, 501, 524, 486, 493, 550, 490, 497, 533, 533, 498, 488, 531, 509, 487, 500, 489, 502, 513, 531, 531, 502, 510, 515, 549, 503, 497, 509, 542, 535, 496, 504, 486, 546, 528, 517, 523, 474, 501, 509, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True + [2026-08-26 15:25:31] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 946.1 | throughput per GPU (TFLOP/s/GPU): 191.1 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.700154E+00 | loss scale: 1.0 | grad norm: 69.503 | number of skipped iterations: 0 | number of nan iterations: 0 | WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[506, 498, 526, 520, 503, 478, 513, 507, 505, 530, 518, 460, 521, 562, 547, 461, 513, 487, 497, 470, 505, 561, 531, 538, 513, 488, 525, 508, 513, 535, 499, 516, 519, 509, 503, 525, 489, 486, 537, 487, 489, 501, 516, 498, 520, 490, 456, 517, 504, 534, 553, 480, 481, 536, 532, 498, 507, 500, 504, 521, 470, 506, 513, 509, 508, 540, 477, 486, 499, 548, 543, 518, 522, 497, 531, 503, 500, 528, 496, 527, 523, 495, 513, 509, 531, 531, 513, 484, 502, 537, 506, 481, 514, 557, 516, 515, 491, 502, 535, 552, 522, 539, 502, 537, 514, 492, 527, 511, 513, 501, 509, 481, 552, 496, 489, 519, 507, 505, 533, 571, 472, 516, 548, 452, 536, 569, 516, 533] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[504, 545, 512, 533, 507, 487, 485, 496, 534, 501, 519, 484, 523, 494, 520, 524, 492, 515, 518, 482, 540, 481, 527, 511, 504, 528, 519, 539, 480, 509, 508, 487, 508, 528, 511, 449, 499, 522, 478, 520, 536, 555, 529, 523, 492, 482, 541, 535, 484, 540, 523, 521, 530, 535, 492, 509, 483, 526, 539, 511, 480, 507, 496, 527, 532, 535, 485, 476, 504, 486, 497, 517, 501, 492, 534, 578, 527, 494, 533, 533, 503, 487, 497, 475, 506, 516, 476, 528, 491, 510, 494, 478, 529, 530, 486, 543, 538, 541, 527, 534, 508, 527, 525, 492, 494, 521, 518, 494, 484, 519, 497, 479, 488, 513, 529, 578, 484, 477, 542, 526, 504, 558, 513, 523, 521, 546, 511, 533] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[515, 522, 527, 507, 462, 541, 575, 533, 504, 523, 498, 513, 516, 511, 499, 566, 541, 515, 525, 527, 517, 499, 514, 532, 489, 491, 488, 509, 502, 521, 551, 490, 507, 499, 518, 474, 562, 535, 494, 484, 523, 496, 544, 511, 493, 471, 481, 489, 516, 575, 497, 515, 472, 512, 482, 532, 507, 500, 523, 510, 543, 488, 511, 524, 532, 508, 517, 551, 472, 541, 525, 539, 560, 520, 527, 541, 531, 528, 513, 522, 540, 487, 475, 508, 511, 484, 511, 467, 524, 502, 508, 535, 482, 466, 467, 526, 504, 529, 491, 496, 500, 470, 483, 524, 486, 481, 513, 477, 517, 557, 509, 527, 536, 552, 502, 502, 532, 525, 515, 531, 504, 490, 525, 498, 501, 468, 530, 530] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[532, 529, 504, 478, 514, 495, 495, 532, 493, 513, 525, 486, 532, 507, 536, 490, 502, 500, 517, 482, 525, 527, 539, 530, 484, 510, 547, 503, 500, 493, 532, 497, 526, 514, 519, 494, 458, 535, 502, 513, 490, 505, 506, 539, 514, 535, 540, 522, 531, 525, 499, 508, 525, 502, 486, 547, 483, 547, 487, 456, 524, 488, 528, 504, 519, 510, 460, 522, 494, 522, 479, 562, 502, 535, 486, 509, 531, 496, 545, 553, 526, 508, 510, 544, 497, 505, 526, 520, 494, 480, 487, 520, 505, 520, 553, 471, 495, 540, 521, 510, 529, 525, 516, 524, 521, 498, 479, 520, 495, 541, 523, 475, 533, 541, 526, 528, 521, 520, 508, 509, 470, 523, 508, 478, 524, 556, 486, 501] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[568, 468, 501, 550, 512, 485, 489, 531, 496, 530, 519, 491, 507, 514, 491, 489, 464, 561, 523, 481, 503, 541, 488, 504, 501, 524, 511, 518, 534, 499, 522, 528, 501, 531, 477, 505, 511, 513, 532, 512, 501, 509, 507, 508, 502, 519, 497, 550, 500, 514, 507, 501, 553, 501, 542, 499, 531, 490, 503, 510, 502, 537, 561, 507, 506, 486, 492, 567, 523, 506, 533, 532, 512, 574, 490, 525, 508, 474, 500, 487, 499, 519, 510, 485, 507, 484, 506, 544, 505, 480, 512, 515, 494, 527, 502, 504, 504, 540, 555, 506, 543, 525, 505, 458, 523, 546, 514, 469, 543, 536, 503, 479, 504, 497, 533, 485, 551, 510, 508, 522, 492, 515, 481, 523, 509, 509, 553, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[530, 488, 514, 463, 503, 528, 503, 486, 547, 526, 501, 495, 528, 532, 464, 511, 470, 530, 555, 506, 501, 537, 495, 542, 507, 521, 521, 514, 510, 530, 491, 502, 502, 513, 508, 511, 492, 505, 511, 504, 502, 525, 511, 483, 516, 498, 511, 497, 526, 507, 515, 541, 502, 515, 522, 539, 506, 532, 549, 571, 475, 530, 489, 488, 505, 521, 529, 514, 516, 496, 485, 545, 499, 521, 495, 541, 486, 566, 509, 510, 515, 510, 553, 518, 568, 524, 509, 531, 508, 480, 512, 493, 513, 505, 529, 500, 513, 527, 499, 513, 511, 531, 499, 524, 559, 495, 476, 514, 499, 489, 495, 531, 553, 494, 455, 471, 483, 517, 495, 498, 506, 491, 549, 523, 480, 477, 519, 597] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[499, 509, 493, 501, 560, 508, 505, 478, 560, 542, 527, 522, 513, 532, 535, 517, 498, 491, 520, 530, 513, 551, 535, 516, 512, 513, 548, 523, 518, 485, 511, 509, 529, 517, 495, 501, 479, 513, 488, 529, 545, 470, 557, 493, 476, 504, 564, 499, 565, 553, 480, 457, 509, 543, 512, 509, 492, 509, 507, 524, 485, 552, 472, 484, 550, 497, 497, 514, 508, 505, 517, 486, 526, 518, 486, 481, 549, 502, 511, 498, 487, 550, 512, 545, 509, 485, 483, 499, 501, 524, 486, 493, 550, 490, 497, 533, 533, 498, 488, 531, 509, 487, 500, 489, 502, 513, 531, 531, 502, 510, 515, 549, 503, 497, 509, 542, 535, 496, 504, 486, 546, 528, 517, 523, 474, 501, 509, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[532, 529, 504, 478, 514, 495, 495, 532, 493, 513, 525, 486, 532, 507, 536, 490, 502, 500, 517, 482, 525, 527, 539, 530, 484, 510, 547, 503, 500, 493, 532, 497, 526, 514, 519, 494, 458, 535, 502, 513, 490, 505, 506, 539, 514, 535, 540, 522, 531, 525, 499, 508, 525, 502, 486, 547, 483, 547, 487, 456, 524, 488, 528, 504, 519, 510, 460, 522, 494, 522, 479, 562, 502, 535, 486, 509, 531, 496, 545, 553, 526, 508, 510, 544, 497, 505, 526, 520, 494, 480, 487, 520, 505, 520, 553, 471, 495, 540, 521, 510, 529, 525, 516, 524, 521, 498, 479, 520, 495, 541, 523, 475, 533, 541, 526, 528, 521, 520, 508, 509, 470, 523, 508, 478, 524, 556, 486, 501] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[496, 480, 537, 538, 515, 514, 546, 498, 490, 506, 481, 507, 505, 534, 512, 529, 503, 511, 493, 507, 512, 522, 482, 491, 471, 496, 528, 519, 528, 542, 511, 530, 496, 524, 535, 479, 507, 495, 492, 532, 520, 541, 538, 537, 544, 492, 523, 519, 517, 487, 497, 493, 503, 534, 529, 557, 528, 504, 524, 476, 534, 523, 487, 525, 531, 504, 521, 537, 495, 462, 506, 535, 493, 512, 506, 498, 491, 500, 469, 550, 557, 529, 512, 521, 491, 517, 499, 523, 544, 484, 511, 490, 526, 507, 509, 533, 528, 531, 507, 485, 514, 485, 496, 479, 526, 530, 505, 539, 477, 503, 535, 490, 497, 495, 502, 499, 519, 535, 484, 537, 542, 491, 506, 501, 532, 504, 535, 538] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[568, 468, 501, 550, 512, 485, 489, 531, 496, 530, 519, 491, 507, 514, 491, 489, 464, 561, 523, 481, 503, 541, 488, 504, 501, 524, 511, 518, 534, 499, 522, 528, 501, 531, 477, 505, 511, 513, 532, 512, 501, 509, 507, 508, 502, 519, 497, 550, 500, 514, 507, 501, 553, 501, 542, 499, 531, 490, 503, 510, 502, 537, 561, 507, 506, 486, 492, 567, 523, 506, 533, 532, 512, 574, 490, 525, 508, 474, 500, 487, 499, 519, 510, 485, 507, 484, 506, 544, 505, 480, 512, 515, 494, 527, 502, 504, 504, 540, 555, 506, 543, 525, 505, 458, 523, 546, 514, 469, 543, 536, 503, 479, 504, 497, 533, 485, 551, 510, 508, 522, 492, 515, 481, 523, 509, 509, 553, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[504, 545, 512, 533, 507, 487, 485, 496, 534, 501, 519, 484, 523, 494, 520, 524, 492, 515, 518, 482, 540, 481, 527, 511, 504, 528, 519, 539, 480, 509, 508, 487, 508, 528, 511, 449, 499, 522, 478, 520, 536, 555, 529, 523, 492, 482, 541, 535, 484, 540, 523, 521, 530, 535, 492, 509, 483, 526, 539, 511, 480, 507, 496, 527, 532, 535, 485, 476, 504, 486, 497, 517, 501, 492, 534, 578, 527, 494, 533, 533, 503, 487, 497, 475, 506, 516, 476, 528, 491, 510, 494, 478, 529, 530, 486, 543, 538, 541, 527, 534, 508, 527, 525, 492, 494, 521, 518, 494, 484, 519, 497, 479, 488, 513, 529, 578, 484, 477, 542, 526, 504, 558, 513, 523, 521, 546, 511, 533] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[515, 522, 527, 507, 462, 541, 575, 533, 504, 523, 498, 513, 516, 509, 499, 566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 515, 525, 527, 517, 490, 514, 532, 489, 491, 488, 509, 502, 521, 551, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 499, 518, 474, 562, 535, 494, 484, 523, 496, 544, 511, 493, 471, 481, 489, 33, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 555, 497, 515, 472, 512, 482, 532, 507, 500, 523, 510, 543, 488, 511, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 508, 517, 551, 472, 541, 525, 496, 560, 520, 527, 541, 531, 528, 513, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 487, 475, 508, 511, 484, 511, 467, 524, 502, 508, 535, 482, 466, 467, 526, 41, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 529, 491, 496, 500, 470, 483, 524, 486, 481, 513, 477, 517, 557, 509, 527, 9, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 552, 502, 502, 532, 525, 515, 531, 504, 490, 525, 498, 501, 468, 530, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8310, 8202, 8127, 8187, 8384, 8044, 8082, 8200] avg=8192.0000 max=8384 max_rank=4 min=8044 min_rank=5 imbalance_ratio=1.023438 max_load_reduction=57152 max_load_reduction_pct=0.872070 moved_to_echo=32710 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[506, 498, 526, 520, 503, 478, 513, 507, 505, 530, 518, 460, 521, 561, 547, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 487, 497, 470, 505, 554, 531, 538, 513, 488, 525, 508, 513, 535, 499, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 509, 503, 525, 489, 486, 537, 487, 489, 501, 516, 498, 520, 490, 456, 517, 27, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 523, 553, 480, 481, 536, 532, 498, 507, 500, 504, 521, 470, 506, 513, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 540, 477, 486, 499, 548, 543, 486, 522, 497, 531, 503, 500, 528, 496, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 495, 513, 509, 531, 531, 513, 484, 502, 537, 506, 481, 514, 557, 516, 515, 37, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 502, 535, 552, 522, 539, 502, 537, 514, 492, 527, 511, 513, 501, 509, 481, 7, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 496, 489, 519, 507, 505, 533, 571, 472, 516, 548, 452, 536, 569, 516, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8192, 8077, 8137, 8191, 8269, 8239, 8277] avg=8192.0000 max=8277 max_rank=7 min=8077 min_rank=2 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32976 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[499, 509, 493, 501, 560, 508, 505, 478, 560, 542, 527, 522, 513, 531, 535, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 491, 520, 530, 513, 544, 535, 516, 512, 513, 548, 523, 518, 485, 511, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 517, 495, 501, 479, 513, 488, 529, 545, 470, 557, 493, 476, 504, 564, 499, 25, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 565, 541, 480, 457, 509, 543, 512, 509, 492, 509, 507, 524, 485, 552, 472, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 497, 497, 514, 508, 505, 517, 456, 526, 518, 486, 481, 549, 502, 511, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 550, 512, 545, 509, 485, 483, 499, 501, 524, 486, 493, 550, 490, 497, 533, 35, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 498, 488, 531, 509, 487, 500, 489, 502, 513, 531, 531, 502, 510, 515, 549, 7, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 497, 509, 542, 535, 496, 504, 486, 546, 528, 517, 523, 474, 501, 509, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8300, 8266, 8193, 8141, 8115, 8184, 8199, 8138] avg=8192.0000 max=8300 max_rank=0 min=8115 min_rank=4 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32636 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[568, 468, 501, 550, 512, 485, 489, 531, 496, 530, 519, 491, 507, 513, 491, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 464, 561, 523, 481, 503, 534, 488, 504, 501, 524, 511, 518, 534, 499, 522, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 531, 477, 505, 511, 513, 532, 512, 501, 509, 507, 508, 502, 519, 497, 550, 28, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 503, 507, 501, 553, 501, 542, 499, 531, 490, 503, 510, 502, 537, 561, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 486, 492, 567, 523, 506, 533, 499, 512, 574, 490, 525, 508, 474, 500, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 519, 510, 485, 507, 484, 506, 544, 505, 480, 512, 515, 494, 527, 502, 504, 35, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 540, 555, 506, 543, 525, 505, 458, 523, 546, 514, 469, 543, 536, 503, 479, 7, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 497, 533, 485, 551, 510, 508, 522, 492, 515, 481, 523, 509, 509, 553, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8140, 8195, 8211, 8247, 8182, 8133, 8260, 8168] avg=8192.0000 max=8260 max_rank=6 min=8133 min_rank=5 imbalance_ratio=1.008301 max_load_reduction=57276 max_load_reduction_pct=0.873962 moved_to_echo=32743 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[532, 529, 504, 478, 514, 495, 495, 532, 493, 513, 525, 486, 532, 506, 536, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 500, 517, 482, 525, 521, 539, 530, 484, 510, 547, 503, 500, 493, 532, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 514, 519, 494, 458, 535, 502, 513, 490, 505, 506, 539, 514, 535, 540, 522, 29, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 514, 499, 508, 525, 502, 486, 547, 483, 547, 487, 456, 524, 488, 528, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 510, 460, 522, 494, 522, 479, 528, 502, 535, 486, 509, 531, 496, 545, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 508, 510, 544, 497, 505, 526, 520, 494, 480, 487, 520, 505, 520, 553, 471, 35, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 540, 521, 510, 529, 525, 516, 524, 521, 498, 479, 520, 495, 541, 523, 475, 6, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 541, 526, 528, 521, 520, 508, 509, 470, 523, 508, 478, 524, 556, 486, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8160, 8182, 8249, 8129, 8191, 8206, 8222, 8197] avg=8192.0000 max=8249 max_rank=2 min=8129 min_rank=3 imbalance_ratio=1.006958 max_load_reduction=57287 max_load_reduction_pct=0.874130 moved_to_echo=32816 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[504, 545, 512, 533, 507, 487, 485, 496, 534, 501, 519, 484, 523, 493, 520, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 515, 518, 482, 540, 475, 527, 511, 504, 528, 519, 539, 480, 509, 508, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 528, 511, 449, 499, 522, 478, 520, 536, 555, 529, 523, 492, 482, 541, 535, 27, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 529, 523, 521, 530, 535, 492, 509, 483, 526, 539, 511, 480, 507, 496, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 535, 485, 476, 504, 486, 497, 485, 501, 492, 534, 578, 527, 494, 533, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 487, 497, 475, 506, 516, 476, 528, 491, 510, 494, 478, 529, 530, 486, 543, 37, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 541, 527, 534, 508, 527, 525, 492, 494, 521, 518, 494, 484, 519, 497, 479, 6, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 513, 529, 578, 484, 477, 542, 526, 504, 558, 513, 523, 521, 546, 511, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8134, 8243, 8192, 8192, 8091, 8208, 8309] avg=8192.0000 max=8309 max_rank=7 min=8091 min_rank=5 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32800 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[506, 498, 526, 520, 503, 478, 513, 507, 505, 530, 518, 460, 521, 561, 547, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 487, 497, 470, 505, 554, 531, 538, 513, 488, 525, 508, 513, 535, 499, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 509, 503, 525, 489, 486, 537, 487, 489, 501, 516, 498, 520, 490, 456, 517, 27, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 523, 553, 480, 481, 536, 532, 498, 507, 500, 504, 521, 470, 506, 513, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 540, 477, 486, 499, 548, 543, 486, 522, 497, 531, 503, 500, 528, 496, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 495, 513, 509, 531, 531, 513, 484, 502, 537, 506, 481, 514, 557, 516, 515, 37, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 502, 535, 552, 522, 539, 502, 537, 514, 492, 527, 511, 513, 501, 509, 481, 7, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 496, 489, 519, 507, 505, 533, 571, 472, 516, 548, 452, 536, 569, 516, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8192, 8077, 8137, 8191, 8269, 8239, 8277] avg=8192.0000 max=8277 max_rank=7 min=8077 min_rank=2 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32976 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[515, 522, 527, 507, 462, 541, 575, 533, 504, 523, 498, 513, 516, 509, 499, 566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 515, 525, 527, 517, 490, 514, 532, 489, 491, 488, 509, 502, 521, 551, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 499, 518, 474, 562, 535, 494, 484, 523, 496, 544, 511, 493, 471, 481, 489, 33, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 555, 497, 515, 472, 512, 482, 532, 507, 500, 523, 510, 543, 488, 511, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 508, 517, 551, 472, 541, 525, 496, 560, 520, 527, 541, 531, 528, 513, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 487, 475, 508, 511, 484, 511, 467, 524, 502, 508, 535, 482, 466, 467, 526, 41, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 529, 491, 496, 500, 470, 483, 524, 486, 481, 513, 477, 517, 557, 509, 527, 9, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 552, 502, 502, 532, 525, 515, 531, 504, 490, 525, 498, 501, 468, 530, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8310, 8202, 8127, 8187, 8384, 8044, 8082, 8200] avg=8192.0000 max=8384 max_rank=4 min=8044 min_rank=5 imbalance_ratio=1.023438 max_load_reduction=57152 max_load_reduction_pct=0.872070 moved_to_echo=32710 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[496, 480, 537, 538, 515, 514, 546, 498, 490, 506, 481, 507, 505, 533, 512, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 511, 493, 507, 512, 516, 482, 491, 471, 496, 528, 519, 528, 542, 511, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 524, 535, 479, 507, 495, 492, 532, 520, 541, 538, 537, 544, 492, 523, 519, 28, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 476, 497, 493, 503, 534, 529, 557, 528, 504, 524, 476, 534, 523, 487, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 504, 521, 537, 495, 462, 506, 502, 493, 512, 506, 498, 491, 500, 469, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 557, 529, 512, 521, 491, 517, 499, 523, 544, 484, 511, 490, 526, 507, 509, 533, 37, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 531, 507, 485, 514, 485, 496, 479, 526, 530, 505, 539, 477, 503, 535, 490, 6, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 495, 502, 499, 519, 535, 484, 537, 542, 491, 506, 501, 532, 504, 535, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8140, 8310, 8207, 8077, 8295, 8140, 8180] avg=8192.0000 max=8310 max_rank=2 min=8077 min_rank=4 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32692 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[530, 488, 514, 463, 503, 528, 503, 486, 547, 526, 501, 495, 528, 531, 464, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 530, 555, 506, 501, 530, 495, 542, 507, 521, 521, 514, 510, 530, 491, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 513, 508, 511, 492, 505, 511, 504, 502, 525, 511, 483, 516, 498, 511, 497, 29, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 496, 515, 541, 502, 515, 522, 539, 506, 532, 549, 571, 475, 530, 489, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 521, 529, 514, 516, 496, 485, 511, 499, 521, 495, 541, 486, 566, 509, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 510, 553, 518, 568, 524, 509, 531, 508, 480, 512, 493, 513, 505, 529, 500, 41, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 527, 499, 513, 511, 531, 499, 524, 559, 495, 476, 514, 499, 489, 495, 531, 7, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 494, 455, 471, 483, 517, 495, 498, 506, 491, 549, 523, 480, 477, 519, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8118, 8225, 8126, 8296, 8204, 8314, 8186, 8067] avg=8192.0000 max=8314 max_rank=5 min=8067 min_rank=7 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32771 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[568, 468, 501, 550, 512, 485, 489, 531, 496, 530, 519, 491, 507, 513, 491, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 464, 561, 523, 481, 503, 534, 488, 504, 501, 524, 511, 518, 534, 499, 522, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 531, 477, 505, 511, 513, 532, 512, 501, 509, 507, 508, 502, 519, 497, 550, 28, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 503, 507, 501, 553, 501, 542, 499, 531, 490, 503, 510, 502, 537, 561, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 486, 492, 567, 523, 506, 533, 499, 512, 574, 490, 525, 508, 474, 500, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 519, 510, 485, 507, 484, 506, 544, 505, 480, 512, 515, 494, 527, 502, 504, 35, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 540, 555, 506, 543, 525, 505, 458, 523, 546, 514, 469, 543, 536, 503, 479, 7, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 497, 533, 485, 551, 510, 508, 522, 492, 515, 481, 523, 509, 509, 553, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8140, 8195, 8211, 8247, 8182, 8133, 8260, 8168] avg=8192.0000 max=8260 max_rank=6 min=8133 min_rank=5 imbalance_ratio=1.008301 max_load_reduction=57276 max_load_reduction_pct=0.873962 moved_to_echo=32743 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[496, 480, 537, 538, 515, 514, 546, 498, 490, 506, 481, 507, 505, 533, 512, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 511, 493, 507, 512, 516, 482, 491, 471, 496, 528, 519, 528, 542, 511, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 524, 535, 479, 507, 495, 492, 532, 520, 541, 538, 537, 544, 492, 523, 519, 28, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 476, 497, 493, 503, 534, 529, 557, 528, 504, 524, 476, 534, 523, 487, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 504, 521, 537, 495, 462, 506, 502, 493, 512, 506, 498, 491, 500, 469, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 557, 529, 512, 521, 491, 517, 499, 523, 544, 484, 511, 490, 526, 507, 509, 533, 37, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 531, 507, 485, 514, 485, 496, 479, 526, 530, 505, 539, 477, 503, 535, 490, 6, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 495, 502, 499, 519, 535, 484, 537, 542, 491, 506, 501, 532, 504, 535, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8140, 8310, 8207, 8077, 8295, 8140, 8180] avg=8192.0000 max=8310 max_rank=2 min=8077 min_rank=4 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32692 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[504, 545, 512, 533, 507, 487, 485, 496, 534, 501, 519, 484, 523, 493, 520, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 515, 518, 482, 540, 475, 527, 511, 504, 528, 519, 539, 480, 509, 508, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 528, 511, 449, 499, 522, 478, 520, 536, 555, 529, 523, 492, 482, 541, 535, 27, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 529, 523, 521, 530, 535, 492, 509, 483, 526, 539, 511, 480, 507, 496, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 535, 485, 476, 504, 486, 497, 485, 501, 492, 534, 578, 527, 494, 533, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 487, 497, 475, 506, 516, 476, 528, 491, 510, 494, 478, 529, 530, 486, 543, 37, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 541, 527, 534, 508, 527, 525, 492, 494, 521, 518, 494, 484, 519, 497, 479, 6, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 513, 529, 578, 484, 477, 542, 526, 504, 558, 513, 523, 521, 546, 511, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8134, 8243, 8192, 8192, 8091, 8208, 8309] avg=8192.0000 max=8309 max_rank=7 min=8091 min_rank=5 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32800 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4096, 3968, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 3968, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 3968, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4096, 3968, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 3968, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[541, 502, 517, 507, 484, 523, 514, 505, 545, 528, 535, 495, 583, 486, 516, 456, 482, 543, 503, 472, 503, 534, 567, 522, 507, 524, 498, 520, 470, 506, 508, 535, 522, 496, 531, 500, 489, 508, 509, 519, 516, 516, 489, 509, 512, 528, 505, 531, 520, 513, 513, 532, 518, 507, 513, 535, 494, 489, 487, 504, 524, 508, 492, 528, 484, 481, 510, 486, 514, 491, 546, 495, 535, 530, 513, 472, 562, 518, 534, 458, 505, 516, 503, 503, 494, 529, 499, 482, 514, 533, 528, 494, 513, 497, 540, 452, 487, 528, 533, 556, 540, 530, 519, 533, 495, 509, 500, 501, 536, 509, 512, 487, 507, 500, 487, 521, 505, 567, 515, 522, 529, 525, 508, 517, 489, 529, 517, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[490, 510, 540, 510, 519, 521, 517, 517, 435, 526, 498, 524, 509, 475, 544, 518, 482, 497, 486, 486, 469, 549, 557, 534, 503, 506, 490, 493, 516, 496, 525, 488, 505, 485, 542, 495, 546, 530, 552, 522, 528, 527, 519, 509, 481, 539, 517, 460, 515, 527, 498, 516, 523, 497, 508, 521, 545, 533, 544, 499, 529, 518, 518, 477, 573, 509, 589, 520, 507, 513, 482, 529, 551, 490, 525, 528, 508, 496, 520, 522, 508, 491, 481, 497, 529, 474, 504, 545, 519, 571, 514, 537, 478, 533, 530, 491, 481, 504, 508, 470, 493, 479, 517, 504, 476, 496, 491, 495, 520, 514, 525, 538, 553, 517, 517, 517, 524, 518, 543, 498, 544, 463, 519, 478, 494, 500, 487, 534] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[480, 533, 545, 508, 510, 526, 492, 518, 523, 517, 529, 523, 546, 483, 539, 482, 516, 480, 519, 485, 524, 548, 500, 482, 536, 493, 526, 487, 511, 442, 544, 507, 493, 543, 533, 568, 541, 510, 494, 484, 538, 463, 515, 527, 476, 502, 510, 507, 530, 494, 487, 502, 546, 498, 543, 504, 531, 500, 472, 508, 490, 549, 541, 537, 511, 530, 509, 488, 520, 508, 521, 490, 522, 516, 513, 567, 497, 523, 560, 521, 495, 524, 557, 500, 534, 519, 493, 486, 512, 516, 556, 529, 526, 529, 513, 530, 476, 555, 489, 516, 538, 471, 485, 513, 517, 504, 495, 512, 492, 510, 495, 472, 500, 528, 520, 507, 501, 514, 471, 516, 505, 474, 510, 511, 499, 513, 522, 500] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[546, 531, 518, 496, 531, 525, 506, 528, 515, 557, 534, 480, 467, 528, 511, 470, 517, 519, 531, 540, 480, 509, 510, 535, 502, 509, 551, 525, 508, 500, 490, 480, 525, 497, 506, 508, 525, 509, 526, 504, 548, 502, 522, 527, 517, 490, 491, 483, 521, 532, 491, 523, 493, 507, 528, 467, 525, 498, 489, 469, 491, 542, 540, 544, 532, 508, 488, 519, 525, 527, 483, 507, 485, 503, 531, 555, 525, 530, 500, 542, 539, 532, 497, 510, 530, 495, 506, 477, 489, 512, 496, 540, 523, 510, 493, 517, 555, 519, 502, 548, 500, 491, 515, 543, 468, 513, 506, 514, 525, 516, 470, 528, 491, 501, 477, 513, 545, 492, 491, 513, 504, 504, 492, 495, 538, 493, 544, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[500, 474, 534, 498, 520, 517, 498, 532, 489, 545, 514, 536, 499, 479, 512, 505, 494, 522, 521, 474, 509, 500, 506, 479, 544, 518, 523, 491, 498, 529, 525, 497, 532, 521, 502, 522, 532, 533, 537, 508, 476, 497, 517, 529, 512, 517, 536, 511, 496, 531, 556, 513, 484, 519, 509, 501, 499, 509, 486, 515, 505, 541, 511, 470, 493, 523, 486, 511, 491, 501, 533, 523, 511, 537, 517, 506, 527, 515, 522, 502, 533, 541, 556, 570, 528, 515, 545, 507, 518, 528, 491, 503, 539, 476, 524, 495, 510, 453, 507, 509, 503, 515, 490, 543, 514, 475, 516, 543, 469, 508, 521, 534, 487, 563, 499, 480, 530, 498, 520, 518, 518, 499, 497, 498, 530, 472, 507, 534] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[460, 484, 534, 465, 515, 501, 530, 510, 522, 529, 539, 479, 485, 508, 543, 474, 490, 498, 539, 479, 508, 538, 494, 529, 533, 531, 522, 514, 528, 507, 549, 531, 514, 512, 561, 500, 482, 506, 518, 542, 523, 514, 522, 477, 492, 540, 515, 510, 535, 562, 511, 580, 512, 498, 508, 513, 515, 509, 508, 494, 512, 498, 538, 508, 499, 482, 488, 556, 501, 485, 508, 524, 503, 509, 520, 509, 532, 507, 503, 497, 531, 518, 476, 490, 478, 521, 528, 505, 491, 482, 479, 508, 499, 508, 509, 505, 508, 512, 540, 521, 525, 480, 511, 477, 493, 536, 514, 530, 527, 549, 541, 460, 545, 519, 508, 517, 507, 512, 509, 546, 525, 524, 555, 503, 508, 522, 467, 497] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[484, 499, 478, 516, 514, 491, 545, 503, 539, 516, 453, 509, 503, 472, 530, 531, 531, 485, 524, 516, 500, 484, 511, 513, 483, 492, 506, 508, 565, 540, 534, 507, 513, 520, 509, 528, 521, 518, 493, 512, 530, 525, 515, 489, 530, 515, 512, 497, 505, 499, 545, 509, 506, 494, 511, 544, 504, 487, 488, 514, 479, 537, 510, 517, 513, 540, 503, 536, 491, 487, 498, 497, 492, 544, 527, 502, 499, 520, 535, 487, 523, 494, 524, 483, 517, 519, 517, 502, 527, 502, 510, 518, 508, 511, 498, 496, 552, 518, 523, 494, 540, 520, 520, 528, 517, 529, 516, 496, 512, 480, 528, 530, 519, 512, 513, 522, 515, 513, 534, 523, 501, 527, 510, 509, 501, 539, 497, 520] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[521, 526, 517, 543, 499, 513, 521, 509, 488, 498, 517, 518, 513, 525, 506, 510, 520, 514, 515, 511, 523, 520, 526, 506, 525, 516, 509, 523, 495, 501, 568, 503, 542, 477, 557, 537, 484, 516, 495, 489, 480, 492, 518, 473, 495, 507, 513, 504, 562, 566, 512, 514, 550, 487, 529, 495, 510, 482, 538, 473, 490, 513, 501, 535, 543, 505, 500, 523, 555, 493, 542, 504, 507, 522, 500, 539, 529, 478, 512, 493, 497, 508, 519, 503, 534, 509, 527, 506, 486, 489, 494, 529, 542, 547, 522, 516, 468, 520, 502, 523, 507, 532, 560, 492, 541, 534, 493, 506, 471, 474, 525, 542, 543, 504, 462, 494, 451, 540, 507, 449, 532, 470, 508, 532, 532, 497, 519, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[490, 510, 540, 510, 519, 521, 517, 517, 435, 526, 498, 524, 509, 475, 544, 518, 8, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 497, 486, 486, 469, 549, 557, 534, 503, 506, 490, 493, 516, 496, 525, 488, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 485, 530, 495, 546, 530, 552, 522, 528, 527, 519, 509, 481, 539, 517, 460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 509, 498, 516, 523, 497, 508, 521, 545, 533, 544, 499, 529, 518, 518, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 573, 509, 589, 520, 507, 513, 482, 529, 551, 490, 525, 528, 508, 496, 490, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 491, 481, 497, 529, 474, 504, 545, 519, 571, 514, 534, 478, 533, 530, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 504, 508, 470, 493, 479, 517, 504, 476, 496, 491, 495, 520, 514, 525, 538, 12, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 517, 517, 517, 524, 518, 543, 498, 544, 463, 519, 478, 494, 500, 487, 534, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8084, 8245, 8250, 8332, 8199, 8031, 8228] avg=8192.0000 max=8332 max_rank=4 min=8031 min_rank=6 imbalance_ratio=1.017090 max_load_reduction=57204 max_load_reduction_pct=0.872864 moved_to_echo=32790 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[484, 499, 478, 516, 514, 491, 545, 503, 539, 516, 453, 509, 503, 472, 530, 531, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 485, 524, 516, 500, 484, 511, 513, 483, 492, 506, 508, 565, 540, 534, 507, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 520, 498, 528, 521, 518, 493, 512, 530, 525, 515, 489, 530, 515, 512, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 482, 545, 509, 506, 494, 511, 544, 504, 487, 488, 514, 479, 537, 510, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 540, 503, 536, 491, 487, 498, 497, 492, 544, 527, 502, 499, 520, 504, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 494, 524, 483, 517, 519, 517, 502, 527, 502, 510, 515, 508, 511, 498, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 518, 523, 494, 540, 520, 520, 528, 517, 529, 516, 496, 512, 480, 528, 530, 12, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 512, 513, 522, 515, 513, 534, 523, 501, 527, 510, 509, 501, 539, 497, 520, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8095, 8206, 8216, 8132, 8140, 8146, 8323, 8278] avg=8192.0000 max=8323 max_rank=6 min=8095 min_rank=0 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32887 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[541, 502, 517, 507, 484, 523, 514, 505, 545, 528, 535, 495, 583, 486, 516, 456, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 543, 503, 472, 503, 534, 567, 522, 507, 524, 498, 520, 470, 506, 508, 535, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 496, 520, 500, 489, 508, 509, 519, 516, 516, 489, 509, 512, 528, 505, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 496, 513, 532, 518, 507, 513, 535, 494, 489, 487, 504, 524, 508, 492, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 481, 510, 486, 514, 491, 546, 495, 535, 530, 513, 472, 562, 518, 503, 458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 516, 503, 503, 494, 529, 499, 482, 514, 533, 528, 491, 513, 497, 540, 452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 528, 533, 556, 540, 530, 519, 533, 495, 509, 500, 501, 536, 509, 512, 487, 12, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 500, 487, 521, 505, 567, 515, 522, 529, 525, 508, 517, 489, 529, 517, 504, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8249, 8201, 8169, 8160, 8098, 8099, 8295, 8265] avg=8192.0000 max=8295 max_rank=6 min=8098 min_rank=4 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32757 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[500, 474, 534, 498, 520, 517, 498, 532, 489, 545, 514, 536, 499, 479, 512, 505, 494, 522, 521, 474, 509, 500, 506, 479, 544, 518, 523, 491, 498, 529, 525, 497, 532, 521, 502, 522, 532, 533, 537, 508, 476, 497, 517, 529, 512, 517, 536, 511, 496, 531, 556, 513, 484, 519, 509, 501, 499, 509, 486, 515, 505, 541, 511, 470, 493, 523, 486, 511, 491, 501, 533, 523, 511, 537, 517, 506, 527, 515, 522, 502, 533, 541, 556, 570, 528, 515, 545, 507, 518, 528, 491, 503, 539, 476, 524, 495, 510, 453, 507, 509, 503, 515, 490, 543, 514, 475, 516, 543, 469, 508, 521, 534, 487, 563, 499, 480, 530, 498, 520, 518, 518, 499, 497, 498, 530, 472, 507, 534] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[490, 510, 540, 510, 519, 521, 517, 517, 435, 526, 498, 524, 509, 475, 544, 518, 482, 497, 486, 486, 469, 549, 557, 534, 503, 506, 490, 493, 516, 496, 525, 488, 505, 485, 542, 495, 546, 530, 552, 522, 528, 527, 519, 509, 481, 539, 517, 460, 515, 527, 498, 516, 523, 497, 508, 521, 545, 533, 544, 499, 529, 518, 518, 477, 573, 509, 589, 520, 507, 513, 482, 529, 551, 490, 525, 528, 508, 496, 520, 522, 508, 491, 481, 497, 529, 474, 504, 545, 519, 571, 514, 537, 478, 533, 530, 491, 481, 504, 508, 470, 493, 479, 517, 504, 476, 496, 491, 495, 520, 514, 525, 538, 553, 517, 517, 517, 524, 518, 543, 498, 544, 463, 519, 478, 494, 500, 487, 534] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[460, 484, 534, 465, 515, 501, 530, 510, 522, 529, 539, 479, 485, 508, 543, 474, 490, 498, 539, 479, 508, 538, 494, 529, 533, 531, 522, 514, 528, 507, 549, 531, 514, 512, 561, 500, 482, 506, 518, 542, 523, 514, 522, 477, 492, 540, 515, 510, 535, 562, 511, 580, 512, 498, 508, 513, 515, 509, 508, 494, 512, 498, 538, 508, 499, 482, 488, 556, 501, 485, 508, 524, 503, 509, 520, 509, 532, 507, 503, 497, 531, 518, 476, 490, 478, 521, 528, 505, 491, 482, 479, 508, 499, 508, 509, 505, 508, 512, 540, 521, 525, 480, 511, 477, 493, 536, 514, 530, 527, 549, 541, 460, 545, 519, 508, 517, 507, 512, 509, 546, 525, 524, 555, 503, 508, 522, 467, 497] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[480, 533, 545, 508, 510, 526, 492, 518, 523, 517, 529, 523, 546, 483, 539, 482, 516, 480, 519, 485, 524, 548, 500, 482, 536, 493, 526, 487, 511, 442, 544, 507, 493, 543, 533, 568, 541, 510, 494, 484, 538, 463, 515, 527, 476, 502, 510, 507, 530, 494, 487, 502, 546, 498, 543, 504, 531, 500, 472, 508, 490, 549, 541, 537, 511, 530, 509, 488, 520, 508, 521, 490, 522, 516, 513, 567, 497, 523, 560, 521, 495, 524, 557, 500, 534, 519, 493, 486, 512, 516, 556, 529, 526, 529, 513, 530, 476, 555, 489, 516, 538, 471, 485, 513, 517, 504, 495, 512, 492, 510, 495, 472, 500, 528, 520, 507, 501, 514, 471, 516, 505, 474, 510, 511, 499, 513, 522, 500] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[541, 502, 517, 507, 484, 523, 514, 505, 545, 528, 535, 495, 583, 486, 516, 456, 482, 543, 503, 472, 503, 534, 567, 522, 507, 524, 498, 520, 470, 506, 508, 535, 522, 496, 531, 500, 489, 508, 509, 519, 516, 516, 489, 509, 512, 528, 505, 531, 520, 513, 513, 532, 518, 507, 513, 535, 494, 489, 487, 504, 524, 508, 492, 528, 484, 481, 510, 486, 514, 491, 546, 495, 535, 530, 513, 472, 562, 518, 534, 458, 505, 516, 503, 503, 494, 529, 499, 482, 514, 533, 528, 494, 513, 497, 540, 452, 487, 528, 533, 556, 540, 530, 519, 533, 495, 509, 500, 501, 536, 509, 512, 487, 507, 500, 487, 521, 505, 567, 515, 522, 529, 525, 508, 517, 489, 529, 517, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[546, 531, 518, 496, 531, 525, 506, 528, 515, 557, 534, 480, 467, 528, 511, 470, 11, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 519, 531, 540, 480, 509, 510, 535, 502, 509, 551, 525, 508, 500, 490, 480, 9, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 497, 486, 508, 525, 509, 526, 504, 548, 502, 522, 527, 517, 490, 491, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 505, 491, 523, 493, 507, 528, 467, 525, 498, 489, 469, 491, 542, 540, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 508, 488, 519, 525, 527, 483, 507, 485, 503, 531, 555, 525, 530, 464, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 532, 497, 510, 530, 495, 506, 477, 489, 512, 496, 534, 523, 510, 493, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 519, 502, 548, 500, 491, 515, 543, 468, 513, 506, 514, 525, 516, 470, 528, 17, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 501, 477, 513, 545, 492, 491, 513, 504, 504, 492, 495, 538, 493, 544, 515, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8264, 8221, 8160, 8133, 8224, 8160, 8241, 8133] avg=8192.0000 max=8264 max_rank=0 min=8133 min_rank=3 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32758 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[480, 533, 545, 508, 510, 526, 492, 518, 523, 517, 529, 523, 546, 483, 539, 482, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 480, 519, 485, 524, 548, 500, 482, 536, 493, 526, 487, 511, 442, 544, 507, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 543, 522, 568, 541, 510, 494, 484, 538, 463, 515, 527, 476, 502, 510, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 477, 487, 502, 546, 498, 543, 504, 531, 500, 472, 508, 490, 549, 541, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 530, 509, 488, 520, 508, 521, 490, 522, 516, 513, 567, 497, 523, 528, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 524, 557, 500, 534, 519, 493, 486, 512, 516, 556, 526, 526, 529, 513, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 555, 489, 516, 538, 471, 485, 513, 517, 504, 495, 512, 492, 510, 495, 472, 12, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 528, 520, 507, 501, 514, 471, 516, 505, 474, 510, 511, 499, 513, 522, 500, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8266, 8107, 8193, 8215, 8264, 8316, 8060, 8115] avg=8192.0000 max=8316 max_rank=5 min=8060 min_rank=6 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32755 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[500, 474, 534, 498, 520, 517, 498, 532, 489, 545, 514, 536, 499, 479, 512, 505, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 522, 521, 474, 509, 500, 506, 479, 544, 518, 523, 491, 498, 529, 525, 497, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 521, 491, 522, 532, 533, 537, 508, 476, 497, 517, 529, 512, 517, 536, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 512, 556, 513, 484, 519, 509, 501, 499, 509, 486, 515, 505, 541, 511, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 523, 486, 511, 491, 501, 533, 523, 511, 537, 517, 506, 527, 515, 492, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 541, 556, 570, 528, 515, 545, 507, 518, 528, 491, 500, 539, 476, 524, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 453, 507, 509, 503, 515, 490, 543, 514, 475, 516, 543, 469, 508, 521, 534, 13, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 563, 499, 480, 530, 498, 520, 518, 518, 499, 497, 498, 530, 472, 507, 534, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8165, 8137, 8271, 8126, 8168, 8366, 8131, 8172] avg=8192.0000 max=8366 max_rank=5 min=8126 min_rank=3 imbalance_ratio=1.021240 max_load_reduction=57170 max_load_reduction_pct=0.872345 moved_to_echo=32837 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[484, 499, 478, 516, 514, 491, 545, 503, 539, 516, 453, 509, 503, 472, 530, 531, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 485, 524, 516, 500, 484, 511, 513, 483, 492, 506, 508, 565, 540, 534, 507, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 520, 498, 528, 521, 518, 493, 512, 530, 525, 515, 489, 530, 515, 512, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 482, 545, 509, 506, 494, 511, 544, 504, 487, 488, 514, 479, 537, 510, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 540, 503, 536, 491, 487, 498, 497, 492, 544, 527, 502, 499, 520, 504, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 494, 524, 483, 517, 519, 517, 502, 527, 502, 510, 515, 508, 511, 498, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 518, 523, 494, 540, 520, 520, 528, 517, 529, 516, 496, 512, 480, 528, 530, 12, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 512, 513, 522, 515, 513, 534, 523, 501, 527, 510, 509, 501, 539, 497, 520, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8095, 8206, 8216, 8132, 8140, 8146, 8323, 8278] avg=8192.0000 max=8323 max_rank=6 min=8095 min_rank=0 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32887 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[490, 510, 540, 510, 519, 521, 517, 517, 435, 526, 498, 524, 509, 475, 544, 518, 8, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 497, 486, 486, 469, 549, 557, 534, 503, 506, 490, 493, 516, 496, 525, 488, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 485, 530, 495, 546, 530, 552, 522, 528, 527, 519, 509, 481, 539, 517, 460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 509, 498, 516, 523, 497, 508, 521, 545, 533, 544, 499, 529, 518, 518, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 573, 509, 589, 520, 507, 513, 482, 529, 551, 490, 525, 528, 508, 496, 490, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 491, 481, 497, 529, 474, 504, 545, 519, 571, 514, 534, 478, 533, 530, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 504, 508, 470, 493, 479, 517, 504, 476, 496, 491, 495, 520, 514, 525, 538, 12, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 517, 517, 517, 524, 518, 543, 498, 544, 463, 519, 478, 494, 500, 487, 534, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8084, 8245, 8250, 8332, 8199, 8031, 8228] avg=8192.0000 max=8332 max_rank=4 min=8031 min_rank=6 imbalance_ratio=1.017090 max_load_reduction=57204 max_load_reduction_pct=0.872864 moved_to_echo=32790 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[521, 526, 517, 543, 499, 513, 521, 509, 488, 498, 517, 518, 513, 525, 506, 510, 8, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 514, 515, 511, 523, 520, 526, 506, 525, 516, 509, 523, 495, 501, 568, 503, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 477, 545, 537, 484, 516, 495, 489, 480, 492, 518, 473, 495, 507, 513, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 562, 547, 512, 514, 550, 487, 529, 495, 510, 482, 538, 473, 490, 513, 501, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 505, 500, 523, 555, 493, 542, 504, 507, 522, 500, 539, 529, 478, 482, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 508, 519, 503, 534, 509, 527, 506, 486, 489, 494, 526, 542, 547, 522, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 520, 502, 523, 507, 532, 560, 492, 541, 534, 493, 506, 471, 474, 525, 542, 13, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 504, 462, 494, 451, 540, 507, 449, 532, 470, 508, 532, 532, 497, 519, 498, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8282, 8067, 8238, 8215, 8225, 8211, 8060] avg=8192.0000 max=8282 max_rank=1 min=8060 min_rank=7 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32711 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[500, 474, 534, 498, 520, 517, 498, 532, 489, 545, 514, 536, 499, 479, 512, 505, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 522, 521, 474, 509, 500, 506, 479, 544, 518, 523, 491, 498, 529, 525, 497, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 521, 491, 522, 532, 533, 537, 508, 476, 497, 517, 529, 512, 517, 536, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 512, 556, 513, 484, 519, 509, 501, 499, 509, 486, 515, 505, 541, 511, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 523, 486, 511, 491, 501, 533, 523, 511, 537, 517, 506, 527, 515, 492, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 541, 556, 570, 528, 515, 545, 507, 518, 528, 491, 500, 539, 476, 524, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 453, 507, 509, 503, 515, 490, 543, 514, 475, 516, 543, 469, 508, 521, 534, 13, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 563, 499, 480, 530, 498, 520, 518, 518, 499, 497, 498, 530, 472, 507, 534, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8165, 8137, 8271, 8126, 8168, 8366, 8131, 8172] avg=8192.0000 max=8366 max_rank=5 min=8126 min_rank=3 imbalance_ratio=1.021240 max_load_reduction=57170 max_load_reduction_pct=0.872345 moved_to_echo=32837 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[480, 533, 545, 508, 510, 526, 492, 518, 523, 517, 529, 523, 546, 483, 539, 482, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 480, 519, 485, 524, 548, 500, 482, 536, 493, 526, 487, 511, 442, 544, 507, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 543, 522, 568, 541, 510, 494, 484, 538, 463, 515, 527, 476, 502, 510, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 477, 487, 502, 546, 498, 543, 504, 531, 500, 472, 508, 490, 549, 541, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 530, 509, 488, 520, 508, 521, 490, 522, 516, 513, 567, 497, 523, 528, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 524, 557, 500, 534, 519, 493, 486, 512, 516, 556, 526, 526, 529, 513, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 555, 489, 516, 538, 471, 485, 513, 517, 504, 495, 512, 492, 510, 495, 472, 12, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 528, 520, 507, 501, 514, 471, 516, 505, 474, 510, 511, 499, 513, 522, 500, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8266, 8107, 8193, 8215, 8264, 8316, 8060, 8115] avg=8192.0000 max=8316 max_rank=5 min=8060 min_rank=6 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32755 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[460, 484, 534, 465, 515, 501, 530, 510, 522, 529, 539, 479, 485, 508, 543, 474, 8, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 498, 539, 479, 508, 538, 494, 529, 533, 531, 522, 514, 528, 507, 549, 531, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 512, 548, 500, 482, 506, 518, 542, 523, 514, 522, 477, 492, 540, 515, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 543, 511, 580, 512, 498, 508, 513, 515, 509, 508, 494, 512, 498, 538, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 482, 488, 556, 501, 485, 508, 524, 503, 509, 520, 509, 532, 507, 475, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 518, 476, 490, 478, 521, 528, 505, 491, 482, 479, 505, 499, 508, 509, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 512, 540, 521, 525, 480, 511, 477, 493, 536, 514, 530, 527, 549, 541, 460, 13, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 519, 508, 517, 507, 512, 509, 546, 525, 524, 555, 503, 508, 522, 467, 497, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8092, 8298, 8215, 8282, 8095, 8025, 8244, 8285] avg=8192.0000 max=8298 max_rank=1 min=8025 min_rank=5 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32649 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[541, 502, 517, 507, 484, 523, 514, 505, 545, 528, 535, 495, 583, 486, 516, 456, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 543, 503, 472, 503, 534, 567, 522, 507, 524, 498, 520, 470, 506, 508, 535, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 496, 520, 500, 489, 508, 509, 519, 516, 516, 489, 509, 512, 528, 505, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 496, 513, 532, 518, 507, 513, 535, 494, 489, 487, 504, 524, 508, 492, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 481, 510, 486, 514, 491, 546, 495, 535, 530, 513, 472, 562, 518, 503, 458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 516, 503, 503, 494, 529, 499, 482, 514, 533, 528, 491, 513, 497, 540, 452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 528, 533, 556, 540, 530, 519, 533, 495, 509, 500, 501, 536, 509, 512, 487, 12, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 500, 487, 521, 505, 567, 515, 522, 529, 525, 508, 517, 489, 529, 517, 504, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8249, 8201, 8169, 8160, 8098, 8099, 8295, 8265] avg=8192.0000 max=8295 max_rank=6 min=8098 min_rank=4 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32757 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 3968, 4224, 3968, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 3968, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 3968, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4352, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 3968, 4224, 3968, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 3968, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4352, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 3968, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[516, 517, 509, 488, 490, 507, 523, 502, 505, 548, 479, 530, 518, 516, 526, 510, 518, 513, 560, 536, 489, 492, 498, 485, 507, 499, 527, 503, 491, 512, 494, 496, 454, 527, 544, 492, 511, 498, 528, 504, 513, 490, 498, 492, 504, 516, 491, 527, 539, 529, 530, 523, 520, 519, 472, 532, 473, 521, 536, 517, 504, 470, 525, 496, 496, 569, 558, 531, 542, 513, 494, 513, 526, 532, 522, 529, 512, 484, 474, 511, 552, 467, 533, 520, 518, 428, 531, 521, 515, 506, 476, 470, 500, 505, 494, 523, 499, 506, 537, 534, 508, 580, 478, 507, 519, 497, 524, 480, 526, 539, 490, 503, 538, 509, 511, 461, 542, 563, 535, 495, 524, 493, 528, 494, 557, 527, 528, 540] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[537, 489, 544, 461, 576, 509, 541, 537, 507, 541, 525, 516, 495, 490, 504, 510, 504, 518, 537, 531, 551, 498, 517, 533, 511, 511, 512, 490, 524, 504, 541, 489, 562, 527, 513, 508, 499, 514, 518, 506, 525, 504, 512, 535, 506, 483, 522, 536, 511, 512, 512, 523, 541, 512, 534, 553, 475, 505, 541, 544, 524, 478, 523, 506, 521, 534, 534, 511, 525, 498, 497, 476, 551, 482, 544, 488, 502, 474, 533, 475, 467, 519, 475, 479, 511, 543, 507, 480, 501, 488, 465, 528, 541, 509, 493, 509, 478, 530, 526, 526, 518, 522, 496, 502, 565, 524, 475, 481, 473, 525, 467, 532, 522, 518, 465, 521, 523, 534, 502, 516, 529, 512, 474, 503, 500, 536, 491, 473] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[495, 502, 517, 549, 521, 525, 454, 481, 510, 520, 528, 486, 532, 490, 499, 527, 552, 526, 558, 517, 533, 538, 531, 491, 523, 492, 542, 524, 491, 478, 503, 507, 509, 521, 527, 494, 506, 500, 548, 528, 520, 494, 502, 521, 495, 495, 502, 528, 511, 494, 485, 518, 510, 563, 518, 505, 505, 503, 545, 507, 515, 495, 515, 455, 503, 522, 491, 529, 512, 466, 498, 533, 508, 534, 558, 474, 522, 541, 517, 545, 569, 489, 486, 493, 501, 562, 473, 513, 498, 488, 513, 480, 535, 524, 547, 496, 528, 517, 495, 504, 517, 510, 544, 528, 528, 507, 544, 508, 477, 523, 544, 504, 511, 537, 531, 513, 514, 482, 509, 481, 491, 515, 452, 495, 492, 524, 520, 495] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[497, 468, 534, 534, 558, 511, 522, 502, 510, 531, 494, 520, 465, 510, 518, 551, 514, 472, 488, 510, 535, 487, 532, 529, 509, 539, 528, 554, 499, 525, 525, 514, 524, 516, 517, 521, 499, 478, 504, 494, 492, 516, 537, 536, 507, 519, 499, 500, 484, 533, 533, 471, 519, 511, 501, 495, 562, 493, 501, 535, 500, 497, 515, 511, 487, 561, 516, 509, 524, 507, 510, 514, 476, 524, 552, 557, 522, 509, 515, 549, 506, 499, 488, 503, 510, 504, 501, 504, 544, 522, 506, 482, 521, 528, 530, 545, 498, 512, 525, 548, 537, 539, 507, 509, 463, 478, 540, 463, 539, 517, 497, 518, 488, 474, 466, 551, 529, 532, 513, 493, 520, 486, 475, 511, 504, 473, 507, 494] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[489, 526, 510, 515, 514, 543, 511, 500, 531, 485, 497, 533, 478, 504, 504, 512, 536, 528, 476, 530, 455, 496, 498, 503, 516, 521, 566, 526, 470, 498, 503, 504, 462, 541, 524, 504, 503, 513, 517, 511, 502, 499, 507, 512, 507, 513, 473, 492, 518, 529, 505, 524, 534, 548, 561, 511, 512, 514, 522, 512, 542, 505, 514, 485, 514, 550, 468, 521, 517, 448, 502, 518, 496, 500, 539, 485, 541, 528, 520, 477, 533, 474, 523, 506, 513, 521, 550, 486, 502, 518, 522, 532, 532, 486, 545, 467, 499, 493, 473, 491, 527, 502, 551, 512, 512, 503, 492, 522, 478, 515, 560, 551, 544, 492, 487, 515, 487, 505, 516, 529, 534, 508, 554, 514, 529, 550, 549, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[482, 519, 487, 530, 498, 477, 522, 535, 524, 521, 522, 529, 537, 458, 509, 485, 521, 510, 473, 516, 524, 513, 517, 505, 531, 502, 490, 520, 527, 531, 518, 475, 525, 522, 532, 528, 499, 527, 443, 482, 548, 516, 535, 550, 525, 519, 491, 481, 451, 494, 502, 521, 513, 539, 493, 492, 513, 509, 539, 519, 504, 533, 525, 498, 556, 522, 510, 494, 547, 515, 507, 532, 484, 488, 475, 512, 532, 500, 543, 494, 470, 506, 493, 475, 521, 526, 506, 463, 490, 474, 540, 474, 538, 524, 536, 496, 495, 522, 532, 465, 463, 496, 542, 495, 534, 508, 531, 544, 527, 489, 514, 535, 563, 512, 566, 497, 519, 519, 521, 499, 486, 551, 512, 537, 548, 572, 537, 486] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[492, 538, 485, 498, 491, 512, 502, 493, 508, 479, 469, 466, 488, 485, 513, 521, 504, 519, 502, 520, 525, 519, 512, 489, 500, 489, 519, 522, 493, 492, 495, 519, 535, 536, 515, 511, 520, 548, 522, 528, 514, 493, 514, 520, 525, 523, 519, 499, 503, 526, 510, 475, 534, 534, 498, 551, 527, 480, 526, 526, 507, 495, 541, 510, 494, 528, 559, 519, 498, 496, 491, 521, 505, 502, 533, 519, 526, 501, 533, 493, 530, 488, 515, 524, 518, 531, 500, 497, 546, 514, 517, 514, 509, 511, 488, 457, 529, 504, 518, 526, 516, 542, 508, 525, 533, 554, 528, 481, 529, 471, 528, 523, 508, 501, 527, 549, 547, 515, 504, 502, 519, 500, 491, 539, 509, 498, 494, 517] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[497, 468, 534, 534, 558, 511, 522, 502, 510, 531, 494, 520, 465, 510, 518, 551, 514, 472, 488, 510, 535, 487, 532, 529, 509, 539, 528, 554, 499, 525, 525, 514, 524, 516, 517, 521, 499, 478, 504, 494, 492, 516, 537, 536, 507, 519, 499, 500, 484, 533, 533, 471, 519, 511, 501, 495, 562, 493, 501, 535, 500, 497, 515, 511, 487, 561, 516, 509, 524, 507, 510, 514, 476, 524, 552, 557, 522, 509, 515, 549, 506, 499, 488, 503, 510, 504, 501, 504, 544, 522, 506, 482, 521, 528, 530, 545, 498, 512, 525, 548, 537, 539, 507, 509, 463, 478, 540, 463, 539, 517, 497, 518, 488, 474, 466, 551, 529, 532, 513, 493, 520, 486, 475, 511, 504, 473, 507, 494] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[495, 502, 517, 549, 521, 525, 454, 481, 510, 520, 528, 486, 532, 490, 499, 527, 552, 526, 558, 517, 533, 538, 531, 491, 523, 492, 542, 524, 491, 478, 503, 507, 509, 521, 527, 494, 506, 500, 548, 528, 520, 494, 502, 521, 495, 495, 502, 528, 511, 494, 485, 518, 510, 563, 518, 505, 505, 503, 545, 507, 515, 495, 515, 455, 503, 522, 491, 529, 512, 466, 498, 533, 508, 534, 558, 474, 522, 541, 517, 545, 569, 489, 486, 493, 501, 562, 473, 513, 498, 488, 513, 480, 535, 524, 547, 496, 528, 517, 495, 504, 517, 510, 544, 528, 528, 507, 544, 508, 477, 523, 544, 504, 511, 537, 531, 513, 514, 482, 509, 481, 491, 515, 452, 495, 492, 524, 520, 495] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[497, 534, 522, 503, 504, 534, 504, 547, 535, 518, 500, 506, 537, 520, 514, 492, 503, 526, 496, 481, 492, 540, 513, 502, 533, 514, 516, 476, 506, 526, 532, 482, 510, 495, 532, 526, 554, 515, 482, 540, 495, 483, 487, 503, 536, 496, 510, 535, 520, 518, 493, 541, 499, 504, 500, 555, 532, 524, 548, 516, 487, 518, 517, 525, 515, 495, 517, 504, 468, 546, 543, 507, 470, 521, 523, 579, 471, 505, 527, 516, 493, 506, 489, 510, 500, 502, 484, 478, 501, 519, 506, 528, 492, 488, 490, 493, 501, 550, 546, 510, 529, 482, 523, 486, 523, 533, 479, 504, 476, 466, 512, 522, 531, 545, 529, 546, 499, 501, 492, 486, 504, 496, 542, 518, 509, 535, 533, 541] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[537, 489, 544, 461, 576, 509, 541, 537, 507, 541, 525, 516, 495, 490, 504, 510, 504, 518, 537, 531, 551, 498, 517, 533, 511, 511, 512, 490, 524, 504, 541, 489, 562, 527, 513, 508, 499, 514, 518, 506, 525, 504, 512, 535, 506, 483, 522, 536, 511, 512, 512, 523, 541, 512, 534, 553, 475, 505, 541, 544, 524, 478, 523, 506, 521, 534, 534, 511, 525, 498, 497, 476, 551, 482, 544, 488, 502, 474, 533, 475, 467, 519, 475, 479, 511, 543, 507, 480, 501, 488, 465, 528, 541, 509, 493, 509, 478, 530, 526, 526, 518, 522, 496, 502, 565, 524, 475, 481, 473, 525, 467, 532, 522, 518, 465, 521, 523, 534, 502, 516, 529, 512, 474, 503, 500, 536, 491, 473] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[489, 526, 510, 515, 514, 543, 511, 500, 531, 485, 497, 533, 478, 504, 504, 512, 14, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 528, 476, 530, 455, 496, 498, 503, 516, 521, 566, 526, 470, 498, 503, 504, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 541, 524, 504, 503, 513, 517, 511, 502, 499, 507, 512, 507, 513, 473, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 529, 505, 524, 534, 548, 561, 511, 512, 514, 487, 512, 542, 505, 514, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 518, 468, 521, 517, 448, 502, 518, 496, 500, 539, 485, 541, 528, 520, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 474, 523, 506, 513, 521, 550, 486, 502, 518, 522, 532, 532, 486, 545, 467, 35, 37, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 493, 473, 491, 527, 502, 551, 512, 512, 503, 492, 522, 478, 515, 560, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 492, 487, 515, 487, 505, 516, 529, 534, 508, 554, 514, 529, 513, 549, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8129, 8080, 8301, 8092, 8300, 8165, 8290] avg=8192.0000 max=8301 max_rank=3 min=8080 min_rank=2 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32847 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[516, 517, 509, 488, 490, 507, 523, 502, 505, 548, 479, 530, 518, 516, 526, 510, 15, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 513, 560, 536, 489, 492, 498, 485, 507, 499, 527, 503, 491, 512, 494, 496, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 454, 527, 544, 492, 511, 498, 528, 504, 513, 490, 498, 492, 504, 516, 491, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 529, 530, 523, 520, 519, 472, 532, 473, 521, 500, 517, 504, 470, 525, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 535, 558, 531, 542, 513, 494, 513, 526, 532, 522, 529, 512, 484, 474, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 467, 533, 520, 518, 428, 531, 521, 515, 506, 476, 470, 500, 505, 494, 523, 36, 35, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 506, 537, 534, 508, 580, 478, 507, 519, 497, 524, 480, 526, 539, 490, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 509, 511, 461, 542, 563, 535, 495, 524, 493, 528, 494, 557, 492, 528, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8211, 8122, 8089, 8170, 8272, 8149, 8213, 8310] avg=8192.0000 max=8310 max_rank=7 min=8089 min_rank=2 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32944 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[482, 519, 487, 530, 498, 477, 522, 535, 524, 521, 522, 529, 537, 458, 509, 485, 13, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 510, 473, 516, 524, 513, 517, 505, 531, 502, 490, 520, 527, 531, 518, 475, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 522, 532, 528, 499, 527, 443, 482, 548, 516, 535, 550, 525, 519, 491, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 451, 494, 502, 521, 513, 539, 493, 492, 513, 509, 503, 519, 504, 533, 525, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 492, 510, 494, 547, 515, 507, 532, 484, 488, 475, 512, 532, 500, 543, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 506, 493, 475, 521, 526, 506, 463, 490, 474, 540, 474, 538, 524, 536, 496, 36, 38, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 522, 532, 465, 463, 496, 542, 495, 534, 508, 531, 544, 527, 489, 514, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 563, 512, 566, 497, 519, 519, 521, 499, 486, 551, 512, 537, 548, 534, 537, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8175, 8223, 8109, 8181, 8123, 8177, 8387] avg=8192.0000 max=8387 max_rank=7 min=8109 min_rank=3 imbalance_ratio=1.023804 max_load_reduction=57149 max_load_reduction_pct=0.872025 moved_to_echo=32868 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[516, 517, 509, 488, 490, 507, 523, 502, 505, 548, 479, 530, 518, 516, 526, 510, 518, 513, 560, 536, 489, 492, 498, 485, 507, 499, 527, 503, 491, 512, 494, 496, 454, 527, 544, 492, 511, 498, 528, 504, 513, 490, 498, 492, 504, 516, 491, 527, 539, 529, 530, 523, 520, 519, 472, 532, 473, 521, 536, 517, 504, 470, 525, 496, 496, 569, 558, 531, 542, 513, 494, 513, 526, 532, 522, 529, 512, 484, 474, 511, 552, 467, 533, 520, 518, 428, 531, 521, 515, 506, 476, 470, 500, 505, 494, 523, 499, 506, 537, 534, 508, 580, 478, 507, 519, 497, 524, 480, 526, 539, 490, 503, 538, 509, 511, 461, 542, 563, 535, 495, 524, 493, 528, 494, 557, 527, 528, 540] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[482, 519, 487, 530, 498, 477, 522, 535, 524, 521, 522, 529, 537, 458, 509, 485, 521, 510, 473, 516, 524, 513, 517, 505, 531, 502, 490, 520, 527, 531, 518, 475, 525, 522, 532, 528, 499, 527, 443, 482, 548, 516, 535, 550, 525, 519, 491, 481, 451, 494, 502, 521, 513, 539, 493, 492, 513, 509, 539, 519, 504, 533, 525, 498, 556, 522, 510, 494, 547, 515, 507, 532, 484, 488, 475, 512, 532, 500, 543, 494, 470, 506, 493, 475, 521, 526, 506, 463, 490, 474, 540, 474, 538, 524, 536, 496, 495, 522, 532, 465, 463, 496, 542, 495, 534, 508, 531, 544, 527, 489, 514, 535, 563, 512, 566, 497, 519, 519, 521, 499, 486, 551, 512, 537, 548, 572, 537, 486] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[497, 468, 534, 534, 558, 511, 522, 502, 510, 531, 494, 520, 465, 510, 518, 551, 18, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 472, 488, 510, 535, 487, 532, 529, 509, 539, 528, 554, 499, 525, 525, 514, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 516, 517, 521, 499, 478, 504, 494, 492, 516, 537, 536, 507, 519, 499, 500, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 533, 533, 471, 519, 511, 501, 495, 562, 493, 463, 535, 500, 497, 515, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 519, 516, 509, 524, 507, 510, 514, 476, 524, 552, 557, 522, 509, 515, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 499, 488, 503, 510, 504, 501, 504, 544, 522, 506, 482, 521, 528, 530, 545, 38, 35, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 512, 525, 548, 537, 539, 507, 509, 463, 478, 540, 463, 539, 517, 497, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 474, 466, 551, 529, 532, 513, 493, 520, 486, 475, 511, 504, 438, 507, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8259, 8268, 8163, 8123, 8290, 8290, 8162, 7981] avg=8192.0000 max=8290 max_rank=4 min=7981 min_rank=7 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32723 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[492, 538, 485, 498, 491, 512, 502, 493, 508, 479, 469, 466, 488, 485, 513, 521, 13, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 519, 502, 520, 525, 519, 512, 489, 500, 489, 519, 522, 493, 492, 495, 519, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 536, 515, 511, 520, 548, 522, 528, 514, 493, 514, 520, 525, 523, 519, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 526, 510, 475, 534, 534, 498, 551, 527, 480, 491, 526, 507, 495, 541, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 497, 559, 519, 498, 496, 491, 521, 505, 502, 533, 519, 526, 501, 533, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 488, 515, 524, 518, 531, 500, 497, 546, 514, 517, 514, 509, 511, 488, 457, 35, 33, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 504, 518, 526, 516, 542, 508, 525, 533, 554, 528, 481, 529, 471, 528, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 501, 527, 549, 547, 515, 504, 502, 519, 500, 491, 539, 509, 465, 494, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7965, 8121, 8322, 8208, 8187, 8245, 8301, 8187] avg=8192.0000 max=8322 max_rank=2 min=7965 min_rank=0 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32920 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[516, 517, 509, 488, 490, 507, 523, 502, 505, 548, 479, 530, 518, 516, 526, 510, 15, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 513, 560, 536, 489, 492, 498, 485, 507, 499, 527, 503, 491, 512, 494, 496, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 454, 527, 544, 492, 511, 498, 528, 504, 513, 490, 498, 492, 504, 516, 491, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 529, 530, 523, 520, 519, 472, 532, 473, 521, 500, 517, 504, 470, 525, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 535, 558, 531, 542, 513, 494, 513, 526, 532, 522, 529, 512, 484, 474, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 467, 533, 520, 518, 428, 531, 521, 515, 506, 476, 470, 500, 505, 494, 523, 36, 35, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 506, 537, 534, 508, 580, 478, 507, 519, 497, 524, 480, 526, 539, 490, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 509, 511, 461, 542, 563, 535, 495, 524, 493, 528, 494, 557, 492, 528, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8211, 8122, 8089, 8170, 8272, 8149, 8213, 8310] avg=8192.0000 max=8310 max_rank=7 min=8089 min_rank=2 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32944 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[537, 489, 544, 461, 576, 509, 541, 537, 507, 541, 525, 516, 495, 490, 504, 510, 14, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 518, 537, 531, 551, 498, 517, 533, 511, 511, 512, 490, 524, 504, 541, 489, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 562, 527, 513, 508, 499, 514, 518, 506, 525, 504, 512, 535, 506, 483, 522, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 512, 512, 523, 541, 512, 534, 553, 475, 505, 505, 544, 524, 478, 523, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 502, 534, 511, 525, 498, 497, 476, 551, 482, 544, 488, 502, 474, 533, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 519, 475, 479, 511, 543, 507, 480, 501, 488, 465, 528, 541, 509, 493, 509, 36, 36, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 530, 526, 526, 518, 522, 496, 502, 565, 524, 475, 481, 473, 525, 467, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 518, 465, 521, 523, 534, 502, 516, 529, 512, 474, 503, 500, 500, 491, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8308, 8273, 8270, 8258, 8113, 8105, 8126, 8083] avg=8192.0000 max=8308 max_rank=0 min=8083 min_rank=7 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32427 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[495, 502, 517, 549, 521, 525, 454, 481, 510, 520, 528, 486, 532, 490, 499, 527, 13, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 526, 558, 517, 533, 538, 531, 491, 523, 492, 542, 524, 491, 478, 503, 507, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 521, 527, 494, 506, 500, 548, 528, 520, 494, 502, 521, 495, 495, 502, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 494, 485, 518, 510, 563, 518, 505, 505, 503, 508, 507, 515, 495, 515, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 492, 491, 529, 512, 466, 498, 533, 508, 534, 558, 474, 522, 541, 517, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 569, 489, 486, 493, 501, 562, 473, 513, 498, 488, 513, 480, 535, 524, 547, 496, 37, 35, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 517, 495, 504, 517, 510, 544, 528, 528, 507, 544, 508, 477, 523, 544, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 537, 531, 513, 514, 482, 509, 481, 491, 515, 452, 495, 492, 489, 520, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8308, 8190, 8107, 8223, 8256, 8264, 8027] avg=8192.0000 max=8308 max_rank=1 min=8027 min_rank=7 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32770 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[489, 526, 510, 515, 514, 543, 511, 500, 531, 485, 497, 533, 478, 504, 504, 512, 14, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 528, 476, 530, 455, 496, 498, 503, 516, 521, 566, 526, 470, 498, 503, 504, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 541, 524, 504, 503, 513, 517, 511, 502, 499, 507, 512, 507, 513, 473, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 529, 505, 524, 534, 548, 561, 511, 512, 514, 487, 512, 542, 505, 514, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 518, 468, 521, 517, 448, 502, 518, 496, 500, 539, 485, 541, 528, 520, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 474, 523, 506, 513, 521, 550, 486, 502, 518, 522, 532, 532, 486, 545, 467, 35, 37, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 493, 473, 491, 527, 502, 551, 512, 512, 503, 492, 522, 478, 515, 560, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 492, 487, 515, 487, 505, 516, 529, 534, 508, 554, 514, 529, 513, 549, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8129, 8080, 8301, 8092, 8300, 8165, 8290] avg=8192.0000 max=8301 max_rank=3 min=8080 min_rank=2 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32847 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[497, 534, 522, 503, 504, 534, 504, 547, 535, 518, 500, 506, 537, 520, 514, 492, 13, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 526, 496, 481, 492, 540, 513, 502, 533, 514, 516, 476, 506, 526, 532, 482, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 495, 532, 526, 554, 515, 482, 540, 495, 483, 487, 503, 536, 496, 510, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 518, 493, 541, 499, 504, 500, 555, 532, 524, 511, 516, 487, 518, 517, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 466, 517, 504, 468, 546, 543, 507, 470, 521, 523, 579, 471, 505, 527, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 506, 489, 510, 500, 502, 484, 478, 501, 519, 506, 528, 492, 488, 490, 493, 37, 36, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 550, 546, 510, 529, 482, 523, 486, 523, 533, 479, 504, 476, 466, 512, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 545, 529, 546, 499, 501, 492, 486, 504, 496, 542, 518, 509, 499, 533, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8292, 8140, 8199, 8260, 8178, 8068, 8128, 8271] avg=8192.0000 max=8292 max_rank=0 min=8068 min_rank=5 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32645 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[537, 489, 544, 461, 576, 509, 541, 537, 507, 541, 525, 516, 495, 490, 504, 510, 14, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 518, 537, 531, 551, 498, 517, 533, 511, 511, 512, 490, 524, 504, 541, 489, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 562, 527, 513, 508, 499, 514, 518, 506, 525, 504, 512, 535, 506, 483, 522, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 512, 512, 523, 541, 512, 534, 553, 475, 505, 505, 544, 524, 478, 523, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 502, 534, 511, 525, 498, 497, 476, 551, 482, 544, 488, 502, 474, 533, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 519, 475, 479, 511, 543, 507, 480, 501, 488, 465, 528, 541, 509, 493, 509, 36, 36, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 530, 526, 526, 518, 522, 496, 502, 565, 524, 475, 481, 473, 525, 467, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 518, 465, 521, 523, 534, 502, 516, 529, 512, 474, 503, 500, 500, 491, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8308, 8273, 8270, 8258, 8113, 8105, 8126, 8083] avg=8192.0000 max=8308 max_rank=0 min=8083 min_rank=7 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32427 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[492, 538, 485, 498, 491, 512, 502, 493, 508, 479, 469, 466, 488, 485, 513, 521, 13, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 519, 502, 520, 525, 519, 512, 489, 500, 489, 519, 522, 493, 492, 495, 519, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 536, 515, 511, 520, 548, 522, 528, 514, 493, 514, 520, 525, 523, 519, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 526, 510, 475, 534, 534, 498, 551, 527, 480, 491, 526, 507, 495, 541, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 497, 559, 519, 498, 496, 491, 521, 505, 502, 533, 519, 526, 501, 533, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 488, 515, 524, 518, 531, 500, 497, 546, 514, 517, 514, 509, 511, 488, 457, 35, 33, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 504, 518, 526, 516, 542, 508, 525, 533, 554, 528, 481, 529, 471, 528, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 501, 527, 549, 547, 515, 504, 502, 519, 500, 491, 539, 509, 465, 494, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7965, 8121, 8322, 8208, 8187, 8245, 8301, 8187] avg=8192.0000 max=8322 max_rank=2 min=7965 min_rank=0 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32920 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[482, 519, 487, 530, 498, 477, 522, 535, 524, 521, 522, 529, 537, 458, 509, 485, 13, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 510, 473, 516, 524, 513, 517, 505, 531, 502, 490, 520, 527, 531, 518, 475, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 522, 532, 528, 499, 527, 443, 482, 548, 516, 535, 550, 525, 519, 491, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 451, 494, 502, 521, 513, 539, 493, 492, 513, 509, 503, 519, 504, 533, 525, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 492, 510, 494, 547, 515, 507, 532, 484, 488, 475, 512, 532, 500, 543, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 506, 493, 475, 521, 526, 506, 463, 490, 474, 540, 474, 538, 524, 536, 496, 36, 38, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 522, 532, 465, 463, 496, 542, 495, 534, 508, 531, 544, 527, 489, 514, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 563, 512, 566, 497, 519, 519, 521, 499, 486, 551, 512, 537, 548, 534, 537, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8175, 8223, 8109, 8181, 8123, 8177, 8387] avg=8192.0000 max=8387 max_rank=7 min=8109 min_rank=3 imbalance_ratio=1.023804 max_load_reduction=57149 max_load_reduction_pct=0.872025 moved_to_echo=32868 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4352, 4096, 4224, 4224, 4096, 3968, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4352, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 3968, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4352, 4096, 4224, 4224, 4096, 3968, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 3968, 4096, 4096, 4096, 4224, 4096, 3968, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 384, 384, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4352, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[530, 518, 501, 528, 570, 519, 505, 494, 532, 490, 493, 512, 541, 508, 521, 451, 508, 497, 501, 536, 504, 482, 526, 507, 535, 507, 543, 526, 507, 498, 523, 498, 527, 502, 490, 500, 544, 485, 465, 511, 476, 474, 522, 511, 533, 495, 545, 483, 514, 541, 482, 488, 499, 546, 565, 524, 477, 522, 506, 533, 550, 500, 537, 531, 524, 522, 511, 527, 526, 531, 525, 507, 525, 529, 517, 508, 485, 515, 549, 448, 500, 492, 473, 473, 509, 539, 489, 505, 506, 512, 509, 534, 543, 510, 546, 541, 535, 510, 484, 497, 508, 477, 509, 520, 483, 503, 490, 507, 506, 487, 579, 536, 517, 492, 492, 490, 540, 536, 505, 549, 502, 482, 549, 469, 510, 509, 515, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[537, 504, 536, 548, 502, 539, 514, 494, 499, 529, 498, 478, 521, 545, 521, 530, 556, 512, 506, 496, 486, 491, 508, 530, 517, 512, 498, 505, 484, 511, 506, 503, 542, 502, 527, 499, 532, 483, 494, 518, 478, 536, 497, 495, 513, 499, 500, 530, 509, 500, 531, 506, 531, 503, 482, 487, 515, 537, 539, 504, 485, 541, 522, 510, 515, 530, 502, 486, 533, 513, 492, 542, 498, 533, 506, 501, 525, 503, 497, 540, 518, 500, 510, 478, 512, 533, 506, 506, 537, 506, 488, 535, 510, 512, 541, 517, 486, 513, 498, 494, 523, 496, 536, 497, 520, 500, 493, 561, 526, 518, 501, 505, 492, 526, 533, 464, 522, 468, 526, 509, 509, 548, 487, 511, 509, 485, 528, 564] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[495, 511, 481, 487, 482, 510, 513, 526, 503, 511, 560, 477, 507, 520, 544, 466, 520, 496, 517, 481, 517, 506, 481, 479, 494, 530, 541, 523, 516, 542, 499, 527, 528, 529, 480, 527, 541, 539, 524, 489, 535, 509, 531, 503, 495, 538, 519, 547, 548, 481, 508, 536, 531, 491, 481, 518, 494, 537, 514, 526, 513, 472, 541, 470, 481, 502, 520, 500, 513, 506, 477, 520, 529, 517, 523, 522, 529, 524, 525, 486, 556, 509, 523, 515, 519, 517, 489, 497, 546, 521, 517, 517, 498, 503, 539, 495, 501, 488, 503, 494, 534, 514, 530, 520, 511, 504, 494, 549, 534, 541, 486, 502, 508, 511, 525, 474, 561, 515, 506, 506, 524, 509, 533, 481, 499, 512, 478, 497] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[492, 519, 535, 518, 502, 519, 520, 511, 499, 515, 507, 504, 472, 489, 489, 557, 498, 523, 520, 523, 511, 512, 511, 521, 554, 513, 527, 483, 515, 537, 498, 498, 510, 502, 514, 527, 491, 543, 562, 502, 482, 465, 516, 519, 524, 525, 514, 466, 507, 465, 512, 516, 506, 508, 535, 520, 509, 517, 553, 517, 516, 500, 516, 499, 498, 532, 519, 527, 511, 499, 529, 497, 508, 488, 496, 516, 527, 531, 464, 519, 481, 510, 557, 522, 514, 510, 524, 550, 503, 533, 557, 496, 505, 470, 528, 517, 483, 485, 513, 529, 471, 501, 514, 492, 471, 523, 520, 537, 530, 545, 526, 504, 532, 515, 506, 501, 473, 516, 508, 493, 522, 502, 519, 508, 507, 531, 535, 536] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[527, 517, 507, 500, 550, 522, 520, 544, 518, 530, 505, 532, 472, 505, 495, 512, 471, 501, 472, 514, 532, 489, 503, 510, 501, 494, 487, 494, 498, 480, 487, 522, 503, 540, 521, 469, 547, 484, 539, 532, 533, 504, 521, 505, 525, 503, 509, 549, 499, 514, 514, 522, 501, 504, 526, 497, 537, 535, 537, 500, 488, 511, 482, 503, 501, 509, 525, 525, 522, 506, 498, 489, 538, 485, 523, 499, 489, 514, 478, 501, 522, 509, 533, 516, 516, 533, 513, 508, 491, 500, 484, 550, 534, 556, 522, 532, 500, 536, 483, 479, 529, 515, 507, 533, 483, 505, 550, 542, 519, 514, 495, 491, 541, 517, 504, 511, 496, 491, 532, 499, 517, 512, 524, 518, 505, 538, 520, 544] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[537, 504, 536, 548, 502, 539, 514, 494, 499, 529, 498, 478, 521, 545, 521, 530, 556, 512, 506, 496, 486, 491, 508, 530, 517, 512, 498, 505, 484, 511, 506, 503, 542, 502, 527, 499, 532, 483, 494, 518, 478, 536, 497, 495, 513, 499, 500, 530, 509, 500, 531, 506, 531, 503, 482, 487, 515, 537, 539, 504, 485, 541, 522, 510, 515, 530, 502, 486, 533, 513, 492, 542, 498, 533, 506, 501, 525, 503, 497, 540, 518, 500, 510, 478, 512, 533, 506, 506, 537, 506, 488, 535, 510, 512, 541, 517, 486, 513, 498, 494, 523, 496, 536, 497, 520, 500, 493, 561, 526, 518, 501, 505, 492, 526, 533, 464, 522, 468, 526, 509, 509, 548, 487, 511, 509, 485, 528, 564] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[516, 509, 549, 530, 491, 488, 490, 498, 509, 485, 515, 514, 508, 507, 540, 502, 510, 494, 522, 491, 480, 520, 497, 504, 518, 491, 543, 505, 489, 518, 518, 510, 508, 532, 476, 549, 530, 529, 513, 559, 494, 576, 508, 532, 479, 538, 514, 551, 518, 500, 482, 460, 520, 509, 501, 537, 494, 482, 509, 505, 530, 497, 519, 511, 519, 520, 483, 498, 504, 487, 491, 471, 513, 503, 519, 558, 528, 481, 507, 534, 585, 486, 501, 501, 514, 478, 516, 502, 509, 549, 536, 472, 539, 509, 555, 482, 553, 494, 541, 476, 526, 502, 456, 484, 506, 508, 537, 511, 509, 468, 552, 550, 524, 514, 515, 540, 505, 532, 537, 495, 485, 490, 473, 517, 549, 545, 534, 535] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[554, 504, 503, 537, 485, 494, 520, 501, 519, 489, 548, 494, 512, 537, 497, 528, 504, 525, 537, 491, 535, 525, 515, 537, 530, 495, 500, 517, 518, 515, 507, 507, 470, 500, 507, 525, 558, 500, 516, 537, 500, 528, 498, 504, 529, 503, 516, 530, 493, 492, 513, 520, 541, 522, 501, 522, 483, 539, 507, 498, 513, 531, 530, 506, 483, 560, 491, 492, 512, 485, 514, 467, 464, 543, 503, 534, 503, 503, 527, 502, 488, 528, 477, 547, 460, 477, 488, 481, 480, 503, 494, 521, 505, 504, 532, 524, 512, 526, 490, 503, 512, 515, 495, 515, 501, 492, 463, 520, 523, 534, 519, 512, 535, 554, 539, 502, 524, 537, 534, 543, 543, 512, 541, 532, 516, 499, 481, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[492, 519, 535, 518, 502, 519, 520, 511, 499, 515, 507, 504, 472, 489, 489, 557, 498, 523, 520, 523, 511, 512, 511, 521, 554, 513, 527, 483, 515, 537, 498, 498, 510, 502, 514, 527, 491, 543, 562, 502, 482, 465, 516, 519, 524, 525, 514, 466, 507, 465, 512, 516, 506, 508, 535, 520, 509, 517, 553, 517, 516, 500, 516, 499, 498, 532, 519, 527, 511, 499, 529, 497, 508, 488, 496, 516, 527, 531, 464, 519, 481, 510, 557, 522, 514, 510, 524, 550, 503, 533, 557, 496, 505, 470, 528, 517, 483, 485, 513, 529, 471, 501, 514, 492, 471, 523, 520, 537, 530, 545, 526, 504, 532, 515, 506, 501, 473, 516, 508, 493, 522, 502, 519, 508, 507, 531, 535, 536] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[513, 527, 509, 511, 509, 549, 522, 508, 520, 527, 521, 488, 542, 529, 510, 487, 513, 548, 497, 542, 439, 481, 467, 462, 498, 481, 501, 509, 525, 520, 486, 532, 491, 494, 499, 496, 528, 507, 524, 528, 496, 515, 507, 489, 554, 514, 519, 507, 479, 524, 523, 498, 500, 535, 487, 500, 529, 528, 550, 523, 499, 527, 508, 540, 510, 507, 509, 513, 531, 544, 526, 499, 533, 516, 532, 516, 519, 513, 520, 491, 486, 533, 530, 488, 530, 544, 487, 508, 537, 490, 550, 489, 525, 534, 488, 512, 507, 540, 506, 530, 518, 521, 531, 539, 503, 519, 481, 520, 492, 478, 518, 504, 517, 537, 503, 484, 455, 514, 488, 514, 513, 481, 544, 463, 549, 542, 490, 534] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[495, 511, 481, 487, 482, 510, 513, 526, 503, 511, 560, 477, 507, 520, 544, 466, 520, 496, 517, 481, 517, 506, 481, 479, 494, 530, 541, 523, 516, 542, 499, 527, 528, 529, 480, 527, 541, 539, 524, 489, 535, 509, 531, 503, 495, 538, 519, 547, 548, 481, 508, 536, 531, 491, 481, 518, 494, 537, 514, 526, 513, 472, 541, 470, 481, 502, 520, 500, 513, 506, 477, 520, 529, 517, 523, 522, 529, 524, 525, 486, 556, 509, 523, 515, 519, 517, 489, 497, 546, 521, 517, 517, 498, 503, 539, 495, 501, 488, 503, 494, 534, 514, 530, 520, 511, 504, 494, 549, 534, 541, 486, 502, 508, 511, 525, 474, 561, 515, 506, 506, 524, 509, 533, 481, 499, 512, 478, 497] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[530, 518, 501, 528, 570, 519, 505, 494, 532, 490, 493, 512, 541, 508, 521, 451, 508, 497, 501, 536, 504, 482, 526, 507, 535, 507, 543, 526, 507, 498, 523, 498, 527, 502, 490, 500, 544, 485, 465, 511, 476, 474, 522, 511, 533, 495, 545, 483, 514, 541, 482, 488, 499, 546, 565, 524, 477, 522, 506, 533, 550, 500, 537, 531, 524, 522, 511, 527, 526, 531, 525, 507, 525, 529, 517, 508, 485, 515, 549, 448, 500, 492, 473, 473, 509, 539, 489, 505, 506, 512, 509, 534, 543, 510, 546, 541, 535, 510, 484, 497, 508, 477, 509, 520, 483, 503, 490, 507, 506, 487, 579, 536, 517, 492, 492, 490, 540, 536, 505, 549, 502, 482, 549, 469, 510, 509, 515, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[554, 504, 503, 537, 485, 494, 520, 501, 519, 489, 548, 494, 512, 537, 497, 528, 504, 525, 537, 491, 535, 525, 515, 537, 530, 495, 500, 517, 518, 515, 507, 507, 470, 500, 507, 525, 558, 500, 516, 537, 500, 528, 498, 504, 529, 503, 516, 530, 493, 492, 513, 520, 541, 522, 501, 522, 483, 539, 507, 498, 513, 531, 530, 506, 483, 560, 491, 492, 512, 485, 514, 467, 464, 543, 503, 534, 503, 503, 527, 502, 488, 528, 477, 547, 460, 477, 488, 481, 480, 503, 494, 521, 505, 504, 532, 524, 512, 526, 490, 503, 512, 515, 495, 515, 501, 492, 463, 520, 523, 534, 519, 512, 535, 554, 539, 502, 524, 537, 534, 543, 543, 512, 541, 532, 516, 499, 481, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[519, 504, 536, 548, 502, 539, 514, 494, 499, 529, 498, 478, 521, 545, 521, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 512, 506, 496, 486, 491, 508, 530, 517, 512, 498, 505, 484, 511, 506, 503, 39, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 502, 527, 499, 499, 483, 494, 518, 478, 536, 497, 495, 513, 499, 500, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 500, 531, 506, 531, 503, 482, 487, 515, 537, 531, 504, 485, 541, 522, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 530, 502, 486, 533, 513, 492, 542, 498, 533, 506, 501, 525, 503, 497, 540, 17, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 500, 510, 478, 512, 533, 506, 506, 537, 506, 488, 535, 510, 512, 514, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 513, 498, 494, 523, 496, 536, 497, 520, 500, 493, 561, 526, 518, 501, 505, 27, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 526, 533, 464, 522, 468, 526, 509, 509, 548, 487, 511, 509, 485, 528, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8277, 8190, 8112, 8194, 8241, 8182, 8198, 8142] avg=8192.0000 max=8277 max_rank=0 min=8112 min_rank=2 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32763 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[513, 517, 507, 500, 550, 522, 520, 544, 518, 530, 505, 532, 472, 505, 495, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 501, 472, 514, 532, 489, 503, 510, 501, 494, 487, 494, 498, 480, 487, 522, 33, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 540, 521, 469, 518, 484, 539, 532, 533, 504, 521, 505, 525, 503, 509, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 514, 514, 522, 501, 504, 526, 497, 537, 535, 532, 500, 488, 511, 482, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 509, 525, 525, 522, 506, 498, 489, 538, 485, 523, 499, 489, 514, 478, 501, 14, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 509, 533, 516, 516, 533, 513, 508, 491, 500, 484, 550, 534, 556, 500, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 536, 483, 479, 529, 515, 507, 533, 483, 505, 550, 542, 519, 514, 495, 491, 22, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 517, 504, 511, 496, 491, 532, 499, 517, 512, 524, 518, 505, 538, 520, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8242, 8016, 8255, 8165, 8121, 8297, 8204, 8236] avg=8192.0000 max=8297 max_rank=5 min=8016 min_rank=1 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32858 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[502, 509, 549, 530, 491, 488, 490, 498, 509, 485, 515, 514, 508, 507, 540, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 494, 522, 491, 480, 520, 497, 504, 518, 491, 543, 505, 489, 518, 518, 510, 32, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 532, 476, 549, 502, 529, 513, 559, 494, 576, 508, 532, 479, 538, 514, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 500, 482, 460, 520, 509, 501, 537, 494, 482, 504, 505, 530, 497, 519, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 520, 483, 498, 504, 487, 491, 471, 513, 503, 519, 558, 528, 481, 507, 534, 14, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 585, 486, 501, 501, 514, 478, 516, 502, 509, 549, 536, 472, 539, 509, 531, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 494, 541, 476, 526, 502, 456, 484, 506, 508, 537, 511, 509, 468, 552, 550, 24, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 514, 515, 540, 505, 532, 537, 495, 485, 490, 473, 517, 549, 545, 534, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8137, 8169, 8360, 8069, 8135, 8210, 8198, 8258] avg=8192.0000 max=8360 max_rank=2 min=8069 min_rank=3 imbalance_ratio=1.020508 max_load_reduction=57176 max_load_reduction_pct=0.872437 moved_to_echo=32801 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[539, 504, 503, 537, 485, 494, 520, 501, 519, 489, 548, 494, 512, 537, 497, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 525, 537, 491, 535, 525, 515, 537, 530, 495, 500, 517, 518, 515, 507, 507, 31, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 500, 507, 525, 529, 500, 516, 537, 500, 528, 498, 504, 529, 503, 516, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 492, 513, 520, 541, 522, 501, 522, 483, 539, 502, 498, 513, 531, 530, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 560, 491, 492, 512, 485, 514, 467, 464, 543, 503, 534, 503, 503, 527, 502, 15, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 528, 477, 547, 460, 477, 488, 481, 480, 503, 494, 521, 505, 504, 509, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 526, 490, 503, 512, 515, 495, 515, 501, 492, 463, 520, 523, 534, 519, 512, 23, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 554, 539, 502, 524, 537, 534, 543, 543, 512, 541, 532, 516, 499, 481, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8317, 8192, 8206, 8103, 7986, 8156, 8369] avg=8192.0000 max=8369 max_rank=7 min=7986 min_rank=5 imbalance_ratio=1.021606 max_load_reduction=57167 max_load_reduction_pct=0.872299 moved_to_echo=32614 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[479, 519, 535, 518, 502, 519, 520, 511, 499, 515, 507, 504, 472, 489, 489, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 523, 520, 523, 511, 512, 511, 521, 554, 513, 527, 483, 515, 537, 498, 498, 32, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 502, 514, 527, 465, 543, 562, 502, 482, 465, 516, 519, 524, 525, 514, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 465, 512, 516, 506, 508, 535, 520, 509, 517, 548, 517, 516, 500, 516, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 532, 519, 527, 511, 499, 529, 497, 508, 488, 496, 516, 527, 531, 464, 519, 13, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 510, 557, 522, 514, 510, 524, 550, 503, 533, 557, 496, 505, 470, 506, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 485, 513, 529, 471, 501, 514, 492, 471, 523, 520, 537, 530, 545, 526, 504, 22, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 515, 506, 501, 473, 516, 508, 493, 522, 502, 519, 508, 507, 531, 535, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8135, 8301, 8136, 8191, 8179, 8255, 8167, 8172] avg=8192.0000 max=8301 max_rank=1 min=8135 min_rank=0 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32773 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[500, 527, 509, 511, 509, 549, 522, 508, 520, 527, 521, 488, 542, 529, 510, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 548, 497, 542, 439, 481, 467, 462, 498, 481, 501, 509, 525, 520, 486, 532, 32, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 494, 499, 496, 500, 507, 524, 528, 496, 515, 507, 489, 554, 514, 519, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 524, 523, 498, 500, 535, 487, 500, 529, 528, 545, 523, 499, 527, 508, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 507, 509, 513, 531, 544, 526, 499, 533, 516, 532, 516, 519, 513, 520, 491, 13, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 533, 530, 488, 530, 544, 487, 508, 537, 490, 550, 489, 525, 534, 467, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 540, 506, 530, 518, 521, 531, 539, 503, 519, 481, 520, 492, 478, 518, 504, 21, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 537, 503, 484, 455, 514, 488, 514, 513, 481, 544, 463, 549, 542, 490, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8259, 8060, 8140, 8245, 8297, 8210, 8229, 8096] avg=8192.0000 max=8297 max_rank=4 min=8060 min_rank=1 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32832 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[482, 511, 481, 487, 482, 510, 513, 526, 503, 511, 560, 477, 507, 520, 544, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 496, 517, 481, 517, 506, 481, 479, 494, 530, 541, 523, 516, 542, 499, 527, 30, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 529, 480, 527, 513, 539, 524, 489, 535, 509, 531, 503, 495, 538, 519, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 481, 508, 536, 531, 491, 481, 518, 494, 537, 509, 526, 513, 472, 541, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 502, 520, 500, 513, 506, 477, 520, 529, 517, 523, 522, 529, 524, 525, 486, 13, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 509, 523, 515, 519, 517, 489, 497, 546, 521, 517, 517, 498, 503, 516, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 488, 503, 494, 534, 514, 530, 520, 511, 504, 494, 549, 534, 541, 486, 502, 23, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 511, 525, 474, 561, 515, 506, 506, 524, 509, 533, 481, 499, 512, 478, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8080, 8226, 8306, 8156, 8192, 8238, 8229, 8109] avg=8192.0000 max=8306 max_rank=2 min=8080 min_rank=0 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32768 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[502, 509, 549, 530, 491, 488, 490, 498, 509, 485, 515, 514, 508, 507, 540, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 494, 522, 491, 480, 520, 497, 504, 518, 491, 543, 505, 489, 518, 518, 510, 32, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 532, 476, 549, 502, 529, 513, 559, 494, 576, 508, 532, 479, 538, 514, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 500, 482, 460, 520, 509, 501, 537, 494, 482, 504, 505, 530, 497, 519, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 520, 483, 498, 504, 487, 491, 471, 513, 503, 519, 558, 528, 481, 507, 534, 14, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 585, 486, 501, 501, 514, 478, 516, 502, 509, 549, 536, 472, 539, 509, 531, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 494, 541, 476, 526, 502, 456, 484, 506, 508, 537, 511, 509, 468, 552, 550, 24, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 514, 515, 540, 505, 532, 537, 495, 485, 490, 473, 517, 549, 545, 534, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8137, 8169, 8360, 8069, 8135, 8210, 8198, 8258] avg=8192.0000 max=8360 max_rank=2 min=8069 min_rank=3 imbalance_ratio=1.020508 max_load_reduction=57176 max_load_reduction_pct=0.872437 moved_to_echo=32801 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[516, 518, 501, 528, 570, 519, 505, 494, 532, 490, 493, 512, 541, 508, 521, 451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 497, 501, 536, 504, 482, 526, 507, 535, 507, 543, 526, 507, 498, 523, 498, 32, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 502, 490, 500, 516, 485, 465, 511, 476, 474, 522, 511, 533, 495, 545, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 541, 482, 488, 499, 546, 565, 524, 477, 522, 501, 533, 550, 500, 537, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 522, 511, 527, 526, 531, 525, 507, 525, 529, 517, 508, 485, 515, 549, 448, 14, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 492, 473, 473, 509, 539, 489, 505, 506, 512, 509, 534, 543, 510, 523, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 510, 484, 497, 508, 477, 509, 520, 483, 503, 490, 507, 506, 487, 579, 536, 23, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 492, 492, 490, 540, 536, 505, 549, 502, 482, 549, 469, 510, 509, 515, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8257, 8035, 8310, 8268, 8158, 8155, 8154] avg=8192.0000 max=8310 max_rank=3 min=8035 min_rank=2 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32735 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[539, 504, 503, 537, 485, 494, 520, 501, 519, 489, 548, 494, 512, 537, 497, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 525, 537, 491, 535, 525, 515, 537, 530, 495, 500, 517, 518, 515, 507, 507, 31, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 500, 507, 525, 529, 500, 516, 537, 500, 528, 498, 504, 529, 503, 516, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 492, 513, 520, 541, 522, 501, 522, 483, 539, 502, 498, 513, 531, 530, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 560, 491, 492, 512, 485, 514, 467, 464, 543, 503, 534, 503, 503, 527, 502, 15, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 528, 477, 547, 460, 477, 488, 481, 480, 503, 494, 521, 505, 504, 509, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 526, 490, 503, 512, 515, 495, 515, 501, 492, 463, 520, 523, 534, 519, 512, 23, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 554, 539, 502, 524, 537, 534, 543, 543, 512, 541, 532, 516, 499, 481, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8317, 8192, 8206, 8103, 7986, 8156, 8369] avg=8192.0000 max=8369 max_rank=7 min=7986 min_rank=5 imbalance_ratio=1.021606 max_load_reduction=57167 max_load_reduction_pct=0.872299 moved_to_echo=32614 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[513, 517, 507, 500, 550, 522, 520, 544, 518, 530, 505, 532, 472, 505, 495, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 501, 472, 514, 532, 489, 503, 510, 501, 494, 487, 494, 498, 480, 487, 522, 33, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 540, 521, 469, 518, 484, 539, 532, 533, 504, 521, 505, 525, 503, 509, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 514, 514, 522, 501, 504, 526, 497, 537, 535, 532, 500, 488, 511, 482, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 509, 525, 525, 522, 506, 498, 489, 538, 485, 523, 499, 489, 514, 478, 501, 14, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 509, 533, 516, 516, 533, 513, 508, 491, 500, 484, 550, 534, 556, 500, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 536, 483, 479, 529, 515, 507, 533, 483, 505, 550, 542, 519, 514, 495, 491, 22, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 517, 504, 511, 496, 491, 532, 499, 517, 512, 524, 518, 505, 538, 520, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8242, 8016, 8255, 8165, 8121, 8297, 8204, 8236] avg=8192.0000 max=8297 max_rank=5 min=8016 min_rank=1 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32858 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 384, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 3968, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4352, 4224, 4096, 4224, 4224, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4352, 4224, 4096, 4224, 4224, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 3968, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[506, 512, 534, 471, 497, 535, 513, 474, 482, 490, 531, 505, 513, 489, 546, 490, 511, 552, 494, 532, 523, 547, 481, 507, 510, 528, 508, 498, 503, 526, 599, 495, 522, 503, 502, 507, 503, 546, 503, 510, 516, 548, 487, 503, 487, 500, 529, 505, 508, 496, 478, 528, 519, 500, 512, 490, 486, 530, 514, 479, 481, 508, 520, 505, 480, 534, 500, 508, 478, 510, 534, 495, 516, 491, 509, 510, 549, 497, 512, 503, 510, 509, 504, 515, 531, 535, 514, 514, 479, 529, 529, 517, 517, 508, 498, 505, 516, 558, 546, 470, 520, 516, 532, 532, 561, 535, 450, 563, 523, 516, 512, 524, 508, 527, 496, 491, 511, 520, 507, 528, 529, 491, 524, 514, 486, 530, 525, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[553, 525, 523, 504, 513, 505, 466, 527, 508, 532, 522, 512, 522, 494, 508, 520, 510, 534, 510, 481, 481, 500, 490, 503, 528, 520, 535, 520, 492, 455, 546, 514, 525, 512, 466, 488, 500, 527, 537, 510, 532, 505, 487, 537, 570, 524, 568, 488, 516, 544, 495, 508, 522, 490, 522, 527, 529, 520, 504, 531, 537, 491, 515, 500, 500, 506, 489, 489, 525, 502, 516, 548, 514, 513, 517, 540, 517, 498, 508, 478, 532, 498, 549, 507, 516, 495, 509, 493, 509, 513, 506, 538, 489, 536, 500, 523, 486, 524, 515, 468, 491, 487, 524, 521, 483, 495, 564, 516, 480, 553, 507, 521, 522, 498, 469, 513, 483, 501, 493, 466, 580, 504, 494, 518, 504, 572, 493, 538] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[508, 515, 522, 501, 526, 531, 482, 504, 554, 480, 494, 500, 539, 528, 526, 548, 535, 497, 502, 468, 501, 509, 503, 527, 514, 511, 514, 502, 536, 522, 492, 571, 547, 499, 488, 495, 495, 493, 498, 553, 521, 482, 493, 495, 513, 509, 551, 512, 496, 497, 527, 524, 527, 536, 488, 526, 539, 515, 522, 520, 527, 530, 556, 513, 530, 503, 478, 532, 548, 508, 507, 515, 505, 528, 513, 491, 501, 534, 525, 518, 522, 534, 493, 504, 509, 512, 502, 509, 500, 493, 493, 465, 507, 506, 537, 524, 515, 534, 481, 498, 502, 516, 518, 469, 551, 547, 495, 494, 474, 499, 534, 509, 511, 484, 508, 491, 525, 478, 484, 516, 530, 476, 568, 526, 478, 496, 535, 499] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[519, 473, 491, 492, 511, 532, 483, 492, 508, 518, 542, 527, 507, 543, 531, 471, 558, 502, 519, 528, 529, 504, 545, 536, 509, 505, 513, 503, 493, 497, 528, 536, 529, 533, 545, 496, 490, 502, 530, 501, 491, 506, 521, 508, 514, 506, 493, 541, 476, 495, 497, 499, 559, 502, 478, 539, 505, 519, 509, 492, 514, 522, 507, 548, 505, 554, 520, 517, 533, 502, 498, 509, 516, 505, 514, 501, 498, 492, 487, 522, 514, 466, 545, 491, 524, 441, 529, 528, 528, 520, 520, 502, 516, 504, 561, 505, 499, 516, 453, 512, 509, 520, 532, 541, 506, 546, 517, 487, 467, 494, 537, 488, 537, 512, 517, 505, 509, 540, 491, 496, 522, 504, 527, 544, 475, 505, 512, 537] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[487, 525, 472, 552, 527, 516, 516, 483, 522, 537, 547, 531, 522, 512, 561, 531, 514, 535, 518, 530, 487, 501, 490, 485, 513, 550, 470, 545, 500, 526, 505, 496, 514, 477, 535, 509, 527, 471, 513, 450, 491, 504, 503, 481, 532, 514, 481, 505, 522, 522, 486, 526, 531, 508, 527, 508, 558, 530, 487, 523, 474, 531, 459, 519, 505, 537, 501, 486, 566, 515, 542, 489, 503, 528, 508, 473, 476, 550, 528, 493, 518, 516, 496, 520, 521, 488, 518, 559, 495, 500, 539, 565, 516, 505, 505, 526, 517, 509, 475, 511, 498, 515, 483, 565, 535, 503, 489, 516, 487, 538, 532, 497, 535, 518, 507, 511, 490, 525, 494, 536, 480, 513, 507, 483, 520, 521, 505, 510] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[506, 512, 534, 471, 497, 535, 513, 474, 482, 490, 531, 505, 513, 489, 546, 490, 511, 552, 494, 532, 523, 547, 481, 507, 510, 528, 508, 498, 503, 526, 599, 495, 522, 503, 502, 507, 503, 546, 503, 510, 516, 548, 487, 503, 487, 500, 529, 505, 508, 496, 478, 528, 519, 500, 512, 490, 486, 530, 514, 479, 481, 508, 520, 505, 480, 534, 500, 508, 478, 510, 534, 495, 516, 491, 509, 510, 549, 497, 512, 503, 510, 509, 504, 515, 531, 535, 514, 514, 479, 529, 529, 517, 517, 508, 498, 505, 516, 558, 546, 470, 520, 516, 532, 532, 561, 535, 450, 563, 523, 516, 512, 524, 508, 527, 496, 491, 511, 520, 507, 528, 529, 491, 524, 514, 486, 530, 525, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[531, 481, 536, 477, 517, 471, 504, 524, 535, 485, 517, 494, 520, 497, 486, 469, 494, 516, 521, 514, 496, 534, 493, 519, 497, 530, 454, 510, 523, 514, 550, 515, 521, 510, 522, 507, 510, 530, 487, 554, 480, 524, 498, 502, 513, 502, 560, 479, 480, 523, 481, 477, 490, 545, 526, 522, 492, 498, 541, 509, 490, 477, 523, 513, 519, 554, 469, 522, 573, 520, 523, 511, 513, 507, 505, 493, 519, 525, 551, 541, 471, 530, 512, 534, 564, 505, 504, 477, 526, 486, 475, 524, 480, 526, 525, 510, 511, 527, 553, 505, 480, 542, 494, 556, 507, 500, 531, 468, 493, 518, 502, 473, 557, 514, 503, 519, 530, 520, 562, 488, 501, 530, 542, 517, 506, 501, 544, 538] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[508, 515, 522, 501, 526, 531, 482, 504, 554, 480, 494, 500, 539, 528, 526, 548, 535, 497, 502, 468, 501, 509, 503, 527, 514, 511, 514, 502, 536, 522, 492, 571, 547, 499, 488, 495, 495, 493, 498, 553, 521, 482, 493, 495, 513, 509, 551, 512, 496, 497, 527, 524, 527, 536, 488, 526, 539, 515, 522, 520, 527, 530, 556, 513, 530, 503, 478, 532, 548, 508, 507, 515, 505, 528, 513, 491, 501, 534, 525, 518, 522, 534, 493, 504, 509, 512, 502, 509, 500, 493, 493, 465, 507, 506, 537, 524, 515, 534, 481, 498, 502, 516, 518, 469, 551, 547, 495, 494, 474, 499, 534, 509, 511, 484, 508, 491, 525, 478, 484, 516, 530, 476, 568, 526, 478, 496, 535, 499] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[519, 473, 491, 492, 511, 532, 483, 492, 508, 518, 542, 527, 507, 543, 531, 471, 558, 502, 519, 528, 529, 504, 545, 536, 509, 505, 513, 503, 493, 497, 528, 536, 529, 533, 545, 496, 490, 502, 530, 501, 491, 506, 521, 508, 514, 506, 493, 541, 476, 495, 497, 499, 559, 502, 478, 539, 505, 519, 509, 492, 514, 522, 507, 548, 505, 554, 520, 517, 533, 502, 498, 509, 516, 505, 514, 501, 498, 492, 487, 522, 514, 466, 545, 491, 524, 441, 529, 528, 528, 520, 520, 502, 516, 504, 561, 505, 499, 516, 453, 512, 509, 520, 532, 541, 506, 546, 517, 487, 467, 494, 537, 488, 537, 512, 517, 505, 509, 540, 491, 496, 522, 504, 527, 544, 475, 505, 512, 537] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[484, 525, 528, 440, 504, 500, 490, 539, 507, 561, 490, 521, 542, 503, 526, 511, 548, 471, 500, 512, 531, 534, 510, 521, 524, 501, 470, 503, 485, 509, 518, 521, 501, 535, 545, 522, 523, 541, 549, 527, 538, 513, 517, 528, 490, 532, 545, 498, 520, 556, 524, 490, 511, 486, 514, 466, 495, 525, 493, 494, 517, 505, 532, 539, 506, 517, 516, 528, 484, 547, 533, 512, 527, 508, 486, 508, 484, 530, 516, 515, 533, 506, 560, 504, 495, 478, 516, 545, 513, 532, 511, 501, 512, 512, 482, 499, 498, 471, 517, 481, 526, 543, 519, 529, 517, 510, 493, 503, 497, 531, 541, 522, 514, 518, 487, 516, 529, 512, 471, 504, 527, 441, 522, 492, 514, 498, 494, 483] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[503, 514, 510, 542, 502, 548, 499, 509, 558, 505, 527, 499, 482, 534, 490, 517, 513, 505, 555, 512, 507, 510, 515, 517, 520, 484, 514, 513, 504, 504, 490, 487, 555, 553, 542, 514, 526, 484, 461, 484, 504, 508, 511, 521, 522, 525, 518, 506, 481, 512, 553, 523, 480, 482, 495, 522, 547, 480, 506, 504, 505, 522, 502, 501, 511, 536, 530, 528, 498, 530, 531, 526, 574, 487, 499, 493, 495, 532, 495, 495, 491, 494, 553, 512, 488, 537, 522, 482, 500, 521, 485, 501, 492, 532, 515, 537, 516, 500, 512, 504, 502, 490, 531, 507, 519, 512, 527, 499, 510, 515, 495, 518, 507, 484, 508, 542, 501, 533, 537, 491, 505, 520, 502, 529, 502, 481, 549, 528] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[508, 515, 522, 501, 526, 531, 482, 504, 554, 480, 494, 500, 539, 528, 526, 548, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 497, 502, 468, 501, 509, 503, 527, 514, 511, 514, 502, 536, 522, 487, 571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 499, 488, 495, 495, 493, 498, 553, 521, 482, 493, 495, 513, 509, 539, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 497, 527, 524, 527, 536, 488, 526, 539, 515, 522, 520, 527, 530, 556, 513, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 482, 478, 532, 548, 508, 507, 515, 505, 528, 513, 491, 501, 534, 525, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 534, 493, 504, 509, 512, 502, 509, 500, 493, 493, 465, 507, 506, 537, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 534, 481, 498, 502, 516, 518, 469, 551, 547, 495, 494, 474, 499, 534, 509, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 484, 508, 491, 525, 478, 484, 516, 530, 476, 568, 526, 478, 496, 535, 499, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8260, 8199, 8132, 8360, 8215, 8110, 8145, 8115] avg=8192.0000 max=8360 max_rank=3 min=8110 min_rank=5 imbalance_ratio=1.020508 max_load_reduction=57176 max_load_reduction_pct=0.872437 moved_to_echo=32585 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[553, 525, 523, 504, 513, 505, 466, 527, 508, 532, 522, 512, 522, 494, 508, 520, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 534, 510, 481, 481, 500, 490, 503, 528, 520, 535, 520, 492, 455, 525, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 512, 466, 488, 500, 527, 537, 510, 532, 505, 487, 537, 570, 524, 546, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 544, 495, 508, 522, 490, 522, 527, 529, 520, 504, 531, 537, 491, 515, 500, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 479, 489, 489, 525, 502, 516, 548, 514, 513, 517, 540, 517, 498, 508, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 498, 549, 507, 516, 495, 509, 493, 509, 513, 506, 538, 489, 536, 500, 523, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 524, 515, 468, 491, 487, 524, 521, 483, 495, 564, 516, 480, 553, 507, 521, 11, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 498, 469, 513, 483, 501, 493, 466, 580, 504, 494, 518, 504, 572, 493, 538, 11, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8241, 8098, 8254, 8272, 8133, 8219, 8154, 8165] avg=8192.0000 max=8272 max_rank=3 min=8098 min_rank=1 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32671 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[506, 512, 534, 471, 497, 535, 513, 474, 482, 490, 531, 505, 513, 489, 546, 490, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 552, 494, 532, 523, 547, 481, 507, 510, 528, 508, 498, 503, 526, 592, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 503, 502, 507, 503, 546, 503, 510, 516, 548, 487, 503, 487, 500, 517, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 496, 478, 528, 519, 500, 512, 490, 486, 530, 514, 479, 481, 508, 520, 505, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 512, 500, 508, 478, 510, 534, 495, 516, 491, 509, 510, 549, 497, 512, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 509, 504, 515, 531, 535, 514, 514, 479, 529, 529, 517, 517, 508, 498, 505, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 558, 546, 470, 520, 516, 532, 532, 561, 535, 450, 563, 523, 516, 512, 524, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 527, 496, 491, 511, 520, 507, 528, 529, 491, 524, 514, 486, 530, 525, 508, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8090, 8307, 8159, 8072, 8104, 8215, 8384, 8205] avg=8192.0000 max=8384 max_rank=6 min=8072 min_rank=3 imbalance_ratio=1.023438 max_load_reduction=57152 max_load_reduction_pct=0.872070 moved_to_echo=32908 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[531, 481, 536, 477, 517, 471, 504, 524, 535, 485, 517, 494, 520, 497, 486, 469, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 516, 521, 514, 496, 534, 493, 519, 497, 530, 454, 510, 523, 514, 544, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 510, 522, 507, 510, 530, 487, 554, 480, 524, 498, 502, 513, 502, 548, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 523, 481, 477, 490, 545, 526, 522, 492, 498, 541, 509, 490, 477, 523, 513, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 531, 469, 522, 573, 520, 523, 511, 513, 507, 505, 493, 519, 525, 551, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 530, 512, 534, 564, 505, 504, 477, 526, 486, 475, 524, 480, 526, 525, 510, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 527, 553, 505, 480, 542, 494, 556, 507, 500, 531, 468, 493, 518, 502, 473, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 557, 514, 503, 519, 530, 520, 562, 488, 501, 530, 542, 517, 506, 501, 544, 538, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8046, 8174, 8187, 8106, 8322, 8150, 8169, 8382] avg=8192.0000 max=8382 max_rank=7 min=8046 min_rank=0 imbalance_ratio=1.023193 max_load_reduction=57154 max_load_reduction_pct=0.872101 moved_to_echo=33023 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[487, 525, 472, 552, 527, 516, 516, 483, 522, 537, 547, 531, 522, 512, 561, 531, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 535, 518, 530, 487, 501, 490, 485, 513, 550, 470, 545, 500, 526, 500, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 477, 535, 509, 527, 471, 513, 450, 491, 504, 503, 481, 532, 514, 470, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 522, 486, 526, 531, 508, 527, 508, 558, 530, 487, 523, 474, 531, 459, 519, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 515, 501, 486, 566, 515, 542, 489, 503, 528, 508, 473, 476, 550, 528, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 516, 496, 520, 521, 488, 518, 559, 495, 500, 539, 565, 516, 505, 505, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 509, 475, 511, 498, 515, 483, 565, 535, 503, 489, 516, 487, 538, 532, 497, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 518, 507, 511, 490, 525, 494, 536, 480, 513, 507, 483, 520, 521, 505, 510, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8343, 8160, 7996, 8229, 8178, 8287, 8178, 8165] avg=8192.0000 max=8343 max_rank=0 min=7996 min_rank=2 imbalance_ratio=1.018433 max_load_reduction=57193 max_load_reduction_pct=0.872696 moved_to_echo=32808 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[503, 514, 510, 542, 502, 548, 499, 509, 558, 505, 527, 499, 482, 534, 490, 517, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 505, 555, 512, 507, 510, 515, 517, 520, 484, 514, 513, 504, 504, 485, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 553, 542, 514, 526, 484, 461, 484, 504, 508, 511, 521, 522, 525, 506, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 512, 553, 523, 480, 482, 495, 522, 547, 480, 506, 504, 505, 522, 502, 501, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 514, 530, 528, 498, 530, 531, 526, 574, 487, 499, 493, 495, 532, 495, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 494, 553, 512, 488, 537, 522, 482, 500, 521, 485, 501, 492, 532, 515, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 500, 512, 504, 502, 490, 531, 507, 519, 512, 527, 499, 510, 515, 495, 518, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 484, 508, 542, 501, 533, 537, 491, 505, 520, 502, 529, 502, 481, 549, 528, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8241, 8145, 8222, 8133, 8238, 8162, 8166, 8229] avg=8192.0000 max=8241 max_rank=0 min=8133 min_rank=3 imbalance_ratio=1.005981 max_load_reduction=57295 max_load_reduction_pct=0.874252 moved_to_echo=32795 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[506, 512, 534, 471, 497, 535, 513, 474, 482, 490, 531, 505, 513, 489, 546, 490, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 552, 494, 532, 523, 547, 481, 507, 510, 528, 508, 498, 503, 526, 592, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 503, 502, 507, 503, 546, 503, 510, 516, 548, 487, 503, 487, 500, 517, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 496, 478, 528, 519, 500, 512, 490, 486, 530, 514, 479, 481, 508, 520, 505, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 512, 500, 508, 478, 510, 534, 495, 516, 491, 509, 510, 549, 497, 512, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 509, 504, 515, 531, 535, 514, 514, 479, 529, 529, 517, 517, 508, 498, 505, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 558, 546, 470, 520, 516, 532, 532, 561, 535, 450, 563, 523, 516, 512, 524, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 527, 496, 491, 511, 520, 507, 528, 529, 491, 524, 514, 486, 530, 525, 508, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8090, 8307, 8159, 8072, 8104, 8215, 8384, 8205] avg=8192.0000 max=8384 max_rank=6 min=8072 min_rank=3 imbalance_ratio=1.023438 max_load_reduction=57152 max_load_reduction_pct=0.872070 moved_to_echo=32908 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[508, 515, 522, 501, 526, 531, 482, 504, 554, 480, 494, 500, 539, 528, 526, 548, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 497, 502, 468, 501, 509, 503, 527, 514, 511, 514, 502, 536, 522, 487, 571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 499, 488, 495, 495, 493, 498, 553, 521, 482, 493, 495, 513, 509, 539, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 497, 527, 524, 527, 536, 488, 526, 539, 515, 522, 520, 527, 530, 556, 513, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 482, 478, 532, 548, 508, 507, 515, 505, 528, 513, 491, 501, 534, 525, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 534, 493, 504, 509, 512, 502, 509, 500, 493, 493, 465, 507, 506, 537, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 534, 481, 498, 502, 516, 518, 469, 551, 547, 495, 494, 474, 499, 534, 509, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 484, 508, 491, 525, 478, 484, 516, 530, 476, 568, 526, 478, 496, 535, 499, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8260, 8199, 8132, 8360, 8215, 8110, 8145, 8115] avg=8192.0000 max=8360 max_rank=3 min=8110 min_rank=5 imbalance_ratio=1.020508 max_load_reduction=57176 max_load_reduction_pct=0.872437 moved_to_echo=32585 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[519, 473, 491, 492, 511, 532, 483, 492, 508, 518, 542, 527, 507, 543, 531, 471, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 558, 502, 519, 528, 529, 504, 545, 536, 509, 505, 513, 503, 493, 497, 523, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 533, 545, 496, 490, 502, 530, 501, 491, 506, 521, 508, 514, 506, 481, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 495, 497, 499, 559, 502, 478, 539, 505, 519, 509, 492, 514, 522, 507, 548, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 531, 520, 517, 533, 502, 498, 509, 516, 505, 514, 501, 498, 492, 487, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 466, 545, 491, 524, 441, 529, 528, 528, 520, 520, 502, 516, 504, 561, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 516, 453, 512, 509, 520, 532, 541, 506, 546, 517, 487, 467, 494, 537, 488, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 512, 517, 505, 509, 540, 491, 496, 522, 504, 527, 544, 475, 505, 512, 537, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8142, 8300, 8194, 8180, 8150, 8194, 8133, 8243] avg=8192.0000 max=8300 max_rank=1 min=8133 min_rank=6 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32720 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[484, 525, 528, 440, 504, 500, 490, 539, 507, 561, 490, 521, 542, 503, 526, 511, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 471, 500, 512, 531, 534, 510, 521, 524, 501, 470, 503, 485, 509, 513, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 535, 545, 522, 523, 541, 549, 527, 538, 513, 517, 528, 490, 532, 533, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 556, 524, 490, 511, 486, 514, 466, 495, 525, 493, 494, 517, 505, 532, 539, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 496, 516, 528, 484, 547, 533, 512, 527, 508, 486, 508, 484, 530, 516, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 506, 560, 504, 495, 478, 516, 545, 513, 532, 511, 501, 512, 512, 482, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 471, 517, 481, 526, 543, 519, 529, 517, 510, 493, 503, 497, 531, 541, 522, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 518, 487, 516, 529, 512, 471, 504, 527, 441, 522, 492, 514, 498, 494, 483, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8173, 8153, 8392, 8184, 8196, 8199, 8207, 8032] avg=8192.0000 max=8392 max_rank=2 min=8032 min_rank=7 imbalance_ratio=1.024414 max_load_reduction=57144 max_load_reduction_pct=0.871948 moved_to_echo=32634 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[503, 514, 510, 542, 502, 548, 499, 509, 558, 505, 527, 499, 482, 534, 490, 517, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 505, 555, 512, 507, 510, 515, 517, 520, 484, 514, 513, 504, 504, 485, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 553, 542, 514, 526, 484, 461, 484, 504, 508, 511, 521, 522, 525, 506, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 512, 553, 523, 480, 482, 495, 522, 547, 480, 506, 504, 505, 522, 502, 501, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 514, 530, 528, 498, 530, 531, 526, 574, 487, 499, 493, 495, 532, 495, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 494, 553, 512, 488, 537, 522, 482, 500, 521, 485, 501, 492, 532, 515, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 500, 512, 504, 502, 490, 531, 507, 519, 512, 527, 499, 510, 515, 495, 518, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 484, 508, 542, 501, 533, 537, 491, 505, 520, 502, 529, 502, 481, 549, 528, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8241, 8145, 8222, 8133, 8238, 8162, 8166, 8229] avg=8192.0000 max=8241 max_rank=0 min=8133 min_rank=3 imbalance_ratio=1.005981 max_load_reduction=57295 max_load_reduction_pct=0.874252 moved_to_echo=32795 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 3968, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 3968, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 3968, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 3968, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[500, 515, 551, 479, 518, 546, 486, 480, 520, 526, 505, 488, 506, 523, 544, 509, 498, 524, 510, 503, 500, 531, 508, 490, 528, 494, 494, 507, 509, 464, 496, 500, 524, 502, 461, 537, 530, 515, 550, 494, 486, 505, 485, 518, 464, 497, 481, 512, 537, 531, 514, 532, 521, 552, 527, 495, 514, 477, 458, 527, 475, 482, 571, 485, 508, 505, 519, 506, 519, 507, 521, 541, 513, 478, 479, 529, 579, 535, 556, 540, 513, 518, 530, 490, 504, 527, 497, 528, 525, 480, 480, 562, 564, 534, 508, 441, 493, 493, 489, 528, 516, 499, 527, 530, 540, 536, 499, 494, 546, 524, 504, 545, 532, 482, 529, 529, 511, 493, 495, 530, 539, 496, 538, 529, 508, 481, 516, 518] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[498, 512, 524, 533, 531, 534, 503, 519, 512, 508, 529, 491, 532, 502, 548, 485, 516, 507, 490, 470, 513, 509, 552, 508, 516, 475, 535, 513, 480, 513, 475, 542, 491, 547, 510, 521, 529, 526, 510, 516, 527, 525, 496, 529, 531, 516, 528, 505, 532, 531, 495, 561, 526, 519, 511, 492, 517, 489, 494, 492, 508, 509, 542, 557, 500, 521, 492, 492, 512, 518, 510, 482, 484, 510, 504, 496, 507, 499, 527, 513, 509, 494, 511, 502, 537, 509, 486, 507, 508, 503, 524, 500, 491, 506, 521, 487, 508, 529, 493, 532, 519, 533, 527, 503, 480, 483, 536, 527, 523, 519, 512, 509, 497, 528, 523, 463, 515, 561, 552, 511, 499, 480, 503, 558, 501, 499, 487, 507] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[482, 530, 512, 524, 503, 461, 493, 522, 502, 545, 524, 487, 490, 514, 521, 486, 486, 487, 506, 464, 486, 494, 472, 504, 502, 506, 539, 550, 537, 490, 559, 521, 523, 516, 522, 478, 532, 514, 489, 513, 478, 515, 532, 536, 513, 474, 500, 528, 492, 510, 493, 558, 490, 527, 491, 490, 531, 518, 509, 542, 489, 508, 536, 550, 581, 514, 489, 489, 520, 522, 516, 473, 517, 502, 533, 470, 502, 539, 514, 542, 533, 487, 526, 512, 484, 484, 536, 488, 568, 526, 515, 552, 536, 506, 537, 497, 490, 516, 526, 514, 528, 519, 532, 533, 455, 525, 505, 485, 481, 526, 535, 507, 542, 510, 536, 487, 518, 503, 531, 487, 540, 519, 507, 493, 502, 521, 541, 516] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[455, 506, 458, 529, 512, 498, 555, 508, 491, 480, 526, 515, 502, 544, 514, 487, 497, 527, 521, 496, 496, 489, 514, 519, 506, 508, 539, 542, 524, 526, 479, 512, 528, 523, 573, 502, 533, 510, 529, 533, 491, 547, 488, 487, 532, 543, 487, 534, 511, 527, 473, 519, 550, 483, 522, 509, 551, 472, 497, 532, 517, 524, 513, 515, 539, 499, 527, 512, 497, 536, 529, 532, 499, 520, 510, 492, 540, 519, 507, 486, 521, 461, 489, 512, 523, 494, 509, 532, 520, 520, 517, 479, 492, 530, 485, 524, 561, 490, 483, 493, 512, 502, 494, 501, 551, 516, 526, 493, 478, 514, 523, 496, 556, 492, 484, 524, 461, 529, 527, 497, 513, 540, 545, 509, 531, 516, 495, 502] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[499, 531, 492, 504, 497, 470, 535, 503, 486, 500, 539, 536, 542, 527, 475, 476, 497, 480, 529, 537, 496, 526, 547, 464, 540, 511, 540, 519, 540, 582, 491, 524, 487, 470, 504, 526, 535, 516, 525, 498, 528, 504, 519, 499, 526, 516, 541, 516, 515, 519, 501, 513, 495, 499, 490, 503, 490, 519, 524, 494, 503, 545, 514, 521, 519, 505, 498, 531, 481, 518, 530, 521, 503, 518, 479, 505, 529, 520, 515, 510, 510, 513, 474, 495, 537, 512, 511, 485, 558, 509, 543, 524, 496, 519, 527, 487, 495, 535, 505, 496, 527, 512, 499, 526, 462, 496, 519, 527, 532, 526, 521, 530, 500, 520, 531, 541, 483, 503, 520, 532, 482, 487, 545, 447, 514, 511, 526, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[477, 548, 533, 505, 515, 505, 519, 500, 478, 532, 497, 473, 469, 504, 488, 511, 501, 470, 522, 526, 510, 529, 503, 493, 531, 486, 532, 483, 508, 463, 532, 512, 497, 498, 538, 518, 477, 540, 551, 543, 511, 513, 541, 551, 522, 481, 536, 569, 540, 515, 468, 499, 519, 472, 524, 500, 502, 531, 529, 530, 514, 505, 498, 520, 527, 480, 497, 512, 525, 477, 515, 482, 497, 500, 494, 488, 562, 475, 508, 542, 482, 486, 493, 533, 594, 509, 519, 547, 541, 527, 529, 536, 537, 507, 506, 458, 526, 528, 530, 524, 529, 497, 487, 536, 508, 546, 526, 485, 558, 491, 505, 520, 529, 478, 483, 528, 547, 492, 503, 497, 500, 505, 513, 520, 504, 499, 497, 553] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[469, 498, 521, 519, 487, 500, 487, 529, 532, 475, 514, 529, 530, 552, 481, 542, 561, 478, 489, 521, 486, 553, 535, 501, 528, 519, 520, 536, 508, 536, 513, 515, 486, 510, 527, 486, 504, 487, 513, 512, 548, 491, 491, 488, 514, 506, 526, 486, 495, 523, 528, 536, 530, 495, 537, 555, 480, 498, 510, 517, 475, 485, 531, 505, 512, 522, 529, 548, 503, 544, 496, 525, 518, 499, 505, 496, 496, 520, 517, 546, 489, 578, 557, 520, 552, 521, 497, 508, 525, 544, 506, 513, 538, 511, 511, 498, 503, 474, 525, 511, 529, 536, 470, 539, 511, 490, 489, 461, 514, 503, 496, 478, 525, 507, 468, 490, 470, 482, 546, 522, 519, 521, 520, 516, 532, 525, 473, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[498, 512, 524, 533, 531, 534, 503, 519, 512, 508, 529, 491, 532, 502, 548, 485, 516, 507, 490, 470, 513, 509, 552, 508, 516, 475, 535, 513, 480, 513, 475, 542, 491, 547, 510, 521, 529, 526, 510, 516, 527, 525, 496, 529, 531, 516, 528, 505, 532, 531, 495, 561, 526, 519, 511, 492, 517, 489, 494, 492, 508, 509, 542, 557, 500, 521, 492, 492, 512, 518, 510, 482, 484, 510, 504, 496, 507, 499, 527, 513, 509, 494, 511, 502, 537, 509, 486, 507, 508, 503, 524, 500, 491, 506, 521, 487, 508, 529, 493, 532, 519, 533, 527, 503, 480, 483, 536, 527, 523, 519, 512, 509, 497, 528, 523, 463, 515, 561, 552, 511, 499, 480, 503, 558, 501, 499, 487, 507] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[499, 531, 492, 504, 497, 470, 535, 503, 486, 500, 539, 536, 542, 527, 475, 476, 497, 480, 529, 537, 496, 526, 547, 464, 540, 511, 540, 519, 540, 582, 491, 524, 487, 470, 504, 526, 535, 516, 525, 498, 528, 504, 519, 499, 526, 516, 541, 516, 515, 519, 501, 513, 495, 499, 490, 503, 490, 519, 524, 494, 503, 545, 514, 521, 519, 505, 498, 531, 481, 518, 530, 521, 503, 518, 479, 505, 529, 520, 515, 510, 510, 513, 474, 495, 537, 512, 511, 485, 558, 509, 543, 524, 496, 519, 527, 487, 495, 535, 505, 496, 527, 512, 499, 526, 462, 496, 519, 527, 532, 526, 521, 530, 500, 520, 531, 541, 483, 503, 520, 532, 482, 487, 545, 447, 514, 511, 526, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[466, 494, 543, 553, 504, 550, 452, 546, 510, 516, 506, 524, 522, 513, 527, 522, 508, 507, 499, 543, 518, 536, 503, 538, 515, 502, 557, 507, 504, 474, 490, 508, 493, 473, 463, 510, 527, 519, 526, 508, 480, 499, 480, 549, 552, 525, 495, 466, 504, 500, 487, 502, 529, 512, 519, 533, 512, 522, 490, 503, 499, 541, 534, 500, 520, 517, 518, 494, 505, 525, 524, 546, 571, 529, 540, 514, 519, 518, 479, 507, 476, 483, 544, 546, 519, 519, 472, 509, 507, 518, 520, 528, 490, 511, 478, 537, 516, 510, 447, 497, 507, 529, 536, 503, 500, 512, 498, 486, 548, 562, 537, 492, 540, 530, 531, 501, 502, 518, 525, 496, 496, 483, 526, 505, 474, 536, 492, 509] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[500, 515, 551, 479, 518, 546, 486, 480, 520, 526, 505, 488, 506, 523, 544, 509, 27, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 524, 510, 503, 500, 531, 508, 490, 528, 494, 494, 507, 509, 464, 496, 500, 10, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 502, 461, 537, 530, 515, 541, 494, 486, 505, 485, 518, 464, 497, 481, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 531, 514, 532, 521, 552, 527, 495, 514, 477, 458, 527, 475, 482, 561, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 505, 519, 506, 519, 507, 521, 541, 513, 478, 479, 529, 552, 535, 556, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 518, 530, 490, 504, 527, 497, 528, 503, 480, 480, 562, 564, 534, 508, 441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 493, 489, 528, 516, 499, 527, 530, 540, 536, 499, 494, 546, 524, 504, 545, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 482, 529, 529, 511, 493, 495, 530, 539, 496, 538, 529, 508, 481, 516, 518, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8073, 8052, 8188, 8308, 8179, 8265, 8233] avg=8192.0000 max=8308 max_rank=4 min=8052 min_rank=2 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32985 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[482, 530, 512, 524, 503, 461, 493, 522, 502, 545, 524, 487, 490, 514, 521, 486, 23, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 487, 506, 464, 486, 494, 472, 504, 502, 506, 539, 550, 537, 490, 559, 521, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 516, 522, 478, 532, 514, 482, 513, 478, 515, 532, 536, 513, 474, 500, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 510, 493, 558, 490, 527, 491, 490, 531, 518, 509, 542, 489, 508, 527, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 581, 514, 489, 489, 520, 522, 516, 473, 517, 502, 533, 470, 479, 539, 514, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 487, 526, 512, 484, 484, 536, 488, 545, 526, 515, 552, 536, 506, 537, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 516, 526, 514, 528, 519, 532, 533, 455, 525, 505, 485, 481, 526, 535, 507, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 510, 536, 487, 518, 503, 531, 487, 540, 519, 507, 493, 502, 521, 541, 516, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8135, 8119, 8156, 8225, 8200, 8264, 8178, 8259] avg=8192.0000 max=8264 max_rank=5 min=8119 min_rank=1 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32901 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[477, 548, 533, 505, 515, 505, 519, 500, 478, 532, 497, 473, 469, 504, 488, 511, 501, 470, 522, 526, 510, 529, 503, 493, 531, 486, 532, 483, 508, 463, 532, 512, 497, 498, 538, 518, 477, 540, 551, 543, 511, 513, 541, 551, 522, 481, 536, 569, 540, 515, 468, 499, 519, 472, 524, 500, 502, 531, 529, 530, 514, 505, 498, 520, 527, 480, 497, 512, 525, 477, 515, 482, 497, 500, 494, 488, 562, 475, 508, 542, 482, 486, 493, 533, 594, 509, 519, 547, 541, 527, 529, 536, 537, 507, 506, 458, 526, 528, 530, 524, 529, 497, 487, 536, 508, 546, 526, 485, 558, 491, 505, 520, 529, 478, 483, 528, 547, 492, 503, 497, 500, 505, 513, 520, 504, 499, 497, 553] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[482, 530, 512, 524, 503, 461, 493, 522, 502, 545, 524, 487, 490, 514, 521, 486, 486, 487, 506, 464, 486, 494, 472, 504, 502, 506, 539, 550, 537, 490, 559, 521, 523, 516, 522, 478, 532, 514, 489, 513, 478, 515, 532, 536, 513, 474, 500, 528, 492, 510, 493, 558, 490, 527, 491, 490, 531, 518, 509, 542, 489, 508, 536, 550, 581, 514, 489, 489, 520, 522, 516, 473, 517, 502, 533, 470, 502, 539, 514, 542, 533, 487, 526, 512, 484, 484, 536, 488, 568, 526, 515, 552, 536, 506, 537, 497, 490, 516, 526, 514, 528, 519, 532, 533, 455, 525, 505, 485, 481, 526, 535, 507, 542, 510, 536, 487, 518, 503, 531, 487, 540, 519, 507, 493, 502, 521, 541, 516] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[500, 515, 551, 479, 518, 546, 486, 480, 520, 526, 505, 488, 506, 523, 544, 509, 498, 524, 510, 503, 500, 531, 508, 490, 528, 494, 494, 507, 509, 464, 496, 500, 524, 502, 461, 537, 530, 515, 550, 494, 486, 505, 485, 518, 464, 497, 481, 512, 537, 531, 514, 532, 521, 552, 527, 495, 514, 477, 458, 527, 475, 482, 571, 485, 508, 505, 519, 506, 519, 507, 521, 541, 513, 478, 479, 529, 579, 535, 556, 540, 513, 518, 530, 490, 504, 527, 497, 528, 525, 480, 480, 562, 564, 534, 508, 441, 493, 493, 489, 528, 516, 499, 527, 530, 540, 536, 499, 494, 546, 524, 504, 545, 532, 482, 529, 529, 511, 493, 495, 530, 539, 496, 538, 529, 508, 481, 516, 518] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[469, 498, 521, 519, 487, 500, 487, 529, 532, 475, 514, 529, 530, 552, 481, 542, 23, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 478, 489, 521, 486, 553, 535, 501, 528, 519, 520, 536, 508, 536, 513, 515, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 510, 527, 486, 504, 487, 505, 512, 548, 491, 491, 488, 514, 506, 526, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 523, 528, 536, 530, 495, 537, 555, 480, 498, 510, 517, 475, 485, 522, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 522, 529, 548, 503, 544, 496, 525, 518, 499, 505, 496, 473, 520, 517, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 578, 557, 520, 552, 521, 497, 508, 503, 544, 506, 513, 538, 511, 511, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 474, 525, 511, 529, 536, 470, 539, 511, 490, 489, 461, 514, 503, 496, 478, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 507, 468, 490, 470, 482, 546, 522, 519, 521, 520, 516, 532, 525, 473, 508, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8315, 8067, 8191, 8253, 8346, 8031, 8130] avg=8192.0000 max=8346 max_rank=5 min=8031 min_rank=6 imbalance_ratio=1.018799 max_load_reduction=57190 max_load_reduction_pct=0.872650 moved_to_echo=32760 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[499, 531, 492, 504, 497, 470, 535, 503, 486, 500, 539, 536, 542, 527, 475, 476, 24, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 480, 529, 537, 496, 526, 547, 464, 540, 511, 540, 519, 540, 582, 491, 524, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 470, 504, 526, 535, 516, 517, 498, 528, 504, 519, 499, 526, 516, 541, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 519, 501, 513, 495, 499, 490, 503, 490, 519, 524, 494, 503, 545, 505, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 505, 498, 531, 481, 518, 530, 521, 503, 518, 479, 505, 505, 520, 515, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 513, 474, 495, 537, 512, 511, 485, 535, 509, 543, 524, 496, 519, 527, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 535, 505, 496, 527, 512, 499, 526, 462, 496, 519, 527, 532, 526, 521, 530, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 520, 531, 541, 483, 503, 520, 532, 482, 487, 545, 447, 514, 511, 526, 514, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8152, 8339, 8202, 8136, 8158, 8177, 8210, 8162] avg=8192.0000 max=8339 max_rank=1 min=8136 min_rank=3 imbalance_ratio=1.017944 max_load_reduction=57197 max_load_reduction_pct=0.872757 moved_to_echo=32707 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[466, 494, 543, 553, 504, 550, 452, 546, 510, 516, 506, 524, 522, 513, 527, 522, 27, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 507, 499, 543, 518, 536, 503, 538, 515, 502, 557, 507, 504, 474, 490, 508, 14, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 473, 463, 510, 527, 519, 512, 508, 480, 499, 480, 549, 552, 525, 495, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 500, 487, 502, 529, 512, 519, 533, 512, 522, 490, 503, 499, 541, 514, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 517, 518, 494, 505, 525, 524, 546, 571, 529, 540, 514, 492, 518, 479, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 483, 544, 546, 519, 519, 472, 509, 479, 518, 520, 528, 490, 511, 478, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 510, 447, 497, 507, 529, 536, 503, 500, 512, 498, 486, 548, 562, 537, 492, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 530, 531, 501, 502, 518, 525, 496, 496, 483, 526, 505, 474, 536, 492, 509, 10, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8293, 8233, 8051, 8167, 8299, 8129, 8184, 8180] avg=8192.0000 max=8299 max_rank=4 min=8051 min_rank=2 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32792 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[477, 548, 533, 505, 515, 505, 519, 500, 478, 532, 497, 473, 469, 504, 488, 511, 26, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 470, 522, 526, 510, 529, 503, 493, 531, 486, 532, 483, 508, 463, 532, 512, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 498, 538, 518, 477, 540, 542, 543, 511, 513, 541, 551, 522, 481, 536, 569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 515, 468, 499, 519, 472, 524, 500, 502, 531, 529, 530, 514, 505, 489, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 480, 497, 512, 525, 477, 515, 482, 497, 500, 494, 488, 536, 475, 508, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 486, 493, 533, 594, 509, 519, 547, 518, 527, 529, 536, 537, 507, 506, 458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 528, 530, 524, 529, 497, 487, 536, 508, 546, 526, 485, 558, 491, 505, 520, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 478, 483, 528, 547, 492, 503, 497, 500, 505, 513, 520, 504, 499, 497, 553, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8096, 8117, 8377, 8157, 8055, 8281, 8298, 8155] avg=8192.0000 max=8377 max_rank=2 min=8055 min_rank=4 imbalance_ratio=1.022583 max_load_reduction=57159 max_load_reduction_pct=0.872177 moved_to_echo=32789 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[500, 515, 551, 479, 518, 546, 486, 480, 520, 526, 505, 488, 506, 523, 544, 509, 27, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 524, 510, 503, 500, 531, 508, 490, 528, 494, 494, 507, 509, 464, 496, 500, 10, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 502, 461, 537, 530, 515, 541, 494, 486, 505, 485, 518, 464, 497, 481, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 531, 514, 532, 521, 552, 527, 495, 514, 477, 458, 527, 475, 482, 561, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 505, 519, 506, 519, 507, 521, 541, 513, 478, 479, 529, 552, 535, 556, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 518, 530, 490, 504, 527, 497, 528, 503, 480, 480, 562, 564, 534, 508, 441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 493, 489, 528, 516, 499, 527, 530, 540, 536, 499, 494, 546, 524, 504, 545, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 482, 529, 529, 511, 493, 495, 530, 539, 496, 538, 529, 508, 481, 516, 518, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8073, 8052, 8188, 8308, 8179, 8265, 8233] avg=8192.0000 max=8308 max_rank=4 min=8052 min_rank=2 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32985 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[455, 506, 458, 529, 512, 498, 555, 508, 491, 480, 526, 515, 502, 544, 514, 487, 25, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 527, 521, 496, 496, 489, 514, 519, 506, 508, 539, 542, 524, 526, 479, 512, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 523, 573, 502, 533, 510, 521, 533, 491, 547, 488, 487, 532, 543, 487, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 527, 473, 519, 550, 483, 522, 509, 551, 472, 497, 532, 517, 524, 504, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 499, 527, 512, 497, 536, 529, 532, 499, 520, 510, 492, 515, 519, 507, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 461, 489, 512, 523, 494, 509, 532, 498, 520, 517, 479, 492, 530, 485, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 490, 483, 493, 512, 502, 494, 501, 551, 516, 526, 493, 478, 514, 523, 496, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 492, 484, 524, 461, 529, 527, 497, 513, 540, 545, 509, 531, 516, 495, 502, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8120, 8211, 8332, 8206, 8219, 8086, 8135, 8227] avg=8192.0000 max=8332 max_rank=2 min=8086 min_rank=5 imbalance_ratio=1.017090 max_load_reduction=57204 max_load_reduction_pct=0.872864 moved_to_echo=32667 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[498, 512, 524, 533, 531, 534, 503, 519, 512, 508, 529, 491, 532, 502, 548, 485, 23, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 507, 490, 470, 513, 509, 552, 508, 516, 475, 535, 513, 480, 513, 475, 542, 9, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 547, 510, 521, 529, 526, 502, 516, 527, 525, 496, 529, 531, 516, 528, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 531, 495, 561, 526, 519, 511, 492, 517, 489, 494, 492, 508, 509, 533, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 521, 492, 492, 512, 518, 510, 482, 484, 510, 504, 496, 484, 499, 527, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 494, 511, 502, 537, 509, 486, 507, 487, 503, 524, 500, 491, 506, 521, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 529, 493, 532, 519, 533, 527, 503, 480, 483, 536, 527, 523, 519, 512, 509, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 528, 523, 463, 515, 561, 552, 511, 499, 480, 503, 558, 501, 499, 487, 507, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8299, 8129, 8299, 8266, 8044, 8074, 8235, 8190] avg=8192.0000 max=8299 max_rank=0 min=8044 min_rank=4 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32543 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[477, 548, 533, 505, 515, 505, 519, 500, 478, 532, 497, 473, 469, 504, 488, 511, 26, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 470, 522, 526, 510, 529, 503, 493, 531, 486, 532, 483, 508, 463, 532, 512, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 498, 538, 518, 477, 540, 542, 543, 511, 513, 541, 551, 522, 481, 536, 569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 515, 468, 499, 519, 472, 524, 500, 502, 531, 529, 530, 514, 505, 489, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 480, 497, 512, 525, 477, 515, 482, 497, 500, 494, 488, 536, 475, 508, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 486, 493, 533, 594, 509, 519, 547, 518, 527, 529, 536, 537, 507, 506, 458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 528, 530, 524, 529, 497, 487, 536, 508, 546, 526, 485, 558, 491, 505, 520, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 478, 483, 528, 547, 492, 503, 497, 500, 505, 513, 520, 504, 499, 497, 553, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8096, 8117, 8377, 8157, 8055, 8281, 8298, 8155] avg=8192.0000 max=8377 max_rank=2 min=8055 min_rank=4 imbalance_ratio=1.022583 max_load_reduction=57159 max_load_reduction_pct=0.872177 moved_to_echo=32789 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[469, 498, 521, 519, 487, 500, 487, 529, 532, 475, 514, 529, 530, 552, 481, 542, 23, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 478, 489, 521, 486, 553, 535, 501, 528, 519, 520, 536, 508, 536, 513, 515, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 510, 527, 486, 504, 487, 505, 512, 548, 491, 491, 488, 514, 506, 526, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 523, 528, 536, 530, 495, 537, 555, 480, 498, 510, 517, 475, 485, 522, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 522, 529, 548, 503, 544, 496, 525, 518, 499, 505, 496, 473, 520, 517, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 578, 557, 520, 552, 521, 497, 508, 503, 544, 506, 513, 538, 511, 511, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 474, 525, 511, 529, 536, 470, 539, 511, 490, 489, 461, 514, 503, 496, 478, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 507, 468, 490, 470, 482, 546, 522, 519, 521, 520, 516, 532, 525, 473, 508, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8315, 8067, 8191, 8253, 8346, 8031, 8130] avg=8192.0000 max=8346 max_rank=5 min=8031 min_rank=6 imbalance_ratio=1.018799 max_load_reduction=57190 max_load_reduction_pct=0.872650 moved_to_echo=32760 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[466, 494, 543, 553, 504, 550, 452, 546, 510, 516, 506, 524, 522, 513, 527, 522, 27, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 507, 499, 543, 518, 536, 503, 538, 515, 502, 557, 507, 504, 474, 490, 508, 14, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 473, 463, 510, 527, 519, 512, 508, 480, 499, 480, 549, 552, 525, 495, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 500, 487, 502, 529, 512, 519, 533, 512, 522, 490, 503, 499, 541, 514, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 517, 518, 494, 505, 525, 524, 546, 571, 529, 540, 514, 492, 518, 479, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 483, 544, 546, 519, 519, 472, 509, 479, 518, 520, 528, 490, 511, 478, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 510, 447, 497, 507, 529, 536, 503, 500, 512, 498, 486, 548, 562, 537, 492, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 530, 531, 501, 502, 518, 525, 496, 496, 483, 526, 505, 474, 536, 492, 509, 10, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8293, 8233, 8051, 8167, 8299, 8129, 8184, 8180] avg=8192.0000 max=8299 max_rank=4 min=8051 min_rank=2 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32792 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[482, 530, 512, 524, 503, 461, 493, 522, 502, 545, 524, 487, 490, 514, 521, 486, 23, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 487, 506, 464, 486, 494, 472, 504, 502, 506, 539, 550, 537, 490, 559, 521, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 516, 522, 478, 532, 514, 482, 513, 478, 515, 532, 536, 513, 474, 500, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 510, 493, 558, 490, 527, 491, 490, 531, 518, 509, 542, 489, 508, 527, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 581, 514, 489, 489, 520, 522, 516, 473, 517, 502, 533, 470, 479, 539, 514, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 487, 526, 512, 484, 484, 536, 488, 545, 526, 515, 552, 536, 506, 537, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 516, 526, 514, 528, 519, 532, 533, 455, 525, 505, 485, 481, 526, 535, 507, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 510, 536, 487, 518, 503, 531, 487, 540, 519, 507, 493, 502, 521, 541, 516, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8135, 8119, 8156, 8225, 8200, 8264, 8178, 8259] avg=8192.0000 max=8264 max_rank=5 min=8119 min_rank=1 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32901 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 3968, 4224, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[3968, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 3968, 4224, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4352, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 3968, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4352, 4224, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4352, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[533, 555, 512, 540, 521, 505, 499, 481, 471, 524, 543, 557, 509, 477, 513, 510, 504, 534, 494, 513, 506, 521, 538, 514, 516, 494, 509, 497, 539, 506, 504, 519, 530, 505, 503, 511, 534, 497, 534, 513, 524, 514, 546, 542, 499, 535, 496, 558, 505, 533, 530, 529, 489, 474, 570, 503, 502, 510, 543, 476, 514, 542, 511, 507, 511, 506, 515, 525, 488, 474, 526, 500, 471, 487, 510, 504, 523, 511, 516, 530, 507, 521, 500, 528, 552, 535, 500, 502, 522, 529, 515, 512, 523, 466, 475, 498, 537, 473, 520, 467, 517, 502, 499, 556, 507, 498, 544, 479, 516, 512, 485, 522, 490, 508, 470, 522, 568, 469, 520, 519, 507, 473, 514, 477, 537, 518, 477, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[493, 503, 494, 498, 533, 508, 543, 507, 506, 540, 500, 481, 501, 532, 508, 556, 518, 495, 508, 538, 530, 538, 491, 496, 502, 516, 472, 542, 519, 529, 520, 489, 515, 482, 515, 469, 504, 487, 502, 541, 499, 497, 497, 469, 516, 523, 505, 545, 505, 513, 498, 550, 512, 480, 511, 546, 521, 494, 519, 529, 506, 507, 488, 526, 525, 530, 511, 502, 558, 493, 521, 534, 513, 522, 545, 508, 536, 539, 487, 469, 509, 492, 516, 537, 524, 537, 519, 521, 482, 503, 484, 509, 515, 507, 478, 502, 510, 505, 534, 492, 504, 528, 520, 518, 537, 521, 535, 490, 507, 523, 525, 534, 487, 538, 504, 517, 500, 525, 504, 541, 511, 536, 507, 494, 475, 509, 492, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[552, 512, 497, 538, 482, 520, 510, 488, 507, 532, 484, 523, 479, 500, 534, 511, 534, 474, 495, 487, 506, 493, 509, 492, 530, 529, 514, 498, 539, 510, 530, 539, 564, 515, 512, 521, 520, 543, 499, 524, 536, 508, 486, 488, 529, 527, 467, 472, 508, 525, 516, 504, 486, 542, 501, 524, 500, 511, 537, 509, 497, 523, 496, 544, 506, 472, 519, 496, 521, 509, 572, 554, 500, 497, 522, 508, 511, 546, 516, 506, 504, 542, 522, 508, 495, 504, 476, 530, 538, 508, 490, 493, 514, 523, 533, 497, 526, 524, 494, 476, 568, 528, 499, 526, 504, 548, 477, 492, 513, 473, 504, 533, 504, 494, 518, 504, 489, 500, 476, 469, 499, 478, 519, 551, 514, 567, 558, 497] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[499, 499, 518, 553, 520, 529, 473, 553, 518, 533, 479, 544, 516, 563, 524, 486, 506, 497, 520, 493, 505, 521, 511, 494, 520, 485, 503, 508, 535, 513, 580, 499, 499, 524, 539, 530, 496, 470, 527, 529, 528, 502, 508, 531, 519, 536, 508, 493, 511, 490, 536, 521, 494, 536, 509, 549, 514, 522, 501, 566, 507, 497, 484, 537, 478, 520, 521, 517, 524, 546, 522, 494, 509, 485, 489, 516, 480, 484, 515, 478, 501, 524, 520, 561, 510, 515, 496, 475, 504, 533, 509, 548, 493, 506, 510, 498, 514, 523, 514, 509, 491, 505, 490, 507, 499, 538, 503, 495, 524, 494, 523, 508, 489, 540, 508, 508, 526, 502, 528, 497, 484, 495, 511, 515, 492, 526, 471, 516] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[497, 536, 505, 563, 509, 526, 499, 496, 500, 499, 521, 517, 537, 494, 529, 535, 490, 506, 516, 489, 534, 467, 547, 527, 495, 493, 481, 486, 532, 522, 555, 538, 495, 513, 516, 555, 506, 512, 512, 497, 511, 512, 524, 529, 517, 490, 470, 505, 539, 510, 511, 515, 518, 497, 503, 522, 505, 496, 503, 561, 505, 484, 534, 486, 506, 504, 498, 537, 537, 478, 548, 532, 507, 532, 494, 529, 499, 500, 503, 465, 513, 513, 504, 527, 517, 472, 514, 484, 508, 501, 456, 537, 492, 512, 550, 533, 515, 499, 518, 499, 520, 549, 512, 496, 500, 530, 527, 517, 530, 513, 548, 495, 489, 542, 543, 502, 496, 519, 510, 485, 501, 496, 502, 511, 507, 544, 528, 497] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[503, 511, 543, 504, 527, 512, 502, 523, 531, 523, 520, 484, 538, 499, 514, 509, 460, 469, 482, 526, 539, 523, 483, 532, 550, 486, 513, 550, 509, 470, 554, 500, 559, 524, 511, 529, 505, 503, 498, 488, 515, 523, 495, 484, 496, 497, 489, 535, 522, 481, 524, 517, 550, 521, 555, 488, 513, 523, 469, 537, 506, 503, 501, 488, 523, 511, 531, 456, 521, 513, 470, 470, 516, 505, 542, 507, 504, 496, 535, 479, 514, 514, 512, 540, 496, 503, 486, 515, 516, 501, 534, 523, 508, 510, 507, 505, 510, 536, 504, 472, 489, 556, 533, 511, 494, 533, 501, 546, 533, 527, 497, 509, 558, 507, 490, 508, 529, 513, 496, 498, 517, 565, 550, 513, 534, 519, 491, 496] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[510, 512, 498, 501, 498, 483, 460, 521, 488, 494, 481, 519, 523, 531, 496, 543, 532, 481, 468, 537, 536, 511, 518, 535, 512, 516, 510, 499, 510, 518, 488, 485, 517, 513, 469, 519, 518, 507, 524, 526, 479, 494, 522, 513, 498, 508, 519, 471, 559, 515, 484, 528, 479, 524, 535, 494, 533, 516, 517, 526, 563, 514, 534, 519, 503, 478, 497, 506, 536, 497, 495, 510, 493, 544, 508, 513, 546, 538, 499, 531, 519, 525, 505, 504, 502, 529, 539, 510, 474, 506, 532, 509, 523, 519, 525, 513, 532, 496, 513, 515, 508, 508, 522, 503, 500, 512, 531, 524, 504, 533, 503, 529, 522, 488, 538, 498, 573, 502, 514, 515, 530, 517, 489, 519, 489, 484, 518, 528] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[497, 536, 505, 563, 509, 526, 499, 496, 500, 499, 521, 517, 537, 494, 529, 535, 490, 506, 516, 489, 534, 467, 547, 527, 495, 493, 481, 486, 532, 522, 555, 538, 495, 513, 516, 555, 506, 512, 512, 497, 511, 512, 524, 529, 517, 490, 470, 505, 539, 510, 511, 515, 518, 497, 503, 522, 505, 496, 503, 561, 505, 484, 534, 486, 506, 504, 498, 537, 537, 478, 548, 532, 507, 532, 494, 529, 499, 500, 503, 465, 513, 513, 504, 527, 517, 472, 514, 484, 508, 501, 456, 537, 492, 512, 550, 533, 515, 499, 518, 499, 520, 549, 512, 496, 500, 530, 527, 517, 530, 513, 548, 495, 489, 542, 543, 502, 496, 519, 510, 485, 501, 496, 502, 511, 507, 544, 528, 497] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[531, 549, 559, 465, 496, 518, 507, 544, 501, 482, 509, 498, 523, 488, 559, 505, 521, 527, 505, 476, 492, 478, 499, 525, 509, 550, 513, 472, 527, 509, 533, 471, 502, 529, 511, 532, 498, 494, 519, 536, 489, 511, 501, 513, 483, 515, 491, 481, 462, 536, 505, 519, 552, 518, 504, 491, 514, 509, 509, 546, 511, 522, 563, 568, 506, 499, 521, 504, 453, 500, 495, 499, 478, 525, 478, 510, 512, 471, 512, 523, 523, 506, 544, 527, 543, 466, 488, 498, 499, 509, 526, 522, 521, 529, 543, 532, 541, 526, 503, 505, 548, 500, 544, 549, 490, 532, 509, 518, 510, 476, 511, 494, 518, 538, 531, 509, 489, 498, 542, 513, 511, 488, 514, 548, 495, 532, 495, 522] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[552, 490, 497, 538, 482, 520, 510, 488, 507, 532, 484, 523, 479, 500, 534, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 474, 495, 487, 506, 493, 509, 492, 530, 529, 514, 498, 539, 510, 530, 539, 12, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 564, 515, 512, 521, 520, 543, 499, 524, 536, 508, 486, 488, 529, 527, 467, 472, 15, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 525, 516, 504, 486, 542, 501, 524, 500, 511, 537, 455, 497, 523, 496, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 472, 519, 496, 521, 509, 572, 554, 500, 497, 522, 508, 511, 546, 516, 506, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 542, 522, 508, 495, 504, 476, 530, 538, 508, 490, 493, 514, 523, 533, 497, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 524, 494, 476, 568, 528, 499, 526, 504, 521, 477, 492, 513, 473, 504, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 494, 518, 504, 489, 500, 476, 469, 499, 478, 519, 551, 514, 567, 558, 497, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8147, 8199, 8231, 8169, 8301, 8178, 8158, 8153] avg=8192.0000 max=8301 max_rank=4 min=8147 min_rank=0 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32790 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[510, 512, 498, 501, 498, 483, 460, 521, 488, 494, 481, 519, 523, 531, 496, 543, 532, 481, 468, 537, 536, 511, 518, 535, 512, 516, 510, 499, 510, 518, 488, 485, 517, 513, 469, 519, 518, 507, 524, 526, 479, 494, 522, 513, 498, 508, 519, 471, 559, 515, 484, 528, 479, 524, 535, 494, 533, 516, 517, 526, 563, 514, 534, 519, 503, 478, 497, 506, 536, 497, 495, 510, 493, 544, 508, 513, 546, 538, 499, 531, 519, 525, 505, 504, 502, 529, 539, 510, 474, 506, 532, 509, 523, 519, 525, 513, 532, 496, 513, 515, 508, 508, 522, 503, 500, 512, 531, 524, 504, 533, 503, 529, 522, 488, 538, 498, 573, 502, 514, 515, 530, 517, 489, 519, 489, 484, 518, 528] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[499, 499, 518, 553, 520, 529, 473, 553, 518, 533, 479, 544, 516, 563, 524, 486, 506, 497, 520, 493, 505, 521, 511, 494, 520, 485, 503, 508, 535, 513, 580, 499, 499, 524, 539, 530, 496, 470, 527, 529, 528, 502, 508, 531, 519, 536, 508, 493, 511, 490, 536, 521, 494, 536, 509, 549, 514, 522, 501, 566, 507, 497, 484, 537, 478, 520, 521, 517, 524, 546, 522, 494, 509, 485, 489, 516, 480, 484, 515, 478, 501, 524, 520, 561, 510, 515, 496, 475, 504, 533, 509, 548, 493, 506, 510, 498, 514, 523, 514, 509, 491, 505, 490, 507, 499, 538, 503, 495, 524, 494, 523, 508, 489, 540, 508, 508, 526, 502, 528, 497, 484, 495, 511, 515, 492, 526, 471, 516] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[552, 512, 497, 538, 482, 520, 510, 488, 507, 532, 484, 523, 479, 500, 534, 511, 534, 474, 495, 487, 506, 493, 509, 492, 530, 529, 514, 498, 539, 510, 530, 539, 564, 515, 512, 521, 520, 543, 499, 524, 536, 508, 486, 488, 529, 527, 467, 472, 508, 525, 516, 504, 486, 542, 501, 524, 500, 511, 537, 509, 497, 523, 496, 544, 506, 472, 519, 496, 521, 509, 572, 554, 500, 497, 522, 508, 511, 546, 516, 506, 504, 542, 522, 508, 495, 504, 476, 530, 538, 508, 490, 493, 514, 523, 533, 497, 526, 524, 494, 476, 568, 528, 499, 526, 504, 548, 477, 492, 513, 473, 504, 533, 504, 494, 518, 504, 489, 500, 476, 469, 499, 478, 519, 551, 514, 567, 558, 497] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[493, 503, 494, 498, 533, 508, 543, 507, 506, 540, 500, 481, 501, 532, 508, 556, 518, 495, 508, 538, 530, 538, 491, 496, 502, 516, 472, 542, 519, 529, 520, 489, 515, 482, 515, 469, 504, 487, 502, 541, 499, 497, 497, 469, 516, 523, 505, 545, 505, 513, 498, 550, 512, 480, 511, 546, 521, 494, 519, 529, 506, 507, 488, 526, 525, 530, 511, 502, 558, 493, 521, 534, 513, 522, 545, 508, 536, 539, 487, 469, 509, 492, 516, 537, 524, 537, 519, 521, 482, 503, 484, 509, 515, 507, 478, 502, 510, 505, 534, 492, 504, 528, 520, 518, 537, 521, 535, 490, 507, 523, 525, 534, 487, 538, 504, 517, 500, 525, 504, 541, 511, 536, 507, 494, 475, 509, 492, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[533, 531, 512, 540, 521, 505, 499, 481, 471, 524, 543, 557, 509, 477, 513, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 534, 494, 513, 506, 521, 538, 514, 516, 494, 509, 497, 539, 506, 504, 519, 11, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 505, 503, 511, 534, 497, 534, 513, 524, 514, 546, 542, 499, 535, 496, 558, 13, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 533, 530, 529, 489, 474, 570, 503, 502, 510, 543, 425, 514, 542, 511, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 506, 515, 525, 488, 474, 526, 500, 471, 487, 510, 504, 523, 511, 516, 530, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 521, 500, 528, 552, 535, 500, 502, 522, 529, 515, 512, 523, 466, 475, 498, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 473, 520, 467, 517, 502, 499, 556, 507, 474, 544, 479, 516, 512, 485, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 508, 470, 522, 568, 469, 520, 519, 507, 473, 514, 477, 537, 518, 477, 514, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 8227, 8359, 8187, 8140, 8186, 8110, 8101] avg=8192.0000 max=8359 max_rank=2 min=8101 min_rank=7 imbalance_ratio=1.020386 max_load_reduction=57177 max_load_reduction_pct=0.872452 moved_to_echo=32537 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[493, 481, 494, 498, 533, 508, 543, 507, 506, 540, 500, 481, 501, 532, 508, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 495, 508, 538, 530, 538, 491, 496, 502, 516, 472, 542, 519, 529, 520, 489, 11, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 482, 515, 469, 504, 487, 502, 541, 499, 497, 497, 469, 516, 523, 505, 545, 14, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 513, 498, 550, 512, 480, 511, 546, 521, 494, 519, 473, 506, 507, 488, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 530, 511, 502, 558, 493, 521, 534, 513, 522, 545, 508, 536, 539, 487, 469, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 492, 516, 537, 524, 537, 519, 521, 482, 503, 484, 509, 515, 507, 478, 502, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 505, 534, 492, 504, 528, 520, 518, 537, 496, 535, 490, 507, 523, 525, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 538, 504, 517, 500, 525, 504, 541, 511, 536, 507, 494, 475, 509, 492, 508, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8223, 8085, 8149, 8340, 8136, 8258, 8164] avg=8192.0000 max=8340 max_rank=4 min=8085 min_rank=2 imbalance_ratio=1.018066 max_load_reduction=57196 max_load_reduction_pct=0.872742 moved_to_echo=32898 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[499, 467, 518, 553, 520, 529, 473, 553, 518, 533, 479, 544, 516, 563, 524, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 497, 520, 493, 505, 521, 511, 494, 520, 485, 503, 508, 535, 513, 580, 499, 16, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 524, 539, 530, 496, 470, 527, 529, 528, 502, 508, 531, 519, 536, 508, 493, 19, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 490, 536, 521, 494, 536, 509, 549, 514, 522, 501, 497, 507, 497, 484, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 520, 521, 517, 524, 546, 522, 494, 509, 485, 489, 516, 480, 484, 515, 478, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 524, 520, 561, 510, 515, 496, 475, 504, 533, 509, 548, 493, 506, 510, 498, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 523, 514, 509, 491, 505, 490, 507, 499, 503, 503, 495, 524, 494, 523, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 540, 508, 508, 526, 502, 528, 497, 484, 495, 511, 515, 492, 526, 471, 516, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8275, 8220, 8268, 8205, 8133, 8205, 8102, 8128] avg=8192.0000 max=8275 max_rank=0 min=8102 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32568 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[503, 489, 543, 504, 527, 512, 502, 523, 531, 523, 520, 484, 538, 499, 514, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 460, 469, 482, 526, 539, 523, 483, 532, 550, 486, 513, 550, 509, 470, 554, 500, 11, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 524, 511, 529, 505, 503, 498, 488, 515, 523, 495, 484, 496, 497, 489, 535, 14, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 481, 524, 517, 550, 521, 555, 488, 513, 523, 469, 480, 506, 503, 501, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 511, 531, 456, 521, 513, 470, 470, 516, 505, 542, 507, 504, 496, 535, 479, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 514, 512, 540, 496, 503, 486, 515, 516, 501, 534, 523, 508, 510, 507, 505, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 536, 504, 472, 489, 556, 533, 511, 494, 508, 501, 546, 533, 527, 497, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 558, 507, 490, 508, 529, 513, 496, 498, 517, 565, 550, 513, 534, 519, 491, 496, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8166, 8170, 8141, 8127, 8185, 8226, 8300] avg=8192.0000 max=8300 max_rank=7 min=8127 min_rank=4 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32838 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[510, 490, 498, 501, 498, 483, 460, 521, 488, 494, 481, 519, 523, 531, 496, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 481, 468, 537, 536, 511, 518, 535, 512, 516, 510, 499, 510, 518, 488, 485, 11, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 513, 469, 519, 518, 507, 524, 526, 479, 494, 522, 513, 498, 508, 519, 471, 14, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 515, 484, 528, 479, 524, 535, 494, 533, 516, 517, 470, 563, 514, 534, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 478, 497, 506, 536, 497, 495, 510, 493, 544, 508, 513, 546, 538, 499, 531, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 525, 505, 504, 502, 529, 539, 510, 474, 506, 532, 509, 523, 519, 525, 513, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 496, 513, 515, 508, 508, 522, 503, 500, 487, 531, 524, 504, 533, 503, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 488, 538, 498, 573, 502, 514, 515, 530, 517, 489, 519, 489, 484, 518, 528, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8036, 8176, 8116, 8284, 8241, 8235, 8208, 8240] avg=8192.0000 max=8284 max_rank=3 min=8036 min_rank=0 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32924 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[497, 513, 505, 563, 509, 526, 499, 496, 500, 499, 521, 517, 537, 494, 529, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 506, 516, 489, 534, 467, 547, 527, 495, 493, 481, 486, 532, 522, 555, 538, 11, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 513, 516, 555, 506, 512, 512, 497, 511, 512, 524, 529, 517, 490, 470, 505, 14, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 510, 511, 515, 518, 497, 503, 522, 505, 496, 503, 502, 505, 484, 534, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 504, 498, 537, 537, 478, 548, 532, 507, 532, 494, 529, 499, 500, 503, 465, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 513, 504, 527, 517, 472, 514, 484, 508, 501, 456, 537, 492, 512, 550, 533, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 499, 518, 499, 520, 549, 512, 496, 500, 505, 527, 517, 530, 513, 548, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 542, 543, 502, 496, 519, 510, 485, 501, 496, 502, 511, 507, 544, 528, 497, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8240, 8198, 8183, 8130, 8219, 8134, 8243, 8189] avg=8192.0000 max=8243 max_rank=6 min=8130 min_rank=3 imbalance_ratio=1.006226 max_load_reduction=57293 max_load_reduction_pct=0.874222 moved_to_echo=32785 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[499, 467, 518, 553, 520, 529, 473, 553, 518, 533, 479, 544, 516, 563, 524, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 497, 520, 493, 505, 521, 511, 494, 520, 485, 503, 508, 535, 513, 580, 499, 16, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 524, 539, 530, 496, 470, 527, 529, 528, 502, 508, 531, 519, 536, 508, 493, 19, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 490, 536, 521, 494, 536, 509, 549, 514, 522, 501, 497, 507, 497, 484, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 520, 521, 517, 524, 546, 522, 494, 509, 485, 489, 516, 480, 484, 515, 478, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 524, 520, 561, 510, 515, 496, 475, 504, 533, 509, 548, 493, 506, 510, 498, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 523, 514, 509, 491, 505, 490, 507, 499, 503, 503, 495, 524, 494, 523, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 540, 508, 508, 526, 502, 528, 497, 484, 495, 511, 515, 492, 526, 471, 516, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8275, 8220, 8268, 8205, 8133, 8205, 8102, 8128] avg=8192.0000 max=8275 max_rank=0 min=8102 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32568 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[531, 525, 559, 465, 496, 518, 507, 544, 501, 482, 509, 498, 523, 488, 559, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 527, 505, 476, 492, 478, 499, 525, 509, 550, 513, 472, 527, 509, 533, 471, 11, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 529, 511, 532, 498, 494, 519, 536, 489, 511, 501, 513, 483, 515, 491, 481, 14, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 536, 505, 519, 552, 518, 504, 491, 514, 509, 509, 488, 511, 522, 563, 568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 499, 521, 504, 453, 500, 495, 499, 478, 525, 478, 510, 512, 471, 512, 523, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 506, 544, 527, 543, 466, 488, 498, 499, 509, 526, 522, 521, 529, 543, 532, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 526, 503, 505, 548, 500, 544, 549, 490, 507, 509, 518, 510, 476, 511, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 538, 531, 509, 489, 498, 542, 513, 511, 488, 514, 548, 495, 532, 495, 522, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8127, 8124, 8271, 8035, 8277, 8231, 8261] avg=8192.0000 max=8277 max_rank=5 min=8035 min_rank=4 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32804 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[497, 513, 505, 563, 509, 526, 499, 496, 500, 499, 521, 517, 537, 494, 529, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 506, 516, 489, 534, 467, 547, 527, 495, 493, 481, 486, 532, 522, 555, 538, 11, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 513, 516, 555, 506, 512, 512, 497, 511, 512, 524, 529, 517, 490, 470, 505, 14, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 510, 511, 515, 518, 497, 503, 522, 505, 496, 503, 502, 505, 484, 534, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 504, 498, 537, 537, 478, 548, 532, 507, 532, 494, 529, 499, 500, 503, 465, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 513, 504, 527, 517, 472, 514, 484, 508, 501, 456, 537, 492, 512, 550, 533, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 499, 518, 499, 520, 549, 512, 496, 500, 505, 527, 517, 530, 513, 548, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 542, 543, 502, 496, 519, 510, 485, 501, 496, 502, 511, 507, 544, 528, 497, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8240, 8198, 8183, 8130, 8219, 8134, 8243, 8189] avg=8192.0000 max=8243 max_rank=6 min=8130 min_rank=3 imbalance_ratio=1.006226 max_load_reduction=57293 max_load_reduction_pct=0.874222 moved_to_echo=32785 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[552, 490, 497, 538, 482, 520, 510, 488, 507, 532, 484, 523, 479, 500, 534, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 474, 495, 487, 506, 493, 509, 492, 530, 529, 514, 498, 539, 510, 530, 539, 12, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 564, 515, 512, 521, 520, 543, 499, 524, 536, 508, 486, 488, 529, 527, 467, 472, 15, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 525, 516, 504, 486, 542, 501, 524, 500, 511, 537, 455, 497, 523, 496, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 472, 519, 496, 521, 509, 572, 554, 500, 497, 522, 508, 511, 546, 516, 506, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 542, 522, 508, 495, 504, 476, 530, 538, 508, 490, 493, 514, 523, 533, 497, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 524, 494, 476, 568, 528, 499, 526, 504, 521, 477, 492, 513, 473, 504, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 494, 518, 504, 489, 500, 476, 469, 499, 478, 519, 551, 514, 567, 558, 497, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8147, 8199, 8231, 8169, 8301, 8178, 8158, 8153] avg=8192.0000 max=8301 max_rank=4 min=8147 min_rank=0 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32790 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[493, 481, 494, 498, 533, 508, 543, 507, 506, 540, 500, 481, 501, 532, 508, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 495, 508, 538, 530, 538, 491, 496, 502, 516, 472, 542, 519, 529, 520, 489, 11, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 482, 515, 469, 504, 487, 502, 541, 499, 497, 497, 469, 516, 523, 505, 545, 14, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 513, 498, 550, 512, 480, 511, 546, 521, 494, 519, 473, 506, 507, 488, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 530, 511, 502, 558, 493, 521, 534, 513, 522, 545, 508, 536, 539, 487, 469, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 492, 516, 537, 524, 537, 519, 521, 482, 503, 484, 509, 515, 507, 478, 502, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 505, 534, 492, 504, 528, 520, 518, 537, 496, 535, 490, 507, 523, 525, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 538, 504, 517, 500, 525, 504, 541, 511, 536, 507, 494, 475, 509, 492, 508, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8223, 8085, 8149, 8340, 8136, 8258, 8164] avg=8192.0000 max=8340 max_rank=4 min=8085 min_rank=2 imbalance_ratio=1.018066 max_load_reduction=57196 max_load_reduction_pct=0.872742 moved_to_echo=32898 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4352, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4352, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 3840, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4352, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 3968, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 3968, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[493, 553, 530, 507, 475, 524, 542, 499, 536, 560, 506, 532, 513, 471, 495, 502, 528, 534, 508, 493, 513, 521, 515, 470, 551, 506, 504, 477, 506, 497, 519, 518, 541, 503, 489, 509, 484, 484, 488, 512, 486, 497, 523, 517, 494, 528, 518, 504, 510, 521, 504, 514, 510, 512, 491, 520, 499, 537, 467, 517, 513, 488, 466, 558, 509, 503, 508, 499, 504, 503, 507, 513, 547, 497, 525, 487, 520, 500, 496, 529, 548, 527, 487, 544, 495, 544, 517, 532, 491, 522, 521, 486, 528, 510, 531, 502, 485, 514, 524, 519, 524, 557, 505, 522, 548, 526, 533, 551, 519, 537, 474, 523, 489, 493, 519, 476, 535, 492, 519, 501, 557, 497, 495, 497, 499, 529, 562, 481] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[514, 535, 477, 567, 486, 515, 536, 510, 517, 488, 511, 513, 488, 558, 492, 486, 524, 508, 508, 510, 552, 538, 543, 525, 522, 532, 513, 536, 520, 530, 470, 545, 521, 476, 530, 537, 514, 525, 492, 467, 518, 509, 506, 499, 538, 496, 507, 523, 500, 483, 495, 524, 504, 499, 509, 498, 534, 529, 507, 543, 500, 543, 524, 468, 492, 567, 513, 528, 495, 495, 480, 521, 529, 509, 490, 508, 520, 533, 493, 484, 510, 508, 509, 526, 485, 506, 508, 500, 511, 494, 512, 552, 519, 492, 516, 525, 519, 512, 518, 445, 500, 533, 490, 518, 518, 498, 479, 478, 511, 503, 492, 520, 504, 507, 553, 527, 496, 517, 511, 510, 521, 537, 504, 528, 506, 522, 519, 523] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[465, 497, 522, 454, 504, 529, 519, 514, 533, 476, 513, 527, 506, 530, 547, 502, 540, 475, 518, 498, 571, 521, 542, 522, 531, 494, 506, 495, 514, 519, 508, 539, 506, 491, 469, 475, 498, 518, 502, 522, 488, 520, 511, 534, 510, 528, 544, 523, 496, 537, 498, 491, 481, 516, 512, 504, 527, 504, 504, 520, 506, 524, 451, 507, 533, 541, 506, 539, 509, 492, 540, 485, 532, 501, 518, 503, 478, 534, 548, 523, 535, 521, 539, 492, 541, 533, 525, 541, 538, 523, 476, 528, 517, 530, 490, 496, 503, 527, 467, 544, 517, 516, 487, 498, 500, 530, 491, 537, 535, 527, 513, 495, 543, 525, 490, 496, 528, 471, 517, 486, 526, 484, 493, 553, 519, 488, 484, 491] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[510, 522, 540, 499, 547, 506, 500, 479, 510, 507, 528, 535, 483, 504, 515, 480, 523, 483, 532, 516, 506, 504, 514, 494, 493, 533, 556, 530, 543, 512, 540, 484, 537, 505, 514, 496, 504, 499, 511, 512, 520, 499, 484, 513, 537, 487, 507, 496, 530, 491, 530, 513, 522, 527, 520, 521, 520, 527, 470, 506, 505, 547, 548, 527, 513, 519, 519, 542, 491, 506, 528, 515, 508, 496, 500, 491, 529, 514, 513, 527, 526, 514, 477, 495, 497, 530, 477, 487, 466, 513, 501, 499, 494, 502, 498, 499, 524, 523, 527, 494, 487, 493, 536, 544, 507, 555, 518, 532, 524, 513, 474, 513, 473, 533, 506, 532, 522, 516, 503, 493, 513, 514, 532, 511, 506, 537, 532, 510] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[528, 526, 531, 476, 495, 519, 525, 516, 489, 532, 525, 509, 530, 558, 538, 537, 532, 495, 526, 517, 534, 518, 524, 541, 565, 498, 519, 493, 530, 517, 535, 565, 526, 526, 489, 497, 500, 529, 515, 537, 524, 516, 516, 529, 509, 502, 467, 491, 484, 497, 511, 548, 481, 532, 507, 507, 514, 461, 478, 519, 470, 546, 501, 493, 501, 505, 539, 489, 527, 476, 543, 494, 484, 507, 513, 513, 510, 501, 512, 516, 541, 474, 518, 455, 486, 532, 515, 526, 460, 543, 520, 529, 548, 487, 540, 534, 461, 478, 530, 528, 467, 498, 497, 517, 531, 525, 507, 514, 548, 503, 538, 494, 552, 494, 510, 492, 496, 534, 509, 502, 458, 520, 515, 517, 519, 499, 487, 493] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[511, 504, 517, 518, 484, 486, 543, 524, 510, 498, 489, 537, 484, 466, 519, 531, 527, 520, 501, 516, 522, 515, 544, 499, 504, 517, 491, 516, 525, 492, 517, 543, 489, 495, 517, 507, 511, 510, 511, 486, 499, 497, 517, 525, 528, 505, 533, 498, 498, 552, 517, 510, 486, 500, 554, 520, 544, 479, 478, 515, 546, 556, 509, 508, 506, 517, 522, 509, 515, 535, 519, 481, 516, 540, 495, 505, 498, 520, 531, 530, 496, 511, 511, 533, 503, 502, 502, 518, 501, 480, 515, 514, 551, 507, 528, 502, 448, 522, 503, 533, 499, 494, 486, 538, 500, 480, 547, 507, 527, 499, 489, 554, 499, 534, 512, 485, 525, 532, 528, 503, 537, 516, 523, 525, 486, 482, 490, 550] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[514, 535, 477, 567, 486, 515, 536, 510, 517, 488, 511, 513, 488, 558, 492, 486, 524, 508, 508, 510, 552, 538, 543, 525, 522, 532, 513, 536, 520, 530, 470, 545, 521, 476, 530, 537, 514, 525, 492, 467, 518, 509, 506, 499, 538, 496, 507, 523, 500, 483, 495, 524, 504, 499, 509, 498, 534, 529, 507, 543, 500, 543, 524, 468, 492, 567, 513, 528, 495, 495, 480, 521, 529, 509, 490, 508, 520, 533, 493, 484, 510, 508, 509, 526, 485, 506, 508, 500, 511, 494, 512, 552, 519, 492, 516, 525, 519, 512, 518, 445, 500, 533, 490, 518, 518, 498, 479, 478, 511, 503, 492, 520, 504, 507, 553, 527, 496, 517, 511, 510, 521, 537, 504, 528, 506, 522, 519, 523] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[492, 482, 543, 503, 515, 500, 517, 477, 536, 494, 484, 554, 499, 526, 499, 477, 468, 507, 519, 501, 485, 510, 495, 481, 510, 498, 525, 544, 530, 507, 520, 529, 524, 479, 507, 517, 545, 528, 519, 497, 495, 522, 502, 524, 526, 483, 504, 523, 477, 510, 524, 515, 504, 522, 533, 483, 492, 507, 520, 483, 528, 516, 504, 568, 507, 553, 519, 516, 486, 536, 516, 492, 563, 498, 510, 480, 525, 513, 509, 517, 509, 494, 555, 565, 518, 532, 498, 505, 506, 488, 533, 530, 545, 504, 491, 513, 495, 520, 539, 506, 495, 506, 470, 526, 515, 536, 537, 538, 519, 536, 510, 522, 531, 493, 553, 485, 521, 459, 494, 511, 492, 551, 509, 519, 559, 499, 469, 487] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[528, 526, 531, 476, 495, 519, 525, 516, 489, 532, 525, 509, 530, 558, 538, 537, 532, 495, 526, 517, 534, 518, 524, 541, 565, 498, 519, 493, 530, 517, 535, 565, 526, 526, 489, 497, 500, 529, 515, 537, 524, 516, 516, 529, 509, 502, 467, 491, 484, 497, 511, 548, 481, 532, 507, 507, 514, 461, 478, 519, 470, 546, 501, 493, 501, 505, 539, 489, 527, 476, 543, 494, 484, 507, 513, 513, 510, 501, 512, 516, 541, 474, 518, 455, 486, 532, 515, 526, 460, 543, 520, 529, 548, 487, 540, 534, 461, 478, 530, 528, 467, 498, 497, 517, 531, 525, 507, 514, 548, 503, 538, 494, 552, 494, 510, 492, 496, 534, 509, 502, 458, 520, 515, 517, 519, 499, 487, 493] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[493, 553, 530, 507, 475, 524, 542, 499, 536, 560, 506, 532, 513, 471, 495, 502, 528, 534, 508, 493, 513, 521, 515, 470, 551, 506, 504, 477, 506, 497, 519, 518, 541, 503, 489, 509, 484, 484, 488, 512, 486, 497, 523, 517, 494, 528, 518, 504, 510, 521, 504, 514, 510, 512, 491, 520, 499, 537, 467, 517, 513, 488, 466, 558, 509, 503, 508, 499, 504, 503, 507, 513, 547, 497, 525, 487, 520, 500, 496, 529, 548, 527, 487, 544, 495, 544, 517, 532, 491, 522, 521, 486, 528, 510, 531, 502, 485, 514, 524, 519, 524, 557, 505, 522, 548, 526, 533, 551, 519, 537, 474, 523, 489, 493, 519, 476, 535, 492, 519, 501, 557, 497, 495, 497, 499, 529, 562, 481] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[530, 554, 491, 537, 509, 470, 533, 518, 496, 526, 501, 512, 482, 522, 536, 526, 524, 498, 489, 563, 541, 548, 497, 524, 560, 452, 492, 538, 490, 537, 491, 498, 509, 518, 547, 537, 507, 537, 503, 507, 476, 481, 511, 513, 499, 509, 507, 488, 494, 491, 479, 536, 505, 516, 516, 530, 543, 500, 529, 504, 502, 534, 503, 519, 463, 523, 532, 518, 555, 505, 493, 468, 506, 500, 495, 523, 534, 528, 506, 491, 539, 547, 511, 484, 477, 514, 547, 477, 535, 525, 487, 477, 524, 487, 536, 523, 495, 511, 524, 516, 471, 531, 521, 518, 492, 550, 476, 497, 520, 485, 500, 518, 530, 532, 501, 505, 493, 534, 505, 522, 540, 528, 484, 536, 492, 527, 510, 507] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[492, 482, 543, 503, 515, 500, 517, 477, 536, 494, 484, 554, 499, 526, 499, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 507, 519, 501, 485, 510, 495, 481, 441, 498, 525, 544, 530, 507, 520, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 479, 507, 517, 545, 528, 519, 497, 495, 522, 502, 524, 526, 483, 504, 523, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 510, 524, 515, 504, 522, 533, 483, 492, 507, 520, 483, 528, 516, 504, 568, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 553, 519, 516, 486, 536, 516, 492, 563, 498, 510, 480, 525, 513, 509, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 494, 555, 565, 518, 532, 498, 505, 506, 488, 533, 530, 536, 504, 491, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 520, 539, 506, 495, 506, 470, 526, 515, 536, 537, 538, 519, 536, 510, 522, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 493, 553, 485, 521, 459, 494, 511, 492, 551, 509, 519, 559, 499, 469, 487, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8098, 8060, 8242, 8205, 8240, 8277, 8273, 8141] avg=8192.0000 max=8277 max_rank=5 min=8060 min_rank=1 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32931 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[465, 497, 522, 454, 504, 529, 519, 514, 533, 476, 513, 527, 506, 530, 547, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 475, 518, 498, 571, 521, 542, 522, 460, 494, 506, 495, 514, 519, 508, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 491, 469, 475, 498, 518, 502, 522, 488, 520, 511, 534, 510, 528, 544, 523, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 537, 498, 491, 481, 516, 512, 504, 527, 504, 504, 520, 506, 524, 451, 507, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 541, 506, 539, 509, 492, 540, 485, 532, 501, 518, 503, 478, 534, 548, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 521, 539, 492, 541, 533, 525, 541, 538, 523, 476, 528, 508, 530, 490, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 527, 467, 544, 517, 516, 487, 498, 500, 530, 491, 537, 535, 527, 513, 495, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 525, 490, 496, 528, 471, 517, 486, 526, 484, 493, 553, 519, 488, 484, 491, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8138, 8222, 8188, 8097, 8282, 8316, 8190, 8103] avg=8192.0000 max=8316 max_rank=5 min=8097 min_rank=3 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32891 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[510, 522, 540, 499, 547, 506, 500, 479, 510, 507, 528, 535, 483, 504, 515, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 483, 532, 516, 506, 504, 514, 494, 426, 533, 556, 530, 543, 512, 540, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 505, 514, 496, 504, 499, 511, 512, 520, 499, 484, 513, 537, 487, 507, 496, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 491, 530, 513, 522, 527, 520, 521, 520, 527, 470, 506, 505, 547, 548, 527, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 519, 519, 542, 491, 506, 528, 515, 508, 496, 500, 491, 529, 514, 513, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 514, 477, 495, 497, 530, 477, 487, 466, 513, 501, 499, 486, 502, 498, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 523, 527, 494, 487, 493, 536, 544, 507, 555, 518, 532, 524, 513, 474, 513, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 533, 506, 532, 522, 516, 503, 493, 513, 514, 532, 511, 506, 537, 532, 510, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8165, 8196, 8167, 8322, 8211, 7967, 8267, 8241] avg=8192.0000 max=8322 max_rank=3 min=7967 min_rank=5 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32686 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[514, 535, 477, 567, 486, 515, 536, 510, 517, 488, 511, 513, 488, 558, 492, 486, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 508, 508, 510, 552, 538, 543, 525, 439, 532, 513, 536, 520, 530, 470, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 476, 530, 537, 514, 525, 492, 467, 518, 509, 506, 499, 538, 496, 507, 523, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 483, 495, 524, 504, 499, 509, 498, 534, 529, 507, 543, 500, 543, 524, 468, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 557, 513, 528, 495, 495, 480, 521, 529, 509, 490, 508, 520, 533, 493, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 508, 509, 526, 485, 506, 508, 500, 511, 494, 512, 552, 501, 492, 516, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 512, 518, 445, 500, 533, 490, 518, 518, 498, 479, 478, 511, 503, 492, 520, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 507, 553, 527, 496, 517, 511, 510, 521, 537, 504, 528, 506, 522, 519, 523, 10, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8293, 8210, 8183, 8147, 8155, 8046, 8303] avg=8192.0000 max=8303 max_rank=7 min=8046 min_rank=6 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32651 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[510, 522, 540, 499, 547, 506, 500, 479, 510, 507, 528, 535, 483, 504, 515, 480, 523, 483, 532, 516, 506, 504, 514, 494, 493, 533, 556, 530, 543, 512, 540, 484, 537, 505, 514, 496, 504, 499, 511, 512, 520, 499, 484, 513, 537, 487, 507, 496, 530, 491, 530, 513, 522, 527, 520, 521, 520, 527, 470, 506, 505, 547, 548, 527, 513, 519, 519, 542, 491, 506, 528, 515, 508, 496, 500, 491, 529, 514, 513, 527, 526, 514, 477, 495, 497, 530, 477, 487, 466, 513, 501, 499, 494, 502, 498, 499, 524, 523, 527, 494, 487, 493, 536, 544, 507, 555, 518, 532, 524, 513, 474, 513, 473, 533, 506, 532, 522, 516, 503, 493, 513, 514, 532, 511, 506, 537, 532, 510] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[493, 553, 530, 507, 475, 524, 542, 499, 536, 560, 506, 532, 513, 471, 495, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 534, 508, 493, 513, 521, 515, 470, 477, 506, 504, 477, 506, 497, 519, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 503, 489, 509, 484, 484, 488, 512, 486, 497, 523, 517, 494, 528, 518, 504, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 521, 504, 514, 510, 512, 491, 520, 499, 537, 467, 517, 513, 488, 466, 558, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 503, 508, 499, 504, 503, 507, 513, 547, 497, 525, 487, 520, 500, 496, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 527, 487, 544, 495, 544, 517, 532, 491, 522, 521, 486, 519, 510, 531, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 514, 524, 519, 524, 557, 505, 522, 548, 526, 533, 551, 519, 537, 474, 523, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 493, 519, 476, 535, 492, 519, 501, 557, 497, 495, 497, 499, 529, 562, 481, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8086, 8128, 8147, 8147, 8276, 8364, 8150] avg=8192.0000 max=8364 max_rank=6 min=8086 min_rank=1 imbalance_ratio=1.020996 max_load_reduction=57172 max_load_reduction_pct=0.872375 moved_to_echo=32937 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[514, 535, 477, 567, 486, 515, 536, 510, 517, 488, 511, 513, 488, 558, 492, 486, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 508, 508, 510, 552, 538, 543, 525, 439, 532, 513, 536, 520, 530, 470, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 476, 530, 537, 514, 525, 492, 467, 518, 509, 506, 499, 538, 496, 507, 523, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 483, 495, 524, 504, 499, 509, 498, 534, 529, 507, 543, 500, 543, 524, 468, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 557, 513, 528, 495, 495, 480, 521, 529, 509, 490, 508, 520, 533, 493, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 508, 509, 526, 485, 506, 508, 500, 511, 494, 512, 552, 501, 492, 516, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 512, 518, 445, 500, 533, 490, 518, 518, 498, 479, 478, 511, 503, 492, 520, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 507, 553, 527, 496, 517, 511, 510, 521, 537, 504, 528, 506, 522, 519, 523, 10, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8293, 8210, 8183, 8147, 8155, 8046, 8303] avg=8192.0000 max=8303 max_rank=7 min=8046 min_rank=6 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32651 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[465, 497, 522, 454, 504, 529, 519, 514, 533, 476, 513, 527, 506, 530, 547, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 475, 518, 498, 571, 521, 542, 522, 460, 494, 506, 495, 514, 519, 508, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 491, 469, 475, 498, 518, 502, 522, 488, 520, 511, 534, 510, 528, 544, 523, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 537, 498, 491, 481, 516, 512, 504, 527, 504, 504, 520, 506, 524, 451, 507, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 541, 506, 539, 509, 492, 540, 485, 532, 501, 518, 503, 478, 534, 548, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 521, 539, 492, 541, 533, 525, 541, 538, 523, 476, 528, 508, 530, 490, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 527, 467, 544, 517, 516, 487, 498, 500, 530, 491, 537, 535, 527, 513, 495, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 525, 490, 496, 528, 471, 517, 486, 526, 484, 493, 553, 519, 488, 484, 491, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8138, 8222, 8188, 8097, 8282, 8316, 8190, 8103] avg=8192.0000 max=8316 max_rank=5 min=8097 min_rank=3 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32891 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[511, 504, 517, 518, 484, 486, 543, 524, 510, 498, 489, 537, 484, 466, 519, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 520, 501, 516, 522, 515, 544, 499, 436, 517, 491, 516, 525, 492, 517, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 495, 517, 507, 511, 510, 511, 486, 499, 497, 517, 525, 528, 505, 533, 498, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 552, 517, 510, 486, 500, 554, 520, 544, 479, 478, 515, 546, 556, 509, 508, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 517, 522, 509, 515, 535, 519, 481, 516, 540, 495, 505, 498, 520, 531, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 511, 511, 533, 503, 502, 502, 518, 501, 480, 515, 514, 542, 507, 528, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 448, 522, 503, 533, 499, 494, 486, 538, 500, 480, 547, 507, 527, 499, 489, 554, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 534, 512, 485, 525, 532, 528, 503, 537, 516, 523, 525, 486, 482, 490, 550, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8121, 8181, 8175, 8290, 8239, 8165, 8129, 8236] avg=8192.0000 max=8290 max_rank=3 min=8121 min_rank=0 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32769 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[528, 526, 531, 476, 495, 519, 525, 516, 489, 532, 525, 509, 530, 558, 538, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 495, 526, 517, 534, 518, 524, 541, 488, 498, 519, 493, 530, 517, 535, 565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 526, 489, 497, 500, 529, 515, 537, 524, 516, 516, 529, 509, 502, 467, 491, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 497, 511, 548, 481, 532, 507, 507, 514, 461, 478, 519, 470, 546, 501, 493, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 505, 539, 489, 527, 476, 543, 494, 484, 507, 513, 513, 510, 501, 512, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 474, 518, 455, 486, 532, 515, 526, 460, 543, 520, 529, 539, 487, 540, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 478, 530, 528, 467, 498, 497, 517, 531, 525, 507, 514, 548, 503, 538, 494, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 494, 510, 492, 496, 534, 509, 502, 458, 520, 515, 517, 519, 499, 487, 493, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8334, 8332, 8225, 8070, 8130, 8199, 8139, 8107] avg=8192.0000 max=8334 max_rank=0 min=8070 min_rank=3 imbalance_ratio=1.017334 max_load_reduction=57202 max_load_reduction_pct=0.872833 moved_to_echo=32575 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[530, 554, 491, 537, 509, 470, 533, 518, 496, 526, 501, 512, 482, 522, 536, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 498, 489, 563, 541, 548, 497, 524, 484, 452, 492, 538, 490, 537, 491, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 518, 547, 537, 507, 537, 503, 507, 476, 481, 511, 513, 499, 509, 507, 488, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 491, 479, 536, 505, 516, 516, 530, 543, 500, 529, 504, 502, 534, 503, 519, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 463, 523, 532, 518, 555, 505, 493, 468, 506, 500, 495, 523, 534, 528, 506, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 547, 511, 484, 477, 514, 547, 477, 535, 525, 487, 477, 515, 487, 536, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 511, 524, 516, 471, 531, 521, 518, 492, 550, 476, 497, 520, 485, 500, 518, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 532, 501, 505, 493, 534, 505, 522, 540, 528, 484, 536, 492, 527, 510, 507, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8243, 8166, 8201, 8222, 8140, 8181, 8128, 8255] avg=8192.0000 max=8255 max_rank=7 min=8128 min_rank=6 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32704 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[492, 482, 543, 503, 515, 500, 517, 477, 536, 494, 484, 554, 499, 526, 499, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 507, 519, 501, 485, 510, 495, 481, 441, 498, 525, 544, 530, 507, 520, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 479, 507, 517, 545, 528, 519, 497, 495, 522, 502, 524, 526, 483, 504, 523, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 510, 524, 515, 504, 522, 533, 483, 492, 507, 520, 483, 528, 516, 504, 568, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 553, 519, 516, 486, 536, 516, 492, 563, 498, 510, 480, 525, 513, 509, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 494, 555, 565, 518, 532, 498, 505, 506, 488, 533, 530, 536, 504, 491, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 520, 539, 506, 495, 506, 470, 526, 515, 536, 537, 538, 519, 536, 510, 522, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 493, 553, 485, 521, 459, 494, 511, 492, 551, 509, 519, 559, 499, 469, 487, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8098, 8060, 8242, 8205, 8240, 8277, 8273, 8141] avg=8192.0000 max=8277 max_rank=5 min=8060 min_rank=1 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32931 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[528, 526, 531, 476, 495, 519, 525, 516, 489, 532, 525, 509, 530, 558, 538, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 495, 526, 517, 534, 518, 524, 541, 488, 498, 519, 493, 530, 517, 535, 565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 526, 489, 497, 500, 529, 515, 537, 524, 516, 516, 529, 509, 502, 467, 491, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 497, 511, 548, 481, 532, 507, 507, 514, 461, 478, 519, 470, 546, 501, 493, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 505, 539, 489, 527, 476, 543, 494, 484, 507, 513, 513, 510, 501, 512, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 474, 518, 455, 486, 532, 515, 526, 460, 543, 520, 529, 539, 487, 540, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 478, 530, 528, 467, 498, 497, 517, 531, 525, 507, 514, 548, 503, 538, 494, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 494, 510, 492, 496, 534, 509, 502, 458, 520, 515, 517, 519, 499, 487, 493, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8334, 8332, 8225, 8070, 8130, 8199, 8139, 8107] avg=8192.0000 max=8334 max_rank=0 min=8070 min_rank=3 imbalance_ratio=1.017334 max_load_reduction=57202 max_load_reduction_pct=0.872833 moved_to_echo=32575 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[510, 522, 540, 499, 547, 506, 500, 479, 510, 507, 528, 535, 483, 504, 515, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 483, 532, 516, 506, 504, 514, 494, 426, 533, 556, 530, 543, 512, 540, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 505, 514, 496, 504, 499, 511, 512, 520, 499, 484, 513, 537, 487, 507, 496, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 491, 530, 513, 522, 527, 520, 521, 520, 527, 470, 506, 505, 547, 548, 527, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 519, 519, 542, 491, 506, 528, 515, 508, 496, 500, 491, 529, 514, 513, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 514, 477, 495, 497, 530, 477, 487, 466, 513, 501, 499, 486, 502, 498, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 523, 527, 494, 487, 493, 536, 544, 507, 555, 518, 532, 524, 513, 474, 513, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 533, 506, 532, 522, 516, 503, 493, 513, 514, 532, 511, 506, 537, 532, 510, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8165, 8196, 8167, 8322, 8211, 7967, 8267, 8241] avg=8192.0000 max=8322 max_rank=3 min=7967 min_rank=5 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32686 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[3968, 4224, 4224, 4096, 3968, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 3712, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[3968, 4224, 4224, 4096, 3968, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 3968, 4224, 4096, 4352, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[472, 514, 527, 488, 505, 504, 486, 500, 528, 497, 542, 506, 546, 508, 506, 524, 554, 524, 505, 527, 556, 500, 549, 511, 493, 508, 510, 490, 504, 542, 559, 540, 537, 497, 536, 522, 514, 511, 535, 514, 516, 493, 543, 520, 525, 497, 524, 489, 548, 521, 528, 509, 516, 534, 545, 513, 487, 512, 521, 497, 502, 478, 502, 468, 499, 510, 510, 515, 474, 535, 527, 523, 523, 510, 530, 492, 491, 504, 526, 516, 511, 513, 552, 520, 538, 511, 478, 553, 490, 552, 493, 498, 501, 514, 492, 464, 510, 523, 495, 512, 499, 463, 514, 496, 490, 525, 533, 508, 515, 500, 518, 510, 522, 485, 510, 510, 511, 531, 502, 492, 499, 508, 500, 492, 525, 520, 487, 487] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[463, 527, 521, 541, 513, 507, 533, 534, 538, 487, 466, 502, 528, 529, 497, 487, 513, 535, 540, 558, 514, 519, 519, 503, 551, 511, 508, 519, 513, 526, 541, 535, 493, 482, 507, 496, 515, 516, 528, 507, 521, 519, 516, 513, 539, 510, 494, 516, 552, 446, 527, 479, 498, 503, 541, 514, 516, 523, 525, 493, 506, 524, 562, 518, 485, 499, 515, 526, 543, 524, 470, 527, 516, 516, 575, 470, 535, 485, 531, 504, 519, 488, 516, 508, 498, 490, 542, 516, 480, 517, 466, 495, 478, 482, 504, 508, 493, 517, 534, 520, 476, 503, 525, 520, 544, 521, 500, 528, 542, 497, 526, 477, 571, 507, 501, 464, 506, 507, 519, 539, 497, 498, 516, 500, 482, 491, 496, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[546, 507, 541, 514, 491, 492, 522, 524, 544, 494, 513, 487, 498, 510, 512, 500, 517, 524, 530, 521, 509, 521, 524, 521, 534, 543, 498, 476, 479, 536, 559, 538, 532, 479, 481, 520, 487, 487, 526, 526, 502, 497, 517, 495, 528, 550, 499, 484, 519, 531, 447, 498, 515, 526, 532, 523, 520, 518, 552, 518, 513, 499, 490, 503, 506, 517, 514, 489, 560, 465, 501, 482, 478, 518, 520, 530, 489, 543, 470, 528, 517, 515, 496, 494, 528, 475, 508, 482, 534, 512, 515, 517, 518, 556, 521, 521, 526, 479, 494, 540, 494, 505, 510, 508, 510, 519, 503, 483, 519, 475, 529, 514, 518, 540, 540, 522, 508, 543, 477, 528, 522, 502, 491, 488, 501, 553, 511, 526] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[539, 542, 469, 525, 494, 508, 458, 537, 488, 524, 480, 511, 485, 477, 483, 518, 487, 516, 507, 480, 571, 514, 488, 505, 514, 500, 474, 500, 504, 536, 469, 513, 522, 534, 507, 470, 518, 528, 536, 517, 488, 523, 495, 484, 509, 533, 529, 502, 510, 544, 507, 534, 516, 508, 532, 505, 506, 527, 529, 487, 494, 532, 525, 486, 523, 487, 496, 509, 519, 519, 521, 539, 519, 549, 515, 479, 480, 476, 483, 514, 556, 535, 522, 509, 510, 542, 517, 530, 532, 529, 513, 511, 548, 534, 498, 562, 516, 517, 506, 526, 518, 511, 539, 522, 515, 502, 512, 513, 548, 503, 518, 498, 500, 501, 526, 517, 538, 503, 500, 531, 537, 474, 497, 520, 480, 518, 485, 516] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[466, 516, 494, 495, 545, 548, 524, 544, 550, 540, 525, 538, 507, 494, 481, 528, 506, 516, 514, 552, 480, 498, 523, 551, 554, 512, 477, 498, 480, 534, 542, 539, 550, 513, 524, 524, 458, 535, 512, 529, 515, 499, 541, 501, 473, 484, 539, 507, 511, 513, 529, 506, 510, 514, 505, 468, 494, 529, 496, 500, 550, 510, 542, 485, 534, 491, 536, 509, 497, 497, 538, 502, 526, 493, 522, 561, 488, 513, 503, 517, 473, 548, 527, 499, 526, 507, 510, 521, 512, 498, 512, 528, 506, 489, 541, 501, 520, 501, 467, 512, 519, 555, 509, 470, 534, 518, 519, 533, 473, 535, 511, 514, 526, 513, 508, 466, 503, 516, 496, 469, 494, 509, 524, 532, 500, 457, 474, 497] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[506, 503, 480, 531, 496, 495, 492, 490, 491, 513, 538, 509, 476, 478, 540, 524, 550, 549, 514, 482, 492, 538, 512, 512, 490, 511, 537, 505, 493, 497, 530, 496, 541, 482, 510, 495, 473, 538, 542, 532, 548, 510, 496, 507, 510, 494, 525, 500, 551, 512, 532, 540, 498, 528, 504, 543, 500, 493, 531, 497, 525, 518, 504, 524, 531, 543, 517, 516, 500, 479, 490, 508, 517, 516, 502, 508, 547, 523, 533, 530, 531, 513, 524, 447, 504, 524, 496, 482, 510, 483, 455, 535, 525, 537, 549, 543, 489, 521, 526, 502, 484, 519, 511, 531, 533, 503, 502, 501, 502, 496, 502, 517, 513, 491, 525, 535, 478, 524, 518, 512, 482, 510, 533, 486, 522, 542, 490, 545] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[504, 510, 513, 532, 506, 538, 481, 498, 542, 509, 468, 494, 532, 499, 493, 463, 539, 525, 522, 503, 503, 510, 505, 504, 503, 531, 547, 520, 525, 516, 508, 523, 534, 507, 517, 525, 470, 502, 487, 500, 542, 495, 515, 493, 511, 517, 555, 477, 536, 497, 495, 516, 529, 498, 507, 459, 503, 503, 514, 533, 518, 526, 491, 519, 550, 493, 494, 521, 552, 533, 499, 502, 521, 498, 510, 506, 521, 551, 492, 540, 491, 512, 518, 479, 519, 510, 501, 525, 564, 513, 512, 528, 492, 497, 530, 496, 519, 510, 491, 514, 526, 498, 498, 504, 502, 512, 509, 519, 487, 542, 520, 496, 483, 511, 523, 493, 501, 551, 479, 529, 499, 515, 539, 520, 537, 515, 547, 520] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[546, 507, 541, 514, 491, 492, 522, 524, 544, 494, 513, 487, 498, 510, 512, 500, 517, 524, 530, 521, 509, 521, 524, 521, 534, 543, 498, 476, 479, 536, 559, 538, 532, 479, 481, 520, 487, 487, 526, 526, 502, 497, 517, 495, 528, 550, 499, 484, 519, 531, 447, 498, 515, 526, 532, 523, 520, 518, 552, 518, 513, 499, 490, 503, 506, 517, 514, 489, 560, 465, 501, 482, 478, 518, 520, 530, 489, 543, 470, 528, 517, 515, 496, 494, 528, 475, 508, 482, 534, 512, 515, 517, 518, 556, 521, 521, 526, 479, 494, 540, 494, 505, 510, 508, 510, 519, 503, 483, 519, 475, 529, 514, 518, 540, 540, 522, 508, 543, 477, 528, 522, 502, 491, 488, 501, 553, 511, 526] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[473, 493, 525, 476, 513, 493, 507, 531, 473, 515, 502, 495, 529, 517, 474, 521, 503, 526, 516, 494, 508, 493, 511, 512, 501, 509, 540, 556, 541, 504, 514, 503, 539, 517, 532, 543, 547, 500, 487, 491, 540, 493, 545, 480, 524, 481, 539, 520, 487, 491, 521, 510, 475, 520, 548, 528, 534, 529, 541, 481, 515, 482, 511, 546, 511, 528, 525, 540, 471, 535, 498, 495, 487, 494, 505, 533, 530, 521, 458, 520, 500, 521, 497, 552, 520, 521, 526, 492, 486, 494, 497, 549, 492, 519, 558, 504, 532, 535, 528, 507, 508, 544, 523, 484, 536, 523, 522, 484, 498, 513, 492, 463, 501, 501, 527, 555, 509, 529, 511, 525, 472, 482, 512, 559, 492, 490, 538, 497] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[466, 516, 494, 495, 545, 548, 524, 544, 550, 540, 525, 538, 507, 494, 481, 528, 506, 516, 514, 552, 480, 498, 523, 551, 554, 512, 477, 498, 480, 534, 542, 539, 550, 513, 524, 524, 458, 535, 512, 529, 515, 499, 541, 501, 473, 484, 539, 507, 511, 513, 529, 506, 510, 514, 505, 468, 494, 529, 496, 500, 550, 510, 542, 485, 534, 491, 536, 509, 497, 497, 538, 502, 526, 493, 522, 561, 488, 513, 503, 517, 473, 548, 527, 499, 526, 507, 510, 521, 512, 498, 512, 528, 506, 489, 541, 501, 520, 501, 467, 512, 519, 555, 509, 470, 534, 518, 519, 533, 473, 535, 511, 514, 526, 513, 508, 466, 503, 516, 496, 469, 494, 509, 524, 532, 500, 457, 474, 497] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[539, 542, 469, 525, 494, 508, 458, 537, 488, 524, 480, 511, 485, 477, 483, 518, 487, 516, 507, 480, 571, 514, 488, 505, 514, 500, 474, 500, 504, 536, 469, 513, 522, 534, 507, 470, 518, 528, 536, 517, 488, 523, 495, 484, 509, 533, 529, 502, 510, 544, 507, 534, 516, 508, 532, 505, 506, 527, 529, 487, 494, 532, 525, 486, 523, 487, 496, 509, 519, 519, 521, 539, 519, 549, 515, 479, 480, 476, 483, 514, 556, 535, 522, 509, 510, 542, 517, 530, 532, 529, 513, 511, 548, 534, 498, 562, 516, 517, 506, 526, 518, 511, 539, 522, 515, 502, 512, 513, 548, 503, 518, 498, 500, 501, 526, 517, 538, 503, 500, 531, 537, 474, 497, 520, 480, 518, 485, 516] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[463, 527, 521, 541, 513, 507, 533, 534, 538, 487, 466, 502, 528, 529, 497, 487, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 535, 540, 558, 514, 519, 519, 503, 551, 511, 508, 519, 513, 526, 450, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 482, 507, 496, 515, 516, 528, 507, 521, 519, 516, 513, 539, 510, 494, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 446, 527, 479, 498, 503, 518, 514, 516, 523, 525, 493, 506, 524, 562, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 499, 515, 526, 543, 524, 470, 527, 516, 516, 567, 470, 535, 485, 531, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 488, 516, 508, 498, 490, 542, 516, 480, 517, 466, 495, 478, 482, 491, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 517, 534, 520, 476, 503, 525, 520, 544, 521, 500, 528, 542, 497, 526, 477, 13, 11, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 571, 507, 501, 464, 506, 507, 519, 539, 497, 498, 516, 500, 482, 491, 496, 514, 23, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8241, 8314, 8161, 8204, 8213, 7994, 8259, 8150] avg=8192.0000 max=8314 max_rank=1 min=7994 min_rank=5 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32616 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[472, 514, 527, 488, 505, 504, 486, 500, 528, 497, 542, 506, 546, 508, 506, 524, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 524, 505, 527, 556, 500, 549, 511, 493, 508, 510, 490, 504, 542, 474, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 497, 536, 522, 514, 511, 535, 514, 516, 493, 543, 520, 525, 497, 524, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 521, 528, 509, 516, 534, 527, 513, 487, 512, 521, 497, 502, 478, 502, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 510, 510, 515, 474, 535, 527, 523, 523, 510, 527, 492, 491, 504, 526, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 513, 552, 520, 538, 511, 478, 553, 490, 552, 493, 498, 501, 514, 483, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 523, 495, 512, 499, 463, 514, 496, 490, 525, 533, 508, 515, 500, 518, 510, 9, 5, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 485, 510, 510, 511, 531, 502, 492, 499, 508, 500, 492, 525, 520, 487, 487, 19, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 8287, 8268, 8163, 8182, 8171, 8129, 8117] avg=8192.0000 max=8287 max_rank=1 min=8117 min_rank=7 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32599 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[506, 503, 480, 531, 496, 495, 492, 490, 491, 513, 538, 509, 476, 478, 540, 524, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 549, 514, 482, 492, 538, 512, 512, 490, 511, 537, 505, 493, 497, 450, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 482, 510, 495, 473, 538, 542, 532, 548, 510, 496, 507, 510, 494, 525, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 512, 532, 540, 498, 528, 488, 543, 500, 493, 531, 497, 525, 518, 504, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 543, 517, 516, 500, 479, 490, 508, 517, 516, 499, 508, 547, 523, 533, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 513, 524, 447, 504, 524, 496, 482, 510, 483, 455, 535, 525, 537, 539, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 521, 526, 502, 484, 519, 511, 531, 533, 503, 502, 501, 502, 496, 502, 517, 10, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 491, 525, 535, 478, 524, 518, 512, 482, 510, 533, 486, 522, 542, 490, 545, 18, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8124, 8128, 8198, 8284, 8257, 8148, 8157, 8240] avg=8192.0000 max=8284 max_rank=3 min=8124 min_rank=0 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32802 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[539, 542, 469, 525, 494, 508, 458, 537, 488, 524, 480, 511, 485, 477, 483, 518, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 516, 507, 480, 571, 514, 488, 505, 514, 500, 474, 500, 504, 536, 398, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 534, 507, 470, 518, 528, 536, 517, 488, 523, 495, 484, 509, 533, 529, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 544, 507, 534, 516, 508, 514, 505, 506, 527, 529, 487, 494, 532, 525, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 487, 496, 509, 519, 519, 521, 539, 519, 549, 512, 479, 480, 476, 483, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 535, 522, 509, 510, 542, 517, 530, 532, 529, 513, 511, 548, 534, 489, 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 517, 506, 526, 518, 511, 539, 522, 515, 502, 512, 513, 548, 503, 518, 498, 9, 5, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 501, 526, 517, 538, 503, 500, 531, 537, 474, 497, 520, 480, 518, 485, 516, 16, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8093, 8007, 8190, 8224, 8125, 8439, 8282, 8176] avg=8192.0000 max=8439 max_rank=5 min=8007 min_rank=1 imbalance_ratio=1.030151 max_load_reduction=57097 max_load_reduction_pct=0.871231 moved_to_echo=33022 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[466, 516, 494, 495, 545, 548, 524, 544, 550, 540, 525, 538, 507, 494, 481, 528, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 516, 514, 552, 480, 498, 523, 551, 554, 512, 477, 498, 480, 534, 460, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 513, 524, 524, 458, 535, 512, 529, 515, 499, 541, 501, 473, 484, 539, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 513, 529, 506, 510, 514, 489, 468, 494, 529, 496, 500, 550, 510, 542, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 491, 536, 509, 497, 497, 538, 502, 526, 493, 519, 561, 488, 513, 503, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 548, 527, 499, 526, 507, 510, 521, 512, 498, 512, 528, 506, 489, 531, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 501, 467, 512, 519, 555, 509, 470, 534, 518, 519, 533, 473, 535, 511, 514, 10, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 513, 508, 466, 503, 516, 496, 469, 494, 509, 524, 532, 500, 457, 474, 497, 18, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8359, 8194, 8199, 8146, 8224, 8188, 8208, 8018] avg=8192.0000 max=8359 max_rank=0 min=8018 min_rank=7 imbalance_ratio=1.020386 max_load_reduction=57177 max_load_reduction_pct=0.872452 moved_to_echo=32638 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[504, 510, 513, 532, 506, 538, 481, 498, 542, 509, 468, 494, 532, 499, 493, 463, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 525, 522, 503, 503, 510, 505, 504, 503, 531, 547, 520, 525, 516, 431, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 507, 517, 525, 470, 502, 487, 500, 542, 495, 515, 493, 511, 517, 555, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 497, 495, 516, 529, 498, 491, 459, 503, 503, 514, 533, 518, 526, 491, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 493, 494, 521, 552, 533, 499, 502, 521, 498, 507, 506, 521, 551, 492, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 512, 518, 479, 519, 510, 501, 525, 564, 513, 512, 528, 492, 497, 521, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 510, 491, 514, 526, 498, 498, 504, 502, 512, 509, 519, 487, 542, 520, 496, 9, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 511, 523, 493, 501, 551, 479, 529, 499, 515, 539, 520, 537, 515, 547, 520, 17, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8142, 8207, 8142, 8128, 8280, 8178, 8164, 8295] avg=8192.0000 max=8295 max_rank=7 min=8128 min_rank=3 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32917 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[546, 507, 541, 514, 491, 492, 522, 524, 544, 494, 513, 487, 498, 510, 512, 500, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 524, 530, 521, 509, 521, 524, 521, 534, 543, 498, 476, 479, 536, 474, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 479, 481, 520, 487, 487, 526, 526, 502, 497, 517, 495, 528, 550, 499, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 531, 447, 498, 515, 526, 514, 523, 520, 518, 552, 518, 513, 499, 490, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 517, 514, 489, 560, 465, 501, 482, 478, 518, 517, 530, 489, 543, 470, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 515, 496, 494, 528, 475, 508, 482, 534, 512, 515, 517, 518, 556, 512, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 479, 494, 540, 494, 505, 510, 508, 510, 519, 503, 483, 519, 475, 529, 514, 9, 5, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 540, 540, 522, 508, 543, 477, 528, 522, 502, 491, 488, 501, 553, 511, 526, 19, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8261, 8245, 8105, 8186, 8107, 8200, 8126, 8306] avg=8192.0000 max=8306 max_rank=7 min=8105 min_rank=2 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32739 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[473, 493, 525, 476, 513, 493, 507, 531, 473, 515, 502, 495, 529, 517, 474, 521, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 526, 516, 494, 508, 493, 511, 512, 501, 509, 540, 556, 541, 504, 437, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 517, 532, 543, 547, 500, 487, 491, 540, 493, 545, 480, 524, 481, 539, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 491, 521, 510, 475, 520, 530, 528, 534, 529, 541, 481, 515, 482, 511, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 528, 525, 540, 471, 535, 498, 495, 487, 494, 502, 533, 530, 521, 458, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 521, 497, 552, 520, 521, 526, 492, 486, 494, 497, 549, 492, 519, 548, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 535, 528, 507, 508, 544, 523, 484, 536, 523, 522, 484, 498, 513, 492, 463, 10, 5, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 501, 527, 555, 509, 529, 511, 525, 472, 482, 512, 559, 492, 490, 538, 497, 17, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8097, 8154, 8273, 8201, 8148, 8218, 8211, 8234] avg=8192.0000 max=8273 max_rank=2 min=8097 min_rank=0 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32811 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[506, 503, 480, 531, 496, 495, 492, 490, 491, 513, 538, 509, 476, 478, 540, 524, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 549, 514, 482, 492, 538, 512, 512, 490, 511, 537, 505, 493, 497, 450, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 482, 510, 495, 473, 538, 542, 532, 548, 510, 496, 507, 510, 494, 525, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 512, 532, 540, 498, 528, 488, 543, 500, 493, 531, 497, 525, 518, 504, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 543, 517, 516, 500, 479, 490, 508, 517, 516, 499, 508, 547, 523, 533, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 513, 524, 447, 504, 524, 496, 482, 510, 483, 455, 535, 525, 537, 539, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 521, 526, 502, 484, 519, 511, 531, 533, 503, 502, 501, 502, 496, 502, 517, 10, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 491, 525, 535, 478, 524, 518, 512, 482, 510, 533, 486, 522, 542, 490, 545, 18, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8124, 8128, 8198, 8284, 8257, 8148, 8157, 8240] avg=8192.0000 max=8284 max_rank=3 min=8124 min_rank=0 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32802 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[466, 516, 494, 495, 545, 548, 524, 544, 550, 540, 525, 538, 507, 494, 481, 528, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 516, 514, 552, 480, 498, 523, 551, 554, 512, 477, 498, 480, 534, 460, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 513, 524, 524, 458, 535, 512, 529, 515, 499, 541, 501, 473, 484, 539, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 513, 529, 506, 510, 514, 489, 468, 494, 529, 496, 500, 550, 510, 542, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 491, 536, 509, 497, 497, 538, 502, 526, 493, 519, 561, 488, 513, 503, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 548, 527, 499, 526, 507, 510, 521, 512, 498, 512, 528, 506, 489, 531, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 501, 467, 512, 519, 555, 509, 470, 534, 518, 519, 533, 473, 535, 511, 514, 10, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 513, 508, 466, 503, 516, 496, 469, 494, 509, 524, 532, 500, 457, 474, 497, 18, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8359, 8194, 8199, 8146, 8224, 8188, 8208, 8018] avg=8192.0000 max=8359 max_rank=0 min=8018 min_rank=7 imbalance_ratio=1.020386 max_load_reduction=57177 max_load_reduction_pct=0.872452 moved_to_echo=32638 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[539, 542, 469, 525, 494, 508, 458, 537, 488, 524, 480, 511, 485, 477, 483, 518, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 516, 507, 480, 571, 514, 488, 505, 514, 500, 474, 500, 504, 536, 398, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 534, 507, 470, 518, 528, 536, 517, 488, 523, 495, 484, 509, 533, 529, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 544, 507, 534, 516, 508, 514, 505, 506, 527, 529, 487, 494, 532, 525, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 487, 496, 509, 519, 519, 521, 539, 519, 549, 512, 479, 480, 476, 483, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 535, 522, 509, 510, 542, 517, 530, 532, 529, 513, 511, 548, 534, 489, 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 517, 506, 526, 518, 511, 539, 522, 515, 502, 512, 513, 548, 503, 518, 498, 9, 5, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 501, 526, 517, 538, 503, 500, 531, 537, 474, 497, 520, 480, 518, 485, 516, 16, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8093, 8007, 8190, 8224, 8125, 8439, 8282, 8176] avg=8192.0000 max=8439 max_rank=5 min=8007 min_rank=1 imbalance_ratio=1.030151 max_load_reduction=57097 max_load_reduction_pct=0.871231 moved_to_echo=33022 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 3584, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 3968, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 3968, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 256, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=67200 observed_loss=-1664 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=67200 slack_slots=63872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[515, 539, 498, 473, 489, 552, 482, 517, 518, 536, 480, 531, 498, 510, 567, 483, 518, 526, 510, 516, 511, 503, 485, 499, 527, 509, 509, 513, 506, 536, 555, 513, 485, 477, 530, 501, 531, 526, 519, 547, 514, 535, 498, 502, 487, 474, 521, 535, 533, 531, 502, 504, 536, 493, 517, 523, 505, 487, 537, 547, 512, 495, 485, 477, 499, 512, 514, 521, 513, 500, 513, 519, 513, 529, 514, 482, 492, 527, 516, 506, 519, 499, 510, 469, 515, 552, 517, 492, 527, 495, 524, 518, 545, 485, 502, 475, 497, 535, 516, 503, 515, 506, 503, 503, 551, 514, 551, 512, 526, 538, 510, 521, 489, 488, 468, 538, 504, 519, 490, 470, 488, 532, 510, 518, 553, 519, 516, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[528, 509, 517, 478, 475, 514, 519, 550, 529, 504, 496, 477, 487, 510, 538, 502, 493, 520, 524, 511, 499, 551, 521, 560, 490, 499, 525, 513, 510, 513, 514, 504, 497, 491, 493, 561, 520, 516, 525, 510, 521, 509, 477, 505, 494, 526, 523, 503, 490, 487, 505, 508, 484, 511, 530, 506, 534, 536, 502, 516, 511, 469, 514, 529, 516, 498, 513, 486, 520, 529, 494, 511, 549, 530, 513, 507, 545, 515, 557, 498, 504, 508, 502, 473, 523, 464, 498, 504, 522, 529, 523, 523, 498, 548, 505, 550, 512, 513, 509, 507, 532, 516, 536, 529, 492, 516, 537, 503, 504, 491, 480, 503, 504, 533, 524, 513, 499, 520, 521, 511, 485, 531, 535, 498, 533, 500, 523, 488] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[506, 586, 508, 500, 510, 498, 542, 510, 526, 531, 467, 515, 514, 499, 462, 537, 467, 478, 532, 515, 549, 514, 529, 513, 490, 566, 484, 533, 509, 549, 549, 496, 503, 516, 532, 545, 510, 520, 533, 539, 520, 510, 493, 496, 495, 488, 521, 524, 502, 505, 496, 508, 517, 513, 495, 487, 532, 464, 491, 479, 533, 496, 502, 514, 494, 508, 540, 510, 503, 505, 502, 520, 503, 539, 557, 507, 541, 529, 482, 545, 533, 522, 487, 524, 507, 519, 506, 503, 521, 470, 492, 529, 524, 523, 489, 528, 534, 528, 517, 524, 505, 498, 487, 479, 527, 495, 505, 523, 508, 496, 535, 538, 493, 484, 505, 513, 520, 506, 518, 505, 539, 483, 513, 485, 500, 509, 521, 518] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[531, 496, 483, 545, 499, 505, 557, 467, 476, 519, 508, 546, 476, 493, 522, 528, 528, 493, 527, 516, 520, 514, 461, 523, 509, 497, 499, 476, 507, 534, 514, 533, 529, 540, 536, 497, 496, 555, 566, 500, 499, 551, 495, 485, 510, 532, 509, 515, 485, 495, 506, 516, 543, 504, 512, 495, 498, 482, 540, 491, 533, 513, 494, 525, 539, 491, 534, 522, 513, 520, 540, 502, 496, 495, 509, 521, 477, 488, 495, 475, 515, 486, 503, 478, 503, 539, 540, 495, 497, 536, 511, 503, 507, 523, 546, 497, 499, 522, 509, 506, 512, 457, 531, 491, 515, 486, 492, 511, 522, 521, 508, 496, 498, 501, 531, 550, 519, 548, 548, 517, 538, 548, 540, 502, 509, 517, 552, 495] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[505, 489, 519, 525, 522, 481, 474, 533, 539, 540, 498, 539, 521, 529, 516, 490, 510, 542, 503, 506, 511, 478, 494, 477, 532, 482, 539, 487, 537, 527, 539, 512, 491, 523, 520, 513, 531, 522, 497, 498, 483, 510, 554, 540, 505, 521, 519, 528, 496, 501, 523, 545, 482, 493, 492, 510, 489, 496, 481, 530, 505, 514, 480, 520, 506, 546, 497, 504, 541, 481, 491, 493, 518, 489, 464, 564, 529, 514, 522, 478, 518, 514, 490, 519, 501, 499, 525, 518, 503, 483, 511, 524, 501, 524, 516, 502, 520, 499, 505, 551, 550, 531, 502, 503, 530, 551, 524, 573, 510, 508, 498, 546, 575, 513, 459, 524, 528, 510, 510, 534, 507, 475, 498, 476, 532, 463, 532, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[512, 497, 485, 505, 484, 499, 477, 494, 557, 543, 487, 511, 503, 558, 469, 508, 531, 490, 494, 542, 474, 460, 511, 509, 513, 510, 538, 469, 546, 513, 493, 480, 472, 502, 558, 533, 543, 519, 505, 525, 514, 549, 517, 486, 487, 532, 531, 523, 491, 485, 493, 527, 495, 517, 477, 493, 571, 515, 461, 493, 523, 536, 511, 517, 496, 514, 501, 539, 534, 521, 556, 497, 541, 551, 525, 504, 512, 503, 497, 525, 536, 490, 551, 531, 519, 524, 508, 490, 517, 494, 501, 509, 498, 494, 498, 508, 505, 540, 496, 544, 494, 474, 519, 485, 540, 533, 523, 526, 467, 513, 530, 524, 487, 530, 519, 516, 504, 517, 535, 536, 493, 482, 491, 537, 541, 526, 526, 536] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[505, 489, 519, 525, 522, 481, 474, 533, 539, 540, 498, 539, 521, 529, 516, 490, 510, 542, 503, 506, 511, 478, 494, 477, 532, 482, 539, 487, 537, 527, 539, 512, 491, 523, 520, 513, 531, 522, 497, 498, 483, 510, 554, 540, 505, 521, 519, 528, 496, 501, 523, 545, 482, 493, 492, 510, 489, 496, 481, 530, 505, 514, 480, 520, 506, 546, 497, 504, 541, 481, 491, 493, 518, 489, 464, 564, 529, 514, 522, 478, 518, 514, 490, 519, 501, 499, 525, 518, 503, 483, 511, 524, 501, 524, 516, 502, 520, 499, 505, 551, 550, 531, 502, 503, 530, 551, 524, 573, 510, 508, 498, 546, 575, 513, 459, 524, 528, 510, 510, 534, 507, 475, 498, 476, 532, 463, 532, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[523, 531, 493, 534, 492, 529, 494, 479, 516, 537, 497, 536, 482, 511, 515, 524, 462, 474, 506, 492, 522, 510, 529, 525, 504, 508, 509, 498, 500, 517, 558, 481, 509, 495, 532, 519, 520, 522, 491, 522, 513, 502, 562, 521, 472, 521, 498, 509, 504, 486, 485, 503, 526, 520, 505, 525, 511, 499, 506, 512, 537, 505, 546, 525, 484, 488, 533, 545, 496, 544, 487, 511, 535, 532, 491, 494, 512, 524, 516, 471, 499, 541, 480, 501, 499, 496, 538, 518, 544, 500, 522, 473, 531, 529, 525, 500, 510, 547, 549, 511, 506, 494, 519, 535, 497, 493, 479, 503, 503, 531, 498, 532, 525, 509, 494, 510, 543, 539, 490, 529, 536, 575, 518, 531, 506, 495, 475, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[506, 586, 508, 500, 510, 498, 542, 510, 526, 531, 467, 515, 514, 499, 462, 537, 467, 478, 532, 515, 549, 514, 529, 513, 490, 566, 484, 533, 509, 549, 549, 496, 503, 516, 532, 545, 510, 520, 533, 539, 520, 510, 493, 496, 495, 488, 521, 524, 502, 505, 496, 508, 517, 513, 495, 487, 532, 464, 491, 479, 533, 496, 502, 514, 494, 508, 540, 510, 503, 505, 502, 520, 503, 539, 557, 507, 541, 529, 482, 545, 533, 522, 487, 524, 507, 519, 506, 503, 521, 470, 492, 529, 524, 523, 489, 528, 534, 528, 517, 524, 505, 498, 487, 479, 527, 495, 505, 523, 508, 496, 535, 538, 493, 484, 505, 513, 520, 506, 518, 505, 539, 483, 513, 485, 500, 509, 521, 518] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[528, 509, 517, 478, 475, 514, 519, 550, 529, 504, 496, 477, 487, 510, 538, 502, 493, 520, 524, 511, 499, 551, 521, 560, 490, 499, 525, 513, 510, 513, 514, 504, 497, 491, 493, 561, 520, 516, 525, 510, 521, 509, 477, 505, 494, 526, 523, 503, 490, 487, 505, 508, 484, 511, 530, 506, 534, 536, 502, 516, 511, 469, 514, 529, 516, 498, 513, 486, 520, 529, 494, 511, 549, 530, 513, 507, 545, 515, 557, 498, 504, 508, 502, 473, 523, 464, 498, 504, 522, 529, 523, 523, 498, 548, 505, 550, 512, 513, 509, 507, 532, 516, 536, 529, 492, 516, 537, 503, 504, 491, 480, 503, 504, 533, 524, 513, 499, 520, 521, 511, 485, 531, 535, 498, 533, 500, 523, 488] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[505, 529, 542, 526, 525, 537, 522, 522, 477, 493, 555, 540, 498, 500, 529, 494, 514, 518, 502, 451, 520, 498, 492, 518, 558, 476, 580, 540, 538, 534, 490, 462, 512, 476, 487, 525, 528, 541, 518, 513, 536, 523, 514, 505, 525, 544, 521, 524, 501, 497, 531, 535, 499, 522, 520, 488, 497, 514, 524, 513, 513, 483, 530, 501, 515, 494, 534, 513, 494, 532, 514, 529, 505, 532, 503, 484, 506, 532, 465, 547, 508, 546, 504, 479, 509, 517, 453, 481, 501, 511, 539, 526, 498, 500, 484, 507, 519, 527, 521, 545, 497, 502, 501, 513, 513, 504, 498, 538, 510, 483, 511, 521, 509, 500, 523, 479, 518, 514, 497, 545, 524, 483, 509, 484, 515, 510, 502, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[515, 539, 498, 473, 489, 552, 482, 517, 518, 536, 480, 531, 498, 510, 567, 483, 518, 526, 510, 516, 511, 503, 485, 499, 527, 509, 509, 513, 506, 536, 555, 513, 485, 477, 530, 501, 531, 526, 519, 547, 514, 535, 498, 502, 487, 474, 521, 535, 533, 531, 502, 504, 536, 493, 517, 523, 505, 487, 537, 547, 512, 495, 485, 477, 499, 512, 514, 521, 513, 500, 513, 519, 513, 529, 514, 482, 492, 527, 516, 506, 519, 499, 510, 469, 515, 552, 517, 492, 527, 495, 524, 518, 545, 485, 502, 475, 497, 535, 516, 503, 515, 506, 503, 503, 551, 514, 551, 512, 526, 538, 510, 521, 489, 488, 468, 538, 504, 519, 490, 470, 488, 532, 510, 518, 553, 519, 516, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[512, 497, 485, 505, 484, 499, 477, 494, 557, 543, 487, 511, 503, 558, 469, 508, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 490, 494, 542, 474, 460, 511, 509, 513, 510, 538, 469, 546, 513, 493, 480, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 502, 558, 533, 543, 467, 505, 525, 514, 549, 517, 486, 487, 532, 531, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 485, 493, 527, 495, 517, 477, 493, 571, 515, 461, 493, 523, 536, 511, 517, 52, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 514, 501, 539, 534, 521, 556, 497, 541, 535, 525, 504, 512, 503, 497, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 490, 551, 531, 519, 524, 508, 490, 517, 494, 501, 509, 498, 494, 498, 508, 18, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 510, 496, 544, 494, 474, 519, 485, 540, 533, 523, 526, 467, 513, 530, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 530, 519, 516, 504, 517, 535, 536, 493, 482, 491, 537, 521, 526, 526, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8096, 8084, 8244, 8169, 8300, 8204, 8183, 8256] avg=8192.0000 max=8300 max_rank=4 min=8084 min_rank=1 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32943 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[505, 489, 519, 525, 522, 481, 474, 533, 539, 540, 498, 539, 521, 529, 516, 490, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 542, 503, 506, 511, 478, 494, 477, 532, 482, 539, 487, 537, 527, 539, 512, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 523, 520, 513, 531, 470, 497, 498, 483, 510, 554, 540, 505, 521, 519, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 501, 523, 545, 482, 493, 492, 510, 489, 496, 481, 530, 505, 514, 480, 520, 52, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 546, 497, 504, 541, 481, 491, 493, 518, 475, 464, 564, 529, 514, 522, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 514, 490, 519, 501, 499, 525, 518, 503, 483, 511, 524, 501, 524, 516, 502, 17, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 471, 505, 551, 550, 531, 502, 503, 530, 551, 524, 573, 510, 508, 498, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 575, 513, 459, 524, 528, 510, 510, 534, 507, 475, 498, 476, 512, 463, 532, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 8186, 8203, 8120, 8123, 8183, 8373, 8122] avg=8192.0000 max=8373 max_rank=6 min=8120 min_rank=3 imbalance_ratio=1.022095 max_load_reduction=57163 max_load_reduction_pct=0.872238 moved_to_echo=32801 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[528, 509, 517, 478, 475, 514, 519, 550, 529, 504, 496, 477, 487, 510, 538, 502, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 520, 524, 511, 499, 551, 521, 560, 490, 499, 525, 513, 510, 513, 514, 504, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 491, 493, 561, 520, 464, 525, 510, 521, 509, 477, 505, 494, 526, 523, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 487, 505, 508, 484, 511, 530, 506, 534, 536, 502, 516, 511, 469, 514, 529, 52, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 498, 513, 486, 520, 529, 494, 511, 549, 514, 513, 507, 545, 515, 557, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 508, 502, 473, 523, 464, 498, 504, 522, 529, 523, 523, 498, 548, 505, 550, 17, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 484, 509, 507, 532, 516, 536, 529, 492, 516, 537, 503, 504, 491, 480, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 533, 524, 513, 499, 520, 521, 511, 485, 531, 535, 498, 513, 500, 523, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8140, 8258, 8119, 8196, 8265, 8209, 8151, 8198] avg=8192.0000 max=8265 max_rank=4 min=8119 min_rank=2 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32823 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[515, 539, 498, 473, 489, 552, 482, 517, 518, 536, 480, 531, 498, 510, 567, 483, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 526, 510, 516, 511, 503, 485, 499, 527, 509, 509, 513, 506, 536, 555, 513, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 477, 530, 501, 531, 473, 519, 547, 514, 535, 498, 502, 487, 474, 521, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 531, 502, 504, 536, 493, 517, 523, 505, 487, 537, 547, 512, 495, 485, 477, 53, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 512, 514, 521, 513, 500, 513, 519, 513, 513, 514, 482, 492, 527, 516, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 499, 510, 469, 515, 552, 517, 492, 527, 495, 524, 518, 545, 485, 502, 475, 18, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 505, 516, 503, 515, 506, 503, 503, 551, 514, 551, 512, 526, 538, 510, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 488, 468, 538, 504, 519, 490, 470, 488, 532, 510, 518, 533, 519, 516, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8247, 8129, 8249, 8154, 8180, 8271, 8111] avg=8192.0000 max=8271 max_rank=6 min=8111 min_rank=7 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32716 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[506, 586, 508, 500, 510, 498, 542, 510, 526, 531, 467, 515, 514, 499, 462, 537, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 478, 532, 515, 549, 514, 529, 513, 490, 566, 484, 533, 509, 549, 549, 496, 14, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 516, 532, 545, 510, 463, 533, 539, 520, 510, 493, 496, 495, 488, 521, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 505, 496, 508, 517, 513, 495, 487, 532, 464, 491, 479, 533, 496, 502, 514, 57, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 508, 540, 510, 503, 505, 502, 520, 503, 515, 557, 507, 541, 529, 482, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 522, 487, 524, 507, 519, 506, 503, 521, 470, 492, 529, 524, 523, 489, 528, 22, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 489, 517, 524, 505, 498, 487, 479, 527, 495, 505, 523, 508, 496, 535, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 484, 505, 513, 520, 506, 518, 505, 539, 483, 513, 485, 476, 509, 521, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8292, 8188, 8108, 8261, 8218, 8160, 8088] avg=8192.0000 max=8292 max_rank=1 min=8088 min_rank=7 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32727 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[505, 489, 519, 525, 522, 481, 474, 533, 539, 540, 498, 539, 521, 529, 516, 490, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 542, 503, 506, 511, 478, 494, 477, 532, 482, 539, 487, 537, 527, 539, 512, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 523, 520, 513, 531, 470, 497, 498, 483, 510, 554, 540, 505, 521, 519, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 501, 523, 545, 482, 493, 492, 510, 489, 496, 481, 530, 505, 514, 480, 520, 52, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 546, 497, 504, 541, 481, 491, 493, 518, 475, 464, 564, 529, 514, 522, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 514, 490, 519, 501, 499, 525, 518, 503, 483, 511, 524, 501, 524, 516, 502, 17, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 471, 505, 551, 550, 531, 502, 503, 530, 551, 524, 573, 510, 508, 498, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 575, 513, 459, 524, 528, 510, 510, 534, 507, 475, 498, 476, 512, 463, 532, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 8186, 8203, 8120, 8123, 8183, 8373, 8122] avg=8192.0000 max=8373 max_rank=6 min=8120 min_rank=3 imbalance_ratio=1.022095 max_load_reduction=57163 max_load_reduction_pct=0.872238 moved_to_echo=32801 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[512, 497, 485, 505, 484, 499, 477, 494, 557, 543, 487, 511, 503, 558, 469, 508, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 490, 494, 542, 474, 460, 511, 509, 513, 510, 538, 469, 546, 513, 493, 480, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 502, 558, 533, 543, 467, 505, 525, 514, 549, 517, 486, 487, 532, 531, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 485, 493, 527, 495, 517, 477, 493, 571, 515, 461, 493, 523, 536, 511, 517, 52, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 514, 501, 539, 534, 521, 556, 497, 541, 535, 525, 504, 512, 503, 497, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 490, 551, 531, 519, 524, 508, 490, 517, 494, 501, 509, 498, 494, 498, 508, 18, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 510, 496, 544, 494, 474, 519, 485, 540, 533, 523, 526, 467, 513, 530, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 530, 519, 516, 504, 517, 535, 536, 493, 482, 491, 537, 521, 526, 526, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8096, 8084, 8244, 8169, 8300, 8204, 8183, 8256] avg=8192.0000 max=8300 max_rank=4 min=8084 min_rank=1 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32943 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[531, 496, 483, 545, 499, 505, 557, 467, 476, 519, 508, 546, 476, 493, 522, 528, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 493, 527, 516, 520, 514, 461, 523, 509, 497, 499, 476, 507, 534, 514, 533, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 540, 536, 497, 496, 499, 566, 500, 499, 551, 495, 485, 510, 532, 509, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 495, 506, 516, 543, 504, 512, 495, 498, 482, 540, 491, 533, 513, 494, 525, 56, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 491, 534, 522, 513, 520, 540, 502, 496, 481, 509, 521, 477, 488, 495, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 486, 503, 478, 503, 539, 540, 495, 497, 536, 511, 503, 507, 523, 546, 497, 17, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 493, 509, 506, 512, 457, 531, 491, 515, 486, 492, 511, 522, 521, 508, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 501, 531, 550, 519, 548, 548, 517, 538, 548, 540, 502, 490, 517, 552, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8161, 8259, 8200, 8103, 8213, 8049, 8394] avg=8192.0000 max=8394 max_rank=7 min=8049 min_rank=6 imbalance_ratio=1.024658 max_load_reduction=57142 max_load_reduction_pct=0.871918 moved_to_echo=32759 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[505, 529, 542, 526, 525, 537, 522, 522, 477, 493, 555, 540, 498, 500, 529, 494, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 518, 502, 451, 520, 498, 492, 518, 558, 476, 580, 540, 538, 534, 490, 462, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 476, 487, 525, 528, 487, 518, 513, 536, 523, 514, 505, 525, 544, 521, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 497, 531, 535, 499, 522, 520, 488, 497, 514, 524, 513, 513, 483, 530, 501, 54, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 494, 534, 513, 494, 532, 514, 529, 505, 516, 503, 484, 506, 532, 465, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 546, 504, 479, 509, 517, 453, 481, 501, 511, 539, 526, 498, 500, 484, 507, 18, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 497, 521, 545, 497, 502, 501, 513, 513, 504, 498, 538, 510, 483, 511, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 500, 523, 479, 518, 514, 497, 545, 524, 483, 509, 484, 496, 510, 502, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8301, 8202, 8238, 8234, 8183, 8098, 8173, 8107] avg=8192.0000 max=8301 max_rank=0 min=8098 min_rank=5 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32561 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[523, 531, 493, 534, 492, 529, 494, 479, 516, 537, 497, 536, 482, 511, 515, 524, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 474, 506, 492, 522, 510, 529, 525, 504, 508, 509, 498, 500, 517, 558, 481, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 495, 532, 519, 520, 470, 491, 522, 513, 502, 562, 521, 472, 521, 498, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 486, 485, 503, 526, 520, 505, 525, 511, 499, 506, 512, 537, 505, 546, 525, 52, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 488, 533, 545, 496, 544, 487, 511, 535, 516, 491, 494, 512, 524, 516, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 541, 480, 501, 499, 496, 538, 518, 544, 500, 522, 473, 531, 529, 525, 500, 18, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 516, 549, 511, 506, 494, 519, 535, 497, 493, 479, 503, 503, 531, 498, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 509, 494, 510, 543, 539, 490, 529, 536, 575, 518, 531, 487, 495, 475, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8106, 8156, 8260, 8147, 8231, 8176, 8260] avg=8192.0000 max=8260 max_rank=3 min=8106 min_rank=1 imbalance_ratio=1.008301 max_load_reduction=57276 max_load_reduction_pct=0.873962 moved_to_echo=32814 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[515, 539, 498, 473, 489, 552, 482, 517, 518, 536, 480, 531, 498, 510, 567, 483, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 526, 510, 516, 511, 503, 485, 499, 527, 509, 509, 513, 506, 536, 555, 513, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 477, 530, 501, 531, 473, 519, 547, 514, 535, 498, 502, 487, 474, 521, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 531, 502, 504, 536, 493, 517, 523, 505, 487, 537, 547, 512, 495, 485, 477, 53, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 512, 514, 521, 513, 500, 513, 519, 513, 513, 514, 482, 492, 527, 516, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 499, 510, 469, 515, 552, 517, 492, 527, 495, 524, 518, 545, 485, 502, 475, 18, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 505, 516, 503, 515, 506, 503, 503, 551, 514, 551, 512, 526, 538, 510, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 488, 468, 538, 504, 519, 490, 470, 488, 532, 510, 518, 533, 519, 516, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8247, 8129, 8249, 8154, 8180, 8271, 8111] avg=8192.0000 max=8271 max_rank=6 min=8111 min_rank=7 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32716 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[505, 529, 542, 526, 525, 537, 522, 522, 477, 493, 555, 540, 498, 500, 529, 494, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 518, 502, 451, 520, 498, 492, 518, 558, 476, 580, 540, 538, 534, 490, 462, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 476, 487, 525, 528, 487, 518, 513, 536, 523, 514, 505, 525, 544, 521, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 497, 531, 535, 499, 522, 520, 488, 497, 514, 524, 513, 513, 483, 530, 501, 54, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 494, 534, 513, 494, 532, 514, 529, 505, 516, 503, 484, 506, 532, 465, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 546, 504, 479, 509, 517, 453, 481, 501, 511, 539, 526, 498, 500, 484, 507, 18, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 497, 521, 545, 497, 502, 501, 513, 513, 504, 498, 538, 510, 483, 511, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 500, 523, 479, 518, 514, 497, 545, 524, 483, 509, 484, 496, 510, 502, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8301, 8202, 8238, 8234, 8183, 8098, 8173, 8107] avg=8192.0000 max=8301 max_rank=0 min=8098 min_rank=5 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32561 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 3968, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 256, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 3840, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 3968, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 3840, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 256, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[522, 470, 515, 525, 493, 543, 516, 515, 506, 483, 558, 540, 536, 536, 477, 554, 561, 497, 492, 511, 509, 454, 540, 517, 508, 495, 510, 498, 553, 494, 502, 544, 557, 514, 530, 501, 507, 534, 528, 489, 515, 525, 512, 517, 539, 544, 511, 512, 477, 496, 527, 497, 527, 517, 516, 504, 515, 546, 508, 508, 483, 492, 510, 539, 535, 526, 504, 554, 529, 508, 510, 494, 481, 496, 533, 511, 477, 521, 492, 547, 485, 492, 530, 531, 529, 526, 482, 505, 473, 534, 508, 524, 509, 507, 517, 526, 486, 488, 516, 481, 483, 528, 492, 512, 515, 542, 476, 487, 517, 511, 503, 534, 470, 524, 503, 504, 487, 507, 500, 498, 511, 469, 519, 522, 490, 525, 538, 531] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[486, 543, 496, 483, 513, 523, 487, 535, 499, 497, 513, 490, 499, 518, 539, 530, 496, 504, 544, 540, 514, 521, 492, 508, 502, 497, 505, 482, 523, 559, 494, 493, 540, 520, 490, 505, 487, 501, 487, 504, 536, 543, 510, 505, 507, 513, 474, 521, 510, 551, 507, 518, 501, 498, 514, 539, 518, 507, 503, 560, 507, 497, 507, 507, 484, 493, 512, 495, 479, 546, 505, 500, 545, 545, 505, 496, 542, 503, 547, 495, 508, 541, 541, 538, 540, 525, 512, 513, 520, 487, 519, 506, 494, 498, 542, 492, 507, 532, 497, 514, 473, 508, 519, 513, 507, 494, 497, 482, 489, 520, 519, 498, 493, 518, 487, 552, 522, 526, 472, 525, 497, 520, 523, 548, 522, 525, 562, 495] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[472, 531, 508, 509, 537, 512, 501, 500, 497, 485, 539, 567, 513, 532, 492, 509, 514, 479, 550, 474, 479, 491, 534, 516, 504, 556, 534, 488, 519, 470, 498, 526, 542, 543, 463, 487, 456, 509, 483, 488, 520, 484, 502, 492, 493, 480, 543, 519, 522, 541, 540, 500, 529, 530, 520, 481, 515, 516, 527, 531, 526, 522, 486, 502, 523, 537, 531, 527, 573, 493, 537, 509, 480, 484, 520, 524, 522, 501, 524, 507, 530, 562, 514, 537, 510, 491, 508, 538, 531, 495, 506, 512, 521, 538, 520, 519, 534, 495, 464, 511, 510, 546, 520, 520, 536, 523, 512, 526, 489, 519, 529, 496, 483, 481, 488, 493, 484, 517, 510, 502, 551, 509, 493, 480, 504, 529, 502, 528] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[476, 485, 506, 509, 521, 526, 575, 495, 482, 511, 513, 546, 515, 548, 507, 510, 535, 516, 497, 507, 492, 522, 493, 534, 541, 468, 465, 501, 478, 516, 495, 517, 501, 489, 477, 509, 510, 515, 489, 531, 520, 500, 553, 521, 492, 515, 499, 524, 538, 489, 469, 518, 492, 509, 495, 507, 491, 485, 486, 490, 532, 490, 538, 498, 532, 504, 535, 507, 504, 506, 516, 532, 520, 524, 550, 493, 507, 536, 538, 502, 562, 490, 547, 512, 520, 528, 529, 529, 518, 506, 545, 506, 534, 538, 487, 552, 505, 521, 537, 533, 501, 500, 506, 473, 556, 514, 507, 496, 496, 490, 514, 526, 496, 532, 554, 524, 512, 463, 492, 473, 533, 541, 495, 475, 540, 518, 485, 545] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[519, 551, 568, 509, 495, 491, 518, 541, 545, 537, 518, 537, 511, 510, 522, 558, 523, 523, 469, 525, 470, 504, 480, 517, 504, 499, 527, 484, 517, 524, 507, 528, 518, 485, 529, 496, 480, 523, 499, 494, 486, 522, 495, 470, 537, 507, 564, 500, 504, 491, 515, 508, 514, 503, 517, 507, 538, 519, 480, 517, 491, 509, 538, 494, 492, 540, 488, 528, 517, 539, 541, 515, 519, 537, 525, 513, 463, 531, 542, 511, 509, 511, 510, 521, 545, 507, 558, 502, 519, 506, 518, 516, 496, 575, 486, 505, 505, 497, 476, 490, 505, 518, 495, 517, 477, 545, 502, 513, 498, 520, 490, 524, 526, 519, 481, 537, 482, 537, 511, 500, 483, 511, 450, 507, 528, 506, 533, 487] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[487, 482, 475, 539, 494, 521, 504, 512, 577, 515, 534, 518, 536, 537, 507, 538, 491, 540, 526, 477, 523, 508, 511, 458, 492, 572, 494, 519, 485, 514, 501, 489, 480, 544, 487, 547, 506, 502, 513, 525, 492, 547, 549, 486, 524, 515, 479, 495, 487, 537, 531, 481, 526, 536, 526, 506, 534, 516, 489, 558, 524, 511, 511, 522, 541, 493, 551, 483, 522, 498, 523, 496, 548, 536, 497, 552, 530, 529, 508, 479, 499, 476, 517, 486, 514, 493, 499, 494, 519, 486, 518, 486, 505, 514, 497, 518, 510, 490, 493, 546, 513, 510, 502, 531, 493, 530, 507, 479, 530, 488, 542, 532, 520, 510, 481, 506, 539, 513, 513, 499, 499, 515, 530, 504, 541, 488, 509, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[507, 555, 495, 509, 486, 551, 539, 503, 514, 504, 512, 509, 505, 499, 544, 515, 513, 517, 480, 500, 542, 481, 517, 534, 549, 534, 488, 509, 529, 534, 496, 507, 520, 528, 539, 516, 533, 505, 529, 477, 498, 477, 506, 531, 535, 483, 478, 565, 485, 495, 520, 520, 506, 485, 505, 530, 486, 498, 522, 506, 516, 509, 512, 520, 511, 547, 508, 526, 508, 532, 561, 461, 464, 495, 543, 540, 511, 504, 519, 501, 488, 500, 525, 556, 504, 518, 521, 517, 539, 488, 529, 471, 547, 502, 526, 492, 545, 478, 496, 529, 491, 492, 489, 489, 493, 503, 504, 517, 524, 511, 522, 475, 520, 479, 522, 530, 528, 531, 517, 509, 490, 534, 493, 504, 478, 546, 519, 512] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[566, 517, 522, 492, 517, 557, 501, 534, 522, 517, 500, 514, 503, 472, 472, 524, 527, 505, 540, 535, 505, 512, 567, 489, 540, 513, 502, 474, 543, 503, 487, 535, 547, 521, 540, 501, 497, 508, 489, 491, 516, 518, 505, 523, 475, 484, 529, 507, 503, 513, 520, 521, 557, 495, 552, 446, 494, 551, 472, 505, 513, 506, 498, 577, 544, 518, 496, 499, 491, 501, 485, 507, 504, 511, 513, 504, 495, 520, 527, 533, 518, 486, 495, 511, 499, 508, 524, 502, 517, 519, 531, 535, 522, 533, 526, 504, 515, 552, 547, 501, 531, 503, 525, 490, 511, 505, 488, 499, 551, 524, 501, 505, 483, 520, 517, 510, 499, 478, 529, 487, 508, 500, 528, 504, 504, 467, 497, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[472, 531, 508, 509, 537, 512, 501, 500, 497, 485, 539, 567, 513, 532, 492, 448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 479, 550, 474, 479, 491, 534, 516, 504, 556, 534, 488, 519, 470, 498, 526, 16, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 543, 463, 487, 456, 509, 483, 488, 520, 484, 502, 492, 493, 480, 543, 519, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 541, 540, 500, 529, 530, 520, 481, 515, 516, 527, 531, 526, 522, 486, 502, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 537, 531, 527, 573, 493, 537, 509, 480, 484, 520, 524, 522, 501, 471, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 562, 514, 537, 510, 491, 508, 538, 531, 495, 506, 512, 521, 488, 520, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 495, 464, 511, 510, 546, 520, 520, 536, 523, 512, 526, 489, 519, 529, 496, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 481, 488, 493, 484, 517, 510, 502, 551, 509, 493, 480, 504, 529, 502, 528, 38, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8143, 8163, 8034, 8292, 8239, 8282, 8280, 8103] avg=8192.0000 max=8292 max_rank=3 min=8034 min_rank=2 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32904 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[507, 555, 495, 509, 486, 551, 539, 503, 514, 504, 512, 509, 505, 499, 544, 515, 513, 517, 480, 500, 542, 481, 517, 534, 549, 534, 488, 509, 529, 534, 496, 507, 520, 528, 539, 516, 533, 505, 529, 477, 498, 477, 506, 531, 535, 483, 478, 565, 485, 495, 520, 520, 506, 485, 505, 530, 486, 498, 522, 506, 516, 509, 512, 520, 511, 547, 508, 526, 508, 532, 561, 461, 464, 495, 543, 540, 511, 504, 519, 501, 488, 500, 525, 556, 504, 518, 521, 517, 539, 488, 529, 471, 547, 502, 526, 492, 545, 478, 496, 529, 491, 492, 489, 489, 493, 503, 504, 517, 524, 511, 522, 475, 520, 479, 522, 530, 528, 531, 517, 509, 490, 534, 493, 504, 478, 546, 519, 512] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[486, 543, 496, 483, 513, 523, 487, 535, 499, 497, 513, 490, 499, 518, 539, 530, 496, 504, 544, 540, 514, 521, 492, 508, 502, 497, 505, 482, 523, 559, 494, 493, 540, 520, 490, 505, 487, 501, 487, 504, 536, 543, 510, 505, 507, 513, 474, 521, 510, 551, 507, 518, 501, 498, 514, 539, 518, 507, 503, 560, 507, 497, 507, 507, 484, 493, 512, 495, 479, 546, 505, 500, 545, 545, 505, 496, 542, 503, 547, 495, 508, 541, 541, 538, 540, 525, 512, 513, 520, 487, 519, 506, 494, 498, 542, 492, 507, 532, 497, 514, 473, 508, 519, 513, 507, 494, 497, 482, 489, 520, 519, 498, 493, 518, 487, 552, 522, 526, 472, 525, 497, 520, 523, 548, 522, 525, 562, 495] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[487, 482, 475, 539, 494, 521, 504, 512, 577, 515, 534, 518, 536, 537, 507, 538, 491, 540, 526, 477, 523, 508, 511, 458, 492, 572, 494, 519, 485, 514, 501, 489, 480, 544, 487, 547, 506, 502, 513, 525, 492, 547, 549, 486, 524, 515, 479, 495, 487, 537, 531, 481, 526, 536, 526, 506, 534, 516, 489, 558, 524, 511, 511, 522, 541, 493, 551, 483, 522, 498, 523, 496, 548, 536, 497, 552, 530, 529, 508, 479, 499, 476, 517, 486, 514, 493, 499, 494, 519, 486, 518, 486, 505, 514, 497, 518, 510, 490, 493, 546, 513, 510, 502, 531, 493, 530, 507, 479, 530, 488, 542, 532, 520, 510, 481, 506, 539, 513, 513, 499, 499, 515, 530, 504, 541, 488, 509, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[472, 531, 508, 509, 537, 512, 501, 500, 497, 485, 539, 567, 513, 532, 492, 509, 514, 479, 550, 474, 479, 491, 534, 516, 504, 556, 534, 488, 519, 470, 498, 526, 542, 543, 463, 487, 456, 509, 483, 488, 520, 484, 502, 492, 493, 480, 543, 519, 522, 541, 540, 500, 529, 530, 520, 481, 515, 516, 527, 531, 526, 522, 486, 502, 523, 537, 531, 527, 573, 493, 537, 509, 480, 484, 520, 524, 522, 501, 524, 507, 530, 562, 514, 537, 510, 491, 508, 538, 531, 495, 506, 512, 521, 538, 520, 519, 534, 495, 464, 511, 510, 546, 520, 520, 536, 523, 512, 526, 489, 519, 529, 496, 483, 481, 488, 493, 484, 517, 510, 502, 551, 509, 493, 480, 504, 529, 502, 528] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[522, 470, 515, 525, 493, 543, 516, 515, 506, 483, 558, 540, 536, 536, 477, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 497, 492, 511, 509, 454, 540, 517, 508, 495, 510, 498, 553, 494, 502, 544, 15, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 557, 514, 530, 501, 507, 534, 528, 489, 515, 525, 512, 517, 539, 544, 511, 512, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 496, 527, 497, 527, 517, 516, 504, 515, 546, 508, 508, 483, 492, 510, 539, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 526, 504, 554, 529, 508, 510, 494, 481, 496, 533, 511, 477, 521, 441, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 492, 530, 531, 529, 526, 482, 505, 473, 534, 508, 524, 509, 459, 517, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 488, 516, 481, 483, 528, 492, 512, 515, 542, 476, 487, 517, 511, 503, 534, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 524, 503, 504, 487, 507, 500, 498, 511, 469, 519, 522, 490, 525, 538, 531, 36, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8223, 8215, 8364, 8166, 8167, 8130, 8125, 8146] avg=8192.0000 max=8364 max_rank=2 min=8125 min_rank=6 imbalance_ratio=1.020996 max_load_reduction=57172 max_load_reduction_pct=0.872375 moved_to_echo=32568 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[519, 551, 568, 509, 495, 491, 518, 541, 545, 537, 518, 537, 511, 510, 522, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 523, 469, 525, 470, 504, 480, 517, 504, 499, 527, 484, 517, 524, 507, 528, 23, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 485, 529, 496, 480, 523, 499, 494, 486, 522, 495, 470, 537, 507, 564, 500, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 491, 515, 508, 514, 503, 517, 507, 538, 519, 480, 517, 491, 509, 538, 494, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 540, 488, 528, 517, 539, 541, 515, 519, 537, 525, 513, 463, 531, 478, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 511, 510, 521, 545, 507, 558, 502, 519, 506, 518, 516, 496, 505, 486, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 497, 476, 490, 505, 518, 495, 517, 477, 545, 502, 513, 498, 520, 490, 524, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 519, 481, 537, 482, 537, 511, 500, 483, 511, 450, 507, 528, 506, 533, 487, 44, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8355, 8144, 8143, 8154, 8237, 8214, 8130, 8159] avg=8192.0000 max=8355 max_rank=0 min=8130 min_rank=6 imbalance_ratio=1.019897 max_load_reduction=57181 max_load_reduction_pct=0.872513 moved_to_echo=32740 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[566, 517, 522, 492, 517, 557, 501, 534, 522, 517, 500, 514, 503, 472, 472, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 505, 540, 535, 505, 512, 567, 489, 540, 513, 502, 474, 543, 503, 487, 535, 16, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 521, 540, 501, 497, 508, 489, 491, 516, 518, 505, 523, 475, 484, 529, 507, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 513, 520, 521, 557, 495, 552, 446, 494, 551, 472, 505, 513, 506, 498, 577, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 518, 496, 499, 491, 501, 485, 507, 504, 511, 513, 504, 495, 520, 473, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 486, 495, 511, 499, 508, 524, 502, 517, 519, 531, 535, 522, 483, 526, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 552, 547, 501, 531, 503, 525, 490, 511, 505, 488, 499, 551, 524, 501, 505, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 520, 517, 510, 499, 478, 529, 487, 508, 500, 528, 504, 504, 467, 497, 498, 38, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8309, 8181, 8227, 8094, 8180, 8299, 8079] avg=8192.0000 max=8309 max_rank=1 min=8079 min_rank=7 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32652 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[476, 485, 506, 509, 521, 526, 575, 495, 482, 511, 513, 546, 515, 548, 507, 449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 516, 497, 507, 492, 522, 493, 534, 541, 468, 465, 501, 478, 516, 495, 517, 16, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 489, 477, 509, 510, 515, 489, 531, 520, 500, 553, 521, 492, 515, 499, 524, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 489, 469, 518, 492, 509, 495, 507, 491, 485, 486, 490, 532, 490, 538, 498, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 504, 535, 507, 504, 506, 516, 532, 520, 524, 550, 493, 507, 536, 483, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 562, 490, 547, 512, 520, 528, 529, 529, 518, 506, 545, 506, 534, 488, 487, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 521, 537, 533, 501, 500, 506, 473, 556, 514, 507, 496, 496, 490, 514, 526, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 532, 554, 524, 512, 463, 492, 473, 533, 541, 495, 475, 540, 518, 485, 545, 39, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8109, 8175, 8031, 8251, 8353, 8225, 8228] avg=8192.0000 max=8353 max_rank=5 min=8031 min_rank=3 imbalance_ratio=1.019653 max_load_reduction=57183 max_load_reduction_pct=0.872543 moved_to_echo=33057 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[487, 482, 475, 539, 494, 521, 504, 512, 577, 515, 534, 518, 536, 537, 507, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 540, 526, 477, 523, 508, 511, 458, 492, 572, 494, 519, 485, 514, 501, 489, 16, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 544, 487, 547, 506, 502, 513, 525, 492, 547, 549, 486, 524, 515, 479, 495, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 537, 531, 481, 526, 536, 526, 506, 534, 516, 489, 558, 524, 511, 511, 522, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 493, 551, 483, 522, 498, 523, 496, 548, 536, 497, 552, 530, 529, 456, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 476, 517, 486, 514, 493, 499, 494, 519, 486, 518, 486, 505, 465, 497, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 490, 493, 546, 513, 510, 502, 531, 493, 530, 507, 479, 530, 488, 542, 532, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 510, 481, 506, 539, 513, 513, 499, 499, 515, 530, 504, 541, 488, 509, 504, 37, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8131, 8220, 8299, 8234, 7972, 8248, 8220] avg=8192.0000 max=8299 max_rank=3 min=7972 min_rank=5 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32674 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[486, 543, 496, 483, 513, 523, 487, 535, 499, 497, 513, 490, 499, 518, 539, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 504, 544, 540, 514, 521, 492, 508, 502, 497, 505, 482, 523, 559, 494, 493, 15, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 520, 490, 505, 487, 501, 487, 504, 536, 543, 510, 505, 507, 513, 474, 521, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 551, 507, 518, 501, 498, 514, 539, 518, 507, 503, 560, 507, 497, 507, 507, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 493, 512, 495, 479, 546, 505, 500, 545, 545, 505, 496, 542, 503, 491, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 541, 541, 538, 540, 525, 512, 513, 520, 487, 519, 506, 494, 451, 542, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 532, 497, 514, 473, 508, 519, 513, 507, 494, 497, 482, 489, 520, 519, 498, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 518, 487, 552, 522, 526, 472, 525, 497, 520, 523, 548, 522, 525, 562, 495, 40, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8087, 8205, 8171, 8248, 8136, 8229, 8121, 8339] avg=8192.0000 max=8339 max_rank=7 min=8087 min_rank=0 imbalance_ratio=1.017944 max_load_reduction=57197 max_load_reduction_pct=0.872757 moved_to_echo=32825 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[507, 555, 495, 509, 486, 551, 539, 503, 514, 504, 512, 509, 505, 499, 544, 453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 517, 480, 500, 542, 481, 517, 534, 549, 534, 488, 509, 529, 534, 496, 507, 15, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 528, 539, 516, 533, 505, 529, 477, 498, 477, 506, 531, 535, 483, 478, 565, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 495, 520, 520, 506, 485, 505, 530, 486, 498, 522, 506, 516, 509, 512, 520, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 547, 508, 526, 508, 532, 561, 461, 464, 495, 543, 540, 511, 504, 466, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 500, 525, 556, 504, 518, 521, 517, 539, 488, 529, 471, 547, 455, 526, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 478, 496, 529, 491, 492, 489, 489, 493, 503, 504, 517, 524, 511, 522, 475, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 479, 522, 530, 528, 531, 517, 509, 490, 534, 493, 504, 478, 546, 519, 512, 38, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8260, 8248, 8119, 8178, 8176, 8108, 8262] avg=8192.0000 max=8262 max_rank=7 min=8108 min_rank=6 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32724 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[566, 517, 522, 492, 517, 557, 501, 534, 522, 517, 500, 514, 503, 472, 472, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 505, 540, 535, 505, 512, 567, 489, 540, 513, 502, 474, 543, 503, 487, 535, 16, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 521, 540, 501, 497, 508, 489, 491, 516, 518, 505, 523, 475, 484, 529, 507, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 513, 520, 521, 557, 495, 552, 446, 494, 551, 472, 505, 513, 506, 498, 577, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 518, 496, 499, 491, 501, 485, 507, 504, 511, 513, 504, 495, 520, 473, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 486, 495, 511, 499, 508, 524, 502, 517, 519, 531, 535, 522, 483, 526, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 552, 547, 501, 531, 503, 525, 490, 511, 505, 488, 499, 551, 524, 501, 505, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 520, 517, 510, 499, 478, 529, 487, 508, 500, 528, 504, 504, 467, 497, 498, 38, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8309, 8181, 8227, 8094, 8180, 8299, 8079] avg=8192.0000 max=8309 max_rank=1 min=8079 min_rank=7 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32652 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[472, 531, 508, 509, 537, 512, 501, 500, 497, 485, 539, 567, 513, 532, 492, 448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 479, 550, 474, 479, 491, 534, 516, 504, 556, 534, 488, 519, 470, 498, 526, 16, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 543, 463, 487, 456, 509, 483, 488, 520, 484, 502, 492, 493, 480, 543, 519, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 541, 540, 500, 529, 530, 520, 481, 515, 516, 527, 531, 526, 522, 486, 502, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 537, 531, 527, 573, 493, 537, 509, 480, 484, 520, 524, 522, 501, 471, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 562, 514, 537, 510, 491, 508, 538, 531, 495, 506, 512, 521, 488, 520, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 495, 464, 511, 510, 546, 520, 520, 536, 523, 512, 526, 489, 519, 529, 496, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 481, 488, 493, 484, 517, 510, 502, 551, 509, 493, 480, 504, 529, 502, 528, 38, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8143, 8163, 8034, 8292, 8239, 8282, 8280, 8103] avg=8192.0000 max=8292 max_rank=3 min=8034 min_rank=2 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32904 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[486, 543, 496, 483, 513, 523, 487, 535, 499, 497, 513, 490, 499, 518, 539, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 504, 544, 540, 514, 521, 492, 508, 502, 497, 505, 482, 523, 559, 494, 493, 15, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 520, 490, 505, 487, 501, 487, 504, 536, 543, 510, 505, 507, 513, 474, 521, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 551, 507, 518, 501, 498, 514, 539, 518, 507, 503, 560, 507, 497, 507, 507, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 493, 512, 495, 479, 546, 505, 500, 545, 545, 505, 496, 542, 503, 491, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 541, 541, 538, 540, 525, 512, 513, 520, 487, 519, 506, 494, 451, 542, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 532, 497, 514, 473, 508, 519, 513, 507, 494, 497, 482, 489, 520, 519, 498, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 518, 487, 552, 522, 526, 472, 525, 497, 520, 523, 548, 522, 525, 562, 495, 40, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8087, 8205, 8171, 8248, 8136, 8229, 8121, 8339] avg=8192.0000 max=8339 max_rank=7 min=8087 min_rank=0 imbalance_ratio=1.017944 max_load_reduction=57197 max_load_reduction_pct=0.872757 moved_to_echo=32825 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[487, 482, 475, 539, 494, 521, 504, 512, 577, 515, 534, 518, 536, 537, 507, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 540, 526, 477, 523, 508, 511, 458, 492, 572, 494, 519, 485, 514, 501, 489, 16, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 544, 487, 547, 506, 502, 513, 525, 492, 547, 549, 486, 524, 515, 479, 495, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 537, 531, 481, 526, 536, 526, 506, 534, 516, 489, 558, 524, 511, 511, 522, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 493, 551, 483, 522, 498, 523, 496, 548, 536, 497, 552, 530, 529, 456, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 476, 517, 486, 514, 493, 499, 494, 519, 486, 518, 486, 505, 465, 497, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 490, 493, 546, 513, 510, 502, 531, 493, 530, 507, 479, 530, 488, 542, 532, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 510, 481, 506, 539, 513, 513, 499, 499, 515, 530, 504, 541, 488, 509, 504, 37, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8131, 8220, 8299, 8234, 7972, 8248, 8220] avg=8192.0000 max=8299 max_rank=3 min=7972 min_rank=5 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32674 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 3968, 4224, 4224, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 3840, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 3840, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 3840, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 3840, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[494, 525, 499, 503, 540, 499, 521, 570, 499, 491, 528, 504, 486, 506, 527, 485, 537, 547, 470, 519, 492, 536, 506, 528, 522, 491, 513, 500, 523, 529, 514, 556, 496, 512, 511, 523, 490, 523, 485, 529, 558, 502, 504, 521, 509, 546, 566, 492, 513, 485, 526, 502, 485, 482, 521, 526, 561, 492, 499, 481, 512, 486, 514, 497, 510, 506, 490, 519, 501, 460, 518, 524, 497, 548, 513, 494, 540, 555, 548, 460, 536, 498, 489, 512, 517, 488, 498, 541, 538, 532, 501, 532, 529, 491, 537, 521, 495, 477, 541, 501, 563, 487, 485, 498, 508, 509, 535, 514, 531, 533, 525, 517, 484, 501, 474, 486, 504, 523, 518, 511, 490, 507, 483, 500, 520, 497, 530, 537] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[491, 511, 555, 489, 506, 522, 506, 509, 532, 482, 475, 474, 489, 512, 515, 514, 543, 537, 516, 477, 505, 510, 496, 517, 529, 553, 522, 537, 517, 501, 517, 505, 525, 508, 517, 501, 504, 504, 510, 480, 490, 532, 547, 535, 514, 501, 480, 519, 523, 497, 538, 543, 527, 495, 489, 509, 537, 496, 527, 527, 523, 504, 500, 555, 531, 526, 504, 491, 494, 528, 503, 498, 495, 500, 562, 522, 483, 529, 524, 514, 503, 517, 515, 527, 492, 492, 495, 518, 519, 497, 499, 495, 527, 516, 521, 512, 530, 537, 536, 516, 503, 525, 478, 488, 485, 504, 538, 535, 519, 493, 523, 494, 545, 480, 460, 486, 496, 530, 521, 534, 526, 536, 509, 456, 495, 542, 540, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[552, 484, 520, 513, 508, 568, 502, 515, 491, 523, 518, 530, 534, 526, 515, 522, 520, 546, 496, 488, 491, 491, 501, 516, 483, 503, 504, 550, 523, 491, 521, 479, 532, 488, 557, 481, 521, 510, 502, 534, 486, 487, 520, 525, 525, 489, 514, 511, 478, 516, 503, 527, 502, 501, 507, 525, 531, 534, 469, 488, 520, 488, 508, 500, 494, 492, 530, 538, 498, 538, 502, 526, 526, 519, 540, 482, 521, 473, 496, 537, 511, 537, 500, 525, 544, 503, 552, 522, 546, 497, 476, 486, 489, 513, 522, 525, 498, 514, 481, 490, 486, 554, 491, 519, 503, 479, 504, 527, 508, 516, 543, 531, 514, 569, 505, 509, 514, 486, 511, 492, 497, 516, 510, 517, 525, 520, 536, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[514, 486, 524, 555, 516, 527, 549, 487, 520, 484, 524, 495, 467, 532, 492, 514, 547, 512, 530, 478, 504, 533, 493, 498, 536, 501, 511, 523, 510, 517, 500, 474, 531, 483, 460, 486, 508, 510, 541, 536, 540, 490, 506, 506, 506, 502, 496, 503, 505, 535, 530, 538, 518, 491, 544, 495, 503, 512, 497, 529, 493, 538, 509, 529, 522, 542, 508, 525, 517, 484, 511, 519, 499, 537, 565, 489, 476, 556, 481, 526, 472, 477, 518, 515, 494, 513, 482, 526, 501, 509, 494, 516, 487, 560, 531, 508, 542, 517, 473, 489, 509, 496, 512, 504, 494, 496, 514, 528, 506, 489, 508, 499, 493, 502, 533, 541, 525, 518, 521, 563, 484, 527, 522, 518, 544, 510, 527, 549] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[526, 534, 526, 516, 500, 523, 542, 533, 494, 521, 501, 537, 509, 501, 494, 521, 517, 494, 469, 539, 517, 515, 487, 508, 521, 517, 491, 513, 475, 513, 521, 517, 517, 507, 504, 480, 499, 554, 505, 529, 533, 525, 512, 494, 490, 523, 512, 488, 550, 542, 544, 481, 524, 536, 542, 501, 513, 506, 525, 530, 508, 504, 525, 493, 463, 511, 489, 540, 543, 470, 563, 517, 518, 539, 533, 480, 495, 523, 516, 525, 504, 516, 537, 536, 513, 512, 452, 494, 506, 482, 487, 480, 547, 488, 531, 496, 519, 495, 504, 507, 460, 474, 514, 494, 568, 556, 515, 499, 501, 485, 525, 485, 482, 550, 525, 490, 506, 495, 532, 458, 558, 499, 484, 537, 546, 557, 508, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[479, 567, 508, 500, 464, 466, 553, 536, 517, 516, 478, 500, 486, 556, 515, 521, 525, 475, 497, 523, 564, 492, 524, 525, 508, 483, 477, 525, 531, 530, 550, 505, 467, 478, 498, 483, 460, 516, 509, 518, 527, 514, 499, 533, 472, 539, 545, 491, 510, 506, 540, 539, 499, 514, 513, 517, 475, 526, 500, 480, 531, 510, 500, 514, 502, 501, 554, 502, 509, 508, 501, 498, 517, 490, 522, 496, 485, 490, 518, 512, 540, 503, 510, 552, 525, 518, 509, 492, 548, 513, 540, 486, 505, 492, 516, 529, 572, 537, 486, 524, 515, 517, 496, 509, 465, 528, 535, 503, 525, 525, 530, 513, 546, 519, 507, 492, 533, 505, 546, 538, 529, 521, 517, 471, 485, 546, 496, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[516, 500, 503, 509, 565, 537, 506, 486, 536, 525, 501, 503, 519, 514, 511, 516, 558, 517, 497, 507, 503, 518, 518, 474, 527, 484, 508, 532, 560, 528, 517, 545, 516, 481, 502, 462, 487, 501, 544, 494, 515, 482, 541, 521, 492, 502, 480, 498, 529, 549, 531, 521, 495, 553, 508, 503, 489, 475, 474, 479, 546, 488, 533, 529, 497, 485, 475, 546, 525, 533, 516, 488, 504, 517, 478, 502, 502, 514, 517, 551, 534, 540, 489, 511, 504, 507, 514, 478, 489, 522, 513, 531, 553, 534, 527, 510, 507, 530, 518, 487, 530, 494, 512, 498, 478, 542, 514, 518, 538, 519, 509, 534, 510, 516, 470, 491, 521, 499, 536, 512, 533, 484, 537, 491, 477, 502, 534, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[552, 484, 520, 513, 508, 568, 502, 515, 491, 523, 518, 530, 534, 526, 515, 522, 520, 546, 496, 488, 491, 491, 501, 516, 483, 503, 504, 550, 523, 491, 521, 479, 532, 488, 557, 481, 521, 510, 502, 534, 486, 487, 520, 525, 525, 489, 514, 511, 478, 516, 503, 527, 502, 501, 507, 525, 531, 534, 469, 488, 520, 488, 508, 500, 494, 492, 530, 538, 498, 538, 502, 526, 526, 519, 540, 482, 521, 473, 496, 537, 511, 537, 500, 525, 544, 503, 552, 522, 546, 497, 476, 486, 489, 513, 522, 525, 498, 514, 481, 490, 486, 554, 491, 519, 503, 479, 504, 527, 508, 516, 543, 531, 514, 569, 505, 509, 514, 486, 511, 492, 497, 516, 510, 517, 525, 520, 536, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[494, 525, 499, 503, 540, 499, 521, 570, 499, 491, 528, 504, 486, 506, 527, 485, 537, 547, 470, 519, 492, 536, 506, 528, 522, 491, 513, 500, 523, 529, 514, 556, 496, 512, 511, 523, 490, 523, 485, 529, 558, 502, 504, 521, 509, 546, 566, 492, 513, 485, 526, 502, 485, 482, 521, 526, 561, 492, 499, 481, 512, 486, 514, 497, 510, 506, 490, 519, 501, 460, 518, 524, 497, 548, 513, 494, 540, 555, 548, 460, 536, 498, 489, 512, 517, 488, 498, 541, 538, 532, 501, 532, 529, 491, 537, 521, 495, 477, 541, 501, 563, 487, 485, 498, 508, 509, 535, 514, 531, 533, 525, 517, 484, 501, 474, 486, 504, 523, 518, 511, 490, 507, 483, 500, 520, 497, 530, 537] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[539, 534, 550, 510, 497, 519, 491, 509, 521, 499, 500, 543, 522, 537, 510, 497, 509, 540, 509, 518, 513, 487, 493, 524, 519, 512, 509, 507, 510, 528, 516, 547, 517, 520, 525, 531, 522, 503, 535, 497, 495, 548, 490, 505, 525, 507, 540, 495, 493, 501, 495, 513, 519, 520, 498, 537, 502, 520, 508, 568, 529, 505, 491, 495, 492, 494, 509, 497, 498, 503, 501, 528, 511, 469, 512, 476, 495, 512, 519, 480, 478, 559, 516, 508, 523, 517, 484, 517, 483, 540, 524, 515, 507, 513, 471, 528, 520, 531, 505, 531, 525, 546, 461, 492, 528, 541, 488, 501, 519, 519, 541, 473, 492, 500, 534, 479, 500, 480, 493, 531, 518, 529, 504, 514, 511, 516, 533, 534] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[526, 534, 526, 516, 500, 523, 542, 533, 494, 521, 501, 537, 509, 501, 494, 521, 517, 494, 469, 539, 517, 515, 487, 508, 521, 517, 491, 513, 475, 513, 521, 517, 517, 507, 504, 480, 499, 554, 505, 529, 533, 525, 512, 494, 490, 523, 512, 488, 550, 542, 544, 481, 524, 536, 542, 501, 513, 506, 525, 530, 508, 504, 525, 493, 463, 511, 489, 540, 543, 470, 563, 517, 518, 539, 533, 480, 495, 523, 516, 525, 504, 516, 537, 536, 513, 512, 452, 494, 506, 482, 487, 480, 547, 488, 531, 496, 519, 495, 504, 507, 460, 474, 514, 494, 568, 556, 515, 499, 501, 485, 525, 485, 482, 550, 525, 490, 506, 495, 532, 458, 558, 499, 484, 537, 546, 557, 508, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[494, 525, 476, 503, 540, 499, 521, 570, 499, 491, 528, 504, 486, 506, 527, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 547, 470, 519, 492, 536, 506, 528, 522, 491, 513, 500, 523, 529, 514, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 512, 511, 523, 490, 523, 485, 529, 558, 502, 504, 521, 509, 546, 566, 492, 23, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 485, 515, 502, 485, 482, 521, 526, 561, 492, 499, 481, 512, 486, 514, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 506, 490, 519, 501, 460, 518, 524, 497, 548, 513, 494, 540, 555, 548, 460, 12, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 498, 489, 510, 517, 488, 498, 541, 538, 532, 501, 532, 529, 491, 537, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 477, 541, 501, 563, 487, 485, 498, 508, 509, 535, 514, 531, 533, 525, 517, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 501, 474, 486, 504, 523, 518, 511, 490, 507, 483, 500, 520, 497, 518, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8261, 8306, 8071, 8207, 8258, 8226, 8053] avg=8192.0000 max=8306 max_rank=2 min=8053 min_rank=7 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32744 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[491, 511, 526, 489, 506, 522, 506, 509, 532, 482, 475, 474, 489, 512, 515, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 537, 516, 477, 505, 510, 496, 517, 529, 553, 522, 537, 517, 501, 517, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 508, 517, 501, 504, 504, 510, 480, 490, 532, 547, 535, 514, 501, 480, 519, 29, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 497, 520, 543, 527, 495, 489, 509, 537, 496, 527, 527, 523, 504, 500, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 526, 504, 491, 494, 528, 503, 498, 495, 500, 562, 522, 483, 529, 524, 514, 17, 11, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 517, 515, 523, 492, 492, 495, 518, 519, 497, 499, 495, 527, 516, 521, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 537, 536, 516, 503, 525, 478, 488, 485, 504, 538, 535, 519, 493, 523, 494, 10, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 480, 460, 486, 496, 530, 521, 534, 526, 536, 509, 456, 495, 542, 523, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8053, 8252, 8215, 8272, 8240, 8141, 8218, 8145] avg=8192.0000 max=8272 max_rank=3 min=8053 min_rank=0 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32744 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[516, 500, 480, 509, 565, 537, 506, 486, 536, 525, 501, 503, 519, 514, 511, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 517, 497, 507, 503, 518, 518, 474, 527, 484, 508, 532, 560, 528, 517, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 481, 502, 462, 487, 501, 544, 494, 515, 482, 541, 521, 492, 502, 480, 498, 23, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 549, 520, 521, 495, 553, 508, 503, 489, 475, 474, 479, 546, 488, 533, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 485, 475, 546, 525, 533, 516, 488, 504, 517, 478, 502, 502, 514, 517, 551, 12, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 540, 489, 509, 504, 507, 514, 478, 489, 522, 513, 531, 553, 534, 527, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 530, 518, 487, 530, 494, 512, 498, 478, 542, 514, 518, 538, 519, 509, 534, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 516, 470, 491, 521, 499, 536, 512, 533, 484, 537, 491, 477, 502, 522, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8224, 8270, 8057, 8191, 8175, 8254, 8235, 8130] avg=8192.0000 max=8270 max_rank=1 min=8057 min_rank=2 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32794 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[552, 484, 496, 513, 508, 568, 502, 515, 491, 523, 518, 530, 534, 526, 515, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 546, 496, 488, 491, 491, 501, 516, 483, 503, 504, 550, 523, 491, 521, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 488, 557, 481, 521, 510, 502, 534, 486, 487, 520, 525, 525, 489, 514, 511, 24, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 516, 493, 527, 502, 501, 507, 525, 531, 534, 469, 488, 520, 488, 508, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 492, 530, 538, 498, 538, 502, 526, 526, 519, 540, 482, 521, 473, 496, 537, 13, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 537, 500, 523, 544, 503, 552, 522, 546, 497, 476, 486, 489, 513, 522, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 514, 481, 490, 486, 554, 491, 519, 503, 479, 504, 527, 508, 516, 543, 531, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 569, 505, 509, 514, 486, 511, 492, 497, 516, 510, 517, 525, 520, 523, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8297, 8082, 8221, 8087, 8236, 8246, 8151, 8216] avg=8192.0000 max=8297 max_rank=0 min=8082 min_rank=1 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32849 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[539, 534, 525, 510, 497, 519, 491, 509, 521, 499, 500, 543, 522, 537, 510, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 540, 509, 518, 513, 487, 493, 524, 519, 512, 509, 507, 510, 528, 516, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 520, 525, 531, 522, 503, 535, 497, 495, 548, 490, 505, 525, 507, 540, 495, 25, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 501, 485, 513, 519, 520, 498, 537, 502, 520, 508, 568, 529, 505, 491, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 494, 509, 497, 498, 503, 501, 528, 511, 469, 512, 476, 495, 512, 519, 480, 12, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 559, 516, 506, 523, 517, 484, 517, 483, 540, 524, 515, 507, 513, 471, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 531, 505, 531, 525, 546, 461, 492, 528, 541, 488, 501, 519, 519, 541, 473, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 500, 534, 479, 500, 480, 493, 531, 518, 529, 504, 514, 511, 516, 521, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8253, 8220, 8295, 8184, 8019, 8181, 8228, 8156] avg=8192.0000 max=8295 max_rank=2 min=8019 min_rank=4 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32584 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[526, 534, 502, 516, 500, 523, 542, 533, 494, 521, 501, 537, 509, 501, 494, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 494, 469, 539, 517, 515, 487, 508, 521, 517, 491, 513, 475, 513, 521, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 507, 504, 480, 499, 554, 505, 529, 533, 525, 512, 494, 490, 523, 512, 488, 24, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 542, 533, 481, 524, 536, 542, 501, 513, 506, 525, 530, 508, 504, 525, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 463, 511, 489, 540, 543, 470, 563, 517, 518, 539, 533, 480, 495, 523, 516, 525, 12, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 516, 537, 534, 513, 512, 452, 494, 506, 482, 487, 480, 547, 488, 531, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 495, 504, 507, 460, 474, 514, 494, 568, 556, 515, 499, 501, 485, 525, 485, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 550, 525, 490, 506, 495, 532, 458, 558, 499, 484, 537, 546, 557, 496, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8254, 8093, 8211, 8313, 8249, 8079, 8108, 8229] avg=8192.0000 max=8313 max_rank=3 min=8079 min_rank=5 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32665 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[514, 486, 500, 555, 516, 527, 549, 487, 520, 484, 524, 495, 467, 532, 492, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 512, 530, 478, 504, 533, 493, 498, 536, 501, 511, 523, 510, 517, 500, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 483, 460, 486, 508, 510, 541, 536, 540, 490, 506, 506, 506, 502, 496, 503, 24, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 535, 519, 538, 518, 491, 544, 495, 503, 512, 497, 529, 493, 538, 509, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 542, 508, 525, 517, 484, 511, 519, 499, 537, 565, 489, 476, 556, 481, 526, 12, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 477, 518, 513, 494, 513, 482, 526, 501, 509, 494, 516, 487, 560, 531, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 517, 473, 489, 509, 496, 512, 504, 494, 496, 514, 528, 506, 489, 508, 499, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 502, 533, 541, 525, 518, 521, 563, 484, 527, 522, 518, 544, 510, 515, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8162, 8145, 8144, 8255, 8281, 8101, 8083, 8365] avg=8192.0000 max=8365 max_rank=7 min=8083 min_rank=6 imbalance_ratio=1.021118 max_load_reduction=57171 max_load_reduction_pct=0.872360 moved_to_echo=32830 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[494, 525, 476, 503, 540, 499, 521, 570, 499, 491, 528, 504, 486, 506, 527, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 547, 470, 519, 492, 536, 506, 528, 522, 491, 513, 500, 523, 529, 514, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 512, 511, 523, 490, 523, 485, 529, 558, 502, 504, 521, 509, 546, 566, 492, 23, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 485, 515, 502, 485, 482, 521, 526, 561, 492, 499, 481, 512, 486, 514, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 506, 490, 519, 501, 460, 518, 524, 497, 548, 513, 494, 540, 555, 548, 460, 12, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 498, 489, 510, 517, 488, 498, 541, 538, 532, 501, 532, 529, 491, 537, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 477, 541, 501, 563, 487, 485, 498, 508, 509, 535, 514, 531, 533, 525, 517, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 501, 474, 486, 504, 523, 518, 511, 490, 507, 483, 500, 520, 497, 518, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8261, 8306, 8071, 8207, 8258, 8226, 8053] avg=8192.0000 max=8306 max_rank=2 min=8053 min_rank=7 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32744 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[552, 484, 496, 513, 508, 568, 502, 515, 491, 523, 518, 530, 534, 526, 515, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 546, 496, 488, 491, 491, 501, 516, 483, 503, 504, 550, 523, 491, 521, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 488, 557, 481, 521, 510, 502, 534, 486, 487, 520, 525, 525, 489, 514, 511, 24, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 516, 493, 527, 502, 501, 507, 525, 531, 534, 469, 488, 520, 488, 508, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 492, 530, 538, 498, 538, 502, 526, 526, 519, 540, 482, 521, 473, 496, 537, 13, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 537, 500, 523, 544, 503, 552, 522, 546, 497, 476, 486, 489, 513, 522, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 514, 481, 490, 486, 554, 491, 519, 503, 479, 504, 527, 508, 516, 543, 531, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 569, 505, 509, 514, 486, 511, 492, 497, 516, 510, 517, 525, 520, 523, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8297, 8082, 8221, 8087, 8236, 8246, 8151, 8216] avg=8192.0000 max=8297 max_rank=0 min=8082 min_rank=1 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32849 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[479, 567, 485, 500, 464, 466, 553, 536, 517, 516, 478, 500, 486, 556, 515, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 475, 497, 523, 564, 492, 524, 525, 508, 483, 477, 525, 531, 530, 550, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 478, 498, 483, 460, 516, 509, 518, 527, 514, 499, 533, 472, 539, 545, 491, 23, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 506, 529, 539, 499, 514, 513, 517, 475, 526, 500, 480, 531, 510, 500, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 501, 554, 502, 509, 508, 501, 498, 517, 490, 522, 496, 485, 490, 518, 512, 12, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 503, 510, 550, 525, 518, 509, 492, 548, 513, 540, 486, 505, 492, 516, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 572, 537, 486, 524, 515, 517, 496, 509, 465, 528, 535, 503, 525, 525, 530, 513, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 519, 507, 492, 533, 505, 546, 538, 529, 521, 517, 471, 485, 546, 484, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8139, 8213, 8087, 8163, 8129, 8276, 8287, 8242] avg=8192.0000 max=8287 max_rank=6 min=8087 min_rank=2 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32934 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[526, 534, 502, 516, 500, 523, 542, 533, 494, 521, 501, 537, 509, 501, 494, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 494, 469, 539, 517, 515, 487, 508, 521, 517, 491, 513, 475, 513, 521, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 507, 504, 480, 499, 554, 505, 529, 533, 525, 512, 494, 490, 523, 512, 488, 24, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 542, 533, 481, 524, 536, 542, 501, 513, 506, 525, 530, 508, 504, 525, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 463, 511, 489, 540, 543, 470, 563, 517, 518, 539, 533, 480, 495, 523, 516, 525, 12, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 516, 537, 534, 513, 512, 452, 494, 506, 482, 487, 480, 547, 488, 531, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 495, 504, 507, 460, 474, 514, 494, 568, 556, 515, 499, 501, 485, 525, 485, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 550, 525, 490, 506, 495, 532, 458, 558, 499, 484, 537, 546, 557, 496, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8254, 8093, 8211, 8313, 8249, 8079, 8108, 8229] avg=8192.0000 max=8313 max_rank=3 min=8079 min_rank=5 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32665 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[516, 500, 480, 509, 565, 537, 506, 486, 536, 525, 501, 503, 519, 514, 511, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 517, 497, 507, 503, 518, 518, 474, 527, 484, 508, 532, 560, 528, 517, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 481, 502, 462, 487, 501, 544, 494, 515, 482, 541, 521, 492, 502, 480, 498, 23, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 549, 520, 521, 495, 553, 508, 503, 489, 475, 474, 479, 546, 488, 533, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 485, 475, 546, 525, 533, 516, 488, 504, 517, 478, 502, 502, 514, 517, 551, 12, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 540, 489, 509, 504, 507, 514, 478, 489, 522, 513, 531, 553, 534, 527, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 530, 518, 487, 530, 494, 512, 498, 478, 542, 514, 518, 538, 519, 509, 534, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 516, 470, 491, 521, 499, 536, 512, 533, 484, 537, 491, 477, 502, 522, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8224, 8270, 8057, 8191, 8175, 8254, 8235, 8130] avg=8192.0000 max=8270 max_rank=1 min=8057 min_rank=2 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32794 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[539, 534, 525, 510, 497, 519, 491, 509, 521, 499, 500, 543, 522, 537, 510, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 540, 509, 518, 513, 487, 493, 524, 519, 512, 509, 507, 510, 528, 516, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 520, 525, 531, 522, 503, 535, 497, 495, 548, 490, 505, 525, 507, 540, 495, 25, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 501, 485, 513, 519, 520, 498, 537, 502, 520, 508, 568, 529, 505, 491, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 494, 509, 497, 498, 503, 501, 528, 511, 469, 512, 476, 495, 512, 519, 480, 12, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 559, 516, 506, 523, 517, 484, 517, 483, 540, 524, 515, 507, 513, 471, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 531, 505, 531, 525, 546, 461, 492, 528, 541, 488, 501, 519, 519, 541, 473, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 500, 534, 479, 500, 480, 493, 531, 518, 529, 504, 514, 511, 516, 521, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8253, 8220, 8295, 8184, 8019, 8181, 8228, 8156] avg=8192.0000 max=8295 max_rank=2 min=8019 min_rank=4 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32584 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 3968, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4352, 3968, 4096, 4224, 4224, 4224, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4096, 4096, 3968, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 3968, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4096, 4096, 3968, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4352, 3968, 4096, 4224, 4224, 4224, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True - [2026-08-26 14:49:28] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 415.0 | throughput per GPU (TFLOP/s/GPU): 435.7 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.340171E+01 | loss scale: 1.0 | grad norm: 19.518 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[478, 547, 500, 506, 512, 505, 527, 531, 507, 478, 498, 534, 532, 510, 508, 522, 496, 480, 549, 490, 537, 516, 493, 533, 486, 533, 509, 554, 553, 471, 543, 509, 507, 487, 514, 515, 550, 516, 516, 495, 482, 510, 545, 503, 551, 508, 509, 519, 508, 515, 511, 480, 501, 511, 502, 505, 532, 506, 512, 527, 537, 501, 510, 536, 518, 480, 541, 507, 489, 518, 522, 505, 551, 487, 493, 500, 472, 488, 488, 542, 511, 536, 489, 497, 557, 488, 503, 518, 518, 495, 561, 524, 500, 520, 509, 492, 540, 497, 506, 511, 500, 502, 543, 542, 535, 535, 538, 523, 458, 526, 521, 488, 530, 516, 529, 504, 455, 463, 510, 514, 513, 483, 520, 494, 482, 532, 514, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[522, 525, 503, 513, 540, 494, 527, 511, 497, 519, 500, 535, 483, 497, 493, 478, 515, 533, 518, 550, 493, 511, 510, 503, 539, 516, 492, 518, 498, 509, 489, 497, 560, 529, 498, 501, 528, 505, 488, 503, 512, 531, 527, 519, 465, 464, 480, 538, 503, 503, 527, 484, 506, 538, 480, 524, 456, 522, 534, 476, 562, 507, 482, 515, 515, 539, 529, 509, 519, 528, 508, 540, 577, 532, 536, 505, 500, 530, 536, 528, 493, 512, 451, 545, 545, 502, 497, 561, 537, 512, 507, 492, 497, 526, 496, 483, 503, 494, 497, 553, 548, 474, 498, 480, 529, 502, 520, 516, 490, 502, 524, 505, 492, 486, 532, 520, 464, 538, 531, 517, 503, 526, 510, 523, 572, 497, 518, 490] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[498, 499, 520, 523, 511, 540, 490, 504, 493, 529, 525, 509, 493, 531, 502, 506, 553, 564, 528, 498, 496, 493, 527, 475, 520, 540, 489, 473, 545, 511, 465, 485, 524, 525, 507, 532, 543, 493, 467, 510, 488, 493, 468, 529, 494, 513, 530, 510, 507, 517, 498, 483, 540, 538, 484, 486, 529, 510, 506, 519, 530, 496, 486, 536, 504, 511, 518, 526, 526, 507, 485, 512, 479, 552, 506, 516, 491, 522, 512, 504, 525, 522, 514, 499, 544, 523, 512, 501, 516, 489, 496, 505, 561, 499, 489, 523, 517, 533, 526, 551, 524, 518, 509, 508, 528, 478, 552, 536, 485, 544, 514, 508, 507, 504, 525, 525, 516, 499, 525, 547, 518, 513, 482, 530, 520, 498, 479, 502] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True + [2026-08-26 15:25:31] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 432.6 | throughput per GPU (TFLOP/s/GPU): 418.0 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.340218E+01 | loss scale: 1.0 | grad norm: 19.503 | number of skipped iterations: 0 | number of nan iterations: 0 | WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[520, 504, 526, 483, 541, 510, 522, 515, 504, 497, 483, 511, 509, 538, 540, 529, 516, 499, 475, 485, 502, 474, 520, 477, 511, 523, 526, 535, 469, 526, 519, 539, 527, 516, 522, 531, 527, 505, 472, 560, 536, 527, 501, 510, 522, 552, 555, 479, 517, 502, 498, 551, 482, 448, 461, 524, 502, 521, 495, 517, 510, 496, 515, 482, 502, 537, 517, 468, 458, 548, 525, 538, 527, 498, 497, 558, 516, 487, 520, 517, 520, 534, 507, 552, 494, 554, 468, 522, 527, 494, 531, 506, 521, 492, 520, 492, 501, 509, 502, 525, 515, 526, 524, 488, 491, 497, 522, 530, 486, 514, 526, 516, 520, 527, 509, 475, 541, 505, 520, 511, 519, 512, 523, 535, 524, 485, 518, 502] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[526, 503, 512, 484, 531, 497, 508, 521, 502, 481, 539, 489, 557, 483, 521, 499, 493, 546, 485, 488, 474, 500, 517, 495, 537, 477, 476, 521, 490, 512, 497, 517, 510, 491, 518, 555, 530, 528, 509, 518, 513, 534, 503, 523, 499, 482, 481, 477, 479, 557, 515, 558, 515, 522, 521, 533, 525, 525, 509, 513, 514, 498, 484, 512, 518, 511, 532, 564, 500, 518, 537, 523, 510, 488, 490, 503, 496, 545, 478, 503, 480, 479, 544, 504, 558, 518, 512, 553, 477, 525, 553, 496, 497, 510, 531, 502, 534, 514, 505, 510, 508, 489, 510, 543, 551, 521, 480, 541, 513, 520, 518, 505, 543, 495, 502, 499, 511, 540, 511, 487, 481, 504, 540, 540, 516, 530, 505, 486] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[515, 547, 503, 528, 481, 511, 519, 536, 522, 510, 492, 530, 538, 464, 523, 515, 529, 489, 502, 513, 529, 487, 529, 508, 513, 535, 501, 533, 525, 519, 502, 493, 509, 501, 537, 509, 500, 541, 514, 514, 510, 488, 519, 544, 542, 537, 520, 473, 548, 485, 526, 547, 517, 510, 540, 511, 518, 493, 480, 449, 517, 535, 510, 503, 506, 536, 531, 527, 510, 541, 488, 500, 480, 497, 500, 526, 503, 521, 512, 504, 507, 505, 500, 548, 522, 498, 484, 542, 536, 484, 521, 511, 488, 508, 497, 489, 536, 494, 480, 531, 492, 528, 515, 510, 486, 494, 533, 503, 503, 500, 511, 528, 514, 508, 526, 490, 503, 507, 552, 523, 495, 533, 529, 518, 525, 483, 510, 466] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[488, 506, 510, 500, 533, 487, 532, 514, 526, 471, 519, 539, 535, 484, 467, 492, 505, 508, 497, 482, 527, 495, 505, 508, 555, 507, 519, 520, 567, 494, 516, 508, 512, 491, 482, 484, 504, 542, 538, 514, 494, 501, 505, 527, 527, 532, 506, 519, 497, 479, 547, 549, 509, 482, 529, 461, 474, 493, 500, 503, 512, 531, 503, 503, 492, 500, 510, 533, 506, 542, 493, 458, 505, 499, 547, 498, 522, 514, 510, 556, 482, 552, 524, 507, 491, 553, 491, 483, 546, 487, 511, 513, 540, 538, 572, 522, 531, 530, 477, 538, 525, 493, 516, 529, 540, 525, 531, 481, 561, 537, 495, 517, 499, 503, 500, 489, 518, 490, 557, 549, 521, 506, 489, 471, 520, 510, 512, 513] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[498, 499, 520, 523, 511, 540, 490, 504, 493, 529, 525, 509, 493, 531, 502, 506, 553, 564, 528, 498, 496, 493, 527, 475, 520, 540, 489, 473, 545, 511, 465, 485, 524, 525, 507, 532, 543, 493, 467, 510, 488, 493, 468, 529, 494, 513, 530, 510, 507, 517, 498, 483, 540, 538, 484, 486, 529, 510, 506, 519, 530, 496, 486, 536, 504, 511, 518, 526, 526, 507, 485, 512, 479, 552, 506, 516, 491, 522, 512, 504, 525, 522, 514, 499, 544, 523, 512, 501, 516, 489, 496, 505, 561, 499, 489, 523, 517, 533, 526, 551, 524, 518, 509, 508, 528, 478, 552, 536, 485, 544, 514, 508, 507, 504, 525, 525, 516, 499, 525, 547, 518, 513, 482, 530, 520, 498, 479, 502] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[478, 547, 500, 506, 512, 505, 527, 531, 507, 478, 498, 534, 532, 510, 508, 522, 496, 480, 549, 490, 537, 516, 493, 533, 486, 533, 509, 554, 553, 471, 543, 509, 507, 487, 514, 515, 550, 516, 516, 495, 482, 510, 545, 503, 551, 508, 509, 519, 508, 515, 511, 480, 501, 511, 502, 505, 532, 506, 512, 527, 537, 501, 510, 536, 518, 480, 541, 507, 489, 518, 522, 505, 551, 487, 493, 500, 472, 488, 488, 542, 511, 536, 489, 497, 557, 488, 503, 518, 518, 495, 561, 524, 500, 520, 509, 492, 540, 497, 506, 511, 500, 502, 543, 542, 535, 535, 538, 523, 458, 526, 521, 488, 530, 516, 529, 504, 455, 463, 510, 514, 513, 483, 520, 494, 482, 532, 514, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[540, 497, 513, 512, 475, 504, 511, 522, 519, 533, 497, 514, 476, 501, 521, 524, 525, 511, 545, 526, 520, 513, 549, 521, 494, 517, 510, 510, 537, 529, 505, 544, 478, 543, 522, 516, 545, 523, 527, 526, 486, 515, 489, 518, 492, 498, 462, 528, 452, 506, 527, 528, 531, 489, 507, 501, 497, 524, 537, 487, 534, 516, 455, 502, 498, 525, 514, 525, 497, 489, 532, 525, 520, 497, 533, 545, 522, 523, 493, 539, 490, 497, 484, 537, 521, 524, 517, 497, 521, 527, 485, 527, 493, 473, 504, 475, 538, 492, 523, 496, 514, 524, 520, 500, 533, 493, 514, 479, 502, 520, 546, 488, 540, 495, 555, 507, 464, 510, 495, 532, 517, 502, 517, 518, 514, 519, 497, 547] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[526, 503, 512, 484, 531, 497, 508, 521, 502, 481, 539, 489, 557, 483, 521, 499, 14, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 546, 485, 488, 474, 500, 517, 495, 537, 477, 476, 521, 490, 512, 497, 517, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 491, 518, 555, 520, 528, 509, 518, 513, 534, 503, 523, 499, 482, 481, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 557, 515, 558, 515, 522, 521, 533, 525, 525, 509, 513, 514, 498, 484, 512, 34, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 511, 532, 564, 500, 518, 537, 523, 510, 488, 490, 503, 496, 545, 478, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 479, 544, 504, 552, 518, 512, 553, 477, 525, 553, 496, 497, 510, 531, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 514, 505, 476, 508, 489, 510, 543, 551, 521, 480, 541, 513, 520, 518, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 495, 502, 499, 511, 540, 511, 487, 481, 504, 540, 540, 516, 530, 505, 486, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 8029, 8161, 8328, 8188, 8233, 8228, 8198] avg=8192.0000 max=8328 max_rank=3 min=8029 min_rank=1 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=32847 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[522, 525, 503, 513, 540, 494, 527, 511, 497, 519, 500, 535, 483, 497, 493, 478, 14, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 533, 518, 550, 493, 511, 510, 503, 539, 516, 492, 518, 498, 509, 489, 497, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 560, 529, 498, 501, 519, 505, 488, 503, 512, 531, 527, 519, 465, 464, 480, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 503, 527, 484, 506, 538, 480, 524, 456, 522, 534, 476, 562, 507, 482, 515, 36, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 539, 529, 509, 519, 528, 508, 540, 577, 532, 536, 505, 500, 530, 536, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 512, 451, 545, 539, 502, 497, 561, 537, 512, 507, 492, 497, 526, 496, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 494, 497, 517, 548, 474, 498, 480, 529, 502, 520, 516, 490, 502, 524, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 486, 532, 520, 464, 538, 531, 517, 503, 526, 510, 523, 572, 497, 518, 490, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8195, 8139, 8170, 8402, 8150, 8099, 8227] avg=8192.0000 max=8402 max_rank=4 min=8099 min_rank=6 imbalance_ratio=1.025635 max_load_reduction=57134 max_load_reduction_pct=0.871796 moved_to_echo=32878 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[488, 506, 510, 500, 533, 487, 532, 514, 526, 471, 519, 539, 535, 484, 467, 492, 505, 508, 497, 482, 527, 495, 505, 508, 555, 507, 519, 520, 567, 494, 516, 508, 512, 491, 482, 484, 504, 542, 538, 514, 494, 501, 505, 527, 527, 532, 506, 519, 497, 479, 547, 549, 509, 482, 529, 461, 474, 493, 500, 503, 512, 531, 503, 503, 492, 500, 510, 533, 506, 542, 493, 458, 505, 499, 547, 498, 522, 514, 510, 556, 482, 552, 524, 507, 491, 553, 491, 483, 546, 487, 511, 513, 540, 538, 572, 522, 531, 530, 477, 538, 525, 493, 516, 529, 540, 525, 531, 481, 561, 537, 495, 517, 499, 503, 500, 489, 518, 490, 557, 549, 521, 506, 489, 471, 520, 510, 512, 513] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[526, 503, 512, 484, 531, 497, 508, 521, 502, 481, 539, 489, 557, 483, 521, 499, 493, 546, 485, 488, 474, 500, 517, 495, 537, 477, 476, 521, 490, 512, 497, 517, 510, 491, 518, 555, 530, 528, 509, 518, 513, 534, 503, 523, 499, 482, 481, 477, 479, 557, 515, 558, 515, 522, 521, 533, 525, 525, 509, 513, 514, 498, 484, 512, 518, 511, 532, 564, 500, 518, 537, 523, 510, 488, 490, 503, 496, 545, 478, 503, 480, 479, 544, 504, 558, 518, 512, 553, 477, 525, 553, 496, 497, 510, 531, 502, 534, 514, 505, 510, 508, 489, 510, 543, 551, 521, 480, 541, 513, 520, 518, 505, 543, 495, 502, 499, 511, 540, 511, 487, 481, 504, 540, 540, 516, 530, 505, 486] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[522, 525, 503, 513, 540, 494, 527, 511, 497, 519, 500, 535, 483, 497, 493, 478, 515, 533, 518, 550, 493, 511, 510, 503, 539, 516, 492, 518, 498, 509, 489, 497, 560, 529, 498, 501, 528, 505, 488, 503, 512, 531, 527, 519, 465, 464, 480, 538, 503, 503, 527, 484, 506, 538, 480, 524, 456, 522, 534, 476, 562, 507, 482, 515, 515, 539, 529, 509, 519, 528, 508, 540, 577, 532, 536, 505, 500, 530, 536, 528, 493, 512, 451, 545, 545, 502, 497, 561, 537, 512, 507, 492, 497, 526, 496, 483, 503, 494, 497, 553, 548, 474, 498, 480, 529, 502, 520, 516, 490, 502, 524, 505, 492, 486, 532, 520, 464, 538, 531, 517, 503, 526, 510, 523, 572, 497, 518, 490] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[520, 504, 526, 483, 541, 510, 522, 515, 504, 497, 483, 511, 509, 538, 540, 529, 17, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 499, 475, 485, 502, 474, 520, 477, 511, 523, 526, 535, 469, 526, 519, 539, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 516, 522, 531, 510, 505, 472, 560, 536, 527, 501, 510, 522, 552, 555, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 502, 498, 551, 482, 448, 461, 524, 502, 521, 495, 517, 510, 496, 515, 482, 38, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 537, 517, 468, 458, 548, 525, 538, 527, 498, 497, 558, 516, 487, 520, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 534, 507, 552, 482, 554, 468, 522, 527, 494, 531, 506, 521, 492, 520, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 509, 502, 487, 515, 526, 524, 488, 491, 497, 522, 530, 486, 514, 526, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 527, 509, 475, 541, 505, 520, 511, 519, 512, 523, 535, 524, 485, 518, 502, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8256, 8103, 8325, 8078, 8177, 8222, 8134, 8241] avg=8192.0000 max=8325 max_rank=2 min=8078 min_rank=3 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32774 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[540, 497, 513, 512, 475, 504, 511, 522, 519, 533, 497, 514, 476, 501, 521, 524, 15, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 511, 545, 526, 520, 513, 549, 521, 494, 517, 510, 510, 537, 529, 505, 544, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 543, 522, 516, 535, 523, 527, 526, 486, 515, 489, 518, 492, 498, 462, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 452, 506, 527, 528, 531, 489, 507, 501, 497, 524, 537, 487, 534, 516, 455, 502, 33, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 525, 514, 525, 497, 489, 532, 525, 520, 497, 533, 545, 522, 523, 493, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 497, 484, 537, 515, 524, 517, 497, 521, 527, 485, 527, 493, 473, 504, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 492, 523, 463, 514, 524, 520, 500, 533, 493, 514, 479, 502, 520, 546, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 495, 555, 507, 464, 510, 495, 532, 517, 502, 517, 518, 514, 519, 497, 547, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8360, 8158, 8141, 8247, 8066, 8149, 8237] avg=8192.0000 max=8360 max_rank=1 min=8066 min_rank=5 imbalance_ratio=1.020508 max_load_reduction=57176 max_load_reduction_pct=0.872437 moved_to_echo=32699 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[478, 547, 500, 506, 512, 505, 527, 531, 507, 478, 498, 534, 532, 510, 508, 522, 15, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 480, 549, 490, 537, 516, 493, 533, 486, 533, 509, 554, 553, 471, 543, 509, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 487, 514, 515, 540, 516, 516, 495, 482, 510, 545, 503, 551, 508, 509, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 515, 511, 480, 501, 511, 502, 505, 532, 506, 512, 527, 537, 501, 510, 536, 34, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 480, 541, 507, 489, 518, 522, 505, 551, 487, 493, 500, 472, 488, 488, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 536, 489, 497, 551, 488, 503, 518, 518, 495, 561, 524, 500, 520, 509, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 497, 506, 477, 500, 502, 543, 542, 535, 535, 538, 523, 458, 526, 521, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 516, 529, 504, 455, 463, 510, 514, 513, 483, 520, 494, 482, 532, 514, 525, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8214, 8256, 8217, 8243, 8071, 8212, 8231, 8092] avg=8192.0000 max=8256 max_rank=1 min=8071 min_rank=4 imbalance_ratio=1.007812 max_load_reduction=57280 max_load_reduction_pct=0.874023 moved_to_echo=32606 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[498, 499, 520, 523, 511, 540, 490, 504, 493, 529, 525, 509, 493, 531, 502, 506, 14, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 564, 528, 498, 496, 493, 527, 475, 520, 540, 489, 473, 545, 511, 465, 485, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 525, 507, 532, 533, 493, 467, 510, 488, 493, 468, 529, 494, 513, 530, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 517, 498, 483, 540, 538, 484, 486, 529, 510, 506, 519, 530, 496, 486, 536, 36, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 511, 518, 526, 526, 507, 485, 512, 479, 552, 506, 516, 491, 522, 512, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 522, 514, 499, 538, 523, 512, 501, 516, 489, 496, 505, 561, 499, 489, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 533, 526, 515, 524, 518, 509, 508, 528, 478, 552, 536, 485, 544, 514, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 504, 525, 525, 516, 499, 525, 547, 518, 513, 482, 530, 520, 498, 479, 502, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8166, 8116, 8215, 8143, 8212, 8295, 8198] avg=8192.0000 max=8295 max_rank=6 min=8116 min_rank=2 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32848 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[515, 547, 503, 528, 481, 511, 519, 536, 522, 510, 492, 530, 538, 464, 523, 515, 14, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 489, 502, 513, 529, 487, 529, 508, 513, 535, 501, 533, 525, 519, 502, 493, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 501, 537, 509, 492, 541, 514, 514, 510, 488, 519, 544, 542, 537, 520, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 485, 526, 547, 517, 510, 540, 511, 518, 493, 480, 449, 517, 535, 510, 503, 35, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 536, 531, 527, 510, 541, 488, 500, 480, 497, 500, 526, 503, 521, 512, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 505, 500, 548, 516, 498, 484, 542, 536, 484, 521, 511, 488, 508, 497, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 494, 480, 496, 492, 528, 515, 510, 486, 494, 533, 503, 503, 500, 511, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 508, 526, 490, 503, 507, 552, 523, 495, 533, 529, 518, 525, 483, 510, 466, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8251, 8211, 8250, 8238, 8154, 8134, 8109, 8189] avg=8192.0000 max=8251 max_rank=0 min=8109 min_rank=6 imbalance_ratio=1.007202 max_load_reduction=57285 max_load_reduction_pct=0.874100 moved_to_echo=32586 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[488, 506, 510, 500, 533, 487, 532, 514, 526, 471, 519, 539, 535, 484, 467, 492, 15, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 508, 497, 482, 527, 495, 505, 508, 555, 507, 519, 520, 567, 494, 516, 508, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 491, 482, 484, 496, 542, 538, 514, 494, 501, 505, 527, 527, 532, 506, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 479, 547, 549, 509, 482, 529, 461, 474, 493, 500, 503, 512, 531, 503, 503, 35, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 500, 510, 533, 506, 542, 493, 458, 505, 499, 547, 498, 522, 514, 510, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 552, 524, 507, 486, 553, 491, 483, 546, 487, 511, 513, 540, 538, 572, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 530, 477, 503, 525, 493, 516, 529, 540, 525, 531, 481, 561, 537, 495, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 503, 500, 489, 518, 490, 557, 549, 521, 506, 489, 471, 520, 510, 512, 513, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8121, 8216, 8170, 8123, 8154, 8307, 8291, 8154] avg=8192.0000 max=8307 max_rank=5 min=8121 min_rank=0 imbalance_ratio=1.014038 max_load_reduction=57229 max_load_reduction_pct=0.873245 moved_to_echo=32906 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[520, 504, 526, 483, 541, 510, 522, 515, 504, 497, 483, 511, 509, 538, 540, 529, 17, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 499, 475, 485, 502, 474, 520, 477, 511, 523, 526, 535, 469, 526, 519, 539, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 516, 522, 531, 510, 505, 472, 560, 536, 527, 501, 510, 522, 552, 555, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 502, 498, 551, 482, 448, 461, 524, 502, 521, 495, 517, 510, 496, 515, 482, 38, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 537, 517, 468, 458, 548, 525, 538, 527, 498, 497, 558, 516, 487, 520, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 534, 507, 552, 482, 554, 468, 522, 527, 494, 531, 506, 521, 492, 520, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 509, 502, 487, 515, 526, 524, 488, 491, 497, 522, 530, 486, 514, 526, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 527, 509, 475, 541, 505, 520, 511, 519, 512, 523, 535, 524, 485, 518, 502, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8256, 8103, 8325, 8078, 8177, 8222, 8134, 8241] avg=8192.0000 max=8325 max_rank=2 min=8078 min_rank=3 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32774 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[540, 497, 513, 512, 475, 504, 511, 522, 519, 533, 497, 514, 476, 501, 521, 524, 15, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 511, 545, 526, 520, 513, 549, 521, 494, 517, 510, 510, 537, 529, 505, 544, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 543, 522, 516, 535, 523, 527, 526, 486, 515, 489, 518, 492, 498, 462, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 452, 506, 527, 528, 531, 489, 507, 501, 497, 524, 537, 487, 534, 516, 455, 502, 33, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 525, 514, 525, 497, 489, 532, 525, 520, 497, 533, 545, 522, 523, 493, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 497, 484, 537, 515, 524, 517, 497, 521, 527, 485, 527, 493, 473, 504, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 492, 523, 463, 514, 524, 520, 500, 533, 493, 514, 479, 502, 520, 546, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 495, 555, 507, 464, 510, 495, 532, 517, 502, 517, 518, 514, 519, 497, 547, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8360, 8158, 8141, 8247, 8066, 8149, 8237] avg=8192.0000 max=8360 max_rank=1 min=8066 min_rank=5 imbalance_ratio=1.020508 max_load_reduction=57176 max_load_reduction_pct=0.872437 moved_to_echo=32699 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[526, 503, 512, 484, 531, 497, 508, 521, 502, 481, 539, 489, 557, 483, 521, 499, 14, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 546, 485, 488, 474, 500, 517, 495, 537, 477, 476, 521, 490, 512, 497, 517, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 491, 518, 555, 520, 528, 509, 518, 513, 534, 503, 523, 499, 482, 481, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 557, 515, 558, 515, 522, 521, 533, 525, 525, 509, 513, 514, 498, 484, 512, 34, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 511, 532, 564, 500, 518, 537, 523, 510, 488, 490, 503, 496, 545, 478, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 479, 544, 504, 552, 518, 512, 553, 477, 525, 553, 496, 497, 510, 531, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 514, 505, 476, 508, 489, 510, 543, 551, 521, 480, 541, 513, 520, 518, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 495, 502, 499, 511, 540, 511, 487, 481, 504, 540, 540, 516, 530, 505, 486, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 8029, 8161, 8328, 8188, 8233, 8228, 8198] avg=8192.0000 max=8328 max_rank=3 min=8029 min_rank=1 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=32847 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[522, 525, 503, 513, 540, 494, 527, 511, 497, 519, 500, 535, 483, 497, 493, 478, 14, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 533, 518, 550, 493, 511, 510, 503, 539, 516, 492, 518, 498, 509, 489, 497, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 560, 529, 498, 501, 519, 505, 488, 503, 512, 531, 527, 519, 465, 464, 480, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 503, 527, 484, 506, 538, 480, 524, 456, 522, 534, 476, 562, 507, 482, 515, 36, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 539, 529, 509, 519, 528, 508, 540, 577, 532, 536, 505, 500, 530, 536, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 512, 451, 545, 539, 502, 497, 561, 537, 512, 507, 492, 497, 526, 496, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 494, 497, 517, 548, 474, 498, 480, 529, 502, 520, 516, 490, 502, 524, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 486, 532, 520, 464, 538, 531, 517, 503, 526, 510, 523, 572, 497, 518, 490, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8195, 8139, 8170, 8402, 8150, 8099, 8227] avg=8192.0000 max=8402 max_rank=4 min=8099 min_rank=6 imbalance_ratio=1.025635 max_load_reduction=57134 max_load_reduction_pct=0.871796 moved_to_echo=32878 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 3968, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 3968, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 3968, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[510, 503, 513, 523, 514, 499, 551, 487, 506, 485, 529, 516, 507, 569, 509, 557, 505, 559, 522, 521, 510, 485, 519, 482, 524, 487, 490, 478, 495, 545, 504, 481, 487, 529, 537, 468, 519, 564, 505, 485, 528, 524, 493, 507, 532, 523, 524, 511, 533, 485, 532, 493, 519, 474, 522, 497, 526, 513, 520, 472, 499, 550, 504, 572, 474, 563, 475, 518, 490, 511, 535, 504, 511, 474, 527, 496, 507, 483, 494, 523, 518, 486, 569, 509, 477, 535, 518, 483, 565, 528, 476, 498, 527, 559, 507, 539, 470, 501, 501, 523, 561, 483, 499, 488, 500, 515, 496, 487, 521, 466, 506, 534, 482, 499, 495, 522, 501, 528, 533, 511, 530, 516, 509, 515, 565, 515, 538, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[543, 553, 512, 483, 494, 537, 511, 522, 518, 502, 530, 496, 464, 492, 553, 511, 488, 551, 472, 507, 561, 539, 488, 493, 508, 500, 529, 513, 518, 508, 529, 517, 525, 543, 485, 524, 482, 496, 500, 506, 530, 483, 531, 478, 518, 518, 501, 517, 493, 513, 491, 487, 540, 509, 506, 491, 518, 517, 514, 493, 555, 550, 482, 534, 509, 535, 486, 505, 503, 529, 528, 521, 585, 521, 514, 473, 509, 494, 498, 473, 523, 524, 494, 485, 522, 489, 550, 504, 505, 513, 535, 476, 501, 519, 533, 516, 515, 528, 511, 534, 493, 512, 541, 538, 572, 528, 495, 518, 504, 531, 507, 499, 501, 488, 505, 526, 486, 502, 465, 509, 483, 526, 511, 488, 534, 501, 554, 487] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[452, 530, 532, 495, 497, 521, 531, 575, 537, 470, 512, 525, 539, 494, 503, 516, 467, 523, 516, 506, 476, 468, 504, 573, 500, 519, 514, 506, 529, 498, 532, 506, 519, 518, 512, 508, 507, 516, 503, 531, 486, 536, 542, 497, 486, 546, 516, 537, 519, 521, 522, 531, 503, 509, 523, 511, 504, 522, 498, 513, 528, 474, 507, 469, 512, 504, 473, 521, 484, 516, 545, 506, 477, 518, 524, 522, 503, 489, 485, 544, 476, 502, 502, 519, 495, 521, 527, 524, 566, 524, 525, 512, 525, 512, 529, 528, 513, 485, 494, 511, 520, 510, 528, 526, 508, 492, 524, 540, 503, 518, 510, 473, 484, 529, 502, 498, 512, 505, 499, 513, 466, 547, 523, 509, 521, 535, 519, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[531, 544, 519, 502, 520, 528, 487, 504, 547, 497, 467, 562, 505, 502, 529, 499, 501, 502, 504, 535, 482, 480, 522, 495, 498, 559, 505, 459, 497, 558, 521, 534, 530, 517, 517, 516, 536, 530, 517, 509, 499, 504, 483, 507, 501, 507, 525, 486, 512, 549, 498, 524, 479, 499, 486, 529, 498, 491, 517, 564, 496, 491, 519, 506, 491, 513, 484, 482, 550, 538, 514, 507, 526, 501, 482, 489, 557, 512, 493, 483, 480, 522, 501, 486, 506, 520, 476, 491, 510, 564, 534, 494, 508, 524, 536, 511, 512, 505, 515, 538, 465, 531, 520, 505, 522, 514, 491, 491, 546, 531, 542, 486, 501, 522, 551, 467, 496, 510, 496, 530, 528, 566, 527, 564, 523, 503, 526, 490] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[527, 502, 494, 536, 529, 498, 524, 560, 488, 502, 532, 539, 517, 498, 528, 513, 456, 494, 511, 500, 512, 548, 516, 474, 509, 497, 490, 513, 531, 530, 540, 552, 462, 510, 572, 513, 545, 488, 501, 540, 508, 509, 488, 509, 458, 499, 508, 525, 542, 537, 474, 518, 509, 534, 526, 536, 506, 495, 518, 499, 475, 518, 514, 485, 535, 515, 530, 498, 496, 468, 514, 488, 510, 512, 514, 508, 505, 536, 499, 501, 490, 509, 542, 533, 488, 500, 538, 524, 524, 522, 502, 500, 521, 489, 505, 547, 485, 528, 486, 505, 506, 518, 490, 521, 562, 524, 500, 488, 523, 504, 523, 527, 508, 499, 496, 498, 539, 476, 526, 520, 530, 526, 542, 493, 495, 521, 546, 487] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[495, 490, 528, 486, 498, 498, 505, 463, 498, 502, 552, 500, 518, 511, 528, 569, 523, 523, 498, 537, 551, 495, 530, 474, 528, 477, 518, 507, 473, 519, 471, 535, 481, 521, 550, 507, 519, 488, 544, 529, 509, 520, 529, 524, 529, 487, 499, 485, 502, 499, 521, 464, 507, 544, 535, 479, 481, 517, 508, 513, 542, 531, 501, 506, 496, 496, 488, 497, 487, 487, 536, 513, 510, 540, 531, 523, 502, 556, 523, 488, 540, 533, 496, 508, 517, 538, 476, 523, 499, 492, 529, 501, 500, 531, 496, 562, 505, 512, 540, 535, 519, 491, 485, 475, 510, 510, 527, 494, 541, 516, 531, 508, 510, 520, 529, 511, 492, 566, 482, 526, 541, 526, 482, 527, 481, 501, 544, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[511, 540, 532, 515, 475, 495, 525, 541, 496, 486, 490, 511, 517, 501, 537, 517, 535, 507, 539, 466, 498, 500, 505, 533, 535, 545, 530, 551, 481, 531, 478, 558, 509, 479, 495, 542, 499, 499, 501, 554, 502, 499, 494, 492, 509, 494, 509, 514, 537, 483, 517, 523, 528, 555, 524, 498, 488, 474, 505, 510, 499, 518, 490, 507, 527, 449, 524, 516, 522, 548, 549, 519, 518, 530, 478, 479, 530, 514, 514, 526, 540, 515, 489, 518, 506, 477, 498, 497, 504, 501, 505, 501, 520, 512, 545, 533, 521, 506, 497, 529, 534, 506, 538, 496, 514, 524, 490, 533, 505, 554, 487, 541, 526, 506, 499, 483, 478, 527, 516, 525, 504, 522, 525, 560, 458, 516, 487, 497] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[531, 544, 519, 502, 520, 528, 487, 504, 547, 497, 467, 562, 505, 502, 529, 499, 501, 502, 504, 535, 482, 480, 522, 495, 498, 559, 505, 459, 497, 558, 521, 534, 530, 517, 517, 516, 536, 530, 517, 509, 499, 504, 483, 507, 501, 507, 525, 486, 512, 549, 498, 524, 479, 499, 486, 529, 498, 491, 517, 564, 496, 491, 519, 506, 491, 513, 484, 482, 550, 538, 514, 507, 526, 501, 482, 489, 557, 512, 493, 483, 480, 522, 501, 486, 506, 520, 476, 491, 510, 564, 534, 494, 508, 524, 536, 511, 512, 505, 515, 538, 465, 531, 520, 505, 522, 514, 491, 491, 546, 531, 542, 486, 501, 522, 551, 467, 496, 510, 496, 530, 528, 566, 527, 564, 523, 503, 526, 490] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[527, 502, 494, 536, 529, 498, 524, 560, 488, 502, 532, 539, 517, 498, 528, 513, 456, 494, 511, 500, 512, 548, 516, 474, 509, 497, 490, 513, 531, 530, 540, 552, 462, 510, 572, 513, 545, 488, 501, 540, 508, 509, 488, 509, 458, 499, 508, 525, 542, 537, 474, 518, 509, 534, 526, 536, 506, 495, 518, 499, 475, 518, 514, 485, 535, 515, 530, 498, 496, 468, 514, 488, 510, 512, 514, 508, 505, 536, 499, 501, 490, 509, 542, 533, 488, 500, 538, 524, 524, 522, 502, 500, 521, 489, 505, 547, 485, 528, 486, 505, 506, 518, 490, 521, 562, 524, 500, 488, 523, 504, 523, 527, 508, 499, 496, 498, 539, 476, 526, 520, 530, 526, 542, 493, 495, 521, 546, 487] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[499, 547, 522, 523, 534, 533, 569, 538, 530, 486, 508, 533, 500, 539, 538, 525, 504, 502, 507, 482, 510, 497, 534, 509, 521, 485, 505, 550, 504, 516, 530, 472, 540, 566, 498, 503, 493, 492, 524, 529, 519, 497, 517, 499, 489, 508, 537, 473, 533, 516, 539, 528, 525, 497, 521, 562, 549, 524, 497, 530, 494, 510, 508, 522, 510, 539, 493, 534, 452, 484, 539, 549, 520, 506, 509, 497, 496, 504, 521, 534, 514, 506, 485, 540, 520, 479, 509, 497, 537, 528, 491, 475, 505, 478, 519, 504, 485, 479, 487, 487, 496, 503, 503, 510, 515, 461, 504, 570, 492, 494, 537, 487, 541, 502, 533, 527, 534, 504, 491, 513, 450, 513, 508, 519, 480, 538, 495, 513] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[543, 553, 512, 483, 494, 537, 511, 522, 518, 502, 530, 496, 464, 492, 492, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 551, 472, 507, 561, 539, 488, 493, 508, 500, 529, 513, 518, 508, 529, 517, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 543, 485, 524, 482, 496, 500, 506, 530, 483, 531, 478, 518, 518, 501, 517, 4, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 513, 491, 487, 540, 509, 506, 491, 518, 517, 514, 493, 555, 550, 482, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 535, 486, 505, 503, 529, 528, 521, 585, 521, 514, 473, 509, 494, 498, 473, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 524, 494, 485, 522, 489, 550, 504, 505, 513, 535, 476, 501, 519, 533, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 528, 511, 534, 493, 512, 541, 538, 572, 528, 495, 518, 504, 531, 507, 499, 11, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 488, 505, 526, 486, 502, 465, 509, 483, 522, 511, 488, 534, 501, 554, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8160, 8242, 8146, 8190, 8221, 8176, 8339, 8062] avg=8192.0000 max=8339 max_rank=6 min=8062 min_rank=7 imbalance_ratio=1.017944 max_load_reduction=57197 max_load_reduction_pct=0.872757 moved_to_echo=32798 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[510, 503, 513, 523, 514, 499, 551, 487, 506, 485, 529, 516, 507, 569, 452, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 559, 522, 521, 510, 485, 519, 482, 524, 487, 490, 478, 495, 545, 504, 481, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 529, 537, 468, 519, 564, 505, 485, 528, 524, 493, 507, 532, 523, 524, 511, 4, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 485, 532, 493, 519, 474, 522, 497, 526, 513, 520, 472, 499, 550, 504, 572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 563, 475, 518, 490, 511, 535, 504, 511, 474, 527, 496, 507, 483, 494, 523, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 486, 569, 509, 477, 535, 518, 483, 565, 528, 476, 498, 527, 559, 507, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 501, 501, 523, 561, 483, 499, 488, 500, 515, 496, 487, 521, 466, 506, 534, 12, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 499, 495, 522, 501, 528, 533, 511, 530, 512, 509, 515, 565, 515, 538, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8127, 8245, 8208, 8120, 8280, 8065, 8270] avg=8192.0000 max=8280 max_rank=5 min=8065 min_rank=6 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32735 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[510, 503, 513, 523, 514, 499, 551, 487, 506, 485, 529, 516, 507, 569, 509, 557, 505, 559, 522, 521, 510, 485, 519, 482, 524, 487, 490, 478, 495, 545, 504, 481, 487, 529, 537, 468, 519, 564, 505, 485, 528, 524, 493, 507, 532, 523, 524, 511, 533, 485, 532, 493, 519, 474, 522, 497, 526, 513, 520, 472, 499, 550, 504, 572, 474, 563, 475, 518, 490, 511, 535, 504, 511, 474, 527, 496, 507, 483, 494, 523, 518, 486, 569, 509, 477, 535, 518, 483, 565, 528, 476, 498, 527, 559, 507, 539, 470, 501, 501, 523, 561, 483, 499, 488, 500, 515, 496, 487, 521, 466, 506, 534, 482, 499, 495, 522, 501, 528, 533, 511, 530, 516, 509, 515, 565, 515, 538, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[543, 553, 512, 483, 494, 537, 511, 522, 518, 502, 530, 496, 464, 492, 553, 511, 488, 551, 472, 507, 561, 539, 488, 493, 508, 500, 529, 513, 518, 508, 529, 517, 525, 543, 485, 524, 482, 496, 500, 506, 530, 483, 531, 478, 518, 518, 501, 517, 493, 513, 491, 487, 540, 509, 506, 491, 518, 517, 514, 493, 555, 550, 482, 534, 509, 535, 486, 505, 503, 529, 528, 521, 585, 521, 514, 473, 509, 494, 498, 473, 523, 524, 494, 485, 522, 489, 550, 504, 505, 513, 535, 476, 501, 519, 533, 516, 515, 528, 511, 534, 493, 512, 541, 538, 572, 528, 495, 518, 504, 531, 507, 499, 501, 488, 505, 526, 486, 502, 465, 509, 483, 526, 511, 488, 534, 501, 554, 487] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[452, 530, 532, 495, 497, 521, 531, 575, 537, 470, 512, 525, 539, 494, 503, 516, 467, 523, 516, 506, 476, 468, 504, 573, 500, 519, 514, 506, 529, 498, 532, 506, 519, 518, 512, 508, 507, 516, 503, 531, 486, 536, 542, 497, 486, 546, 516, 537, 519, 521, 522, 531, 503, 509, 523, 511, 504, 522, 498, 513, 528, 474, 507, 469, 512, 504, 473, 521, 484, 516, 545, 506, 477, 518, 524, 522, 503, 489, 485, 544, 476, 502, 502, 519, 495, 521, 527, 524, 566, 524, 525, 512, 525, 512, 529, 528, 513, 485, 494, 511, 520, 510, 528, 526, 508, 492, 524, 540, 503, 518, 510, 473, 484, 529, 502, 498, 512, 505, 499, 513, 466, 547, 523, 509, 521, 535, 519, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[531, 544, 519, 502, 520, 528, 487, 504, 547, 497, 467, 562, 505, 502, 471, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 502, 504, 535, 482, 480, 522, 495, 498, 559, 505, 459, 497, 558, 521, 534, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 517, 517, 516, 536, 530, 517, 509, 499, 504, 483, 507, 501, 507, 525, 486, 5, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 549, 498, 524, 479, 499, 486, 529, 498, 491, 517, 564, 496, 491, 519, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 513, 484, 482, 550, 538, 514, 507, 526, 501, 482, 489, 557, 512, 493, 483, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 522, 501, 486, 506, 520, 476, 491, 510, 564, 534, 494, 508, 524, 536, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 505, 515, 538, 465, 531, 520, 505, 522, 514, 491, 491, 546, 531, 542, 486, 11, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 522, 551, 467, 496, 510, 496, 530, 528, 561, 527, 564, 523, 503, 526, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8172, 8194, 8155, 8158, 8150, 8227, 8295] avg=8192.0000 max=8295 max_rank=7 min=8150 min_rank=5 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32830 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[452, 530, 532, 495, 497, 521, 531, 575, 537, 470, 512, 525, 539, 494, 448, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 523, 516, 506, 476, 468, 504, 573, 500, 519, 514, 506, 529, 498, 532, 506, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 518, 512, 508, 507, 516, 503, 531, 486, 536, 542, 497, 486, 546, 516, 537, 5, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 521, 522, 531, 503, 509, 523, 511, 504, 522, 498, 513, 528, 474, 507, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 504, 473, 521, 484, 516, 545, 506, 477, 518, 524, 522, 503, 489, 485, 544, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 502, 502, 519, 495, 521, 527, 524, 566, 524, 525, 512, 525, 512, 529, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 485, 494, 511, 520, 510, 528, 526, 508, 492, 524, 540, 503, 518, 510, 473, 12, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 529, 502, 498, 512, 505, 499, 513, 466, 542, 523, 509, 521, 535, 519, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 8156, 8270, 8151, 8157, 8273, 8169, 8186] avg=8192.0000 max=8273 max_rank=5 min=8151 min_rank=3 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32785 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[511, 540, 532, 515, 475, 495, 525, 541, 496, 486, 490, 511, 517, 501, 468, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 507, 539, 466, 498, 500, 505, 533, 535, 545, 530, 551, 481, 531, 478, 558, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 479, 495, 542, 499, 499, 501, 554, 502, 499, 494, 492, 509, 494, 509, 514, 9, 6, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 483, 517, 523, 528, 555, 524, 498, 488, 474, 505, 510, 499, 518, 490, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 449, 524, 516, 522, 548, 549, 519, 518, 530, 478, 479, 530, 514, 514, 526, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 515, 489, 518, 506, 477, 498, 497, 504, 501, 505, 501, 520, 512, 545, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 506, 497, 529, 534, 506, 538, 496, 514, 524, 490, 533, 505, 554, 487, 541, 17, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 506, 499, 483, 478, 527, 516, 525, 504, 513, 525, 560, 458, 516, 487, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8120, 8318, 8114, 8150, 8282, 8136, 8296, 8120] avg=8192.0000 max=8318 max_rank=1 min=8114 min_rank=2 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32834 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[527, 502, 494, 536, 529, 498, 524, 560, 488, 502, 532, 539, 517, 498, 470, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 456, 494, 511, 500, 512, 548, 516, 474, 509, 497, 490, 513, 531, 530, 540, 552, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 510, 572, 513, 545, 488, 501, 540, 508, 509, 488, 509, 458, 499, 508, 525, 4, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 537, 474, 518, 509, 534, 526, 536, 506, 495, 518, 499, 475, 518, 514, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 515, 530, 498, 496, 468, 514, 488, 510, 512, 514, 508, 505, 536, 499, 501, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 509, 542, 533, 488, 500, 538, 524, 524, 522, 502, 500, 521, 489, 505, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 528, 486, 505, 506, 518, 490, 521, 562, 524, 500, 488, 523, 504, 523, 527, 12, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 499, 496, 498, 539, 476, 526, 520, 530, 522, 542, 493, 495, 521, 546, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8229, 8193, 8144, 8183, 8165, 8220, 8204, 8198] avg=8192.0000 max=8229 max_rank=0 min=8144 min_rank=2 imbalance_ratio=1.004517 max_load_reduction=57307 max_load_reduction_pct=0.874435 moved_to_echo=32787 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[495, 490, 528, 486, 498, 498, 505, 463, 498, 502, 552, 500, 518, 511, 470, 569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 523, 498, 537, 551, 495, 530, 474, 528, 477, 518, 507, 473, 519, 471, 535, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 521, 550, 507, 519, 488, 544, 529, 509, 520, 529, 524, 529, 487, 499, 485, 4, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 499, 521, 464, 507, 544, 535, 479, 481, 517, 508, 513, 542, 531, 501, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 496, 488, 497, 487, 487, 536, 513, 510, 540, 531, 523, 502, 556, 523, 488, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 533, 496, 508, 517, 538, 476, 523, 499, 492, 529, 501, 500, 531, 496, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 512, 540, 535, 519, 491, 485, 475, 510, 510, 527, 494, 541, 516, 531, 508, 12, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 520, 529, 511, 492, 566, 482, 526, 541, 522, 482, 527, 481, 501, 544, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8083, 8179, 8230, 8147, 8209, 8227, 8213, 8248] avg=8192.0000 max=8248 max_rank=7 min=8083 min_rank=0 imbalance_ratio=1.006836 max_load_reduction=57288 max_load_reduction_pct=0.874146 moved_to_echo=32897 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[511, 540, 532, 515, 475, 495, 525, 541, 496, 486, 490, 511, 517, 501, 468, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 507, 539, 466, 498, 500, 505, 533, 535, 545, 530, 551, 481, 531, 478, 558, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 479, 495, 542, 499, 499, 501, 554, 502, 499, 494, 492, 509, 494, 509, 514, 9, 6, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 483, 517, 523, 528, 555, 524, 498, 488, 474, 505, 510, 499, 518, 490, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 449, 524, 516, 522, 548, 549, 519, 518, 530, 478, 479, 530, 514, 514, 526, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 515, 489, 518, 506, 477, 498, 497, 504, 501, 505, 501, 520, 512, 545, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 506, 497, 529, 534, 506, 538, 496, 514, 524, 490, 533, 505, 554, 487, 541, 17, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 506, 499, 483, 478, 527, 516, 525, 504, 513, 525, 560, 458, 516, 487, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8120, 8318, 8114, 8150, 8282, 8136, 8296, 8120] avg=8192.0000 max=8318 max_rank=1 min=8114 min_rank=2 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32834 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[499, 547, 522, 523, 534, 533, 569, 538, 530, 486, 508, 533, 500, 539, 478, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 502, 507, 482, 510, 497, 534, 509, 521, 485, 505, 550, 504, 516, 530, 472, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 566, 498, 503, 493, 492, 524, 529, 519, 497, 517, 499, 489, 508, 537, 473, 4, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 516, 539, 528, 525, 497, 521, 562, 549, 524, 497, 530, 494, 510, 508, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 539, 493, 534, 452, 484, 539, 549, 520, 506, 509, 497, 496, 504, 521, 534, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 506, 485, 540, 520, 479, 509, 497, 537, 528, 491, 475, 505, 478, 519, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 479, 487, 487, 496, 503, 503, 510, 515, 461, 504, 570, 492, 494, 537, 487, 11, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 502, 533, 527, 534, 504, 491, 513, 450, 509, 508, 519, 480, 538, 495, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8364, 8149, 8193, 8352, 8224, 8074, 8023, 8157] avg=8192.0000 max=8364 max_rank=0 min=8023 min_rank=6 imbalance_ratio=1.020996 max_load_reduction=57172 max_load_reduction_pct=0.872375 moved_to_echo=32478 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[543, 553, 512, 483, 494, 537, 511, 522, 518, 502, 530, 496, 464, 492, 492, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 551, 472, 507, 561, 539, 488, 493, 508, 500, 529, 513, 518, 508, 529, 517, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 543, 485, 524, 482, 496, 500, 506, 530, 483, 531, 478, 518, 518, 501, 517, 4, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 513, 491, 487, 540, 509, 506, 491, 518, 517, 514, 493, 555, 550, 482, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 535, 486, 505, 503, 529, 528, 521, 585, 521, 514, 473, 509, 494, 498, 473, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 524, 494, 485, 522, 489, 550, 504, 505, 513, 535, 476, 501, 519, 533, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 528, 511, 534, 493, 512, 541, 538, 572, 528, 495, 518, 504, 531, 507, 499, 11, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 488, 505, 526, 486, 502, 465, 509, 483, 522, 511, 488, 534, 501, 554, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8160, 8242, 8146, 8190, 8221, 8176, 8339, 8062] avg=8192.0000 max=8339 max_rank=6 min=8062 min_rank=7 imbalance_ratio=1.017944 max_load_reduction=57197 max_load_reduction_pct=0.872757 moved_to_echo=32798 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[510, 503, 513, 523, 514, 499, 551, 487, 506, 485, 529, 516, 507, 569, 452, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 559, 522, 521, 510, 485, 519, 482, 524, 487, 490, 478, 495, 545, 504, 481, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 529, 537, 468, 519, 564, 505, 485, 528, 524, 493, 507, 532, 523, 524, 511, 4, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 485, 532, 493, 519, 474, 522, 497, 526, 513, 520, 472, 499, 550, 504, 572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 563, 475, 518, 490, 511, 535, 504, 511, 474, 527, 496, 507, 483, 494, 523, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 486, 569, 509, 477, 535, 518, 483, 565, 528, 476, 498, 527, 559, 507, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 501, 501, 523, 561, 483, 499, 488, 500, 515, 496, 487, 521, 466, 506, 534, 12, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 499, 495, 522, 501, 528, 533, 511, 530, 512, 509, 515, 565, 515, 538, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8127, 8245, 8208, 8120, 8280, 8065, 8270] avg=8192.0000 max=8280 max_rank=5 min=8065 min_rank=6 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32735 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[452, 530, 532, 495, 497, 521, 531, 575, 537, 470, 512, 525, 539, 494, 448, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 523, 516, 506, 476, 468, 504, 573, 500, 519, 514, 506, 529, 498, 532, 506, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 518, 512, 508, 507, 516, 503, 531, 486, 536, 542, 497, 486, 546, 516, 537, 5, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 521, 522, 531, 503, 509, 523, 511, 504, 522, 498, 513, 528, 474, 507, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 504, 473, 521, 484, 516, 545, 506, 477, 518, 524, 522, 503, 489, 485, 544, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 502, 502, 519, 495, 521, 527, 524, 566, 524, 525, 512, 525, 512, 529, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 485, 494, 511, 520, 510, 528, 526, 508, 492, 524, 540, 503, 518, 510, 473, 12, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 529, 502, 498, 512, 505, 499, 513, 466, 542, 523, 509, 521, 535, 519, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 8156, 8270, 8151, 8157, 8273, 8169, 8186] avg=8192.0000 max=8273 max_rank=5 min=8151 min_rank=3 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32785 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 3968, 4224, 4224, 4096, 4224, 3840, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 3968, 4096, 4096, 4096, 4352, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 3968, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[535, 467, 499, 512, 490, 481, 507, 514, 514, 540, 484, 520, 511, 542, 507, 499, 545, 499, 507, 493, 554, 523, 527, 522, 512, 495, 494, 554, 553, 488, 544, 533, 503, 524, 506, 500, 519, 485, 480, 481, 532, 530, 501, 490, 512, 490, 507, 514, 537, 540, 489, 514, 549, 522, 533, 471, 485, 482, 506, 500, 526, 552, 522, 500, 506, 477, 477, 523, 528, 504, 516, 521, 517, 528, 515, 532, 513, 521, 495, 518, 470, 472, 510, 507, 554, 546, 492, 529, 502, 553, 455, 543, 505, 516, 574, 489, 525, 493, 490, 514, 519, 508, 502, 498, 525, 540, 522, 544, 483, 530, 530, 505, 570, 542, 554, 495, 504, 510, 483, 502, 510, 505, 493, 492, 493, 509, 490, 481] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[506, 497, 478, 522, 481, 480, 509, 505, 498, 516, 534, 475, 535, 518, 509, 536, 487, 529, 527, 518, 494, 539, 501, 482, 538, 529, 494, 518, 516, 511, 537, 500, 546, 500, 491, 519, 494, 516, 564, 503, 532, 520, 528, 527, 504, 505, 537, 497, 491, 541, 508, 527, 504, 531, 487, 511, 504, 551, 514, 485, 505, 544, 492, 501, 507, 506, 512, 515, 483, 503, 553, 545, 516, 533, 529, 501, 494, 493, 519, 477, 478, 525, 508, 500, 537, 533, 486, 512, 486, 554, 515, 505, 518, 557, 498, 486, 504, 513, 489, 556, 502, 500, 502, 507, 541, 524, 524, 510, 477, 496, 538, 538, 509, 528, 507, 489, 503, 499, 528, 494, 498, 510, 498, 496, 493, 489, 559, 533] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[521, 524, 499, 509, 522, 492, 532, 529, 530, 455, 520, 501, 512, 545, 510, 537, 506, 500, 491, 507, 487, 554, 530, 571, 506, 521, 524, 509, 517, 522, 501, 545, 524, 516, 520, 546, 491, 556, 529, 521, 492, 503, 498, 522, 516, 489, 514, 478, 520, 564, 537, 466, 479, 488, 513, 521, 512, 510, 536, 535, 486, 494, 495, 498, 527, 550, 513, 540, 507, 498, 531, 503, 463, 504, 483, 488, 499, 489, 509, 521, 536, 529, 506, 485, 501, 527, 517, 500, 527, 539, 515, 485, 509, 486, 530, 509, 500, 490, 512, 489, 535, 522, 541, 539, 540, 491, 479, 507, 486, 492, 506, 511, 484, 488, 513, 514, 519, 538, 523, 503, 506, 487, 544, 491, 523, 509, 525, 505] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[537, 555, 531, 461, 555, 509, 518, 502, 532, 495, 511, 466, 485, 505, 489, 524, 522, 480, 533, 507, 485, 465, 492, 474, 498, 542, 505, 504, 475, 496, 547, 534, 508, 518, 497, 503, 518, 494, 551, 570, 491, 502, 486, 507, 572, 511, 554, 528, 500, 510, 524, 507, 516, 524, 517, 550, 531, 543, 526, 479, 500, 505, 489, 497, 518, 474, 525, 520, 523, 521, 508, 501, 539, 507, 525, 538, 523, 519, 503, 479, 503, 497, 523, 494, 515, 481, 514, 513, 476, 481, 467, 524, 501, 504, 509, 534, 524, 521, 527, 534, 516, 522, 545, 516, 501, 487, 508, 541, 519, 534, 502, 540, 506, 523, 491, 531, 500, 530, 542, 538, 487, 509, 488, 523, 517, 507, 487, 499] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[504, 501, 531, 519, 481, 551, 507, 530, 517, 545, 546, 492, 551, 527, 517, 475, 477, 510, 503, 481, 489, 557, 501, 479, 513, 488, 499, 515, 515, 556, 531, 510, 501, 496, 491, 530, 485, 520, 517, 503, 484, 495, 513, 538, 480, 517, 467, 508, 499, 524, 531, 498, 514, 520, 496, 518, 518, 558, 512, 532, 521, 516, 518, 536, 528, 462, 498, 519, 538, 521, 506, 497, 515, 529, 523, 491, 488, 512, 512, 494, 510, 487, 493, 526, 530, 490, 532, 536, 524, 515, 520, 503, 489, 535, 516, 519, 474, 500, 527, 518, 514, 521, 477, 508, 494, 529, 481, 522, 520, 503, 482, 479, 499, 495, 527, 535, 505, 491, 521, 545, 487, 560, 563, 518, 537, 520, 524, 528] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[493, 525, 510, 496, 484, 535, 493, 505, 529, 496, 507, 525, 533, 529, 511, 515, 518, 526, 479, 510, 532, 507, 473, 497, 529, 506, 506, 509, 530, 518, 507, 542, 517, 475, 546, 486, 544, 510, 500, 504, 502, 490, 510, 519, 519, 544, 551, 509, 545, 541, 520, 523, 525, 546, 499, 536, 507, 538, 487, 535, 497, 517, 479, 522, 511, 489, 529, 503, 504, 532, 503, 485, 493, 494, 550, 513, 517, 504, 519, 533, 524, 496, 489, 522, 470, 471, 478, 512, 490, 527, 544, 550, 537, 524, 509, 516, 503, 499, 483, 462, 560, 538, 466, 508, 485, 480, 505, 483, 525, 539, 498, 469, 568, 523, 525, 499, 508, 503, 476, 551, 515, 520, 517, 520, 496, 520, 507, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[506, 497, 478, 522, 481, 480, 509, 505, 498, 516, 534, 475, 535, 518, 509, 536, 487, 529, 527, 518, 494, 539, 501, 482, 538, 529, 494, 518, 516, 511, 537, 500, 546, 500, 491, 519, 494, 516, 564, 503, 532, 520, 528, 527, 504, 505, 537, 497, 491, 541, 508, 527, 504, 531, 487, 511, 504, 551, 514, 485, 505, 544, 492, 501, 507, 506, 512, 515, 483, 503, 553, 545, 516, 533, 529, 501, 494, 493, 519, 477, 478, 525, 508, 500, 537, 533, 486, 512, 486, 554, 515, 505, 518, 557, 498, 486, 504, 513, 489, 556, 502, 500, 502, 507, 541, 524, 524, 510, 477, 496, 538, 538, 509, 528, 507, 489, 503, 499, 528, 494, 498, 510, 498, 496, 493, 489, 559, 533] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[516, 502, 544, 526, 515, 481, 529, 507, 543, 497, 523, 501, 514, 499, 508, 512, 497, 513, 512, 481, 484, 514, 522, 509, 516, 521, 514, 485, 508, 503, 499, 516, 478, 534, 569, 525, 481, 489, 540, 519, 494, 504, 527, 529, 534, 531, 479, 498, 529, 488, 501, 458, 494, 532, 504, 499, 511, 535, 481, 552, 543, 521, 470, 499, 500, 490, 506, 513, 488, 461, 545, 579, 528, 542, 553, 540, 527, 493, 497, 526, 550, 491, 487, 492, 533, 514, 490, 487, 503, 523, 517, 497, 490, 532, 519, 522, 503, 530, 518, 497, 512, 500, 561, 513, 515, 537, 506, 544, 497, 507, 519, 519, 497, 458, 516, 525, 515, 516, 515, 519, 485, 502, 496, 494, 546, 510, 533, 537] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[476, 505, 535, 515, 528, 515, 481, 489, 472, 533, 484, 504, 525, 505, 536, 497, 516, 510, 513, 544, 503, 528, 502, 521, 546, 505, 499, 507, 535, 517, 525, 571, 528, 506, 489, 465, 499, 493, 520, 525, 512, 470, 509, 508, 550, 486, 519, 564, 556, 481, 499, 468, 480, 515, 499, 496, 496, 496, 543, 537, 504, 534, 526, 524, 522, 496, 516, 486, 526, 529, 528, 508, 495, 521, 495, 508, 491, 472, 519, 524, 487, 469, 495, 509, 559, 522, 482, 509, 524, 549, 516, 543, 524, 533, 515, 474, 519, 505, 521, 460, 526, 505, 507, 529, 470, 517, 527, 481, 513, 509, 498, 565, 514, 524, 556, 530, 508, 501, 515, 506, 565, 544, 478, 529, 517, 484, 516, 512] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[516, 502, 544, 526, 515, 481, 529, 507, 543, 497, 523, 501, 514, 499, 508, 512, 497, 513, 512, 481, 484, 514, 522, 509, 516, 521, 514, 485, 508, 503, 499, 516, 478, 534, 569, 525, 481, 489, 540, 519, 494, 504, 527, 529, 534, 531, 479, 498, 529, 488, 501, 458, 494, 532, 504, 499, 511, 535, 481, 552, 543, 521, 470, 499, 500, 490, 506, 513, 488, 461, 545, 579, 528, 542, 553, 540, 527, 493, 497, 526, 550, 491, 487, 492, 533, 514, 490, 487, 503, 523, 517, 497, 490, 532, 519, 522, 503, 530, 518, 497, 512, 500, 561, 513, 515, 537, 506, 544, 497, 507, 519, 519, 497, 458, 516, 525, 515, 516, 515, 519, 485, 502, 496, 494, 546, 510, 533, 537] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[537, 555, 531, 461, 555, 509, 518, 502, 532, 495, 511, 466, 485, 505, 489, 524, 522, 480, 533, 507, 485, 465, 492, 474, 498, 542, 505, 504, 475, 496, 547, 534, 508, 518, 497, 503, 518, 494, 551, 570, 491, 502, 486, 507, 572, 511, 554, 528, 500, 510, 524, 507, 516, 524, 517, 550, 531, 543, 526, 479, 500, 505, 489, 497, 518, 474, 525, 520, 523, 521, 508, 501, 539, 507, 525, 538, 523, 519, 503, 479, 503, 497, 523, 494, 515, 481, 514, 513, 476, 481, 467, 524, 501, 504, 509, 534, 524, 521, 527, 534, 516, 522, 545, 516, 501, 487, 508, 541, 519, 534, 502, 540, 506, 523, 491, 531, 500, 530, 542, 538, 487, 509, 488, 523, 517, 507, 487, 499] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[535, 467, 499, 512, 490, 481, 507, 514, 514, 540, 484, 520, 511, 542, 507, 499, 545, 499, 507, 493, 554, 523, 527, 522, 512, 495, 494, 554, 553, 488, 544, 533, 503, 524, 506, 500, 519, 485, 480, 481, 532, 530, 501, 490, 512, 490, 507, 514, 537, 540, 489, 514, 549, 522, 533, 471, 485, 482, 506, 500, 526, 552, 522, 500, 506, 477, 477, 523, 528, 504, 516, 521, 517, 528, 515, 532, 513, 521, 495, 518, 470, 472, 510, 507, 554, 546, 492, 529, 502, 553, 455, 543, 505, 516, 574, 489, 525, 493, 490, 514, 519, 508, 502, 498, 525, 540, 522, 544, 483, 530, 530, 505, 570, 542, 554, 495, 504, 510, 483, 502, 510, 505, 493, 492, 493, 509, 490, 481] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[521, 524, 499, 509, 522, 492, 532, 529, 530, 455, 520, 501, 512, 545, 510, 537, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 500, 491, 507, 487, 554, 530, 571, 506, 521, 524, 509, 517, 522, 501, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 516, 520, 546, 491, 556, 529, 521, 492, 503, 498, 522, 516, 489, 514, 478, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 564, 537, 466, 479, 488, 513, 521, 512, 492, 536, 535, 486, 494, 495, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 550, 513, 540, 507, 498, 531, 503, 463, 504, 483, 488, 499, 489, 509, 521, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 529, 506, 485, 501, 527, 517, 500, 527, 539, 515, 485, 509, 486, 530, 509, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 490, 512, 489, 535, 522, 541, 539, 540, 491, 479, 507, 486, 492, 506, 511, 11, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 488, 493, 514, 519, 538, 523, 503, 506, 487, 544, 491, 523, 509, 525, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8250, 8276, 8216, 8136, 8134, 8218, 8154, 8152] avg=8192.0000 max=8276 max_rank=1 min=8134 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32658 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[537, 555, 531, 461, 555, 509, 518, 502, 532, 495, 511, 466, 485, 505, 489, 524, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 480, 533, 507, 485, 465, 492, 474, 498, 542, 505, 504, 475, 496, 547, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 518, 497, 503, 518, 494, 551, 570, 491, 502, 486, 507, 572, 511, 554, 528, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 510, 524, 507, 516, 524, 517, 550, 531, 523, 526, 479, 500, 505, 489, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 474, 525, 520, 523, 521, 508, 501, 539, 507, 525, 538, 523, 519, 503, 479, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 497, 523, 494, 515, 481, 514, 513, 476, 481, 467, 524, 501, 504, 509, 534, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 521, 527, 534, 516, 522, 545, 516, 501, 487, 508, 541, 519, 534, 502, 540, 12, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 523, 472, 531, 500, 530, 542, 538, 487, 509, 488, 523, 517, 507, 487, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8044, 8311, 8198, 8232, 8052, 8352, 8159] avg=8192.0000 max=8352 max_rank=6 min=8044 min_rank=1 imbalance_ratio=1.019531 max_load_reduction=57184 max_load_reduction_pct=0.872559 moved_to_echo=32795 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[535, 467, 499, 512, 490, 481, 507, 514, 514, 540, 484, 520, 511, 542, 507, 499, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 499, 507, 493, 554, 523, 527, 522, 512, 495, 494, 554, 553, 488, 544, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 524, 506, 500, 519, 485, 480, 481, 532, 530, 501, 490, 512, 490, 507, 514, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 540, 489, 514, 549, 522, 533, 471, 485, 465, 506, 500, 526, 552, 522, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 477, 477, 523, 528, 504, 516, 521, 517, 528, 515, 532, 513, 521, 495, 518, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 472, 510, 507, 554, 546, 492, 529, 502, 553, 455, 543, 505, 516, 574, 489, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 493, 490, 514, 519, 508, 502, 498, 525, 540, 522, 544, 483, 530, 530, 505, 10, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 570, 542, 532, 495, 504, 510, 483, 502, 510, 505, 493, 492, 493, 509, 490, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8134, 8328, 8075, 8211, 8200, 8235, 8242, 8111] avg=8192.0000 max=8328 max_rank=1 min=8075 min_rank=2 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=32788 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[504, 501, 531, 519, 481, 551, 507, 530, 517, 545, 546, 492, 551, 527, 517, 475, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 510, 503, 481, 489, 557, 501, 479, 513, 488, 499, 515, 515, 556, 531, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 496, 491, 530, 485, 520, 517, 503, 484, 495, 513, 538, 480, 517, 467, 508, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 524, 531, 498, 514, 520, 496, 518, 518, 538, 512, 532, 521, 516, 518, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 462, 498, 519, 538, 521, 506, 497, 515, 529, 523, 491, 488, 512, 512, 494, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 487, 493, 526, 530, 490, 532, 536, 524, 515, 520, 503, 489, 535, 516, 519, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 500, 527, 518, 514, 521, 477, 508, 494, 529, 481, 522, 520, 503, 482, 479, 12, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 495, 506, 535, 505, 491, 521, 545, 487, 560, 563, 518, 537, 520, 524, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8307, 8110, 8046, 8291, 8141, 8242, 8065, 8334] avg=8192.0000 max=8334 max_rank=7 min=8046 min_rank=2 imbalance_ratio=1.017334 max_load_reduction=57202 max_load_reduction_pct=0.872833 moved_to_echo=32782 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[506, 497, 478, 522, 481, 480, 509, 505, 498, 516, 534, 475, 535, 518, 509, 536, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 529, 527, 518, 494, 539, 501, 482, 538, 529, 494, 518, 516, 511, 537, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 500, 491, 519, 494, 516, 564, 503, 532, 520, 528, 527, 504, 505, 537, 497, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 541, 508, 527, 504, 531, 487, 511, 504, 531, 514, 485, 505, 544, 492, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 506, 512, 515, 483, 503, 553, 545, 516, 533, 529, 501, 494, 493, 519, 477, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 525, 508, 500, 537, 533, 486, 512, 486, 554, 515, 505, 518, 557, 498, 486, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 513, 489, 556, 502, 500, 502, 507, 541, 524, 524, 510, 477, 496, 538, 538, 12, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 528, 487, 489, 503, 499, 528, 494, 498, 510, 498, 496, 493, 489, 559, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8111, 8207, 8284, 8176, 8194, 8215, 8236, 8113] avg=8192.0000 max=8284 max_rank=2 min=8111 min_rank=0 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32758 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[493, 525, 510, 496, 484, 535, 493, 505, 529, 496, 507, 525, 533, 529, 511, 515, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 526, 479, 510, 532, 507, 473, 497, 529, 506, 506, 509, 530, 518, 507, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 475, 546, 486, 544, 510, 500, 504, 502, 490, 510, 519, 519, 544, 551, 509, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 541, 520, 523, 525, 546, 499, 536, 507, 518, 487, 535, 497, 517, 479, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 489, 529, 503, 504, 532, 503, 485, 493, 494, 550, 513, 517, 504, 519, 533, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 496, 489, 522, 470, 471, 478, 512, 490, 527, 544, 550, 537, 524, 509, 516, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 499, 483, 462, 560, 538, 466, 508, 485, 480, 505, 483, 525, 539, 498, 469, 12, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 568, 523, 504, 499, 508, 503, 476, 551, 515, 520, 517, 520, 496, 520, 507, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8174, 8227, 8297, 8188, 8176, 8019, 8256] avg=8192.0000 max=8297 max_rank=3 min=8019 min_rank=6 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32639 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[535, 467, 499, 512, 490, 481, 507, 514, 514, 540, 484, 520, 511, 542, 507, 499, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 499, 507, 493, 554, 523, 527, 522, 512, 495, 494, 554, 553, 488, 544, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 524, 506, 500, 519, 485, 480, 481, 532, 530, 501, 490, 512, 490, 507, 514, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 540, 489, 514, 549, 522, 533, 471, 485, 465, 506, 500, 526, 552, 522, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 477, 477, 523, 528, 504, 516, 521, 517, 528, 515, 532, 513, 521, 495, 518, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 472, 510, 507, 554, 546, 492, 529, 502, 553, 455, 543, 505, 516, 574, 489, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 493, 490, 514, 519, 508, 502, 498, 525, 540, 522, 544, 483, 530, 530, 505, 10, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 570, 542, 532, 495, 504, 510, 483, 502, 510, 505, 493, 492, 493, 509, 490, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8134, 8328, 8075, 8211, 8200, 8235, 8242, 8111] avg=8192.0000 max=8328 max_rank=1 min=8075 min_rank=2 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=32788 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[516, 502, 544, 526, 515, 481, 529, 507, 543, 497, 523, 501, 514, 499, 508, 512, 10, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 513, 512, 481, 484, 514, 522, 509, 516, 521, 514, 485, 508, 503, 499, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 534, 569, 525, 481, 489, 540, 519, 494, 504, 527, 529, 534, 531, 479, 498, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 488, 501, 458, 494, 532, 504, 499, 511, 509, 481, 552, 543, 521, 470, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 490, 506, 513, 488, 461, 545, 579, 528, 542, 553, 540, 527, 493, 497, 526, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 491, 487, 492, 533, 514, 490, 487, 503, 523, 517, 497, 490, 532, 519, 522, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 530, 518, 497, 512, 500, 561, 513, 515, 537, 506, 544, 497, 507, 519, 519, 16, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 458, 486, 525, 515, 516, 515, 519, 485, 502, 496, 494, 546, 510, 533, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8071, 8233, 8091, 8301, 8170, 8301, 8134] avg=8192.0000 max=8301 max_rank=4 min=8071 min_rank=1 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32906 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[506, 497, 478, 522, 481, 480, 509, 505, 498, 516, 534, 475, 535, 518, 509, 536, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 529, 527, 518, 494, 539, 501, 482, 538, 529, 494, 518, 516, 511, 537, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 500, 491, 519, 494, 516, 564, 503, 532, 520, 528, 527, 504, 505, 537, 497, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 541, 508, 527, 504, 531, 487, 511, 504, 531, 514, 485, 505, 544, 492, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 506, 512, 515, 483, 503, 553, 545, 516, 533, 529, 501, 494, 493, 519, 477, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 525, 508, 500, 537, 533, 486, 512, 486, 554, 515, 505, 518, 557, 498, 486, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 513, 489, 556, 502, 500, 502, 507, 541, 524, 524, 510, 477, 496, 538, 538, 12, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 528, 487, 489, 503, 499, 528, 494, 498, 510, 498, 496, 493, 489, 559, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8111, 8207, 8284, 8176, 8194, 8215, 8236, 8113] avg=8192.0000 max=8284 max_rank=2 min=8111 min_rank=0 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32758 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[537, 555, 531, 461, 555, 509, 518, 502, 532, 495, 511, 466, 485, 505, 489, 524, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 480, 533, 507, 485, 465, 492, 474, 498, 542, 505, 504, 475, 496, 547, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 518, 497, 503, 518, 494, 551, 570, 491, 502, 486, 507, 572, 511, 554, 528, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 510, 524, 507, 516, 524, 517, 550, 531, 523, 526, 479, 500, 505, 489, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 474, 525, 520, 523, 521, 508, 501, 539, 507, 525, 538, 523, 519, 503, 479, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 497, 523, 494, 515, 481, 514, 513, 476, 481, 467, 524, 501, 504, 509, 534, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 521, 527, 534, 516, 522, 545, 516, 501, 487, 508, 541, 519, 534, 502, 540, 12, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 523, 472, 531, 500, 530, 542, 538, 487, 509, 488, 523, 517, 507, 487, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8044, 8311, 8198, 8232, 8052, 8352, 8159] avg=8192.0000 max=8352 max_rank=6 min=8044 min_rank=1 imbalance_ratio=1.019531 max_load_reduction=57184 max_load_reduction_pct=0.872559 moved_to_echo=32795 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[476, 505, 535, 515, 528, 515, 481, 489, 472, 533, 484, 504, 525, 505, 536, 497, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 510, 513, 544, 503, 528, 502, 521, 546, 505, 499, 507, 535, 517, 525, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 506, 489, 465, 499, 493, 520, 525, 512, 470, 509, 508, 550, 486, 519, 564, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 481, 499, 468, 480, 515, 499, 496, 496, 478, 543, 537, 504, 534, 526, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 496, 516, 486, 526, 529, 528, 508, 495, 521, 495, 508, 491, 472, 519, 524, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 469, 495, 509, 559, 522, 482, 509, 524, 549, 516, 543, 524, 533, 515, 474, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 505, 521, 460, 526, 505, 507, 529, 470, 517, 527, 481, 513, 509, 498, 565, 11, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 524, 534, 530, 508, 501, 515, 506, 565, 544, 478, 529, 517, 484, 516, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8112, 8326, 8144, 8136, 8146, 8228, 8167, 8277] avg=8192.0000 max=8326 max_rank=1 min=8112 min_rank=0 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32818 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 3968, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4352, 4096, 4224, 4096, 4224, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 3968, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 3968, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4352, 4096, 4224, 4096, 4224, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 3968, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[492, 510, 499, 502, 514, 512, 509, 482, 515, 520, 491, 529, 490, 512, 502, 524, 499, 493, 479, 543, 581, 493, 500, 518, 525, 555, 521, 489, 492, 455, 527, 525, 487, 498, 519, 536, 506, 515, 544, 499, 502, 527, 542, 504, 534, 544, 516, 550, 517, 511, 469, 563, 499, 532, 519, 520, 501, 528, 463, 485, 506, 478, 486, 474, 522, 473, 513, 522, 516, 527, 549, 481, 484, 535, 531, 494, 523, 501, 509, 512, 536, 522, 536, 471, 495, 533, 471, 526, 508, 501, 526, 511, 499, 531, 535, 533, 516, 518, 513, 524, 526, 504, 491, 534, 512, 519, 508, 550, 499, 496, 518, 482, 535, 502, 517, 539, 488, 554, 479, 503, 504, 527, 526, 509, 471, 511, 520, 543] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[502, 523, 510, 512, 518, 504, 504, 546, 483, 525, 529, 526, 480, 485, 517, 482, 544, 481, 523, 531, 504, 504, 548, 497, 484, 523, 527, 552, 513, 465, 535, 505, 522, 492, 516, 508, 511, 498, 518, 506, 527, 516, 518, 516, 549, 536, 482, 493, 546, 505, 516, 498, 536, 566, 496, 489, 493, 522, 526, 505, 509, 522, 492, 505, 516, 510, 506, 529, 519, 510, 493, 505, 531, 518, 522, 496, 525, 499, 493, 533, 530, 511, 553, 545, 460, 512, 536, 483, 506, 511, 521, 525, 466, 512, 490, 504, 467, 443, 552, 475, 506, 504, 483, 549, 520, 557, 550, 496, 530, 498, 574, 507, 502, 519, 481, 481, 502, 512, 489, 513, 565, 504, 491, 522, 468, 494, 585, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[491, 503, 547, 509, 519, 501, 473, 515, 532, 488, 511, 497, 538, 504, 516, 505, 512, 492, 484, 482, 486, 484, 483, 509, 542, 510, 504, 519, 544, 534, 523, 480, 511, 496, 504, 526, 452, 530, 506, 528, 512, 531, 491, 504, 526, 527, 513, 513, 510, 519, 514, 494, 473, 497, 533, 512, 490, 517, 485, 542, 524, 546, 504, 472, 555, 511, 556, 483, 511, 504, 526, 510, 501, 503, 460, 496, 506, 522, 501, 511, 529, 514, 534, 522, 478, 541, 542, 501, 530, 498, 524, 524, 462, 505, 560, 526, 520, 488, 507, 519, 481, 506, 543, 516, 500, 538, 558, 491, 536, 492, 521, 515, 555, 559, 504, 515, 497, 547, 528, 547, 500, 504, 509, 514, 509, 501, 513, 518] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[499, 532, 516, 544, 528, 498, 504, 517, 504, 543, 495, 491, 547, 511, 542, 471, 512, 463, 482, 502, 542, 536, 529, 490, 474, 508, 512, 494, 494, 544, 535, 497, 531, 524, 519, 466, 548, 512, 521, 491, 481, 512, 517, 535, 513, 496, 519, 563, 502, 510, 538, 544, 508, 500, 507, 498, 503, 492, 509, 512, 488, 480, 485, 519, 497, 529, 506, 538, 467, 467, 458, 480, 487, 552, 538, 503, 515, 503, 523, 459, 512, 495, 525, 529, 529, 546, 526, 541, 541, 513, 510, 502, 519, 540, 501, 516, 517, 495, 503, 477, 517, 529, 503, 529, 537, 514, 547, 497, 506, 500, 539, 515, 510, 527, 544, 507, 519, 545, 501, 531, 493, 515, 534, 520, 501, 483, 536, 479] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[515, 474, 524, 551, 511, 503, 495, 516, 532, 538, 533, 508, 533, 546, 520, 535, 523, 503, 535, 521, 478, 498, 467, 479, 479, 511, 512, 476, 477, 530, 480, 510, 524, 527, 483, 545, 524, 503, 496, 510, 504, 517, 496, 514, 519, 550, 539, 485, 526, 526, 509, 526, 503, 497, 534, 515, 515, 536, 529, 507, 476, 511, 512, 495, 502, 473, 485, 542, 503, 484, 504, 504, 520, 507, 515, 485, 503, 525, 525, 497, 552, 522, 546, 547, 516, 546, 444, 494, 511, 518, 482, 518, 544, 519, 558, 460, 481, 505, 535, 509, 525, 488, 524, 519, 478, 500, 535, 520, 505, 561, 485, 509, 511, 521, 529, 484, 516, 497, 540, 492, 534, 533, 502, 520, 519, 546, 507, 489] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[562, 520, 506, 526, 490, 488, 530, 508, 518, 483, 547, 559, 514, 526, 559, 517, 523, 507, 520, 500, 494, 452, 515, 507, 521, 501, 532, 517, 528, 545, 531, 538, 568, 495, 485, 535, 505, 500, 510, 498, 510, 515, 508, 509, 520, 523, 549, 512, 508, 497, 494, 501, 486, 518, 493, 500, 561, 502, 494, 556, 528, 515, 511, 509, 532, 514, 544, 468, 498, 529, 493, 523, 481, 501, 495, 467, 521, 526, 511, 512, 497, 503, 454, 475, 511, 510, 524, 482, 536, 534, 533, 536, 507, 548, 469, 487, 534, 545, 502, 488, 552, 510, 508, 502, 488, 511, 451, 537, 505, 503, 479, 536, 486, 530, 512, 516, 489, 522, 490, 500, 477, 539, 541, 525, 490, 498, 516, 534] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[537, 513, 521, 519, 536, 524, 494, 500, 525, 520, 515, 527, 518, 470, 512, 515, 533, 469, 534, 496, 515, 490, 542, 497, 525, 506, 510, 526, 513, 549, 483, 501, 497, 567, 503, 510, 505, 505, 494, 523, 535, 537, 501, 503, 531, 477, 472, 500, 489, 517, 488, 505, 517, 476, 490, 542, 499, 521, 544, 528, 482, 492, 492, 499, 534, 534, 513, 511, 515, 501, 526, 514, 504, 514, 500, 471, 511, 551, 512, 533, 535, 526, 477, 517, 515, 521, 569, 524, 492, 555, 507, 527, 506, 482, 563, 503, 475, 492, 500, 530, 524, 518, 508, 496, 512, 480, 501, 498, 536, 473, 513, 523, 501, 533, 524, 506, 492, 483, 525, 533, 529, 500, 516, 509, 523, 514, 535, 495] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[499, 532, 516, 544, 528, 498, 504, 517, 504, 543, 495, 491, 547, 511, 542, 471, 512, 463, 482, 502, 542, 536, 529, 490, 474, 508, 512, 494, 494, 544, 535, 497, 531, 524, 519, 466, 548, 512, 521, 491, 481, 512, 517, 535, 513, 496, 519, 563, 502, 510, 538, 544, 508, 500, 507, 498, 503, 492, 509, 512, 488, 480, 485, 519, 497, 529, 506, 538, 467, 467, 458, 480, 487, 552, 538, 503, 515, 503, 523, 459, 512, 495, 525, 529, 529, 546, 526, 541, 541, 513, 510, 502, 519, 540, 501, 516, 517, 495, 503, 477, 517, 529, 503, 529, 537, 514, 547, 497, 506, 500, 539, 515, 510, 527, 544, 507, 519, 545, 501, 531, 493, 515, 534, 520, 501, 483, 536, 479] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[491, 520, 498, 490, 513, 521, 497, 467, 490, 500, 493, 500, 487, 537, 528, 546, 486, 531, 487, 523, 485, 547, 520, 520, 509, 529, 503, 489, 493, 533, 523, 508, 516, 547, 522, 482, 505, 503, 532, 483, 515, 504, 558, 504, 534, 566, 513, 526, 475, 534, 508, 462, 521, 516, 494, 528, 509, 496, 498, 520, 538, 510, 536, 501, 552, 535, 534, 550, 556, 550, 513, 525, 493, 555, 500, 513, 550, 509, 479, 499, 480, 517, 505, 504, 517, 438, 531, 507, 530, 518, 516, 546, 497, 510, 512, 481, 522, 510, 481, 561, 464, 480, 481, 554, 491, 499, 517, 480, 494, 495, 512, 460, 543, 525, 533, 543, 516, 500, 526, 492, 523, 532, 495, 529, 509, 517, 535, 475] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[502, 523, 510, 512, 518, 504, 504, 546, 483, 525, 529, 526, 480, 485, 503, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 481, 523, 531, 504, 504, 548, 497, 484, 523, 527, 552, 513, 465, 535, 505, 35, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 492, 516, 508, 511, 498, 518, 506, 527, 516, 518, 516, 503, 536, 482, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 505, 516, 498, 536, 566, 496, 489, 493, 522, 526, 505, 509, 522, 492, 505, 46, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 510, 506, 529, 519, 510, 493, 505, 531, 518, 522, 496, 525, 499, 493, 533, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 511, 553, 545, 460, 512, 536, 483, 506, 511, 521, 487, 466, 512, 490, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 443, 552, 475, 506, 504, 483, 549, 520, 557, 550, 496, 530, 498, 574, 507, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 519, 481, 481, 502, 512, 489, 513, 565, 504, 491, 522, 468, 494, 543, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8132, 8278, 8162, 8279, 8219, 8127, 8242, 8097] avg=8192.0000 max=8279 max_rank=3 min=8097 min_rank=7 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32685 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[492, 510, 499, 502, 514, 512, 509, 482, 515, 520, 491, 529, 490, 512, 489, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 493, 479, 543, 581, 493, 500, 518, 525, 555, 521, 489, 492, 455, 527, 525, 31, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 498, 519, 536, 506, 515, 544, 499, 502, 527, 542, 504, 489, 544, 516, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 511, 469, 563, 499, 532, 519, 520, 501, 528, 463, 485, 506, 478, 486, 474, 45, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 473, 513, 522, 516, 527, 549, 481, 484, 535, 531, 494, 523, 501, 509, 512, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 522, 536, 471, 495, 533, 471, 526, 508, 501, 526, 474, 499, 531, 535, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 518, 513, 524, 526, 504, 491, 534, 512, 519, 508, 550, 499, 496, 518, 482, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 502, 517, 539, 488, 554, 479, 503, 504, 527, 526, 509, 471, 511, 483, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8090, 8233, 8278, 8102, 8205, 8197, 8240, 8191] avg=8192.0000 max=8278 max_rank=2 min=8090 min_rank=0 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32833 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[537, 513, 521, 519, 536, 524, 494, 500, 525, 520, 515, 527, 518, 470, 498, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 469, 534, 496, 515, 490, 542, 497, 525, 506, 510, 526, 513, 549, 483, 501, 32, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 567, 503, 510, 505, 505, 494, 523, 535, 537, 501, 503, 486, 477, 472, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 517, 488, 505, 517, 476, 490, 542, 499, 521, 544, 528, 482, 492, 492, 499, 45, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 534, 513, 511, 515, 501, 526, 514, 504, 514, 500, 471, 511, 551, 512, 533, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 526, 477, 517, 515, 521, 569, 524, 492, 555, 507, 489, 506, 482, 563, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 492, 500, 530, 524, 518, 508, 496, 512, 480, 501, 498, 536, 473, 513, 523, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 533, 524, 506, 492, 483, 525, 533, 529, 500, 516, 509, 523, 514, 497, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8232, 8228, 8115, 8132, 8258, 8281, 8110, 8180] avg=8192.0000 max=8281 max_rank=5 min=8110 min_rank=6 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32829 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[515, 474, 524, 551, 511, 503, 495, 516, 532, 538, 533, 508, 533, 546, 506, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 503, 535, 521, 478, 498, 467, 479, 479, 511, 512, 476, 477, 530, 480, 510, 30, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 527, 483, 545, 524, 503, 496, 510, 504, 517, 496, 514, 475, 550, 539, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 526, 509, 526, 503, 497, 534, 515, 515, 536, 529, 507, 476, 511, 512, 495, 44, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 473, 485, 542, 503, 484, 504, 504, 520, 507, 515, 485, 503, 525, 525, 497, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 522, 546, 547, 516, 546, 444, 494, 511, 518, 482, 481, 544, 519, 558, 460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 505, 535, 509, 525, 488, 524, 519, 478, 500, 535, 520, 505, 561, 485, 509, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 521, 529, 484, 516, 497, 540, 492, 534, 533, 502, 520, 519, 546, 471, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8320, 8016, 8192, 8267, 8088, 8240, 8209, 8204] avg=8192.0000 max=8320 max_rank=0 min=8016 min_rank=1 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32741 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[492, 510, 499, 502, 514, 512, 509, 482, 515, 520, 491, 529, 490, 512, 502, 524, 499, 493, 479, 543, 581, 493, 500, 518, 525, 555, 521, 489, 492, 455, 527, 525, 487, 498, 519, 536, 506, 515, 544, 499, 502, 527, 542, 504, 534, 544, 516, 550, 517, 511, 469, 563, 499, 532, 519, 520, 501, 528, 463, 485, 506, 478, 486, 474, 522, 473, 513, 522, 516, 527, 549, 481, 484, 535, 531, 494, 523, 501, 509, 512, 536, 522, 536, 471, 495, 533, 471, 526, 508, 501, 526, 511, 499, 531, 535, 533, 516, 518, 513, 524, 526, 504, 491, 534, 512, 519, 508, 550, 499, 496, 518, 482, 535, 502, 517, 539, 488, 554, 479, 503, 504, 527, 526, 509, 471, 511, 520, 543] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[537, 513, 521, 519, 536, 524, 494, 500, 525, 520, 515, 527, 518, 470, 512, 515, 533, 469, 534, 496, 515, 490, 542, 497, 525, 506, 510, 526, 513, 549, 483, 501, 497, 567, 503, 510, 505, 505, 494, 523, 535, 537, 501, 503, 531, 477, 472, 500, 489, 517, 488, 505, 517, 476, 490, 542, 499, 521, 544, 528, 482, 492, 492, 499, 534, 534, 513, 511, 515, 501, 526, 514, 504, 514, 500, 471, 511, 551, 512, 533, 535, 526, 477, 517, 515, 521, 569, 524, 492, 555, 507, 527, 506, 482, 563, 503, 475, 492, 500, 530, 524, 518, 508, 496, 512, 480, 501, 498, 536, 473, 513, 523, 501, 533, 524, 506, 492, 483, 525, 533, 529, 500, 516, 509, 523, 514, 535, 495] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[502, 523, 510, 512, 518, 504, 504, 546, 483, 525, 529, 526, 480, 485, 517, 482, 544, 481, 523, 531, 504, 504, 548, 497, 484, 523, 527, 552, 513, 465, 535, 505, 522, 492, 516, 508, 511, 498, 518, 506, 527, 516, 518, 516, 549, 536, 482, 493, 546, 505, 516, 498, 536, 566, 496, 489, 493, 522, 526, 505, 509, 522, 492, 505, 516, 510, 506, 529, 519, 510, 493, 505, 531, 518, 522, 496, 525, 499, 493, 533, 530, 511, 553, 545, 460, 512, 536, 483, 506, 511, 521, 525, 466, 512, 490, 504, 467, 443, 552, 475, 506, 504, 483, 549, 520, 557, 550, 496, 530, 498, 574, 507, 502, 519, 481, 481, 502, 512, 489, 513, 565, 504, 491, 522, 468, 494, 585, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[491, 503, 547, 509, 519, 501, 473, 515, 532, 488, 511, 497, 538, 504, 499, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 492, 484, 482, 486, 484, 483, 509, 542, 510, 504, 519, 544, 534, 523, 480, 35, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 496, 504, 526, 452, 530, 506, 528, 512, 531, 491, 504, 477, 527, 513, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 519, 514, 494, 473, 497, 533, 512, 490, 517, 485, 542, 524, 546, 504, 472, 49, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 511, 556, 483, 511, 504, 526, 510, 501, 503, 460, 496, 506, 522, 501, 511, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 514, 534, 522, 478, 541, 542, 501, 530, 498, 524, 478, 462, 505, 560, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 488, 507, 519, 481, 506, 543, 516, 500, 538, 558, 491, 536, 492, 521, 515, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 559, 504, 515, 497, 547, 528, 547, 500, 504, 509, 514, 509, 501, 467, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8132, 8134, 8121, 8192, 8173, 8244, 8266, 8274] avg=8192.0000 max=8274 max_rank=7 min=8121 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32957 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[562, 520, 506, 526, 490, 488, 530, 508, 518, 483, 547, 559, 514, 526, 544, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 507, 520, 500, 494, 452, 515, 507, 521, 501, 532, 517, 528, 545, 531, 538, 31, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 568, 495, 485, 535, 505, 500, 510, 498, 510, 515, 508, 509, 476, 523, 549, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 497, 494, 501, 486, 518, 493, 500, 561, 502, 494, 556, 528, 515, 511, 509, 44, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 514, 544, 468, 498, 529, 493, 523, 481, 501, 495, 467, 521, 526, 511, 512, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 503, 454, 475, 511, 510, 524, 482, 536, 534, 533, 498, 507, 548, 469, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 545, 502, 488, 552, 510, 508, 502, 488, 511, 451, 537, 505, 503, 479, 536, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 530, 512, 516, 489, 522, 490, 500, 477, 539, 541, 525, 490, 498, 479, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8338, 8269, 8198, 8223, 8130, 8068, 8182, 8128] avg=8192.0000 max=8338 max_rank=0 min=8068 min_rank=5 imbalance_ratio=1.017822 max_load_reduction=57198 max_load_reduction_pct=0.872772 moved_to_echo=32508 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[537, 513, 521, 519, 536, 524, 494, 500, 525, 520, 515, 527, 518, 470, 498, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 469, 534, 496, 515, 490, 542, 497, 525, 506, 510, 526, 513, 549, 483, 501, 32, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 567, 503, 510, 505, 505, 494, 523, 535, 537, 501, 503, 486, 477, 472, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 517, 488, 505, 517, 476, 490, 542, 499, 521, 544, 528, 482, 492, 492, 499, 45, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 534, 513, 511, 515, 501, 526, 514, 504, 514, 500, 471, 511, 551, 512, 533, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 526, 477, 517, 515, 521, 569, 524, 492, 555, 507, 489, 506, 482, 563, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 492, 500, 530, 524, 518, 508, 496, 512, 480, 501, 498, 536, 473, 513, 523, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 533, 524, 506, 492, 483, 525, 533, 529, 500, 516, 509, 523, 514, 497, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8232, 8228, 8115, 8132, 8258, 8281, 8110, 8180] avg=8192.0000 max=8281 max_rank=5 min=8110 min_rank=6 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32829 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[499, 532, 516, 544, 528, 498, 504, 517, 504, 543, 495, 491, 547, 511, 528, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 463, 482, 502, 542, 536, 529, 490, 474, 508, 512, 494, 494, 544, 535, 497, 32, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 524, 519, 466, 548, 512, 521, 491, 481, 512, 517, 535, 470, 496, 519, 563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 510, 538, 544, 508, 500, 507, 498, 503, 492, 509, 512, 488, 480, 485, 519, 43, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 529, 506, 538, 467, 467, 458, 480, 487, 552, 538, 503, 515, 503, 523, 459, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 495, 525, 529, 529, 546, 526, 541, 541, 513, 510, 466, 519, 540, 501, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 495, 503, 477, 517, 529, 503, 529, 537, 514, 547, 497, 506, 500, 539, 515, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 527, 544, 507, 519, 545, 501, 531, 493, 515, 534, 520, 501, 483, 498, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8228, 8153, 8205, 8144, 8036, 8309, 8254, 8207] avg=8192.0000 max=8309 max_rank=5 min=8036 min_rank=4 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32806 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[491, 520, 498, 490, 513, 521, 497, 467, 490, 500, 493, 500, 487, 537, 514, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 531, 487, 523, 485, 547, 520, 520, 509, 529, 503, 489, 493, 533, 523, 508, 32, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 547, 522, 482, 505, 503, 532, 483, 515, 504, 558, 504, 489, 566, 513, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 534, 508, 462, 521, 516, 494, 528, 509, 496, 498, 520, 538, 510, 536, 501, 45, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 535, 534, 550, 556, 550, 513, 525, 493, 555, 500, 513, 550, 509, 479, 499, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 517, 505, 504, 517, 438, 531, 507, 530, 518, 516, 507, 497, 510, 512, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 510, 481, 561, 464, 480, 481, 554, 491, 499, 517, 480, 494, 495, 512, 460, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 525, 533, 543, 516, 500, 526, 492, 523, 532, 495, 529, 509, 517, 497, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8064, 8225, 8265, 8197, 8427, 8070, 8033, 8255] avg=8192.0000 max=8427 max_rank=4 min=8033 min_rank=6 imbalance_ratio=1.028687 max_load_reduction=57109 max_load_reduction_pct=0.871414 moved_to_echo=32785 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[515, 474, 524, 551, 511, 503, 495, 516, 532, 538, 533, 508, 533, 546, 506, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 503, 535, 521, 478, 498, 467, 479, 479, 511, 512, 476, 477, 530, 480, 510, 30, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 527, 483, 545, 524, 503, 496, 510, 504, 517, 496, 514, 475, 550, 539, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 526, 509, 526, 503, 497, 534, 515, 515, 536, 529, 507, 476, 511, 512, 495, 44, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 473, 485, 542, 503, 484, 504, 504, 520, 507, 515, 485, 503, 525, 525, 497, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 522, 546, 547, 516, 546, 444, 494, 511, 518, 482, 481, 544, 519, 558, 460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 505, 535, 509, 525, 488, 524, 519, 478, 500, 535, 520, 505, 561, 485, 509, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 521, 529, 484, 516, 497, 540, 492, 534, 533, 502, 520, 519, 546, 471, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8320, 8016, 8192, 8267, 8088, 8240, 8209, 8204] avg=8192.0000 max=8320 max_rank=0 min=8016 min_rank=1 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32741 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[502, 523, 510, 512, 518, 504, 504, 546, 483, 525, 529, 526, 480, 485, 503, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 481, 523, 531, 504, 504, 548, 497, 484, 523, 527, 552, 513, 465, 535, 505, 35, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 492, 516, 508, 511, 498, 518, 506, 527, 516, 518, 516, 503, 536, 482, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 505, 516, 498, 536, 566, 496, 489, 493, 522, 526, 505, 509, 522, 492, 505, 46, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 510, 506, 529, 519, 510, 493, 505, 531, 518, 522, 496, 525, 499, 493, 533, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 511, 553, 545, 460, 512, 536, 483, 506, 511, 521, 487, 466, 512, 490, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 443, 552, 475, 506, 504, 483, 549, 520, 557, 550, 496, 530, 498, 574, 507, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 519, 481, 481, 502, 512, 489, 513, 565, 504, 491, 522, 468, 494, 543, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8132, 8278, 8162, 8279, 8219, 8127, 8242, 8097] avg=8192.0000 max=8279 max_rank=3 min=8097 min_rank=7 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32685 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[492, 510, 499, 502, 514, 512, 509, 482, 515, 520, 491, 529, 490, 512, 489, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 493, 479, 543, 581, 493, 500, 518, 525, 555, 521, 489, 492, 455, 527, 525, 31, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 498, 519, 536, 506, 515, 544, 499, 502, 527, 542, 504, 489, 544, 516, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 511, 469, 563, 499, 532, 519, 520, 501, 528, 463, 485, 506, 478, 486, 474, 45, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 473, 513, 522, 516, 527, 549, 481, 484, 535, 531, 494, 523, 501, 509, 512, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 522, 536, 471, 495, 533, 471, 526, 508, 501, 526, 474, 499, 531, 535, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 518, 513, 524, 526, 504, 491, 534, 512, 519, 508, 550, 499, 496, 518, 482, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 502, 517, 539, 488, 554, 479, 503, 504, 527, 526, 509, 471, 511, 483, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8090, 8233, 8278, 8102, 8205, 8197, 8240, 8191] avg=8192.0000 max=8278 max_rank=2 min=8090 min_rank=0 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32833 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 3968, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 3968, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 3968, 4224, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 3968, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 3968, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 3968, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 3968, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 3968, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 3968, 4224, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[506, 530, 525, 524, 547, 493, 490, 538, 499, 546, 489, 518, 485, 461, 499, 545, 539, 509, 541, 513, 524, 514, 534, 505, 493, 505, 511, 488, 532, 551, 484, 490, 521, 491, 506, 497, 519, 516, 510, 502, 459, 496, 502, 505, 506, 511, 483, 516, 509, 518, 500, 536, 510, 478, 504, 496, 536, 520, 533, 516, 542, 539, 504, 508, 528, 525, 524, 503, 522, 530, 468, 534, 495, 523, 544, 547, 507, 505, 473, 489, 522, 481, 535, 540, 530, 461, 507, 497, 529, 557, 543, 495, 479, 517, 527, 513, 501, 486, 517, 515, 527, 526, 530, 507, 494, 484, 486, 513, 489, 544, 500, 508, 506, 459, 511, 483, 510, 550, 539, 523, 499, 534, 530, 545, 527, 520, 486, 520] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[477, 491, 533, 475, 502, 510, 509, 537, 543, 492, 527, 567, 471, 504, 521, 520, 495, 519, 510, 520, 473, 511, 486, 529, 492, 489, 505, 522, 471, 528, 517, 536, 550, 508, 558, 477, 520, 509, 484, 521, 495, 528, 506, 489, 545, 536, 513, 446, 511, 484, 517, 509, 527, 485, 466, 508, 480, 497, 523, 491, 482, 484, 517, 533, 535, 500, 540, 534, 520, 526, 493, 474, 536, 510, 552, 545, 526, 518, 500, 501, 536, 611, 495, 489, 498, 511, 502, 512, 485, 495, 508, 540, 512, 535, 538, 518, 509, 526, 555, 507, 482, 556, 485, 521, 498, 515, 516, 520, 499, 521, 502, 485, 514, 549, 505, 546, 527, 517, 522, 530, 502, 457, 530, 517, 496, 522, 527, 502] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[498, 500, 528, 497, 471, 518, 516, 526, 492, 515, 534, 541, 519, 511, 507, 533, 513, 505, 539, 469, 474, 522, 520, 536, 529, 494, 514, 489, 555, 521, 486, 561, 517, 539, 533, 498, 528, 508, 523, 495, 476, 473, 504, 527, 499, 511, 485, 555, 509, 509, 527, 519, 526, 487, 525, 533, 499, 520, 520, 492, 516, 512, 491, 489, 534, 541, 534, 536, 497, 489, 499, 495, 500, 560, 481, 530, 492, 495, 489, 526, 459, 482, 511, 522, 514, 548, 545, 546, 513, 539, 542, 545, 485, 505, 511, 526, 557, 515, 502, 504, 537, 486, 503, 464, 509, 527, 483, 494, 516, 549, 484, 533, 511, 461, 489, 517, 465, 523, 520, 509, 576, 526, 533, 500, 482, 495, 497, 500] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[542, 542, 500, 501, 522, 501, 528, 483, 543, 544, 501, 604, 517, 488, 520, 555, 533, 477, 525, 489, 490, 511, 503, 530, 525, 527, 489, 515, 481, 463, 465, 497, 498, 494, 466, 505, 521, 514, 540, 520, 503, 476, 489, 574, 524, 528, 525, 494, 481, 526, 528, 515, 547, 486, 520, 534, 488, 484, 483, 521, 513, 490, 498, 531, 463, 514, 492, 510, 539, 507, 527, 505, 512, 506, 555, 517, 456, 516, 487, 508, 485, 503, 546, 520, 495, 570, 488, 491, 476, 538, 491, 534, 491, 504, 528, 518, 515, 547, 513, 524, 534, 491, 515, 489, 527, 542, 533, 519, 494, 468, 519, 480, 500, 507, 502, 537, 491, 510, 515, 521, 488, 508, 557, 564, 527, 535, 509, 536] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[516, 507, 510, 529, 482, 500, 538, 504, 518, 483, 520, 470, 503, 494, 513, 503, 483, 486, 544, 498, 509, 508, 528, 546, 529, 501, 495, 533, 503, 523, 534, 486, 506, 534, 521, 508, 502, 485, 596, 496, 508, 531, 474, 500, 508, 507, 519, 528, 516, 516, 480, 508, 487, 529, 482, 504, 520, 497, 497, 527, 489, 539, 488, 479, 508, 512, 487, 514, 575, 482, 505, 516, 507, 495, 574, 521, 530, 530, 563, 505, 525, 470, 500, 487, 548, 520, 499, 536, 556, 490, 516, 496, 551, 495, 479, 504, 528, 503, 509, 524, 468, 499, 528, 475, 471, 534, 523, 518, 488, 526, 538, 484, 553, 526, 522, 529, 487, 559, 527, 536, 556, 477, 490, 530, 514, 493, 520, 528] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[516, 481, 498, 541, 502, 496, 508, 487, 537, 513, 540, 508, 486, 535, 500, 545, 536, 476, 528, 532, 517, 547, 521, 485, 489, 534, 519, 534, 524, 470, 503, 513, 532, 499, 492, 492, 505, 515, 511, 529, 502, 552, 514, 464, 500, 521, 513, 522, 542, 512, 529, 530, 519, 524, 522, 546, 510, 519, 512, 525, 517, 486, 526, 510, 502, 492, 482, 495, 507, 490, 494, 526, 531, 476, 533, 521, 505, 497, 501, 504, 523, 540, 489, 517, 512, 512, 484, 536, 516, 518, 516, 491, 523, 517, 529, 491, 544, 503, 476, 474, 513, 513, 512, 537, 506, 497, 497, 516, 528, 495, 517, 514, 518, 536, 509, 533, 553, 537, 468, 502, 505, 500, 504, 528, 501, 511, 518, 488] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[477, 511, 468, 511, 527, 553, 496, 497, 479, 515, 499, 502, 477, 515, 524, 478, 523, 510, 517, 508, 520, 510, 525, 499, 476, 490, 523, 514, 495, 505, 527, 550, 519, 518, 533, 522, 525, 536, 495, 531, 511, 480, 489, 501, 513, 524, 498, 512, 553, 516, 509, 554, 528, 494, 504, 477, 503, 529, 535, 507, 543, 493, 505, 532, 528, 514, 495, 506, 512, 528, 498, 507, 516, 511, 540, 499, 469, 504, 548, 526, 519, 523, 546, 485, 527, 498, 529, 500, 477, 492, 524, 498, 523, 503, 507, 510, 505, 523, 526, 477, 516, 528, 529, 500, 520, 498, 496, 515, 533, 492, 519, 495, 506, 498, 524, 508, 545, 552, 520, 522, 521, 528, 485, 549, 505, 546, 498, 485] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[542, 542, 500, 501, 522, 501, 528, 483, 543, 544, 501, 604, 517, 488, 520, 555, 533, 477, 525, 489, 490, 511, 503, 530, 525, 527, 489, 515, 481, 463, 465, 497, 498, 494, 466, 505, 521, 514, 540, 520, 503, 476, 489, 574, 524, 528, 525, 494, 481, 526, 528, 515, 547, 486, 520, 534, 488, 484, 483, 521, 513, 490, 498, 531, 463, 514, 492, 510, 539, 507, 527, 505, 512, 506, 555, 517, 456, 516, 487, 508, 485, 503, 546, 520, 495, 570, 488, 491, 476, 538, 491, 534, 491, 504, 528, 518, 515, 547, 513, 524, 534, 491, 515, 489, 527, 542, 533, 519, 494, 468, 519, 480, 500, 507, 502, 537, 491, 510, 515, 521, 488, 508, 557, 564, 527, 535, 509, 536] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[517, 521, 542, 499, 508, 540, 539, 483, 493, 519, 543, 512, 515, 523, 516, 537, 479, 525, 549, 550, 502, 532, 486, 508, 516, 514, 508, 527, 499, 529, 541, 491, 512, 504, 503, 489, 525, 550, 501, 517, 511, 513, 536, 493, 529, 517, 476, 488, 489, 510, 495, 538, 496, 544, 474, 527, 514, 497, 533, 508, 508, 505, 544, 519, 483, 480, 516, 511, 513, 521, 532, 501, 518, 513, 501, 480, 516, 485, 503, 508, 477, 506, 524, 526, 514, 467, 558, 517, 528, 489, 494, 513, 483, 538, 521, 488, 487, 517, 492, 511, 504, 493, 527, 512, 505, 487, 530, 490, 527, 488, 529, 540, 441, 545, 516, 542, 504, 507, 492, 515, 491, 511, 510, 521, 562, 535, 525, 528] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[477, 491, 533, 475, 502, 510, 509, 537, 543, 492, 527, 561, 471, 504, 521, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 519, 510, 520, 473, 511, 486, 529, 492, 489, 505, 522, 471, 528, 517, 536, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 508, 558, 477, 520, 509, 484, 521, 495, 528, 506, 489, 545, 536, 513, 446, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 484, 517, 509, 527, 485, 466, 508, 480, 497, 523, 491, 482, 484, 517, 533, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 500, 540, 534, 520, 526, 493, 474, 536, 510, 552, 545, 526, 518, 500, 501, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 611, 479, 489, 498, 511, 502, 512, 485, 495, 508, 540, 512, 535, 538, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 526, 555, 507, 482, 556, 485, 521, 498, 515, 516, 520, 499, 521, 502, 485, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 549, 505, 546, 527, 461, 522, 530, 502, 457, 530, 517, 496, 522, 527, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8173, 8111, 8209, 8024, 8314, 8269, 8229, 8207] avg=8192.0000 max=8314 max_rank=4 min=8024 min_rank=3 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=33019 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[477, 511, 468, 511, 527, 553, 496, 497, 479, 515, 499, 502, 477, 515, 524, 478, 523, 510, 517, 508, 520, 510, 525, 499, 476, 490, 523, 514, 495, 505, 527, 550, 519, 518, 533, 522, 525, 536, 495, 531, 511, 480, 489, 501, 513, 524, 498, 512, 553, 516, 509, 554, 528, 494, 504, 477, 503, 529, 535, 507, 543, 493, 505, 532, 528, 514, 495, 506, 512, 528, 498, 507, 516, 511, 540, 499, 469, 504, 548, 526, 519, 523, 546, 485, 527, 498, 529, 500, 477, 492, 524, 498, 523, 503, 507, 510, 505, 523, 526, 477, 516, 528, 529, 500, 520, 498, 496, 515, 533, 492, 519, 495, 506, 498, 524, 508, 545, 552, 520, 522, 521, 528, 485, 549, 505, 546, 498, 485] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[477, 491, 533, 475, 502, 510, 509, 537, 543, 492, 527, 567, 471, 504, 521, 520, 495, 519, 510, 520, 473, 511, 486, 529, 492, 489, 505, 522, 471, 528, 517, 536, 550, 508, 558, 477, 520, 509, 484, 521, 495, 528, 506, 489, 545, 536, 513, 446, 511, 484, 517, 509, 527, 485, 466, 508, 480, 497, 523, 491, 482, 484, 517, 533, 535, 500, 540, 534, 520, 526, 493, 474, 536, 510, 552, 545, 526, 518, 500, 501, 536, 611, 495, 489, 498, 511, 502, 512, 485, 495, 508, 540, 512, 535, 538, 518, 509, 526, 555, 507, 482, 556, 485, 521, 498, 515, 516, 520, 499, 521, 502, 485, 514, 549, 505, 546, 527, 517, 522, 530, 502, 457, 530, 517, 496, 522, 527, 502] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[498, 500, 528, 497, 471, 518, 516, 526, 492, 515, 534, 541, 519, 511, 507, 533, 513, 505, 539, 469, 474, 522, 520, 536, 529, 494, 514, 489, 555, 521, 486, 561, 517, 539, 533, 498, 528, 508, 523, 495, 476, 473, 504, 527, 499, 511, 485, 555, 509, 509, 527, 519, 526, 487, 525, 533, 499, 520, 520, 492, 516, 512, 491, 489, 534, 541, 534, 536, 497, 489, 499, 495, 500, 560, 481, 530, 492, 495, 489, 526, 459, 482, 511, 522, 514, 548, 545, 546, 513, 539, 542, 545, 485, 505, 511, 526, 557, 515, 502, 504, 537, 486, 503, 464, 509, 527, 483, 494, 516, 549, 484, 533, 511, 461, 489, 517, 465, 523, 520, 509, 576, 526, 533, 500, 482, 495, 497, 500] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[506, 530, 525, 524, 547, 493, 490, 538, 499, 546, 489, 512, 485, 461, 499, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 509, 541, 513, 524, 514, 534, 505, 493, 505, 511, 488, 532, 551, 484, 490, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 491, 506, 497, 519, 516, 510, 502, 459, 496, 502, 505, 506, 511, 483, 516, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 518, 500, 536, 510, 478, 504, 496, 536, 520, 533, 516, 542, 539, 504, 508, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 525, 524, 503, 522, 530, 468, 534, 495, 523, 544, 547, 507, 505, 473, 489, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 481, 519, 540, 530, 461, 507, 497, 529, 557, 543, 495, 479, 517, 527, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 486, 517, 515, 527, 526, 530, 507, 494, 484, 486, 513, 489, 544, 500, 508, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 459, 511, 483, 510, 490, 539, 523, 499, 534, 530, 545, 527, 520, 486, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8241, 8066, 8259, 8221, 8217, 8161, 8182] avg=8192.0000 max=8259 max_rank=3 min=8066 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=57277 max_load_reduction_pct=0.873978 moved_to_echo=32781 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[498, 500, 528, 497, 471, 518, 516, 526, 492, 515, 534, 535, 519, 511, 507, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 505, 539, 469, 474, 522, 520, 536, 529, 494, 514, 489, 555, 521, 486, 561, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 539, 533, 498, 528, 508, 523, 495, 476, 473, 504, 527, 499, 511, 485, 555, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 509, 527, 519, 526, 487, 525, 533, 499, 520, 520, 492, 516, 512, 491, 489, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 541, 534, 536, 497, 489, 499, 495, 500, 560, 481, 530, 492, 495, 489, 526, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 459, 482, 495, 522, 514, 548, 545, 546, 513, 539, 542, 545, 485, 505, 511, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 557, 515, 502, 504, 537, 486, 503, 464, 509, 527, 483, 494, 516, 549, 484, 533, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 461, 489, 517, 465, 465, 520, 509, 576, 526, 533, 500, 482, 495, 497, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8235, 8196, 8184, 8202, 8277, 8196, 8046] avg=8192.0000 max=8277 max_rank=5 min=8046 min_rank=7 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32721 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[542, 542, 500, 501, 522, 501, 528, 483, 543, 544, 501, 591, 517, 488, 520, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 477, 525, 489, 490, 511, 503, 530, 525, 527, 489, 515, 481, 463, 465, 497, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 494, 466, 505, 521, 514, 540, 520, 503, 476, 489, 574, 524, 528, 525, 494, 29, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 526, 528, 515, 547, 486, 520, 534, 488, 484, 483, 521, 513, 490, 498, 531, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 463, 514, 492, 510, 539, 507, 527, 505, 512, 506, 555, 517, 456, 516, 487, 508, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 503, 515, 520, 495, 570, 488, 491, 476, 538, 491, 534, 491, 504, 528, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 547, 513, 524, 534, 491, 515, 489, 527, 542, 533, 519, 494, 468, 519, 480, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 507, 502, 537, 491, 446, 515, 521, 488, 508, 557, 564, 527, 535, 509, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8378, 8035, 8207, 8159, 8122, 8147, 8245, 8243] avg=8192.0000 max=8378 max_rank=0 min=8035 min_rank=1 imbalance_ratio=1.022705 max_load_reduction=57158 max_load_reduction_pct=0.872162 moved_to_echo=32757 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[516, 507, 510, 529, 482, 500, 538, 504, 518, 483, 520, 465, 503, 494, 513, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 486, 544, 498, 509, 508, 528, 546, 529, 501, 495, 533, 503, 523, 534, 486, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 534, 521, 508, 502, 485, 596, 496, 508, 531, 474, 500, 508, 507, 519, 528, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 516, 480, 508, 487, 529, 482, 504, 520, 497, 497, 527, 489, 539, 488, 479, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 512, 487, 514, 575, 482, 505, 516, 507, 495, 574, 521, 530, 530, 563, 505, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 470, 484, 487, 548, 520, 499, 536, 556, 490, 516, 496, 551, 495, 479, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 503, 509, 524, 468, 499, 528, 475, 471, 534, 523, 518, 488, 526, 538, 484, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 526, 522, 529, 487, 498, 527, 536, 556, 477, 490, 530, 514, 493, 520, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8085, 8214, 8249, 8068, 8327, 8156, 8151, 8286] avg=8192.0000 max=8327 max_rank=4 min=8068 min_rank=3 imbalance_ratio=1.016479 max_load_reduction=57209 max_load_reduction_pct=0.872940 moved_to_echo=32920 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[516, 481, 498, 541, 502, 496, 508, 487, 537, 513, 540, 502, 486, 535, 500, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 476, 528, 532, 517, 547, 521, 485, 489, 534, 519, 534, 524, 470, 503, 513, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 499, 492, 492, 505, 515, 511, 529, 502, 552, 514, 464, 500, 521, 513, 522, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 512, 529, 530, 519, 524, 522, 546, 510, 519, 512, 525, 517, 486, 526, 510, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 492, 482, 495, 507, 490, 494, 526, 531, 476, 533, 521, 505, 497, 501, 504, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 540, 474, 517, 512, 512, 484, 536, 516, 518, 516, 491, 523, 517, 529, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 503, 476, 474, 513, 513, 512, 537, 506, 497, 497, 516, 528, 495, 517, 514, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 536, 509, 533, 553, 478, 468, 502, 505, 500, 504, 528, 501, 511, 518, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8236, 8188, 8338, 8060, 8199, 8176, 8152] avg=8192.0000 max=8338 max_rank=3 min=8060 min_rank=4 imbalance_ratio=1.017822 max_load_reduction=57198 max_load_reduction_pct=0.872772 moved_to_echo=32587 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[477, 511, 468, 511, 527, 553, 496, 497, 479, 515, 499, 496, 477, 515, 524, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 510, 517, 508, 520, 510, 525, 499, 476, 490, 523, 514, 495, 505, 527, 550, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 518, 533, 522, 525, 536, 495, 531, 511, 480, 489, 501, 513, 524, 498, 512, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 516, 509, 554, 528, 494, 504, 477, 503, 529, 535, 507, 543, 493, 505, 532, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 514, 495, 506, 512, 528, 498, 507, 516, 511, 540, 499, 469, 504, 548, 526, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 523, 529, 485, 527, 498, 529, 500, 477, 492, 524, 498, 523, 503, 507, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 523, 526, 477, 516, 528, 529, 500, 520, 498, 496, 515, 533, 492, 519, 495, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 498, 524, 508, 545, 491, 520, 522, 521, 528, 485, 549, 505, 546, 498, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8023, 8200, 8233, 8293, 8205, 8144, 8207, 8231] avg=8192.0000 max=8293 max_rank=3 min=8023 min_rank=0 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32787 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[517, 521, 542, 499, 508, 540, 539, 483, 493, 519, 543, 506, 515, 523, 516, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 525, 549, 550, 502, 532, 486, 508, 516, 514, 508, 527, 499, 529, 541, 491, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 504, 503, 489, 525, 550, 501, 517, 511, 513, 536, 493, 529, 517, 476, 488, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 510, 495, 538, 496, 544, 474, 527, 514, 497, 533, 508, 508, 505, 544, 519, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 480, 516, 511, 513, 521, 532, 501, 518, 513, 501, 480, 516, 485, 503, 508, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 506, 508, 526, 514, 467, 558, 517, 528, 489, 494, 513, 483, 538, 521, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 517, 492, 511, 504, 493, 527, 512, 505, 487, 530, 490, 527, 488, 529, 540, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 441, 545, 516, 542, 504, 451, 492, 515, 491, 511, 510, 521, 562, 535, 525, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8301, 8264, 8188, 8211, 8085, 8127, 8171, 8189] avg=8192.0000 max=8301 max_rank=0 min=8085 min_rank=4 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32572 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[477, 511, 468, 511, 527, 553, 496, 497, 479, 515, 499, 496, 477, 515, 524, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 510, 517, 508, 520, 510, 525, 499, 476, 490, 523, 514, 495, 505, 527, 550, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 518, 533, 522, 525, 536, 495, 531, 511, 480, 489, 501, 513, 524, 498, 512, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 516, 509, 554, 528, 494, 504, 477, 503, 529, 535, 507, 543, 493, 505, 532, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 514, 495, 506, 512, 528, 498, 507, 516, 511, 540, 499, 469, 504, 548, 526, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 523, 529, 485, 527, 498, 529, 500, 477, 492, 524, 498, 523, 503, 507, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 523, 526, 477, 516, 528, 529, 500, 520, 498, 496, 515, 533, 492, 519, 495, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 498, 524, 508, 545, 491, 520, 522, 521, 528, 485, 549, 505, 546, 498, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8023, 8200, 8233, 8293, 8205, 8144, 8207, 8231] avg=8192.0000 max=8293 max_rank=3 min=8023 min_rank=0 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32787 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[516, 507, 510, 529, 482, 500, 538, 504, 518, 483, 520, 465, 503, 494, 513, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 486, 544, 498, 509, 508, 528, 546, 529, 501, 495, 533, 503, 523, 534, 486, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 534, 521, 508, 502, 485, 596, 496, 508, 531, 474, 500, 508, 507, 519, 528, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 516, 480, 508, 487, 529, 482, 504, 520, 497, 497, 527, 489, 539, 488, 479, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 512, 487, 514, 575, 482, 505, 516, 507, 495, 574, 521, 530, 530, 563, 505, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 470, 484, 487, 548, 520, 499, 536, 556, 490, 516, 496, 551, 495, 479, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 503, 509, 524, 468, 499, 528, 475, 471, 534, 523, 518, 488, 526, 538, 484, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 526, 522, 529, 487, 498, 527, 536, 556, 477, 490, 530, 514, 493, 520, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8085, 8214, 8249, 8068, 8327, 8156, 8151, 8286] avg=8192.0000 max=8327 max_rank=4 min=8068 min_rank=3 imbalance_ratio=1.016479 max_load_reduction=57209 max_load_reduction_pct=0.872940 moved_to_echo=32920 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[477, 491, 533, 475, 502, 510, 509, 537, 543, 492, 527, 561, 471, 504, 521, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 519, 510, 520, 473, 511, 486, 529, 492, 489, 505, 522, 471, 528, 517, 536, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 508, 558, 477, 520, 509, 484, 521, 495, 528, 506, 489, 545, 536, 513, 446, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 484, 517, 509, 527, 485, 466, 508, 480, 497, 523, 491, 482, 484, 517, 533, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 500, 540, 534, 520, 526, 493, 474, 536, 510, 552, 545, 526, 518, 500, 501, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 611, 479, 489, 498, 511, 502, 512, 485, 495, 508, 540, 512, 535, 538, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 526, 555, 507, 482, 556, 485, 521, 498, 515, 516, 520, 499, 521, 502, 485, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 549, 505, 546, 527, 461, 522, 530, 502, 457, 530, 517, 496, 522, 527, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8173, 8111, 8209, 8024, 8314, 8269, 8229, 8207] avg=8192.0000 max=8314 max_rank=4 min=8024 min_rank=3 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=33019 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[498, 500, 528, 497, 471, 518, 516, 526, 492, 515, 534, 535, 519, 511, 507, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 505, 539, 469, 474, 522, 520, 536, 529, 494, 514, 489, 555, 521, 486, 561, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 539, 533, 498, 528, 508, 523, 495, 476, 473, 504, 527, 499, 511, 485, 555, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 509, 527, 519, 526, 487, 525, 533, 499, 520, 520, 492, 516, 512, 491, 489, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 541, 534, 536, 497, 489, 499, 495, 500, 560, 481, 530, 492, 495, 489, 526, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 459, 482, 495, 522, 514, 548, 545, 546, 513, 539, 542, 545, 485, 505, 511, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 557, 515, 502, 504, 537, 486, 503, 464, 509, 527, 483, 494, 516, 549, 484, 533, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 461, 489, 517, 465, 465, 520, 509, 576, 526, 533, 500, 482, 495, 497, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8235, 8196, 8184, 8202, 8277, 8196, 8046] avg=8192.0000 max=8277 max_rank=5 min=8046 min_rank=7 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32721 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4352, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4352, 4224, 4096, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 3840, 4224, 4224, 4224, 4096, 4224, 4352, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 3968, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 3968, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 3840, 4224, 4224, 4224, 4096, 4224, 4352, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4352, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4352, 4224, 4096, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[533, 545, 487, 504, 521, 476, 488, 525, 511, 520, 487, 504, 498, 508, 505, 469, 550, 481, 484, 518, 483, 535, 508, 504, 500, 521, 553, 534, 546, 510, 518, 479, 524, 550, 533, 499, 522, 519, 513, 517, 511, 484, 562, 493, 530, 526, 500, 531, 541, 507, 498, 551, 476, 526, 531, 525, 511, 511, 486, 536, 463, 524, 529, 514, 492, 470, 498, 508, 550, 544, 504, 530, 485, 477, 538, 531, 536, 519, 512, 493, 535, 523, 506, 489, 512, 513, 491, 495, 474, 533, 494, 538, 550, 440, 498, 501, 522, 562, 533, 509, 507, 506, 528, 501, 533, 514, 503, 513, 480, 511, 512, 512, 489, 514, 581, 532, 517, 520, 489, 530, 459, 486, 536, 486, 512, 491, 490, 531] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[525, 509, 508, 497, 493, 514, 522, 511, 518, 494, 530, 497, 511, 499, 545, 517, 514, 564, 500, 523, 512, 506, 547, 505, 546, 516, 493, 517, 508, 522, 493, 515, 520, 523, 493, 475, 486, 546, 471, 494, 541, 537, 525, 534, 528, 525, 514, 524, 531, 496, 487, 490, 519, 537, 500, 511, 515, 514, 526, 569, 534, 487, 467, 513, 528, 513, 466, 553, 533, 505, 509, 493, 472, 515, 520, 528, 534, 531, 543, 517, 500, 534, 502, 532, 473, 465, 499, 533, 511, 517, 526, 513, 523, 482, 526, 490, 521, 517, 477, 541, 519, 482, 545, 481, 526, 532, 497, 499, 521, 550, 501, 499, 516, 511, 541, 528, 483, 507, 484, 522, 460, 511, 499, 528, 492, 510, 486, 461] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[522, 524, 522, 491, 512, 465, 498, 505, 525, 529, 511, 540, 446, 538, 499, 523, 481, 528, 527, 510, 502, 516, 535, 545, 508, 471, 505, 482, 493, 505, 506, 516, 496, 539, 536, 537, 552, 556, 533, 502, 523, 491, 475, 553, 493, 555, 503, 547, 507, 518, 497, 511, 510, 516, 488, 523, 498, 490, 501, 511, 516, 504, 506, 467, 562, 500, 518, 518, 527, 491, 517, 538, 533, 555, 509, 472, 537, 509, 496, 523, 520, 505, 522, 478, 535, 534, 528, 528, 512, 471, 552, 495, 484, 523, 529, 488, 528, 514, 502, 502, 548, 494, 469, 482, 524, 499, 470, 507, 528, 484, 533, 515, 478, 523, 533, 519, 511, 537, 510, 484, 513, 541, 492, 523, 507, 508, 523, 492] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[510, 500, 518, 534, 550, 520, 488, 515, 492, 520, 496, 541, 543, 494, 502, 510, 503, 514, 558, 534, 525, 518, 528, 538, 528, 506, 482, 573, 543, 516, 544, 513, 510, 476, 510, 476, 552, 518, 528, 536, 503, 526, 521, 507, 513, 486, 506, 530, 513, 489, 534, 520, 502, 506, 498, 515, 488, 487, 494, 525, 555, 542, 525, 542, 514, 514, 489, 540, 554, 564, 481, 538, 516, 512, 512, 505, 495, 507, 504, 523, 517, 527, 505, 515, 500, 526, 553, 528, 517, 504, 493, 501, 524, 506, 513, 473, 526, 516, 484, 514, 474, 499, 543, 508, 481, 496, 481, 476, 503, 488, 534, 520, 470, 502, 521, 466, 502, 517, 486, 518, 480, 482, 515, 533, 494, 481, 481, 486] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[477, 500, 559, 497, 535, 491, 488, 563, 517, 520, 519, 504, 499, 526, 520, 499, 523, 522, 534, 495, 493, 520, 548, 496, 497, 505, 514, 544, 535, 492, 553, 504, 535, 516, 551, 459, 529, 530, 511, 533, 548, 536, 498, 515, 503, 546, 495, 511, 501, 516, 515, 497, 521, 514, 529, 496, 496, 515, 498, 490, 524, 509, 514, 525, 534, 494, 517, 563, 523, 508, 545, 516, 510, 532, 503, 497, 499, 460, 497, 529, 465, 493, 498, 517, 502, 500, 512, 484, 538, 533, 514, 488, 507, 542, 473, 502, 485, 516, 524, 484, 499, 506, 549, 515, 539, 468, 497, 510, 474, 523, 479, 528, 497, 530, 530, 483, 523, 522, 521, 472, 508, 506, 517, 538, 519, 508, 532, 474] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[493, 563, 548, 509, 490, 506, 522, 492, 531, 533, 492, 512, 511, 490, 506, 485, 529, 531, 534, 520, 503, 524, 524, 522, 527, 494, 511, 498, 468, 531, 495, 531, 528, 553, 558, 490, 504, 513, 500, 506, 512, 531, 498, 497, 554, 522, 508, 525, 501, 501, 536, 500, 515, 500, 475, 552, 506, 510, 520, 505, 493, 494, 526, 497, 561, 535, 524, 509, 530, 491, 520, 521, 507, 530, 507, 489, 526, 498, 497, 533, 526, 525, 492, 485, 519, 487, 491, 510, 519, 516, 531, 514, 500, 458, 515, 499, 498, 532, 524, 490, 508, 527, 499, 506, 517, 521, 517, 499, 508, 530, 509, 518, 549, 493, 507, 475, 488, 524, 457, 487, 497, 504, 533, 503, 490, 561, 508, 537] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[509, 537, 522, 533, 521, 528, 530, 476, 534, 493, 512, 502, 530, 500, 555, 491, 499, 530, 521, 538, 502, 524, 484, 490, 556, 503, 587, 528, 516, 501, 565, 503, 546, 500, 520, 502, 518, 521, 498, 503, 527, 471, 511, 536, 535, 502, 539, 485, 487, 480, 503, 507, 525, 483, 495, 555, 513, 526, 522, 449, 462, 500, 496, 522, 504, 506, 505, 542, 531, 526, 498, 528, 510, 512, 530, 513, 493, 517, 477, 508, 525, 503, 579, 502, 475, 516, 519, 501, 536, 532, 525, 550, 471, 488, 517, 508, 502, 527, 508, 524, 511, 502, 495, 536, 513, 526, 520, 512, 530, 456, 530, 487, 500, 497, 493, 481, 505, 546, 500, 511, 521, 503, 509, 524, 482, 479, 502, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[499, 486, 481, 528, 531, 526, 555, 539, 513, 530, 498, 552, 517, 509, 518, 477, 522, 514, 542, 499, 517, 530, 490, 513, 510, 517, 491, 491, 528, 514, 491, 498, 519, 500, 532, 521, 492, 498, 527, 533, 526, 500, 525, 506, 459, 536, 545, 534, 521, 506, 495, 459, 506, 521, 498, 464, 520, 477, 540, 509, 480, 528, 533, 485, 467, 559, 495, 539, 508, 502, 505, 479, 513, 501, 525, 531, 520, 497, 535, 524, 545, 519, 490, 517, 484, 530, 540, 484, 482, 490, 478, 512, 551, 526, 522, 528, 532, 522, 473, 501, 500, 501, 551, 544, 448, 493, 532, 480, 459, 519, 531, 554, 502, 520, 501, 538, 499, 517, 534, 519, 517, 489, 528, 510, 531, 524, 505, 543] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[510, 500, 518, 534, 550, 520, 488, 515, 492, 520, 496, 541, 543, 494, 502, 510, 503, 514, 558, 534, 525, 518, 528, 538, 528, 506, 482, 573, 543, 516, 544, 513, 510, 476, 510, 476, 552, 518, 528, 536, 503, 526, 521, 507, 513, 486, 506, 530, 513, 489, 534, 520, 502, 506, 498, 515, 488, 487, 494, 525, 555, 542, 525, 542, 514, 514, 489, 540, 554, 564, 481, 538, 516, 512, 512, 505, 495, 507, 504, 523, 517, 527, 505, 515, 500, 526, 553, 528, 517, 504, 493, 501, 524, 506, 513, 473, 526, 516, 484, 514, 474, 499, 543, 508, 481, 496, 481, 476, 503, 488, 534, 520, 470, 502, 521, 466, 502, 517, 486, 518, 480, 482, 515, 533, 494, 481, 481, 486] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[525, 504, 508, 497, 493, 514, 522, 511, 518, 494, 530, 497, 511, 499, 545, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 564, 435, 523, 512, 506, 547, 505, 546, 516, 493, 517, 508, 522, 493, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 523, 414, 475, 486, 546, 471, 494, 541, 537, 525, 534, 528, 525, 514, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 496, 487, 490, 519, 537, 500, 511, 515, 514, 526, 569, 534, 487, 467, 513, 42, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 513, 466, 503, 533, 505, 509, 493, 472, 515, 520, 528, 534, 531, 543, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 534, 502, 532, 473, 465, 499, 533, 511, 517, 526, 513, 523, 482, 526, 490, 34, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 517, 477, 541, 519, 482, 545, 481, 526, 532, 497, 499, 521, 550, 501, 499, 23, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 511, 541, 528, 483, 507, 484, 522, 460, 511, 499, 528, 492, 510, 486, 461, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8216, 8157, 8249, 8210, 8165, 8247, 8107] avg=8192.0000 max=8249 max_rank=3 min=8107 min_rank=7 imbalance_ratio=1.006958 max_load_reduction=57287 max_load_reduction_pct=0.874130 moved_to_echo=32729 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[533, 535, 487, 504, 521, 476, 488, 525, 511, 520, 487, 504, 498, 508, 505, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 481, 415, 518, 483, 535, 508, 504, 500, 521, 553, 534, 546, 510, 518, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 550, 442, 499, 522, 519, 513, 517, 511, 484, 562, 493, 530, 526, 500, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 507, 498, 551, 476, 526, 531, 525, 511, 511, 486, 536, 463, 524, 529, 514, 43, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 470, 498, 455, 550, 544, 504, 530, 485, 477, 538, 531, 536, 519, 512, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 523, 506, 489, 512, 513, 491, 495, 474, 533, 494, 538, 550, 440, 498, 501, 35, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 562, 533, 509, 507, 506, 528, 501, 533, 514, 503, 513, 480, 511, 512, 512, 26, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 514, 581, 532, 517, 520, 489, 530, 459, 486, 536, 486, 512, 491, 490, 531, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8071, 8155, 8223, 8287, 8134, 8137, 8290, 8239] avg=8192.0000 max=8290 max_rank=6 min=8071 min_rank=0 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32800 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[509, 531, 522, 533, 521, 528, 530, 476, 534, 493, 512, 502, 530, 500, 555, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 530, 453, 538, 502, 524, 484, 490, 556, 503, 587, 528, 516, 501, 565, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 500, 437, 502, 518, 521, 498, 503, 527, 471, 511, 536, 535, 502, 539, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 480, 503, 507, 525, 483, 495, 555, 513, 526, 522, 449, 462, 500, 496, 522, 44, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 506, 505, 494, 531, 526, 498, 528, 510, 512, 530, 513, 493, 517, 477, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 503, 579, 502, 475, 516, 519, 501, 536, 532, 525, 550, 471, 488, 517, 508, 33, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 527, 508, 524, 511, 502, 495, 536, 513, 526, 520, 512, 530, 456, 530, 487, 24, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 497, 493, 481, 505, 546, 500, 511, 521, 503, 509, 524, 482, 479, 502, 498, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8267, 8279, 8131, 8081, 8152, 8286, 8218, 8122] avg=8192.0000 max=8286 max_rank=5 min=8081 min_rank=3 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32778 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[525, 509, 508, 497, 493, 514, 522, 511, 518, 494, 530, 497, 511, 499, 545, 517, 514, 564, 500, 523, 512, 506, 547, 505, 546, 516, 493, 517, 508, 522, 493, 515, 520, 523, 493, 475, 486, 546, 471, 494, 541, 537, 525, 534, 528, 525, 514, 524, 531, 496, 487, 490, 519, 537, 500, 511, 515, 514, 526, 569, 534, 487, 467, 513, 528, 513, 466, 553, 533, 505, 509, 493, 472, 515, 520, 528, 534, 531, 543, 517, 500, 534, 502, 532, 473, 465, 499, 533, 511, 517, 526, 513, 523, 482, 526, 490, 521, 517, 477, 541, 519, 482, 545, 481, 526, 532, 497, 499, 521, 550, 501, 499, 516, 511, 541, 528, 483, 507, 484, 522, 460, 511, 499, 528, 492, 510, 486, 461] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[522, 524, 522, 491, 512, 465, 498, 505, 525, 529, 511, 540, 446, 538, 499, 523, 481, 528, 527, 510, 502, 516, 535, 545, 508, 471, 505, 482, 493, 505, 506, 516, 496, 539, 536, 537, 552, 556, 533, 502, 523, 491, 475, 553, 493, 555, 503, 547, 507, 518, 497, 511, 510, 516, 488, 523, 498, 490, 501, 511, 516, 504, 506, 467, 562, 500, 518, 518, 527, 491, 517, 538, 533, 555, 509, 472, 537, 509, 496, 523, 520, 505, 522, 478, 535, 534, 528, 528, 512, 471, 552, 495, 484, 523, 529, 488, 528, 514, 502, 502, 548, 494, 469, 482, 524, 499, 470, 507, 528, 484, 533, 515, 478, 523, 533, 519, 511, 537, 510, 484, 513, 541, 492, 523, 507, 508, 523, 492] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[533, 545, 487, 504, 521, 476, 488, 525, 511, 520, 487, 504, 498, 508, 505, 469, 550, 481, 484, 518, 483, 535, 508, 504, 500, 521, 553, 534, 546, 510, 518, 479, 524, 550, 533, 499, 522, 519, 513, 517, 511, 484, 562, 493, 530, 526, 500, 531, 541, 507, 498, 551, 476, 526, 531, 525, 511, 511, 486, 536, 463, 524, 529, 514, 492, 470, 498, 508, 550, 544, 504, 530, 485, 477, 538, 531, 536, 519, 512, 493, 535, 523, 506, 489, 512, 513, 491, 495, 474, 533, 494, 538, 550, 440, 498, 501, 522, 562, 533, 509, 507, 506, 528, 501, 533, 514, 503, 513, 480, 511, 512, 512, 489, 514, 581, 532, 517, 520, 489, 530, 459, 486, 536, 486, 512, 491, 490, 531] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[509, 537, 522, 533, 521, 528, 530, 476, 534, 493, 512, 502, 530, 500, 555, 491, 499, 530, 521, 538, 502, 524, 484, 490, 556, 503, 587, 528, 516, 501, 565, 503, 546, 500, 520, 502, 518, 521, 498, 503, 527, 471, 511, 536, 535, 502, 539, 485, 487, 480, 503, 507, 525, 483, 495, 555, 513, 526, 522, 449, 462, 500, 496, 522, 504, 506, 505, 542, 531, 526, 498, 528, 510, 512, 530, 513, 493, 517, 477, 508, 525, 503, 579, 502, 475, 516, 519, 501, 536, 532, 525, 550, 471, 488, 517, 508, 502, 527, 508, 524, 511, 502, 495, 536, 513, 526, 520, 512, 530, 456, 530, 487, 500, 497, 493, 481, 505, 546, 500, 511, 521, 503, 509, 524, 482, 479, 502, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[510, 495, 518, 534, 550, 520, 488, 515, 492, 520, 496, 541, 543, 494, 502, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 514, 485, 534, 525, 518, 528, 538, 528, 506, 482, 573, 543, 516, 544, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 476, 428, 476, 552, 518, 528, 536, 503, 526, 521, 507, 513, 486, 506, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 489, 534, 520, 502, 506, 498, 515, 488, 487, 494, 525, 555, 542, 525, 542, 47, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 514, 489, 492, 554, 564, 481, 538, 516, 512, 512, 505, 495, 507, 504, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 527, 505, 515, 500, 526, 553, 528, 517, 504, 493, 501, 524, 506, 513, 473, 33, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 516, 484, 514, 474, 499, 543, 508, 481, 496, 481, 476, 503, 488, 534, 520, 26, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 502, 521, 466, 502, 517, 486, 518, 480, 482, 515, 533, 494, 481, 481, 486, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8228, 8350, 8116, 8294, 8220, 8240, 8084, 8004] avg=8192.0000 max=8350 max_rank=1 min=8004 min_rank=7 imbalance_ratio=1.019287 max_load_reduction=57186 max_load_reduction_pct=0.872589 moved_to_echo=32548 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[493, 557, 548, 509, 490, 506, 522, 492, 531, 533, 492, 512, 511, 490, 506, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 531, 464, 520, 503, 524, 524, 522, 527, 494, 511, 498, 468, 531, 495, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 553, 468, 490, 504, 513, 500, 506, 512, 531, 498, 497, 554, 522, 508, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 501, 536, 500, 515, 500, 475, 552, 506, 510, 520, 505, 493, 494, 526, 497, 45, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 535, 524, 464, 530, 491, 520, 521, 507, 530, 507, 489, 526, 498, 497, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 525, 492, 485, 519, 487, 491, 510, 519, 516, 531, 514, 500, 458, 515, 499, 31, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 532, 524, 490, 508, 527, 499, 506, 517, 521, 517, 499, 508, 530, 509, 518, 25, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 493, 507, 475, 488, 524, 457, 487, 497, 504, 533, 503, 490, 561, 508, 537, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8172, 8209, 8189, 8233, 8124, 8242, 8190] avg=8192.0000 max=8242 max_rank=6 min=8124 min_rank=5 imbalance_ratio=1.006104 max_load_reduction=57294 max_load_reduction_pct=0.874237 moved_to_echo=32789 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[499, 481, 481, 528, 531, 526, 555, 539, 513, 530, 498, 552, 517, 509, 518, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 514, 472, 499, 517, 530, 490, 513, 510, 517, 491, 491, 528, 514, 491, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 500, 447, 521, 492, 498, 527, 533, 526, 500, 525, 506, 459, 536, 545, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 506, 495, 459, 506, 521, 498, 464, 520, 477, 540, 509, 480, 528, 533, 485, 45, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 559, 495, 491, 508, 502, 505, 479, 513, 501, 525, 531, 520, 497, 535, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 519, 490, 517, 484, 530, 540, 484, 482, 490, 478, 512, 551, 526, 522, 528, 33, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 522, 473, 501, 500, 501, 551, 544, 448, 493, 532, 480, 459, 519, 531, 554, 25, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 520, 501, 538, 499, 517, 534, 519, 517, 489, 528, 510, 531, 524, 505, 543, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8254, 8097, 8168, 8099, 8152, 8236, 8180, 8350] avg=8192.0000 max=8350 max_rank=7 min=8097 min_rank=1 imbalance_ratio=1.019287 max_load_reduction=57186 max_load_reduction_pct=0.872589 moved_to_echo=32918 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[533, 535, 487, 504, 521, 476, 488, 525, 511, 520, 487, 504, 498, 508, 505, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 481, 415, 518, 483, 535, 508, 504, 500, 521, 553, 534, 546, 510, 518, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 550, 442, 499, 522, 519, 513, 517, 511, 484, 562, 493, 530, 526, 500, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 507, 498, 551, 476, 526, 531, 525, 511, 511, 486, 536, 463, 524, 529, 514, 43, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 470, 498, 455, 550, 544, 504, 530, 485, 477, 538, 531, 536, 519, 512, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 523, 506, 489, 512, 513, 491, 495, 474, 533, 494, 538, 550, 440, 498, 501, 35, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 562, 533, 509, 507, 506, 528, 501, 533, 514, 503, 513, 480, 511, 512, 512, 26, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 514, 581, 532, 517, 520, 489, 530, 459, 486, 536, 486, 512, 491, 490, 531, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8071, 8155, 8223, 8287, 8134, 8137, 8290, 8239] avg=8192.0000 max=8290 max_rank=6 min=8071 min_rank=0 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32800 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[522, 519, 522, 491, 512, 465, 498, 505, 525, 529, 511, 540, 446, 538, 499, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 528, 459, 510, 502, 516, 535, 545, 508, 471, 505, 482, 493, 505, 506, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 539, 450, 537, 552, 556, 533, 502, 523, 491, 475, 553, 493, 555, 503, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 518, 497, 511, 510, 516, 488, 523, 498, 490, 501, 511, 516, 504, 506, 467, 44, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 562, 500, 518, 471, 527, 491, 517, 538, 533, 555, 509, 472, 537, 509, 496, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 505, 522, 478, 535, 534, 528, 528, 512, 471, 552, 495, 484, 523, 529, 488, 32, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 514, 502, 502, 548, 494, 469, 482, 524, 499, 470, 507, 528, 484, 533, 515, 24, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 523, 533, 519, 511, 537, 510, 484, 513, 541, 492, 523, 507, 508, 523, 492, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8145, 8062, 8305, 8119, 8258, 8241, 8138, 8268] avg=8192.0000 max=8305 max_rank=2 min=8062 min_rank=1 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32905 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[510, 495, 518, 534, 550, 520, 488, 515, 492, 520, 496, 541, 543, 494, 502, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 514, 485, 534, 525, 518, 528, 538, 528, 506, 482, 573, 543, 516, 544, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 476, 428, 476, 552, 518, 528, 536, 503, 526, 521, 507, 513, 486, 506, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 489, 534, 520, 502, 506, 498, 515, 488, 487, 494, 525, 555, 542, 525, 542, 47, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 514, 489, 492, 554, 564, 481, 538, 516, 512, 512, 505, 495, 507, 504, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 527, 505, 515, 500, 526, 553, 528, 517, 504, 493, 501, 524, 506, 513, 473, 33, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 516, 484, 514, 474, 499, 543, 508, 481, 496, 481, 476, 503, 488, 534, 520, 26, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 502, 521, 466, 502, 517, 486, 518, 480, 482, 515, 533, 494, 481, 481, 486, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8228, 8350, 8116, 8294, 8220, 8240, 8084, 8004] avg=8192.0000 max=8350 max_rank=1 min=8004 min_rank=7 imbalance_ratio=1.019287 max_load_reduction=57186 max_load_reduction_pct=0.872589 moved_to_echo=32548 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[477, 495, 559, 497, 535, 491, 488, 563, 517, 520, 519, 504, 499, 526, 520, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 522, 464, 495, 493, 520, 548, 496, 497, 505, 514, 544, 535, 492, 553, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 516, 462, 459, 529, 530, 511, 533, 548, 536, 498, 515, 503, 546, 495, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 516, 515, 497, 521, 514, 529, 496, 496, 515, 498, 490, 524, 509, 514, 525, 45, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 494, 517, 513, 523, 508, 545, 516, 510, 532, 503, 497, 499, 460, 497, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 493, 498, 517, 502, 500, 512, 484, 538, 533, 514, 488, 507, 542, 473, 502, 34, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 516, 524, 484, 499, 506, 549, 515, 539, 468, 497, 510, 474, 523, 479, 528, 25, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 530, 530, 483, 523, 522, 521, 472, 508, 506, 517, 538, 519, 508, 532, 474, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8205, 8227, 8218, 8177, 8107, 8137, 8256] avg=8192.0000 max=8256 max_rank=7 min=8107 min_rank=5 imbalance_ratio=1.007812 max_load_reduction=57280 max_load_reduction_pct=0.874023 moved_to_echo=32677 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[493, 557, 548, 509, 490, 506, 522, 492, 531, 533, 492, 512, 511, 490, 506, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 531, 464, 520, 503, 524, 524, 522, 527, 494, 511, 498, 468, 531, 495, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 553, 468, 490, 504, 513, 500, 506, 512, 531, 498, 497, 554, 522, 508, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 501, 536, 500, 515, 500, 475, 552, 506, 510, 520, 505, 493, 494, 526, 497, 45, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 535, 524, 464, 530, 491, 520, 521, 507, 530, 507, 489, 526, 498, 497, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 525, 492, 485, 519, 487, 491, 510, 519, 516, 531, 514, 500, 458, 515, 499, 31, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 532, 524, 490, 508, 527, 499, 506, 517, 521, 517, 499, 508, 530, 509, 518, 25, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 493, 507, 475, 488, 524, 457, 487, 497, 504, 533, 503, 490, 561, 508, 537, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8172, 8209, 8189, 8233, 8124, 8242, 8190] avg=8192.0000 max=8242 max_rank=6 min=8124 min_rank=5 imbalance_ratio=1.006104 max_load_reduction=57294 max_load_reduction_pct=0.874237 moved_to_echo=32789 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[525, 504, 508, 497, 493, 514, 522, 511, 518, 494, 530, 497, 511, 499, 545, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 564, 435, 523, 512, 506, 547, 505, 546, 516, 493, 517, 508, 522, 493, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 523, 414, 475, 486, 546, 471, 494, 541, 537, 525, 534, 528, 525, 514, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 496, 487, 490, 519, 537, 500, 511, 515, 514, 526, 569, 534, 487, 467, 513, 42, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 513, 466, 503, 533, 505, 509, 493, 472, 515, 520, 528, 534, 531, 543, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 534, 502, 532, 473, 465, 499, 533, 511, 517, 526, 513, 523, 482, 526, 490, 34, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 517, 477, 541, 519, 482, 545, 481, 526, 532, 497, 499, 521, 550, 501, 499, 23, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 511, 541, 528, 483, 507, 484, 522, 460, 511, 499, 528, 492, 510, 486, 461, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8216, 8157, 8249, 8210, 8165, 8247, 8107] avg=8192.0000 max=8249 max_rank=3 min=8107 min_rank=7 imbalance_ratio=1.006958 max_load_reduction=57287 max_load_reduction_pct=0.874130 moved_to_echo=32729 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[509, 531, 522, 533, 521, 528, 530, 476, 534, 493, 512, 502, 530, 500, 555, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 530, 453, 538, 502, 524, 484, 490, 556, 503, 587, 528, 516, 501, 565, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 500, 437, 502, 518, 521, 498, 503, 527, 471, 511, 536, 535, 502, 539, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 480, 503, 507, 525, 483, 495, 555, 513, 526, 522, 449, 462, 500, 496, 522, 44, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 506, 505, 494, 531, 526, 498, 528, 510, 512, 530, 513, 493, 517, 477, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 503, 579, 502, 475, 516, 519, 501, 536, 532, 525, 550, 471, 488, 517, 508, 33, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 527, 508, 524, 511, 502, 495, 536, 513, 526, 520, 512, 530, 456, 530, 487, 24, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 497, 493, 481, 505, 546, 500, 511, 521, 503, 509, 524, 482, 479, 502, 498, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8267, 8279, 8131, 8081, 8152, 8286, 8218, 8122] avg=8192.0000 max=8286 max_rank=5 min=8081 min_rank=3 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32778 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 3712, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 3968, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 3968, 4352, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 3968, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 3584, 3968, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 3712, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 3968, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[539, 511, 547, 515, 546, 538, 476, 475, 545, 535, 502, 532, 538, 517, 533, 501, 518, 515, 513, 521, 522, 480, 520, 499, 530, 540, 514, 481, 530, 543, 493, 500, 508, 532, 522, 521, 486, 508, 532, 499, 506, 518, 536, 537, 490, 512, 523, 530, 492, 498, 532, 538, 463, 479, 534, 493, 512, 538, 480, 530, 526, 514, 512, 503, 515, 509, 502, 487, 498, 491, 524, 498, 514, 541, 478, 508, 515, 524, 487, 499, 504, 495, 536, 493, 480, 488, 481, 475, 519, 508, 511, 534, 502, 474, 516, 521, 515, 519, 478, 532, 479, 520, 495, 489, 511, 478, 563, 504, 574, 518, 522, 509, 500, 525, 508, 503, 516, 535, 547, 509, 507, 530, 504, 540, 498, 505, 509, 494] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[491, 500, 507, 505, 511, 522, 518, 498, 540, 535, 500, 530, 522, 502, 519, 529, 493, 513, 512, 551, 552, 488, 514, 510, 483, 509, 511, 530, 514, 498, 555, 543, 501, 501, 529, 520, 499, 511, 514, 468, 460, 520, 518, 571, 530, 498, 528, 489, 486, 545, 507, 486, 497, 489, 537, 527, 519, 525, 477, 518, 501, 488, 519, 544, 484, 489, 492, 501, 535, 506, 505, 477, 505, 509, 522, 495, 518, 551, 533, 516, 509, 486, 504, 547, 514, 508, 507, 514, 495, 526, 505, 535, 488, 510, 551, 536, 529, 516, 510, 487, 497, 495, 519, 521, 505, 487, 515, 524, 507, 559, 460, 542, 520, 487, 530, 515, 509, 496, 549, 497, 516, 473, 510, 499, 503, 504, 541, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[497, 523, 496, 522, 476, 519, 528, 518, 539, 500, 524, 552, 539, 477, 514, 520, 545, 479, 520, 527, 501, 520, 476, 525, 482, 480, 499, 486, 521, 512, 471, 515, 518, 516, 497, 526, 501, 468, 502, 527, 509, 464, 535, 496, 509, 514, 499, 504, 529, 505, 479, 526, 532, 503, 500, 509, 535, 549, 516, 543, 511, 525, 516, 509, 574, 506, 483, 499, 483, 517, 512, 531, 544, 525, 490, 522, 528, 523, 521, 536, 535, 480, 502, 484, 536, 494, 523, 483, 467, 517, 518, 530, 537, 521, 544, 479, 540, 471, 554, 485, 505, 557, 540, 523, 535, 556, 516, 510, 512, 475, 487, 487, 510, 516, 535, 516, 496, 558, 521, 468, 502, 471, 512, 495, 535, 503, 528, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[511, 534, 525, 493, 525, 535, 520, 553, 530, 505, 509, 487, 547, 542, 493, 526, 549, 513, 546, 533, 522, 505, 505, 490, 536, 514, 513, 517, 518, 476, 514, 520, 464, 501, 522, 498, 480, 522, 573, 505, 493, 504, 503, 530, 507, 515, 471, 511, 539, 496, 536, 456, 502, 505, 484, 485, 511, 536, 517, 532, 519, 521, 496, 537, 528, 547, 538, 493, 483, 526, 561, 547, 555, 517, 444, 523, 536, 492, 511, 511, 529, 507, 490, 505, 493, 520, 454, 482, 503, 475, 494, 494, 501, 515, 521, 548, 497, 538, 525, 506, 496, 492, 499, 495, 504, 513, 529, 503, 495, 476, 532, 512, 532, 506, 530, 551, 524, 531, 494, 527, 483, 523, 524, 505, 498, 494, 492, 490] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[519, 534, 490, 494, 529, 513, 531, 477, 523, 513, 529, 511, 491, 495, 495, 521, 544, 519, 548, 510, 496, 506, 488, 535, 547, 491, 555, 527, 482, 528, 515, 509, 500, 477, 518, 509, 538, 495, 520, 517, 492, 460, 461, 500, 531, 500, 487, 527, 508, 513, 544, 525, 518, 530, 479, 500, 533, 495, 532, 507, 510, 481, 487, 520, 519, 507, 534, 519, 530, 508, 508, 508, 500, 509, 534, 510, 534, 497, 507, 536, 518, 501, 489, 530, 536, 529, 494, 536, 493, 524, 534, 494, 542, 529, 527, 533, 510, 506, 507, 533, 513, 517, 493, 523, 491, 519, 503, 530, 543, 534, 479, 530, 486, 520, 488, 467, 537, 485, 485, 511, 513, 542, 530, 471, 507, 484, 495, 536] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[539, 511, 547, 515, 546, 538, 476, 475, 545, 535, 502, 532, 538, 517, 533, 501, 518, 515, 513, 521, 522, 480, 520, 499, 530, 540, 514, 481, 530, 543, 493, 500, 508, 532, 522, 521, 486, 508, 532, 499, 506, 518, 536, 537, 490, 512, 523, 530, 492, 498, 532, 538, 463, 479, 534, 493, 512, 538, 480, 530, 526, 514, 512, 503, 515, 509, 502, 487, 498, 491, 524, 498, 514, 541, 478, 508, 515, 524, 487, 499, 504, 495, 536, 493, 480, 488, 481, 475, 519, 508, 511, 534, 502, 474, 516, 521, 515, 519, 478, 532, 479, 520, 495, 489, 511, 478, 563, 504, 574, 518, 522, 509, 500, 525, 508, 503, 516, 535, 547, 509, 507, 530, 504, 540, 498, 505, 509, 494] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[514, 490, 502, 545, 512, 538, 512, 519, 498, 562, 530, 506, 535, 521, 500, 513, 514, 519, 493, 514, 537, 486, 536, 503, 518, 502, 527, 542, 477, 543, 512, 498, 484, 475, 551, 500, 530, 514, 515, 489, 472, 530, 543, 503, 514, 533, 543, 521, 493, 533, 498, 458, 511, 482, 475, 499, 495, 505, 555, 507, 491, 516, 523, 530, 475, 516, 533, 574, 494, 527, 504, 531, 512, 557, 524, 522, 480, 536, 537, 505, 503, 510, 487, 487, 505, 500, 536, 478, 542, 493, 491, 482, 496, 517, 493, 485, 518, 526, 510, 498, 532, 519, 541, 522, 505, 534, 513, 518, 509, 512, 542, 505, 506, 506, 533, 469, 489, 494, 512, 541, 514, 498, 494, 532, 477, 512, 477, 540] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[481, 522, 497, 471, 523, 512, 547, 498, 493, 504, 493, 518, 481, 491, 519, 502, 547, 481, 498, 511, 553, 513, 498, 529, 542, 549, 540, 516, 517, 506, 528, 532, 463, 488, 541, 521, 525, 524, 485, 491, 503, 518, 521, 462, 529, 529, 527, 539, 468, 548, 556, 541, 489, 469, 513, 505, 537, 529, 518, 499, 505, 497, 510, 539, 535, 500, 526, 484, 479, 464, 524, 485, 521, 490, 527, 512, 531, 550, 477, 521, 541, 522, 506, 528, 554, 515, 498, 476, 510, 548, 538, 534, 477, 521, 507, 500, 519, 510, 495, 526, 512, 503, 480, 499, 547, 532, 532, 486, 509, 521, 504, 513, 528, 491, 512, 508, 492, 510, 534, 529, 484, 519, 477, 531, 489, 529, 531, 482] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[519, 534, 490, 494, 529, 513, 531, 477, 523, 513, 529, 511, 491, 495, 495, 521, 544, 519, 548, 510, 496, 506, 488, 535, 547, 491, 555, 527, 482, 528, 515, 509, 500, 477, 518, 509, 538, 495, 520, 517, 492, 460, 461, 500, 531, 500, 487, 527, 508, 513, 544, 525, 518, 530, 479, 500, 533, 495, 532, 507, 510, 481, 487, 520, 519, 507, 534, 519, 530, 508, 508, 508, 500, 509, 534, 510, 534, 497, 507, 536, 518, 501, 489, 530, 536, 529, 494, 536, 493, 524, 534, 494, 542, 529, 527, 533, 510, 506, 507, 533, 513, 517, 493, 523, 491, 519, 503, 530, 543, 534, 479, 530, 486, 520, 488, 467, 537, 485, 485, 511, 513, 542, 530, 471, 507, 484, 495, 536] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[545, 515, 541, 506, 534, 539, 493, 481, 539, 499, 479, 488, 486, 525, 506, 512, 529, 492, 469, 487, 507, 527, 520, 514, 554, 562, 490, 493, 514, 508, 528, 506, 505, 519, 545, 498, 519, 543, 573, 526, 502, 528, 469, 522, 502, 520, 524, 474, 491, 513, 505, 571, 471, 486, 518, 551, 513, 529, 567, 527, 507, 499, 514, 503, 476, 489, 505, 485, 577, 492, 521, 493, 487, 530, 517, 481, 530, 544, 520, 515, 487, 505, 473, 559, 504, 500, 494, 496, 528, 524, 533, 496, 505, 501, 510, 542, 487, 535, 509, 505, 515, 530, 518, 496, 525, 506, 510, 513, 491, 478, 506, 513, 522, 504, 482, 521, 514, 534, 495, 495, 528, 523, 511, 512, 514, 477, 510, 516] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[497, 523, 496, 522, 476, 519, 528, 518, 539, 500, 524, 552, 539, 477, 514, 520, 545, 479, 520, 527, 501, 520, 476, 525, 482, 480, 499, 486, 521, 512, 471, 515, 518, 516, 497, 526, 501, 468, 502, 527, 509, 464, 535, 496, 509, 514, 499, 504, 529, 505, 479, 526, 532, 503, 500, 509, 535, 549, 516, 543, 511, 525, 516, 509, 574, 506, 483, 499, 483, 517, 512, 531, 544, 525, 490, 522, 528, 523, 521, 536, 535, 480, 502, 484, 536, 494, 523, 483, 467, 517, 518, 530, 537, 521, 544, 479, 540, 471, 554, 485, 505, 557, 540, 523, 535, 556, 516, 510, 512, 475, 487, 487, 510, 516, 535, 516, 496, 558, 521, 468, 502, 471, 512, 495, 535, 503, 528, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[491, 500, 507, 505, 511, 483, 518, 498, 540, 535, 500, 530, 522, 502, 519, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 451, 513, 512, 551, 552, 488, 514, 510, 483, 509, 511, 530, 514, 498, 555, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 501, 529, 520, 499, 511, 514, 468, 460, 520, 518, 571, 530, 498, 528, 489, 22, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 545, 507, 486, 497, 489, 537, 527, 519, 525, 477, 518, 501, 488, 519, 544, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 489, 492, 501, 535, 506, 505, 477, 505, 509, 522, 495, 518, 529, 533, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 486, 504, 547, 514, 508, 507, 514, 495, 526, 505, 535, 488, 510, 551, 536, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 516, 510, 487, 497, 495, 519, 521, 505, 487, 507, 524, 507, 559, 460, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 487, 530, 515, 509, 496, 549, 497, 516, 473, 510, 499, 503, 504, 541, 514, 35, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8234, 8188, 8168, 8116, 8274, 8165, 8201] avg=8192.0000 max=8274 max_rank=5 min=8116 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32756 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[519, 534, 490, 494, 529, 465, 531, 477, 523, 513, 529, 511, 491, 495, 495, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 519, 548, 510, 496, 506, 488, 535, 547, 491, 555, 527, 482, 528, 515, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 477, 518, 509, 538, 495, 520, 517, 492, 460, 461, 500, 531, 500, 487, 527, 24, 6, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 513, 544, 525, 518, 530, 479, 500, 533, 495, 532, 507, 510, 481, 487, 520, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 507, 534, 519, 530, 508, 508, 508, 500, 509, 534, 510, 534, 473, 507, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 501, 489, 530, 536, 529, 494, 536, 493, 524, 534, 494, 542, 529, 527, 533, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 506, 507, 533, 513, 517, 493, 523, 491, 519, 491, 530, 543, 534, 479, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 520, 488, 467, 537, 485, 485, 511, 513, 542, 530, 471, 507, 484, 495, 536, 39, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8117, 8248, 8071, 8188, 8236, 8355, 8219, 8102] avg=8192.0000 max=8355 max_rank=5 min=8071 min_rank=2 imbalance_ratio=1.019897 max_load_reduction=57181 max_load_reduction_pct=0.872513 moved_to_echo=32912 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[497, 523, 496, 522, 476, 480, 528, 518, 539, 500, 524, 552, 539, 477, 514, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 479, 520, 527, 501, 520, 476, 525, 482, 480, 499, 486, 521, 512, 471, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 516, 497, 526, 501, 468, 502, 527, 509, 464, 535, 496, 509, 514, 499, 504, 21, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 505, 479, 526, 532, 503, 500, 509, 535, 549, 516, 543, 511, 525, 516, 509, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 574, 506, 483, 499, 483, 517, 512, 531, 544, 525, 490, 522, 528, 502, 521, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 480, 502, 484, 536, 494, 523, 483, 467, 517, 518, 530, 537, 521, 544, 479, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 471, 554, 485, 505, 557, 540, 523, 535, 556, 508, 510, 512, 475, 487, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 516, 535, 516, 496, 558, 521, 468, 502, 471, 512, 495, 535, 503, 528, 498, 35, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8012, 8115, 8290, 8273, 8193, 8245, 8203] avg=8192.0000 max=8290 max_rank=3 min=8012 min_rank=1 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32914 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[481, 522, 497, 471, 523, 474, 547, 498, 493, 504, 493, 518, 481, 491, 519, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 481, 498, 511, 553, 513, 498, 529, 542, 549, 540, 516, 517, 506, 528, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 463, 488, 541, 521, 525, 524, 485, 491, 503, 518, 521, 462, 529, 529, 527, 539, 22, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 548, 556, 541, 489, 469, 513, 505, 537, 529, 518, 499, 505, 497, 510, 539, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 500, 526, 484, 479, 464, 524, 485, 521, 490, 527, 512, 531, 528, 477, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 522, 506, 528, 554, 515, 498, 476, 510, 548, 538, 534, 477, 521, 507, 500, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 510, 495, 526, 512, 503, 480, 499, 547, 532, 524, 486, 509, 521, 504, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 491, 512, 508, 492, 510, 534, 529, 484, 519, 477, 531, 489, 529, 531, 482, 34, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8014, 8313, 8197, 8226, 8104, 8318, 8180, 8184] avg=8192.0000 max=8318 max_rank=5 min=8014 min_rank=0 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32786 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[519, 534, 490, 494, 529, 465, 531, 477, 523, 513, 529, 511, 491, 495, 495, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 519, 548, 510, 496, 506, 488, 535, 547, 491, 555, 527, 482, 528, 515, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 477, 518, 509, 538, 495, 520, 517, 492, 460, 461, 500, 531, 500, 487, 527, 24, 6, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 513, 544, 525, 518, 530, 479, 500, 533, 495, 532, 507, 510, 481, 487, 520, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 507, 534, 519, 530, 508, 508, 508, 500, 509, 534, 510, 534, 473, 507, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 501, 489, 530, 536, 529, 494, 536, 493, 524, 534, 494, 542, 529, 527, 533, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 506, 507, 533, 513, 517, 493, 523, 491, 519, 491, 530, 543, 534, 479, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 520, 488, 467, 537, 485, 485, 511, 513, 542, 530, 471, 507, 484, 495, 536, 39, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8117, 8248, 8071, 8188, 8236, 8355, 8219, 8102] avg=8192.0000 max=8355 max_rank=5 min=8071 min_rank=2 imbalance_ratio=1.019897 max_load_reduction=57181 max_load_reduction_pct=0.872513 moved_to_echo=32912 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[491, 500, 507, 505, 511, 483, 518, 498, 540, 535, 500, 530, 522, 502, 519, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 451, 513, 512, 551, 552, 488, 514, 510, 483, 509, 511, 530, 514, 498, 555, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 501, 529, 520, 499, 511, 514, 468, 460, 520, 518, 571, 530, 498, 528, 489, 22, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 545, 507, 486, 497, 489, 537, 527, 519, 525, 477, 518, 501, 488, 519, 544, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 489, 492, 501, 535, 506, 505, 477, 505, 509, 522, 495, 518, 529, 533, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 486, 504, 547, 514, 508, 507, 514, 495, 526, 505, 535, 488, 510, 551, 536, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 516, 510, 487, 497, 495, 519, 521, 505, 487, 507, 524, 507, 559, 460, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 487, 530, 515, 509, 496, 549, 497, 516, 473, 510, 499, 503, 504, 541, 514, 35, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8234, 8188, 8168, 8116, 8274, 8165, 8201] avg=8192.0000 max=8274 max_rank=5 min=8116 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32756 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[539, 511, 547, 515, 546, 498, 476, 475, 545, 535, 502, 532, 538, 517, 533, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 515, 513, 521, 522, 480, 520, 499, 530, 540, 514, 481, 530, 543, 493, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 532, 522, 521, 486, 508, 532, 499, 506, 518, 536, 537, 490, 512, 523, 530, 21, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 498, 532, 538, 463, 479, 534, 493, 512, 538, 480, 530, 526, 514, 512, 503, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 509, 502, 487, 498, 491, 524, 498, 514, 541, 478, 508, 515, 503, 487, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 495, 536, 493, 480, 488, 481, 475, 519, 508, 511, 534, 502, 474, 516, 521, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 519, 478, 532, 479, 520, 495, 489, 511, 478, 555, 504, 574, 518, 522, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 525, 508, 503, 516, 535, 547, 509, 507, 530, 504, 540, 498, 505, 509, 494, 36, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8310, 8174, 8290, 8147, 8069, 8078, 8198, 8270] avg=8192.0000 max=8310 max_rank=0 min=8069 min_rank=4 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32615 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[545, 515, 541, 506, 534, 499, 493, 481, 539, 499, 479, 488, 486, 525, 506, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 492, 469, 487, 507, 527, 520, 514, 554, 562, 490, 493, 514, 508, 528, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 519, 545, 498, 519, 543, 573, 526, 502, 528, 469, 522, 502, 520, 524, 474, 22, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 513, 505, 571, 471, 486, 518, 551, 513, 529, 567, 527, 507, 499, 514, 503, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 489, 505, 485, 577, 492, 521, 493, 487, 530, 517, 481, 530, 522, 520, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 505, 473, 559, 504, 500, 494, 496, 528, 524, 533, 496, 505, 501, 510, 542, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 535, 509, 505, 515, 530, 518, 496, 525, 506, 502, 513, 491, 478, 506, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 504, 482, 521, 514, 534, 495, 495, 528, 523, 511, 512, 514, 477, 510, 516, 36, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8148, 8154, 8300, 8268, 8140, 8199, 8129, 8198] avg=8192.0000 max=8300 max_rank=2 min=8129 min_rank=6 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32666 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[511, 534, 525, 493, 525, 495, 520, 553, 530, 505, 509, 487, 547, 542, 493, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 513, 546, 533, 522, 505, 505, 490, 536, 514, 513, 517, 518, 476, 514, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 464, 501, 522, 498, 480, 522, 573, 505, 493, 504, 503, 530, 507, 515, 471, 511, 20, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 496, 536, 456, 502, 505, 484, 485, 511, 536, 517, 532, 519, 521, 496, 537, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 547, 538, 493, 483, 526, 561, 547, 555, 517, 444, 523, 536, 472, 511, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 507, 490, 505, 493, 520, 454, 482, 503, 475, 494, 494, 501, 515, 521, 548, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 538, 525, 506, 496, 492, 499, 495, 504, 513, 521, 503, 495, 476, 532, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 506, 530, 551, 524, 531, 494, 527, 483, 523, 524, 505, 498, 494, 492, 490, 36, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8295, 8224, 8128, 8175, 8292, 8074, 8104, 8244] avg=8192.0000 max=8295 max_rank=0 min=8074 min_rank=5 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32714 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[514, 490, 502, 545, 512, 498, 512, 519, 498, 562, 530, 506, 535, 521, 500, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 519, 493, 514, 537, 486, 536, 503, 518, 502, 527, 542, 477, 543, 512, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 475, 551, 500, 530, 514, 515, 489, 472, 530, 543, 503, 514, 533, 543, 521, 21, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 533, 498, 458, 511, 482, 475, 499, 495, 505, 555, 507, 491, 516, 523, 530, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 516, 533, 574, 494, 527, 504, 531, 512, 557, 524, 522, 480, 515, 537, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 510, 487, 487, 505, 500, 536, 478, 542, 493, 491, 482, 496, 517, 493, 485, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 526, 510, 498, 532, 519, 541, 522, 505, 534, 505, 518, 509, 512, 542, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 506, 533, 469, 489, 494, 512, 541, 514, 498, 494, 532, 477, 512, 477, 540, 36, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8257, 8177, 8247, 8074, 8306, 8045, 8296, 8134] avg=8192.0000 max=8306 max_rank=4 min=8045 min_rank=5 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32781 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[497, 523, 496, 522, 476, 480, 528, 518, 539, 500, 524, 552, 539, 477, 514, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 479, 520, 527, 501, 520, 476, 525, 482, 480, 499, 486, 521, 512, 471, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 516, 497, 526, 501, 468, 502, 527, 509, 464, 535, 496, 509, 514, 499, 504, 21, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 505, 479, 526, 532, 503, 500, 509, 535, 549, 516, 543, 511, 525, 516, 509, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 574, 506, 483, 499, 483, 517, 512, 531, 544, 525, 490, 522, 528, 502, 521, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 480, 502, 484, 536, 494, 523, 483, 467, 517, 518, 530, 537, 521, 544, 479, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 471, 554, 485, 505, 557, 540, 523, 535, 556, 508, 510, 512, 475, 487, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 516, 535, 516, 496, 558, 521, 468, 502, 471, 512, 495, 535, 503, 528, 498, 35, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8012, 8115, 8290, 8273, 8193, 8245, 8203] avg=8192.0000 max=8290 max_rank=3 min=8012 min_rank=1 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32914 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[545, 515, 541, 506, 534, 499, 493, 481, 539, 499, 479, 488, 486, 525, 506, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 492, 469, 487, 507, 527, 520, 514, 554, 562, 490, 493, 514, 508, 528, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 519, 545, 498, 519, 543, 573, 526, 502, 528, 469, 522, 502, 520, 524, 474, 22, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 513, 505, 571, 471, 486, 518, 551, 513, 529, 567, 527, 507, 499, 514, 503, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 489, 505, 485, 577, 492, 521, 493, 487, 530, 517, 481, 530, 522, 520, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 505, 473, 559, 504, 500, 494, 496, 528, 524, 533, 496, 505, 501, 510, 542, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 535, 509, 505, 515, 530, 518, 496, 525, 506, 502, 513, 491, 478, 506, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 504, 482, 521, 514, 534, 495, 495, 528, 523, 511, 512, 514, 477, 510, 516, 36, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8148, 8154, 8300, 8268, 8140, 8199, 8129, 8198] avg=8192.0000 max=8300 max_rank=2 min=8129 min_rank=6 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32666 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[3968, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 3968, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[3968, 4096, 4352, 4096, 4096, 4096, 4224, 4096, 3968, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 3968, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[3968, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[3968, 4096, 4352, 4096, 4096, 4096, 4224, 4096, 3968, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 3968, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 3968, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[512, 502, 490, 528, 496, 551, 509, 540, 493, 493, 483, 533, 472, 505, 552, 505, 491, 551, 537, 510, 520, 489, 551, 487, 495, 487, 507, 463, 501, 528, 514, 545, 513, 495, 518, 483, 506, 519, 495, 519, 545, 516, 505, 541, 481, 529, 495, 523, 514, 556, 511, 524, 553, 499, 500, 512, 509, 494, 498, 526, 551, 533, 484, 487, 510, 515, 530, 512, 495, 515, 529, 493, 461, 520, 501, 536, 473, 509, 515, 501, 473, 520, 522, 508, 500, 526, 535, 448, 525, 501, 508, 519, 492, 509, 486, 534, 512, 522, 514, 515, 491, 542, 563, 534, 544, 507, 507, 541, 505, 515, 513, 552, 576, 494, 481, 518, 530, 486, 536, 506, 539, 499, 533, 524, 460, 510, 480, 492] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[543, 513, 551, 500, 496, 535, 544, 518, 495, 533, 545, 511, 460, 526, 522, 502, 515, 496, 525, 538, 469, 526, 516, 498, 523, 485, 488, 516, 523, 487, 522, 493, 514, 516, 531, 510, 531, 498, 526, 515, 492, 522, 552, 527, 511, 503, 512, 491, 504, 475, 486, 491, 519, 487, 516, 513, 497, 572, 524, 517, 529, 531, 528, 514, 548, 548, 517, 515, 501, 480, 523, 510, 491, 508, 526, 540, 529, 497, 509, 506, 494, 513, 495, 504, 509, 533, 508, 520, 547, 513, 495, 535, 493, 510, 468, 482, 488, 515, 541, 522, 518, 521, 542, 500, 447, 503, 504, 509, 531, 525, 497, 516, 501, 479, 545, 489, 507, 531, 495, 521, 489, 491, 513, 510, 539, 469, 531, 512] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[511, 540, 483, 498, 515, 549, 532, 507, 526, 536, 501, 512, 514, 503, 513, 499, 501, 552, 446, 525, 543, 514, 543, 480, 551, 521, 516, 537, 502, 494, 528, 528, 508, 549, 523, 538, 506, 547, 497, 504, 500, 547, 485, 528, 522, 507, 510, 512, 486, 497, 525, 554, 514, 519, 550, 507, 532, 474, 497, 506, 512, 493, 494, 537, 500, 503, 505, 522, 529, 510, 524, 515, 518, 506, 508, 502, 536, 465, 525, 504, 533, 482, 492, 474, 517, 512, 550, 540, 506, 498, 503, 514, 487, 518, 516, 503, 502, 514, 509, 473, 526, 480, 475, 498, 466, 511, 547, 502, 525, 526, 522, 517, 502, 530, 499, 498, 503, 527, 494, 500, 553, 520, 488, 493, 541, 471, 512, 495] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[507, 519, 501, 517, 527, 530, 497, 553, 533, 541, 498, 560, 472, 552, 527, 511, 490, 522, 491, 512, 521, 491, 509, 556, 474, 477, 512, 509, 551, 496, 501, 509, 528, 491, 473, 483, 468, 475, 517, 489, 505, 529, 505, 522, 525, 519, 546, 474, 497, 496, 512, 529, 485, 482, 519, 519, 503, 533, 533, 513, 547, 535, 522, 520, 515, 491, 496, 526, 538, 563, 515, 481, 502, 518, 507, 480, 521, 502, 518, 507, 473, 492, 490, 508, 508, 581, 500, 535, 493, 501, 492, 516, 516, 464, 499, 511, 521, 534, 491, 518, 520, 545, 500, 504, 507, 529, 555, 482, 489, 516, 500, 502, 513, 501, 536, 501, 550, 493, 516, 538, 553, 503, 535, 483, 521, 506, 535, 520] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[530, 516, 505, 512, 475, 537, 531, 489, 508, 538, 555, 503, 548, 509, 539, 487, 514, 520, 523, 535, 478, 471, 526, 513, 504, 522, 472, 507, 547, 536, 532, 497, 494, 508, 517, 500, 519, 526, 510, 504, 507, 486, 505, 532, 499, 491, 558, 501, 507, 531, 499, 491, 488, 519, 530, 476, 475, 532, 470, 509, 494, 525, 520, 525, 482, 472, 474, 519, 496, 525, 467, 508, 531, 540, 527, 480, 516, 473, 516, 539, 516, 491, 530, 525, 554, 545, 539, 546, 505, 500, 516, 499, 479, 484, 501, 483, 532, 512, 503, 511, 516, 532, 507, 501, 500, 507, 510, 496, 546, 548, 487, 511, 512, 527, 526, 487, 487, 523, 538, 541, 507, 518, 505, 549, 504, 525, 534, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[518, 507, 570, 466, 494, 494, 523, 506, 522, 513, 481, 503, 483, 564, 512, 502, 496, 510, 491, 498, 512, 545, 477, 548, 560, 529, 510, 521, 523, 511, 487, 500, 538, 507, 479, 542, 484, 531, 477, 500, 513, 542, 499, 491, 510, 473, 511, 494, 526, 529, 509, 500, 527, 516, 506, 518, 485, 526, 509, 506, 507, 491, 493, 502, 489, 531, 480, 564, 513, 488, 540, 480, 503, 518, 517, 536, 515, 517, 487, 499, 507, 523, 545, 546, 542, 518, 487, 496, 520, 496, 517, 516, 526, 482, 517, 528, 501, 529, 504, 517, 493, 503, 481, 504, 508, 509, 540, 571, 508, 524, 507, 485, 482, 536, 552, 520, 545, 542, 521, 503, 534, 483, 530, 523, 510, 491, 526, 494] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[506, 521, 490, 506, 531, 489, 538, 543, 511, 493, 515, 474, 508, 466, 473, 520, 507, 541, 524, 516, 508, 503, 521, 526, 483, 541, 518, 492, 522, 508, 511, 499, 485, 532, 509, 532, 516, 545, 506, 533, 543, 528, 481, 528, 509, 492, 547, 542, 507, 539, 505, 531, 524, 518, 520, 519, 509, 510, 503, 500, 539, 519, 531, 506, 470, 489, 529, 532, 542, 500, 492, 486, 511, 468, 538, 503, 491, 506, 523, 486, 522, 495, 567, 507, 519, 478, 513, 527, 536, 505, 531, 518, 505, 483, 491, 460, 477, 515, 497, 513, 526, 557, 511, 511, 542, 536, 512, 502, 461, 506, 498, 521, 504, 521, 478, 513, 507, 522, 501, 497, 553, 520, 525, 536, 514, 495, 476, 554] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[512, 502, 490, 528, 496, 551, 509, 540, 493, 493, 483, 533, 472, 505, 552, 505, 491, 551, 537, 510, 520, 489, 551, 487, 495, 487, 507, 463, 501, 528, 514, 545, 513, 495, 518, 483, 506, 519, 495, 519, 545, 516, 505, 541, 481, 529, 495, 523, 514, 556, 511, 524, 553, 499, 500, 512, 509, 494, 498, 526, 551, 533, 484, 487, 510, 515, 530, 512, 495, 515, 529, 493, 461, 520, 501, 536, 473, 509, 515, 501, 473, 520, 522, 508, 500, 526, 535, 448, 525, 501, 508, 519, 492, 509, 486, 534, 512, 522, 514, 515, 491, 542, 563, 534, 544, 507, 507, 541, 505, 515, 513, 552, 576, 494, 481, 518, 530, 486, 536, 506, 539, 499, 533, 524, 460, 510, 480, 492] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[504, 506, 534, 511, 444, 539, 492, 515, 515, 508, 493, 527, 493, 504, 516, 526, 514, 469, 552, 532, 500, 539, 498, 485, 514, 520, 505, 505, 531, 498, 552, 520, 483, 532, 548, 504, 495, 506, 511, 525, 487, 497, 507, 527, 517, 512, 496, 507, 513, 537, 468, 544, 531, 516, 530, 508, 496, 474, 513, 457, 487, 497, 513, 520, 520, 534, 504, 528, 506, 519, 524, 541, 537, 540, 539, 531, 485, 527, 497, 500, 539, 486, 513, 526, 505, 542, 491, 506, 527, 588, 500, 517, 489, 527, 525, 496, 493, 524, 542, 512, 507, 506, 473, 524, 518, 515, 524, 521, 500, 544, 495, 517, 513, 488, 531, 504, 507, 488, 507, 506, 522, 489, 525, 517, 506, 481, 502, 507] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[530, 516, 505, 512, 475, 537, 531, 489, 508, 538, 555, 503, 548, 509, 539, 487, 514, 520, 523, 535, 478, 471, 526, 513, 504, 522, 472, 507, 547, 536, 532, 497, 494, 508, 517, 500, 519, 526, 510, 504, 507, 486, 505, 532, 499, 491, 558, 501, 507, 531, 499, 491, 488, 519, 530, 476, 475, 532, 470, 509, 494, 525, 520, 525, 482, 472, 474, 519, 496, 525, 467, 508, 531, 540, 527, 480, 516, 473, 516, 539, 516, 491, 530, 525, 554, 545, 539, 546, 505, 500, 516, 499, 479, 484, 501, 483, 532, 512, 503, 511, 516, 532, 507, 501, 500, 507, 510, 496, 546, 548, 487, 511, 512, 527, 526, 487, 487, 523, 538, 541, 507, 518, 505, 549, 504, 525, 534, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[506, 521, 490, 506, 531, 489, 538, 543, 511, 493, 515, 474, 508, 466, 473, 520, 507, 541, 524, 516, 508, 503, 521, 526, 483, 541, 518, 492, 522, 508, 511, 499, 485, 532, 509, 532, 516, 545, 506, 533, 543, 528, 481, 528, 509, 492, 547, 542, 507, 539, 505, 531, 524, 518, 520, 519, 509, 510, 503, 500, 539, 519, 531, 506, 470, 489, 529, 532, 542, 500, 492, 486, 511, 468, 538, 503, 491, 506, 523, 486, 522, 495, 567, 507, 519, 478, 513, 527, 536, 505, 531, 518, 505, 483, 491, 460, 477, 515, 497, 513, 526, 557, 511, 511, 542, 536, 512, 502, 461, 506, 498, 521, 504, 521, 478, 513, 507, 522, 501, 497, 553, 520, 525, 536, 514, 495, 476, 554] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[511, 540, 483, 498, 515, 549, 532, 507, 526, 536, 501, 512, 514, 503, 513, 499, 501, 552, 446, 525, 543, 514, 543, 480, 551, 521, 516, 537, 502, 494, 528, 528, 508, 549, 523, 538, 506, 547, 497, 504, 500, 547, 485, 528, 522, 507, 510, 512, 486, 497, 525, 554, 514, 519, 550, 507, 532, 474, 497, 506, 512, 493, 494, 537, 500, 503, 505, 522, 529, 510, 524, 515, 518, 506, 508, 502, 536, 465, 525, 504, 533, 482, 492, 474, 517, 512, 550, 540, 506, 498, 503, 514, 487, 518, 516, 503, 502, 514, 509, 473, 526, 480, 475, 498, 466, 511, 547, 502, 525, 526, 522, 517, 502, 530, 499, 498, 503, 527, 494, 500, 553, 520, 488, 493, 541, 471, 512, 495] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[507, 519, 501, 517, 527, 511, 497, 553, 533, 541, 498, 560, 472, 552, 527, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 522, 491, 512, 521, 491, 509, 556, 474, 477, 512, 509, 548, 496, 501, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 491, 473, 483, 468, 475, 517, 489, 505, 529, 505, 522, 525, 519, 546, 474, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 496, 512, 529, 485, 482, 519, 519, 503, 533, 533, 513, 547, 535, 522, 520, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 491, 496, 526, 538, 563, 515, 481, 502, 518, 507, 480, 521, 502, 518, 507, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 492, 490, 508, 508, 581, 500, 535, 493, 501, 492, 516, 516, 464, 499, 511, 12, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 534, 491, 518, 520, 545, 500, 504, 507, 529, 555, 482, 489, 502, 500, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 501, 536, 501, 550, 493, 516, 538, 542, 503, 535, 483, 521, 506, 535, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8326, 8118, 8054, 8246, 8201, 8099, 8199, 8293] avg=8192.0000 max=8326 max_rank=0 min=8054 min_rank=2 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32792 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[543, 513, 551, 500, 496, 512, 544, 518, 495, 533, 545, 511, 460, 526, 522, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 496, 525, 538, 469, 526, 516, 498, 523, 485, 488, 516, 509, 487, 522, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 516, 531, 510, 531, 498, 526, 515, 492, 522, 552, 527, 511, 503, 512, 491, 5, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 475, 486, 491, 519, 487, 516, 513, 497, 572, 524, 517, 529, 531, 528, 514, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 548, 517, 515, 501, 480, 523, 510, 491, 508, 526, 540, 529, 497, 509, 506, 23, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 513, 495, 504, 509, 533, 508, 520, 547, 513, 495, 535, 493, 510, 468, 482, 17, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 515, 541, 522, 518, 521, 542, 500, 447, 503, 504, 509, 531, 501, 497, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 479, 545, 489, 507, 531, 495, 521, 471, 491, 513, 510, 539, 469, 531, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8271, 8106, 8262, 8211, 8278, 8149, 8155, 8104] avg=8192.0000 max=8278 max_rank=4 min=8104 min_rank=7 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32686 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[506, 521, 490, 506, 531, 471, 538, 543, 511, 493, 515, 474, 508, 466, 473, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 541, 524, 516, 508, 503, 521, 526, 483, 541, 518, 492, 520, 508, 511, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 532, 509, 532, 516, 545, 506, 533, 543, 528, 481, 528, 509, 492, 547, 542, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 539, 505, 531, 524, 518, 520, 519, 509, 510, 503, 500, 539, 519, 531, 506, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 489, 529, 532, 542, 500, 492, 486, 511, 468, 538, 503, 491, 506, 523, 486, 18, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 495, 567, 507, 519, 478, 513, 527, 536, 505, 531, 518, 505, 483, 491, 460, 12, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 515, 497, 513, 526, 557, 511, 511, 542, 536, 512, 502, 461, 492, 498, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 521, 478, 513, 507, 522, 501, 497, 542, 520, 525, 536, 514, 495, 476, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8066, 8218, 8332, 8281, 8086, 8177, 8171, 8205] avg=8192.0000 max=8332 max_rank=2 min=8066 min_rank=0 imbalance_ratio=1.017090 max_load_reduction=57204 max_load_reduction_pct=0.872864 moved_to_echo=32639 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[530, 516, 505, 512, 475, 518, 531, 489, 508, 538, 555, 503, 548, 509, 539, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 520, 523, 535, 478, 471, 526, 513, 504, 522, 472, 507, 544, 536, 532, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 508, 517, 500, 519, 526, 510, 504, 507, 486, 505, 532, 499, 491, 558, 501, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 531, 499, 491, 488, 519, 530, 476, 475, 532, 470, 509, 494, 525, 520, 525, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 472, 474, 519, 496, 525, 467, 508, 531, 540, 527, 480, 516, 473, 516, 539, 19, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 491, 530, 525, 554, 545, 539, 546, 505, 500, 516, 499, 479, 484, 501, 483, 13, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 512, 503, 511, 516, 532, 507, 501, 500, 507, 510, 496, 546, 532, 487, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 527, 526, 487, 487, 523, 538, 541, 497, 518, 505, 549, 504, 525, 534, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8263, 8194, 8161, 8092, 8087, 8234, 8203, 8302] avg=8192.0000 max=8302 max_rank=7 min=8087 min_rank=4 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32826 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[518, 507, 570, 466, 494, 476, 523, 506, 522, 513, 481, 503, 483, 564, 512, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 510, 491, 498, 512, 545, 477, 548, 560, 529, 510, 521, 521, 511, 487, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 507, 479, 542, 484, 531, 477, 500, 513, 542, 499, 491, 510, 473, 511, 494, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 529, 509, 500, 527, 516, 506, 518, 485, 526, 509, 506, 507, 491, 493, 502, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 531, 480, 564, 513, 488, 540, 480, 503, 518, 517, 536, 515, 517, 487, 499, 18, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 523, 545, 546, 542, 518, 487, 496, 520, 496, 517, 516, 526, 482, 517, 528, 13, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 529, 504, 517, 493, 503, 481, 504, 508, 509, 540, 571, 508, 509, 507, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 536, 552, 520, 545, 542, 521, 503, 523, 483, 530, 523, 510, 491, 526, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8140, 8216, 8095, 8151, 8197, 8287, 8169, 8281] avg=8192.0000 max=8287 max_rank=5 min=8095 min_rank=2 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32934 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[504, 506, 534, 511, 444, 519, 492, 515, 515, 508, 493, 527, 493, 504, 516, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 469, 552, 532, 500, 539, 498, 485, 514, 520, 505, 505, 528, 498, 552, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 532, 548, 504, 495, 506, 511, 525, 487, 497, 507, 527, 517, 512, 496, 507, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 537, 468, 544, 531, 516, 530, 508, 496, 474, 513, 457, 487, 497, 513, 520, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 534, 504, 528, 506, 519, 524, 541, 537, 540, 539, 531, 485, 527, 497, 500, 20, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 486, 513, 526, 505, 542, 491, 506, 527, 588, 500, 517, 489, 527, 525, 496, 13, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 524, 542, 512, 507, 506, 473, 524, 518, 515, 524, 521, 500, 528, 495, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 488, 531, 504, 507, 488, 507, 506, 512, 489, 525, 517, 506, 481, 502, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8107, 8231, 8158, 8105, 8355, 8298, 8199, 8083] avg=8192.0000 max=8355 max_rank=4 min=8083 min_rank=7 imbalance_ratio=1.019897 max_load_reduction=57181 max_load_reduction_pct=0.872513 moved_to_echo=32935 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[512, 502, 490, 528, 496, 531, 509, 540, 493, 493, 483, 533, 472, 505, 552, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 551, 537, 510, 520, 489, 551, 487, 495, 487, 507, 463, 499, 528, 514, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 495, 518, 483, 506, 519, 495, 519, 545, 516, 505, 541, 481, 529, 495, 523, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 556, 511, 524, 553, 499, 500, 512, 509, 494, 498, 526, 551, 533, 484, 487, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 515, 530, 512, 495, 515, 529, 493, 461, 520, 501, 536, 473, 509, 515, 501, 20, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 520, 522, 508, 500, 526, 535, 448, 525, 501, 508, 519, 492, 509, 486, 534, 12, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 522, 514, 515, 491, 542, 563, 534, 544, 507, 507, 541, 505, 501, 513, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 576, 494, 481, 518, 530, 486, 536, 506, 528, 499, 533, 524, 460, 510, 480, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8144, 8174, 8187, 8252, 8137, 8126, 8363, 8153] avg=8192.0000 max=8363 max_rank=6 min=8126 min_rank=5 imbalance_ratio=1.020874 max_load_reduction=57173 max_load_reduction_pct=0.872391 moved_to_echo=32779 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[507, 519, 501, 517, 527, 511, 497, 553, 533, 541, 498, 560, 472, 552, 527, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 522, 491, 512, 521, 491, 509, 556, 474, 477, 512, 509, 548, 496, 501, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 491, 473, 483, 468, 475, 517, 489, 505, 529, 505, 522, 525, 519, 546, 474, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 496, 512, 529, 485, 482, 519, 519, 503, 533, 533, 513, 547, 535, 522, 520, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 491, 496, 526, 538, 563, 515, 481, 502, 518, 507, 480, 521, 502, 518, 507, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 492, 490, 508, 508, 581, 500, 535, 493, 501, 492, 516, 516, 464, 499, 511, 12, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 534, 491, 518, 520, 545, 500, 504, 507, 529, 555, 482, 489, 502, 500, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 501, 536, 501, 550, 493, 516, 538, 542, 503, 535, 483, 521, 506, 535, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8326, 8118, 8054, 8246, 8201, 8099, 8199, 8293] avg=8192.0000 max=8326 max_rank=0 min=8054 min_rank=2 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32792 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[506, 521, 490, 506, 531, 471, 538, 543, 511, 493, 515, 474, 508, 466, 473, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 541, 524, 516, 508, 503, 521, 526, 483, 541, 518, 492, 520, 508, 511, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 532, 509, 532, 516, 545, 506, 533, 543, 528, 481, 528, 509, 492, 547, 542, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 539, 505, 531, 524, 518, 520, 519, 509, 510, 503, 500, 539, 519, 531, 506, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 489, 529, 532, 542, 500, 492, 486, 511, 468, 538, 503, 491, 506, 523, 486, 18, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 495, 567, 507, 519, 478, 513, 527, 536, 505, 531, 518, 505, 483, 491, 460, 12, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 515, 497, 513, 526, 557, 511, 511, 542, 536, 512, 502, 461, 492, 498, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 521, 478, 513, 507, 522, 501, 497, 542, 520, 525, 536, 514, 495, 476, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8066, 8218, 8332, 8281, 8086, 8177, 8171, 8205] avg=8192.0000 max=8332 max_rank=2 min=8066 min_rank=0 imbalance_ratio=1.017090 max_load_reduction=57204 max_load_reduction_pct=0.872864 moved_to_echo=32639 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[511, 540, 483, 498, 515, 529, 532, 507, 526, 536, 501, 512, 514, 503, 513, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 552, 446, 525, 543, 514, 543, 480, 551, 521, 516, 537, 500, 494, 528, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 549, 523, 538, 506, 547, 497, 504, 500, 547, 485, 528, 522, 507, 510, 512, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 497, 525, 554, 514, 519, 550, 507, 532, 474, 497, 506, 512, 493, 494, 537, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 503, 505, 522, 529, 510, 524, 515, 518, 506, 508, 502, 536, 465, 525, 504, 20, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 482, 492, 474, 517, 512, 550, 540, 506, 498, 503, 514, 487, 518, 516, 503, 13, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 514, 509, 473, 526, 480, 475, 498, 466, 511, 547, 502, 525, 510, 522, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 530, 499, 498, 503, 527, 494, 500, 542, 520, 488, 493, 541, 471, 512, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 8279, 8287, 8198, 8195, 8166, 8077, 8115] avg=8192.0000 max=8287 max_rank=2 min=8077 min_rank=6 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32553 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[504, 506, 534, 511, 444, 519, 492, 515, 515, 508, 493, 527, 493, 504, 516, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 469, 552, 532, 500, 539, 498, 485, 514, 520, 505, 505, 528, 498, 552, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 532, 548, 504, 495, 506, 511, 525, 487, 497, 507, 527, 517, 512, 496, 507, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 537, 468, 544, 531, 516, 530, 508, 496, 474, 513, 457, 487, 497, 513, 520, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 534, 504, 528, 506, 519, 524, 541, 537, 540, 539, 531, 485, 527, 497, 500, 20, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 486, 513, 526, 505, 542, 491, 506, 527, 588, 500, 517, 489, 527, 525, 496, 13, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 524, 542, 512, 507, 506, 473, 524, 518, 515, 524, 521, 500, 528, 495, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 488, 531, 504, 507, 488, 507, 506, 512, 489, 525, 517, 506, 481, 502, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8107, 8231, 8158, 8105, 8355, 8298, 8199, 8083] avg=8192.0000 max=8355 max_rank=4 min=8083 min_rank=7 imbalance_ratio=1.019897 max_load_reduction=57181 max_load_reduction_pct=0.872513 moved_to_echo=32935 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[518, 507, 570, 466, 494, 476, 523, 506, 522, 513, 481, 503, 483, 564, 512, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 510, 491, 498, 512, 545, 477, 548, 560, 529, 510, 521, 521, 511, 487, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 507, 479, 542, 484, 531, 477, 500, 513, 542, 499, 491, 510, 473, 511, 494, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 529, 509, 500, 527, 516, 506, 518, 485, 526, 509, 506, 507, 491, 493, 502, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 531, 480, 564, 513, 488, 540, 480, 503, 518, 517, 536, 515, 517, 487, 499, 18, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 523, 545, 546, 542, 518, 487, 496, 520, 496, 517, 516, 526, 482, 517, 528, 13, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 529, 504, 517, 493, 503, 481, 504, 508, 509, 540, 571, 508, 509, 507, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 536, 552, 520, 545, 542, 521, 503, 523, 483, 530, 523, 510, 491, 526, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8140, 8216, 8095, 8151, 8197, 8287, 8169, 8281] avg=8192.0000 max=8287 max_rank=5 min=8095 min_rank=2 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32934 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[530, 516, 505, 512, 475, 518, 531, 489, 508, 538, 555, 503, 548, 509, 539, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 520, 523, 535, 478, 471, 526, 513, 504, 522, 472, 507, 544, 536, 532, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 508, 517, 500, 519, 526, 510, 504, 507, 486, 505, 532, 499, 491, 558, 501, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 531, 499, 491, 488, 519, 530, 476, 475, 532, 470, 509, 494, 525, 520, 525, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 472, 474, 519, 496, 525, 467, 508, 531, 540, 527, 480, 516, 473, 516, 539, 19, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 491, 530, 525, 554, 545, 539, 546, 505, 500, 516, 499, 479, 484, 501, 483, 13, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 512, 503, 511, 516, 532, 507, 501, 500, 507, 510, 496, 546, 532, 487, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 527, 526, 487, 487, 523, 538, 541, 497, 518, 505, 549, 504, 525, 534, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8263, 8194, 8161, 8092, 8087, 8234, 8203, 8302] avg=8192.0000 max=8302 max_rank=7 min=8087 min_rank=4 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32826 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 3968, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4352, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 3968, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 3968, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4352, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[488, 503, 484, 552, 531, 501, 457, 527, 502, 493, 496, 536, 523, 514, 521, 476, 507, 492, 447, 528, 543, 538, 515, 504, 541, 525, 535, 483, 539, 526, 532, 516, 509, 507, 552, 546, 480, 492, 541, 511, 529, 524, 497, 499, 502, 483, 486, 524, 490, 507, 531, 517, 527, 536, 523, 503, 546, 509, 464, 531, 466, 498, 542, 559, 496, 540, 507, 501, 523, 522, 492, 558, 518, 527, 487, 483, 531, 520, 523, 525, 496, 526, 534, 533, 499, 453, 471, 521, 474, 515, 520, 470, 527, 470, 504, 523, 513, 515, 496, 521, 551, 503, 514, 528, 490, 493, 511, 516, 480, 525, 505, 508, 503, 482, 498, 514, 517, 493, 545, 538, 492, 511, 487, 549, 497, 518, 585, 543] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[500, 539, 493, 501, 541, 548, 523, 518, 532, 518, 517, 549, 479, 506, 571, 479, 516, 583, 497, 529, 479, 531, 518, 518, 506, 509, 507, 506, 493, 521, 487, 474, 539, 497, 505, 538, 513, 467, 516, 544, 521, 483, 462, 498, 546, 521, 507, 512, 528, 462, 490, 543, 498, 504, 485, 506, 520, 495, 525, 520, 515, 507, 513, 480, 524, 501, 500, 538, 505, 505, 508, 513, 510, 535, 497, 513, 515, 511, 501, 516, 502, 524, 511, 526, 506, 519, 530, 514, 534, 542, 511, 508, 499, 516, 510, 495, 533, 516, 541, 507, 511, 503, 511, 512, 512, 523, 488, 506, 464, 526, 486, 483, 491, 470, 529, 489, 499, 495, 538, 538, 556, 505, 539, 503, 536, 503, 495, 541] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[513, 550, 485, 534, 509, 481, 506, 481, 548, 476, 546, 518, 519, 518, 513, 502, 516, 528, 513, 502, 531, 522, 518, 510, 484, 538, 496, 514, 515, 521, 485, 497, 491, 502, 463, 515, 502, 512, 508, 544, 525, 526, 509, 502, 543, 531, 505, 484, 518, 534, 498, 508, 563, 492, 537, 478, 543, 502, 532, 517, 470, 544, 491, 474, 501, 506, 522, 569, 539, 470, 509, 495, 496, 492, 493, 498, 542, 496, 525, 532, 535, 512, 536, 515, 549, 523, 507, 517, 506, 475, 504, 513, 533, 553, 530, 505, 488, 507, 526, 496, 513, 504, 506, 530, 514, 514, 481, 506, 482, 464, 556, 530, 567, 488, 520, 482, 528, 508, 509, 480, 489, 522, 511, 474, 557, 505, 519, 510] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[489, 475, 523, 512, 491, 464, 511, 531, 526, 546, 509, 541, 516, 533, 510, 523, 548, 544, 495, 532, 554, 514, 545, 485, 506, 546, 493, 518, 518, 530, 543, 501, 532, 497, 498, 523, 499, 537, 510, 479, 508, 469, 486, 556, 516, 511, 503, 523, 504, 501, 546, 501, 483, 486, 513, 515, 482, 523, 528, 536, 508, 481, 576, 496, 518, 494, 519, 499, 488, 487, 530, 481, 531, 494, 511, 529, 517, 519, 535, 527, 488, 499, 499, 543, 524, 492, 516, 503, 508, 458, 528, 465, 489, 490, 536, 517, 528, 475, 488, 529, 520, 491, 489, 483, 508, 534, 512, 544, 533, 525, 513, 523, 541, 484, 495, 483, 508, 527, 504, 519, 563, 515, 512, 491, 507, 530, 535, 495] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[553, 533, 525, 467, 527, 554, 538, 502, 532, 486, 496, 530, 518, 540, 502, 520, 544, 486, 520, 528, 519, 475, 508, 495, 534, 514, 519, 497, 503, 499, 489, 549, 501, 482, 459, 539, 495, 537, 479, 551, 497, 528, 541, 488, 538, 515, 511, 508, 488, 486, 506, 471, 519, 526, 510, 525, 546, 480, 489, 515, 518, 520, 566, 501, 548, 498, 467, 505, 526, 546, 505, 554, 502, 526, 526, 498, 484, 511, 542, 545, 493, 515, 489, 496, 508, 487, 515, 488, 506, 530, 519, 497, 498, 494, 511, 480, 548, 479, 491, 496, 492, 566, 538, 504, 500, 491, 512, 515, 495, 526, 538, 555, 487, 510, 519, 533, 477, 528, 519, 526, 540, 522, 480, 514, 519, 480, 515, 475] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[495, 507, 489, 494, 567, 505, 502, 502, 498, 538, 505, 515, 503, 476, 524, 511, 511, 508, 510, 538, 516, 488, 517, 490, 517, 478, 540, 529, 537, 536, 539, 487, 478, 499, 506, 550, 529, 509, 500, 522, 528, 524, 538, 521, 550, 456, 513, 543, 525, 546, 502, 499, 502, 511, 494, 529, 490, 511, 507, 496, 556, 498, 506, 488, 521, 527, 480, 515, 526, 525, 524, 515, 522, 506, 535, 501, 492, 517, 514, 544, 524, 520, 507, 502, 498, 491, 516, 512, 522, 498, 496, 514, 473, 525, 475, 558, 542, 510, 494, 469, 520, 514, 497, 480, 529, 533, 516, 505, 536, 518, 498, 516, 470, 521, 516, 552, 487, 513, 541, 474, 514, 527, 467, 513, 538, 520, 505, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[486, 465, 490, 483, 548, 516, 525, 538, 534, 552, 501, 498, 520, 532, 499, 520, 495, 496, 500, 506, 536, 517, 540, 531, 527, 508, 526, 504, 516, 531, 512, 528, 509, 526, 485, 540, 531, 511, 526, 511, 508, 509, 510, 496, 538, 535, 514, 484, 502, 554, 520, 516, 497, 502, 508, 521, 533, 503, 554, 521, 491, 500, 515, 483, 488, 511, 564, 520, 497, 495, 507, 557, 493, 517, 512, 543, 509, 476, 511, 510, 519, 491, 468, 532, 509, 510, 503, 546, 513, 493, 471, 521, 507, 468, 534, 510, 501, 501, 515, 488, 497, 499, 519, 529, 518, 540, 531, 530, 548, 502, 524, 512, 493, 489, 510, 528, 480, 537, 496, 505, 509, 509, 502, 493, 498, 492, 494, 509] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[500, 539, 493, 501, 541, 548, 523, 518, 532, 518, 517, 549, 479, 506, 571, 479, 516, 583, 497, 529, 479, 531, 518, 518, 506, 509, 507, 506, 493, 521, 487, 474, 539, 497, 505, 538, 513, 467, 516, 544, 521, 483, 462, 498, 546, 521, 507, 512, 528, 462, 490, 543, 498, 504, 485, 506, 520, 495, 525, 520, 515, 507, 513, 480, 524, 501, 500, 538, 505, 505, 508, 513, 510, 535, 497, 513, 515, 511, 501, 516, 502, 524, 511, 526, 506, 519, 530, 514, 534, 542, 511, 508, 499, 516, 510, 495, 533, 516, 541, 507, 511, 503, 511, 512, 512, 523, 488, 506, 464, 526, 486, 483, 491, 470, 529, 489, 499, 495, 538, 538, 556, 505, 539, 503, 536, 503, 495, 541] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[565, 504, 522, 503, 530, 522, 516, 488, 537, 534, 521, 516, 483, 491, 484, 490, 538, 509, 527, 529, 527, 471, 537, 507, 508, 492, 535, 543, 485, 517, 526, 533, 496, 565, 513, 512, 491, 502, 526, 526, 483, 507, 499, 523, 516, 490, 493, 519, 485, 511, 507, 457, 516, 494, 538, 494, 532, 547, 518, 510, 488, 506, 527, 491, 462, 512, 496, 503, 480, 494, 522, 533, 488, 513, 542, 505, 522, 553, 510, 519, 516, 506, 482, 499, 507, 500, 472, 465, 515, 528, 477, 484, 554, 519, 497, 515, 526, 504, 490, 498, 515, 519, 471, 548, 527, 520, 530, 514, 495, 538, 501, 542, 535, 532, 548, 511, 547, 510, 503, 510, 516, 531, 502, 527, 513, 535, 515, 501] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[486, 465, 490, 483, 530, 516, 525, 538, 534, 552, 501, 498, 520, 532, 499, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 496, 500, 506, 479, 517, 540, 531, 527, 508, 526, 504, 516, 531, 512, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 526, 485, 540, 531, 511, 526, 511, 508, 509, 510, 496, 538, 535, 514, 484, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 554, 520, 516, 497, 502, 508, 521, 533, 503, 554, 521, 491, 500, 515, 483, 14, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 511, 564, 520, 497, 495, 507, 557, 493, 517, 512, 543, 509, 476, 511, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 491, 468, 532, 509, 510, 503, 546, 513, 493, 471, 521, 507, 468, 534, 510, 57, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 501, 515, 488, 497, 499, 519, 529, 518, 540, 531, 530, 548, 502, 524, 512, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 489, 510, 528, 480, 537, 496, 505, 506, 509, 502, 493, 498, 492, 494, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8216, 8238, 8238, 8188, 8170, 8256, 8041] avg=8192.0000 max=8256 max_rank=6 min=8041 min_rank=7 imbalance_ratio=1.007812 max_load_reduction=57280 max_load_reduction_pct=0.874023 moved_to_echo=32655 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[495, 507, 489, 494, 548, 505, 502, 502, 498, 538, 505, 515, 503, 476, 524, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 508, 510, 538, 462, 488, 517, 490, 517, 478, 540, 529, 537, 536, 539, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 499, 506, 550, 529, 509, 500, 522, 528, 524, 538, 521, 550, 456, 513, 543, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 546, 502, 499, 502, 511, 494, 529, 490, 511, 507, 496, 556, 498, 506, 488, 15, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 527, 480, 515, 526, 525, 524, 515, 522, 506, 535, 501, 492, 517, 514, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 520, 507, 502, 498, 491, 516, 512, 522, 498, 496, 514, 473, 525, 475, 558, 54, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 510, 494, 469, 520, 514, 497, 480, 529, 533, 516, 505, 536, 518, 498, 516, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 521, 516, 552, 487, 513, 541, 474, 511, 527, 467, 513, 538, 520, 505, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8112, 8187, 8271, 8179, 8241, 8204, 8179, 8163] avg=8192.0000 max=8271 max_rank=2 min=8112 min_rank=0 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32787 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[488, 503, 484, 552, 531, 501, 457, 527, 502, 493, 496, 536, 523, 514, 521, 476, 507, 492, 447, 528, 543, 538, 515, 504, 541, 525, 535, 483, 539, 526, 532, 516, 509, 507, 552, 546, 480, 492, 541, 511, 529, 524, 497, 499, 502, 483, 486, 524, 490, 507, 531, 517, 527, 536, 523, 503, 546, 509, 464, 531, 466, 498, 542, 559, 496, 540, 507, 501, 523, 522, 492, 558, 518, 527, 487, 483, 531, 520, 523, 525, 496, 526, 534, 533, 499, 453, 471, 521, 474, 515, 520, 470, 527, 470, 504, 523, 513, 515, 496, 521, 551, 503, 514, 528, 490, 493, 511, 516, 480, 525, 505, 508, 503, 482, 498, 514, 517, 493, 545, 538, 492, 511, 487, 549, 497, 518, 585, 543] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[486, 465, 490, 483, 548, 516, 525, 538, 534, 552, 501, 498, 520, 532, 499, 520, 495, 496, 500, 506, 536, 517, 540, 531, 527, 508, 526, 504, 516, 531, 512, 528, 509, 526, 485, 540, 531, 511, 526, 511, 508, 509, 510, 496, 538, 535, 514, 484, 502, 554, 520, 516, 497, 502, 508, 521, 533, 503, 554, 521, 491, 500, 515, 483, 488, 511, 564, 520, 497, 495, 507, 557, 493, 517, 512, 543, 509, 476, 511, 510, 519, 491, 468, 532, 509, 510, 503, 546, 513, 493, 471, 521, 507, 468, 534, 510, 501, 501, 515, 488, 497, 499, 519, 529, 518, 540, 531, 530, 548, 502, 524, 512, 493, 489, 510, 528, 480, 537, 496, 505, 509, 509, 502, 493, 498, 492, 494, 509] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[489, 475, 523, 512, 491, 464, 511, 531, 526, 546, 509, 541, 516, 533, 510, 523, 548, 544, 495, 532, 554, 514, 545, 485, 506, 546, 493, 518, 518, 530, 543, 501, 532, 497, 498, 523, 499, 537, 510, 479, 508, 469, 486, 556, 516, 511, 503, 523, 504, 501, 546, 501, 483, 486, 513, 515, 482, 523, 528, 536, 508, 481, 576, 496, 518, 494, 519, 499, 488, 487, 530, 481, 531, 494, 511, 529, 517, 519, 535, 527, 488, 499, 499, 543, 524, 492, 516, 503, 508, 458, 528, 465, 489, 490, 536, 517, 528, 475, 488, 529, 520, 491, 489, 483, 508, 534, 512, 544, 533, 525, 513, 523, 541, 484, 495, 483, 508, 527, 504, 519, 563, 515, 512, 491, 507, 530, 535, 495] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[553, 533, 525, 467, 527, 554, 538, 502, 532, 486, 496, 530, 518, 540, 502, 520, 544, 486, 520, 528, 519, 475, 508, 495, 534, 514, 519, 497, 503, 499, 489, 549, 501, 482, 459, 539, 495, 537, 479, 551, 497, 528, 541, 488, 538, 515, 511, 508, 488, 486, 506, 471, 519, 526, 510, 525, 546, 480, 489, 515, 518, 520, 566, 501, 548, 498, 467, 505, 526, 546, 505, 554, 502, 526, 526, 498, 484, 511, 542, 545, 493, 515, 489, 496, 508, 487, 515, 488, 506, 530, 519, 497, 498, 494, 511, 480, 548, 479, 491, 496, 492, 566, 538, 504, 500, 491, 512, 515, 495, 526, 538, 555, 487, 510, 519, 533, 477, 528, 519, 526, 540, 522, 480, 514, 519, 480, 515, 475] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[513, 550, 485, 534, 486, 481, 506, 481, 548, 476, 546, 518, 519, 518, 513, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 528, 513, 502, 472, 522, 518, 510, 484, 538, 496, 514, 515, 521, 485, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 502, 463, 515, 502, 512, 508, 544, 525, 526, 509, 502, 543, 531, 505, 484, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 534, 498, 508, 563, 492, 537, 478, 543, 502, 532, 517, 470, 544, 491, 474, 16, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 506, 522, 569, 539, 470, 509, 495, 496, 492, 493, 498, 542, 496, 525, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 512, 536, 515, 549, 523, 507, 517, 506, 475, 504, 513, 533, 553, 530, 505, 59, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 507, 526, 496, 513, 504, 506, 530, 514, 514, 481, 506, 482, 464, 556, 530, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 567, 488, 520, 482, 528, 508, 509, 480, 479, 522, 511, 474, 557, 505, 519, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8131, 8175, 8224, 8157, 8393, 8121, 8159] avg=8192.0000 max=8393 max_rank=5 min=8121 min_rank=6 imbalance_ratio=1.024536 max_load_reduction=57143 max_load_reduction_pct=0.871933 moved_to_echo=32830 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[488, 503, 484, 552, 513, 501, 457, 527, 502, 493, 496, 536, 523, 514, 521, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 492, 447, 528, 486, 538, 515, 504, 541, 525, 535, 483, 539, 526, 532, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 507, 552, 546, 480, 492, 541, 511, 529, 524, 497, 499, 502, 483, 486, 524, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 507, 531, 517, 527, 536, 523, 503, 546, 509, 464, 531, 466, 498, 542, 559, 14, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 540, 507, 501, 523, 522, 492, 558, 518, 527, 487, 483, 531, 520, 523, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 526, 534, 533, 499, 453, 471, 521, 474, 515, 520, 470, 527, 470, 504, 523, 57, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 515, 496, 521, 551, 503, 514, 528, 490, 493, 511, 516, 480, 525, 505, 508, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 482, 498, 514, 517, 493, 545, 538, 489, 511, 487, 549, 497, 518, 585, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8086, 8214, 8187, 8267, 8231, 8111, 8171, 8269] avg=8192.0000 max=8269 max_rank=7 min=8086 min_rank=0 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32782 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[489, 475, 523, 512, 475, 464, 511, 531, 526, 546, 509, 541, 516, 533, 510, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 544, 495, 532, 495, 514, 545, 485, 506, 546, 493, 518, 518, 530, 543, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 497, 498, 523, 499, 537, 510, 479, 508, 469, 486, 556, 516, 511, 503, 523, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 501, 546, 501, 483, 486, 513, 515, 482, 523, 528, 536, 508, 481, 576, 496, 13, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 494, 519, 499, 488, 487, 530, 481, 531, 494, 511, 529, 517, 519, 535, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 499, 499, 543, 524, 492, 516, 503, 508, 458, 528, 465, 489, 490, 536, 517, 59, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 475, 488, 529, 520, 491, 489, 483, 508, 534, 512, 544, 533, 525, 513, 523, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 484, 495, 483, 508, 527, 504, 519, 560, 515, 512, 491, 507, 530, 535, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8313, 8151, 8196, 8157, 8132, 8197, 8206] avg=8192.0000 max=8313 max_rank=1 min=8132 min_rank=5 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32692 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[553, 533, 525, 467, 509, 554, 538, 502, 532, 486, 496, 530, 518, 540, 502, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 486, 520, 528, 464, 475, 508, 495, 534, 514, 519, 497, 503, 499, 489, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 482, 459, 539, 495, 537, 479, 551, 497, 528, 541, 488, 538, 515, 511, 508, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 486, 506, 471, 519, 526, 510, 525, 546, 480, 489, 515, 518, 520, 566, 501, 14, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 498, 467, 505, 526, 546, 505, 554, 502, 526, 526, 498, 484, 511, 542, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 515, 489, 496, 508, 487, 515, 488, 506, 530, 519, 497, 498, 494, 511, 480, 55, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 479, 491, 496, 492, 566, 538, 504, 500, 491, 512, 515, 495, 526, 538, 555, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 510, 519, 533, 477, 528, 519, 526, 537, 522, 480, 514, 519, 480, 515, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8305, 8124, 8174, 8184, 8260, 8100, 8248, 8141] avg=8192.0000 max=8305 max_rank=0 min=8100 min_rank=5 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32749 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[495, 507, 489, 494, 548, 505, 502, 502, 498, 538, 505, 515, 503, 476, 524, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 508, 510, 538, 462, 488, 517, 490, 517, 478, 540, 529, 537, 536, 539, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 499, 506, 550, 529, 509, 500, 522, 528, 524, 538, 521, 550, 456, 513, 543, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 546, 502, 499, 502, 511, 494, 529, 490, 511, 507, 496, 556, 498, 506, 488, 15, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 527, 480, 515, 526, 525, 524, 515, 522, 506, 535, 501, 492, 517, 514, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 520, 507, 502, 498, 491, 516, 512, 522, 498, 496, 514, 473, 525, 475, 558, 54, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 510, 494, 469, 520, 514, 497, 480, 529, 533, 516, 505, 536, 518, 498, 516, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 521, 516, 552, 487, 513, 541, 474, 511, 527, 467, 513, 538, 520, 505, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8112, 8187, 8271, 8179, 8241, 8204, 8179, 8163] avg=8192.0000 max=8271 max_rank=2 min=8112 min_rank=0 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32787 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[500, 539, 493, 501, 523, 548, 523, 518, 532, 518, 517, 549, 479, 506, 571, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 583, 497, 529, 428, 531, 518, 518, 506, 509, 507, 506, 493, 521, 487, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 497, 505, 538, 513, 467, 516, 544, 521, 483, 462, 498, 546, 521, 507, 512, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 462, 490, 543, 498, 504, 485, 506, 520, 495, 525, 520, 515, 507, 513, 480, 14, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 501, 500, 538, 505, 505, 508, 513, 510, 535, 497, 513, 515, 511, 501, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 524, 511, 526, 506, 519, 530, 514, 534, 542, 511, 508, 499, 516, 510, 495, 51, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 516, 541, 507, 511, 503, 511, 512, 512, 523, 488, 506, 464, 526, 486, 483, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 470, 529, 489, 499, 495, 538, 538, 553, 505, 539, 503, 536, 503, 495, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8296, 8123, 8174, 8109, 8170, 8316, 8124, 8224] avg=8192.0000 max=8316 max_rank=5 min=8109 min_rank=3 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32834 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[565, 504, 522, 503, 512, 522, 516, 488, 537, 534, 521, 516, 483, 491, 484, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 509, 527, 529, 471, 471, 537, 507, 508, 492, 535, 543, 485, 517, 526, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 565, 513, 512, 491, 502, 526, 526, 483, 507, 499, 523, 516, 490, 493, 519, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 511, 507, 457, 516, 494, 538, 494, 532, 547, 518, 510, 488, 506, 527, 491, 14, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 512, 496, 503, 480, 494, 522, 533, 488, 513, 542, 505, 522, 553, 510, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 506, 482, 499, 507, 500, 472, 465, 515, 528, 477, 484, 554, 519, 497, 515, 56, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 504, 490, 498, 515, 519, 471, 548, 527, 520, 530, 514, 495, 538, 501, 542, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 532, 548, 511, 547, 510, 503, 510, 513, 531, 502, 527, 513, 535, 515, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8228, 8166, 8139, 8132, 8110, 8240, 8333] avg=8192.0000 max=8333 max_rank=7 min=8110 min_rank=5 imbalance_ratio=1.017212 max_load_reduction=57203 max_load_reduction_pct=0.872849 moved_to_echo=32815 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[488, 503, 484, 552, 513, 501, 457, 527, 502, 493, 496, 536, 523, 514, 521, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 492, 447, 528, 486, 538, 515, 504, 541, 525, 535, 483, 539, 526, 532, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 507, 552, 546, 480, 492, 541, 511, 529, 524, 497, 499, 502, 483, 486, 524, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 507, 531, 517, 527, 536, 523, 503, 546, 509, 464, 531, 466, 498, 542, 559, 14, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 540, 507, 501, 523, 522, 492, 558, 518, 527, 487, 483, 531, 520, 523, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 526, 534, 533, 499, 453, 471, 521, 474, 515, 520, 470, 527, 470, 504, 523, 57, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 515, 496, 521, 551, 503, 514, 528, 490, 493, 511, 516, 480, 525, 505, 508, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 482, 498, 514, 517, 493, 545, 538, 489, 511, 487, 549, 497, 518, 585, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8086, 8214, 8187, 8267, 8231, 8111, 8171, 8269] avg=8192.0000 max=8269 max_rank=7 min=8086 min_rank=0 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32782 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[553, 533, 525, 467, 509, 554, 538, 502, 532, 486, 496, 530, 518, 540, 502, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 486, 520, 528, 464, 475, 508, 495, 534, 514, 519, 497, 503, 499, 489, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 482, 459, 539, 495, 537, 479, 551, 497, 528, 541, 488, 538, 515, 511, 508, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 486, 506, 471, 519, 526, 510, 525, 546, 480, 489, 515, 518, 520, 566, 501, 14, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 498, 467, 505, 526, 546, 505, 554, 502, 526, 526, 498, 484, 511, 542, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 515, 489, 496, 508, 487, 515, 488, 506, 530, 519, 497, 498, 494, 511, 480, 55, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 479, 491, 496, 492, 566, 538, 504, 500, 491, 512, 515, 495, 526, 538, 555, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 510, 519, 533, 477, 528, 519, 526, 537, 522, 480, 514, 519, 480, 515, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8305, 8124, 8174, 8184, 8260, 8100, 8248, 8141] avg=8192.0000 max=8305 max_rank=0 min=8100 min_rank=5 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32749 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[486, 465, 490, 483, 530, 516, 525, 538, 534, 552, 501, 498, 520, 532, 499, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 496, 500, 506, 479, 517, 540, 531, 527, 508, 526, 504, 516, 531, 512, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 526, 485, 540, 531, 511, 526, 511, 508, 509, 510, 496, 538, 535, 514, 484, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 554, 520, 516, 497, 502, 508, 521, 533, 503, 554, 521, 491, 500, 515, 483, 14, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 511, 564, 520, 497, 495, 507, 557, 493, 517, 512, 543, 509, 476, 511, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 491, 468, 532, 509, 510, 503, 546, 513, 493, 471, 521, 507, 468, 534, 510, 57, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 501, 515, 488, 497, 499, 519, 529, 518, 540, 531, 530, 548, 502, 524, 512, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 489, 510, 528, 480, 537, 496, 505, 506, 509, 502, 493, 498, 492, 494, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8216, 8238, 8238, 8188, 8170, 8256, 8041] avg=8192.0000 max=8256 max_rank=6 min=8041 min_rank=7 imbalance_ratio=1.007812 max_load_reduction=57280 max_load_reduction_pct=0.874023 moved_to_echo=32655 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[489, 475, 523, 512, 475, 464, 511, 531, 526, 546, 509, 541, 516, 533, 510, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 544, 495, 532, 495, 514, 545, 485, 506, 546, 493, 518, 518, 530, 543, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 497, 498, 523, 499, 537, 510, 479, 508, 469, 486, 556, 516, 511, 503, 523, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 501, 546, 501, 483, 486, 513, 515, 482, 523, 528, 536, 508, 481, 576, 496, 13, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 494, 519, 499, 488, 487, 530, 481, 531, 494, 511, 529, 517, 519, 535, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 499, 499, 543, 524, 492, 516, 503, 508, 458, 528, 465, 489, 490, 536, 517, 59, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 475, 488, 529, 520, 491, 489, 483, 508, 534, 512, 544, 533, 525, 513, 523, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 484, 495, 483, 508, 527, 504, 519, 560, 515, 512, 491, 507, 530, 535, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8313, 8151, 8196, 8157, 8132, 8197, 8206] avg=8192.0000 max=8313 max_rank=1 min=8132 min_rank=5 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32692 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4224, 3840, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 4352, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4352, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4352, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 512, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4224, 3840, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4352, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[511, 505, 531, 551, 513, 517, 535, 505, 486, 504, 514, 515, 471, 538, 488, 536, 519, 486, 516, 548, 506, 497, 502, 527, 463, 557, 528, 486, 534, 523, 500, 525, 492, 517, 509, 537, 511, 518, 506, 554, 527, 522, 524, 473, 510, 516, 575, 523, 501, 498, 477, 517, 534, 523, 480, 500, 512, 538, 496, 506, 519, 484, 484, 484, 495, 524, 496, 519, 478, 531, 493, 500, 526, 534, 556, 514, 472, 503, 486, 585, 487, 512, 510, 505, 505, 472, 532, 522, 559, 488, 486, 500, 488, 529, 543, 530, 538, 522, 554, 484, 534, 467, 479, 493, 520, 537, 498, 552, 493, 501, 511, 501, 532, 510, 510, 531, 498, 493, 508, 513, 506, 508, 561, 516, 509, 495, 494, 484] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[501, 507, 498, 516, 515, 526, 512, 534, 541, 516, 550, 509, 508, 537, 510, 486, 494, 509, 538, 549, 509, 509, 518, 493, 520, 506, 516, 512, 499, 486, 513, 513, 514, 497, 494, 500, 526, 525, 511, 514, 505, 488, 527, 514, 530, 572, 489, 502, 517, 521, 478, 508, 510, 501, 518, 521, 527, 553, 529, 514, 520, 520, 487, 506, 516, 497, 478, 544, 510, 524, 494, 494, 512, 503, 539, 504, 478, 551, 514, 513, 472, 493, 519, 512, 561, 511, 536, 498, 526, 489, 540, 540, 489, 498, 509, 504, 513, 532, 465, 512, 503, 485, 505, 497, 498, 510, 460, 513, 521, 562, 501, 569, 494, 549, 528, 556, 517, 525, 527, 473, 502, 482, 491, 488, 487, 510, 510, 495] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[511, 505, 531, 551, 513, 517, 535, 505, 486, 504, 514, 515, 471, 538, 488, 536, 519, 486, 516, 548, 506, 497, 502, 527, 463, 557, 528, 486, 534, 523, 500, 525, 492, 517, 509, 537, 511, 518, 506, 554, 527, 522, 524, 473, 510, 516, 575, 523, 501, 498, 477, 517, 534, 523, 480, 500, 512, 538, 496, 506, 519, 484, 484, 484, 495, 524, 496, 519, 478, 531, 493, 500, 526, 534, 556, 514, 472, 503, 486, 585, 487, 512, 510, 505, 505, 472, 532, 522, 559, 488, 486, 500, 488, 529, 543, 530, 538, 522, 554, 484, 534, 467, 479, 493, 520, 537, 498, 552, 493, 501, 511, 501, 532, 510, 510, 531, 498, 493, 508, 513, 506, 508, 561, 516, 509, 495, 494, 484] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[521, 497, 508, 528, 499, 514, 452, 488, 510, 511, 484, 509, 501, 550, 521, 532, 491, 441, 535, 489, 521, 533, 538, 543, 503, 499, 525, 531, 524, 491, 486, 505, 516, 515, 520, 460, 505, 513, 516, 510, 464, 507, 470, 525, 520, 503, 498, 494, 518, 536, 496, 516, 518, 508, 528, 521, 469, 485, 495, 502, 528, 507, 482, 528, 554, 564, 502, 497, 540, 509, 495, 572, 496, 524, 528, 520, 511, 528, 542, 468, 553, 555, 506, 467, 511, 497, 544, 527, 480, 514, 517, 496, 512, 491, 490, 526, 547, 482, 555, 543, 499, 467, 506, 539, 528, 508, 491, 504, 518, 523, 516, 525, 469, 546, 534, 525, 532, 527, 493, 507, 489, 498, 581, 487, 493, 566, 534, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[570, 535, 501, 524, 469, 537, 541, 497, 497, 516, 508, 512, 490, 536, 533, 505, 489, 471, 526, 488, 499, 492, 497, 497, 548, 526, 523, 526, 521, 497, 509, 520, 533, 538, 488, 492, 507, 464, 507, 517, 551, 527, 501, 531, 484, 507, 478, 483, 545, 514, 516, 517, 552, 519, 527, 522, 530, 516, 496, 522, 488, 543, 537, 530, 508, 523, 488, 497, 478, 490, 511, 552, 485, 488, 506, 492, 519, 562, 481, 558, 484, 535, 515, 521, 526, 475, 515, 539, 455, 513, 529, 524, 538, 530, 495, 509, 524, 520, 501, 494, 518, 518, 534, 505, 456, 493, 494, 515, 468, 525, 503, 517, 502, 514, 541, 534, 510, 510, 545, 534, 507, 500, 540, 483, 497, 517, 496, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[490, 503, 543, 493, 501, 552, 531, 508, 517, 496, 548, 503, 496, 502, 527, 503, 497, 541, 494, 491, 500, 504, 517, 522, 490, 529, 505, 523, 518, 502, 531, 511, 482, 504, 553, 508, 478, 531, 506, 482, 528, 501, 507, 539, 514, 497, 475, 518, 468, 534, 476, 521, 533, 534, 509, 499, 504, 528, 538, 510, 525, 520, 491, 510, 513, 497, 534, 474, 513, 512, 509, 498, 547, 505, 516, 530, 543, 536, 511, 523, 545, 475, 506, 540, 529, 514, 507, 521, 532, 517, 513, 507, 541, 506, 481, 527, 559, 523, 518, 523, 522, 531, 531, 508, 502, 486, 500, 524, 544, 559, 499, 497, 485, 522, 515, 497, 478, 481, 480, 480, 486, 500, 509, 511, 513, 549, 481, 490] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[517, 502, 534, 495, 502, 508, 517, 484, 474, 477, 530, 499, 518, 553, 512, 455, 527, 541, 541, 560, 514, 505, 487, 505, 531, 522, 522, 534, 495, 472, 534, 542, 508, 506, 535, 534, 490, 509, 497, 519, 543, 490, 545, 505, 575, 477, 501, 500, 502, 520, 487, 484, 489, 520, 512, 497, 500, 533, 499, 524, 518, 487, 542, 475, 539, 501, 495, 502, 521, 534, 502, 503, 533, 540, 489, 509, 511, 523, 524, 524, 487, 496, 525, 508, 519, 501, 520, 483, 488, 521, 489, 528, 478, 522, 516, 514, 506, 538, 480, 514, 511, 531, 523, 532, 495, 539, 458, 536, 514, 553, 479, 493, 498, 540, 495, 490, 534, 505, 513, 507, 500, 507, 501, 542, 527, 547, 504, 547] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[505, 469, 547, 525, 528, 533, 508, 499, 518, 487, 536, 524, 550, 544, 508, 511, 524, 543, 529, 517, 489, 503, 499, 476, 525, 522, 536, 524, 467, 523, 481, 490, 464, 527, 522, 485, 499, 525, 547, 514, 507, 538, 500, 491, 511, 524, 505, 527, 513, 496, 477, 499, 520, 556, 486, 543, 459, 565, 505, 511, 496, 495, 519, 492, 515, 485, 515, 565, 491, 507, 541, 518, 508, 486, 490, 536, 495, 539, 496, 497, 508, 510, 494, 490, 509, 519, 491, 517, 496, 516, 552, 460, 486, 525, 515, 531, 500, 494, 523, 477, 512, 554, 514, 500, 524, 516, 545, 518, 512, 501, 496, 555, 499, 538, 503, 508, 537, 475, 542, 494, 522, 532, 473, 511, 512, 530, 514, 544] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[490, 503, 543, 493, 501, 552, 531, 508, 517, 496, 548, 503, 496, 502, 527, 503, 497, 541, 494, 491, 500, 504, 517, 522, 490, 529, 505, 523, 518, 502, 531, 511, 482, 504, 553, 508, 478, 531, 506, 482, 528, 501, 507, 539, 514, 497, 475, 518, 468, 534, 476, 521, 533, 534, 509, 499, 504, 528, 538, 510, 525, 520, 491, 510, 513, 497, 534, 474, 513, 512, 509, 498, 547, 505, 516, 530, 543, 536, 511, 523, 545, 475, 506, 540, 529, 514, 507, 521, 532, 517, 513, 507, 541, 506, 481, 527, 559, 523, 518, 523, 522, 531, 531, 508, 502, 486, 500, 524, 544, 559, 499, 497, 485, 522, 515, 497, 478, 481, 480, 480, 486, 500, 509, 511, 513, 549, 481, 490] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[513, 486, 502, 507, 496, 517, 522, 498, 502, 501, 528, 540, 558, 495, 504, 522, 498, 531, 523, 560, 496, 486, 562, 503, 516, 508, 516, 517, 518, 530, 512, 506, 494, 507, 511, 512, 536, 489, 508, 501, 502, 495, 479, 515, 468, 474, 542, 519, 512, 506, 489, 532, 496, 542, 498, 507, 544, 526, 482, 482, 531, 503, 512, 517, 526, 548, 493, 516, 554, 539, 538, 510, 503, 461, 543, 467, 524, 542, 506, 476, 549, 497, 507, 521, 538, 520, 559, 543, 467, 537, 532, 512, 465, 513, 529, 480, 472, 553, 501, 525, 563, 518, 521, 507, 519, 493, 481, 491, 500, 542, 481, 468, 538, 467, 509, 511, 549, 494, 555, 488, 516, 502, 514, 505, 519, 518, 491, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[570, 535, 501, 524, 469, 537, 541, 497, 497, 516, 508, 512, 490, 536, 533, 505, 489, 471, 526, 488, 499, 492, 497, 497, 548, 526, 523, 526, 521, 497, 509, 520, 533, 538, 488, 492, 507, 464, 507, 517, 551, 527, 501, 531, 484, 507, 478, 483, 545, 514, 516, 517, 552, 519, 527, 522, 530, 516, 496, 522, 488, 543, 537, 530, 508, 523, 488, 497, 478, 490, 511, 552, 485, 488, 506, 492, 519, 562, 481, 558, 484, 535, 515, 521, 526, 475, 515, 539, 455, 513, 529, 524, 538, 530, 495, 509, 524, 520, 501, 494, 518, 518, 534, 505, 456, 493, 494, 515, 468, 525, 503, 517, 502, 514, 541, 534, 510, 510, 545, 534, 507, 500, 540, 483, 497, 517, 496, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[521, 497, 508, 528, 499, 514, 452, 488, 510, 511, 484, 509, 501, 550, 521, 532, 491, 441, 535, 489, 521, 533, 538, 543, 503, 499, 525, 531, 524, 491, 486, 505, 516, 515, 520, 460, 505, 513, 516, 510, 464, 507, 470, 525, 520, 503, 498, 494, 518, 536, 496, 516, 518, 508, 528, 521, 469, 485, 495, 502, 528, 507, 482, 528, 554, 564, 502, 497, 540, 509, 495, 572, 496, 524, 528, 520, 511, 528, 542, 468, 553, 555, 506, 467, 511, 497, 544, 527, 480, 514, 517, 496, 512, 491, 490, 526, 547, 482, 555, 543, 499, 467, 506, 539, 528, 508, 491, 504, 518, 523, 516, 525, 469, 546, 534, 525, 532, 527, 493, 507, 489, 498, 581, 487, 493, 566, 534, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[511, 505, 531, 551, 513, 517, 535, 505, 486, 504, 514, 515, 471, 520, 488, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 486, 516, 523, 506, 497, 502, 527, 463, 557, 528, 486, 534, 523, 500, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 517, 509, 537, 511, 518, 506, 554, 527, 522, 524, 473, 510, 516, 575, 523, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 498, 477, 517, 534, 523, 480, 500, 512, 538, 496, 506, 519, 484, 484, 484, 18, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 524, 496, 519, 478, 531, 493, 500, 526, 534, 556, 514, 472, 466, 486, 585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 512, 510, 505, 505, 472, 532, 522, 559, 488, 486, 500, 488, 529, 543, 530, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 522, 554, 484, 534, 467, 479, 493, 520, 537, 498, 552, 493, 494, 511, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 510, 510, 531, 498, 493, 508, 513, 506, 508, 561, 516, 509, 495, 494, 484, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8192, 8350, 8080, 8175, 8179, 8177, 8181] avg=8192.0000 max=8350 max_rank=2 min=8080 min_rank=3 imbalance_ratio=1.019287 max_load_reduction=57186 max_load_reduction_pct=0.872589 moved_to_echo=32712 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[513, 486, 502, 507, 496, 517, 522, 498, 502, 501, 528, 540, 558, 482, 504, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 531, 523, 551, 496, 486, 562, 503, 516, 508, 516, 517, 518, 530, 512, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 507, 511, 512, 536, 489, 508, 501, 502, 495, 479, 515, 468, 474, 542, 519, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 506, 489, 532, 496, 542, 498, 507, 544, 526, 482, 482, 531, 503, 512, 517, 13, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 548, 493, 516, 554, 539, 538, 510, 503, 461, 543, 467, 524, 508, 506, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 497, 507, 521, 538, 520, 559, 543, 467, 537, 532, 512, 465, 513, 529, 480, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 553, 501, 525, 563, 518, 521, 507, 519, 493, 481, 491, 500, 538, 481, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 467, 509, 511, 549, 494, 555, 488, 516, 502, 514, 505, 519, 518, 491, 506, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8273, 8086, 8194, 8212, 8273, 8131, 8189] avg=8192.0000 max=8273 max_rank=1 min=8086 min_rank=2 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32805 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[501, 507, 498, 516, 515, 526, 512, 534, 541, 516, 550, 509, 508, 522, 510, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 509, 538, 540, 509, 509, 518, 493, 520, 506, 516, 512, 499, 486, 513, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 497, 494, 500, 526, 525, 511, 514, 505, 488, 527, 514, 530, 572, 489, 502, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 521, 478, 508, 510, 501, 518, 521, 527, 553, 529, 514, 520, 520, 487, 506, 15, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 497, 478, 544, 510, 524, 494, 494, 512, 503, 539, 504, 478, 516, 514, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 493, 519, 512, 561, 511, 536, 498, 526, 489, 540, 540, 489, 498, 509, 504, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 532, 465, 512, 503, 485, 505, 497, 498, 510, 460, 513, 521, 558, 501, 569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 549, 528, 556, 517, 525, 527, 473, 502, 482, 491, 488, 487, 510, 510, 495, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8251, 8175, 8243, 8247, 8136, 8201, 8142, 8141] avg=8192.0000 max=8251 max_rank=0 min=8136 min_rank=4 imbalance_ratio=1.007202 max_load_reduction=57285 max_load_reduction_pct=0.874100 moved_to_echo=32620 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[505, 469, 547, 525, 528, 533, 508, 499, 518, 487, 536, 524, 550, 529, 508, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 543, 529, 508, 489, 503, 499, 476, 525, 522, 536, 524, 467, 523, 481, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 464, 527, 522, 485, 499, 525, 547, 514, 507, 538, 500, 491, 511, 524, 505, 527, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 496, 477, 499, 520, 556, 486, 543, 459, 565, 505, 511, 496, 495, 519, 492, 15, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 485, 515, 565, 491, 507, 541, 518, 508, 486, 490, 536, 495, 505, 496, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 510, 494, 490, 509, 519, 491, 517, 496, 516, 552, 460, 486, 525, 515, 531, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 494, 523, 477, 512, 554, 514, 500, 524, 516, 545, 518, 512, 498, 496, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 538, 503, 508, 537, 475, 542, 494, 522, 532, 473, 511, 512, 530, 514, 544, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8277, 8139, 8220, 8149, 8150, 8122, 8238, 8241] avg=8192.0000 max=8277 max_rank=0 min=8122 min_rank=5 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32751 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[517, 502, 534, 495, 502, 508, 517, 484, 474, 477, 530, 499, 518, 538, 512, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 541, 541, 551, 514, 505, 487, 505, 531, 522, 522, 534, 495, 472, 534, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 506, 535, 534, 490, 509, 497, 519, 543, 490, 545, 505, 575, 477, 501, 500, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 520, 487, 484, 489, 520, 512, 497, 500, 533, 499, 524, 518, 487, 542, 475, 15, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 501, 495, 502, 521, 534, 502, 503, 533, 540, 489, 509, 511, 490, 524, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 496, 525, 508, 519, 501, 520, 483, 488, 521, 489, 528, 478, 522, 516, 514, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 538, 480, 514, 511, 531, 523, 532, 495, 539, 458, 536, 514, 549, 479, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 540, 495, 490, 534, 505, 513, 507, 500, 507, 501, 542, 527, 547, 504, 547, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8062, 8323, 8267, 8106, 8217, 8099, 8198, 8264] avg=8192.0000 max=8323 max_rank=1 min=8062 min_rank=0 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32778 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[490, 503, 543, 493, 501, 552, 531, 508, 517, 496, 548, 503, 496, 488, 527, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 541, 494, 482, 500, 504, 517, 522, 490, 529, 505, 523, 518, 502, 531, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 504, 553, 508, 478, 531, 506, 482, 528, 501, 507, 539, 514, 497, 475, 518, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 534, 476, 521, 533, 534, 509, 499, 504, 528, 538, 510, 525, 520, 491, 510, 14, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 497, 534, 474, 513, 512, 509, 498, 547, 505, 516, 530, 543, 502, 511, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 475, 506, 540, 529, 514, 507, 521, 532, 517, 513, 507, 541, 506, 481, 527, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 523, 518, 523, 522, 531, 531, 508, 502, 486, 500, 524, 544, 555, 499, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 522, 515, 497, 478, 481, 480, 480, 486, 500, 509, 511, 513, 549, 481, 490, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8166, 8157, 8216, 8227, 8265, 8322, 7984] avg=8192.0000 max=8322 max_rank=6 min=7984 min_rank=7 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32798 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[521, 497, 508, 528, 499, 514, 452, 488, 510, 511, 484, 509, 501, 535, 521, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 441, 535, 481, 521, 533, 538, 543, 503, 499, 525, 531, 524, 491, 486, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 515, 520, 460, 505, 513, 516, 510, 464, 507, 470, 525, 520, 503, 498, 494, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 536, 496, 516, 518, 508, 528, 521, 469, 485, 495, 502, 528, 507, 482, 528, 15, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 564, 502, 497, 540, 509, 495, 572, 496, 524, 528, 520, 511, 495, 542, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 555, 506, 467, 511, 497, 544, 527, 480, 514, 517, 496, 512, 491, 490, 526, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 482, 555, 543, 499, 467, 506, 539, 528, 508, 491, 504, 518, 519, 516, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 546, 534, 525, 532, 527, 493, 507, 489, 498, 581, 487, 493, 566, 534, 515, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8110, 8147, 8069, 8154, 8317, 8190, 8247, 8302] avg=8192.0000 max=8317 max_rank=4 min=8069 min_rank=2 imbalance_ratio=1.015259 max_load_reduction=57219 max_load_reduction_pct=0.873093 moved_to_echo=33056 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[490, 503, 543, 493, 501, 552, 531, 508, 517, 496, 548, 503, 496, 488, 527, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 541, 494, 482, 500, 504, 517, 522, 490, 529, 505, 523, 518, 502, 531, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 504, 553, 508, 478, 531, 506, 482, 528, 501, 507, 539, 514, 497, 475, 518, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 534, 476, 521, 533, 534, 509, 499, 504, 528, 538, 510, 525, 520, 491, 510, 14, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 497, 534, 474, 513, 512, 509, 498, 547, 505, 516, 530, 543, 502, 511, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 475, 506, 540, 529, 514, 507, 521, 532, 517, 513, 507, 541, 506, 481, 527, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 523, 518, 523, 522, 531, 531, 508, 502, 486, 500, 524, 544, 555, 499, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 522, 515, 497, 478, 481, 480, 480, 486, 500, 509, 511, 513, 549, 481, 490, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8166, 8157, 8216, 8227, 8265, 8322, 7984] avg=8192.0000 max=8322 max_rank=6 min=7984 min_rank=7 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32798 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[517, 502, 534, 495, 502, 508, 517, 484, 474, 477, 530, 499, 518, 538, 512, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 541, 541, 551, 514, 505, 487, 505, 531, 522, 522, 534, 495, 472, 534, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 506, 535, 534, 490, 509, 497, 519, 543, 490, 545, 505, 575, 477, 501, 500, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 520, 487, 484, 489, 520, 512, 497, 500, 533, 499, 524, 518, 487, 542, 475, 15, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 501, 495, 502, 521, 534, 502, 503, 533, 540, 489, 509, 511, 490, 524, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 496, 525, 508, 519, 501, 520, 483, 488, 521, 489, 528, 478, 522, 516, 514, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 538, 480, 514, 511, 531, 523, 532, 495, 539, 458, 536, 514, 549, 479, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 540, 495, 490, 534, 505, 513, 507, 500, 507, 501, 542, 527, 547, 504, 547, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8062, 8323, 8267, 8106, 8217, 8099, 8198, 8264] avg=8192.0000 max=8323 max_rank=1 min=8062 min_rank=0 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32778 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[570, 535, 501, 524, 469, 537, 541, 497, 497, 516, 508, 512, 490, 522, 533, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 471, 526, 480, 499, 492, 497, 497, 548, 526, 523, 526, 521, 497, 509, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 538, 488, 492, 507, 464, 507, 517, 551, 527, 501, 531, 484, 507, 478, 483, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 514, 516, 517, 552, 519, 527, 522, 530, 516, 496, 522, 488, 543, 537, 530, 14, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 523, 488, 497, 478, 490, 511, 552, 485, 488, 506, 492, 519, 526, 481, 558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 535, 515, 521, 526, 475, 515, 539, 455, 513, 529, 524, 538, 530, 495, 509, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 520, 501, 494, 518, 518, 534, 505, 456, 493, 494, 515, 468, 521, 503, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 514, 541, 534, 510, 510, 545, 534, 507, 500, 540, 483, 497, 517, 496, 498, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8257, 8121, 8144, 8390, 8102, 8207, 8081, 8234] avg=8192.0000 max=8390 max_rank=3 min=8081 min_rank=6 imbalance_ratio=1.024170 max_load_reduction=57146 max_load_reduction_pct=0.871979 moved_to_echo=32624 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[505, 469, 547, 525, 528, 533, 508, 499, 518, 487, 536, 524, 550, 529, 508, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 543, 529, 508, 489, 503, 499, 476, 525, 522, 536, 524, 467, 523, 481, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 464, 527, 522, 485, 499, 525, 547, 514, 507, 538, 500, 491, 511, 524, 505, 527, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 496, 477, 499, 520, 556, 486, 543, 459, 565, 505, 511, 496, 495, 519, 492, 15, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 485, 515, 565, 491, 507, 541, 518, 508, 486, 490, 536, 495, 505, 496, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 510, 494, 490, 509, 519, 491, 517, 496, 516, 552, 460, 486, 525, 515, 531, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 494, 523, 477, 512, 554, 514, 500, 524, 516, 545, 518, 512, 498, 496, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 538, 503, 508, 537, 475, 542, 494, 522, 532, 473, 511, 512, 530, 514, 544, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8277, 8139, 8220, 8149, 8150, 8122, 8238, 8241] avg=8192.0000 max=8277 max_rank=0 min=8122 min_rank=5 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32751 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[513, 486, 502, 507, 496, 517, 522, 498, 502, 501, 528, 540, 558, 482, 504, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 531, 523, 551, 496, 486, 562, 503, 516, 508, 516, 517, 518, 530, 512, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 507, 511, 512, 536, 489, 508, 501, 502, 495, 479, 515, 468, 474, 542, 519, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 506, 489, 532, 496, 542, 498, 507, 544, 526, 482, 482, 531, 503, 512, 517, 13, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 548, 493, 516, 554, 539, 538, 510, 503, 461, 543, 467, 524, 508, 506, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 497, 507, 521, 538, 520, 559, 543, 467, 537, 532, 512, 465, 513, 529, 480, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 553, 501, 525, 563, 518, 521, 507, 519, 493, 481, 491, 500, 538, 481, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 467, 509, 511, 549, 494, 555, 488, 516, 502, 514, 505, 519, 518, 491, 506, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8273, 8086, 8194, 8212, 8273, 8131, 8189] avg=8192.0000 max=8273 max_rank=1 min=8086 min_rank=2 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32805 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 3968, 4096, 4224, 4224, 4096, 4224, 4096, 4352, 4096, 4096, 4224, 4096, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 3968, 4224, 4096, 4352, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4352, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 3968, 4096, 4224, 4224, 4096, 4224, 4096, 4352, 4096, 4096, 4224, 4096, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[467, 511, 521, 516, 518, 524, 554, 542, 507, 495, 530, 522, 513, 563, 489, 535, 544, 495, 539, 488, 473, 521, 504, 481, 516, 488, 492, 531, 481, 557, 495, 492, 556, 517, 494, 488, 543, 517, 498, 501, 514, 510, 509, 518, 521, 533, 506, 493, 518, 501, 504, 515, 497, 496, 523, 514, 538, 518, 508, 502, 461, 524, 521, 537, 534, 522, 520, 529, 511, 474, 521, 546, 503, 493, 548, 493, 500, 508, 544, 505, 484, 500, 516, 501, 496, 480, 529, 491, 505, 506, 530, 512, 497, 524, 493, 512, 523, 538, 514, 528, 510, 512, 511, 514, 536, 522, 503, 476, 514, 503, 496, 536, 570, 497, 545, 524, 500, 517, 516, 508, 527, 491, 503, 501, 512, 479, 480, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[495, 504, 506, 485, 507, 557, 527, 511, 493, 550, 529, 516, 493, 537, 545, 557, 490, 500, 511, 489, 506, 502, 554, 539, 540, 513, 485, 517, 529, 487, 512, 517, 498, 499, 508, 511, 479, 495, 510, 511, 515, 550, 508, 488, 495, 516, 501, 499, 493, 521, 509, 488, 511, 492, 470, 509, 528, 533, 513, 535, 515, 528, 489, 511, 464, 486, 555, 534, 520, 489, 516, 507, 519, 544, 533, 487, 524, 550, 530, 519, 546, 520, 464, 472, 523, 514, 481, 549, 540, 521, 456, 521, 510, 547, 476, 525, 487, 516, 541, 511, 522, 530, 501, 490, 503, 513, 515, 520, 497, 512, 507, 548, 508, 564, 477, 506, 501, 522, 521, 486, 504, 513, 510, 455, 486, 507, 519, 571] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[509, 520, 530, 553, 521, 529, 510, 506, 470, 514, 502, 543, 549, 511, 488, 483, 492, 532, 516, 524, 536, 520, 512, 505, 525, 499, 551, 510, 501, 509, 516, 493, 526, 542, 480, 544, 483, 461, 477, 522, 507, 501, 531, 535, 520, 523, 523, 540, 513, 523, 503, 488, 507, 515, 541, 491, 510, 538, 533, 493, 517, 487, 537, 537, 527, 494, 488, 552, 523, 506, 518, 520, 492, 550, 513, 560, 514, 486, 537, 526, 480, 489, 467, 474, 490, 551, 509, 486, 525, 530, 483, 525, 504, 493, 552, 496, 497, 512, 507, 472, 535, 480, 501, 506, 528, 512, 513, 493, 562, 544, 503, 488, 523, 507, 505, 516, 512, 509, 517, 535, 507, 488, 511, 495, 456, 504, 492, 519] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[487, 506, 508, 486, 521, 508, 502, 529, 478, 498, 524, 492, 499, 499, 487, 504, 485, 489, 501, 470, 527, 499, 535, 531, 517, 565, 541, 514, 488, 506, 512, 504, 509, 473, 516, 511, 552, 529, 529, 523, 538, 477, 491, 563, 491, 494, 473, 529, 511, 468, 538, 488, 484, 523, 546, 525, 511, 530, 506, 552, 495, 481, 548, 452, 522, 549, 486, 513, 543, 501, 506, 494, 545, 538, 504, 516, 490, 539, 484, 529, 533, 490, 488, 504, 495, 556, 504, 522, 556, 517, 531, 510, 544, 527, 513, 529, 527, 490, 535, 530, 506, 449, 536, 530, 481, 514, 468, 513, 483, 521, 527, 533, 517, 496, 516, 529, 515, 542, 522, 500, 500, 520, 524, 529, 504, 510, 516, 507] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[522, 514, 534, 516, 528, 494, 507, 507, 506, 476, 500, 490, 493, 531, 481, 516, 479, 490, 551, 518, 516, 483, 538, 532, 534, 507, 469, 520, 480, 533, 506, 552, 510, 500, 501, 484, 522, 510, 537, 506, 505, 551, 487, 490, 506, 512, 544, 526, 502, 530, 537, 523, 514, 525, 485, 536, 517, 486, 508, 509, 542, 506, 522, 503, 533, 527, 531, 530, 560, 506, 534, 536, 509, 519, 469, 512, 501, 528, 516, 525, 492, 503, 489, 515, 532, 510, 531, 517, 499, 514, 572, 481, 493, 517, 510, 517, 546, 509, 528, 508, 520, 524, 477, 482, 530, 506, 481, 513, 516, 506, 499, 508, 517, 489, 515, 512, 529, 554, 511, 490, 511, 521, 489, 490, 469, 495, 496, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[498, 503, 509, 489, 539, 485, 506, 510, 565, 512, 505, 541, 502, 522, 470, 519, 495, 508, 525, 536, 518, 476, 535, 521, 474, 518, 477, 506, 543, 516, 534, 523, 525, 495, 500, 531, 502, 509, 527, 481, 489, 516, 518, 556, 490, 480, 540, 499, 514, 528, 533, 548, 533, 540, 537, 483, 499, 525, 525, 507, 524, 523, 516, 512, 518, 463, 492, 502, 493, 488, 552, 513, 520, 532, 518, 485, 526, 554, 512, 492, 473, 495, 499, 513, 527, 500, 496, 523, 498, 492, 487, 516, 504, 500, 468, 538, 490, 531, 500, 516, 476, 499, 502, 487, 487, 531, 491, 521, 504, 524, 500, 557, 521, 533, 557, 533, 527, 484, 510, 503, 520, 523, 525, 573, 528, 503, 512, 494] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[537, 497, 489, 526, 512, 508, 523, 506, 551, 483, 524, 525, 514, 559, 507, 487, 475, 497, 495, 512, 471, 510, 514, 492, 533, 500, 524, 521, 503, 502, 517, 561, 540, 503, 504, 513, 511, 495, 519, 511, 526, 555, 539, 537, 511, 516, 476, 518, 481, 525, 493, 511, 494, 490, 509, 537, 541, 474, 516, 538, 493, 526, 512, 524, 491, 524, 496, 515, 505, 497, 511, 533, 493, 521, 525, 487, 503, 525, 486, 518, 523, 510, 511, 510, 502, 517, 514, 539, 484, 547, 514, 525, 513, 519, 505, 489, 507, 509, 504, 469, 551, 499, 500, 539, 466, 476, 519, 542, 540, 492, 497, 496, 508, 496, 541, 516, 516, 517, 496, 505, 554, 506, 478, 538, 520, 538, 539, 497] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[487, 506, 508, 486, 521, 508, 502, 529, 478, 498, 524, 492, 499, 499, 487, 504, 485, 489, 501, 470, 527, 499, 535, 531, 517, 565, 541, 514, 488, 506, 512, 504, 509, 473, 516, 511, 552, 529, 529, 523, 538, 477, 491, 563, 491, 494, 473, 529, 511, 468, 538, 488, 484, 523, 546, 525, 511, 530, 506, 552, 495, 481, 548, 452, 522, 549, 486, 513, 543, 501, 506, 494, 545, 538, 504, 516, 490, 539, 484, 529, 533, 490, 488, 504, 495, 556, 504, 522, 556, 517, 531, 510, 544, 527, 513, 529, 527, 490, 535, 530, 506, 449, 536, 530, 481, 514, 468, 513, 483, 521, 527, 533, 517, 496, 516, 529, 515, 542, 522, 500, 500, 520, 524, 529, 504, 510, 516, 507] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[495, 542, 484, 540, 501, 467, 513, 516, 522, 510, 498, 521, 536, 519, 502, 501, 518, 502, 527, 493, 447, 529, 514, 532, 523, 517, 488, 502, 487, 565, 507, 550, 522, 556, 482, 482, 495, 520, 526, 511, 516, 518, 482, 522, 544, 540, 497, 552, 566, 521, 511, 494, 536, 503, 470, 492, 490, 510, 554, 517, 489, 492, 492, 538, 519, 508, 524, 502, 516, 496, 519, 507, 497, 488, 547, 497, 493, 494, 527, 507, 517, 476, 513, 528, 520, 519, 503, 533, 521, 507, 518, 500, 548, 510, 532, 485, 511, 502, 493, 560, 510, 538, 489, 507, 529, 494, 493, 465, 495, 481, 526, 531, 476, 520, 532, 485, 514, 506, 558, 528, 520, 506, 508, 492, 521, 550, 510, 507] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[467, 511, 521, 516, 518, 524, 554, 542, 507, 495, 530, 522, 513, 556, 489, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 495, 539, 488, 473, 521, 504, 481, 516, 488, 492, 531, 481, 557, 495, 492, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 517, 494, 488, 543, 517, 498, 501, 514, 510, 509, 511, 521, 533, 506, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 501, 504, 515, 497, 496, 523, 514, 538, 518, 495, 502, 461, 524, 521, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 522, 520, 529, 511, 474, 521, 546, 503, 456, 548, 493, 500, 508, 544, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 500, 516, 501, 496, 480, 529, 491, 505, 506, 530, 512, 497, 524, 493, 512, 13, 9, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 538, 514, 528, 510, 512, 511, 514, 536, 522, 503, 476, 514, 503, 496, 536, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 570, 497, 536, 524, 500, 517, 516, 508, 527, 491, 503, 501, 512, 479, 480, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8300, 8107, 8211, 8164, 8214, 8105, 8270, 8165] avg=8192.0000 max=8300 max_rank=0 min=8105 min_rank=5 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32754 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[495, 504, 506, 485, 507, 557, 527, 511, 493, 550, 529, 516, 493, 531, 545, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 500, 511, 489, 506, 502, 554, 539, 540, 513, 485, 517, 529, 487, 512, 517, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 499, 508, 511, 479, 495, 510, 511, 515, 550, 508, 481, 495, 516, 501, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 521, 509, 488, 511, 492, 470, 509, 528, 533, 500, 535, 515, 528, 489, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 464, 486, 555, 534, 520, 489, 516, 507, 519, 503, 533, 487, 524, 550, 530, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 520, 464, 472, 523, 514, 481, 549, 540, 521, 456, 521, 510, 547, 476, 525, 13, 8, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 516, 541, 511, 522, 530, 501, 490, 503, 513, 515, 520, 497, 512, 507, 548, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 564, 469, 506, 501, 522, 521, 486, 504, 513, 510, 455, 486, 507, 519, 571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8306, 8200, 8076, 8132, 8236, 8194, 8250, 8142] avg=8192.0000 max=8306 max_rank=0 min=8076 min_rank=2 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32822 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[467, 511, 521, 516, 518, 524, 554, 542, 507, 495, 530, 522, 513, 563, 489, 535, 544, 495, 539, 488, 473, 521, 504, 481, 516, 488, 492, 531, 481, 557, 495, 492, 556, 517, 494, 488, 543, 517, 498, 501, 514, 510, 509, 518, 521, 533, 506, 493, 518, 501, 504, 515, 497, 496, 523, 514, 538, 518, 508, 502, 461, 524, 521, 537, 534, 522, 520, 529, 511, 474, 521, 546, 503, 493, 548, 493, 500, 508, 544, 505, 484, 500, 516, 501, 496, 480, 529, 491, 505, 506, 530, 512, 497, 524, 493, 512, 523, 538, 514, 528, 510, 512, 511, 514, 536, 522, 503, 476, 514, 503, 496, 536, 570, 497, 545, 524, 500, 517, 516, 508, 527, 491, 503, 501, 512, 479, 480, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[495, 504, 506, 485, 507, 557, 527, 511, 493, 550, 529, 516, 493, 537, 545, 557, 490, 500, 511, 489, 506, 502, 554, 539, 540, 513, 485, 517, 529, 487, 512, 517, 498, 499, 508, 511, 479, 495, 510, 511, 515, 550, 508, 488, 495, 516, 501, 499, 493, 521, 509, 488, 511, 492, 470, 509, 528, 533, 513, 535, 515, 528, 489, 511, 464, 486, 555, 534, 520, 489, 516, 507, 519, 544, 533, 487, 524, 550, 530, 519, 546, 520, 464, 472, 523, 514, 481, 549, 540, 521, 456, 521, 510, 547, 476, 525, 487, 516, 541, 511, 522, 530, 501, 490, 503, 513, 515, 520, 497, 512, 507, 548, 508, 564, 477, 506, 501, 522, 521, 486, 504, 513, 510, 455, 486, 507, 519, 571] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[498, 503, 509, 489, 539, 485, 506, 510, 565, 512, 505, 541, 502, 522, 470, 519, 495, 508, 525, 536, 518, 476, 535, 521, 474, 518, 477, 506, 543, 516, 534, 523, 525, 495, 500, 531, 502, 509, 527, 481, 489, 516, 518, 556, 490, 480, 540, 499, 514, 528, 533, 548, 533, 540, 537, 483, 499, 525, 525, 507, 524, 523, 516, 512, 518, 463, 492, 502, 493, 488, 552, 513, 520, 532, 518, 485, 526, 554, 512, 492, 473, 495, 499, 513, 527, 500, 496, 523, 498, 492, 487, 516, 504, 500, 468, 538, 490, 531, 500, 516, 476, 499, 502, 487, 487, 531, 491, 521, 504, 524, 500, 557, 521, 533, 557, 533, 527, 484, 510, 503, 520, 523, 525, 573, 528, 503, 512, 494] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[487, 506, 508, 486, 521, 508, 502, 529, 478, 498, 524, 492, 499, 489, 487, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 489, 501, 470, 527, 499, 535, 531, 517, 565, 541, 514, 488, 506, 512, 504, 10, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 473, 516, 511, 552, 529, 529, 523, 538, 477, 491, 547, 491, 494, 473, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 468, 538, 488, 484, 523, 546, 525, 511, 530, 490, 552, 495, 481, 548, 452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 549, 486, 513, 543, 501, 506, 494, 545, 490, 504, 516, 490, 539, 484, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 490, 488, 504, 495, 556, 504, 522, 556, 517, 531, 510, 544, 527, 513, 529, 16, 11, 9, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 490, 535, 530, 506, 449, 536, 530, 481, 514, 468, 513, 483, 521, 527, 533, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 496, 505, 529, 515, 542, 522, 500, 500, 520, 524, 529, 504, 510, 516, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8018, 8201, 8182, 8142, 8211, 8363, 8183, 8236] avg=8192.0000 max=8363 max_rank=5 min=8018 min_rank=0 imbalance_ratio=1.020874 max_load_reduction=57173 max_load_reduction_pct=0.872391 moved_to_echo=32993 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[509, 520, 530, 553, 521, 529, 510, 506, 470, 514, 502, 543, 549, 505, 488, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 532, 516, 524, 536, 520, 512, 505, 525, 499, 551, 510, 501, 509, 516, 493, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 542, 480, 544, 483, 461, 477, 522, 507, 501, 531, 528, 520, 523, 523, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 523, 503, 488, 507, 515, 541, 491, 510, 538, 520, 493, 517, 487, 537, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 494, 488, 552, 523, 506, 518, 520, 492, 508, 513, 560, 514, 486, 537, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 489, 467, 474, 490, 551, 509, 486, 525, 530, 483, 525, 504, 493, 552, 496, 13, 8, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 512, 507, 472, 535, 480, 501, 506, 528, 512, 513, 493, 562, 544, 503, 488, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 507, 497, 516, 512, 509, 517, 535, 507, 488, 511, 495, 456, 504, 492, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8232, 8250, 8208, 8220, 8264, 8083, 8191, 8088] avg=8192.0000 max=8264 max_rank=4 min=8083 min_rank=5 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32626 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[537, 497, 489, 526, 512, 508, 523, 506, 551, 483, 524, 525, 514, 552, 507, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 497, 495, 512, 471, 510, 514, 492, 533, 500, 524, 521, 503, 502, 517, 561, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 503, 504, 513, 511, 495, 519, 511, 526, 555, 539, 530, 511, 516, 476, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 525, 493, 511, 494, 490, 509, 537, 541, 474, 503, 538, 493, 526, 512, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 524, 496, 515, 505, 497, 511, 533, 493, 482, 525, 487, 503, 525, 486, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 510, 511, 510, 502, 517, 514, 539, 484, 547, 514, 525, 513, 519, 505, 489, 13, 9, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 509, 504, 469, 551, 499, 500, 539, 466, 476, 519, 542, 540, 492, 497, 496, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 496, 532, 516, 516, 517, 496, 505, 554, 506, 478, 538, 520, 538, 539, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8241, 8137, 8267, 8151, 8091, 8251, 8142, 8256] avg=8192.0000 max=8267 max_rank=2 min=8091 min_rank=4 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32740 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[522, 514, 534, 516, 528, 494, 507, 507, 506, 476, 500, 490, 493, 525, 481, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 490, 551, 518, 516, 483, 538, 532, 534, 507, 469, 520, 480, 533, 506, 552, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 500, 501, 484, 522, 510, 537, 506, 505, 551, 487, 483, 506, 512, 544, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 530, 537, 523, 514, 525, 485, 536, 517, 486, 495, 509, 542, 506, 522, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 527, 531, 530, 560, 506, 534, 536, 509, 480, 469, 512, 501, 528, 516, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 503, 489, 515, 532, 510, 531, 517, 499, 514, 572, 481, 493, 517, 510, 517, 13, 8, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 509, 528, 508, 520, 524, 477, 482, 530, 506, 481, 513, 516, 506, 499, 508, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 489, 507, 512, 529, 554, 511, 490, 511, 521, 489, 490, 469, 495, 496, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8109, 8217, 8184, 8232, 8297, 8220, 8189, 8088] avg=8192.0000 max=8297 max_rank=4 min=8088 min_rank=7 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32794 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[495, 542, 484, 540, 501, 467, 513, 516, 522, 510, 498, 521, 536, 513, 502, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 502, 527, 493, 447, 529, 514, 532, 523, 517, 488, 502, 487, 565, 507, 550, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 556, 482, 482, 495, 520, 526, 511, 516, 518, 482, 515, 544, 540, 497, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 566, 521, 511, 494, 536, 503, 470, 492, 490, 510, 540, 517, 489, 492, 492, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 508, 524, 502, 516, 496, 519, 507, 497, 451, 547, 497, 493, 494, 527, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 476, 513, 528, 520, 519, 503, 533, 521, 507, 518, 500, 548, 510, 532, 485, 14, 9, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 502, 493, 560, 510, 538, 489, 507, 529, 494, 493, 465, 495, 481, 526, 531, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 520, 523, 485, 514, 506, 558, 528, 520, 506, 508, 492, 521, 550, 510, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8210, 8258, 8161, 8104, 8260, 8158, 8224] avg=8192.0000 max=8260 max_rank=5 min=8104 min_rank=4 imbalance_ratio=1.008301 max_load_reduction=57276 max_load_reduction_pct=0.873962 moved_to_echo=32746 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[467, 511, 521, 516, 518, 524, 554, 542, 507, 495, 530, 522, 513, 556, 489, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 495, 539, 488, 473, 521, 504, 481, 516, 488, 492, 531, 481, 557, 495, 492, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 517, 494, 488, 543, 517, 498, 501, 514, 510, 509, 511, 521, 533, 506, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 501, 504, 515, 497, 496, 523, 514, 538, 518, 495, 502, 461, 524, 521, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 522, 520, 529, 511, 474, 521, 546, 503, 456, 548, 493, 500, 508, 544, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 500, 516, 501, 496, 480, 529, 491, 505, 506, 530, 512, 497, 524, 493, 512, 13, 9, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 538, 514, 528, 510, 512, 511, 514, 536, 522, 503, 476, 514, 503, 496, 536, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 570, 497, 536, 524, 500, 517, 516, 508, 527, 491, 503, 501, 512, 479, 480, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8300, 8107, 8211, 8164, 8214, 8105, 8270, 8165] avg=8192.0000 max=8300 max_rank=0 min=8105 min_rank=5 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32754 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[537, 497, 489, 526, 512, 508, 523, 506, 551, 483, 524, 525, 514, 552, 507, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 497, 495, 512, 471, 510, 514, 492, 533, 500, 524, 521, 503, 502, 517, 561, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 503, 504, 513, 511, 495, 519, 511, 526, 555, 539, 530, 511, 516, 476, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 525, 493, 511, 494, 490, 509, 537, 541, 474, 503, 538, 493, 526, 512, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 524, 496, 515, 505, 497, 511, 533, 493, 482, 525, 487, 503, 525, 486, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 510, 511, 510, 502, 517, 514, 539, 484, 547, 514, 525, 513, 519, 505, 489, 13, 9, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 509, 504, 469, 551, 499, 500, 539, 466, 476, 519, 542, 540, 492, 497, 496, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 496, 532, 516, 516, 517, 496, 505, 554, 506, 478, 538, 520, 538, 539, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8241, 8137, 8267, 8151, 8091, 8251, 8142, 8256] avg=8192.0000 max=8267 max_rank=2 min=8091 min_rank=4 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32740 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[498, 503, 509, 489, 539, 485, 506, 510, 565, 512, 505, 541, 502, 516, 470, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 508, 525, 536, 518, 476, 535, 521, 474, 518, 477, 506, 543, 516, 534, 523, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 495, 500, 531, 502, 509, 527, 481, 489, 516, 518, 548, 490, 480, 540, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 528, 533, 548, 533, 540, 537, 483, 499, 525, 512, 507, 524, 523, 516, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 463, 492, 502, 493, 488, 552, 513, 520, 491, 518, 485, 526, 554, 512, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 495, 499, 513, 527, 500, 496, 523, 498, 492, 487, 516, 504, 500, 468, 538, 13, 9, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 531, 500, 516, 476, 499, 502, 487, 487, 531, 491, 521, 504, 524, 500, 557, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 533, 548, 533, 527, 484, 510, 503, 520, 523, 525, 573, 528, 503, 512, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8169, 8214, 8150, 8334, 8119, 8060, 8153, 8337] avg=8192.0000 max=8337 max_rank=7 min=8060 min_rank=5 imbalance_ratio=1.017700 max_load_reduction=57199 max_load_reduction_pct=0.872787 moved_to_echo=32669 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[487, 506, 508, 486, 521, 508, 502, 529, 478, 498, 524, 492, 499, 489, 487, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 489, 501, 470, 527, 499, 535, 531, 517, 565, 541, 514, 488, 506, 512, 504, 10, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 473, 516, 511, 552, 529, 529, 523, 538, 477, 491, 547, 491, 494, 473, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 468, 538, 488, 484, 523, 546, 525, 511, 530, 490, 552, 495, 481, 548, 452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 549, 486, 513, 543, 501, 506, 494, 545, 490, 504, 516, 490, 539, 484, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 490, 488, 504, 495, 556, 504, 522, 556, 517, 531, 510, 544, 527, 513, 529, 16, 11, 9, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 490, 535, 530, 506, 449, 536, 530, 481, 514, 468, 513, 483, 521, 527, 533, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 496, 505, 529, 515, 542, 522, 500, 500, 520, 524, 529, 504, 510, 516, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8018, 8201, 8182, 8142, 8211, 8363, 8183, 8236] avg=8192.0000 max=8363 max_rank=5 min=8018 min_rank=0 imbalance_ratio=1.020874 max_load_reduction=57173 max_load_reduction_pct=0.872391 moved_to_echo=32993 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[495, 504, 506, 485, 507, 557, 527, 511, 493, 550, 529, 516, 493, 531, 545, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 500, 511, 489, 506, 502, 554, 539, 540, 513, 485, 517, 529, 487, 512, 517, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 499, 508, 511, 479, 495, 510, 511, 515, 550, 508, 481, 495, 516, 501, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 521, 509, 488, 511, 492, 470, 509, 528, 533, 500, 535, 515, 528, 489, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 464, 486, 555, 534, 520, 489, 516, 507, 519, 503, 533, 487, 524, 550, 530, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 520, 464, 472, 523, 514, 481, 549, 540, 521, 456, 521, 510, 547, 476, 525, 13, 8, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 516, 541, 511, 522, 530, 501, 490, 503, 513, 515, 520, 497, 512, 507, 548, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 564, 469, 506, 501, 522, 521, 486, 504, 513, 510, 455, 486, 507, 519, 571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8306, 8200, 8076, 8132, 8236, 8194, 8250, 8142] avg=8192.0000 max=8306 max_rank=0 min=8076 min_rank=2 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32822 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[495, 542, 484, 540, 501, 467, 513, 516, 522, 510, 498, 521, 536, 513, 502, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 502, 527, 493, 447, 529, 514, 532, 523, 517, 488, 502, 487, 565, 507, 550, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 556, 482, 482, 495, 520, 526, 511, 516, 518, 482, 515, 544, 540, 497, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 566, 521, 511, 494, 536, 503, 470, 492, 490, 510, 540, 517, 489, 492, 492, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 508, 524, 502, 516, 496, 519, 507, 497, 451, 547, 497, 493, 494, 527, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 476, 513, 528, 520, 519, 503, 533, 521, 507, 518, 500, 548, 510, 532, 485, 14, 9, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 502, 493, 560, 510, 538, 489, 507, 529, 494, 493, 465, 495, 481, 526, 531, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 520, 523, 485, 514, 506, 558, 528, 520, 506, 508, 492, 521, 550, 510, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8210, 8258, 8161, 8104, 8260, 8158, 8224] avg=8192.0000 max=8260 max_rank=5 min=8104 min_rank=4 imbalance_ratio=1.008301 max_load_reduction=57276 max_load_reduction_pct=0.873962 moved_to_echo=32746 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 3968, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 3968, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 3968, 4224, 4224, 4096, 3968, 4224, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[527, 544, 490, 480, 479, 509, 502, 507, 519, 504, 491, 543, 528, 550, 533, 528, 520, 526, 505, 520, 491, 515, 521, 520, 538, 502, 518, 529, 527, 490, 514, 475, 511, 500, 539, 499, 533, 477, 516, 527, 504, 491, 539, 521, 482, 516, 519, 499, 520, 527, 498, 495, 521, 506, 470, 500, 495, 508, 528, 520, 525, 477, 513, 500, 510, 540, 502, 486, 541, 524, 537, 504, 518, 523, 529, 521, 513, 518, 495, 525, 514, 537, 506, 513, 463, 499, 516, 533, 509, 516, 532, 502, 538, 532, 489, 502, 504, 487, 505, 466, 497, 471, 573, 501, 520, 501, 514, 534, 524, 491, 536, 502, 530, 536, 523, 494, 532, 500, 498, 482, 534, 517, 514, 479, 515, 539, 502, 507] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[498, 490, 529, 538, 514, 538, 486, 503, 520, 506, 540, 540, 488, 502, 494, 492, 477, 512, 516, 523, 501, 534, 517, 499, 502, 496, 495, 522, 512, 500, 529, 509, 463, 510, 510, 503, 511, 487, 488, 536, 514, 540, 555, 538, 472, 528, 494, 479, 535, 537, 518, 509, 501, 493, 561, 488, 507, 506, 478, 544, 511, 474, 468, 515, 538, 557, 490, 541, 502, 514, 531, 519, 520, 555, 522, 525, 563, 529, 525, 510, 454, 512, 509, 534, 503, 528, 531, 538, 517, 487, 510, 509, 520, 531, 512, 522, 504, 505, 510, 526, 516, 478, 492, 471, 465, 519, 540, 524, 519, 499, 528, 512, 503, 498, 536, 523, 494, 507, 558, 520, 480, 499, 503, 456, 539, 527, 500, 532] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[480, 479, 500, 509, 533, 512, 492, 547, 452, 505, 477, 475, 523, 510, 534, 514, 507, 487, 514, 459, 536, 522, 511, 488, 523, 525, 563, 451, 505, 497, 530, 483, 456, 485, 517, 512, 499, 554, 552, 501, 520, 523, 466, 506, 498, 508, 552, 537, 521, 505, 511, 514, 545, 522, 502, 501, 529, 521, 507, 547, 525, 519, 471, 522, 498, 534, 522, 481, 525, 511, 507, 474, 512, 496, 510, 545, 496, 512, 519, 519, 465, 520, 540, 527, 503, 545, 489, 529, 495, 477, 562, 485, 512, 479, 516, 524, 522, 527, 495, 520, 505, 562, 547, 498, 538, 518, 533, 478, 501, 584, 531, 510, 511, 514, 510, 562, 494, 481, 519, 527, 546, 489, 490, 531, 528, 533, 510, 502] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[514, 508, 510, 522, 504, 514, 523, 500, 499, 504, 494, 469, 520, 490, 536, 478, 518, 520, 502, 517, 515, 515, 542, 523, 512, 480, 489, 514, 501, 515, 483, 539, 505, 531, 538, 531, 519, 556, 519, 511, 502, 513, 520, 558, 510, 536, 514, 498, 534, 510, 483, 486, 495, 521, 455, 481, 492, 483, 515, 512, 515, 501, 544, 463, 535, 501, 511, 573, 503, 503, 495, 514, 509, 517, 509, 491, 484, 491, 509, 513, 544, 487, 551, 475, 487, 503, 488, 522, 490, 505, 506, 531, 468, 544, 495, 447, 568, 539, 534, 479, 552, 519, 522, 521, 552, 525, 525, 506, 516, 556, 498, 514, 519, 517, 522, 534, 521, 544, 501, 510, 520, 517, 503, 519, 516, 529, 480, 536] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[520, 465, 536, 557, 498, 470, 503, 539, 502, 559, 541, 473, 508, 475, 535, 532, 517, 491, 533, 496, 515, 509, 566, 502, 486, 496, 497, 540, 494, 563, 517, 498, 550, 519, 521, 501, 505, 521, 506, 494, 547, 536, 493, 499, 521, 497, 502, 506, 482, 499, 513, 516, 548, 503, 486, 489, 514, 514, 535, 536, 482, 490, 525, 533, 534, 510, 527, 552, 537, 492, 505, 520, 478, 555, 497, 517, 501, 527, 525, 510, 522, 470, 462, 520, 491, 541, 523, 500, 522, 499, 510, 498, 505, 535, 506, 554, 500, 532, 524, 513, 495, 515, 554, 506, 503, 532, 522, 481, 481, 537, 542, 475, 496, 521, 511, 525, 491, 473, 562, 520, 504, 494, 476, 501, 479, 507, 503, 500] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[505, 502, 497, 538, 534, 467, 557, 523, 543, 513, 500, 529, 524, 519, 539, 494, 517, 540, 459, 510, 514, 511, 518, 512, 507, 502, 526, 499, 482, 569, 521, 464, 520, 472, 504, 498, 532, 521, 502, 524, 534, 517, 509, 533, 518, 565, 528, 499, 511, 502, 538, 521, 501, 543, 507, 515, 472, 493, 546, 538, 480, 483, 508, 513, 478, 515, 481, 516, 492, 498, 505, 497, 509, 503, 538, 442, 529, 507, 516, 475, 527, 522, 510, 514, 481, 513, 495, 551, 510, 480, 518, 507, 517, 496, 536, 504, 515, 521, 513, 507, 482, 497, 501, 540, 524, 509, 507, 542, 531, 495, 534, 505, 493, 505, 500, 522, 524, 498, 533, 520, 495, 532, 523, 539, 491, 532, 557, 485] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[520, 465, 536, 557, 498, 470, 503, 539, 502, 559, 541, 473, 508, 475, 535, 532, 517, 491, 533, 496, 515, 509, 566, 502, 486, 496, 497, 540, 494, 563, 517, 498, 550, 519, 521, 501, 505, 521, 506, 494, 547, 536, 493, 499, 521, 497, 502, 506, 482, 499, 513, 516, 548, 503, 486, 489, 514, 514, 535, 536, 482, 490, 525, 533, 534, 510, 527, 552, 537, 492, 505, 520, 478, 555, 497, 517, 501, 527, 525, 510, 522, 470, 462, 520, 491, 541, 523, 500, 522, 499, 510, 498, 505, 535, 506, 554, 500, 532, 524, 513, 495, 515, 554, 506, 503, 532, 522, 481, 481, 537, 542, 475, 496, 521, 511, 525, 491, 473, 562, 520, 504, 494, 476, 501, 479, 507, 503, 500] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[509, 475, 534, 487, 503, 494, 505, 484, 509, 501, 562, 520, 509, 501, 499, 502, 528, 536, 495, 520, 495, 548, 528, 524, 513, 519, 548, 547, 518, 473, 505, 533, 471, 514, 524, 505, 458, 524, 516, 527, 538, 501, 534, 501, 525, 520, 521, 563, 482, 521, 484, 507, 522, 492, 498, 481, 564, 497, 532, 502, 506, 494, 503, 542, 529, 511, 503, 545, 514, 511, 527, 533, 491, 498, 489, 507, 523, 538, 503, 490, 526, 524, 516, 487, 516, 509, 489, 484, 542, 502, 501, 520, 540, 484, 519, 512, 517, 523, 536, 495, 512, 557, 508, 498, 542, 525, 497, 493, 502, 515, 498, 460, 509, 494, 527, 535, 505, 495, 566, 522, 494, 505, 515, 488, 470, 519, 517, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[538, 487, 470, 548, 527, 490, 525, 505, 518, 522, 498, 501, 480, 529, 564, 553, 502, 549, 506, 526, 485, 527, 507, 511, 506, 548, 503, 501, 496, 534, 529, 497, 524, 492, 542, 515, 492, 497, 495, 525, 540, 538, 510, 518, 500, 479, 524, 499, 522, 531, 534, 501, 499, 504, 554, 498, 523, 489, 499, 507, 481, 467, 499, 505, 532, 522, 504, 515, 523, 494, 540, 521, 493, 473, 496, 511, 514, 551, 533, 501, 521, 505, 533, 524, 534, 499, 532, 546, 518, 512, 525, 503, 483, 517, 488, 482, 516, 556, 517, 473, 538, 555, 487, 468, 524, 567, 500, 509, 498, 517, 542, 455, 488, 528, 505, 518, 498, 508, 530, 536, 516, 501, 479, 507, 496, 492, 526, 456] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[514, 508, 510, 522, 504, 514, 523, 500, 499, 504, 494, 469, 520, 490, 536, 478, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 520, 502, 517, 515, 515, 538, 523, 512, 480, 489, 514, 501, 515, 483, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 531, 538, 531, 519, 556, 519, 511, 475, 513, 520, 558, 510, 536, 514, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 510, 483, 486, 495, 521, 455, 481, 492, 483, 515, 512, 515, 501, 544, 463, 38, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 501, 511, 543, 503, 503, 495, 514, 509, 517, 509, 491, 484, 491, 509, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 487, 551, 475, 487, 503, 488, 522, 490, 505, 506, 531, 468, 544, 495, 447, 12, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 568, 539, 534, 479, 552, 519, 522, 521, 552, 525, 525, 506, 516, 556, 460, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 517, 522, 534, 521, 544, 501, 510, 520, 517, 503, 519, 516, 529, 480, 536, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8105, 8181, 8334, 8043, 8128, 8064, 8388, 8293] avg=8192.0000 max=8388 max_rank=6 min=8043 min_rank=3 imbalance_ratio=1.023926 max_load_reduction=57148 max_load_reduction_pct=0.872009 moved_to_echo=32873 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[480, 479, 500, 509, 533, 512, 492, 547, 452, 505, 477, 475, 523, 510, 534, 514, 507, 487, 514, 459, 536, 522, 511, 488, 523, 525, 563, 451, 505, 497, 530, 483, 456, 485, 517, 512, 499, 554, 552, 501, 520, 523, 466, 506, 498, 508, 552, 537, 521, 505, 511, 514, 545, 522, 502, 501, 529, 521, 507, 547, 525, 519, 471, 522, 498, 534, 522, 481, 525, 511, 507, 474, 512, 496, 510, 545, 496, 512, 519, 519, 465, 520, 540, 527, 503, 545, 489, 529, 495, 477, 562, 485, 512, 479, 516, 524, 522, 527, 495, 520, 505, 562, 547, 498, 538, 518, 533, 478, 501, 584, 531, 510, 511, 514, 510, 562, 494, 481, 519, 527, 546, 489, 490, 531, 528, 533, 510, 502] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[498, 490, 529, 538, 514, 538, 486, 503, 520, 506, 540, 540, 488, 502, 494, 492, 477, 512, 516, 523, 501, 534, 517, 499, 502, 496, 495, 522, 512, 500, 529, 509, 463, 510, 510, 503, 511, 487, 488, 536, 514, 540, 555, 538, 472, 528, 494, 479, 535, 537, 518, 509, 501, 493, 561, 488, 507, 506, 478, 544, 511, 474, 468, 515, 538, 557, 490, 541, 502, 514, 531, 519, 520, 555, 522, 525, 563, 529, 525, 510, 454, 512, 509, 534, 503, 528, 531, 538, 517, 487, 510, 509, 520, 531, 512, 522, 504, 505, 510, 526, 516, 478, 492, 471, 465, 519, 540, 524, 519, 499, 528, 512, 503, 498, 536, 523, 494, 507, 558, 520, 480, 499, 503, 456, 539, 527, 500, 532] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[527, 544, 490, 480, 479, 509, 502, 507, 519, 504, 491, 543, 528, 550, 533, 528, 520, 526, 505, 520, 491, 515, 521, 520, 538, 502, 518, 529, 527, 490, 514, 475, 511, 500, 539, 499, 533, 477, 516, 527, 504, 491, 539, 521, 482, 516, 519, 499, 520, 527, 498, 495, 521, 506, 470, 500, 495, 508, 528, 520, 525, 477, 513, 500, 510, 540, 502, 486, 541, 524, 537, 504, 518, 523, 529, 521, 513, 518, 495, 525, 514, 537, 506, 513, 463, 499, 516, 533, 509, 516, 532, 502, 538, 532, 489, 502, 504, 487, 505, 466, 497, 471, 573, 501, 520, 501, 514, 534, 524, 491, 536, 502, 530, 536, 523, 494, 532, 500, 498, 482, 534, 517, 514, 479, 515, 539, 502, 507] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[514, 508, 510, 522, 504, 514, 523, 500, 499, 504, 494, 469, 520, 490, 536, 478, 518, 520, 502, 517, 515, 515, 542, 523, 512, 480, 489, 514, 501, 515, 483, 539, 505, 531, 538, 531, 519, 556, 519, 511, 502, 513, 520, 558, 510, 536, 514, 498, 534, 510, 483, 486, 495, 521, 455, 481, 492, 483, 515, 512, 515, 501, 544, 463, 535, 501, 511, 573, 503, 503, 495, 514, 509, 517, 509, 491, 484, 491, 509, 513, 544, 487, 551, 475, 487, 503, 488, 522, 490, 505, 506, 531, 468, 544, 495, 447, 568, 539, 534, 479, 552, 519, 522, 521, 552, 525, 525, 506, 516, 556, 498, 514, 519, 517, 522, 534, 521, 544, 501, 510, 520, 517, 503, 519, 516, 529, 480, 536] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[480, 479, 500, 509, 533, 512, 492, 547, 452, 505, 477, 475, 523, 510, 534, 514, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 487, 514, 459, 536, 522, 507, 488, 523, 525, 563, 451, 505, 497, 530, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 456, 485, 517, 512, 499, 554, 552, 501, 491, 523, 466, 506, 498, 508, 552, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 505, 511, 514, 545, 522, 502, 501, 529, 521, 507, 547, 525, 519, 471, 522, 41, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 534, 522, 456, 525, 511, 507, 474, 512, 496, 510, 545, 496, 512, 519, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 520, 540, 527, 503, 545, 489, 529, 495, 477, 562, 485, 512, 479, 516, 524, 13, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 527, 495, 520, 505, 562, 547, 498, 538, 518, 533, 478, 501, 584, 490, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 514, 510, 562, 494, 481, 519, 527, 546, 489, 490, 531, 528, 533, 510, 502, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8059, 8097, 8157, 8319, 8136, 8188, 8328, 8252] avg=8192.0000 max=8328 max_rank=6 min=8059 min_rank=0 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=32904 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[527, 544, 490, 480, 479, 509, 502, 507, 519, 504, 491, 543, 528, 550, 533, 528, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 526, 505, 520, 491, 515, 515, 520, 538, 502, 518, 529, 527, 490, 514, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 500, 539, 499, 533, 477, 516, 527, 467, 491, 539, 521, 482, 516, 519, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 527, 498, 495, 521, 506, 470, 500, 495, 508, 528, 520, 525, 477, 513, 500, 45, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 540, 502, 447, 541, 524, 537, 504, 518, 523, 529, 521, 513, 518, 495, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 537, 506, 513, 463, 499, 516, 533, 509, 516, 532, 502, 538, 532, 489, 502, 17, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 487, 505, 466, 497, 471, 573, 501, 520, 501, 514, 534, 524, 491, 491, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 536, 523, 494, 532, 500, 498, 482, 534, 517, 514, 479, 515, 539, 502, 507, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8255, 8205, 8136, 8168, 8247, 8230, 8081, 8214] avg=8192.0000 max=8255 max_rank=0 min=8081 min_rank=6 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32772 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[498, 490, 529, 538, 514, 538, 486, 503, 520, 506, 540, 540, 488, 502, 494, 492, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 512, 516, 523, 501, 534, 513, 499, 502, 496, 495, 522, 512, 500, 529, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 463, 510, 510, 503, 511, 487, 488, 536, 486, 540, 555, 538, 472, 528, 494, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 537, 518, 509, 501, 493, 561, 488, 507, 506, 478, 544, 511, 474, 468, 515, 41, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 557, 490, 513, 502, 514, 531, 519, 520, 555, 522, 525, 563, 529, 525, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 454, 512, 509, 534, 503, 528, 531, 538, 517, 487, 510, 509, 520, 531, 512, 522, 12, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 505, 510, 526, 516, 478, 492, 471, 465, 519, 540, 524, 519, 499, 487, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 498, 536, 523, 494, 507, 558, 520, 480, 499, 503, 456, 539, 527, 500, 532, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8140, 8100, 8202, 8413, 8237, 8067, 8180] avg=8192.0000 max=8413 max_rank=4 min=8067 min_rank=6 imbalance_ratio=1.026978 max_load_reduction=57123 max_load_reduction_pct=0.871628 moved_to_echo=32897 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[520, 465, 536, 557, 498, 470, 503, 539, 502, 559, 541, 473, 508, 475, 535, 532, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 491, 533, 496, 515, 509, 562, 502, 486, 496, 497, 540, 494, 563, 517, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 519, 521, 501, 505, 521, 506, 494, 517, 536, 493, 499, 521, 497, 502, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 499, 513, 516, 548, 503, 486, 489, 514, 514, 535, 536, 482, 490, 525, 533, 42, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 510, 527, 523, 537, 492, 505, 520, 478, 555, 497, 517, 501, 527, 525, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 470, 462, 520, 491, 541, 523, 500, 522, 499, 510, 498, 505, 535, 506, 554, 13, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 532, 524, 513, 495, 515, 554, 506, 503, 532, 522, 481, 481, 537, 500, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 521, 511, 525, 491, 473, 562, 520, 504, 494, 476, 501, 479, 507, 503, 500, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8232, 8216, 8188, 8224, 8258, 8180, 8170, 8068] avg=8192.0000 max=8258 max_rank=4 min=8068 min_rank=7 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32676 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[509, 475, 534, 487, 503, 494, 505, 484, 509, 501, 562, 520, 509, 501, 499, 502, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 536, 495, 520, 495, 548, 524, 524, 513, 519, 548, 547, 518, 473, 505, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 514, 524, 505, 458, 524, 516, 527, 509, 501, 534, 501, 525, 520, 521, 563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 521, 484, 507, 522, 492, 498, 481, 564, 497, 532, 502, 506, 494, 503, 542, 38, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 511, 503, 517, 514, 511, 527, 533, 491, 498, 489, 507, 523, 538, 503, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 524, 516, 487, 516, 509, 489, 484, 542, 502, 501, 520, 540, 484, 519, 512, 13, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 523, 536, 495, 512, 557, 508, 498, 542, 525, 497, 493, 502, 515, 460, 460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 494, 527, 535, 505, 495, 566, 522, 494, 505, 515, 488, 470, 519, 517, 521, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8113, 8326, 8213, 8181, 8184, 8192, 8140, 8187] avg=8192.0000 max=8326 max_rank=1 min=8113 min_rank=0 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32703 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[505, 502, 497, 538, 534, 467, 557, 523, 543, 513, 500, 529, 524, 519, 539, 494, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 540, 459, 510, 514, 511, 514, 512, 507, 502, 526, 499, 482, 569, 521, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 472, 504, 498, 532, 521, 502, 524, 505, 517, 509, 533, 518, 565, 528, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 502, 538, 521, 501, 543, 507, 515, 472, 493, 546, 538, 480, 483, 508, 513, 41, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 515, 481, 489, 492, 498, 505, 497, 509, 503, 538, 442, 529, 507, 516, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 522, 510, 514, 481, 513, 495, 551, 510, 480, 518, 507, 517, 496, 536, 504, 13, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 521, 513, 507, 482, 497, 501, 540, 524, 509, 507, 542, 531, 495, 493, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 505, 500, 522, 524, 498, 533, 520, 495, 532, 523, 539, 491, 532, 557, 485, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8302, 8147, 8247, 8228, 7974, 8202, 8182, 8254] avg=8192.0000 max=8302 max_rank=0 min=7974 min_rank=4 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32612 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[520, 465, 536, 557, 498, 470, 503, 539, 502, 559, 541, 473, 508, 475, 535, 532, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 491, 533, 496, 515, 509, 562, 502, 486, 496, 497, 540, 494, 563, 517, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 519, 521, 501, 505, 521, 506, 494, 517, 536, 493, 499, 521, 497, 502, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 499, 513, 516, 548, 503, 486, 489, 514, 514, 535, 536, 482, 490, 525, 533, 42, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 510, 527, 523, 537, 492, 505, 520, 478, 555, 497, 517, 501, 527, 525, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 470, 462, 520, 491, 541, 523, 500, 522, 499, 510, 498, 505, 535, 506, 554, 13, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 532, 524, 513, 495, 515, 554, 506, 503, 532, 522, 481, 481, 537, 500, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 521, 511, 525, 491, 473, 562, 520, 504, 494, 476, 501, 479, 507, 503, 500, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8232, 8216, 8188, 8224, 8258, 8180, 8170, 8068] avg=8192.0000 max=8258 max_rank=4 min=8068 min_rank=7 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32676 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[538, 487, 470, 548, 527, 490, 525, 505, 518, 522, 498, 501, 480, 529, 564, 553, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 549, 506, 526, 485, 527, 504, 511, 506, 548, 503, 501, 496, 534, 529, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 492, 542, 515, 492, 497, 495, 525, 511, 538, 510, 518, 500, 479, 524, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 531, 534, 501, 499, 504, 554, 498, 523, 489, 499, 507, 481, 467, 499, 505, 42, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 522, 504, 488, 523, 494, 540, 521, 493, 473, 496, 511, 514, 551, 533, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 505, 533, 524, 534, 499, 532, 546, 518, 512, 525, 503, 483, 517, 488, 482, 13, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 556, 517, 473, 538, 555, 487, 468, 524, 567, 500, 509, 498, 517, 500, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 528, 505, 518, 498, 508, 530, 536, 516, 501, 479, 507, 496, 492, 526, 456, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8273, 8224, 8161, 8171, 8196, 8243, 8180, 8088] avg=8192.0000 max=8273 max_rank=0 min=8088 min_rank=7 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32707 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[509, 475, 534, 487, 503, 494, 505, 484, 509, 501, 562, 520, 509, 501, 499, 502, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 536, 495, 520, 495, 548, 524, 524, 513, 519, 548, 547, 518, 473, 505, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 514, 524, 505, 458, 524, 516, 527, 509, 501, 534, 501, 525, 520, 521, 563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 521, 484, 507, 522, 492, 498, 481, 564, 497, 532, 502, 506, 494, 503, 542, 38, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 511, 503, 517, 514, 511, 527, 533, 491, 498, 489, 507, 523, 538, 503, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 524, 516, 487, 516, 509, 489, 484, 542, 502, 501, 520, 540, 484, 519, 512, 13, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 523, 536, 495, 512, 557, 508, 498, 542, 525, 497, 493, 502, 515, 460, 460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 494, 527, 535, 505, 495, 566, 522, 494, 505, 515, 488, 470, 519, 517, 521, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8113, 8326, 8213, 8181, 8184, 8192, 8140, 8187] avg=8192.0000 max=8326 max_rank=1 min=8113 min_rank=0 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32703 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[527, 544, 490, 480, 479, 509, 502, 507, 519, 504, 491, 543, 528, 550, 533, 528, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 526, 505, 520, 491, 515, 515, 520, 538, 502, 518, 529, 527, 490, 514, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 500, 539, 499, 533, 477, 516, 527, 467, 491, 539, 521, 482, 516, 519, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 527, 498, 495, 521, 506, 470, 500, 495, 508, 528, 520, 525, 477, 513, 500, 45, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 540, 502, 447, 541, 524, 537, 504, 518, 523, 529, 521, 513, 518, 495, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 537, 506, 513, 463, 499, 516, 533, 509, 516, 532, 502, 538, 532, 489, 502, 17, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 487, 505, 466, 497, 471, 573, 501, 520, 501, 514, 534, 524, 491, 491, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 536, 523, 494, 532, 500, 498, 482, 534, 517, 514, 479, 515, 539, 502, 507, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8255, 8205, 8136, 8168, 8247, 8230, 8081, 8214] avg=8192.0000 max=8255 max_rank=0 min=8081 min_rank=6 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32772 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[498, 490, 529, 538, 514, 538, 486, 503, 520, 506, 540, 540, 488, 502, 494, 492, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 512, 516, 523, 501, 534, 513, 499, 502, 496, 495, 522, 512, 500, 529, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 463, 510, 510, 503, 511, 487, 488, 536, 486, 540, 555, 538, 472, 528, 494, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 537, 518, 509, 501, 493, 561, 488, 507, 506, 478, 544, 511, 474, 468, 515, 41, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 557, 490, 513, 502, 514, 531, 519, 520, 555, 522, 525, 563, 529, 525, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 454, 512, 509, 534, 503, 528, 531, 538, 517, 487, 510, 509, 520, 531, 512, 522, 12, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 505, 510, 526, 516, 478, 492, 471, 465, 519, 540, 524, 519, 499, 487, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 498, 536, 523, 494, 507, 558, 520, 480, 499, 503, 456, 539, 527, 500, 532, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8140, 8100, 8202, 8413, 8237, 8067, 8180] avg=8192.0000 max=8413 max_rank=4 min=8067 min_rank=6 imbalance_ratio=1.026978 max_load_reduction=57123 max_load_reduction_pct=0.871628 moved_to_echo=32897 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[514, 508, 510, 522, 504, 514, 523, 500, 499, 504, 494, 469, 520, 490, 536, 478, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 520, 502, 517, 515, 515, 538, 523, 512, 480, 489, 514, 501, 515, 483, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 531, 538, 531, 519, 556, 519, 511, 475, 513, 520, 558, 510, 536, 514, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 510, 483, 486, 495, 521, 455, 481, 492, 483, 515, 512, 515, 501, 544, 463, 38, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 501, 511, 543, 503, 503, 495, 514, 509, 517, 509, 491, 484, 491, 509, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 487, 551, 475, 487, 503, 488, 522, 490, 505, 506, 531, 468, 544, 495, 447, 12, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 568, 539, 534, 479, 552, 519, 522, 521, 552, 525, 525, 506, 516, 556, 460, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 517, 522, 534, 521, 544, 501, 510, 520, 517, 503, 519, 516, 529, 480, 536, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8105, 8181, 8334, 8043, 8128, 8064, 8388, 8293] avg=8192.0000 max=8388 max_rank=6 min=8043 min_rank=3 imbalance_ratio=1.023926 max_load_reduction=57148 max_load_reduction_pct=0.872009 moved_to_echo=32873 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 3968, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4352, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4096, 3968, 4096, 4096, 4224, 4224, 4096, 3968, 4096, 4096, 384, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4352, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 3968, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 3968, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4096, 3968, 4096, 4096, 4224, 4224, 4096, 3968, 4096, 4096, 384, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4352, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True - [2026-08-26 14:49:28] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 418.9 | throughput per GPU (TFLOP/s/GPU): 431.6 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.040528E+01 | loss scale: 1.0 | grad norm: 88.634 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[518, 513, 523, 508, 509, 532, 532, 524, 476, 507, 493, 491, 511, 527, 520, 515, 522, 476, 537, 526, 521, 503, 520, 495, 515, 499, 492, 536, 469, 507, 531, 488, 513, 535, 527, 507, 476, 502, 478, 515, 505, 485, 484, 496, 537, 512, 496, 507, 522, 484, 542, 507, 527, 544, 493, 517, 483, 485, 480, 517, 521, 502, 501, 500, 532, 504, 515, 502, 524, 505, 544, 513, 491, 520, 509, 493, 529, 522, 524, 528, 510, 510, 495, 574, 494, 481, 500, 544, 499, 500, 500, 527, 523, 559, 499, 502, 543, 580, 512, 514, 545, 524, 509, 501, 516, 490, 503, 505, 525, 469, 535, 533, 545, 519, 497, 505, 509, 516, 484, 551, 509, 531, 521, 520, 497, 509, 492, 519] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[517, 520, 523, 553, 501, 483, 519, 480, 503, 518, 528, 516, 539, 526, 482, 512, 511, 519, 524, 506, 551, 521, 495, 461, 511, 524, 525, 493, 516, 511, 527, 519, 538, 539, 539, 535, 536, 563, 519, 499, 488, 513, 518, 519, 462, 510, 508, 522, 507, 560, 532, 524, 528, 507, 557, 514, 490, 525, 511, 528, 486, 477, 519, 507, 524, 506, 505, 485, 491, 525, 510, 502, 501, 510, 503, 494, 540, 490, 488, 501, 527, 537, 496, 528, 531, 526, 496, 522, 491, 512, 511, 508, 501, 518, 524, 516, 512, 477, 504, 530, 526, 524, 515, 485, 493, 506, 537, 489, 501, 497, 493, 477, 525, 551, 499, 505, 520, 535, 550, 527, 496, 481, 466, 475, 509, 493, 486, 519] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[532, 555, 511, 553, 502, 472, 538, 512, 537, 501, 508, 491, 564, 496, 521, 517, 519, 479, 521, 492, 467, 513, 470, 507, 546, 512, 517, 531, 513, 521, 520, 518, 524, 545, 510, 493, 548, 553, 525, 502, 531, 518, 534, 516, 513, 528, 511, 488, 544, 531, 498, 520, 522, 527, 513, 486, 506, 531, 516, 500, 511, 497, 505, 533, 525, 475, 491, 474, 505, 486, 532, 490, 533, 494, 519, 531, 484, 520, 553, 513, 536, 502, 496, 513, 464, 572, 474, 494, 521, 486, 491, 499, 540, 507, 469, 527, 527, 505, 550, 495, 516, 507, 511, 502, 502, 482, 519, 530, 494, 518, 511, 472, 523, 538, 489, 544, 521, 506, 487, 476, 520, 518, 514, 501, 509, 478, 502, 518] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[470, 494, 545, 585, 521, 514, 522, 536, 528, 507, 549, 544, 502, 510, 448, 517, 516, 530, 523, 482, 495, 513, 506, 532, 512, 527, 507, 494, 502, 502, 513, 498, 499, 515, 509, 497, 517, 558, 558, 488, 525, 520, 537, 520, 542, 480, 514, 522, 532, 504, 547, 478, 522, 547, 531, 500, 503, 482, 508, 523, 476, 491, 508, 469, 568, 462, 481, 483, 528, 507, 508, 503, 508, 531, 562, 496, 529, 500, 501, 491, 500, 494, 525, 506, 519, 498, 526, 487, 499, 487, 485, 560, 484, 491, 506, 507, 477, 548, 497, 509, 548, 480, 513, 504, 530, 488, 532, 519, 505, 504, 493, 543, 519, 500, 547, 503, 510, 546, 544, 532, 511, 503, 513, 498, 527, 489, 500, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[486, 473, 495, 497, 481, 497, 532, 534, 526, 537, 486, 488, 544, 521, 495, 549, 479, 527, 520, 514, 515, 495, 498, 489, 532, 503, 520, 508, 504, 532, 520, 498, 539, 506, 516, 520, 483, 501, 508, 503, 479, 525, 503, 571, 517, 521, 511, 528, 480, 511, 546, 514, 582, 518, 532, 511, 499, 530, 572, 530, 515, 491, 554, 504, 471, 506, 503, 495, 530, 525, 535, 497, 506, 488, 483, 511, 543, 539, 555, 492, 538, 500, 517, 500, 472, 569, 489, 527, 503, 490, 513, 513, 477, 466, 475, 472, 510, 533, 510, 523, 487, 503, 532, 542, 501, 503, 507, 476, 514, 512, 501, 525, 527, 524, 488, 521, 522, 521, 513, 551, 529, 513, 514, 504, 481, 533, 491, 510] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 + [2026-08-26 15:25:32] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 431.0 | throughput per GPU (TFLOP/s/GPU): 419.6 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.039367E+01 | loss scale: 1.0 | grad norm: 88.230 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[553, 517, 468, 560, 540, 519, 509, 503, 507, 535, 523, 487, 502, 548, 501, 506, 541, 533, 513, 539, 502, 523, 564, 453, 495, 518, 451, 497, 501, 530, 517, 529, 486, 502, 517, 488, 502, 498, 471, 507, 514, 529, 555, 554, 508, 505, 529, 535, 528, 511, 526, 532, 492, 502, 505, 519, 521, 476, 512, 496, 509, 523, 504, 501, 524, 536, 553, 510, 520, 520, 518, 499, 517, 527, 493, 561, 523, 491, 486, 515, 516, 520, 499, 517, 526, 521, 490, 548, 521, 474, 488, 481, 492, 481, 539, 491, 538, 528, 495, 500, 510, 518, 508, 515, 482, 492, 559, 478, 524, 512, 526, 502, 485, 523, 541, 518, 502, 511, 529, 504, 504, 486, 498, 497, 513, 517, 489, 494] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[505, 467, 492, 500, 506, 523, 546, 503, 514, 525, 516, 538, 487, 483, 530, 557, 500, 512, 537, 523, 536, 523, 469, 550, 491, 538, 473, 569, 469, 527, 520, 492, 562, 517, 531, 483, 486, 536, 531, 506, 514, 501, 534, 526, 508, 523, 443, 465, 556, 561, 503, 523, 495, 545, 509, 476, 488, 502, 532, 538, 519, 552, 537, 522, 508, 545, 510, 507, 496, 504, 512, 526, 520, 509, 515, 500, 497, 496, 520, 533, 475, 486, 513, 516, 503, 480, 520, 505, 489, 522, 509, 509, 513, 525, 540, 495, 535, 502, 505, 487, 493, 509, 481, 521, 529, 479, 523, 514, 471, 522, 515, 529, 538, 504, 515, 519, 521, 508, 512, 529, 512, 482, 536, 512, 493, 488, 515, 494] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[470, 494, 545, 585, 521, 514, 522, 536, 528, 507, 549, 544, 502, 510, 448, 517, 516, 530, 523, 482, 495, 513, 506, 532, 512, 527, 507, 494, 502, 502, 513, 498, 499, 515, 509, 497, 517, 558, 558, 488, 525, 520, 537, 520, 542, 480, 514, 522, 532, 504, 547, 478, 522, 547, 531, 500, 503, 482, 508, 523, 476, 491, 508, 469, 568, 462, 481, 483, 528, 507, 508, 503, 508, 531, 562, 496, 529, 500, 501, 491, 500, 494, 525, 506, 519, 498, 526, 487, 499, 487, 485, 560, 484, 491, 506, 507, 477, 548, 497, 509, 548, 480, 513, 504, 530, 488, 532, 519, 505, 504, 493, 543, 519, 500, 547, 503, 510, 546, 544, 532, 511, 503, 513, 498, 527, 489, 500, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[532, 555, 511, 553, 502, 472, 538, 512, 537, 501, 508, 491, 564, 496, 521, 517, 519, 479, 521, 492, 467, 513, 470, 507, 546, 512, 517, 531, 513, 521, 520, 518, 524, 545, 510, 493, 548, 553, 525, 502, 531, 518, 534, 516, 513, 528, 511, 488, 544, 531, 498, 520, 522, 527, 513, 486, 506, 531, 516, 500, 511, 497, 505, 533, 525, 475, 491, 474, 505, 486, 532, 490, 533, 494, 519, 531, 484, 520, 553, 513, 536, 502, 496, 513, 464, 572, 474, 494, 521, 486, 491, 499, 540, 507, 469, 527, 527, 505, 550, 495, 516, 507, 511, 502, 502, 482, 519, 530, 494, 518, 511, 472, 523, 538, 489, 544, 521, 506, 487, 476, 520, 518, 514, 501, 509, 478, 502, 518] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[546, 479, 500, 533, 536, 487, 494, 535, 502, 488, 559, 511, 531, 519, 548, 512, 506, 510, 498, 476, 496, 519, 500, 501, 524, 511, 512, 491, 487, 512, 545, 518, 496, 537, 503, 496, 524, 522, 467, 490, 505, 524, 530, 498, 534, 519, 530, 507, 514, 495, 548, 500, 539, 530, 511, 519, 508, 550, 497, 523, 526, 497, 471, 513, 521, 530, 537, 558, 488, 517, 532, 532, 523, 532, 460, 523, 532, 498, 554, 484, 496, 514, 487, 496, 532, 530, 486, 501, 517, 502, 509, 502, 479, 486, 488, 525, 504, 496, 523, 506, 509, 505, 498, 504, 550, 481, 486, 541, 541, 499, 531, 522, 506, 513, 521, 503, 528, 497, 549, 489, 484, 497, 549, 526, 468, 497, 510, 523] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[553, 517, 468, 560, 540, 519, 509, 503, 507, 535, 523, 487, 502, 548, 501, 506, 541, 533, 513, 539, 502, 523, 564, 453, 495, 518, 451, 497, 501, 530, 517, 529, 486, 502, 517, 488, 502, 498, 471, 507, 514, 529, 555, 554, 508, 505, 529, 535, 528, 511, 526, 532, 492, 502, 505, 519, 521, 476, 512, 496, 509, 523, 504, 501, 524, 536, 553, 510, 520, 520, 518, 499, 517, 527, 493, 561, 523, 491, 486, 515, 516, 520, 499, 517, 526, 521, 490, 548, 521, 474, 488, 481, 492, 481, 539, 491, 538, 528, 495, 500, 510, 518, 508, 515, 482, 492, 559, 478, 524, 512, 526, 502, 485, 523, 541, 518, 502, 511, 529, 504, 504, 486, 498, 497, 513, 517, 489, 494] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[517, 520, 523, 505, 501, 483, 519, 480, 503, 518, 528, 516, 539, 526, 482, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 519, 524, 506, 551, 521, 495, 461, 511, 524, 525, 493, 516, 511, 527, 519, 13, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 539, 539, 535, 536, 528, 519, 499, 488, 513, 518, 519, 462, 510, 508, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 560, 487, 524, 528, 507, 557, 514, 490, 525, 511, 528, 486, 477, 519, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 506, 505, 485, 491, 525, 502, 502, 501, 510, 503, 494, 540, 490, 488, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 537, 496, 528, 531, 526, 496, 522, 491, 512, 511, 508, 501, 518, 524, 516, 48, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 477, 504, 530, 526, 524, 515, 485, 493, 506, 537, 489, 501, 497, 493, 477, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 551, 499, 505, 520, 535, 550, 527, 496, 481, 466, 475, 509, 493, 486, 519, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8238, 8273, 8227, 8067, 8324, 8087, 8148] avg=8192.0000 max=8324 max_rank=5 min=8067 min_rank=4 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32626 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[486, 473, 495, 497, 481, 497, 532, 534, 526, 537, 486, 488, 544, 521, 495, 549, 479, 527, 520, 514, 515, 495, 498, 489, 532, 503, 520, 508, 504, 532, 520, 498, 539, 506, 516, 520, 483, 501, 508, 503, 479, 525, 503, 571, 517, 521, 511, 528, 480, 511, 546, 514, 582, 518, 532, 511, 499, 530, 572, 530, 515, 491, 554, 504, 471, 506, 503, 495, 530, 525, 535, 497, 506, 488, 483, 511, 543, 539, 555, 492, 538, 500, 517, 500, 472, 569, 489, 527, 503, 490, 513, 513, 477, 466, 475, 472, 510, 533, 510, 523, 487, 503, 532, 542, 501, 503, 507, 476, 514, 512, 501, 525, 527, 524, 488, 521, 522, 521, 513, 551, 529, 513, 514, 504, 481, 533, 491, 510] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[517, 520, 523, 553, 501, 483, 519, 480, 503, 518, 528, 516, 539, 526, 482, 512, 511, 519, 524, 506, 551, 521, 495, 461, 511, 524, 525, 493, 516, 511, 527, 519, 538, 539, 539, 535, 536, 563, 519, 499, 488, 513, 518, 519, 462, 510, 508, 522, 507, 560, 532, 524, 528, 507, 557, 514, 490, 525, 511, 528, 486, 477, 519, 507, 524, 506, 505, 485, 491, 525, 510, 502, 501, 510, 503, 494, 540, 490, 488, 501, 527, 537, 496, 528, 531, 526, 496, 522, 491, 512, 511, 508, 501, 518, 524, 516, 512, 477, 504, 530, 526, 524, 515, 485, 493, 506, 537, 489, 501, 497, 493, 477, 525, 551, 499, 505, 520, 535, 550, 527, 496, 481, 466, 475, 509, 493, 486, 519] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[518, 513, 523, 508, 509, 532, 532, 524, 476, 507, 493, 491, 511, 527, 520, 515, 522, 476, 537, 526, 521, 503, 520, 495, 515, 499, 492, 536, 469, 507, 531, 488, 513, 535, 527, 507, 476, 502, 478, 515, 505, 485, 484, 496, 537, 512, 496, 507, 522, 484, 542, 507, 527, 544, 493, 517, 483, 485, 480, 517, 521, 502, 501, 500, 532, 504, 515, 502, 524, 505, 544, 513, 491, 520, 509, 493, 529, 522, 524, 528, 510, 510, 495, 574, 494, 481, 500, 544, 499, 500, 500, 527, 523, 559, 499, 502, 543, 580, 512, 514, 545, 524, 509, 501, 516, 490, 503, 505, 525, 469, 535, 533, 545, 519, 497, 505, 509, 516, 484, 551, 509, 531, 521, 520, 497, 509, 492, 519] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[553, 517, 468, 511, 540, 519, 509, 503, 507, 535, 523, 487, 502, 548, 501, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 533, 513, 539, 502, 523, 564, 453, 495, 518, 451, 497, 501, 530, 517, 529, 13, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 502, 517, 488, 502, 469, 471, 507, 514, 529, 555, 554, 508, 505, 529, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 511, 481, 532, 492, 502, 505, 519, 521, 476, 512, 496, 509, 523, 504, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 536, 553, 510, 520, 520, 510, 499, 517, 527, 493, 561, 523, 491, 486, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 520, 499, 517, 526, 521, 490, 548, 521, 474, 488, 481, 492, 481, 539, 491, 49, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 528, 495, 500, 510, 518, 508, 515, 482, 492, 559, 478, 524, 512, 526, 502, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 523, 541, 518, 502, 511, 529, 504, 504, 486, 498, 497, 513, 517, 489, 494, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8229, 8228, 8171, 8112, 8285, 8185, 8205, 8121] avg=8192.0000 max=8285 max_rank=4 min=8112 min_rank=3 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32796 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[532, 555, 511, 501, 502, 472, 538, 512, 537, 501, 508, 491, 564, 496, 521, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 479, 521, 492, 467, 513, 470, 507, 546, 512, 517, 531, 513, 521, 520, 518, 18, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 545, 510, 493, 548, 507, 525, 502, 531, 518, 534, 516, 513, 528, 511, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 531, 447, 520, 522, 527, 513, 486, 506, 531, 516, 500, 511, 497, 505, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 475, 491, 474, 505, 486, 520, 490, 533, 494, 519, 531, 484, 520, 553, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 502, 496, 513, 464, 572, 474, 494, 521, 486, 491, 499, 540, 507, 469, 527, 52, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 505, 550, 495, 516, 507, 511, 502, 502, 482, 519, 530, 494, 518, 511, 472, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 538, 489, 544, 521, 506, 487, 476, 520, 518, 514, 501, 509, 478, 502, 518, 12, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8258, 8179, 8293, 8189, 8113, 8176, 8165, 8163] avg=8192.0000 max=8293 max_rank=2 min=8113 min_rank=4 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32617 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[518, 513, 523, 464, 509, 532, 532, 524, 476, 507, 493, 491, 511, 527, 520, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 476, 537, 526, 521, 503, 520, 495, 515, 499, 492, 536, 469, 507, 531, 488, 13, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 535, 527, 507, 476, 473, 478, 515, 505, 485, 484, 496, 537, 512, 496, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 484, 496, 507, 527, 544, 493, 517, 483, 485, 480, 517, 521, 502, 501, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 504, 515, 502, 524, 505, 536, 513, 491, 520, 509, 493, 529, 522, 524, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 510, 495, 574, 494, 481, 500, 544, 499, 500, 500, 527, 523, 559, 499, 502, 44, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 580, 512, 514, 545, 524, 509, 501, 516, 490, 503, 505, 525, 469, 535, 533, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 519, 497, 505, 509, 516, 484, 551, 509, 531, 521, 520, 497, 509, 492, 519, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8155, 8159, 8046, 8079, 8247, 8294, 8322, 8234] avg=8192.0000 max=8322 max_rank=6 min=8046 min_rank=2 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=33097 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[505, 467, 492, 457, 506, 523, 546, 503, 514, 525, 516, 538, 487, 483, 530, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 512, 537, 523, 536, 523, 469, 550, 491, 538, 473, 569, 469, 527, 520, 492, 12, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 562, 517, 531, 483, 486, 503, 531, 506, 514, 501, 534, 526, 508, 523, 443, 465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 561, 461, 523, 495, 545, 509, 476, 488, 502, 532, 538, 519, 552, 537, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 545, 510, 507, 496, 504, 504, 526, 520, 509, 515, 500, 497, 496, 520, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 486, 513, 516, 503, 480, 520, 505, 489, 522, 509, 509, 513, 525, 540, 495, 43, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 502, 505, 487, 493, 509, 481, 521, 529, 479, 523, 514, 471, 522, 515, 529, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 504, 515, 519, 521, 508, 512, 529, 512, 482, 536, 512, 493, 488, 515, 494, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8149, 8251, 8133, 8316, 8190, 8173, 8135, 8189] avg=8192.0000 max=8316 max_rank=3 min=8133 min_rank=2 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32687 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[470, 494, 545, 534, 521, 514, 522, 536, 528, 507, 549, 544, 502, 510, 448, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 530, 523, 482, 495, 513, 506, 532, 512, 527, 507, 494, 502, 502, 513, 498, 13, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 515, 509, 497, 517, 524, 558, 488, 525, 520, 537, 520, 542, 480, 514, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 504, 501, 478, 522, 547, 531, 500, 503, 482, 508, 523, 476, 491, 508, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 568, 462, 481, 483, 528, 507, 500, 503, 508, 531, 562, 496, 529, 500, 501, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 494, 525, 506, 519, 498, 526, 487, 499, 487, 485, 560, 484, 491, 506, 507, 51, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 548, 497, 509, 548, 480, 513, 504, 530, 488, 532, 519, 505, 504, 493, 543, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 500, 547, 503, 510, 546, 544, 532, 511, 503, 513, 498, 527, 489, 500, 506, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8241, 8176, 8267, 8075, 8150, 8158, 8210, 8259] avg=8192.0000 max=8267 max_rank=2 min=8075 min_rank=3 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32777 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[486, 473, 495, 454, 481, 497, 532, 534, 526, 537, 486, 488, 544, 521, 495, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 527, 520, 514, 515, 495, 498, 489, 532, 503, 520, 508, 504, 532, 520, 498, 13, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 506, 516, 520, 483, 472, 508, 503, 479, 525, 503, 571, 517, 521, 511, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 511, 500, 514, 582, 518, 532, 511, 499, 530, 572, 530, 515, 491, 554, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 506, 503, 495, 530, 525, 527, 497, 506, 488, 483, 511, 543, 539, 555, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 500, 517, 500, 472, 569, 489, 527, 503, 490, 513, 513, 477, 466, 475, 472, 43, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 533, 510, 523, 487, 503, 532, 542, 501, 503, 507, 476, 514, 512, 501, 525, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 524, 488, 521, 522, 521, 513, 551, 529, 513, 514, 504, 481, 533, 491, 510, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8098, 8176, 8202, 8343, 8171, 8097, 8197, 8252] avg=8192.0000 max=8343 max_rank=3 min=8097 min_rank=5 imbalance_ratio=1.018433 max_load_reduction=57193 max_load_reduction_pct=0.872696 moved_to_echo=32717 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[546, 479, 500, 487, 536, 487, 494, 535, 502, 488, 559, 511, 531, 519, 548, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 510, 498, 476, 496, 519, 500, 501, 524, 511, 512, 491, 487, 512, 545, 518, 13, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 537, 503, 496, 524, 491, 467, 490, 505, 524, 530, 498, 534, 519, 530, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 495, 502, 500, 539, 530, 511, 519, 508, 550, 497, 523, 526, 497, 471, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 530, 537, 558, 488, 517, 524, 532, 523, 532, 460, 523, 532, 498, 554, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 514, 487, 496, 532, 530, 486, 501, 517, 502, 509, 502, 479, 486, 488, 525, 46, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 496, 523, 506, 509, 505, 498, 504, 550, 481, 486, 541, 541, 499, 531, 522, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 513, 521, 503, 528, 497, 549, 489, 484, 497, 549, 526, 468, 497, 510, 523, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8234, 8129, 8151, 8195, 8313, 8129, 8215, 8170] avg=8192.0000 max=8313 max_rank=4 min=8129 min_rank=1 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32827 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[505, 467, 492, 457, 506, 523, 546, 503, 514, 525, 516, 538, 487, 483, 530, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 512, 537, 523, 536, 523, 469, 550, 491, 538, 473, 569, 469, 527, 520, 492, 12, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 562, 517, 531, 483, 486, 503, 531, 506, 514, 501, 534, 526, 508, 523, 443, 465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 561, 461, 523, 495, 545, 509, 476, 488, 502, 532, 538, 519, 552, 537, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 545, 510, 507, 496, 504, 504, 526, 520, 509, 515, 500, 497, 496, 520, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 486, 513, 516, 503, 480, 520, 505, 489, 522, 509, 509, 513, 525, 540, 495, 43, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 502, 505, 487, 493, 509, 481, 521, 529, 479, 523, 514, 471, 522, 515, 529, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 504, 515, 519, 521, 508, 512, 529, 512, 482, 536, 512, 493, 488, 515, 494, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8149, 8251, 8133, 8316, 8190, 8173, 8135, 8189] avg=8192.0000 max=8316 max_rank=3 min=8133 min_rank=2 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32687 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[553, 517, 468, 511, 540, 519, 509, 503, 507, 535, 523, 487, 502, 548, 501, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 533, 513, 539, 502, 523, 564, 453, 495, 518, 451, 497, 501, 530, 517, 529, 13, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 502, 517, 488, 502, 469, 471, 507, 514, 529, 555, 554, 508, 505, 529, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 511, 481, 532, 492, 502, 505, 519, 521, 476, 512, 496, 509, 523, 504, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 536, 553, 510, 520, 520, 510, 499, 517, 527, 493, 561, 523, 491, 486, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 520, 499, 517, 526, 521, 490, 548, 521, 474, 488, 481, 492, 481, 539, 491, 49, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 528, 495, 500, 510, 518, 508, 515, 482, 492, 559, 478, 524, 512, 526, 502, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 523, 541, 518, 502, 511, 529, 504, 504, 486, 498, 497, 513, 517, 489, 494, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8229, 8228, 8171, 8112, 8285, 8185, 8205, 8121] avg=8192.0000 max=8285 max_rank=4 min=8112 min_rank=3 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32796 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[486, 473, 495, 454, 481, 497, 532, 534, 526, 537, 486, 488, 544, 521, 495, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 527, 520, 514, 515, 495, 498, 489, 532, 503, 520, 508, 504, 532, 520, 498, 13, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 506, 516, 520, 483, 472, 508, 503, 479, 525, 503, 571, 517, 521, 511, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 511, 500, 514, 582, 518, 532, 511, 499, 530, 572, 530, 515, 491, 554, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 506, 503, 495, 530, 525, 527, 497, 506, 488, 483, 511, 543, 539, 555, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 500, 517, 500, 472, 569, 489, 527, 503, 490, 513, 513, 477, 466, 475, 472, 43, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 533, 510, 523, 487, 503, 532, 542, 501, 503, 507, 476, 514, 512, 501, 525, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 524, 488, 521, 522, 521, 513, 551, 529, 513, 514, 504, 481, 533, 491, 510, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8098, 8176, 8202, 8343, 8171, 8097, 8197, 8252] avg=8192.0000 max=8343 max_rank=3 min=8097 min_rank=5 imbalance_ratio=1.018433 max_load_reduction=57193 max_load_reduction_pct=0.872696 moved_to_echo=32717 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[517, 520, 523, 505, 501, 483, 519, 480, 503, 518, 528, 516, 539, 526, 482, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 519, 524, 506, 551, 521, 495, 461, 511, 524, 525, 493, 516, 511, 527, 519, 13, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 539, 539, 535, 536, 528, 519, 499, 488, 513, 518, 519, 462, 510, 508, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 560, 487, 524, 528, 507, 557, 514, 490, 525, 511, 528, 486, 477, 519, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 506, 505, 485, 491, 525, 502, 502, 501, 510, 503, 494, 540, 490, 488, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 537, 496, 528, 531, 526, 496, 522, 491, 512, 511, 508, 501, 518, 524, 516, 48, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 477, 504, 530, 526, 524, 515, 485, 493, 506, 537, 489, 501, 497, 493, 477, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 551, 499, 505, 520, 535, 550, 527, 496, 481, 466, 475, 509, 493, 486, 519, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8238, 8273, 8227, 8067, 8324, 8087, 8148] avg=8192.0000 max=8324 max_rank=5 min=8067 min_rank=4 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32626 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[518, 513, 523, 464, 509, 532, 532, 524, 476, 507, 493, 491, 511, 527, 520, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 476, 537, 526, 521, 503, 520, 495, 515, 499, 492, 536, 469, 507, 531, 488, 13, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 535, 527, 507, 476, 473, 478, 515, 505, 485, 484, 496, 537, 512, 496, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 484, 496, 507, 527, 544, 493, 517, 483, 485, 480, 517, 521, 502, 501, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 504, 515, 502, 524, 505, 536, 513, 491, 520, 509, 493, 529, 522, 524, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 510, 495, 574, 494, 481, 500, 544, 499, 500, 500, 527, 523, 559, 499, 502, 44, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 580, 512, 514, 545, 524, 509, 501, 516, 490, 503, 505, 525, 469, 535, 533, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 519, 497, 505, 509, 516, 484, 551, 509, 531, 521, 520, 497, 509, 492, 519, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8155, 8159, 8046, 8079, 8247, 8294, 8322, 8234] avg=8192.0000 max=8322 max_rank=6 min=8046 min_rank=2 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=33097 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 384, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 3968, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 3968, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 3968, 4224, 4096, 4096, 4096, 4224, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 3968, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 3968, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 3968, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 384, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 3968, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[540, 545, 562, 524, 510, 535, 474, 492, 501, 533, 491, 510, 530, 491, 468, 510, 498, 507, 548, 484, 440, 510, 499, 500, 550, 505, 511, 519, 522, 591, 529, 506, 512, 530, 526, 559, 544, 517, 480, 526, 538, 506, 498, 538, 511, 501, 501, 496, 535, 509, 534, 503, 504, 486, 512, 461, 501, 554, 528, 486, 509, 480, 526, 498, 529, 456, 533, 484, 485, 462, 472, 533, 477, 545, 503, 541, 520, 468, 528, 552, 514, 533, 495, 530, 553, 513, 547, 504, 523, 537, 463, 535, 504, 491, 495, 487, 468, 528, 565, 486, 537, 497, 492, 511, 522, 496, 514, 492, 534, 521, 498, 517, 535, 513, 554, 507, 512, 526, 499, 503, 511, 515, 513, 506, 485, 511, 542, 470] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[502, 526, 497, 497, 498, 502, 539, 493, 517, 540, 497, 525, 509, 462, 535, 494, 508, 504, 525, 528, 492, 497, 516, 538, 520, 541, 516, 507, 491, 483, 524, 479, 510, 517, 507, 491, 531, 524, 515, 533, 478, 527, 497, 529, 560, 507, 518, 525, 507, 523, 498, 497, 548, 516, 529, 530, 464, 517, 514, 498, 509, 476, 497, 482, 534, 532, 539, 550, 514, 495, 505, 502, 502, 513, 535, 513, 513, 571, 507, 494, 451, 515, 530, 531, 496, 486, 507, 503, 498, 515, 526, 563, 572, 490, 512, 508, 528, 506, 537, 538, 507, 503, 470, 503, 477, 497, 527, 496, 491, 490, 534, 551, 494, 517, 509, 498, 511, 550, 471, 496, 549, 502, 509, 490, 541, 506, 524, 516] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[492, 519, 547, 533, 523, 484, 519, 481, 506, 481, 539, 524, 503, 533, 540, 550, 520, 524, 518, 485, 522, 553, 505, 503, 563, 497, 529, 503, 538, 527, 540, 529, 509, 491, 525, 531, 486, 491, 545, 518, 530, 510, 508, 505, 514, 499, 538, 489, 518, 507, 484, 517, 523, 514, 487, 478, 490, 478, 484, 534, 479, 493, 555, 531, 518, 547, 495, 485, 517, 522, 489, 513, 537, 490, 516, 517, 520, 552, 518, 509, 516, 494, 490, 507, 514, 498, 514, 466, 520, 544, 545, 492, 498, 497, 534, 488, 505, 490, 494, 483, 509, 520, 455, 528, 514, 504, 520, 532, 530, 518, 543, 469, 507, 525, 535, 511, 542, 493, 511, 527, 477, 519, 538, 500, 500, 513, 504, 467] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[508, 523, 556, 540, 522, 520, 498, 478, 515, 540, 497, 520, 530, 512, 502, 516, 513, 483, 537, 528, 510, 532, 509, 511, 548, 523, 516, 542, 518, 506, 483, 506, 492, 522, 495, 498, 509, 528, 478, 500, 508, 490, 506, 520, 497, 516, 493, 534, 517, 496, 521, 495, 532, 540, 521, 500, 478, 505, 505, 501, 527, 512, 507, 534, 541, 539, 528, 493, 508, 506, 529, 494, 512, 506, 533, 514, 521, 487, 534, 512, 501, 507, 484, 502, 550, 485, 530, 556, 521, 485, 466, 521, 492, 484, 507, 482, 517, 526, 480, 530, 539, 539, 520, 530, 528, 493, 503, 503, 512, 484, 497, 520, 540, 463, 555, 494, 496, 513, 491, 522, 498, 555, 499, 498, 518, 503, 515, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[531, 508, 497, 515, 511, 520, 523, 548, 524, 492, 514, 530, 551, 518, 501, 485, 521, 511, 451, 502, 515, 542, 478, 536, 494, 532, 496, 508, 515, 520, 538, 518, 509, 498, 518, 473, 492, 515, 510, 504, 514, 534, 498, 508, 540, 499, 510, 510, 508, 525, 517, 515, 512, 528, 517, 532, 527, 531, 546, 537, 518, 495, 492, 510, 499, 549, 482, 535, 522, 502, 481, 536, 551, 475, 513, 519, 498, 505, 533, 479, 554, 533, 549, 540, 481, 494, 508, 520, 513, 482, 514, 493, 511, 507, 495, 521, 514, 496, 463, 515, 512, 516, 518, 512, 514, 507, 550, 512, 540, 505, 481, 526, 472, 514, 515, 498, 503, 518, 511, 483, 478, 559, 506, 491, 525, 495, 511, 495] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[540, 545, 562, 524, 510, 535, 474, 492, 501, 533, 491, 510, 530, 491, 468, 510, 498, 507, 548, 484, 440, 510, 499, 500, 550, 505, 511, 519, 522, 591, 529, 506, 512, 530, 526, 559, 544, 517, 480, 526, 538, 506, 498, 538, 511, 501, 501, 496, 535, 509, 534, 503, 504, 486, 512, 461, 501, 554, 528, 486, 509, 480, 526, 498, 529, 456, 533, 484, 485, 462, 472, 533, 477, 545, 503, 541, 520, 468, 528, 552, 514, 533, 495, 530, 553, 513, 547, 504, 523, 537, 463, 535, 504, 491, 495, 487, 468, 528, 565, 486, 537, 497, 492, 511, 522, 496, 514, 492, 534, 521, 498, 517, 535, 513, 554, 507, 512, 526, 499, 503, 511, 515, 513, 506, 485, 511, 542, 470] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[492, 519, 547, 533, 523, 484, 519, 481, 506, 481, 539, 524, 503, 533, 540, 550, 520, 524, 518, 485, 522, 553, 505, 503, 563, 497, 529, 503, 538, 527, 540, 529, 509, 491, 525, 531, 486, 491, 545, 518, 530, 510, 508, 505, 514, 499, 538, 489, 518, 507, 484, 517, 523, 514, 487, 478, 490, 478, 484, 534, 479, 493, 555, 531, 518, 547, 495, 485, 517, 522, 489, 513, 537, 490, 516, 517, 520, 552, 518, 509, 516, 494, 490, 507, 514, 498, 514, 466, 520, 544, 545, 492, 498, 497, 534, 488, 505, 490, 494, 483, 509, 520, 455, 528, 514, 504, 520, 532, 530, 518, 543, 469, 507, 525, 535, 511, 542, 493, 511, 527, 477, 519, 538, 500, 500, 513, 504, 467] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[494, 500, 526, 539, 517, 547, 467, 542, 534, 524, 515, 542, 530, 514, 520, 522, 530, 508, 510, 495, 497, 481, 507, 499, 537, 529, 542, 530, 531, 547, 514, 536, 522, 504, 495, 492, 505, 510, 479, 575, 510, 535, 510, 550, 543, 505, 509, 504, 526, 496, 536, 513, 521, 531, 470, 474, 497, 491, 525, 495, 513, 498, 528, 483, 493, 480, 535, 512, 505, 476, 516, 515, 509, 483, 544, 473, 546, 505, 507, 525, 524, 519, 555, 525, 497, 524, 492, 479, 480, 507, 507, 536, 486, 503, 510, 518, 524, 496, 542, 508, 512, 488, 463, 502, 532, 485, 510, 511, 500, 507, 483, 519, 504, 513, 496, 492, 490, 512, 523, 514, 542, 517, 527, 532, 509, 509, 509, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[520, 520, 534, 480, 458, 510, 462, 512, 512, 533, 574, 518, 520, 543, 518, 535, 492, 533, 481, 565, 541, 515, 497, 519, 506, 517, 497, 516, 499, 516, 518, 516, 547, 533, 498, 496, 522, 526, 504, 516, 528, 552, 538, 537, 542, 526, 517, 520, 486, 502, 495, 529, 537, 504, 498, 509, 535, 483, 524, 515, 495, 477, 487, 513, 518, 503, 492, 531, 456, 508, 520, 494, 478, 500, 520, 528, 497, 509, 510, 524, 534, 532, 532, 507, 513, 516, 453, 484, 512, 489, 464, 509, 494, 534, 521, 537, 506, 522, 522, 529, 543, 527, 468, 527, 522, 493, 551, 467, 521, 497, 511, 508, 520, 493, 518, 543, 518, 499, 489, 554, 517, 514, 520, 480, 481, 492, 489, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[530, 519, 488, 515, 493, 488, 514, 519, 492, 498, 501, 492, 522, 515, 515, 509, 465, 495, 462, 476, 496, 538, 536, 511, 529, 525, 493, 488, 517, 499, 489, 494, 518, 523, 552, 492, 527, 484, 525, 531, 504, 547, 469, 529, 514, 508, 536, 519, 510, 521, 509, 500, 485, 523, 529, 501, 522, 473, 559, 522, 510, 504, 553, 500, 510, 525, 510, 510, 493, 467, 485, 512, 483, 498, 500, 497, 526, 505, 550, 538, 544, 547, 524, 520, 503, 522, 488, 506, 457, 538, 492, 539, 534, 551, 527, 467, 499, 470, 502, 546, 560, 480, 500, 542, 523, 518, 520, 522, 543, 505, 496, 507, 515, 554, 500, 521, 527, 578, 527, 520, 490, 495, 488, 523, 499, 523, 523, 530] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[492, 519, 506, 533, 523, 484, 519, 481, 506, 481, 539, 524, 503, 533, 540, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 524, 518, 485, 522, 553, 505, 503, 540, 497, 529, 503, 538, 527, 540, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 491, 525, 531, 486, 491, 545, 518, 530, 510, 508, 505, 472, 499, 538, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 507, 484, 517, 523, 514, 487, 478, 490, 478, 484, 534, 479, 493, 555, 531, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 547, 495, 485, 517, 522, 489, 513, 537, 490, 516, 517, 520, 552, 518, 509, 11, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 494, 490, 507, 514, 498, 514, 466, 520, 544, 545, 492, 498, 497, 534, 488, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 490, 494, 483, 509, 520, 455, 528, 514, 504, 520, 532, 530, 518, 543, 469, 17, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 525, 535, 511, 542, 493, 511, 527, 477, 519, 538, 500, 500, 513, 504, 467, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8233, 8333, 8147, 8111, 8261, 8136, 8134, 8181] avg=8192.0000 max=8333 max_rank=1 min=8111 min_rank=3 imbalance_ratio=1.017212 max_load_reduction=57203 max_load_reduction_pct=0.872849 moved_to_echo=32712 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[531, 508, 450, 515, 511, 520, 523, 548, 524, 492, 514, 530, 551, 518, 501, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 511, 451, 502, 515, 542, 478, 536, 466, 532, 496, 508, 515, 520, 538, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 498, 518, 473, 492, 515, 510, 504, 514, 534, 498, 508, 490, 499, 510, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 525, 517, 515, 512, 528, 517, 532, 527, 531, 546, 537, 518, 495, 492, 510, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 549, 482, 535, 522, 502, 481, 536, 551, 475, 513, 519, 498, 505, 533, 479, 13, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 533, 549, 540, 481, 494, 508, 520, 513, 482, 514, 493, 511, 507, 495, 521, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 496, 463, 515, 512, 516, 518, 512, 514, 507, 550, 512, 540, 505, 481, 526, 19, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 514, 515, 498, 503, 518, 511, 483, 478, 559, 506, 491, 525, 495, 511, 495, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8149, 8082, 8355, 8199, 8236, 8205, 8089] avg=8192.0000 max=8355 max_rank=3 min=8082 min_rank=2 imbalance_ratio=1.019897 max_load_reduction=57181 max_load_reduction_pct=0.872513 moved_to_echo=32729 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[508, 523, 556, 540, 522, 520, 498, 478, 515, 540, 497, 520, 530, 512, 502, 516, 513, 483, 537, 528, 510, 532, 509, 511, 548, 523, 516, 542, 518, 506, 483, 506, 492, 522, 495, 498, 509, 528, 478, 500, 508, 490, 506, 520, 497, 516, 493, 534, 517, 496, 521, 495, 532, 540, 521, 500, 478, 505, 505, 501, 527, 512, 507, 534, 541, 539, 528, 493, 508, 506, 529, 494, 512, 506, 533, 514, 521, 487, 534, 512, 501, 507, 484, 502, 550, 485, 530, 556, 521, 485, 466, 521, 492, 484, 507, 482, 517, 526, 480, 530, 539, 539, 520, 530, 528, 493, 503, 503, 512, 484, 497, 520, 540, 463, 555, 494, 496, 513, 491, 522, 498, 555, 499, 498, 518, 503, 515, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[540, 545, 520, 524, 510, 535, 474, 492, 501, 533, 491, 510, 530, 491, 468, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 507, 548, 484, 440, 510, 499, 500, 527, 505, 511, 519, 522, 591, 529, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 530, 526, 559, 544, 517, 480, 526, 538, 506, 498, 538, 469, 501, 501, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 509, 534, 503, 504, 486, 512, 461, 501, 554, 528, 486, 509, 480, 526, 498, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 456, 533, 484, 485, 462, 472, 533, 477, 545, 503, 541, 520, 468, 528, 552, 11, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 533, 495, 530, 553, 513, 547, 504, 523, 537, 463, 535, 504, 491, 495, 487, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 528, 565, 486, 537, 497, 492, 511, 522, 496, 514, 492, 534, 521, 498, 517, 17, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 513, 554, 507, 512, 526, 499, 503, 511, 515, 513, 506, 485, 511, 542, 470, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 8196, 8241, 8165, 8104, 8244, 8198, 8214] avg=8192.0000 max=8244 max_rank=5 min=8104 min_rank=4 imbalance_ratio=1.006348 max_load_reduction=57292 max_load_reduction_pct=0.874207 moved_to_echo=32760 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[508, 523, 514, 540, 522, 520, 498, 478, 515, 540, 497, 520, 530, 512, 502, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 483, 537, 528, 510, 532, 509, 511, 525, 523, 516, 542, 518, 506, 483, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 522, 495, 498, 509, 528, 478, 500, 508, 490, 506, 520, 456, 516, 493, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 496, 521, 495, 532, 540, 521, 500, 478, 505, 505, 501, 527, 512, 507, 534, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 539, 528, 493, 508, 506, 529, 494, 512, 506, 533, 514, 521, 487, 534, 512, 11, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 507, 484, 502, 550, 485, 530, 556, 521, 485, 466, 521, 492, 484, 507, 482, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 526, 480, 530, 539, 539, 520, 530, 528, 493, 503, 503, 512, 484, 497, 520, 17, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 463, 555, 494, 496, 513, 491, 522, 498, 555, 499, 498, 518, 503, 515, 506, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8242, 8045, 8229, 8273, 8093, 8241, 8178] avg=8192.0000 max=8273 max_rank=4 min=8045 min_rank=2 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32785 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[531, 508, 450, 515, 511, 520, 523, 548, 524, 492, 514, 530, 551, 518, 501, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 511, 451, 502, 515, 542, 478, 536, 466, 532, 496, 508, 515, 520, 538, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 498, 518, 473, 492, 515, 510, 504, 514, 534, 498, 508, 490, 499, 510, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 525, 517, 515, 512, 528, 517, 532, 527, 531, 546, 537, 518, 495, 492, 510, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 549, 482, 535, 522, 502, 481, 536, 551, 475, 513, 519, 498, 505, 533, 479, 13, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 533, 549, 540, 481, 494, 508, 520, 513, 482, 514, 493, 511, 507, 495, 521, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 496, 463, 515, 512, 516, 518, 512, 514, 507, 550, 512, 540, 505, 481, 526, 19, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 514, 515, 498, 503, 518, 511, 483, 478, 559, 506, 491, 525, 495, 511, 495, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8149, 8082, 8355, 8199, 8236, 8205, 8089] avg=8192.0000 max=8355 max_rank=3 min=8082 min_rank=2 imbalance_ratio=1.019897 max_load_reduction=57181 max_load_reduction_pct=0.872513 moved_to_echo=32729 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[502, 526, 461, 497, 498, 502, 539, 493, 517, 540, 497, 525, 509, 462, 535, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 504, 525, 528, 492, 497, 516, 538, 499, 541, 516, 507, 491, 483, 524, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 517, 507, 491, 531, 524, 515, 533, 478, 527, 497, 529, 514, 507, 518, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 523, 498, 497, 548, 516, 529, 530, 464, 517, 514, 498, 509, 476, 497, 482, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 532, 539, 550, 514, 495, 505, 502, 502, 513, 535, 513, 513, 571, 507, 494, 10, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 451, 515, 530, 531, 496, 486, 507, 503, 498, 515, 526, 563, 572, 490, 512, 508, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 506, 537, 538, 507, 503, 470, 503, 477, 497, 527, 496, 491, 490, 534, 551, 15, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 517, 509, 498, 511, 550, 471, 496, 549, 502, 509, 490, 541, 506, 524, 516, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8097, 8148, 8223, 8148, 8333, 8220, 8173, 8194] avg=8192.0000 max=8333 max_rank=4 min=8097 min_rank=0 imbalance_ratio=1.017212 max_load_reduction=57203 max_load_reduction_pct=0.872849 moved_to_echo=32920 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[492, 519, 506, 533, 523, 484, 519, 481, 506, 481, 539, 524, 503, 533, 540, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 524, 518, 485, 522, 553, 505, 503, 540, 497, 529, 503, 538, 527, 540, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 491, 525, 531, 486, 491, 545, 518, 530, 510, 508, 505, 472, 499, 538, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 507, 484, 517, 523, 514, 487, 478, 490, 478, 484, 534, 479, 493, 555, 531, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 547, 495, 485, 517, 522, 489, 513, 537, 490, 516, 517, 520, 552, 518, 509, 11, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 494, 490, 507, 514, 498, 514, 466, 520, 544, 545, 492, 498, 497, 534, 488, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 490, 494, 483, 509, 520, 455, 528, 514, 504, 520, 532, 530, 518, 543, 469, 17, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 525, 535, 511, 542, 493, 511, 527, 477, 519, 538, 500, 500, 513, 504, 467, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8233, 8333, 8147, 8111, 8261, 8136, 8134, 8181] avg=8192.0000 max=8333 max_rank=1 min=8111 min_rank=3 imbalance_ratio=1.017212 max_load_reduction=57203 max_load_reduction_pct=0.872849 moved_to_echo=32712 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[520, 520, 494, 480, 458, 510, 462, 512, 512, 533, 574, 518, 520, 543, 518, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 533, 481, 565, 541, 515, 497, 519, 485, 517, 497, 516, 499, 516, 518, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 533, 498, 496, 522, 526, 504, 516, 528, 552, 538, 537, 498, 526, 517, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 502, 495, 529, 537, 504, 498, 509, 535, 483, 524, 515, 495, 477, 487, 513, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 503, 492, 531, 456, 508, 520, 494, 478, 500, 520, 528, 497, 509, 510, 524, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 532, 532, 507, 513, 516, 453, 484, 512, 489, 464, 509, 494, 534, 521, 537, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 522, 522, 529, 543, 527, 468, 527, 522, 493, 551, 467, 521, 497, 511, 508, 16, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 493, 518, 543, 518, 499, 489, 554, 517, 514, 520, 480, 481, 492, 489, 508, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8207, 8358, 8130, 8103, 8150, 8233, 8146] avg=8192.0000 max=8358 max_rank=2 min=8103 min_rank=4 imbalance_ratio=1.020264 max_load_reduction=57178 max_load_reduction_pct=0.872467 moved_to_echo=32632 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[494, 500, 486, 539, 517, 547, 467, 542, 534, 524, 515, 542, 530, 514, 520, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 508, 510, 495, 497, 481, 507, 499, 514, 529, 542, 530, 531, 547, 514, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 504, 495, 492, 505, 510, 479, 575, 510, 535, 510, 550, 499, 505, 509, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 496, 536, 513, 521, 531, 470, 474, 497, 491, 525, 495, 513, 498, 528, 483, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 480, 535, 512, 505, 476, 516, 515, 509, 483, 544, 473, 546, 505, 507, 525, 11, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 519, 555, 525, 497, 524, 492, 479, 480, 507, 507, 536, 486, 503, 510, 518, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 496, 542, 508, 512, 488, 463, 502, 532, 485, 510, 511, 500, 507, 483, 519, 16, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 513, 496, 492, 490, 512, 523, 514, 542, 517, 527, 532, 509, 509, 509, 508, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8293, 8270, 8204, 8138, 8140, 8181, 8101, 8209] avg=8192.0000 max=8293 max_rank=0 min=8101 min_rank=6 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32631 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[530, 519, 452, 515, 493, 488, 514, 519, 492, 498, 501, 492, 522, 515, 515, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 495, 462, 476, 496, 538, 536, 511, 507, 525, 493, 488, 517, 499, 489, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 523, 552, 492, 527, 484, 525, 531, 504, 547, 469, 529, 472, 508, 536, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 521, 509, 500, 485, 523, 529, 501, 522, 473, 559, 522, 510, 504, 553, 500, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 525, 510, 510, 493, 467, 485, 512, 483, 498, 500, 497, 526, 505, 550, 538, 10, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 547, 524, 520, 503, 522, 488, 506, 457, 538, 492, 539, 534, 551, 527, 467, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 470, 502, 546, 560, 480, 500, 542, 523, 518, 520, 522, 543, 505, 496, 507, 15, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 554, 500, 521, 527, 578, 527, 520, 490, 495, 488, 523, 499, 523, 523, 530, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8074, 7991, 8236, 8260, 8123, 8276, 8251, 8325] avg=8192.0000 max=8325 max_rank=7 min=7991 min_rank=1 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32975 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[508, 523, 514, 540, 522, 520, 498, 478, 515, 540, 497, 520, 530, 512, 502, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 483, 537, 528, 510, 532, 509, 511, 525, 523, 516, 542, 518, 506, 483, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 522, 495, 498, 509, 528, 478, 500, 508, 490, 506, 520, 456, 516, 493, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 496, 521, 495, 532, 540, 521, 500, 478, 505, 505, 501, 527, 512, 507, 534, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 539, 528, 493, 508, 506, 529, 494, 512, 506, 533, 514, 521, 487, 534, 512, 11, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 507, 484, 502, 550, 485, 530, 556, 521, 485, 466, 521, 492, 484, 507, 482, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 526, 480, 530, 539, 539, 520, 530, 528, 493, 503, 503, 512, 484, 497, 520, 17, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 463, 555, 494, 496, 513, 491, 522, 498, 555, 499, 498, 518, 503, 515, 506, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8242, 8045, 8229, 8273, 8093, 8241, 8178] avg=8192.0000 max=8273 max_rank=4 min=8045 min_rank=2 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32785 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 3968, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 3968, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 3968, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 3968, 4224, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 3968, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 3968, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[509, 559, 520, 494, 499, 490, 497, 503, 499, 514, 512, 465, 525, 524, 534, 523, 520, 526, 501, 512, 537, 503, 502, 464, 484, 506, 532, 523, 539, 514, 496, 513, 556, 523, 507, 517, 540, 510, 536, 504, 499, 559, 505, 504, 515, 485, 538, 505, 515, 510, 497, 538, 491, 478, 505, 534, 505, 535, 507, 516, 519, 530, 521, 468, 485, 530, 485, 500, 507, 510, 535, 524, 509, 486, 487, 474, 496, 534, 491, 507, 514, 460, 493, 492, 497, 484, 518, 527, 554, 510, 535, 525, 511, 507, 543, 509, 520, 479, 546, 494, 542, 521, 517, 524, 517, 530, 537, 472, 565, 534, 515, 508, 519, 539, 509, 488, 478, 519, 480, 511, 541, 523, 493, 523, 539, 525, 487, 491] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[512, 468, 528, 521, 502, 499, 495, 497, 478, 522, 521, 498, 496, 498, 509, 533, 489, 522, 524, 495, 500, 536, 515, 516, 493, 533, 531, 480, 503, 515, 525, 474, 537, 514, 485, 520, 520, 494, 514, 485, 518, 484, 518, 504, 517, 526, 491, 496, 492, 541, 533, 496, 513, 499, 567, 531, 509, 507, 535, 524, 494, 524, 472, 537, 534, 490, 515, 495, 531, 482, 514, 536, 515, 477, 468, 516, 522, 482, 508, 540, 489, 537, 520, 519, 550, 537, 528, 517, 545, 481, 484, 516, 514, 543, 490, 498, 500, 499, 534, 468, 512, 524, 546, 492, 506, 518, 516, 521, 533, 514, 516, 542, 538, 478, 521, 536, 537, 504, 516, 488, 538, 519, 517, 517, 515, 502, 496, 555] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[512, 482, 473, 517, 479, 492, 538, 476, 471, 512, 496, 537, 514, 533, 482, 496, 526, 521, 535, 462, 507, 515, 497, 529, 470, 497, 513, 490, 504, 491, 547, 517, 502, 503, 525, 466, 495, 521, 489, 551, 520, 498, 529, 506, 514, 517, 520, 549, 533, 487, 518, 515, 552, 514, 495, 515, 525, 499, 532, 520, 522, 538, 562, 539, 530, 533, 474, 494, 521, 539, 520, 497, 521, 486, 491, 505, 544, 487, 511, 489, 510, 566, 556, 527, 531, 483, 500, 520, 518, 525, 547, 496, 508, 520, 527, 522, 510, 491, 549, 519, 507, 455, 510, 513, 513, 525, 508, 536, 487, 556, 522, 561, 513, 485, 517, 496, 522, 487, 513, 514, 522, 516, 504, 461, 513, 506, 522, 483] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[503, 550, 515, 460, 511, 503, 533, 513, 518, 520, 521, 458, 537, 504, 503, 520, 528, 540, 514, 545, 478, 495, 496, 516, 535, 537, 508, 492, 532, 543, 478, 527, 485, 517, 503, 526, 494, 519, 524, 505, 496, 507, 524, 530, 528, 501, 509, 508, 520, 507, 504, 545, 537, 503, 494, 539, 486, 504, 546, 522, 472, 540, 507, 517, 512, 501, 503, 488, 493, 515, 522, 470, 482, 520, 519, 460, 510, 508, 537, 492, 494, 526, 528, 502, 528, 472, 507, 499, 500, 518, 515, 505, 487, 494, 476, 500, 496, 547, 468, 519, 517, 521, 482, 515, 545, 530, 499, 509, 539, 506, 501, 509, 500, 515, 546, 518, 534, 516, 521, 505, 514, 554, 497, 538, 530, 538, 524, 548] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[492, 540, 481, 500, 514, 482, 509, 562, 512, 539, 493, 519, 506, 507, 539, 512, 551, 523, 512, 494, 503, 518, 498, 493, 502, 471, 518, 497, 518, 483, 526, 479, 501, 529, 488, 498, 478, 493, 532, 531, 499, 513, 511, 487, 495, 495, 507, 566, 553, 541, 470, 547, 543, 519, 502, 538, 518, 510, 500, 514, 500, 528, 527, 496, 514, 494, 510, 490, 526, 511, 498, 532, 480, 534, 507, 498, 502, 552, 508, 487, 516, 493, 525, 484, 485, 519, 483, 511, 502, 480, 508, 502, 549, 503, 538, 483, 489, 549, 549, 538, 561, 539, 488, 507, 499, 515, 496, 494, 519, 552, 498, 516, 508, 567, 494, 506, 504, 459, 527, 502, 530, 528, 538, 513, 507, 530, 528, 540] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[520, 532, 509, 500, 505, 502, 492, 499, 520, 519, 522, 506, 518, 499, 520, 490, 507, 504, 495, 517, 511, 516, 557, 484, 487, 526, 532, 523, 507, 528, 490, 529, 477, 501, 510, 510, 527, 514, 491, 517, 528, 519, 547, 486, 511, 496, 500, 501, 512, 471, 501, 527, 539, 478, 514, 520, 522, 514, 527, 485, 530, 561, 507, 477, 551, 487, 513, 526, 485, 531, 530, 527, 502, 540, 508, 512, 518, 490, 511, 515, 521, 539, 494, 506, 509, 498, 513, 538, 503, 472, 540, 546, 508, 515, 529, 516, 576, 501, 495, 536, 531, 533, 480, 530, 518, 535, 529, 533, 533, 464, 481, 482, 480, 523, 511, 524, 493, 496, 497, 497, 490, 520, 527, 506, 534, 522, 494, 486] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[524, 515, 490, 519, 495, 521, 519, 486, 474, 574, 503, 510, 493, 536, 565, 488, 543, 499, 538, 531, 544, 535, 500, 521, 491, 504, 502, 526, 542, 491, 474, 512, 495, 496, 494, 474, 458, 503, 480, 509, 509, 490, 503, 473, 535, 525, 498, 520, 497, 486, 532, 520, 486, 516, 517, 510, 559, 499, 482, 510, 498, 519, 527, 542, 510, 545, 514, 513, 524, 543, 491, 499, 497, 520, 484, 539, 524, 503, 516, 488, 487, 528, 553, 526, 536, 481, 573, 499, 496, 477, 520, 528, 500, 470, 561, 563, 534, 521, 471, 533, 496, 504, 534, 541, 499, 505, 572, 478, 507, 563, 505, 514, 505, 507, 504, 512, 507, 530, 493, 525, 503, 512, 494, 488, 542, 508, 531, 463] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[509, 559, 520, 494, 499, 490, 497, 503, 499, 514, 512, 465, 525, 524, 534, 523, 520, 526, 501, 512, 537, 503, 502, 464, 484, 506, 532, 523, 539, 514, 496, 513, 556, 523, 507, 517, 540, 510, 536, 504, 499, 559, 505, 504, 515, 485, 538, 505, 515, 510, 497, 538, 491, 478, 505, 534, 505, 535, 507, 516, 519, 530, 521, 468, 485, 530, 485, 500, 507, 510, 535, 524, 509, 486, 487, 474, 496, 534, 491, 507, 514, 460, 493, 492, 497, 484, 518, 527, 554, 510, 535, 525, 511, 507, 543, 509, 520, 479, 546, 494, 542, 521, 517, 524, 517, 530, 537, 472, 565, 534, 515, 508, 519, 539, 509, 488, 478, 519, 480, 511, 541, 523, 493, 523, 539, 525, 487, 491] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[477, 495, 505, 515, 487, 490, 539, 543, 548, 534, 487, 503, 500, 500, 512, 513, 506, 495, 552, 500, 495, 491, 525, 500, 527, 521, 548, 537, 522, 510, 488, 530, 556, 504, 523, 532, 537, 539, 530, 549, 537, 509, 512, 491, 531, 499, 503, 538, 539, 520, 507, 462, 517, 512, 482, 512, 528, 522, 488, 502, 494, 521, 504, 474, 503, 489, 516, 527, 507, 505, 519, 507, 552, 504, 533, 513, 519, 519, 504, 497, 501, 504, 548, 517, 483, 503, 491, 514, 507, 509, 520, 454, 516, 519, 503, 515, 514, 530, 542, 501, 515, 515, 532, 514, 501, 510, 538, 563, 489, 469, 506, 535, 508, 489, 517, 502, 525, 458, 518, 540, 520, 499, 492, 545, 459, 449, 537, 517] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[520, 532, 509, 500, 505, 502, 492, 499, 520, 519, 522, 506, 518, 499, 520, 490, 507, 504, 495, 517, 511, 516, 557, 484, 487, 526, 532, 523, 507, 528, 490, 529, 477, 501, 510, 510, 527, 514, 491, 517, 528, 519, 547, 486, 511, 496, 500, 501, 512, 471, 501, 527, 539, 478, 514, 520, 522, 514, 527, 485, 530, 561, 507, 477, 551, 487, 513, 526, 485, 531, 530, 527, 502, 540, 508, 512, 518, 490, 511, 515, 521, 539, 494, 506, 509, 498, 513, 538, 503, 472, 540, 546, 508, 515, 529, 516, 576, 501, 495, 536, 531, 533, 480, 530, 518, 535, 529, 533, 533, 464, 481, 482, 480, 523, 511, 524, 493, 496, 497, 497, 490, 520, 527, 506, 534, 522, 494, 486] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[492, 540, 481, 500, 514, 482, 509, 562, 512, 539, 493, 519, 506, 507, 539, 512, 551, 523, 512, 494, 503, 518, 498, 493, 502, 471, 518, 497, 518, 483, 526, 479, 501, 529, 488, 498, 478, 493, 532, 531, 499, 513, 511, 487, 495, 495, 507, 566, 553, 541, 470, 547, 543, 519, 502, 538, 518, 510, 500, 514, 500, 528, 527, 496, 514, 494, 510, 490, 526, 511, 498, 532, 480, 534, 507, 498, 502, 552, 508, 487, 516, 493, 525, 484, 485, 519, 483, 511, 502, 480, 508, 502, 549, 503, 538, 483, 489, 549, 549, 538, 561, 539, 488, 507, 499, 515, 496, 494, 519, 552, 498, 516, 508, 567, 494, 506, 504, 459, 527, 502, 530, 528, 538, 513, 507, 530, 528, 540] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[503, 550, 515, 460, 511, 503, 533, 513, 518, 520, 521, 458, 537, 504, 503, 520, 528, 540, 514, 545, 478, 495, 496, 516, 535, 537, 508, 492, 532, 543, 478, 527, 485, 517, 503, 526, 494, 519, 524, 505, 496, 507, 524, 530, 528, 501, 509, 508, 520, 507, 504, 545, 537, 503, 494, 539, 486, 504, 546, 522, 472, 540, 507, 517, 512, 501, 503, 488, 493, 515, 522, 470, 482, 520, 519, 460, 510, 508, 537, 492, 494, 526, 528, 502, 528, 472, 507, 499, 500, 518, 515, 505, 487, 494, 476, 500, 496, 547, 468, 519, 517, 521, 482, 515, 545, 530, 499, 509, 539, 506, 501, 509, 500, 515, 546, 518, 534, 516, 521, 505, 514, 554, 497, 538, 530, 538, 524, 548] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[512, 482, 473, 517, 479, 492, 538, 476, 471, 512, 496, 537, 514, 533, 482, 496, 526, 521, 535, 462, 507, 515, 497, 529, 470, 497, 513, 490, 504, 491, 547, 517, 502, 503, 525, 466, 495, 521, 489, 551, 520, 498, 529, 506, 514, 517, 520, 549, 533, 487, 518, 515, 552, 514, 495, 515, 525, 499, 532, 520, 522, 538, 562, 539, 530, 533, 474, 494, 521, 539, 520, 497, 521, 486, 491, 505, 544, 487, 511, 489, 510, 566, 556, 527, 531, 483, 500, 520, 518, 525, 547, 496, 508, 520, 527, 522, 510, 491, 549, 519, 507, 455, 510, 513, 513, 525, 508, 536, 487, 556, 522, 561, 513, 485, 517, 496, 522, 487, 513, 514, 522, 516, 504, 461, 513, 506, 522, 483] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[524, 515, 490, 519, 495, 521, 519, 486, 474, 574, 503, 510, 493, 536, 565, 488, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 499, 538, 531, 544, 535, 500, 521, 491, 504, 502, 526, 542, 491, 474, 512, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 496, 494, 474, 458, 503, 480, 509, 509, 490, 503, 473, 535, 525, 498, 520, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 486, 532, 520, 486, 516, 517, 510, 559, 499, 482, 510, 498, 485, 527, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 545, 514, 513, 524, 543, 491, 499, 497, 520, 484, 539, 524, 503, 516, 488, 29, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 528, 548, 526, 536, 481, 573, 499, 496, 477, 520, 528, 500, 470, 561, 563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 521, 471, 533, 496, 504, 534, 541, 499, 505, 490, 478, 507, 563, 505, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 507, 504, 512, 507, 530, 493, 525, 503, 512, 494, 488, 542, 508, 531, 463, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8265, 8256, 7976, 8166, 8257, 8293, 8195, 8128] avg=8192.0000 max=8293 max_rank=5 min=7976 min_rank=2 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32873 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[512, 468, 528, 521, 502, 499, 495, 497, 478, 522, 521, 498, 496, 498, 509, 533, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 522, 524, 495, 500, 536, 515, 516, 493, 533, 531, 480, 503, 515, 525, 474, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 514, 485, 520, 520, 494, 514, 485, 518, 484, 518, 504, 517, 526, 491, 496, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 541, 533, 496, 513, 499, 567, 531, 509, 507, 535, 524, 494, 490, 472, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 490, 515, 495, 531, 482, 514, 536, 515, 477, 468, 516, 522, 482, 508, 540, 26, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 537, 515, 519, 550, 537, 528, 517, 545, 481, 484, 516, 514, 543, 490, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 499, 534, 468, 512, 524, 546, 492, 506, 518, 442, 521, 533, 514, 516, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 478, 521, 536, 537, 504, 516, 488, 538, 519, 517, 517, 515, 502, 496, 555, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8125, 8154, 8137, 8240, 8169, 8263, 8167, 8281] avg=8192.0000 max=8281 max_rank=7 min=8125 min_rank=0 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32880 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[477, 495, 505, 515, 487, 490, 539, 543, 548, 534, 487, 503, 500, 500, 512, 513, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 495, 552, 500, 495, 491, 525, 500, 527, 521, 548, 537, 522, 510, 488, 530, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 504, 523, 532, 537, 539, 530, 549, 537, 509, 512, 491, 531, 499, 503, 538, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 520, 507, 462, 517, 512, 482, 512, 528, 522, 488, 502, 494, 487, 504, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 489, 516, 527, 507, 505, 519, 507, 552, 504, 533, 513, 519, 519, 504, 497, 27, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 504, 543, 517, 483, 503, 491, 514, 507, 509, 520, 454, 516, 519, 503, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 530, 542, 501, 515, 515, 532, 514, 501, 510, 461, 563, 489, 469, 506, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 489, 517, 502, 525, 458, 518, 540, 520, 499, 492, 545, 459, 449, 537, 517, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8250, 8404, 8050, 8259, 8099, 8197, 8079] avg=8192.0000 max=8404 max_rank=2 min=8050 min_rank=3 imbalance_ratio=1.025879 max_load_reduction=57132 max_load_reduction_pct=0.871765 moved_to_echo=32634 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[503, 550, 515, 460, 511, 503, 533, 513, 518, 520, 521, 458, 537, 504, 503, 520, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 540, 514, 545, 478, 495, 496, 516, 535, 537, 508, 492, 532, 543, 478, 527, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 517, 503, 526, 494, 519, 524, 505, 496, 507, 524, 530, 528, 501, 509, 508, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 507, 504, 545, 537, 503, 494, 539, 486, 504, 546, 522, 472, 491, 507, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 501, 503, 488, 493, 515, 522, 470, 482, 520, 519, 460, 510, 508, 537, 492, 28, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 526, 515, 502, 528, 472, 507, 499, 500, 518, 515, 505, 487, 494, 476, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 547, 468, 519, 517, 521, 482, 515, 545, 530, 421, 509, 539, 506, 501, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 515, 546, 518, 534, 516, 521, 505, 514, 554, 497, 538, 530, 538, 524, 548, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 8272, 8195, 8194, 8082, 8038, 8125, 8411] avg=8192.0000 max=8411 max_rank=7 min=8038 min_rank=5 imbalance_ratio=1.026733 max_load_reduction=57125 max_load_reduction_pct=0.871658 moved_to_echo=32656 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[509, 559, 520, 494, 499, 490, 497, 503, 499, 514, 512, 465, 525, 524, 534, 523, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 526, 501, 512, 537, 503, 502, 464, 484, 506, 532, 523, 539, 514, 496, 513, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 523, 507, 517, 540, 510, 536, 504, 499, 559, 505, 504, 515, 485, 538, 505, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 510, 497, 538, 491, 478, 505, 534, 505, 535, 507, 516, 519, 496, 521, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 530, 485, 500, 507, 510, 535, 524, 509, 486, 487, 474, 496, 534, 491, 507, 27, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 460, 488, 492, 497, 484, 518, 527, 554, 510, 535, 525, 511, 507, 543, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 479, 546, 494, 542, 521, 517, 524, 517, 530, 460, 472, 565, 534, 515, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 539, 509, 488, 478, 519, 480, 511, 541, 523, 493, 523, 539, 525, 487, 491, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8217, 8175, 8317, 8135, 8105, 8174, 8244, 8169] avg=8192.0000 max=8317 max_rank=2 min=8105 min_rank=4 imbalance_ratio=1.015259 max_load_reduction=57219 max_load_reduction_pct=0.873093 moved_to_echo=32692 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[520, 532, 509, 500, 505, 502, 492, 499, 520, 519, 522, 506, 518, 499, 520, 490, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 504, 495, 517, 511, 516, 557, 484, 487, 526, 532, 523, 507, 528, 490, 529, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 501, 510, 510, 527, 514, 491, 517, 528, 519, 547, 486, 511, 496, 500, 501, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 471, 501, 527, 539, 478, 514, 520, 522, 514, 527, 485, 530, 524, 507, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 487, 513, 526, 485, 531, 530, 527, 502, 540, 508, 512, 518, 490, 511, 515, 27, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 539, 489, 506, 509, 498, 513, 538, 503, 472, 540, 546, 508, 515, 529, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 576, 501, 495, 536, 531, 533, 480, 530, 518, 535, 453, 533, 533, 464, 481, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 523, 511, 524, 493, 496, 497, 497, 490, 520, 527, 506, 534, 522, 494, 486, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8216, 8150, 8148, 8292, 8242, 8181, 8105] avg=8192.0000 max=8292 max_rank=4 min=8105 min_rank=7 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32820 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[492, 540, 481, 500, 514, 482, 509, 562, 512, 539, 493, 519, 506, 507, 539, 512, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 523, 512, 494, 503, 518, 498, 493, 502, 471, 518, 497, 518, 483, 526, 479, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 529, 488, 498, 478, 493, 532, 531, 499, 513, 511, 487, 495, 495, 507, 566, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 541, 470, 547, 543, 519, 502, 538, 518, 510, 500, 514, 500, 494, 527, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 494, 510, 490, 526, 511, 498, 532, 480, 534, 507, 498, 502, 552, 508, 487, 25, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 493, 520, 484, 485, 519, 483, 511, 502, 480, 508, 502, 549, 503, 538, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 549, 549, 538, 561, 539, 488, 507, 499, 515, 425, 494, 519, 552, 498, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 567, 494, 506, 504, 459, 527, 502, 530, 528, 538, 513, 507, 530, 528, 540, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8253, 8089, 8137, 8272, 8186, 8076, 8238, 8285] avg=8192.0000 max=8285 max_rank=7 min=8076 min_rank=5 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32785 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[512, 482, 473, 517, 479, 492, 538, 476, 471, 512, 496, 537, 514, 533, 482, 496, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 521, 535, 462, 507, 515, 497, 529, 470, 497, 513, 490, 504, 491, 547, 517, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 503, 525, 466, 495, 521, 489, 551, 520, 498, 529, 506, 514, 517, 520, 549, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 487, 518, 515, 552, 514, 495, 515, 525, 499, 532, 520, 522, 503, 562, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 533, 474, 494, 521, 539, 520, 497, 521, 486, 491, 505, 544, 487, 511, 489, 26, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 566, 551, 527, 531, 483, 500, 520, 518, 525, 547, 496, 508, 520, 527, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 491, 549, 519, 507, 455, 510, 513, 513, 525, 435, 536, 487, 556, 522, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 485, 517, 496, 522, 487, 513, 514, 522, 516, 504, 461, 513, 506, 522, 483, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8057, 8124, 8220, 8331, 8186, 8351, 8189, 8078] avg=8192.0000 max=8351 max_rank=5 min=8057 min_rank=0 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32804 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[492, 540, 481, 500, 514, 482, 509, 562, 512, 539, 493, 519, 506, 507, 539, 512, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 523, 512, 494, 503, 518, 498, 493, 502, 471, 518, 497, 518, 483, 526, 479, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 529, 488, 498, 478, 493, 532, 531, 499, 513, 511, 487, 495, 495, 507, 566, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 541, 470, 547, 543, 519, 502, 538, 518, 510, 500, 514, 500, 494, 527, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 494, 510, 490, 526, 511, 498, 532, 480, 534, 507, 498, 502, 552, 508, 487, 25, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 493, 520, 484, 485, 519, 483, 511, 502, 480, 508, 502, 549, 503, 538, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 549, 549, 538, 561, 539, 488, 507, 499, 515, 425, 494, 519, 552, 498, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 567, 494, 506, 504, 459, 527, 502, 530, 528, 538, 513, 507, 530, 528, 540, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8253, 8089, 8137, 8272, 8186, 8076, 8238, 8285] avg=8192.0000 max=8285 max_rank=7 min=8076 min_rank=5 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32785 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[524, 515, 490, 519, 495, 521, 519, 486, 474, 574, 503, 510, 493, 536, 565, 488, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 499, 538, 531, 544, 535, 500, 521, 491, 504, 502, 526, 542, 491, 474, 512, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 496, 494, 474, 458, 503, 480, 509, 509, 490, 503, 473, 535, 525, 498, 520, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 486, 532, 520, 486, 516, 517, 510, 559, 499, 482, 510, 498, 485, 527, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 545, 514, 513, 524, 543, 491, 499, 497, 520, 484, 539, 524, 503, 516, 488, 29, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 528, 548, 526, 536, 481, 573, 499, 496, 477, 520, 528, 500, 470, 561, 563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 521, 471, 533, 496, 504, 534, 541, 499, 505, 490, 478, 507, 563, 505, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 507, 504, 512, 507, 530, 493, 525, 503, 512, 494, 488, 542, 508, 531, 463, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8265, 8256, 7976, 8166, 8257, 8293, 8195, 8128] avg=8192.0000 max=8293 max_rank=5 min=7976 min_rank=2 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32873 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[520, 532, 509, 500, 505, 502, 492, 499, 520, 519, 522, 506, 518, 499, 520, 490, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 504, 495, 517, 511, 516, 557, 484, 487, 526, 532, 523, 507, 528, 490, 529, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 501, 510, 510, 527, 514, 491, 517, 528, 519, 547, 486, 511, 496, 500, 501, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 471, 501, 527, 539, 478, 514, 520, 522, 514, 527, 485, 530, 524, 507, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 487, 513, 526, 485, 531, 530, 527, 502, 540, 508, 512, 518, 490, 511, 515, 27, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 539, 489, 506, 509, 498, 513, 538, 503, 472, 540, 546, 508, 515, 529, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 576, 501, 495, 536, 531, 533, 480, 530, 518, 535, 453, 533, 533, 464, 481, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 523, 511, 524, 493, 496, 497, 497, 490, 520, 527, 506, 534, 522, 494, 486, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8216, 8150, 8148, 8292, 8242, 8181, 8105] avg=8192.0000 max=8292 max_rank=4 min=8105 min_rank=7 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32820 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[477, 495, 505, 515, 487, 490, 539, 543, 548, 534, 487, 503, 500, 500, 512, 513, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 495, 552, 500, 495, 491, 525, 500, 527, 521, 548, 537, 522, 510, 488, 530, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 504, 523, 532, 537, 539, 530, 549, 537, 509, 512, 491, 531, 499, 503, 538, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 520, 507, 462, 517, 512, 482, 512, 528, 522, 488, 502, 494, 487, 504, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 489, 516, 527, 507, 505, 519, 507, 552, 504, 533, 513, 519, 519, 504, 497, 27, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 504, 543, 517, 483, 503, 491, 514, 507, 509, 520, 454, 516, 519, 503, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 530, 542, 501, 515, 515, 532, 514, 501, 510, 461, 563, 489, 469, 506, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 489, 517, 502, 525, 458, 518, 540, 520, 499, 492, 545, 459, 449, 537, 517, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8250, 8404, 8050, 8259, 8099, 8197, 8079] avg=8192.0000 max=8404 max_rank=2 min=8050 min_rank=3 imbalance_ratio=1.025879 max_load_reduction=57132 max_load_reduction_pct=0.871765 moved_to_echo=32634 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4352, 4096, 4096, 4096, 4224, 4224, 4096, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 256, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 3712, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 3712, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 256, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[487, 515, 488, 537, 506, 505, 527, 544, 508, 492, 515, 510, 506, 517, 486, 491, 487, 514, 544, 498, 487, 536, 543, 556, 489, 487, 500, 474, 470, 485, 511, 498, 566, 547, 542, 523, 554, 526, 519, 522, 497, 509, 516, 486, 522, 521, 544, 525, 507, 475, 557, 510, 530, 525, 534, 514, 490, 506, 563, 521, 523, 508, 536, 575, 539, 488, 492, 502, 497, 502, 525, 508, 492, 497, 473, 461, 523, 497, 478, 501, 519, 490, 509, 540, 519, 539, 512, 529, 512, 528, 498, 499, 515, 500, 493, 526, 514, 487, 490, 525, 537, 511, 539, 487, 513, 491, 515, 513, 527, 489, 504, 496, 489, 545, 523, 507, 526, 505, 488, 538, 535, 514, 511, 518, 517, 517, 452, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[476, 511, 511, 520, 490, 528, 495, 543, 499, 497, 524, 516, 507, 507, 555, 493, 485, 515, 535, 528, 529, 518, 520, 484, 476, 522, 519, 481, 542, 494, 531, 501, 539, 520, 498, 530, 548, 533, 514, 526, 528, 499, 501, 471, 482, 530, 473, 506, 519, 530, 494, 513, 510, 544, 526, 501, 501, 517, 522, 501, 555, 499, 510, 525, 547, 534, 566, 532, 525, 501, 520, 496, 560, 497, 495, 507, 575, 527, 526, 522, 506, 501, 510, 515, 496, 533, 499, 495, 472, 508, 547, 510, 471, 508, 530, 520, 496, 525, 488, 524, 501, 479, 483, 494, 529, 486, 492, 516, 524, 544, 477, 507, 526, 465, 527, 514, 512, 532, 524, 464, 555, 511, 502, 469, 507, 505, 480, 510] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[519, 516, 501, 518, 570, 526, 502, 522, 503, 495, 524, 496, 527, 519, 495, 470, 503, 482, 479, 503, 510, 502, 535, 540, 514, 506, 519, 488, 514, 552, 501, 489, 527, 509, 542, 512, 498, 529, 532, 533, 486, 531, 530, 502, 491, 516, 527, 514, 535, 501, 536, 468, 489, 511, 505, 525, 509, 504, 529, 495, 505, 532, 500, 496, 537, 515, 513, 544, 570, 531, 549, 511, 523, 492, 520, 495, 493, 497, 506, 488, 500, 523, 530, 534, 498, 470, 513, 475, 516, 525, 437, 526, 537, 513, 544, 489, 506, 474, 533, 479, 530, 530, 490, 510, 476, 557, 542, 508, 530, 508, 486, 489, 492, 522, 513, 498, 517, 481, 492, 511, 489, 528, 545, 534, 540, 518, 523, 512] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[488, 521, 572, 521, 504, 538, 485, 537, 512, 533, 474, 501, 539, 529, 488, 517, 543, 541, 522, 511, 491, 476, 486, 505, 516, 492, 527, 546, 521, 521, 473, 498, 510, 520, 516, 532, 499, 503, 493, 531, 493, 512, 496, 503, 564, 491, 510, 557, 524, 512, 532, 520, 504, 528, 555, 542, 521, 548, 518, 514, 514, 516, 518, 509, 477, 491, 514, 518, 473, 520, 521, 520, 501, 504, 496, 514, 540, 526, 472, 502, 523, 513, 521, 466, 517, 504, 506, 521, 530, 499, 518, 484, 531, 521, 492, 504, 502, 514, 508, 506, 508, 519, 457, 489, 479, 514, 499, 501, 506, 506, 498, 570, 534, 502, 527, 507, 513, 517, 452, 503, 521, 490, 513, 539, 517, 535, 509, 509] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[526, 523, 468, 473, 527, 537, 511, 518, 491, 505, 513, 545, 516, 516, 515, 485, 521, 519, 583, 494, 511, 539, 506, 492, 506, 527, 519, 496, 527, 511, 491, 505, 536, 510, 513, 535, 507, 500, 467, 499, 469, 484, 498, 525, 522, 493, 530, 509, 518, 518, 512, 504, 531, 499, 545, 513, 511, 493, 530, 494, 519, 506, 498, 485, 533, 547, 542, 518, 537, 473, 469, 514, 475, 489, 488, 530, 511, 525, 467, 541, 505, 542, 514, 526, 582, 520, 523, 497, 519, 538, 516, 525, 530, 499, 550, 526, 502, 484, 456, 490, 532, 523, 533, 523, 525, 510, 502, 507, 521, 521, 505, 505, 520, 517, 533, 534, 485, 519, 503, 510, 508, 469, 526, 460, 491, 495, 543, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[514, 499, 482, 507, 511, 510, 495, 527, 516, 502, 482, 514, 476, 492, 521, 501, 565, 509, 479, 512, 473, 517, 502, 475, 553, 490, 530, 515, 531, 507, 540, 518, 490, 489, 505, 515, 521, 529, 505, 486, 476, 504, 523, 517, 488, 541, 559, 502, 526, 527, 564, 469, 504, 512, 545, 500, 510, 549, 505, 526, 478, 527, 521, 515, 480, 550, 511, 494, 476, 517, 493, 489, 484, 522, 509, 533, 537, 545, 519, 505, 492, 530, 551, 515, 507, 491, 501, 506, 531, 528, 542, 537, 522, 471, 516, 544, 471, 524, 482, 505, 493, 546, 552, 511, 543, 507, 514, 512, 510, 513, 495, 525, 536, 513, 498, 517, 494, 523, 520, 535, 499, 497, 485, 528, 543, 501, 521, 482] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[487, 515, 488, 537, 506, 505, 527, 544, 508, 492, 515, 510, 506, 517, 486, 491, 487, 514, 544, 498, 487, 536, 543, 556, 489, 487, 500, 474, 470, 485, 511, 498, 566, 547, 542, 523, 554, 526, 519, 522, 497, 509, 516, 486, 522, 521, 544, 525, 507, 475, 557, 510, 530, 525, 534, 514, 490, 506, 563, 521, 523, 508, 536, 575, 539, 488, 492, 502, 497, 502, 525, 508, 492, 497, 473, 461, 523, 497, 478, 501, 519, 490, 509, 540, 519, 539, 512, 529, 512, 528, 498, 499, 515, 500, 493, 526, 514, 487, 490, 525, 537, 511, 539, 487, 513, 491, 515, 513, 527, 489, 504, 496, 489, 545, 523, 507, 526, 505, 488, 538, 535, 514, 511, 518, 517, 517, 452, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[528, 505, 489, 490, 513, 477, 538, 508, 530, 517, 547, 485, 525, 505, 484, 544, 525, 505, 543, 547, 494, 491, 489, 483, 461, 509, 536, 508, 523, 526, 523, 475, 492, 515, 501, 501, 517, 512, 522, 502, 508, 524, 552, 514, 457, 528, 527, 545, 512, 535, 449, 490, 523, 516, 519, 504, 506, 517, 530, 491, 513, 441, 501, 469, 540, 534, 553, 534, 523, 487, 551, 509, 514, 516, 507, 503, 495, 491, 556, 473, 490, 541, 505, 478, 483, 495, 512, 533, 554, 500, 536, 534, 553, 545, 536, 491, 512, 493, 522, 504, 492, 509, 493, 509, 466, 534, 531, 520, 513, 528, 502, 565, 511, 532, 518, 534, 526, 507, 517, 507, 505, 488, 515, 492, 531, 536, 485, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[519, 516, 501, 518, 570, 526, 502, 522, 503, 495, 524, 496, 527, 519, 495, 470, 503, 482, 479, 503, 510, 502, 535, 540, 514, 506, 519, 488, 514, 552, 501, 489, 527, 509, 542, 512, 498, 529, 532, 533, 486, 531, 530, 502, 491, 516, 527, 514, 535, 501, 536, 468, 489, 511, 505, 525, 509, 504, 529, 495, 505, 532, 500, 496, 537, 515, 513, 544, 570, 531, 549, 511, 523, 492, 520, 495, 493, 497, 506, 488, 500, 523, 530, 534, 498, 470, 513, 475, 516, 525, 437, 526, 537, 513, 544, 489, 506, 474, 533, 479, 530, 530, 490, 510, 476, 557, 542, 508, 530, 508, 486, 489, 492, 522, 513, 498, 517, 481, 492, 511, 489, 528, 545, 534, 540, 518, 523, 512] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[500, 515, 510, 553, 467, 514, 501, 509, 504, 534, 462, 507, 530, 521, 508, 522, 532, 514, 514, 500, 524, 511, 512, 486, 536, 482, 525, 503, 511, 525, 556, 518, 487, 487, 525, 519, 522, 522, 532, 523, 516, 518, 528, 504, 559, 509, 473, 528, 528, 521, 551, 494, 530, 532, 521, 515, 537, 513, 509, 486, 535, 516, 504, 511, 531, 518, 511, 526, 465, 512, 513, 489, 508, 484, 475, 534, 511, 503, 527, 519, 543, 507, 528, 504, 481, 525, 519, 513, 521, 545, 486, 504, 533, 482, 495, 479, 532, 493, 477, 500, 518, 510, 502, 489, 524, 475, 507, 526, 506, 540, 516, 485, 519, 483, 560, 501, 543, 486, 489, 490, 536, 513, 492, 525, 497, 487, 529, 534] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[526, 523, 468, 473, 527, 537, 511, 518, 491, 505, 513, 545, 516, 516, 515, 485, 521, 519, 583, 494, 511, 539, 506, 492, 506, 527, 519, 496, 527, 511, 491, 505, 536, 510, 513, 535, 507, 500, 467, 499, 469, 484, 498, 525, 522, 493, 530, 509, 518, 518, 512, 504, 531, 499, 545, 513, 511, 493, 530, 494, 519, 506, 498, 485, 533, 547, 542, 518, 537, 473, 469, 514, 475, 489, 488, 530, 511, 525, 467, 541, 505, 542, 514, 526, 582, 520, 523, 497, 519, 538, 516, 525, 530, 499, 550, 526, 502, 484, 456, 490, 532, 523, 533, 523, 525, 510, 502, 507, 521, 521, 505, 505, 520, 517, 533, 534, 485, 519, 503, 510, 508, 469, 526, 460, 491, 495, 543, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[476, 511, 511, 520, 490, 528, 495, 543, 499, 497, 524, 516, 507, 507, 555, 493, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 515, 535, 528, 529, 518, 520, 484, 476, 522, 519, 481, 542, 494, 531, 501, 11, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 520, 498, 530, 548, 533, 514, 526, 528, 499, 501, 471, 482, 530, 473, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 530, 494, 513, 510, 544, 478, 501, 501, 517, 522, 501, 555, 499, 510, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 534, 566, 532, 525, 501, 520, 496, 560, 497, 495, 507, 575, 527, 526, 522, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 501, 510, 515, 496, 533, 499, 495, 472, 508, 547, 510, 446, 508, 530, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 525, 488, 524, 501, 479, 483, 494, 529, 486, 492, 516, 524, 544, 477, 507, 48, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 465, 527, 514, 512, 532, 524, 464, 555, 511, 502, 469, 507, 505, 480, 510, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8193, 8162, 8219, 8432, 8096, 8122, 8115] avg=8192.0000 max=8432 max_rank=4 min=8096 min_rank=5 imbalance_ratio=1.029297 max_load_reduction=57104 max_load_reduction_pct=0.871338 moved_to_echo=32765 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[488, 521, 572, 521, 504, 538, 485, 537, 512, 533, 474, 501, 539, 529, 488, 517, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 541, 522, 511, 491, 476, 486, 505, 516, 492, 527, 546, 521, 521, 473, 498, 13, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 520, 516, 532, 499, 503, 493, 531, 493, 512, 496, 503, 564, 491, 510, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 512, 532, 520, 504, 528, 504, 542, 521, 548, 518, 514, 514, 516, 518, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 491, 514, 518, 473, 520, 521, 520, 501, 504, 496, 514, 540, 526, 472, 502, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 513, 521, 466, 517, 504, 506, 521, 530, 499, 518, 484, 502, 521, 492, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 514, 508, 506, 508, 519, 457, 489, 479, 514, 499, 501, 506, 506, 498, 570, 51, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 502, 527, 507, 513, 517, 452, 503, 521, 490, 513, 539, 517, 535, 509, 509, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8286, 8184, 8191, 8324, 8091, 8121, 8137, 8202] avg=8192.0000 max=8324 max_rank=3 min=8091 min_rank=4 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32551 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[519, 516, 501, 518, 570, 526, 502, 522, 503, 495, 524, 496, 527, 519, 495, 470, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 482, 479, 503, 510, 502, 535, 540, 514, 506, 519, 488, 514, 552, 501, 489, 13, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 509, 542, 512, 498, 529, 532, 533, 486, 531, 530, 502, 491, 516, 527, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 501, 536, 468, 489, 511, 459, 525, 509, 504, 529, 495, 505, 532, 500, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 515, 513, 544, 570, 531, 549, 511, 523, 492, 520, 495, 493, 497, 506, 488, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 523, 530, 534, 498, 470, 513, 475, 516, 525, 437, 526, 508, 513, 544, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 474, 533, 479, 530, 530, 490, 510, 476, 557, 542, 508, 530, 508, 486, 489, 46, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 522, 513, 498, 517, 481, 492, 511, 489, 528, 545, 534, 540, 518, 523, 512, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8228, 8152, 8243, 8094, 8286, 8101, 8203, 8229] avg=8192.0000 max=8286 max_rank=4 min=8094 min_rank=3 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32819 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[528, 505, 489, 490, 513, 477, 538, 508, 530, 517, 547, 485, 525, 505, 484, 544, 525, 505, 543, 547, 494, 491, 489, 483, 461, 509, 536, 508, 523, 526, 523, 475, 492, 515, 501, 501, 517, 512, 522, 502, 508, 524, 552, 514, 457, 528, 527, 545, 512, 535, 449, 490, 523, 516, 519, 504, 506, 517, 530, 491, 513, 441, 501, 469, 540, 534, 553, 534, 523, 487, 551, 509, 514, 516, 507, 503, 495, 491, 556, 473, 490, 541, 505, 478, 483, 495, 512, 533, 554, 500, 536, 534, 553, 545, 536, 491, 512, 493, 522, 504, 492, 509, 493, 509, 466, 534, 531, 520, 513, 528, 502, 565, 511, 532, 518, 534, 526, 507, 517, 507, 505, 488, 515, 492, 531, 536, 485, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[487, 515, 488, 537, 506, 505, 527, 544, 508, 492, 515, 510, 506, 517, 486, 491, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 514, 544, 498, 487, 536, 543, 556, 489, 487, 500, 474, 470, 485, 511, 498, 16, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 566, 547, 542, 523, 554, 526, 519, 522, 497, 509, 516, 486, 522, 521, 544, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 475, 557, 510, 530, 525, 481, 514, 490, 506, 563, 521, 523, 508, 536, 575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 488, 492, 502, 497, 502, 525, 508, 492, 497, 473, 461, 523, 497, 478, 501, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 490, 509, 540, 519, 539, 512, 529, 512, 528, 498, 499, 474, 500, 493, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 487, 490, 525, 537, 511, 539, 487, 513, 491, 515, 513, 527, 489, 504, 496, 53, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 545, 523, 507, 526, 505, 488, 538, 535, 514, 511, 518, 517, 517, 452, 504, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8163, 8098, 8372, 8321, 7983, 8187, 8206, 8206] avg=8192.0000 max=8372 max_rank=2 min=7983 min_rank=4 imbalance_ratio=1.021973 max_load_reduction=57164 max_load_reduction_pct=0.872253 moved_to_echo=32582 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[514, 499, 482, 507, 511, 510, 495, 527, 516, 502, 482, 514, 476, 492, 521, 501, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 565, 509, 479, 512, 473, 517, 502, 475, 553, 490, 530, 515, 531, 507, 540, 518, 12, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 489, 505, 515, 521, 529, 505, 486, 476, 504, 523, 517, 488, 541, 559, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 527, 564, 469, 504, 512, 495, 500, 510, 549, 505, 526, 478, 527, 521, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 550, 511, 494, 476, 517, 493, 489, 484, 522, 509, 533, 537, 545, 519, 505, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 530, 551, 515, 507, 491, 501, 506, 531, 528, 542, 537, 494, 471, 516, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 524, 482, 505, 493, 546, 552, 511, 543, 507, 514, 512, 510, 513, 495, 525, 50, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 513, 498, 517, 494, 523, 520, 535, 499, 497, 485, 528, 543, 501, 521, 482, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8073, 8230, 8115, 8228, 8166, 8256, 8262, 8206] avg=8192.0000 max=8262 max_rank=6 min=8073 min_rank=0 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32890 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[528, 505, 489, 490, 513, 477, 538, 508, 530, 517, 547, 485, 525, 505, 484, 544, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 505, 543, 547, 494, 491, 489, 483, 461, 509, 536, 508, 523, 526, 523, 475, 13, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 515, 501, 501, 517, 512, 522, 502, 508, 524, 552, 514, 457, 528, 527, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 535, 449, 490, 523, 516, 472, 504, 506, 517, 530, 491, 513, 441, 501, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 534, 553, 534, 523, 487, 551, 509, 514, 516, 507, 503, 495, 491, 556, 473, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 541, 505, 478, 483, 495, 512, 533, 554, 500, 536, 534, 523, 545, 536, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 493, 522, 504, 492, 509, 493, 509, 466, 534, 531, 520, 513, 528, 502, 565, 47, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 532, 518, 534, 526, 507, 517, 507, 505, 488, 515, 492, 531, 536, 485, 511, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8153, 8178, 7969, 8289, 8256, 8250, 8229] avg=8192.0000 max=8289 max_rank=4 min=7969 min_rank=3 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=33024 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[488, 521, 572, 521, 504, 538, 485, 537, 512, 533, 474, 501, 539, 529, 488, 517, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 541, 522, 511, 491, 476, 486, 505, 516, 492, 527, 546, 521, 521, 473, 498, 13, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 520, 516, 532, 499, 503, 493, 531, 493, 512, 496, 503, 564, 491, 510, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 512, 532, 520, 504, 528, 504, 542, 521, 548, 518, 514, 514, 516, 518, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 491, 514, 518, 473, 520, 521, 520, 501, 504, 496, 514, 540, 526, 472, 502, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 513, 521, 466, 517, 504, 506, 521, 530, 499, 518, 484, 502, 521, 492, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 514, 508, 506, 508, 519, 457, 489, 479, 514, 499, 501, 506, 506, 498, 570, 51, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 502, 527, 507, 513, 517, 452, 503, 521, 490, 513, 539, 517, 535, 509, 509, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8286, 8184, 8191, 8324, 8091, 8121, 8137, 8202] avg=8192.0000 max=8324 max_rank=3 min=8091 min_rank=4 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32551 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[526, 523, 468, 473, 527, 537, 511, 518, 491, 505, 513, 545, 516, 516, 515, 485, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 519, 583, 494, 511, 539, 506, 492, 506, 527, 519, 496, 527, 511, 491, 505, 13, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 510, 513, 535, 507, 500, 467, 499, 469, 484, 498, 525, 522, 493, 530, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 518, 512, 504, 531, 499, 495, 513, 511, 493, 530, 494, 519, 506, 498, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 547, 542, 518, 537, 473, 469, 514, 475, 489, 488, 530, 511, 525, 467, 541, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 542, 514, 526, 582, 520, 523, 497, 519, 538, 516, 525, 501, 499, 550, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 484, 456, 490, 532, 523, 533, 523, 525, 510, 502, 507, 521, 521, 505, 505, 50, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 517, 533, 534, 485, 519, 503, 510, 508, 469, 526, 460, 491, 495, 543, 524, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8262, 8061, 8126, 8161, 8383, 8198, 8151] avg=8192.0000 max=8383 max_rank=5 min=8061 min_rank=2 imbalance_ratio=1.023315 max_load_reduction=57153 max_load_reduction_pct=0.872086 moved_to_echo=32893 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[476, 511, 511, 520, 490, 528, 495, 543, 499, 497, 524, 516, 507, 507, 555, 493, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 515, 535, 528, 529, 518, 520, 484, 476, 522, 519, 481, 542, 494, 531, 501, 11, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 520, 498, 530, 548, 533, 514, 526, 528, 499, 501, 471, 482, 530, 473, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 530, 494, 513, 510, 544, 478, 501, 501, 517, 522, 501, 555, 499, 510, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 534, 566, 532, 525, 501, 520, 496, 560, 497, 495, 507, 575, 527, 526, 522, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 501, 510, 515, 496, 533, 499, 495, 472, 508, 547, 510, 446, 508, 530, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 525, 488, 524, 501, 479, 483, 494, 529, 486, 492, 516, 524, 544, 477, 507, 48, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 465, 527, 514, 512, 532, 524, 464, 555, 511, 502, 469, 507, 505, 480, 510, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8193, 8162, 8219, 8432, 8096, 8122, 8115] avg=8192.0000 max=8432 max_rank=4 min=8096 min_rank=5 imbalance_ratio=1.029297 max_load_reduction=57104 max_load_reduction_pct=0.871338 moved_to_echo=32765 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[528, 505, 489, 490, 513, 477, 538, 508, 530, 517, 547, 485, 525, 505, 484, 544, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 505, 543, 547, 494, 491, 489, 483, 461, 509, 536, 508, 523, 526, 523, 475, 13, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 515, 501, 501, 517, 512, 522, 502, 508, 524, 552, 514, 457, 528, 527, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 535, 449, 490, 523, 516, 472, 504, 506, 517, 530, 491, 513, 441, 501, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 534, 553, 534, 523, 487, 551, 509, 514, 516, 507, 503, 495, 491, 556, 473, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 541, 505, 478, 483, 495, 512, 533, 554, 500, 536, 534, 523, 545, 536, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 493, 522, 504, 492, 509, 493, 509, 466, 534, 531, 520, 513, 528, 502, 565, 47, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 532, 518, 534, 526, 507, 517, 507, 505, 488, 515, 492, 531, 536, 485, 511, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8153, 8178, 7969, 8289, 8256, 8250, 8229] avg=8192.0000 max=8289 max_rank=4 min=7969 min_rank=3 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=33024 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[514, 499, 482, 507, 511, 510, 495, 527, 516, 502, 482, 514, 476, 492, 521, 501, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 565, 509, 479, 512, 473, 517, 502, 475, 553, 490, 530, 515, 531, 507, 540, 518, 12, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 489, 505, 515, 521, 529, 505, 486, 476, 504, 523, 517, 488, 541, 559, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 527, 564, 469, 504, 512, 495, 500, 510, 549, 505, 526, 478, 527, 521, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 550, 511, 494, 476, 517, 493, 489, 484, 522, 509, 533, 537, 545, 519, 505, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 530, 551, 515, 507, 491, 501, 506, 531, 528, 542, 537, 494, 471, 516, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 524, 482, 505, 493, 546, 552, 511, 543, 507, 514, 512, 510, 513, 495, 525, 50, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 513, 498, 517, 494, 523, 520, 535, 499, 497, 485, 528, 543, 501, 521, 482, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8073, 8230, 8115, 8228, 8166, 8256, 8262, 8206] avg=8192.0000 max=8262 max_rank=6 min=8073 min_rank=0 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32890 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[519, 516, 501, 518, 570, 526, 502, 522, 503, 495, 524, 496, 527, 519, 495, 470, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 482, 479, 503, 510, 502, 535, 540, 514, 506, 519, 488, 514, 552, 501, 489, 13, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 509, 542, 512, 498, 529, 532, 533, 486, 531, 530, 502, 491, 516, 527, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 501, 536, 468, 489, 511, 459, 525, 509, 504, 529, 495, 505, 532, 500, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 515, 513, 544, 570, 531, 549, 511, 523, 492, 520, 495, 493, 497, 506, 488, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 523, 530, 534, 498, 470, 513, 475, 516, 525, 437, 526, 508, 513, 544, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 474, 533, 479, 530, 530, 490, 510, 476, 557, 542, 508, 530, 508, 486, 489, 46, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 522, 513, 498, 517, 481, 492, 511, 489, 528, 545, 534, 540, 518, 523, 512, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8228, 8152, 8243, 8094, 8286, 8101, 8203, 8229] avg=8192.0000 max=8286 max_rank=4 min=8094 min_rank=3 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32819 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[500, 515, 510, 553, 467, 514, 501, 509, 504, 534, 462, 507, 530, 521, 508, 522, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 514, 514, 500, 524, 511, 512, 486, 536, 482, 525, 503, 511, 525, 556, 518, 13, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 487, 525, 519, 522, 522, 532, 523, 516, 518, 528, 504, 559, 509, 473, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 521, 551, 494, 530, 532, 473, 515, 537, 513, 509, 486, 535, 516, 504, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 518, 511, 526, 465, 512, 513, 489, 508, 484, 475, 534, 511, 503, 527, 519, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 507, 528, 504, 481, 525, 519, 513, 521, 545, 486, 504, 504, 482, 495, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 493, 477, 500, 518, 510, 502, 489, 524, 475, 507, 526, 506, 540, 516, 485, 48, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 483, 560, 501, 543, 486, 489, 490, 536, 513, 492, 525, 497, 487, 529, 534, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8264, 8214, 8255, 8128, 8136, 8158, 8198] avg=8192.0000 max=8264 max_rank=1 min=8128 min_rank=4 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32620 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 3968, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4096, 3968, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 3968, 4224, 4224, 3968, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 3968, 4096, 4224, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4096, 3968, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 3968, 4096, 4224, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 3968, 4224, 4224, 3968, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 3968, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[533, 501, 522, 543, 531, 530, 541, 527, 515, 501, 497, 495, 525, 517, 532, 529, 553, 526, 514, 477, 504, 545, 538, 524, 511, 538, 522, 509, 537, 489, 484, 539, 504, 494, 508, 545, 556, 529, 513, 516, 512, 505, 512, 504, 561, 505, 556, 474, 526, 488, 527, 515, 518, 508, 472, 538, 520, 499, 532, 480, 480, 503, 519, 480, 503, 510, 525, 494, 475, 501, 522, 491, 509, 503, 519, 499, 482, 503, 508, 495, 490, 480, 543, 522, 526, 508, 503, 498, 542, 507, 522, 537, 496, 509, 520, 534, 505, 527, 524, 516, 527, 542, 475, 520, 494, 505, 509, 523, 490, 485, 517, 503, 506, 529, 498, 506, 493, 508, 488, 503, 479, 514, 525, 492, 513, 512, 527, 457] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[496, 509, 510, 524, 500, 513, 543, 528, 500, 505, 499, 484, 493, 503, 495, 503, 520, 472, 553, 476, 523, 526, 512, 519, 551, 483, 490, 515, 547, 541, 498, 512, 559, 488, 475, 513, 473, 510, 508, 505, 471, 532, 507, 480, 533, 530, 524, 515, 489, 508, 505, 502, 568, 506, 543, 582, 537, 469, 539, 518, 512, 560, 518, 496, 517, 501, 523, 519, 497, 506, 528, 447, 483, 523, 539, 544, 524, 518, 511, 474, 549, 530, 535, 472, 546, 553, 508, 490, 509, 494, 498, 511, 426, 530, 514, 551, 513, 500, 497, 538, 500, 497, 504, 513, 525, 510, 516, 466, 515, 503, 520, 526, 524, 505, 494, 509, 520, 563, 518, 462, 540, 480, 524, 527, 528, 462, 501, 548] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[546, 487, 498, 508, 470, 485, 472, 524, 528, 498, 533, 505, 529, 514, 491, 515, 543, 481, 501, 476, 481, 487, 502, 570, 519, 511, 559, 477, 496, 491, 505, 483, 535, 512, 517, 528, 507, 506, 499, 556, 491, 487, 524, 496, 508, 544, 509, 530, 530, 509, 518, 501, 521, 540, 492, 471, 558, 533, 510, 493, 512, 519, 495, 533, 504, 559, 505, 497, 516, 528, 461, 504, 496, 460, 502, 488, 494, 548, 508, 522, 485, 535, 525, 512, 537, 539, 527, 526, 499, 515, 530, 503, 534, 524, 482, 519, 483, 516, 543, 484, 495, 501, 495, 547, 533, 492, 528, 509, 552, 547, 513, 518, 485, 538, 483, 526, 528, 470, 498, 542, 480, 498, 524, 551, 530, 511, 539, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[533, 501, 522, 543, 531, 530, 541, 527, 515, 501, 497, 495, 525, 517, 532, 529, 553, 526, 514, 477, 504, 545, 538, 524, 511, 538, 522, 509, 537, 489, 484, 539, 504, 494, 508, 545, 556, 529, 513, 516, 512, 505, 512, 504, 561, 505, 556, 474, 526, 488, 527, 515, 518, 508, 472, 538, 520, 499, 532, 480, 480, 503, 519, 480, 503, 510, 525, 494, 475, 501, 522, 491, 509, 503, 519, 499, 482, 503, 508, 495, 490, 480, 543, 522, 526, 508, 503, 498, 542, 507, 522, 537, 496, 509, 520, 534, 505, 527, 524, 516, 527, 542, 475, 520, 494, 505, 509, 523, 490, 485, 517, 503, 506, 529, 498, 506, 493, 508, 488, 503, 479, 514, 525, 492, 513, 512, 527, 457] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[495, 518, 498, 557, 547, 486, 541, 552, 495, 506, 526, 562, 504, 495, 496, 507, 524, 536, 547, 511, 496, 519, 512, 486, 508, 516, 464, 511, 511, 528, 488, 500, 531, 494, 528, 512, 527, 517, 462, 518, 479, 532, 516, 532, 538, 502, 504, 532, 498, 472, 479, 472, 535, 516, 512, 503, 525, 485, 520, 527, 525, 518, 458, 487, 485, 516, 541, 505, 487, 502, 552, 497, 473, 547, 539, 560, 498, 504, 504, 509, 520, 505, 523, 539, 512, 532, 490, 515, 525, 515, 500, 524, 498, 471, 507, 564, 502, 505, 505, 511, 529, 559, 503, 522, 507, 503, 546, 534, 511, 498, 502, 508, 531, 516, 482, 529, 514, 515, 535, 508, 492, 496, 491, 472, 537, 512, 517, 487] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[513, 511, 501, 514, 509, 502, 502, 531, 494, 507, 503, 494, 488, 533, 493, 472, 507, 539, 496, 529, 503, 522, 514, 500, 505, 492, 516, 497, 496, 512, 507, 489, 546, 519, 492, 495, 509, 493, 536, 516, 500, 514, 522, 557, 489, 518, 512, 515, 483, 509, 510, 559, 507, 535, 542, 522, 530, 511, 525, 477, 510, 502, 537, 505, 482, 521, 499, 527, 478, 520, 493, 515, 518, 511, 528, 486, 525, 545, 551, 488, 541, 511, 474, 476, 530, 527, 504, 478, 515, 523, 519, 491, 519, 566, 492, 500, 538, 522, 494, 479, 490, 486, 531, 524, 556, 530, 543, 494, 525, 513, 543, 512, 477, 492, 504, 501, 510, 554, 548, 516, 505, 528, 489, 520, 523, 553, 524, 471] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[530, 546, 464, 484, 550, 460, 509, 482, 535, 530, 504, 490, 513, 508, 464, 474, 513, 524, 550, 486, 491, 490, 525, 474, 532, 503, 533, 555, 539, 502, 499, 512, 536, 532, 534, 530, 494, 491, 519, 527, 518, 537, 522, 495, 510, 474, 505, 493, 508, 470, 530, 504, 542, 453, 543, 493, 519, 510, 469, 522, 542, 544, 523, 487, 464, 501, 538, 442, 527, 495, 519, 549, 477, 519, 493, 533, 499, 529, 537, 520, 509, 513, 514, 524, 507, 511, 512, 509, 517, 513, 521, 464, 551, 548, 536, 518, 480, 524, 511, 518, 455, 500, 524, 470, 479, 541, 506, 526, 511, 497, 565, 514, 531, 521, 533, 488, 543, 512, 538, 520, 506, 503, 515, 536, 490, 536, 542, 545] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[514, 472, 461, 501, 536, 562, 515, 537, 559, 509, 494, 492, 522, 500, 504, 518, 511, 547, 498, 493, 519, 526, 520, 507, 504, 504, 503, 543, 505, 503, 492, 501, 504, 500, 509, 492, 522, 511, 483, 495, 516, 530, 516, 506, 510, 493, 510, 501, 524, 497, 492, 554, 524, 506, 512, 453, 487, 483, 515, 517, 507, 518, 510, 526, 499, 514, 522, 535, 531, 490, 509, 505, 503, 483, 537, 543, 522, 520, 519, 493, 494, 548, 517, 509, 517, 545, 496, 473, 470, 517, 502, 498, 522, 539, 497, 538, 527, 509, 528, 509, 510, 496, 491, 546, 522, 534, 521, 518, 508, 526, 485, 500, 517, 530, 532, 510, 565, 492, 548, 488, 490, 534, 495, 502, 520, 511, 543, 527] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[524, 490, 494, 522, 549, 509, 525, 512, 507, 524, 527, 541, 481, 508, 567, 540, 542, 499, 524, 509, 495, 525, 507, 532, 501, 530, 491, 492, 542, 528, 500, 499, 514, 492, 557, 491, 520, 502, 496, 486, 532, 496, 521, 462, 506, 511, 533, 510, 504, 539, 526, 495, 489, 521, 527, 507, 526, 489, 490, 521, 527, 522, 518, 497, 491, 457, 517, 506, 528, 483, 530, 541, 522, 528, 489, 465, 550, 543, 525, 500, 526, 530, 527, 530, 506, 511, 466, 470, 531, 519, 454, 471, 507, 505, 518, 535, 525, 534, 532, 543, 512, 501, 492, 477, 506, 544, 532, 525, 505, 547, 512, 498, 511, 510, 497, 511, 493, 511, 532, 514, 493, 497, 547, 506, 499, 515, 473, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[530, 546, 464, 484, 550, 460, 509, 482, 535, 530, 504, 490, 513, 508, 464, 474, 513, 524, 550, 486, 491, 490, 525, 474, 532, 503, 533, 555, 539, 502, 499, 512, 536, 532, 534, 530, 494, 491, 519, 527, 518, 537, 522, 495, 510, 474, 505, 493, 508, 470, 530, 504, 542, 453, 543, 493, 519, 510, 469, 522, 542, 544, 523, 487, 464, 501, 538, 442, 527, 495, 519, 549, 477, 519, 493, 533, 499, 529, 537, 520, 509, 513, 514, 524, 507, 511, 512, 509, 517, 513, 521, 464, 551, 548, 536, 518, 480, 524, 511, 518, 455, 500, 524, 470, 479, 541, 506, 526, 511, 497, 565, 514, 531, 521, 533, 488, 543, 512, 538, 520, 506, 503, 515, 536, 490, 536, 542, 545] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[513, 511, 501, 514, 509, 502, 502, 531, 494, 507, 503, 494, 488, 533, 493, 472, 507, 539, 496, 529, 503, 522, 514, 500, 505, 492, 516, 497, 496, 512, 507, 489, 546, 519, 492, 495, 509, 493, 536, 516, 500, 514, 522, 557, 489, 518, 512, 515, 483, 509, 510, 559, 507, 535, 542, 522, 530, 511, 525, 477, 510, 502, 537, 505, 482, 521, 499, 527, 478, 520, 493, 515, 518, 511, 528, 486, 525, 545, 551, 488, 541, 511, 474, 476, 530, 527, 504, 478, 515, 523, 519, 491, 519, 566, 492, 500, 538, 522, 494, 479, 490, 486, 531, 524, 556, 530, 543, 494, 525, 513, 543, 512, 477, 492, 504, 501, 510, 554, 548, 516, 505, 528, 489, 520, 523, 553, 524, 471] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[546, 487, 498, 508, 470, 485, 472, 524, 528, 498, 533, 505, 529, 514, 491, 515, 543, 481, 501, 476, 481, 487, 502, 570, 519, 511, 559, 477, 496, 491, 505, 483, 535, 512, 517, 528, 507, 506, 499, 556, 491, 487, 524, 496, 508, 544, 509, 530, 530, 509, 518, 501, 521, 540, 492, 471, 558, 533, 510, 493, 512, 519, 495, 533, 504, 559, 505, 497, 516, 528, 461, 504, 496, 460, 502, 488, 494, 548, 508, 522, 485, 535, 525, 512, 537, 539, 527, 526, 499, 515, 530, 503, 534, 524, 482, 519, 483, 516, 543, 484, 495, 501, 495, 547, 533, 492, 528, 509, 552, 547, 513, 518, 485, 538, 483, 526, 528, 470, 498, 542, 480, 498, 524, 551, 530, 511, 539, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[496, 509, 510, 524, 500, 513, 543, 528, 500, 505, 499, 484, 493, 503, 495, 503, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 472, 553, 476, 523, 526, 512, 519, 551, 483, 490, 515, 547, 541, 498, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 488, 475, 513, 473, 510, 508, 505, 471, 532, 507, 480, 533, 530, 524, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 508, 505, 502, 568, 506, 543, 582, 537, 469, 539, 518, 512, 560, 518, 496, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 501, 523, 519, 497, 506, 528, 447, 483, 523, 539, 544, 524, 518, 511, 474, 22, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 530, 535, 472, 546, 553, 508, 490, 509, 494, 498, 511, 426, 530, 514, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 500, 497, 538, 500, 497, 504, 513, 525, 510, 494, 466, 515, 503, 520, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 505, 494, 509, 520, 563, 511, 462, 540, 480, 524, 527, 528, 462, 501, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8114, 8238, 8120, 8359, 8191, 8195, 8121, 8198] avg=8192.0000 max=8359 max_rank=3 min=8114 min_rank=0 imbalance_ratio=1.020386 max_load_reduction=57177 max_load_reduction_pct=0.872452 moved_to_echo=32705 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[533, 501, 522, 543, 531, 530, 541, 527, 515, 501, 497, 495, 525, 517, 532, 529, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 526, 514, 477, 504, 545, 538, 524, 511, 538, 522, 509, 537, 489, 484, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 494, 508, 545, 556, 529, 513, 516, 512, 505, 512, 504, 561, 505, 556, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 488, 527, 515, 518, 508, 472, 538, 520, 499, 532, 480, 480, 503, 519, 480, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 510, 525, 494, 475, 501, 522, 491, 509, 503, 519, 499, 482, 503, 508, 495, 22, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 480, 543, 522, 526, 508, 503, 498, 542, 507, 522, 537, 496, 509, 520, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 527, 524, 516, 527, 542, 475, 520, 494, 505, 487, 523, 490, 485, 517, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 529, 498, 506, 493, 508, 482, 503, 479, 514, 525, 492, 513, 512, 527, 457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8347, 8310, 8291, 8112, 8075, 8217, 8140, 8044] avg=8192.0000 max=8347 max_rank=0 min=8044 min_rank=7 imbalance_ratio=1.018921 max_load_reduction=57189 max_load_reduction_pct=0.872635 moved_to_echo=32476 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[524, 490, 494, 522, 549, 509, 525, 512, 507, 524, 527, 541, 481, 508, 567, 540, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 499, 524, 509, 495, 525, 507, 532, 501, 530, 491, 492, 542, 528, 500, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 492, 557, 491, 520, 502, 496, 486, 532, 496, 521, 462, 506, 511, 533, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 539, 526, 495, 489, 521, 527, 507, 526, 489, 490, 521, 527, 522, 518, 497, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 457, 517, 506, 528, 483, 530, 541, 522, 528, 489, 465, 550, 543, 525, 500, 23, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 530, 527, 530, 506, 511, 466, 470, 531, 519, 454, 471, 507, 505, 518, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 534, 532, 543, 512, 501, 492, 477, 506, 544, 509, 525, 505, 547, 512, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 510, 497, 511, 493, 511, 524, 514, 493, 497, 547, 506, 499, 515, 473, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8329, 8216, 8126, 8206, 8212, 8086, 8262, 8099] avg=8192.0000 max=8329 max_rank=0 min=8086 min_rank=5 imbalance_ratio=1.016724 max_load_reduction=57207 max_load_reduction_pct=0.872910 moved_to_echo=32659 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[514, 472, 461, 501, 536, 562, 515, 537, 559, 509, 494, 492, 522, 500, 504, 518, 10, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 547, 498, 493, 519, 526, 520, 507, 504, 504, 503, 543, 505, 503, 492, 501, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 500, 509, 492, 522, 511, 483, 495, 516, 530, 516, 506, 510, 493, 510, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 497, 492, 554, 524, 506, 512, 453, 487, 483, 515, 517, 507, 518, 510, 526, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 514, 522, 535, 531, 490, 509, 505, 503, 483, 537, 543, 522, 520, 519, 493, 26, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 548, 517, 509, 517, 545, 496, 473, 470, 517, 502, 498, 522, 539, 497, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 509, 528, 509, 510, 496, 491, 546, 522, 534, 495, 518, 508, 526, 485, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 530, 532, 510, 565, 492, 534, 488, 490, 534, 495, 502, 520, 511, 543, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8181, 8088, 8138, 8269, 8154, 8204, 8290] avg=8192.0000 max=8290 max_rank=7 min=8088 min_rank=2 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32917 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[495, 518, 498, 557, 547, 486, 541, 552, 495, 506, 526, 562, 504, 495, 496, 507, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 536, 547, 511, 496, 519, 512, 486, 508, 516, 464, 511, 511, 528, 488, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 494, 528, 512, 527, 517, 462, 518, 479, 532, 516, 532, 538, 502, 504, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 472, 479, 472, 535, 516, 512, 503, 525, 485, 520, 527, 525, 518, 458, 487, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 516, 541, 505, 487, 502, 552, 497, 473, 547, 539, 560, 498, 504, 504, 509, 24, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 505, 523, 539, 512, 532, 490, 515, 525, 515, 500, 524, 498, 471, 507, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 505, 505, 511, 529, 559, 503, 522, 507, 503, 522, 534, 511, 498, 502, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 516, 482, 529, 514, 515, 527, 508, 492, 496, 491, 472, 537, 512, 517, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8295, 8157, 8221, 8040, 8258, 8218, 8221, 8126] avg=8192.0000 max=8295 max_rank=0 min=8040 min_rank=3 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32823 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[530, 546, 464, 484, 550, 460, 509, 482, 535, 530, 504, 490, 513, 508, 464, 474, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 524, 550, 486, 491, 490, 525, 474, 532, 503, 533, 555, 539, 502, 499, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 532, 534, 530, 494, 491, 519, 527, 518, 537, 522, 495, 510, 474, 505, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 470, 530, 504, 542, 453, 543, 493, 519, 510, 469, 522, 542, 544, 523, 487, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 464, 501, 538, 442, 527, 495, 519, 549, 477, 519, 493, 533, 499, 529, 537, 520, 22, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 513, 514, 524, 507, 511, 512, 509, 517, 513, 521, 464, 551, 548, 536, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 524, 511, 518, 455, 500, 524, 470, 479, 541, 484, 526, 511, 497, 565, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 521, 533, 488, 543, 512, 530, 520, 506, 503, 515, 536, 490, 536, 542, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8052, 8228, 8214, 8167, 8178, 8247, 8099, 8351] avg=8192.0000 max=8351 max_rank=7 min=8052 min_rank=0 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32875 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[513, 511, 501, 514, 509, 502, 502, 531, 494, 507, 503, 494, 488, 533, 493, 472, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 539, 496, 529, 503, 522, 514, 500, 505, 492, 516, 497, 496, 512, 507, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 519, 492, 495, 509, 493, 536, 516, 500, 514, 522, 557, 489, 518, 512, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 509, 510, 559, 507, 535, 542, 522, 530, 511, 525, 477, 510, 502, 537, 505, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 521, 499, 527, 478, 520, 493, 515, 518, 511, 528, 486, 525, 545, 551, 488, 24, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 511, 474, 476, 530, 527, 504, 478, 515, 523, 519, 491, 519, 566, 492, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 522, 494, 479, 490, 486, 531, 524, 556, 530, 519, 494, 525, 513, 543, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 492, 504, 501, 510, 554, 540, 516, 505, 528, 489, 520, 523, 553, 524, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8076, 8124, 8230, 8272, 8224, 8147, 8256, 8207] avg=8192.0000 max=8272 max_rank=3 min=8076 min_rank=0 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32834 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[546, 487, 498, 508, 470, 485, 472, 524, 528, 498, 533, 505, 529, 514, 491, 515, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 481, 501, 476, 481, 487, 502, 570, 519, 511, 559, 477, 496, 491, 505, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 512, 517, 528, 507, 506, 499, 556, 491, 487, 524, 496, 508, 544, 509, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 509, 518, 501, 521, 540, 492, 471, 558, 533, 510, 493, 512, 519, 495, 533, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 559, 505, 497, 516, 528, 461, 504, 496, 460, 502, 488, 494, 548, 508, 522, 23, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 535, 525, 512, 537, 539, 527, 526, 499, 515, 530, 503, 534, 524, 482, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 516, 543, 484, 495, 501, 495, 547, 533, 492, 505, 509, 552, 547, 513, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 538, 483, 526, 528, 470, 492, 542, 480, 498, 524, 551, 530, 511, 539, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8111, 8082, 8246, 8242, 8129, 8272, 8233, 8221] avg=8192.0000 max=8272 max_rank=5 min=8082 min_rank=1 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32855 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[513, 511, 501, 514, 509, 502, 502, 531, 494, 507, 503, 494, 488, 533, 493, 472, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 539, 496, 529, 503, 522, 514, 500, 505, 492, 516, 497, 496, 512, 507, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 519, 492, 495, 509, 493, 536, 516, 500, 514, 522, 557, 489, 518, 512, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 509, 510, 559, 507, 535, 542, 522, 530, 511, 525, 477, 510, 502, 537, 505, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 521, 499, 527, 478, 520, 493, 515, 518, 511, 528, 486, 525, 545, 551, 488, 24, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 511, 474, 476, 530, 527, 504, 478, 515, 523, 519, 491, 519, 566, 492, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 522, 494, 479, 490, 486, 531, 524, 556, 530, 519, 494, 525, 513, 543, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 492, 504, 501, 510, 554, 540, 516, 505, 528, 489, 520, 523, 553, 524, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8076, 8124, 8230, 8272, 8224, 8147, 8256, 8207] avg=8192.0000 max=8272 max_rank=3 min=8076 min_rank=0 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32834 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[496, 509, 510, 524, 500, 513, 543, 528, 500, 505, 499, 484, 493, 503, 495, 503, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 472, 553, 476, 523, 526, 512, 519, 551, 483, 490, 515, 547, 541, 498, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 488, 475, 513, 473, 510, 508, 505, 471, 532, 507, 480, 533, 530, 524, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 508, 505, 502, 568, 506, 543, 582, 537, 469, 539, 518, 512, 560, 518, 496, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 501, 523, 519, 497, 506, 528, 447, 483, 523, 539, 544, 524, 518, 511, 474, 22, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 530, 535, 472, 546, 553, 508, 490, 509, 494, 498, 511, 426, 530, 514, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 500, 497, 538, 500, 497, 504, 513, 525, 510, 494, 466, 515, 503, 520, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 505, 494, 509, 520, 563, 511, 462, 540, 480, 524, 527, 528, 462, 501, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8114, 8238, 8120, 8359, 8191, 8195, 8121, 8198] avg=8192.0000 max=8359 max_rank=3 min=8114 min_rank=0 imbalance_ratio=1.020386 max_load_reduction=57177 max_load_reduction_pct=0.872452 moved_to_echo=32705 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[533, 501, 522, 543, 531, 530, 541, 527, 515, 501, 497, 495, 525, 517, 532, 529, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 526, 514, 477, 504, 545, 538, 524, 511, 538, 522, 509, 537, 489, 484, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 494, 508, 545, 556, 529, 513, 516, 512, 505, 512, 504, 561, 505, 556, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 488, 527, 515, 518, 508, 472, 538, 520, 499, 532, 480, 480, 503, 519, 480, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 510, 525, 494, 475, 501, 522, 491, 509, 503, 519, 499, 482, 503, 508, 495, 22, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 480, 543, 522, 526, 508, 503, 498, 542, 507, 522, 537, 496, 509, 520, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 527, 524, 516, 527, 542, 475, 520, 494, 505, 487, 523, 490, 485, 517, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 529, 498, 506, 493, 508, 482, 503, 479, 514, 525, 492, 513, 512, 527, 457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8347, 8310, 8291, 8112, 8075, 8217, 8140, 8044] avg=8192.0000 max=8347 max_rank=0 min=8044 min_rank=7 imbalance_ratio=1.018921 max_load_reduction=57189 max_load_reduction_pct=0.872635 moved_to_echo=32476 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[530, 546, 464, 484, 550, 460, 509, 482, 535, 530, 504, 490, 513, 508, 464, 474, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 524, 550, 486, 491, 490, 525, 474, 532, 503, 533, 555, 539, 502, 499, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 532, 534, 530, 494, 491, 519, 527, 518, 537, 522, 495, 510, 474, 505, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 470, 530, 504, 542, 453, 543, 493, 519, 510, 469, 522, 542, 544, 523, 487, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 464, 501, 538, 442, 527, 495, 519, 549, 477, 519, 493, 533, 499, 529, 537, 520, 22, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 513, 514, 524, 507, 511, 512, 509, 517, 513, 521, 464, 551, 548, 536, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 524, 511, 518, 455, 500, 524, 470, 479, 541, 484, 526, 511, 497, 565, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 521, 533, 488, 543, 512, 530, 520, 506, 503, 515, 536, 490, 536, 542, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8052, 8228, 8214, 8167, 8178, 8247, 8099, 8351] avg=8192.0000 max=8351 max_rank=7 min=8052 min_rank=0 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32875 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[524, 490, 494, 522, 549, 509, 525, 512, 507, 524, 527, 541, 481, 508, 567, 540, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 499, 524, 509, 495, 525, 507, 532, 501, 530, 491, 492, 542, 528, 500, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 492, 557, 491, 520, 502, 496, 486, 532, 496, 521, 462, 506, 511, 533, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 539, 526, 495, 489, 521, 527, 507, 526, 489, 490, 521, 527, 522, 518, 497, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 457, 517, 506, 528, 483, 530, 541, 522, 528, 489, 465, 550, 543, 525, 500, 23, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 530, 527, 530, 506, 511, 466, 470, 531, 519, 454, 471, 507, 505, 518, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 534, 532, 543, 512, 501, 492, 477, 506, 544, 509, 525, 505, 547, 512, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 510, 497, 511, 493, 511, 524, 514, 493, 497, 547, 506, 499, 515, 473, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8329, 8216, 8126, 8206, 8212, 8086, 8262, 8099] avg=8192.0000 max=8329 max_rank=0 min=8086 min_rank=5 imbalance_ratio=1.016724 max_load_reduction=57207 max_load_reduction_pct=0.872910 moved_to_echo=32659 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 3968, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[3968, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 3968, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4352, 4096, 3968, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[506, 497, 474, 532, 513, 564, 511, 491, 511, 499, 522, 502, 482, 509, 507, 500, 524, 547, 523, 512, 532, 531, 484, 493, 493, 528, 544, 551, 541, 477, 523, 504, 521, 513, 505, 503, 524, 503, 482, 542, 576, 500, 515, 495, 512, 505, 535, 503, 545, 557, 493, 536, 525, 504, 521, 493, 492, 497, 516, 486, 553, 548, 502, 500, 523, 513, 464, 510, 479, 505, 511, 494, 534, 516, 496, 511, 515, 502, 499, 535, 477, 520, 503, 481, 504, 528, 539, 515, 505, 522, 538, 513, 538, 547, 524, 527, 518, 472, 550, 499, 485, 519, 503, 490, 539, 522, 500, 488, 524, 512, 545, 501, 495, 506, 485, 502, 546, 505, 508, 492, 538, 489, 482, 529, 492, 503, 475, 505] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[497, 499, 502, 478, 554, 505, 512, 547, 499, 538, 505, 515, 510, 508, 517, 530, 534, 508, 558, 499, 527, 527, 499, 540, 529, 473, 472, 543, 534, 471, 518, 553, 509, 508, 517, 511, 510, 488, 514, 509, 548, 485, 524, 498, 517, 503, 484, 486, 523, 521, 559, 527, 557, 504, 531, 551, 494, 506, 534, 474, 531, 464, 547, 545, 532, 500, 506, 480, 526, 481, 530, 525, 518, 523, 555, 520, 512, 513, 507, 533, 517, 495, 470, 484, 514, 512, 501, 516, 507, 525, 516, 529, 500, 534, 459, 523, 511, 489, 471, 495, 527, 494, 514, 491, 525, 475, 496, 513, 487, 472, 496, 537, 514, 536, 502, 517, 508, 495, 474, 495, 508, 544, 548, 508, 462, 541, 525, 523] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[530, 500, 509, 499, 532, 522, 522, 532, 501, 553, 528, 533, 498, 508, 513, 496, 477, 533, 515, 522, 496, 536, 508, 503, 503, 524, 520, 480, 528, 512, 467, 514, 534, 501, 501, 523, 496, 502, 521, 521, 526, 536, 523, 541, 519, 521, 543, 519, 475, 532, 540, 492, 465, 485, 471, 525, 503, 497, 527, 461, 547, 503, 515, 524, 517, 444, 491, 566, 516, 530, 496, 535, 547, 495, 510, 468, 532, 524, 523, 482, 500, 533, 473, 493, 564, 517, 460, 519, 508, 531, 517, 498, 492, 506, 513, 509, 515, 509, 494, 476, 489, 523, 504, 518, 536, 513, 505, 524, 524, 511, 487, 491, 545, 521, 520, 521, 515, 483, 533, 551, 514, 505, 514, 491, 538, 539, 518, 497] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[483, 523, 469, 537, 555, 513, 492, 554, 527, 519, 496, 514, 517, 487, 527, 503, 499, 531, 478, 465, 511, 497, 513, 484, 498, 521, 523, 530, 487, 517, 524, 527, 477, 506, 524, 475, 525, 520, 530, 507, 508, 543, 536, 512, 532, 521, 521, 499, 533, 465, 511, 549, 499, 539, 518, 544, 530, 539, 468, 512, 506, 500, 480, 547, 558, 509, 471, 518, 492, 520, 525, 494, 546, 523, 501, 518, 513, 526, 505, 554, 495, 530, 526, 477, 534, 493, 526, 531, 505, 513, 516, 479, 499, 505, 484, 508, 516, 505, 501, 501, 480, 486, 527, 504, 507, 539, 557, 558, 483, 515, 491, 540, 534, 526, 495, 532, 542, 500, 521, 507, 493, 500, 486, 493, 476, 538, 485, 507] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[490, 488, 510, 513, 528, 502, 525, 506, 503, 522, 530, 487, 507, 481, 528, 526, 515, 522, 512, 558, 541, 548, 477, 492, 506, 506, 525, 493, 515, 536, 501, 461, 513, 489, 505, 575, 513, 526, 544, 533, 501, 504, 466, 558, 529, 510, 541, 507, 546, 516, 501, 502, 504, 519, 518, 516, 481, 485, 517, 512, 476, 487, 510, 527, 501, 539, 564, 501, 508, 525, 535, 483, 523, 561, 484, 521, 504, 536, 506, 522, 501, 472, 512, 512, 493, 516, 515, 506, 524, 517, 523, 459, 516, 513, 498, 499, 521, 457, 492, 543, 526, 508, 486, 519, 527, 517, 522, 528, 498, 532, 508, 516, 519, 489, 508, 473, 491, 498, 480, 513, 548, 531, 510, 514, 554, 522, 526, 486] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[488, 540, 461, 510, 533, 514, 504, 487, 497, 500, 472, 519, 507, 468, 562, 477, 500, 502, 515, 537, 545, 510, 523, 533, 559, 495, 493, 497, 530, 472, 495, 504, 513, 519, 498, 497, 528, 500, 485, 559, 512, 492, 509, 530, 488, 509, 545, 515, 475, 504, 511, 507, 522, 507, 556, 498, 534, 485, 528, 538, 539, 527, 526, 528, 482, 527, 506, 560, 498, 509, 516, 519, 498, 525, 545, 530, 496, 546, 515, 480, 546, 486, 484, 520, 494, 486, 529, 470, 511, 521, 464, 477, 529, 496, 507, 508, 526, 487, 511, 513, 526, 529, 514, 501, 503, 526, 542, 483, 562, 508, 477, 567, 576, 537, 508, 485, 548, 498, 512, 520, 500, 500, 525, 502, 466, 501, 528, 542] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[530, 500, 509, 499, 532, 522, 522, 532, 501, 553, 528, 533, 498, 508, 513, 496, 477, 533, 515, 522, 496, 536, 508, 503, 503, 524, 520, 480, 528, 512, 467, 514, 534, 501, 501, 523, 496, 502, 521, 521, 526, 536, 523, 541, 519, 521, 543, 519, 475, 532, 540, 492, 465, 485, 471, 525, 503, 497, 527, 461, 547, 503, 515, 524, 517, 444, 491, 566, 516, 530, 496, 535, 547, 495, 510, 468, 532, 524, 523, 482, 500, 533, 473, 493, 564, 517, 460, 519, 508, 531, 517, 498, 492, 506, 513, 509, 515, 509, 494, 476, 489, 523, 504, 518, 536, 513, 505, 524, 524, 511, 487, 491, 545, 521, 520, 521, 515, 483, 533, 551, 514, 505, 514, 491, 538, 539, 518, 497] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[497, 499, 502, 478, 554, 505, 512, 547, 499, 538, 505, 515, 510, 508, 517, 530, 534, 508, 558, 499, 527, 527, 499, 540, 529, 473, 472, 543, 534, 471, 518, 553, 509, 508, 517, 511, 510, 488, 514, 509, 548, 485, 524, 498, 517, 503, 484, 486, 523, 521, 559, 527, 557, 504, 531, 551, 494, 506, 534, 474, 531, 464, 547, 545, 532, 500, 506, 480, 526, 481, 530, 525, 518, 523, 555, 520, 512, 513, 507, 533, 517, 495, 470, 484, 514, 512, 501, 516, 507, 525, 516, 529, 500, 534, 459, 523, 511, 489, 471, 495, 527, 494, 514, 491, 525, 475, 496, 513, 487, 472, 496, 537, 514, 536, 502, 517, 508, 495, 474, 495, 508, 544, 548, 508, 462, 541, 525, 523] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[502, 503, 484, 496, 471, 480, 472, 517, 484, 540, 515, 526, 520, 522, 506, 451, 524, 482, 531, 520, 491, 478, 513, 505, 515, 504, 502, 506, 524, 519, 573, 533, 543, 488, 488, 533, 463, 492, 509, 517, 520, 514, 542, 482, 514, 534, 529, 503, 518, 521, 522, 482, 513, 520, 508, 505, 496, 480, 555, 510, 538, 521, 537, 477, 491, 512, 542, 553, 520, 476, 469, 553, 517, 507, 531, 497, 548, 537, 532, 539, 549, 503, 513, 536, 557, 498, 506, 509, 518, 565, 507, 512, 467, 548, 511, 521, 487, 510, 490, 510, 540, 508, 526, 500, 561, 489, 532, 479, 500, 511, 515, 505, 527, 466, 515, 497, 462, 508, 530, 501, 516, 513, 544, 451, 499, 532, 511, 574] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[483, 523, 469, 537, 555, 513, 492, 554, 527, 519, 496, 514, 517, 487, 527, 503, 499, 531, 478, 465, 511, 497, 513, 484, 498, 521, 523, 530, 487, 517, 524, 527, 477, 506, 524, 475, 525, 520, 530, 507, 508, 543, 536, 512, 532, 521, 521, 499, 533, 465, 511, 549, 499, 539, 518, 544, 530, 539, 468, 512, 506, 500, 480, 547, 558, 509, 471, 518, 492, 520, 525, 494, 546, 523, 501, 518, 513, 526, 505, 554, 495, 530, 526, 477, 534, 493, 526, 531, 505, 513, 516, 479, 499, 505, 484, 508, 516, 505, 501, 501, 480, 486, 527, 504, 507, 539, 557, 558, 483, 515, 491, 540, 534, 526, 495, 532, 542, 500, 521, 507, 493, 500, 486, 493, 476, 538, 485, 507] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[545, 515, 499, 509, 477, 533, 509, 513, 522, 520, 518, 449, 496, 501, 544, 536, 484, 532, 529, 541, 563, 551, 495, 480, 485, 464, 568, 524, 503, 532, 543, 510, 514, 536, 509, 534, 536, 505, 504, 526, 529, 490, 506, 499, 489, 540, 524, 524, 515, 508, 521, 527, 542, 493, 556, 510, 505, 544, 550, 506, 539, 529, 530, 515, 532, 491, 504, 508, 504, 495, 494, 515, 500, 494, 505, 503, 482, 492, 513, 521, 507, 495, 542, 514, 505, 518, 518, 512, 516, 529, 529, 486, 504, 506, 510, 491, 521, 452, 535, 496, 507, 493, 505, 523, 501, 545, 513, 485, 524, 472, 510, 517, 514, 507, 500, 499, 505, 529, 459, 510, 493, 508, 517, 495, 488, 572, 473, 488] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[488, 540, 461, 510, 533, 514, 504, 487, 497, 500, 472, 519, 507, 468, 562, 477, 500, 502, 515, 537, 545, 510, 523, 533, 559, 495, 493, 497, 530, 472, 495, 504, 513, 519, 498, 497, 528, 500, 485, 559, 512, 492, 509, 530, 488, 509, 545, 515, 475, 504, 511, 507, 522, 507, 556, 498, 534, 485, 528, 538, 539, 527, 526, 528, 482, 527, 506, 560, 498, 509, 516, 519, 498, 525, 545, 530, 496, 546, 515, 480, 546, 486, 484, 520, 494, 486, 529, 470, 511, 521, 464, 477, 529, 496, 507, 508, 526, 487, 511, 513, 526, 529, 514, 501, 503, 526, 542, 483, 562, 508, 477, 567, 576, 537, 508, 485, 548, 498, 512, 520, 500, 500, 525, 502, 466, 501, 528, 542] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[530, 500, 509, 499, 532, 522, 522, 532, 501, 553, 528, 533, 498, 508, 513, 496, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 533, 515, 522, 469, 536, 508, 503, 503, 524, 520, 480, 528, 512, 467, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 501, 501, 523, 496, 502, 521, 521, 526, 536, 523, 541, 519, 521, 503, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 532, 540, 492, 465, 485, 471, 525, 503, 497, 527, 461, 496, 503, 515, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 444, 491, 536, 516, 530, 496, 535, 547, 495, 510, 468, 532, 524, 523, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 533, 473, 493, 564, 517, 460, 519, 508, 531, 517, 498, 492, 506, 513, 509, 27, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 509, 494, 476, 489, 523, 504, 518, 536, 513, 505, 524, 524, 511, 487, 491, 33, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 521, 520, 521, 515, 483, 533, 551, 514, 505, 514, 491, 538, 539, 518, 497, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8321, 8111, 8287, 8011, 8146, 8167, 8158, 8335] avg=8192.0000 max=8335 max_rank=7 min=8011 min_rank=3 imbalance_ratio=1.017456 max_load_reduction=57201 max_load_reduction_pct=0.872818 moved_to_echo=32806 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[506, 497, 474, 532, 513, 564, 511, 491, 511, 499, 522, 502, 482, 509, 507, 500, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 547, 523, 512, 502, 531, 484, 493, 493, 528, 544, 551, 541, 477, 523, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 513, 505, 503, 524, 503, 482, 542, 576, 500, 515, 495, 512, 505, 495, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 557, 493, 536, 525, 504, 521, 493, 492, 497, 516, 486, 502, 548, 502, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 513, 464, 483, 479, 505, 511, 494, 534, 516, 496, 511, 515, 502, 499, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 520, 503, 481, 504, 528, 539, 515, 505, 522, 538, 513, 538, 547, 524, 527, 29, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 472, 550, 499, 485, 519, 503, 490, 539, 522, 500, 488, 524, 512, 545, 501, 33, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 506, 485, 502, 546, 505, 508, 492, 538, 489, 482, 529, 492, 503, 475, 505, 27, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8165, 8277, 8194, 8217, 8080, 8317, 8206, 8080] avg=8192.0000 max=8317 max_rank=5 min=8080 min_rank=4 imbalance_ratio=1.015259 max_load_reduction=57219 max_load_reduction_pct=0.873093 moved_to_echo=32683 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[490, 488, 510, 513, 528, 502, 525, 506, 503, 522, 530, 487, 507, 481, 528, 526, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 522, 512, 558, 511, 548, 477, 492, 506, 506, 525, 493, 515, 536, 501, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 489, 505, 575, 513, 526, 544, 533, 501, 504, 466, 558, 529, 510, 501, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 516, 501, 502, 504, 519, 518, 516, 481, 485, 517, 512, 432, 487, 510, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 539, 564, 475, 508, 525, 535, 483, 523, 561, 484, 521, 504, 536, 506, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 472, 512, 512, 493, 516, 515, 506, 524, 517, 523, 459, 516, 513, 498, 499, 29, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 457, 492, 543, 526, 508, 486, 519, 527, 517, 522, 528, 498, 532, 508, 516, 33, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 489, 508, 473, 491, 498, 480, 513, 548, 531, 510, 514, 554, 522, 526, 486, 26, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8178, 8274, 8073, 8287, 8112, 8238, 8189] avg=8192.0000 max=8287 max_rank=4 min=8073 min_rank=3 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32826 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[488, 540, 461, 510, 533, 514, 504, 487, 497, 500, 472, 519, 507, 468, 562, 477, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 502, 515, 537, 508, 510, 523, 533, 559, 495, 493, 497, 530, 472, 495, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 519, 498, 497, 528, 500, 485, 559, 512, 492, 509, 530, 488, 509, 496, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 504, 511, 507, 522, 507, 556, 498, 534, 485, 528, 538, 483, 527, 526, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 527, 506, 527, 498, 509, 516, 519, 498, 525, 545, 530, 496, 546, 515, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 486, 484, 520, 494, 486, 529, 470, 511, 521, 464, 477, 529, 496, 507, 508, 33, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 487, 511, 513, 526, 529, 514, 501, 503, 526, 542, 483, 562, 508, 477, 567, 37, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 576, 537, 508, 485, 548, 498, 512, 520, 500, 500, 525, 502, 466, 501, 528, 542, 33, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8086, 8173, 8150, 8229, 8219, 8073, 8321, 8285] avg=8192.0000 max=8321 max_rank=6 min=8073 min_rank=5 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32898 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[490, 488, 510, 513, 528, 502, 525, 506, 503, 522, 530, 487, 507, 481, 528, 526, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 522, 512, 558, 511, 548, 477, 492, 506, 506, 525, 493, 515, 536, 501, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 489, 505, 575, 513, 526, 544, 533, 501, 504, 466, 558, 529, 510, 501, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 516, 501, 502, 504, 519, 518, 516, 481, 485, 517, 512, 432, 487, 510, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 539, 564, 475, 508, 525, 535, 483, 523, 561, 484, 521, 504, 536, 506, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 472, 512, 512, 493, 516, 515, 506, 524, 517, 523, 459, 516, 513, 498, 499, 29, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 457, 492, 543, 526, 508, 486, 519, 527, 517, 522, 528, 498, 532, 508, 516, 33, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 489, 508, 473, 491, 498, 480, 513, 548, 531, 510, 514, 554, 522, 526, 486, 26, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8178, 8274, 8073, 8287, 8112, 8238, 8189] avg=8192.0000 max=8287 max_rank=4 min=8073 min_rank=3 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32826 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[483, 523, 469, 537, 555, 513, 492, 554, 527, 519, 496, 514, 517, 487, 527, 503, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 531, 478, 465, 483, 497, 513, 484, 498, 521, 523, 530, 487, 517, 524, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 506, 524, 475, 525, 520, 530, 507, 508, 543, 536, 512, 532, 521, 482, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 465, 511, 549, 499, 539, 518, 544, 530, 539, 468, 512, 460, 500, 480, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 558, 509, 471, 491, 492, 520, 525, 494, 546, 523, 501, 518, 513, 526, 505, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 530, 526, 477, 534, 493, 526, 531, 505, 513, 516, 479, 499, 505, 484, 508, 28, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 505, 501, 501, 480, 486, 527, 504, 507, 539, 557, 558, 483, 515, 491, 540, 32, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 526, 495, 532, 542, 500, 521, 507, 493, 500, 486, 493, 476, 538, 485, 507, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8257, 8077, 8197, 8194, 8246, 8156, 8247, 8162] avg=8192.0000 max=8257 max_rank=0 min=8077 min_rank=1 imbalance_ratio=1.007935 max_load_reduction=57279 max_load_reduction_pct=0.874008 moved_to_echo=32811 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[502, 503, 484, 496, 471, 480, 472, 517, 484, 540, 515, 526, 520, 522, 506, 451, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 482, 531, 520, 464, 478, 513, 505, 515, 504, 502, 506, 524, 519, 573, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 488, 488, 533, 463, 492, 509, 517, 520, 514, 542, 482, 514, 534, 490, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 521, 522, 482, 513, 520, 508, 505, 496, 480, 555, 510, 488, 521, 537, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 512, 542, 524, 520, 476, 469, 553, 517, 507, 531, 497, 548, 537, 532, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 503, 513, 536, 557, 498, 506, 509, 518, 565, 507, 512, 467, 548, 511, 521, 27, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 510, 490, 510, 540, 508, 526, 500, 561, 489, 532, 479, 500, 511, 515, 505, 32, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 466, 515, 497, 462, 508, 530, 501, 516, 513, 544, 451, 499, 532, 511, 574, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8033, 8193, 8132, 8153, 8295, 8354, 8201, 8175] avg=8192.0000 max=8354 max_rank=5 min=8033 min_rank=0 imbalance_ratio=1.019775 max_load_reduction=57182 max_load_reduction_pct=0.872528 moved_to_echo=33025 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[530, 500, 509, 499, 532, 522, 522, 532, 501, 553, 528, 533, 498, 508, 513, 496, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 533, 515, 522, 469, 536, 508, 503, 503, 524, 520, 480, 528, 512, 467, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 501, 501, 523, 496, 502, 521, 521, 526, 536, 523, 541, 519, 521, 503, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 532, 540, 492, 465, 485, 471, 525, 503, 497, 527, 461, 496, 503, 515, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 444, 491, 536, 516, 530, 496, 535, 547, 495, 510, 468, 532, 524, 523, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 533, 473, 493, 564, 517, 460, 519, 508, 531, 517, 498, 492, 506, 513, 509, 27, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 509, 494, 476, 489, 523, 504, 518, 536, 513, 505, 524, 524, 511, 487, 491, 33, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 521, 520, 521, 515, 483, 533, 551, 514, 505, 514, 491, 538, 539, 518, 497, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8321, 8111, 8287, 8011, 8146, 8167, 8158, 8335] avg=8192.0000 max=8335 max_rank=7 min=8011 min_rank=3 imbalance_ratio=1.017456 max_load_reduction=57201 max_load_reduction_pct=0.872818 moved_to_echo=32806 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[545, 515, 499, 509, 477, 533, 509, 513, 522, 520, 518, 449, 496, 501, 544, 536, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 532, 529, 541, 531, 551, 495, 480, 485, 464, 568, 524, 503, 532, 543, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 536, 509, 534, 536, 505, 504, 526, 529, 490, 506, 499, 489, 540, 485, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 508, 521, 527, 542, 493, 556, 510, 505, 544, 550, 506, 489, 529, 530, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 491, 504, 482, 504, 495, 494, 515, 500, 494, 505, 503, 482, 492, 513, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 495, 542, 514, 505, 518, 518, 512, 516, 529, 529, 486, 504, 506, 510, 491, 31, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 452, 535, 496, 507, 493, 505, 523, 501, 545, 513, 485, 524, 472, 510, 517, 32, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 507, 500, 499, 505, 529, 459, 510, 493, 508, 517, 495, 488, 572, 473, 488, 26, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 8272, 8226, 8340, 8027, 8220, 8137, 8084] avg=8192.0000 max=8340 max_rank=3 min=8027 min_rank=4 imbalance_ratio=1.018066 max_load_reduction=57196 max_load_reduction_pct=0.872742 moved_to_echo=32468 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[497, 499, 502, 478, 554, 505, 512, 547, 499, 538, 505, 515, 510, 508, 517, 530, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 508, 558, 499, 497, 527, 499, 540, 529, 473, 472, 543, 534, 471, 518, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 508, 517, 511, 510, 488, 514, 509, 548, 485, 524, 498, 517, 503, 449, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 521, 559, 527, 557, 504, 531, 551, 494, 506, 534, 474, 482, 464, 547, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 500, 506, 455, 526, 481, 530, 525, 518, 523, 555, 520, 512, 513, 507, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 495, 470, 484, 514, 512, 501, 516, 507, 525, 516, 529, 500, 534, 459, 523, 29, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 489, 471, 495, 527, 494, 514, 491, 525, 475, 496, 513, 487, 472, 496, 537, 29, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 536, 502, 517, 508, 495, 474, 495, 508, 544, 548, 508, 462, 541, 525, 523, 25, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8259, 8255, 8076, 8319, 8236, 8137, 8028, 8226] avg=8192.0000 max=8319 max_rank=3 min=8028 min_rank=6 imbalance_ratio=1.015503 max_load_reduction=57217 max_load_reduction_pct=0.873062 moved_to_echo=32627 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4224, 3968, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 3968, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 3968, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 3968, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 3840, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4352, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 3968, 4096, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 3968, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 3968, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 3840, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[485, 479, 494, 550, 552, 518, 535, 449, 487, 496, 485, 518, 491, 513, 537, 519, 489, 488, 532, 521, 486, 524, 505, 526, 549, 506, 521, 504, 544, 487, 518, 496, 514, 508, 493, 501, 526, 519, 538, 518, 565, 533, 493, 478, 499, 548, 506, 516, 533, 545, 513, 549, 493, 488, 528, 512, 488, 518, 527, 553, 498, 556, 507, 458, 493, 493, 484, 480, 478, 518, 538, 505, 516, 524, 529, 543, 477, 526, 530, 493, 508, 512, 561, 522, 499, 514, 539, 512, 542, 503, 521, 518, 514, 510, 515, 525, 515, 497, 504, 504, 479, 475, 522, 520, 548, 462, 517, 509, 518, 514, 495, 524, 477, 537, 515, 524, 501, 498, 532, 492, 535, 536, 483, 534, 503, 535, 480, 484] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[544, 527, 527, 488, 522, 536, 556, 504, 531, 489, 514, 507, 508, 527, 485, 528, 492, 480, 523, 528, 525, 532, 508, 527, 544, 505, 532, 549, 510, 526, 509, 530, 498, 522, 474, 507, 507, 537, 537, 550, 508, 511, 507, 504, 514, 497, 538, 461, 502, 524, 488, 478, 488, 523, 483, 517, 547, 545, 504, 503, 485, 503, 496, 511, 513, 516, 465, 488, 532, 497, 484, 454, 529, 512, 512, 498, 495, 525, 507, 489, 492, 523, 481, 519, 498, 509, 474, 516, 518, 513, 512, 530, 533, 508, 557, 456, 507, 509, 509, 513, 545, 530, 500, 516, 463, 516, 519, 544, 513, 544, 567, 520, 506, 523, 493, 520, 517, 542, 511, 504, 527, 487, 520, 493, 489, 497, 521, 534] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[528, 508, 506, 517, 484, 526, 492, 501, 490, 495, 502, 531, 515, 527, 530, 484, 518, 464, 474, 521, 498, 483, 487, 527, 521, 526, 500, 522, 503, 541, 514, 513, 488, 523, 487, 479, 521, 509, 504, 486, 559, 519, 555, 477, 498, 517, 546, 519, 533, 495, 560, 541, 481, 495, 520, 539, 504, 527, 516, 562, 505, 504, 490, 493, 503, 505, 515, 474, 524, 473, 541, 500, 540, 531, 523, 485, 516, 512, 531, 518, 512, 497, 560, 501, 530, 536, 521, 497, 511, 507, 537, 495, 527, 492, 531, 517, 492, 512, 536, 465, 498, 556, 543, 535, 520, 500, 466, 512, 514, 525, 544, 502, 515, 517, 499, 480, 486, 497, 534, 540, 509, 527, 509, 530, 526, 480, 541, 464] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[504, 528, 508, 549, 489, 472, 484, 484, 500, 530, 550, 485, 501, 506, 471, 501, 512, 523, 541, 537, 473, 505, 519, 494, 533, 508, 532, 521, 499, 507, 532, 502, 486, 558, 495, 501, 489, 507, 511, 514, 551, 521, 560, 536, 482, 505, 539, 518, 547, 546, 552, 543, 510, 500, 491, 523, 537, 506, 513, 512, 508, 533, 552, 522, 491, 527, 505, 517, 510, 517, 516, 476, 540, 516, 517, 493, 480, 545, 508, 501, 509, 480, 521, 522, 522, 511, 481, 505, 503, 512, 501, 492, 497, 526, 525, 474, 451, 511, 479, 536, 528, 549, 516, 503, 519, 503, 469, 513, 516, 491, 491, 494, 503, 483, 487, 522, 546, 514, 518, 505, 537, 531, 539, 493, 486, 543, 534, 518] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[537, 513, 510, 526, 508, 510, 528, 513, 514, 501, 493, 495, 513, 547, 544, 517, 503, 526, 540, 466, 526, 536, 511, 541, 517, 490, 557, 491, 496, 511, 470, 514, 487, 502, 527, 526, 547, 547, 496, 524, 490, 514, 526, 493, 538, 518, 499, 488, 549, 468, 509, 567, 509, 512, 492, 528, 509, 480, 489, 495, 532, 489, 504, 453, 501, 484, 481, 474, 529, 536, 496, 515, 496, 494, 516, 481, 512, 514, 512, 526, 533, 484, 517, 536, 551, 547, 513, 501, 485, 475, 529, 488, 522, 517, 533, 489, 469, 485, 478, 524, 489, 497, 515, 511, 509, 511, 543, 508, 530, 512, 544, 555, 527, 499, 596, 461, 512, 509, 502, 526, 548, 519, 490, 524, 506, 531, 532, 516] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[528, 508, 506, 517, 484, 526, 492, 501, 490, 495, 502, 531, 515, 527, 530, 484, 518, 464, 474, 521, 498, 483, 487, 527, 521, 526, 500, 522, 503, 541, 514, 513, 488, 523, 487, 479, 521, 509, 504, 486, 559, 519, 555, 477, 498, 517, 546, 519, 533, 495, 560, 541, 481, 495, 520, 539, 504, 527, 516, 562, 505, 504, 490, 493, 503, 505, 515, 474, 524, 473, 541, 500, 540, 531, 523, 485, 516, 512, 531, 518, 512, 497, 560, 501, 530, 536, 521, 497, 511, 507, 537, 495, 527, 492, 531, 517, 492, 512, 536, 465, 498, 556, 543, 535, 520, 500, 466, 512, 514, 525, 544, 502, 515, 517, 499, 480, 486, 497, 534, 540, 509, 527, 509, 530, 526, 480, 541, 464] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[522, 501, 549, 520, 531, 510, 538, 550, 549, 510, 511, 483, 488, 495, 529, 510, 496, 471, 532, 528, 553, 517, 498, 493, 528, 489, 514, 483, 484, 497, 502, 490, 475, 509, 481, 513, 501, 533, 570, 501, 484, 490, 524, 474, 498, 490, 512, 550, 487, 495, 527, 456, 498, 478, 525, 514, 518, 537, 497, 525, 510, 526, 509, 573, 493, 505, 529, 499, 508, 534, 508, 495, 483, 512, 529, 519, 528, 504, 512, 542, 482, 549, 490, 504, 508, 537, 490, 518, 519, 515, 544, 528, 518, 524, 506, 539, 527, 498, 526, 500, 554, 546, 545, 522, 509, 517, 494, 539, 522, 497, 508, 515, 483, 485, 480, 504, 522, 508, 535, 510, 527, 499, 516, 491, 512, 529, 483, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[525, 509, 526, 475, 491, 546, 520, 525, 506, 473, 511, 515, 523, 518, 495, 540, 521, 522, 492, 521, 522, 491, 510, 531, 498, 557, 517, 512, 507, 535, 535, 537, 504, 516, 536, 546, 516, 490, 535, 517, 486, 514, 556, 530, 508, 515, 495, 536, 485, 492, 500, 534, 518, 487, 509, 503, 527, 506, 492, 524, 536, 511, 568, 517, 508, 517, 533, 489, 534, 498, 522, 494, 475, 524, 531, 502, 505, 523, 488, 501, 497, 467, 504, 517, 481, 523, 473, 489, 506, 541, 541, 509, 511, 515, 459, 506, 519, 514, 496, 461, 505, 507, 507, 506, 504, 514, 529, 507, 521, 549, 514, 511, 502, 540, 516, 473, 516, 516, 515, 497, 508, 484, 529, 525, 537, 508, 495, 513] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[518, 547, 509, 560, 492, 534, 511, 514, 479, 522, 473, 497, 528, 498, 542, 523, 550, 502, 523, 487, 516, 510, 467, 553, 479, 471, 528, 486, 534, 488, 544, 467, 501, 498, 530, 516, 509, 482, 520, 532, 492, 519, 508, 503, 527, 521, 545, 506, 549, 523, 483, 529, 505, 534, 514, 507, 513, 521, 547, 494, 537, 516, 520, 502, 504, 482, 543, 470, 510, 499, 553, 480, 544, 538, 514, 514, 514, 476, 518, 497, 517, 495, 499, 504, 520, 485, 507, 521, 532, 507, 527, 533, 523, 525, 497, 503, 511, 504, 526, 512, 492, 528, 507, 531, 503, 475, 535, 472, 522, 515, 499, 541, 493, 535, 514, 497, 501, 497, 540, 524, 517, 512, 491, 499, 498, 516, 520, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[485, 479, 494, 550, 552, 518, 535, 449, 487, 496, 485, 518, 491, 513, 537, 519, 489, 488, 532, 521, 486, 524, 505, 526, 549, 506, 521, 504, 544, 487, 518, 496, 514, 508, 493, 501, 526, 519, 538, 518, 565, 533, 493, 478, 499, 548, 506, 516, 533, 545, 513, 549, 493, 488, 528, 512, 488, 518, 527, 553, 498, 556, 507, 458, 493, 493, 484, 480, 478, 518, 538, 505, 516, 524, 529, 543, 477, 526, 530, 493, 508, 512, 561, 522, 499, 514, 539, 512, 542, 503, 521, 518, 514, 510, 515, 525, 515, 497, 504, 504, 479, 475, 522, 520, 548, 462, 517, 509, 518, 514, 495, 524, 477, 537, 515, 524, 501, 498, 532, 492, 535, 536, 483, 534, 503, 535, 480, 484] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[504, 528, 508, 549, 489, 472, 484, 484, 500, 530, 550, 485, 501, 506, 471, 501, 512, 523, 541, 537, 473, 505, 519, 494, 533, 508, 532, 521, 499, 507, 532, 502, 486, 558, 495, 501, 489, 507, 511, 514, 551, 521, 560, 536, 482, 505, 539, 518, 547, 546, 552, 543, 510, 500, 491, 523, 537, 506, 513, 512, 508, 533, 552, 522, 491, 527, 505, 517, 510, 517, 516, 476, 540, 516, 517, 493, 480, 545, 508, 501, 509, 480, 521, 522, 522, 511, 481, 505, 503, 512, 501, 492, 497, 526, 525, 474, 451, 511, 479, 536, 528, 549, 516, 503, 519, 503, 469, 513, 516, 491, 491, 494, 503, 483, 487, 522, 546, 514, 518, 505, 537, 531, 539, 493, 486, 543, 534, 518] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[522, 501, 549, 520, 531, 510, 538, 550, 549, 510, 511, 483, 488, 495, 529, 510, 496, 471, 532, 528, 553, 517, 498, 493, 528, 489, 514, 483, 484, 497, 502, 490, 475, 509, 481, 513, 501, 533, 570, 501, 484, 490, 524, 474, 498, 490, 512, 550, 487, 495, 527, 456, 498, 478, 525, 514, 518, 537, 497, 525, 510, 526, 509, 573, 493, 505, 529, 499, 508, 534, 508, 495, 483, 512, 529, 519, 528, 504, 512, 542, 482, 549, 490, 504, 508, 537, 490, 518, 519, 515, 544, 528, 518, 524, 506, 539, 527, 498, 526, 500, 554, 546, 545, 522, 509, 517, 494, 539, 522, 497, 508, 515, 483, 485, 480, 504, 522, 508, 535, 510, 527, 499, 516, 491, 512, 529, 483, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[544, 527, 527, 481, 522, 536, 556, 504, 531, 489, 514, 507, 508, 527, 485, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 480, 523, 528, 525, 532, 508, 527, 544, 505, 531, 549, 510, 526, 509, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 522, 474, 507, 507, 537, 537, 550, 508, 511, 485, 504, 514, 497, 538, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 524, 488, 450, 488, 523, 483, 517, 547, 545, 504, 503, 485, 503, 496, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 516, 465, 488, 532, 497, 484, 454, 529, 512, 512, 498, 495, 525, 507, 489, 28, 22, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 523, 481, 519, 498, 509, 474, 516, 518, 513, 512, 530, 533, 508, 557, 456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 509, 509, 513, 545, 530, 500, 516, 463, 516, 519, 544, 513, 544, 567, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 523, 493, 520, 517, 542, 511, 504, 527, 487, 520, 493, 489, 497, 521, 534, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8286, 8319, 8150, 8069, 8070, 8139, 8315, 8188] avg=8192.0000 max=8319 max_rank=1 min=8069 min_rank=3 imbalance_ratio=1.015503 max_load_reduction=57217 max_load_reduction_pct=0.873062 moved_to_echo=32712 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[518, 547, 509, 551, 492, 534, 511, 514, 479, 522, 473, 497, 528, 498, 542, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 502, 523, 487, 516, 510, 467, 553, 479, 471, 527, 486, 534, 488, 544, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 498, 530, 516, 509, 482, 520, 532, 492, 519, 486, 503, 527, 521, 545, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 523, 483, 498, 505, 534, 514, 507, 513, 521, 547, 494, 537, 516, 520, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 482, 543, 470, 510, 499, 553, 480, 544, 538, 514, 514, 514, 476, 518, 497, 31, 22, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 495, 499, 504, 520, 485, 507, 521, 532, 507, 527, 533, 523, 525, 497, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 504, 526, 512, 492, 528, 507, 531, 503, 475, 535, 472, 522, 515, 499, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 535, 514, 497, 501, 497, 540, 524, 517, 512, 491, 499, 498, 516, 520, 503, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8104, 8187, 8263, 8214, 8195, 8173, 8162] avg=8192.0000 max=8263 max_rank=3 min=8104 min_rank=1 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32744 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[485, 479, 494, 541, 552, 518, 535, 449, 487, 496, 485, 518, 491, 513, 537, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 488, 532, 521, 486, 524, 505, 526, 549, 506, 520, 504, 544, 487, 518, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 508, 493, 501, 526, 519, 538, 518, 565, 533, 472, 478, 499, 548, 506, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 545, 513, 517, 493, 488, 528, 512, 488, 518, 527, 553, 498, 556, 507, 458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 493, 484, 480, 478, 518, 538, 505, 516, 524, 529, 543, 477, 526, 530, 493, 32, 21, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 512, 561, 522, 499, 514, 539, 512, 542, 503, 521, 518, 514, 510, 515, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 497, 504, 504, 479, 475, 522, 520, 548, 462, 517, 509, 518, 514, 495, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 537, 515, 524, 501, 498, 532, 492, 535, 536, 483, 534, 503, 535, 480, 484, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8099, 8195, 8234, 8234, 8185, 8315, 8103, 8171] avg=8192.0000 max=8315 max_rank=5 min=8099 min_rank=0 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32774 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[504, 528, 508, 540, 489, 472, 484, 484, 500, 530, 550, 485, 501, 506, 471, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 523, 541, 537, 473, 505, 519, 494, 533, 508, 531, 521, 499, 507, 532, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 558, 495, 501, 489, 507, 511, 514, 551, 521, 536, 536, 482, 505, 539, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 546, 552, 511, 510, 500, 491, 523, 537, 506, 513, 512, 508, 533, 552, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 527, 505, 517, 510, 517, 516, 476, 540, 516, 517, 493, 480, 545, 508, 501, 32, 24, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 480, 521, 522, 522, 511, 481, 505, 503, 512, 501, 492, 497, 526, 525, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 451, 511, 479, 536, 528, 549, 516, 503, 519, 503, 469, 513, 516, 491, 491, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 483, 487, 522, 546, 514, 518, 505, 537, 531, 539, 493, 486, 543, 534, 518, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8053, 8237, 8249, 8363, 8220, 8081, 8069, 8264] avg=8192.0000 max=8363 max_rank=3 min=8053 min_rank=0 imbalance_ratio=1.020874 max_load_reduction=57173 max_load_reduction_pct=0.872391 moved_to_echo=32634 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[528, 508, 506, 501, 484, 526, 492, 501, 490, 495, 502, 531, 515, 527, 530, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 464, 474, 521, 498, 483, 487, 527, 521, 526, 494, 522, 503, 541, 514, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 523, 487, 479, 521, 509, 504, 486, 559, 519, 527, 477, 498, 517, 546, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 495, 560, 505, 481, 495, 520, 539, 504, 527, 516, 562, 505, 504, 490, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 505, 515, 474, 524, 473, 541, 500, 540, 531, 523, 485, 516, 512, 531, 518, 36, 28, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 497, 560, 501, 530, 536, 521, 497, 511, 507, 537, 495, 527, 492, 531, 517, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 512, 536, 465, 498, 549, 543, 535, 520, 500, 466, 512, 514, 525, 544, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 517, 499, 480, 486, 497, 534, 540, 509, 527, 509, 530, 526, 480, 541, 464, 8, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8120, 8106, 8159, 8229, 8263, 8276, 8213, 8170] avg=8192.0000 max=8276 max_rank=5 min=8106 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32922 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[544, 527, 527, 481, 522, 536, 556, 504, 531, 489, 514, 507, 508, 527, 485, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 480, 523, 528, 525, 532, 508, 527, 544, 505, 531, 549, 510, 526, 509, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 522, 474, 507, 507, 537, 537, 550, 508, 511, 485, 504, 514, 497, 538, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 524, 488, 450, 488, 523, 483, 517, 547, 545, 504, 503, 485, 503, 496, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 516, 465, 488, 532, 497, 484, 454, 529, 512, 512, 498, 495, 525, 507, 489, 28, 22, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 523, 481, 519, 498, 509, 474, 516, 518, 513, 512, 530, 533, 508, 557, 456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 509, 509, 513, 545, 530, 500, 516, 463, 516, 519, 544, 513, 544, 567, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 523, 493, 520, 517, 542, 511, 504, 527, 487, 520, 493, 489, 497, 521, 534, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8286, 8319, 8150, 8069, 8070, 8139, 8315, 8188] avg=8192.0000 max=8319 max_rank=1 min=8069 min_rank=3 imbalance_ratio=1.015503 max_load_reduction=57217 max_load_reduction_pct=0.873062 moved_to_echo=32712 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[537, 513, 510, 518, 508, 510, 528, 513, 514, 501, 493, 495, 513, 547, 544, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 526, 540, 466, 526, 536, 511, 541, 517, 490, 556, 491, 496, 511, 470, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 502, 527, 526, 547, 547, 496, 524, 490, 514, 503, 493, 538, 518, 499, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 468, 509, 534, 509, 512, 492, 528, 509, 480, 489, 495, 532, 489, 504, 453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 484, 481, 474, 529, 536, 496, 515, 496, 494, 516, 481, 512, 514, 512, 526, 33, 23, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 484, 517, 536, 551, 547, 513, 501, 485, 475, 529, 488, 522, 517, 533, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 485, 478, 524, 489, 497, 515, 511, 509, 511, 543, 508, 530, 512, 544, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 499, 596, 461, 512, 509, 502, 526, 548, 519, 490, 524, 506, 531, 532, 516, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8261, 8194, 8199, 8052, 8127, 8220, 8180, 8303] avg=8192.0000 max=8303 max_rank=7 min=8052 min_rank=3 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32830 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[485, 479, 494, 541, 552, 518, 535, 449, 487, 496, 485, 518, 491, 513, 537, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 488, 532, 521, 486, 524, 505, 526, 549, 506, 520, 504, 544, 487, 518, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 508, 493, 501, 526, 519, 538, 518, 565, 533, 472, 478, 499, 548, 506, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 545, 513, 517, 493, 488, 528, 512, 488, 518, 527, 553, 498, 556, 507, 458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 493, 484, 480, 478, 518, 538, 505, 516, 524, 529, 543, 477, 526, 530, 493, 32, 21, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 512, 561, 522, 499, 514, 539, 512, 542, 503, 521, 518, 514, 510, 515, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 497, 504, 504, 479, 475, 522, 520, 548, 462, 517, 509, 518, 514, 495, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 537, 515, 524, 501, 498, 532, 492, 535, 536, 483, 534, 503, 535, 480, 484, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8099, 8195, 8234, 8234, 8185, 8315, 8103, 8171] avg=8192.0000 max=8315 max_rank=5 min=8099 min_rank=0 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32774 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[525, 509, 526, 468, 491, 546, 520, 525, 506, 473, 511, 515, 523, 518, 495, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 522, 492, 521, 522, 491, 510, 531, 498, 557, 516, 512, 507, 535, 535, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 516, 536, 546, 516, 490, 535, 517, 486, 514, 532, 530, 508, 515, 495, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 492, 500, 503, 518, 487, 509, 503, 527, 506, 492, 524, 536, 511, 568, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 517, 533, 489, 534, 498, 522, 494, 475, 524, 531, 502, 505, 523, 488, 501, 31, 24, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 467, 504, 517, 481, 523, 473, 489, 506, 541, 541, 509, 511, 515, 459, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 514, 496, 461, 505, 507, 507, 506, 504, 514, 529, 507, 521, 549, 514, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 540, 516, 473, 516, 516, 515, 497, 508, 484, 529, 525, 537, 508, 495, 513, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8307, 8276, 8178, 8203, 8039, 8164, 8178] avg=8192.0000 max=8307 max_rank=1 min=8039 min_rank=5 imbalance_ratio=1.014038 max_load_reduction=57229 max_load_reduction_pct=0.873245 moved_to_echo=32584 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[504, 528, 508, 540, 489, 472, 484, 484, 500, 530, 550, 485, 501, 506, 471, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 523, 541, 537, 473, 505, 519, 494, 533, 508, 531, 521, 499, 507, 532, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 558, 495, 501, 489, 507, 511, 514, 551, 521, 536, 536, 482, 505, 539, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 546, 552, 511, 510, 500, 491, 523, 537, 506, 513, 512, 508, 533, 552, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 527, 505, 517, 510, 517, 516, 476, 540, 516, 517, 493, 480, 545, 508, 501, 32, 24, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 480, 521, 522, 522, 511, 481, 505, 503, 512, 501, 492, 497, 526, 525, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 451, 511, 479, 536, 528, 549, 516, 503, 519, 503, 469, 513, 516, 491, 491, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 483, 487, 522, 546, 514, 518, 505, 537, 531, 539, 493, 486, 543, 534, 518, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8053, 8237, 8249, 8363, 8220, 8081, 8069, 8264] avg=8192.0000 max=8363 max_rank=3 min=8053 min_rank=0 imbalance_ratio=1.020874 max_load_reduction=57173 max_load_reduction_pct=0.872391 moved_to_echo=32634 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[522, 501, 549, 512, 531, 510, 538, 550, 549, 510, 511, 483, 488, 495, 529, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 471, 532, 528, 553, 517, 498, 493, 528, 489, 513, 483, 484, 497, 502, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 509, 481, 513, 501, 533, 570, 501, 484, 490, 501, 474, 498, 490, 512, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 495, 527, 429, 498, 478, 525, 514, 518, 537, 497, 525, 510, 526, 509, 573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 505, 529, 499, 508, 534, 508, 495, 483, 512, 529, 519, 528, 504, 512, 542, 27, 23, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 549, 490, 504, 508, 537, 490, 518, 519, 515, 544, 528, 518, 524, 506, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 498, 526, 500, 554, 546, 545, 522, 509, 517, 494, 539, 522, 497, 508, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 485, 480, 504, 522, 508, 535, 510, 527, 499, 516, 491, 512, 529, 483, 511, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8288, 8074, 8082, 8148, 8254, 8271, 8319, 8100] avg=8192.0000 max=8319 max_rank=6 min=8074 min_rank=1 imbalance_ratio=1.015503 max_load_reduction=57217 max_load_reduction_pct=0.873062 moved_to_echo=32944 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[518, 547, 509, 551, 492, 534, 511, 514, 479, 522, 473, 497, 528, 498, 542, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 502, 523, 487, 516, 510, 467, 553, 479, 471, 527, 486, 534, 488, 544, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 498, 530, 516, 509, 482, 520, 532, 492, 519, 486, 503, 527, 521, 545, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 523, 483, 498, 505, 534, 514, 507, 513, 521, 547, 494, 537, 516, 520, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 482, 543, 470, 510, 499, 553, 480, 544, 538, 514, 514, 514, 476, 518, 497, 31, 22, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 495, 499, 504, 520, 485, 507, 521, 532, 507, 527, 533, 523, 525, 497, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 504, 526, 512, 492, 528, 507, 531, 503, 475, 535, 472, 522, 515, 499, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 535, 514, 497, 501, 497, 540, 524, 517, 512, 491, 499, 498, 516, 520, 503, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8104, 8187, 8263, 8214, 8195, 8173, 8162] avg=8192.0000 max=8263 max_rank=3 min=8104 min_rank=1 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32744 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 3968, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[3968, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 3968, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 3968, 4224, 4096, 4224, 3968, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 256, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[521, 545, 477, 506, 523, 503, 499, 501, 523, 554, 491, 512, 528, 552, 553, 505, 537, 544, 534, 498, 497, 492, 472, 509, 486, 480, 512, 550, 525, 455, 526, 510, 530, 506, 512, 499, 497, 483, 505, 512, 490, 493, 514, 514, 488, 540, 537, 516, 488, 482, 500, 547, 508, 518, 521, 507, 502, 455, 531, 486, 484, 461, 533, 574, 560, 487, 511, 508, 536, 515, 490, 523, 538, 542, 480, 511, 473, 488, 486, 531, 543, 555, 507, 478, 544, 516, 496, 487, 510, 528, 480, 517, 501, 522, 474, 514, 490, 488, 585, 520, 521, 508, 487, 510, 516, 560, 521, 496, 535, 496, 505, 545, 502, 525, 548, 525, 533, 507, 500, 500, 524, 509, 495, 544, 489, 507, 505, 536] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[500, 534, 513, 533, 557, 526, 506, 517, 516, 525, 540, 485, 532, 494, 542, 499, 506, 529, 494, 517, 477, 530, 531, 532, 489, 495, 524, 513, 497, 505, 550, 488, 502, 512, 478, 489, 542, 477, 524, 508, 504, 512, 516, 554, 526, 501, 492, 484, 529, 513, 486, 539, 534, 526, 526, 488, 522, 480, 506, 500, 508, 510, 531, 521, 523, 530, 525, 505, 496, 522, 543, 511, 506, 504, 466, 509, 482, 516, 503, 526, 536, 499, 495, 512, 537, 503, 522, 467, 475, 507, 517, 538, 502, 509, 506, 506, 511, 524, 530, 550, 511, 510, 526, 480, 544, 528, 501, 519, 511, 519, 484, 490, 483, 513, 496, 486, 541, 516, 497, 509, 570, 476, 529, 512, 505, 509, 507, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[522, 472, 513, 503, 495, 478, 491, 510, 526, 490, 543, 556, 508, 517, 540, 532, 501, 501, 495, 522, 526, 518, 498, 498, 502, 528, 513, 503, 514, 521, 536, 481, 514, 505, 527, 504, 515, 498, 501, 562, 531, 508, 575, 493, 539, 543, 534, 529, 542, 510, 491, 552, 522, 519, 481, 516, 560, 500, 538, 514, 554, 454, 522, 465, 506, 514, 514, 502, 547, 524, 518, 527, 489, 503, 526, 504, 492, 503, 506, 480, 505, 529, 477, 513, 490, 503, 499, 501, 521, 492, 519, 507, 525, 499, 503, 502, 530, 494, 496, 511, 536, 530, 516, 535, 506, 532, 485, 541, 530, 495, 511, 518, 481, 511, 514, 512, 492, 515, 521, 518, 452, 548, 515, 486, 480, 470, 533, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[522, 511, 538, 493, 481, 484, 532, 524, 525, 507, 489, 534, 492, 532, 529, 512, 526, 515, 528, 497, 508, 550, 469, 526, 492, 472, 546, 541, 503, 508, 505, 522, 512, 489, 509, 491, 534, 526, 491, 478, 509, 509, 482, 549, 512, 494, 487, 521, 458, 501, 483, 513, 536, 520, 505, 540, 516, 529, 510, 479, 502, 520, 542, 513, 505, 481, 500, 516, 511, 534, 468, 532, 546, 520, 550, 550, 505, 516, 492, 466, 540, 523, 519, 508, 504, 489, 502, 482, 490, 525, 505, 505, 512, 508, 510, 511, 546, 533, 486, 541, 505, 535, 518, 484, 533, 525, 521, 589, 524, 495, 532, 488, 497, 512, 530, 485, 501, 540, 509, 505, 502, 498, 515, 517, 534, 496, 526, 516] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[481, 506, 497, 494, 524, 491, 484, 506, 544, 499, 559, 513, 487, 533, 534, 497, 521, 490, 481, 529, 499, 501, 521, 491, 505, 480, 515, 488, 492, 498, 508, 513, 518, 511, 470, 523, 545, 510, 507, 496, 559, 512, 498, 532, 489, 512, 533, 477, 488, 472, 521, 505, 490, 486, 531, 586, 525, 515, 478, 513, 525, 510, 502, 521, 503, 505, 502, 553, 478, 474, 526, 551, 555, 493, 524, 506, 509, 543, 515, 516, 539, 532, 523, 510, 509, 527, 520, 509, 534, 514, 495, 515, 540, 537, 531, 543, 516, 541, 535, 524, 481, 519, 523, 468, 480, 506, 492, 480, 520, 485, 538, 516, 498, 515, 492, 524, 500, 498, 500, 517, 504, 551, 534, 538, 501, 479, 524, 565] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[518, 509, 546, 486, 519, 526, 482, 463, 510, 513, 488, 502, 497, 527, 514, 487, 539, 491, 487, 514, 503, 496, 493, 494, 524, 496, 539, 527, 551, 482, 532, 499, 532, 538, 512, 521, 506, 532, 528, 506, 506, 505, 498, 508, 496, 549, 540, 508, 513, 525, 523, 489, 527, 534, 541, 536, 506, 512, 514, 529, 490, 479, 513, 520, 517, 474, 496, 509, 522, 530, 473, 577, 545, 478, 494, 534, 558, 490, 478, 516, 536, 509, 503, 511, 513, 567, 535, 521, 522, 453, 486, 495, 509, 528, 481, 483, 492, 547, 494, 511, 562, 494, 489, 500, 502, 494, 528, 544, 520, 546, 529, 524, 509, 491, 510, 532, 523, 485, 530, 472, 464, 508, 506, 521, 518, 511, 519, 528] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[486, 539, 528, 509, 530, 547, 499, 510, 513, 504, 527, 469, 525, 503, 511, 552, 512, 567, 498, 501, 463, 512, 551, 500, 497, 507, 532, 491, 493, 535, 494, 496, 512, 503, 506, 497, 520, 525, 466, 500, 517, 520, 520, 538, 504, 493, 484, 488, 533, 518, 516, 495, 528, 557, 526, 498, 501, 526, 475, 500, 518, 530, 511, 537, 508, 485, 500, 514, 484, 500, 472, 510, 520, 487, 529, 523, 539, 490, 521, 516, 533, 527, 542, 524, 473, 513, 516, 512, 502, 522, 509, 498, 527, 524, 523, 514, 520, 515, 515, 510, 497, 511, 521, 504, 489, 503, 524, 533, 498, 549, 508, 552, 490, 480, 527, 534, 510, 525, 486, 499, 513, 519, 539, 492, 520, 499, 514, 520] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[481, 506, 497, 494, 524, 491, 484, 506, 544, 499, 559, 513, 487, 533, 534, 497, 521, 490, 481, 529, 499, 501, 521, 491, 505, 480, 515, 488, 492, 498, 508, 513, 518, 511, 470, 523, 545, 510, 507, 496, 559, 512, 498, 532, 489, 512, 533, 477, 488, 472, 521, 505, 490, 486, 531, 586, 525, 515, 478, 513, 525, 510, 502, 521, 503, 505, 502, 553, 478, 474, 526, 551, 555, 493, 524, 506, 509, 543, 515, 516, 539, 532, 523, 510, 509, 527, 520, 509, 534, 514, 495, 515, 540, 537, 531, 543, 516, 541, 535, 524, 481, 519, 523, 468, 480, 506, 492, 480, 520, 485, 538, 516, 498, 515, 492, 524, 500, 498, 500, 517, 504, 551, 534, 538, 501, 479, 524, 565] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[522, 472, 513, 503, 495, 478, 491, 510, 526, 490, 543, 556, 508, 517, 540, 532, 501, 501, 495, 522, 526, 518, 498, 498, 502, 528, 513, 503, 514, 521, 536, 481, 514, 505, 527, 504, 515, 498, 501, 562, 531, 508, 575, 493, 539, 543, 534, 529, 542, 510, 491, 552, 522, 519, 481, 516, 560, 500, 538, 514, 554, 454, 522, 465, 506, 514, 514, 502, 547, 524, 518, 527, 489, 503, 526, 504, 492, 503, 506, 480, 505, 529, 477, 513, 490, 503, 499, 501, 521, 492, 519, 507, 525, 499, 503, 502, 530, 494, 496, 511, 536, 530, 516, 535, 506, 532, 485, 541, 530, 495, 511, 518, 481, 511, 514, 512, 492, 515, 521, 518, 452, 548, 515, 486, 480, 470, 533, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[499, 542, 519, 514, 517, 519, 459, 507, 515, 538, 506, 506, 549, 547, 543, 494, 481, 509, 517, 523, 557, 500, 509, 535, 497, 509, 504, 509, 529, 473, 506, 489, 470, 525, 490, 509, 546, 475, 490, 497, 512, 505, 528, 501, 514, 526, 503, 485, 495, 510, 546, 518, 509, 474, 519, 468, 528, 513, 518, 482, 557, 504, 494, 532, 541, 512, 451, 541, 494, 505, 533, 487, 505, 509, 495, 506, 512, 532, 541, 538, 527, 476, 522, 504, 540, 540, 449, 528, 533, 520, 524, 526, 517, 533, 504, 536, 497, 543, 503, 508, 507, 487, 506, 528, 520, 523, 525, 500, 497, 522, 547, 475, 533, 500, 558, 531, 493, 490, 519, 510, 512, 501, 500, 513, 530, 494, 489, 530] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[521, 545, 477, 506, 523, 503, 499, 501, 523, 554, 491, 512, 528, 552, 524, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 544, 534, 498, 497, 492, 472, 509, 486, 480, 512, 550, 525, 455, 526, 510, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 506, 512, 499, 497, 483, 505, 512, 490, 493, 514, 514, 488, 540, 537, 516, 11, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 482, 500, 547, 508, 518, 521, 507, 502, 455, 531, 486, 484, 461, 533, 569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 560, 487, 511, 508, 536, 515, 490, 523, 538, 542, 480, 511, 473, 488, 486, 531, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 555, 507, 478, 544, 516, 496, 487, 510, 528, 480, 517, 501, 522, 474, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 488, 585, 520, 521, 508, 487, 510, 516, 560, 521, 445, 535, 496, 505, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 525, 548, 525, 533, 507, 500, 500, 524, 509, 495, 544, 489, 507, 505, 536, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8264, 8170, 8155, 8092, 8190, 8166, 8232, 8267] avg=8192.0000 max=8267 max_rank=7 min=8092 min_rank=3 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32855 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[521, 545, 477, 506, 523, 503, 499, 501, 523, 554, 491, 512, 528, 552, 553, 505, 537, 544, 534, 498, 497, 492, 472, 509, 486, 480, 512, 550, 525, 455, 526, 510, 530, 506, 512, 499, 497, 483, 505, 512, 490, 493, 514, 514, 488, 540, 537, 516, 488, 482, 500, 547, 508, 518, 521, 507, 502, 455, 531, 486, 484, 461, 533, 574, 560, 487, 511, 508, 536, 515, 490, 523, 538, 542, 480, 511, 473, 488, 486, 531, 543, 555, 507, 478, 544, 516, 496, 487, 510, 528, 480, 517, 501, 522, 474, 514, 490, 488, 585, 520, 521, 508, 487, 510, 516, 560, 521, 496, 535, 496, 505, 545, 502, 525, 548, 525, 533, 507, 500, 500, 524, 509, 495, 544, 489, 507, 505, 536] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[500, 534, 513, 533, 557, 526, 506, 517, 516, 525, 540, 485, 532, 494, 513, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 529, 494, 517, 477, 530, 531, 532, 489, 495, 524, 513, 497, 505, 550, 488, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 512, 478, 489, 542, 477, 524, 508, 504, 512, 516, 554, 526, 501, 492, 484, 11, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 513, 486, 539, 534, 526, 526, 488, 522, 480, 506, 500, 508, 510, 531, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 530, 525, 505, 496, 522, 543, 511, 506, 504, 466, 509, 482, 516, 503, 526, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 499, 495, 512, 537, 503, 522, 467, 475, 507, 517, 538, 502, 509, 506, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 524, 530, 550, 511, 510, 526, 480, 544, 528, 501, 466, 511, 519, 484, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 513, 496, 486, 541, 516, 497, 509, 570, 476, 529, 512, 505, 509, 507, 515, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8290, 8222, 8140, 8214, 8178, 8125, 8185, 8182] avg=8192.0000 max=8290 max_rank=0 min=8125 min_rank=5 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32670 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[486, 539, 528, 509, 530, 547, 499, 510, 513, 504, 527, 469, 525, 503, 484, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 567, 498, 501, 463, 512, 551, 500, 497, 507, 532, 491, 493, 535, 494, 496, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 503, 506, 497, 520, 525, 466, 500, 517, 520, 520, 538, 504, 493, 484, 488, 10, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 518, 516, 495, 528, 557, 526, 498, 501, 526, 475, 500, 518, 530, 511, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 485, 500, 514, 484, 500, 472, 510, 520, 487, 529, 523, 539, 490, 521, 516, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 527, 542, 524, 473, 513, 516, 512, 502, 522, 509, 498, 527, 524, 523, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 515, 515, 510, 497, 511, 521, 504, 489, 503, 524, 477, 498, 549, 508, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 480, 527, 534, 510, 525, 486, 499, 513, 519, 539, 492, 520, 499, 514, 520, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 8196, 8112, 8264, 8109, 8253, 8193, 8184] avg=8192.0000 max=8264 max_rank=3 min=8109 min_rank=4 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32739 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[518, 509, 546, 486, 519, 526, 482, 463, 510, 513, 488, 502, 497, 527, 475, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 491, 487, 514, 503, 496, 493, 494, 524, 496, 539, 527, 551, 482, 532, 499, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 538, 512, 521, 506, 532, 528, 506, 506, 505, 498, 508, 496, 549, 540, 508, 15, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 525, 523, 489, 527, 534, 541, 536, 506, 512, 514, 529, 490, 479, 513, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 474, 496, 509, 522, 530, 473, 577, 545, 478, 494, 534, 558, 490, 478, 516, 11, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 509, 503, 511, 513, 567, 535, 521, 522, 453, 486, 495, 509, 528, 481, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 547, 494, 511, 562, 494, 489, 500, 502, 494, 528, 480, 520, 546, 529, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 491, 510, 532, 523, 485, 530, 472, 464, 508, 506, 521, 518, 511, 519, 528, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8048, 8219, 8312, 8243, 8214, 8141, 8212, 8147] avg=8192.0000 max=8312 max_rank=2 min=8048 min_rank=0 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32714 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[522, 472, 513, 503, 495, 478, 491, 510, 526, 490, 543, 556, 508, 517, 511, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 501, 495, 522, 526, 518, 498, 498, 502, 528, 513, 503, 514, 521, 536, 481, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 505, 527, 504, 515, 498, 501, 562, 531, 508, 575, 493, 539, 543, 534, 529, 11, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 510, 491, 552, 522, 519, 481, 516, 560, 500, 538, 514, 554, 454, 522, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 514, 514, 502, 547, 524, 518, 527, 489, 503, 526, 504, 492, 503, 506, 480, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 529, 477, 513, 490, 503, 499, 501, 521, 492, 519, 507, 525, 499, 503, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 494, 496, 511, 536, 530, 516, 535, 506, 532, 485, 485, 530, 495, 511, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 511, 514, 512, 492, 515, 521, 518, 452, 548, 515, 486, 480, 470, 533, 511, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8204, 8398, 8236, 8165, 8079, 8210, 8077] avg=8192.0000 max=8398 max_rank=2 min=8077 min_rank=7 imbalance_ratio=1.025146 max_load_reduction=57138 max_load_reduction_pct=0.871857 moved_to_echo=32531 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[522, 511, 538, 493, 481, 484, 532, 524, 525, 507, 489, 534, 492, 532, 501, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 515, 528, 497, 508, 550, 469, 526, 492, 472, 546, 541, 503, 508, 505, 522, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 489, 509, 491, 534, 526, 491, 478, 509, 509, 482, 549, 512, 494, 487, 521, 11, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 458, 501, 483, 513, 536, 520, 505, 540, 516, 529, 510, 479, 502, 520, 542, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 481, 500, 516, 511, 534, 468, 532, 546, 520, 550, 550, 505, 516, 492, 466, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 523, 519, 508, 504, 489, 502, 482, 490, 525, 505, 505, 512, 508, 510, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 533, 486, 541, 505, 535, 518, 484, 533, 525, 521, 528, 524, 495, 532, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 512, 530, 485, 501, 540, 509, 505, 502, 498, 515, 517, 534, 496, 526, 516, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8260, 8113, 8162, 8203, 8127, 8294, 8200] avg=8192.0000 max=8294 max_rank=6 min=8113 min_rank=2 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32824 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[481, 506, 497, 494, 524, 491, 484, 506, 544, 499, 559, 513, 487, 533, 505, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 490, 481, 529, 499, 501, 521, 491, 505, 480, 515, 488, 492, 498, 508, 513, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 511, 470, 523, 545, 510, 507, 496, 559, 512, 498, 532, 489, 512, 533, 477, 11, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 472, 521, 505, 490, 486, 531, 586, 525, 515, 478, 513, 525, 510, 502, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 505, 502, 553, 478, 474, 526, 551, 555, 493, 524, 506, 509, 543, 515, 516, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 532, 523, 510, 509, 527, 520, 509, 534, 514, 495, 515, 540, 537, 531, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 541, 535, 524, 481, 519, 523, 468, 480, 506, 492, 430, 520, 485, 538, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 515, 492, 524, 500, 498, 500, 517, 504, 551, 534, 538, 501, 479, 524, 565, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8120, 8074, 8211, 8163, 8264, 8372, 8074, 8258] avg=8192.0000 max=8372 max_rank=5 min=8074 min_rank=1 imbalance_ratio=1.021973 max_load_reduction=57164 max_load_reduction_pct=0.872253 moved_to_echo=32968 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[522, 511, 538, 493, 481, 484, 532, 524, 525, 507, 489, 534, 492, 532, 501, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 515, 528, 497, 508, 550, 469, 526, 492, 472, 546, 541, 503, 508, 505, 522, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 489, 509, 491, 534, 526, 491, 478, 509, 509, 482, 549, 512, 494, 487, 521, 11, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 458, 501, 483, 513, 536, 520, 505, 540, 516, 529, 510, 479, 502, 520, 542, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 481, 500, 516, 511, 534, 468, 532, 546, 520, 550, 550, 505, 516, 492, 466, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 523, 519, 508, 504, 489, 502, 482, 490, 525, 505, 505, 512, 508, 510, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 533, 486, 541, 505, 535, 518, 484, 533, 525, 521, 528, 524, 495, 532, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 512, 530, 485, 501, 540, 509, 505, 502, 498, 515, 517, 534, 496, 526, 516, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8260, 8113, 8162, 8203, 8127, 8294, 8200] avg=8192.0000 max=8294 max_rank=6 min=8113 min_rank=2 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32824 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[518, 509, 546, 486, 519, 526, 482, 463, 510, 513, 488, 502, 497, 527, 475, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 491, 487, 514, 503, 496, 493, 494, 524, 496, 539, 527, 551, 482, 532, 499, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 538, 512, 521, 506, 532, 528, 506, 506, 505, 498, 508, 496, 549, 540, 508, 15, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 525, 523, 489, 527, 534, 541, 536, 506, 512, 514, 529, 490, 479, 513, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 474, 496, 509, 522, 530, 473, 577, 545, 478, 494, 534, 558, 490, 478, 516, 11, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 509, 503, 511, 513, 567, 535, 521, 522, 453, 486, 495, 509, 528, 481, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 547, 494, 511, 562, 494, 489, 500, 502, 494, 528, 480, 520, 546, 529, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 491, 510, 532, 523, 485, 530, 472, 464, 508, 506, 521, 518, 511, 519, 528, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8048, 8219, 8312, 8243, 8214, 8141, 8212, 8147] avg=8192.0000 max=8312 max_rank=2 min=8048 min_rank=0 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32714 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[486, 539, 528, 509, 530, 547, 499, 510, 513, 504, 527, 469, 525, 503, 484, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 567, 498, 501, 463, 512, 551, 500, 497, 507, 532, 491, 493, 535, 494, 496, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 503, 506, 497, 520, 525, 466, 500, 517, 520, 520, 538, 504, 493, 484, 488, 10, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 518, 516, 495, 528, 557, 526, 498, 501, 526, 475, 500, 518, 530, 511, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 485, 500, 514, 484, 500, 472, 510, 520, 487, 529, 523, 539, 490, 521, 516, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 527, 542, 524, 473, 513, 516, 512, 502, 522, 509, 498, 527, 524, 523, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 515, 515, 510, 497, 511, 521, 504, 489, 503, 524, 477, 498, 549, 508, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 480, 527, 534, 510, 525, 486, 499, 513, 519, 539, 492, 520, 499, 514, 520, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 8196, 8112, 8264, 8109, 8253, 8193, 8184] avg=8192.0000 max=8264 max_rank=3 min=8109 min_rank=4 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32739 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[499, 542, 519, 514, 517, 519, 459, 507, 515, 538, 506, 506, 549, 547, 514, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 509, 517, 523, 557, 500, 509, 535, 497, 509, 504, 509, 529, 473, 506, 489, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 525, 490, 509, 546, 475, 490, 497, 512, 505, 528, 501, 514, 526, 503, 485, 11, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 510, 546, 518, 509, 474, 519, 468, 528, 513, 518, 482, 557, 504, 494, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 512, 451, 541, 494, 505, 533, 487, 505, 509, 495, 506, 512, 532, 541, 538, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 476, 522, 504, 540, 540, 449, 528, 533, 520, 524, 526, 517, 533, 504, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 543, 503, 508, 507, 487, 506, 528, 520, 523, 525, 448, 497, 522, 547, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 500, 558, 531, 493, 490, 519, 510, 512, 501, 500, 513, 530, 494, 489, 530, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 8191, 8095, 8162, 8213, 8273, 8136, 8221] avg=8192.0000 max=8273 max_rank=5 min=8095 min_rank=2 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32843 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[522, 472, 513, 503, 495, 478, 491, 510, 526, 490, 543, 556, 508, 517, 511, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 501, 495, 522, 526, 518, 498, 498, 502, 528, 513, 503, 514, 521, 536, 481, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 505, 527, 504, 515, 498, 501, 562, 531, 508, 575, 493, 539, 543, 534, 529, 11, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 510, 491, 552, 522, 519, 481, 516, 560, 500, 538, 514, 554, 454, 522, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 514, 514, 502, 547, 524, 518, 527, 489, 503, 526, 504, 492, 503, 506, 480, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 529, 477, 513, 490, 503, 499, 501, 521, 492, 519, 507, 525, 499, 503, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 494, 496, 511, 536, 530, 516, 535, 506, 532, 485, 485, 530, 495, 511, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 511, 514, 512, 492, 515, 521, 518, 452, 548, 515, 486, 480, 470, 533, 511, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8204, 8398, 8236, 8165, 8079, 8210, 8077] avg=8192.0000 max=8398 max_rank=2 min=8077 min_rank=7 imbalance_ratio=1.025146 max_load_reduction=57138 max_load_reduction_pct=0.871857 moved_to_echo=32531 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[521, 545, 477, 506, 523, 503, 499, 501, 523, 554, 491, 512, 528, 552, 524, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 544, 534, 498, 497, 492, 472, 509, 486, 480, 512, 550, 525, 455, 526, 510, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 506, 512, 499, 497, 483, 505, 512, 490, 493, 514, 514, 488, 540, 537, 516, 11, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 482, 500, 547, 508, 518, 521, 507, 502, 455, 531, 486, 484, 461, 533, 569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 560, 487, 511, 508, 536, 515, 490, 523, 538, 542, 480, 511, 473, 488, 486, 531, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 555, 507, 478, 544, 516, 496, 487, 510, 528, 480, 517, 501, 522, 474, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 488, 585, 520, 521, 508, 487, 510, 516, 560, 521, 445, 535, 496, 505, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 525, 548, 525, 533, 507, 500, 500, 524, 509, 495, 544, 489, 507, 505, 536, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8264, 8170, 8155, 8092, 8190, 8166, 8232, 8267] avg=8192.0000 max=8267 max_rank=7 min=8092 min_rank=3 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32855 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 3968, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 3968, 4224, 4224, 4224, 4096, 4224, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 3840, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 3968, 4224, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 3968, 4224, 4224, 4224, 4096, 4224, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 3968, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 3968, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[480, 539, 536, 533, 508, 531, 505, 526, 502, 558, 493, 496, 504, 511, 524, 511, 491, 506, 536, 533, 485, 528, 506, 509, 514, 504, 474, 479, 491, 482, 522, 530, 478, 532, 526, 515, 535, 518, 526, 508, 496, 543, 517, 510, 515, 495, 518, 508, 504, 515, 579, 529, 546, 530, 540, 514, 518, 531, 490, 502, 546, 529, 481, 523, 502, 510, 501, 518, 493, 522, 514, 515, 502, 547, 498, 458, 503, 473, 538, 529, 462, 487, 508, 516, 517, 522, 491, 518, 508, 491, 482, 535, 507, 526, 516, 506, 533, 499, 513, 509, 503, 491, 545, 520, 530, 530, 538, 518, 503, 512, 497, 508, 499, 509, 505, 518, 493, 507, 498, 503, 505, 533, 458, 539, 528, 504, 492, 517] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[481, 534, 522, 493, 479, 514, 539, 495, 521, 530, 524, 506, 508, 510, 530, 499, 496, 516, 516, 494, 552, 507, 515, 503, 498, 546, 515, 501, 503, 528, 504, 508, 514, 500, 553, 548, 493, 506, 473, 526, 533, 528, 503, 529, 474, 492, 516, 527, 514, 505, 504, 553, 445, 473, 509, 527, 493, 518, 489, 506, 557, 508, 517, 511, 497, 511, 482, 476, 518, 531, 541, 490, 508, 527, 519, 497, 525, 500, 517, 521, 512, 517, 527, 490, 525, 547, 514, 539, 553, 504, 496, 491, 455, 533, 512, 528, 526, 482, 500, 517, 548, 541, 514, 518, 487, 514, 530, 530, 542, 496, 511, 515, 492, 503, 517, 544, 487, 485, 523, 524, 526, 509, 489, 495, 514, 483, 504, 536] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[521, 493, 525, 520, 505, 517, 506, 495, 508, 508, 502, 524, 520, 494, 525, 486, 522, 505, 529, 529, 525, 503, 501, 493, 544, 526, 484, 482, 506, 518, 490, 547, 488, 509, 507, 514, 483, 504, 469, 493, 466, 505, 517, 520, 554, 521, 502, 543, 525, 480, 530, 517, 540, 556, 562, 466, 512, 513, 477, 518, 508, 521, 540, 544, 558, 507, 519, 530, 529, 484, 497, 515, 511, 491, 532, 542, 533, 507, 525, 519, 517, 544, 507, 510, 488, 506, 497, 461, 510, 501, 502, 522, 528, 486, 484, 490, 490, 541, 499, 557, 554, 511, 499, 515, 489, 516, 498, 537, 538, 545, 537, 505, 522, 480, 486, 499, 520, 506, 494, 572, 483, 518, 461, 498, 521, 519, 488, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[484, 529, 489, 504, 544, 488, 487, 501, 496, 532, 495, 501, 520, 484, 479, 512, 507, 463, 522, 496, 501, 537, 509, 482, 522, 482, 507, 512, 492, 535, 536, 518, 522, 493, 545, 526, 557, 487, 524, 488, 472, 542, 515, 523, 561, 495, 515, 501, 505, 516, 507, 464, 529, 500, 516, 499, 549, 518, 529, 519, 491, 502, 516, 525, 553, 525, 498, 519, 505, 521, 503, 525, 518, 550, 536, 508, 543, 505, 555, 446, 493, 478, 503, 508, 520, 549, 524, 503, 508, 537, 565, 529, 493, 456, 525, 536, 449, 524, 507, 520, 535, 526, 535, 460, 474, 528, 554, 515, 519, 485, 484, 496, 517, 560, 528, 489, 518, 518, 551, 514, 511, 488, 489, 574, 485, 520, 487, 522] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[542, 485, 520, 527, 525, 491, 517, 501, 552, 509, 511, 494, 537, 540, 503, 499, 482, 513, 507, 540, 488, 551, 521, 528, 573, 502, 515, 492, 499, 515, 462, 520, 517, 501, 488, 516, 518, 505, 501, 548, 481, 493, 519, 524, 529, 522, 523, 512, 516, 531, 502, 497, 525, 496, 500, 501, 515, 496, 502, 500, 510, 487, 535, 528, 474, 498, 505, 532, 537, 468, 503, 513, 506, 514, 502, 536, 504, 498, 485, 516, 531, 497, 498, 483, 520, 483, 501, 536, 508, 495, 528, 517, 502, 477, 511, 551, 501, 519, 487, 483, 464, 524, 547, 533, 519, 504, 524, 529, 512, 518, 509, 548, 552, 509, 508, 501, 504, 486, 552, 522, 519, 543, 499, 549, 527, 506, 509, 501] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[504, 514, 516, 485, 506, 506, 494, 465, 498, 523, 489, 483, 502, 514, 490, 491, 505, 521, 531, 505, 533, 496, 486, 470, 541, 484, 524, 495, 489, 503, 532, 531, 504, 566, 534, 508, 549, 529, 531, 485, 473, 518, 506, 522, 491, 543, 465, 480, 573, 529, 486, 505, 561, 515, 493, 542, 500, 506, 511, 520, 538, 526, 516, 529, 497, 482, 532, 481, 490, 499, 515, 537, 532, 489, 508, 494, 497, 482, 510, 515, 540, 493, 509, 535, 506, 505, 537, 525, 542, 529, 487, 503, 478, 497, 501, 518, 506, 511, 529, 530, 509, 499, 501, 540, 463, 542, 557, 517, 508, 505, 535, 491, 506, 513, 504, 526, 527, 524, 557, 527, 546, 530, 510, 503, 523, 512, 512, 528] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[480, 539, 536, 533, 508, 531, 505, 526, 502, 558, 493, 496, 504, 511, 524, 511, 491, 506, 536, 533, 485, 528, 506, 509, 514, 504, 474, 479, 491, 482, 522, 530, 478, 532, 526, 515, 535, 518, 526, 508, 496, 543, 517, 510, 515, 495, 518, 508, 504, 515, 579, 529, 546, 530, 540, 514, 518, 531, 490, 502, 546, 529, 481, 523, 502, 510, 501, 518, 493, 522, 514, 515, 502, 547, 498, 458, 503, 473, 538, 529, 462, 487, 508, 516, 517, 522, 491, 518, 508, 491, 482, 535, 507, 526, 516, 506, 533, 499, 513, 509, 503, 491, 545, 520, 530, 530, 538, 518, 503, 512, 497, 508, 499, 509, 505, 518, 493, 507, 498, 503, 505, 533, 458, 539, 528, 504, 492, 517] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[513, 546, 496, 513, 504, 486, 493, 490, 507, 506, 480, 510, 546, 493, 513, 508, 538, 493, 542, 493, 499, 531, 505, 517, 487, 500, 498, 475, 572, 524, 522, 528, 525, 537, 493, 535, 482, 509, 538, 496, 494, 520, 507, 503, 493, 538, 501, 511, 516, 528, 486, 517, 519, 477, 493, 516, 519, 540, 506, 513, 532, 502, 512, 506, 522, 489, 493, 548, 492, 512, 495, 530, 538, 515, 509, 480, 508, 503, 527, 473, 540, 510, 509, 550, 469, 495, 518, 547, 538, 463, 554, 554, 497, 512, 510, 530, 479, 468, 495, 547, 482, 494, 499, 495, 509, 530, 504, 518, 533, 502, 516, 502, 518, 497, 543, 546, 524, 532, 527, 545, 518, 499, 516, 489, 523, 530, 518, 516] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[484, 529, 489, 504, 544, 488, 487, 501, 496, 532, 495, 501, 520, 484, 479, 512, 507, 463, 522, 496, 501, 537, 509, 482, 522, 482, 507, 512, 492, 535, 536, 518, 522, 493, 545, 526, 557, 487, 524, 488, 472, 542, 515, 523, 561, 495, 515, 501, 505, 516, 507, 464, 529, 500, 516, 499, 549, 518, 529, 519, 491, 502, 516, 525, 553, 525, 498, 519, 505, 521, 503, 525, 518, 550, 536, 508, 543, 505, 555, 446, 493, 478, 503, 508, 520, 549, 524, 503, 508, 537, 565, 529, 493, 456, 525, 536, 449, 524, 507, 520, 535, 526, 535, 460, 474, 528, 554, 515, 519, 485, 484, 496, 517, 560, 528, 489, 518, 518, 551, 514, 511, 488, 489, 574, 485, 520, 487, 522] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[472, 526, 541, 525, 497, 503, 534, 483, 530, 532, 527, 524, 516, 511, 549, 498, 522, 488, 529, 528, 527, 513, 514, 507, 530, 515, 508, 465, 499, 520, 496, 513, 566, 521, 509, 519, 521, 510, 499, 525, 512, 526, 557, 500, 518, 474, 501, 538, 515, 489, 534, 540, 509, 525, 500, 504, 471, 485, 477, 522, 502, 483, 519, 528, 493, 510, 519, 502, 492, 532, 524, 500, 511, 493, 529, 474, 500, 488, 541, 485, 504, 538, 525, 520, 518, 515, 499, 478, 468, 528, 492, 466, 539, 503, 519, 518, 502, 500, 550, 555, 546, 508, 505, 515, 560, 536, 487, 483, 515, 477, 507, 505, 490, 528, 504, 516, 515, 541, 508, 550, 527, 490, 515, 491, 519, 522, 484, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[481, 534, 522, 493, 479, 514, 539, 495, 521, 530, 524, 506, 508, 510, 530, 499, 36, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 516, 516, 494, 552, 507, 515, 503, 498, 546, 515, 501, 503, 528, 504, 508, 22, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 500, 553, 517, 493, 506, 473, 526, 533, 528, 503, 529, 474, 492, 516, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 505, 504, 553, 445, 473, 509, 527, 493, 518, 489, 506, 557, 508, 517, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 511, 482, 476, 518, 531, 541, 490, 508, 527, 519, 497, 525, 500, 517, 521, 30, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 517, 527, 490, 525, 547, 514, 539, 553, 504, 496, 491, 455, 533, 512, 528, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 482, 500, 486, 548, 541, 514, 518, 487, 514, 530, 530, 542, 496, 511, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 503, 517, 544, 487, 485, 523, 488, 526, 509, 489, 495, 514, 483, 504, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8228, 8231, 8184, 8092, 8199, 8267, 8240, 8095] avg=8192.0000 max=8267 max_rank=5 min=8092 min_rank=3 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32801 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[484, 529, 489, 504, 544, 488, 487, 501, 496, 532, 495, 501, 520, 484, 479, 512, 32, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 463, 522, 496, 501, 537, 509, 482, 522, 482, 507, 512, 492, 535, 536, 518, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 493, 545, 507, 557, 487, 524, 488, 472, 542, 515, 523, 561, 495, 515, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 516, 507, 464, 529, 500, 516, 499, 549, 518, 529, 519, 491, 502, 516, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 525, 498, 519, 505, 521, 503, 525, 518, 550, 536, 508, 543, 505, 555, 446, 26, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 478, 503, 508, 520, 549, 524, 503, 508, 537, 565, 529, 493, 456, 525, 536, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 449, 524, 507, 495, 535, 526, 535, 460, 474, 528, 554, 515, 519, 485, 484, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 560, 528, 489, 518, 518, 551, 482, 511, 488, 489, 574, 485, 520, 487, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8080, 8140, 8247, 8156, 8342, 8246, 8086, 8239] avg=8192.0000 max=8342 max_rank=4 min=8080 min_rank=0 imbalance_ratio=1.018311 max_load_reduction=57194 max_load_reduction_pct=0.872711 moved_to_echo=32913 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[521, 493, 525, 520, 505, 517, 506, 495, 508, 508, 502, 524, 520, 494, 525, 486, 35, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 505, 529, 529, 525, 503, 501, 493, 544, 526, 484, 482, 506, 518, 490, 547, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 509, 507, 496, 483, 504, 469, 493, 466, 505, 517, 520, 554, 521, 502, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 480, 530, 517, 540, 556, 562, 466, 512, 513, 477, 518, 508, 521, 540, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 558, 507, 519, 530, 529, 484, 497, 515, 511, 491, 532, 542, 533, 507, 525, 519, 27, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 544, 507, 510, 488, 506, 497, 461, 510, 501, 502, 522, 528, 486, 484, 490, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 541, 499, 529, 554, 511, 499, 515, 489, 516, 498, 537, 538, 545, 537, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 480, 486, 499, 520, 506, 494, 537, 483, 518, 461, 498, 521, 519, 488, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8225, 8077, 8279, 8333, 8071, 8303, 8061] avg=8192.0000 max=8333 max_rank=4 min=8061 min_rank=7 imbalance_ratio=1.017212 max_load_reduction=57203 max_load_reduction_pct=0.872849 moved_to_echo=32768 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[513, 546, 496, 513, 504, 486, 493, 490, 507, 506, 480, 510, 546, 493, 513, 508, 538, 493, 542, 493, 499, 531, 505, 517, 487, 500, 498, 475, 572, 524, 522, 528, 525, 537, 493, 535, 482, 509, 538, 496, 494, 520, 507, 503, 493, 538, 501, 511, 516, 528, 486, 517, 519, 477, 493, 516, 519, 540, 506, 513, 532, 502, 512, 506, 522, 489, 493, 548, 492, 512, 495, 530, 538, 515, 509, 480, 508, 503, 527, 473, 540, 510, 509, 550, 469, 495, 518, 547, 538, 463, 554, 554, 497, 512, 510, 530, 479, 468, 495, 547, 482, 494, 499, 495, 509, 530, 504, 518, 533, 502, 516, 502, 518, 497, 543, 546, 524, 532, 527, 545, 518, 499, 516, 489, 523, 530, 518, 516] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[521, 493, 525, 520, 505, 517, 506, 495, 508, 508, 502, 524, 520, 494, 525, 486, 522, 505, 529, 529, 525, 503, 501, 493, 544, 526, 484, 482, 506, 518, 490, 547, 488, 509, 507, 514, 483, 504, 469, 493, 466, 505, 517, 520, 554, 521, 502, 543, 525, 480, 530, 517, 540, 556, 562, 466, 512, 513, 477, 518, 508, 521, 540, 544, 558, 507, 519, 530, 529, 484, 497, 515, 511, 491, 532, 542, 533, 507, 525, 519, 517, 544, 507, 510, 488, 506, 497, 461, 510, 501, 502, 522, 528, 486, 484, 490, 490, 541, 499, 557, 554, 511, 499, 515, 489, 516, 498, 537, 538, 545, 537, 505, 522, 480, 486, 499, 520, 506, 494, 572, 483, 518, 461, 498, 521, 519, 488, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[480, 539, 536, 533, 508, 531, 505, 526, 502, 558, 493, 496, 504, 511, 524, 511, 31, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 506, 536, 533, 485, 528, 506, 509, 514, 504, 474, 479, 491, 482, 522, 530, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 532, 526, 497, 535, 518, 526, 508, 496, 543, 517, 510, 515, 495, 518, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 515, 579, 529, 546, 530, 540, 514, 518, 531, 490, 502, 546, 529, 481, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 510, 501, 518, 493, 522, 514, 515, 502, 547, 498, 458, 503, 473, 538, 529, 26, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 487, 508, 516, 517, 522, 491, 518, 508, 491, 482, 535, 507, 526, 516, 506, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 499, 513, 484, 503, 491, 545, 520, 530, 530, 538, 518, 503, 512, 497, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 509, 505, 518, 493, 507, 498, 472, 505, 533, 458, 539, 528, 504, 492, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8291, 8109, 8222, 8348, 8155, 8110, 8224, 8077] avg=8192.0000 max=8348 max_rank=3 min=8077 min_rank=7 imbalance_ratio=1.019043 max_load_reduction=57188 max_load_reduction_pct=0.872620 moved_to_echo=32566 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[481, 534, 522, 493, 479, 514, 539, 495, 521, 530, 524, 506, 508, 510, 530, 499, 36, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 516, 516, 494, 552, 507, 515, 503, 498, 546, 515, 501, 503, 528, 504, 508, 22, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 500, 553, 517, 493, 506, 473, 526, 533, 528, 503, 529, 474, 492, 516, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 505, 504, 553, 445, 473, 509, 527, 493, 518, 489, 506, 557, 508, 517, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 511, 482, 476, 518, 531, 541, 490, 508, 527, 519, 497, 525, 500, 517, 521, 30, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 517, 527, 490, 525, 547, 514, 539, 553, 504, 496, 491, 455, 533, 512, 528, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 482, 500, 486, 548, 541, 514, 518, 487, 514, 530, 530, 542, 496, 511, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 503, 517, 544, 487, 485, 523, 488, 526, 509, 489, 495, 514, 483, 504, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8228, 8231, 8184, 8092, 8199, 8267, 8240, 8095] avg=8192.0000 max=8267 max_rank=5 min=8092 min_rank=3 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32801 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[542, 485, 520, 527, 525, 491, 517, 501, 552, 509, 511, 494, 537, 540, 503, 499, 32, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 513, 507, 540, 488, 551, 521, 528, 573, 502, 515, 492, 499, 515, 462, 520, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 501, 488, 498, 518, 505, 501, 548, 481, 493, 519, 524, 529, 522, 523, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 531, 502, 497, 525, 496, 500, 501, 515, 496, 502, 500, 510, 487, 535, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 498, 505, 532, 537, 468, 503, 513, 506, 514, 502, 536, 504, 498, 485, 516, 26, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 497, 498, 483, 520, 483, 501, 536, 508, 495, 528, 517, 502, 477, 511, 551, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 519, 487, 459, 464, 524, 547, 533, 519, 504, 524, 529, 512, 518, 509, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 509, 508, 501, 504, 486, 552, 490, 519, 543, 499, 549, 527, 506, 509, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8288, 8226, 8179, 8112, 8123, 8156, 8197, 8255] avg=8192.0000 max=8288 max_rank=0 min=8112 min_rank=3 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32731 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[504, 514, 516, 485, 506, 506, 494, 465, 498, 523, 489, 483, 502, 514, 490, 491, 33, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 521, 531, 505, 533, 496, 486, 470, 541, 484, 524, 495, 489, 503, 532, 531, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 566, 534, 490, 549, 529, 531, 485, 473, 518, 506, 522, 491, 543, 465, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 573, 529, 486, 505, 561, 515, 493, 542, 500, 506, 511, 520, 538, 526, 516, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 482, 532, 481, 490, 499, 515, 537, 532, 489, 508, 494, 497, 482, 510, 515, 26, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 493, 509, 535, 506, 505, 537, 525, 542, 529, 487, 503, 478, 497, 501, 518, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 511, 529, 504, 509, 499, 501, 540, 463, 542, 557, 517, 508, 505, 535, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 513, 504, 526, 527, 524, 557, 494, 546, 530, 510, 503, 523, 512, 512, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8016, 8166, 8186, 8321, 8092, 8223, 8217, 8315] avg=8192.0000 max=8321 max_rank=3 min=8016 min_rank=0 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32847 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[513, 546, 496, 513, 504, 486, 493, 490, 507, 506, 480, 510, 546, 493, 513, 508, 34, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 493, 542, 493, 499, 531, 505, 517, 487, 500, 498, 475, 572, 524, 522, 528, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 537, 493, 516, 482, 509, 538, 496, 494, 520, 507, 503, 493, 538, 501, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 528, 486, 517, 519, 477, 493, 516, 519, 540, 506, 513, 532, 502, 512, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 489, 493, 548, 492, 512, 495, 530, 538, 515, 509, 480, 508, 503, 527, 473, 25, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 510, 509, 550, 469, 495, 518, 547, 538, 463, 554, 554, 497, 512, 510, 530, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 468, 495, 520, 482, 494, 499, 495, 509, 530, 504, 518, 533, 502, 516, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 497, 543, 546, 524, 532, 527, 511, 518, 499, 516, 489, 523, 530, 518, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8141, 8244, 8163, 8154, 8166, 8315, 8046, 8307] avg=8192.0000 max=8315 max_rank=5 min=8046 min_rank=6 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32834 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[472, 526, 541, 525, 497, 503, 534, 483, 530, 532, 527, 524, 516, 511, 549, 498, 34, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 488, 529, 528, 527, 513, 514, 507, 530, 515, 508, 465, 499, 520, 496, 513, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 566, 521, 509, 501, 521, 510, 499, 525, 512, 526, 557, 500, 518, 474, 501, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 489, 534, 540, 509, 525, 500, 504, 471, 485, 477, 522, 502, 483, 519, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 510, 519, 502, 492, 532, 524, 500, 511, 493, 529, 474, 500, 488, 541, 485, 26, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 538, 525, 520, 518, 515, 499, 478, 468, 528, 492, 466, 539, 503, 519, 518, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 500, 550, 527, 546, 508, 505, 515, 560, 536, 487, 483, 515, 477, 507, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 528, 504, 516, 515, 541, 508, 516, 527, 490, 515, 491, 519, 522, 484, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8305, 8195, 8278, 8074, 8126, 8148, 8223, 8187] avg=8192.0000 max=8305 max_rank=0 min=8074 min_rank=3 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32684 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[484, 529, 489, 504, 544, 488, 487, 501, 496, 532, 495, 501, 520, 484, 479, 512, 32, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 463, 522, 496, 501, 537, 509, 482, 522, 482, 507, 512, 492, 535, 536, 518, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 493, 545, 507, 557, 487, 524, 488, 472, 542, 515, 523, 561, 495, 515, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 516, 507, 464, 529, 500, 516, 499, 549, 518, 529, 519, 491, 502, 516, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 525, 498, 519, 505, 521, 503, 525, 518, 550, 536, 508, 543, 505, 555, 446, 26, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 478, 503, 508, 520, 549, 524, 503, 508, 537, 565, 529, 493, 456, 525, 536, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 449, 524, 507, 495, 535, 526, 535, 460, 474, 528, 554, 515, 519, 485, 484, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 560, 528, 489, 518, 518, 551, 482, 511, 488, 489, 574, 485, 520, 487, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8080, 8140, 8247, 8156, 8342, 8246, 8086, 8239] avg=8192.0000 max=8342 max_rank=4 min=8080 min_rank=0 imbalance_ratio=1.018311 max_load_reduction=57194 max_load_reduction_pct=0.872711 moved_to_echo=32913 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[513, 546, 496, 513, 504, 486, 493, 490, 507, 506, 480, 510, 546, 493, 513, 508, 34, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 493, 542, 493, 499, 531, 505, 517, 487, 500, 498, 475, 572, 524, 522, 528, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 537, 493, 516, 482, 509, 538, 496, 494, 520, 507, 503, 493, 538, 501, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 528, 486, 517, 519, 477, 493, 516, 519, 540, 506, 513, 532, 502, 512, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 489, 493, 548, 492, 512, 495, 530, 538, 515, 509, 480, 508, 503, 527, 473, 25, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 510, 509, 550, 469, 495, 518, 547, 538, 463, 554, 554, 497, 512, 510, 530, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 468, 495, 520, 482, 494, 499, 495, 509, 530, 504, 518, 533, 502, 516, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 497, 543, 546, 524, 532, 527, 511, 518, 499, 516, 489, 523, 530, 518, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8141, 8244, 8163, 8154, 8166, 8315, 8046, 8307] avg=8192.0000 max=8315 max_rank=5 min=8046 min_rank=6 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32834 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[504, 514, 516, 485, 506, 506, 494, 465, 498, 523, 489, 483, 502, 514, 490, 491, 33, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 521, 531, 505, 533, 496, 486, 470, 541, 484, 524, 495, 489, 503, 532, 531, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 566, 534, 490, 549, 529, 531, 485, 473, 518, 506, 522, 491, 543, 465, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 573, 529, 486, 505, 561, 515, 493, 542, 500, 506, 511, 520, 538, 526, 516, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 482, 532, 481, 490, 499, 515, 537, 532, 489, 508, 494, 497, 482, 510, 515, 26, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 493, 509, 535, 506, 505, 537, 525, 542, 529, 487, 503, 478, 497, 501, 518, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 511, 529, 504, 509, 499, 501, 540, 463, 542, 557, 517, 508, 505, 535, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 513, 504, 526, 527, 524, 557, 494, 546, 530, 510, 503, 523, 512, 512, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8016, 8166, 8186, 8321, 8092, 8223, 8217, 8315] avg=8192.0000 max=8321 max_rank=3 min=8016 min_rank=0 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32847 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[521, 493, 525, 520, 505, 517, 506, 495, 508, 508, 502, 524, 520, 494, 525, 486, 35, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 505, 529, 529, 525, 503, 501, 493, 544, 526, 484, 482, 506, 518, 490, 547, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 509, 507, 496, 483, 504, 469, 493, 466, 505, 517, 520, 554, 521, 502, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 480, 530, 517, 540, 556, 562, 466, 512, 513, 477, 518, 508, 521, 540, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 558, 507, 519, 530, 529, 484, 497, 515, 511, 491, 532, 542, 533, 507, 525, 519, 27, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 544, 507, 510, 488, 506, 497, 461, 510, 501, 502, 522, 528, 486, 484, 490, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 541, 499, 529, 554, 511, 499, 515, 489, 516, 498, 537, 538, 545, 537, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 480, 486, 499, 520, 506, 494, 537, 483, 518, 461, 498, 521, 519, 488, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8225, 8077, 8279, 8333, 8071, 8303, 8061] avg=8192.0000 max=8333 max_rank=4 min=8061 min_rank=7 imbalance_ratio=1.017212 max_load_reduction=57203 max_load_reduction_pct=0.872849 moved_to_echo=32768 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 3968, 4096, 4224, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 3968, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4096, 3968, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 3968, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 3968, 4096, 4224, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 3968, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 3968, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[502, 530, 523, 526, 517, 492, 469, 541, 503, 502, 520, 521, 504, 473, 512, 542, 543, 542, 497, 525, 502, 483, 528, 500, 514, 496, 517, 517, 553, 519, 493, 524, 530, 519, 506, 537, 486, 508, 562, 521, 550, 532, 484, 495, 575, 504, 523, 504, 519, 532, 468, 547, 514, 508, 505, 524, 495, 545, 475, 501, 517, 481, 478, 540, 491, 514, 518, 512, 541, 513, 547, 540, 471, 546, 541, 482, 515, 498, 553, 497, 509, 495, 490, 490, 532, 539, 491, 520, 491, 524, 537, 501, 500, 462, 475, 498, 507, 509, 506, 491, 522, 503, 491, 517, 531, 494, 517, 492, 490, 522, 540, 550, 497, 541, 500, 496, 492, 547, 531, 492, 456, 466, 497, 507, 512, 538, 513, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[498, 518, 536, 522, 566, 502, 521, 501, 506, 512, 537, 492, 485, 512, 563, 503, 519, 519, 494, 494, 464, 506, 507, 552, 515, 539, 482, 503, 538, 493, 513, 534, 486, 492, 483, 513, 475, 516, 515, 503, 489, 521, 559, 518, 533, 535, 498, 502, 543, 476, 539, 545, 465, 504, 491, 503, 519, 516, 484, 518, 498, 535, 537, 529, 536, 478, 487, 514, 525, 535, 534, 502, 505, 487, 521, 515, 518, 550, 508, 501, 469, 504, 490, 518, 489, 550, 520, 524, 508, 547, 540, 534, 532, 511, 502, 517, 537, 530, 512, 487, 528, 508, 512, 480, 497, 506, 531, 525, 531, 515, 529, 488, 531, 515, 519, 492, 484, 480, 509, 542, 512, 488, 479, 488, 526, 465, 525, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[493, 509, 551, 495, 545, 501, 504, 475, 508, 518, 489, 519, 505, 521, 551, 469, 522, 519, 526, 520, 454, 488, 511, 504, 508, 553, 537, 512, 495, 486, 499, 487, 506, 517, 543, 518, 509, 504, 475, 551, 499, 587, 521, 516, 477, 529, 493, 539, 515, 551, 503, 510, 488, 531, 532, 484, 509, 539, 500, 498, 501, 532, 533, 551, 511, 514, 528, 489, 517, 529, 525, 523, 469, 536, 521, 521, 478, 471, 492, 485, 505, 508, 497, 513, 463, 518, 495, 536, 473, 532, 492, 512, 524, 491, 517, 524, 545, 488, 497, 513, 523, 506, 527, 509, 471, 568, 540, 504, 509, 512, 553, 493, 489, 489, 528, 447, 519, 510, 556, 514, 517, 501, 545, 490, 533, 563, 508, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[502, 530, 523, 526, 517, 492, 469, 541, 503, 502, 520, 521, 504, 473, 512, 542, 543, 542, 497, 525, 502, 483, 528, 500, 514, 496, 517, 517, 553, 519, 493, 524, 530, 519, 506, 537, 486, 508, 562, 521, 550, 532, 484, 495, 575, 504, 523, 504, 519, 532, 468, 547, 514, 508, 505, 524, 495, 545, 475, 501, 517, 481, 478, 540, 491, 514, 518, 512, 541, 513, 547, 540, 471, 546, 541, 482, 515, 498, 553, 497, 509, 495, 490, 490, 532, 539, 491, 520, 491, 524, 537, 501, 500, 462, 475, 498, 507, 509, 506, 491, 522, 503, 491, 517, 531, 494, 517, 492, 490, 522, 540, 550, 497, 541, 500, 496, 492, 547, 531, 492, 456, 466, 497, 507, 512, 538, 513, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[496, 500, 522, 522, 500, 519, 505, 510, 499, 537, 523, 531, 505, 510, 516, 539, 541, 512, 546, 525, 490, 489, 489, 526, 496, 514, 520, 475, 462, 490, 461, 525, 498, 506, 505, 512, 491, 497, 519, 481, 514, 559, 532, 552, 511, 498, 497, 509, 488, 493, 523, 490, 508, 560, 506, 516, 527, 525, 508, 504, 514, 526, 530, 549, 495, 498, 502, 538, 522, 513, 518, 543, 527, 500, 509, 537, 493, 499, 544, 522, 493, 526, 542, 520, 552, 520, 517, 501, 498, 484, 508, 505, 497, 533, 523, 531, 535, 559, 486, 517, 540, 531, 493, 539, 496, 511, 500, 505, 531, 484, 498, 485, 494, 500, 515, 484, 553, 497, 483, 498, 502, 501, 499, 540, 453, 500, 550, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[536, 519, 543, 492, 566, 495, 507, 499, 488, 510, 513, 496, 530, 482, 515, 509, 495, 512, 511, 522, 525, 519, 507, 512, 481, 503, 516, 531, 569, 535, 535, 501, 534, 466, 478, 553, 512, 504, 522, 487, 481, 494, 480, 509, 499, 506, 489, 546, 518, 498, 513, 513, 510, 501, 531, 495, 531, 528, 493, 539, 490, 551, 531, 501, 503, 522, 512, 490, 520, 513, 521, 518, 498, 505, 528, 521, 526, 531, 491, 523, 502, 515, 492, 535, 526, 502, 562, 514, 510, 498, 505, 503, 520, 531, 516, 511, 524, 512, 543, 450, 538, 511, 509, 532, 499, 533, 517, 549, 538, 476, 480, 501, 465, 532, 522, 524, 493, 483, 471, 523, 466, 511, 497, 503, 537, 508, 533, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[504, 503, 500, 542, 518, 515, 488, 516, 508, 493, 532, 514, 515, 524, 494, 478, 518, 525, 514, 536, 512, 560, 539, 513, 527, 497, 537, 537, 528, 532, 521, 543, 541, 525, 557, 522, 498, 531, 513, 532, 491, 541, 522, 493, 504, 493, 517, 491, 519, 506, 509, 521, 503, 518, 508, 532, 512, 498, 480, 464, 501, 516, 526, 512, 501, 547, 529, 491, 502, 521, 492, 498, 485, 510, 519, 489, 519, 497, 520, 535, 480, 480, 512, 526, 514, 503, 469, 473, 520, 502, 537, 529, 516, 504, 525, 481, 516, 529, 493, 521, 532, 524, 513, 498, 503, 440, 484, 518, 548, 537, 456, 511, 543, 497, 483, 519, 515, 512, 502, 497, 481, 514, 522, 524, 544, 512, 529, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[535, 501, 481, 498, 519, 516, 499, 531, 526, 540, 542, 512, 514, 510, 558, 483, 530, 514, 527, 485, 501, 479, 500, 514, 515, 501, 521, 498, 508, 519, 489, 497, 528, 504, 537, 519, 562, 447, 537, 517, 500, 512, 499, 497, 503, 542, 542, 512, 486, 526, 524, 491, 493, 488, 506, 492, 513, 511, 532, 539, 518, 559, 471, 505, 497, 514, 515, 506, 513, 573, 497, 499, 495, 507, 517, 509, 537, 531, 521, 513, 547, 487, 530, 491, 503, 538, 522, 501, 501, 479, 530, 539, 509, 494, 542, 508, 542, 530, 517, 511, 505, 488, 458, 488, 503, 515, 525, 512, 523, 523, 531, 498, 507, 530, 487, 495, 498, 509, 524, 521, 513, 509, 472, 515, 483, 507, 529, 528] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[504, 503, 500, 542, 518, 515, 488, 516, 508, 493, 532, 514, 515, 524, 494, 478, 518, 525, 514, 536, 512, 560, 539, 513, 527, 497, 537, 537, 528, 532, 521, 543, 541, 525, 557, 522, 498, 531, 513, 532, 491, 541, 522, 493, 504, 493, 517, 491, 519, 506, 509, 521, 503, 518, 508, 532, 512, 498, 480, 464, 501, 516, 526, 512, 501, 547, 529, 491, 502, 521, 492, 498, 485, 510, 519, 489, 519, 497, 520, 535, 480, 480, 512, 526, 514, 503, 469, 473, 520, 502, 537, 529, 516, 504, 525, 481, 516, 529, 493, 521, 532, 524, 513, 498, 503, 440, 484, 518, 548, 537, 456, 511, 543, 497, 483, 519, 515, 512, 502, 497, 481, 514, 522, 524, 544, 512, 529, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[493, 509, 551, 495, 545, 501, 504, 475, 508, 518, 489, 519, 505, 521, 551, 469, 522, 519, 526, 520, 454, 488, 511, 504, 508, 553, 537, 512, 495, 486, 499, 487, 506, 517, 543, 518, 509, 504, 475, 551, 499, 587, 521, 516, 477, 529, 493, 539, 515, 551, 503, 510, 488, 531, 532, 484, 509, 539, 500, 498, 501, 532, 533, 551, 511, 514, 528, 489, 517, 529, 525, 523, 469, 536, 521, 521, 478, 471, 492, 485, 505, 508, 497, 513, 463, 518, 495, 536, 473, 532, 492, 512, 524, 491, 517, 524, 545, 488, 497, 513, 523, 506, 527, 509, 471, 568, 540, 504, 509, 512, 553, 493, 489, 489, 528, 447, 519, 510, 556, 514, 517, 501, 545, 490, 533, 563, 508, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[536, 519, 543, 492, 566, 495, 507, 499, 488, 510, 513, 496, 530, 482, 515, 509, 495, 512, 511, 522, 525, 519, 507, 512, 481, 503, 516, 531, 569, 535, 535, 501, 534, 466, 478, 553, 512, 504, 522, 487, 481, 494, 480, 509, 499, 506, 489, 546, 518, 498, 513, 513, 510, 501, 531, 495, 531, 528, 493, 539, 490, 551, 531, 501, 503, 522, 512, 490, 520, 513, 521, 518, 498, 505, 528, 521, 526, 531, 491, 523, 502, 515, 492, 535, 526, 502, 562, 514, 510, 498, 505, 503, 520, 531, 516, 511, 524, 512, 543, 450, 538, 511, 509, 532, 499, 533, 517, 549, 538, 476, 480, 501, 465, 532, 522, 524, 493, 483, 471, 523, 466, 511, 497, 503, 537, 508, 533, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[505, 548, 508, 508, 513, 539, 521, 510, 496, 516, 513, 513, 498, 497, 477, 491, 501, 536, 483, 494, 509, 515, 501, 484, 485, 486, 542, 521, 511, 520, 476, 495, 493, 486, 550, 569, 501, 539, 528, 489, 496, 521, 493, 498, 523, 527, 523, 521, 482, 504, 573, 527, 568, 533, 513, 517, 499, 542, 532, 539, 550, 505, 516, 511, 534, 517, 487, 539, 533, 514, 522, 510, 509, 488, 477, 485, 513, 497, 501, 475, 505, 505, 533, 472, 496, 501, 489, 491, 498, 523, 524, 526, 521, 551, 525, 522, 513, 550, 549, 539, 533, 510, 515, 485, 506, 519, 534, 503, 545, 468, 543, 501, 501, 529, 505, 491, 484, 493, 503, 497, 500, 523, 467, 499, 508, 524, 505, 531] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[493, 509, 551, 495, 537, 501, 504, 475, 508, 518, 489, 519, 505, 521, 551, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 519, 526, 520, 454, 488, 511, 504, 508, 553, 537, 512, 495, 486, 499, 487, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 517, 543, 518, 509, 504, 475, 551, 499, 553, 521, 516, 477, 529, 493, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 551, 503, 510, 488, 531, 532, 484, 509, 539, 500, 498, 501, 496, 533, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 514, 528, 489, 517, 523, 525, 523, 469, 536, 521, 521, 478, 471, 492, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 508, 497, 513, 463, 518, 495, 536, 473, 532, 492, 512, 524, 491, 517, 524, 13, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 488, 497, 513, 505, 506, 527, 509, 471, 568, 540, 504, 509, 512, 553, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 489, 528, 447, 519, 510, 556, 514, 517, 501, 545, 490, 533, 563, 508, 525, 36, 34, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8145, 8128, 8250, 8241, 8103, 8120, 8240, 8309] avg=8192.0000 max=8309 max_rank=7 min=8103 min_rank=4 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32772 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[502, 530, 523, 526, 510, 492, 469, 541, 503, 502, 520, 521, 504, 473, 512, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 542, 497, 525, 502, 483, 528, 500, 514, 496, 517, 517, 553, 519, 493, 524, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 519, 506, 537, 486, 508, 562, 521, 550, 501, 484, 495, 575, 504, 523, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 532, 468, 547, 514, 508, 505, 524, 495, 545, 475, 501, 517, 448, 478, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 514, 518, 512, 541, 507, 547, 540, 471, 546, 541, 482, 515, 498, 553, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 495, 490, 490, 532, 539, 491, 520, 491, 524, 537, 501, 500, 462, 475, 498, 13, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 509, 506, 491, 504, 503, 491, 517, 531, 494, 517, 492, 490, 522, 540, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 541, 500, 496, 492, 547, 531, 492, 456, 466, 497, 507, 512, 538, 513, 521, 33, 31, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8170, 8260, 8305, 8116, 8273, 8073, 8164, 8175] avg=8192.0000 max=8305 max_rank=2 min=8073 min_rank=5 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32685 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[496, 500, 522, 522, 488, 519, 505, 510, 499, 537, 523, 531, 505, 510, 516, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 512, 546, 525, 490, 489, 489, 526, 496, 514, 520, 475, 462, 490, 461, 525, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 506, 505, 512, 491, 497, 519, 481, 514, 523, 532, 552, 511, 498, 497, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 493, 523, 490, 508, 560, 506, 516, 527, 525, 508, 504, 514, 486, 530, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 498, 502, 538, 522, 505, 518, 543, 527, 500, 509, 537, 493, 499, 544, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 526, 542, 520, 552, 520, 517, 501, 498, 484, 508, 505, 497, 533, 523, 531, 16, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 559, 486, 517, 517, 531, 493, 539, 496, 511, 500, 505, 531, 484, 498, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 500, 515, 484, 553, 497, 483, 498, 502, 501, 499, 540, 453, 500, 550, 504, 40, 36, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8071, 8145, 8227, 8252, 8276, 8187, 8156] avg=8192.0000 max=8276 max_rank=5 min=8071 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32871 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[502, 530, 523, 526, 510, 492, 469, 541, 503, 502, 520, 521, 504, 473, 512, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 542, 497, 525, 502, 483, 528, 500, 514, 496, 517, 517, 553, 519, 493, 524, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 519, 506, 537, 486, 508, 562, 521, 550, 501, 484, 495, 575, 504, 523, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 532, 468, 547, 514, 508, 505, 524, 495, 545, 475, 501, 517, 448, 478, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 514, 518, 512, 541, 507, 547, 540, 471, 546, 541, 482, 515, 498, 553, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 495, 490, 490, 532, 539, 491, 520, 491, 524, 537, 501, 500, 462, 475, 498, 13, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 509, 506, 491, 504, 503, 491, 517, 531, 494, 517, 492, 490, 522, 540, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 541, 500, 496, 492, 547, 531, 492, 456, 466, 497, 507, 512, 538, 513, 521, 33, 31, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8170, 8260, 8305, 8116, 8273, 8073, 8164, 8175] avg=8192.0000 max=8305 max_rank=2 min=8073 min_rank=5 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32685 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[498, 518, 536, 522, 558, 502, 521, 501, 506, 512, 537, 492, 485, 512, 563, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 519, 494, 494, 464, 506, 507, 552, 515, 539, 482, 503, 538, 493, 513, 534, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 492, 483, 513, 475, 516, 515, 503, 489, 491, 559, 518, 533, 535, 498, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 476, 539, 545, 465, 504, 491, 503, 519, 516, 484, 518, 498, 498, 537, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 478, 487, 514, 525, 529, 534, 502, 505, 487, 521, 515, 518, 550, 508, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 504, 490, 518, 489, 550, 520, 524, 508, 547, 540, 534, 532, 511, 502, 517, 13, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 530, 512, 487, 510, 508, 512, 480, 497, 506, 531, 525, 531, 515, 529, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 515, 519, 492, 484, 480, 509, 542, 512, 488, 479, 488, 526, 465, 525, 508, 37, 30, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8266, 8179, 8108, 8165, 8210, 8275, 8198, 8135] avg=8192.0000 max=8275 max_rank=5 min=8108 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32818 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[504, 503, 500, 542, 511, 515, 488, 516, 508, 493, 532, 514, 515, 524, 494, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 525, 514, 536, 512, 560, 539, 513, 527, 497, 537, 537, 528, 532, 521, 543, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 525, 557, 522, 498, 531, 513, 532, 491, 510, 522, 493, 504, 493, 517, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 506, 509, 521, 503, 518, 508, 532, 512, 498, 480, 464, 501, 481, 526, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 547, 529, 491, 502, 515, 492, 498, 485, 510, 519, 489, 519, 497, 520, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 480, 512, 526, 514, 503, 469, 473, 520, 502, 537, 529, 516, 504, 525, 481, 13, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 529, 493, 521, 514, 524, 513, 498, 503, 440, 484, 518, 548, 537, 456, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 497, 483, 519, 515, 512, 502, 497, 481, 514, 522, 524, 544, 512, 529, 514, 35, 31, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8137, 8446, 8240, 8090, 8149, 8090, 8105, 8279] avg=8192.0000 max=8446 max_rank=1 min=8090 min_rank=3 imbalance_ratio=1.031006 max_load_reduction=57090 max_load_reduction_pct=0.871124 moved_to_echo=32623 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[505, 548, 508, 508, 506, 539, 521, 510, 496, 516, 513, 513, 498, 497, 477, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 536, 483, 494, 509, 515, 501, 484, 485, 486, 542, 521, 511, 520, 476, 495, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 486, 550, 569, 501, 539, 528, 489, 496, 491, 493, 498, 523, 527, 523, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 504, 573, 527, 568, 533, 513, 517, 499, 542, 532, 539, 550, 470, 516, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 517, 487, 539, 533, 508, 522, 510, 509, 488, 477, 485, 513, 497, 501, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 505, 533, 472, 496, 501, 489, 491, 498, 523, 524, 526, 521, 551, 525, 522, 13, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 550, 549, 539, 515, 510, 515, 485, 506, 519, 534, 503, 545, 468, 543, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 529, 505, 491, 484, 493, 503, 497, 500, 523, 467, 499, 508, 524, 505, 531, 35, 30, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8146, 8066, 8227, 8376, 8095, 8201, 8295, 8130] avg=8192.0000 max=8376 max_rank=3 min=8066 min_rank=1 imbalance_ratio=1.022461 max_load_reduction=57160 max_load_reduction_pct=0.872192 moved_to_echo=32721 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[536, 519, 543, 492, 558, 495, 507, 499, 488, 510, 513, 496, 530, 482, 515, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 512, 511, 522, 525, 519, 507, 512, 481, 503, 516, 531, 569, 535, 535, 501, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 466, 478, 553, 512, 504, 522, 487, 481, 466, 480, 509, 499, 506, 489, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 498, 513, 513, 510, 501, 531, 495, 531, 528, 493, 539, 490, 513, 531, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 522, 512, 490, 520, 507, 521, 518, 498, 505, 528, 521, 526, 531, 491, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 515, 492, 535, 526, 502, 562, 514, 510, 498, 505, 503, 520, 531, 516, 511, 13, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 512, 543, 450, 520, 511, 509, 532, 499, 533, 517, 549, 538, 476, 480, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 532, 522, 524, 493, 483, 471, 523, 466, 511, 497, 503, 537, 508, 533, 515, 38, 28, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8281, 8032, 8205, 8216, 8262, 8194, 8154] avg=8192.0000 max=8281 max_rank=1 min=8032 min_rank=2 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32826 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[493, 509, 551, 495, 537, 501, 504, 475, 508, 518, 489, 519, 505, 521, 551, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 519, 526, 520, 454, 488, 511, 504, 508, 553, 537, 512, 495, 486, 499, 487, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 517, 543, 518, 509, 504, 475, 551, 499, 553, 521, 516, 477, 529, 493, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 551, 503, 510, 488, 531, 532, 484, 509, 539, 500, 498, 501, 496, 533, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 514, 528, 489, 517, 523, 525, 523, 469, 536, 521, 521, 478, 471, 492, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 508, 497, 513, 463, 518, 495, 536, 473, 532, 492, 512, 524, 491, 517, 524, 13, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 488, 497, 513, 505, 506, 527, 509, 471, 568, 540, 504, 509, 512, 553, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 489, 528, 447, 519, 510, 556, 514, 517, 501, 545, 490, 533, 563, 508, 525, 36, 34, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8145, 8128, 8250, 8241, 8103, 8120, 8240, 8309] avg=8192.0000 max=8309 max_rank=7 min=8103 min_rank=4 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32772 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[535, 501, 481, 498, 512, 516, 499, 531, 526, 540, 542, 512, 514, 510, 558, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 514, 527, 485, 501, 479, 500, 514, 515, 501, 521, 498, 508, 519, 489, 497, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 504, 537, 519, 562, 447, 537, 517, 500, 483, 499, 497, 503, 542, 542, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 526, 524, 491, 493, 488, 506, 492, 513, 511, 532, 539, 518, 521, 471, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 514, 515, 506, 513, 567, 497, 499, 495, 507, 517, 509, 537, 531, 521, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 487, 530, 491, 503, 538, 522, 501, 501, 479, 530, 539, 509, 494, 542, 508, 12, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 530, 517, 511, 489, 488, 458, 488, 503, 515, 525, 512, 523, 523, 531, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 530, 487, 495, 498, 509, 524, 521, 513, 509, 472, 515, 483, 507, 529, 528, 38, 29, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8258, 8105, 8229, 8116, 8238, 8239, 8153, 8198] avg=8192.0000 max=8258 max_rank=0 min=8105 min_rank=1 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32828 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[505, 548, 508, 508, 506, 539, 521, 510, 496, 516, 513, 513, 498, 497, 477, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 536, 483, 494, 509, 515, 501, 484, 485, 486, 542, 521, 511, 520, 476, 495, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 486, 550, 569, 501, 539, 528, 489, 496, 491, 493, 498, 523, 527, 523, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 504, 573, 527, 568, 533, 513, 517, 499, 542, 532, 539, 550, 470, 516, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 517, 487, 539, 533, 508, 522, 510, 509, 488, 477, 485, 513, 497, 501, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 505, 533, 472, 496, 501, 489, 491, 498, 523, 524, 526, 521, 551, 525, 522, 13, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 550, 549, 539, 515, 510, 515, 485, 506, 519, 534, 503, 545, 468, 543, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 529, 505, 491, 484, 493, 503, 497, 500, 523, 467, 499, 508, 524, 505, 531, 35, 30, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8146, 8066, 8227, 8376, 8095, 8201, 8295, 8130] avg=8192.0000 max=8376 max_rank=3 min=8066 min_rank=1 imbalance_ratio=1.022461 max_load_reduction=57160 max_load_reduction_pct=0.872192 moved_to_echo=32721 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 3968, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 3968, 4096, 4096, 4096, 4096, 3968, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 384, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 3968, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4352, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 3968, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4352, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 3968, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 3968, 4096, 4096, 4096, 4096, 3968, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 384, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[495, 481, 496, 459, 530, 500, 504, 514, 528, 509, 499, 494, 523, 511, 515, 555, 484, 542, 474, 517, 590, 510, 497, 519, 494, 473, 538, 495, 503, 556, 508, 500, 533, 506, 550, 540, 515, 498, 490, 475, 542, 494, 516, 507, 504, 511, 508, 536, 544, 527, 509, 501, 501, 498, 541, 531, 510, 498, 550, 517, 493, 508, 539, 487, 469, 532, 525, 498, 562, 489, 502, 489, 537, 471, 503, 519, 536, 509, 550, 529, 521, 511, 564, 503, 555, 494, 500, 481, 510, 516, 489, 518, 547, 490, 443, 510, 537, 519, 485, 524, 522, 522, 534, 493, 521, 487, 504, 528, 511, 524, 514, 505, 516, 486, 514, 561, 498, 538, 504, 472, 536, 470, 523, 509, 510, 490, 503, 512] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[490, 498, 523, 522, 531, 504, 516, 489, 497, 526, 523, 511, 468, 499, 533, 508, 514, 530, 497, 492, 516, 514, 486, 482, 526, 510, 549, 500, 482, 529, 530, 517, 509, 484, 482, 522, 491, 544, 487, 527, 477, 512, 527, 502, 504, 503, 541, 527, 547, 527, 565, 505, 495, 470, 531, 505, 502, 520, 502, 465, 531, 507, 526, 512, 496, 497, 509, 538, 519, 555, 497, 503, 537, 469, 533, 504, 484, 471, 529, 511, 507, 484, 500, 522, 540, 499, 505, 543, 513, 553, 529, 501, 509, 567, 503, 516, 513, 546, 504, 496, 527, 519, 504, 535, 503, 528, 540, 493, 505, 514, 480, 531, 480, 531, 497, 512, 496, 511, 530, 527, 516, 508, 480, 500, 544, 541, 509, 512] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[562, 497, 509, 512, 529, 488, 429, 521, 538, 533, 549, 501, 507, 534, 475, 480, 519, 490, 498, 519, 503, 520, 550, 520, 497, 542, 539, 518, 552, 532, 519, 494, 503, 556, 549, 510, 517, 490, 538, 493, 536, 519, 509, 506, 504, 490, 508, 478, 496, 499, 482, 460, 515, 536, 526, 502, 533, 530, 513, 545, 483, 531, 477, 511, 506, 487, 517, 530, 493, 529, 512, 505, 486, 537, 524, 516, 506, 488, 509, 519, 511, 503, 550, 488, 528, 499, 536, 529, 464, 501, 514, 486, 518, 511, 502, 506, 510, 499, 494, 503, 511, 536, 528, 509, 507, 517, 522, 522, 495, 532, 496, 500, 509, 483, 485, 501, 519, 497, 551, 521, 515, 519, 541, 515, 540, 520, 540, 468] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[506, 523, 505, 506, 510, 509, 495, 483, 466, 506, 522, 569, 511, 522, 508, 489, 485, 540, 508, 503, 485, 497, 520, 510, 497, 496, 529, 474, 533, 490, 507, 542, 500, 514, 545, 551, 500, 507, 494, 508, 529, 529, 524, 507, 523, 474, 480, 475, 487, 501, 543, 534, 514, 537, 495, 590, 500, 513, 491, 520, 516, 521, 508, 523, 517, 504, 482, 509, 489, 508, 503, 524, 526, 489, 504, 534, 515, 516, 533, 525, 510, 537, 502, 490, 521, 535, 516, 484, 520, 536, 513, 512, 522, 499, 513, 527, 522, 481, 525, 510, 533, 486, 483, 501, 546, 518, 546, 484, 521, 550, 520, 518, 512, 505, 511, 515, 508, 532, 497, 522, 511, 462, 520, 507, 543, 490, 525, 518] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[543, 501, 504, 483, 511, 524, 499, 483, 498, 472, 501, 502, 569, 531, 521, 515, 505, 526, 540, 478, 510, 515, 506, 501, 547, 473, 488, 495, 502, 496, 561, 538, 503, 524, 458, 552, 514, 535, 513, 536, 483, 525, 483, 538, 470, 519, 527, 492, 477, 487, 512, 512, 506, 495, 520, 515, 502, 495, 534, 482, 531, 499, 512, 522, 500, 471, 554, 554, 514, 514, 517, 518, 507, 510, 508, 531, 504, 541, 527, 502, 510, 498, 488, 515, 537, 540, 526, 527, 511, 545, 497, 513, 503, 538, 525, 508, 519, 492, 494, 509, 500, 525, 579, 508, 526, 550, 567, 512, 536, 511, 502, 483, 539, 508, 520, 507, 507, 512, 502, 447, 506, 472, 528, 501, 536, 500, 514, 460] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[554, 532, 500, 499, 488, 547, 525, 528, 556, 469, 512, 526, 486, 520, 469, 479, 537, 490, 509, 559, 523, 573, 507, 472, 517, 523, 477, 537, 480, 516, 521, 491, 524, 558, 471, 529, 497, 500, 464, 508, 467, 497, 514, 512, 516, 512, 499, 455, 537, 507, 535, 538, 544, 514, 488, 491, 544, 478, 500, 487, 511, 521, 495, 512, 503, 501, 512, 502, 483, 530, 499, 497, 517, 514, 499, 529, 535, 535, 510, 545, 480, 535, 521, 528, 496, 507, 505, 486, 478, 493, 562, 516, 528, 497, 538, 519, 522, 508, 517, 588, 502, 512, 496, 516, 526, 499, 495, 513, 536, 515, 523, 507, 506, 495, 498, 517, 553, 488, 485, 496, 522, 516, 547, 504, 492, 503, 528, 564] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[516, 505, 500, 491, 510, 448, 476, 480, 453, 516, 568, 517, 518, 481, 489, 489, 488, 532, 547, 511, 526, 509, 533, 491, 555, 502, 492, 527, 525, 523, 530, 504, 486, 481, 508, 535, 566, 476, 495, 534, 539, 508, 501, 500, 581, 502, 521, 482, 511, 513, 496, 515, 516, 515, 500, 493, 525, 518, 507, 538, 496, 514, 515, 541, 499, 489, 491, 486, 505, 480, 513, 499, 512, 518, 493, 521, 467, 525, 516, 504, 518, 533, 518, 526, 510, 510, 525, 483, 508, 492, 523, 544, 484, 545, 565, 564, 511, 531, 508, 501, 527, 525, 489, 552, 516, 492, 526, 532, 503, 496, 472, 514, 505, 495, 563, 510, 516, 495, 528, 518, 516, 540, 487, 528, 523, 501, 512, 558] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[543, 501, 504, 483, 511, 524, 499, 483, 498, 472, 501, 502, 569, 531, 521, 515, 505, 526, 540, 478, 510, 515, 506, 501, 547, 473, 488, 495, 502, 496, 561, 538, 503, 524, 458, 552, 514, 535, 513, 536, 483, 525, 483, 538, 470, 519, 527, 492, 477, 487, 512, 512, 506, 495, 520, 515, 502, 495, 534, 482, 531, 499, 512, 522, 500, 471, 554, 554, 514, 514, 517, 518, 507, 510, 508, 531, 504, 541, 527, 502, 510, 498, 488, 515, 537, 540, 526, 527, 511, 545, 497, 513, 503, 538, 525, 508, 519, 492, 494, 509, 500, 525, 579, 508, 526, 550, 567, 512, 536, 511, 502, 483, 539, 508, 520, 507, 507, 512, 502, 447, 506, 472, 528, 501, 536, 500, 514, 460] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[562, 497, 509, 512, 529, 488, 429, 521, 538, 533, 549, 501, 507, 534, 475, 480, 519, 490, 498, 519, 503, 520, 550, 520, 497, 542, 539, 518, 552, 532, 519, 494, 503, 556, 549, 510, 517, 490, 538, 493, 536, 519, 509, 506, 504, 490, 508, 478, 496, 499, 482, 460, 515, 536, 526, 502, 533, 530, 513, 545, 483, 531, 477, 511, 506, 487, 517, 530, 493, 529, 512, 505, 486, 537, 524, 516, 506, 488, 509, 519, 511, 503, 550, 488, 528, 499, 536, 529, 464, 501, 514, 486, 518, 511, 502, 506, 510, 499, 494, 503, 511, 536, 528, 509, 507, 517, 522, 522, 495, 532, 496, 500, 509, 483, 485, 501, 519, 497, 551, 521, 515, 519, 541, 515, 540, 520, 540, 468] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[490, 498, 523, 522, 531, 504, 516, 489, 497, 526, 523, 511, 468, 499, 533, 508, 514, 530, 497, 492, 516, 514, 486, 482, 526, 510, 549, 500, 482, 529, 530, 517, 509, 484, 482, 522, 491, 544, 487, 527, 477, 512, 527, 502, 504, 503, 541, 527, 547, 527, 565, 505, 495, 470, 531, 505, 502, 520, 502, 465, 531, 507, 526, 512, 496, 497, 509, 538, 519, 555, 497, 503, 537, 469, 533, 504, 484, 471, 529, 511, 507, 484, 500, 522, 540, 499, 505, 543, 513, 553, 529, 501, 509, 567, 503, 516, 513, 546, 504, 496, 527, 519, 504, 535, 503, 528, 540, 493, 505, 514, 480, 531, 480, 531, 497, 512, 496, 511, 530, 527, 516, 508, 480, 500, 544, 541, 509, 512] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[499, 505, 502, 507, 487, 503, 487, 538, 526, 514, 550, 508, 534, 517, 478, 498, 536, 545, 531, 518, 505, 487, 556, 524, 504, 532, 516, 568, 514, 507, 469, 535, 519, 506, 495, 517, 446, 502, 513, 487, 545, 514, 526, 518, 501, 529, 514, 566, 513, 512, 546, 522, 526, 489, 530, 512, 525, 509, 491, 495, 463, 526, 526, 536, 496, 533, 483, 514, 540, 474, 482, 539, 481, 520, 532, 516, 498, 505, 495, 517, 525, 504, 522, 531, 520, 519, 482, 509, 536, 511, 531, 467, 504, 459, 511, 558, 532, 452, 516, 549, 502, 491, 504, 506, 525, 507, 551, 454, 490, 512, 511, 514, 534, 495, 504, 506, 522, 521, 507, 503, 476, 512, 504, 502, 568, 493, 512, 528] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[495, 481, 496, 459, 530, 500, 504, 514, 528, 509, 499, 494, 523, 511, 515, 555, 41, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 501, 474, 517, 590, 510, 497, 519, 494, 473, 538, 495, 503, 556, 508, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 506, 550, 540, 515, 498, 490, 475, 542, 494, 516, 507, 504, 511, 508, 536, 13, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 527, 498, 501, 501, 498, 541, 531, 510, 498, 550, 517, 493, 508, 539, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 532, 525, 498, 562, 489, 502, 489, 537, 471, 503, 519, 536, 509, 550, 529, 16, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 511, 564, 503, 555, 494, 500, 481, 510, 516, 489, 518, 547, 490, 443, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 519, 485, 524, 522, 522, 534, 493, 521, 487, 475, 528, 511, 524, 514, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 486, 514, 561, 498, 538, 504, 472, 536, 470, 523, 509, 510, 490, 503, 512, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8159, 8248, 8243, 8242, 8117, 8201, 8147] avg=8192.0000 max=8248 max_rank=2 min=8117 min_rank=5 imbalance_ratio=1.006836 max_load_reduction=57288 max_load_reduction_pct=0.874146 moved_to_echo=32707 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[554, 532, 500, 499, 488, 547, 525, 528, 556, 469, 512, 526, 486, 520, 469, 479, 37, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 453, 509, 559, 523, 573, 507, 472, 517, 523, 477, 537, 480, 516, 521, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 558, 471, 529, 497, 500, 464, 508, 467, 497, 514, 512, 516, 512, 499, 455, 13, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 507, 524, 538, 544, 514, 488, 491, 544, 478, 500, 487, 511, 521, 495, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 501, 512, 502, 483, 530, 499, 497, 517, 514, 499, 529, 535, 535, 510, 545, 16, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 535, 521, 528, 496, 507, 505, 486, 478, 493, 562, 516, 528, 497, 538, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 508, 517, 588, 502, 512, 496, 516, 526, 499, 466, 513, 536, 515, 523, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 495, 498, 517, 553, 488, 485, 496, 522, 516, 547, 504, 492, 503, 528, 564, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8253, 8195, 8046, 8191, 8233, 8153, 8246, 8219] avg=8192.0000 max=8253 max_rank=0 min=8046 min_rank=2 imbalance_ratio=1.007446 max_load_reduction=57283 max_load_reduction_pct=0.874069 moved_to_echo=32851 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[495, 481, 496, 459, 530, 500, 504, 514, 528, 509, 499, 494, 523, 511, 515, 555, 484, 542, 474, 517, 590, 510, 497, 519, 494, 473, 538, 495, 503, 556, 508, 500, 533, 506, 550, 540, 515, 498, 490, 475, 542, 494, 516, 507, 504, 511, 508, 536, 544, 527, 509, 501, 501, 498, 541, 531, 510, 498, 550, 517, 493, 508, 539, 487, 469, 532, 525, 498, 562, 489, 502, 489, 537, 471, 503, 519, 536, 509, 550, 529, 521, 511, 564, 503, 555, 494, 500, 481, 510, 516, 489, 518, 547, 490, 443, 510, 537, 519, 485, 524, 522, 522, 534, 493, 521, 487, 504, 528, 511, 524, 514, 505, 516, 486, 514, 561, 498, 538, 504, 472, 536, 470, 523, 509, 510, 490, 503, 512] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[490, 498, 523, 522, 531, 504, 516, 489, 497, 526, 523, 511, 468, 499, 533, 508, 40, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 490, 497, 492, 516, 514, 486, 482, 526, 510, 549, 500, 482, 529, 530, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 484, 482, 522, 491, 544, 487, 527, 477, 512, 527, 502, 504, 503, 541, 527, 14, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 527, 553, 505, 495, 470, 531, 505, 502, 520, 502, 465, 531, 507, 526, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 497, 509, 538, 519, 555, 497, 503, 537, 469, 533, 504, 484, 471, 529, 511, 18, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 484, 500, 522, 540, 499, 505, 543, 513, 553, 529, 501, 509, 567, 503, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 546, 504, 496, 527, 519, 504, 535, 503, 528, 508, 493, 505, 514, 480, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 531, 497, 512, 496, 511, 530, 527, 516, 508, 480, 500, 544, 541, 509, 512, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8134, 8163, 8198, 8177, 8255, 8206, 8199] avg=8192.0000 max=8255 max_rank=5 min=8134 min_rank=1 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32837 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[506, 523, 505, 506, 510, 509, 495, 483, 466, 506, 522, 569, 511, 522, 508, 489, 41, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 499, 508, 503, 485, 497, 520, 510, 497, 496, 529, 474, 533, 490, 507, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 514, 545, 551, 500, 507, 494, 508, 529, 529, 524, 507, 523, 474, 480, 475, 14, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 501, 531, 534, 514, 537, 495, 590, 500, 513, 491, 520, 516, 521, 508, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 504, 482, 509, 489, 508, 503, 524, 526, 489, 504, 534, 515, 516, 533, 525, 18, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 537, 502, 490, 521, 535, 516, 484, 520, 536, 513, 512, 522, 499, 513, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 481, 525, 510, 533, 486, 483, 501, 546, 518, 514, 484, 521, 550, 520, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 505, 511, 515, 508, 532, 497, 522, 511, 462, 520, 507, 543, 490, 525, 518, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8075, 8184, 8281, 8203, 8201, 8212, 8183] avg=8192.0000 max=8281 max_rank=3 min=8075 min_rank=1 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32799 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[543, 501, 504, 483, 511, 524, 499, 483, 498, 472, 501, 502, 569, 531, 521, 515, 44, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 482, 540, 478, 510, 515, 506, 501, 547, 473, 488, 495, 502, 496, 561, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 524, 458, 552, 514, 535, 513, 536, 483, 525, 483, 538, 470, 519, 527, 492, 19, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 487, 494, 512, 506, 495, 520, 515, 502, 495, 534, 482, 531, 499, 512, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 471, 554, 554, 514, 514, 517, 518, 507, 510, 508, 531, 504, 541, 527, 502, 22, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 498, 488, 515, 537, 540, 526, 527, 511, 545, 497, 513, 503, 538, 525, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 492, 494, 509, 500, 525, 579, 508, 526, 550, 526, 512, 536, 511, 502, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 508, 520, 507, 507, 512, 502, 447, 506, 472, 528, 501, 536, 500, 514, 460, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 8137, 8203, 8083, 8303, 8240, 8272, 8068] avg=8192.0000 max=8303 max_rank=4 min=8068 min_rank=7 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32883 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[516, 505, 500, 491, 510, 448, 476, 480, 453, 516, 568, 517, 518, 481, 489, 489, 40, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 492, 547, 511, 526, 509, 533, 491, 555, 502, 492, 527, 525, 523, 530, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 481, 508, 535, 566, 476, 495, 534, 539, 508, 501, 500, 581, 502, 521, 482, 14, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 513, 485, 515, 516, 515, 500, 493, 525, 518, 507, 538, 496, 514, 515, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 489, 491, 486, 505, 480, 513, 499, 512, 518, 493, 521, 467, 525, 516, 504, 17, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 533, 518, 526, 510, 510, 525, 483, 508, 492, 523, 544, 484, 545, 565, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 531, 508, 501, 527, 525, 489, 552, 516, 492, 495, 532, 503, 496, 472, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 495, 563, 510, 516, 495, 528, 518, 516, 540, 487, 528, 523, 501, 512, 558, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8025, 8255, 8240, 8202, 8041, 8309, 8164, 8300] avg=8192.0000 max=8309 max_rank=5 min=8025 min_rank=0 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32814 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[562, 497, 509, 512, 529, 488, 429, 521, 538, 533, 549, 501, 507, 534, 475, 480, 37, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 453, 498, 519, 503, 520, 550, 520, 497, 542, 539, 518, 552, 532, 519, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 556, 549, 510, 517, 490, 538, 493, 536, 519, 509, 506, 504, 490, 508, 478, 13, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 499, 472, 460, 515, 536, 526, 502, 533, 530, 513, 545, 483, 531, 477, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 487, 517, 530, 493, 529, 512, 505, 486, 537, 524, 516, 506, 488, 509, 519, 17, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 503, 550, 488, 528, 499, 536, 529, 464, 501, 514, 486, 518, 511, 502, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 499, 494, 503, 511, 536, 528, 509, 507, 517, 492, 522, 495, 532, 496, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 483, 485, 501, 519, 497, 551, 521, 515, 519, 541, 515, 540, 520, 540, 468, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 8275, 8229, 8129, 8187, 8111, 8151, 8228] avg=8192.0000 max=8275 max_rank=1 min=8111 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32677 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[490, 498, 523, 522, 531, 504, 516, 489, 497, 526, 523, 511, 468, 499, 533, 508, 40, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 490, 497, 492, 516, 514, 486, 482, 526, 510, 549, 500, 482, 529, 530, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 484, 482, 522, 491, 544, 487, 527, 477, 512, 527, 502, 504, 503, 541, 527, 14, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 527, 553, 505, 495, 470, 531, 505, 502, 520, 502, 465, 531, 507, 526, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 497, 509, 538, 519, 555, 497, 503, 537, 469, 533, 504, 484, 471, 529, 511, 18, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 484, 500, 522, 540, 499, 505, 543, 513, 553, 529, 501, 509, 567, 503, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 546, 504, 496, 527, 519, 504, 535, 503, 528, 508, 493, 505, 514, 480, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 531, 497, 512, 496, 511, 530, 527, 516, 508, 480, 500, 544, 541, 509, 512, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8134, 8163, 8198, 8177, 8255, 8206, 8199] avg=8192.0000 max=8255 max_rank=5 min=8134 min_rank=1 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32837 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[516, 505, 500, 491, 510, 448, 476, 480, 453, 516, 568, 517, 518, 481, 489, 489, 40, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 492, 547, 511, 526, 509, 533, 491, 555, 502, 492, 527, 525, 523, 530, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 481, 508, 535, 566, 476, 495, 534, 539, 508, 501, 500, 581, 502, 521, 482, 14, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 513, 485, 515, 516, 515, 500, 493, 525, 518, 507, 538, 496, 514, 515, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 489, 491, 486, 505, 480, 513, 499, 512, 518, 493, 521, 467, 525, 516, 504, 17, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 533, 518, 526, 510, 510, 525, 483, 508, 492, 523, 544, 484, 545, 565, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 531, 508, 501, 527, 525, 489, 552, 516, 492, 495, 532, 503, 496, 472, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 495, 563, 510, 516, 495, 528, 518, 516, 540, 487, 528, 523, 501, 512, 558, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8025, 8255, 8240, 8202, 8041, 8309, 8164, 8300] avg=8192.0000 max=8309 max_rank=5 min=8025 min_rank=0 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32814 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[499, 505, 502, 507, 487, 503, 487, 538, 526, 514, 550, 508, 534, 517, 478, 498, 41, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 504, 531, 518, 505, 487, 556, 524, 504, 532, 516, 568, 514, 507, 469, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 506, 495, 517, 446, 502, 513, 487, 545, 514, 526, 518, 501, 529, 514, 566, 14, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 512, 534, 522, 526, 489, 530, 512, 525, 509, 491, 495, 463, 526, 526, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 533, 483, 514, 540, 474, 482, 539, 481, 520, 532, 516, 498, 505, 495, 517, 18, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 504, 522, 531, 520, 519, 482, 509, 536, 511, 531, 467, 504, 459, 511, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 452, 516, 549, 502, 491, 504, 506, 525, 507, 519, 454, 490, 512, 511, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 495, 504, 506, 522, 521, 507, 503, 476, 512, 504, 502, 568, 493, 512, 528, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8306, 8223, 8209, 8150, 8150, 8084, 8192] avg=8192.0000 max=8306 max_rank=1 min=8084 min_rank=6 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32576 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[543, 501, 504, 483, 511, 524, 499, 483, 498, 472, 501, 502, 569, 531, 521, 515, 44, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 482, 540, 478, 510, 515, 506, 501, 547, 473, 488, 495, 502, 496, 561, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 524, 458, 552, 514, 535, 513, 536, 483, 525, 483, 538, 470, 519, 527, 492, 19, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 487, 494, 512, 506, 495, 520, 515, 502, 495, 534, 482, 531, 499, 512, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 471, 554, 554, 514, 514, 517, 518, 507, 510, 508, 531, 504, 541, 527, 502, 22, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 498, 488, 515, 537, 540, 526, 527, 511, 545, 497, 513, 503, 538, 525, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 492, 494, 509, 500, 525, 579, 508, 526, 550, 526, 512, 536, 511, 502, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 508, 520, 507, 507, 512, 502, 447, 506, 472, 528, 501, 536, 500, 514, 460, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 8137, 8203, 8083, 8303, 8240, 8272, 8068] avg=8192.0000 max=8303 max_rank=4 min=8068 min_rank=7 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32883 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[554, 532, 500, 499, 488, 547, 525, 528, 556, 469, 512, 526, 486, 520, 469, 479, 37, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 453, 509, 559, 523, 573, 507, 472, 517, 523, 477, 537, 480, 516, 521, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 558, 471, 529, 497, 500, 464, 508, 467, 497, 514, 512, 516, 512, 499, 455, 13, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 507, 524, 538, 544, 514, 488, 491, 544, 478, 500, 487, 511, 521, 495, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 501, 512, 502, 483, 530, 499, 497, 517, 514, 499, 529, 535, 535, 510, 545, 16, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 535, 521, 528, 496, 507, 505, 486, 478, 493, 562, 516, 528, 497, 538, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 508, 517, 588, 502, 512, 496, 516, 526, 499, 466, 513, 536, 515, 523, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 495, 498, 517, 553, 488, 485, 496, 522, 516, 547, 504, 492, 503, 528, 564, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8253, 8195, 8046, 8191, 8233, 8153, 8246, 8219] avg=8192.0000 max=8253 max_rank=0 min=8046 min_rank=2 imbalance_ratio=1.007446 max_load_reduction=57283 max_load_reduction_pct=0.874069 moved_to_echo=32851 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[495, 481, 496, 459, 530, 500, 504, 514, 528, 509, 499, 494, 523, 511, 515, 555, 41, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 501, 474, 517, 590, 510, 497, 519, 494, 473, 538, 495, 503, 556, 508, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 506, 550, 540, 515, 498, 490, 475, 542, 494, 516, 507, 504, 511, 508, 536, 13, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 527, 498, 501, 501, 498, 541, 531, 510, 498, 550, 517, 493, 508, 539, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 532, 525, 498, 562, 489, 502, 489, 537, 471, 503, 519, 536, 509, 550, 529, 16, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 511, 564, 503, 555, 494, 500, 481, 510, 516, 489, 518, 547, 490, 443, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 519, 485, 524, 522, 522, 534, 493, 521, 487, 475, 528, 511, 524, 514, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 486, 514, 561, 498, 538, 504, 472, 536, 470, 523, 509, 510, 490, 503, 512, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8159, 8248, 8243, 8242, 8117, 8201, 8147] avg=8192.0000 max=8248 max_rank=2 min=8117 min_rank=5 imbalance_ratio=1.006836 max_load_reduction=57288 max_load_reduction_pct=0.874146 moved_to_echo=32707 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 3968, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 384, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 3968, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4096, 4352, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 3968, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4352, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4096, 4352, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[515, 506, 530, 524, 511, 526, 507, 527, 524, 525, 516, 491, 527, 530, 502, 556, 520, 555, 491, 515, 519, 538, 473, 487, 539, 494, 510, 514, 505, 525, 557, 492, 451, 540, 466, 549, 510, 485, 528, 522, 492, 511, 527, 509, 524, 545, 506, 555, 491, 511, 504, 501, 499, 499, 485, 496, 507, 502, 499, 544, 544, 530, 499, 500, 486, 483, 495, 521, 533, 492, 487, 516, 516, 505, 497, 501, 536, 485, 477, 547, 508, 507, 528, 528, 510, 524, 500, 507, 507, 531, 508, 473, 482, 513, 472, 516, 516, 519, 528, 502, 530, 523, 524, 537, 510, 513, 524, 544, 505, 544, 507, 482, 510, 499, 509, 531, 517, 507, 486, 510, 510, 498, 459, 520, 553, 518, 516, 512] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[536, 556, 457, 499, 573, 504, 522, 500, 523, 502, 509, 509, 490, 518, 565, 526, 510, 508, 484, 474, 506, 534, 494, 536, 487, 536, 551, 484, 555, 524, 530, 528, 485, 519, 519, 510, 530, 496, 552, 520, 510, 527, 543, 504, 482, 520, 533, 507, 507, 529, 512, 516, 547, 493, 508, 521, 527, 500, 489, 508, 513, 505, 513, 495, 505, 514, 512, 538, 536, 529, 509, 487, 525, 532, 534, 491, 521, 526, 520, 522, 508, 490, 494, 481, 508, 497, 467, 516, 511, 521, 513, 473, 466, 504, 538, 516, 531, 505, 525, 526, 520, 515, 541, 501, 478, 487, 514, 506, 457, 497, 507, 528, 509, 481, 492, 515, 576, 497, 516, 513, 511, 533, 497, 510, 499, 494, 509, 472] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[517, 452, 523, 537, 498, 478, 501, 484, 483, 513, 506, 508, 493, 499, 532, 517, 526, 504, 535, 486, 503, 506, 527, 576, 502, 546, 511, 493, 484, 529, 491, 511, 499, 535, 516, 513, 545, 514, 475, 509, 552, 484, 558, 462, 503, 499, 519, 540, 506, 511, 510, 497, 517, 520, 506, 513, 503, 517, 505, 500, 529, 502, 493, 496, 510, 505, 555, 515, 500, 520, 537, 508, 526, 516, 534, 487, 504, 501, 494, 542, 525, 518, 492, 497, 540, 544, 533, 479, 502, 474, 535, 545, 503, 525, 506, 526, 514, 481, 511, 516, 500, 492, 502, 540, 530, 512, 503, 536, 501, 497, 530, 504, 522, 528, 500, 465, 493, 530, 547, 507, 514, 515, 523, 508, 531, 500, 536, 531] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[519, 517, 492, 521, 562, 515, 507, 518, 476, 522, 525, 534, 508, 509, 537, 487, 562, 543, 512, 530, 499, 519, 513, 496, 502, 532, 493, 509, 510, 519, 513, 522, 509, 536, 557, 474, 504, 516, 537, 503, 467, 502, 472, 515, 532, 557, 520, 492, 509, 473, 495, 457, 516, 488, 537, 554, 529, 516, 536, 518, 517, 518, 509, 521, 497, 519, 467, 519, 552, 504, 490, 491, 496, 501, 500, 534, 500, 527, 468, 484, 510, 521, 526, 530, 529, 537, 507, 521, 499, 524, 560, 573, 503, 483, 489, 533, 501, 528, 520, 514, 516, 491, 503, 489, 506, 500, 524, 552, 498, 502, 533, 498, 515, 509, 513, 478, 536, 525, 521, 456, 508, 550, 502, 481, 496, 491, 507, 470] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[508, 509, 531, 480, 524, 534, 496, 519, 511, 531, 499, 477, 509, 522, 494, 524, 510, 514, 531, 533, 505, 530, 569, 551, 493, 516, 530, 522, 502, 480, 476, 538, 517, 498, 503, 475, 492, 558, 494, 574, 473, 491, 533, 510, 550, 508, 527, 480, 502, 512, 515, 534, 506, 497, 494, 558, 518, 515, 522, 524, 505, 505, 531, 514, 552, 527, 504, 492, 500, 486, 522, 490, 504, 499, 510, 519, 493, 507, 534, 519, 495, 507, 520, 511, 548, 493, 503, 496, 492, 529, 477, 515, 493, 528, 492, 504, 541, 495, 522, 521, 484, 507, 517, 520, 533, 506, 537, 523, 521, 502, 542, 502, 543, 527, 517, 504, 486, 520, 481, 472, 509, 492, 494, 454, 520, 525, 526, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[528, 506, 578, 548, 509, 501, 528, 527, 543, 476, 574, 484, 519, 471, 509, 552, 554, 516, 509, 496, 477, 527, 486, 475, 520, 523, 535, 528, 507, 484, 542, 493, 523, 524, 533, 472, 516, 505, 498, 493, 480, 507, 495, 514, 505, 496, 494, 493, 496, 513, 504, 524, 537, 528, 506, 532, 500, 492, 462, 551, 529, 538, 539, 477, 528, 490, 456, 487, 558, 517, 492, 535, 530, 530, 480, 523, 507, 548, 482, 487, 501, 508, 477, 510, 499, 491, 515, 518, 495, 501, 496, 470, 533, 453, 512, 508, 519, 518, 535, 528, 515, 493, 510, 535, 527, 519, 510, 509, 532, 524, 531, 539, 519, 514, 547, 510, 524, 483, 526, 542, 527, 480, 544, 501, 517, 520, 501, 499] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[536, 513, 525, 522, 513, 455, 517, 516, 468, 505, 509, 523, 513, 509, 494, 503, 525, 472, 559, 517, 518, 490, 520, 495, 497, 494, 506, 500, 522, 515, 542, 500, 511, 527, 535, 511, 507, 485, 485, 557, 518, 515, 489, 485, 525, 482, 513, 523, 536, 492, 519, 502, 542, 530, 495, 489, 524, 533, 508, 510, 497, 544, 512, 502, 490, 550, 504, 505, 531, 554, 488, 533, 502, 480, 523, 546, 527, 533, 542, 484, 517, 539, 532, 526, 490, 535, 470, 534, 525, 485, 503, 488, 498, 536, 510, 555, 531, 506, 511, 497, 515, 505, 523, 532, 507, 503, 537, 514, 534, 551, 463, 490, 523, 497, 508, 463, 535, 546, 477, 517, 539, 493, 483, 529, 447, 521, 508, 500] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[517, 452, 523, 537, 498, 478, 501, 484, 483, 513, 506, 508, 493, 499, 532, 517, 526, 504, 535, 486, 503, 506, 527, 576, 502, 546, 511, 493, 484, 529, 491, 511, 499, 535, 516, 513, 545, 514, 475, 509, 552, 484, 558, 462, 503, 499, 519, 540, 506, 511, 510, 497, 517, 520, 506, 513, 503, 517, 505, 500, 529, 502, 493, 496, 510, 505, 555, 515, 500, 520, 537, 508, 526, 516, 534, 487, 504, 501, 494, 542, 525, 518, 492, 497, 540, 544, 533, 479, 502, 474, 535, 545, 503, 525, 506, 526, 514, 481, 511, 516, 500, 492, 502, 540, 530, 512, 503, 536, 501, 497, 530, 504, 522, 528, 500, 465, 493, 530, 547, 507, 514, 515, 523, 508, 531, 500, 536, 531] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[536, 556, 457, 499, 573, 504, 522, 500, 523, 502, 509, 509, 490, 518, 565, 526, 510, 508, 484, 474, 506, 534, 494, 536, 487, 536, 551, 484, 555, 524, 530, 528, 485, 519, 519, 510, 530, 496, 552, 520, 510, 527, 543, 504, 482, 520, 533, 507, 507, 529, 512, 516, 547, 493, 508, 521, 527, 500, 489, 508, 513, 505, 513, 495, 505, 514, 512, 538, 536, 529, 509, 487, 525, 532, 534, 491, 521, 526, 520, 522, 508, 490, 494, 481, 508, 497, 467, 516, 511, 521, 513, 473, 466, 504, 538, 516, 531, 505, 525, 526, 520, 515, 541, 501, 478, 487, 514, 506, 457, 497, 507, 528, 509, 481, 492, 515, 576, 497, 516, 513, 511, 533, 497, 510, 499, 494, 509, 472] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[494, 499, 499, 533, 559, 514, 537, 471, 555, 513, 498, 526, 538, 514, 478, 527, 495, 498, 506, 507, 513, 504, 502, 509, 528, 517, 477, 518, 519, 540, 480, 541, 490, 515, 490, 512, 512, 495, 535, 498, 545, 447, 502, 489, 527, 533, 501, 529, 472, 476, 573, 542, 546, 520, 540, 535, 475, 562, 472, 528, 509, 509, 499, 533, 550, 539, 519, 523, 504, 502, 525, 527, 502, 489, 469, 447, 509, 501, 508, 530, 519, 566, 538, 514, 483, 474, 539, 511, 538, 537, 497, 531, 536, 554, 505, 508, 464, 485, 493, 477, 527, 485, 526, 480, 517, 529, 508, 520, 518, 538, 507, 498, 508, 516, 501, 502, 484, 520, 540, 547, 538, 464, 517, 501, 449, 524, 527, 512] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[528, 506, 578, 548, 509, 501, 528, 527, 543, 476, 574, 484, 519, 471, 509, 552, 554, 516, 509, 496, 477, 527, 486, 475, 520, 523, 535, 528, 507, 484, 542, 493, 523, 524, 533, 472, 516, 505, 498, 493, 480, 507, 495, 514, 505, 496, 494, 493, 496, 513, 504, 524, 537, 528, 506, 532, 500, 492, 462, 551, 529, 538, 539, 477, 528, 490, 456, 487, 558, 517, 492, 535, 530, 530, 480, 523, 507, 548, 482, 487, 501, 508, 477, 510, 499, 491, 515, 518, 495, 501, 496, 470, 533, 453, 512, 508, 519, 518, 535, 528, 515, 493, 510, 535, 527, 519, 510, 509, 532, 524, 531, 539, 519, 514, 547, 510, 524, 483, 526, 542, 527, 480, 544, 501, 517, 520, 501, 499] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[519, 517, 492, 521, 562, 515, 507, 518, 476, 522, 525, 534, 508, 509, 537, 487, 562, 543, 512, 530, 499, 519, 513, 496, 502, 532, 493, 509, 510, 519, 513, 522, 509, 536, 557, 474, 504, 516, 537, 503, 467, 502, 472, 515, 532, 557, 520, 492, 509, 473, 495, 457, 516, 488, 537, 554, 529, 516, 536, 518, 517, 518, 509, 521, 497, 519, 467, 519, 552, 504, 490, 491, 496, 501, 500, 534, 500, 527, 468, 484, 510, 521, 526, 530, 529, 537, 507, 521, 499, 524, 560, 573, 503, 483, 489, 533, 501, 528, 520, 514, 516, 491, 503, 489, 506, 500, 524, 552, 498, 502, 533, 498, 515, 509, 513, 478, 536, 525, 521, 456, 508, 550, 502, 481, 496, 491, 507, 470] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[515, 506, 530, 524, 511, 526, 507, 527, 524, 525, 516, 491, 527, 530, 502, 556, 520, 555, 491, 515, 519, 538, 473, 487, 539, 494, 510, 514, 505, 525, 557, 492, 451, 540, 466, 549, 510, 485, 528, 522, 492, 511, 527, 509, 524, 545, 506, 555, 491, 511, 504, 501, 499, 499, 485, 496, 507, 502, 499, 544, 544, 530, 499, 500, 486, 483, 495, 521, 533, 492, 487, 516, 516, 505, 497, 501, 536, 485, 477, 547, 508, 507, 528, 528, 510, 524, 500, 507, 507, 531, 508, 473, 482, 513, 472, 516, 516, 519, 528, 502, 530, 523, 524, 537, 510, 513, 524, 544, 505, 544, 507, 482, 510, 499, 509, 531, 517, 507, 486, 510, 510, 498, 459, 520, 553, 518, 516, 512] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[536, 513, 525, 522, 482, 455, 517, 516, 468, 505, 509, 523, 513, 509, 494, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 472, 559, 517, 518, 490, 520, 495, 497, 494, 506, 500, 522, 515, 542, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 527, 535, 511, 507, 485, 485, 557, 518, 515, 489, 485, 525, 482, 513, 523, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 492, 519, 502, 532, 530, 495, 489, 524, 533, 508, 510, 497, 544, 512, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 550, 504, 505, 531, 554, 488, 533, 502, 480, 523, 546, 527, 533, 542, 484, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 539, 532, 526, 490, 535, 470, 534, 525, 485, 503, 488, 498, 536, 510, 555, 17, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 506, 511, 497, 515, 505, 523, 532, 507, 503, 537, 495, 534, 551, 463, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 497, 508, 463, 535, 546, 477, 517, 539, 493, 483, 529, 447, 521, 508, 500, 31, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8090, 8143, 8183, 8225, 8305, 8261, 8200, 8129] avg=8192.0000 max=8305 max_rank=4 min=8090 min_rank=0 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32895 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[508, 509, 531, 480, 488, 534, 496, 519, 511, 531, 499, 477, 509, 522, 494, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 514, 531, 533, 505, 530, 569, 551, 493, 516, 530, 522, 502, 480, 476, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 498, 503, 475, 492, 558, 494, 574, 473, 491, 533, 510, 550, 508, 527, 480, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 512, 515, 534, 493, 497, 494, 558, 518, 515, 522, 524, 505, 505, 531, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 527, 504, 492, 500, 486, 522, 490, 504, 499, 510, 519, 493, 507, 534, 519, 13, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 507, 520, 511, 548, 493, 503, 496, 492, 529, 477, 515, 493, 528, 492, 504, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 495, 522, 521, 484, 507, 517, 520, 533, 506, 537, 495, 521, 502, 542, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 527, 517, 504, 486, 520, 481, 472, 509, 492, 494, 454, 520, 525, 526, 529, 36, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8132, 8264, 8201, 8239, 8179, 8124, 8245, 8152] avg=8192.0000 max=8264 max_rank=1 min=8124 min_rank=5 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32700 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[517, 452, 523, 537, 468, 478, 501, 484, 483, 513, 506, 508, 493, 499, 532, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 504, 535, 486, 503, 506, 527, 576, 502, 546, 511, 493, 484, 529, 491, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 535, 516, 513, 545, 514, 475, 509, 552, 484, 558, 462, 503, 499, 519, 540, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 511, 510, 497, 507, 520, 506, 513, 503, 517, 505, 500, 529, 502, 493, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 505, 555, 515, 500, 520, 537, 508, 526, 516, 534, 487, 504, 501, 494, 542, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 518, 492, 497, 540, 544, 533, 479, 502, 474, 535, 545, 503, 525, 506, 526, 17, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 481, 511, 516, 500, 492, 502, 540, 530, 512, 503, 517, 501, 497, 530, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 528, 500, 465, 493, 530, 547, 507, 514, 515, 523, 508, 531, 500, 536, 531, 30, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8011, 8201, 8238, 8115, 8267, 8262, 8150, 8292] avg=8192.0000 max=8292 max_rank=7 min=8011 min_rank=0 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32971 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[494, 499, 499, 533, 526, 514, 537, 471, 555, 513, 498, 526, 538, 514, 478, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 498, 506, 507, 513, 504, 502, 509, 528, 517, 477, 518, 519, 540, 480, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 515, 490, 512, 512, 495, 535, 498, 545, 447, 502, 489, 527, 533, 501, 529, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 476, 573, 542, 536, 520, 540, 535, 475, 562, 472, 528, 509, 509, 499, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 539, 519, 523, 504, 502, 525, 527, 502, 489, 469, 447, 509, 501, 508, 530, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 566, 538, 514, 483, 474, 539, 511, 538, 537, 497, 531, 536, 554, 505, 508, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 464, 485, 493, 477, 527, 485, 526, 480, 517, 529, 508, 501, 518, 538, 507, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 516, 501, 502, 484, 520, 540, 547, 538, 464, 517, 501, 449, 524, 527, 512, 33, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8126, 8135, 8281, 8157, 8367, 8053, 8195] avg=8192.0000 max=8367 max_rank=5 min=8053 min_rank=6 imbalance_ratio=1.021362 max_load_reduction=57169 max_load_reduction_pct=0.872330 moved_to_echo=32772 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[536, 556, 457, 499, 539, 504, 522, 500, 523, 502, 509, 509, 490, 518, 565, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 508, 484, 474, 506, 534, 494, 536, 487, 536, 551, 484, 555, 524, 530, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 519, 519, 510, 530, 496, 552, 520, 510, 527, 543, 504, 482, 520, 533, 507, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 529, 512, 516, 537, 493, 508, 521, 527, 500, 489, 508, 513, 505, 513, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 514, 512, 538, 536, 529, 509, 487, 525, 532, 534, 491, 521, 526, 520, 522, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 490, 494, 481, 508, 497, 467, 516, 511, 521, 513, 473, 466, 504, 538, 516, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 505, 525, 526, 520, 515, 541, 501, 478, 487, 514, 488, 457, 497, 507, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 481, 492, 515, 576, 497, 516, 513, 511, 533, 497, 510, 499, 494, 509, 472, 34, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8255, 8213, 8271, 8173, 8314, 8020, 8120, 8170] avg=8192.0000 max=8314 max_rank=4 min=8020 min_rank=5 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32624 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[508, 509, 531, 480, 488, 534, 496, 519, 511, 531, 499, 477, 509, 522, 494, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 514, 531, 533, 505, 530, 569, 551, 493, 516, 530, 522, 502, 480, 476, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 498, 503, 475, 492, 558, 494, 574, 473, 491, 533, 510, 550, 508, 527, 480, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 512, 515, 534, 493, 497, 494, 558, 518, 515, 522, 524, 505, 505, 531, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 527, 504, 492, 500, 486, 522, 490, 504, 499, 510, 519, 493, 507, 534, 519, 13, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 507, 520, 511, 548, 493, 503, 496, 492, 529, 477, 515, 493, 528, 492, 504, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 495, 522, 521, 484, 507, 517, 520, 533, 506, 537, 495, 521, 502, 542, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 527, 517, 504, 486, 520, 481, 472, 509, 492, 494, 454, 520, 525, 526, 529, 36, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8132, 8264, 8201, 8239, 8179, 8124, 8245, 8152] avg=8192.0000 max=8264 max_rank=1 min=8124 min_rank=5 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32700 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[515, 506, 530, 524, 481, 526, 507, 527, 524, 525, 516, 491, 527, 530, 502, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 555, 491, 515, 519, 538, 473, 487, 539, 494, 510, 514, 505, 525, 557, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 451, 540, 466, 549, 510, 485, 528, 522, 492, 511, 527, 509, 524, 545, 506, 555, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 511, 504, 501, 490, 499, 485, 496, 507, 502, 499, 544, 544, 530, 499, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 483, 495, 521, 533, 492, 487, 516, 516, 505, 497, 501, 536, 485, 477, 547, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 507, 528, 528, 510, 524, 500, 507, 507, 531, 508, 473, 482, 513, 472, 516, 17, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 519, 528, 502, 530, 523, 524, 537, 510, 513, 524, 524, 505, 544, 507, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 499, 509, 531, 517, 507, 486, 510, 510, 498, 459, 520, 553, 518, 516, 512, 30, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8287, 8205, 8236, 8102, 8089, 8132, 8288, 8197] avg=8192.0000 max=8288 max_rank=6 min=8089 min_rank=4 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32706 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[528, 506, 578, 548, 479, 501, 528, 527, 543, 476, 574, 484, 519, 471, 509, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 516, 509, 496, 477, 527, 486, 475, 520, 523, 535, 528, 507, 484, 542, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 524, 533, 472, 516, 505, 498, 493, 480, 507, 495, 514, 505, 496, 494, 493, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 513, 504, 524, 527, 528, 506, 532, 500, 492, 462, 551, 529, 538, 539, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 490, 456, 487, 558, 517, 492, 535, 530, 530, 480, 523, 507, 548, 482, 487, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 508, 477, 510, 499, 491, 515, 518, 495, 501, 496, 470, 533, 453, 512, 508, 18, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 518, 535, 528, 515, 493, 510, 535, 527, 519, 510, 490, 532, 524, 531, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 514, 547, 510, 524, 483, 526, 542, 527, 480, 544, 501, 517, 520, 501, 499, 30, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8323, 8141, 8063, 8218, 8163, 8006, 8325, 8297] avg=8192.0000 max=8325 max_rank=6 min=8006 min_rank=5 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32791 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[519, 517, 492, 521, 529, 515, 507, 518, 476, 522, 525, 534, 508, 509, 537, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 543, 512, 530, 499, 519, 513, 496, 502, 532, 493, 509, 510, 519, 513, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 536, 557, 474, 504, 516, 537, 503, 467, 502, 472, 515, 532, 557, 520, 492, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 473, 495, 457, 506, 488, 537, 554, 529, 516, 536, 518, 517, 518, 509, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 519, 467, 519, 552, 504, 490, 491, 496, 501, 500, 534, 500, 527, 468, 484, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 521, 526, 530, 529, 537, 507, 521, 499, 524, 560, 573, 503, 483, 489, 533, 18, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 528, 520, 514, 516, 491, 503, 489, 506, 500, 524, 532, 498, 502, 533, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 509, 513, 478, 536, 525, 521, 456, 508, 550, 502, 481, 496, 491, 507, 470, 33, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8243, 8209, 8183, 8062, 8364, 8155, 8104] avg=8192.0000 max=8364 max_rank=5 min=8062 min_rank=4 imbalance_ratio=1.020996 max_load_reduction=57172 max_load_reduction_pct=0.872375 moved_to_echo=32685 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[536, 556, 457, 499, 539, 504, 522, 500, 523, 502, 509, 509, 490, 518, 565, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 508, 484, 474, 506, 534, 494, 536, 487, 536, 551, 484, 555, 524, 530, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 519, 519, 510, 530, 496, 552, 520, 510, 527, 543, 504, 482, 520, 533, 507, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 529, 512, 516, 537, 493, 508, 521, 527, 500, 489, 508, 513, 505, 513, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 514, 512, 538, 536, 529, 509, 487, 525, 532, 534, 491, 521, 526, 520, 522, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 490, 494, 481, 508, 497, 467, 516, 511, 521, 513, 473, 466, 504, 538, 516, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 505, 525, 526, 520, 515, 541, 501, 478, 487, 514, 488, 457, 497, 507, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 481, 492, 515, 576, 497, 516, 513, 511, 533, 497, 510, 499, 494, 509, 472, 34, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8255, 8213, 8271, 8173, 8314, 8020, 8120, 8170] avg=8192.0000 max=8314 max_rank=4 min=8020 min_rank=5 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32624 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[528, 506, 578, 548, 479, 501, 528, 527, 543, 476, 574, 484, 519, 471, 509, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 516, 509, 496, 477, 527, 486, 475, 520, 523, 535, 528, 507, 484, 542, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 524, 533, 472, 516, 505, 498, 493, 480, 507, 495, 514, 505, 496, 494, 493, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 513, 504, 524, 527, 528, 506, 532, 500, 492, 462, 551, 529, 538, 539, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 490, 456, 487, 558, 517, 492, 535, 530, 530, 480, 523, 507, 548, 482, 487, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 508, 477, 510, 499, 491, 515, 518, 495, 501, 496, 470, 533, 453, 512, 508, 18, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 518, 535, 528, 515, 493, 510, 535, 527, 519, 510, 490, 532, 524, 531, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 514, 547, 510, 524, 483, 526, 542, 527, 480, 544, 501, 517, 520, 501, 499, 30, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8323, 8141, 8063, 8218, 8163, 8006, 8325, 8297] avg=8192.0000 max=8325 max_rank=6 min=8006 min_rank=5 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32791 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[536, 513, 525, 522, 482, 455, 517, 516, 468, 505, 509, 523, 513, 509, 494, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 472, 559, 517, 518, 490, 520, 495, 497, 494, 506, 500, 522, 515, 542, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 527, 535, 511, 507, 485, 485, 557, 518, 515, 489, 485, 525, 482, 513, 523, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 492, 519, 502, 532, 530, 495, 489, 524, 533, 508, 510, 497, 544, 512, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 550, 504, 505, 531, 554, 488, 533, 502, 480, 523, 546, 527, 533, 542, 484, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 539, 532, 526, 490, 535, 470, 534, 525, 485, 503, 488, 498, 536, 510, 555, 17, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 506, 511, 497, 515, 505, 523, 532, 507, 503, 537, 495, 534, 551, 463, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 497, 508, 463, 535, 546, 477, 517, 539, 493, 483, 529, 447, 521, 508, 500, 31, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8090, 8143, 8183, 8225, 8305, 8261, 8200, 8129] avg=8192.0000 max=8305 max_rank=4 min=8090 min_rank=0 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32895 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[494, 499, 499, 533, 526, 514, 537, 471, 555, 513, 498, 526, 538, 514, 478, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 498, 506, 507, 513, 504, 502, 509, 528, 517, 477, 518, 519, 540, 480, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 515, 490, 512, 512, 495, 535, 498, 545, 447, 502, 489, 527, 533, 501, 529, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 476, 573, 542, 536, 520, 540, 535, 475, 562, 472, 528, 509, 509, 499, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 539, 519, 523, 504, 502, 525, 527, 502, 489, 469, 447, 509, 501, 508, 530, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 566, 538, 514, 483, 474, 539, 511, 538, 537, 497, 531, 536, 554, 505, 508, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 464, 485, 493, 477, 527, 485, 526, 480, 517, 529, 508, 501, 518, 538, 507, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 516, 501, 502, 484, 520, 540, 547, 538, 464, 517, 501, 449, 524, 527, 512, 33, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8126, 8135, 8281, 8157, 8367, 8053, 8195] avg=8192.0000 max=8367 max_rank=5 min=8053 min_rank=6 imbalance_ratio=1.021362 max_load_reduction=57169 max_load_reduction_pct=0.872330 moved_to_echo=32772 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[3968, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 3968, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[3968, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 3968, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True - [2026-08-26 14:49:29] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 422.2 | throughput per GPU (TFLOP/s/GPU): 428.3 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.012798E+01 | loss scale: 1.0 | grad norm: 26.763 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[526, 490, 501, 510, 449, 527, 548, 526, 492, 505, 505, 512, 502, 522, 512, 519, 536, 479, 492, 485, 514, 493, 498, 505, 537, 555, 510, 506, 539, 543, 522, 556, 508, 539, 517, 507, 458, 538, 494, 520, 513, 539, 507, 505, 488, 505, 503, 520, 523, 484, 523, 516, 520, 530, 511, 500, 483, 514, 530, 548, 516, 500, 498, 501, 503, 496, 518, 491, 519, 517, 519, 471, 523, 509, 549, 524, 474, 490, 502, 463, 578, 525, 492, 535, 550, 511, 516, 544, 544, 542, 518, 494, 519, 514, 508, 509, 513, 519, 523, 536, 491, 509, 519, 493, 514, 530, 528, 499, 520, 535, 493, 533, 535, 520, 496, 513, 478, 542, 483, 523, 512, 486, 479, 479, 488, 493, 498, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[525, 533, 520, 519, 510, 543, 524, 527, 524, 548, 526, 498, 508, 512, 497, 510, 545, 522, 471, 519, 489, 515, 538, 505, 515, 504, 485, 543, 558, 499, 501, 505, 526, 491, 490, 507, 505, 542, 486, 477, 505, 523, 470, 489, 488, 510, 492, 504, 510, 525, 538, 529, 510, 488, 482, 552, 507, 492, 527, 502, 519, 534, 544, 551, 507, 495, 515, 518, 499, 518, 519, 491, 500, 501, 545, 504, 547, 472, 523, 521, 511, 496, 540, 514, 491, 494, 521, 500, 535, 508, 518, 522, 477, 505, 506, 477, 520, 503, 526, 534, 471, 535, 527, 491, 520, 531, 497, 489, 511, 508, 498, 505, 518, 495, 489, 490, 497, 487, 571, 558, 463, 517, 492, 516, 509, 517, 541, 567] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True + [2026-08-26 15:25:32] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 433.7 | throughput per GPU (TFLOP/s/GPU): 416.9 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.015221E+01 | loss scale: 1.0 | grad norm: 28.346 | number of skipped iterations: 0 | number of nan iterations: 0 | WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[492, 525, 527, 487, 481, 535, 527, 493, 526, 494, 465, 499, 520, 514, 527, 494, 480, 522, 486, 495, 542, 510, 486, 521, 470, 532, 518, 505, 513, 544, 548, 543, 487, 526, 502, 537, 507, 497, 515, 498, 515, 504, 489, 485, 510, 511, 528, 540, 497, 504, 500, 531, 517, 500, 495, 494, 523, 507, 509, 546, 479, 504, 528, 487, 515, 512, 511, 499, 512, 501, 454, 492, 544, 496, 537, 493, 515, 515, 539, 536, 540, 446, 519, 536, 528, 509, 485, 533, 540, 493, 515, 500, 478, 530, 539, 511, 527, 531, 501, 527, 497, 503, 548, 518, 546, 500, 485, 543, 461, 492, 466, 504, 507, 528, 529, 528, 545, 505, 518, 500, 514, 515, 573, 494, 505, 574, 540, 546] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[550, 543, 511, 517, 518, 505, 541, 465, 496, 546, 558, 532, 525, 516, 516, 515, 519, 523, 567, 492, 488, 534, 507, 512, 544, 456, 525, 541, 494, 509, 519, 531, 508, 506, 517, 524, 513, 483, 519, 484, 498, 539, 509, 496, 553, 490, 496, 515, 503, 507, 532, 548, 514, 467, 504, 494, 512, 514, 514, 517, 525, 532, 495, 562, 554, 496, 491, 514, 505, 505, 537, 499, 493, 517, 547, 518, 467, 508, 496, 512, 511, 509, 491, 495, 503, 519, 524, 524, 494, 534, 505, 486, 525, 492, 494, 546, 508, 474, 509, 532, 493, 490, 534, 489, 529, 467, 488, 498, 489, 515, 522, 533, 550, 533, 463, 495, 481, 542, 495, 511, 510, 544, 522, 470, 488, 521, 512, 513] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[529, 524, 520, 538, 526, 508, 528, 511, 555, 542, 492, 517, 513, 559, 505, 500, 533, 548, 486, 521, 535, 506, 541, 512, 526, 487, 558, 490, 525, 469, 535, 510, 487, 510, 473, 548, 498, 510, 504, 504, 500, 540, 518, 496, 500, 564, 499, 505, 512, 493, 504, 479, 475, 534, 529, 493, 527, 530, 556, 497, 521, 540, 541, 496, 470, 535, 542, 501, 498, 497, 500, 501, 485, 495, 541, 502, 515, 493, 533, 515, 517, 507, 476, 520, 557, 479, 502, 502, 472, 500, 522, 513, 502, 524, 510, 454, 546, 513, 500, 520, 523, 501, 487, 512, 503, 502, 495, 497, 482, 477, 482, 539, 533, 486, 508, 557, 537, 527, 533, 509, 557, 468, 499, 468, 540, 536, 511, 476] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[526, 445, 508, 504, 487, 511, 560, 518, 532, 520, 504, 547, 557, 527, 510, 519, 518, 486, 524, 475, 493, 530, 509, 511, 552, 493, 539, 542, 496, 506, 555, 542, 513, 537, 526, 495, 504, 519, 480, 516, 507, 515, 522, 524, 482, 511, 501, 548, 508, 525, 518, 493, 527, 530, 484, 494, 517, 496, 510, 528, 529, 498, 455, 516, 520, 519, 505, 507, 492, 549, 496, 572, 511, 507, 508, 499, 503, 541, 488, 480, 552, 524, 498, 539, 507, 480, 520, 495, 519, 512, 492, 488, 524, 486, 521, 513, 494, 507, 547, 508, 518, 510, 540, 523, 515, 506, 489, 524, 537, 497, 514, 482, 540, 520, 531, 533, 452, 524, 539, 502, 469, 493, 532, 484, 507, 500, 483, 475] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[525, 533, 520, 519, 510, 543, 524, 527, 524, 548, 526, 498, 508, 512, 497, 510, 545, 522, 471, 519, 489, 515, 538, 505, 515, 504, 485, 543, 558, 499, 501, 505, 526, 491, 490, 507, 505, 542, 486, 477, 505, 523, 470, 489, 488, 510, 492, 504, 510, 525, 538, 529, 510, 488, 482, 552, 507, 492, 527, 502, 519, 534, 544, 551, 507, 495, 515, 518, 499, 518, 519, 491, 500, 501, 545, 504, 547, 472, 523, 521, 511, 496, 540, 514, 491, 494, 521, 500, 535, 508, 518, 522, 477, 505, 506, 477, 520, 503, 526, 534, 471, 535, 527, 491, 520, 531, 497, 489, 511, 508, 498, 505, 518, 495, 489, 490, 497, 487, 571, 558, 463, 517, 492, 516, 509, 517, 541, 567] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[529, 524, 520, 538, 526, 508, 528, 511, 555, 542, 492, 517, 513, 559, 505, 500, 533, 548, 486, 521, 535, 506, 541, 512, 526, 487, 558, 490, 525, 469, 535, 510, 487, 510, 473, 548, 498, 510, 504, 504, 500, 540, 518, 496, 500, 564, 499, 505, 512, 493, 504, 479, 475, 534, 529, 493, 527, 530, 556, 497, 521, 540, 541, 496, 470, 535, 542, 501, 498, 497, 500, 501, 485, 495, 541, 502, 515, 493, 533, 515, 517, 507, 476, 520, 557, 479, 502, 502, 472, 500, 522, 513, 502, 524, 510, 454, 546, 513, 500, 520, 523, 501, 487, 512, 503, 502, 495, 497, 482, 477, 482, 539, 533, 486, 508, 557, 537, 527, 533, 509, 557, 468, 499, 468, 540, 536, 511, 476] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[525, 508, 505, 483, 526, 468, 566, 526, 525, 521, 551, 536, 525, 522, 508, 550, 473, 522, 506, 504, 491, 504, 505, 506, 507, 488, 523, 531, 485, 495, 533, 490, 513, 534, 521, 484, 505, 506, 489, 498, 468, 500, 474, 556, 547, 523, 533, 506, 452, 518, 508, 517, 521, 539, 538, 525, 482, 479, 504, 514, 530, 494, 522, 503, 538, 491, 520, 497, 530, 532, 508, 523, 528, 543, 525, 519, 549, 527, 531, 535, 481, 506, 533, 522, 505, 488, 505, 500, 491, 523, 542, 513, 500, 548, 461, 539, 509, 447, 543, 533, 499, 497, 529, 527, 478, 523, 514, 502, 498, 520, 518, 512, 511, 487, 480, 499, 492, 528, 518, 491, 518, 533, 504, 511, 511, 499, 533, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[526, 490, 501, 510, 449, 527, 548, 526, 492, 505, 505, 512, 502, 522, 512, 519, 536, 479, 492, 485, 514, 493, 498, 505, 537, 555, 510, 506, 539, 543, 522, 556, 508, 539, 517, 507, 458, 538, 494, 520, 513, 539, 507, 505, 488, 505, 503, 520, 523, 484, 523, 516, 520, 530, 511, 500, 483, 514, 530, 548, 516, 500, 498, 501, 503, 496, 518, 491, 519, 517, 519, 471, 523, 509, 549, 524, 474, 490, 502, 463, 578, 525, 492, 535, 550, 511, 516, 544, 544, 542, 518, 494, 519, 514, 508, 509, 513, 519, 523, 536, 491, 509, 519, 493, 514, 530, 528, 499, 520, 535, 493, 533, 535, 520, 496, 513, 478, 542, 483, 523, 512, 486, 479, 479, 488, 493, 498, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[553, 513, 559, 496, 489, 496, 512, 544, 494, 524, 518, 498, 517, 535, 504, 533, 518, 498, 540, 509, 501, 522, 514, 475, 501, 482, 534, 553, 512, 537, 493, 488, 548, 475, 532, 534, 482, 473, 516, 514, 509, 477, 517, 535, 513, 488, 499, 512, 521, 496, 482, 515, 480, 518, 527, 494, 543, 524, 507, 513, 510, 510, 487, 513, 504, 536, 504, 466, 495, 540, 553, 511, 498, 500, 538, 504, 490, 508, 533, 506, 511, 507, 505, 485, 499, 530, 504, 508, 542, 519, 524, 543, 540, 472, 520, 494, 514, 542, 540, 525, 515, 492, 538, 477, 561, 524, 534, 525, 481, 492, 505, 497, 521, 530, 487, 464, 497, 518, 514, 502, 541, 546, 498, 475, 502, 530, 497, 537] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[492, 525, 527, 487, 481, 535, 448, 493, 526, 494, 465, 499, 520, 514, 527, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 522, 486, 495, 542, 510, 486, 521, 470, 532, 518, 481, 513, 544, 548, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 526, 502, 537, 507, 497, 515, 498, 515, 504, 489, 485, 510, 511, 528, 540, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 504, 500, 531, 517, 500, 495, 494, 523, 507, 509, 546, 479, 504, 528, 487, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 512, 511, 499, 512, 501, 454, 492, 544, 496, 537, 493, 515, 515, 539, 536, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 446, 519, 536, 528, 509, 485, 533, 540, 493, 515, 500, 478, 530, 539, 511, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 531, 501, 527, 497, 503, 548, 518, 546, 500, 485, 543, 461, 492, 466, 504, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 528, 529, 528, 545, 505, 518, 500, 514, 515, 573, 494, 505, 574, 540, 546, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8027, 8191, 8203, 8124, 8178, 8211, 8172, 8430] avg=8192.0000 max=8430 max_rank=7 min=8027 min_rank=0 imbalance_ratio=1.029053 max_load_reduction=57106 max_load_reduction_pct=0.871368 moved_to_echo=32991 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[550, 543, 511, 517, 518, 505, 459, 465, 496, 546, 558, 532, 525, 516, 516, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 523, 567, 492, 488, 534, 507, 512, 544, 456, 525, 515, 494, 509, 519, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 506, 517, 524, 513, 483, 519, 484, 498, 539, 509, 496, 553, 490, 496, 515, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 507, 532, 548, 514, 467, 504, 494, 512, 514, 514, 517, 525, 532, 495, 562, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 496, 491, 514, 505, 505, 537, 499, 493, 517, 547, 518, 467, 508, 496, 512, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 509, 491, 495, 503, 519, 524, 524, 494, 534, 505, 486, 525, 492, 494, 546, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 474, 509, 532, 493, 490, 534, 489, 529, 467, 488, 498, 489, 515, 522, 533, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 533, 463, 495, 481, 542, 495, 511, 510, 544, 522, 470, 488, 521, 512, 513, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8272, 8235, 8204, 8243, 8166, 8162, 8094, 8160] avg=8192.0000 max=8272 max_rank=0 min=8094 min_rank=6 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32582 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[525, 533, 520, 519, 510, 543, 445, 527, 524, 548, 526, 498, 508, 512, 497, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 522, 471, 519, 489, 515, 538, 505, 515, 504, 485, 517, 558, 499, 501, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 491, 490, 507, 505, 542, 486, 477, 505, 523, 470, 489, 488, 510, 492, 504, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 525, 538, 529, 510, 488, 482, 552, 507, 492, 527, 502, 519, 534, 544, 551, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 495, 515, 518, 499, 518, 519, 491, 500, 501, 545, 504, 547, 472, 523, 521, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 496, 540, 514, 491, 494, 521, 500, 535, 508, 518, 522, 477, 505, 506, 477, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 503, 526, 534, 471, 535, 527, 491, 520, 531, 497, 489, 511, 508, 498, 505, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 495, 489, 490, 497, 487, 571, 558, 463, 517, 492, 516, 509, 517, 541, 567, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 8188, 8057, 8313, 8182, 8125, 8189, 8237] avg=8192.0000 max=8313 max_rank=3 min=8057 min_rank=2 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32733 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[526, 490, 501, 510, 449, 527, 451, 526, 492, 505, 505, 512, 502, 522, 512, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 479, 492, 485, 514, 493, 498, 505, 537, 555, 510, 468, 539, 543, 522, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 539, 517, 507, 458, 538, 494, 520, 513, 539, 507, 505, 488, 505, 503, 520, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 484, 523, 516, 520, 530, 511, 500, 483, 514, 530, 548, 516, 500, 498, 501, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 496, 518, 491, 519, 517, 519, 471, 523, 509, 549, 524, 474, 490, 502, 463, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 578, 525, 492, 535, 550, 511, 516, 544, 544, 542, 518, 494, 519, 514, 508, 509, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 519, 523, 536, 491, 509, 519, 493, 514, 530, 528, 499, 520, 535, 493, 533, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 520, 496, 513, 478, 542, 483, 523, 512, 486, 479, 479, 488, 493, 498, 515, 8, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8049, 8232, 8220, 8203, 8079, 8412, 8284, 8057] avg=8192.0000 max=8412 max_rank=5 min=8049 min_rank=0 imbalance_ratio=1.026855 max_load_reduction=57124 max_load_reduction_pct=0.871643 moved_to_echo=32832 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[492, 525, 527, 487, 481, 535, 448, 493, 526, 494, 465, 499, 520, 514, 527, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 522, 486, 495, 542, 510, 486, 521, 470, 532, 518, 481, 513, 544, 548, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 526, 502, 537, 507, 497, 515, 498, 515, 504, 489, 485, 510, 511, 528, 540, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 504, 500, 531, 517, 500, 495, 494, 523, 507, 509, 546, 479, 504, 528, 487, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 512, 511, 499, 512, 501, 454, 492, 544, 496, 537, 493, 515, 515, 539, 536, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 446, 519, 536, 528, 509, 485, 533, 540, 493, 515, 500, 478, 530, 539, 511, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 531, 501, 527, 497, 503, 548, 518, 546, 500, 485, 543, 461, 492, 466, 504, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 528, 529, 528, 545, 505, 518, 500, 514, 515, 573, 494, 505, 574, 540, 546, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8027, 8191, 8203, 8124, 8178, 8211, 8172, 8430] avg=8192.0000 max=8430 max_rank=7 min=8027 min_rank=0 imbalance_ratio=1.029053 max_load_reduction=57106 max_load_reduction_pct=0.871368 moved_to_echo=32991 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[529, 524, 520, 538, 526, 508, 449, 511, 555, 542, 492, 517, 513, 559, 505, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 548, 486, 521, 535, 506, 541, 512, 526, 487, 558, 466, 525, 469, 535, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 510, 473, 548, 498, 510, 504, 504, 500, 540, 518, 496, 500, 564, 499, 505, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 493, 504, 479, 475, 534, 529, 493, 527, 530, 556, 497, 521, 540, 541, 496, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 535, 542, 501, 498, 497, 500, 501, 485, 495, 541, 502, 515, 493, 533, 515, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 507, 476, 520, 557, 479, 502, 502, 472, 500, 522, 513, 502, 524, 510, 454, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 513, 500, 520, 523, 501, 487, 512, 503, 502, 495, 497, 482, 477, 482, 539, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 486, 508, 557, 537, 527, 533, 509, 557, 468, 499, 468, 540, 536, 511, 476, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8288, 8258, 8208, 8230, 8130, 8066, 8102, 8254] avg=8192.0000 max=8288 max_rank=0 min=8066 min_rank=5 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32552 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[525, 533, 520, 519, 510, 543, 445, 527, 524, 548, 526, 498, 508, 512, 497, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 522, 471, 519, 489, 515, 538, 505, 515, 504, 485, 517, 558, 499, 501, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 491, 490, 507, 505, 542, 486, 477, 505, 523, 470, 489, 488, 510, 492, 504, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 525, 538, 529, 510, 488, 482, 552, 507, 492, 527, 502, 519, 534, 544, 551, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 495, 515, 518, 499, 518, 519, 491, 500, 501, 545, 504, 547, 472, 523, 521, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 496, 540, 514, 491, 494, 521, 500, 535, 508, 518, 522, 477, 505, 506, 477, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 503, 526, 534, 471, 535, 527, 491, 520, 531, 497, 489, 511, 508, 498, 505, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 495, 489, 490, 497, 487, 571, 558, 463, 517, 492, 516, 509, 517, 541, 567, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 8188, 8057, 8313, 8182, 8125, 8189, 8237] avg=8192.0000 max=8313 max_rank=3 min=8057 min_rank=2 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32733 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[526, 445, 508, 504, 487, 511, 474, 518, 532, 520, 504, 547, 557, 527, 510, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 486, 524, 475, 493, 530, 509, 511, 552, 493, 539, 516, 496, 506, 555, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 537, 526, 495, 504, 519, 480, 516, 507, 515, 522, 524, 482, 511, 501, 548, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 525, 518, 493, 527, 530, 484, 494, 517, 496, 510, 528, 529, 498, 455, 516, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 519, 505, 507, 492, 549, 496, 572, 511, 507, 508, 499, 503, 541, 488, 480, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 524, 498, 539, 507, 480, 520, 495, 519, 512, 492, 488, 524, 486, 521, 513, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 507, 547, 508, 518, 510, 540, 523, 515, 506, 489, 524, 537, 497, 514, 482, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 520, 531, 533, 452, 524, 539, 502, 469, 493, 532, 484, 507, 500, 483, 475, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8245, 8256, 8131, 8204, 8180, 8236, 8095] avg=8192.0000 max=8256 max_rank=2 min=8095 min_rank=7 imbalance_ratio=1.007812 max_load_reduction=57280 max_load_reduction_pct=0.874023 moved_to_echo=32715 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[525, 508, 505, 483, 526, 468, 480, 526, 525, 521, 551, 536, 525, 522, 508, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 522, 506, 504, 491, 504, 505, 506, 507, 488, 523, 505, 485, 495, 533, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 534, 521, 484, 505, 506, 489, 498, 468, 500, 474, 556, 547, 523, 533, 506, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 452, 518, 508, 517, 521, 539, 538, 525, 482, 479, 504, 514, 530, 494, 522, 503, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 491, 520, 497, 530, 532, 508, 523, 528, 543, 525, 519, 549, 527, 531, 535, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 506, 533, 522, 505, 488, 505, 500, 491, 523, 542, 513, 500, 548, 461, 539, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 447, 543, 533, 499, 497, 529, 527, 478, 523, 514, 502, 498, 520, 518, 512, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 487, 480, 499, 492, 528, 518, 491, 518, 533, 504, 511, 511, 499, 533, 508, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8259, 8037, 8213, 8149, 8403, 8167, 8174, 8134] avg=8192.0000 max=8403 max_rank=4 min=8037 min_rank=1 imbalance_ratio=1.025757 max_load_reduction=57133 max_load_reduction_pct=0.871780 moved_to_echo=32878 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[526, 445, 508, 504, 487, 511, 474, 518, 532, 520, 504, 547, 557, 527, 510, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 486, 524, 475, 493, 530, 509, 511, 552, 493, 539, 516, 496, 506, 555, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 537, 526, 495, 504, 519, 480, 516, 507, 515, 522, 524, 482, 511, 501, 548, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 525, 518, 493, 527, 530, 484, 494, 517, 496, 510, 528, 529, 498, 455, 516, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 519, 505, 507, 492, 549, 496, 572, 511, 507, 508, 499, 503, 541, 488, 480, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 524, 498, 539, 507, 480, 520, 495, 519, 512, 492, 488, 524, 486, 521, 513, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 507, 547, 508, 518, 510, 540, 523, 515, 506, 489, 524, 537, 497, 514, 482, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 520, 531, 533, 452, 524, 539, 502, 469, 493, 532, 484, 507, 500, 483, 475, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8245, 8256, 8131, 8204, 8180, 8236, 8095] avg=8192.0000 max=8256 max_rank=2 min=8095 min_rank=7 imbalance_ratio=1.007812 max_load_reduction=57280 max_load_reduction_pct=0.874023 moved_to_echo=32715 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[553, 513, 559, 496, 489, 496, 434, 544, 494, 524, 518, 498, 517, 535, 504, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 498, 540, 509, 501, 522, 514, 475, 501, 482, 534, 526, 512, 537, 493, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 475, 532, 534, 482, 473, 516, 514, 509, 477, 517, 535, 513, 488, 499, 512, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 496, 482, 515, 480, 518, 527, 494, 543, 524, 507, 513, 510, 510, 487, 513, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 536, 504, 466, 495, 540, 553, 511, 498, 500, 538, 504, 490, 508, 533, 506, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 507, 505, 485, 499, 530, 504, 508, 542, 519, 524, 543, 540, 472, 520, 494, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 542, 540, 525, 515, 492, 538, 477, 561, 524, 534, 525, 481, 492, 505, 497, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 530, 487, 464, 497, 518, 514, 502, 541, 546, 498, 475, 502, 530, 497, 537, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8150, 8175, 8143, 8194, 8213, 8285, 8169] avg=8192.0000 max=8285 max_rank=6 min=8143 min_rank=3 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32861 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4352, 4096, 4224, 4096, 4096, 4096, 3712, 4224, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4352, 4096, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 3968, 4096, 4224, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 3968, 4096, 4224, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4352, 4096, 4224, 4096, 4096, 4096, 3712, 4224, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4352, 4096, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[521, 484, 505, 495, 524, 543, 508, 537, 493, 562, 484, 568, 518, 476, 521, 499, 540, 541, 516, 493, 511, 532, 470, 499, 539, 541, 543, 499, 512, 532, 506, 499, 497, 517, 483, 519, 514, 533, 515, 533, 522, 508, 513, 529, 496, 514, 505, 510, 517, 494, 526, 512, 518, 523, 475, 491, 526, 516, 508, 528, 530, 471, 505, 548, 504, 522, 505, 494, 505, 515, 507, 455, 486, 529, 514, 531, 490, 496, 510, 545, 505, 507, 506, 524, 487, 513, 520, 522, 519, 524, 530, 507, 508, 526, 489, 459, 531, 500, 527, 534, 514, 498, 526, 532, 536, 502, 477, 534, 494, 469, 532, 548, 494, 530, 527, 513, 508, 493, 482, 506, 512, 499, 505, 500, 520, 482, 536, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[517, 530, 516, 483, 456, 488, 504, 506, 468, 531, 565, 545, 531, 503, 491, 533, 535, 491, 521, 507, 540, 492, 485, 521, 488, 509, 488, 545, 557, 490, 539, 492, 528, 552, 521, 528, 480, 508, 473, 529, 522, 558, 530, 515, 554, 477, 497, 466, 479, 516, 533, 521, 503, 522, 528, 539, 519, 515, 527, 491, 498, 512, 508, 516, 523, 521, 480, 543, 537, 549, 475, 514, 480, 497, 482, 510, 522, 498, 487, 538, 514, 519, 489, 517, 501, 555, 463, 510, 530, 514, 505, 524, 523, 524, 539, 488, 520, 481, 495, 511, 448, 528, 502, 518, 565, 564, 526, 523, 532, 511, 510, 503, 520, 489, 490, 511, 505, 500, 527, 504, 488, 483, 484, 522, 498, 527, 532, 516] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[526, 503, 555, 536, 506, 520, 554, 496, 537, 493, 484, 495, 544, 496, 475, 514, 497, 514, 509, 508, 526, 526, 488, 503, 520, 510, 559, 519, 538, 543, 482, 521, 554, 505, 485, 497, 512, 488, 529, 527, 539, 508, 516, 506, 514, 532, 551, 502, 523, 495, 522, 503, 520, 527, 456, 526, 537, 497, 503, 487, 481, 554, 489, 532, 486, 529, 520, 526, 469, 526, 536, 550, 528, 483, 489, 505, 551, 480, 543, 502, 493, 535, 528, 509, 478, 565, 490, 507, 500, 521, 554, 495, 494, 513, 526, 479, 530, 487, 525, 503, 492, 508, 485, 462, 515, 471, 492, 507, 481, 528, 552, 513, 533, 492, 506, 482, 487, 481, 502, 487, 521, 528, 518, 515, 542, 520, 550, 497] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[528, 506, 553, 530, 521, 506, 516, 525, 528, 540, 520, 485, 447, 531, 529, 476, 526, 528, 539, 510, 500, 522, 486, 497, 522, 518, 504, 508, 488, 527, 538, 455, 516, 509, 538, 507, 551, 521, 515, 539, 500, 562, 519, 558, 502, 515, 483, 518, 518, 483, 533, 531, 509, 519, 537, 518, 508, 512, 474, 489, 493, 499, 529, 536, 499, 506, 531, 521, 507, 539, 490, 529, 481, 496, 488, 535, 511, 522, 502, 483, 518, 505, 509, 467, 528, 498, 498, 493, 501, 497, 545, 495, 502, 533, 475, 474, 507, 476, 494, 508, 541, 489, 563, 535, 541, 467, 482, 515, 511, 519, 530, 535, 534, 491, 467, 518, 516, 535, 533, 501, 525, 513, 529, 511, 470, 508, 511, 533] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[496, 486, 514, 482, 512, 535, 500, 534, 540, 466, 487, 510, 510, 500, 513, 516, 491, 477, 551, 480, 489, 465, 542, 505, 543, 522, 489, 550, 482, 504, 522, 521, 496, 497, 539, 452, 550, 513, 471, 515, 484, 521, 550, 536, 524, 520, 494, 530, 512, 576, 480, 529, 467, 528, 527, 505, 531, 512, 495, 503, 494, 520, 491, 513, 501, 573, 521, 474, 505, 489, 539, 477, 544, 531, 554, 486, 540, 535, 530, 527, 469, 484, 492, 531, 544, 498, 532, 481, 485, 491, 521, 532, 513, 501, 490, 490, 521, 494, 534, 535, 534, 531, 522, 496, 513, 532, 512, 505, 500, 530, 511, 519, 491, 539, 565, 554, 510, 491, 491, 512, 533, 487, 517, 493, 508, 520, 523, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[513, 519, 530, 511, 477, 471, 507, 524, 512, 501, 539, 495, 542, 519, 513, 511, 499, 529, 456, 540, 524, 524, 488, 554, 468, 524, 518, 496, 530, 507, 458, 534, 525, 525, 568, 519, 513, 493, 505, 544, 525, 485, 528, 485, 492, 544, 537, 515, 524, 536, 524, 520, 558, 511, 513, 536, 505, 527, 479, 523, 483, 528, 530, 531, 557, 476, 513, 536, 505, 523, 491, 478, 486, 501, 508, 484, 523, 495, 523, 534, 495, 491, 500, 518, 528, 455, 551, 522, 483, 522, 502, 495, 549, 513, 525, 510, 508, 497, 519, 526, 510, 490, 523, 538, 507, 529, 490, 565, 493, 491, 497, 463, 505, 530, 519, 518, 510, 499, 505, 494, 488, 494, 487, 498, 568, 476, 504, 539] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[518, 508, 492, 516, 497, 517, 518, 509, 463, 528, 524, 482, 540, 507, 519, 542, 537, 506, 532, 526, 509, 519, 519, 513, 484, 512, 532, 500, 513, 476, 522, 484, 474, 481, 525, 526, 533, 504, 526, 506, 505, 520, 499, 520, 494, 490, 521, 483, 484, 508, 523, 514, 513, 537, 514, 510, 540, 515, 515, 526, 522, 520, 545, 516, 539, 517, 537, 501, 548, 519, 542, 508, 495, 484, 503, 482, 537, 486, 504, 529, 521, 518, 530, 499, 496, 502, 528, 508, 543, 505, 496, 531, 511, 509, 515, 491, 507, 528, 524, 536, 536, 518, 506, 478, 522, 525, 512, 492, 512, 492, 532, 519, 495, 528, 526, 462, 495, 551, 469, 485, 500, 514, 514, 509, 485, 515, 518, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[517, 530, 516, 483, 456, 488, 504, 506, 468, 531, 565, 545, 531, 503, 491, 533, 535, 491, 521, 507, 540, 492, 485, 521, 488, 509, 488, 545, 557, 490, 539, 492, 528, 552, 521, 528, 480, 508, 473, 529, 522, 558, 530, 515, 554, 477, 497, 466, 479, 516, 533, 521, 503, 522, 528, 539, 519, 515, 527, 491, 498, 512, 508, 516, 523, 521, 480, 543, 537, 549, 475, 514, 480, 497, 482, 510, 522, 498, 487, 538, 514, 519, 489, 517, 501, 555, 463, 510, 530, 514, 505, 524, 523, 524, 539, 488, 520, 481, 495, 511, 448, 528, 502, 518, 565, 564, 526, 523, 532, 511, 510, 503, 520, 489, 490, 511, 505, 500, 527, 504, 488, 483, 484, 522, 498, 527, 532, 516] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[513, 519, 530, 511, 477, 471, 507, 524, 512, 501, 539, 495, 542, 519, 513, 511, 499, 529, 456, 540, 524, 524, 488, 554, 468, 524, 518, 496, 530, 507, 458, 534, 525, 525, 568, 519, 513, 493, 505, 544, 525, 485, 528, 485, 492, 544, 537, 515, 524, 536, 524, 520, 558, 511, 513, 536, 505, 527, 479, 523, 483, 528, 530, 531, 557, 476, 513, 536, 505, 523, 491, 478, 486, 501, 508, 484, 523, 495, 523, 534, 495, 491, 500, 518, 528, 455, 551, 522, 483, 522, 502, 495, 549, 513, 525, 510, 508, 497, 519, 526, 510, 490, 523, 538, 507, 529, 490, 565, 493, 491, 497, 463, 505, 530, 519, 518, 510, 499, 505, 494, 488, 494, 487, 498, 568, 476, 504, 539] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[521, 519, 515, 544, 477, 508, 502, 526, 492, 483, 539, 478, 516, 508, 488, 508, 521, 498, 487, 553, 544, 524, 530, 509, 494, 516, 482, 517, 479, 507, 491, 479, 499, 504, 495, 482, 489, 524, 520, 521, 500, 528, 544, 495, 501, 501, 516, 518, 497, 498, 516, 512, 547, 514, 507, 505, 517, 491, 494, 526, 533, 509, 466, 518, 534, 508, 518, 516, 515, 546, 499, 528, 505, 504, 540, 503, 524, 520, 543, 502, 506, 519, 548, 518, 505, 517, 504, 536, 524, 489, 549, 497, 514, 542, 504, 557, 493, 479, 458, 500, 513, 506, 531, 508, 573, 503, 526, 493, 505, 520, 489, 499, 516, 502, 489, 501, 522, 542, 525, 517, 556, 515, 508, 507, 537, 510, 526, 491] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[521, 484, 505, 495, 524, 543, 508, 537, 493, 562, 484, 568, 518, 476, 521, 499, 540, 541, 516, 493, 511, 532, 470, 499, 539, 541, 543, 499, 512, 532, 506, 499, 497, 517, 483, 519, 514, 533, 515, 533, 522, 508, 513, 529, 496, 514, 505, 510, 517, 494, 526, 512, 518, 523, 475, 491, 526, 516, 508, 528, 530, 471, 505, 548, 504, 522, 505, 494, 505, 515, 507, 455, 486, 529, 514, 531, 490, 496, 510, 545, 505, 507, 506, 524, 487, 513, 520, 522, 519, 524, 530, 507, 508, 526, 489, 459, 531, 500, 527, 534, 514, 498, 526, 532, 536, 502, 477, 534, 494, 469, 532, 548, 494, 530, 527, 513, 508, 493, 482, 506, 512, 499, 505, 500, 520, 482, 536, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[528, 506, 553, 530, 521, 506, 516, 525, 528, 540, 520, 485, 447, 531, 529, 476, 526, 528, 539, 510, 500, 522, 486, 497, 522, 518, 504, 508, 488, 527, 538, 455, 516, 509, 538, 507, 551, 521, 515, 539, 500, 562, 519, 558, 502, 515, 483, 518, 518, 483, 533, 531, 509, 519, 537, 518, 508, 512, 474, 489, 493, 499, 529, 536, 499, 506, 531, 521, 507, 539, 490, 529, 481, 496, 488, 535, 511, 522, 502, 483, 518, 505, 509, 467, 528, 498, 498, 493, 501, 497, 545, 495, 502, 533, 475, 474, 507, 476, 494, 508, 541, 489, 563, 535, 541, 467, 482, 515, 511, 519, 530, 535, 534, 491, 467, 518, 516, 535, 533, 501, 525, 513, 529, 511, 470, 508, 511, 533] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[496, 486, 514, 482, 512, 535, 500, 534, 540, 466, 487, 510, 510, 500, 513, 516, 491, 477, 551, 480, 489, 465, 542, 505, 543, 522, 489, 550, 482, 504, 522, 521, 496, 497, 539, 452, 550, 513, 471, 515, 484, 521, 550, 536, 524, 520, 494, 530, 512, 576, 480, 529, 467, 528, 527, 505, 531, 512, 495, 503, 494, 520, 491, 513, 501, 573, 521, 474, 505, 489, 539, 477, 544, 531, 554, 486, 540, 535, 530, 527, 469, 484, 492, 531, 544, 498, 532, 481, 485, 491, 521, 532, 513, 501, 490, 490, 521, 494, 534, 535, 534, 531, 522, 496, 513, 532, 512, 505, 500, 530, 511, 519, 491, 539, 565, 554, 510, 491, 491, 512, 533, 487, 517, 493, 508, 520, 523, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[517, 530, 516, 483, 456, 488, 504, 506, 468, 531, 565, 545, 531, 503, 491, 533, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 491, 521, 507, 540, 492, 485, 521, 488, 509, 488, 545, 557, 490, 539, 492, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 552, 521, 528, 480, 508, 473, 496, 522, 558, 530, 515, 554, 477, 497, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 516, 533, 521, 503, 522, 528, 539, 519, 515, 527, 491, 498, 512, 508, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 521, 480, 543, 537, 539, 475, 514, 480, 497, 482, 510, 522, 498, 487, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 519, 489, 517, 501, 555, 463, 510, 530, 514, 505, 524, 523, 524, 539, 488, 18, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 481, 495, 511, 448, 528, 502, 518, 565, 564, 526, 523, 532, 511, 510, 503, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 489, 490, 511, 505, 500, 527, 504, 488, 483, 484, 522, 498, 527, 532, 516, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8204, 8205, 8206, 8146, 8239, 8238, 8123] avg=8192.0000 max=8239 max_rank=5 min=8123 min_rank=7 imbalance_ratio=1.005737 max_load_reduction=57297 max_load_reduction_pct=0.874283 moved_to_echo=32746 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[521, 484, 505, 495, 524, 543, 508, 537, 493, 562, 484, 568, 518, 476, 521, 499, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 541, 516, 493, 511, 532, 470, 499, 539, 541, 543, 499, 512, 532, 506, 499, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 517, 483, 519, 514, 533, 515, 500, 522, 508, 513, 529, 496, 514, 505, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 494, 526, 512, 518, 523, 475, 491, 526, 516, 508, 528, 530, 471, 505, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 522, 505, 494, 505, 506, 507, 455, 486, 529, 514, 531, 490, 496, 510, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 507, 506, 524, 487, 513, 520, 522, 519, 524, 530, 507, 508, 526, 489, 459, 20, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 500, 527, 534, 514, 498, 526, 532, 536, 502, 477, 534, 494, 469, 532, 548, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 530, 527, 513, 508, 493, 482, 506, 512, 499, 505, 500, 520, 482, 536, 514, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 8277, 8175, 8165, 8099, 8172, 8255, 8148] avg=8192.0000 max=8277 max_rank=1 min=8099 min_rank=4 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32674 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[528, 506, 553, 530, 521, 506, 516, 525, 528, 540, 520, 485, 447, 531, 529, 476, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 528, 539, 510, 500, 522, 486, 497, 522, 518, 504, 508, 488, 527, 538, 455, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 509, 538, 507, 551, 521, 515, 506, 500, 562, 519, 558, 502, 515, 483, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 483, 533, 531, 509, 519, 537, 518, 508, 512, 474, 489, 493, 499, 529, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 506, 531, 521, 507, 529, 490, 529, 481, 496, 488, 535, 511, 522, 502, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 505, 509, 467, 528, 498, 498, 493, 501, 497, 545, 495, 502, 533, 475, 474, 19, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 476, 494, 508, 541, 489, 563, 535, 541, 467, 482, 515, 511, 519, 530, 535, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 491, 467, 518, 516, 535, 533, 501, 525, 513, 529, 511, 470, 508, 511, 533, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8249, 8172, 8320, 8166, 8130, 8063, 8214, 8222] avg=8192.0000 max=8320 max_rank=2 min=8063 min_rank=5 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32629 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[513, 519, 530, 511, 477, 471, 507, 524, 512, 501, 539, 495, 542, 519, 513, 511, 8, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 529, 456, 540, 524, 524, 488, 554, 468, 524, 518, 496, 530, 507, 458, 534, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 525, 568, 519, 513, 493, 505, 505, 525, 485, 528, 485, 492, 544, 537, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 536, 524, 520, 558, 511, 513, 536, 505, 527, 479, 523, 483, 528, 530, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 557, 476, 513, 536, 505, 504, 491, 478, 486, 501, 508, 484, 523, 495, 523, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 491, 500, 518, 528, 455, 551, 522, 483, 522, 502, 495, 549, 513, 525, 510, 24, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 497, 519, 526, 510, 490, 523, 538, 507, 529, 490, 565, 493, 491, 497, 463, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 530, 519, 518, 510, 499, 505, 494, 488, 494, 487, 498, 568, 476, 504, 539, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8156, 8264, 8298, 8114, 8191, 8150, 8165] avg=8192.0000 max=8298 max_rank=3 min=8114 min_rank=4 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32620 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[517, 530, 516, 483, 456, 488, 504, 506, 468, 531, 565, 545, 531, 503, 491, 533, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 491, 521, 507, 540, 492, 485, 521, 488, 509, 488, 545, 557, 490, 539, 492, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 552, 521, 528, 480, 508, 473, 496, 522, 558, 530, 515, 554, 477, 497, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 516, 533, 521, 503, 522, 528, 539, 519, 515, 527, 491, 498, 512, 508, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 521, 480, 543, 537, 539, 475, 514, 480, 497, 482, 510, 522, 498, 487, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 519, 489, 517, 501, 555, 463, 510, 530, 514, 505, 524, 523, 524, 539, 488, 18, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 481, 495, 511, 448, 528, 502, 518, 565, 564, 526, 523, 532, 511, 510, 503, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 489, 490, 511, 505, 500, 527, 504, 488, 483, 484, 522, 498, 527, 532, 516, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8204, 8205, 8206, 8146, 8239, 8238, 8123] avg=8192.0000 max=8239 max_rank=5 min=8123 min_rank=7 imbalance_ratio=1.005737 max_load_reduction=57297 max_load_reduction_pct=0.874283 moved_to_echo=32746 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[518, 508, 492, 516, 497, 517, 518, 509, 463, 528, 524, 482, 540, 507, 519, 542, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 506, 532, 526, 509, 519, 519, 513, 484, 512, 532, 500, 513, 476, 522, 484, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 481, 525, 526, 533, 504, 526, 474, 505, 520, 499, 520, 494, 490, 521, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 508, 523, 514, 513, 537, 514, 510, 540, 515, 515, 526, 522, 520, 545, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 517, 537, 501, 548, 510, 542, 508, 495, 484, 503, 482, 537, 486, 504, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 518, 530, 499, 496, 502, 528, 508, 543, 505, 496, 531, 511, 509, 515, 491, 18, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 528, 524, 536, 536, 518, 506, 478, 522, 525, 512, 492, 512, 492, 532, 519, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 528, 526, 462, 495, 551, 469, 485, 500, 514, 514, 509, 485, 515, 518, 524, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8188, 8075, 8281, 8222, 8227, 8240, 8116] avg=8192.0000 max=8281 max_rank=3 min=8075 min_rank=2 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32805 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[521, 519, 515, 544, 477, 508, 502, 526, 492, 483, 539, 478, 516, 508, 488, 508, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 498, 487, 553, 544, 524, 530, 509, 494, 516, 482, 517, 479, 507, 491, 479, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 504, 495, 482, 489, 524, 520, 489, 500, 528, 544, 495, 501, 501, 516, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 498, 516, 512, 547, 514, 507, 505, 517, 491, 494, 526, 533, 509, 466, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 508, 518, 516, 515, 536, 499, 528, 505, 504, 540, 503, 524, 520, 543, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 519, 548, 518, 505, 517, 504, 536, 524, 489, 549, 497, 514, 542, 504, 557, 19, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 479, 458, 500, 513, 506, 531, 508, 573, 503, 526, 493, 505, 520, 489, 499, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 502, 489, 501, 522, 542, 525, 517, 556, 515, 508, 507, 537, 510, 526, 491, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8132, 8135, 8105, 8128, 8295, 8354, 8097, 8290] avg=8192.0000 max=8354 max_rank=5 min=8097 min_rank=6 imbalance_ratio=1.019775 max_load_reduction=57182 max_load_reduction_pct=0.872528 moved_to_echo=33036 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[528, 506, 553, 530, 521, 506, 516, 525, 528, 540, 520, 485, 447, 531, 529, 476, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 528, 539, 510, 500, 522, 486, 497, 522, 518, 504, 508, 488, 527, 538, 455, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 509, 538, 507, 551, 521, 515, 506, 500, 562, 519, 558, 502, 515, 483, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 483, 533, 531, 509, 519, 537, 518, 508, 512, 474, 489, 493, 499, 529, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 506, 531, 521, 507, 529, 490, 529, 481, 496, 488, 535, 511, 522, 502, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 505, 509, 467, 528, 498, 498, 493, 501, 497, 545, 495, 502, 533, 475, 474, 19, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 476, 494, 508, 541, 489, 563, 535, 541, 467, 482, 515, 511, 519, 530, 535, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 491, 467, 518, 516, 535, 533, 501, 525, 513, 529, 511, 470, 508, 511, 533, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8249, 8172, 8320, 8166, 8130, 8063, 8214, 8222] avg=8192.0000 max=8320 max_rank=2 min=8063 min_rank=5 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32629 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[496, 486, 514, 482, 512, 535, 500, 534, 540, 466, 487, 510, 510, 500, 513, 516, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 477, 551, 480, 489, 465, 542, 505, 543, 522, 489, 550, 482, 504, 522, 521, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 497, 539, 452, 550, 513, 471, 483, 484, 521, 550, 536, 524, 520, 494, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 576, 480, 529, 467, 528, 527, 505, 531, 512, 495, 503, 494, 520, 491, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 573, 521, 474, 505, 480, 539, 477, 544, 531, 554, 486, 540, 535, 530, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 484, 492, 531, 544, 498, 532, 481, 485, 491, 521, 532, 513, 501, 490, 490, 18, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 494, 534, 535, 534, 531, 522, 496, 513, 532, 512, 505, 500, 530, 511, 519, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 539, 565, 554, 510, 491, 491, 512, 533, 487, 517, 493, 508, 520, 523, 524, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8108, 8137, 8160, 8162, 8317, 8078, 8290, 8284] avg=8192.0000 max=8317 max_rank=4 min=8078 min_rank=5 imbalance_ratio=1.015259 max_load_reduction=57219 max_load_reduction_pct=0.873093 moved_to_echo=32969 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[526, 503, 555, 536, 506, 520, 554, 496, 537, 493, 484, 495, 544, 496, 475, 514, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 514, 509, 508, 526, 526, 488, 503, 520, 510, 559, 519, 538, 543, 482, 521, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 505, 485, 497, 512, 488, 529, 494, 539, 508, 516, 506, 514, 532, 551, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 495, 522, 503, 520, 527, 456, 526, 537, 497, 503, 487, 481, 554, 489, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 529, 520, 526, 469, 516, 536, 550, 528, 483, 489, 505, 551, 480, 543, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 535, 528, 509, 478, 565, 490, 507, 500, 521, 554, 495, 494, 513, 526, 479, 19, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 487, 525, 503, 492, 508, 485, 462, 515, 471, 492, 507, 481, 528, 552, 513, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 492, 506, 482, 487, 481, 502, 487, 521, 528, 518, 515, 542, 520, 550, 497, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8242, 8267, 8232, 8130, 8213, 8212, 8052, 8188] avg=8192.0000 max=8267 max_rank=1 min=8052 min_rank=6 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32665 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[521, 519, 515, 544, 477, 508, 502, 526, 492, 483, 539, 478, 516, 508, 488, 508, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 498, 487, 553, 544, 524, 530, 509, 494, 516, 482, 517, 479, 507, 491, 479, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 504, 495, 482, 489, 524, 520, 489, 500, 528, 544, 495, 501, 501, 516, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 498, 516, 512, 547, 514, 507, 505, 517, 491, 494, 526, 533, 509, 466, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 508, 518, 516, 515, 536, 499, 528, 505, 504, 540, 503, 524, 520, 543, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 519, 548, 518, 505, 517, 504, 536, 524, 489, 549, 497, 514, 542, 504, 557, 19, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 479, 458, 500, 513, 506, 531, 508, 573, 503, 526, 493, 505, 520, 489, 499, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 502, 489, 501, 522, 542, 525, 517, 556, 515, 508, 507, 537, 510, 526, 491, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8132, 8135, 8105, 8128, 8295, 8354, 8097, 8290] avg=8192.0000 max=8354 max_rank=5 min=8097 min_rank=6 imbalance_ratio=1.019775 max_load_reduction=57182 max_load_reduction_pct=0.872528 moved_to_echo=33036 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[518, 508, 492, 516, 497, 517, 518, 509, 463, 528, 524, 482, 540, 507, 519, 542, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 506, 532, 526, 509, 519, 519, 513, 484, 512, 532, 500, 513, 476, 522, 484, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 481, 525, 526, 533, 504, 526, 474, 505, 520, 499, 520, 494, 490, 521, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 508, 523, 514, 513, 537, 514, 510, 540, 515, 515, 526, 522, 520, 545, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 517, 537, 501, 548, 510, 542, 508, 495, 484, 503, 482, 537, 486, 504, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 518, 530, 499, 496, 502, 528, 508, 543, 505, 496, 531, 511, 509, 515, 491, 18, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 528, 524, 536, 536, 518, 506, 478, 522, 525, 512, 492, 512, 492, 532, 519, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 528, 526, 462, 495, 551, 469, 485, 500, 514, 514, 509, 485, 515, 518, 524, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8188, 8075, 8281, 8222, 8227, 8240, 8116] avg=8192.0000 max=8281 max_rank=3 min=8075 min_rank=2 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32805 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4096, 3968, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4096, 3968, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 3968, 4096, 4224, 4096, 4096, 4224, 4096, 4352, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 3968, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[516, 483, 469, 527, 473, 518, 486, 551, 515, 521, 541, 516, 532, 497, 502, 502, 512, 538, 498, 514, 507, 481, 531, 502, 486, 506, 525, 560, 507, 508, 509, 524, 507, 533, 465, 490, 506, 547, 525, 515, 496, 500, 494, 482, 517, 491, 512, 551, 495, 497, 544, 519, 550, 515, 483, 502, 537, 488, 533, 489, 535, 489, 473, 519, 530, 500, 509, 475, 510, 512, 513, 517, 515, 528, 532, 482, 468, 516, 521, 535, 493, 542, 521, 555, 502, 558, 480, 538, 519, 504, 512, 536, 530, 471, 498, 516, 529, 530, 479, 472, 525, 522, 535, 543, 476, 530, 508, 553, 541, 509, 527, 475, 533, 494, 483, 525, 513, 525, 504, 494, 506, 509, 479, 546, 495, 546, 511, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[514, 510, 528, 516, 560, 504, 508, 520, 459, 497, 485, 517, 512, 469, 532, 529, 507, 512, 508, 505, 515, 526, 536, 489, 515, 486, 517, 465, 462, 536, 545, 493, 548, 499, 554, 476, 491, 506, 541, 540, 491, 465, 506, 526, 526, 559, 506, 534, 498, 459, 548, 523, 500, 499, 505, 493, 510, 557, 520, 505, 498, 522, 506, 510, 513, 495, 511, 524, 510, 533, 501, 515, 532, 507, 535, 502, 519, 527, 495, 500, 497, 543, 532, 519, 517, 520, 517, 490, 513, 496, 507, 505, 485, 544, 487, 547, 490, 510, 501, 462, 515, 501, 476, 505, 526, 498, 532, 502, 521, 547, 497, 539, 514, 491, 504, 504, 510, 503, 515, 515, 531, 539, 522, 525, 505, 521, 539, 540] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[492, 499, 481, 496, 519, 530, 502, 489, 514, 514, 485, 528, 517, 496, 503, 507, 527, 515, 531, 566, 481, 512, 513, 499, 463, 493, 533, 481, 529, 512, 537, 495, 486, 549, 508, 523, 505, 510, 500, 530, 498, 502, 540, 524, 515, 534, 517, 533, 539, 510, 515, 528, 518, 524, 538, 467, 517, 494, 525, 507, 500, 525, 503, 515, 479, 467, 508, 527, 487, 545, 528, 518, 470, 451, 521, 514, 508, 550, 503, 525, 521, 525, 493, 501, 489, 525, 572, 552, 503, 475, 477, 503, 490, 492, 468, 505, 511, 521, 521, 537, 512, 528, 480, 500, 549, 497, 524, 514, 497, 520, 528, 509, 557, 497, 548, 499, 506, 520, 482, 533, 526, 536, 539, 530, 509, 537, 508, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[492, 506, 511, 525, 510, 529, 515, 533, 518, 500, 499, 481, 478, 496, 523, 474, 529, 488, 482, 541, 481, 522, 522, 529, 565, 489, 521, 499, 500, 462, 489, 509, 491, 548, 534, 492, 542, 508, 502, 523, 498, 533, 546, 535, 496, 491, 474, 506, 490, 526, 521, 513, 515, 502, 504, 530, 506, 516, 534, 506, 510, 503, 505, 488, 504, 509, 544, 511, 501, 527, 510, 465, 473, 549, 497, 555, 471, 494, 480, 499, 530, 529, 507, 535, 544, 512, 516, 527, 536, 497, 520, 534, 490, 535, 500, 538, 512, 554, 542, 522, 560, 500, 529, 502, 530, 549, 510, 542, 556, 485, 474, 530, 548, 493, 497, 493, 518, 504, 477, 521, 505, 529, 497, 540, 502, 507, 473, 490] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[484, 547, 461, 547, 533, 528, 515, 487, 520, 483, 515, 488, 496, 536, 531, 554, 523, 497, 559, 541, 474, 489, 514, 500, 499, 518, 522, 517, 493, 492, 539, 520, 494, 517, 519, 488, 491, 486, 526, 521, 495, 537, 518, 512, 474, 521, 542, 525, 520, 494, 545, 491, 529, 506, 498, 505, 540, 513, 512, 483, 543, 503, 503, 516, 508, 471, 462, 530, 516, 459, 506, 534, 489, 519, 494, 520, 534, 494, 529, 518, 530, 512, 557, 531, 511, 552, 508, 514, 541, 488, 514, 487, 522, 534, 471, 527, 524, 549, 512, 504, 522, 503, 527, 480, 499, 496, 517, 496, 523, 484, 504, 475, 513, 488, 536, 489, 528, 544, 513, 490, 536, 539, 527, 503, 508, 533, 500, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[490, 512, 540, 509, 542, 486, 505, 520, 495, 534, 483, 496, 488, 499, 551, 515, 526, 548, 528, 475, 510, 503, 513, 481, 505, 553, 521, 516, 521, 515, 522, 494, 530, 485, 508, 538, 502, 532, 503, 508, 473, 558, 523, 524, 496, 519, 510, 517, 517, 510, 518, 519, 537, 567, 485, 535, 525, 533, 531, 506, 542, 512, 487, 557, 501, 477, 527, 503, 553, 482, 500, 457, 539, 498, 520, 504, 536, 538, 517, 498, 506, 551, 537, 471, 518, 518, 506, 488, 485, 504, 513, 464, 523, 493, 499, 496, 475, 542, 482, 513, 517, 502, 463, 495, 520, 474, 527, 486, 560, 516, 501, 513, 503, 530, 522, 529, 525, 542, 522, 521, 489, 508, 476, 521, 534, 492, 499, 512] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[514, 510, 528, 516, 560, 504, 508, 520, 459, 497, 485, 517, 512, 469, 532, 529, 507, 512, 508, 505, 515, 526, 536, 489, 515, 486, 517, 465, 462, 536, 545, 493, 548, 499, 554, 476, 491, 506, 541, 540, 491, 465, 506, 526, 526, 559, 506, 534, 498, 459, 548, 523, 500, 499, 505, 493, 510, 557, 520, 505, 498, 522, 506, 510, 513, 495, 511, 524, 510, 533, 501, 515, 532, 507, 535, 502, 519, 527, 495, 500, 497, 543, 532, 519, 517, 520, 517, 490, 513, 496, 507, 505, 485, 544, 487, 547, 490, 510, 501, 462, 515, 501, 476, 505, 526, 498, 532, 502, 521, 547, 497, 539, 514, 491, 504, 504, 510, 503, 515, 515, 531, 539, 522, 525, 505, 521, 539, 540] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[492, 506, 511, 525, 510, 529, 515, 533, 518, 500, 499, 481, 478, 496, 523, 474, 529, 488, 482, 541, 481, 522, 522, 529, 565, 489, 521, 499, 500, 462, 489, 509, 491, 548, 534, 492, 542, 508, 502, 523, 498, 533, 546, 535, 496, 491, 474, 506, 490, 526, 521, 513, 515, 502, 504, 530, 506, 516, 534, 506, 510, 503, 505, 488, 504, 509, 544, 511, 501, 527, 510, 465, 473, 549, 497, 555, 471, 494, 480, 499, 530, 529, 507, 535, 544, 512, 516, 527, 536, 497, 520, 534, 490, 535, 500, 538, 512, 554, 542, 522, 560, 500, 529, 502, 530, 549, 510, 542, 556, 485, 474, 530, 548, 493, 497, 493, 518, 504, 477, 521, 505, 529, 497, 540, 502, 507, 473, 490] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[529, 499, 501, 489, 517, 566, 528, 492, 516, 469, 458, 511, 521, 516, 515, 548, 545, 509, 499, 508, 535, 523, 515, 481, 492, 498, 518, 491, 528, 557, 492, 465, 482, 501, 482, 506, 479, 524, 539, 542, 506, 511, 529, 500, 536, 505, 504, 494, 495, 499, 546, 476, 546, 517, 508, 532, 522, 524, 490, 521, 498, 508, 471, 536, 508, 460, 513, 546, 502, 537, 519, 468, 552, 498, 495, 514, 500, 499, 530, 555, 521, 467, 495, 534, 526, 512, 491, 519, 519, 527, 479, 511, 501, 534, 507, 536, 505, 503, 508, 538, 503, 515, 505, 526, 518, 529, 531, 545, 524, 514, 479, 543, 525, 470, 510, 522, 480, 511, 529, 518, 531, 516, 577, 466, 509, 532, 504, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[536, 499, 523, 477, 529, 458, 485, 537, 562, 537, 523, 486, 517, 495, 535, 491, 555, 511, 510, 526, 524, 541, 535, 489, 470, 468, 512, 528, 490, 560, 517, 488, 468, 525, 471, 536, 504, 522, 482, 516, 507, 500, 477, 522, 498, 499, 508, 523, 507, 537, 522, 500, 469, 526, 521, 533, 508, 540, 547, 509, 523, 487, 533, 543, 484, 495, 523, 522, 570, 500, 499, 540, 496, 515, 504, 513, 541, 508, 516, 491, 484, 504, 537, 514, 496, 547, 518, 530, 529, 512, 480, 547, 547, 506, 481, 550, 501, 482, 533, 510, 515, 511, 520, 491, 538, 463, 539, 544, 519, 498, 493, 491, 476, 530, 520, 508, 515, 509, 528, 518, 544, 461, 511, 485, 505, 497, 486, 519] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[516, 483, 469, 527, 473, 518, 486, 551, 515, 521, 541, 516, 532, 497, 502, 502, 22, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 538, 498, 514, 507, 481, 531, 502, 486, 506, 525, 560, 507, 508, 509, 524, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 533, 465, 490, 506, 547, 525, 515, 496, 500, 494, 482, 517, 491, 512, 551, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 497, 512, 519, 550, 515, 483, 502, 537, 488, 533, 489, 535, 489, 473, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 500, 509, 475, 510, 512, 513, 517, 515, 528, 532, 482, 468, 516, 521, 535, 32, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 542, 521, 555, 502, 528, 480, 538, 519, 504, 512, 536, 530, 471, 498, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 530, 479, 472, 525, 522, 535, 543, 476, 530, 508, 553, 527, 509, 527, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 494, 483, 525, 513, 525, 504, 494, 506, 509, 479, 546, 495, 546, 511, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8218, 8137, 8136, 8203, 8245, 8240, 8168] avg=8192.0000 max=8245 max_rank=5 min=8136 min_rank=3 imbalance_ratio=1.006470 max_load_reduction=57291 max_load_reduction_pct=0.874191 moved_to_echo=32856 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[490, 512, 540, 509, 542, 486, 505, 520, 495, 534, 483, 496, 488, 499, 551, 515, 526, 548, 528, 475, 510, 503, 513, 481, 505, 553, 521, 516, 521, 515, 522, 494, 530, 485, 508, 538, 502, 532, 503, 508, 473, 558, 523, 524, 496, 519, 510, 517, 517, 510, 518, 519, 537, 567, 485, 535, 525, 533, 531, 506, 542, 512, 487, 557, 501, 477, 527, 503, 553, 482, 500, 457, 539, 498, 520, 504, 536, 538, 517, 498, 506, 551, 537, 471, 518, 518, 506, 488, 485, 504, 513, 464, 523, 493, 499, 496, 475, 542, 482, 513, 517, 502, 463, 495, 520, 474, 527, 486, 560, 516, 501, 513, 503, 530, 522, 529, 525, 542, 522, 521, 489, 508, 476, 521, 534, 492, 499, 512] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[516, 483, 469, 527, 473, 518, 486, 551, 515, 521, 541, 516, 532, 497, 502, 502, 512, 538, 498, 514, 507, 481, 531, 502, 486, 506, 525, 560, 507, 508, 509, 524, 507, 533, 465, 490, 506, 547, 525, 515, 496, 500, 494, 482, 517, 491, 512, 551, 495, 497, 544, 519, 550, 515, 483, 502, 537, 488, 533, 489, 535, 489, 473, 519, 530, 500, 509, 475, 510, 512, 513, 517, 515, 528, 532, 482, 468, 516, 521, 535, 493, 542, 521, 555, 502, 558, 480, 538, 519, 504, 512, 536, 530, 471, 498, 516, 529, 530, 479, 472, 525, 522, 535, 543, 476, 530, 508, 553, 541, 509, 527, 475, 533, 494, 483, 525, 513, 525, 504, 494, 506, 509, 479, 546, 495, 546, 511, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[492, 499, 481, 496, 519, 530, 502, 489, 514, 514, 485, 528, 517, 496, 503, 507, 527, 515, 531, 566, 481, 512, 513, 499, 463, 493, 533, 481, 529, 512, 537, 495, 486, 549, 508, 523, 505, 510, 500, 530, 498, 502, 540, 524, 515, 534, 517, 533, 539, 510, 515, 528, 518, 524, 538, 467, 517, 494, 525, 507, 500, 525, 503, 515, 479, 467, 508, 527, 487, 545, 528, 518, 470, 451, 521, 514, 508, 550, 503, 525, 521, 525, 493, 501, 489, 525, 572, 552, 503, 475, 477, 503, 490, 492, 468, 505, 511, 521, 521, 537, 512, 528, 480, 500, 549, 497, 524, 514, 497, 520, 528, 509, 557, 497, 548, 499, 506, 520, 482, 533, 526, 536, 539, 530, 509, 537, 508, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[514, 510, 528, 516, 560, 504, 508, 520, 459, 497, 485, 517, 512, 469, 532, 529, 20, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 512, 508, 505, 515, 526, 536, 489, 515, 486, 517, 465, 462, 536, 545, 493, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 499, 554, 476, 491, 506, 541, 540, 491, 465, 506, 526, 526, 559, 506, 534, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 459, 516, 523, 500, 499, 505, 493, 510, 557, 520, 505, 498, 522, 506, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 495, 511, 524, 510, 533, 501, 515, 532, 507, 535, 502, 519, 527, 495, 500, 32, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 543, 532, 519, 517, 493, 517, 490, 513, 496, 507, 505, 485, 544, 487, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 510, 501, 462, 515, 501, 476, 505, 526, 498, 532, 502, 507, 547, 497, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 491, 504, 504, 510, 503, 515, 515, 531, 539, 522, 525, 505, 521, 539, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8127, 8274, 8121, 8258, 8192, 8108, 8259] avg=8192.0000 max=8274 max_rank=2 min=8108 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32817 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[529, 499, 501, 489, 517, 566, 528, 492, 516, 469, 458, 511, 521, 516, 515, 548, 20, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 509, 499, 508, 535, 523, 515, 481, 492, 498, 518, 491, 528, 557, 492, 465, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 501, 482, 506, 479, 524, 539, 542, 506, 511, 529, 500, 536, 505, 504, 494, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 499, 514, 476, 546, 517, 508, 532, 522, 524, 490, 521, 498, 508, 471, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 460, 513, 546, 502, 537, 519, 468, 552, 498, 495, 514, 500, 499, 530, 555, 32, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 467, 495, 534, 526, 485, 491, 519, 519, 527, 479, 511, 501, 534, 507, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 503, 508, 538, 503, 515, 505, 526, 518, 529, 531, 545, 510, 514, 479, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 470, 510, 522, 480, 511, 529, 518, 531, 516, 577, 466, 509, 532, 504, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8166, 8146, 8157, 8235, 8152, 8272, 8196] avg=8192.0000 max=8272 max_rank=6 min=8146 min_rank=2 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32855 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[484, 547, 461, 547, 533, 528, 515, 487, 520, 483, 515, 488, 496, 536, 531, 554, 21, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 497, 559, 541, 474, 489, 514, 500, 499, 518, 522, 517, 493, 492, 539, 520, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 517, 519, 488, 491, 486, 526, 521, 495, 537, 518, 512, 474, 521, 542, 525, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 494, 513, 491, 529, 506, 498, 505, 540, 513, 512, 483, 543, 503, 503, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 471, 462, 530, 516, 459, 506, 534, 489, 519, 494, 520, 534, 494, 529, 518, 32, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 512, 557, 531, 511, 523, 508, 514, 541, 488, 514, 487, 522, 534, 471, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 549, 512, 504, 522, 503, 527, 480, 499, 496, 517, 496, 509, 484, 504, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 488, 536, 489, 528, 544, 513, 490, 536, 539, 527, 503, 508, 533, 500, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8263, 8207, 8172, 8169, 8123, 8270, 8101, 8231] avg=8192.0000 max=8270 max_rank=5 min=8101 min_rank=6 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32725 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[490, 512, 540, 509, 542, 486, 505, 520, 495, 534, 483, 496, 488, 499, 551, 515, 24, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 548, 528, 475, 510, 503, 513, 481, 505, 553, 521, 516, 521, 515, 522, 494, 11, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 485, 508, 538, 502, 532, 503, 508, 473, 558, 523, 524, 496, 519, 510, 517, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 510, 483, 519, 537, 567, 485, 535, 525, 533, 531, 506, 542, 512, 487, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 477, 527, 503, 553, 482, 500, 457, 539, 498, 520, 504, 536, 538, 517, 498, 35, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 551, 537, 471, 518, 483, 506, 488, 485, 504, 513, 464, 523, 493, 499, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 542, 482, 513, 517, 502, 463, 495, 520, 474, 527, 486, 538, 516, 501, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 530, 522, 529, 525, 542, 522, 521, 489, 508, 476, 521, 534, 492, 499, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8250, 8237, 8346, 8196, 8037, 8064, 8196] avg=8192.0000 max=8346 max_rank=3 min=8037 min_rank=5 imbalance_ratio=1.018799 max_load_reduction=57190 max_load_reduction_pct=0.872650 moved_to_echo=32493 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[536, 499, 523, 477, 529, 458, 485, 537, 562, 537, 523, 486, 517, 495, 535, 491, 21, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 511, 510, 526, 524, 541, 535, 489, 470, 468, 512, 528, 490, 560, 517, 488, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 525, 471, 536, 504, 522, 482, 516, 507, 500, 477, 522, 498, 499, 508, 523, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 537, 491, 500, 469, 526, 521, 533, 508, 540, 547, 509, 523, 487, 533, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 495, 523, 522, 570, 500, 499, 540, 496, 515, 504, 513, 541, 508, 516, 491, 31, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 504, 537, 514, 496, 518, 518, 530, 529, 512, 480, 547, 547, 506, 481, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 482, 533, 510, 515, 511, 520, 491, 538, 463, 539, 544, 505, 498, 493, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 458, 530, 520, 508, 515, 509, 528, 518, 544, 461, 511, 485, 505, 497, 486, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8227, 8234, 8064, 8274, 8256, 8253, 8134, 8094] avg=8192.0000 max=8274 max_rank=3 min=8064 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32737 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[492, 499, 481, 496, 519, 530, 502, 489, 514, 514, 485, 528, 517, 496, 503, 507, 20, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 515, 531, 566, 481, 512, 513, 499, 463, 493, 533, 481, 529, 512, 537, 495, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 549, 508, 523, 505, 510, 500, 530, 498, 502, 540, 524, 515, 534, 517, 533, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 510, 485, 528, 518, 524, 538, 467, 517, 494, 525, 507, 500, 525, 503, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 467, 508, 527, 487, 545, 528, 518, 470, 451, 521, 514, 508, 550, 503, 525, 30, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 525, 493, 501, 489, 498, 572, 552, 503, 475, 477, 503, 490, 492, 468, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 521, 521, 537, 512, 528, 480, 500, 549, 497, 524, 514, 484, 520, 528, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 497, 548, 499, 506, 520, 482, 533, 526, 536, 539, 530, 509, 537, 508, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8110, 8196, 8280, 8195, 8138, 8064, 8235, 8318] avg=8192.0000 max=8318 max_rank=7 min=8064 min_rank=5 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32755 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[516, 483, 469, 527, 473, 518, 486, 551, 515, 521, 541, 516, 532, 497, 502, 502, 22, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 538, 498, 514, 507, 481, 531, 502, 486, 506, 525, 560, 507, 508, 509, 524, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 533, 465, 490, 506, 547, 525, 515, 496, 500, 494, 482, 517, 491, 512, 551, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 497, 512, 519, 550, 515, 483, 502, 537, 488, 533, 489, 535, 489, 473, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 500, 509, 475, 510, 512, 513, 517, 515, 528, 532, 482, 468, 516, 521, 535, 32, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 542, 521, 555, 502, 528, 480, 538, 519, 504, 512, 536, 530, 471, 498, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 530, 479, 472, 525, 522, 535, 543, 476, 530, 508, 553, 527, 509, 527, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 494, 483, 525, 513, 525, 504, 494, 506, 509, 479, 546, 495, 546, 511, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8218, 8137, 8136, 8203, 8245, 8240, 8168] avg=8192.0000 max=8245 max_rank=5 min=8136 min_rank=3 imbalance_ratio=1.006470 max_load_reduction=57291 max_load_reduction_pct=0.874191 moved_to_echo=32856 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[492, 506, 511, 525, 510, 529, 515, 533, 518, 500, 499, 481, 478, 496, 523, 474, 20, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 488, 482, 541, 481, 522, 522, 529, 565, 489, 521, 499, 500, 462, 489, 509, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 548, 534, 492, 542, 508, 502, 523, 498, 533, 546, 535, 496, 491, 474, 506, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 526, 490, 513, 515, 502, 504, 530, 506, 516, 534, 506, 510, 503, 505, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 509, 544, 511, 501, 527, 510, 465, 473, 549, 497, 555, 471, 494, 480, 499, 31, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 529, 507, 535, 544, 485, 516, 527, 536, 497, 520, 534, 490, 535, 500, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 554, 542, 522, 560, 500, 529, 502, 530, 549, 510, 542, 541, 485, 474, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 493, 497, 493, 518, 504, 477, 521, 505, 529, 497, 540, 502, 507, 473, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8128, 8138, 8226, 8138, 8127, 8323, 8382, 8074] avg=8192.0000 max=8382 max_rank=6 min=8074 min_rank=7 imbalance_ratio=1.023193 max_load_reduction=57154 max_load_reduction_pct=0.872101 moved_to_echo=32906 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[490, 512, 540, 509, 542, 486, 505, 520, 495, 534, 483, 496, 488, 499, 551, 515, 24, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 548, 528, 475, 510, 503, 513, 481, 505, 553, 521, 516, 521, 515, 522, 494, 11, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 485, 508, 538, 502, 532, 503, 508, 473, 558, 523, 524, 496, 519, 510, 517, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 510, 483, 519, 537, 567, 485, 535, 525, 533, 531, 506, 542, 512, 487, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 477, 527, 503, 553, 482, 500, 457, 539, 498, 520, 504, 536, 538, 517, 498, 35, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 551, 537, 471, 518, 483, 506, 488, 485, 504, 513, 464, 523, 493, 499, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 542, 482, 513, 517, 502, 463, 495, 520, 474, 527, 486, 538, 516, 501, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 530, 522, 529, 525, 542, 522, 521, 489, 508, 476, 521, 534, 492, 499, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8250, 8237, 8346, 8196, 8037, 8064, 8196] avg=8192.0000 max=8346 max_rank=3 min=8037 min_rank=5 imbalance_ratio=1.018799 max_load_reduction=57190 max_load_reduction_pct=0.872650 moved_to_echo=32493 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[484, 547, 461, 547, 533, 528, 515, 487, 520, 483, 515, 488, 496, 536, 531, 554, 21, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 497, 559, 541, 474, 489, 514, 500, 499, 518, 522, 517, 493, 492, 539, 520, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 517, 519, 488, 491, 486, 526, 521, 495, 537, 518, 512, 474, 521, 542, 525, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 494, 513, 491, 529, 506, 498, 505, 540, 513, 512, 483, 543, 503, 503, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 471, 462, 530, 516, 459, 506, 534, 489, 519, 494, 520, 534, 494, 529, 518, 32, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 512, 557, 531, 511, 523, 508, 514, 541, 488, 514, 487, 522, 534, 471, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 549, 512, 504, 522, 503, 527, 480, 499, 496, 517, 496, 509, 484, 504, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 488, 536, 489, 528, 544, 513, 490, 536, 539, 527, 503, 508, 533, 500, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8263, 8207, 8172, 8169, 8123, 8270, 8101, 8231] avg=8192.0000 max=8270 max_rank=5 min=8101 min_rank=6 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32725 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[529, 499, 501, 489, 517, 566, 528, 492, 516, 469, 458, 511, 521, 516, 515, 548, 20, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 509, 499, 508, 535, 523, 515, 481, 492, 498, 518, 491, 528, 557, 492, 465, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 501, 482, 506, 479, 524, 539, 542, 506, 511, 529, 500, 536, 505, 504, 494, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 499, 514, 476, 546, 517, 508, 532, 522, 524, 490, 521, 498, 508, 471, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 460, 513, 546, 502, 537, 519, 468, 552, 498, 495, 514, 500, 499, 530, 555, 32, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 467, 495, 534, 526, 485, 491, 519, 519, 527, 479, 511, 501, 534, 507, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 503, 508, 538, 503, 515, 505, 526, 518, 529, 531, 545, 510, 514, 479, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 470, 510, 522, 480, 511, 529, 518, 531, 516, 577, 466, 509, 532, 504, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8166, 8146, 8157, 8235, 8152, 8272, 8196] avg=8192.0000 max=8272 max_rank=6 min=8146 min_rank=2 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32855 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 256, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 3968, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 3968, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 3968, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[549, 520, 496, 494, 517, 496, 533, 520, 479, 536, 505, 505, 498, 548, 532, 566, 510, 518, 509, 544, 525, 476, 533, 532, 456, 522, 520, 522, 478, 478, 467, 491, 448, 527, 508, 497, 493, 540, 485, 524, 541, 502, 542, 544, 566, 470, 499, 465, 514, 547, 551, 529, 505, 496, 532, 516, 525, 511, 524, 529, 534, 501, 491, 513, 541, 495, 538, 491, 506, 498, 502, 519, 508, 464, 505, 495, 505, 502, 523, 572, 471, 527, 528, 466, 534, 471, 480, 500, 512, 512, 510, 511, 527, 543, 499, 518, 520, 508, 482, 522, 492, 487, 494, 525, 504, 510, 511, 534, 505, 531, 536, 510, 520, 497, 533, 532, 469, 526, 499, 539, 501, 506, 540, 535, 514, 532, 496, 509] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[536, 551, 507, 509, 483, 533, 535, 521, 515, 537, 555, 508, 510, 497, 528, 549, 471, 497, 542, 498, 520, 507, 509, 483, 493, 519, 472, 544, 519, 492, 468, 520, 505, 550, 503, 527, 523, 523, 532, 489, 522, 524, 505, 490, 492, 510, 503, 522, 527, 506, 509, 519, 471, 499, 474, 537, 514, 486, 471, 518, 504, 518, 533, 520, 485, 505, 497, 483, 530, 486, 504, 570, 508, 500, 481, 477, 478, 502, 527, 488, 474, 530, 528, 544, 495, 567, 522, 521, 511, 482, 521, 500, 488, 498, 509, 562, 511, 475, 509, 535, 513, 568, 567, 465, 473, 492, 508, 535, 526, 518, 551, 507, 526, 472, 551, 503, 485, 511, 514, 557, 504, 517, 513, 524, 509, 546, 499, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[523, 527, 521, 520, 514, 482, 525, 506, 497, 506, 493, 511, 502, 488, 520, 489, 518, 529, 515, 522, 512, 485, 506, 503, 500, 497, 513, 535, 497, 498, 497, 520, 537, 527, 517, 501, 518, 540, 557, 524, 495, 499, 484, 532, 478, 476, 486, 502, 485, 485, 512, 493, 480, 525, 541, 511, 547, 556, 503, 551, 536, 530, 508, 498, 498, 506, 552, 506, 485, 505, 496, 542, 495, 540, 507, 549, 531, 517, 514, 498, 500, 511, 500, 494, 511, 488, 512, 481, 550, 512, 509, 531, 521, 504, 500, 513, 489, 469, 511, 539, 510, 518, 523, 523, 520, 525, 522, 492, 569, 484, 502, 497, 519, 518, 515, 573, 439, 520, 508, 518, 500, 519, 506, 470, 577, 555, 499, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[502, 525, 492, 506, 536, 494, 530, 504, 499, 520, 494, 514, 542, 522, 536, 490, 514, 501, 506, 525, 506, 526, 544, 498, 507, 525, 504, 562, 513, 531, 499, 531, 523, 523, 498, 497, 484, 501, 516, 501, 500, 529, 520, 501, 486, 521, 517, 530, 486, 527, 532, 504, 523, 522, 514, 492, 495, 566, 558, 505, 517, 495, 511, 529, 534, 480, 519, 541, 524, 470, 504, 520, 524, 543, 526, 474, 507, 511, 479, 515, 534, 517, 483, 492, 523, 535, 531, 504, 489, 530, 503, 496, 500, 510, 517, 506, 543, 520, 477, 507, 508, 492, 507, 530, 531, 446, 502, 570, 509, 485, 518, 488, 530, 493, 506, 553, 514, 509, 440, 493, 528, 511, 555, 522, 492, 504, 464, 527] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[536, 551, 507, 509, 483, 533, 535, 521, 515, 537, 555, 508, 510, 497, 528, 549, 471, 497, 542, 498, 520, 507, 509, 483, 493, 519, 472, 544, 519, 492, 468, 520, 505, 550, 503, 527, 523, 523, 532, 489, 522, 524, 505, 490, 492, 510, 503, 522, 527, 506, 509, 519, 471, 499, 474, 537, 514, 486, 471, 518, 504, 518, 533, 520, 485, 505, 497, 483, 530, 486, 504, 570, 508, 500, 481, 477, 478, 502, 527, 488, 474, 530, 528, 544, 495, 567, 522, 521, 511, 482, 521, 500, 488, 498, 509, 562, 511, 475, 509, 535, 513, 568, 567, 465, 473, 492, 508, 535, 526, 518, 551, 507, 526, 472, 551, 503, 485, 511, 514, 557, 504, 517, 513, 524, 509, 546, 499, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[525, 537, 480, 523, 519, 549, 490, 504, 516, 537, 526, 505, 503, 506, 489, 494, 542, 487, 522, 511, 531, 484, 527, 520, 530, 533, 480, 522, 527, 476, 517, 489, 459, 516, 503, 507, 501, 489, 526, 537, 537, 464, 527, 501, 516, 511, 512, 516, 565, 534, 520, 477, 488, 518, 493, 490, 543, 505, 504, 495, 512, 512, 486, 515, 522, 495, 511, 511, 505, 526, 497, 532, 513, 528, 495, 511, 531, 476, 498, 494, 537, 496, 493, 502, 497, 563, 489, 537, 506, 516, 525, 492, 554, 461, 515, 549, 502, 524, 494, 540, 493, 514, 542, 507, 537, 531, 530, 507, 501, 495, 535, 489, 497, 511, 515, 551, 486, 544, 514, 503, 513, 520, 481, 512, 556, 533, 477, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[522, 492, 483, 509, 506, 540, 534, 527, 522, 510, 505, 497, 547, 485, 494, 532, 526, 528, 542, 532, 509, 514, 499, 513, 522, 528, 497, 492, 529, 487, 536, 498, 493, 537, 525, 449, 515, 525, 522, 512, 557, 533, 550, 504, 490, 499, 517, 505, 492, 507, 528, 544, 515, 449, 474, 491, 476, 509, 552, 488, 494, 519, 543, 524, 506, 490, 537, 501, 534, 487, 513, 521, 543, 527, 491, 512, 526, 494, 488, 543, 518, 517, 507, 529, 550, 491, 516, 535, 484, 484, 518, 522, 508, 515, 489, 546, 463, 498, 511, 523, 557, 475, 466, 548, 515, 518, 491, 508, 529, 497, 506, 495, 517, 476, 485, 504, 501, 488, 502, 527, 529, 533, 543, 492, 494, 555, 528, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[515, 501, 543, 476, 496, 532, 526, 539, 489, 514, 511, 548, 538, 509, 548, 492, 520, 475, 516, 528, 533, 491, 504, 472, 501, 474, 503, 495, 475, 464, 477, 502, 528, 530, 492, 471, 505, 503, 469, 528, 533, 518, 523, 526, 553, 531, 535, 490, 526, 548, 490, 557, 522, 509, 514, 524, 483, 535, 508, 498, 513, 518, 525, 519, 514, 544, 537, 520, 513, 518, 496, 505, 526, 478, 498, 498, 497, 499, 545, 507, 538, 534, 521, 532, 488, 498, 522, 575, 498, 520, 482, 531, 478, 504, 532, 494, 470, 518, 509, 524, 509, 513, 546, 531, 507, 484, 509, 520, 519, 491, 504, 535, 514, 514, 501, 536, 524, 515, 519, 512, 529, 492, 487, 473, 526, 499, 526, 507] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[490, 538, 489, 523, 495, 520, 496, 485, 496, 521, 542, 515, 509, 523, 524, 476, 521, 498, 519, 517, 559, 483, 484, 506, 503, 499, 519, 536, 514, 526, 546, 492, 514, 510, 513, 510, 527, 512, 545, 500, 506, 519, 533, 489, 492, 517, 555, 519, 555, 530, 509, 490, 463, 471, 487, 544, 509, 527, 476, 537, 483, 530, 474, 534, 505, 492, 544, 505, 526, 522, 521, 502, 520, 509, 550, 497, 481, 550, 524, 496, 544, 511, 483, 534, 469, 494, 523, 520, 500, 482, 505, 536, 521, 574, 464, 527, 507, 512, 503, 504, 543, 539, 505, 531, 499, 527, 508, 496, 550, 491, 500, 476, 517, 518, 533, 500, 498, 506, 500, 499, 516, 499, 515, 522, 541, 482, 513, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[525, 537, 480, 523, 519, 549, 490, 504, 516, 537, 526, 505, 503, 506, 489, 494, 542, 487, 522, 511, 531, 484, 527, 520, 530, 533, 480, 522, 527, 476, 517, 489, 459, 516, 503, 507, 501, 489, 526, 537, 537, 464, 527, 501, 516, 511, 512, 516, 565, 534, 520, 477, 488, 518, 493, 490, 543, 505, 504, 495, 512, 512, 486, 515, 522, 495, 511, 511, 505, 526, 497, 532, 513, 528, 495, 511, 531, 476, 498, 494, 537, 496, 493, 502, 497, 563, 489, 537, 506, 516, 525, 492, 554, 461, 515, 549, 502, 524, 494, 540, 493, 514, 542, 507, 537, 531, 530, 507, 501, 495, 535, 489, 497, 511, 515, 551, 486, 544, 514, 503, 513, 520, 481, 512, 556, 533, 477, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[515, 467, 543, 476, 496, 532, 526, 539, 489, 514, 511, 548, 538, 509, 548, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 475, 516, 528, 533, 491, 504, 472, 501, 474, 503, 495, 475, 464, 477, 502, 34, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 530, 492, 471, 505, 503, 469, 528, 533, 518, 523, 526, 553, 531, 535, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 548, 490, 557, 522, 509, 514, 524, 483, 523, 508, 498, 513, 518, 525, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 544, 537, 520, 513, 518, 496, 505, 526, 478, 498, 498, 497, 499, 545, 507, 9, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 534, 521, 532, 488, 498, 522, 575, 498, 520, 482, 531, 478, 504, 532, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 518, 509, 524, 509, 513, 546, 531, 507, 484, 509, 520, 519, 491, 504, 535, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 514, 501, 519, 524, 515, 519, 512, 529, 492, 487, 473, 526, 499, 526, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8243, 7972, 8235, 8277, 8212, 8244, 8196, 8157] avg=8192.0000 max=8277 max_rank=3 min=7972 min_rank=1 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32809 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[502, 489, 492, 506, 536, 494, 530, 504, 499, 520, 494, 514, 542, 522, 536, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 501, 506, 525, 506, 526, 544, 498, 507, 525, 504, 562, 513, 531, 499, 531, 36, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 523, 498, 497, 484, 501, 516, 501, 500, 529, 520, 501, 486, 521, 517, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 527, 532, 504, 523, 522, 514, 492, 495, 554, 558, 505, 517, 495, 511, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 480, 519, 541, 524, 470, 504, 520, 524, 543, 526, 474, 507, 511, 479, 515, 10, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 517, 483, 492, 523, 535, 531, 504, 489, 530, 503, 496, 500, 510, 517, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 520, 477, 507, 508, 492, 507, 530, 531, 446, 502, 570, 509, 485, 518, 488, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 493, 506, 534, 514, 509, 440, 493, 528, 511, 555, 522, 492, 504, 464, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8170, 8337, 8147, 8264, 8189, 8167, 8140, 8122] avg=8192.0000 max=8337 max_rank=1 min=8122 min_rank=7 imbalance_ratio=1.017700 max_load_reduction=57199 max_load_reduction_pct=0.872787 moved_to_echo=32618 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[515, 501, 543, 476, 496, 532, 526, 539, 489, 514, 511, 548, 538, 509, 548, 492, 520, 475, 516, 528, 533, 491, 504, 472, 501, 474, 503, 495, 475, 464, 477, 502, 528, 530, 492, 471, 505, 503, 469, 528, 533, 518, 523, 526, 553, 531, 535, 490, 526, 548, 490, 557, 522, 509, 514, 524, 483, 535, 508, 498, 513, 518, 525, 519, 514, 544, 537, 520, 513, 518, 496, 505, 526, 478, 498, 498, 497, 499, 545, 507, 538, 534, 521, 532, 488, 498, 522, 575, 498, 520, 482, 531, 478, 504, 532, 494, 470, 518, 509, 524, 509, 513, 546, 531, 507, 484, 509, 520, 519, 491, 504, 535, 514, 514, 501, 536, 524, 515, 519, 512, 529, 492, 487, 473, 526, 499, 526, 507] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[549, 481, 496, 494, 517, 496, 533, 520, 479, 536, 505, 505, 498, 548, 532, 566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 518, 509, 544, 525, 476, 533, 532, 456, 522, 520, 522, 478, 478, 467, 491, 39, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 448, 521, 508, 497, 493, 540, 485, 524, 541, 502, 542, 544, 566, 470, 499, 465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 547, 551, 529, 505, 496, 532, 516, 525, 493, 524, 529, 534, 501, 491, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 495, 538, 491, 506, 498, 502, 519, 508, 464, 505, 495, 505, 502, 523, 572, 13, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 527, 528, 466, 534, 471, 480, 500, 512, 512, 510, 511, 527, 543, 499, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 508, 482, 522, 492, 487, 494, 525, 504, 510, 511, 534, 505, 531, 536, 510, 7, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 497, 533, 507, 469, 526, 499, 539, 501, 506, 540, 535, 514, 532, 496, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8255, 8132, 8145, 8300, 8188, 8103, 8190, 8223] avg=8192.0000 max=8300 max_rank=3 min=8103 min_rank=5 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32704 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[536, 514, 507, 509, 483, 533, 535, 521, 515, 537, 555, 508, 510, 497, 528, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 497, 542, 498, 520, 507, 509, 483, 493, 519, 472, 544, 519, 492, 468, 520, 37, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 550, 503, 527, 523, 523, 532, 489, 522, 524, 505, 490, 492, 510, 503, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 506, 509, 519, 471, 499, 474, 537, 514, 476, 471, 518, 504, 518, 533, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 505, 497, 483, 530, 486, 504, 570, 508, 500, 481, 477, 478, 502, 527, 488, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 530, 528, 544, 495, 567, 522, 521, 511, 482, 521, 500, 488, 498, 509, 559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 475, 509, 535, 513, 568, 567, 465, 473, 492, 508, 535, 526, 518, 551, 507, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 472, 551, 486, 485, 511, 514, 557, 504, 517, 513, 524, 509, 546, 499, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8337, 8099, 8220, 8096, 8037, 8249, 8259, 8239] avg=8192.0000 max=8337 max_rank=0 min=8037 min_rank=4 imbalance_ratio=1.017700 max_load_reduction=57199 max_load_reduction_pct=0.872787 moved_to_echo=32784 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[502, 489, 492, 506, 536, 494, 530, 504, 499, 520, 494, 514, 542, 522, 536, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 501, 506, 525, 506, 526, 544, 498, 507, 525, 504, 562, 513, 531, 499, 531, 36, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 523, 498, 497, 484, 501, 516, 501, 500, 529, 520, 501, 486, 521, 517, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 527, 532, 504, 523, 522, 514, 492, 495, 554, 558, 505, 517, 495, 511, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 480, 519, 541, 524, 470, 504, 520, 524, 543, 526, 474, 507, 511, 479, 515, 10, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 517, 483, 492, 523, 535, 531, 504, 489, 530, 503, 496, 500, 510, 517, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 520, 477, 507, 508, 492, 507, 530, 531, 446, 502, 570, 509, 485, 518, 488, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 493, 506, 534, 514, 509, 440, 493, 528, 511, 555, 522, 492, 504, 464, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8170, 8337, 8147, 8264, 8189, 8167, 8140, 8122] avg=8192.0000 max=8337 max_rank=1 min=8122 min_rank=7 imbalance_ratio=1.017700 max_load_reduction=57199 max_load_reduction_pct=0.872787 moved_to_echo=32618 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[523, 491, 521, 520, 514, 482, 525, 506, 497, 506, 493, 511, 502, 488, 520, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 529, 515, 522, 512, 485, 506, 503, 500, 497, 513, 535, 497, 498, 497, 520, 36, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 527, 517, 501, 518, 540, 557, 524, 495, 499, 484, 532, 478, 476, 486, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 485, 512, 493, 480, 525, 541, 511, 547, 544, 503, 551, 536, 530, 508, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 506, 552, 506, 485, 505, 496, 542, 495, 540, 507, 549, 531, 517, 514, 498, 10, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 511, 500, 494, 511, 488, 512, 481, 550, 512, 509, 531, 521, 504, 500, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 469, 511, 539, 510, 518, 523, 523, 520, 525, 522, 492, 569, 484, 502, 497, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 518, 515, 554, 439, 520, 508, 518, 500, 519, 506, 470, 577, 555, 499, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8088, 8192, 8173, 8249, 8259, 8134, 8200, 8241] avg=8192.0000 max=8259 max_rank=4 min=8088 min_rank=0 imbalance_ratio=1.008179 max_load_reduction=57277 max_load_reduction_pct=0.873978 moved_to_echo=32834 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[525, 500, 480, 523, 519, 549, 490, 504, 516, 537, 526, 505, 503, 506, 489, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 487, 522, 511, 531, 484, 527, 520, 530, 533, 480, 522, 527, 476, 517, 489, 37, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 459, 516, 503, 507, 501, 489, 526, 537, 537, 464, 527, 501, 516, 511, 512, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 565, 534, 520, 477, 488, 518, 493, 490, 543, 495, 504, 495, 512, 512, 486, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 495, 511, 511, 505, 526, 497, 532, 513, 528, 495, 511, 531, 476, 498, 494, 10, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 496, 493, 502, 497, 563, 489, 537, 506, 516, 525, 492, 554, 461, 515, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 524, 494, 540, 493, 514, 542, 507, 537, 531, 530, 507, 501, 495, 535, 489, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 511, 515, 532, 486, 544, 514, 503, 513, 520, 481, 512, 556, 533, 477, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8166, 8244, 8122, 8147, 8162, 8229, 8247, 8219] avg=8192.0000 max=8247 max_rank=6 min=8122 min_rank=2 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32857 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[490, 501, 489, 523, 495, 520, 496, 485, 496, 521, 542, 515, 509, 523, 524, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 498, 519, 517, 559, 483, 484, 506, 503, 499, 519, 536, 514, 526, 546, 492, 37, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 510, 513, 510, 527, 512, 545, 500, 506, 519, 533, 489, 492, 517, 555, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 530, 509, 490, 463, 471, 487, 544, 509, 516, 476, 537, 483, 530, 474, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 492, 544, 505, 526, 522, 521, 502, 520, 509, 550, 497, 481, 550, 524, 496, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 511, 483, 534, 469, 494, 523, 520, 500, 482, 505, 536, 521, 574, 464, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 512, 503, 504, 543, 539, 505, 531, 499, 527, 508, 496, 550, 491, 500, 476, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 518, 533, 483, 498, 506, 500, 499, 516, 499, 515, 522, 541, 482, 513, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8105, 8267, 8261, 8108, 8260, 8184, 8198, 8153] avg=8192.0000 max=8267 max_rank=1 min=8105 min_rank=0 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32795 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[549, 481, 496, 494, 517, 496, 533, 520, 479, 536, 505, 505, 498, 548, 532, 566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 518, 509, 544, 525, 476, 533, 532, 456, 522, 520, 522, 478, 478, 467, 491, 39, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 448, 521, 508, 497, 493, 540, 485, 524, 541, 502, 542, 544, 566, 470, 499, 465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 547, 551, 529, 505, 496, 532, 516, 525, 493, 524, 529, 534, 501, 491, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 495, 538, 491, 506, 498, 502, 519, 508, 464, 505, 495, 505, 502, 523, 572, 13, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 527, 528, 466, 534, 471, 480, 500, 512, 512, 510, 511, 527, 543, 499, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 508, 482, 522, 492, 487, 494, 525, 504, 510, 511, 534, 505, 531, 536, 510, 7, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 497, 533, 507, 469, 526, 499, 539, 501, 506, 540, 535, 514, 532, 496, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8255, 8132, 8145, 8300, 8188, 8103, 8190, 8223] avg=8192.0000 max=8300 max_rank=3 min=8103 min_rank=5 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32704 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[522, 459, 483, 509, 506, 540, 534, 527, 522, 510, 505, 497, 547, 485, 494, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 528, 542, 532, 509, 514, 499, 513, 522, 528, 497, 492, 529, 487, 536, 498, 33, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 537, 525, 449, 515, 525, 522, 512, 557, 533, 550, 504, 490, 499, 517, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 507, 528, 544, 515, 449, 474, 491, 476, 499, 552, 488, 494, 519, 543, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 490, 537, 501, 534, 487, 513, 521, 543, 527, 491, 512, 526, 494, 488, 543, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 517, 507, 529, 550, 491, 516, 535, 484, 484, 518, 522, 508, 515, 489, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 463, 498, 511, 523, 557, 475, 466, 548, 515, 518, 491, 508, 529, 497, 506, 495, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 476, 485, 487, 501, 488, 502, 527, 529, 533, 543, 492, 494, 555, 528, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8293, 8233, 8095, 8229, 8226, 8106, 8182] avg=8192.0000 max=8293 max_rank=1 min=8095 min_rank=3 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32743 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[525, 500, 480, 523, 519, 549, 490, 504, 516, 537, 526, 505, 503, 506, 489, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 487, 522, 511, 531, 484, 527, 520, 530, 533, 480, 522, 527, 476, 517, 489, 37, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 459, 516, 503, 507, 501, 489, 526, 537, 537, 464, 527, 501, 516, 511, 512, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 565, 534, 520, 477, 488, 518, 493, 490, 543, 495, 504, 495, 512, 512, 486, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 495, 511, 511, 505, 526, 497, 532, 513, 528, 495, 511, 531, 476, 498, 494, 10, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 496, 493, 502, 497, 563, 489, 537, 506, 516, 525, 492, 554, 461, 515, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 524, 494, 540, 493, 514, 542, 507, 537, 531, 530, 507, 501, 495, 535, 489, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 511, 515, 532, 486, 544, 514, 503, 513, 520, 481, 512, 556, 533, 477, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8166, 8244, 8122, 8147, 8162, 8229, 8247, 8219] avg=8192.0000 max=8247 max_rank=6 min=8122 min_rank=2 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32857 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[515, 467, 543, 476, 496, 532, 526, 539, 489, 514, 511, 548, 538, 509, 548, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 475, 516, 528, 533, 491, 504, 472, 501, 474, 503, 495, 475, 464, 477, 502, 34, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 530, 492, 471, 505, 503, 469, 528, 533, 518, 523, 526, 553, 531, 535, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 548, 490, 557, 522, 509, 514, 524, 483, 523, 508, 498, 513, 518, 525, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 544, 537, 520, 513, 518, 496, 505, 526, 478, 498, 498, 497, 499, 545, 507, 9, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 534, 521, 532, 488, 498, 522, 575, 498, 520, 482, 531, 478, 504, 532, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 518, 509, 524, 509, 513, 546, 531, 507, 484, 509, 520, 519, 491, 504, 535, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 514, 501, 519, 524, 515, 519, 512, 529, 492, 487, 473, 526, 499, 526, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8243, 7972, 8235, 8277, 8212, 8244, 8196, 8157] avg=8192.0000 max=8277 max_rank=3 min=7972 min_rank=1 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32809 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 3968, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 3968, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 3968, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4224, 4224, 3968, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 3968, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 3968, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 3968, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4224, 3968, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 3968, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4224, 4224, 3968, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4352, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4224, 3968, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[516, 530, 570, 521, 484, 515, 500, 499, 525, 546, 491, 506, 496, 524, 525, 494, 476, 510, 502, 506, 543, 510, 504, 480, 487, 545, 508, 514, 486, 503, 536, 522, 529, 525, 525, 502, 541, 534, 513, 544, 489, 529, 494, 490, 490, 532, 526, 564, 495, 513, 478, 463, 472, 513, 499, 501, 542, 513, 463, 543, 500, 494, 511, 496, 510, 526, 500, 543, 511, 542, 497, 506, 477, 581, 487, 527, 481, 503, 529, 519, 539, 500, 527, 474, 520, 498, 537, 528, 532, 485, 497, 515, 513, 493, 513, 478, 527, 514, 485, 556, 488, 525, 533, 491, 505, 526, 520, 529, 529, 461, 508, 517, 520, 510, 543, 476, 496, 482, 521, 540, 530, 509, 529, 548, 536, 505, 500, 492] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[483, 473, 520, 508, 475, 521, 522, 502, 540, 457, 486, 513, 511, 530, 523, 494, 519, 501, 523, 532, 508, 522, 513, 528, 490, 511, 513, 489, 527, 526, 496, 524, 542, 535, 525, 494, 476, 526, 484, 505, 548, 530, 499, 486, 499, 552, 518, 524, 537, 486, 534, 518, 507, 522, 482, 506, 523, 488, 532, 560, 535, 499, 534, 516, 522, 554, 508, 514, 545, 495, 494, 448, 495, 553, 543, 521, 494, 506, 535, 553, 524, 527, 561, 482, 497, 535, 531, 521, 508, 544, 520, 508, 521, 512, 502, 506, 521, 516, 498, 551, 535, 484, 471, 531, 452, 501, 502, 485, 482, 473, 499, 487, 527, 523, 499, 499, 479, 530, 530, 523, 526, 505, 507, 503, 526, 518, 466, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[510, 555, 502, 506, 509, 500, 494, 524, 477, 485, 507, 484, 502, 503, 566, 516, 504, 518, 494, 497, 536, 522, 512, 518, 506, 533, 534, 527, 481, 507, 539, 522, 568, 524, 506, 502, 508, 498, 564, 520, 534, 510, 491, 516, 523, 501, 508, 519, 511, 526, 507, 497, 490, 483, 521, 524, 500, 520, 507, 520, 503, 494, 500, 492, 513, 503, 520, 487, 519, 485, 456, 484, 487, 530, 501, 514, 511, 536, 520, 503, 537, 544, 468, 534, 517, 539, 506, 498, 558, 534, 519, 480, 509, 510, 530, 488, 542, 486, 484, 502, 545, 499, 509, 495, 548, 497, 544, 541, 531, 494, 499, 493, 540, 483, 477, 523, 534, 504, 525, 463, 528, 496, 511, 504, 530, 539, 523, 530] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[505, 534, 503, 509, 548, 498, 493, 507, 548, 493, 502, 539, 505, 495, 523, 552, 496, 527, 549, 524, 533, 511, 465, 479, 493, 495, 517, 513, 478, 523, 500, 530, 509, 533, 500, 541, 515, 524, 514, 482, 497, 530, 541, 542, 498, 473, 515, 525, 514, 508, 508, 501, 539, 549, 519, 492, 520, 543, 502, 516, 508, 489, 540, 490, 491, 460, 491, 533, 532, 515, 530, 513, 539, 520, 513, 509, 518, 558, 566, 495, 482, 528, 497, 544, 516, 523, 541, 472, 523, 507, 538, 487, 523, 490, 466, 511, 524, 511, 539, 499, 504, 486, 511, 508, 523, 430, 517, 494, 524, 472, 499, 497, 527, 500, 565, 501, 511, 479, 543, 503, 514, 513, 535, 517, 496, 507, 511, 481] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[426, 512, 519, 540, 534, 529, 496, 475, 483, 522, 494, 533, 523, 511, 514, 521, 506, 481, 503, 513, 518, 494, 522, 520, 506, 511, 537, 540, 514, 484, 494, 488, 497, 517, 511, 514, 517, 525, 467, 530, 548, 510, 534, 498, 505, 550, 555, 504, 497, 484, 545, 471, 538, 496, 528, 517, 530, 489, 503, 487, 498, 500, 505, 552, 485, 509, 503, 525, 525, 528, 542, 557, 507, 511, 541, 512, 504, 530, 539, 523, 539, 520, 527, 517, 496, 530, 520, 481, 477, 490, 500, 529, 486, 498, 519, 505, 547, 526, 497, 506, 493, 548, 529, 513, 484, 508, 459, 561, 488, 520, 445, 543, 530, 493, 507, 536, 522, 514, 509, 523, 495, 488, 477, 523, 533, 512, 520, 527] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[529, 525, 530, 519, 483, 512, 502, 497, 506, 484, 525, 485, 509, 481, 493, 561, 558, 509, 492, 486, 520, 489, 465, 500, 520, 515, 500, 509, 492, 560, 511, 501, 526, 471, 509, 501, 498, 531, 517, 501, 505, 541, 516, 472, 523, 503, 509, 563, 512, 488, 530, 475, 481, 528, 529, 521, 497, 511, 497, 520, 499, 494, 525, 486, 500, 520, 511, 490, 547, 535, 472, 513, 524, 537, 515, 563, 501, 550, 469, 522, 492, 526, 526, 560, 524, 496, 523, 497, 563, 497, 505, 516, 486, 512, 528, 505, 518, 491, 547, 523, 485, 491, 526, 519, 491, 545, 529, 498, 508, 515, 536, 496, 498, 513, 527, 480, 462, 555, 539, 516, 543, 531, 514, 496, 534, 566, 479, 493] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[483, 473, 520, 508, 475, 521, 522, 502, 540, 457, 486, 513, 511, 530, 523, 494, 519, 501, 523, 532, 508, 522, 513, 528, 490, 511, 513, 489, 527, 526, 496, 524, 542, 535, 525, 494, 476, 526, 484, 505, 548, 530, 499, 486, 499, 552, 518, 524, 537, 486, 534, 518, 507, 522, 482, 506, 523, 488, 532, 560, 535, 499, 534, 516, 522, 554, 508, 514, 545, 495, 494, 448, 495, 553, 543, 521, 494, 506, 535, 553, 524, 527, 561, 482, 497, 535, 531, 521, 508, 544, 520, 508, 521, 512, 502, 506, 521, 516, 498, 551, 535, 484, 471, 531, 452, 501, 502, 485, 482, 473, 499, 487, 527, 523, 499, 499, 479, 530, 530, 523, 526, 505, 507, 503, 526, 518, 466, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[512, 437, 542, 491, 505, 483, 554, 463, 498, 524, 531, 515, 522, 498, 527, 504, 492, 480, 478, 526, 514, 499, 536, 542, 527, 539, 544, 502, 528, 527, 489, 508, 500, 497, 514, 522, 511, 470, 541, 513, 483, 509, 512, 493, 534, 552, 536, 496, 523, 541, 505, 523, 509, 520, 487, 539, 493, 456, 492, 515, 523, 513, 474, 513, 507, 504, 505, 477, 529, 444, 478, 493, 490, 509, 540, 560, 506, 516, 519, 521, 535, 511, 526, 491, 512, 517, 520, 506, 515, 502, 508, 532, 525, 498, 536, 536, 487, 526, 503, 493, 555, 494, 534, 532, 501, 488, 570, 479, 551, 534, 497, 496, 557, 522, 496, 517, 520, 543, 540, 510, 545, 504, 505, 490, 506, 506, 513, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[426, 512, 519, 540, 534, 529, 496, 475, 483, 522, 494, 533, 523, 511, 514, 521, 506, 481, 503, 513, 518, 494, 522, 520, 506, 511, 537, 540, 514, 484, 494, 488, 497, 517, 511, 514, 517, 525, 467, 530, 548, 510, 534, 498, 505, 550, 555, 504, 497, 484, 545, 471, 538, 496, 528, 517, 530, 489, 503, 487, 498, 500, 505, 552, 485, 509, 503, 525, 525, 528, 542, 557, 507, 511, 541, 512, 504, 530, 539, 523, 539, 520, 527, 517, 496, 530, 520, 481, 477, 490, 500, 529, 486, 498, 519, 505, 547, 526, 497, 506, 493, 548, 529, 513, 484, 508, 459, 561, 488, 520, 445, 543, 530, 493, 507, 536, 522, 514, 509, 523, 495, 488, 477, 523, 533, 512, 520, 527] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[510, 555, 502, 506, 509, 500, 494, 524, 477, 485, 507, 484, 502, 503, 566, 516, 504, 518, 494, 497, 536, 522, 512, 518, 506, 533, 534, 527, 481, 507, 539, 522, 568, 524, 506, 502, 508, 498, 564, 520, 534, 510, 491, 516, 523, 501, 508, 519, 511, 526, 507, 497, 490, 483, 521, 524, 500, 520, 507, 520, 503, 494, 500, 492, 513, 503, 520, 487, 519, 485, 456, 484, 487, 530, 501, 514, 511, 536, 520, 503, 537, 544, 468, 534, 517, 539, 506, 498, 558, 534, 519, 480, 509, 510, 530, 488, 542, 486, 484, 502, 545, 499, 509, 495, 548, 497, 544, 541, 531, 494, 499, 493, 540, 483, 477, 523, 534, 504, 525, 463, 528, 496, 511, 504, 530, 539, 523, 530] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[503, 513, 489, 520, 520, 506, 485, 524, 516, 498, 503, 486, 524, 503, 481, 501, 502, 529, 534, 490, 532, 509, 498, 489, 491, 503, 504, 555, 527, 506, 503, 548, 509, 496, 522, 517, 521, 525, 531, 525, 523, 524, 509, 534, 504, 492, 517, 534, 535, 482, 510, 545, 540, 520, 507, 565, 521, 517, 518, 502, 448, 494, 554, 473, 487, 546, 517, 472, 552, 524, 539, 542, 562, 531, 513, 528, 473, 497, 474, 459, 516, 538, 520, 520, 505, 498, 516, 506, 522, 544, 510, 517, 556, 514, 497, 484, 504, 508, 511, 508, 510, 490, 478, 506, 496, 484, 491, 500, 520, 525, 513, 493, 492, 500, 508, 523, 549, 540, 498, 519, 511, 509, 490, 528, 484, 511, 528, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[510, 555, 502, 506, 509, 500, 494, 524, 477, 485, 507, 484, 502, 503, 566, 516, 32, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 518, 494, 497, 536, 522, 512, 518, 506, 533, 534, 527, 481, 507, 539, 522, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 568, 524, 506, 502, 508, 498, 564, 520, 534, 510, 491, 516, 523, 501, 508, 458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 526, 507, 497, 490, 483, 521, 524, 500, 520, 507, 520, 503, 494, 500, 492, 26, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 503, 520, 487, 519, 485, 456, 484, 487, 498, 501, 514, 511, 536, 520, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 544, 468, 534, 517, 539, 506, 498, 525, 534, 519, 480, 509, 510, 530, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 486, 484, 502, 545, 499, 509, 495, 548, 497, 544, 541, 531, 494, 499, 493, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 483, 477, 523, 534, 504, 497, 463, 528, 496, 511, 504, 530, 539, 523, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8261, 8231, 8138, 8037, 8238, 8261, 8182] avg=8192.0000 max=8261 max_rank=1 min=8037 min_rank=4 imbalance_ratio=1.008423 max_load_reduction=57275 max_load_reduction_pct=0.873947 moved_to_echo=32718 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[516, 530, 570, 521, 484, 515, 500, 499, 525, 546, 491, 506, 496, 524, 525, 494, 476, 510, 502, 506, 543, 510, 504, 480, 487, 545, 508, 514, 486, 503, 536, 522, 529, 525, 525, 502, 541, 534, 513, 544, 489, 529, 494, 490, 490, 532, 526, 564, 495, 513, 478, 463, 472, 513, 499, 501, 542, 513, 463, 543, 500, 494, 511, 496, 510, 526, 500, 543, 511, 542, 497, 506, 477, 581, 487, 527, 481, 503, 529, 519, 539, 500, 527, 474, 520, 498, 537, 528, 532, 485, 497, 515, 513, 493, 513, 478, 527, 514, 485, 556, 488, 525, 533, 491, 505, 526, 520, 529, 529, 461, 508, 517, 520, 510, 543, 476, 496, 482, 521, 540, 530, 509, 529, 548, 536, 505, 500, 492] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[529, 525, 530, 519, 483, 512, 502, 497, 506, 484, 525, 485, 509, 481, 493, 561, 32, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 558, 509, 492, 486, 520, 489, 465, 500, 520, 515, 500, 509, 492, 560, 511, 501, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 471, 509, 501, 498, 531, 517, 501, 505, 541, 516, 472, 523, 503, 509, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 488, 530, 475, 481, 528, 529, 521, 497, 511, 497, 520, 499, 494, 525, 486, 26, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 520, 511, 490, 547, 535, 472, 513, 524, 505, 515, 563, 501, 550, 469, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 526, 526, 560, 524, 496, 523, 497, 530, 497, 505, 516, 486, 512, 528, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 491, 547, 523, 485, 491, 526, 519, 491, 545, 529, 498, 508, 515, 536, 496, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 513, 527, 480, 462, 555, 510, 516, 543, 531, 514, 496, 534, 566, 479, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8138, 8121, 8137, 8237, 8223, 8274, 8217] avg=8192.0000 max=8274 max_rank=6 min=8121 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32951 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[505, 534, 503, 509, 548, 498, 493, 507, 548, 493, 502, 539, 505, 495, 523, 552, 31, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 527, 549, 524, 533, 511, 465, 479, 493, 495, 517, 513, 478, 523, 500, 530, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 533, 500, 541, 515, 524, 514, 482, 497, 530, 541, 542, 498, 473, 515, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 508, 508, 501, 539, 549, 519, 492, 520, 543, 502, 516, 508, 489, 540, 490, 24, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 460, 491, 533, 532, 515, 530, 513, 539, 489, 513, 509, 518, 558, 566, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 528, 497, 544, 516, 523, 541, 472, 492, 507, 538, 487, 523, 490, 466, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 511, 539, 499, 504, 486, 511, 508, 523, 430, 517, 494, 524, 472, 499, 497, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 500, 565, 501, 511, 479, 514, 503, 514, 513, 535, 517, 496, 507, 511, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8301, 8144, 8178, 8280, 8252, 8117, 8090, 8174] avg=8192.0000 max=8301 max_rank=0 min=8090 min_rank=6 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32633 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[483, 473, 520, 508, 475, 521, 522, 502, 540, 457, 486, 513, 511, 530, 523, 494, 37, 13, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 501, 523, 532, 508, 522, 513, 528, 490, 511, 513, 489, 527, 526, 496, 524, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 535, 525, 494, 476, 526, 484, 505, 548, 530, 499, 486, 499, 552, 518, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 486, 534, 518, 507, 522, 482, 506, 523, 488, 532, 560, 535, 499, 534, 516, 26, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 554, 508, 514, 545, 495, 494, 448, 495, 516, 543, 521, 494, 506, 535, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 527, 561, 482, 497, 535, 531, 521, 472, 544, 520, 508, 521, 512, 502, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 516, 498, 551, 535, 484, 471, 531, 452, 501, 502, 485, 482, 473, 499, 487, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 523, 499, 499, 479, 530, 494, 523, 526, 505, 507, 503, 526, 518, 466, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8118, 8237, 8173, 8326, 8243, 8263, 8045, 8131] avg=8192.0000 max=8326 max_rank=3 min=8045 min_rank=6 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32682 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[516, 530, 570, 521, 484, 515, 500, 499, 525, 546, 491, 506, 496, 524, 525, 494, 35, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 510, 502, 506, 543, 510, 504, 480, 487, 545, 508, 514, 486, 503, 536, 522, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 525, 525, 502, 541, 534, 513, 544, 489, 529, 494, 490, 490, 532, 526, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 513, 478, 463, 472, 513, 499, 501, 542, 513, 463, 543, 500, 494, 511, 496, 24, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 526, 500, 543, 511, 542, 497, 506, 477, 546, 487, 527, 481, 503, 529, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 500, 527, 474, 520, 498, 537, 528, 501, 485, 497, 515, 513, 493, 513, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 514, 485, 556, 488, 525, 533, 491, 505, 526, 520, 529, 529, 461, 508, 517, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 510, 543, 476, 496, 482, 493, 540, 530, 509, 529, 548, 536, 505, 500, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8293, 8143, 8262, 8037, 8204, 8118, 8270, 8209] avg=8192.0000 max=8293 max_rank=0 min=8037 min_rank=3 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32801 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[426, 512, 519, 540, 534, 529, 496, 475, 483, 522, 494, 533, 523, 511, 514, 521, 30, 8, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 481, 503, 513, 518, 494, 522, 520, 506, 511, 537, 540, 514, 484, 494, 488, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 517, 511, 514, 517, 525, 467, 530, 548, 510, 534, 498, 505, 550, 555, 446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 484, 545, 471, 538, 496, 528, 517, 530, 489, 503, 487, 498, 500, 505, 552, 22, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 509, 503, 525, 525, 528, 542, 557, 507, 481, 541, 512, 504, 530, 539, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 520, 527, 517, 496, 530, 520, 481, 449, 490, 500, 529, 486, 498, 519, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 526, 497, 506, 493, 548, 529, 513, 484, 508, 459, 561, 488, 520, 445, 543, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 493, 507, 536, 522, 514, 482, 523, 495, 488, 477, 523, 533, 512, 520, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8141, 8224, 8179, 8311, 8106, 8217, 8182] avg=8192.0000 max=8311 max_rank=4 min=8106 min_rank=5 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32816 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[512, 437, 542, 491, 505, 483, 554, 463, 498, 524, 531, 515, 522, 498, 527, 504, 30, 8, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 480, 478, 526, 514, 499, 536, 542, 527, 539, 544, 502, 528, 527, 489, 508, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 497, 514, 522, 511, 470, 541, 513, 483, 509, 512, 493, 534, 552, 536, 439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 541, 505, 523, 509, 520, 487, 539, 493, 456, 492, 515, 523, 513, 474, 513, 24, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 504, 505, 477, 529, 444, 478, 493, 490, 479, 540, 560, 506, 516, 519, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 511, 526, 491, 512, 517, 520, 506, 484, 502, 508, 532, 525, 498, 536, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 526, 503, 493, 555, 494, 534, 532, 501, 488, 570, 479, 551, 534, 497, 496, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 557, 522, 496, 517, 520, 543, 511, 510, 545, 504, 505, 490, 506, 506, 513, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 8242, 8126, 8168, 8068, 8239, 8289, 8253] avg=8192.0000 max=8289 max_rank=6 min=8068 min_rank=4 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32849 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[483, 473, 520, 508, 475, 521, 522, 502, 540, 457, 486, 513, 511, 530, 523, 494, 37, 13, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 501, 523, 532, 508, 522, 513, 528, 490, 511, 513, 489, 527, 526, 496, 524, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 535, 525, 494, 476, 526, 484, 505, 548, 530, 499, 486, 499, 552, 518, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 486, 534, 518, 507, 522, 482, 506, 523, 488, 532, 560, 535, 499, 534, 516, 26, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 554, 508, 514, 545, 495, 494, 448, 495, 516, 543, 521, 494, 506, 535, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 527, 561, 482, 497, 535, 531, 521, 472, 544, 520, 508, 521, 512, 502, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 516, 498, 551, 535, 484, 471, 531, 452, 501, 502, 485, 482, 473, 499, 487, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 523, 499, 499, 479, 530, 494, 523, 526, 505, 507, 503, 526, 518, 466, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8118, 8237, 8173, 8326, 8243, 8263, 8045, 8131] avg=8192.0000 max=8326 max_rank=3 min=8045 min_rank=6 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32682 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[503, 513, 489, 520, 520, 506, 485, 524, 516, 498, 503, 486, 524, 503, 481, 501, 32, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 529, 534, 490, 532, 509, 498, 489, 491, 503, 504, 555, 527, 506, 503, 548, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 496, 522, 517, 521, 525, 531, 525, 523, 524, 509, 534, 504, 492, 517, 472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 482, 510, 545, 540, 520, 507, 565, 521, 517, 518, 502, 448, 494, 554, 473, 24, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 546, 517, 472, 552, 524, 539, 542, 562, 499, 513, 528, 473, 497, 474, 459, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 538, 520, 520, 505, 498, 516, 506, 491, 544, 510, 517, 556, 514, 497, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 508, 511, 508, 510, 490, 478, 506, 496, 484, 491, 500, 520, 525, 513, 493, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 500, 508, 523, 549, 540, 472, 519, 511, 509, 490, 528, 484, 511, 528, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8120, 8230, 8221, 8271, 8184, 8232, 8090, 8188] avg=8192.0000 max=8271 max_rank=3 min=8090 min_rank=6 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32694 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[505, 534, 503, 509, 548, 498, 493, 507, 548, 493, 502, 539, 505, 495, 523, 552, 31, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 527, 549, 524, 533, 511, 465, 479, 493, 495, 517, 513, 478, 523, 500, 530, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 533, 500, 541, 515, 524, 514, 482, 497, 530, 541, 542, 498, 473, 515, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 508, 508, 501, 539, 549, 519, 492, 520, 543, 502, 516, 508, 489, 540, 490, 24, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 460, 491, 533, 532, 515, 530, 513, 539, 489, 513, 509, 518, 558, 566, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 528, 497, 544, 516, 523, 541, 472, 492, 507, 538, 487, 523, 490, 466, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 511, 539, 499, 504, 486, 511, 508, 523, 430, 517, 494, 524, 472, 499, 497, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 500, 565, 501, 511, 479, 514, 503, 514, 513, 535, 517, 496, 507, 511, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8301, 8144, 8178, 8280, 8252, 8117, 8090, 8174] avg=8192.0000 max=8301 max_rank=0 min=8090 min_rank=6 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32633 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[512, 437, 542, 491, 505, 483, 554, 463, 498, 524, 531, 515, 522, 498, 527, 504, 30, 8, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 480, 478, 526, 514, 499, 536, 542, 527, 539, 544, 502, 528, 527, 489, 508, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 497, 514, 522, 511, 470, 541, 513, 483, 509, 512, 493, 534, 552, 536, 439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 541, 505, 523, 509, 520, 487, 539, 493, 456, 492, 515, 523, 513, 474, 513, 24, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 504, 505, 477, 529, 444, 478, 493, 490, 479, 540, 560, 506, 516, 519, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 511, 526, 491, 512, 517, 520, 506, 484, 502, 508, 532, 525, 498, 536, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 526, 503, 493, 555, 494, 534, 532, 501, 488, 570, 479, 551, 534, 497, 496, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 557, 522, 496, 517, 520, 543, 511, 510, 545, 504, 505, 490, 506, 506, 513, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 8242, 8126, 8168, 8068, 8239, 8289, 8253] avg=8192.0000 max=8289 max_rank=6 min=8068 min_rank=4 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32849 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[510, 555, 502, 506, 509, 500, 494, 524, 477, 485, 507, 484, 502, 503, 566, 516, 32, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 518, 494, 497, 536, 522, 512, 518, 506, 533, 534, 527, 481, 507, 539, 522, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 568, 524, 506, 502, 508, 498, 564, 520, 534, 510, 491, 516, 523, 501, 508, 458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 526, 507, 497, 490, 483, 521, 524, 500, 520, 507, 520, 503, 494, 500, 492, 26, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 503, 520, 487, 519, 485, 456, 484, 487, 498, 501, 514, 511, 536, 520, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 544, 468, 534, 517, 539, 506, 498, 525, 534, 519, 480, 509, 510, 530, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 486, 484, 502, 545, 499, 509, 495, 548, 497, 544, 541, 531, 494, 499, 493, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 483, 477, 523, 534, 504, 497, 463, 528, 496, 511, 504, 530, 539, 523, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8261, 8231, 8138, 8037, 8238, 8261, 8182] avg=8192.0000 max=8261 max_rank=1 min=8037 min_rank=4 imbalance_ratio=1.008423 max_load_reduction=57275 max_load_reduction_pct=0.873947 moved_to_echo=32718 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[516, 530, 570, 521, 484, 515, 500, 499, 525, 546, 491, 506, 496, 524, 525, 494, 35, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 510, 502, 506, 543, 510, 504, 480, 487, 545, 508, 514, 486, 503, 536, 522, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 525, 525, 502, 541, 534, 513, 544, 489, 529, 494, 490, 490, 532, 526, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 513, 478, 463, 472, 513, 499, 501, 542, 513, 463, 543, 500, 494, 511, 496, 24, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 526, 500, 543, 511, 542, 497, 506, 477, 546, 487, 527, 481, 503, 529, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 500, 527, 474, 520, 498, 537, 528, 501, 485, 497, 515, 513, 493, 513, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 514, 485, 556, 488, 525, 533, 491, 505, 526, 520, 529, 529, 461, 508, 517, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 510, 543, 476, 496, 482, 493, 540, 530, 509, 529, 548, 536, 505, 500, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8293, 8143, 8262, 8037, 8204, 8118, 8270, 8209] avg=8192.0000 max=8293 max_rank=0 min=8037 min_rank=3 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32801 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 384, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4096, 4352, 4096, 4096, 4096, 4096, 4096, 4224, 4352, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 384, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 3968, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4096, 4352, 4096, 4096, 4096, 4096, 4096, 4224, 4352, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 256, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[500, 517, 530, 482, 509, 519, 485, 550, 476, 512, 470, 492, 495, 550, 569, 506, 509, 518, 534, 531, 482, 496, 489, 506, 522, 552, 531, 513, 523, 551, 540, 494, 515, 546, 506, 507, 532, 553, 519, 528, 451, 531, 526, 527, 508, 500, 523, 517, 492, 534, 509, 503, 509, 525, 516, 487, 495, 474, 484, 529, 504, 505, 488, 459, 494, 526, 515, 513, 526, 484, 497, 540, 495, 497, 516, 476, 474, 499, 504, 548, 523, 487, 518, 528, 483, 512, 512, 485, 524, 514, 555, 550, 533, 536, 491, 517, 532, 488, 500, 486, 533, 504, 494, 531, 481, 496, 500, 513, 528, 560, 533, 516, 530, 555, 491, 472, 503, 536, 481, 518, 516, 512, 499, 499, 518, 536, 504, 544] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[545, 500, 512, 507, 503, 515, 503, 506, 528, 508, 526, 529, 506, 498, 492, 504, 521, 504, 488, 551, 469, 494, 503, 556, 481, 509, 530, 502, 525, 510, 514, 537, 528, 506, 584, 522, 514, 518, 513, 527, 527, 523, 493, 491, 469, 533, 493, 510, 494, 481, 506, 486, 505, 519, 510, 512, 515, 507, 570, 524, 483, 536, 511, 530, 524, 520, 525, 527, 531, 526, 487, 512, 498, 508, 536, 526, 484, 548, 512, 531, 515, 507, 540, 501, 466, 507, 523, 564, 510, 493, 463, 521, 497, 513, 512, 524, 546, 487, 514, 516, 483, 513, 497, 489, 497, 495, 469, 505, 535, 539, 500, 535, 522, 517, 474, 515, 504, 541, 503, 461, 536, 501, 457, 528, 494, 528, 551, 517] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[519, 544, 475, 489, 520, 518, 544, 502, 488, 510, 472, 503, 503, 502, 472, 548, 524, 527, 520, 536, 508, 489, 484, 540, 515, 526, 522, 499, 523, 528, 524, 532, 514, 541, 512, 537, 503, 540, 491, 538, 501, 504, 475, 471, 551, 500, 497, 518, 536, 501, 498, 506, 514, 559, 513, 494, 519, 503, 511, 472, 477, 566, 510, 539, 535, 497, 511, 505, 495, 496, 459, 504, 523, 509, 530, 515, 483, 538, 461, 503, 491, 500, 534, 518, 479, 536, 564, 521, 508, 531, 487, 515, 528, 545, 526, 538, 496, 551, 495, 547, 546, 467, 508, 497, 525, 505, 532, 490, 540, 501, 508, 481, 530, 500, 494, 509, 521, 554, 480, 524, 525, 481, 498, 499, 537, 471, 489, 533] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[509, 564, 508, 499, 502, 510, 508, 484, 512, 524, 508, 491, 526, 468, 518, 536, 512, 509, 504, 488, 512, 506, 509, 528, 472, 487, 520, 508, 511, 484, 504, 511, 470, 493, 478, 509, 524, 514, 573, 528, 495, 528, 497, 506, 535, 486, 536, 493, 510, 509, 504, 496, 527, 520, 543, 537, 480, 535, 501, 520, 491, 523, 531, 539, 521, 554, 484, 503, 456, 521, 560, 540, 532, 436, 518, 497, 504, 548, 500, 511, 523, 507, 522, 539, 507, 504, 522, 517, 506, 514, 555, 529, 531, 492, 505, 526, 508, 522, 505, 554, 503, 542, 515, 506, 490, 512, 523, 474, 517, 518, 536, 494, 520, 509, 482, 483, 537, 502, 528, 496, 493, 507, 473, 487, 522, 538, 572, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[486, 521, 518, 552, 475, 539, 534, 503, 508, 518, 543, 516, 507, 514, 504, 465, 497, 528, 511, 487, 493, 499, 477, 525, 512, 571, 534, 493, 517, 461, 496, 512, 528, 505, 518, 480, 515, 538, 509, 532, 547, 515, 489, 567, 544, 500, 499, 530, 516, 480, 525, 471, 527, 513, 534, 482, 524, 503, 514, 504, 555, 535, 487, 537, 531, 495, 547, 485, 505, 514, 527, 531, 505, 515, 522, 489, 492, 506, 530, 497, 503, 509, 502, 515, 513, 526, 482, 517, 519, 493, 496, 544, 490, 513, 513, 496, 557, 531, 476, 492, 513, 513, 523, 521, 548, 495, 510, 540, 524, 489, 493, 507, 484, 508, 486, 488, 511, 509, 500, 551, 538, 493, 495, 521, 510, 518, 506, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[494, 510, 485, 501, 500, 490, 481, 502, 480, 508, 548, 510, 478, 506, 509, 505, 525, 477, 539, 507, 550, 487, 531, 493, 511, 530, 511, 500, 511, 503, 560, 512, 530, 473, 526, 520, 525, 478, 504, 529, 554, 458, 451, 532, 553, 491, 502, 522, 522, 540, 495, 517, 509, 515, 484, 492, 523, 513, 537, 523, 533, 487, 540, 530, 470, 530, 504, 536, 534, 504, 476, 517, 477, 543, 501, 453, 508, 507, 472, 477, 519, 510, 548, 514, 533, 521, 526, 510, 520, 518, 501, 538, 512, 519, 530, 522, 503, 516, 524, 512, 492, 506, 538, 533, 518, 503, 494, 517, 505, 528, 537, 514, 509, 516, 505, 534, 470, 527, 507, 527, 548, 519, 520, 525, 516, 520, 527, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[528, 499, 549, 517, 488, 465, 550, 529, 491, 528, 470, 480, 533, 527, 523, 506, 474, 519, 530, 513, 519, 558, 476, 513, 536, 521, 522, 529, 532, 512, 499, 499, 502, 492, 513, 533, 485, 544, 446, 494, 473, 524, 547, 523, 502, 524, 563, 497, 527, 502, 507, 521, 507, 537, 497, 519, 532, 508, 550, 494, 477, 527, 524, 522, 482, 510, 532, 511, 530, 514, 520, 534, 528, 514, 491, 491, 494, 512, 532, 515, 520, 489, 503, 539, 505, 562, 493, 526, 514, 515, 500, 532, 533, 446, 547, 526, 501, 532, 496, 510, 481, 496, 519, 519, 485, 516, 515, 501, 513, 490, 476, 513, 494, 496, 520, 501, 511, 517, 514, 500, 497, 483, 535, 556, 518, 510, 522, 491] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[486, 521, 518, 552, 475, 539, 534, 503, 508, 518, 543, 516, 507, 514, 504, 465, 497, 528, 511, 487, 493, 499, 477, 525, 512, 571, 534, 493, 517, 461, 496, 512, 528, 505, 518, 480, 515, 538, 509, 532, 547, 515, 489, 567, 544, 500, 499, 530, 516, 480, 525, 471, 527, 513, 534, 482, 524, 503, 514, 504, 555, 535, 487, 537, 531, 495, 547, 485, 505, 514, 527, 531, 505, 515, 522, 489, 492, 506, 530, 497, 503, 509, 502, 515, 513, 526, 482, 517, 519, 493, 496, 544, 490, 513, 513, 496, 557, 531, 476, 492, 513, 513, 523, 521, 548, 495, 510, 540, 524, 489, 493, 507, 484, 508, 486, 488, 511, 509, 500, 551, 538, 493, 495, 521, 510, 518, 506, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[509, 564, 508, 499, 502, 510, 508, 484, 512, 524, 508, 491, 526, 468, 518, 536, 512, 509, 504, 488, 512, 506, 509, 528, 472, 487, 520, 508, 511, 484, 504, 511, 470, 493, 478, 509, 524, 514, 573, 528, 495, 528, 497, 506, 535, 486, 536, 493, 510, 509, 504, 496, 527, 520, 543, 537, 480, 535, 501, 520, 491, 523, 531, 539, 521, 554, 484, 503, 456, 521, 560, 540, 532, 436, 518, 497, 504, 548, 500, 511, 523, 507, 522, 539, 507, 504, 522, 517, 506, 514, 555, 529, 531, 492, 505, 526, 508, 522, 505, 554, 503, 542, 515, 506, 490, 512, 523, 474, 517, 518, 536, 494, 520, 509, 482, 483, 537, 502, 528, 496, 493, 507, 473, 487, 522, 538, 572, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[511, 494, 509, 516, 525, 523, 504, 518, 527, 525, 481, 523, 547, 541, 512, 509, 485, 527, 533, 538, 531, 511, 509, 519, 497, 530, 530, 502, 529, 531, 495, 480, 531, 530, 506, 508, 472, 558, 492, 522, 526, 481, 491, 542, 517, 493, 509, 499, 512, 531, 469, 526, 551, 483, 515, 506, 489, 505, 508, 485, 502, 494, 483, 523, 533, 515, 489, 464, 529, 517, 502, 513, 535, 547, 538, 514, 504, 498, 507, 513, 501, 535, 528, 446, 543, 539, 517, 563, 484, 539, 513, 510, 512, 540, 499, 506, 502, 533, 483, 509, 525, 546, 535, 498, 493, 538, 515, 498, 508, 463, 486, 495, 478, 506, 491, 540, 492, 485, 491, 576, 509, 521, 509, 504, 516, 487, 514, 526] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[486, 521, 518, 552, 475, 539, 534, 503, 508, 518, 543, 516, 507, 514, 504, 465, 31, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 528, 511, 487, 493, 499, 477, 525, 512, 550, 534, 493, 517, 461, 496, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 505, 518, 480, 515, 518, 509, 532, 547, 515, 489, 567, 544, 500, 499, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 480, 525, 471, 527, 513, 534, 482, 524, 503, 514, 504, 555, 535, 487, 537, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 495, 547, 485, 505, 514, 527, 531, 505, 515, 522, 489, 492, 506, 530, 497, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 509, 502, 515, 513, 526, 482, 517, 519, 493, 496, 480, 490, 513, 513, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 557, 531, 476, 492, 513, 513, 523, 521, 548, 495, 510, 540, 524, 489, 493, 507, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 508, 486, 488, 511, 509, 500, 551, 538, 493, 495, 521, 510, 518, 506, 525, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8092, 8296, 8213, 8224, 8067, 8252, 8157] avg=8192.0000 max=8296 max_rank=2 min=8067 min_rank=5 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32700 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[500, 517, 530, 482, 509, 519, 485, 550, 476, 512, 470, 492, 495, 550, 569, 506, 31, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 518, 534, 531, 482, 496, 489, 506, 522, 532, 531, 513, 523, 551, 540, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 546, 506, 507, 532, 533, 519, 528, 451, 531, 526, 527, 508, 500, 523, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 534, 509, 503, 509, 525, 516, 487, 495, 474, 484, 529, 504, 505, 488, 459, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 526, 515, 513, 526, 484, 497, 540, 495, 497, 516, 476, 474, 499, 504, 548, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 487, 518, 528, 483, 512, 512, 485, 524, 514, 555, 486, 533, 536, 491, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 488, 500, 486, 533, 504, 494, 531, 481, 496, 500, 513, 528, 560, 533, 516, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 555, 491, 472, 503, 536, 481, 518, 516, 512, 499, 499, 518, 536, 504, 544, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8271, 8269, 8019, 8137, 8204, 8214, 8228] avg=8192.0000 max=8271 max_rank=1 min=8019 min_rank=3 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32783 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[500, 517, 530, 482, 509, 519, 485, 550, 476, 512, 470, 492, 495, 550, 569, 506, 509, 518, 534, 531, 482, 496, 489, 506, 522, 552, 531, 513, 523, 551, 540, 494, 515, 546, 506, 507, 532, 553, 519, 528, 451, 531, 526, 527, 508, 500, 523, 517, 492, 534, 509, 503, 509, 525, 516, 487, 495, 474, 484, 529, 504, 505, 488, 459, 494, 526, 515, 513, 526, 484, 497, 540, 495, 497, 516, 476, 474, 499, 504, 548, 523, 487, 518, 528, 483, 512, 512, 485, 524, 514, 555, 550, 533, 536, 491, 517, 532, 488, 500, 486, 533, 504, 494, 531, 481, 496, 500, 513, 528, 560, 533, 516, 530, 555, 491, 472, 503, 536, 481, 518, 516, 512, 499, 499, 518, 536, 504, 544] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[494, 510, 485, 501, 500, 490, 481, 502, 480, 508, 548, 510, 478, 506, 509, 505, 525, 477, 539, 507, 550, 487, 531, 493, 511, 530, 511, 500, 511, 503, 560, 512, 530, 473, 526, 520, 525, 478, 504, 529, 554, 458, 451, 532, 553, 491, 502, 522, 522, 540, 495, 517, 509, 515, 484, 492, 523, 513, 537, 523, 533, 487, 540, 530, 470, 530, 504, 536, 534, 504, 476, 517, 477, 543, 501, 453, 508, 507, 472, 477, 519, 510, 548, 514, 533, 521, 526, 510, 520, 518, 501, 538, 512, 519, 530, 522, 503, 516, 524, 512, 492, 506, 538, 533, 518, 503, 494, 517, 505, 528, 537, 514, 509, 516, 505, 534, 470, 527, 507, 527, 548, 519, 520, 525, 516, 520, 527, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[545, 500, 512, 507, 503, 515, 503, 506, 528, 508, 526, 529, 506, 498, 492, 504, 34, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 504, 488, 551, 469, 494, 503, 556, 481, 474, 530, 502, 525, 510, 514, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 506, 584, 522, 514, 490, 513, 527, 527, 523, 493, 491, 469, 533, 493, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 481, 506, 486, 505, 519, 510, 512, 515, 507, 570, 524, 483, 536, 511, 530, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 520, 525, 527, 531, 526, 487, 512, 498, 508, 536, 526, 484, 548, 512, 531, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 507, 540, 501, 466, 507, 523, 564, 510, 493, 463, 451, 497, 513, 512, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 487, 514, 516, 483, 513, 497, 489, 497, 495, 469, 505, 535, 539, 500, 535, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 517, 474, 515, 504, 541, 503, 461, 536, 501, 457, 528, 494, 528, 551, 517, 19, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8159, 8223, 8198, 8331, 8086, 8143, 8174] avg=8192.0000 max=8331 max_rank=4 min=8086 min_rank=5 imbalance_ratio=1.016968 max_load_reduction=57205 max_load_reduction_pct=0.872879 moved_to_echo=32734 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[519, 544, 475, 489, 520, 518, 544, 502, 488, 510, 472, 503, 503, 502, 472, 548, 29, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 527, 520, 536, 508, 489, 484, 540, 515, 507, 522, 499, 523, 528, 524, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 541, 512, 537, 503, 520, 491, 538, 501, 504, 475, 471, 551, 500, 497, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 501, 498, 506, 514, 559, 513, 494, 519, 503, 511, 472, 477, 566, 510, 539, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 497, 511, 505, 495, 496, 459, 504, 523, 509, 530, 515, 483, 538, 461, 503, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 500, 534, 518, 479, 536, 564, 521, 508, 531, 487, 455, 528, 545, 526, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 551, 495, 547, 546, 467, 508, 497, 525, 505, 532, 490, 540, 501, 508, 481, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 500, 494, 509, 521, 554, 480, 524, 525, 481, 498, 499, 537, 471, 489, 533, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8139, 8278, 8173, 8224, 8095, 8261, 8207, 8159] avg=8192.0000 max=8278 max_rank=1 min=8095 min_rank=4 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32722 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[500, 517, 530, 482, 509, 519, 485, 550, 476, 512, 470, 492, 495, 550, 569, 506, 31, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 518, 534, 531, 482, 496, 489, 506, 522, 532, 531, 513, 523, 551, 540, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 546, 506, 507, 532, 533, 519, 528, 451, 531, 526, 527, 508, 500, 523, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 534, 509, 503, 509, 525, 516, 487, 495, 474, 484, 529, 504, 505, 488, 459, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 526, 515, 513, 526, 484, 497, 540, 495, 497, 516, 476, 474, 499, 504, 548, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 487, 518, 528, 483, 512, 512, 485, 524, 514, 555, 486, 533, 536, 491, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 488, 500, 486, 533, 504, 494, 531, 481, 496, 500, 513, 528, 560, 533, 516, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 555, 491, 472, 503, 536, 481, 518, 516, 512, 499, 499, 518, 536, 504, 544, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8271, 8269, 8019, 8137, 8204, 8214, 8228] avg=8192.0000 max=8271 max_rank=1 min=8019 min_rank=3 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32783 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[528, 499, 549, 517, 488, 465, 550, 529, 491, 528, 470, 480, 533, 527, 523, 506, 30, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 519, 530, 513, 519, 558, 476, 513, 536, 502, 522, 529, 532, 512, 499, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 492, 513, 533, 485, 524, 446, 494, 473, 524, 547, 523, 502, 524, 563, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 502, 507, 521, 507, 537, 497, 519, 532, 508, 550, 494, 477, 527, 524, 522, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 510, 532, 511, 530, 514, 520, 534, 528, 514, 491, 491, 494, 512, 532, 515, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 489, 503, 539, 505, 562, 493, 526, 514, 515, 500, 470, 533, 446, 547, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 532, 496, 510, 481, 496, 519, 519, 485, 516, 515, 501, 513, 490, 476, 513, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 496, 520, 501, 511, 517, 514, 500, 497, 483, 535, 556, 518, 510, 522, 491, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8214, 8233, 8142, 8257, 8242, 8188, 8081, 8179] avg=8192.0000 max=8257 max_rank=3 min=8081 min_rank=6 imbalance_ratio=1.007935 max_load_reduction=57279 max_load_reduction_pct=0.874008 moved_to_echo=32690 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[509, 564, 508, 499, 502, 510, 508, 484, 512, 524, 508, 491, 526, 468, 518, 536, 30, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 509, 504, 488, 512, 506, 509, 528, 472, 469, 520, 508, 511, 484, 504, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 493, 478, 509, 524, 495, 573, 528, 495, 528, 497, 506, 535, 486, 536, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 509, 504, 496, 527, 520, 543, 537, 480, 535, 501, 520, 491, 523, 531, 539, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 554, 484, 503, 456, 521, 560, 540, 532, 436, 518, 497, 504, 548, 500, 511, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 507, 522, 539, 507, 504, 522, 517, 506, 514, 555, 467, 531, 492, 505, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 522, 505, 554, 503, 542, 515, 506, 490, 512, 523, 474, 517, 518, 536, 494, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 509, 482, 483, 537, 502, 528, 496, 493, 507, 473, 487, 522, 538, 572, 521, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8047, 8146, 8272, 8217, 8237, 8236, 8183] avg=8192.0000 max=8272 max_rank=3 min=8047 min_rank=1 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32873 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[511, 494, 509, 516, 525, 523, 504, 518, 527, 525, 481, 523, 547, 541, 512, 509, 29, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 527, 533, 538, 531, 511, 509, 519, 497, 511, 530, 502, 529, 531, 495, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 530, 506, 508, 472, 537, 492, 522, 526, 481, 491, 542, 517, 493, 509, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 531, 469, 526, 551, 483, 515, 506, 489, 505, 508, 485, 502, 494, 483, 523, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 515, 489, 464, 529, 517, 502, 513, 535, 547, 538, 514, 504, 498, 507, 513, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 535, 528, 446, 543, 539, 517, 563, 484, 539, 513, 450, 512, 540, 499, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 533, 483, 509, 525, 546, 535, 498, 493, 538, 515, 498, 508, 463, 486, 495, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 506, 491, 540, 492, 485, 491, 576, 509, 521, 509, 504, 516, 487, 514, 526, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8295, 8228, 8156, 8088, 8249, 8215, 8145, 8160] avg=8192.0000 max=8295 max_rank=0 min=8088 min_rank=3 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32769 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[486, 521, 518, 552, 475, 539, 534, 503, 508, 518, 543, 516, 507, 514, 504, 465, 31, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 528, 511, 487, 493, 499, 477, 525, 512, 550, 534, 493, 517, 461, 496, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 505, 518, 480, 515, 518, 509, 532, 547, 515, 489, 567, 544, 500, 499, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 480, 525, 471, 527, 513, 534, 482, 524, 503, 514, 504, 555, 535, 487, 537, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 495, 547, 485, 505, 514, 527, 531, 505, 515, 522, 489, 492, 506, 530, 497, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 509, 502, 515, 513, 526, 482, 517, 519, 493, 496, 480, 490, 513, 513, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 557, 531, 476, 492, 513, 513, 523, 521, 548, 495, 510, 540, 524, 489, 493, 507, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 508, 486, 488, 511, 509, 500, 551, 538, 493, 495, 521, 510, 518, 506, 525, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8092, 8296, 8213, 8224, 8067, 8252, 8157] avg=8192.0000 max=8296 max_rank=2 min=8067 min_rank=5 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32700 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[494, 510, 485, 501, 500, 490, 481, 502, 480, 508, 548, 510, 478, 506, 509, 505, 31, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 477, 539, 507, 550, 487, 531, 493, 511, 511, 511, 500, 511, 503, 560, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 473, 526, 520, 525, 461, 504, 529, 554, 458, 451, 532, 553, 491, 502, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 540, 495, 517, 509, 515, 484, 492, 523, 513, 537, 523, 533, 487, 540, 530, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 530, 504, 536, 534, 504, 476, 517, 477, 543, 501, 453, 508, 507, 472, 477, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 510, 548, 514, 533, 521, 526, 510, 520, 518, 501, 475, 512, 519, 530, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 516, 524, 512, 492, 506, 538, 533, 518, 503, 494, 517, 505, 528, 537, 514, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 516, 505, 534, 470, 527, 507, 527, 548, 519, 520, 525, 516, 520, 527, 514, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8039, 8228, 8131, 8265, 8041, 8278, 8258, 8296] avg=8192.0000 max=8296 max_rank=7 min=8039 min_rank=0 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32873 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[509, 564, 508, 499, 502, 510, 508, 484, 512, 524, 508, 491, 526, 468, 518, 536, 30, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 509, 504, 488, 512, 506, 509, 528, 472, 469, 520, 508, 511, 484, 504, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 493, 478, 509, 524, 495, 573, 528, 495, 528, 497, 506, 535, 486, 536, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 509, 504, 496, 527, 520, 543, 537, 480, 535, 501, 520, 491, 523, 531, 539, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 554, 484, 503, 456, 521, 560, 540, 532, 436, 518, 497, 504, 548, 500, 511, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 507, 522, 539, 507, 504, 522, 517, 506, 514, 555, 467, 531, 492, 505, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 522, 505, 554, 503, 542, 515, 506, 490, 512, 523, 474, 517, 518, 536, 494, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 509, 482, 483, 537, 502, 528, 496, 493, 507, 473, 487, 522, 538, 572, 521, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8047, 8146, 8272, 8217, 8237, 8236, 8183] avg=8192.0000 max=8272 max_rank=3 min=8047 min_rank=1 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32873 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 3968, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 3840, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 3968, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 3968, 3968, 4224, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[484, 562, 519, 497, 527, 510, 532, 517, 497, 487, 516, 517, 501, 496, 522, 516, 521, 549, 509, 488, 518, 498, 517, 490, 524, 551, 502, 497, 561, 491, 509, 539, 503, 490, 544, 510, 526, 466, 508, 529, 512, 531, 510, 503, 492, 500, 550, 531, 524, 519, 521, 517, 525, 489, 512, 479, 481, 493, 476, 498, 500, 536, 494, 493, 496, 484, 493, 526, 478, 462, 503, 541, 521, 512, 515, 508, 507, 512, 526, 475, 554, 492, 498, 542, 508, 532, 538, 536, 511, 501, 493, 562, 520, 506, 522, 506, 522, 484, 530, 505, 523, 506, 525, 518, 495, 518, 543, 493, 486, 533, 559, 515, 518, 481, 506, 566, 504, 509, 536, 495, 486, 487, 543, 475, 515, 521, 513, 520] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[508, 503, 492, 512, 517, 504, 531, 500, 529, 541, 502, 518, 511, 509, 519, 485, 519, 509, 494, 489, 506, 508, 557, 502, 514, 514, 538, 561, 495, 494, 514, 550, 527, 509, 502, 524, 518, 510, 486, 516, 538, 509, 504, 544, 520, 502, 515, 521, 517, 525, 505, 514, 490, 533, 526, 500, 504, 473, 455, 520, 522, 520, 519, 560, 510, 465, 508, 535, 516, 497, 560, 524, 487, 507, 555, 509, 478, 545, 465, 518, 510, 536, 514, 513, 473, 494, 488, 537, 518, 461, 515, 539, 538, 484, 534, 527, 483, 512, 500, 531, 497, 480, 509, 512, 531, 543, 554, 513, 505, 508, 524, 467, 545, 480, 504, 551, 507, 470, 520, 495, 502, 512, 492, 521, 511, 514, 482, 528] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[518, 483, 498, 505, 501, 509, 524, 493, 478, 508, 533, 510, 505, 524, 526, 480, 492, 499, 517, 511, 521, 504, 543, 491, 560, 498, 499, 529, 502, 531, 520, 513, 550, 512, 486, 490, 495, 512, 494, 510, 519, 523, 536, 529, 510, 508, 529, 536, 503, 502, 526, 508, 493, 508, 511, 474, 536, 539, 514, 503, 437, 530, 496, 500, 482, 532, 508, 457, 535, 504, 530, 532, 498, 521, 510, 533, 494, 527, 520, 513, 479, 514, 510, 499, 503, 513, 547, 550, 524, 505, 500, 543, 541, 505, 515, 501, 555, 535, 521, 556, 514, 511, 537, 482, 498, 529, 493, 534, 521, 507, 499, 520, 525, 499, 493, 529, 524, 503, 527, 508, 523, 486, 492, 498, 487, 485, 529, 527] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[535, 521, 501, 491, 479, 507, 538, 492, 523, 508, 535, 498, 508, 482, 522, 540, 515, 516, 476, 490, 500, 543, 501, 496, 536, 499, 498, 492, 484, 565, 520, 551, 529, 536, 481, 500, 538, 500, 512, 490, 507, 475, 480, 461, 512, 501, 533, 503, 505, 493, 504, 516, 486, 511, 485, 555, 513, 549, 479, 539, 531, 531, 516, 518, 498, 497, 534, 499, 472, 476, 506, 516, 486, 528, 512, 494, 496, 505, 510, 502, 560, 519, 519, 513, 524, 511, 526, 502, 551, 489, 519, 515, 534, 524, 518, 518, 487, 527, 567, 505, 531, 521, 506, 540, 538, 532, 538, 480, 521, 497, 544, 536, 547, 520, 476, 526, 515, 502, 436, 480, 515, 484, 503, 522, 533, 541, 513, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[536, 534, 516, 513, 514, 566, 525, 473, 493, 510, 468, 548, 497, 526, 552, 507, 484, 489, 498, 500, 523, 493, 548, 485, 535, 529, 491, 476, 475, 481, 549, 503, 543, 503, 498, 517, 523, 476, 540, 478, 534, 559, 512, 508, 470, 512, 511, 483, 541, 516, 488, 539, 517, 501, 455, 567, 474, 524, 483, 558, 476, 484, 509, 498, 531, 510, 502, 522, 490, 524, 528, 487, 496, 528, 516, 499, 502, 497, 513, 538, 539, 513, 495, 512, 524, 522, 495, 509, 527, 540, 503, 530, 517, 517, 520, 520, 500, 489, 544, 527, 482, 487, 515, 542, 500, 518, 474, 525, 531, 543, 499, 505, 550, 501, 484, 524, 525, 532, 546, 515, 510, 500, 521, 535, 492, 509, 483, 528] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[485, 519, 513, 485, 505, 536, 498, 524, 511, 498, 504, 509, 512, 495, 495, 503, 522, 551, 512, 504, 537, 480, 492, 536, 554, 519, 494, 493, 540, 553, 527, 469, 500, 516, 507, 515, 494, 538, 496, 493, 532, 532, 508, 521, 492, 478, 531, 496, 501, 516, 531, 554, 495, 496, 487, 526, 513, 502, 514, 491, 493, 513, 508, 528, 526, 549, 525, 526, 515, 519, 504, 521, 502, 519, 473, 519, 492, 534, 520, 507, 473, 509, 515, 512, 571, 518, 509, 492, 472, 481, 509, 528, 504, 506, 513, 545, 511, 496, 527, 536, 504, 506, 513, 521, 508, 529, 535, 504, 497, 515, 529, 494, 573, 576, 524, 529, 490, 501, 514, 514, 496, 518, 478, 523, 501, 487, 490, 497] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[508, 503, 492, 512, 517, 504, 531, 500, 529, 541, 502, 518, 511, 509, 519, 485, 519, 509, 494, 489, 506, 508, 557, 502, 514, 514, 538, 561, 495, 494, 514, 550, 527, 509, 502, 524, 518, 510, 486, 516, 538, 509, 504, 544, 520, 502, 515, 521, 517, 525, 505, 514, 490, 533, 526, 500, 504, 473, 455, 520, 522, 520, 519, 560, 510, 465, 508, 535, 516, 497, 560, 524, 487, 507, 555, 509, 478, 545, 465, 518, 510, 536, 514, 513, 473, 494, 488, 537, 518, 461, 515, 539, 538, 484, 534, 527, 483, 512, 500, 531, 497, 480, 509, 512, 531, 543, 554, 513, 505, 508, 524, 467, 545, 480, 504, 551, 507, 470, 520, 495, 502, 512, 492, 521, 511, 514, 482, 528] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[484, 562, 519, 497, 527, 510, 532, 517, 497, 487, 516, 517, 501, 496, 522, 516, 521, 549, 509, 488, 518, 498, 517, 490, 524, 551, 502, 497, 561, 491, 509, 539, 503, 490, 544, 510, 526, 466, 508, 529, 512, 531, 510, 503, 492, 500, 550, 531, 524, 519, 521, 517, 525, 489, 512, 479, 481, 493, 476, 498, 500, 536, 494, 493, 496, 484, 493, 526, 478, 462, 503, 541, 521, 512, 515, 508, 507, 512, 526, 475, 554, 492, 498, 542, 508, 532, 538, 536, 511, 501, 493, 562, 520, 506, 522, 506, 522, 484, 530, 505, 523, 506, 525, 518, 495, 518, 543, 493, 486, 533, 559, 515, 518, 481, 506, 566, 504, 509, 536, 495, 486, 487, 543, 475, 515, 521, 513, 520] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[515, 547, 511, 533, 521, 550, 535, 497, 464, 484, 485, 492, 520, 510, 535, 524, 540, 525, 512, 481, 493, 505, 479, 501, 510, 521, 532, 553, 537, 497, 514, 549, 473, 516, 485, 469, 454, 505, 506, 487, 533, 523, 535, 521, 510, 518, 479, 496, 522, 518, 483, 491, 550, 511, 531, 497, 528, 511, 535, 497, 510, 518, 469, 517, 497, 502, 536, 506, 488, 499, 477, 485, 524, 517, 532, 503, 549, 465, 549, 506, 551, 560, 539, 547, 484, 556, 533, 478, 535, 497, 525, 495, 551, 520, 529, 503, 498, 537, 503, 493, 532, 502, 496, 544, 514, 522, 505, 494, 528, 502, 477, 498, 516, 493, 530, 499, 499, 530, 541, 536, 510, 475, 493, 547, 488, 503, 510, 513] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[485, 535, 509, 482, 478, 467, 503, 494, 500, 509, 522, 521, 483, 509, 506, 508, 505, 524, 518, 535, 494, 519, 520, 466, 501, 516, 510, 540, 518, 521, 516, 521, 509, 512, 501, 504, 508, 545, 525, 532, 520, 512, 535, 521, 554, 519, 503, 537, 485, 533, 530, 519, 477, 512, 538, 532, 496, 514, 494, 546, 490, 564, 487, 541, 533, 516, 496, 490, 541, 519, 523, 522, 509, 519, 514, 517, 530, 487, 506, 496, 492, 543, 538, 532, 500, 520, 493, 538, 557, 492, 531, 517, 521, 498, 496, 495, 505, 476, 485, 500, 535, 493, 521, 514, 492, 502, 519, 514, 480, 531, 506, 516, 514, 531, 493, 476, 508, 503, 523, 515, 518, 462, 534, 551, 503, 492, 517, 496] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[515, 547, 511, 533, 521, 550, 535, 497, 464, 484, 485, 492, 520, 510, 535, 524, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 525, 512, 481, 493, 505, 479, 501, 485, 521, 532, 553, 537, 497, 514, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 516, 485, 469, 454, 505, 506, 487, 533, 523, 535, 521, 510, 518, 479, 496, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 518, 483, 491, 550, 511, 531, 497, 528, 511, 535, 497, 510, 518, 469, 517, 16, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 502, 536, 506, 488, 499, 477, 485, 524, 517, 532, 503, 549, 465, 549, 506, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 560, 539, 547, 484, 556, 533, 478, 535, 497, 525, 418, 551, 520, 529, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 537, 503, 493, 532, 502, 496, 544, 514, 497, 505, 494, 528, 502, 477, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 493, 530, 499, 499, 530, 541, 536, 510, 475, 493, 547, 488, 503, 510, 513, 10, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8255, 8224, 8025, 8216, 8168, 8326, 8120, 8202] avg=8192.0000 max=8326 max_rank=5 min=8025 min_rank=2 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32816 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[535, 521, 501, 491, 479, 507, 538, 492, 523, 508, 535, 498, 508, 482, 522, 540, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 516, 476, 490, 500, 543, 501, 496, 509, 499, 498, 492, 484, 565, 520, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 536, 481, 500, 538, 500, 512, 490, 507, 475, 480, 461, 512, 501, 533, 503, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 493, 504, 516, 486, 511, 485, 555, 513, 549, 479, 539, 531, 531, 516, 518, 17, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 497, 534, 499, 472, 476, 506, 516, 486, 528, 512, 494, 496, 505, 510, 502, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 560, 519, 519, 513, 524, 511, 526, 502, 551, 489, 519, 436, 534, 524, 518, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 527, 567, 505, 531, 521, 506, 540, 538, 507, 538, 480, 521, 497, 544, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 520, 476, 526, 515, 502, 436, 480, 515, 484, 503, 522, 533, 541, 513, 529, 10, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8155, 8073, 8260, 8065, 8263, 8345, 8162] avg=8192.0000 max=8345 max_rank=6 min=8065 min_rank=4 imbalance_ratio=1.018677 max_load_reduction=57191 max_load_reduction_pct=0.872665 moved_to_echo=32835 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[536, 534, 516, 513, 514, 566, 525, 473, 493, 510, 468, 548, 497, 526, 552, 507, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 489, 498, 500, 523, 493, 548, 485, 508, 529, 491, 476, 475, 481, 549, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 503, 498, 517, 523, 476, 540, 478, 534, 559, 512, 508, 470, 512, 511, 483, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 516, 488, 539, 517, 501, 455, 567, 474, 524, 483, 558, 476, 484, 509, 498, 17, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 510, 502, 522, 490, 524, 528, 487, 496, 528, 516, 499, 502, 497, 513, 538, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 513, 495, 512, 524, 522, 495, 509, 527, 540, 503, 449, 517, 517, 520, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 489, 544, 527, 482, 487, 515, 542, 500, 493, 474, 525, 531, 543, 499, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 501, 484, 524, 525, 532, 546, 515, 510, 500, 521, 535, 492, 509, 483, 528, 10, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8312, 8032, 8182, 8159, 8218, 8202, 8156, 8275] avg=8192.0000 max=8312 max_rank=0 min=8032 min_rank=1 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32851 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[485, 519, 513, 485, 505, 536, 498, 524, 511, 498, 504, 509, 512, 495, 495, 503, 522, 551, 512, 504, 537, 480, 492, 536, 554, 519, 494, 493, 540, 553, 527, 469, 500, 516, 507, 515, 494, 538, 496, 493, 532, 532, 508, 521, 492, 478, 531, 496, 501, 516, 531, 554, 495, 496, 487, 526, 513, 502, 514, 491, 493, 513, 508, 528, 526, 549, 525, 526, 515, 519, 504, 521, 502, 519, 473, 519, 492, 534, 520, 507, 473, 509, 515, 512, 571, 518, 509, 492, 472, 481, 509, 528, 504, 506, 513, 545, 511, 496, 527, 536, 504, 506, 513, 521, 508, 529, 535, 504, 497, 515, 529, 494, 573, 576, 524, 529, 490, 501, 514, 514, 496, 518, 478, 523, 501, 487, 490, 497] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[535, 521, 501, 491, 479, 507, 538, 492, 523, 508, 535, 498, 508, 482, 522, 540, 515, 516, 476, 490, 500, 543, 501, 496, 536, 499, 498, 492, 484, 565, 520, 551, 529, 536, 481, 500, 538, 500, 512, 490, 507, 475, 480, 461, 512, 501, 533, 503, 505, 493, 504, 516, 486, 511, 485, 555, 513, 549, 479, 539, 531, 531, 516, 518, 498, 497, 534, 499, 472, 476, 506, 516, 486, 528, 512, 494, 496, 505, 510, 502, 560, 519, 519, 513, 524, 511, 526, 502, 551, 489, 519, 515, 534, 524, 518, 518, 487, 527, 567, 505, 531, 521, 506, 540, 538, 532, 538, 480, 521, 497, 544, 536, 547, 520, 476, 526, 515, 502, 436, 480, 515, 484, 503, 522, 533, 541, 513, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[484, 562, 519, 497, 527, 510, 532, 517, 497, 487, 516, 517, 501, 496, 522, 516, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 549, 509, 488, 518, 498, 517, 490, 490, 551, 502, 497, 561, 491, 509, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 490, 544, 510, 526, 466, 508, 529, 512, 531, 510, 503, 492, 500, 550, 531, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 519, 521, 517, 525, 489, 512, 479, 481, 493, 476, 498, 500, 536, 494, 493, 20, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 484, 493, 526, 478, 462, 503, 541, 521, 512, 515, 508, 507, 512, 526, 475, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 492, 498, 542, 508, 532, 538, 536, 511, 501, 493, 464, 520, 506, 522, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 484, 530, 505, 523, 506, 525, 518, 495, 484, 543, 493, 486, 533, 559, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 481, 506, 566, 504, 509, 536, 495, 486, 487, 543, 475, 515, 521, 513, 520, 14, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8240, 8230, 8225, 8094, 8100, 8223, 8221, 8203] avg=8192.0000 max=8240 max_rank=0 min=8094 min_rank=3 imbalance_ratio=1.005859 max_load_reduction=57296 max_load_reduction_pct=0.874268 moved_to_echo=32747 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[485, 519, 513, 485, 505, 536, 498, 524, 511, 498, 504, 509, 512, 495, 495, 503, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 551, 512, 504, 537, 480, 492, 536, 526, 519, 494, 493, 540, 553, 527, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 516, 507, 515, 494, 538, 496, 493, 532, 532, 508, 521, 492, 478, 531, 496, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 516, 531, 554, 495, 496, 487, 526, 513, 502, 514, 491, 493, 513, 508, 528, 18, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 549, 525, 526, 515, 519, 504, 521, 502, 519, 473, 519, 492, 534, 520, 507, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 509, 515, 512, 571, 518, 509, 492, 472, 481, 509, 447, 504, 506, 513, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 496, 527, 536, 504, 506, 513, 521, 508, 504, 535, 504, 497, 515, 529, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 573, 576, 524, 529, 490, 501, 514, 514, 496, 518, 478, 523, 501, 487, 490, 497, 10, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8126, 8255, 8164, 8198, 8286, 8076, 8200, 8231] avg=8192.0000 max=8286 max_rank=4 min=8076 min_rank=5 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32793 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[485, 535, 509, 482, 478, 467, 503, 494, 500, 509, 522, 521, 483, 509, 506, 508, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 524, 518, 535, 494, 519, 520, 466, 476, 516, 510, 540, 518, 521, 516, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 512, 501, 504, 508, 545, 525, 532, 520, 512, 535, 521, 554, 519, 503, 537, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 533, 530, 519, 477, 512, 538, 532, 496, 514, 494, 546, 490, 564, 487, 541, 16, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 516, 496, 490, 541, 519, 523, 522, 509, 519, 514, 517, 530, 487, 506, 496, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 543, 538, 532, 500, 520, 493, 538, 557, 492, 531, 438, 521, 498, 496, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 476, 485, 500, 535, 493, 521, 514, 492, 477, 519, 514, 480, 531, 506, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 531, 493, 476, 508, 503, 523, 515, 518, 462, 534, 551, 503, 492, 517, 496, 10, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8044, 8199, 8352, 8286, 8252, 8184, 8064, 8155] avg=8192.0000 max=8352 max_rank=2 min=8044 min_rank=0 imbalance_ratio=1.019531 max_load_reduction=57184 max_load_reduction_pct=0.872559 moved_to_echo=32655 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[518, 483, 498, 505, 501, 509, 524, 493, 478, 508, 533, 510, 505, 524, 526, 480, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 499, 517, 511, 521, 504, 543, 491, 532, 498, 499, 529, 502, 531, 520, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 512, 486, 490, 495, 512, 494, 510, 519, 523, 536, 529, 510, 508, 529, 536, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 502, 526, 508, 493, 508, 511, 474, 536, 539, 514, 503, 437, 530, 496, 500, 18, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 532, 508, 457, 535, 504, 530, 532, 498, 521, 510, 533, 494, 527, 520, 513, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 514, 510, 499, 503, 513, 547, 550, 524, 505, 500, 459, 541, 505, 515, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 535, 521, 556, 514, 511, 537, 482, 498, 504, 493, 534, 521, 507, 499, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 499, 493, 529, 524, 503, 527, 508, 523, 486, 492, 498, 487, 485, 529, 527, 10, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8130, 8202, 8254, 8111, 8232, 8165, 8287, 8155] avg=8192.0000 max=8287 max_rank=6 min=8111 min_rank=3 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32839 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[485, 519, 513, 485, 505, 536, 498, 524, 511, 498, 504, 509, 512, 495, 495, 503, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 551, 512, 504, 537, 480, 492, 536, 526, 519, 494, 493, 540, 553, 527, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 516, 507, 515, 494, 538, 496, 493, 532, 532, 508, 521, 492, 478, 531, 496, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 516, 531, 554, 495, 496, 487, 526, 513, 502, 514, 491, 493, 513, 508, 528, 18, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 549, 525, 526, 515, 519, 504, 521, 502, 519, 473, 519, 492, 534, 520, 507, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 509, 515, 512, 571, 518, 509, 492, 472, 481, 509, 447, 504, 506, 513, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 496, 527, 536, 504, 506, 513, 521, 508, 504, 535, 504, 497, 515, 529, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 573, 576, 524, 529, 490, 501, 514, 514, 496, 518, 478, 523, 501, 487, 490, 497, 10, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8126, 8255, 8164, 8198, 8286, 8076, 8200, 8231] avg=8192.0000 max=8286 max_rank=4 min=8076 min_rank=5 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32793 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[536, 534, 516, 513, 514, 566, 525, 473, 493, 510, 468, 548, 497, 526, 552, 507, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 489, 498, 500, 523, 493, 548, 485, 508, 529, 491, 476, 475, 481, 549, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 503, 498, 517, 523, 476, 540, 478, 534, 559, 512, 508, 470, 512, 511, 483, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 516, 488, 539, 517, 501, 455, 567, 474, 524, 483, 558, 476, 484, 509, 498, 17, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 510, 502, 522, 490, 524, 528, 487, 496, 528, 516, 499, 502, 497, 513, 538, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 513, 495, 512, 524, 522, 495, 509, 527, 540, 503, 449, 517, 517, 520, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 489, 544, 527, 482, 487, 515, 542, 500, 493, 474, 525, 531, 543, 499, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 501, 484, 524, 525, 532, 546, 515, 510, 500, 521, 535, 492, 509, 483, 528, 10, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8312, 8032, 8182, 8159, 8218, 8202, 8156, 8275] avg=8192.0000 max=8312 max_rank=0 min=8032 min_rank=1 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32851 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[535, 521, 501, 491, 479, 507, 538, 492, 523, 508, 535, 498, 508, 482, 522, 540, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 516, 476, 490, 500, 543, 501, 496, 509, 499, 498, 492, 484, 565, 520, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 536, 481, 500, 538, 500, 512, 490, 507, 475, 480, 461, 512, 501, 533, 503, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 493, 504, 516, 486, 511, 485, 555, 513, 549, 479, 539, 531, 531, 516, 518, 17, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 497, 534, 499, 472, 476, 506, 516, 486, 528, 512, 494, 496, 505, 510, 502, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 560, 519, 519, 513, 524, 511, 526, 502, 551, 489, 519, 436, 534, 524, 518, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 527, 567, 505, 531, 521, 506, 540, 538, 507, 538, 480, 521, 497, 544, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 520, 476, 526, 515, 502, 436, 480, 515, 484, 503, 522, 533, 541, 513, 529, 10, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8155, 8073, 8260, 8065, 8263, 8345, 8162] avg=8192.0000 max=8345 max_rank=6 min=8065 min_rank=4 imbalance_ratio=1.018677 max_load_reduction=57191 max_load_reduction_pct=0.872665 moved_to_echo=32835 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[508, 503, 492, 512, 517, 504, 531, 500, 529, 541, 502, 518, 511, 509, 519, 485, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 509, 494, 489, 506, 508, 557, 502, 489, 514, 538, 561, 495, 494, 514, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 509, 502, 524, 518, 510, 486, 516, 538, 509, 504, 544, 520, 502, 515, 521, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 525, 505, 514, 490, 533, 526, 500, 504, 473, 455, 520, 522, 520, 519, 560, 16, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 465, 508, 535, 516, 497, 560, 524, 487, 507, 555, 509, 478, 545, 465, 518, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 536, 514, 513, 473, 494, 488, 537, 518, 461, 515, 455, 538, 484, 534, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 512, 500, 531, 497, 480, 509, 512, 531, 517, 554, 513, 505, 508, 524, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 480, 504, 551, 507, 470, 520, 495, 502, 512, 492, 521, 511, 514, 482, 528, 10, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8239, 8261, 8212, 8215, 8097, 8143, 8153] avg=8192.0000 max=8261 max_rank=2 min=8097 min_rank=5 imbalance_ratio=1.008423 max_load_reduction=57275 max_load_reduction_pct=0.873947 moved_to_echo=32608 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[485, 535, 509, 482, 478, 467, 503, 494, 500, 509, 522, 521, 483, 509, 506, 508, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 524, 518, 535, 494, 519, 520, 466, 476, 516, 510, 540, 518, 521, 516, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 512, 501, 504, 508, 545, 525, 532, 520, 512, 535, 521, 554, 519, 503, 537, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 533, 530, 519, 477, 512, 538, 532, 496, 514, 494, 546, 490, 564, 487, 541, 16, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 516, 496, 490, 541, 519, 523, 522, 509, 519, 514, 517, 530, 487, 506, 496, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 543, 538, 532, 500, 520, 493, 538, 557, 492, 531, 438, 521, 498, 496, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 476, 485, 500, 535, 493, 521, 514, 492, 477, 519, 514, 480, 531, 506, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 531, 493, 476, 508, 503, 523, 515, 518, 462, 534, 551, 503, 492, 517, 496, 10, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8044, 8199, 8352, 8286, 8252, 8184, 8064, 8155] avg=8192.0000 max=8352 max_rank=2 min=8044 min_rank=0 imbalance_ratio=1.019531 max_load_reduction=57184 max_load_reduction_pct=0.872559 moved_to_echo=32655 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[515, 547, 511, 533, 521, 550, 535, 497, 464, 484, 485, 492, 520, 510, 535, 524, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 525, 512, 481, 493, 505, 479, 501, 485, 521, 532, 553, 537, 497, 514, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 516, 485, 469, 454, 505, 506, 487, 533, 523, 535, 521, 510, 518, 479, 496, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 518, 483, 491, 550, 511, 531, 497, 528, 511, 535, 497, 510, 518, 469, 517, 16, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 502, 536, 506, 488, 499, 477, 485, 524, 517, 532, 503, 549, 465, 549, 506, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 560, 539, 547, 484, 556, 533, 478, 535, 497, 525, 418, 551, 520, 529, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 537, 503, 493, 532, 502, 496, 544, 514, 497, 505, 494, 528, 502, 477, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 493, 530, 499, 499, 530, 541, 536, 510, 475, 493, 547, 488, 503, 510, 513, 10, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8255, 8224, 8025, 8216, 8168, 8326, 8120, 8202] avg=8192.0000 max=8326 max_rank=5 min=8025 min_rank=2 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32816 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 3968, 4096, 3584, 4352, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 3968, 4224, 3968, 4224, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 3968, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 3968, 4096, 3584, 4352, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 3968, 4224, 3968, 4224, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 3968, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4352, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 3968, 4096, 4224, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[498, 489, 500, 511, 530, 546, 542, 466, 509, 512, 549, 470, 530, 503, 514, 466, 493, 504, 520, 507, 482, 525, 480, 514, 563, 529, 526, 523, 537, 542, 535, 491, 537, 490, 469, 515, 448, 520, 518, 518, 493, 509, 542, 536, 536, 503, 503, 516, 497, 496, 466, 533, 578, 518, 539, 562, 501, 537, 513, 550, 500, 503, 506, 529, 475, 506, 511, 489, 527, 499, 518, 524, 530, 498, 504, 511, 519, 516, 526, 488, 516, 516, 513, 515, 523, 511, 519, 510, 541, 492, 490, 488, 539, 465, 449, 519, 509, 517, 507, 486, 469, 490, 506, 532, 532, 546, 555, 527, 486, 483, 504, 505, 524, 479, 551, 524, 516, 507, 507, 532, 493, 522, 527, 510, 512, 510, 501, 533] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[507, 474, 493, 520, 533, 502, 486, 510, 548, 521, 498, 525, 478, 442, 524, 531, 504, 514, 527, 524, 532, 525, 525, 494, 523, 520, 494, 539, 539, 511, 492, 520, 488, 536, 484, 535, 544, 495, 482, 559, 514, 497, 509, 505, 537, 526, 521, 504, 524, 488, 533, 517, 525, 510, 532, 497, 501, 499, 531, 546, 507, 525, 498, 501, 513, 473, 569, 519, 507, 508, 520, 496, 485, 490, 519, 526, 504, 533, 486, 491, 475, 506, 526, 523, 540, 521, 529, 521, 513, 476, 502, 497, 493, 502, 505, 480, 488, 509, 477, 508, 530, 503, 532, 526, 561, 477, 555, 475, 542, 531, 494, 566, 499, 525, 479, 528, 532, 515, 498, 484, 491, 536, 488, 524, 522, 495, 509, 544] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[552, 516, 580, 515, 516, 509, 506, 514, 477, 501, 496, 517, 553, 514, 519, 509, 523, 512, 515, 506, 509, 461, 486, 506, 482, 520, 504, 529, 477, 530, 473, 544, 529, 542, 496, 512, 522, 544, 515, 494, 525, 530, 519, 566, 514, 480, 519, 517, 547, 506, 505, 502, 487, 485, 516, 553, 500, 517, 520, 468, 514, 545, 542, 519, 482, 501, 501, 475, 524, 495, 475, 499, 517, 536, 513, 518, 514, 512, 515, 525, 487, 539, 486, 464, 523, 548, 500, 491, 533, 498, 530, 513, 478, 501, 512, 509, 515, 501, 529, 517, 506, 502, 538, 557, 510, 541, 493, 511, 486, 485, 495, 494, 508, 551, 495, 531, 541, 524, 488, 512, 473, 503, 546, 503, 518, 520, 493, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[489, 552, 507, 490, 518, 522, 558, 487, 535, 505, 538, 464, 502, 551, 511, 493, 531, 471, 522, 527, 514, 489, 504, 465, 485, 516, 483, 534, 521, 456, 506, 536, 525, 526, 548, 508, 485, 532, 515, 523, 502, 499, 520, 502, 484, 517, 454, 501, 514, 525, 509, 506, 495, 535, 493, 501, 488, 486, 503, 546, 551, 510, 548, 544, 507, 552, 518, 525, 518, 486, 512, 512, 487, 524, 500, 521, 506, 513, 522, 536, 500, 511, 517, 496, 510, 499, 495, 505, 516, 508, 543, 490, 540, 539, 479, 543, 524, 513, 545, 510, 487, 536, 504, 513, 506, 501, 487, 532, 567, 537, 526, 520, 510, 506, 527, 486, 496, 543, 458, 528, 515, 522, 493, 477, 504, 515, 530, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[514, 516, 487, 511, 524, 535, 494, 534, 525, 539, 531, 525, 526, 525, 521, 453, 540, 537, 559, 517, 518, 500, 511, 549, 528, 506, 491, 519, 508, 499, 515, 525, 495, 523, 561, 488, 503, 507, 510, 535, 521, 512, 481, 469, 456, 489, 530, 502, 494, 504, 474, 518, 520, 519, 512, 515, 539, 482, 512, 464, 527, 511, 531, 532, 516, 474, 495, 462, 524, 526, 520, 498, 510, 526, 488, 531, 529, 525, 506, 520, 520, 495, 531, 522, 528, 497, 558, 520, 506, 533, 489, 513, 536, 488, 545, 505, 538, 489, 553, 506, 519, 518, 493, 504, 488, 485, 499, 521, 490, 556, 504, 510, 498, 504, 517, 489, 499, 489, 468, 523, 485, 530, 536, 510, 530, 492, 541, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[521, 509, 503, 490, 490, 500, 510, 469, 518, 528, 525, 510, 546, 529, 496, 521, 470, 501, 499, 503, 536, 480, 498, 528, 534, 480, 451, 555, 456, 512, 521, 516, 485, 520, 524, 519, 488, 514, 510, 521, 503, 518, 510, 481, 508, 513, 558, 511, 525, 551, 449, 484, 540, 508, 527, 545, 474, 532, 489, 547, 511, 497, 512, 542, 482, 513, 539, 512, 542, 539, 544, 521, 519, 540, 475, 512, 462, 487, 523, 504, 537, 537, 540, 523, 498, 509, 518, 492, 533, 500, 540, 506, 477, 535, 505, 529, 498, 508, 497, 516, 524, 520, 549, 513, 504, 487, 505, 502, 485, 497, 517, 474, 520, 508, 526, 547, 505, 515, 548, 546, 533, 520, 517, 478, 526, 512, 520, 505] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[519, 520, 502, 553, 486, 540, 505, 502, 519, 494, 510, 514, 525, 508, 545, 522, 493, 513, 527, 478, 506, 532, 534, 499, 523, 488, 489, 499, 502, 547, 522, 531, 522, 519, 508, 506, 481, 545, 495, 526, 515, 495, 526, 505, 496, 499, 552, 540, 506, 509, 528, 517, 579, 497, 491, 514, 558, 522, 504, 521, 484, 510, 530, 535, 525, 536, 528, 484, 513, 483, 514, 556, 518, 551, 468, 513, 552, 524, 516, 539, 490, 524, 509, 483, 475, 498, 473, 504, 487, 504, 489, 498, 495, 492, 527, 494, 514, 496, 499, 498, 506, 490, 506, 504, 551, 517, 505, 523, 525, 506, 516, 485, 498, 502, 525, 524, 536, 515, 516, 489, 486, 500, 516, 531, 496, 476, 519, 522] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[552, 516, 580, 515, 516, 509, 506, 514, 477, 501, 496, 517, 553, 514, 519, 509, 523, 512, 515, 506, 509, 461, 486, 506, 482, 520, 504, 529, 477, 530, 473, 544, 529, 542, 496, 512, 522, 544, 515, 494, 525, 530, 519, 566, 514, 480, 519, 517, 547, 506, 505, 502, 487, 485, 516, 553, 500, 517, 520, 468, 514, 545, 542, 519, 482, 501, 501, 475, 524, 495, 475, 499, 517, 536, 513, 518, 514, 512, 515, 525, 487, 539, 486, 464, 523, 548, 500, 491, 533, 498, 530, 513, 478, 501, 512, 509, 515, 501, 529, 517, 506, 502, 538, 557, 510, 541, 493, 511, 486, 485, 495, 494, 508, 551, 495, 531, 541, 524, 488, 512, 473, 503, 546, 503, 518, 520, 493, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[507, 474, 493, 520, 533, 502, 486, 510, 548, 521, 498, 525, 478, 442, 524, 531, 504, 514, 527, 524, 532, 525, 525, 494, 523, 520, 494, 539, 539, 511, 492, 520, 488, 536, 484, 535, 544, 495, 482, 559, 514, 497, 509, 505, 537, 526, 521, 504, 524, 488, 533, 517, 525, 510, 532, 497, 501, 499, 531, 546, 507, 525, 498, 501, 513, 473, 569, 519, 507, 508, 520, 496, 485, 490, 519, 526, 504, 533, 486, 491, 475, 506, 526, 523, 540, 521, 529, 521, 513, 476, 502, 497, 493, 502, 505, 480, 488, 509, 477, 508, 530, 503, 532, 526, 561, 477, 555, 475, 542, 531, 494, 566, 499, 525, 479, 528, 532, 515, 498, 484, 491, 536, 488, 524, 522, 495, 509, 544] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[500, 522, 517, 526, 509, 508, 514, 538, 521, 480, 506, 557, 489, 480, 521, 534, 506, 515, 525, 536, 463, 550, 517, 514, 525, 515, 537, 539, 499, 512, 500, 486, 491, 511, 514, 511, 487, 535, 469, 477, 470, 510, 504, 503, 549, 526, 533, 512, 530, 502, 509, 498, 526, 495, 521, 502, 523, 538, 530, 508, 526, 542, 538, 520, 521, 507, 517, 549, 516, 556, 529, 508, 496, 481, 502, 489, 505, 500, 511, 508, 487, 559, 520, 506, 509, 505, 539, 519, 507, 498, 510, 548, 467, 533, 546, 508, 500, 512, 496, 570, 540, 516, 482, 523, 454, 511, 520, 467, 482, 528, 497, 494, 504, 521, 481, 523, 465, 514, 512, 526, 477, 499, 509, 494, 548, 498, 546, 500] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[489, 552, 507, 490, 518, 496, 558, 487, 535, 505, 538, 464, 502, 551, 511, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 471, 522, 527, 514, 489, 504, 465, 485, 516, 483, 534, 521, 456, 506, 536, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 526, 548, 508, 485, 532, 515, 523, 502, 499, 520, 502, 484, 517, 454, 501, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 525, 509, 506, 422, 535, 493, 501, 488, 486, 503, 546, 551, 510, 548, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 552, 518, 525, 518, 486, 512, 512, 487, 524, 500, 521, 506, 513, 522, 536, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 511, 517, 496, 510, 499, 495, 505, 516, 508, 543, 490, 540, 539, 479, 543, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 513, 545, 510, 487, 536, 504, 513, 506, 501, 487, 532, 567, 537, 526, 520, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 506, 527, 486, 496, 543, 458, 528, 515, 522, 493, 477, 504, 515, 530, 511, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8071, 8155, 8181, 8247, 8224, 8325, 8137] avg=8192.0000 max=8325 max_rank=6 min=8071 min_rank=1 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32933 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[498, 489, 500, 511, 530, 546, 542, 466, 509, 512, 549, 470, 530, 503, 514, 466, 493, 504, 520, 507, 482, 525, 480, 514, 563, 529, 526, 523, 537, 542, 535, 491, 537, 490, 469, 515, 448, 520, 518, 518, 493, 509, 542, 536, 536, 503, 503, 516, 497, 496, 466, 533, 578, 518, 539, 562, 501, 537, 513, 550, 500, 503, 506, 529, 475, 506, 511, 489, 527, 499, 518, 524, 530, 498, 504, 511, 519, 516, 526, 488, 516, 516, 513, 515, 523, 511, 519, 510, 541, 492, 490, 488, 539, 465, 449, 519, 509, 517, 507, 486, 469, 490, 506, 532, 532, 546, 555, 527, 486, 483, 504, 505, 524, 479, 551, 524, 516, 507, 507, 532, 493, 522, 527, 510, 512, 510, 501, 533] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[521, 509, 503, 490, 490, 500, 510, 469, 518, 528, 525, 510, 546, 529, 496, 521, 470, 501, 499, 503, 536, 480, 498, 528, 534, 480, 451, 555, 456, 512, 521, 516, 485, 520, 524, 519, 488, 514, 510, 521, 503, 518, 510, 481, 508, 513, 558, 511, 525, 551, 449, 484, 540, 508, 527, 545, 474, 532, 489, 547, 511, 497, 512, 542, 482, 513, 539, 512, 542, 539, 544, 521, 519, 540, 475, 512, 462, 487, 523, 504, 537, 537, 540, 523, 498, 509, 518, 492, 533, 500, 540, 506, 477, 535, 505, 529, 498, 508, 497, 516, 524, 520, 549, 513, 504, 487, 505, 502, 485, 497, 517, 474, 520, 508, 526, 547, 505, 515, 548, 546, 533, 520, 517, 478, 526, 512, 520, 505] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[514, 516, 487, 511, 524, 522, 494, 534, 525, 539, 531, 525, 526, 525, 521, 453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 537, 559, 517, 518, 500, 511, 549, 528, 506, 491, 519, 508, 499, 515, 525, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 523, 561, 488, 503, 507, 510, 535, 521, 512, 481, 469, 456, 489, 530, 502, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 504, 474, 518, 460, 519, 512, 515, 539, 482, 512, 464, 527, 511, 531, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 474, 495, 462, 524, 526, 520, 498, 510, 526, 488, 531, 529, 525, 506, 520, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 495, 531, 522, 528, 497, 558, 520, 506, 533, 489, 513, 536, 488, 545, 505, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 489, 553, 506, 519, 518, 493, 504, 488, 485, 499, 521, 490, 556, 504, 510, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 504, 517, 489, 499, 489, 468, 523, 485, 530, 536, 510, 530, 492, 541, 498, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8247, 8329, 8092, 8094, 8154, 8316, 8187, 8117] avg=8192.0000 max=8329 max_rank=1 min=8092 min_rank=2 imbalance_ratio=1.016724 max_load_reduction=57207 max_load_reduction_pct=0.872910 moved_to_echo=32774 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[498, 489, 500, 511, 530, 532, 542, 466, 509, 512, 549, 470, 530, 503, 514, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 504, 520, 507, 482, 525, 480, 514, 563, 529, 526, 523, 537, 542, 535, 491, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 490, 469, 515, 448, 520, 518, 518, 493, 509, 542, 536, 536, 503, 503, 516, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 496, 466, 533, 510, 518, 539, 562, 501, 537, 513, 550, 500, 503, 506, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 506, 511, 489, 527, 499, 518, 524, 530, 498, 504, 511, 519, 516, 526, 488, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 516, 513, 515, 523, 511, 519, 510, 541, 492, 490, 488, 539, 465, 449, 519, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 517, 507, 486, 469, 490, 506, 532, 532, 546, 555, 527, 486, 483, 504, 505, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 479, 551, 524, 516, 507, 507, 532, 493, 522, 527, 510, 512, 510, 501, 533, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8121, 8279, 8164, 8260, 8145, 8140, 8170, 8257] avg=8192.0000 max=8279 max_rank=1 min=8121 min_rank=0 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32712 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[500, 522, 517, 526, 509, 495, 514, 538, 521, 480, 506, 557, 489, 480, 521, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 515, 525, 536, 463, 550, 517, 514, 525, 515, 537, 539, 499, 512, 500, 486, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 511, 514, 511, 487, 535, 469, 477, 470, 510, 504, 503, 549, 526, 533, 512, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 502, 509, 498, 466, 495, 521, 502, 523, 538, 530, 508, 526, 542, 538, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 507, 517, 549, 516, 556, 529, 508, 496, 481, 502, 489, 505, 500, 511, 508, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 559, 520, 506, 509, 505, 539, 519, 507, 498, 510, 548, 467, 533, 546, 508, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 512, 496, 570, 540, 516, 482, 523, 454, 511, 520, 467, 482, 528, 497, 494, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 521, 481, 523, 465, 514, 512, 526, 477, 499, 509, 494, 548, 498, 546, 500, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8246, 8112, 8248, 8199, 8291, 8106, 8125] avg=8192.0000 max=8291 max_rank=5 min=8106 min_rank=6 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32721 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[519, 520, 502, 553, 486, 527, 505, 502, 519, 494, 510, 514, 525, 508, 545, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 513, 527, 478, 506, 532, 534, 499, 523, 488, 489, 499, 502, 547, 522, 531, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 519, 508, 506, 481, 545, 495, 526, 515, 495, 526, 505, 496, 499, 552, 540, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 509, 528, 517, 511, 497, 491, 514, 558, 522, 504, 521, 484, 510, 530, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 536, 528, 484, 513, 483, 514, 556, 518, 551, 468, 513, 552, 524, 516, 539, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 524, 509, 483, 475, 498, 473, 504, 487, 504, 489, 498, 495, 492, 527, 494, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 496, 499, 498, 506, 490, 506, 504, 551, 517, 505, 523, 525, 506, 516, 485, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 502, 525, 524, 536, 515, 516, 489, 486, 500, 516, 531, 496, 476, 519, 522, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8251, 8190, 8241, 8237, 8324, 7976, 8157, 8160] avg=8192.0000 max=8324 max_rank=4 min=7976 min_rank=5 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32617 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[552, 516, 580, 515, 516, 496, 506, 514, 477, 501, 496, 517, 553, 514, 519, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 512, 515, 506, 509, 461, 486, 506, 482, 520, 504, 529, 477, 530, 473, 544, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 542, 496, 512, 522, 544, 515, 494, 525, 530, 519, 566, 514, 480, 519, 517, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 506, 505, 502, 431, 485, 516, 553, 500, 517, 520, 468, 514, 545, 542, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 501, 501, 475, 524, 495, 475, 499, 517, 536, 513, 518, 514, 512, 515, 525, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 539, 486, 464, 523, 548, 500, 491, 533, 498, 530, 513, 478, 501, 512, 509, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 501, 529, 517, 506, 502, 538, 557, 510, 541, 493, 511, 486, 485, 495, 494, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 551, 495, 531, 541, 524, 488, 512, 473, 503, 546, 503, 518, 520, 493, 515, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8281, 8084, 8333, 8170, 8106, 8140, 8193, 8229] avg=8192.0000 max=8333 max_rank=2 min=8084 min_rank=1 imbalance_ratio=1.017212 max_load_reduction=57203 max_load_reduction_pct=0.872849 moved_to_echo=32668 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[521, 509, 503, 490, 490, 487, 510, 469, 518, 528, 525, 510, 546, 529, 496, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 501, 499, 503, 536, 480, 498, 528, 534, 480, 451, 555, 456, 512, 521, 516, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 520, 524, 519, 488, 514, 510, 521, 503, 518, 510, 481, 508, 513, 558, 511, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 551, 449, 484, 479, 508, 527, 545, 474, 532, 489, 547, 511, 497, 512, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 513, 539, 512, 542, 539, 544, 521, 519, 540, 475, 512, 462, 487, 523, 504, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 537, 540, 523, 498, 509, 518, 492, 533, 500, 540, 506, 477, 535, 505, 529, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 508, 497, 516, 524, 520, 549, 513, 504, 487, 505, 502, 485, 497, 517, 474, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 508, 526, 547, 505, 515, 548, 546, 533, 520, 517, 478, 526, 512, 520, 505, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8152, 8047, 8193, 8172, 8218, 8310, 8110, 8334] avg=8192.0000 max=8334 max_rank=7 min=8047 min_rank=1 imbalance_ratio=1.017334 max_load_reduction=57202 max_load_reduction_pct=0.872833 moved_to_echo=32972 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[489, 552, 507, 490, 518, 496, 558, 487, 535, 505, 538, 464, 502, 551, 511, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 471, 522, 527, 514, 489, 504, 465, 485, 516, 483, 534, 521, 456, 506, 536, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 526, 548, 508, 485, 532, 515, 523, 502, 499, 520, 502, 484, 517, 454, 501, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 525, 509, 506, 422, 535, 493, 501, 488, 486, 503, 546, 551, 510, 548, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 552, 518, 525, 518, 486, 512, 512, 487, 524, 500, 521, 506, 513, 522, 536, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 511, 517, 496, 510, 499, 495, 505, 516, 508, 543, 490, 540, 539, 479, 543, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 513, 545, 510, 487, 536, 504, 513, 506, 501, 487, 532, 567, 537, 526, 520, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 506, 527, 486, 496, 543, 458, 528, 515, 522, 493, 477, 504, 515, 530, 511, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8071, 8155, 8181, 8247, 8224, 8325, 8137] avg=8192.0000 max=8325 max_rank=6 min=8071 min_rank=1 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32933 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[519, 520, 502, 553, 486, 527, 505, 502, 519, 494, 510, 514, 525, 508, 545, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 513, 527, 478, 506, 532, 534, 499, 523, 488, 489, 499, 502, 547, 522, 531, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 519, 508, 506, 481, 545, 495, 526, 515, 495, 526, 505, 496, 499, 552, 540, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 509, 528, 517, 511, 497, 491, 514, 558, 522, 504, 521, 484, 510, 530, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 536, 528, 484, 513, 483, 514, 556, 518, 551, 468, 513, 552, 524, 516, 539, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 524, 509, 483, 475, 498, 473, 504, 487, 504, 489, 498, 495, 492, 527, 494, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 496, 499, 498, 506, 490, 506, 504, 551, 517, 505, 523, 525, 506, 516, 485, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 502, 525, 524, 536, 515, 516, 489, 486, 500, 516, 531, 496, 476, 519, 522, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8251, 8190, 8241, 8237, 8324, 7976, 8157, 8160] avg=8192.0000 max=8324 max_rank=4 min=7976 min_rank=5 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32617 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[500, 522, 517, 526, 509, 495, 514, 538, 521, 480, 506, 557, 489, 480, 521, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 515, 525, 536, 463, 550, 517, 514, 525, 515, 537, 539, 499, 512, 500, 486, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 511, 514, 511, 487, 535, 469, 477, 470, 510, 504, 503, 549, 526, 533, 512, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 502, 509, 498, 466, 495, 521, 502, 523, 538, 530, 508, 526, 542, 538, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 507, 517, 549, 516, 556, 529, 508, 496, 481, 502, 489, 505, 500, 511, 508, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 559, 520, 506, 509, 505, 539, 519, 507, 498, 510, 548, 467, 533, 546, 508, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 512, 496, 570, 540, 516, 482, 523, 454, 511, 520, 467, 482, 528, 497, 494, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 521, 481, 523, 465, 514, 512, 526, 477, 499, 509, 494, 548, 498, 546, 500, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8246, 8112, 8248, 8199, 8291, 8106, 8125] avg=8192.0000 max=8291 max_rank=5 min=8106 min_rank=6 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32721 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[498, 489, 500, 511, 530, 532, 542, 466, 509, 512, 549, 470, 530, 503, 514, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 504, 520, 507, 482, 525, 480, 514, 563, 529, 526, 523, 537, 542, 535, 491, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 490, 469, 515, 448, 520, 518, 518, 493, 509, 542, 536, 536, 503, 503, 516, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 496, 466, 533, 510, 518, 539, 562, 501, 537, 513, 550, 500, 503, 506, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 506, 511, 489, 527, 499, 518, 524, 530, 498, 504, 511, 519, 516, 526, 488, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 516, 513, 515, 523, 511, 519, 510, 541, 492, 490, 488, 539, 465, 449, 519, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 517, 507, 486, 469, 490, 506, 532, 532, 546, 555, 527, 486, 483, 504, 505, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 479, 551, 524, 516, 507, 507, 532, 493, 522, 527, 510, 512, 510, 501, 533, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8121, 8279, 8164, 8260, 8145, 8140, 8170, 8257] avg=8192.0000 max=8279 max_rank=1 min=8121 min_rank=0 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32712 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[507, 474, 493, 520, 533, 489, 486, 510, 548, 521, 498, 525, 478, 442, 524, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 514, 527, 524, 532, 525, 525, 494, 523, 520, 494, 539, 539, 511, 492, 520, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 536, 484, 535, 544, 495, 482, 559, 514, 497, 509, 505, 537, 526, 521, 504, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 488, 533, 517, 465, 510, 532, 497, 501, 499, 531, 546, 507, 525, 498, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 473, 569, 519, 507, 508, 520, 496, 485, 490, 519, 526, 504, 533, 486, 491, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 506, 526, 523, 540, 521, 529, 521, 513, 476, 502, 497, 493, 502, 505, 480, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 509, 477, 508, 530, 503, 532, 526, 561, 477, 555, 475, 542, 531, 494, 566, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 525, 479, 528, 532, 515, 498, 484, 491, 536, 488, 524, 522, 495, 509, 544, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8079, 8290, 8246, 8174, 8143, 8139, 8288, 8177] avg=8192.0000 max=8290 max_rank=1 min=8079 min_rank=0 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32747 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[521, 509, 503, 490, 490, 487, 510, 469, 518, 528, 525, 510, 546, 529, 496, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 501, 499, 503, 536, 480, 498, 528, 534, 480, 451, 555, 456, 512, 521, 516, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 520, 524, 519, 488, 514, 510, 521, 503, 518, 510, 481, 508, 513, 558, 511, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 551, 449, 484, 479, 508, 527, 545, 474, 532, 489, 547, 511, 497, 512, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 513, 539, 512, 542, 539, 544, 521, 519, 540, 475, 512, 462, 487, 523, 504, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 537, 540, 523, 498, 509, 518, 492, 533, 500, 540, 506, 477, 535, 505, 529, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 508, 497, 516, 524, 520, 549, 513, 504, 487, 505, 502, 485, 497, 517, 474, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 508, 526, 547, 505, 515, 548, 546, 533, 520, 517, 478, 526, 512, 520, 505, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8152, 8047, 8193, 8172, 8218, 8310, 8110, 8334] avg=8192.0000 max=8334 max_rank=7 min=8047 min_rank=1 imbalance_ratio=1.017334 max_load_reduction=57202 max_load_reduction_pct=0.872833 moved_to_echo=32972 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4352, 4096, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4096, 3968, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4096, 3840, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4352, 4096, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4096, 3968, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 3968, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[524, 520, 489, 468, 501, 509, 484, 517, 467, 503, 510, 501, 556, 503, 529, 502, 477, 522, 536, 519, 554, 512, 496, 516, 527, 510, 536, 507, 486, 508, 519, 495, 536, 517, 490, 503, 474, 514, 540, 469, 499, 504, 477, 511, 517, 494, 546, 499, 495, 486, 517, 511, 484, 501, 488, 508, 542, 526, 512, 512, 489, 446, 518, 481, 481, 520, 500, 513, 493, 575, 511, 509, 521, 491, 478, 520, 482, 512, 532, 540, 535, 468, 513, 499, 509, 533, 515, 522, 511, 550, 501, 510, 526, 517, 546, 522, 534, 520, 511, 542, 560, 551, 525, 541, 514, 482, 530, 512, 487, 503, 494, 551, 506, 545, 501, 521, 498, 492, 491, 511, 534, 538, 541, 532, 508, 516, 543, 538] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[527, 550, 476, 484, 503, 505, 503, 499, 497, 513, 498, 559, 532, 491, 498, 483, 489, 503, 555, 493, 497, 526, 527, 528, 529, 496, 528, 515, 545, 477, 499, 487, 513, 513, 540, 500, 551, 514, 516, 488, 461, 499, 495, 526, 499, 486, 523, 485, 502, 564, 503, 531, 523, 523, 527, 478, 512, 508, 546, 530, 499, 494, 512, 493, 522, 516, 491, 496, 502, 480, 508, 535, 543, 504, 502, 521, 496, 487, 559, 501, 527, 546, 500, 499, 504, 523, 480, 473, 527, 554, 486, 507, 509, 516, 535, 473, 541, 504, 528, 527, 525, 518, 497, 535, 494, 505, 521, 497, 530, 508, 503, 556, 513, 481, 505, 506, 559, 529, 508, 482, 541, 545, 528, 524, 492, 514, 511, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[517, 527, 523, 489, 543, 485, 512, 459, 523, 484, 503, 480, 527, 486, 523, 550, 478, 526, 485, 542, 484, 524, 521, 532, 455, 537, 509, 485, 544, 524, 492, 500, 510, 507, 514, 506, 554, 533, 512, 493, 516, 542, 470, 511, 538, 482, 520, 499, 517, 525, 527, 555, 562, 542, 511, 522, 510, 553, 498, 507, 513, 538, 489, 516, 541, 526, 458, 510, 553, 516, 513, 504, 490, 520, 524, 515, 522, 512, 539, 523, 497, 505, 530, 524, 497, 492, 503, 536, 501, 511, 515, 491, 478, 496, 482, 512, 529, 546, 495, 528, 493, 528, 517, 523, 522, 502, 514, 493, 518, 509, 456, 515, 528, 505, 528, 499, 525, 493, 502, 522, 500, 478, 509, 543, 486, 480, 514, 539] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[498, 560, 521, 509, 524, 521, 526, 535, 540, 505, 502, 556, 513, 508, 513, 495, 543, 502, 529, 506, 509, 506, 480, 502, 519, 483, 515, 534, 524, 499, 513, 514, 516, 507, 478, 513, 531, 488, 468, 520, 510, 511, 525, 533, 489, 540, 488, 493, 543, 503, 520, 468, 497, 533, 478, 529, 512, 495, 527, 535, 509, 493, 511, 520, 496, 502, 459, 502, 506, 487, 509, 490, 519, 527, 532, 537, 490, 476, 527, 546, 542, 493, 517, 535, 484, 525, 519, 502, 498, 484, 474, 501, 510, 533, 488, 534, 529, 490, 496, 529, 516, 533, 506, 530, 536, 530, 521, 474, 502, 529, 521, 510, 468, 540, 530, 528, 526, 525, 515, 538, 503, 511, 508, 487, 537, 510, 538, 489] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[502, 524, 512, 471, 479, 503, 501, 492, 557, 529, 493, 566, 531, 534, 538, 482, 483, 545, 534, 545, 496, 543, 504, 546, 478, 530, 532, 486, 452, 499, 502, 480, 497, 516, 515, 514, 492, 511, 517, 478, 484, 517, 475, 490, 514, 473, 508, 533, 488, 492, 545, 506, 554, 492, 528, 521, 547, 494, 495, 538, 509, 539, 481, 484, 490, 504, 519, 537, 537, 491, 505, 460, 526, 562, 512, 557, 476, 472, 547, 501, 526, 511, 504, 525, 542, 502, 537, 519, 513, 549, 517, 505, 503, 566, 504, 489, 548, 533, 470, 512, 517, 482, 540, 475, 488, 550, 484, 500, 476, 480, 494, 568, 510, 490, 556, 506, 517, 540, 482, 510, 570, 533, 492, 511, 542, 520, 543, 473] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[498, 560, 521, 509, 524, 521, 526, 535, 540, 505, 502, 556, 513, 508, 513, 495, 543, 502, 529, 506, 509, 506, 480, 502, 519, 483, 515, 534, 524, 499, 513, 514, 516, 507, 478, 513, 531, 488, 468, 520, 510, 511, 525, 533, 489, 540, 488, 493, 543, 503, 520, 468, 497, 533, 478, 529, 512, 495, 527, 535, 509, 493, 511, 520, 496, 502, 459, 502, 506, 487, 509, 490, 519, 527, 532, 537, 490, 476, 527, 546, 542, 493, 517, 535, 484, 525, 519, 502, 498, 484, 474, 501, 510, 533, 488, 534, 529, 490, 496, 529, 516, 533, 506, 530, 536, 530, 521, 474, 502, 529, 521, 510, 468, 540, 530, 528, 526, 525, 515, 538, 503, 511, 508, 487, 537, 510, 538, 489] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[507, 541, 515, 554, 506, 515, 520, 489, 575, 525, 468, 479, 544, 518, 521, 527, 514, 496, 550, 483, 507, 507, 518, 510, 494, 511, 495, 493, 481, 518, 504, 512, 506, 527, 491, 485, 553, 537, 488, 509, 494, 510, 506, 492, 531, 490, 477, 505, 496, 489, 496, 522, 502, 525, 511, 561, 507, 506, 478, 530, 531, 530, 522, 537, 525, 505, 518, 475, 538, 489, 546, 509, 502, 508, 483, 491, 502, 520, 544, 523, 518, 564, 533, 545, 508, 532, 535, 473, 518, 547, 546, 519, 511, 480, 513, 516, 511, 478, 493, 496, 454, 552, 491, 558, 491, 488, 480, 508, 540, 507, 476, 526, 523, 517, 526, 517, 517, 462, 532, 532, 535, 522, 498, 507, 481, 509, 485, 547] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[533, 503, 517, 502, 526, 499, 558, 528, 504, 542, 521, 475, 533, 501, 492, 551, 485, 478, 502, 507, 514, 512, 516, 527, 466, 499, 505, 488, 494, 522, 503, 501, 536, 483, 546, 484, 578, 504, 538, 512, 477, 476, 502, 519, 530, 522, 475, 506, 459, 499, 525, 497, 538, 558, 498, 524, 475, 517, 493, 499, 515, 503, 557, 502, 500, 529, 516, 539, 477, 504, 492, 519, 513, 527, 539, 557, 517, 499, 509, 473, 542, 522, 523, 455, 512, 532, 501, 516, 542, 492, 472, 495, 523, 543, 536, 543, 519, 522, 533, 508, 531, 566, 528, 507, 536, 486, 518, 507, 462, 499, 490, 479, 507, 517, 537, 515, 498, 520, 539, 514, 522, 562, 494, 479, 536, 476, 513, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[521, 504, 515, 474, 503, 498, 527, 512, 543, 487, 540, 497, 499, 499, 537, 487, 511, 525, 495, 506, 488, 501, 540, 494, 481, 482, 515, 552, 530, 542, 483, 511, 499, 481, 494, 535, 522, 491, 493, 554, 506, 499, 513, 519, 508, 535, 502, 536, 540, 530, 513, 527, 527, 493, 535, 521, 520, 485, 500, 493, 521, 532, 476, 503, 541, 512, 503, 533, 527, 507, 465, 500, 541, 479, 523, 498, 502, 498, 510, 525, 481, 529, 493, 504, 522, 520, 495, 528, 527, 528, 490, 552, 546, 567, 462, 502, 494, 536, 484, 526, 533, 544, 503, 523, 496, 523, 521, 515, 526, 525, 486, 505, 494, 530, 542, 502, 512, 528, 515, 478, 502, 539, 526, 490, 491, 518, 493, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[527, 550, 476, 484, 503, 505, 503, 499, 497, 513, 498, 559, 524, 491, 498, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 503, 555, 493, 497, 526, 527, 528, 529, 496, 528, 515, 545, 477, 499, 487, 17, 15, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 513, 540, 500, 551, 514, 516, 488, 461, 499, 495, 526, 499, 486, 523, 485, 47, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 564, 503, 531, 510, 523, 527, 478, 512, 508, 546, 530, 499, 494, 512, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 516, 491, 496, 502, 480, 508, 535, 543, 504, 502, 521, 496, 487, 559, 501, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 546, 500, 499, 504, 523, 480, 473, 527, 554, 486, 507, 509, 484, 535, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 504, 528, 527, 525, 501, 497, 535, 494, 505, 521, 497, 530, 508, 503, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 481, 505, 506, 559, 529, 508, 482, 541, 498, 528, 524, 492, 514, 511, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8110, 8239, 8173, 8232, 8171, 8127, 8272, 8212] avg=8192.0000 max=8272 max_rank=6 min=8110 min_rank=0 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32782 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[533, 503, 517, 502, 526, 499, 558, 528, 504, 542, 521, 475, 533, 501, 492, 551, 485, 478, 502, 507, 514, 512, 516, 527, 466, 499, 505, 488, 494, 522, 503, 501, 536, 483, 546, 484, 578, 504, 538, 512, 477, 476, 502, 519, 530, 522, 475, 506, 459, 499, 525, 497, 538, 558, 498, 524, 475, 517, 493, 499, 515, 503, 557, 502, 500, 529, 516, 539, 477, 504, 492, 519, 513, 527, 539, 557, 517, 499, 509, 473, 542, 522, 523, 455, 512, 532, 501, 516, 542, 492, 472, 495, 523, 543, 536, 543, 519, 522, 533, 508, 531, 566, 528, 507, 536, 486, 518, 507, 462, 499, 490, 479, 507, 517, 537, 515, 498, 520, 539, 514, 522, 562, 494, 479, 536, 476, 513, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[524, 520, 489, 468, 501, 509, 484, 517, 467, 503, 510, 501, 556, 503, 529, 502, 477, 522, 536, 519, 554, 512, 496, 516, 527, 510, 536, 507, 486, 508, 519, 495, 536, 517, 490, 503, 474, 514, 540, 469, 499, 504, 477, 511, 517, 494, 546, 499, 495, 486, 517, 511, 484, 501, 488, 508, 542, 526, 512, 512, 489, 446, 518, 481, 481, 520, 500, 513, 493, 575, 511, 509, 521, 491, 478, 520, 482, 512, 532, 540, 535, 468, 513, 499, 509, 533, 515, 522, 511, 550, 501, 510, 526, 517, 546, 522, 534, 520, 511, 542, 560, 551, 525, 541, 514, 482, 530, 512, 487, 503, 494, 551, 506, 545, 501, 521, 498, 492, 491, 511, 534, 538, 541, 532, 508, 516, 543, 538] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[527, 550, 476, 484, 503, 505, 503, 499, 497, 513, 498, 559, 532, 491, 498, 483, 489, 503, 555, 493, 497, 526, 527, 528, 529, 496, 528, 515, 545, 477, 499, 487, 513, 513, 540, 500, 551, 514, 516, 488, 461, 499, 495, 526, 499, 486, 523, 485, 502, 564, 503, 531, 523, 523, 527, 478, 512, 508, 546, 530, 499, 494, 512, 493, 522, 516, 491, 496, 502, 480, 508, 535, 543, 504, 502, 521, 496, 487, 559, 501, 527, 546, 500, 499, 504, 523, 480, 473, 527, 554, 486, 507, 509, 516, 535, 473, 541, 504, 528, 527, 525, 518, 497, 535, 494, 505, 521, 497, 530, 508, 503, 556, 513, 481, 505, 506, 559, 529, 508, 482, 541, 545, 528, 524, 492, 514, 511, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[517, 527, 523, 489, 543, 485, 512, 459, 523, 484, 503, 480, 519, 486, 523, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 526, 485, 542, 484, 524, 521, 532, 455, 537, 509, 485, 544, 524, 492, 500, 18, 15, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 507, 514, 506, 554, 533, 512, 493, 516, 542, 470, 511, 538, 482, 520, 499, 41, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 525, 527, 555, 547, 542, 511, 522, 510, 553, 498, 507, 513, 538, 489, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 526, 458, 510, 553, 516, 513, 504, 490, 520, 524, 515, 522, 512, 539, 523, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 505, 530, 524, 497, 492, 503, 536, 501, 511, 515, 491, 478, 465, 482, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 546, 495, 528, 493, 510, 517, 523, 522, 502, 514, 493, 518, 509, 456, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 505, 528, 499, 525, 493, 502, 522, 500, 437, 509, 543, 486, 480, 514, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8123, 8185, 8264, 8370, 8275, 8039, 8170, 8110] avg=8192.0000 max=8370 max_rank=3 min=8039 min_rank=5 imbalance_ratio=1.021729 max_load_reduction=57166 max_load_reduction_pct=0.872284 moved_to_echo=32594 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[498, 560, 521, 509, 524, 521, 526, 535, 540, 505, 502, 556, 501, 508, 513, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 502, 529, 506, 509, 506, 480, 502, 519, 483, 515, 534, 524, 499, 513, 514, 23, 21, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 507, 478, 513, 531, 488, 468, 520, 510, 511, 525, 533, 489, 540, 488, 493, 47, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 503, 520, 468, 478, 533, 478, 529, 512, 495, 527, 535, 509, 493, 511, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 502, 459, 502, 506, 487, 509, 490, 519, 527, 532, 537, 490, 476, 527, 546, 12, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 493, 517, 535, 484, 525, 519, 502, 498, 484, 474, 501, 510, 490, 488, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 490, 496, 529, 516, 510, 506, 530, 536, 530, 521, 474, 502, 529, 521, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 540, 530, 528, 526, 525, 515, 538, 503, 464, 508, 487, 537, 510, 538, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8314, 8237, 8179, 8154, 8121, 8096, 8229, 8206] avg=8192.0000 max=8314 max_rank=0 min=8096 min_rank=5 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32652 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[524, 520, 489, 468, 501, 509, 484, 517, 467, 503, 510, 501, 548, 503, 529, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 522, 536, 519, 554, 512, 496, 516, 527, 510, 536, 507, 486, 508, 519, 495, 18, 15, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 517, 490, 503, 474, 514, 540, 469, 499, 504, 477, 511, 517, 494, 546, 499, 46, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 486, 517, 511, 472, 501, 488, 508, 542, 526, 512, 512, 489, 446, 518, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 520, 500, 513, 493, 575, 511, 509, 521, 491, 478, 520, 482, 512, 532, 540, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 468, 513, 499, 509, 533, 515, 522, 511, 550, 501, 510, 526, 485, 546, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 520, 511, 542, 560, 533, 525, 541, 514, 482, 530, 512, 487, 503, 494, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 545, 501, 521, 498, 492, 491, 511, 534, 492, 541, 532, 508, 516, 543, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8075, 8265, 8153, 8004, 8186, 8245, 8339, 8269] avg=8192.0000 max=8339 max_rank=6 min=8004 min_rank=3 imbalance_ratio=1.017944 max_load_reduction=57197 max_load_reduction_pct=0.872757 moved_to_echo=33039 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[533, 503, 517, 502, 526, 499, 558, 528, 504, 542, 521, 475, 525, 501, 492, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 478, 502, 507, 514, 512, 516, 527, 466, 499, 505, 488, 494, 522, 503, 501, 19, 16, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 483, 546, 484, 578, 504, 538, 512, 477, 476, 502, 519, 530, 522, 475, 506, 48, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 459, 499, 525, 497, 524, 558, 498, 524, 475, 517, 493, 499, 515, 503, 557, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 529, 516, 539, 477, 504, 492, 519, 513, 527, 539, 557, 517, 499, 509, 473, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 522, 523, 455, 512, 532, 501, 516, 542, 492, 472, 495, 523, 509, 536, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 522, 533, 508, 531, 547, 528, 507, 536, 486, 518, 507, 462, 499, 490, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 517, 537, 515, 498, 520, 539, 514, 522, 514, 494, 479, 536, 476, 513, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8277, 8067, 8254, 8145, 8219, 8215, 8172, 8187] avg=8192.0000 max=8277 max_rank=0 min=8067 min_rank=1 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32793 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[507, 541, 515, 554, 506, 515, 520, 489, 575, 525, 468, 479, 536, 518, 521, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 496, 550, 483, 507, 507, 518, 510, 494, 511, 495, 493, 481, 518, 504, 512, 18, 14, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 527, 491, 485, 553, 537, 488, 509, 494, 510, 506, 492, 531, 490, 477, 505, 45, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 489, 496, 522, 490, 525, 511, 561, 507, 506, 478, 530, 531, 530, 522, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 505, 518, 475, 538, 489, 546, 509, 502, 508, 483, 491, 502, 520, 544, 523, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 564, 533, 545, 508, 532, 535, 473, 518, 547, 546, 519, 511, 450, 513, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 478, 493, 496, 454, 534, 491, 558, 491, 488, 480, 508, 540, 507, 476, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 517, 526, 517, 517, 462, 532, 532, 535, 477, 498, 507, 481, 509, 485, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8296, 8137, 8162, 8231, 8186, 8328, 8031, 8165] avg=8192.0000 max=8328 max_rank=5 min=8031 min_rank=6 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=32710 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[502, 524, 512, 471, 479, 503, 501, 492, 557, 529, 493, 566, 523, 534, 538, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 545, 534, 545, 496, 543, 504, 546, 478, 530, 532, 486, 452, 499, 502, 480, 16, 17, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 516, 515, 514, 492, 511, 517, 478, 484, 517, 475, 490, 514, 473, 508, 533, 46, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 492, 545, 506, 539, 492, 528, 521, 547, 494, 495, 538, 509, 539, 481, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 504, 519, 537, 537, 491, 505, 460, 526, 562, 512, 557, 476, 472, 547, 501, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 511, 504, 525, 542, 502, 537, 519, 513, 549, 517, 505, 503, 530, 504, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 533, 470, 512, 517, 466, 540, 475, 488, 550, 484, 500, 476, 480, 494, 568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 490, 556, 506, 517, 540, 482, 510, 570, 487, 492, 511, 542, 520, 543, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8202, 8099, 8198, 8205, 8276, 8101, 8249] avg=8192.0000 max=8276 max_rank=5 min=8099 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32831 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[521, 504, 515, 474, 503, 498, 527, 512, 543, 487, 540, 497, 491, 499, 537, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 525, 495, 506, 488, 501, 540, 494, 481, 482, 515, 552, 530, 542, 483, 511, 18, 17, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 481, 494, 535, 522, 491, 493, 554, 506, 499, 513, 519, 508, 535, 502, 536, 46, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 530, 513, 527, 513, 493, 535, 521, 520, 485, 500, 493, 521, 532, 476, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 512, 503, 533, 527, 507, 465, 500, 541, 479, 523, 498, 502, 498, 510, 525, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 529, 493, 504, 522, 520, 495, 528, 527, 528, 490, 552, 546, 531, 462, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 536, 484, 526, 533, 526, 503, 523, 496, 523, 521, 515, 526, 525, 486, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 530, 542, 502, 512, 528, 515, 478, 502, 493, 526, 490, 491, 518, 493, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8135, 8204, 8252, 8202, 8173, 8210, 8222, 8138] avg=8192.0000 max=8252 max_rank=2 min=8135 min_rank=0 imbalance_ratio=1.007324 max_load_reduction=57284 max_load_reduction_pct=0.874084 moved_to_echo=32743 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[533, 503, 517, 502, 526, 499, 558, 528, 504, 542, 521, 475, 525, 501, 492, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 478, 502, 507, 514, 512, 516, 527, 466, 499, 505, 488, 494, 522, 503, 501, 19, 16, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 483, 546, 484, 578, 504, 538, 512, 477, 476, 502, 519, 530, 522, 475, 506, 48, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 459, 499, 525, 497, 524, 558, 498, 524, 475, 517, 493, 499, 515, 503, 557, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 529, 516, 539, 477, 504, 492, 519, 513, 527, 539, 557, 517, 499, 509, 473, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 522, 523, 455, 512, 532, 501, 516, 542, 492, 472, 495, 523, 509, 536, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 522, 533, 508, 531, 547, 528, 507, 536, 486, 518, 507, 462, 499, 490, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 517, 537, 515, 498, 520, 539, 514, 522, 514, 494, 479, 536, 476, 513, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8277, 8067, 8254, 8145, 8219, 8215, 8172, 8187] avg=8192.0000 max=8277 max_rank=0 min=8067 min_rank=1 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32793 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[524, 520, 489, 468, 501, 509, 484, 517, 467, 503, 510, 501, 548, 503, 529, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 522, 536, 519, 554, 512, 496, 516, 527, 510, 536, 507, 486, 508, 519, 495, 18, 15, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 517, 490, 503, 474, 514, 540, 469, 499, 504, 477, 511, 517, 494, 546, 499, 46, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 486, 517, 511, 472, 501, 488, 508, 542, 526, 512, 512, 489, 446, 518, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 520, 500, 513, 493, 575, 511, 509, 521, 491, 478, 520, 482, 512, 532, 540, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 468, 513, 499, 509, 533, 515, 522, 511, 550, 501, 510, 526, 485, 546, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 520, 511, 542, 560, 533, 525, 541, 514, 482, 530, 512, 487, 503, 494, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 545, 501, 521, 498, 492, 491, 511, 534, 492, 541, 532, 508, 516, 543, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8075, 8265, 8153, 8004, 8186, 8245, 8339, 8269] avg=8192.0000 max=8339 max_rank=6 min=8004 min_rank=3 imbalance_ratio=1.017944 max_load_reduction=57197 max_load_reduction_pct=0.872757 moved_to_echo=33039 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[527, 550, 476, 484, 503, 505, 503, 499, 497, 513, 498, 559, 524, 491, 498, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 503, 555, 493, 497, 526, 527, 528, 529, 496, 528, 515, 545, 477, 499, 487, 17, 15, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 513, 540, 500, 551, 514, 516, 488, 461, 499, 495, 526, 499, 486, 523, 485, 47, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 564, 503, 531, 510, 523, 527, 478, 512, 508, 546, 530, 499, 494, 512, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 516, 491, 496, 502, 480, 508, 535, 543, 504, 502, 521, 496, 487, 559, 501, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 546, 500, 499, 504, 523, 480, 473, 527, 554, 486, 507, 509, 484, 535, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 504, 528, 527, 525, 501, 497, 535, 494, 505, 521, 497, 530, 508, 503, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 481, 505, 506, 559, 529, 508, 482, 541, 498, 528, 524, 492, 514, 511, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8110, 8239, 8173, 8232, 8171, 8127, 8272, 8212] avg=8192.0000 max=8272 max_rank=6 min=8110 min_rank=0 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32782 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[507, 541, 515, 554, 506, 515, 520, 489, 575, 525, 468, 479, 536, 518, 521, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 496, 550, 483, 507, 507, 518, 510, 494, 511, 495, 493, 481, 518, 504, 512, 18, 14, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 527, 491, 485, 553, 537, 488, 509, 494, 510, 506, 492, 531, 490, 477, 505, 45, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 489, 496, 522, 490, 525, 511, 561, 507, 506, 478, 530, 531, 530, 522, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 505, 518, 475, 538, 489, 546, 509, 502, 508, 483, 491, 502, 520, 544, 523, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 564, 533, 545, 508, 532, 535, 473, 518, 547, 546, 519, 511, 450, 513, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 478, 493, 496, 454, 534, 491, 558, 491, 488, 480, 508, 540, 507, 476, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 517, 526, 517, 517, 462, 532, 532, 535, 477, 498, 507, 481, 509, 485, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8296, 8137, 8162, 8231, 8186, 8328, 8031, 8165] avg=8192.0000 max=8328 max_rank=5 min=8031 min_rank=6 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=32710 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4224, 4352, 4096, 4224, 4096, 4096, 4096, 4224, 3968, 4096, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4352, 4096, 3968, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 3968, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 3968, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 256, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4096, 4352, 4096, 4096, 4096, 3968, 4096, 3968, 4224, 4224, 4096, 4096, 4096, 384, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 3968, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4352, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 3968, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4224, 4352, 4096, 4224, 4096, 4096, 4096, 4224, 3968, 4096, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[574, 509, 525, 535, 544, 550, 521, 455, 529, 512, 546, 512, 492, 553, 499, 520, 560, 490, 543, 538, 508, 486, 479, 523, 525, 525, 513, 499, 561, 548, 509, 489, 538, 500, 471, 513, 511, 534, 506, 503, 511, 519, 544, 512, 504, 512, 508, 499, 515, 510, 491, 506, 500, 535, 475, 553, 512, 498, 518, 539, 543, 498, 490, 539, 526, 513, 461, 486, 505, 489, 559, 490, 505, 523, 519, 539, 531, 551, 487, 521, 458, 513, 473, 463, 504, 541, 504, 502, 521, 479, 508, 524, 485, 532, 499, 471, 474, 512, 522, 518, 499, 544, 484, 536, 525, 519, 481, 496, 504, 524, 493, 546, 520, 485, 511, 496, 520, 514, 533, 514, 528, 500, 500, 512, 485, 499, 491, 490] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[504, 491, 507, 528, 521, 503, 521, 477, 501, 514, 504, 494, 536, 564, 522, 506, 488, 494, 545, 493, 536, 532, 499, 512, 502, 505, 500, 514, 528, 506, 540, 486, 550, 491, 481, 513, 466, 516, 539, 494, 513, 521, 544, 506, 551, 545, 514, 502, 502, 472, 493, 510, 522, 511, 492, 486, 513, 519, 496, 509, 515, 513, 550, 485, 530, 502, 488, 469, 498, 537, 494, 520, 506, 538, 503, 557, 493, 524, 557, 503, 505, 527, 503, 511, 487, 495, 530, 512, 523, 489, 524, 516, 498, 510, 515, 510, 498, 511, 509, 512, 515, 487, 507, 518, 510, 501, 531, 553, 512, 537, 521, 498, 534, 487, 546, 512, 559, 476, 556, 534, 507, 515, 551, 492, 516, 500, 479, 471] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[548, 516, 533, 565, 461, 509, 545, 507, 517, 509, 523, 511, 479, 485, 494, 525, 544, 561, 513, 490, 512, 494, 491, 496, 508, 529, 499, 527, 484, 507, 489, 481, 513, 483, 526, 544, 518, 473, 497, 509, 517, 541, 503, 526, 512, 490, 492, 462, 506, 513, 536, 481, 525, 484, 512, 494, 494, 529, 524, 503, 500, 522, 517, 507, 502, 515, 498, 491, 514, 527, 504, 511, 478, 517, 539, 511, 515, 499, 486, 525, 508, 529, 520, 518, 573, 522, 536, 546, 506, 537, 545, 504, 542, 497, 487, 518, 529, 507, 493, 506, 469, 539, 526, 491, 520, 526, 505, 537, 499, 520, 527, 536, 505, 484, 509, 531, 476, 504, 513, 508, 485, 531, 558, 504, 518, 511, 513, 531] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[499, 505, 516, 514, 555, 527, 503, 516, 514, 472, 549, 501, 527, 514, 496, 508, 504, 550, 489, 553, 513, 488, 527, 501, 475, 496, 470, 538, 502, 495, 497, 519, 487, 489, 494, 530, 532, 504, 465, 524, 519, 534, 523, 537, 478, 504, 505, 507, 508, 494, 516, 498, 485, 510, 502, 533, 518, 503, 552, 519, 492, 548, 534, 499, 537, 510, 481, 506, 493, 525, 552, 500, 522, 477, 490, 509, 506, 536, 521, 529, 525, 531, 505, 525, 506, 519, 487, 534, 504, 579, 533, 502, 489, 528, 495, 530, 490, 481, 516, 520, 499, 514, 537, 516, 475, 521, 525, 532, 500, 512, 484, 502, 497, 508, 524, 522, 509, 498, 508, 529, 515, 522, 539, 529, 497, 533, 523, 497] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[500, 499, 532, 526, 480, 506, 538, 507, 471, 516, 511, 499, 473, 523, 496, 512, 516, 537, 549, 516, 508, 481, 524, 501, 508, 502, 501, 471, 518, 508, 519, 524, 531, 522, 514, 549, 534, 520, 500, 542, 493, 531, 466, 515, 501, 474, 490, 517, 494, 542, 528, 489, 517, 525, 482, 529, 547, 507, 523, 468, 514, 495, 554, 481, 531, 567, 537, 485, 501, 519, 535, 529, 513, 498, 476, 498, 502, 509, 501, 525, 499, 534, 526, 531, 486, 480, 511, 497, 497, 500, 521, 532, 526, 519, 516, 484, 520, 499, 550, 498, 499, 542, 533, 510, 487, 532, 538, 490, 507, 497, 509, 521, 554, 530, 546, 487, 489, 526, 491, 487, 493, 532, 518, 538, 548, 471, 491, 552] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[526, 485, 533, 517, 480, 520, 500, 541, 491, 497, 514, 493, 529, 465, 533, 498, 516, 537, 510, 530, 477, 480, 567, 489, 491, 534, 484, 521, 513, 526, 561, 548, 514, 526, 533, 492, 516, 482, 489, 511, 516, 470, 489, 528, 491, 513, 503, 516, 532, 473, 506, 508, 504, 510, 519, 522, 543, 537, 458, 511, 495, 538, 492, 516, 555, 543, 531, 468, 476, 486, 527, 515, 499, 504, 517, 495, 480, 524, 500, 527, 489, 535, 515, 485, 541, 486, 542, 535, 502, 571, 498, 524, 501, 523, 515, 528, 518, 513, 531, 489, 498, 515, 491, 502, 511, 509, 531, 521, 527, 543, 499, 515, 543, 528, 514, 550, 497, 541, 488, 498, 485, 509, 513, 540, 511, 500, 463, 547] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[481, 515, 478, 513, 468, 530, 511, 484, 499, 519, 518, 493, 539, 492, 524, 470, 562, 520, 506, 476, 487, 454, 483, 500, 509, 507, 510, 548, 517, 526, 524, 531, 543, 525, 516, 541, 507, 518, 548, 528, 511, 476, 509, 518, 504, 556, 514, 554, 499, 508, 511, 492, 516, 498, 517, 534, 523, 493, 502, 497, 520, 528, 490, 566, 504, 497, 496, 526, 511, 543, 472, 486, 507, 475, 527, 535, 534, 498, 497, 506, 489, 510, 511, 523, 548, 510, 481, 487, 506, 509, 537, 496, 554, 485, 536, 510, 528, 517, 522, 503, 526, 495, 497, 498, 548, 456, 547, 481, 543, 537, 519, 536, 509, 498, 545, 520, 489, 519, 522, 500, 491, 492, 506, 523, 572, 500, 529, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[504, 491, 507, 528, 521, 503, 521, 477, 501, 514, 504, 494, 536, 564, 522, 506, 488, 494, 545, 493, 536, 532, 499, 512, 502, 505, 500, 514, 528, 506, 540, 486, 550, 491, 481, 513, 466, 516, 539, 494, 513, 521, 544, 506, 551, 545, 514, 502, 502, 472, 493, 510, 522, 511, 492, 486, 513, 519, 496, 509, 515, 513, 550, 485, 530, 502, 488, 469, 498, 537, 494, 520, 506, 538, 503, 557, 493, 524, 557, 503, 505, 527, 503, 511, 487, 495, 530, 512, 523, 489, 524, 516, 498, 510, 515, 510, 498, 511, 509, 512, 515, 487, 507, 518, 510, 501, 531, 553, 512, 537, 521, 498, 534, 487, 546, 512, 559, 476, 556, 534, 507, 515, 551, 492, 516, 500, 479, 471] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[499, 505, 516, 514, 555, 527, 503, 516, 514, 472, 549, 501, 527, 514, 496, 508, 504, 550, 489, 553, 513, 488, 527, 501, 475, 496, 470, 538, 502, 495, 497, 519, 487, 489, 494, 530, 532, 504, 465, 524, 519, 534, 523, 537, 478, 504, 505, 507, 508, 494, 516, 498, 485, 510, 502, 533, 518, 503, 552, 519, 492, 548, 534, 499, 537, 510, 481, 506, 493, 525, 552, 500, 522, 477, 490, 509, 506, 536, 521, 529, 525, 531, 505, 525, 506, 519, 487, 534, 504, 579, 533, 502, 489, 528, 495, 530, 490, 481, 516, 520, 499, 514, 537, 516, 475, 521, 525, 532, 500, 512, 484, 502, 497, 508, 524, 522, 509, 498, 508, 529, 515, 522, 539, 529, 497, 533, 523, 497] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[574, 509, 525, 535, 544, 550, 521, 455, 529, 512, 546, 512, 492, 553, 499, 520, 560, 490, 543, 538, 508, 486, 479, 523, 525, 525, 513, 499, 561, 548, 509, 489, 538, 500, 471, 513, 511, 534, 506, 503, 511, 519, 544, 512, 504, 512, 508, 499, 515, 510, 491, 506, 500, 535, 475, 553, 512, 498, 518, 539, 543, 498, 490, 539, 526, 513, 461, 486, 505, 489, 559, 490, 505, 523, 519, 539, 531, 551, 487, 521, 458, 513, 473, 463, 504, 541, 504, 502, 521, 479, 508, 524, 485, 532, 499, 471, 474, 512, 522, 518, 499, 544, 484, 536, 525, 519, 481, 496, 504, 524, 493, 546, 520, 485, 511, 496, 520, 514, 533, 514, 528, 500, 500, 512, 485, 499, 491, 490] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[502, 508, 533, 508, 523, 516, 530, 530, 538, 508, 472, 527, 504, 492, 527, 488, 512, 518, 497, 557, 491, 492, 516, 538, 516, 501, 517, 521, 463, 542, 519, 478, 525, 530, 506, 516, 489, 496, 519, 493, 476, 500, 539, 510, 509, 530, 496, 502, 491, 533, 516, 552, 521, 507, 498, 521, 500, 511, 535, 520, 521, 531, 507, 555, 501, 595, 529, 520, 507, 464, 504, 507, 528, 517, 513, 491, 495, 507, 503, 520, 528, 477, 506, 489, 501, 505, 548, 510, 507, 545, 510, 527, 506, 502, 516, 562, 528, 487, 452, 548, 485, 549, 509, 500, 519, 489, 521, 508, 454, 505, 497, 530, 525, 473, 493, 508, 518, 509, 538, 525, 506, 524, 501, 517, 530, 503, 523, 483] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[500, 499, 532, 526, 480, 506, 538, 507, 471, 516, 511, 499, 473, 523, 496, 512, 516, 537, 549, 516, 508, 481, 524, 501, 508, 502, 501, 471, 518, 508, 519, 524, 531, 522, 514, 549, 534, 520, 500, 542, 493, 531, 466, 515, 501, 474, 490, 517, 494, 542, 528, 489, 517, 525, 482, 529, 547, 507, 523, 468, 514, 495, 554, 481, 531, 567, 537, 485, 501, 519, 535, 529, 513, 498, 476, 498, 502, 509, 501, 525, 499, 534, 526, 531, 486, 480, 511, 497, 497, 500, 521, 532, 526, 519, 516, 484, 520, 499, 550, 498, 499, 542, 533, 510, 487, 532, 538, 490, 507, 497, 509, 521, 554, 530, 546, 487, 489, 526, 491, 487, 493, 532, 518, 538, 548, 471, 491, 552] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[499, 505, 516, 514, 555, 527, 503, 516, 514, 472, 549, 501, 527, 514, 496, 508, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 550, 489, 553, 513, 488, 527, 501, 475, 496, 470, 538, 502, 495, 497, 519, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 489, 494, 530, 532, 504, 465, 524, 519, 534, 523, 537, 478, 504, 505, 507, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 494, 516, 498, 485, 510, 502, 533, 518, 503, 552, 519, 492, 548, 534, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 510, 481, 506, 493, 525, 552, 500, 522, 477, 490, 509, 506, 536, 521, 529, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 531, 505, 525, 506, 519, 487, 534, 504, 559, 533, 502, 489, 528, 495, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 481, 516, 520, 499, 514, 537, 516, 475, 521, 525, 532, 500, 512, 484, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 508, 512, 522, 509, 498, 508, 529, 515, 522, 539, 529, 497, 533, 523, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 8118, 8141, 8211, 8207, 8272, 8124, 8238] avg=8192.0000 max=8272 max_rank=5 min=8118 min_rank=1 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32841 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[481, 515, 478, 513, 468, 530, 511, 484, 499, 519, 518, 493, 539, 492, 524, 470, 562, 520, 506, 476, 487, 454, 483, 500, 509, 507, 510, 548, 517, 526, 524, 531, 543, 525, 516, 541, 507, 518, 548, 528, 511, 476, 509, 518, 504, 556, 514, 554, 499, 508, 511, 492, 516, 498, 517, 534, 523, 493, 502, 497, 520, 528, 490, 566, 504, 497, 496, 526, 511, 543, 472, 486, 507, 475, 527, 535, 534, 498, 497, 506, 489, 510, 511, 523, 548, 510, 481, 487, 506, 509, 537, 496, 554, 485, 536, 510, 528, 517, 522, 503, 526, 495, 497, 498, 548, 456, 547, 481, 543, 537, 519, 536, 509, 498, 545, 520, 489, 519, 522, 500, 491, 492, 506, 523, 572, 500, 529, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[504, 491, 507, 528, 521, 503, 521, 477, 501, 514, 504, 494, 536, 564, 522, 506, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 494, 545, 493, 536, 532, 499, 512, 502, 505, 500, 514, 528, 506, 540, 486, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 491, 481, 513, 466, 516, 539, 494, 513, 521, 544, 506, 551, 545, 514, 502, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 472, 493, 510, 522, 511, 492, 486, 513, 519, 496, 509, 515, 513, 550, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 502, 488, 469, 498, 537, 494, 520, 506, 538, 503, 557, 493, 524, 557, 503, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 527, 503, 511, 487, 495, 530, 512, 523, 472, 524, 516, 498, 510, 515, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 511, 509, 512, 515, 487, 507, 518, 510, 501, 531, 553, 512, 537, 521, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 487, 534, 512, 559, 476, 556, 534, 507, 515, 551, 492, 516, 500, 479, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8181, 8254, 8088, 8230, 8138, 8220, 8223] avg=8192.0000 max=8254 max_rank=2 min=8088 min_rank=3 imbalance_ratio=1.007568 max_load_reduction=57282 max_load_reduction_pct=0.874054 moved_to_echo=32811 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[574, 509, 525, 535, 544, 550, 521, 455, 529, 512, 546, 512, 492, 553, 499, 520, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 560, 490, 543, 538, 508, 486, 479, 523, 525, 525, 513, 499, 561, 548, 509, 489, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 500, 471, 513, 511, 534, 506, 503, 511, 519, 544, 512, 504, 512, 508, 499, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 510, 491, 506, 500, 535, 475, 553, 512, 498, 518, 539, 543, 498, 490, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 513, 461, 486, 505, 489, 559, 490, 505, 523, 519, 539, 531, 551, 487, 521, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 458, 513, 473, 463, 504, 541, 504, 502, 521, 462, 508, 524, 485, 532, 499, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 512, 522, 518, 499, 544, 484, 536, 525, 519, 481, 496, 504, 524, 493, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 485, 500, 496, 520, 514, 533, 514, 528, 500, 500, 512, 485, 499, 491, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8384, 8297, 8193, 8222, 8216, 7960, 8177, 8087] avg=8192.0000 max=8384 max_rank=0 min=7960 min_rank=5 imbalance_ratio=1.023438 max_load_reduction=57152 max_load_reduction_pct=0.872070 moved_to_echo=32440 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[548, 516, 533, 565, 461, 509, 545, 507, 517, 509, 523, 511, 479, 485, 494, 525, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 561, 513, 490, 512, 494, 491, 496, 508, 529, 499, 527, 484, 507, 489, 481, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 483, 526, 544, 518, 473, 497, 509, 517, 541, 503, 526, 512, 490, 492, 462, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 513, 536, 481, 525, 484, 512, 494, 494, 529, 524, 503, 500, 522, 517, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 515, 498, 491, 514, 527, 504, 511, 478, 517, 539, 511, 515, 499, 486, 525, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 529, 520, 518, 573, 522, 536, 546, 506, 518, 545, 504, 542, 497, 487, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 507, 493, 506, 469, 539, 526, 491, 520, 526, 505, 537, 499, 520, 527, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 484, 498, 531, 476, 504, 513, 508, 485, 531, 558, 504, 518, 511, 513, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8126, 8115, 8147, 8144, 8369, 8230, 8170] avg=8192.0000 max=8369 max_rank=5 min=8115 min_rank=2 imbalance_ratio=1.021606 max_load_reduction=57167 max_load_reduction_pct=0.872299 moved_to_echo=32913 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[481, 515, 478, 513, 468, 530, 511, 484, 499, 519, 518, 493, 539, 492, 524, 470, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 562, 520, 506, 476, 487, 454, 483, 500, 509, 507, 510, 548, 517, 526, 524, 531, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 525, 516, 541, 507, 518, 548, 528, 511, 476, 509, 518, 504, 556, 514, 554, 12, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 508, 511, 492, 516, 498, 517, 534, 523, 493, 502, 497, 520, 524, 490, 566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 497, 496, 526, 511, 543, 472, 486, 507, 475, 527, 535, 534, 498, 497, 506, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 510, 511, 523, 548, 510, 481, 487, 506, 482, 537, 496, 554, 485, 536, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 517, 522, 503, 526, 495, 497, 498, 548, 456, 547, 481, 543, 537, 519, 536, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 498, 520, 520, 489, 519, 522, 500, 491, 492, 506, 523, 572, 500, 529, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8048, 8166, 8383, 8190, 8129, 8165, 8259, 8196] avg=8192.0000 max=8383 max_rank=2 min=8048 min_rank=0 imbalance_ratio=1.023315 max_load_reduction=57153 max_load_reduction_pct=0.872086 moved_to_echo=32749 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[526, 485, 533, 517, 480, 520, 500, 541, 491, 497, 514, 493, 529, 465, 533, 498, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 537, 510, 530, 477, 480, 567, 489, 491, 534, 484, 521, 513, 526, 561, 548, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 526, 533, 492, 516, 482, 489, 511, 516, 470, 489, 528, 491, 513, 503, 516, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 473, 506, 508, 504, 510, 519, 522, 543, 537, 458, 511, 495, 538, 492, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 543, 531, 468, 476, 486, 527, 515, 499, 504, 517, 495, 480, 524, 500, 527, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 535, 515, 485, 541, 486, 542, 535, 502, 551, 498, 524, 501, 523, 515, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 513, 531, 489, 498, 515, 491, 502, 511, 509, 531, 521, 527, 543, 499, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 528, 503, 550, 497, 541, 488, 498, 485, 509, 513, 540, 511, 500, 463, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8130, 8285, 8098, 8164, 8160, 8270, 8213, 8216] avg=8192.0000 max=8285 max_rank=1 min=8098 min_rank=2 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32859 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[504, 491, 507, 528, 521, 503, 521, 477, 501, 514, 504, 494, 536, 564, 522, 506, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 494, 545, 493, 536, 532, 499, 512, 502, 505, 500, 514, 528, 506, 540, 486, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 491, 481, 513, 466, 516, 539, 494, 513, 521, 544, 506, 551, 545, 514, 502, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 472, 493, 510, 522, 511, 492, 486, 513, 519, 496, 509, 515, 513, 550, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 502, 488, 469, 498, 537, 494, 520, 506, 538, 503, 557, 493, 524, 557, 503, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 527, 503, 511, 487, 495, 530, 512, 523, 472, 524, 516, 498, 510, 515, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 511, 509, 512, 515, 487, 507, 518, 510, 501, 531, 553, 512, 537, 521, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 487, 534, 512, 559, 476, 556, 534, 507, 515, 551, 492, 516, 500, 479, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8181, 8254, 8088, 8230, 8138, 8220, 8223] avg=8192.0000 max=8254 max_rank=2 min=8088 min_rank=3 imbalance_ratio=1.007568 max_load_reduction=57282 max_load_reduction_pct=0.874054 moved_to_echo=32811 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[499, 505, 516, 514, 555, 527, 503, 516, 514, 472, 549, 501, 527, 514, 496, 508, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 550, 489, 553, 513, 488, 527, 501, 475, 496, 470, 538, 502, 495, 497, 519, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 489, 494, 530, 532, 504, 465, 524, 519, 534, 523, 537, 478, 504, 505, 507, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 494, 516, 498, 485, 510, 502, 533, 518, 503, 552, 519, 492, 548, 534, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 510, 481, 506, 493, 525, 552, 500, 522, 477, 490, 509, 506, 536, 521, 529, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 531, 505, 525, 506, 519, 487, 534, 504, 559, 533, 502, 489, 528, 495, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 481, 516, 520, 499, 514, 537, 516, 475, 521, 525, 532, 500, 512, 484, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 508, 512, 522, 509, 498, 508, 529, 515, 522, 539, 529, 497, 533, 523, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 8118, 8141, 8211, 8207, 8272, 8124, 8238] avg=8192.0000 max=8272 max_rank=5 min=8118 min_rank=1 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32841 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[502, 508, 533, 508, 523, 516, 530, 530, 538, 508, 472, 527, 504, 492, 527, 488, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 518, 497, 557, 491, 492, 516, 538, 516, 501, 517, 521, 463, 542, 519, 478, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 530, 506, 516, 489, 496, 519, 493, 476, 500, 539, 510, 509, 530, 496, 502, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 533, 516, 552, 521, 507, 498, 521, 500, 511, 535, 520, 521, 531, 507, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 595, 529, 520, 507, 464, 504, 507, 528, 517, 513, 491, 495, 507, 503, 520, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 477, 506, 489, 501, 505, 548, 510, 507, 526, 510, 527, 506, 502, 516, 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 487, 452, 548, 485, 549, 509, 500, 519, 489, 521, 508, 454, 505, 497, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 473, 482, 508, 518, 509, 538, 525, 506, 524, 501, 517, 530, 503, 523, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8214, 8179, 8145, 8319, 8213, 8220, 8081, 8165] avg=8192.0000 max=8319 max_rank=3 min=8081 min_rank=6 imbalance_ratio=1.015503 max_load_reduction=57217 max_load_reduction_pct=0.873062 moved_to_echo=32679 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[526, 485, 533, 517, 480, 520, 500, 541, 491, 497, 514, 493, 529, 465, 533, 498, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 537, 510, 530, 477, 480, 567, 489, 491, 534, 484, 521, 513, 526, 561, 548, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 526, 533, 492, 516, 482, 489, 511, 516, 470, 489, 528, 491, 513, 503, 516, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 473, 506, 508, 504, 510, 519, 522, 543, 537, 458, 511, 495, 538, 492, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 543, 531, 468, 476, 486, 527, 515, 499, 504, 517, 495, 480, 524, 500, 527, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 535, 515, 485, 541, 486, 542, 535, 502, 551, 498, 524, 501, 523, 515, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 513, 531, 489, 498, 515, 491, 502, 511, 509, 531, 521, 527, 543, 499, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 528, 503, 550, 497, 541, 488, 498, 485, 509, 513, 540, 511, 500, 463, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8130, 8285, 8098, 8164, 8160, 8270, 8213, 8216] avg=8192.0000 max=8285 max_rank=1 min=8098 min_rank=2 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32859 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[500, 499, 532, 526, 480, 506, 538, 507, 471, 516, 511, 499, 473, 523, 496, 512, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 537, 549, 516, 508, 481, 524, 501, 508, 502, 501, 471, 518, 508, 519, 524, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 522, 514, 549, 534, 520, 500, 542, 493, 531, 466, 515, 501, 474, 490, 517, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 542, 528, 489, 517, 525, 482, 529, 547, 507, 523, 468, 514, 495, 554, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 567, 537, 485, 501, 519, 535, 529, 513, 498, 476, 498, 502, 509, 501, 525, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 534, 526, 531, 486, 480, 511, 497, 497, 483, 521, 532, 526, 519, 516, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 499, 550, 498, 499, 542, 533, 510, 487, 532, 538, 490, 507, 497, 509, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 530, 534, 487, 489, 526, 491, 487, 493, 532, 518, 538, 548, 471, 491, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8098, 8184, 8207, 8195, 8237, 8142, 8232, 8241] avg=8192.0000 max=8241 max_rank=7 min=8098 min_rank=0 imbalance_ratio=1.005981 max_load_reduction=57295 max_load_reduction_pct=0.874252 moved_to_echo=32852 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 3968, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4352, 4096, 3968, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[495, 565, 535, 514, 481, 504, 526, 552, 526, 562, 504, 547, 544, 516, 497, 496, 532, 512, 497, 499, 491, 543, 539, 505, 483, 539, 548, 519, 511, 520, 516, 495, 531, 513, 468, 529, 507, 487, 538, 481, 472, 518, 535, 523, 480, 518, 510, 501, 551, 514, 471, 500, 503, 551, 528, 506, 507, 494, 490, 498, 533, 523, 517, 506, 495, 530, 552, 522, 492, 500, 534, 481, 499, 555, 476, 518, 466, 520, 529, 562, 450, 550, 520, 533, 479, 517, 501, 499, 511, 493, 467, 494, 494, 510, 477, 502, 493, 464, 487, 524, 498, 520, 507, 485, 488, 515, 534, 514, 522, 479, 477, 493, 482, 558, 495, 524, 523, 534, 541, 511, 533, 537, 551, 496, 542, 520, 537, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[495, 485, 495, 489, 497, 517, 512, 491, 516, 513, 534, 518, 518, 509, 513, 529, 495, 540, 516, 519, 488, 507, 524, 510, 501, 502, 496, 521, 492, 506, 548, 531, 475, 474, 555, 502, 505, 485, 522, 530, 486, 484, 492, 460, 519, 502, 492, 523, 489, 498, 512, 497, 528, 538, 513, 535, 554, 518, 516, 496, 507, 501, 552, 491, 510, 524, 501, 540, 516, 523, 489, 489, 517, 508, 499, 514, 539, 472, 515, 515, 488, 520, 532, 516, 502, 518, 501, 524, 521, 481, 514, 531, 505, 553, 539, 508, 543, 490, 509, 519, 521, 535, 513, 523, 501, 485, 572, 520, 510, 523, 542, 522, 489, 510, 517, 516, 492, 513, 516, 508, 483, 535, 503, 536, 528, 508, 500, 552] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[507, 532, 485, 504, 512, 500, 521, 503, 525, 511, 522, 500, 476, 504, 507, 522, 502, 507, 505, 487, 532, 503, 534, 531, 506, 507, 476, 514, 513, 505, 504, 527, 511, 523, 484, 552, 498, 520, 497, 524, 510, 508, 509, 568, 501, 497, 502, 527, 530, 499, 534, 452, 511, 534, 487, 524, 482, 492, 513, 486, 484, 499, 510, 533, 528, 531, 492, 525, 532, 517, 539, 572, 496, 529, 531, 509, 478, 481, 499, 512, 504, 517, 515, 516, 500, 522, 526, 523, 460, 457, 532, 540, 493, 503, 520, 519, 493, 535, 524, 534, 525, 493, 530, 551, 520, 499, 543, 535, 531, 539, 521, 491, 502, 510, 541, 537, 471, 539, 491, 510, 536, 489, 476, 489, 524, 513, 520, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[512, 549, 520, 519, 484, 472, 509, 503, 529, 484, 524, 505, 503, 530, 490, 543, 525, 536, 499, 535, 482, 542, 519, 526, 527, 496, 483, 495, 489, 537, 510, 505, 464, 525, 553, 527, 476, 513, 529, 527, 489, 515, 500, 537, 489, 552, 484, 491, 557, 540, 509, 516, 506, 466, 482, 506, 511, 513, 523, 553, 519, 516, 476, 483, 490, 514, 538, 511, 535, 489, 484, 559, 527, 510, 535, 523, 499, 481, 490, 491, 490, 500, 532, 560, 476, 541, 494, 527, 540, 520, 494, 531, 498, 531, 499, 515, 482, 501, 504, 486, 532, 511, 519, 472, 475, 525, 554, 516, 504, 499, 517, 500, 501, 465, 500, 506, 522, 489, 504, 538, 519, 569, 534, 530, 531, 531, 523, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[511, 508, 509, 507, 499, 512, 490, 490, 527, 536, 456, 501, 480, 523, 529, 498, 506, 481, 535, 487, 531, 456, 507, 490, 539, 536, 485, 513, 481, 529, 518, 558, 487, 496, 551, 496, 526, 508, 526, 547, 511, 530, 506, 501, 492, 507, 519, 493, 512, 524, 485, 521, 528, 546, 523, 529, 540, 529, 495, 526, 516, 479, 523, 499, 474, 529, 495, 489, 513, 543, 559, 531, 489, 519, 510, 506, 485, 522, 533, 495, 501, 519, 535, 506, 512, 482, 512, 527, 491, 493, 532, 489, 505, 466, 547, 533, 527, 515, 530, 492, 540, 482, 504, 545, 512, 498, 512, 531, 515, 484, 510, 518, 511, 478, 500, 526, 519, 510, 514, 548, 501, 519, 518, 528, 521, 569, 499, 519] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[514, 508, 509, 520, 517, 469, 474, 524, 490, 507, 531, 496, 470, 522, 490, 531, 545, 534, 518, 485, 514, 540, 549, 527, 495, 484, 465, 490, 508, 543, 537, 555, 536, 461, 484, 510, 503, 517, 518, 520, 479, 492, 510, 567, 523, 497, 524, 533, 520, 492, 487, 550, 500, 525, 506, 500, 543, 524, 478, 528, 538, 498, 503, 525, 513, 513, 487, 504, 535, 509, 544, 507, 493, 546, 515, 531, 512, 528, 505, 491, 488, 497, 534, 490, 530, 512, 495, 513, 476, 525, 520, 496, 504, 509, 509, 475, 506, 529, 508, 491, 506, 514, 527, 557, 499, 501, 541, 534, 534, 541, 488, 531, 503, 534, 442, 515, 539, 513, 516, 497, 503, 524, 537, 452, 515, 528, 544, 509] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[512, 549, 520, 519, 484, 472, 509, 503, 529, 484, 524, 505, 503, 530, 490, 543, 525, 536, 499, 535, 482, 542, 519, 526, 527, 496, 483, 495, 489, 537, 510, 505, 464, 525, 553, 527, 476, 513, 529, 527, 489, 515, 500, 537, 489, 552, 484, 491, 557, 540, 509, 516, 506, 466, 482, 506, 511, 513, 523, 553, 519, 516, 476, 483, 490, 514, 538, 511, 535, 489, 484, 559, 527, 510, 535, 523, 499, 481, 490, 491, 490, 500, 532, 560, 476, 541, 494, 527, 540, 520, 494, 531, 498, 531, 499, 515, 482, 501, 504, 486, 532, 511, 519, 472, 475, 525, 554, 516, 504, 499, 517, 500, 501, 465, 500, 506, 522, 489, 504, 538, 519, 569, 534, 530, 531, 531, 523, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[507, 532, 485, 504, 512, 500, 521, 503, 525, 511, 522, 500, 476, 504, 507, 522, 502, 507, 505, 487, 532, 503, 534, 531, 506, 507, 476, 514, 513, 505, 504, 527, 511, 523, 484, 552, 498, 520, 497, 524, 510, 508, 509, 568, 501, 497, 502, 527, 530, 499, 534, 452, 511, 534, 487, 524, 482, 492, 513, 486, 484, 499, 510, 533, 528, 531, 492, 525, 532, 517, 539, 572, 496, 529, 531, 509, 478, 481, 499, 512, 504, 517, 515, 516, 500, 522, 526, 523, 460, 457, 532, 540, 493, 503, 520, 519, 493, 535, 524, 534, 525, 493, 530, 551, 520, 499, 543, 535, 531, 539, 521, 491, 502, 510, 541, 537, 471, 539, 491, 510, 536, 489, 476, 489, 524, 513, 520, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[504, 511, 524, 500, 542, 481, 504, 551, 500, 510, 513, 526, 484, 492, 513, 502, 503, 512, 460, 511, 530, 502, 463, 517, 484, 526, 518, 534, 530, 500, 499, 529, 483, 473, 532, 501, 539, 544, 490, 507, 533, 553, 483, 527, 545, 505, 550, 497, 489, 522, 518, 531, 559, 465, 517, 572, 463, 486, 508, 498, 491, 484, 502, 523, 510, 525, 497, 525, 532, 498, 506, 516, 517, 521, 506, 508, 504, 517, 507, 511, 515, 503, 494, 533, 516, 518, 504, 512, 549, 545, 523, 498, 541, 514, 516, 514, 479, 534, 509, 506, 502, 495, 487, 461, 493, 502, 517, 495, 542, 535, 535, 513, 518, 507, 541, 517, 497, 523, 518, 504, 533, 476, 478, 489, 520, 522, 544, 584] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[495, 485, 495, 489, 497, 517, 512, 491, 516, 513, 534, 518, 518, 509, 513, 529, 495, 540, 516, 519, 488, 507, 524, 510, 501, 502, 496, 521, 492, 506, 548, 531, 475, 474, 555, 502, 505, 485, 522, 530, 486, 484, 492, 460, 519, 502, 492, 523, 489, 498, 512, 497, 528, 538, 513, 535, 554, 518, 516, 496, 507, 501, 552, 491, 510, 524, 501, 540, 516, 523, 489, 489, 517, 508, 499, 514, 539, 472, 515, 515, 488, 520, 532, 516, 502, 518, 501, 524, 521, 481, 514, 531, 505, 553, 539, 508, 543, 490, 509, 519, 521, 535, 513, 523, 501, 485, 572, 520, 510, 523, 542, 522, 489, 510, 517, 516, 492, 513, 516, 508, 483, 535, 503, 536, 528, 508, 500, 552] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[495, 565, 535, 514, 481, 504, 526, 552, 526, 562, 504, 547, 544, 516, 497, 496, 532, 512, 497, 499, 491, 543, 539, 505, 483, 539, 548, 519, 511, 520, 516, 495, 531, 513, 468, 529, 507, 487, 538, 481, 472, 518, 535, 523, 480, 518, 510, 501, 551, 514, 471, 500, 503, 551, 528, 506, 507, 494, 490, 498, 533, 523, 517, 506, 495, 530, 552, 522, 492, 500, 534, 481, 499, 555, 476, 518, 466, 520, 529, 562, 450, 550, 520, 533, 479, 517, 501, 499, 511, 493, 467, 494, 494, 510, 477, 502, 493, 464, 487, 524, 498, 520, 507, 485, 488, 515, 534, 514, 522, 479, 477, 493, 482, 558, 495, 524, 523, 534, 541, 511, 533, 537, 551, 496, 542, 520, 537, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[483, 545, 497, 503, 513, 553, 507, 498, 486, 515, 502, 485, 512, 501, 540, 520, 467, 502, 512, 475, 543, 516, 481, 502, 522, 530, 523, 469, 542, 514, 535, 527, 516, 518, 474, 486, 529, 513, 491, 503, 509, 504, 536, 519, 535, 524, 532, 510, 513, 478, 501, 520, 547, 514, 511, 514, 518, 530, 534, 537, 549, 518, 549, 492, 503, 519, 529, 492, 486, 482, 529, 532, 521, 550, 502, 522, 494, 514, 502, 516, 499, 524, 528, 493, 462, 544, 530, 495, 490, 522, 502, 541, 491, 470, 482, 493, 504, 507, 536, 477, 509, 535, 508, 501, 520, 505, 512, 553, 511, 525, 515, 518, 531, 546, 526, 493, 478, 496, 513, 518, 503, 504, 503, 490, 577, 489, 524, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[495, 565, 535, 514, 481, 504, 526, 552, 526, 562, 504, 547, 544, 516, 497, 496, 17, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 512, 497, 499, 491, 543, 539, 505, 483, 539, 548, 519, 511, 520, 516, 495, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 513, 468, 529, 507, 487, 538, 481, 472, 518, 535, 523, 480, 518, 510, 501, 13, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 514, 471, 500, 503, 551, 528, 496, 507, 494, 490, 498, 533, 523, 517, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 530, 552, 522, 492, 500, 534, 481, 499, 538, 476, 518, 466, 520, 529, 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 450, 550, 520, 533, 479, 517, 501, 499, 511, 493, 467, 494, 494, 510, 477, 502, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 464, 487, 524, 498, 520, 507, 485, 488, 515, 520, 514, 522, 479, 477, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 558, 495, 524, 523, 534, 541, 511, 533, 537, 551, 496, 487, 520, 537, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8398, 8250, 8133, 8182, 8214, 8036, 7986, 8337] avg=8192.0000 max=8398 max_rank=0 min=7986 min_rank=6 imbalance_ratio=1.025146 max_load_reduction=57138 max_load_reduction_pct=0.871857 moved_to_echo=32573 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[504, 511, 524, 500, 542, 481, 504, 551, 500, 510, 513, 526, 484, 492, 513, 502, 16, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 512, 460, 511, 530, 502, 463, 517, 484, 526, 518, 534, 530, 500, 499, 529, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 473, 532, 501, 539, 544, 490, 507, 533, 553, 483, 527, 545, 505, 550, 497, 12, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 522, 518, 531, 559, 465, 517, 561, 463, 486, 508, 498, 491, 484, 502, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 525, 497, 525, 532, 498, 506, 516, 517, 505, 506, 508, 504, 517, 507, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 503, 494, 533, 516, 518, 504, 512, 549, 545, 523, 498, 541, 514, 516, 514, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 534, 509, 506, 502, 495, 487, 461, 493, 502, 504, 495, 542, 535, 535, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 507, 541, 517, 497, 523, 518, 504, 533, 476, 478, 489, 468, 522, 544, 584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8119, 8284, 8117, 8184, 8332, 8092, 8219] avg=8192.0000 max=8332 max_rank=5 min=8092 min_rank=6 imbalance_ratio=1.017090 max_load_reduction=57204 max_load_reduction_pct=0.872864 moved_to_echo=32827 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[504, 511, 524, 500, 542, 481, 504, 551, 500, 510, 513, 526, 484, 492, 513, 502, 503, 512, 460, 511, 530, 502, 463, 517, 484, 526, 518, 534, 530, 500, 499, 529, 483, 473, 532, 501, 539, 544, 490, 507, 533, 553, 483, 527, 545, 505, 550, 497, 489, 522, 518, 531, 559, 465, 517, 572, 463, 486, 508, 498, 491, 484, 502, 523, 510, 525, 497, 525, 532, 498, 506, 516, 517, 521, 506, 508, 504, 517, 507, 511, 515, 503, 494, 533, 516, 518, 504, 512, 549, 545, 523, 498, 541, 514, 516, 514, 479, 534, 509, 506, 502, 495, 487, 461, 493, 502, 517, 495, 542, 535, 535, 513, 518, 507, 541, 517, 497, 523, 518, 504, 533, 476, 478, 489, 520, 522, 544, 584] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[512, 549, 520, 519, 484, 472, 509, 503, 529, 484, 524, 505, 503, 530, 490, 543, 18, 21, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 536, 499, 535, 482, 542, 519, 526, 527, 496, 483, 495, 489, 537, 510, 505, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 464, 525, 553, 527, 476, 513, 529, 527, 489, 515, 500, 537, 489, 552, 484, 491, 18, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 557, 540, 509, 516, 506, 466, 482, 486, 511, 513, 523, 553, 519, 516, 476, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 514, 538, 511, 535, 489, 484, 559, 527, 492, 535, 523, 499, 481, 490, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 500, 532, 560, 476, 541, 494, 527, 540, 520, 494, 531, 498, 531, 499, 515, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 501, 504, 486, 532, 511, 519, 472, 475, 525, 534, 516, 504, 499, 517, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 465, 500, 506, 522, 489, 504, 538, 519, 569, 534, 530, 469, 531, 523, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8217, 8212, 8203, 8156, 8158, 8289, 8077, 8224] avg=8192.0000 max=8289 max_rank=5 min=8077 min_rank=6 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32748 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[495, 485, 495, 489, 497, 517, 512, 491, 516, 513, 534, 518, 518, 509, 513, 529, 15, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 540, 516, 519, 488, 507, 524, 510, 501, 502, 496, 521, 492, 506, 548, 531, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 474, 555, 502, 505, 485, 522, 530, 486, 484, 492, 460, 519, 502, 492, 523, 14, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 498, 512, 497, 528, 538, 513, 524, 554, 518, 516, 496, 507, 501, 552, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 524, 501, 540, 516, 523, 489, 489, 517, 493, 499, 514, 539, 472, 515, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 520, 532, 516, 502, 518, 501, 524, 521, 481, 514, 531, 505, 553, 539, 508, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 490, 509, 519, 521, 535, 513, 523, 501, 485, 557, 520, 510, 523, 542, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 510, 517, 516, 492, 513, 516, 508, 483, 535, 503, 536, 475, 508, 500, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8162, 8197, 8030, 8234, 8156, 8291, 8313, 8153] avg=8192.0000 max=8313 max_rank=6 min=8030 min_rank=2 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32913 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[483, 545, 497, 503, 513, 553, 507, 498, 486, 515, 502, 485, 512, 501, 540, 520, 17, 17, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 502, 512, 475, 543, 516, 481, 502, 522, 530, 523, 469, 542, 514, 535, 527, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 518, 474, 486, 529, 513, 491, 503, 509, 504, 536, 519, 535, 524, 532, 510, 12, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 478, 501, 520, 547, 514, 511, 504, 518, 530, 534, 537, 549, 518, 549, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 519, 529, 492, 486, 482, 529, 532, 521, 533, 502, 522, 494, 514, 502, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 524, 528, 493, 462, 544, 530, 495, 490, 522, 502, 541, 491, 470, 482, 493, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 507, 536, 477, 509, 535, 508, 501, 520, 505, 499, 553, 511, 525, 515, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 546, 526, 493, 478, 496, 513, 518, 503, 504, 503, 490, 519, 489, 524, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8161, 8220, 8315, 8176, 8107, 8223, 8139] avg=8192.0000 max=8315 max_rank=3 min=8107 min_rank=5 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32645 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[507, 532, 485, 504, 512, 500, 521, 503, 525, 511, 522, 500, 476, 504, 507, 522, 16, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 507, 505, 487, 532, 503, 534, 531, 506, 507, 476, 514, 513, 505, 504, 527, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 523, 484, 552, 498, 520, 497, 524, 510, 508, 509, 568, 501, 497, 502, 527, 13, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 499, 534, 452, 511, 534, 487, 514, 482, 492, 513, 486, 484, 499, 510, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 531, 492, 525, 532, 517, 539, 572, 496, 513, 531, 509, 478, 481, 499, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 517, 515, 516, 500, 522, 526, 523, 460, 457, 532, 540, 493, 503, 520, 519, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 535, 524, 534, 525, 493, 530, 551, 520, 499, 529, 535, 531, 539, 521, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 510, 541, 537, 471, 539, 491, 510, 536, 489, 476, 489, 472, 513, 520, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8163, 8154, 8253, 8060, 8255, 8184, 8350, 8117] avg=8192.0000 max=8350 max_rank=6 min=8060 min_rank=3 imbalance_ratio=1.019287 max_load_reduction=57186 max_load_reduction_pct=0.872589 moved_to_echo=32906 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[495, 565, 535, 514, 481, 504, 526, 552, 526, 562, 504, 547, 544, 516, 497, 496, 17, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 512, 497, 499, 491, 543, 539, 505, 483, 539, 548, 519, 511, 520, 516, 495, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 513, 468, 529, 507, 487, 538, 481, 472, 518, 535, 523, 480, 518, 510, 501, 13, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 514, 471, 500, 503, 551, 528, 496, 507, 494, 490, 498, 533, 523, 517, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 530, 552, 522, 492, 500, 534, 481, 499, 538, 476, 518, 466, 520, 529, 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 450, 550, 520, 533, 479, 517, 501, 499, 511, 493, 467, 494, 494, 510, 477, 502, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 464, 487, 524, 498, 520, 507, 485, 488, 515, 520, 514, 522, 479, 477, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 558, 495, 524, 523, 534, 541, 511, 533, 537, 551, 496, 487, 520, 537, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8398, 8250, 8133, 8182, 8214, 8036, 7986, 8337] avg=8192.0000 max=8398 max_rank=0 min=7986 min_rank=6 imbalance_ratio=1.025146 max_load_reduction=57138 max_load_reduction_pct=0.871857 moved_to_echo=32573 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[514, 508, 509, 520, 517, 469, 474, 524, 490, 507, 531, 496, 470, 522, 490, 531, 16, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 534, 518, 485, 514, 540, 549, 527, 495, 484, 465, 490, 508, 543, 537, 555, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 461, 484, 510, 503, 517, 518, 520, 479, 492, 510, 567, 523, 497, 524, 533, 13, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 492, 487, 550, 500, 525, 506, 490, 543, 524, 478, 528, 538, 498, 503, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 513, 487, 504, 535, 509, 544, 507, 493, 530, 515, 531, 512, 528, 505, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 497, 534, 490, 530, 512, 495, 513, 476, 525, 520, 496, 504, 509, 509, 475, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 529, 508, 491, 506, 514, 527, 557, 499, 501, 527, 534, 534, 541, 488, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 534, 442, 515, 539, 513, 516, 497, 503, 524, 537, 452, 463, 528, 544, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8104, 8290, 8196, 8207, 8217, 8110, 8293, 8119] avg=8192.0000 max=8293 max_rank=6 min=8104 min_rank=0 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32739 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[483, 545, 497, 503, 513, 553, 507, 498, 486, 515, 502, 485, 512, 501, 540, 520, 17, 17, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 502, 512, 475, 543, 516, 481, 502, 522, 530, 523, 469, 542, 514, 535, 527, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 518, 474, 486, 529, 513, 491, 503, 509, 504, 536, 519, 535, 524, 532, 510, 12, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 478, 501, 520, 547, 514, 511, 504, 518, 530, 534, 537, 549, 518, 549, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 519, 529, 492, 486, 482, 529, 532, 521, 533, 502, 522, 494, 514, 502, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 524, 528, 493, 462, 544, 530, 495, 490, 522, 502, 541, 491, 470, 482, 493, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 507, 536, 477, 509, 535, 508, 501, 520, 505, 499, 553, 511, 525, 515, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 546, 526, 493, 478, 496, 513, 518, 503, 504, 503, 490, 519, 489, 524, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8161, 8220, 8315, 8176, 8107, 8223, 8139] avg=8192.0000 max=8315 max_rank=3 min=8107 min_rank=5 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32645 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[495, 485, 495, 489, 497, 517, 512, 491, 516, 513, 534, 518, 518, 509, 513, 529, 15, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 540, 516, 519, 488, 507, 524, 510, 501, 502, 496, 521, 492, 506, 548, 531, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 474, 555, 502, 505, 485, 522, 530, 486, 484, 492, 460, 519, 502, 492, 523, 14, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 498, 512, 497, 528, 538, 513, 524, 554, 518, 516, 496, 507, 501, 552, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 524, 501, 540, 516, 523, 489, 489, 517, 493, 499, 514, 539, 472, 515, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 520, 532, 516, 502, 518, 501, 524, 521, 481, 514, 531, 505, 553, 539, 508, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 490, 509, 519, 521, 535, 513, 523, 501, 485, 557, 520, 510, 523, 542, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 510, 517, 516, 492, 513, 516, 508, 483, 535, 503, 536, 475, 508, 500, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8162, 8197, 8030, 8234, 8156, 8291, 8313, 8153] avg=8192.0000 max=8313 max_rank=6 min=8030 min_rank=2 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32913 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[511, 508, 509, 507, 499, 512, 490, 490, 527, 536, 456, 501, 480, 523, 529, 498, 16, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 481, 535, 487, 531, 456, 507, 490, 539, 536, 485, 513, 481, 529, 518, 558, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 496, 551, 496, 526, 508, 526, 547, 511, 530, 506, 501, 492, 507, 519, 493, 12, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 524, 485, 521, 528, 546, 523, 519, 540, 529, 495, 526, 516, 479, 523, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 529, 495, 489, 513, 543, 559, 531, 489, 503, 510, 506, 485, 522, 533, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 519, 535, 506, 512, 482, 512, 527, 491, 493, 532, 489, 505, 466, 547, 533, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 515, 530, 492, 540, 482, 504, 545, 512, 498, 499, 531, 515, 484, 510, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 478, 500, 526, 519, 510, 514, 548, 501, 519, 518, 528, 469, 569, 499, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8108, 8153, 8217, 8265, 8176, 8187, 8202, 8228] avg=8192.0000 max=8265 max_rank=3 min=8108 min_rank=0 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32793 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[507, 532, 485, 504, 512, 500, 521, 503, 525, 511, 522, 500, 476, 504, 507, 522, 16, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 507, 505, 487, 532, 503, 534, 531, 506, 507, 476, 514, 513, 505, 504, 527, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 523, 484, 552, 498, 520, 497, 524, 510, 508, 509, 568, 501, 497, 502, 527, 13, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 499, 534, 452, 511, 534, 487, 514, 482, 492, 513, 486, 484, 499, 510, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 531, 492, 525, 532, 517, 539, 572, 496, 513, 531, 509, 478, 481, 499, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 517, 515, 516, 500, 522, 526, 523, 460, 457, 532, 540, 493, 503, 520, 519, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 535, 524, 534, 525, 493, 530, 551, 520, 499, 529, 535, 531, 539, 521, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 510, 541, 537, 471, 539, 491, 510, 536, 489, 476, 489, 472, 513, 520, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8163, 8154, 8253, 8060, 8255, 8184, 8350, 8117] avg=8192.0000 max=8350 max_rank=6 min=8060 min_rank=3 imbalance_ratio=1.019287 max_load_reduction=57186 max_load_reduction_pct=0.872589 moved_to_echo=32906 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[504, 511, 524, 500, 542, 481, 504, 551, 500, 510, 513, 526, 484, 492, 513, 502, 16, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 512, 460, 511, 530, 502, 463, 517, 484, 526, 518, 534, 530, 500, 499, 529, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 473, 532, 501, 539, 544, 490, 507, 533, 553, 483, 527, 545, 505, 550, 497, 12, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 522, 518, 531, 559, 465, 517, 561, 463, 486, 508, 498, 491, 484, 502, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 525, 497, 525, 532, 498, 506, 516, 517, 505, 506, 508, 504, 517, 507, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 503, 494, 533, 516, 518, 504, 512, 549, 545, 523, 498, 541, 514, 516, 514, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 534, 509, 506, 502, 495, 487, 461, 493, 502, 504, 495, 542, 535, 535, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 507, 541, 517, 497, 523, 518, 504, 533, 476, 478, 489, 468, 522, 544, 584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8119, 8284, 8117, 8184, 8332, 8092, 8219] avg=8192.0000 max=8332 max_rank=5 min=8092 min_rank=6 imbalance_ratio=1.017090 max_load_reduction=57204 max_load_reduction_pct=0.872864 moved_to_echo=32827 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 3840, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4352, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[3968, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 256, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 3840, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 256, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[510, 495, 528, 490, 493, 497, 515, 510, 517, 518, 535, 475, 548, 539, 564, 528, 508, 497, 516, 533, 511, 485, 514, 512, 507, 522, 514, 517, 529, 501, 503, 509, 552, 480, 490, 507, 508, 522, 516, 483, 510, 540, 520, 520, 483, 537, 509, 518, 505, 531, 554, 503, 502, 474, 538, 532, 538, 559, 503, 499, 473, 505, 517, 480, 521, 515, 516, 508, 583, 529, 495, 482, 477, 501, 518, 507, 513, 475, 495, 517, 513, 501, 513, 465, 485, 549, 479, 517, 498, 502, 511, 567, 546, 530, 515, 486, 539, 486, 510, 522, 529, 539, 494, 499, 482, 501, 487, 554, 508, 517, 545, 527, 504, 511, 486, 520, 482, 546, 492, 519, 467, 513, 508, 511, 506, 520, 495, 540] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[500, 505, 531, 501, 495, 535, 466, 499, 505, 537, 521, 475, 510, 513, 509, 516, 524, 471, 488, 517, 516, 531, 537, 484, 535, 552, 504, 510, 489, 480, 506, 494, 540, 484, 535, 498, 540, 514, 515, 533, 517, 489, 502, 517, 534, 530, 445, 482, 448, 493, 507, 507, 506, 527, 516, 499, 497, 506, 474, 512, 525, 533, 496, 500, 540, 535, 508, 557, 498, 499, 491, 505, 513, 485, 503, 512, 523, 515, 498, 510, 548, 531, 575, 475, 501, 508, 548, 518, 537, 534, 525, 549, 483, 481, 521, 519, 536, 508, 540, 551, 515, 507, 514, 546, 493, 495, 528, 488, 537, 507, 494, 500, 520, 524, 495, 515, 512, 488, 525, 520, 503, 509, 532, 553, 478, 531, 524, 526] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[492, 458, 479, 480, 512, 519, 483, 509, 520, 562, 525, 502, 519, 517, 525, 526, 501, 516, 543, 536, 510, 500, 484, 491, 540, 518, 512, 493, 521, 485, 514, 535, 512, 493, 525, 530, 511, 550, 531, 520, 522, 494, 554, 493, 559, 519, 529, 476, 462, 511, 533, 502, 506, 496, 514, 514, 478, 538, 525, 528, 494, 482, 534, 546, 524, 529, 520, 518, 492, 491, 548, 501, 518, 498, 526, 484, 515, 498, 478, 556, 522, 500, 498, 508, 515, 538, 496, 528, 521, 563, 477, 528, 521, 533, 505, 533, 510, 487, 513, 474, 502, 522, 503, 535, 487, 497, 494, 512, 501, 553, 502, 523, 564, 535, 508, 464, 510, 502, 515, 553, 524, 508, 514, 470, 475, 503, 529, 457] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[522, 523, 534, 489, 560, 510, 475, 514, 500, 539, 523, 503, 516, 487, 476, 502, 510, 483, 512, 477, 519, 526, 547, 515, 534, 526, 501, 537, 532, 488, 524, 520, 501, 499, 497, 523, 518, 525, 542, 466, 519, 509, 523, 525, 497, 525, 564, 512, 532, 518, 530, 541, 557, 513, 492, 515, 496, 510, 521, 536, 530, 550, 476, 535, 493, 481, 514, 497, 539, 534, 475, 521, 501, 507, 500, 504, 488, 509, 509, 506, 505, 521, 515, 561, 526, 517, 503, 479, 461, 509, 521, 525, 488, 484, 514, 458, 503, 521, 462, 576, 521, 494, 540, 478, 507, 519, 505, 501, 479, 523, 509, 501, 548, 529, 512, 512, 479, 462, 530, 500, 509, 524, 516, 537, 544, 489, 522, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[511, 532, 530, 505, 529, 483, 527, 537, 542, 522, 523, 505, 529, 517, 483, 503, 476, 527, 455, 469, 505, 510, 520, 506, 538, 478, 483, 488, 531, 521, 503, 487, 536, 496, 495, 506, 529, 499, 522, 496, 530, 543, 516, 504, 542, 493, 540, 528, 543, 553, 521, 508, 506, 499, 534, 500, 545, 533, 529, 529, 522, 527, 515, 518, 504, 498, 518, 505, 501, 512, 537, 522, 520, 503, 538, 547, 471, 511, 524, 488, 522, 490, 476, 498, 541, 507, 538, 523, 493, 512, 483, 510, 471, 519, 525, 481, 495, 527, 484, 539, 545, 496, 535, 524, 520, 514, 541, 505, 521, 490, 533, 488, 529, 479, 483, 496, 525, 496, 512, 517, 475, 532, 519, 520, 521, 459, 508, 488] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[507, 490, 501, 530, 515, 495, 521, 555, 517, 510, 523, 524, 561, 528, 529, 510, 544, 529, 508, 492, 502, 523, 510, 483, 508, 531, 493, 500, 540, 486, 524, 520, 503, 553, 521, 513, 487, 505, 484, 523, 478, 508, 518, 498, 499, 522, 508, 520, 502, 525, 563, 506, 492, 518, 557, 522, 488, 494, 476, 541, 497, 496, 502, 497, 506, 496, 538, 520, 517, 472, 476, 478, 537, 519, 524, 500, 508, 537, 540, 513, 531, 503, 510, 506, 508, 524, 553, 545, 512, 484, 457, 525, 525, 533, 526, 519, 524, 492, 546, 534, 506, 495, 548, 527, 501, 535, 505, 516, 505, 503, 498, 502, 471, 527, 532, 519, 526, 515, 467, 499, 492, 521, 501, 497, 475, 463, 545, 482] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[546, 493, 566, 524, 519, 517, 507, 510, 521, 514, 480, 508, 552, 532, 467, 548, 518, 512, 503, 521, 537, 491, 484, 477, 540, 527, 503, 489, 525, 542, 506, 504, 503, 503, 483, 494, 528, 500, 492, 484, 490, 477, 507, 537, 542, 540, 455, 532, 503, 543, 514, 494, 533, 498, 504, 538, 516, 509, 523, 503, 453, 516, 520, 522, 534, 521, 512, 503, 488, 543, 507, 500, 478, 536, 495, 521, 513, 509, 554, 496, 487, 525, 508, 474, 508, 520, 505, 568, 506, 487, 549, 530, 533, 497, 492, 499, 509, 471, 523, 507, 522, 512, 521, 510, 533, 534, 504, 530, 541, 530, 555, 525, 507, 504, 492, 533, 513, 533, 490, 517, 498, 517, 510, 448, 489, 488, 523, 510] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[510, 495, 528, 490, 493, 497, 515, 510, 517, 518, 535, 475, 548, 539, 564, 528, 508, 497, 516, 533, 511, 485, 514, 512, 507, 522, 514, 517, 529, 501, 503, 509, 552, 480, 490, 507, 508, 522, 516, 483, 510, 540, 520, 520, 483, 537, 509, 518, 505, 531, 554, 503, 502, 474, 538, 532, 538, 559, 503, 499, 473, 505, 517, 480, 521, 515, 516, 508, 583, 529, 495, 482, 477, 501, 518, 507, 513, 475, 495, 517, 513, 501, 513, 465, 485, 549, 479, 517, 498, 502, 511, 567, 546, 530, 515, 486, 539, 486, 510, 522, 529, 539, 494, 499, 482, 501, 487, 554, 508, 517, 545, 527, 504, 511, 486, 520, 482, 546, 492, 519, 467, 513, 508, 511, 506, 520, 495, 540] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[511, 532, 530, 505, 529, 483, 527, 537, 542, 522, 523, 505, 529, 517, 483, 503, 476, 527, 455, 469, 505, 510, 520, 506, 538, 478, 483, 488, 531, 521, 503, 487, 536, 496, 495, 506, 529, 499, 522, 496, 530, 543, 516, 504, 542, 493, 540, 528, 543, 553, 521, 508, 506, 499, 534, 500, 545, 533, 529, 529, 522, 527, 515, 518, 504, 498, 518, 505, 501, 512, 537, 522, 520, 503, 538, 547, 471, 511, 524, 488, 522, 490, 476, 498, 541, 507, 538, 523, 493, 512, 483, 510, 471, 519, 525, 481, 495, 527, 484, 539, 545, 496, 535, 524, 520, 514, 541, 505, 521, 490, 533, 488, 529, 479, 483, 496, 525, 496, 512, 517, 475, 532, 519, 520, 521, 459, 508, 488] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[522, 523, 534, 489, 560, 510, 475, 514, 500, 539, 523, 503, 516, 487, 476, 502, 510, 483, 512, 477, 519, 526, 547, 515, 534, 526, 501, 537, 532, 488, 524, 520, 501, 499, 497, 523, 518, 525, 542, 466, 519, 509, 523, 525, 497, 525, 564, 512, 532, 518, 530, 541, 557, 513, 492, 515, 496, 510, 521, 536, 530, 550, 476, 535, 493, 481, 514, 497, 539, 534, 475, 521, 501, 507, 500, 504, 488, 509, 509, 506, 505, 521, 515, 561, 526, 517, 503, 479, 461, 509, 521, 525, 488, 484, 514, 458, 503, 521, 462, 576, 521, 494, 540, 478, 507, 519, 505, 501, 479, 523, 509, 501, 548, 529, 512, 512, 479, 462, 530, 500, 509, 524, 516, 537, 544, 489, 522, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[500, 505, 531, 501, 495, 535, 466, 499, 505, 537, 521, 475, 510, 513, 509, 516, 524, 471, 488, 517, 516, 531, 537, 484, 535, 552, 504, 510, 489, 480, 506, 494, 540, 484, 535, 498, 540, 514, 515, 533, 517, 489, 502, 517, 534, 530, 445, 482, 448, 493, 507, 507, 506, 527, 516, 499, 497, 506, 474, 512, 525, 533, 496, 500, 540, 535, 508, 557, 498, 499, 491, 505, 513, 485, 503, 512, 523, 515, 498, 510, 548, 531, 575, 475, 501, 508, 548, 518, 537, 534, 525, 549, 483, 481, 521, 519, 536, 508, 540, 551, 515, 507, 514, 546, 493, 495, 528, 488, 537, 507, 494, 500, 520, 524, 495, 515, 512, 488, 525, 520, 503, 509, 532, 553, 478, 531, 524, 526] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[521, 550, 503, 503, 480, 494, 529, 551, 479, 513, 508, 459, 533, 490, 542, 567, 539, 506, 514, 519, 504, 510, 516, 508, 501, 488, 488, 480, 524, 530, 497, 495, 520, 519, 492, 522, 497, 515, 509, 530, 504, 502, 538, 557, 538, 514, 533, 505, 537, 528, 540, 529, 488, 496, 514, 503, 542, 521, 492, 542, 544, 519, 550, 567, 495, 513, 540, 504, 486, 466, 514, 506, 493, 510, 498, 503, 499, 494, 517, 503, 536, 487, 488, 505, 514, 534, 515, 540, 474, 514, 494, 437, 519, 504, 512, 520, 523, 489, 555, 516, 516, 530, 515, 538, 526, 518, 531, 506, 509, 479, 508, 466, 517, 536, 519, 478, 502, 482, 527, 501, 497, 502, 530, 497, 507, 518, 498, 518] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[546, 493, 566, 524, 519, 517, 507, 510, 521, 514, 480, 508, 552, 532, 467, 548, 518, 512, 503, 521, 537, 491, 484, 477, 540, 527, 503, 489, 525, 542, 506, 504, 503, 503, 483, 494, 528, 500, 492, 484, 490, 477, 507, 537, 542, 540, 455, 532, 503, 543, 514, 494, 533, 498, 504, 538, 516, 509, 523, 503, 453, 516, 520, 522, 534, 521, 512, 503, 488, 543, 507, 500, 478, 536, 495, 521, 513, 509, 554, 496, 487, 525, 508, 474, 508, 520, 505, 568, 506, 487, 549, 530, 533, 497, 492, 499, 509, 471, 523, 507, 522, 512, 521, 510, 533, 534, 504, 530, 541, 530, 555, 525, 507, 504, 492, 533, 513, 533, 490, 517, 498, 517, 510, 448, 489, 488, 523, 510] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[510, 495, 528, 490, 493, 497, 515, 510, 517, 518, 535, 475, 515, 539, 564, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 497, 516, 533, 511, 485, 514, 512, 507, 522, 514, 517, 529, 501, 503, 509, 19, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 480, 490, 507, 508, 522, 516, 483, 510, 540, 520, 520, 464, 537, 509, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 531, 503, 503, 502, 474, 538, 532, 538, 559, 503, 499, 473, 505, 517, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 515, 516, 508, 583, 529, 495, 482, 477, 501, 518, 507, 513, 475, 495, 517, 18, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 501, 513, 465, 485, 549, 479, 517, 498, 502, 511, 567, 546, 530, 515, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 486, 510, 491, 529, 539, 494, 499, 482, 501, 487, 554, 508, 517, 545, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 511, 486, 520, 482, 546, 492, 519, 467, 513, 508, 511, 506, 520, 495, 540, 51, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8229, 8210, 8176, 8162, 8187, 8177, 8208, 8187] avg=8192.0000 max=8229 max_rank=0 min=8162 min_rank=3 imbalance_ratio=1.004517 max_load_reduction=57307 max_load_reduction_pct=0.874435 moved_to_echo=32759 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[500, 505, 531, 501, 495, 535, 466, 499, 505, 537, 521, 475, 479, 513, 509, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 471, 488, 517, 516, 531, 537, 484, 535, 552, 504, 510, 489, 480, 506, 494, 21, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 484, 535, 498, 540, 514, 515, 533, 517, 489, 502, 517, 513, 530, 445, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 448, 493, 460, 507, 506, 527, 516, 499, 497, 506, 474, 512, 525, 533, 496, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 535, 508, 557, 498, 499, 491, 505, 513, 485, 503, 512, 523, 515, 498, 510, 19, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 531, 575, 475, 501, 508, 548, 518, 537, 534, 525, 549, 483, 481, 521, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 508, 540, 518, 515, 507, 514, 546, 493, 495, 528, 488, 537, 507, 494, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 524, 495, 515, 512, 488, 525, 520, 503, 509, 532, 553, 478, 531, 524, 526, 47, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8087, 8173, 8154, 7999, 8227, 8353, 8226, 8317] avg=8192.0000 max=8353 max_rank=5 min=7999 min_rank=3 imbalance_ratio=1.019653 max_load_reduction=57183 max_load_reduction_pct=0.872543 moved_to_echo=33123 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[511, 532, 530, 505, 529, 483, 527, 537, 542, 522, 523, 505, 489, 517, 483, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 527, 455, 469, 505, 510, 520, 506, 538, 478, 483, 488, 531, 521, 503, 487, 26, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 496, 495, 506, 529, 499, 522, 496, 530, 543, 516, 504, 514, 493, 540, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 553, 469, 508, 506, 499, 534, 500, 545, 533, 529, 529, 522, 527, 515, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 498, 518, 505, 501, 512, 537, 522, 520, 503, 538, 547, 471, 511, 524, 488, 23, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 490, 476, 498, 541, 507, 538, 523, 493, 512, 483, 510, 471, 519, 525, 481, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 527, 484, 499, 545, 496, 535, 524, 520, 514, 541, 505, 521, 490, 533, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 479, 483, 496, 525, 496, 512, 517, 475, 532, 519, 520, 521, 459, 508, 488, 52, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8040, 8247, 8330, 8242, 8091, 8217, 8131] avg=8192.0000 max=8330 max_rank=3 min=8040 min_rank=1 imbalance_ratio=1.016846 max_load_reduction=57206 max_load_reduction_pct=0.872894 moved_to_echo=32681 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[522, 523, 534, 489, 560, 510, 475, 514, 500, 539, 523, 503, 485, 487, 476, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 483, 512, 477, 519, 526, 547, 515, 534, 526, 501, 537, 532, 488, 524, 520, 20, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 499, 497, 523, 518, 525, 542, 466, 519, 509, 523, 525, 477, 525, 564, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 518, 481, 541, 557, 513, 492, 515, 496, 510, 521, 536, 530, 550, 476, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 481, 514, 497, 539, 534, 475, 521, 501, 507, 500, 504, 488, 509, 509, 506, 20, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 521, 515, 561, 526, 517, 503, 479, 461, 509, 521, 525, 488, 484, 514, 458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 521, 462, 541, 521, 494, 540, 478, 507, 519, 505, 501, 479, 523, 509, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 529, 512, 512, 479, 462, 530, 500, 509, 524, 516, 537, 544, 489, 522, 498, 49, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8142, 8286, 8225, 8303, 8114, 8087, 8104, 8275] avg=8192.0000 max=8303 max_rank=3 min=8087 min_rank=5 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32580 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[507, 490, 501, 530, 515, 495, 521, 555, 517, 510, 523, 524, 527, 528, 529, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 529, 508, 492, 502, 523, 510, 483, 508, 531, 493, 500, 540, 486, 524, 520, 20, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 553, 521, 513, 487, 505, 484, 523, 478, 508, 518, 498, 479, 522, 508, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 525, 511, 506, 492, 518, 557, 522, 488, 494, 476, 541, 497, 496, 502, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 496, 538, 520, 517, 472, 476, 478, 537, 519, 524, 500, 508, 537, 540, 513, 19, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 503, 510, 506, 508, 524, 553, 545, 512, 484, 457, 525, 525, 533, 526, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 492, 546, 502, 506, 495, 548, 527, 501, 535, 505, 516, 505, 503, 498, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 527, 532, 519, 526, 515, 467, 499, 492, 521, 501, 497, 475, 463, 545, 482, 52, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8282, 8226, 8120, 8124, 8217, 8261, 8205, 8101] avg=8192.0000 max=8282 max_rank=0 min=8101 min_rank=7 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32784 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[492, 458, 479, 480, 512, 519, 483, 509, 520, 562, 525, 502, 488, 517, 525, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 516, 543, 536, 510, 500, 484, 491, 540, 518, 512, 493, 521, 485, 514, 535, 22, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 493, 525, 530, 511, 550, 531, 520, 522, 494, 554, 493, 537, 519, 529, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 511, 484, 502, 506, 496, 514, 514, 478, 538, 525, 528, 494, 482, 534, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 529, 520, 518, 492, 491, 548, 501, 518, 498, 526, 484, 515, 498, 478, 556, 17, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 500, 498, 508, 515, 538, 496, 528, 521, 563, 477, 528, 521, 533, 505, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 487, 513, 445, 502, 522, 503, 535, 487, 497, 494, 512, 501, 553, 502, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 564, 535, 508, 464, 510, 502, 515, 553, 524, 508, 514, 470, 475, 503, 529, 457, 49, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8097, 8233, 8296, 8114, 8229, 8286, 8086, 8195] avg=8192.0000 max=8296 max_rank=2 min=8086 min_rank=6 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32796 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[507, 490, 501, 530, 515, 495, 521, 555, 517, 510, 523, 524, 527, 528, 529, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 529, 508, 492, 502, 523, 510, 483, 508, 531, 493, 500, 540, 486, 524, 520, 20, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 553, 521, 513, 487, 505, 484, 523, 478, 508, 518, 498, 479, 522, 508, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 525, 511, 506, 492, 518, 557, 522, 488, 494, 476, 541, 497, 496, 502, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 496, 538, 520, 517, 472, 476, 478, 537, 519, 524, 500, 508, 537, 540, 513, 19, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 503, 510, 506, 508, 524, 553, 545, 512, 484, 457, 525, 525, 533, 526, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 492, 546, 502, 506, 495, 548, 527, 501, 535, 505, 516, 505, 503, 498, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 527, 532, 519, 526, 515, 467, 499, 492, 521, 501, 497, 475, 463, 545, 482, 52, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8282, 8226, 8120, 8124, 8217, 8261, 8205, 8101] avg=8192.0000 max=8282 max_rank=0 min=8101 min_rank=7 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32784 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[522, 523, 534, 489, 560, 510, 475, 514, 500, 539, 523, 503, 485, 487, 476, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 483, 512, 477, 519, 526, 547, 515, 534, 526, 501, 537, 532, 488, 524, 520, 20, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 499, 497, 523, 518, 525, 542, 466, 519, 509, 523, 525, 477, 525, 564, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 518, 481, 541, 557, 513, 492, 515, 496, 510, 521, 536, 530, 550, 476, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 481, 514, 497, 539, 534, 475, 521, 501, 507, 500, 504, 488, 509, 509, 506, 20, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 521, 515, 561, 526, 517, 503, 479, 461, 509, 521, 525, 488, 484, 514, 458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 521, 462, 541, 521, 494, 540, 478, 507, 519, 505, 501, 479, 523, 509, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 529, 512, 512, 479, 462, 530, 500, 509, 524, 516, 537, 544, 489, 522, 498, 49, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8142, 8286, 8225, 8303, 8114, 8087, 8104, 8275] avg=8192.0000 max=8303 max_rank=3 min=8087 min_rank=5 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32580 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[546, 493, 566, 524, 519, 517, 507, 510, 521, 514, 480, 508, 519, 532, 467, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 512, 503, 521, 537, 491, 484, 477, 540, 527, 503, 489, 525, 542, 506, 504, 22, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 503, 483, 494, 528, 500, 492, 484, 490, 477, 507, 537, 520, 540, 455, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 543, 467, 494, 533, 498, 504, 538, 516, 509, 523, 503, 453, 516, 520, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 521, 512, 503, 488, 543, 507, 500, 478, 536, 495, 521, 513, 509, 554, 496, 18, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 525, 508, 474, 508, 520, 505, 568, 506, 487, 549, 530, 533, 497, 492, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 471, 523, 476, 522, 512, 521, 510, 533, 534, 504, 530, 541, 530, 555, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 504, 492, 533, 513, 533, 490, 517, 498, 517, 510, 448, 489, 488, 523, 510, 47, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8271, 8214, 8045, 8142, 8245, 8188, 8296, 8135] avg=8192.0000 max=8296 max_rank=6 min=8045 min_rank=2 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32864 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[500, 505, 531, 501, 495, 535, 466, 499, 505, 537, 521, 475, 479, 513, 509, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 471, 488, 517, 516, 531, 537, 484, 535, 552, 504, 510, 489, 480, 506, 494, 21, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 484, 535, 498, 540, 514, 515, 533, 517, 489, 502, 517, 513, 530, 445, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 448, 493, 460, 507, 506, 527, 516, 499, 497, 506, 474, 512, 525, 533, 496, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 535, 508, 557, 498, 499, 491, 505, 513, 485, 503, 512, 523, 515, 498, 510, 19, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 531, 575, 475, 501, 508, 548, 518, 537, 534, 525, 549, 483, 481, 521, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 508, 540, 518, 515, 507, 514, 546, 493, 495, 528, 488, 537, 507, 494, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 524, 495, 515, 512, 488, 525, 520, 503, 509, 532, 553, 478, 531, 524, 526, 47, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8087, 8173, 8154, 7999, 8227, 8353, 8226, 8317] avg=8192.0000 max=8353 max_rank=5 min=7999 min_rank=3 imbalance_ratio=1.019653 max_load_reduction=57183 max_load_reduction_pct=0.872543 moved_to_echo=33123 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[521, 550, 503, 503, 480, 494, 529, 551, 479, 513, 508, 459, 501, 490, 542, 567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 506, 514, 519, 504, 510, 516, 508, 501, 488, 488, 480, 524, 530, 497, 495, 22, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 519, 492, 522, 497, 515, 509, 530, 504, 502, 538, 557, 516, 514, 533, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 528, 490, 529, 488, 496, 514, 503, 542, 521, 492, 542, 544, 519, 550, 567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 513, 540, 504, 486, 466, 514, 506, 493, 510, 498, 503, 499, 494, 517, 503, 18, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 487, 488, 505, 514, 534, 515, 540, 474, 514, 494, 437, 519, 504, 512, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 489, 555, 485, 516, 530, 515, 538, 526, 518, 531, 506, 509, 479, 508, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 536, 519, 478, 502, 482, 527, 501, 497, 502, 530, 497, 507, 518, 498, 518, 50, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8154, 8273, 8362, 8075, 8093, 8194, 8195] avg=8192.0000 max=8362 max_rank=3 min=8075 min_rank=4 imbalance_ratio=1.020752 max_load_reduction=57174 max_load_reduction_pct=0.872406 moved_to_echo=32557 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 3968, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 3968, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 256, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 3968, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 512, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 3968, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 256, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 3968, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 512, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True - [2026-08-26 14:49:29] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 418.6 | throughput per GPU (TFLOP/s/GPU): 432.0 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 1.024422E+01 | loss scale: 1.0 | grad norm: 6.229 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[481, 512, 460, 475, 544, 512, 506, 489, 473, 543, 553, 471, 541, 490, 510, 506, 493, 543, 493, 469, 519, 495, 515, 538, 507, 521, 521, 506, 583, 514, 529, 537, 510, 490, 540, 529, 481, 507, 547, 531, 544, 537, 502, 552, 503, 497, 506, 508, 539, 534, 549, 546, 521, 497, 506, 530, 527, 520, 510, 542, 476, 557, 503, 528, 542, 523, 496, 503, 504, 508, 531, 503, 518, 505, 490, 536, 496, 545, 517, 532, 545, 485, 508, 508, 488, 482, 506, 534, 525, 488, 518, 516, 520, 526, 509, 521, 512, 470, 527, 540, 495, 542, 456, 489, 489, 501, 495, 494, 513, 526, 519, 520, 490, 469, 496, 517, 499, 470, 503, 491, 518, 506, 528, 513, 531, 507, 479, 485] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[500, 515, 529, 529, 510, 498, 490, 493, 502, 526, 466, 512, 483, 494, 501, 502, 521, 503, 528, 554, 538, 537, 496, 491, 526, 518, 520, 470, 502, 493, 493, 526, 501, 525, 549, 486, 531, 490, 515, 494, 524, 514, 543, 497, 524, 501, 537, 523, 532, 526, 536, 493, 534, 506, 532, 501, 495, 497, 538, 555, 504, 551, 525, 492, 523, 487, 564, 498, 533, 536, 520, 504, 501, 485, 563, 489, 521, 505, 523, 530, 553, 465, 500, 485, 512, 512, 510, 506, 518, 466, 522, 533, 526, 514, 499, 539, 512, 502, 517, 517, 511, 551, 485, 528, 468, 509, 477, 469, 526, 508, 505, 530, 507, 495, 482, 502, 511, 489, 514, 518, 538, 506, 482, 520, 516, 516, 509, 537] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True + [2026-08-26 15:25:32] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 421.1 | throughput per GPU (TFLOP/s/GPU): 429.4 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 1.025074E+01 | loss scale: 1.0 | grad norm: 6.242 | number of skipped iterations: 0 | number of nan iterations: 0 | WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[531, 537, 537, 506, 488, 552, 521, 532, 517, 536, 506, 474, 508, 482, 494, 497, 537, 516, 523, 512, 511, 512, 540, 506, 476, 487, 537, 517, 548, 508, 545, 532, 480, 466, 500, 488, 498, 499, 522, 514, 512, 542, 526, 535, 522, 511, 513, 511, 512, 486, 525, 555, 538, 502, 511, 510, 517, 497, 499, 557, 467, 505, 514, 539, 515, 493, 469, 510, 527, 491, 505, 532, 526, 519, 545, 527, 538, 531, 529, 502, 497, 494, 511, 521, 518, 487, 465, 522, 458, 512, 499, 496, 545, 503, 487, 524, 507, 484, 503, 521, 490, 516, 538, 500, 511, 544, 526, 526, 482, 473, 463, 497, 514, 510, 513, 536, 546, 528, 477, 495, 539, 530, 496, 495, 486, 544, 523, 527] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[472, 523, 502, 514, 492, 497, 523, 493, 514, 510, 522, 502, 534, 467, 518, 503, 530, 575, 525, 530, 527, 501, 521, 506, 574, 531, 517, 489, 509, 473, 541, 523, 498, 500, 486, 489, 505, 523, 501, 492, 492, 504, 550, 544, 493, 523, 497, 500, 505, 511, 508, 511, 513, 526, 481, 543, 511, 510, 512, 520, 508, 465, 504, 500, 511, 535, 510, 498, 484, 484, 523, 558, 490, 509, 532, 508, 482, 553, 508, 518, 490, 500, 489, 540, 521, 537, 532, 538, 517, 500, 509, 551, 461, 513, 534, 489, 526, 547, 516, 501, 507, 560, 513, 524, 572, 505, 475, 518, 533, 497, 531, 494, 520, 509, 507, 482, 524, 540, 527, 486, 505, 504, 504, 487, 490, 533, 507, 485] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[502, 551, 528, 525, 536, 487, 520, 502, 471, 525, 530, 514, 488, 551, 530, 522, 510, 500, 496, 534, 487, 508, 559, 523, 519, 477, 546, 536, 509, 558, 540, 529, 485, 486, 547, 506, 492, 520, 477, 516, 498, 510, 490, 516, 547, 508, 549, 538, 495, 508, 492, 520, 516, 489, 504, 456, 527, 494, 560, 504, 508, 556, 520, 525, 491, 529, 551, 512, 523, 498, 470, 491, 482, 540, 484, 471, 498, 502, 523, 528, 528, 508, 474, 529, 488, 501, 537, 520, 535, 495, 501, 494, 529, 477, 517, 504, 541, 495, 505, 493, 483, 510, 502, 478, 489, 538, 497, 542, 514, 519, 500, 500, 520, 524, 521, 518, 504, 510, 537, 533, 494, 536, 511, 531, 499, 510, 478, 502] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[525, 513, 486, 548, 531, 552, 508, 534, 523, 505, 507, 513, 544, 492, 521, 494, 508, 490, 554, 517, 498, 482, 514, 505, 534, 501, 515, 527, 526, 479, 519, 488, 536, 473, 523, 502, 498, 487, 510, 517, 533, 534, 545, 540, 513, 549, 474, 502, 517, 504, 511, 500, 490, 528, 513, 519, 507, 513, 495, 536, 535, 469, 509, 484, 501, 530, 502, 514, 498, 490, 489, 510, 521, 474, 523, 518, 490, 529, 519, 523, 478, 515, 485, 526, 525, 530, 501, 518, 524, 514, 463, 509, 520, 524, 479, 520, 519, 483, 476, 521, 584, 541, 504, 548, 492, 510, 506, 536, 477, 539, 514, 490, 505, 483, 492, 514, 516, 512, 506, 510, 512, 505, 499, 549, 515, 547, 530, 520] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[547, 508, 526, 468, 505, 513, 515, 513, 485, 499, 523, 526, 484, 553, 489, 515, 517, 490, 496, 521, 513, 503, 513, 523, 504, 533, 524, 541, 484, 498, 496, 501, 504, 540, 525, 514, 489, 536, 526, 498, 481, 492, 488, 489, 490, 485, 527, 545, 493, 537, 546, 551, 483, 542, 516, 509, 567, 534, 515, 496, 510, 532, 518, 492, 497, 532, 518, 545, 500, 531, 482, 501, 531, 523, 507, 467, 520, 498, 496, 535, 469, 488, 504, 554, 523, 503, 502, 515, 523, 561, 517, 501, 497, 499, 527, 562, 490, 517, 554, 489, 518, 525, 483, 482, 519, 512, 514, 510, 505, 514, 494, 502, 544, 527, 514, 528, 507, 481, 479, 506, 504, 483, 504, 488, 533, 528, 551, 507] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[472, 523, 502, 514, 492, 497, 523, 493, 514, 510, 522, 502, 534, 467, 518, 503, 530, 575, 525, 530, 527, 501, 521, 506, 574, 531, 517, 489, 509, 473, 541, 523, 498, 500, 486, 489, 505, 523, 501, 492, 492, 504, 550, 544, 493, 523, 497, 500, 505, 511, 508, 511, 513, 526, 481, 543, 511, 510, 512, 520, 508, 465, 504, 500, 511, 535, 510, 498, 484, 484, 523, 558, 490, 509, 532, 508, 482, 553, 508, 518, 490, 500, 489, 540, 521, 537, 532, 538, 517, 500, 509, 551, 461, 513, 534, 489, 526, 547, 516, 501, 507, 560, 513, 524, 572, 505, 475, 518, 533, 497, 531, 494, 520, 509, 507, 482, 524, 540, 527, 486, 505, 504, 504, 487, 490, 533, 507, 485] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[481, 512, 460, 475, 544, 512, 506, 489, 473, 543, 553, 471, 541, 490, 510, 506, 493, 543, 493, 469, 519, 495, 515, 538, 507, 521, 521, 506, 583, 514, 529, 537, 510, 490, 540, 529, 481, 507, 547, 531, 544, 537, 502, 552, 503, 497, 506, 508, 539, 534, 549, 546, 521, 497, 506, 530, 527, 520, 510, 542, 476, 557, 503, 528, 542, 523, 496, 503, 504, 508, 531, 503, 518, 505, 490, 536, 496, 545, 517, 532, 545, 485, 508, 508, 488, 482, 506, 534, 525, 488, 518, 516, 520, 526, 509, 521, 512, 470, 527, 540, 495, 542, 456, 489, 489, 501, 495, 494, 513, 526, 519, 520, 490, 469, 496, 517, 499, 470, 503, 491, 518, 506, 528, 513, 531, 507, 479, 485] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[500, 515, 529, 529, 510, 498, 490, 493, 502, 526, 466, 512, 483, 494, 501, 502, 521, 503, 528, 554, 538, 537, 496, 491, 526, 518, 520, 470, 502, 493, 493, 526, 501, 525, 549, 486, 531, 490, 515, 494, 524, 514, 543, 497, 524, 501, 537, 523, 532, 526, 536, 493, 534, 506, 532, 501, 495, 497, 538, 555, 504, 551, 525, 492, 523, 487, 564, 498, 533, 536, 520, 504, 501, 485, 563, 489, 521, 505, 523, 530, 553, 465, 500, 485, 512, 512, 510, 506, 518, 466, 522, 533, 526, 514, 499, 539, 512, 502, 517, 517, 511, 551, 485, 528, 468, 509, 477, 469, 526, 508, 505, 530, 507, 495, 482, 502, 511, 489, 514, 518, 538, 506, 482, 520, 516, 516, 509, 537] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[484, 542, 513, 483, 504, 509, 510, 549, 518, 537, 526, 510, 487, 536, 509, 493, 546, 489, 503, 507, 513, 521, 504, 537, 498, 514, 542, 505, 485, 517, 508, 528, 464, 497, 509, 499, 503, 514, 489, 532, 520, 551, 546, 511, 484, 544, 499, 506, 542, 567, 513, 485, 504, 522, 525, 525, 482, 496, 499, 480, 522, 534, 491, 507, 517, 461, 526, 516, 509, 493, 538, 520, 484, 513, 516, 553, 521, 506, 519, 495, 502, 537, 486, 492, 492, 565, 514, 497, 472, 519, 567, 511, 508, 494, 508, 483, 556, 520, 520, 490, 527, 517, 555, 495, 546, 533, 515, 479, 483, 544, 510, 499, 525, 492, 518, 563, 496, 522, 487, 497, 507, 538, 506, 468, 519, 519, 473, 494] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[525, 513, 486, 548, 531, 552, 508, 534, 523, 505, 507, 513, 544, 492, 521, 494, 508, 490, 554, 517, 498, 482, 514, 505, 534, 501, 515, 527, 526, 479, 519, 488, 536, 473, 523, 502, 498, 487, 510, 517, 533, 534, 545, 540, 513, 549, 474, 502, 517, 504, 511, 500, 490, 528, 513, 519, 507, 513, 495, 536, 535, 469, 509, 484, 501, 530, 502, 514, 498, 490, 489, 510, 521, 474, 523, 518, 490, 529, 519, 523, 478, 515, 485, 526, 525, 530, 501, 518, 524, 514, 463, 509, 520, 524, 479, 520, 519, 483, 476, 521, 584, 541, 504, 548, 492, 510, 506, 536, 477, 539, 514, 490, 505, 483, 492, 514, 516, 512, 506, 510, 512, 505, 499, 549, 515, 547, 530, 520] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[481, 512, 460, 475, 544, 512, 506, 489, 473, 543, 553, 471, 541, 490, 510, 506, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 543, 493, 469, 519, 495, 515, 538, 507, 521, 459, 506, 583, 514, 529, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 490, 540, 529, 481, 507, 547, 531, 544, 537, 502, 552, 503, 497, 506, 508, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 534, 549, 546, 521, 497, 506, 530, 527, 520, 510, 497, 476, 557, 503, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 523, 496, 503, 504, 508, 531, 503, 518, 505, 490, 536, 496, 540, 517, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 485, 508, 508, 488, 482, 506, 534, 525, 488, 518, 516, 520, 526, 509, 521, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 470, 527, 540, 495, 542, 456, 489, 489, 501, 495, 494, 513, 526, 519, 520, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 469, 496, 517, 499, 470, 503, 491, 518, 506, 528, 513, 531, 507, 479, 485, 28, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8085, 8221, 8289, 8340, 8244, 8213, 8109, 8035] avg=8192.0000 max=8340 max_rank=3 min=8035 min_rank=7 imbalance_ratio=1.018066 max_load_reduction=57196 max_load_reduction_pct=0.872742 moved_to_echo=32601 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[502, 551, 528, 525, 536, 487, 520, 502, 471, 525, 530, 514, 488, 551, 530, 522, 510, 500, 496, 534, 487, 508, 559, 523, 519, 477, 546, 536, 509, 558, 540, 529, 485, 486, 547, 506, 492, 520, 477, 516, 498, 510, 490, 516, 547, 508, 549, 538, 495, 508, 492, 520, 516, 489, 504, 456, 527, 494, 560, 504, 508, 556, 520, 525, 491, 529, 551, 512, 523, 498, 470, 491, 482, 540, 484, 471, 498, 502, 523, 528, 528, 508, 474, 529, 488, 501, 537, 520, 535, 495, 501, 494, 529, 477, 517, 504, 541, 495, 505, 493, 483, 510, 502, 478, 489, 538, 497, 542, 514, 519, 500, 500, 520, 524, 521, 518, 504, 510, 537, 533, 494, 536, 511, 531, 499, 510, 478, 502] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[531, 537, 537, 506, 488, 552, 521, 532, 517, 536, 506, 474, 508, 482, 494, 497, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 516, 523, 512, 511, 512, 540, 506, 476, 487, 474, 517, 548, 508, 545, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 466, 500, 488, 498, 499, 522, 514, 512, 542, 526, 535, 522, 511, 513, 511, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 486, 525, 555, 538, 502, 511, 510, 517, 497, 499, 510, 467, 505, 514, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 493, 469, 510, 527, 491, 505, 532, 526, 519, 545, 527, 538, 526, 529, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 494, 511, 521, 518, 487, 465, 522, 458, 512, 499, 496, 545, 503, 487, 524, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 484, 503, 521, 490, 516, 538, 500, 511, 544, 526, 526, 482, 473, 463, 497, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 510, 513, 536, 546, 528, 477, 495, 539, 530, 496, 495, 486, 544, 523, 527, 28, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8244, 8144, 8187, 8254, 8074, 8103, 8292] avg=8192.0000 max=8292 max_rank=7 min=8074 min_rank=5 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32723 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[525, 513, 486, 548, 531, 552, 508, 534, 523, 505, 507, 513, 544, 492, 521, 494, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 490, 554, 517, 498, 482, 514, 505, 534, 501, 455, 527, 526, 479, 519, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 473, 523, 502, 498, 487, 510, 517, 533, 534, 545, 540, 513, 549, 474, 502, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 504, 511, 500, 490, 528, 513, 519, 507, 513, 495, 491, 535, 469, 509, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 530, 502, 514, 498, 490, 489, 510, 521, 474, 523, 518, 490, 524, 519, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 515, 485, 526, 525, 530, 501, 518, 524, 514, 463, 509, 520, 524, 479, 520, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 483, 476, 521, 584, 541, 504, 548, 492, 510, 506, 536, 477, 539, 514, 490, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 483, 492, 514, 516, 512, 506, 510, 512, 505, 499, 549, 515, 547, 530, 520, 27, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8315, 8097, 8241, 8085, 8126, 8164, 8261, 8247] avg=8192.0000 max=8315 max_rank=0 min=8085 min_rank=3 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32798 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[500, 515, 529, 529, 510, 498, 490, 493, 502, 526, 466, 512, 483, 494, 501, 502, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 503, 528, 554, 538, 537, 496, 491, 526, 518, 459, 470, 502, 493, 493, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 525, 549, 486, 531, 490, 515, 494, 524, 514, 543, 497, 524, 501, 537, 523, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 526, 536, 493, 534, 506, 532, 501, 495, 497, 538, 508, 504, 551, 525, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 487, 564, 498, 533, 536, 520, 504, 501, 485, 563, 489, 521, 500, 523, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 465, 500, 485, 512, 512, 510, 506, 518, 466, 522, 533, 526, 514, 499, 539, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 502, 517, 517, 511, 551, 485, 528, 468, 509, 477, 469, 526, 508, 505, 530, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 495, 482, 502, 511, 489, 514, 518, 538, 506, 482, 520, 516, 516, 509, 537, 27, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8070, 8155, 8259, 8270, 8277, 8194, 8137, 8174] avg=8192.0000 max=8277 max_rank=4 min=8070 min_rank=0 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32782 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[484, 542, 513, 483, 504, 509, 510, 549, 518, 537, 526, 510, 487, 536, 509, 493, 21, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 489, 503, 507, 513, 521, 504, 537, 498, 514, 470, 505, 485, 517, 508, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 464, 497, 509, 499, 503, 514, 489, 532, 520, 551, 546, 511, 484, 544, 499, 506, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 567, 513, 485, 504, 522, 525, 525, 482, 496, 499, 426, 522, 534, 491, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 461, 526, 516, 509, 493, 538, 520, 484, 513, 516, 553, 521, 490, 519, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 537, 486, 492, 492, 565, 514, 497, 472, 519, 567, 511, 508, 494, 508, 483, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 520, 520, 490, 527, 517, 555, 495, 546, 533, 515, 479, 483, 544, 510, 499, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 492, 518, 563, 496, 522, 487, 497, 507, 538, 506, 468, 519, 519, 473, 494, 33, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8145, 8177, 8140, 8171, 8186, 8312, 8167] avg=8192.0000 max=8312 max_rank=6 min=8140 min_rank=3 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32836 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[472, 523, 502, 514, 492, 497, 523, 493, 514, 510, 522, 502, 534, 467, 518, 503, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 575, 525, 530, 527, 501, 521, 506, 574, 531, 457, 489, 509, 473, 541, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 500, 486, 489, 505, 523, 501, 492, 492, 504, 550, 544, 493, 523, 497, 500, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 511, 508, 511, 513, 526, 481, 543, 511, 510, 512, 476, 508, 465, 504, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 535, 510, 498, 484, 484, 523, 558, 490, 509, 532, 508, 482, 548, 508, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 500, 489, 540, 521, 537, 532, 538, 517, 500, 509, 551, 461, 513, 534, 489, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 547, 516, 501, 507, 560, 513, 524, 572, 505, 475, 518, 533, 497, 531, 494, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 509, 507, 482, 524, 540, 527, 486, 505, 504, 504, 487, 490, 533, 507, 485, 27, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8104, 8312, 8102, 8084, 8198, 8254, 8340, 8142] avg=8192.0000 max=8340 max_rank=6 min=8084 min_rank=3 imbalance_ratio=1.018066 max_load_reduction=57196 max_load_reduction_pct=0.872742 moved_to_echo=32934 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[502, 551, 528, 525, 536, 487, 520, 502, 471, 525, 530, 514, 488, 551, 530, 522, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 500, 496, 534, 487, 508, 559, 523, 519, 477, 482, 536, 509, 558, 540, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 486, 547, 506, 492, 520, 477, 516, 498, 510, 490, 516, 547, 508, 549, 538, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 508, 492, 520, 516, 489, 504, 456, 527, 494, 560, 461, 508, 556, 520, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 529, 551, 512, 523, 498, 470, 491, 482, 540, 484, 471, 498, 497, 523, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 508, 474, 529, 488, 501, 537, 520, 535, 495, 501, 494, 529, 477, 517, 504, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 495, 505, 493, 483, 510, 502, 478, 489, 538, 497, 542, 514, 519, 500, 500, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 524, 521, 518, 504, 510, 537, 533, 494, 536, 511, 531, 499, 510, 478, 502, 29, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8300, 8267, 8190, 8131, 8088, 8172, 8126, 8262] avg=8192.0000 max=8300 max_rank=0 min=8088 min_rank=4 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32648 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[481, 512, 460, 475, 544, 512, 506, 489, 473, 543, 553, 471, 541, 490, 510, 506, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 543, 493, 469, 519, 495, 515, 538, 507, 521, 459, 506, 583, 514, 529, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 490, 540, 529, 481, 507, 547, 531, 544, 537, 502, 552, 503, 497, 506, 508, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 534, 549, 546, 521, 497, 506, 530, 527, 520, 510, 497, 476, 557, 503, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 523, 496, 503, 504, 508, 531, 503, 518, 505, 490, 536, 496, 540, 517, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 485, 508, 508, 488, 482, 506, 534, 525, 488, 518, 516, 520, 526, 509, 521, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 470, 527, 540, 495, 542, 456, 489, 489, 501, 495, 494, 513, 526, 519, 520, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 469, 496, 517, 499, 470, 503, 491, 518, 506, 528, 513, 531, 507, 479, 485, 28, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8085, 8221, 8289, 8340, 8244, 8213, 8109, 8035] avg=8192.0000 max=8340 max_rank=3 min=8035 min_rank=7 imbalance_ratio=1.018066 max_load_reduction=57196 max_load_reduction_pct=0.872742 moved_to_echo=32601 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[547, 508, 526, 468, 505, 513, 515, 513, 485, 499, 523, 526, 484, 553, 489, 515, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 490, 496, 521, 513, 503, 513, 523, 504, 533, 462, 541, 484, 498, 496, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 540, 525, 514, 489, 536, 526, 498, 481, 492, 488, 489, 490, 485, 527, 545, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 537, 546, 551, 483, 542, 516, 509, 567, 534, 515, 454, 510, 532, 518, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 532, 518, 545, 500, 531, 482, 501, 531, 523, 507, 467, 520, 493, 496, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 488, 504, 554, 523, 503, 502, 515, 523, 561, 517, 501, 497, 499, 527, 562, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 517, 554, 489, 518, 525, 483, 482, 519, 512, 514, 510, 505, 514, 494, 502, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 527, 514, 528, 507, 481, 479, 506, 504, 483, 504, 488, 533, 528, 551, 507, 28, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8095, 8134, 8299, 8178, 8279, 8148, 8217] avg=8192.0000 max=8299 max_rank=3 min=8095 min_rank=1 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32822 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[525, 513, 486, 548, 531, 552, 508, 534, 523, 505, 507, 513, 544, 492, 521, 494, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 490, 554, 517, 498, 482, 514, 505, 534, 501, 455, 527, 526, 479, 519, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 473, 523, 502, 498, 487, 510, 517, 533, 534, 545, 540, 513, 549, 474, 502, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 504, 511, 500, 490, 528, 513, 519, 507, 513, 495, 491, 535, 469, 509, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 530, 502, 514, 498, 490, 489, 510, 521, 474, 523, 518, 490, 524, 519, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 515, 485, 526, 525, 530, 501, 518, 524, 514, 463, 509, 520, 524, 479, 520, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 483, 476, 521, 584, 541, 504, 548, 492, 510, 506, 536, 477, 539, 514, 490, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 483, 492, 514, 516, 512, 506, 510, 512, 505, 499, 549, 515, 547, 530, 520, 27, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8315, 8097, 8241, 8085, 8126, 8164, 8261, 8247] avg=8192.0000 max=8315 max_rank=0 min=8085 min_rank=3 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32798 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 3840, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 3840, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 3840, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 3968, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4352, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 3840, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4352, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[507, 489, 506, 511, 491, 522, 526, 509, 519, 511, 459, 471, 537, 508, 496, 527, 527, 530, 525, 534, 507, 498, 483, 517, 508, 518, 492, 552, 497, 459, 494, 502, 504, 529, 496, 525, 521, 523, 521, 507, 505, 475, 473, 503, 535, 512, 544, 509, 506, 554, 520, 502, 480, 554, 495, 489, 543, 524, 529, 505, 500, 477, 530, 537, 506, 510, 547, 504, 526, 469, 530, 508, 493, 493, 519, 554, 497, 511, 528, 520, 519, 478, 531, 479, 518, 506, 496, 485, 538, 509, 516, 521, 547, 509, 478, 483, 540, 526, 516, 499, 538, 479, 506, 494, 534, 521, 537, 509, 516, 496, 551, 485, 497, 548, 519, 487, 508, 515, 515, 508, 519, 512, 527, 538, 555, 539, 488, 527] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[523, 531, 520, 542, 494, 505, 527, 481, 499, 538, 484, 468, 491, 507, 536, 498, 515, 498, 477, 493, 531, 521, 488, 526, 514, 531, 497, 531, 517, 524, 534, 475, 512, 489, 553, 498, 523, 548, 485, 489, 525, 500, 553, 496, 543, 499, 527, 510, 500, 498, 501, 547, 550, 488, 467, 533, 510, 500, 540, 533, 508, 505, 504, 536, 498, 521, 508, 517, 502, 482, 535, 522, 541, 481, 508, 523, 483, 539, 500, 515, 523, 514, 530, 495, 490, 506, 534, 518, 497, 535, 514, 520, 533, 495, 513, 542, 479, 539, 485, 501, 516, 515, 524, 500, 510, 490, 538, 512, 544, 519, 509, 541, 470, 486, 543, 535, 496, 501, 502, 567, 509, 465, 506, 511, 494, 525, 505, 479] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[500, 507, 497, 526, 494, 527, 522, 510, 546, 472, 506, 524, 509, 507, 468, 543, 482, 532, 525, 462, 492, 528, 527, 493, 577, 534, 521, 521, 510, 451, 497, 515, 499, 492, 529, 547, 523, 480, 536, 515, 485, 494, 508, 482, 506, 503, 517, 559, 509, 512, 527, 539, 530, 481, 543, 531, 512, 495, 519, 456, 505, 523, 521, 464, 551, 558, 500, 492, 543, 509, 531, 497, 551, 502, 533, 532, 495, 486, 521, 523, 491, 499, 471, 508, 545, 538, 530, 493, 502, 509, 533, 518, 548, 523, 498, 497, 512, 536, 531, 497, 534, 500, 533, 510, 514, 519, 479, 520, 514, 475, 518, 488, 536, 521, 497, 515, 547, 501, 493, 503, 490, 515, 515, 483, 477, 543, 523, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[495, 478, 470, 508, 502, 525, 495, 493, 538, 539, 523, 505, 547, 537, 509, 510, 533, 474, 491, 494, 495, 498, 486, 509, 496, 496, 475, 547, 499, 507, 542, 548, 494, 541, 511, 553, 494, 521, 532, 516, 491, 512, 496, 503, 498, 493, 522, 481, 470, 527, 512, 498, 536, 499, 536, 517, 522, 503, 502, 493, 555, 508, 521, 481, 545, 497, 516, 515, 497, 526, 512, 501, 509, 552, 507, 534, 564, 535, 452, 502, 482, 543, 507, 508, 490, 501, 509, 534, 508, 493, 514, 508, 521, 481, 512, 498, 556, 554, 557, 483, 538, 525, 554, 511, 522, 514, 527, 508, 485, 512, 537, 496, 524, 485, 525, 514, 511, 493, 508, 515, 548, 534, 525, 501, 483, 488, 494, 534] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[507, 489, 506, 511, 491, 522, 526, 509, 519, 511, 459, 471, 537, 508, 496, 527, 527, 530, 525, 534, 507, 498, 483, 517, 508, 518, 492, 552, 497, 459, 494, 502, 504, 529, 496, 525, 521, 523, 521, 507, 505, 475, 473, 503, 535, 512, 544, 509, 506, 554, 520, 502, 480, 554, 495, 489, 543, 524, 529, 505, 500, 477, 530, 537, 506, 510, 547, 504, 526, 469, 530, 508, 493, 493, 519, 554, 497, 511, 528, 520, 519, 478, 531, 479, 518, 506, 496, 485, 538, 509, 516, 521, 547, 509, 478, 483, 540, 526, 516, 499, 538, 479, 506, 494, 534, 521, 537, 509, 516, 496, 551, 485, 497, 548, 519, 487, 508, 515, 515, 508, 519, 512, 527, 538, 555, 539, 488, 527] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[532, 506, 529, 511, 506, 558, 546, 493, 505, 512, 496, 527, 528, 512, 522, 518, 498, 520, 506, 486, 502, 507, 512, 503, 498, 473, 464, 490, 498, 537, 510, 566, 503, 526, 553, 532, 510, 499, 536, 506, 506, 517, 497, 505, 508, 540, 547, 504, 530, 542, 511, 567, 492, 508, 567, 512, 484, 550, 513, 527, 510, 491, 506, 504, 495, 520, 518, 494, 537, 482, 516, 539, 511, 515, 515, 505, 504, 528, 555, 526, 530, 488, 500, 491, 500, 464, 495, 519, 518, 491, 525, 505, 495, 484, 522, 528, 524, 514, 502, 501, 516, 546, 528, 516, 501, 520, 465, 516, 507, 503, 518, 481, 491, 486, 513, 498, 505, 520, 501, 483, 524, 553, 527, 483, 481, 544, 495, 485] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[517, 513, 535, 531, 520, 503, 505, 516, 506, 546, 517, 526, 534, 555, 527, 529, 519, 498, 494, 507, 536, 491, 488, 519, 478, 478, 512, 496, 502, 550, 484, 511, 522, 549, 509, 484, 482, 515, 538, 483, 508, 546, 523, 511, 498, 499, 509, 516, 485, 524, 506, 490, 529, 490, 475, 496, 534, 518, 517, 491, 500, 497, 499, 503, 520, 548, 491, 498, 510, 532, 500, 483, 523, 483, 490, 527, 505, 528, 515, 515, 495, 532, 516, 528, 481, 509, 502, 508, 516, 534, 504, 493, 520, 512, 505, 524, 522, 497, 486, 544, 532, 509, 490, 531, 539, 502, 546, 543, 519, 507, 481, 508, 488, 485, 512, 500, 494, 519, 545, 522, 543, 487, 524, 544, 522, 498, 542, 519] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[500, 507, 497, 526, 494, 527, 522, 510, 546, 472, 506, 524, 509, 507, 468, 543, 482, 532, 525, 462, 492, 528, 527, 493, 577, 534, 521, 521, 510, 451, 497, 515, 499, 492, 529, 547, 523, 480, 536, 515, 485, 494, 508, 482, 506, 503, 517, 559, 509, 512, 527, 539, 530, 481, 543, 531, 512, 495, 519, 456, 505, 523, 521, 464, 551, 558, 500, 492, 543, 509, 531, 497, 551, 502, 533, 532, 495, 486, 521, 523, 491, 499, 471, 508, 545, 538, 530, 493, 502, 509, 533, 518, 548, 523, 498, 497, 512, 536, 531, 497, 534, 500, 533, 510, 514, 519, 479, 520, 514, 475, 518, 488, 536, 521, 497, 515, 547, 501, 493, 503, 490, 515, 515, 483, 477, 543, 523, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[490, 485, 496, 510, 520, 491, 495, 531, 540, 484, 549, 528, 491, 514, 533, 472, 498, 550, 504, 512, 531, 512, 534, 466, 527, 527, 501, 486, 506, 522, 495, 528, 527, 484, 523, 475, 541, 509, 518, 534, 529, 516, 540, 509, 554, 532, 504, 534, 553, 533, 511, 556, 504, 481, 523, 555, 529, 502, 491, 519, 488, 491, 525, 492, 532, 490, 507, 475, 496, 513, 516, 499, 525, 541, 511, 485, 521, 515, 495, 487, 497, 502, 513, 481, 556, 512, 531, 498, 524, 532, 594, 475, 526, 499, 457, 524, 498, 505, 501, 486, 477, 525, 529, 511, 513, 495, 509, 494, 524, 539, 518, 506, 547, 498, 516, 481, 534, 499, 517, 546, 515, 524, 524, 502, 500, 493, 488, 483] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[532, 506, 529, 511, 506, 558, 546, 493, 505, 512, 496, 527, 528, 512, 522, 518, 498, 520, 506, 486, 502, 507, 512, 503, 498, 473, 464, 490, 498, 537, 510, 566, 503, 526, 553, 532, 510, 499, 536, 506, 506, 517, 497, 505, 508, 540, 547, 504, 530, 542, 511, 567, 492, 508, 567, 512, 484, 550, 513, 527, 510, 491, 506, 504, 495, 520, 518, 494, 537, 482, 516, 539, 511, 515, 515, 505, 504, 528, 555, 526, 530, 488, 500, 491, 500, 464, 495, 519, 518, 491, 525, 505, 495, 484, 522, 528, 524, 514, 502, 501, 516, 546, 528, 516, 501, 520, 465, 516, 507, 503, 518, 481, 491, 486, 513, 498, 505, 520, 501, 483, 524, 553, 527, 483, 481, 544, 495, 485] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[480, 490, 498, 546, 505, 500, 501, 531, 524, 506, 495, 527, 484, 503, 527, 551, 546, 512, 512, 511, 503, 500, 507, 524, 493, 536, 486, 520, 482, 526, 538, 509, 537, 497, 479, 524, 526, 517, 515, 487, 525, 546, 505, 532, 513, 497, 518, 538, 530, 478, 535, 535, 496, 519, 515, 498, 526, 526, 514, 499, 496, 546, 495, 521, 532, 511, 523, 477, 506, 526, 542, 506, 509, 523, 530, 505, 525, 527, 467, 529, 561, 484, 508, 500, 494, 533, 515, 510, 491, 530, 503, 518, 514, 492, 525, 506, 496, 525, 496, 540, 488, 514, 498, 483, 488, 488, 466, 535, 492, 564, 546, 483, 549, 516, 518, 516, 470, 526, 486, 527, 497, 483, 525, 500, 496, 513, 507, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[495, 478, 470, 508, 502, 525, 495, 493, 538, 539, 523, 505, 547, 537, 509, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 474, 491, 494, 495, 498, 486, 509, 496, 496, 475, 547, 499, 507, 542, 548, 36, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 541, 511, 553, 494, 521, 532, 516, 491, 512, 496, 503, 498, 493, 486, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 527, 512, 484, 536, 499, 536, 517, 522, 503, 502, 493, 555, 508, 521, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 497, 516, 515, 497, 526, 486, 501, 509, 552, 507, 534, 564, 535, 452, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 543, 507, 508, 490, 501, 509, 534, 508, 493, 514, 508, 521, 481, 512, 498, 17, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 536, 557, 483, 538, 525, 554, 511, 522, 514, 527, 508, 485, 512, 537, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 485, 525, 514, 511, 493, 508, 515, 548, 534, 525, 501, 483, 488, 494, 534, 14, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 8142, 8122, 8166, 8238, 8136, 8361, 8197] avg=8192.0000 max=8361 max_rank=6 min=8122 min_rank=2 imbalance_ratio=1.020630 max_load_reduction=57175 max_load_reduction_pct=0.872421 moved_to_echo=32932 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[523, 531, 520, 542, 494, 505, 527, 481, 499, 538, 484, 468, 491, 507, 536, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 498, 477, 493, 531, 521, 488, 526, 514, 531, 497, 531, 517, 524, 534, 475, 37, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 489, 553, 498, 523, 548, 485, 489, 525, 500, 553, 496, 543, 499, 490, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 498, 501, 531, 550, 488, 467, 533, 510, 500, 540, 533, 508, 505, 504, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 521, 508, 517, 502, 482, 509, 522, 541, 481, 508, 523, 483, 539, 500, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 514, 530, 495, 490, 506, 534, 518, 497, 535, 514, 520, 533, 495, 513, 542, 16, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 522, 485, 501, 516, 515, 524, 500, 510, 490, 538, 512, 544, 519, 509, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 486, 543, 535, 496, 501, 502, 567, 509, 465, 506, 511, 494, 525, 505, 479, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8144, 8225, 8213, 8204, 8149, 8285, 8205, 8111] avg=8192.0000 max=8285 max_rank=5 min=8111 min_rank=7 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32750 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[500, 507, 497, 526, 494, 527, 522, 510, 546, 472, 506, 524, 509, 507, 468, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 532, 525, 462, 492, 528, 527, 493, 577, 534, 521, 521, 510, 451, 497, 515, 36, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 492, 529, 547, 523, 480, 536, 515, 485, 494, 508, 482, 506, 503, 481, 559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 512, 527, 523, 530, 481, 543, 531, 512, 495, 519, 456, 505, 523, 521, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 558, 500, 492, 543, 509, 505, 497, 551, 502, 533, 532, 495, 486, 521, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 499, 471, 508, 545, 538, 530, 493, 502, 509, 533, 518, 548, 523, 498, 497, 16, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 519, 531, 497, 534, 500, 533, 510, 514, 519, 479, 520, 514, 475, 518, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 521, 497, 515, 547, 501, 493, 503, 490, 515, 515, 483, 477, 543, 523, 503, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8158, 8219, 8139, 8151, 8298, 8229, 8163, 8179] avg=8192.0000 max=8298 max_rank=4 min=8139 min_rank=2 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32869 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[517, 513, 535, 524, 520, 503, 505, 516, 506, 546, 517, 526, 534, 555, 527, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 498, 494, 507, 536, 491, 488, 519, 478, 478, 512, 496, 502, 550, 484, 511, 39, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 549, 509, 484, 482, 515, 538, 483, 508, 546, 523, 511, 498, 499, 470, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 524, 506, 471, 529, 490, 475, 496, 534, 518, 517, 491, 500, 497, 499, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 548, 491, 498, 510, 532, 467, 483, 523, 483, 490, 527, 505, 528, 515, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 532, 516, 528, 481, 509, 502, 508, 516, 534, 504, 493, 520, 512, 505, 524, 18, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 477, 486, 544, 532, 509, 490, 531, 539, 502, 546, 543, 519, 507, 481, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 485, 512, 500, 494, 519, 545, 522, 543, 487, 524, 544, 522, 498, 542, 519, 19, 2, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8373, 8121, 8153, 8035, 8135, 8211, 8236, 8272] avg=8192.0000 max=8373 max_rank=0 min=8035 min_rank=3 imbalance_ratio=1.022095 max_load_reduction=57163 max_load_reduction_pct=0.872238 moved_to_echo=32854 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[517, 513, 535, 531, 520, 503, 505, 516, 506, 546, 517, 526, 534, 555, 527, 529, 519, 498, 494, 507, 536, 491, 488, 519, 478, 478, 512, 496, 502, 550, 484, 511, 522, 549, 509, 484, 482, 515, 538, 483, 508, 546, 523, 511, 498, 499, 509, 516, 485, 524, 506, 490, 529, 490, 475, 496, 534, 518, 517, 491, 500, 497, 499, 503, 520, 548, 491, 498, 510, 532, 500, 483, 523, 483, 490, 527, 505, 528, 515, 515, 495, 532, 516, 528, 481, 509, 502, 508, 516, 534, 504, 493, 520, 512, 505, 524, 522, 497, 486, 544, 532, 509, 490, 531, 539, 502, 546, 543, 519, 507, 481, 508, 488, 485, 512, 500, 494, 519, 545, 522, 543, 487, 524, 544, 522, 498, 542, 519] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[507, 489, 506, 511, 491, 522, 526, 509, 519, 511, 459, 471, 537, 508, 496, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 530, 525, 534, 507, 498, 483, 517, 508, 518, 492, 552, 497, 459, 494, 502, 38, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 529, 496, 525, 521, 523, 521, 507, 505, 475, 473, 503, 535, 512, 506, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 554, 520, 488, 480, 554, 495, 489, 543, 524, 529, 505, 500, 477, 530, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 510, 547, 504, 526, 469, 504, 508, 493, 493, 519, 554, 497, 511, 528, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 478, 531, 479, 518, 506, 496, 485, 538, 509, 516, 521, 547, 509, 478, 483, 16, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 509, 516, 499, 538, 479, 506, 494, 534, 521, 537, 509, 516, 496, 551, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 548, 519, 487, 508, 515, 515, 508, 519, 512, 527, 538, 555, 539, 488, 527, 14, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8089, 8197, 8144, 8231, 8189, 8139, 8230, 8317] avg=8192.0000 max=8317 max_rank=7 min=8089 min_rank=0 imbalance_ratio=1.015259 max_load_reduction=57219 max_load_reduction_pct=0.873093 moved_to_echo=32875 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[517, 513, 535, 524, 520, 503, 505, 516, 506, 546, 517, 526, 534, 555, 527, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 498, 494, 507, 536, 491, 488, 519, 478, 478, 512, 496, 502, 550, 484, 511, 39, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 549, 509, 484, 482, 515, 538, 483, 508, 546, 523, 511, 498, 499, 470, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 524, 506, 471, 529, 490, 475, 496, 534, 518, 517, 491, 500, 497, 499, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 548, 491, 498, 510, 532, 467, 483, 523, 483, 490, 527, 505, 528, 515, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 532, 516, 528, 481, 509, 502, 508, 516, 534, 504, 493, 520, 512, 505, 524, 18, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 477, 486, 544, 532, 509, 490, 531, 539, 502, 546, 543, 519, 507, 481, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 485, 512, 500, 494, 519, 545, 522, 543, 487, 524, 544, 522, 498, 542, 519, 19, 2, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8373, 8121, 8153, 8035, 8135, 8211, 8236, 8272] avg=8192.0000 max=8373 max_rank=0 min=8035 min_rank=3 imbalance_ratio=1.022095 max_load_reduction=57163 max_load_reduction_pct=0.872238 moved_to_echo=32854 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[500, 507, 497, 526, 494, 527, 522, 510, 546, 472, 506, 524, 509, 507, 468, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 532, 525, 462, 492, 528, 527, 493, 577, 534, 521, 521, 510, 451, 497, 515, 36, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 492, 529, 547, 523, 480, 536, 515, 485, 494, 508, 482, 506, 503, 481, 559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 512, 527, 523, 530, 481, 543, 531, 512, 495, 519, 456, 505, 523, 521, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 558, 500, 492, 543, 509, 505, 497, 551, 502, 533, 532, 495, 486, 521, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 499, 471, 508, 545, 538, 530, 493, 502, 509, 533, 518, 548, 523, 498, 497, 16, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 519, 531, 497, 534, 500, 533, 510, 514, 519, 479, 520, 514, 475, 518, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 521, 497, 515, 547, 501, 493, 503, 490, 515, 515, 483, 477, 543, 523, 503, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8158, 8219, 8139, 8151, 8298, 8229, 8163, 8179] avg=8192.0000 max=8298 max_rank=4 min=8139 min_rank=2 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32869 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[490, 485, 496, 510, 520, 491, 495, 531, 540, 484, 549, 528, 491, 514, 533, 472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 550, 504, 512, 531, 512, 534, 466, 527, 527, 501, 486, 506, 522, 495, 528, 35, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 484, 523, 475, 541, 509, 518, 534, 529, 516, 540, 509, 554, 532, 469, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 533, 511, 540, 504, 481, 523, 555, 529, 502, 491, 519, 488, 491, 525, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 490, 507, 475, 496, 513, 490, 499, 525, 541, 511, 485, 521, 515, 495, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 502, 513, 481, 556, 512, 531, 498, 524, 532, 594, 475, 526, 499, 457, 524, 15, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 489, 501, 486, 477, 525, 529, 511, 513, 495, 509, 494, 524, 539, 518, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 498, 516, 481, 534, 499, 517, 546, 515, 524, 524, 502, 500, 493, 488, 483, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8129, 8250, 8294, 8237, 8082, 8246, 8114, 8184] avg=8192.0000 max=8294 max_rank=2 min=8082 min_rank=4 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32626 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[495, 478, 470, 508, 502, 525, 495, 493, 538, 539, 523, 505, 547, 537, 509, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 474, 491, 494, 495, 498, 486, 509, 496, 496, 475, 547, 499, 507, 542, 548, 36, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 541, 511, 553, 494, 521, 532, 516, 491, 512, 496, 503, 498, 493, 486, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 527, 512, 484, 536, 499, 536, 517, 522, 503, 502, 493, 555, 508, 521, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 497, 516, 515, 497, 526, 486, 501, 509, 552, 507, 534, 564, 535, 452, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 543, 507, 508, 490, 501, 509, 534, 508, 493, 514, 508, 521, 481, 512, 498, 17, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 536, 557, 483, 538, 525, 554, 511, 522, 514, 527, 508, 485, 512, 537, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 485, 525, 514, 511, 493, 508, 515, 548, 534, 525, 501, 483, 488, 494, 534, 14, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 8142, 8122, 8166, 8238, 8136, 8361, 8197] avg=8192.0000 max=8361 max_rank=6 min=8122 min_rank=2 imbalance_ratio=1.020630 max_load_reduction=57175 max_load_reduction_pct=0.872421 moved_to_echo=32932 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[532, 506, 529, 511, 506, 558, 546, 493, 505, 512, 496, 527, 528, 512, 522, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 520, 506, 486, 502, 507, 512, 503, 498, 473, 464, 490, 498, 537, 510, 566, 38, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 526, 553, 532, 510, 499, 536, 506, 506, 517, 497, 505, 508, 540, 509, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 542, 511, 551, 492, 508, 567, 512, 484, 550, 513, 527, 510, 491, 506, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 520, 518, 494, 537, 482, 490, 539, 511, 515, 515, 505, 504, 528, 555, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 488, 500, 491, 500, 464, 495, 519, 518, 491, 525, 505, 495, 484, 522, 528, 15, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 498, 502, 501, 516, 546, 528, 516, 501, 520, 465, 516, 507, 503, 518, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 486, 513, 498, 505, 520, 501, 483, 524, 553, 527, 483, 481, 544, 495, 485, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8301, 8124, 8251, 8298, 8234, 8080, 8142, 8106] avg=8192.0000 max=8301 max_rank=0 min=8080 min_rank=5 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32562 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[480, 490, 498, 546, 505, 500, 501, 531, 524, 506, 495, 527, 484, 503, 527, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 512, 512, 511, 503, 500, 507, 524, 493, 536, 486, 520, 482, 526, 538, 509, 36, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 497, 479, 524, 526, 517, 515, 487, 525, 546, 505, 532, 513, 497, 482, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 478, 535, 520, 496, 519, 515, 498, 526, 526, 514, 499, 496, 546, 495, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 511, 523, 477, 506, 526, 515, 506, 509, 523, 530, 505, 525, 527, 467, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 484, 508, 500, 494, 533, 515, 510, 491, 530, 503, 518, 514, 492, 525, 506, 16, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 508, 496, 540, 488, 514, 498, 483, 488, 488, 466, 535, 492, 564, 546, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 516, 518, 516, 470, 526, 486, 527, 497, 483, 525, 500, 496, 513, 507, 525, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8168, 8258, 8220, 8214, 8211, 8210, 8085, 8170] avg=8192.0000 max=8258 max_rank=1 min=8085 min_rank=6 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32676 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[490, 485, 496, 510, 520, 491, 495, 531, 540, 484, 549, 528, 491, 514, 533, 472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 550, 504, 512, 531, 512, 534, 466, 527, 527, 501, 486, 506, 522, 495, 528, 35, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 484, 523, 475, 541, 509, 518, 534, 529, 516, 540, 509, 554, 532, 469, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 533, 511, 540, 504, 481, 523, 555, 529, 502, 491, 519, 488, 491, 525, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 490, 507, 475, 496, 513, 490, 499, 525, 541, 511, 485, 521, 515, 495, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 502, 513, 481, 556, 512, 531, 498, 524, 532, 594, 475, 526, 499, 457, 524, 15, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 489, 501, 486, 477, 525, 529, 511, 513, 495, 509, 494, 524, 539, 518, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 498, 516, 481, 534, 499, 517, 546, 515, 524, 524, 502, 500, 493, 488, 483, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8129, 8250, 8294, 8237, 8082, 8246, 8114, 8184] avg=8192.0000 max=8294 max_rank=2 min=8082 min_rank=4 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32626 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[523, 531, 520, 542, 494, 505, 527, 481, 499, 538, 484, 468, 491, 507, 536, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 498, 477, 493, 531, 521, 488, 526, 514, 531, 497, 531, 517, 524, 534, 475, 37, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 489, 553, 498, 523, 548, 485, 489, 525, 500, 553, 496, 543, 499, 490, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 498, 501, 531, 550, 488, 467, 533, 510, 500, 540, 533, 508, 505, 504, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 521, 508, 517, 502, 482, 509, 522, 541, 481, 508, 523, 483, 539, 500, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 514, 530, 495, 490, 506, 534, 518, 497, 535, 514, 520, 533, 495, 513, 542, 16, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 522, 485, 501, 516, 515, 524, 500, 510, 490, 538, 512, 544, 519, 509, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 486, 543, 535, 496, 501, 502, 567, 509, 465, 506, 511, 494, 525, 505, 479, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8144, 8225, 8213, 8204, 8149, 8285, 8205, 8111] avg=8192.0000 max=8285 max_rank=5 min=8111 min_rank=7 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32750 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 3968, 4224, 4096, 4096, 4096, 4224, 384, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 3968, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 3968, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 3968, 4224, 4096, 4096, 4096, 4224, 384, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[499, 518, 525, 532, 513, 506, 541, 498, 498, 534, 455, 484, 505, 488, 527, 508, 505, 502, 529, 515, 504, 505, 506, 514, 512, 553, 495, 522, 529, 516, 530, 488, 489, 502, 520, 507, 526, 535, 508, 504, 477, 511, 517, 506, 522, 542, 491, 487, 509, 523, 530, 508, 508, 528, 475, 535, 533, 479, 507, 512, 482, 479, 515, 518, 509, 555, 537, 512, 510, 496, 536, 477, 548, 527, 515, 530, 520, 541, 521, 516, 470, 503, 490, 498, 502, 486, 514, 506, 519, 532, 531, 510, 537, 527, 571, 510, 498, 522, 516, 526, 505, 499, 524, 516, 512, 519, 516, 514, 528, 498, 519, 489, 541, 524, 500, 529, 552, 476, 514, 478, 508, 486, 480, 528, 516, 518, 485, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[502, 521, 509, 477, 478, 514, 514, 499, 482, 511, 540, 528, 524, 520, 529, 522, 517, 496, 533, 469, 496, 526, 484, 525, 506, 517, 512, 501, 499, 540, 489, 512, 481, 530, 501, 473, 511, 539, 505, 545, 467, 520, 558, 463, 527, 484, 512, 540, 484, 512, 525, 505, 485, 535, 507, 545, 525, 545, 540, 506, 534, 528, 534, 470, 529, 537, 529, 509, 485, 510, 579, 469, 524, 525, 535, 550, 515, 558, 493, 501, 504, 540, 492, 491, 484, 477, 552, 503, 521, 493, 505, 543, 546, 514, 510, 519, 500, 495, 488, 531, 500, 498, 504, 470, 517, 519, 487, 502, 452, 518, 472, 503, 511, 530, 540, 523, 541, 488, 503, 511, 496, 505, 542, 539, 510, 558, 503, 510] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[554, 504, 499, 514, 469, 495, 520, 499, 526, 518, 555, 542, 515, 485, 468, 518, 515, 486, 489, 485, 499, 523, 506, 508, 508, 539, 517, 514, 502, 507, 506, 509, 482, 543, 524, 526, 537, 503, 498, 505, 478, 498, 493, 491, 516, 504, 551, 532, 535, 488, 460, 521, 527, 538, 526, 553, 508, 510, 521, 539, 559, 507, 506, 517, 521, 478, 523, 552, 490, 473, 499, 493, 432, 542, 508, 538, 437, 548, 460, 534, 563, 511, 569, 498, 593, 514, 501, 481, 550, 514, 499, 480, 488, 536, 521, 503, 490, 521, 505, 492, 500, 516, 511, 528, 529, 511, 481, 540, 510, 556, 530, 526, 506, 539, 535, 492, 504, 524, 505, 521, 482, 490, 492, 494, 526, 489, 538, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[553, 471, 484, 540, 503, 513, 526, 513, 559, 498, 494, 502, 541, 495, 494, 511, 461, 517, 468, 533, 516, 523, 547, 492, 538, 546, 453, 529, 502, 542, 471, 501, 541, 509, 468, 480, 503, 533, 538, 511, 544, 482, 510, 521, 521, 512, 470, 489, 554, 526, 521, 521, 504, 516, 533, 536, 532, 509, 505, 510, 514, 499, 516, 526, 472, 457, 505, 518, 517, 517, 491, 503, 492, 518, 568, 515, 482, 480, 514, 529, 512, 505, 519, 500, 496, 516, 546, 563, 465, 505, 519, 506, 500, 473, 514, 551, 507, 503, 536, 525, 509, 505, 528, 530, 520, 516, 502, 456, 543, 504, 474, 523, 513, 547, 507, 540, 534, 502, 520, 498, 510, 543, 535, 503, 515, 525, 492, 513] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[499, 518, 525, 532, 513, 506, 541, 498, 498, 534, 455, 484, 505, 488, 527, 508, 505, 502, 529, 515, 504, 505, 506, 514, 512, 553, 495, 522, 529, 516, 530, 488, 489, 502, 520, 507, 526, 535, 508, 504, 477, 511, 517, 506, 522, 542, 491, 487, 509, 523, 530, 508, 508, 528, 475, 535, 533, 479, 507, 512, 482, 479, 515, 518, 509, 555, 537, 512, 510, 496, 536, 477, 548, 527, 515, 530, 520, 541, 521, 516, 470, 503, 490, 498, 502, 486, 514, 506, 519, 532, 531, 510, 537, 527, 571, 510, 498, 522, 516, 526, 505, 499, 524, 516, 512, 519, 516, 514, 528, 498, 519, 489, 541, 524, 500, 529, 552, 476, 514, 478, 508, 486, 480, 528, 516, 518, 485, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[531, 473, 543, 521, 510, 497, 531, 497, 520, 530, 480, 556, 512, 512, 487, 528, 509, 558, 532, 511, 512, 497, 498, 485, 515, 504, 513, 498, 518, 537, 540, 512, 563, 460, 524, 471, 519, 483, 494, 501, 516, 507, 487, 471, 549, 517, 520, 531, 479, 519, 529, 471, 520, 552, 523, 507, 472, 562, 526, 544, 468, 514, 479, 519, 536, 505, 520, 524, 522, 558, 517, 485, 466, 556, 505, 530, 515, 512, 508, 522, 469, 552, 473, 540, 524, 538, 510, 502, 503, 519, 504, 546, 489, 530, 512, 509, 528, 527, 494, 506, 527, 503, 547, 494, 496, 511, 508, 527, 462, 523, 517, 484, 489, 488, 512, 466, 484, 547, 532, 502, 520, 504, 485, 470, 551, 532, 538, 497] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[509, 515, 471, 516, 474, 489, 499, 545, 571, 536, 514, 557, 485, 546, 505, 533, 483, 505, 514, 500, 504, 487, 508, 474, 521, 525, 526, 520, 496, 471, 536, 476, 507, 519, 505, 514, 494, 501, 532, 562, 476, 479, 510, 504, 497, 520, 537, 532, 537, 574, 527, 524, 490, 478, 510, 516, 549, 502, 535, 523, 487, 532, 503, 516, 454, 494, 554, 474, 501, 529, 512, 507, 526, 518, 503, 552, 498, 534, 537, 546, 502, 521, 537, 544, 479, 507, 524, 494, 509, 535, 473, 495, 489, 563, 511, 485, 492, 512, 458, 455, 521, 492, 507, 519, 523, 515, 516, 554, 541, 509, 512, 522, 494, 483, 561, 505, 489, 506, 523, 526, 495, 517, 518, 509, 542, 478, 491, 541] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[554, 504, 499, 514, 469, 495, 520, 499, 526, 518, 555, 542, 515, 485, 468, 518, 515, 486, 489, 485, 499, 523, 506, 508, 508, 539, 517, 514, 502, 507, 506, 509, 482, 543, 524, 526, 537, 503, 498, 505, 478, 498, 493, 491, 516, 504, 551, 532, 535, 488, 460, 521, 527, 538, 526, 553, 508, 510, 521, 539, 559, 507, 506, 517, 521, 478, 523, 552, 490, 473, 499, 493, 432, 542, 508, 538, 437, 548, 460, 534, 563, 511, 569, 498, 593, 514, 501, 481, 550, 514, 499, 480, 488, 536, 521, 503, 490, 521, 505, 492, 500, 516, 511, 528, 529, 511, 481, 540, 510, 556, 530, 526, 506, 539, 535, 492, 504, 524, 505, 521, 482, 490, 492, 494, 526, 489, 538, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[550, 541, 529, 505, 550, 521, 496, 516, 514, 539, 512, 481, 523, 495, 501, 494, 529, 519, 493, 508, 530, 541, 475, 520, 487, 508, 515, 480, 526, 497, 514, 441, 524, 518, 500, 513, 518, 535, 493, 508, 509, 505, 469, 520, 518, 498, 522, 552, 512, 501, 502, 505, 504, 484, 527, 470, 508, 512, 560, 553, 506, 537, 519, 522, 503, 513, 483, 515, 516, 533, 484, 503, 549, 537, 537, 480, 534, 520, 534, 510, 518, 514, 503, 504, 494, 470, 498, 547, 477, 546, 527, 478, 487, 520, 515, 502, 542, 507, 537, 467, 497, 518, 520, 501, 498, 531, 490, 520, 542, 485, 528, 541, 507, 538, 495, 520, 473, 542, 527, 489, 523, 517, 493, 510, 521, 512, 504, 516] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[508, 540, 486, 534, 511, 524, 474, 496, 488, 460, 541, 516, 533, 505, 515, 488, 509, 539, 491, 504, 534, 501, 549, 550, 494, 539, 535, 526, 549, 483, 509, 529, 550, 520, 500, 493, 533, 500, 510, 486, 527, 561, 461, 519, 513, 539, 538, 547, 505, 487, 494, 517, 542, 503, 481, 467, 502, 501, 545, 509, 529, 528, 532, 545, 539, 504, 504, 462, 517, 531, 511, 508, 554, 508, 520, 543, 507, 528, 496, 517, 505, 518, 469, 442, 506, 509, 507, 481, 508, 498, 509, 539, 531, 481, 522, 530, 567, 481, 524, 494, 488, 521, 489, 509, 518, 530, 472, 487, 521, 477, 500, 520, 528, 459, 489, 493, 505, 537, 515, 539, 528, 510, 542, 530, 486, 511, 525, 493] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[551, 471, 484, 540, 503, 513, 526, 513, 559, 498, 494, 502, 541, 495, 494, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 517, 468, 533, 516, 523, 547, 492, 538, 546, 453, 529, 502, 542, 471, 501, 16, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 509, 468, 480, 503, 533, 538, 511, 544, 482, 510, 521, 521, 512, 470, 489, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 526, 521, 521, 504, 516, 533, 536, 532, 509, 456, 510, 514, 499, 516, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 457, 505, 518, 517, 517, 491, 503, 492, 518, 568, 482, 482, 480, 514, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 505, 519, 500, 496, 516, 546, 563, 465, 505, 519, 506, 500, 473, 514, 551, 3, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 503, 536, 525, 509, 505, 528, 530, 520, 516, 502, 456, 543, 504, 474, 523, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 547, 507, 540, 534, 502, 520, 498, 510, 543, 535, 503, 512, 525, 492, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8165, 8146, 8273, 8045, 8198, 8220, 8294] avg=8192.0000 max=8294 max_rank=7 min=8045 min_rank=4 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32757 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[550, 541, 529, 505, 550, 521, 496, 516, 514, 539, 512, 481, 523, 495, 501, 494, 529, 519, 493, 508, 530, 541, 475, 520, 487, 508, 515, 480, 526, 497, 514, 441, 524, 518, 500, 513, 518, 535, 493, 508, 509, 505, 469, 520, 518, 498, 522, 552, 512, 501, 502, 505, 504, 484, 527, 470, 508, 512, 560, 553, 506, 537, 519, 522, 503, 513, 483, 515, 516, 533, 484, 503, 549, 537, 537, 480, 534, 520, 534, 510, 518, 514, 503, 504, 494, 470, 498, 547, 477, 546, 527, 478, 487, 520, 515, 502, 542, 507, 537, 467, 497, 518, 520, 501, 498, 531, 490, 520, 542, 485, 528, 541, 507, 538, 495, 520, 473, 542, 527, 489, 523, 517, 493, 510, 521, 512, 504, 516] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[552, 504, 499, 514, 469, 495, 520, 499, 526, 518, 555, 542, 515, 485, 468, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 486, 489, 485, 499, 523, 506, 508, 508, 539, 517, 514, 502, 507, 506, 509, 17, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 543, 524, 526, 537, 503, 498, 505, 478, 498, 493, 491, 516, 504, 551, 532, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 488, 460, 521, 527, 538, 526, 553, 508, 510, 470, 539, 559, 507, 506, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 478, 523, 552, 490, 473, 499, 493, 432, 542, 508, 503, 437, 548, 460, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 563, 511, 569, 498, 593, 514, 501, 481, 550, 514, 499, 480, 488, 536, 521, 503, 4, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 521, 505, 492, 500, 516, 511, 528, 529, 511, 481, 540, 510, 556, 530, 526, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 539, 535, 492, 504, 524, 505, 521, 482, 490, 492, 494, 522, 489, 538, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8141, 8196, 8264, 7993, 8330, 8286, 8147] avg=8192.0000 max=8330 max_rank=5 min=7993 min_rank=4 imbalance_ratio=1.016846 max_load_reduction=57206 max_load_reduction_pct=0.872894 moved_to_echo=32756 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[500, 521, 509, 477, 478, 514, 514, 499, 482, 511, 540, 528, 524, 520, 529, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 496, 533, 469, 496, 526, 484, 525, 506, 517, 512, 501, 499, 540, 489, 512, 17, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 530, 501, 473, 511, 539, 505, 545, 467, 520, 558, 463, 527, 484, 512, 540, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 512, 525, 505, 485, 535, 507, 545, 525, 545, 488, 506, 534, 528, 534, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 537, 529, 509, 485, 510, 579, 469, 524, 525, 535, 515, 515, 558, 493, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 540, 492, 491, 484, 477, 552, 503, 521, 493, 505, 543, 546, 514, 510, 519, 3, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 495, 488, 531, 500, 498, 504, 470, 517, 519, 487, 502, 452, 518, 472, 503, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 530, 540, 523, 541, 488, 503, 511, 496, 505, 542, 539, 507, 558, 503, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8168, 8150, 8171, 8228, 8313, 8202, 7997, 8307] avg=8192.0000 max=8313 max_rank=4 min=7997 min_rank=6 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32819 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[491, 518, 525, 532, 513, 506, 541, 498, 498, 534, 455, 484, 505, 488, 527, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 502, 529, 515, 504, 505, 506, 514, 512, 553, 495, 522, 529, 516, 530, 488, 22, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 502, 520, 507, 526, 535, 508, 504, 477, 511, 517, 506, 522, 542, 491, 487, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 523, 530, 508, 508, 528, 475, 535, 533, 479, 451, 512, 482, 479, 515, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 555, 537, 512, 510, 496, 536, 477, 548, 527, 515, 481, 520, 541, 521, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 503, 490, 498, 502, 486, 514, 506, 519, 532, 531, 510, 537, 527, 571, 510, 7, 7, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 522, 516, 526, 505, 499, 524, 516, 512, 519, 516, 514, 528, 498, 519, 489, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 524, 500, 529, 552, 476, 514, 478, 508, 486, 480, 528, 509, 518, 485, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8123, 8261, 8164, 8085, 8301, 8228, 8243, 8131] avg=8192.0000 max=8301 max_rank=4 min=8085 min_rank=3 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32903 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[552, 504, 499, 514, 469, 495, 520, 499, 526, 518, 555, 542, 515, 485, 468, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 486, 489, 485, 499, 523, 506, 508, 508, 539, 517, 514, 502, 507, 506, 509, 17, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 543, 524, 526, 537, 503, 498, 505, 478, 498, 493, 491, 516, 504, 551, 532, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 488, 460, 521, 527, 538, 526, 553, 508, 510, 470, 539, 559, 507, 506, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 478, 523, 552, 490, 473, 499, 493, 432, 542, 508, 503, 437, 548, 460, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 563, 511, 569, 498, 593, 514, 501, 481, 550, 514, 499, 480, 488, 536, 521, 503, 4, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 521, 505, 492, 500, 516, 511, 528, 529, 511, 481, 540, 510, 556, 530, 526, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 539, 535, 492, 504, 524, 505, 521, 482, 490, 492, 494, 522, 489, 538, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8141, 8196, 8264, 7993, 8330, 8286, 8147] avg=8192.0000 max=8330 max_rank=5 min=7993 min_rank=4 imbalance_ratio=1.016846 max_load_reduction=57206 max_load_reduction_pct=0.872894 moved_to_echo=32756 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[551, 471, 484, 540, 503, 513, 526, 513, 559, 498, 494, 502, 541, 495, 494, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 517, 468, 533, 516, 523, 547, 492, 538, 546, 453, 529, 502, 542, 471, 501, 16, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 509, 468, 480, 503, 533, 538, 511, 544, 482, 510, 521, 521, 512, 470, 489, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 526, 521, 521, 504, 516, 533, 536, 532, 509, 456, 510, 514, 499, 516, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 457, 505, 518, 517, 517, 491, 503, 492, 518, 568, 482, 482, 480, 514, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 505, 519, 500, 496, 516, 546, 563, 465, 505, 519, 506, 500, 473, 514, 551, 3, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 503, 536, 525, 509, 505, 528, 530, 520, 516, 502, 456, 543, 504, 474, 523, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 547, 507, 540, 534, 502, 520, 498, 510, 543, 535, 503, 512, 525, 492, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8165, 8146, 8273, 8045, 8198, 8220, 8294] avg=8192.0000 max=8294 max_rank=7 min=8045 min_rank=4 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32757 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[529, 473, 543, 521, 510, 497, 531, 497, 520, 530, 480, 556, 512, 512, 487, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 558, 532, 511, 512, 497, 498, 485, 515, 504, 513, 498, 518, 537, 540, 512, 17, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 563, 460, 524, 471, 519, 483, 494, 501, 516, 507, 487, 471, 549, 517, 520, 531, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 519, 529, 471, 520, 552, 523, 507, 472, 562, 475, 544, 468, 514, 479, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 505, 520, 524, 522, 558, 517, 485, 466, 556, 505, 495, 515, 512, 508, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 552, 473, 540, 524, 538, 510, 502, 503, 519, 504, 546, 489, 530, 512, 509, 4, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 527, 494, 506, 527, 503, 547, 494, 496, 511, 508, 527, 462, 523, 517, 484, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 488, 512, 466, 484, 547, 532, 502, 520, 504, 485, 470, 547, 532, 538, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 8267, 8128, 8133, 8246, 8229, 8194, 8113] avg=8192.0000 max=8267 max_rank=1 min=8113 min_rank=7 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32782 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[507, 515, 471, 516, 474, 489, 499, 545, 571, 536, 514, 557, 485, 546, 505, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 505, 514, 500, 504, 487, 508, 474, 521, 525, 526, 520, 496, 471, 536, 476, 17, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 519, 505, 514, 494, 501, 532, 562, 476, 479, 510, 504, 497, 520, 537, 532, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 574, 527, 524, 490, 478, 510, 516, 549, 502, 483, 523, 487, 532, 503, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 454, 494, 554, 474, 501, 529, 512, 507, 526, 518, 503, 517, 498, 534, 537, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 521, 537, 544, 479, 507, 524, 494, 509, 535, 473, 495, 489, 563, 511, 485, 4, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 512, 458, 455, 521, 492, 507, 519, 523, 515, 516, 554, 541, 509, 512, 522, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 483, 561, 505, 489, 506, 523, 526, 495, 517, 518, 509, 538, 478, 491, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8263, 8074, 8204, 8251, 8204, 8177, 8189, 8174] avg=8192.0000 max=8263 max_rank=0 min=8074 min_rank=1 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32744 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[548, 541, 529, 505, 550, 521, 496, 516, 514, 539, 512, 481, 523, 495, 501, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 519, 493, 508, 530, 541, 475, 520, 487, 508, 515, 480, 526, 497, 514, 441, 15, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 518, 500, 513, 518, 535, 493, 508, 509, 505, 469, 520, 518, 498, 522, 552, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 501, 502, 505, 504, 484, 527, 470, 508, 512, 506, 553, 506, 537, 519, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 513, 483, 515, 516, 533, 484, 503, 549, 537, 537, 449, 534, 520, 534, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 514, 503, 504, 494, 470, 498, 547, 477, 546, 527, 478, 487, 520, 515, 502, 4, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 507, 537, 467, 497, 518, 520, 501, 498, 531, 490, 520, 542, 485, 528, 541, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 538, 495, 520, 473, 542, 527, 489, 523, 517, 493, 510, 517, 512, 504, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8265, 8109, 8215, 8168, 8220, 8109, 8267, 8183] avg=8192.0000 max=8267 max_rank=6 min=8109 min_rank=1 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32779 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[506, 540, 486, 534, 511, 524, 474, 496, 488, 460, 541, 516, 533, 505, 515, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 539, 491, 504, 534, 501, 549, 550, 494, 539, 535, 526, 549, 483, 509, 529, 17, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 520, 500, 493, 533, 500, 510, 486, 527, 561, 461, 519, 513, 539, 538, 547, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 487, 494, 517, 542, 503, 481, 467, 502, 501, 492, 509, 529, 528, 532, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 504, 504, 462, 517, 531, 511, 508, 554, 508, 520, 508, 507, 528, 496, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 518, 469, 442, 506, 509, 507, 481, 508, 498, 509, 539, 531, 481, 522, 530, 3, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 567, 481, 524, 494, 488, 521, 489, 509, 518, 530, 472, 487, 521, 477, 500, 520, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 459, 489, 493, 505, 537, 515, 539, 528, 510, 542, 530, 483, 511, 525, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8117, 8369, 8312, 8134, 8214, 8063, 8140, 8187] avg=8192.0000 max=8369 max_rank=1 min=8063 min_rank=5 imbalance_ratio=1.021606 max_load_reduction=57167 max_load_reduction_pct=0.872299 moved_to_echo=32604 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4352, 4096, 4096, 4224, 4096, 4096, 3968, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4224, 3968, 4096, 4352, 4224, 3968, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 3840, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4224, 3968, 4096, 4352, 4224, 3968, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 3840, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[494, 503, 501, 578, 514, 524, 538, 500, 542, 529, 512, 504, 537, 495, 511, 474, 508, 476, 539, 510, 492, 504, 521, 495, 507, 544, 526, 531, 543, 534, 521, 496, 497, 508, 539, 501, 519, 483, 488, 512, 501, 525, 505, 528, 536, 517, 503, 510, 504, 490, 519, 506, 498, 505, 503, 497, 527, 528, 498, 444, 517, 507, 485, 522, 522, 512, 490, 486, 500, 478, 515, 501, 528, 524, 492, 524, 504, 504, 527, 475, 563, 536, 528, 484, 484, 504, 493, 464, 504, 538, 538, 544, 491, 507, 527, 477, 522, 520, 510, 516, 528, 517, 514, 551, 492, 509, 516, 539, 514, 500, 500, 523, 485, 527, 499, 473, 502, 557, 517, 505, 516, 532, 536, 527, 537, 522, 516, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[502, 509, 519, 488, 504, 493, 517, 519, 546, 531, 473, 499, 505, 494, 548, 482, 508, 471, 540, 546, 532, 503, 492, 506, 541, 512, 509, 523, 520, 495, 491, 507, 510, 515, 504, 522, 489, 544, 498, 493, 496, 496, 522, 538, 522, 510, 541, 520, 554, 494, 532, 485, 511, 495, 493, 520, 499, 536, 527, 495, 505, 518, 522, 520, 476, 498, 487, 525, 535, 493, 481, 512, 499, 498, 535, 503, 520, 495, 538, 520, 512, 511, 542, 506, 523, 529, 529, 561, 494, 525, 514, 518, 527, 516, 534, 525, 516, 465, 467, 515, 527, 520, 531, 512, 533, 527, 512, 507, 537, 494, 522, 544, 511, 495, 521, 487, 471, 521, 508, 519, 502, 525, 473, 547, 483, 517, 487, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[523, 540, 488, 492, 494, 493, 559, 518, 496, 475, 496, 530, 503, 509, 480, 499, 506, 509, 520, 511, 489, 541, 497, 485, 517, 529, 502, 536, 480, 532, 523, 541, 487, 533, 485, 522, 479, 501, 505, 519, 485, 541, 502, 494, 511, 508, 482, 503, 506, 530, 500, 479, 477, 519, 484, 508, 510, 506, 521, 487, 531, 515, 528, 536, 496, 459, 516, 537, 496, 500, 522, 515, 534, 548, 501, 531, 513, 503, 512, 539, 541, 494, 549, 497, 532, 497, 512, 502, 497, 525, 533, 536, 519, 515, 564, 492, 479, 487, 547, 546, 517, 524, 521, 494, 521, 516, 518, 512, 558, 491, 511, 495, 523, 520, 539, 536, 508, 526, 528, 519, 510, 503, 511, 540, 528, 477, 475, 522] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[545, 533, 540, 500, 489, 522, 483, 501, 469, 511, 495, 513, 491, 540, 525, 473, 536, 478, 494, 540, 500, 483, 512, 510, 547, 504, 510, 530, 504, 504, 513, 512, 534, 516, 507, 534, 492, 492, 528, 496, 450, 533, 537, 516, 482, 520, 513, 520, 548, 516, 511, 549, 530, 519, 517, 529, 516, 503, 531, 484, 509, 479, 514, 479, 497, 510, 478, 493, 516, 518, 527, 537, 531, 537, 531, 553, 481, 533, 509, 496, 518, 542, 541, 517, 504, 476, 528, 518, 492, 529, 528, 502, 492, 553, 493, 473, 508, 496, 507, 502, 473, 535, 532, 480, 491, 501, 516, 488, 532, 490, 554, 520, 537, 489, 497, 517, 522, 542, 535, 506, 523, 474, 536, 522, 512, 508, 523, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[518, 546, 525, 509, 502, 507, 500, 531, 500, 496, 553, 495, 510, 521, 545, 549, 470, 504, 494, 500, 530, 470, 497, 501, 460, 525, 500, 521, 497, 509, 489, 519, 515, 524, 541, 553, 531, 512, 516, 493, 482, 523, 494, 505, 528, 483, 500, 519, 531, 511, 529, 505, 538, 500, 483, 506, 474, 537, 544, 537, 488, 524, 529, 476, 503, 496, 544, 530, 525, 542, 536, 492, 495, 507, 515, 475, 506, 491, 518, 518, 528, 497, 522, 526, 559, 503, 522, 543, 530, 571, 475, 484, 501, 579, 516, 501, 490, 507, 500, 529, 503, 485, 505, 485, 529, 507, 511, 510, 518, 535, 505, 485, 531, 521, 508, 516, 514, 492, 532, 468, 524, 498, 500, 479, 543, 518, 532, 482] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[521, 523, 524, 475, 462, 516, 500, 481, 542, 513, 521, 515, 502, 529, 531, 532, 502, 492, 520, 583, 496, 519, 505, 502, 498, 515, 449, 527, 539, 504, 503, 518, 506, 505, 493, 525, 542, 508, 505, 517, 476, 506, 519, 514, 497, 505, 526, 507, 473, 488, 462, 508, 558, 522, 498, 494, 541, 494, 535, 545, 538, 517, 532, 495, 546, 496, 465, 539, 546, 502, 505, 554, 460, 533, 537, 513, 511, 517, 498, 537, 522, 500, 492, 514, 487, 523, 523, 528, 491, 532, 494, 508, 515, 535, 491, 522, 482, 528, 530, 535, 505, 527, 501, 523, 523, 515, 536, 471, 481, 525, 509, 543, 471, 532, 547, 487, 534, 526, 473, 525, 508, 491, 510, 503, 508, 502, 517, 522] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[494, 517, 528, 503, 517, 517, 511, 533, 509, 504, 534, 512, 498, 514, 487, 512, 503, 492, 482, 547, 483, 515, 491, 557, 506, 503, 496, 521, 507, 506, 533, 473, 510, 524, 532, 516, 495, 524, 487, 496, 501, 525, 512, 512, 507, 480, 545, 484, 529, 512, 513, 518, 531, 459, 515, 505, 522, 503, 534, 521, 505, 524, 492, 545, 538, 531, 558, 498, 479, 546, 520, 530, 502, 510, 545, 510, 536, 537, 513, 514, 518, 508, 524, 515, 485, 559, 514, 501, 506, 473, 465, 496, 530, 478, 516, 498, 485, 531, 527, 513, 492, 510, 531, 528, 533, 550, 494, 486, 502, 500, 527, 497, 518, 506, 547, 494, 503, 485, 534, 529, 524, 509, 489, 500, 542, 535, 489, 490] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[502, 509, 519, 488, 504, 493, 517, 519, 546, 531, 473, 499, 505, 494, 548, 482, 508, 471, 540, 546, 532, 503, 492, 506, 541, 512, 509, 523, 520, 495, 491, 507, 510, 515, 504, 522, 489, 544, 498, 493, 496, 496, 522, 538, 522, 510, 541, 520, 554, 494, 532, 485, 511, 495, 493, 520, 499, 536, 527, 495, 505, 518, 522, 520, 476, 498, 487, 525, 535, 493, 481, 512, 499, 498, 535, 503, 520, 495, 538, 520, 512, 511, 542, 506, 523, 529, 529, 561, 494, 525, 514, 518, 527, 516, 534, 525, 516, 465, 467, 515, 527, 520, 531, 512, 533, 527, 512, 507, 537, 494, 522, 544, 511, 495, 521, 487, 471, 521, 508, 519, 502, 525, 473, 547, 483, 517, 487, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[545, 533, 540, 500, 489, 522, 483, 501, 469, 511, 495, 513, 491, 540, 525, 473, 536, 478, 494, 540, 500, 483, 512, 510, 547, 504, 510, 530, 504, 504, 513, 512, 534, 516, 507, 534, 492, 492, 528, 496, 450, 533, 537, 516, 482, 520, 513, 520, 548, 516, 511, 549, 530, 519, 517, 529, 516, 503, 531, 484, 509, 479, 514, 479, 497, 510, 478, 493, 516, 518, 527, 537, 531, 537, 531, 553, 481, 533, 509, 496, 518, 542, 541, 517, 504, 476, 528, 518, 492, 529, 528, 502, 492, 553, 493, 473, 508, 496, 507, 502, 473, 535, 532, 480, 491, 501, 516, 488, 532, 490, 554, 520, 537, 489, 497, 517, 522, 542, 535, 506, 523, 474, 536, 522, 512, 508, 523, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[494, 503, 501, 578, 514, 524, 538, 500, 542, 529, 512, 504, 537, 495, 511, 474, 508, 476, 539, 510, 492, 504, 521, 495, 507, 544, 526, 531, 543, 534, 521, 496, 497, 508, 539, 501, 519, 483, 488, 512, 501, 525, 505, 528, 536, 517, 503, 510, 504, 490, 519, 506, 498, 505, 503, 497, 527, 528, 498, 444, 517, 507, 485, 522, 522, 512, 490, 486, 500, 478, 515, 501, 528, 524, 492, 524, 504, 504, 527, 475, 563, 536, 528, 484, 484, 504, 493, 464, 504, 538, 538, 544, 491, 507, 527, 477, 522, 520, 510, 516, 528, 517, 514, 551, 492, 509, 516, 539, 514, 500, 500, 523, 485, 527, 499, 473, 502, 557, 517, 505, 516, 532, 536, 527, 537, 522, 516, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[518, 475, 506, 496, 501, 524, 475, 515, 532, 521, 529, 522, 509, 544, 527, 503, 524, 526, 540, 529, 468, 531, 523, 488, 524, 518, 516, 526, 504, 538, 494, 503, 517, 515, 563, 525, 542, 512, 483, 510, 514, 535, 536, 503, 523, 539, 501, 457, 520, 528, 515, 474, 471, 511, 490, 484, 545, 504, 499, 513, 523, 555, 505, 493, 516, 517, 495, 519, 528, 528, 463, 513, 485, 502, 516, 497, 498, 567, 525, 495, 491, 534, 509, 503, 548, 541, 514, 528, 491, 499, 501, 501, 511, 498, 514, 513, 497, 495, 516, 498, 507, 528, 516, 516, 482, 533, 485, 485, 517, 537, 520, 519, 522, 503, 490, 547, 496, 482, 528, 502, 555, 501, 509, 476, 490, 522, 537, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[521, 523, 524, 475, 462, 516, 500, 481, 542, 513, 521, 515, 502, 529, 531, 532, 502, 492, 520, 583, 496, 519, 505, 502, 498, 515, 449, 527, 539, 504, 503, 518, 506, 505, 493, 525, 542, 508, 505, 517, 476, 506, 519, 514, 497, 505, 526, 507, 473, 488, 462, 508, 558, 522, 498, 494, 541, 494, 535, 545, 538, 517, 532, 495, 546, 496, 465, 539, 546, 502, 505, 554, 460, 533, 537, 513, 511, 517, 498, 537, 522, 500, 492, 514, 487, 523, 523, 528, 491, 532, 494, 508, 515, 535, 491, 522, 482, 528, 530, 535, 505, 527, 501, 523, 523, 515, 536, 471, 481, 525, 509, 543, 471, 532, 547, 487, 534, 526, 473, 525, 508, 491, 510, 503, 508, 502, 517, 522] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[502, 509, 519, 488, 504, 493, 517, 519, 546, 531, 473, 499, 505, 494, 548, 482, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 471, 540, 546, 532, 503, 492, 506, 541, 512, 509, 523, 520, 495, 491, 507, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 515, 504, 522, 489, 544, 498, 493, 496, 496, 522, 538, 522, 510, 541, 520, 12, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 494, 532, 485, 511, 495, 493, 520, 499, 536, 527, 495, 505, 518, 522, 520, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 498, 487, 525, 535, 493, 481, 512, 499, 498, 521, 503, 520, 495, 538, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 511, 500, 506, 523, 529, 529, 561, 494, 525, 514, 518, 527, 516, 534, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 465, 467, 515, 527, 520, 531, 512, 533, 527, 512, 507, 535, 494, 522, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 495, 521, 487, 471, 521, 508, 519, 502, 525, 473, 547, 483, 517, 487, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8133, 8218, 8235, 8223, 8101, 8324, 8227, 8075] avg=8192.0000 max=8324 max_rank=5 min=8075 min_rank=7 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32727 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[523, 540, 488, 492, 494, 493, 559, 518, 496, 475, 496, 530, 503, 509, 480, 499, 506, 509, 520, 511, 489, 541, 497, 485, 517, 529, 502, 536, 480, 532, 523, 541, 487, 533, 485, 522, 479, 501, 505, 519, 485, 541, 502, 494, 511, 508, 482, 503, 506, 530, 500, 479, 477, 519, 484, 508, 510, 506, 521, 487, 531, 515, 528, 536, 496, 459, 516, 537, 496, 500, 522, 515, 534, 548, 501, 531, 513, 503, 512, 539, 541, 494, 549, 497, 532, 497, 512, 502, 497, 525, 533, 536, 519, 515, 564, 492, 479, 487, 547, 546, 517, 524, 521, 494, 521, 516, 518, 512, 558, 491, 511, 495, 523, 520, 539, 536, 508, 526, 528, 519, 510, 503, 511, 540, 528, 477, 475, 522] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[494, 517, 528, 503, 517, 517, 511, 533, 509, 504, 534, 512, 498, 514, 487, 512, 7, 4, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 492, 482, 547, 483, 515, 491, 557, 506, 503, 496, 521, 507, 506, 533, 473, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 524, 532, 516, 495, 524, 487, 496, 501, 525, 512, 512, 507, 480, 545, 484, 15, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 512, 513, 518, 531, 459, 515, 505, 522, 503, 534, 521, 505, 524, 492, 545, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 531, 558, 498, 479, 546, 520, 530, 502, 510, 526, 510, 536, 537, 513, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 508, 472, 515, 485, 559, 514, 501, 506, 473, 465, 496, 530, 478, 516, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 531, 527, 513, 492, 510, 531, 528, 533, 550, 494, 486, 495, 500, 527, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 506, 547, 494, 503, 485, 534, 529, 518, 509, 489, 500, 542, 535, 489, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8140, 8171, 8249, 8348, 8034, 8199, 8188] avg=8192.0000 max=8348 max_rank=4 min=8034 min_rank=5 imbalance_ratio=1.019043 max_load_reduction=57188 max_load_reduction_pct=0.872620 moved_to_echo=32769 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[518, 546, 525, 509, 502, 507, 500, 531, 500, 496, 553, 495, 510, 521, 545, 549, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 504, 494, 500, 530, 470, 497, 501, 460, 525, 500, 521, 497, 509, 489, 519, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 524, 541, 553, 531, 512, 516, 493, 482, 523, 494, 505, 528, 483, 500, 519, 11, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 511, 529, 505, 538, 500, 483, 506, 474, 537, 544, 537, 488, 524, 529, 476, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 496, 544, 530, 525, 542, 536, 492, 495, 507, 502, 475, 506, 491, 518, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 497, 481, 526, 559, 503, 522, 543, 530, 571, 475, 484, 501, 579, 516, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 507, 500, 529, 503, 485, 505, 485, 529, 507, 511, 510, 516, 535, 505, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 521, 508, 516, 514, 492, 532, 468, 524, 498, 500, 479, 543, 518, 532, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8311, 8007, 8233, 8229, 8180, 8316, 8102, 8158] avg=8192.0000 max=8316 max_rank=5 min=8007 min_rank=1 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32756 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[545, 533, 540, 500, 489, 522, 483, 501, 469, 511, 495, 513, 491, 540, 525, 473, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 478, 494, 540, 500, 483, 512, 510, 547, 504, 510, 530, 504, 504, 513, 512, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 516, 507, 534, 492, 492, 528, 496, 450, 533, 537, 516, 482, 520, 513, 520, 12, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 516, 511, 549, 530, 519, 517, 529, 516, 503, 531, 484, 509, 479, 514, 479, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 510, 478, 493, 516, 518, 527, 537, 531, 537, 517, 553, 481, 533, 509, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 542, 499, 517, 504, 476, 528, 518, 492, 529, 528, 502, 492, 553, 493, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 496, 507, 502, 473, 535, 532, 480, 491, 501, 516, 488, 530, 490, 554, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 489, 497, 517, 522, 542, 535, 506, 523, 474, 536, 522, 512, 508, 523, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8134, 8199, 8185, 8251, 8233, 8164, 8123, 8247] avg=8192.0000 max=8251 max_rank=3 min=8123 min_rank=6 imbalance_ratio=1.007202 max_load_reduction=57285 max_load_reduction_pct=0.874100 moved_to_echo=32767 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[521, 523, 524, 475, 462, 516, 500, 481, 542, 513, 521, 515, 502, 529, 531, 532, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 492, 520, 583, 496, 519, 505, 502, 498, 515, 449, 527, 539, 504, 503, 518, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 505, 493, 525, 542, 508, 505, 517, 476, 506, 519, 514, 497, 505, 526, 507, 12, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 488, 462, 508, 558, 522, 498, 494, 541, 494, 535, 545, 538, 517, 532, 495, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 496, 465, 539, 546, 502, 505, 554, 460, 533, 523, 513, 511, 517, 498, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 500, 453, 514, 487, 523, 523, 528, 491, 532, 494, 508, 515, 535, 491, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 528, 530, 535, 505, 527, 501, 523, 523, 515, 536, 471, 479, 525, 509, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 532, 547, 487, 534, 526, 473, 525, 508, 491, 510, 503, 508, 502, 517, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8192, 8166, 8216, 8245, 8138, 8232, 8156] avg=8192.0000 max=8245 max_rank=4 min=8138 min_rank=5 imbalance_ratio=1.006470 max_load_reduction=57291 max_load_reduction_pct=0.874191 moved_to_echo=32771 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[494, 503, 501, 578, 514, 524, 538, 500, 542, 529, 512, 504, 537, 495, 511, 474, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 476, 539, 510, 492, 504, 521, 495, 507, 544, 526, 531, 543, 534, 521, 496, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 508, 539, 501, 519, 483, 488, 512, 501, 525, 505, 528, 536, 517, 503, 510, 11, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 490, 519, 506, 498, 505, 503, 497, 527, 528, 498, 444, 517, 507, 485, 522, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 512, 490, 486, 500, 478, 515, 501, 528, 524, 479, 524, 504, 504, 527, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 563, 536, 487, 484, 484, 504, 493, 464, 504, 538, 538, 544, 491, 507, 527, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 520, 510, 516, 528, 517, 514, 551, 492, 509, 516, 539, 512, 500, 500, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 527, 499, 473, 502, 557, 517, 505, 516, 532, 536, 527, 537, 522, 516, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8260, 8268, 8186, 8067, 8069, 8141, 8269, 8276] avg=8192.0000 max=8276 max_rank=7 min=8067 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32755 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[518, 475, 506, 496, 501, 524, 475, 515, 532, 521, 529, 522, 509, 544, 527, 503, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 526, 540, 529, 468, 531, 523, 488, 524, 518, 516, 526, 504, 538, 494, 503, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 515, 563, 525, 542, 512, 483, 510, 514, 535, 536, 503, 523, 539, 501, 457, 11, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 528, 515, 474, 471, 511, 490, 484, 545, 504, 499, 513, 523, 555, 505, 493, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 517, 495, 519, 528, 528, 463, 513, 485, 502, 503, 497, 498, 567, 525, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 534, 470, 503, 548, 541, 514, 528, 491, 499, 501, 501, 511, 498, 514, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 495, 516, 498, 507, 528, 516, 516, 482, 533, 485, 485, 515, 537, 520, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 503, 490, 547, 496, 482, 528, 502, 555, 501, 509, 476, 490, 522, 537, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8272, 8289, 8146, 8151, 8157, 8149, 8171] avg=8192.0000 max=8289 max_rank=2 min=8146 min_rank=3 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32628 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[494, 517, 528, 503, 517, 517, 511, 533, 509, 504, 534, 512, 498, 514, 487, 512, 7, 4, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 492, 482, 547, 483, 515, 491, 557, 506, 503, 496, 521, 507, 506, 533, 473, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 524, 532, 516, 495, 524, 487, 496, 501, 525, 512, 512, 507, 480, 545, 484, 15, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 512, 513, 518, 531, 459, 515, 505, 522, 503, 534, 521, 505, 524, 492, 545, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 531, 558, 498, 479, 546, 520, 530, 502, 510, 526, 510, 536, 537, 513, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 508, 472, 515, 485, 559, 514, 501, 506, 473, 465, 496, 530, 478, 516, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 531, 527, 513, 492, 510, 531, 528, 533, 550, 494, 486, 495, 500, 527, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 506, 547, 494, 503, 485, 534, 529, 518, 509, 489, 500, 542, 535, 489, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8140, 8171, 8249, 8348, 8034, 8199, 8188] avg=8192.0000 max=8348 max_rank=4 min=8034 min_rank=5 imbalance_ratio=1.019043 max_load_reduction=57188 max_load_reduction_pct=0.872620 moved_to_echo=32769 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[523, 540, 488, 492, 494, 493, 559, 518, 496, 475, 496, 530, 503, 509, 480, 499, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 509, 520, 511, 489, 541, 497, 485, 517, 529, 502, 536, 480, 532, 523, 541, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 533, 485, 522, 479, 501, 505, 519, 485, 541, 502, 494, 511, 508, 482, 503, 11, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 530, 500, 479, 477, 519, 484, 508, 510, 506, 521, 487, 531, 515, 528, 536, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 459, 516, 537, 496, 500, 522, 515, 534, 548, 488, 531, 513, 503, 512, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 494, 506, 497, 532, 497, 512, 502, 497, 525, 533, 536, 519, 515, 564, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 487, 547, 546, 517, 524, 521, 494, 521, 516, 518, 512, 556, 491, 511, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 520, 539, 536, 508, 526, 528, 519, 510, 503, 511, 540, 528, 477, 475, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8099, 8240, 8071, 8155, 8209, 8262, 8235, 8265] avg=8192.0000 max=8265 max_rank=7 min=8071 min_rank=2 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32971 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[521, 523, 524, 475, 462, 516, 500, 481, 542, 513, 521, 515, 502, 529, 531, 532, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 492, 520, 583, 496, 519, 505, 502, 498, 515, 449, 527, 539, 504, 503, 518, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 505, 493, 525, 542, 508, 505, 517, 476, 506, 519, 514, 497, 505, 526, 507, 12, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 488, 462, 508, 558, 522, 498, 494, 541, 494, 535, 545, 538, 517, 532, 495, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 496, 465, 539, 546, 502, 505, 554, 460, 533, 523, 513, 511, 517, 498, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 500, 453, 514, 487, 523, 523, 528, 491, 532, 494, 508, 515, 535, 491, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 528, 530, 535, 505, 527, 501, 523, 523, 515, 536, 471, 479, 525, 509, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 532, 547, 487, 534, 526, 473, 525, 508, 491, 510, 503, 508, 502, 517, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8192, 8166, 8216, 8245, 8138, 8232, 8156] avg=8192.0000 max=8245 max_rank=4 min=8138 min_rank=5 imbalance_ratio=1.006470 max_load_reduction=57291 max_load_reduction_pct=0.874191 moved_to_echo=32771 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[502, 509, 519, 488, 504, 493, 517, 519, 546, 531, 473, 499, 505, 494, 548, 482, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 471, 540, 546, 532, 503, 492, 506, 541, 512, 509, 523, 520, 495, 491, 507, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 515, 504, 522, 489, 544, 498, 493, 496, 496, 522, 538, 522, 510, 541, 520, 12, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 494, 532, 485, 511, 495, 493, 520, 499, 536, 527, 495, 505, 518, 522, 520, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 498, 487, 525, 535, 493, 481, 512, 499, 498, 521, 503, 520, 495, 538, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 511, 500, 506, 523, 529, 529, 561, 494, 525, 514, 518, 527, 516, 534, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 465, 467, 515, 527, 520, 531, 512, 533, 527, 512, 507, 535, 494, 522, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 495, 521, 487, 471, 521, 508, 519, 502, 525, 473, 547, 483, 517, 487, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8133, 8218, 8235, 8223, 8101, 8324, 8227, 8075] avg=8192.0000 max=8324 max_rank=5 min=8075 min_rank=7 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32727 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[545, 533, 540, 500, 489, 522, 483, 501, 469, 511, 495, 513, 491, 540, 525, 473, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 478, 494, 540, 500, 483, 512, 510, 547, 504, 510, 530, 504, 504, 513, 512, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 516, 507, 534, 492, 492, 528, 496, 450, 533, 537, 516, 482, 520, 513, 520, 12, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 516, 511, 549, 530, 519, 517, 529, 516, 503, 531, 484, 509, 479, 514, 479, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 510, 478, 493, 516, 518, 527, 537, 531, 537, 517, 553, 481, 533, 509, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 542, 499, 517, 504, 476, 528, 518, 492, 529, 528, 502, 492, 553, 493, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 496, 507, 502, 473, 535, 532, 480, 491, 501, 516, 488, 530, 490, 554, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 489, 497, 517, 522, 542, 535, 506, 523, 474, 536, 522, 512, 508, 523, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8134, 8199, 8185, 8251, 8233, 8164, 8123, 8247] avg=8192.0000 max=8251 max_rank=3 min=8123 min_rank=6 imbalance_ratio=1.007202 max_load_reduction=57285 max_load_reduction_pct=0.874100 moved_to_echo=32767 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[518, 475, 506, 496, 501, 524, 475, 515, 532, 521, 529, 522, 509, 544, 527, 503, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 526, 540, 529, 468, 531, 523, 488, 524, 518, 516, 526, 504, 538, 494, 503, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 515, 563, 525, 542, 512, 483, 510, 514, 535, 536, 503, 523, 539, 501, 457, 11, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 528, 515, 474, 471, 511, 490, 484, 545, 504, 499, 513, 523, 555, 505, 493, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 517, 495, 519, 528, 528, 463, 513, 485, 502, 503, 497, 498, 567, 525, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 534, 470, 503, 548, 541, 514, 528, 491, 499, 501, 501, 511, 498, 514, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 495, 516, 498, 507, 528, 516, 516, 482, 533, 485, 485, 515, 537, 520, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 503, 490, 547, 496, 482, 528, 502, 555, 501, 509, 476, 490, 522, 537, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8272, 8289, 8146, 8151, 8157, 8149, 8171] avg=8192.0000 max=8289 max_rank=2 min=8146 min_rank=3 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32628 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 3968, 4224, 4352, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 3968, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 3968, 4224, 4352, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 3968, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 3968, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[531, 491, 497, 538, 537, 498, 519, 490, 526, 479, 496, 457, 525, 492, 494, 538, 495, 473, 543, 512, 564, 489, 496, 510, 519, 545, 511, 500, 502, 569, 518, 538, 461, 488, 515, 524, 504, 523, 539, 557, 553, 507, 504, 507, 507, 518, 521, 520, 497, 509, 525, 488, 530, 516, 519, 515, 509, 489, 526, 492, 511, 548, 524, 498, 525, 542, 557, 514, 515, 497, 532, 534, 499, 522, 518, 511, 463, 516, 495, 487, 496, 500, 499, 494, 506, 508, 473, 497, 481, 513, 493, 498, 544, 521, 535, 523, 524, 518, 517, 506, 476, 509, 489, 514, 464, 519, 503, 506, 508, 475, 513, 503, 540, 483, 531, 508, 552, 553, 511, 527, 565, 540, 496, 532, 505, 508, 498, 499] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[482, 558, 505, 500, 534, 524, 497, 507, 531, 529, 506, 498, 511, 521, 535, 518, 513, 512, 518, 501, 533, 518, 492, 518, 517, 496, 500, 518, 504, 511, 487, 498, 525, 485, 511, 515, 493, 515, 536, 530, 560, 521, 542, 512, 505, 530, 494, 527, 485, 476, 508, 527, 525, 520, 483, 501, 532, 539, 506, 512, 485, 541, 513, 497, 502, 514, 570, 521, 573, 509, 497, 526, 470, 490, 493, 508, 513, 526, 509, 496, 531, 483, 530, 526, 479, 525, 521, 528, 501, 478, 491, 515, 522, 492, 493, 555, 524, 508, 530, 529, 506, 512, 484, 459, 528, 519, 511, 507, 521, 493, 518, 529, 481, 534, 521, 500, 544, 499, 505, 484, 500, 510, 474, 512, 489, 529, 503, 543] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[576, 520, 524, 506, 519, 523, 481, 559, 518, 522, 522, 474, 494, 509, 502, 555, 504, 517, 510, 528, 521, 510, 517, 470, 504, 484, 508, 511, 493, 529, 461, 490, 515, 540, 519, 496, 500, 545, 518, 526, 528, 529, 483, 540, 509, 508, 499, 513, 540, 505, 473, 533, 532, 509, 515, 522, 504, 543, 487, 495, 536, 521, 549, 521, 472, 542, 489, 502, 475, 521, 478, 483, 455, 524, 493, 520, 518, 499, 504, 497, 504, 538, 541, 496, 491, 550, 558, 525, 535, 495, 483, 470, 529, 500, 498, 517, 515, 499, 535, 526, 547, 507, 502, 537, 495, 511, 526, 467, 536, 546, 528, 524, 487, 526, 553, 563, 526, 507, 504, 508, 491, 518, 479, 499, 498, 483, 475, 502] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[521, 511, 531, 508, 476, 502, 547, 505, 521, 527, 545, 520, 537, 519, 553, 504, 513, 529, 496, 515, 475, 524, 518, 522, 511, 510, 526, 527, 534, 536, 498, 473, 536, 537, 533, 496, 498, 475, 567, 526, 493, 489, 477, 511, 480, 480, 513, 545, 528, 497, 466, 502, 506, 497, 440, 500, 453, 504, 506, 508, 489, 476, 530, 516, 542, 523, 489, 511, 509, 468, 529, 486, 519, 505, 498, 528, 524, 559, 519, 508, 523, 536, 502, 515, 509, 536, 497, 519, 490, 473, 517, 523, 505, 537, 509, 516, 497, 518, 539, 492, 503, 510, 487, 507, 533, 523, 512, 519, 490, 489, 553, 523, 513, 505, 486, 513, 521, 539, 523, 513, 507, 498, 501, 572, 529, 526, 531, 532] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[508, 542, 545, 517, 536, 532, 513, 533, 505, 524, 498, 501, 521, 474, 499, 522, 523, 526, 484, 496, 477, 502, 523, 514, 518, 521, 493, 517, 468, 482, 508, 506, 536, 535, 502, 483, 503, 569, 520, 505, 499, 506, 491, 523, 495, 514, 523, 512, 485, 531, 503, 497, 528, 469, 495, 512, 494, 473, 510, 524, 517, 532, 507, 498, 491, 511, 544, 482, 530, 511, 508, 525, 499, 528, 488, 494, 520, 529, 485, 487, 542, 521, 516, 517, 536, 511, 509, 523, 540, 540, 529, 544, 532, 507, 525, 548, 488, 532, 524, 528, 480, 496, 551, 485, 521, 508, 528, 537, 489, 497, 503, 509, 483, 507, 496, 548, 553, 527, 536, 497, 506, 488, 525, 509, 491, 502, 523, 478] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[502, 525, 512, 522, 494, 503, 470, 514, 502, 501, 509, 520, 514, 517, 484, 521, 542, 510, 528, 497, 498, 505, 520, 542, 536, 529, 537, 492, 528, 547, 498, 486, 487, 550, 542, 496, 539, 533, 520, 511, 537, 502, 494, 533, 546, 489, 541, 484, 512, 497, 462, 514, 539, 481, 500, 528, 544, 542, 520, 504, 494, 528, 519, 549, 545, 510, 509, 521, 501, 500, 492, 561, 465, 525, 493, 524, 549, 495, 498, 535, 475, 516, 478, 492, 513, 501, 500, 492, 470, 493, 500, 550, 543, 512, 499, 508, 500, 540, 510, 530, 526, 512, 526, 481, 554, 514, 523, 509, 471, 498, 490, 481, 542, 517, 503, 481, 502, 539, 529, 484, 512, 532, 516, 485, 509, 517, 485, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[549, 499, 548, 517, 513, 460, 487, 517, 487, 505, 486, 509, 525, 486, 524, 513, 521, 507, 512, 508, 500, 493, 525, 537, 524, 514, 533, 533, 507, 517, 543, 497, 538, 517, 506, 564, 506, 539, 535, 510, 523, 527, 507, 513, 514, 480, 517, 530, 512, 538, 524, 509, 509, 548, 514, 543, 517, 462, 515, 482, 507, 528, 544, 514, 467, 493, 541, 515, 506, 515, 508, 472, 514, 507, 517, 514, 498, 505, 471, 509, 493, 479, 500, 492, 516, 513, 516, 496, 497, 536, 508, 522, 487, 531, 574, 498, 539, 496, 483, 466, 487, 531, 509, 469, 512, 529, 501, 540, 464, 510, 525, 513, 499, 522, 506, 514, 546, 529, 493, 492, 523, 519, 549, 541, 512, 485, 495, 539] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[576, 520, 524, 506, 519, 523, 481, 559, 518, 522, 522, 474, 494, 509, 502, 555, 504, 517, 510, 528, 521, 510, 517, 470, 504, 484, 508, 511, 493, 529, 461, 490, 515, 540, 519, 496, 500, 545, 518, 526, 528, 529, 483, 540, 509, 508, 499, 513, 540, 505, 473, 533, 532, 509, 515, 522, 504, 543, 487, 495, 536, 521, 549, 521, 472, 542, 489, 502, 475, 521, 478, 483, 455, 524, 493, 520, 518, 499, 504, 497, 504, 538, 541, 496, 491, 550, 558, 525, 535, 495, 483, 470, 529, 500, 498, 517, 515, 499, 535, 526, 547, 507, 502, 537, 495, 511, 526, 467, 536, 546, 528, 524, 487, 526, 553, 563, 526, 507, 504, 508, 491, 518, 479, 499, 498, 483, 475, 502] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[508, 542, 545, 517, 536, 532, 513, 533, 505, 524, 498, 501, 521, 474, 499, 522, 523, 526, 484, 496, 477, 502, 523, 514, 518, 521, 493, 517, 468, 482, 508, 506, 536, 535, 502, 483, 503, 569, 520, 505, 499, 506, 491, 523, 495, 514, 523, 512, 485, 531, 503, 497, 528, 469, 495, 512, 494, 473, 510, 524, 517, 532, 507, 498, 491, 511, 544, 482, 530, 511, 508, 525, 499, 528, 488, 494, 520, 529, 485, 487, 542, 521, 516, 517, 536, 511, 509, 523, 540, 540, 529, 544, 532, 507, 525, 548, 488, 532, 524, 528, 480, 496, 551, 485, 521, 508, 528, 537, 489, 497, 503, 509, 483, 507, 496, 548, 553, 527, 536, 497, 506, 488, 525, 509, 491, 502, 523, 478] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[518, 507, 503, 516, 502, 497, 517, 510, 540, 499, 536, 482, 514, 505, 520, 531, 576, 532, 472, 516, 480, 498, 539, 480, 505, 540, 538, 549, 502, 497, 485, 514, 516, 508, 498, 531, 562, 528, 528, 525, 517, 536, 506, 525, 495, 511, 513, 501, 486, 504, 486, 507, 511, 503, 523, 510, 502, 495, 497, 531, 521, 528, 503, 487, 519, 516, 518, 550, 524, 511, 512, 466, 499, 473, 481, 510, 532, 515, 537, 529, 529, 492, 503, 512, 506, 506, 534, 504, 556, 520, 489, 498, 525, 463, 516, 514, 524, 483, 549, 510, 485, 505, 497, 526, 531, 505, 546, 467, 473, 530, 511, 490, 500, 522, 500, 475, 492, 523, 494, 562, 518, 511, 497, 504, 514, 508, 534, 577] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[531, 491, 497, 538, 537, 498, 519, 490, 526, 479, 496, 457, 525, 492, 494, 538, 495, 473, 543, 512, 564, 489, 496, 510, 519, 545, 511, 500, 502, 569, 518, 538, 461, 488, 515, 524, 504, 523, 539, 557, 553, 507, 504, 507, 507, 518, 521, 520, 497, 509, 525, 488, 530, 516, 519, 515, 509, 489, 526, 492, 511, 548, 524, 498, 525, 542, 557, 514, 515, 497, 532, 534, 499, 522, 518, 511, 463, 516, 495, 487, 496, 500, 499, 494, 506, 508, 473, 497, 481, 513, 493, 498, 544, 521, 535, 523, 524, 518, 517, 506, 476, 509, 489, 514, 464, 519, 503, 506, 508, 475, 513, 503, 540, 483, 531, 508, 552, 553, 511, 527, 565, 540, 496, 532, 505, 508, 498, 499] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[482, 558, 505, 500, 534, 524, 497, 507, 531, 529, 506, 498, 511, 521, 535, 518, 513, 512, 518, 501, 533, 518, 492, 518, 517, 496, 500, 518, 504, 511, 487, 498, 525, 485, 511, 515, 493, 515, 536, 530, 560, 521, 542, 512, 505, 530, 494, 527, 485, 476, 508, 527, 525, 520, 483, 501, 532, 539, 506, 512, 485, 541, 513, 497, 502, 514, 570, 521, 573, 509, 497, 526, 470, 490, 493, 508, 513, 526, 509, 496, 531, 483, 530, 526, 479, 525, 521, 528, 501, 478, 491, 515, 522, 492, 493, 555, 524, 508, 530, 529, 506, 512, 484, 459, 528, 519, 511, 507, 521, 493, 518, 529, 481, 534, 521, 500, 544, 499, 505, 484, 500, 510, 474, 512, 489, 529, 503, 543] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[508, 542, 545, 517, 536, 532, 513, 533, 505, 524, 498, 501, 521, 474, 499, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 526, 484, 496, 477, 502, 523, 514, 518, 521, 493, 517, 468, 482, 508, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 535, 502, 483, 503, 569, 450, 505, 499, 506, 491, 523, 495, 514, 523, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 531, 503, 497, 528, 469, 495, 512, 494, 473, 510, 524, 517, 532, 507, 498, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 511, 544, 482, 530, 511, 508, 525, 499, 528, 488, 494, 520, 529, 485, 487, 32, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 521, 516, 517, 536, 511, 509, 523, 540, 540, 529, 544, 532, 507, 525, 548, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 532, 524, 528, 480, 496, 551, 485, 521, 508, 528, 537, 489, 497, 503, 509, 20, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 507, 496, 548, 528, 527, 536, 497, 506, 488, 525, 509, 491, 502, 523, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8251, 8058, 8146, 8113, 8169, 8445, 8210, 8144] avg=8192.0000 max=8445 max_rank=5 min=8058 min_rank=1 imbalance_ratio=1.030884 max_load_reduction=57091 max_load_reduction_pct=0.871140 moved_to_echo=32968 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[521, 511, 531, 508, 476, 502, 547, 505, 521, 527, 545, 520, 537, 519, 553, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 529, 496, 515, 475, 524, 518, 522, 511, 510, 526, 527, 534, 536, 498, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 537, 533, 496, 498, 475, 490, 526, 493, 489, 477, 511, 480, 480, 513, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 497, 466, 502, 506, 497, 440, 500, 453, 504, 506, 508, 489, 476, 530, 516, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 523, 489, 511, 509, 468, 529, 486, 519, 505, 498, 528, 524, 559, 519, 508, 35, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 536, 502, 515, 509, 536, 497, 519, 490, 473, 517, 523, 505, 537, 509, 516, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 518, 539, 492, 503, 510, 487, 507, 533, 523, 512, 519, 490, 489, 553, 523, 19, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 505, 486, 513, 498, 539, 523, 513, 507, 498, 501, 572, 529, 526, 531, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8310, 8207, 8079, 7960, 8256, 8211, 8227, 8286] avg=8192.0000 max=8310 max_rank=0 min=7960 min_rank=3 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32980 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[482, 558, 505, 500, 534, 524, 497, 507, 531, 529, 506, 498, 511, 521, 535, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 512, 518, 501, 533, 518, 492, 518, 517, 496, 500, 518, 504, 511, 487, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 485, 511, 515, 493, 515, 463, 530, 560, 521, 542, 512, 505, 530, 494, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 476, 508, 527, 525, 520, 483, 501, 532, 539, 506, 512, 485, 541, 513, 497, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 514, 570, 521, 573, 509, 497, 526, 470, 490, 493, 508, 513, 526, 509, 496, 33, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 483, 530, 526, 479, 525, 521, 528, 501, 478, 491, 515, 522, 492, 493, 555, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 508, 530, 529, 506, 512, 484, 459, 528, 519, 511, 507, 521, 493, 518, 529, 20, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 534, 521, 500, 519, 499, 505, 484, 500, 510, 474, 512, 489, 529, 503, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8237, 8136, 8228, 8190, 8255, 8175, 8212, 8103] avg=8192.0000 max=8255 max_rank=4 min=8103 min_rank=7 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32745 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[531, 491, 497, 538, 537, 498, 519, 490, 526, 479, 496, 457, 525, 492, 494, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 473, 543, 512, 564, 489, 496, 510, 519, 545, 511, 500, 502, 569, 518, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 488, 515, 524, 504, 523, 466, 557, 553, 507, 504, 507, 507, 518, 521, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 509, 525, 488, 530, 516, 519, 515, 509, 489, 526, 492, 511, 548, 524, 498, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 542, 557, 514, 515, 497, 532, 534, 499, 522, 518, 511, 463, 516, 495, 487, 33, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 500, 499, 494, 506, 508, 473, 497, 481, 513, 493, 498, 544, 521, 535, 523, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 518, 517, 506, 476, 509, 489, 514, 464, 519, 503, 506, 508, 475, 513, 503, 20, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 483, 531, 508, 527, 553, 511, 527, 565, 540, 496, 532, 505, 508, 498, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8089, 8284, 8175, 8236, 8265, 8086, 8078, 8323] avg=8192.0000 max=8323 max_rank=7 min=8078 min_rank=6 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32752 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[502, 525, 512, 522, 494, 503, 470, 514, 502, 501, 509, 520, 514, 517, 484, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 510, 528, 497, 498, 505, 520, 542, 536, 529, 537, 492, 528, 547, 498, 486, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 550, 542, 496, 539, 533, 444, 511, 537, 502, 494, 533, 546, 489, 541, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 497, 462, 514, 539, 481, 500, 528, 544, 542, 520, 504, 494, 528, 519, 549, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 510, 509, 521, 501, 500, 492, 561, 465, 525, 493, 524, 549, 495, 498, 535, 34, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 516, 478, 492, 513, 501, 500, 492, 470, 493, 500, 550, 543, 512, 499, 508, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 540, 510, 530, 526, 512, 526, 481, 554, 514, 523, 509, 471, 498, 490, 481, 20, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 517, 503, 481, 474, 539, 529, 484, 512, 532, 516, 485, 509, 517, 485, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8081, 8300, 8228, 8275, 8265, 8049, 8202, 8136] avg=8192.0000 max=8300 max_rank=1 min=8049 min_rank=5 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32652 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[576, 520, 524, 506, 519, 523, 481, 559, 518, 522, 522, 474, 494, 509, 502, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 517, 510, 528, 521, 510, 517, 470, 504, 484, 508, 511, 493, 529, 461, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 540, 519, 496, 500, 545, 448, 526, 528, 529, 483, 540, 509, 508, 499, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 505, 473, 533, 532, 509, 515, 522, 504, 543, 487, 495, 536, 521, 549, 521, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 542, 489, 502, 475, 521, 478, 483, 455, 524, 493, 520, 518, 499, 504, 497, 32, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 538, 541, 496, 491, 550, 558, 525, 535, 495, 483, 470, 529, 500, 498, 517, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 499, 535, 526, 547, 507, 502, 537, 495, 511, 526, 467, 536, 546, 528, 524, 19, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 526, 553, 563, 503, 507, 504, 508, 491, 518, 479, 499, 498, 483, 475, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8284, 8057, 8198, 8323, 8008, 8235, 8335, 8096] avg=8192.0000 max=8335 max_rank=6 min=8008 min_rank=4 imbalance_ratio=1.017456 max_load_reduction=57201 max_load_reduction_pct=0.872818 moved_to_echo=32674 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[549, 499, 548, 517, 513, 460, 487, 517, 487, 505, 486, 509, 525, 486, 524, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 507, 512, 508, 500, 493, 525, 537, 524, 514, 533, 533, 507, 517, 543, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 517, 506, 564, 506, 539, 462, 510, 523, 527, 507, 513, 514, 480, 517, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 538, 524, 509, 509, 548, 514, 543, 517, 462, 515, 482, 507, 528, 544, 514, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 493, 541, 515, 506, 515, 508, 472, 514, 507, 517, 514, 498, 505, 471, 509, 33, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 479, 500, 492, 516, 513, 516, 496, 497, 536, 508, 522, 487, 531, 574, 498, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 496, 483, 466, 487, 531, 509, 469, 512, 529, 501, 540, 464, 510, 525, 513, 20, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 522, 506, 514, 521, 529, 493, 492, 523, 519, 549, 541, 512, 485, 495, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8106, 8271, 8253, 8306, 8090, 8163, 8108, 8239] avg=8192.0000 max=8306 max_rank=3 min=8090 min_rank=4 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32600 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[531, 491, 497, 538, 537, 498, 519, 490, 526, 479, 496, 457, 525, 492, 494, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 473, 543, 512, 564, 489, 496, 510, 519, 545, 511, 500, 502, 569, 518, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 488, 515, 524, 504, 523, 466, 557, 553, 507, 504, 507, 507, 518, 521, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 509, 525, 488, 530, 516, 519, 515, 509, 489, 526, 492, 511, 548, 524, 498, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 542, 557, 514, 515, 497, 532, 534, 499, 522, 518, 511, 463, 516, 495, 487, 33, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 500, 499, 494, 506, 508, 473, 497, 481, 513, 493, 498, 544, 521, 535, 523, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 518, 517, 506, 476, 509, 489, 514, 464, 519, 503, 506, 508, 475, 513, 503, 20, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 483, 531, 508, 527, 553, 511, 527, 565, 540, 496, 532, 505, 508, 498, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8089, 8284, 8175, 8236, 8265, 8086, 8078, 8323] avg=8192.0000 max=8323 max_rank=7 min=8078 min_rank=6 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32752 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[576, 520, 524, 506, 519, 523, 481, 559, 518, 522, 522, 474, 494, 509, 502, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 517, 510, 528, 521, 510, 517, 470, 504, 484, 508, 511, 493, 529, 461, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 540, 519, 496, 500, 545, 448, 526, 528, 529, 483, 540, 509, 508, 499, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 505, 473, 533, 532, 509, 515, 522, 504, 543, 487, 495, 536, 521, 549, 521, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 542, 489, 502, 475, 521, 478, 483, 455, 524, 493, 520, 518, 499, 504, 497, 32, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 538, 541, 496, 491, 550, 558, 525, 535, 495, 483, 470, 529, 500, 498, 517, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 499, 535, 526, 547, 507, 502, 537, 495, 511, 526, 467, 536, 546, 528, 524, 19, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 526, 553, 563, 503, 507, 504, 508, 491, 518, 479, 499, 498, 483, 475, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8284, 8057, 8198, 8323, 8008, 8235, 8335, 8096] avg=8192.0000 max=8335 max_rank=6 min=8008 min_rank=4 imbalance_ratio=1.017456 max_load_reduction=57201 max_load_reduction_pct=0.872818 moved_to_echo=32674 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[518, 507, 503, 516, 502, 497, 517, 510, 540, 499, 536, 482, 514, 505, 520, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 576, 532, 472, 516, 480, 498, 539, 480, 505, 540, 538, 549, 502, 497, 485, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 508, 498, 531, 562, 528, 457, 525, 517, 536, 506, 525, 495, 511, 513, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 504, 486, 507, 511, 503, 523, 510, 502, 495, 497, 531, 521, 528, 503, 487, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 516, 518, 550, 524, 511, 512, 466, 499, 473, 481, 510, 532, 515, 537, 529, 32, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 492, 503, 512, 506, 506, 534, 504, 556, 520, 489, 498, 525, 463, 516, 514, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 483, 549, 510, 485, 505, 497, 526, 531, 505, 546, 467, 473, 530, 511, 490, 18, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 522, 500, 475, 470, 523, 494, 562, 518, 511, 497, 504, 514, 508, 534, 577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8223, 8229, 8133, 8228, 8172, 8164, 8209] avg=8192.0000 max=8229 max_rank=2 min=8133 min_rank=3 imbalance_ratio=1.004517 max_load_reduction=57307 max_load_reduction_pct=0.874435 moved_to_echo=32773 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[482, 558, 505, 500, 534, 524, 497, 507, 531, 529, 506, 498, 511, 521, 535, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 512, 518, 501, 533, 518, 492, 518, 517, 496, 500, 518, 504, 511, 487, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 485, 511, 515, 493, 515, 463, 530, 560, 521, 542, 512, 505, 530, 494, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 476, 508, 527, 525, 520, 483, 501, 532, 539, 506, 512, 485, 541, 513, 497, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 514, 570, 521, 573, 509, 497, 526, 470, 490, 493, 508, 513, 526, 509, 496, 33, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 483, 530, 526, 479, 525, 521, 528, 501, 478, 491, 515, 522, 492, 493, 555, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 508, 530, 529, 506, 512, 484, 459, 528, 519, 511, 507, 521, 493, 518, 529, 20, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 534, 521, 500, 519, 499, 505, 484, 500, 510, 474, 512, 489, 529, 503, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8237, 8136, 8228, 8190, 8255, 8175, 8212, 8103] avg=8192.0000 max=8255 max_rank=4 min=8103 min_rank=7 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32745 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[508, 542, 545, 517, 536, 532, 513, 533, 505, 524, 498, 501, 521, 474, 499, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 526, 484, 496, 477, 502, 523, 514, 518, 521, 493, 517, 468, 482, 508, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 535, 502, 483, 503, 569, 450, 505, 499, 506, 491, 523, 495, 514, 523, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 531, 503, 497, 528, 469, 495, 512, 494, 473, 510, 524, 517, 532, 507, 498, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 511, 544, 482, 530, 511, 508, 525, 499, 528, 488, 494, 520, 529, 485, 487, 32, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 521, 516, 517, 536, 511, 509, 523, 540, 540, 529, 544, 532, 507, 525, 548, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 532, 524, 528, 480, 496, 551, 485, 521, 508, 528, 537, 489, 497, 503, 509, 20, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 507, 496, 548, 528, 527, 536, 497, 506, 488, 525, 509, 491, 502, 523, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8251, 8058, 8146, 8113, 8169, 8445, 8210, 8144] avg=8192.0000 max=8445 max_rank=5 min=8058 min_rank=1 imbalance_ratio=1.030884 max_load_reduction=57091 max_load_reduction_pct=0.871140 moved_to_echo=32968 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[521, 511, 531, 508, 476, 502, 547, 505, 521, 527, 545, 520, 537, 519, 553, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 529, 496, 515, 475, 524, 518, 522, 511, 510, 526, 527, 534, 536, 498, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 537, 533, 496, 498, 475, 490, 526, 493, 489, 477, 511, 480, 480, 513, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 497, 466, 502, 506, 497, 440, 500, 453, 504, 506, 508, 489, 476, 530, 516, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 523, 489, 511, 509, 468, 529, 486, 519, 505, 498, 528, 524, 559, 519, 508, 35, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 536, 502, 515, 509, 536, 497, 519, 490, 473, 517, 523, 505, 537, 509, 516, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 518, 539, 492, 503, 510, 487, 507, 533, 523, 512, 519, 490, 489, 553, 523, 19, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 505, 486, 513, 498, 539, 523, 513, 507, 498, 501, 572, 529, 526, 531, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8310, 8207, 8079, 7960, 8256, 8211, 8227, 8286] avg=8192.0000 max=8310 max_rank=0 min=7960 min_rank=3 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32980 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 3968, 4096, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4352, 3712, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4096, 3968, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 3968, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 3968, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 3968, 4096, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 3968, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4096, 3968, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[491, 512, 526, 493, 493, 505, 501, 526, 463, 519, 498, 479, 494, 497, 516, 472, 519, 528, 529, 538, 532, 527, 555, 476, 505, 551, 494, 510, 469, 528, 494, 506, 501, 506, 491, 516, 524, 535, 516, 522, 523, 491, 535, 515, 515, 513, 513, 519, 539, 518, 535, 529, 543, 482, 528, 511, 498, 501, 504, 501, 543, 504, 520, 534, 480, 525, 493, 509, 495, 510, 495, 508, 515, 526, 528, 504, 530, 532, 487, 557, 494, 491, 505, 505, 526, 502, 498, 562, 541, 490, 543, 551, 517, 552, 520, 494, 515, 501, 486, 518, 549, 523, 486, 479, 515, 513, 518, 524, 529, 520, 490, 525, 500, 536, 494, 511, 459, 520, 488, 492, 521, 489, 537, 520, 477, 552, 478, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[483, 535, 486, 512, 494, 528, 516, 535, 508, 531, 558, 478, 494, 464, 509, 521, 489, 537, 509, 516, 478, 504, 475, 539, 534, 494, 508, 526, 524, 510, 489, 526, 531, 535, 491, 537, 487, 483, 487, 543, 502, 528, 493, 474, 516, 495, 490, 521, 490, 517, 493, 517, 512, 517, 547, 527, 464, 500, 513, 530, 546, 529, 498, 472, 510, 472, 520, 516, 514, 504, 539, 507, 563, 529, 528, 484, 512, 511, 485, 523, 486, 533, 513, 512, 518, 488, 516, 489, 515, 526, 524, 492, 534, 480, 537, 491, 503, 525, 536, 497, 483, 498, 561, 536, 492, 516, 544, 518, 528, 524, 479, 505, 495, 515, 529, 541, 538, 522, 497, 512, 540, 501, 560, 519, 492, 515, 567, 482] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[509, 480, 512, 530, 549, 522, 526, 542, 527, 527, 460, 498, 477, 509, 498, 507, 498, 536, 477, 538, 528, 469, 538, 557, 501, 471, 507, 512, 519, 495, 510, 533, 520, 538, 484, 522, 485, 478, 524, 489, 490, 541, 488, 535, 524, 507, 503, 528, 502, 484, 553, 518, 508, 511, 517, 549, 484, 499, 503, 502, 521, 530, 519, 476, 514, 500, 491, 503, 525, 512, 535, 505, 526, 523, 470, 537, 533, 529, 525, 541, 504, 523, 521, 502, 498, 552, 524, 503, 506, 508, 508, 500, 523, 514, 532, 511, 474, 507, 539, 489, 487, 528, 563, 501, 483, 492, 509, 516, 520, 496, 526, 491, 515, 490, 506, 545, 488, 500, 479, 539, 505, 558, 510, 520, 549, 487, 527, 505] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[515, 541, 535, 501, 521, 578, 501, 503, 518, 505, 489, 513, 557, 506, 521, 504, 499, 481, 500, 519, 510, 555, 512, 504, 543, 517, 568, 508, 495, 535, 530, 497, 504, 502, 542, 533, 508, 521, 516, 571, 518, 514, 506, 524, 523, 492, 510, 495, 498, 500, 525, 502, 544, 499, 483, 513, 510, 506, 490, 486, 518, 515, 503, 494, 541, 491, 477, 526, 489, 517, 522, 472, 559, 497, 472, 543, 488, 509, 507, 529, 532, 515, 568, 496, 502, 469, 483, 499, 506, 483, 500, 500, 536, 505, 505, 473, 479, 487, 513, 537, 488, 510, 483, 530, 531, 520, 470, 536, 491, 512, 555, 534, 494, 501, 521, 524, 506, 536, 509, 517, 518, 498, 511, 505, 477, 519, 539, 528] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[524, 520, 485, 515, 566, 473, 496, 512, 494, 517, 536, 517, 541, 516, 520, 534, 519, 515, 534, 551, 564, 514, 529, 521, 540, 506, 490, 512, 527, 497, 502, 522, 523, 550, 510, 480, 529, 518, 501, 524, 495, 489, 486, 475, 528, 485, 486, 527, 520, 497, 504, 538, 516, 497, 496, 513, 485, 505, 494, 503, 507, 496, 500, 523, 474, 521, 490, 519, 502, 534, 524, 538, 499, 481, 517, 507, 476, 515, 521, 536, 493, 463, 518, 505, 517, 524, 553, 504, 512, 498, 510, 533, 527, 485, 511, 487, 492, 515, 500, 507, 495, 517, 508, 503, 504, 512, 506, 473, 571, 537, 556, 526, 492, 505, 523, 544, 501, 545, 525, 539, 501, 489, 543, 492, 501, 493, 508, 510] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[546, 474, 536, 525, 505, 503, 505, 537, 524, 562, 534, 500, 481, 503, 536, 515, 518, 507, 535, 502, 530, 528, 499, 524, 503, 500, 523, 514, 539, 527, 486, 469, 502, 514, 528, 517, 495, 526, 507, 474, 537, 504, 481, 513, 482, 522, 521, 510, 498, 537, 514, 569, 539, 496, 558, 552, 528, 504, 501, 507, 495, 509, 527, 508, 515, 477, 502, 517, 509, 521, 490, 532, 507, 513, 511, 498, 518, 522, 512, 517, 503, 543, 500, 515, 465, 504, 529, 506, 527, 527, 518, 535, 531, 494, 489, 519, 523, 512, 482, 507, 498, 473, 505, 522, 499, 497, 494, 509, 526, 500, 499, 518, 527, 510, 511, 519, 502, 480, 485, 483, 505, 543, 494, 523, 497, 523, 497, 542] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[532, 507, 525, 533, 531, 528, 525, 536, 495, 525, 491, 465, 516, 494, 533, 519, 532, 506, 520, 512, 477, 522, 504, 547, 469, 502, 507, 486, 496, 511, 511, 520, 531, 504, 475, 504, 526, 491, 498, 533, 510, 505, 516, 542, 535, 510, 520, 483, 510, 488, 506, 484, 533, 511, 551, 520, 500, 496, 523, 495, 492, 489, 508, 528, 500, 508, 542, 547, 484, 457, 500, 545, 526, 524, 501, 547, 523, 508, 507, 502, 508, 538, 496, 499, 495, 525, 559, 475, 503, 530, 552, 493, 531, 494, 506, 510, 515, 526, 510, 504, 518, 537, 535, 512, 502, 539, 507, 484, 556, 514, 516, 519, 526, 484, 506, 525, 506, 513, 513, 486, 483, 485, 504, 504, 544, 525, 500, 509] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[524, 520, 485, 515, 566, 473, 496, 512, 494, 517, 536, 517, 541, 516, 520, 534, 519, 515, 534, 551, 564, 514, 529, 521, 540, 506, 490, 512, 527, 497, 502, 522, 523, 550, 510, 480, 529, 518, 501, 524, 495, 489, 486, 475, 528, 485, 486, 527, 520, 497, 504, 538, 516, 497, 496, 513, 485, 505, 494, 503, 507, 496, 500, 523, 474, 521, 490, 519, 502, 534, 524, 538, 499, 481, 517, 507, 476, 515, 521, 536, 493, 463, 518, 505, 517, 524, 553, 504, 512, 498, 510, 533, 527, 485, 511, 487, 492, 515, 500, 507, 495, 517, 508, 503, 504, 512, 506, 473, 571, 537, 556, 526, 492, 505, 523, 544, 501, 545, 525, 539, 501, 489, 543, 492, 501, 493, 508, 510] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[491, 512, 526, 493, 493, 505, 501, 526, 463, 519, 498, 479, 494, 497, 516, 472, 519, 528, 529, 538, 532, 527, 555, 476, 505, 551, 494, 510, 469, 528, 494, 506, 501, 506, 491, 516, 524, 535, 516, 522, 523, 491, 535, 515, 515, 513, 513, 519, 539, 518, 535, 529, 543, 482, 528, 511, 498, 501, 504, 501, 543, 504, 520, 534, 480, 525, 493, 509, 495, 510, 495, 508, 515, 526, 528, 504, 530, 532, 487, 557, 494, 491, 505, 505, 526, 502, 498, 562, 541, 490, 543, 551, 517, 552, 520, 494, 515, 501, 486, 518, 549, 523, 486, 479, 515, 513, 518, 524, 529, 520, 490, 525, 500, 536, 494, 511, 459, 520, 488, 492, 521, 489, 537, 520, 477, 552, 478, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[535, 518, 500, 516, 496, 565, 501, 466, 489, 532, 517, 535, 528, 490, 495, 533, 511, 501, 580, 506, 554, 498, 489, 552, 509, 509, 528, 475, 568, 556, 528, 497, 509, 536, 529, 512, 516, 507, 548, 520, 524, 527, 532, 517, 471, 538, 475, 484, 498, 556, 504, 522, 537, 494, 518, 515, 515, 498, 481, 494, 499, 547, 501, 490, 509, 499, 523, 522, 493, 519, 519, 521, 467, 520, 483, 515, 511, 537, 507, 510, 516, 504, 526, 503, 530, 461, 525, 519, 520, 555, 478, 525, 484, 521, 492, 503, 527, 516, 527, 539, 494, 507, 506, 517, 479, 516, 494, 485, 514, 499, 541, 521, 495, 521, 501, 485, 476, 472, 557, 522, 505, 532, 524, 510, 479, 496, 491, 480] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[509, 480, 512, 530, 549, 522, 526, 542, 527, 515, 460, 498, 477, 509, 498, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 536, 477, 538, 528, 469, 538, 509, 501, 471, 507, 512, 519, 495, 510, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 538, 484, 522, 485, 478, 524, 489, 490, 541, 488, 535, 524, 507, 503, 528, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 484, 553, 518, 508, 511, 517, 549, 484, 499, 503, 502, 521, 530, 519, 476, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 500, 491, 503, 525, 512, 535, 505, 526, 523, 470, 537, 533, 529, 525, 541, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 523, 521, 502, 498, 552, 524, 503, 506, 508, 508, 500, 523, 514, 532, 511, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 507, 539, 489, 487, 528, 563, 501, 483, 492, 509, 516, 520, 496, 526, 491, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 490, 506, 545, 488, 500, 479, 539, 505, 558, 510, 520, 549, 487, 527, 505, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8141, 8167, 8185, 8272, 8237, 8126, 8247] avg=8192.0000 max=8272 max_rank=4 min=8126 min_rank=6 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32882 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[509, 480, 512, 530, 549, 522, 526, 542, 527, 527, 460, 498, 477, 509, 498, 507, 498, 536, 477, 538, 528, 469, 538, 557, 501, 471, 507, 512, 519, 495, 510, 533, 520, 538, 484, 522, 485, 478, 524, 489, 490, 541, 488, 535, 524, 507, 503, 528, 502, 484, 553, 518, 508, 511, 517, 549, 484, 499, 503, 502, 521, 530, 519, 476, 514, 500, 491, 503, 525, 512, 535, 505, 526, 523, 470, 537, 533, 529, 525, 541, 504, 523, 521, 502, 498, 552, 524, 503, 506, 508, 508, 500, 523, 514, 532, 511, 474, 507, 539, 489, 487, 528, 563, 501, 483, 492, 509, 516, 520, 496, 526, 491, 515, 490, 506, 545, 488, 500, 479, 539, 505, 558, 510, 520, 549, 487, 527, 505] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[515, 541, 535, 501, 521, 578, 501, 503, 518, 505, 489, 513, 557, 506, 521, 504, 499, 481, 500, 519, 510, 555, 512, 504, 543, 517, 568, 508, 495, 535, 530, 497, 504, 502, 542, 533, 508, 521, 516, 571, 518, 514, 506, 524, 523, 492, 510, 495, 498, 500, 525, 502, 544, 499, 483, 513, 510, 506, 490, 486, 518, 515, 503, 494, 541, 491, 477, 526, 489, 517, 522, 472, 559, 497, 472, 543, 488, 509, 507, 529, 532, 515, 568, 496, 502, 469, 483, 499, 506, 483, 500, 500, 536, 505, 505, 473, 479, 487, 513, 537, 488, 510, 483, 530, 531, 520, 470, 536, 491, 512, 555, 534, 494, 501, 521, 524, 506, 536, 509, 517, 518, 498, 511, 505, 477, 519, 539, 528] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[546, 474, 536, 525, 505, 503, 505, 537, 524, 549, 534, 500, 481, 503, 536, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 507, 535, 502, 530, 528, 499, 480, 503, 500, 523, 514, 539, 527, 486, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 514, 528, 517, 495, 526, 507, 474, 537, 504, 481, 513, 482, 522, 521, 510, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 537, 514, 569, 539, 496, 558, 552, 528, 504, 501, 507, 495, 509, 527, 508, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 477, 502, 517, 509, 521, 490, 532, 507, 513, 511, 498, 518, 522, 512, 517, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 543, 500, 515, 465, 504, 529, 506, 527, 527, 518, 535, 531, 494, 489, 519, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 512, 482, 507, 498, 473, 505, 522, 499, 497, 494, 509, 526, 500, 499, 518, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 510, 511, 519, 502, 480, 485, 483, 505, 543, 494, 523, 497, 523, 497, 542, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8273, 8160, 8143, 8351, 8164, 8213, 8069, 8163] avg=8192.0000 max=8351 max_rank=3 min=8069 min_rank=6 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32609 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[483, 535, 486, 512, 494, 528, 516, 535, 508, 510, 558, 478, 494, 464, 509, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 537, 509, 516, 478, 504, 475, 478, 534, 494, 508, 526, 524, 510, 489, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 535, 491, 537, 487, 483, 487, 543, 502, 528, 493, 474, 516, 495, 490, 521, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 517, 493, 517, 512, 517, 547, 527, 464, 500, 513, 530, 546, 529, 498, 472, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 472, 520, 516, 514, 504, 539, 507, 563, 529, 528, 484, 512, 511, 485, 523, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 533, 513, 512, 518, 488, 516, 489, 515, 526, 524, 492, 534, 480, 537, 491, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 525, 536, 497, 483, 498, 561, 536, 492, 516, 544, 518, 528, 524, 479, 505, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 515, 529, 541, 538, 522, 497, 512, 540, 501, 560, 519, 492, 515, 567, 482, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8131, 8097, 8127, 8184, 8222, 8170, 8252, 8353] avg=8192.0000 max=8353 max_rank=7 min=8097 min_rank=1 imbalance_ratio=1.019653 max_load_reduction=57183 max_load_reduction_pct=0.872543 moved_to_echo=32997 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[491, 512, 526, 493, 493, 505, 501, 526, 463, 507, 498, 479, 494, 497, 516, 472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 528, 529, 538, 532, 527, 555, 436, 505, 551, 494, 510, 469, 528, 494, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 506, 491, 516, 524, 535, 516, 522, 523, 491, 535, 515, 515, 513, 513, 519, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 518, 535, 529, 543, 482, 528, 511, 498, 501, 504, 501, 543, 504, 520, 534, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 525, 493, 509, 495, 510, 495, 508, 515, 526, 528, 504, 530, 532, 487, 557, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 491, 505, 505, 526, 502, 498, 562, 541, 490, 543, 551, 517, 552, 520, 494, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 501, 486, 518, 549, 523, 486, 479, 515, 513, 518, 524, 529, 520, 490, 525, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 536, 494, 511, 459, 520, 488, 492, 521, 489, 537, 520, 477, 552, 478, 515, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7973, 8221, 8244, 8298, 8197, 8298, 8196, 8109] avg=8192.0000 max=8298 max_rank=3 min=7973 min_rank=0 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32800 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[546, 474, 536, 525, 505, 503, 505, 537, 524, 549, 534, 500, 481, 503, 536, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 507, 535, 502, 530, 528, 499, 480, 503, 500, 523, 514, 539, 527, 486, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 514, 528, 517, 495, 526, 507, 474, 537, 504, 481, 513, 482, 522, 521, 510, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 537, 514, 569, 539, 496, 558, 552, 528, 504, 501, 507, 495, 509, 527, 508, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 477, 502, 517, 509, 521, 490, 532, 507, 513, 511, 498, 518, 522, 512, 517, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 543, 500, 515, 465, 504, 529, 506, 527, 527, 518, 535, 531, 494, 489, 519, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 512, 482, 507, 498, 473, 505, 522, 499, 497, 494, 509, 526, 500, 499, 518, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 510, 511, 519, 502, 480, 485, 483, 505, 543, 494, 523, 497, 523, 497, 542, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8273, 8160, 8143, 8351, 8164, 8213, 8069, 8163] avg=8192.0000 max=8351 max_rank=3 min=8069 min_rank=6 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32609 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[524, 520, 485, 515, 566, 473, 496, 512, 494, 505, 536, 517, 541, 516, 520, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 515, 534, 551, 564, 514, 529, 477, 540, 506, 490, 512, 527, 497, 502, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 550, 510, 480, 529, 518, 501, 524, 495, 489, 486, 475, 528, 485, 486, 527, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 497, 504, 538, 516, 497, 496, 513, 485, 505, 494, 503, 507, 496, 500, 523, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 521, 490, 519, 502, 534, 524, 538, 499, 481, 517, 507, 476, 515, 521, 536, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 463, 518, 505, 517, 524, 553, 504, 512, 498, 510, 533, 527, 485, 511, 487, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 515, 500, 507, 495, 517, 508, 503, 504, 512, 506, 473, 571, 537, 556, 526, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 505, 523, 544, 501, 545, 525, 539, 501, 489, 543, 492, 501, 493, 508, 510, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8254, 8299, 8116, 8103, 8157, 8147, 8227, 8233] avg=8192.0000 max=8299 max_rank=1 min=8103 min_rank=3 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32764 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[532, 507, 525, 533, 531, 528, 525, 536, 495, 513, 491, 465, 516, 494, 533, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 506, 520, 512, 477, 522, 504, 500, 469, 502, 507, 486, 496, 511, 511, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 504, 475, 504, 526, 491, 498, 533, 510, 505, 516, 542, 535, 510, 520, 483, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 488, 506, 484, 533, 511, 551, 520, 500, 496, 523, 495, 492, 489, 508, 528, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 508, 542, 547, 484, 457, 500, 545, 526, 524, 501, 547, 523, 508, 507, 502, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 538, 496, 499, 495, 525, 559, 475, 503, 530, 552, 493, 531, 494, 506, 510, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 526, 510, 504, 518, 537, 535, 512, 502, 539, 507, 484, 556, 514, 516, 519, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 484, 506, 525, 506, 513, 513, 486, 483, 485, 504, 504, 544, 525, 500, 509, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8243, 8075, 8194, 8143, 8224, 8222, 8299, 8136] avg=8192.0000 max=8299 max_rank=6 min=8075 min_rank=1 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32881 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[515, 541, 535, 501, 521, 578, 501, 503, 518, 494, 489, 513, 557, 506, 521, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 481, 500, 519, 510, 555, 512, 461, 543, 517, 568, 508, 495, 535, 530, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 502, 542, 533, 508, 521, 516, 571, 518, 514, 506, 524, 523, 492, 510, 495, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 500, 525, 502, 544, 499, 483, 513, 510, 506, 490, 486, 518, 515, 503, 494, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 491, 477, 526, 489, 517, 522, 472, 559, 497, 472, 543, 488, 509, 507, 529, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 515, 568, 496, 502, 469, 483, 499, 506, 483, 500, 500, 536, 505, 505, 473, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 487, 513, 537, 488, 510, 483, 530, 531, 520, 470, 536, 491, 512, 555, 534, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 501, 521, 524, 506, 536, 509, 517, 518, 498, 511, 505, 477, 519, 539, 528, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8297, 8230, 8289, 8094, 8142, 8079, 8180, 8225] avg=8192.0000 max=8297 max_rank=0 min=8079 min_rank=5 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32626 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[535, 518, 500, 516, 496, 565, 501, 466, 489, 520, 517, 535, 528, 490, 495, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 501, 580, 506, 554, 498, 489, 504, 509, 509, 528, 475, 568, 556, 528, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 536, 529, 512, 516, 507, 548, 520, 524, 527, 532, 517, 471, 538, 475, 484, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 556, 504, 522, 537, 494, 518, 515, 515, 498, 481, 494, 499, 547, 501, 490, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 499, 523, 522, 493, 519, 519, 521, 467, 520, 483, 515, 511, 537, 507, 510, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 504, 526, 503, 530, 461, 525, 519, 520, 555, 478, 525, 484, 521, 492, 503, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 516, 527, 539, 494, 507, 506, 517, 479, 516, 494, 485, 514, 499, 541, 521, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 521, 501, 485, 476, 472, 557, 522, 505, 532, 524, 510, 479, 496, 491, 480, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8313, 8256, 8178, 8158, 8170, 8187, 8070] avg=8192.0000 max=8313 max_rank=1 min=8070 min_rank=7 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32585 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[509, 480, 512, 530, 549, 522, 526, 542, 527, 515, 460, 498, 477, 509, 498, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 536, 477, 538, 528, 469, 538, 509, 501, 471, 507, 512, 519, 495, 510, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 538, 484, 522, 485, 478, 524, 489, 490, 541, 488, 535, 524, 507, 503, 528, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 484, 553, 518, 508, 511, 517, 549, 484, 499, 503, 502, 521, 530, 519, 476, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 500, 491, 503, 525, 512, 535, 505, 526, 523, 470, 537, 533, 529, 525, 541, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 523, 521, 502, 498, 552, 524, 503, 506, 508, 508, 500, 523, 514, 532, 511, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 507, 539, 489, 487, 528, 563, 501, 483, 492, 509, 516, 520, 496, 526, 491, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 490, 506, 545, 488, 500, 479, 539, 505, 558, 510, 520, 549, 487, 527, 505, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8141, 8167, 8185, 8272, 8237, 8126, 8247] avg=8192.0000 max=8272 max_rank=4 min=8126 min_rank=6 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32882 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[524, 520, 485, 515, 566, 473, 496, 512, 494, 505, 536, 517, 541, 516, 520, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 515, 534, 551, 564, 514, 529, 477, 540, 506, 490, 512, 527, 497, 502, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 550, 510, 480, 529, 518, 501, 524, 495, 489, 486, 475, 528, 485, 486, 527, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 497, 504, 538, 516, 497, 496, 513, 485, 505, 494, 503, 507, 496, 500, 523, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 521, 490, 519, 502, 534, 524, 538, 499, 481, 517, 507, 476, 515, 521, 536, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 463, 518, 505, 517, 524, 553, 504, 512, 498, 510, 533, 527, 485, 511, 487, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 515, 500, 507, 495, 517, 508, 503, 504, 512, 506, 473, 571, 537, 556, 526, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 505, 523, 544, 501, 545, 525, 539, 501, 489, 543, 492, 501, 493, 508, 510, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8254, 8299, 8116, 8103, 8157, 8147, 8227, 8233] avg=8192.0000 max=8299 max_rank=1 min=8103 min_rank=3 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32764 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[515, 541, 535, 501, 521, 578, 501, 503, 518, 494, 489, 513, 557, 506, 521, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 481, 500, 519, 510, 555, 512, 461, 543, 517, 568, 508, 495, 535, 530, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 502, 542, 533, 508, 521, 516, 571, 518, 514, 506, 524, 523, 492, 510, 495, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 500, 525, 502, 544, 499, 483, 513, 510, 506, 490, 486, 518, 515, 503, 494, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 491, 477, 526, 489, 517, 522, 472, 559, 497, 472, 543, 488, 509, 507, 529, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 515, 568, 496, 502, 469, 483, 499, 506, 483, 500, 500, 536, 505, 505, 473, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 487, 513, 537, 488, 510, 483, 530, 531, 520, 470, 536, 491, 512, 555, 534, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 501, 521, 524, 506, 536, 509, 517, 518, 498, 511, 505, 477, 519, 539, 528, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8297, 8230, 8289, 8094, 8142, 8079, 8180, 8225] avg=8192.0000 max=8297 max_rank=0 min=8079 min_rank=5 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32626 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4224, 4224, 3968, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4224, 4352, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4352, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4352, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4224, 4352, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[484, 517, 564, 505, 511, 556, 486, 482, 476, 506, 500, 545, 465, 515, 539, 533, 482, 506, 499, 493, 505, 478, 538, 483, 505, 541, 496, 494, 504, 505, 494, 538, 519, 492, 504, 506, 507, 519, 484, 536, 507, 535, 451, 516, 527, 511, 488, 517, 501, 501, 510, 538, 484, 496, 513, 548, 510, 531, 524, 524, 532, 503, 515, 539, 520, 491, 502, 553, 490, 479, 551, 533, 533, 512, 510, 502, 520, 508, 518, 461, 528, 536, 546, 547, 486, 521, 513, 510, 512, 543, 522, 528, 523, 504, 503, 530, 521, 517, 536, 514, 493, 528, 540, 495, 515, 505, 512, 535, 533, 500, 442, 500, 502, 507, 484, 491, 523, 472, 523, 537, 511, 509, 523, 516, 540, 497, 525, 522] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[488, 516, 522, 516, 528, 476, 500, 525, 501, 489, 505, 531, 522, 490, 500, 498, 550, 520, 532, 520, 513, 484, 510, 544, 516, 499, 514, 501, 477, 512, 520, 534, 486, 501, 544, 496, 533, 516, 514, 512, 481, 516, 507, 502, 513, 518, 527, 492, 513, 484, 507, 478, 513, 560, 514, 536, 516, 523, 534, 509, 536, 535, 531, 507, 496, 472, 520, 519, 455, 530, 526, 512, 521, 541, 474, 509, 522, 483, 537, 519, 499, 500, 498, 567, 560, 513, 531, 526, 510, 524, 519, 499, 491, 529, 535, 513, 522, 518, 536, 510, 481, 544, 545, 492, 497, 514, 545, 474, 500, 478, 519, 505, 557, 497, 488, 517, 538, 492, 477, 508, 502, 507, 526, 487, 500, 501, 514, 488] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[539, 527, 502, 537, 514, 483, 497, 547, 547, 510, 539, 509, 546, 522, 519, 532, 506, 553, 484, 507, 548, 495, 468, 514, 527, 509, 510, 497, 491, 509, 515, 467, 483, 499, 483, 483, 510, 495, 508, 472, 489, 523, 506, 524, 545, 534, 549, 537, 528, 471, 494, 496, 509, 544, 545, 521, 519, 508, 500, 492, 525, 526, 523, 492, 518, 515, 547, 480, 557, 490, 502, 517, 532, 535, 513, 515, 558, 476, 517, 505, 512, 508, 520, 525, 500, 496, 520, 518, 525, 497, 505, 531, 486, 502, 529, 474, 479, 510, 488, 490, 522, 510, 521, 540, 469, 502, 546, 513, 515, 495, 502, 534, 484, 519, 542, 495, 483, 539, 528, 475, 508, 507, 520, 504, 542, 529, 503, 494] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[511, 534, 516, 529, 579, 495, 510, 528, 523, 488, 483, 511, 529, 488, 495, 524, 499, 510, 461, 509, 491, 512, 487, 500, 544, 568, 506, 523, 518, 536, 476, 547, 486, 515, 491, 515, 533, 514, 530, 548, 536, 536, 533, 564, 528, 523, 541, 476, 541, 508, 530, 532, 480, 555, 481, 509, 528, 539, 501, 522, 519, 502, 533, 511, 481, 525, 480, 488, 518, 523, 480, 500, 496, 526, 536, 476, 515, 475, 512, 516, 516, 488, 514, 539, 555, 511, 498, 513, 523, 486, 505, 522, 511, 516, 548, 492, 490, 494, 501, 488, 521, 462, 480, 485, 455, 563, 494, 480, 509, 503, 502, 531, 503, 552, 534, 534, 521, 537, 501, 498, 500, 502, 524, 545, 529, 463, 474, 487] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[516, 537, 492, 521, 528, 523, 513, 530, 521, 497, 555, 531, 504, 508, 481, 480, 510, 495, 571, 492, 494, 524, 497, 531, 484, 466, 492, 488, 524, 508, 511, 549, 544, 532, 536, 545, 540, 512, 463, 494, 549, 517, 528, 514, 541, 515, 485, 512, 501, 546, 539, 539, 517, 467, 490, 500, 498, 484, 541, 525, 527, 511, 551, 488, 513, 510, 532, 500, 502, 483, 489, 489, 505, 498, 507, 558, 521, 490, 518, 501, 536, 493, 506, 510, 507, 497, 492, 547, 525, 492, 516, 514, 548, 519, 501, 506, 505, 489, 504, 528, 511, 518, 535, 539, 519, 535, 514, 491, 489, 482, 487, 468, 517, 467, 498, 495, 506, 480, 521, 509, 559, 507, 477, 530, 553, 564, 482, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[480, 497, 518, 482, 522, 496, 526, 515, 485, 542, 531, 498, 534, 519, 532, 523, 524, 535, 518, 499, 513, 510, 530, 521, 483, 534, 519, 494, 480, 563, 497, 544, 499, 499, 530, 502, 531, 493, 532, 541, 562, 518, 522, 493, 512, 507, 524, 512, 537, 505, 522, 529, 526, 476, 520, 497, 541, 522, 504, 522, 487, 519, 493, 518, 486, 513, 514, 515, 495, 519, 512, 555, 481, 495, 509, 511, 512, 481, 525, 478, 549, 526, 497, 500, 489, 530, 526, 523, 492, 506, 494, 541, 508, 498, 532, 473, 520, 498, 531, 516, 492, 512, 487, 526, 550, 524, 483, 505, 512, 499, 504, 446, 474, 508, 562, 481, 511, 508, 503, 538, 532, 508, 516, 466, 535, 513, 513, 519] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[539, 527, 502, 537, 514, 483, 497, 547, 547, 510, 539, 509, 546, 522, 519, 532, 506, 553, 484, 507, 548, 495, 468, 514, 527, 509, 510, 497, 491, 509, 515, 467, 483, 499, 483, 483, 510, 495, 508, 472, 489, 523, 506, 524, 545, 534, 549, 537, 528, 471, 494, 496, 509, 544, 545, 521, 519, 508, 500, 492, 525, 526, 523, 492, 518, 515, 547, 480, 557, 490, 502, 517, 532, 535, 513, 515, 558, 476, 517, 505, 512, 508, 520, 525, 500, 496, 520, 518, 525, 497, 505, 531, 486, 502, 529, 474, 479, 510, 488, 490, 522, 510, 521, 540, 469, 502, 546, 513, 515, 495, 502, 534, 484, 519, 542, 495, 483, 539, 528, 475, 508, 507, 520, 504, 542, 529, 503, 494] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[525, 511, 538, 497, 537, 485, 501, 524, 498, 503, 524, 507, 526, 499, 531, 495, 566, 540, 505, 495, 527, 526, 503, 521, 506, 516, 532, 497, 497, 522, 538, 477, 526, 556, 521, 529, 479, 515, 509, 511, 495, 514, 541, 495, 504, 506, 548, 516, 518, 511, 510, 495, 544, 506, 507, 521, 524, 506, 525, 482, 488, 502, 539, 490, 536, 526, 503, 530, 510, 507, 494, 545, 503, 506, 540, 522, 468, 501, 539, 528, 499, 485, 496, 496, 496, 503, 560, 458, 530, 529, 534, 510, 526, 527, 488, 552, 497, 501, 490, 534, 514, 482, 483, 532, 529, 513, 494, 496, 491, 508, 517, 475, 556, 494, 474, 515, 513, 513, 486, 500, 538, 501, 501, 494, 528, 487, 532, 499] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[511, 534, 516, 529, 579, 495, 510, 528, 523, 488, 483, 511, 529, 488, 495, 524, 499, 510, 461, 509, 491, 512, 487, 500, 544, 568, 506, 523, 518, 536, 476, 547, 486, 515, 491, 515, 533, 514, 530, 548, 536, 536, 533, 564, 528, 523, 541, 476, 541, 508, 530, 532, 480, 555, 481, 509, 528, 539, 501, 522, 519, 502, 533, 511, 481, 525, 480, 488, 518, 523, 480, 500, 496, 526, 536, 476, 515, 475, 512, 516, 516, 488, 514, 539, 555, 511, 498, 513, 523, 486, 505, 522, 511, 516, 548, 492, 490, 494, 501, 488, 521, 462, 480, 485, 455, 563, 494, 480, 509, 503, 502, 531, 503, 552, 534, 534, 521, 537, 501, 498, 500, 502, 524, 545, 529, 463, 474, 487] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[517, 495, 485, 494, 520, 502, 484, 474, 523, 542, 532, 505, 527, 525, 490, 499, 495, 538, 548, 511, 499, 473, 512, 522, 539, 468, 533, 531, 513, 516, 517, 500, 504, 513, 510, 514, 576, 496, 526, 525, 521, 492, 531, 549, 514, 493, 489, 513, 532, 499, 490, 484, 478, 497, 543, 554, 537, 561, 535, 503, 502, 467, 523, 502, 519, 510, 500, 512, 499, 491, 516, 520, 521, 472, 527, 501, 531, 493, 500, 512, 538, 520, 465, 526, 526, 523, 491, 514, 501, 496, 485, 478, 547, 519, 475, 515, 581, 506, 491, 491, 476, 537, 520, 481, 506, 513, 509, 511, 504, 545, 505, 515, 501, 532, 543, 549, 507, 518, 484, 511, 486, 544, 512, 546, 509, 514, 517, 527] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[488, 516, 522, 516, 528, 476, 500, 525, 501, 489, 505, 531, 522, 490, 500, 498, 550, 520, 532, 520, 513, 484, 510, 544, 516, 499, 514, 501, 477, 512, 520, 534, 486, 501, 544, 496, 533, 516, 514, 512, 481, 516, 507, 502, 513, 518, 527, 492, 513, 484, 507, 478, 513, 560, 514, 536, 516, 523, 534, 509, 536, 535, 531, 507, 496, 472, 520, 519, 455, 530, 526, 512, 521, 541, 474, 509, 522, 483, 537, 519, 499, 500, 498, 567, 560, 513, 531, 526, 510, 524, 519, 499, 491, 529, 535, 513, 522, 518, 536, 510, 481, 544, 545, 492, 497, 514, 545, 474, 500, 478, 519, 505, 557, 497, 488, 517, 538, 492, 477, 508, 502, 507, 526, 487, 500, 501, 514, 488] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[480, 497, 518, 482, 508, 496, 526, 515, 485, 542, 531, 498, 534, 519, 532, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 535, 518, 499, 513, 510, 530, 521, 483, 534, 519, 494, 480, 563, 497, 544, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 499, 530, 502, 483, 493, 532, 541, 562, 518, 522, 493, 512, 507, 524, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 505, 522, 529, 526, 476, 520, 497, 541, 522, 504, 522, 487, 519, 455, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 513, 514, 515, 495, 519, 512, 555, 481, 495, 509, 511, 512, 481, 525, 478, 22, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 526, 497, 475, 489, 530, 526, 523, 492, 506, 494, 541, 508, 498, 532, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 498, 531, 516, 492, 512, 487, 526, 550, 524, 483, 505, 512, 499, 504, 446, 48, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 508, 562, 481, 511, 508, 503, 538, 532, 508, 516, 466, 535, 513, 513, 519, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8271, 8229, 8180, 8135, 8159, 8179, 8197] avg=8192.0000 max=8271 max_rank=1 min=8135 min_rank=4 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32670 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[484, 517, 564, 505, 497, 556, 486, 482, 476, 506, 500, 545, 465, 515, 539, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 506, 499, 493, 505, 478, 538, 483, 505, 541, 496, 494, 504, 505, 494, 538, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 492, 504, 506, 461, 519, 484, 536, 507, 535, 451, 516, 527, 511, 488, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 501, 510, 538, 484, 496, 513, 548, 510, 531, 524, 524, 532, 503, 476, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 491, 502, 553, 490, 479, 551, 533, 533, 512, 510, 502, 520, 508, 518, 461, 24, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 536, 546, 520, 486, 521, 513, 510, 512, 543, 522, 528, 523, 504, 503, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 517, 536, 514, 493, 528, 540, 495, 515, 505, 512, 535, 533, 500, 442, 500, 46, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 507, 484, 491, 523, 472, 523, 537, 511, 509, 523, 516, 540, 497, 525, 522, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8170, 8068, 8073, 8230, 8219, 8325, 8259, 8192] avg=8192.0000 max=8325 max_rank=5 min=8068 min_rank=1 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32995 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[525, 511, 538, 497, 523, 485, 501, 524, 498, 503, 524, 507, 526, 499, 531, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 566, 540, 505, 495, 527, 526, 503, 521, 506, 516, 532, 497, 497, 522, 538, 477, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 556, 521, 529, 436, 515, 509, 511, 495, 514, 541, 495, 504, 506, 548, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 511, 510, 495, 544, 506, 507, 521, 524, 506, 525, 482, 488, 502, 498, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 526, 503, 530, 510, 507, 494, 545, 503, 506, 540, 522, 468, 501, 539, 528, 22, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 485, 496, 471, 496, 503, 560, 458, 530, 529, 534, 510, 526, 527, 488, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 501, 490, 534, 514, 482, 483, 532, 529, 513, 494, 496, 491, 508, 517, 475, 43, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 494, 474, 515, 513, 513, 486, 500, 538, 501, 501, 494, 528, 487, 532, 499, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8275, 8222, 8127, 8293, 8164, 8127, 8141] avg=8192.0000 max=8293 max_rank=4 min=8127 min_rank=3 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32725 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[539, 527, 502, 537, 494, 483, 497, 547, 547, 510, 539, 509, 546, 522, 519, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 553, 484, 507, 548, 495, 468, 514, 527, 509, 510, 497, 491, 509, 515, 467, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 499, 483, 483, 459, 495, 508, 472, 489, 523, 506, 524, 545, 534, 549, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 471, 494, 496, 509, 544, 545, 521, 519, 508, 500, 492, 525, 526, 475, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 515, 547, 480, 557, 490, 502, 517, 532, 535, 513, 515, 558, 476, 517, 505, 27, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 508, 520, 492, 500, 496, 520, 518, 525, 497, 505, 531, 486, 502, 529, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 510, 488, 490, 522, 510, 521, 540, 469, 502, 546, 513, 515, 495, 502, 534, 51, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 519, 542, 495, 483, 539, 528, 475, 508, 507, 520, 504, 542, 529, 503, 494, 11, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8350, 8109, 8089, 8145, 8320, 8115, 8219, 8189] avg=8192.0000 max=8350 max_rank=0 min=8089 min_rank=2 imbalance_ratio=1.019287 max_load_reduction=57186 max_load_reduction_pct=0.872589 moved_to_echo=32843 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[516, 537, 492, 521, 514, 523, 513, 530, 521, 497, 555, 531, 504, 508, 481, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 495, 571, 492, 494, 524, 497, 531, 484, 466, 492, 488, 524, 508, 511, 549, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 532, 536, 545, 491, 512, 463, 494, 549, 517, 528, 514, 541, 515, 485, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 546, 539, 539, 517, 467, 490, 500, 498, 484, 541, 525, 527, 511, 509, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 510, 532, 500, 502, 483, 489, 489, 505, 498, 507, 558, 521, 490, 518, 501, 22, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 493, 506, 485, 507, 497, 492, 547, 525, 492, 516, 514, 548, 519, 501, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 489, 504, 528, 511, 518, 535, 539, 519, 535, 514, 491, 489, 482, 487, 468, 49, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 467, 498, 495, 506, 480, 521, 509, 559, 507, 477, 530, 553, 564, 482, 508, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8223, 8143, 8278, 8182, 8151, 8184, 8192, 8183] avg=8192.0000 max=8278 max_rank=2 min=8143 min_rank=1 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32710 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[517, 495, 485, 494, 506, 502, 484, 474, 523, 542, 532, 505, 527, 525, 490, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 538, 548, 511, 499, 473, 512, 522, 539, 468, 533, 531, 513, 516, 517, 500, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 513, 510, 514, 524, 496, 526, 525, 521, 492, 531, 549, 514, 493, 489, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 499, 490, 484, 478, 497, 543, 554, 537, 561, 535, 503, 502, 467, 483, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 510, 500, 512, 499, 491, 516, 520, 521, 472, 527, 501, 531, 493, 500, 512, 23, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 520, 465, 500, 526, 523, 491, 514, 501, 496, 485, 478, 547, 519, 475, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 581, 506, 491, 491, 476, 537, 520, 481, 506, 513, 509, 511, 504, 545, 505, 515, 52, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 532, 543, 549, 507, 518, 484, 511, 486, 544, 512, 546, 509, 514, 517, 527, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8100, 8222, 8214, 8167, 8160, 8093, 8270, 8310] avg=8192.0000 max=8310 max_rank=7 min=8093 min_rank=5 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32833 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[511, 534, 516, 529, 563, 495, 510, 528, 523, 488, 483, 511, 529, 488, 495, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 510, 461, 509, 491, 512, 487, 500, 544, 568, 506, 523, 518, 536, 476, 547, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 515, 491, 515, 485, 514, 530, 548, 536, 536, 533, 564, 528, 523, 541, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 508, 530, 532, 480, 555, 481, 509, 528, 539, 501, 522, 519, 502, 492, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 525, 480, 488, 518, 523, 480, 500, 496, 526, 536, 476, 515, 475, 512, 516, 24, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 488, 514, 512, 555, 511, 498, 513, 523, 486, 505, 522, 511, 516, 548, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 494, 501, 488, 521, 462, 480, 485, 455, 563, 494, 480, 509, 503, 502, 531, 48, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 552, 534, 534, 521, 537, 501, 498, 500, 502, 524, 545, 529, 463, 474, 487, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8227, 8195, 8321, 8250, 8084, 8210, 8034, 8215] avg=8192.0000 max=8321 max_rank=2 min=8034 min_rank=6 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32543 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[488, 516, 522, 516, 514, 476, 500, 525, 501, 489, 505, 531, 522, 490, 500, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 520, 532, 520, 513, 484, 510, 544, 516, 499, 514, 501, 477, 512, 520, 534, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 501, 544, 496, 485, 516, 514, 512, 481, 516, 507, 502, 513, 518, 527, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 484, 507, 478, 513, 560, 514, 536, 516, 523, 534, 509, 536, 535, 490, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 472, 520, 519, 455, 530, 526, 512, 521, 541, 474, 509, 522, 483, 537, 519, 25, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 500, 498, 539, 560, 513, 531, 526, 510, 524, 519, 499, 491, 529, 535, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 518, 536, 510, 481, 544, 545, 492, 497, 514, 545, 474, 500, 478, 519, 505, 48, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 557, 497, 488, 517, 538, 492, 477, 508, 502, 507, 526, 487, 500, 501, 514, 488, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8093, 8253, 8110, 8255, 8174, 8286, 8256, 8109] avg=8192.0000 max=8286 max_rank=5 min=8093 min_rank=0 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32825 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[525, 511, 538, 497, 523, 485, 501, 524, 498, 503, 524, 507, 526, 499, 531, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 566, 540, 505, 495, 527, 526, 503, 521, 506, 516, 532, 497, 497, 522, 538, 477, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 556, 521, 529, 436, 515, 509, 511, 495, 514, 541, 495, 504, 506, 548, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 511, 510, 495, 544, 506, 507, 521, 524, 506, 525, 482, 488, 502, 498, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 526, 503, 530, 510, 507, 494, 545, 503, 506, 540, 522, 468, 501, 539, 528, 22, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 485, 496, 471, 496, 503, 560, 458, 530, 529, 534, 510, 526, 527, 488, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 501, 490, 534, 514, 482, 483, 532, 529, 513, 494, 496, 491, 508, 517, 475, 43, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 494, 474, 515, 513, 513, 486, 500, 538, 501, 501, 494, 528, 487, 532, 499, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8275, 8222, 8127, 8293, 8164, 8127, 8141] avg=8192.0000 max=8293 max_rank=4 min=8127 min_rank=3 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32725 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[480, 497, 518, 482, 508, 496, 526, 515, 485, 542, 531, 498, 534, 519, 532, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 535, 518, 499, 513, 510, 530, 521, 483, 534, 519, 494, 480, 563, 497, 544, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 499, 530, 502, 483, 493, 532, 541, 562, 518, 522, 493, 512, 507, 524, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 505, 522, 529, 526, 476, 520, 497, 541, 522, 504, 522, 487, 519, 455, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 513, 514, 515, 495, 519, 512, 555, 481, 495, 509, 511, 512, 481, 525, 478, 22, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 526, 497, 475, 489, 530, 526, 523, 492, 506, 494, 541, 508, 498, 532, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 498, 531, 516, 492, 512, 487, 526, 550, 524, 483, 505, 512, 499, 504, 446, 48, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 508, 562, 481, 511, 508, 503, 538, 532, 508, 516, 466, 535, 513, 513, 519, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8271, 8229, 8180, 8135, 8159, 8179, 8197] avg=8192.0000 max=8271 max_rank=1 min=8135 min_rank=4 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32670 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[517, 495, 485, 494, 506, 502, 484, 474, 523, 542, 532, 505, 527, 525, 490, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 538, 548, 511, 499, 473, 512, 522, 539, 468, 533, 531, 513, 516, 517, 500, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 513, 510, 514, 524, 496, 526, 525, 521, 492, 531, 549, 514, 493, 489, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 499, 490, 484, 478, 497, 543, 554, 537, 561, 535, 503, 502, 467, 483, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 510, 500, 512, 499, 491, 516, 520, 521, 472, 527, 501, 531, 493, 500, 512, 23, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 520, 465, 500, 526, 523, 491, 514, 501, 496, 485, 478, 547, 519, 475, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 581, 506, 491, 491, 476, 537, 520, 481, 506, 513, 509, 511, 504, 545, 505, 515, 52, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 532, 543, 549, 507, 518, 484, 511, 486, 544, 512, 546, 509, 514, 517, 527, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8100, 8222, 8214, 8167, 8160, 8093, 8270, 8310] avg=8192.0000 max=8310 max_rank=7 min=8093 min_rank=5 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32833 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[516, 537, 492, 521, 514, 523, 513, 530, 521, 497, 555, 531, 504, 508, 481, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 495, 571, 492, 494, 524, 497, 531, 484, 466, 492, 488, 524, 508, 511, 549, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 532, 536, 545, 491, 512, 463, 494, 549, 517, 528, 514, 541, 515, 485, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 546, 539, 539, 517, 467, 490, 500, 498, 484, 541, 525, 527, 511, 509, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 510, 532, 500, 502, 483, 489, 489, 505, 498, 507, 558, 521, 490, 518, 501, 22, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 493, 506, 485, 507, 497, 492, 547, 525, 492, 516, 514, 548, 519, 501, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 489, 504, 528, 511, 518, 535, 539, 519, 535, 514, 491, 489, 482, 487, 468, 49, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 467, 498, 495, 506, 480, 521, 509, 559, 507, 477, 530, 553, 564, 482, 508, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8223, 8143, 8278, 8182, 8151, 8184, 8192, 8183] avg=8192.0000 max=8278 max_rank=2 min=8143 min_rank=1 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32710 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 3968, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4352, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 512, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4096, 3840, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4352, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 3968, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 3968, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[483, 495, 541, 560, 482, 524, 508, 522, 512, 480, 515, 513, 529, 516, 496, 549, 470, 493, 509, 533, 487, 501, 505, 507, 520, 493, 497, 534, 464, 481, 564, 491, 519, 520, 525, 498, 544, 501, 499, 513, 508, 538, 552, 521, 518, 510, 502, 489, 501, 498, 485, 547, 524, 525, 518, 499, 502, 493, 505, 508, 476, 532, 527, 516, 475, 487, 489, 482, 539, 520, 518, 513, 519, 493, 562, 540, 500, 526, 536, 546, 511, 552, 500, 519, 537, 507, 522, 552, 517, 538, 571, 521, 518, 492, 524, 510, 496, 518, 499, 524, 528, 495, 491, 483, 508, 487, 494, 507, 484, 491, 530, 493, 494, 515, 508, 482, 495, 507, 537, 516, 506, 496, 529, 500, 513, 542, 523, 522] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[523, 498, 491, 517, 488, 495, 497, 533, 500, 531, 501, 525, 504, 509, 515, 507, 490, 513, 467, 489, 537, 476, 534, 516, 489, 511, 538, 515, 490, 506, 497, 538, 513, 536, 486, 540, 508, 540, 487, 539, 479, 481, 527, 497, 507, 527, 508, 519, 515, 568, 544, 493, 555, 523, 491, 540, 467, 492, 507, 515, 479, 501, 504, 502, 546, 470, 521, 525, 487, 505, 470, 521, 507, 504, 528, 535, 517, 534, 483, 529, 577, 529, 456, 490, 500, 501, 510, 520, 488, 501, 515, 532, 508, 515, 530, 502, 538, 528, 542, 500, 527, 470, 514, 521, 481, 531, 566, 493, 502, 516, 554, 521, 509, 533, 531, 520, 524, 547, 528, 493, 518, 542, 467, 487, 523, 522, 484, 518] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[521, 493, 509, 496, 496, 469, 542, 505, 537, 544, 508, 502, 555, 522, 515, 514, 536, 536, 515, 500, 530, 508, 495, 495, 573, 546, 511, 483, 494, 532, 503, 486, 475, 457, 488, 504, 510, 507, 509, 519, 525, 495, 495, 517, 489, 521, 525, 477, 496, 494, 517, 558, 507, 499, 517, 526, 518, 503, 514, 519, 551, 503, 515, 519, 484, 494, 586, 515, 522, 507, 483, 525, 521, 533, 488, 547, 477, 518, 514, 494, 552, 485, 554, 500, 516, 511, 517, 527, 465, 505, 510, 573, 507, 523, 506, 532, 530, 475, 531, 499, 525, 532, 501, 499, 495, 513, 496, 550, 537, 504, 517, 537, 532, 501, 514, 510, 465, 528, 526, 486, 496, 527, 486, 468, 536, 481, 497, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[526, 524, 566, 532, 522, 506, 503, 535, 511, 478, 505, 475, 511, 505, 526, 542, 507, 558, 490, 541, 509, 514, 496, 507, 496, 550, 489, 558, 508, 495, 529, 516, 493, 492, 507, 515, 466, 513, 531, 552, 517, 498, 515, 510, 534, 569, 511, 506, 555, 526, 554, 492, 538, 519, 517, 493, 560, 488, 512, 543, 534, 550, 474, 507, 495, 522, 515, 450, 557, 528, 514, 476, 489, 512, 484, 515, 509, 519, 507, 489, 504, 503, 489, 513, 506, 508, 513, 475, 503, 462, 525, 537, 488, 509, 510, 493, 519, 533, 518, 491, 505, 533, 473, 531, 500, 528, 522, 487, 494, 484, 497, 510, 545, 475, 510, 503, 492, 451, 568, 490, 564, 520, 533, 525, 482, 493, 537, 483] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[498, 540, 527, 525, 514, 553, 518, 489, 501, 509, 496, 544, 463, 511, 538, 535, 522, 528, 440, 510, 549, 481, 488, 494, 489, 531, 502, 514, 485, 529, 477, 547, 500, 507, 516, 505, 501, 510, 554, 494, 560, 527, 531, 457, 496, 452, 483, 522, 528, 474, 513, 523, 482, 548, 508, 507, 532, 548, 528, 511, 523, 516, 528, 479, 513, 510, 535, 494, 504, 556, 528, 475, 525, 551, 547, 522, 551, 512, 510, 486, 529, 512, 515, 551, 512, 520, 513, 482, 493, 456, 522, 505, 533, 487, 502, 483, 510, 527, 541, 509, 552, 495, 529, 533, 480, 487, 537, 497, 524, 525, 485, 491, 504, 462, 520, 521, 487, 510, 542, 504, 490, 508, 536, 535, 488, 517, 546, 500] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[538, 556, 529, 455, 533, 543, 519, 487, 479, 578, 503, 523, 484, 536, 533, 511, 491, 509, 512, 529, 528, 524, 516, 514, 491, 488, 485, 525, 519, 502, 558, 528, 508, 513, 505, 536, 490, 507, 514, 472, 493, 492, 549, 544, 518, 535, 486, 520, 516, 523, 487, 501, 516, 513, 490, 545, 522, 537, 507, 492, 568, 497, 503, 513, 497, 498, 536, 510, 506, 452, 520, 551, 540, 471, 472, 498, 556, 492, 505, 503, 545, 509, 509, 508, 493, 504, 528, 501, 464, 520, 513, 519, 480, 508, 548, 526, 526, 477, 513, 558, 519, 501, 502, 525, 502, 483, 484, 484, 511, 529, 524, 525, 518, 488, 545, 507, 470, 519, 543, 485, 518, 509, 510, 506, 496, 504, 490, 545] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[526, 524, 566, 532, 522, 506, 503, 535, 511, 478, 505, 475, 511, 505, 526, 542, 507, 558, 490, 541, 509, 514, 496, 507, 496, 550, 489, 558, 508, 495, 529, 516, 493, 492, 507, 515, 466, 513, 531, 552, 517, 498, 515, 510, 534, 569, 511, 506, 555, 526, 554, 492, 538, 519, 517, 493, 560, 488, 512, 543, 534, 550, 474, 507, 495, 522, 515, 450, 557, 528, 514, 476, 489, 512, 484, 515, 509, 519, 507, 489, 504, 503, 489, 513, 506, 508, 513, 475, 503, 462, 525, 537, 488, 509, 510, 493, 519, 533, 518, 491, 505, 533, 473, 531, 500, 528, 522, 487, 494, 484, 497, 510, 545, 475, 510, 503, 492, 451, 568, 490, 564, 520, 533, 525, 482, 493, 537, 483] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[521, 493, 509, 496, 496, 469, 542, 505, 537, 544, 508, 502, 555, 522, 515, 514, 536, 536, 515, 500, 530, 508, 495, 495, 573, 546, 511, 483, 494, 532, 503, 486, 475, 457, 488, 504, 510, 507, 509, 519, 525, 495, 495, 517, 489, 521, 525, 477, 496, 494, 517, 558, 507, 499, 517, 526, 518, 503, 514, 519, 551, 503, 515, 519, 484, 494, 586, 515, 522, 507, 483, 525, 521, 533, 488, 547, 477, 518, 514, 494, 552, 485, 554, 500, 516, 511, 517, 527, 465, 505, 510, 573, 507, 523, 506, 532, 530, 475, 531, 499, 525, 532, 501, 499, 495, 513, 496, 550, 537, 504, 517, 537, 532, 501, 514, 510, 465, 528, 526, 486, 496, 527, 486, 468, 536, 481, 497, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[534, 526, 518, 525, 483, 543, 537, 527, 483, 487, 513, 543, 514, 506, 512, 530, 486, 549, 526, 504, 502, 523, 483, 541, 536, 546, 515, 528, 520, 487, 475, 546, 503, 545, 517, 479, 526, 513, 499, 458, 482, 570, 514, 478, 490, 546, 466, 506, 528, 506, 552, 493, 545, 471, 489, 526, 548, 500, 546, 496, 540, 491, 546, 512, 489, 525, 519, 543, 499, 513, 524, 499, 514, 521, 486, 502, 506, 511, 557, 541, 503, 520, 535, 509, 471, 464, 498, 553, 516, 476, 490, 473, 506, 481, 559, 492, 490, 507, 515, 519, 523, 517, 505, 498, 538, 519, 482, 478, 481, 487, 543, 520, 519, 503, 495, 509, 564, 535, 501, 541, 491, 502, 537, 477, 518, 509, 462, 527] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[483, 495, 541, 560, 482, 524, 508, 522, 512, 480, 515, 513, 529, 516, 496, 549, 470, 493, 509, 533, 487, 501, 505, 507, 520, 493, 497, 534, 464, 481, 564, 491, 519, 520, 525, 498, 544, 501, 499, 513, 508, 538, 552, 521, 518, 510, 502, 489, 501, 498, 485, 547, 524, 525, 518, 499, 502, 493, 505, 508, 476, 532, 527, 516, 475, 487, 489, 482, 539, 520, 518, 513, 519, 493, 562, 540, 500, 526, 536, 546, 511, 552, 500, 519, 537, 507, 522, 552, 517, 538, 571, 521, 518, 492, 524, 510, 496, 518, 499, 524, 528, 495, 491, 483, 508, 487, 494, 507, 484, 491, 530, 493, 494, 515, 508, 482, 495, 507, 537, 516, 506, 496, 529, 500, 513, 542, 523, 522] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[523, 498, 491, 517, 488, 495, 497, 533, 500, 531, 501, 525, 504, 509, 515, 507, 490, 513, 467, 489, 537, 476, 534, 516, 489, 511, 538, 515, 490, 506, 497, 538, 513, 536, 486, 540, 508, 540, 487, 539, 479, 481, 527, 497, 507, 527, 508, 519, 515, 568, 544, 493, 555, 523, 491, 540, 467, 492, 507, 515, 479, 501, 504, 502, 546, 470, 521, 525, 487, 505, 470, 521, 507, 504, 528, 535, 517, 534, 483, 529, 577, 529, 456, 490, 500, 501, 510, 520, 488, 501, 515, 532, 508, 515, 530, 502, 538, 528, 542, 500, 527, 470, 514, 521, 481, 531, 566, 493, 502, 516, 554, 521, 509, 533, 531, 520, 524, 547, 528, 493, 518, 542, 467, 487, 523, 522, 484, 518] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[467, 497, 494, 496, 506, 532, 553, 513, 537, 507, 515, 520, 480, 515, 472, 528, 483, 526, 500, 530, 516, 527, 508, 506, 489, 481, 506, 481, 491, 522, 480, 490, 508, 540, 533, 497, 515, 507, 499, 543, 505, 483, 533, 491, 533, 507, 547, 522, 489, 485, 511, 526, 521, 528, 470, 529, 524, 505, 540, 486, 497, 527, 479, 509, 514, 506, 509, 487, 521, 507, 512, 520, 488, 534, 470, 499, 514, 502, 491, 495, 515, 527, 556, 547, 542, 526, 541, 519, 487, 488, 516, 501, 474, 526, 519, 530, 513, 520, 541, 511, 498, 496, 530, 531, 506, 511, 493, 545, 459, 532, 492, 552, 502, 500, 542, 482, 515, 518, 510, 538, 514, 545, 502, 534, 503, 574, 530, 557] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[483, 495, 541, 560, 482, 524, 508, 522, 512, 480, 515, 513, 529, 516, 496, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 493, 509, 533, 487, 501, 505, 507, 520, 493, 497, 534, 464, 481, 564, 491, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 520, 525, 498, 544, 501, 499, 513, 508, 538, 552, 521, 518, 510, 502, 489, 17, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 498, 485, 547, 484, 525, 518, 499, 502, 493, 505, 508, 476, 532, 527, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 487, 489, 482, 539, 520, 518, 513, 519, 493, 562, 540, 500, 526, 536, 546, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 552, 500, 519, 537, 507, 522, 552, 517, 538, 571, 521, 518, 492, 524, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 518, 499, 524, 528, 495, 491, 483, 508, 487, 494, 507, 484, 491, 530, 493, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 515, 508, 482, 495, 507, 536, 516, 506, 496, 529, 500, 513, 542, 523, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8082, 8281, 8116, 8254, 8391, 8041, 8184] avg=8192.0000 max=8391 max_rank=5 min=8041 min_rank=6 imbalance_ratio=1.024292 max_load_reduction=57145 max_load_reduction_pct=0.871964 moved_to_echo=32870 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[538, 556, 529, 455, 533, 543, 519, 487, 479, 578, 503, 523, 484, 536, 533, 511, 491, 509, 512, 529, 528, 524, 516, 514, 491, 488, 485, 525, 519, 502, 558, 528, 508, 513, 505, 536, 490, 507, 514, 472, 493, 492, 549, 544, 518, 535, 486, 520, 516, 523, 487, 501, 516, 513, 490, 545, 522, 537, 507, 492, 568, 497, 503, 513, 497, 498, 536, 510, 506, 452, 520, 551, 540, 471, 472, 498, 556, 492, 505, 503, 545, 509, 509, 508, 493, 504, 528, 501, 464, 520, 513, 519, 480, 508, 548, 526, 526, 477, 513, 558, 519, 501, 502, 525, 502, 483, 484, 484, 511, 529, 524, 525, 518, 488, 545, 507, 470, 519, 543, 485, 518, 509, 510, 506, 496, 504, 490, 545] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[498, 540, 527, 525, 514, 553, 518, 489, 501, 509, 496, 544, 463, 511, 538, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 528, 440, 510, 549, 481, 488, 494, 489, 531, 502, 514, 485, 529, 477, 547, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 507, 516, 505, 501, 510, 554, 494, 560, 527, 531, 457, 496, 452, 483, 522, 21, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 474, 513, 523, 437, 548, 508, 507, 532, 548, 528, 511, 523, 516, 528, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 510, 535, 494, 504, 556, 528, 475, 525, 551, 547, 522, 551, 512, 510, 486, 11, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 512, 515, 551, 512, 520, 513, 482, 493, 456, 522, 505, 533, 487, 502, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 527, 541, 509, 552, 495, 529, 533, 480, 487, 537, 497, 524, 525, 485, 491, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 462, 520, 521, 487, 510, 540, 504, 490, 508, 536, 535, 488, 517, 546, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8120, 8147, 8203, 8332, 8115, 8238, 8168] avg=8192.0000 max=8332 max_rank=4 min=8115 min_rank=5 imbalance_ratio=1.017090 max_load_reduction=57204 max_load_reduction_pct=0.872864 moved_to_echo=32853 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[534, 526, 518, 525, 483, 543, 537, 527, 483, 487, 513, 543, 514, 506, 512, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 549, 526, 504, 502, 523, 483, 541, 536, 546, 515, 528, 520, 487, 475, 546, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 545, 517, 479, 526, 513, 499, 458, 482, 570, 514, 478, 490, 546, 466, 506, 16, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 506, 552, 493, 504, 471, 489, 526, 548, 500, 546, 496, 540, 491, 546, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 525, 519, 543, 499, 513, 524, 499, 514, 521, 486, 502, 506, 511, 557, 541, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 520, 535, 509, 471, 464, 498, 553, 516, 476, 490, 473, 506, 481, 559, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 507, 515, 519, 523, 517, 505, 498, 538, 519, 482, 478, 481, 487, 543, 520, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 503, 495, 509, 564, 535, 500, 541, 491, 502, 537, 477, 518, 509, 462, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 8301, 8115, 8248, 8258, 8046, 8134, 8189] avg=8192.0000 max=8301 max_rank=1 min=8046 min_rank=5 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32627 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[526, 524, 566, 532, 522, 506, 503, 535, 511, 478, 505, 475, 511, 505, 526, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 558, 490, 541, 509, 514, 496, 507, 496, 550, 489, 558, 508, 495, 529, 516, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 492, 507, 515, 466, 513, 531, 552, 517, 498, 515, 510, 534, 569, 511, 506, 16, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 526, 554, 492, 497, 519, 517, 493, 560, 488, 512, 543, 534, 550, 474, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 522, 515, 450, 557, 528, 514, 476, 489, 512, 484, 515, 509, 519, 507, 489, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 503, 489, 513, 506, 508, 513, 475, 503, 462, 525, 537, 488, 509, 510, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 533, 518, 491, 505, 533, 473, 531, 500, 528, 522, 487, 494, 484, 497, 510, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 475, 510, 503, 492, 451, 567, 490, 564, 520, 533, 525, 482, 493, 537, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8231, 8297, 8252, 8321, 8090, 8038, 8137, 8170] avg=8192.0000 max=8321 max_rank=3 min=8038 min_rank=5 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32435 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[523, 498, 491, 517, 488, 495, 497, 533, 500, 531, 501, 525, 504, 509, 515, 472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 513, 467, 489, 537, 476, 534, 516, 489, 511, 538, 515, 490, 506, 497, 538, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 536, 486, 540, 508, 540, 487, 539, 479, 481, 527, 497, 507, 527, 508, 519, 15, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 568, 544, 493, 513, 523, 491, 540, 467, 492, 507, 515, 479, 501, 504, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 470, 521, 525, 487, 505, 470, 521, 507, 504, 528, 535, 517, 534, 483, 529, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 577, 529, 456, 490, 500, 501, 510, 520, 488, 501, 515, 532, 508, 515, 530, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 528, 542, 500, 527, 470, 514, 521, 481, 531, 566, 493, 502, 516, 554, 521, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 533, 531, 520, 524, 547, 527, 493, 518, 542, 467, 487, 523, 522, 484, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8099, 8141, 8216, 8154, 8191, 8174, 8316, 8245] avg=8192.0000 max=8316 max_rank=6 min=8099 min_rank=0 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32926 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[498, 540, 527, 525, 514, 553, 518, 489, 501, 509, 496, 544, 463, 511, 538, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 528, 440, 510, 549, 481, 488, 494, 489, 531, 502, 514, 485, 529, 477, 547, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 507, 516, 505, 501, 510, 554, 494, 560, 527, 531, 457, 496, 452, 483, 522, 21, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 474, 513, 523, 437, 548, 508, 507, 532, 548, 528, 511, 523, 516, 528, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 510, 535, 494, 504, 556, 528, 475, 525, 551, 547, 522, 551, 512, 510, 486, 11, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 512, 515, 551, 512, 520, 513, 482, 493, 456, 522, 505, 533, 487, 502, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 527, 541, 509, 552, 495, 529, 533, 480, 487, 537, 497, 524, 525, 485, 491, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 462, 520, 521, 487, 510, 540, 504, 490, 508, 536, 535, 488, 517, 546, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8120, 8147, 8203, 8332, 8115, 8238, 8168] avg=8192.0000 max=8332 max_rank=4 min=8115 min_rank=5 imbalance_ratio=1.017090 max_load_reduction=57204 max_load_reduction_pct=0.872864 moved_to_echo=32853 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[521, 493, 509, 496, 496, 469, 542, 505, 537, 544, 508, 502, 555, 522, 515, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 536, 515, 500, 530, 508, 495, 495, 573, 546, 511, 483, 494, 532, 503, 486, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 457, 488, 504, 510, 507, 509, 519, 525, 495, 495, 517, 489, 521, 525, 477, 15, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 494, 517, 558, 468, 499, 517, 526, 518, 503, 514, 519, 551, 503, 515, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 494, 586, 515, 522, 507, 483, 525, 521, 533, 488, 547, 477, 518, 514, 494, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 485, 554, 500, 516, 511, 517, 527, 465, 505, 510, 573, 507, 523, 506, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 475, 531, 499, 525, 532, 501, 499, 495, 513, 496, 550, 537, 504, 517, 537, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 501, 514, 510, 465, 528, 525, 486, 496, 527, 486, 468, 536, 481, 497, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8275, 8035, 8217, 8217, 8283, 8253, 8063] avg=8192.0000 max=8283 max_rank=5 min=8035 min_rank=2 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32816 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[538, 556, 529, 455, 533, 543, 519, 487, 479, 578, 503, 523, 484, 536, 533, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 509, 512, 529, 528, 524, 516, 514, 491, 488, 485, 525, 519, 502, 558, 528, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 513, 505, 536, 490, 507, 514, 472, 493, 492, 549, 544, 518, 535, 486, 520, 15, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 523, 487, 501, 477, 513, 490, 545, 522, 537, 507, 492, 568, 497, 503, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 498, 536, 510, 506, 452, 520, 551, 540, 471, 472, 498, 556, 492, 505, 503, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 509, 509, 508, 493, 504, 528, 501, 464, 520, 513, 519, 480, 508, 548, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 477, 513, 558, 519, 501, 502, 525, 502, 483, 484, 484, 511, 529, 524, 525, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 488, 545, 507, 470, 519, 542, 485, 518, 509, 510, 506, 496, 504, 490, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8272, 8251, 8204, 8191, 8116, 8175, 8175, 8152] avg=8192.0000 max=8272 max_rank=0 min=8116 min_rank=4 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32618 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[521, 493, 509, 496, 496, 469, 542, 505, 537, 544, 508, 502, 555, 522, 515, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 536, 515, 500, 530, 508, 495, 495, 573, 546, 511, 483, 494, 532, 503, 486, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 457, 488, 504, 510, 507, 509, 519, 525, 495, 495, 517, 489, 521, 525, 477, 15, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 494, 517, 558, 468, 499, 517, 526, 518, 503, 514, 519, 551, 503, 515, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 494, 586, 515, 522, 507, 483, 525, 521, 533, 488, 547, 477, 518, 514, 494, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 485, 554, 500, 516, 511, 517, 527, 465, 505, 510, 573, 507, 523, 506, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 475, 531, 499, 525, 532, 501, 499, 495, 513, 496, 550, 537, 504, 517, 537, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 501, 514, 510, 465, 528, 525, 486, 496, 527, 486, 468, 536, 481, 497, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8275, 8035, 8217, 8217, 8283, 8253, 8063] avg=8192.0000 max=8283 max_rank=5 min=8035 min_rank=2 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32816 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[467, 497, 494, 496, 506, 532, 553, 513, 537, 507, 515, 520, 480, 515, 472, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 526, 500, 530, 516, 527, 508, 506, 489, 481, 506, 481, 491, 522, 480, 490, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 540, 533, 497, 515, 507, 499, 543, 505, 483, 533, 491, 533, 507, 547, 522, 16, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 485, 511, 526, 481, 528, 470, 529, 524, 505, 540, 486, 497, 527, 479, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 506, 509, 487, 521, 507, 512, 520, 488, 534, 470, 499, 514, 502, 491, 495, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 527, 556, 547, 542, 526, 541, 519, 487, 488, 516, 501, 474, 526, 519, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 520, 541, 511, 498, 496, 530, 531, 506, 511, 493, 545, 459, 532, 492, 552, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 500, 542, 482, 515, 518, 509, 538, 514, 545, 502, 534, 503, 574, 530, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8096, 8069, 8286, 8086, 8078, 8314, 8242, 8365] avg=8192.0000 max=8365 max_rank=7 min=8069 min_rank=1 imbalance_ratio=1.021118 max_load_reduction=57171 max_load_reduction_pct=0.872360 moved_to_echo=32999 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[534, 526, 518, 525, 483, 543, 537, 527, 483, 487, 513, 543, 514, 506, 512, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 549, 526, 504, 502, 523, 483, 541, 536, 546, 515, 528, 520, 487, 475, 546, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 545, 517, 479, 526, 513, 499, 458, 482, 570, 514, 478, 490, 546, 466, 506, 16, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 506, 552, 493, 504, 471, 489, 526, 548, 500, 546, 496, 540, 491, 546, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 525, 519, 543, 499, 513, 524, 499, 514, 521, 486, 502, 506, 511, 557, 541, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 520, 535, 509, 471, 464, 498, 553, 516, 476, 490, 473, 506, 481, 559, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 507, 515, 519, 523, 517, 505, 498, 538, 519, 482, 478, 481, 487, 543, 520, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 503, 495, 509, 564, 535, 500, 541, 491, 502, 537, 477, 518, 509, 462, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 8301, 8115, 8248, 8258, 8046, 8134, 8189] avg=8192.0000 max=8301 max_rank=1 min=8046 min_rank=5 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32627 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[483, 495, 541, 560, 482, 524, 508, 522, 512, 480, 515, 513, 529, 516, 496, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 493, 509, 533, 487, 501, 505, 507, 520, 493, 497, 534, 464, 481, 564, 491, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 520, 525, 498, 544, 501, 499, 513, 508, 538, 552, 521, 518, 510, 502, 489, 17, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 498, 485, 547, 484, 525, 518, 499, 502, 493, 505, 508, 476, 532, 527, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 487, 489, 482, 539, 520, 518, 513, 519, 493, 562, 540, 500, 526, 536, 546, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 552, 500, 519, 537, 507, 522, 552, 517, 538, 571, 521, 518, 492, 524, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 518, 499, 524, 528, 495, 491, 483, 508, 487, 494, 507, 484, 491, 530, 493, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 515, 508, 482, 495, 507, 536, 516, 506, 496, 529, 500, 513, 542, 523, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8082, 8281, 8116, 8254, 8391, 8041, 8184] avg=8192.0000 max=8391 max_rank=5 min=8041 min_rank=6 imbalance_ratio=1.024292 max_load_reduction=57145 max_load_reduction_pct=0.871964 moved_to_echo=32870 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4352, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 3968, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4352, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 3968, 3968, 4224, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4224, 3968, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4352, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 3968, 3968, 4224, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[460, 540, 525, 500, 524, 517, 456, 508, 525, 461, 543, 522, 495, 539, 485, 503, 506, 492, 536, 491, 542, 505, 534, 517, 460, 556, 490, 527, 516, 514, 547, 513, 479, 495, 518, 474, 517, 506, 535, 506, 496, 521, 537, 524, 532, 504, 498, 512, 518, 529, 504, 528, 543, 526, 533, 481, 524, 494, 540, 516, 488, 512, 509, 538, 528, 528, 508, 515, 519, 531, 516, 509, 546, 555, 491, 524, 521, 527, 508, 454, 515, 528, 507, 533, 519, 514, 531, 495, 485, 537, 502, 436, 511, 503, 459, 519, 461, 491, 514, 480, 522, 517, 526, 502, 501, 492, 511, 538, 560, 516, 525, 540, 525, 534, 535, 504, 518, 502, 504, 524, 548, 496, 494, 518, 505, 505, 496, 472] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[481, 519, 511, 533, 524, 476, 506, 502, 508, 489, 499, 518, 501, 493, 495, 545, 504, 480, 499, 546, 550, 503, 524, 487, 508, 514, 479, 529, 543, 520, 528, 493, 540, 523, 509, 481, 536, 512, 506, 509, 468, 522, 517, 517, 498, 504, 551, 474, 495, 488, 523, 496, 511, 482, 563, 519, 499, 479, 530, 482, 521, 538, 552, 507, 547, 533, 526, 515, 464, 500, 527, 501, 535, 473, 555, 479, 537, 520, 517, 523, 518, 471, 510, 532, 499, 539, 522, 495, 517, 510, 494, 502, 513, 543, 504, 522, 519, 482, 509, 495, 550, 536, 547, 542, 542, 476, 545, 494, 521, 525, 510, 511, 530, 501, 517, 524, 508, 458, 513, 486, 503, 498, 510, 551, 513, 509, 537, 472] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[460, 540, 525, 500, 524, 517, 456, 508, 525, 461, 543, 522, 495, 539, 485, 503, 506, 492, 536, 491, 542, 505, 534, 517, 460, 556, 490, 527, 516, 514, 547, 513, 479, 495, 518, 474, 517, 506, 535, 506, 496, 521, 537, 524, 532, 504, 498, 512, 518, 529, 504, 528, 543, 526, 533, 481, 524, 494, 540, 516, 488, 512, 509, 538, 528, 528, 508, 515, 519, 531, 516, 509, 546, 555, 491, 524, 521, 527, 508, 454, 515, 528, 507, 533, 519, 514, 531, 495, 485, 537, 502, 436, 511, 503, 459, 519, 461, 491, 514, 480, 522, 517, 526, 502, 501, 492, 511, 538, 560, 516, 525, 540, 525, 534, 535, 504, 518, 502, 504, 524, 548, 496, 494, 518, 505, 505, 496, 472] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[502, 519, 515, 539, 536, 496, 498, 516, 520, 498, 504, 506, 497, 507, 506, 525, 529, 496, 490, 505, 494, 547, 515, 510, 511, 484, 498, 469, 513, 515, 540, 529, 540, 505, 530, 528, 503, 506, 487, 479, 496, 507, 506, 481, 503, 537, 516, 554, 516, 541, 492, 487, 520, 497, 524, 500, 518, 505, 527, 504, 508, 522, 514, 481, 481, 519, 502, 531, 543, 543, 525, 513, 568, 508, 550, 517, 479, 517, 490, 529, 515, 501, 463, 513, 539, 515, 538, 504, 528, 525, 506, 528, 505, 503, 497, 530, 513, 523, 493, 500, 503, 512, 489, 545, 477, 520, 507, 495, 509, 492, 514, 454, 508, 486, 496, 538, 509, 543, 532, 524, 528, 514, 527, 513, 503, 529, 528, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[505, 501, 479, 508, 505, 510, 535, 553, 561, 518, 506, 502, 534, 527, 510, 508, 492, 530, 528, 497, 546, 511, 530, 523, 515, 498, 517, 492, 508, 482, 524, 515, 526, 521, 492, 534, 525, 565, 517, 484, 490, 494, 521, 514, 498, 510, 494, 489, 497, 517, 518, 526, 536, 526, 532, 494, 471, 511, 491, 494, 527, 538, 513, 544, 527, 513, 472, 468, 503, 488, 532, 525, 521, 517, 540, 515, 505, 516, 481, 523, 539, 509, 507, 518, 506, 500, 543, 551, 532, 472, 513, 550, 474, 526, 527, 543, 502, 527, 498, 545, 490, 503, 496, 494, 566, 508, 488, 507, 472, 525, 501, 517, 477, 509, 499, 524, 499, 546, 492, 503, 497, 511, 474, 509, 480, 475, 542, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[500, 517, 496, 512, 551, 506, 492, 509, 534, 533, 514, 499, 483, 528, 568, 543, 499, 508, 538, 539, 497, 522, 506, 549, 518, 474, 489, 515, 537, 449, 515, 510, 487, 487, 517, 542, 512, 513, 506, 501, 456, 510, 510, 494, 530, 539, 559, 485, 513, 518, 498, 525, 510, 510, 501, 501, 537, 518, 532, 511, 490, 530, 534, 554, 544, 520, 480, 513, 460, 477, 489, 486, 493, 557, 505, 524, 506, 473, 507, 522, 504, 534, 497, 501, 533, 492, 534, 473, 522, 494, 494, 531, 558, 503, 465, 523, 567, 494, 535, 523, 524, 474, 506, 493, 484, 505, 478, 501, 529, 519, 480, 558, 504, 536, 529, 507, 485, 495, 477, 529, 503, 517, 506, 533, 568, 506, 545, 532] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[528, 487, 530, 520, 488, 524, 507, 505, 531, 544, 530, 482, 487, 536, 492, 523, 528, 537, 514, 509, 521, 512, 550, 525, 499, 471, 526, 498, 523, 502, 503, 520, 490, 497, 526, 491, 553, 550, 532, 503, 510, 508, 538, 486, 490, 519, 525, 505, 488, 531, 504, 491, 528, 501, 447, 518, 536, 506, 525, 505, 502, 529, 477, 530, 502, 511, 513, 538, 468, 501, 514, 531, 537, 512, 507, 506, 515, 509, 503, 517, 487, 482, 459, 501, 509, 526, 497, 535, 511, 490, 481, 573, 549, 494, 524, 518, 534, 521, 543, 496, 513, 509, 489, 529, 502, 554, 506, 523, 524, 501, 541, 510, 504, 503, 519, 483, 486, 475, 486, 509, 528, 500, 509, 552, 492, 534, 496, 552] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[516, 522, 497, 538, 499, 495, 545, 500, 528, 519, 495, 486, 551, 506, 516, 493, 538, 474, 521, 481, 533, 502, 524, 517, 488, 521, 554, 531, 496, 576, 496, 494, 542, 485, 484, 534, 521, 499, 496, 492, 501, 550, 509, 509, 515, 507, 528, 521, 507, 507, 530, 487, 510, 475, 467, 490, 545, 517, 537, 490, 524, 474, 509, 508, 536, 491, 517, 495, 499, 475, 500, 519, 519, 531, 556, 537, 522, 493, 532, 505, 494, 500, 511, 517, 524, 519, 537, 528, 485, 512, 526, 506, 458, 568, 515, 501, 538, 517, 565, 493, 520, 529, 474, 500, 506, 463, 520, 542, 496, 498, 501, 522, 484, 509, 493, 513, 512, 529, 516, 507, 542, 509, 535, 524, 511, 487, 515, 516] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[500, 517, 496, 512, 551, 506, 492, 509, 534, 533, 514, 499, 483, 528, 568, 543, 499, 508, 538, 539, 497, 522, 506, 549, 518, 474, 489, 515, 537, 449, 515, 510, 487, 487, 517, 542, 512, 513, 506, 501, 456, 510, 510, 494, 530, 539, 559, 485, 513, 518, 498, 525, 510, 510, 501, 501, 537, 518, 532, 511, 490, 530, 534, 554, 544, 520, 480, 513, 460, 477, 489, 486, 493, 557, 505, 524, 506, 473, 507, 522, 504, 534, 497, 501, 533, 492, 534, 473, 522, 494, 494, 531, 558, 503, 465, 523, 567, 494, 535, 523, 524, 474, 506, 493, 484, 505, 478, 501, 529, 519, 480, 558, 504, 536, 529, 507, 485, 495, 477, 529, 503, 517, 506, 533, 568, 506, 545, 532] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[519, 504, 536, 522, 495, 518, 497, 485, 503, 515, 506, 484, 527, 526, 495, 523, 511, 541, 532, 491, 518, 528, 509, 515, 475, 508, 460, 551, 501, 515, 557, 514, 508, 495, 515, 511, 560, 521, 489, 529, 523, 550, 524, 502, 511, 500, 529, 531, 563, 511, 478, 543, 495, 537, 542, 493, 451, 521, 539, 501, 543, 542, 509, 529, 516, 557, 501, 514, 487, 564, 508, 487, 511, 497, 528, 526, 484, 516, 507, 540, 480, 502, 449, 520, 468, 526, 512, 525, 518, 495, 503, 474, 483, 534, 509, 490, 511, 526, 510, 503, 530, 494, 466, 527, 515, 489, 514, 517, 518, 504, 492, 537, 528, 490, 495, 537, 502, 537, 523, 497, 509, 515, 530, 515, 532, 491, 476, 499] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[502, 519, 515, 539, 536, 496, 498, 516, 520, 498, 504, 506, 497, 507, 506, 525, 529, 496, 490, 505, 494, 547, 515, 510, 511, 484, 498, 469, 513, 515, 540, 529, 540, 505, 530, 528, 503, 506, 487, 479, 496, 507, 506, 481, 503, 537, 516, 554, 516, 541, 492, 487, 520, 497, 524, 500, 518, 505, 527, 504, 508, 522, 514, 481, 481, 519, 502, 531, 543, 543, 525, 513, 568, 508, 550, 517, 479, 517, 490, 529, 515, 501, 463, 513, 539, 515, 538, 504, 528, 525, 506, 528, 505, 503, 497, 530, 513, 523, 493, 500, 503, 512, 489, 545, 477, 520, 507, 495, 509, 492, 514, 454, 508, 486, 496, 538, 509, 543, 532, 524, 528, 514, 527, 513, 503, 529, 528, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[516, 522, 497, 538, 499, 495, 545, 500, 528, 519, 495, 486, 551, 506, 516, 493, 538, 474, 521, 481, 533, 502, 524, 517, 488, 521, 554, 531, 496, 576, 496, 494, 542, 485, 484, 534, 521, 499, 496, 492, 501, 550, 509, 509, 515, 507, 528, 521, 507, 507, 530, 487, 510, 475, 467, 490, 545, 517, 537, 490, 524, 474, 509, 508, 536, 491, 517, 495, 499, 475, 500, 519, 519, 531, 556, 537, 522, 493, 532, 505, 494, 500, 511, 517, 524, 519, 537, 528, 485, 512, 526, 506, 458, 568, 515, 501, 538, 517, 565, 493, 520, 529, 474, 500, 506, 463, 520, 542, 496, 498, 501, 522, 484, 509, 493, 513, 512, 529, 516, 507, 542, 509, 535, 524, 511, 487, 515, 516] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[481, 519, 511, 533, 524, 476, 506, 502, 508, 489, 499, 518, 501, 493, 495, 545, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 480, 499, 546, 550, 503, 524, 487, 508, 514, 479, 529, 543, 520, 510, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 523, 509, 481, 536, 512, 506, 509, 468, 522, 517, 517, 498, 504, 551, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 488, 523, 496, 511, 482, 563, 519, 499, 479, 519, 482, 521, 538, 552, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 533, 526, 515, 464, 500, 527, 501, 535, 473, 534, 479, 537, 520, 517, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 471, 510, 532, 499, 539, 522, 495, 517, 510, 494, 502, 513, 543, 504, 522, 21, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 482, 509, 495, 550, 536, 547, 542, 542, 476, 545, 494, 521, 525, 510, 511, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 501, 517, 524, 508, 458, 513, 486, 503, 498, 510, 551, 513, 509, 537, 472, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8103, 8189, 8167, 8174, 8231, 8222, 8310, 8140] avg=8192.0000 max=8310 max_rank=6 min=8103 min_rank=0 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32903 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[516, 522, 497, 538, 499, 495, 545, 500, 528, 519, 495, 486, 551, 506, 516, 493, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 474, 521, 481, 533, 502, 524, 517, 488, 521, 554, 531, 496, 576, 480, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 485, 484, 534, 521, 499, 496, 492, 501, 550, 509, 509, 515, 507, 528, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 507, 530, 487, 510, 475, 467, 490, 545, 517, 526, 490, 524, 474, 509, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 491, 517, 495, 499, 475, 500, 519, 519, 531, 535, 537, 522, 493, 532, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 500, 511, 517, 524, 519, 537, 528, 485, 512, 526, 506, 458, 568, 515, 501, 21, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 517, 565, 493, 520, 529, 474, 500, 506, 463, 520, 542, 496, 498, 501, 522, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 509, 493, 513, 512, 529, 516, 507, 542, 509, 535, 524, 511, 487, 515, 516, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8230, 8193, 8066, 8206, 8231, 8190, 8211] avg=8192.0000 max=8231 max_rank=5 min=8066 min_rank=3 imbalance_ratio=1.004761 max_load_reduction=57305 max_load_reduction_pct=0.874405 moved_to_echo=32838 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[505, 501, 479, 508, 505, 510, 535, 553, 561, 518, 506, 502, 534, 527, 510, 508, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 530, 528, 497, 546, 511, 530, 523, 515, 498, 517, 492, 508, 482, 499, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 521, 492, 534, 525, 565, 517, 484, 490, 494, 521, 514, 498, 510, 494, 489, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 517, 518, 526, 536, 526, 532, 494, 471, 511, 471, 494, 527, 538, 513, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 513, 472, 468, 503, 488, 532, 525, 521, 517, 514, 515, 505, 516, 481, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 509, 507, 518, 506, 500, 543, 551, 532, 472, 513, 550, 474, 526, 527, 543, 26, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 527, 498, 545, 490, 503, 496, 494, 566, 508, 488, 507, 472, 525, 501, 517, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 509, 499, 524, 499, 546, 492, 503, 497, 511, 474, 509, 480, 475, 542, 525, 11, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8268, 8183, 8175, 8215, 8120, 8350, 8146, 8079] avg=8192.0000 max=8350 max_rank=5 min=8079 min_rank=7 imbalance_ratio=1.019287 max_load_reduction=57186 max_load_reduction_pct=0.872589 moved_to_echo=32695 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[460, 540, 525, 500, 524, 517, 456, 508, 525, 461, 543, 522, 495, 539, 485, 503, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 492, 536, 491, 542, 505, 534, 517, 460, 556, 490, 527, 516, 514, 529, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 495, 518, 474, 517, 506, 535, 506, 496, 521, 537, 524, 532, 504, 498, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 529, 504, 528, 543, 526, 533, 481, 524, 494, 529, 516, 488, 512, 509, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 528, 508, 515, 519, 531, 516, 509, 546, 555, 472, 524, 521, 527, 508, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 528, 507, 533, 519, 514, 531, 495, 485, 537, 502, 436, 511, 503, 459, 519, 19, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 491, 514, 480, 522, 517, 526, 502, 501, 492, 511, 538, 560, 516, 525, 540, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 534, 535, 504, 518, 502, 504, 524, 548, 496, 494, 518, 505, 505, 496, 472, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8106, 8228, 8154, 8272, 8261, 8123, 8202, 8190] avg=8192.0000 max=8272 max_rank=3 min=8106 min_rank=0 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32776 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[528, 487, 530, 520, 488, 524, 507, 505, 531, 544, 530, 482, 487, 536, 492, 523, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 537, 514, 509, 521, 512, 550, 525, 499, 471, 526, 498, 523, 502, 487, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 497, 526, 491, 553, 550, 532, 503, 510, 508, 538, 486, 490, 519, 525, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 531, 504, 491, 528, 501, 447, 518, 536, 506, 514, 505, 502, 529, 477, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 511, 513, 538, 468, 501, 514, 531, 537, 512, 487, 506, 515, 509, 503, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 482, 459, 501, 509, 526, 497, 535, 511, 490, 481, 573, 549, 494, 524, 518, 20, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 521, 543, 496, 513, 509, 489, 529, 502, 554, 506, 523, 524, 501, 541, 510, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 503, 519, 483, 486, 475, 486, 509, 528, 500, 509, 552, 492, 534, 496, 552, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8217, 8222, 8223, 8107, 8164, 8165, 8301, 8137] avg=8192.0000 max=8301 max_rank=6 min=8107 min_rank=3 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32767 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[505, 501, 479, 508, 505, 510, 535, 553, 561, 518, 506, 502, 534, 527, 510, 508, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 530, 528, 497, 546, 511, 530, 523, 515, 498, 517, 492, 508, 482, 499, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 521, 492, 534, 525, 565, 517, 484, 490, 494, 521, 514, 498, 510, 494, 489, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 517, 518, 526, 536, 526, 532, 494, 471, 511, 471, 494, 527, 538, 513, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 513, 472, 468, 503, 488, 532, 525, 521, 517, 514, 515, 505, 516, 481, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 509, 507, 518, 506, 500, 543, 551, 532, 472, 513, 550, 474, 526, 527, 543, 26, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 527, 498, 545, 490, 503, 496, 494, 566, 508, 488, 507, 472, 525, 501, 517, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 509, 499, 524, 499, 546, 492, 503, 497, 511, 474, 509, 480, 475, 542, 525, 11, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8268, 8183, 8175, 8215, 8120, 8350, 8146, 8079] avg=8192.0000 max=8350 max_rank=5 min=8079 min_rank=7 imbalance_ratio=1.019287 max_load_reduction=57186 max_load_reduction_pct=0.872589 moved_to_echo=32695 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[502, 519, 515, 539, 536, 496, 498, 516, 520, 498, 504, 506, 497, 507, 506, 525, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 496, 490, 505, 494, 547, 515, 510, 511, 484, 498, 469, 513, 515, 522, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 505, 530, 528, 503, 506, 487, 479, 496, 507, 506, 481, 503, 537, 516, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 541, 492, 487, 520, 497, 524, 500, 518, 505, 516, 504, 508, 522, 514, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 519, 502, 531, 543, 543, 525, 513, 568, 508, 529, 517, 479, 517, 490, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 501, 463, 513, 539, 515, 538, 504, 528, 525, 506, 528, 505, 503, 497, 530, 21, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 523, 493, 500, 503, 512, 489, 545, 477, 520, 507, 495, 509, 492, 514, 454, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 486, 496, 538, 509, 543, 532, 524, 528, 514, 527, 513, 503, 529, 528, 524, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8127, 8178, 8145, 8294, 8241, 8052, 8312] avg=8192.0000 max=8312 max_rank=7 min=8052 min_rank=6 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32899 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[500, 517, 496, 512, 551, 506, 492, 509, 534, 533, 514, 499, 483, 528, 568, 543, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 508, 538, 539, 497, 522, 506, 549, 518, 474, 489, 515, 537, 449, 499, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 487, 517, 542, 512, 513, 506, 501, 456, 510, 510, 494, 530, 539, 559, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 518, 498, 525, 510, 510, 501, 501, 537, 518, 521, 511, 490, 530, 534, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 520, 480, 513, 460, 477, 489, 486, 493, 557, 486, 524, 506, 473, 507, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 534, 497, 501, 533, 492, 534, 473, 522, 494, 494, 531, 558, 503, 465, 523, 19, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 567, 494, 535, 523, 524, 474, 506, 493, 484, 505, 478, 501, 529, 519, 480, 558, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 536, 529, 507, 485, 495, 477, 529, 503, 517, 506, 533, 568, 506, 545, 532, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8288, 8149, 8148, 8271, 8037, 8186, 8176, 8281] avg=8192.0000 max=8288 max_rank=0 min=8037 min_rank=4 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32680 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[519, 504, 536, 522, 495, 518, 497, 485, 503, 515, 506, 484, 527, 526, 495, 523, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 541, 532, 491, 518, 528, 509, 515, 475, 508, 460, 551, 501, 515, 539, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 495, 515, 511, 560, 521, 489, 529, 523, 550, 524, 502, 511, 500, 529, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 563, 511, 478, 543, 495, 537, 542, 493, 451, 521, 528, 501, 543, 542, 509, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 557, 501, 514, 487, 564, 508, 487, 511, 497, 508, 526, 484, 516, 507, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 502, 449, 520, 468, 526, 512, 525, 518, 495, 503, 474, 483, 534, 509, 490, 20, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 526, 510, 503, 530, 494, 466, 527, 515, 489, 514, 517, 518, 504, 492, 537, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 490, 495, 537, 502, 537, 523, 497, 509, 515, 530, 515, 532, 491, 476, 499, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8158, 8208, 8298, 8286, 8223, 8018, 8159, 8186] avg=8192.0000 max=8298 max_rank=2 min=8018 min_rank=5 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32586 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[502, 519, 515, 539, 536, 496, 498, 516, 520, 498, 504, 506, 497, 507, 506, 525, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 496, 490, 505, 494, 547, 515, 510, 511, 484, 498, 469, 513, 515, 522, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 505, 530, 528, 503, 506, 487, 479, 496, 507, 506, 481, 503, 537, 516, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 541, 492, 487, 520, 497, 524, 500, 518, 505, 516, 504, 508, 522, 514, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 519, 502, 531, 543, 543, 525, 513, 568, 508, 529, 517, 479, 517, 490, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 501, 463, 513, 539, 515, 538, 504, 528, 525, 506, 528, 505, 503, 497, 530, 21, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 523, 493, 500, 503, 512, 489, 545, 477, 520, 507, 495, 509, 492, 514, 454, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 486, 496, 538, 509, 543, 532, 524, 528, 514, 527, 513, 503, 529, 528, 524, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8127, 8178, 8145, 8294, 8241, 8052, 8312] avg=8192.0000 max=8312 max_rank=7 min=8052 min_rank=6 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32899 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[516, 522, 497, 538, 499, 495, 545, 500, 528, 519, 495, 486, 551, 506, 516, 493, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 474, 521, 481, 533, 502, 524, 517, 488, 521, 554, 531, 496, 576, 480, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 485, 484, 534, 521, 499, 496, 492, 501, 550, 509, 509, 515, 507, 528, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 507, 530, 487, 510, 475, 467, 490, 545, 517, 526, 490, 524, 474, 509, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 491, 517, 495, 499, 475, 500, 519, 519, 531, 535, 537, 522, 493, 532, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 500, 511, 517, 524, 519, 537, 528, 485, 512, 526, 506, 458, 568, 515, 501, 21, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 517, 565, 493, 520, 529, 474, 500, 506, 463, 520, 542, 496, 498, 501, 522, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 509, 493, 513, 512, 529, 516, 507, 542, 509, 535, 524, 511, 487, 515, 516, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8230, 8193, 8066, 8206, 8231, 8190, 8211] avg=8192.0000 max=8231 max_rank=5 min=8066 min_rank=3 imbalance_ratio=1.004761 max_load_reduction=57305 max_load_reduction_pct=0.874405 moved_to_echo=32838 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[528, 487, 530, 520, 488, 524, 507, 505, 531, 544, 530, 482, 487, 536, 492, 523, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 537, 514, 509, 521, 512, 550, 525, 499, 471, 526, 498, 523, 502, 487, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 497, 526, 491, 553, 550, 532, 503, 510, 508, 538, 486, 490, 519, 525, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 531, 504, 491, 528, 501, 447, 518, 536, 506, 514, 505, 502, 529, 477, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 511, 513, 538, 468, 501, 514, 531, 537, 512, 487, 506, 515, 509, 503, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 482, 459, 501, 509, 526, 497, 535, 511, 490, 481, 573, 549, 494, 524, 518, 20, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 521, 543, 496, 513, 509, 489, 529, 502, 554, 506, 523, 524, 501, 541, 510, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 503, 519, 483, 486, 475, 486, 509, 528, 500, 509, 552, 492, 534, 496, 552, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8217, 8222, 8223, 8107, 8164, 8165, 8301, 8137] avg=8192.0000 max=8301 max_rank=6 min=8107 min_rank=3 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32767 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4096, 4352, 4224, 4096, 4096, 3968, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4096, 3968, 4096, 4224, 4096, 4352, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4096, 4352, 4224, 4096, 4096, 3968, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4096, 3968, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4096, 3968, 4096, 4224, 4096, 4352, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[489, 501, 478, 528, 509, 497, 462, 498, 501, 497, 511, 477, 525, 502, 532, 509, 499, 494, 539, 512, 561, 474, 551, 520, 534, 549, 537, 528, 510, 516, 535, 528, 497, 498, 497, 529, 573, 534, 536, 491, 505, 512, 521, 518, 520, 504, 495, 506, 522, 514, 519, 502, 510, 529, 508, 518, 526, 483, 502, 532, 497, 562, 527, 527, 512, 480, 461, 504, 519, 505, 534, 532, 520, 506, 560, 518, 509, 499, 504, 497, 511, 522, 458, 471, 526, 470, 489, 492, 542, 498, 490, 506, 501, 519, 514, 528, 490, 497, 528, 535, 541, 541, 491, 495, 516, 509, 534, 494, 507, 523, 543, 501, 513, 476, 483, 552, 487, 520, 518, 479, 524, 523, 511, 495, 535, 530, 518, 513] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[541, 461, 508, 505, 470, 500, 515, 488, 525, 504, 511, 466, 517, 540, 501, 537, 495, 495, 508, 492, 515, 506, 547, 490, 500, 536, 520, 532, 487, 538, 566, 523, 536, 524, 501, 485, 512, 502, 515, 529, 466, 497, 528, 538, 498, 481, 572, 454, 528, 553, 545, 497, 534, 531, 505, 516, 495, 504, 498, 481, 514, 518, 522, 539, 498, 506, 504, 529, 479, 537, 499, 495, 509, 493, 522, 508, 525, 583, 490, 463, 470, 518, 522, 528, 507, 510, 522, 555, 515, 510, 524, 513, 509, 514, 536, 514, 518, 536, 532, 511, 508, 516, 494, 485, 527, 533, 506, 508, 521, 468, 515, 515, 514, 520, 529, 513, 498, 526, 503, 517, 494, 523, 507, 474, 519, 520, 486, 536] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[502, 528, 506, 493, 490, 507, 508, 552, 530, 521, 506, 555, 497, 477, 531, 505, 544, 496, 525, 505, 522, 515, 484, 492, 486, 518, 510, 497, 537, 499, 540, 569, 494, 538, 519, 483, 520, 504, 484, 529, 500, 525, 525, 501, 542, 498, 540, 495, 522, 490, 513, 511, 479, 538, 492, 523, 529, 508, 496, 536, 525, 506, 498, 491, 541, 478, 517, 515, 511, 496, 526, 485, 536, 518, 543, 495, 527, 507, 519, 521, 508, 487, 529, 511, 520, 542, 502, 554, 501, 497, 513, 502, 489, 536, 536, 483, 465, 468, 520, 499, 458, 506, 531, 518, 497, 531, 500, 523, 486, 514, 524, 511, 487, 512, 507, 523, 531, 513, 487, 482, 538, 519, 496, 523, 525, 572, 534, 490] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[515, 506, 506, 513, 483, 523, 511, 521, 516, 512, 508, 511, 512, 486, 527, 533, 537, 478, 525, 511, 462, 527, 520, 517, 551, 529, 515, 532, 493, 550, 481, 519, 471, 479, 520, 501, 515, 511, 490, 507, 512, 490, 502, 501, 517, 498, 497, 515, 515, 529, 537, 539, 512, 507, 511, 481, 525, 514, 542, 487, 525, 495, 505, 513, 505, 528, 510, 525, 519, 555, 515, 508, 535, 505, 486, 500, 495, 470, 495, 500, 487, 555, 524, 518, 515, 544, 525, 542, 508, 549, 516, 504, 510, 477, 507, 492, 484, 502, 514, 544, 535, 509, 468, 533, 509, 524, 506, 508, 542, 510, 522, 506, 511, 497, 500, 487, 502, 540, 517, 502, 509, 515, 523, 505, 536, 526, 551, 482] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[465, 499, 506, 476, 454, 536, 497, 482, 455, 534, 528, 514, 517, 560, 518, 537, 509, 531, 515, 482, 491, 558, 519, 515, 510, 500, 508, 502, 505, 496, 492, 517, 501, 510, 544, 516, 533, 482, 494, 471, 507, 508, 556, 503, 575, 502, 523, 544, 511, 481, 502, 467, 486, 503, 494, 510, 531, 526, 510, 522, 499, 507, 535, 547, 513, 514, 512, 483, 493, 485, 535, 488, 549, 537, 499, 516, 495, 509, 515, 501, 549, 490, 504, 503, 463, 520, 539, 524, 498, 497, 550, 482, 516, 544, 497, 480, 498, 537, 497, 519, 511, 500, 533, 531, 495, 519, 537, 544, 515, 470, 515, 500, 530, 550, 510, 509, 474, 509, 542, 536, 561, 530, 510, 496, 544, 545, 511, 530] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[502, 512, 525, 504, 472, 480, 536, 500, 512, 493, 539, 489, 493, 540, 515, 496, 493, 517, 483, 541, 484, 508, 494, 492, 516, 497, 537, 512, 555, 529, 545, 535, 500, 588, 472, 500, 521, 527, 503, 497, 540, 502, 526, 516, 523, 501, 501, 512, 510, 488, 504, 532, 494, 499, 519, 510, 544, 453, 529, 511, 541, 501, 490, 540, 469, 528, 524, 487, 511, 512, 497, 551, 504, 509, 501, 519, 570, 493, 525, 555, 473, 511, 548, 529, 532, 545, 478, 499, 516, 470, 508, 533, 491, 498, 494, 534, 518, 508, 525, 527, 497, 512, 530, 536, 541, 527, 488, 493, 555, 509, 514, 501, 525, 501, 505, 537, 500, 515, 486, 490, 496, 508, 502, 501, 499, 507, 520, 509] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[515, 484, 515, 556, 486, 484, 508, 553, 524, 546, 514, 519, 525, 551, 511, 567, 495, 528, 492, 501, 500, 483, 508, 460, 500, 511, 518, 508, 473, 528, 517, 531, 494, 472, 514, 477, 522, 476, 537, 525, 503, 516, 519, 500, 551, 521, 581, 486, 472, 556, 533, 497, 477, 503, 476, 560, 503, 518, 500, 517, 521, 532, 481, 524, 496, 519, 530, 466, 483, 501, 489, 510, 495, 558, 543, 521, 499, 520, 507, 522, 449, 507, 512, 515, 529, 508, 523, 502, 505, 532, 499, 481, 516, 524, 535, 523, 538, 502, 505, 516, 506, 513, 543, 519, 491, 477, 519, 506, 509, 502, 486, 546, 530, 539, 495, 511, 534, 474, 538, 526, 499, 482, 527, 501, 515, 513, 566, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[502, 512, 525, 504, 472, 480, 536, 500, 512, 493, 539, 489, 493, 540, 515, 496, 493, 517, 483, 541, 484, 508, 494, 492, 516, 497, 537, 512, 555, 529, 545, 535, 500, 588, 472, 500, 521, 527, 503, 497, 540, 502, 526, 516, 523, 501, 501, 512, 510, 488, 504, 532, 494, 499, 519, 510, 544, 453, 529, 511, 541, 501, 490, 540, 469, 528, 524, 487, 511, 512, 497, 551, 504, 509, 501, 519, 570, 493, 525, 555, 473, 511, 548, 529, 532, 545, 478, 499, 516, 470, 508, 533, 491, 498, 494, 534, 518, 508, 525, 527, 497, 512, 530, 536, 541, 527, 488, 493, 555, 509, 514, 501, 525, 501, 505, 537, 500, 515, 486, 490, 496, 508, 502, 501, 499, 507, 520, 509] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[510, 507, 490, 510, 502, 499, 491, 506, 517, 544, 506, 527, 521, 533, 565, 519, 530, 507, 510, 548, 525, 513, 535, 510, 484, 507, 519, 519, 498, 537, 475, 533, 478, 515, 492, 507, 534, 525, 519, 528, 511, 529, 527, 547, 510, 488, 533, 515, 513, 504, 518, 526, 527, 511, 479, 528, 496, 486, 536, 506, 493, 517, 502, 494, 502, 541, 506, 489, 523, 499, 496, 486, 550, 515, 520, 517, 492, 523, 528, 504, 534, 521, 554, 504, 523, 504, 529, 495, 519, 522, 496, 526, 497, 524, 483, 512, 490, 535, 505, 498, 522, 522, 489, 509, 519, 492, 490, 535, 481, 504, 500, 528, 515, 508, 551, 482, 506, 496, 489, 506, 540, 537, 485, 509, 500, 474, 506, 488] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[515, 506, 506, 513, 483, 523, 511, 521, 516, 512, 508, 511, 512, 486, 527, 533, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 478, 525, 511, 462, 527, 520, 517, 551, 529, 515, 532, 493, 550, 481, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 479, 520, 501, 515, 511, 490, 507, 512, 490, 502, 501, 517, 498, 496, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 529, 537, 539, 512, 507, 511, 481, 525, 514, 542, 487, 525, 495, 505, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 528, 510, 525, 519, 555, 515, 508, 535, 505, 486, 500, 495, 470, 495, 500, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 555, 524, 518, 515, 544, 525, 542, 508, 549, 516, 504, 510, 477, 507, 492, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 502, 514, 544, 535, 509, 468, 533, 509, 524, 506, 508, 542, 510, 522, 506, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 497, 500, 487, 502, 540, 517, 502, 509, 515, 523, 505, 536, 526, 538, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8214, 8025, 8235, 8163, 8276, 8220, 8190] avg=8192.0000 max=8276 max_rank=5 min=8025 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32849 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[541, 461, 508, 505, 470, 500, 515, 488, 525, 504, 511, 466, 517, 540, 501, 537, 495, 495, 508, 492, 515, 506, 547, 490, 500, 536, 520, 532, 487, 538, 566, 523, 536, 524, 501, 485, 512, 502, 515, 529, 466, 497, 528, 538, 498, 481, 572, 454, 528, 553, 545, 497, 534, 531, 505, 516, 495, 504, 498, 481, 514, 518, 522, 539, 498, 506, 504, 529, 479, 537, 499, 495, 509, 493, 522, 508, 525, 583, 490, 463, 470, 518, 522, 528, 507, 510, 522, 555, 515, 510, 524, 513, 509, 514, 536, 514, 518, 536, 532, 511, 508, 516, 494, 485, 527, 533, 506, 508, 521, 468, 515, 515, 514, 520, 529, 513, 498, 526, 503, 517, 494, 523, 507, 474, 519, 520, 486, 536] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[465, 499, 506, 476, 454, 536, 497, 482, 455, 534, 528, 514, 517, 560, 518, 537, 509, 531, 515, 482, 491, 558, 519, 515, 510, 500, 508, 502, 505, 496, 492, 517, 501, 510, 544, 516, 533, 482, 494, 471, 507, 508, 556, 503, 575, 502, 523, 544, 511, 481, 502, 467, 486, 503, 494, 510, 531, 526, 510, 522, 499, 507, 535, 547, 513, 514, 512, 483, 493, 485, 535, 488, 549, 537, 499, 516, 495, 509, 515, 501, 549, 490, 504, 503, 463, 520, 539, 524, 498, 497, 550, 482, 516, 544, 497, 480, 498, 537, 497, 519, 511, 500, 533, 531, 495, 519, 537, 544, 515, 470, 515, 500, 530, 550, 510, 509, 474, 509, 542, 536, 561, 530, 510, 496, 544, 545, 511, 530] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[515, 506, 506, 513, 483, 523, 511, 521, 516, 512, 508, 511, 512, 486, 527, 533, 537, 478, 525, 511, 462, 527, 520, 517, 551, 529, 515, 532, 493, 550, 481, 519, 471, 479, 520, 501, 515, 511, 490, 507, 512, 490, 502, 501, 517, 498, 497, 515, 515, 529, 537, 539, 512, 507, 511, 481, 525, 514, 542, 487, 525, 495, 505, 513, 505, 528, 510, 525, 519, 555, 515, 508, 535, 505, 486, 500, 495, 470, 495, 500, 487, 555, 524, 518, 515, 544, 525, 542, 508, 549, 516, 504, 510, 477, 507, 492, 484, 502, 514, 544, 535, 509, 468, 533, 509, 524, 506, 508, 542, 510, 522, 506, 511, 497, 500, 487, 502, 540, 517, 502, 509, 515, 523, 505, 536, 526, 551, 482] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[502, 528, 506, 493, 490, 507, 508, 552, 530, 521, 506, 555, 497, 477, 531, 505, 544, 496, 525, 505, 522, 515, 484, 492, 486, 518, 510, 497, 537, 499, 540, 569, 494, 538, 519, 483, 520, 504, 484, 529, 500, 525, 525, 501, 542, 498, 540, 495, 522, 490, 513, 511, 479, 538, 492, 523, 529, 508, 496, 536, 525, 506, 498, 491, 541, 478, 517, 515, 511, 496, 526, 485, 536, 518, 543, 495, 527, 507, 519, 521, 508, 487, 529, 511, 520, 542, 502, 554, 501, 497, 513, 502, 489, 536, 536, 483, 465, 468, 520, 499, 458, 506, 531, 518, 497, 531, 500, 523, 486, 514, 524, 511, 487, 512, 507, 523, 531, 513, 487, 482, 538, 519, 496, 523, 525, 572, 534, 490] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[489, 501, 478, 528, 509, 497, 462, 498, 501, 497, 511, 477, 525, 502, 532, 509, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 494, 539, 512, 561, 474, 551, 520, 534, 549, 537, 528, 510, 516, 535, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 498, 497, 529, 573, 534, 536, 491, 505, 512, 521, 518, 520, 504, 494, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 514, 519, 502, 510, 529, 508, 518, 526, 483, 502, 532, 497, 562, 527, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 480, 461, 504, 519, 505, 534, 532, 520, 506, 560, 518, 509, 499, 504, 497, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 522, 458, 471, 526, 470, 489, 492, 542, 498, 490, 506, 501, 519, 514, 528, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 497, 528, 535, 541, 541, 491, 495, 516, 509, 534, 494, 507, 523, 543, 501, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 476, 483, 552, 487, 520, 518, 479, 524, 523, 511, 495, 535, 530, 507, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8046, 8354, 8235, 8276, 8171, 8040, 8248, 8166] avg=8192.0000 max=8354 max_rank=1 min=8040 min_rank=5 imbalance_ratio=1.019775 max_load_reduction=57182 max_load_reduction_pct=0.872528 moved_to_echo=32625 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[502, 512, 525, 504, 472, 480, 536, 500, 512, 493, 539, 489, 493, 540, 515, 496, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 517, 483, 541, 484, 508, 494, 492, 516, 497, 537, 512, 555, 529, 545, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 588, 472, 500, 521, 527, 503, 497, 540, 502, 526, 516, 523, 501, 500, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 488, 504, 532, 494, 499, 519, 510, 544, 453, 529, 511, 541, 501, 490, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 528, 524, 487, 511, 512, 497, 551, 504, 509, 501, 519, 570, 493, 525, 555, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 511, 548, 529, 532, 545, 478, 499, 516, 470, 508, 533, 491, 498, 494, 534, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 508, 525, 527, 497, 512, 530, 536, 541, 527, 488, 493, 555, 509, 514, 501, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 501, 505, 537, 500, 515, 486, 490, 496, 508, 502, 501, 499, 507, 509, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8139, 8204, 8228, 8163, 8266, 8162, 8284, 8090] avg=8192.0000 max=8284 max_rank=6 min=8090 min_rank=7 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32802 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[541, 461, 508, 505, 470, 500, 515, 488, 525, 504, 511, 466, 517, 540, 501, 537, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 495, 508, 492, 515, 506, 547, 490, 500, 536, 520, 532, 487, 538, 566, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 524, 501, 485, 512, 502, 515, 529, 466, 497, 528, 538, 498, 481, 571, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 553, 545, 497, 534, 531, 505, 516, 495, 504, 498, 481, 514, 518, 522, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 506, 504, 529, 479, 537, 499, 495, 509, 493, 522, 508, 525, 583, 490, 463, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 518, 522, 528, 507, 510, 522, 555, 515, 510, 524, 513, 509, 514, 536, 514, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 536, 532, 511, 508, 516, 494, 485, 527, 533, 506, 508, 521, 468, 515, 515, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 520, 529, 513, 498, 526, 503, 517, 494, 523, 507, 474, 519, 520, 475, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8119, 8217, 8137, 8278, 8151, 8270, 8196, 8168] avg=8192.0000 max=8278 max_rank=3 min=8119 min_rank=0 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32785 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[510, 507, 490, 510, 502, 499, 491, 506, 517, 544, 506, 527, 521, 533, 565, 519, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 507, 510, 548, 525, 513, 535, 510, 484, 507, 519, 519, 498, 537, 475, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 515, 492, 507, 534, 525, 519, 528, 511, 529, 527, 547, 510, 488, 532, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 504, 518, 526, 527, 511, 479, 528, 496, 486, 536, 506, 493, 517, 502, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 541, 506, 489, 523, 499, 496, 486, 550, 515, 520, 517, 492, 523, 528, 504, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 521, 554, 504, 523, 504, 529, 495, 519, 522, 496, 526, 497, 524, 483, 512, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 535, 505, 498, 522, 522, 489, 509, 519, 492, 490, 535, 481, 504, 500, 528, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 508, 551, 482, 506, 496, 489, 506, 540, 537, 485, 509, 500, 474, 495, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8278, 8216, 8257, 8134, 8202, 8246, 8122, 8081] avg=8192.0000 max=8278 max_rank=0 min=8081 min_rank=7 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32651 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[465, 499, 506, 476, 454, 536, 497, 482, 455, 534, 528, 514, 517, 560, 518, 537, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 531, 515, 482, 491, 558, 519, 515, 510, 500, 508, 502, 505, 496, 492, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 510, 544, 516, 533, 482, 494, 471, 507, 508, 556, 503, 575, 502, 519, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 481, 502, 467, 486, 503, 494, 510, 531, 526, 510, 522, 499, 507, 535, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 514, 512, 483, 493, 485, 535, 488, 549, 537, 499, 516, 495, 509, 515, 501, 13, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 490, 504, 503, 463, 520, 539, 524, 498, 497, 550, 482, 516, 544, 497, 480, 4, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 537, 497, 519, 511, 500, 533, 531, 495, 519, 537, 544, 515, 470, 515, 500, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 550, 510, 509, 474, 509, 542, 536, 561, 530, 510, 496, 544, 545, 488, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8111, 8109, 8265, 8127, 8165, 8166, 8229, 8364] avg=8192.0000 max=8364 max_rank=7 min=8109 min_rank=1 imbalance_ratio=1.020996 max_load_reduction=57172 max_load_reduction_pct=0.872375 moved_to_echo=32924 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[502, 528, 506, 493, 490, 507, 508, 552, 530, 521, 506, 555, 497, 477, 531, 505, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 496, 525, 505, 522, 515, 484, 492, 486, 518, 510, 497, 537, 499, 540, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 538, 519, 483, 520, 504, 484, 529, 500, 525, 525, 501, 542, 498, 539, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 490, 513, 511, 479, 538, 492, 523, 529, 508, 496, 536, 525, 506, 498, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 478, 517, 515, 511, 496, 526, 485, 536, 518, 543, 495, 527, 507, 519, 521, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 487, 529, 511, 520, 542, 502, 554, 501, 497, 513, 502, 489, 536, 536, 483, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 468, 520, 499, 458, 506, 531, 518, 497, 531, 500, 523, 486, 514, 524, 511, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 512, 507, 523, 531, 513, 487, 482, 538, 519, 496, 523, 525, 572, 521, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8241, 8203, 8196, 8155, 8247, 8213, 8055, 8226] avg=8192.0000 max=8247 max_rank=4 min=8055 min_rank=6 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32741 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[502, 512, 525, 504, 472, 480, 536, 500, 512, 493, 539, 489, 493, 540, 515, 496, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 517, 483, 541, 484, 508, 494, 492, 516, 497, 537, 512, 555, 529, 545, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 588, 472, 500, 521, 527, 503, 497, 540, 502, 526, 516, 523, 501, 500, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 488, 504, 532, 494, 499, 519, 510, 544, 453, 529, 511, 541, 501, 490, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 528, 524, 487, 511, 512, 497, 551, 504, 509, 501, 519, 570, 493, 525, 555, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 511, 548, 529, 532, 545, 478, 499, 516, 470, 508, 533, 491, 498, 494, 534, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 508, 525, 527, 497, 512, 530, 536, 541, 527, 488, 493, 555, 509, 514, 501, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 501, 505, 537, 500, 515, 486, 490, 496, 508, 502, 501, 499, 507, 509, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8139, 8204, 8228, 8163, 8266, 8162, 8284, 8090] avg=8192.0000 max=8284 max_rank=6 min=8090 min_rank=7 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32802 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[515, 484, 515, 556, 486, 484, 508, 553, 524, 546, 514, 519, 525, 551, 511, 567, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 528, 492, 501, 500, 483, 508, 460, 500, 511, 518, 508, 473, 528, 517, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 472, 514, 477, 522, 476, 537, 525, 503, 516, 519, 500, 551, 521, 580, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 556, 533, 497, 477, 503, 476, 560, 503, 518, 500, 517, 521, 532, 481, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 519, 530, 466, 483, 501, 489, 510, 495, 558, 543, 521, 499, 520, 507, 522, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 449, 507, 512, 515, 529, 508, 523, 502, 505, 532, 499, 481, 516, 524, 535, 523, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 502, 505, 516, 506, 513, 543, 519, 491, 477, 519, 506, 509, 502, 486, 546, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 539, 495, 511, 534, 474, 538, 526, 499, 482, 527, 501, 515, 513, 553, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8389, 8019, 8193, 8168, 8171, 8163, 8182, 8251] avg=8192.0000 max=8389 max_rank=0 min=8019 min_rank=1 imbalance_ratio=1.024048 max_load_reduction=57147 max_load_reduction_pct=0.871994 moved_to_echo=32767 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[510, 507, 490, 510, 502, 499, 491, 506, 517, 544, 506, 527, 521, 533, 565, 519, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 507, 510, 548, 525, 513, 535, 510, 484, 507, 519, 519, 498, 537, 475, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 515, 492, 507, 534, 525, 519, 528, 511, 529, 527, 547, 510, 488, 532, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 504, 518, 526, 527, 511, 479, 528, 496, 486, 536, 506, 493, 517, 502, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 541, 506, 489, 523, 499, 496, 486, 550, 515, 520, 517, 492, 523, 528, 504, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 521, 554, 504, 523, 504, 529, 495, 519, 522, 496, 526, 497, 524, 483, 512, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 535, 505, 498, 522, 522, 489, 509, 519, 492, 490, 535, 481, 504, 500, 528, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 508, 551, 482, 506, 496, 489, 506, 540, 537, 485, 509, 500, 474, 495, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8278, 8216, 8257, 8134, 8202, 8246, 8122, 8081] avg=8192.0000 max=8278 max_rank=0 min=8081 min_rank=7 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32651 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[502, 528, 506, 493, 490, 507, 508, 552, 530, 521, 506, 555, 497, 477, 531, 505, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 496, 525, 505, 522, 515, 484, 492, 486, 518, 510, 497, 537, 499, 540, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 538, 519, 483, 520, 504, 484, 529, 500, 525, 525, 501, 542, 498, 539, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 490, 513, 511, 479, 538, 492, 523, 529, 508, 496, 536, 525, 506, 498, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 478, 517, 515, 511, 496, 526, 485, 536, 518, 543, 495, 527, 507, 519, 521, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 487, 529, 511, 520, 542, 502, 554, 501, 497, 513, 502, 489, 536, 536, 483, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 468, 520, 499, 458, 506, 531, 518, 497, 531, 500, 523, 486, 514, 524, 511, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 512, 507, 523, 531, 513, 487, 482, 538, 519, 496, 523, 525, 572, 521, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8241, 8203, 8196, 8155, 8247, 8213, 8055, 8226] avg=8192.0000 max=8247 max_rank=4 min=8055 min_rank=6 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32741 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[515, 506, 506, 513, 483, 523, 511, 521, 516, 512, 508, 511, 512, 486, 527, 533, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 478, 525, 511, 462, 527, 520, 517, 551, 529, 515, 532, 493, 550, 481, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 479, 520, 501, 515, 511, 490, 507, 512, 490, 502, 501, 517, 498, 496, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 529, 537, 539, 512, 507, 511, 481, 525, 514, 542, 487, 525, 495, 505, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 528, 510, 525, 519, 555, 515, 508, 535, 505, 486, 500, 495, 470, 495, 500, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 555, 524, 518, 515, 544, 525, 542, 508, 549, 516, 504, 510, 477, 507, 492, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 502, 514, 544, 535, 509, 468, 533, 509, 524, 506, 508, 542, 510, 522, 506, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 497, 500, 487, 502, 540, 517, 502, 509, 515, 523, 505, 536, 526, 538, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8214, 8025, 8235, 8163, 8276, 8220, 8190] avg=8192.0000 max=8276 max_rank=5 min=8025 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32849 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 3968, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4352, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4352, 4096, 4352, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4352, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4352, 4096, 4352, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[518, 507, 515, 523, 554, 516, 508, 497, 490, 510, 525, 530, 496, 533, 506, 505, 514, 504, 535, 515, 506, 493, 505, 519, 523, 525, 478, 521, 500, 473, 506, 495, 513, 525, 511, 549, 552, 508, 523, 490, 491, 506, 517, 503, 490, 514, 493, 514, 486, 495, 534, 516, 527, 456, 515, 509, 519, 536, 518, 548, 527, 486, 520, 513, 504, 502, 496, 502, 509, 489, 524, 526, 467, 500, 513, 512, 485, 521, 540, 522, 516, 503, 470, 515, 463, 514, 531, 517, 478, 497, 515, 542, 459, 498, 529, 537, 504, 563, 538, 542, 496, 497, 510, 529, 531, 521, 499, 547, 555, 541, 548, 559, 470, 487, 481, 537, 548, 490, 506, 524, 497, 507, 513, 530, 488, 485, 546, 502] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[506, 519, 527, 508, 489, 517, 506, 564, 527, 479, 510, 524, 499, 511, 498, 504, 491, 534, 539, 520, 519, 497, 509, 483, 511, 560, 509, 519, 526, 498, 494, 553, 521, 499, 546, 513, 495, 511, 519, 530, 525, 535, 493, 515, 487, 529, 517, 462, 507, 518, 482, 517, 536, 514, 502, 521, 519, 493, 491, 522, 549, 518, 531, 530, 503, 536, 491, 520, 503, 535, 524, 509, 562, 487, 502, 524, 504, 491, 507, 552, 505, 508, 537, 503, 512, 516, 491, 487, 528, 473, 505, 505, 571, 529, 498, 516, 526, 485, 539, 505, 487, 466, 519, 525, 488, 520, 468, 523, 549, 458, 502, 494, 519, 499, 520, 500, 500, 526, 495, 529, 537, 501, 499, 542, 519, 468, 494, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[493, 482, 495, 494, 531, 514, 546, 534, 495, 466, 506, 491, 516, 546, 523, 525, 538, 498, 510, 529, 542, 498, 514, 516, 507, 511, 496, 524, 501, 517, 551, 512, 528, 530, 526, 508, 519, 521, 497, 494, 471, 510, 531, 502, 514, 523, 500, 521, 486, 490, 497, 483, 480, 528, 523, 564, 501, 519, 553, 507, 474, 563, 507, 526, 491, 509, 515, 574, 495, 505, 499, 541, 509, 494, 516, 532, 520, 514, 548, 537, 504, 507, 510, 493, 489, 498, 491, 515, 505, 533, 500, 537, 497, 505, 507, 521, 502, 498, 503, 515, 545, 520, 512, 519, 518, 504, 491, 497, 490, 523, 493, 497, 507, 554, 494, 495, 531, 514, 505, 482, 534, 491, 513, 495, 520, 486, 514, 546] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[533, 490, 507, 522, 528, 495, 511, 483, 516, 508, 497, 490, 497, 498, 524, 503, 514, 552, 484, 518, 532, 534, 547, 517, 518, 490, 504, 475, 506, 474, 502, 443, 506, 530, 526, 524, 527, 574, 528, 506, 530, 515, 507, 516, 498, 504, 519, 499, 471, 543, 507, 486, 504, 489, 532, 506, 502, 517, 504, 522, 496, 510, 546, 519, 527, 492, 502, 498, 542, 541, 504, 507, 512, 489, 499, 523, 510, 528, 527, 528, 484, 503, 522, 535, 502, 530, 507, 486, 553, 531, 543, 530, 543, 511, 516, 497, 550, 504, 532, 486, 492, 523, 532, 537, 511, 532, 493, 540, 471, 533, 531, 512, 503, 534, 528, 500, 468, 512, 518, 454, 499, 492, 496, 509, 503, 515, 516, 513] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[543, 572, 534, 541, 516, 514, 527, 513, 512, 526, 487, 512, 530, 507, 512, 525, 538, 507, 531, 500, 514, 540, 525, 482, 529, 526, 520, 516, 513, 525, 509, 461, 503, 555, 503, 513, 512, 482, 507, 547, 502, 509, 521, 541, 499, 522, 498, 514, 515, 475, 524, 536, 499, 523, 491, 543, 499, 506, 488, 524, 477, 525, 511, 488, 529, 533, 493, 501, 543, 549, 500, 548, 504, 483, 480, 505, 511, 506, 491, 475, 481, 455, 503, 553, 514, 512, 507, 487, 511, 517, 481, 513, 482, 485, 496, 517, 534, 538, 488, 536, 512, 544, 523, 499, 494, 509, 501, 482, 486, 506, 551, 516, 473, 488, 529, 471, 496, 526, 522, 528, 498, 514, 519, 517, 520, 530, 516, 546] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[508, 494, 523, 515, 508, 524, 552, 512, 510, 508, 473, 500, 516, 500, 499, 517, 538, 511, 536, 501, 523, 502, 490, 530, 481, 496, 538, 533, 467, 505, 510, 524, 538, 484, 519, 514, 517, 525, 484, 517, 536, 536, 523, 513, 487, 508, 506, 527, 533, 509, 517, 530, 529, 549, 527, 515, 493, 456, 535, 534, 498, 530, 534, 527, 508, 477, 514, 502, 540, 549, 513, 535, 499, 491, 512, 518, 506, 496, 514, 477, 514, 523, 495, 513, 506, 497, 530, 503, 528, 567, 521, 516, 502, 498, 536, 530, 485, 496, 503, 471, 499, 512, 499, 522, 506, 497, 478, 503, 521, 487, 515, 518, 524, 466, 561, 494, 505, 502, 484, 523, 531, 495, 534, 542, 496, 512, 529, 502] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[515, 510, 493, 477, 544, 507, 558, 499, 510, 553, 521, 481, 517, 509, 534, 521, 524, 509, 493, 533, 530, 497, 497, 523, 506, 465, 485, 501, 497, 521, 515, 499, 506, 523, 542, 533, 497, 550, 575, 516, 517, 524, 499, 528, 488, 531, 525, 509, 518, 480, 534, 509, 488, 494, 539, 524, 523, 461, 505, 489, 497, 529, 518, 504, 521, 520, 519, 530, 547, 532, 491, 516, 524, 518, 534, 563, 480, 487, 501, 497, 517, 535, 512, 555, 514, 541, 517, 511, 520, 507, 491, 510, 491, 514, 519, 491, 501, 506, 495, 492, 452, 541, 546, 500, 527, 489, 501, 495, 494, 499, 468, 516, 542, 518, 519, 518, 496, 511, 517, 487, 512, 491, 512, 527, 479, 522, 494, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[533, 490, 507, 522, 528, 495, 511, 483, 516, 508, 497, 490, 497, 498, 524, 503, 514, 552, 484, 518, 532, 534, 547, 517, 518, 490, 504, 475, 506, 474, 502, 443, 506, 530, 526, 524, 527, 574, 528, 506, 530, 515, 507, 516, 498, 504, 519, 499, 471, 543, 507, 486, 504, 489, 532, 506, 502, 517, 504, 522, 496, 510, 546, 519, 527, 492, 502, 498, 542, 541, 504, 507, 512, 489, 499, 523, 510, 528, 527, 528, 484, 503, 522, 535, 502, 530, 507, 486, 553, 531, 543, 530, 543, 511, 516, 497, 550, 504, 532, 486, 492, 523, 532, 537, 511, 532, 493, 540, 471, 533, 531, 512, 503, 534, 528, 500, 468, 512, 518, 454, 499, 492, 496, 509, 503, 515, 516, 513] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[518, 507, 515, 523, 554, 516, 508, 497, 490, 510, 525, 530, 496, 533, 506, 505, 514, 504, 535, 515, 506, 493, 505, 519, 523, 525, 478, 521, 500, 473, 506, 495, 513, 525, 511, 549, 552, 508, 523, 490, 491, 506, 517, 503, 490, 514, 493, 514, 486, 495, 534, 516, 527, 456, 515, 509, 519, 536, 518, 548, 527, 486, 520, 513, 504, 502, 496, 502, 509, 489, 524, 526, 467, 500, 513, 512, 485, 521, 540, 522, 516, 503, 470, 515, 463, 514, 531, 517, 478, 497, 515, 542, 459, 498, 529, 537, 504, 563, 538, 542, 496, 497, 510, 529, 531, 521, 499, 547, 555, 541, 548, 559, 470, 487, 481, 537, 548, 490, 506, 524, 497, 507, 513, 530, 488, 485, 546, 502] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[527, 507, 575, 525, 539, 506, 486, 499, 481, 497, 519, 555, 507, 513, 506, 523, 528, 539, 520, 516, 506, 464, 492, 539, 497, 518, 525, 532, 503, 526, 518, 506, 482, 510, 524, 530, 492, 522, 492, 500, 504, 514, 544, 497, 522, 493, 529, 523, 484, 508, 540, 486, 507, 522, 513, 480, 504, 514, 544, 516, 481, 546, 479, 508, 521, 476, 533, 459, 529, 501, 525, 484, 511, 536, 539, 533, 518, 542, 582, 514, 470, 531, 465, 504, 484, 517, 541, 529, 505, 510, 520, 504, 551, 506, 484, 498, 471, 505, 496, 484, 531, 561, 515, 523, 483, 471, 515, 517, 475, 505, 461, 530, 497, 547, 519, 521, 509, 494, 528, 515, 518, 569, 466, 504, 518, 543, 527, 492] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[506, 519, 527, 508, 470, 517, 506, 564, 527, 479, 510, 524, 499, 511, 498, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 534, 539, 520, 519, 497, 509, 483, 511, 560, 509, 519, 526, 498, 494, 553, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 499, 499, 513, 495, 511, 519, 530, 525, 535, 493, 515, 487, 529, 517, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 518, 482, 517, 536, 514, 502, 521, 519, 493, 491, 522, 549, 518, 531, 530, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 536, 491, 520, 503, 535, 524, 509, 562, 487, 502, 495, 504, 491, 507, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 508, 537, 503, 512, 516, 491, 487, 528, 473, 505, 505, 571, 529, 498, 516, 17, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 485, 539, 505, 487, 466, 519, 525, 488, 520, 468, 523, 549, 458, 502, 494, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 499, 520, 500, 500, 526, 495, 529, 537, 501, 499, 542, 519, 468, 494, 503, 11, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8169, 8266, 8150, 8254, 8221, 8209, 8093, 8174] avg=8192.0000 max=8266 max_rank=1 min=8093 min_rank=6 imbalance_ratio=1.009033 max_load_reduction=57270 max_load_reduction_pct=0.873871 moved_to_echo=32697 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[493, 482, 495, 494, 531, 514, 546, 534, 495, 466, 506, 491, 516, 546, 523, 525, 538, 498, 510, 529, 542, 498, 514, 516, 507, 511, 496, 524, 501, 517, 551, 512, 528, 530, 526, 508, 519, 521, 497, 494, 471, 510, 531, 502, 514, 523, 500, 521, 486, 490, 497, 483, 480, 528, 523, 564, 501, 519, 553, 507, 474, 563, 507, 526, 491, 509, 515, 574, 495, 505, 499, 541, 509, 494, 516, 532, 520, 514, 548, 537, 504, 507, 510, 493, 489, 498, 491, 515, 505, 533, 500, 537, 497, 505, 507, 521, 502, 498, 503, 515, 545, 520, 512, 519, 518, 504, 491, 497, 490, 523, 493, 497, 507, 554, 494, 495, 531, 514, 505, 482, 534, 491, 513, 495, 520, 486, 514, 546] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[508, 494, 523, 515, 508, 524, 552, 512, 510, 508, 473, 500, 516, 500, 499, 517, 538, 511, 536, 501, 523, 502, 490, 530, 481, 496, 538, 533, 467, 505, 510, 524, 538, 484, 519, 514, 517, 525, 484, 517, 536, 536, 523, 513, 487, 508, 506, 527, 533, 509, 517, 530, 529, 549, 527, 515, 493, 456, 535, 534, 498, 530, 534, 527, 508, 477, 514, 502, 540, 549, 513, 535, 499, 491, 512, 518, 506, 496, 514, 477, 514, 523, 495, 513, 506, 497, 530, 503, 528, 567, 521, 516, 502, 498, 536, 530, 485, 496, 503, 471, 499, 512, 499, 522, 506, 497, 478, 503, 521, 487, 515, 518, 524, 466, 561, 494, 505, 502, 484, 523, 531, 495, 534, 542, 496, 512, 529, 502] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[506, 519, 527, 508, 489, 517, 506, 564, 527, 479, 510, 524, 499, 511, 498, 504, 491, 534, 539, 520, 519, 497, 509, 483, 511, 560, 509, 519, 526, 498, 494, 553, 521, 499, 546, 513, 495, 511, 519, 530, 525, 535, 493, 515, 487, 529, 517, 462, 507, 518, 482, 517, 536, 514, 502, 521, 519, 493, 491, 522, 549, 518, 531, 530, 503, 536, 491, 520, 503, 535, 524, 509, 562, 487, 502, 524, 504, 491, 507, 552, 505, 508, 537, 503, 512, 516, 491, 487, 528, 473, 505, 505, 571, 529, 498, 516, 526, 485, 539, 505, 487, 466, 519, 525, 488, 520, 468, 523, 549, 458, 502, 494, 519, 499, 520, 500, 500, 526, 495, 529, 537, 501, 499, 542, 519, 468, 494, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[543, 572, 534, 541, 494, 514, 527, 513, 512, 526, 487, 512, 530, 507, 512, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 507, 531, 500, 514, 540, 525, 482, 529, 526, 520, 516, 513, 525, 509, 461, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 555, 460, 513, 512, 482, 507, 547, 502, 509, 521, 541, 499, 522, 498, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 475, 524, 536, 499, 523, 491, 543, 499, 506, 488, 524, 477, 525, 511, 488, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 533, 493, 501, 543, 549, 500, 548, 504, 483, 480, 478, 511, 506, 491, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 455, 503, 553, 514, 512, 507, 487, 511, 517, 481, 513, 482, 485, 496, 517, 16, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 538, 488, 536, 512, 544, 523, 499, 494, 509, 501, 482, 486, 506, 551, 516, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 488, 529, 471, 496, 526, 522, 528, 498, 514, 519, 517, 520, 530, 516, 546, 12, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8349, 8241, 8185, 8129, 8124, 8038, 8254, 8216] avg=8192.0000 max=8349 max_rank=0 min=8038 min_rank=5 imbalance_ratio=1.019165 max_load_reduction=57187 max_load_reduction_pct=0.872604 moved_to_echo=32632 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[533, 490, 507, 522, 494, 495, 511, 483, 516, 508, 497, 490, 497, 498, 524, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 552, 484, 518, 532, 534, 547, 517, 518, 490, 504, 475, 506, 474, 502, 443, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 530, 474, 524, 527, 574, 528, 506, 530, 515, 507, 516, 498, 504, 519, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 543, 507, 486, 504, 489, 532, 506, 502, 517, 504, 522, 496, 510, 546, 519, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 492, 502, 498, 542, 541, 504, 507, 512, 489, 499, 486, 510, 528, 527, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 503, 522, 535, 502, 530, 507, 486, 553, 531, 543, 530, 543, 511, 516, 497, 21, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 504, 532, 486, 492, 523, 532, 537, 511, 532, 493, 540, 471, 533, 531, 512, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 534, 528, 500, 468, 512, 518, 454, 499, 492, 496, 509, 503, 515, 516, 513, 17, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8068, 8119, 8257, 8162, 8192, 8325, 8320, 8093] avg=8192.0000 max=8325 max_rank=5 min=8068 min_rank=0 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32930 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[515, 510, 493, 477, 521, 507, 558, 499, 510, 553, 521, 481, 517, 509, 534, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 509, 493, 533, 530, 497, 497, 523, 506, 465, 485, 501, 497, 521, 515, 499, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 523, 496, 533, 497, 550, 575, 516, 517, 524, 499, 528, 488, 531, 525, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 480, 534, 509, 488, 494, 539, 524, 523, 461, 505, 489, 497, 529, 518, 504, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 520, 519, 530, 547, 532, 491, 516, 524, 518, 534, 532, 480, 487, 501, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 535, 512, 555, 514, 541, 517, 511, 520, 507, 491, 510, 491, 514, 519, 491, 18, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 506, 495, 492, 452, 541, 546, 500, 527, 489, 501, 495, 494, 499, 468, 516, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 518, 519, 518, 496, 511, 517, 487, 512, 491, 512, 527, 479, 522, 494, 525, 13, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 8100, 8317, 8117, 8249, 8271, 8060, 8196] avg=8192.0000 max=8317 max_rank=2 min=8060 min_rank=6 imbalance_ratio=1.015259 max_load_reduction=57219 max_load_reduction_pct=0.873093 moved_to_echo=32776 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[493, 482, 495, 494, 509, 514, 546, 534, 495, 466, 506, 491, 516, 546, 523, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 498, 510, 529, 542, 498, 514, 516, 507, 511, 496, 524, 501, 517, 551, 512, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 530, 481, 508, 519, 521, 497, 494, 471, 510, 531, 502, 514, 523, 500, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 490, 497, 483, 480, 528, 523, 564, 501, 519, 553, 507, 474, 563, 507, 526, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 509, 515, 574, 495, 505, 499, 541, 509, 494, 516, 503, 520, 514, 548, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 507, 510, 493, 489, 498, 491, 515, 505, 533, 500, 537, 497, 505, 507, 521, 17, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 498, 503, 515, 545, 520, 512, 519, 518, 504, 491, 497, 490, 523, 493, 497, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 554, 494, 495, 531, 514, 505, 482, 534, 491, 513, 495, 520, 486, 514, 546, 12, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8135, 8269, 8150, 8206, 8270, 8137, 8164, 8205] avg=8192.0000 max=8270 max_rank=4 min=8135 min_rank=0 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32776 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[543, 572, 534, 541, 494, 514, 527, 513, 512, 526, 487, 512, 530, 507, 512, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 507, 531, 500, 514, 540, 525, 482, 529, 526, 520, 516, 513, 525, 509, 461, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 555, 460, 513, 512, 482, 507, 547, 502, 509, 521, 541, 499, 522, 498, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 475, 524, 536, 499, 523, 491, 543, 499, 506, 488, 524, 477, 525, 511, 488, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 533, 493, 501, 543, 549, 500, 548, 504, 483, 480, 478, 511, 506, 491, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 455, 503, 553, 514, 512, 507, 487, 511, 517, 481, 513, 482, 485, 496, 517, 16, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 538, 488, 536, 512, 544, 523, 499, 494, 509, 501, 482, 486, 506, 551, 516, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 488, 529, 471, 496, 526, 522, 528, 498, 514, 519, 517, 520, 530, 516, 546, 12, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8349, 8241, 8185, 8129, 8124, 8038, 8254, 8216] avg=8192.0000 max=8349 max_rank=0 min=8038 min_rank=5 imbalance_ratio=1.019165 max_load_reduction=57187 max_load_reduction_pct=0.872604 moved_to_echo=32632 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[518, 507, 515, 523, 531, 516, 508, 497, 490, 510, 525, 530, 496, 533, 506, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 504, 535, 515, 506, 493, 505, 519, 523, 525, 478, 521, 500, 473, 506, 495, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 525, 467, 549, 552, 508, 523, 490, 491, 506, 517, 503, 490, 514, 493, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 495, 534, 516, 527, 456, 515, 509, 519, 536, 518, 548, 527, 486, 520, 513, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 502, 496, 502, 509, 489, 524, 526, 467, 500, 513, 484, 485, 521, 540, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 503, 470, 515, 463, 514, 531, 517, 478, 497, 515, 542, 459, 498, 529, 537, 16, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 563, 538, 542, 496, 497, 510, 529, 531, 521, 499, 547, 555, 541, 548, 559, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 487, 481, 537, 548, 490, 506, 524, 497, 507, 513, 530, 488, 485, 546, 502, 13, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8117, 8155, 8210, 8084, 8108, 8516, 8136] avg=8192.0000 max=8516 max_rank=6 min=8084 min_rank=4 imbalance_ratio=1.039551 max_load_reduction=57020 max_load_reduction_pct=0.870056 moved_to_echo=32844 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[508, 494, 523, 515, 486, 524, 552, 512, 510, 508, 473, 500, 516, 500, 499, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 511, 536, 501, 523, 502, 490, 530, 481, 496, 538, 533, 467, 505, 510, 524, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 484, 474, 514, 517, 525, 484, 517, 536, 536, 523, 513, 487, 508, 506, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 509, 517, 530, 529, 549, 527, 515, 493, 456, 535, 534, 498, 530, 534, 527, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 477, 514, 502, 540, 549, 513, 535, 499, 491, 512, 489, 506, 496, 514, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 523, 495, 513, 506, 497, 530, 503, 528, 567, 521, 516, 502, 498, 536, 530, 17, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 496, 503, 471, 499, 512, 499, 522, 506, 497, 478, 503, 521, 487, 515, 518, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 466, 561, 494, 505, 502, 484, 523, 531, 495, 534, 542, 496, 512, 529, 502, 12, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8137, 8190, 8189, 8321, 8122, 8304, 8049, 8224] avg=8192.0000 max=8321 max_rank=3 min=8049 min_rank=6 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32699 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[527, 507, 575, 525, 516, 506, 486, 499, 481, 497, 519, 555, 507, 513, 506, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 539, 520, 516, 506, 464, 492, 539, 497, 518, 525, 532, 503, 526, 518, 506, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 510, 479, 530, 492, 522, 492, 500, 504, 514, 544, 497, 522, 493, 529, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 508, 540, 486, 507, 522, 513, 480, 504, 514, 544, 516, 481, 546, 479, 508, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 476, 533, 459, 529, 501, 525, 484, 511, 536, 539, 504, 518, 542, 582, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 531, 465, 504, 484, 517, 541, 529, 505, 510, 520, 504, 551, 506, 484, 498, 17, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 505, 496, 484, 531, 561, 515, 523, 483, 471, 515, 517, 475, 505, 461, 530, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 547, 519, 521, 509, 494, 528, 515, 518, 569, 466, 504, 518, 543, 527, 492, 13, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8242, 8234, 8133, 8137, 8274, 8144, 8080, 8292] avg=8192.0000 max=8292 max_rank=7 min=8080 min_rank=6 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32790 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[515, 510, 493, 477, 521, 507, 558, 499, 510, 553, 521, 481, 517, 509, 534, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 509, 493, 533, 530, 497, 497, 523, 506, 465, 485, 501, 497, 521, 515, 499, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 523, 496, 533, 497, 550, 575, 516, 517, 524, 499, 528, 488, 531, 525, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 480, 534, 509, 488, 494, 539, 524, 523, 461, 505, 489, 497, 529, 518, 504, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 520, 519, 530, 547, 532, 491, 516, 524, 518, 534, 532, 480, 487, 501, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 535, 512, 555, 514, 541, 517, 511, 520, 507, 491, 510, 491, 514, 519, 491, 18, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 506, 495, 492, 452, 541, 546, 500, 527, 489, 501, 495, 494, 499, 468, 516, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 518, 519, 518, 496, 511, 517, 487, 512, 491, 512, 527, 479, 522, 494, 525, 13, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 8100, 8317, 8117, 8249, 8271, 8060, 8196] avg=8192.0000 max=8317 max_rank=2 min=8060 min_rank=6 imbalance_ratio=1.015259 max_load_reduction=57219 max_load_reduction_pct=0.873093 moved_to_echo=32776 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[508, 494, 523, 515, 486, 524, 552, 512, 510, 508, 473, 500, 516, 500, 499, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 511, 536, 501, 523, 502, 490, 530, 481, 496, 538, 533, 467, 505, 510, 524, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 484, 474, 514, 517, 525, 484, 517, 536, 536, 523, 513, 487, 508, 506, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 509, 517, 530, 529, 549, 527, 515, 493, 456, 535, 534, 498, 530, 534, 527, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 477, 514, 502, 540, 549, 513, 535, 499, 491, 512, 489, 506, 496, 514, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 523, 495, 513, 506, 497, 530, 503, 528, 567, 521, 516, 502, 498, 536, 530, 17, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 496, 503, 471, 499, 512, 499, 522, 506, 497, 478, 503, 521, 487, 515, 518, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 466, 561, 494, 505, 502, 484, 523, 531, 495, 534, 542, 496, 512, 529, 502, 12, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8137, 8190, 8189, 8321, 8122, 8304, 8049, 8224] avg=8192.0000 max=8321 max_rank=3 min=8049 min_rank=6 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32699 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[506, 519, 527, 508, 470, 517, 506, 564, 527, 479, 510, 524, 499, 511, 498, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 534, 539, 520, 519, 497, 509, 483, 511, 560, 509, 519, 526, 498, 494, 553, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 499, 499, 513, 495, 511, 519, 530, 525, 535, 493, 515, 487, 529, 517, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 518, 482, 517, 536, 514, 502, 521, 519, 493, 491, 522, 549, 518, 531, 530, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 536, 491, 520, 503, 535, 524, 509, 562, 487, 502, 495, 504, 491, 507, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 508, 537, 503, 512, 516, 491, 487, 528, 473, 505, 505, 571, 529, 498, 516, 17, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 485, 539, 505, 487, 466, 519, 525, 488, 520, 468, 523, 549, 458, 502, 494, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 499, 520, 500, 500, 526, 495, 529, 537, 501, 499, 542, 519, 468, 494, 503, 11, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8169, 8266, 8150, 8254, 8221, 8209, 8093, 8174] avg=8192.0000 max=8266 max_rank=1 min=8093 min_rank=6 imbalance_ratio=1.009033 max_load_reduction=57270 max_load_reduction_pct=0.873871 moved_to_echo=32697 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[518, 507, 515, 523, 531, 516, 508, 497, 490, 510, 525, 530, 496, 533, 506, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 504, 535, 515, 506, 493, 505, 519, 523, 525, 478, 521, 500, 473, 506, 495, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 525, 467, 549, 552, 508, 523, 490, 491, 506, 517, 503, 490, 514, 493, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 495, 534, 516, 527, 456, 515, 509, 519, 536, 518, 548, 527, 486, 520, 513, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 502, 496, 502, 509, 489, 524, 526, 467, 500, 513, 484, 485, 521, 540, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 503, 470, 515, 463, 514, 531, 517, 478, 497, 515, 542, 459, 498, 529, 537, 16, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 563, 538, 542, 496, 497, 510, 529, 531, 521, 499, 547, 555, 541, 548, 559, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 487, 481, 537, 548, 490, 506, 524, 497, 507, 513, 530, 488, 485, 546, 502, 13, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8117, 8155, 8210, 8084, 8108, 8516, 8136] avg=8192.0000 max=8516 max_rank=6 min=8084 min_rank=4 imbalance_ratio=1.039551 max_load_reduction=57020 max_load_reduction_pct=0.870056 moved_to_echo=32844 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 3968, 4224, 4096, 4096, 4096, 4224, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 3840, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 3968, 4224, 4096, 4096, 4096, 4224, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[541, 482, 466, 495, 502, 528, 502, 532, 473, 557, 544, 556, 504, 491, 529, 491, 490, 505, 518, 496, 499, 479, 555, 532, 515, 542, 501, 519, 541, 501, 501, 501, 561, 492, 500, 514, 535, 496, 530, 540, 514, 507, 479, 493, 513, 524, 514, 500, 509, 515, 500, 498, 556, 536, 505, 460, 541, 489, 488, 529, 514, 535, 535, 508, 513, 510, 493, 526, 513, 515, 511, 525, 531, 498, 512, 522, 521, 508, 534, 536, 505, 516, 511, 485, 521, 513, 530, 480, 503, 533, 510, 501, 511, 488, 460, 470, 521, 489, 546, 511, 504, 526, 533, 521, 517, 479, 520, 529, 512, 494, 537, 497, 527, 490, 507, 554, 525, 496, 512, 486, 497, 494, 504, 517, 507, 517, 510, 534] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[518, 481, 500, 483, 527, 500, 512, 512, 529, 534, 503, 496, 534, 506, 535, 524, 506, 529, 489, 494, 537, 540, 500, 489, 519, 539, 523, 530, 502, 527, 492, 509, 508, 507, 514, 498, 524, 494, 474, 520, 530, 542, 473, 476, 517, 539, 522, 494, 519, 542, 513, 505, 557, 507, 529, 513, 508, 535, 456, 515, 517, 512, 497, 522, 489, 529, 517, 518, 521, 510, 527, 528, 513, 573, 538, 451, 539, 530, 476, 519, 462, 532, 501, 487, 518, 540, 516, 508, 525, 543, 491, 487, 513, 488, 494, 478, 527, 481, 502, 526, 509, 506, 521, 528, 497, 506, 516, 503, 548, 546, 500, 498, 455, 556, 490, 481, 484, 519, 468, 526, 514, 511, 522, 515, 565, 509, 533, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[483, 534, 509, 472, 514, 478, 535, 516, 485, 520, 483, 492, 489, 538, 516, 542, 524, 527, 514, 506, 501, 479, 485, 525, 531, 494, 519, 524, 500, 538, 521, 484, 522, 520, 497, 553, 507, 519, 542, 516, 533, 486, 560, 521, 503, 517, 512, 490, 519, 517, 473, 517, 550, 503, 539, 524, 530, 481, 525, 543, 532, 511, 511, 556, 527, 476, 508, 540, 544, 522, 573, 479, 507, 515, 513, 524, 550, 509, 564, 486, 499, 510, 517, 486, 502, 474, 487, 514, 471, 519, 505, 520, 495, 502, 520, 485, 523, 504, 497, 527, 528, 483, 502, 508, 527, 494, 506, 538, 478, 520, 487, 501, 505, 540, 513, 471, 497, 501, 510, 485, 490, 548, 528, 522, 491, 520, 517, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[524, 498, 534, 555, 522, 494, 504, 516, 490, 494, 559, 484, 536, 518, 486, 453, 509, 526, 535, 530, 480, 531, 517, 507, 492, 489, 500, 508, 484, 508, 506, 498, 472, 511, 525, 504, 496, 502, 487, 536, 494, 535, 528, 539, 499, 482, 488, 541, 521, 525, 536, 485, 509, 514, 522, 502, 577, 507, 517, 523, 527, 542, 495, 514, 534, 484, 494, 510, 501, 501, 515, 483, 511, 521, 519, 511, 560, 503, 518, 474, 500, 486, 492, 496, 566, 535, 511, 477, 543, 535, 532, 493, 492, 504, 511, 481, 525, 543, 527, 495, 507, 558, 534, 531, 501, 535, 540, 527, 486, 486, 492, 499, 514, 532, 514, 454, 494, 530, 521, 493, 554, 539, 498, 506, 488, 559, 517, 502] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[520, 491, 502, 524, 511, 540, 519, 513, 500, 517, 500, 496, 503, 522, 519, 510, 538, 518, 529, 523, 497, 517, 492, 509, 521, 522, 519, 507, 475, 520, 503, 493, 540, 531, 515, 534, 517, 527, 530, 514, 514, 525, 524, 548, 497, 521, 518, 523, 518, 551, 520, 504, 504, 504, 509, 499, 503, 503, 559, 503, 526, 472, 521, 495, 537, 510, 524, 510, 536, 521, 525, 518, 497, 508, 536, 511, 534, 523, 511, 515, 504, 484, 516, 474, 530, 531, 501, 528, 516, 498, 533, 481, 515, 531, 526, 543, 513, 473, 493, 535, 517, 497, 520, 508, 538, 477, 481, 537, 513, 494, 521, 507, 444, 477, 488, 508, 521, 529, 522, 499, 501, 478, 523, 490, 469, 516, 505, 476] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[502, 553, 491, 488, 524, 543, 521, 512, 553, 519, 476, 511, 540, 470, 547, 510, 477, 516, 520, 514, 484, 525, 519, 516, 532, 477, 495, 550, 490, 524, 526, 543, 504, 541, 471, 545, 512, 523, 498, 532, 514, 514, 513, 525, 546, 488, 518, 491, 505, 491, 514, 524, 501, 496, 497, 515, 524, 486, 504, 512, 515, 553, 502, 506, 503, 517, 524, 516, 520, 509, 508, 554, 525, 504, 472, 494, 528, 499, 491, 483, 531, 494, 511, 530, 501, 508, 526, 505, 501, 518, 521, 504, 554, 501, 488, 518, 540, 482, 528, 517, 513, 505, 521, 476, 553, 514, 542, 504, 524, 535, 526, 524, 513, 522, 508, 491, 524, 468, 466, 497, 473, 484, 519, 512, 490, 532, 476, 551] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[470, 523, 522, 521, 486, 501, 511, 519, 533, 497, 493, 540, 525, 498, 529, 548, 511, 550, 523, 495, 543, 557, 496, 492, 499, 571, 495, 499, 533, 527, 476, 517, 524, 530, 469, 518, 551, 496, 513, 524, 518, 537, 550, 491, 526, 483, 513, 526, 551, 522, 501, 482, 535, 499, 526, 493, 528, 480, 492, 537, 513, 489, 512, 506, 530, 493, 521, 505, 506, 550, 495, 498, 522, 530, 527, 529, 510, 533, 536, 481, 506, 524, 505, 476, 465, 517, 484, 528, 524, 501, 496, 535, 483, 517, 563, 528, 565, 516, 487, 469, 545, 494, 509, 484, 490, 501, 486, 461, 508, 508, 492, 519, 510, 532, 541, 479, 472, 518, 505, 526, 498, 489, 510, 513, 549, 507, 498, 502] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[515, 534, 527, 473, 514, 514, 504, 483, 505, 527, 505, 500, 528, 493, 473, 507, 553, 468, 517, 471, 517, 526, 504, 538, 493, 522, 489, 507, 532, 514, 492, 546, 538, 495, 529, 504, 500, 487, 516, 537, 526, 510, 506, 479, 483, 511, 505, 560, 563, 506, 500, 477, 536, 504, 529, 504, 527, 505, 523, 505, 514, 520, 498, 492, 536, 538, 522, 487, 489, 526, 501, 486, 502, 564, 533, 526, 494, 496, 567, 524, 533, 522, 503, 532, 539, 556, 483, 528, 518, 476, 501, 495, 484, 495, 557, 482, 518, 534, 528, 476, 515, 486, 510, 493, 483, 498, 547, 476, 539, 521, 541, 542, 563, 490, 478, 485, 528, 510, 549, 467, 523, 541, 491, 495, 515, 478, 509, 532] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[520, 491, 502, 524, 511, 540, 519, 513, 500, 517, 500, 496, 503, 522, 519, 510, 538, 518, 529, 523, 497, 517, 492, 509, 521, 522, 519, 507, 475, 520, 503, 493, 540, 531, 515, 534, 517, 527, 530, 514, 514, 525, 524, 548, 497, 521, 518, 523, 518, 551, 520, 504, 504, 504, 509, 499, 503, 503, 559, 503, 526, 472, 521, 495, 537, 510, 524, 510, 536, 521, 525, 518, 497, 508, 536, 511, 534, 523, 511, 515, 504, 484, 516, 474, 530, 531, 501, 528, 516, 498, 533, 481, 515, 531, 526, 543, 513, 473, 493, 535, 517, 497, 520, 508, 538, 477, 481, 537, 513, 494, 521, 507, 444, 477, 488, 508, 521, 529, 522, 499, 501, 478, 523, 490, 469, 516, 505, 476] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[470, 523, 522, 521, 486, 501, 511, 519, 533, 497, 493, 540, 525, 498, 529, 548, 511, 550, 523, 495, 543, 557, 496, 492, 499, 571, 495, 499, 533, 527, 476, 517, 524, 530, 469, 518, 551, 496, 513, 524, 518, 537, 550, 491, 526, 483, 513, 526, 551, 522, 501, 482, 535, 499, 526, 493, 528, 480, 492, 537, 513, 489, 512, 506, 530, 493, 521, 505, 506, 550, 495, 498, 522, 530, 527, 529, 510, 533, 536, 481, 506, 524, 505, 476, 465, 517, 484, 528, 524, 501, 496, 535, 483, 517, 563, 528, 565, 516, 487, 469, 545, 494, 509, 484, 490, 501, 486, 461, 508, 508, 492, 519, 510, 532, 541, 479, 472, 518, 505, 526, 498, 489, 510, 513, 549, 507, 498, 502] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[524, 498, 534, 555, 522, 494, 504, 516, 490, 494, 559, 484, 536, 518, 486, 453, 509, 526, 535, 530, 480, 531, 517, 507, 492, 489, 500, 508, 484, 508, 506, 498, 472, 511, 525, 504, 496, 502, 487, 536, 494, 535, 528, 539, 499, 482, 488, 541, 521, 525, 536, 485, 509, 514, 522, 502, 577, 507, 517, 523, 527, 542, 495, 514, 534, 484, 494, 510, 501, 501, 515, 483, 511, 521, 519, 511, 560, 503, 518, 474, 500, 486, 492, 496, 566, 535, 511, 477, 543, 535, 532, 493, 492, 504, 511, 481, 525, 543, 527, 495, 507, 558, 534, 531, 501, 535, 540, 527, 486, 486, 492, 499, 514, 532, 514, 454, 494, 530, 521, 493, 554, 539, 498, 506, 488, 559, 517, 502] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[518, 481, 500, 483, 527, 500, 512, 512, 529, 534, 503, 496, 534, 506, 535, 524, 506, 529, 489, 494, 537, 540, 500, 489, 519, 539, 523, 530, 502, 527, 492, 509, 508, 507, 514, 498, 524, 494, 474, 520, 530, 542, 473, 476, 517, 539, 522, 494, 519, 542, 513, 505, 557, 507, 529, 513, 508, 535, 456, 515, 517, 512, 497, 522, 489, 529, 517, 518, 521, 510, 527, 528, 513, 573, 538, 451, 539, 530, 476, 519, 462, 532, 501, 487, 518, 540, 516, 508, 525, 543, 491, 487, 513, 488, 494, 478, 527, 481, 502, 526, 509, 506, 521, 528, 497, 506, 516, 503, 548, 546, 500, 498, 455, 556, 490, 481, 484, 519, 468, 526, 514, 511, 522, 515, 565, 509, 533, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[483, 534, 509, 472, 514, 478, 535, 516, 485, 520, 483, 492, 489, 538, 516, 542, 524, 527, 514, 506, 501, 479, 485, 525, 531, 494, 519, 524, 500, 538, 521, 484, 522, 520, 497, 553, 507, 519, 542, 516, 533, 486, 560, 521, 503, 517, 512, 490, 519, 517, 473, 517, 550, 503, 539, 524, 530, 481, 525, 543, 532, 511, 511, 556, 527, 476, 508, 540, 544, 522, 573, 479, 507, 515, 513, 524, 550, 509, 564, 486, 499, 510, 517, 486, 502, 474, 487, 514, 471, 519, 505, 520, 495, 502, 520, 485, 523, 504, 497, 527, 528, 483, 502, 508, 527, 494, 506, 538, 478, 520, 487, 501, 505, 540, 513, 471, 497, 501, 510, 485, 490, 548, 528, 522, 491, 520, 517, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[541, 482, 466, 495, 502, 528, 502, 532, 473, 557, 544, 556, 504, 491, 529, 491, 14, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 505, 518, 496, 499, 479, 555, 532, 515, 542, 501, 519, 541, 492, 501, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 492, 500, 514, 535, 496, 530, 493, 514, 507, 479, 493, 513, 524, 514, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 515, 500, 498, 512, 536, 505, 460, 541, 489, 488, 529, 514, 535, 535, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 510, 493, 526, 513, 515, 511, 525, 531, 498, 512, 522, 455, 508, 534, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 516, 511, 485, 521, 513, 530, 480, 503, 533, 510, 501, 511, 488, 460, 470, 38, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 489, 546, 511, 504, 526, 533, 521, 517, 479, 520, 529, 512, 494, 537, 497, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 490, 507, 554, 525, 496, 512, 486, 497, 494, 504, 517, 507, 517, 510, 534, 66, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8186, 8165, 8174, 8202, 8105, 8239, 8252] avg=8192.0000 max=8252 max_rank=7 min=8105 min_rank=5 imbalance_ratio=1.007324 max_load_reduction=57284 max_load_reduction_pct=0.874084 moved_to_echo=32798 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[502, 553, 491, 488, 524, 543, 521, 512, 553, 519, 476, 511, 540, 470, 547, 510, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 516, 520, 514, 484, 525, 519, 516, 532, 477, 495, 550, 490, 519, 526, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 541, 471, 545, 512, 523, 498, 494, 514, 514, 513, 525, 546, 488, 518, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 491, 514, 524, 469, 496, 497, 515, 524, 486, 504, 512, 515, 553, 502, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 517, 524, 516, 520, 509, 508, 554, 525, 504, 472, 494, 465, 499, 491, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 494, 511, 530, 501, 508, 526, 505, 501, 518, 521, 504, 554, 501, 488, 518, 34, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 482, 528, 517, 513, 505, 521, 476, 553, 514, 542, 504, 524, 535, 526, 524, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 522, 508, 491, 524, 468, 466, 497, 473, 484, 519, 512, 490, 532, 476, 551, 63, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8273, 8203, 8197, 8113, 8084, 8268, 8305, 8093] avg=8192.0000 max=8305 max_rank=6 min=8084 min_rank=4 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32750 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[520, 491, 502, 524, 511, 540, 519, 513, 500, 517, 500, 496, 503, 522, 519, 510, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 518, 529, 523, 497, 517, 492, 509, 521, 522, 519, 507, 475, 515, 503, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 531, 515, 534, 517, 527, 530, 477, 514, 525, 524, 548, 497, 521, 518, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 551, 520, 504, 472, 504, 509, 499, 503, 503, 559, 503, 526, 472, 521, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 510, 524, 510, 536, 521, 525, 518, 497, 508, 536, 511, 471, 523, 511, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 484, 516, 474, 530, 531, 501, 528, 516, 498, 533, 481, 515, 531, 526, 543, 33, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 473, 493, 535, 517, 497, 520, 508, 538, 477, 481, 537, 513, 494, 521, 507, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 444, 477, 488, 508, 521, 529, 522, 499, 501, 478, 523, 490, 469, 516, 505, 476, 63, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8178, 8341, 8159, 8253, 8267, 8125, 8013] avg=8192.0000 max=8341 max_rank=2 min=8013 min_rank=7 imbalance_ratio=1.018188 max_load_reduction=57195 max_load_reduction_pct=0.872726 moved_to_echo=32658 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[515, 534, 527, 473, 514, 514, 504, 483, 505, 527, 505, 500, 528, 493, 473, 507, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 468, 517, 471, 517, 526, 504, 538, 493, 522, 489, 507, 532, 511, 492, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 495, 529, 504, 500, 487, 516, 498, 526, 510, 506, 479, 483, 511, 505, 560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 563, 506, 500, 477, 502, 504, 529, 504, 527, 505, 523, 505, 514, 520, 498, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 538, 522, 487, 489, 526, 501, 486, 502, 564, 533, 526, 435, 496, 567, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 522, 503, 532, 539, 556, 483, 528, 518, 476, 501, 495, 484, 495, 557, 482, 35, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 534, 528, 476, 515, 486, 510, 493, 483, 498, 547, 476, 539, 521, 541, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 563, 490, 478, 485, 528, 510, 549, 467, 523, 541, 491, 495, 515, 478, 509, 532, 59, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8114, 8186, 8147, 8169, 8232, 8264, 8207, 8217] avg=8192.0000 max=8264 max_rank=5 min=8114 min_rank=0 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32920 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[518, 481, 500, 483, 527, 500, 512, 512, 529, 534, 503, 496, 534, 506, 535, 524, 10, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 529, 489, 494, 537, 540, 500, 489, 519, 539, 523, 530, 502, 522, 492, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 507, 514, 498, 524, 494, 474, 482, 530, 542, 473, 476, 517, 539, 522, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 542, 513, 505, 521, 507, 529, 513, 508, 535, 456, 515, 517, 512, 497, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 529, 517, 518, 521, 510, 527, 528, 513, 573, 538, 451, 475, 530, 476, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 532, 501, 487, 518, 540, 516, 508, 525, 543, 491, 487, 513, 488, 494, 478, 34, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 481, 502, 526, 509, 506, 521, 528, 497, 506, 516, 503, 548, 546, 500, 498, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 455, 556, 490, 481, 484, 519, 468, 526, 514, 511, 522, 515, 565, 509, 533, 515, 64, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8220, 8094, 8211, 8214, 8143, 8215, 8231] avg=8192.0000 max=8231 max_rank=7 min=8094 min_rank=2 imbalance_ratio=1.004761 max_load_reduction=57305 max_load_reduction_pct=0.874405 moved_to_echo=32803 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[524, 498, 534, 555, 522, 494, 504, 516, 490, 494, 559, 484, 536, 518, 486, 453, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 526, 535, 530, 480, 531, 517, 507, 492, 489, 500, 508, 484, 505, 506, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 511, 525, 504, 496, 502, 487, 497, 494, 535, 528, 539, 499, 482, 488, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 525, 536, 485, 476, 514, 522, 502, 577, 507, 517, 523, 527, 542, 495, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 484, 494, 510, 501, 501, 515, 483, 511, 521, 519, 511, 494, 503, 518, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 486, 492, 496, 566, 535, 511, 477, 543, 535, 532, 493, 492, 504, 511, 481, 35, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 543, 527, 495, 507, 558, 534, 531, 501, 535, 540, 527, 486, 486, 492, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 532, 514, 454, 494, 530, 521, 493, 554, 539, 498, 506, 488, 559, 517, 502, 66, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8117, 8100, 8283, 8073, 8213, 8286, 8285] avg=8192.0000 max=8286 max_rank=6 min=8073 min_rank=4 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32857 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[483, 534, 509, 472, 514, 478, 535, 516, 485, 520, 483, 492, 489, 538, 516, 542, 10, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 527, 514, 506, 501, 479, 485, 525, 531, 494, 519, 524, 500, 533, 521, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 520, 497, 553, 507, 519, 542, 479, 533, 486, 560, 521, 503, 517, 512, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 517, 473, 517, 514, 503, 539, 524, 530, 481, 525, 543, 532, 511, 511, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 476, 508, 540, 544, 522, 573, 479, 507, 515, 513, 524, 485, 509, 564, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 510, 517, 486, 502, 474, 487, 514, 471, 519, 505, 520, 495, 502, 520, 485, 33, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 504, 497, 527, 528, 483, 502, 508, 527, 494, 506, 538, 478, 520, 487, 501, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 540, 513, 471, 497, 501, 510, 485, 490, 548, 528, 522, 491, 520, 517, 525, 65, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8120, 8167, 8261, 8295, 8272, 8065, 8124, 8232] avg=8192.0000 max=8295 max_rank=3 min=8065 min_rank=5 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32693 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[502, 553, 491, 488, 524, 543, 521, 512, 553, 519, 476, 511, 540, 470, 547, 510, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 516, 520, 514, 484, 525, 519, 516, 532, 477, 495, 550, 490, 519, 526, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 541, 471, 545, 512, 523, 498, 494, 514, 514, 513, 525, 546, 488, 518, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 491, 514, 524, 469, 496, 497, 515, 524, 486, 504, 512, 515, 553, 502, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 517, 524, 516, 520, 509, 508, 554, 525, 504, 472, 494, 465, 499, 491, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 494, 511, 530, 501, 508, 526, 505, 501, 518, 521, 504, 554, 501, 488, 518, 34, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 482, 528, 517, 513, 505, 521, 476, 553, 514, 542, 504, 524, 535, 526, 524, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 522, 508, 491, 524, 468, 466, 497, 473, 484, 519, 512, 490, 532, 476, 551, 63, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8273, 8203, 8197, 8113, 8084, 8268, 8305, 8093] avg=8192.0000 max=8305 max_rank=6 min=8084 min_rank=4 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32750 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[470, 523, 522, 521, 486, 501, 511, 519, 533, 497, 493, 540, 525, 498, 529, 548, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 550, 523, 495, 543, 557, 496, 492, 499, 571, 495, 499, 533, 522, 476, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 530, 469, 518, 551, 496, 513, 486, 518, 537, 550, 491, 526, 483, 513, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 522, 501, 482, 501, 499, 526, 493, 528, 480, 492, 537, 513, 489, 512, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 493, 521, 505, 506, 550, 495, 498, 522, 530, 527, 529, 450, 533, 536, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 524, 505, 476, 465, 517, 484, 528, 524, 501, 496, 535, 483, 517, 563, 528, 34, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 565, 516, 487, 469, 545, 494, 509, 484, 490, 501, 486, 461, 508, 508, 492, 519, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 532, 541, 479, 472, 518, 505, 526, 498, 489, 510, 513, 549, 507, 498, 502, 60, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8229, 8279, 8231, 8132, 8206, 8211, 8035, 8213] avg=8192.0000 max=8279 max_rank=1 min=8035 min_rank=6 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32665 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[515, 534, 527, 473, 514, 514, 504, 483, 505, 527, 505, 500, 528, 493, 473, 507, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 468, 517, 471, 517, 526, 504, 538, 493, 522, 489, 507, 532, 511, 492, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 495, 529, 504, 500, 487, 516, 498, 526, 510, 506, 479, 483, 511, 505, 560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 563, 506, 500, 477, 502, 504, 529, 504, 527, 505, 523, 505, 514, 520, 498, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 538, 522, 487, 489, 526, 501, 486, 502, 564, 533, 526, 435, 496, 567, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 522, 503, 532, 539, 556, 483, 528, 518, 476, 501, 495, 484, 495, 557, 482, 35, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 534, 528, 476, 515, 486, 510, 493, 483, 498, 547, 476, 539, 521, 541, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 563, 490, 478, 485, 528, 510, 549, 467, 523, 541, 491, 495, 515, 478, 509, 532, 59, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8114, 8186, 8147, 8169, 8232, 8264, 8207, 8217] avg=8192.0000 max=8264 max_rank=5 min=8114 min_rank=0 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32920 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[524, 498, 534, 555, 522, 494, 504, 516, 490, 494, 559, 484, 536, 518, 486, 453, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 526, 535, 530, 480, 531, 517, 507, 492, 489, 500, 508, 484, 505, 506, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 511, 525, 504, 496, 502, 487, 497, 494, 535, 528, 539, 499, 482, 488, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 525, 536, 485, 476, 514, 522, 502, 577, 507, 517, 523, 527, 542, 495, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 484, 494, 510, 501, 501, 515, 483, 511, 521, 519, 511, 494, 503, 518, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 486, 492, 496, 566, 535, 511, 477, 543, 535, 532, 493, 492, 504, 511, 481, 35, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 543, 527, 495, 507, 558, 534, 531, 501, 535, 540, 527, 486, 486, 492, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 532, 514, 454, 494, 530, 521, 493, 554, 539, 498, 506, 488, 559, 517, 502, 66, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8117, 8100, 8283, 8073, 8213, 8286, 8285] avg=8192.0000 max=8286 max_rank=6 min=8073 min_rank=4 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32857 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[520, 491, 502, 524, 511, 540, 519, 513, 500, 517, 500, 496, 503, 522, 519, 510, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 518, 529, 523, 497, 517, 492, 509, 521, 522, 519, 507, 475, 515, 503, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 531, 515, 534, 517, 527, 530, 477, 514, 525, 524, 548, 497, 521, 518, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 551, 520, 504, 472, 504, 509, 499, 503, 503, 559, 503, 526, 472, 521, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 510, 524, 510, 536, 521, 525, 518, 497, 508, 536, 511, 471, 523, 511, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 484, 516, 474, 530, 531, 501, 528, 516, 498, 533, 481, 515, 531, 526, 543, 33, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 473, 493, 535, 517, 497, 520, 508, 538, 477, 481, 537, 513, 494, 521, 507, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 444, 477, 488, 508, 521, 529, 522, 499, 501, 478, 523, 490, 469, 516, 505, 476, 63, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8178, 8341, 8159, 8253, 8267, 8125, 8013] avg=8192.0000 max=8341 max_rank=2 min=8013 min_rank=7 imbalance_ratio=1.018188 max_load_reduction=57195 max_load_reduction_pct=0.872726 moved_to_echo=32658 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4352, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 3840, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 3968, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 4096, 3968, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 3840, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 3968, 4096, 4224, 4096, 4352, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 3968, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 384, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True - [2026-08-26 14:49:29] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 421.2 | throughput per GPU (TFLOP/s/GPU): 429.3 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.794035E+00 | loss scale: 1.0 | grad norm: 5.600 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[465, 541, 530, 517, 542, 521, 503, 512, 504, 515, 512, 513, 498, 509, 488, 500, 523, 482, 523, 500, 514, 512, 498, 538, 487, 523, 520, 459, 541, 523, 519, 508, 490, 536, 526, 510, 537, 504, 526, 576, 515, 506, 476, 508, 501, 531, 513, 529, 500, 494, 510, 528, 533, 550, 521, 495, 519, 511, 525, 508, 487, 534, 487, 557, 520, 533, 503, 495, 549, 501, 516, 508, 475, 485, 494, 529, 485, 450, 496, 514, 501, 515, 536, 503, 502, 491, 491, 516, 484, 529, 511, 544, 535, 541, 469, 538, 465, 473, 538, 502, 549, 521, 504, 515, 507, 510, 530, 505, 537, 479, 501, 507, 551, 496, 521, 510, 524, 528, 533, 517, 520, 541, 513, 500, 494, 511, 512, 480] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[510, 526, 528, 519, 559, 500, 500, 484, 529, 514, 487, 510, 511, 513, 480, 507, 532, 530, 532, 473, 516, 463, 511, 507, 528, 516, 506, 489, 523, 509, 485, 544, 485, 509, 482, 553, 557, 481, 526, 548, 514, 520, 553, 524, 496, 501, 560, 514, 490, 505, 510, 510, 546, 503, 476, 541, 516, 515, 476, 494, 533, 503, 556, 501, 516, 487, 550, 531, 526, 545, 530, 522, 510, 513, 546, 488, 489, 486, 526, 521, 528, 509, 500, 507, 497, 530, 485, 526, 511, 504, 502, 496, 476, 466, 529, 487, 496, 524, 498, 478, 489, 530, 503, 497, 497, 525, 516, 520, 536, 503, 533, 523, 542, 517, 526, 511, 503, 496, 505, 519, 510, 497, 476, 519, 562, 513, 507, 487] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 + [2026-08-26 15:25:33] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 413.8 | throughput per GPU (TFLOP/s/GPU): 437.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.801652E+00 | loss scale: 1.0 | grad norm: 5.603 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[510, 496, 502, 545, 487, 513, 433, 513, 500, 475, 531, 524, 504, 534, 529, 495, 481, 504, 536, 523, 506, 532, 553, 498, 500, 504, 506, 517, 528, 502, 519, 483, 500, 530, 533, 525, 512, 479, 533, 517, 538, 533, 495, 499, 515, 495, 524, 518, 494, 514, 504, 518, 555, 528, 551, 524, 492, 490, 455, 552, 519, 523, 536, 566, 552, 555, 496, 515, 521, 488, 490, 551, 488, 485, 480, 515, 512, 495, 500, 537, 500, 482, 513, 520, 537, 468, 518, 481, 499, 507, 518, 523, 529, 539, 482, 527, 473, 492, 524, 514, 535, 522, 545, 538, 486, 523, 476, 492, 527, 491, 530, 544, 519, 531, 492, 497, 494, 488, 512, 517, 467, 553, 485, 522, 509, 527, 504, 534] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[516, 525, 501, 499, 509, 524, 464, 491, 511, 500, 518, 476, 499, 541, 504, 509, 492, 502, 502, 507, 471, 549, 515, 573, 472, 516, 465, 497, 505, 531, 556, 486, 522, 533, 526, 528, 499, 514, 499, 505, 510, 519, 484, 513, 495, 521, 506, 477, 526, 545, 512, 493, 539, 534, 505, 466, 548, 540, 511, 538, 565, 504, 512, 549, 465, 524, 500, 487, 497, 547, 502, 507, 541, 519, 507, 531, 508, 496, 485, 509, 466, 520, 540, 510, 486, 539, 509, 494, 512, 531, 491, 530, 515, 472, 524, 529, 519, 504, 546, 538, 489, 552, 483, 498, 521, 503, 481, 525, 504, 555, 529, 507, 520, 523, 491, 530, 476, 553, 502, 528, 501, 503, 501, 561, 530, 492, 527, 487] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[527, 542, 501, 545, 515, 504, 522, 517, 492, 500, 506, 509, 518, 533, 556, 511, 551, 527, 504, 453, 509, 508, 511, 555, 535, 471, 531, 486, 520, 518, 507, 508, 494, 530, 514, 483, 523, 514, 508, 523, 514, 495, 560, 487, 505, 527, 545, 498, 484, 503, 522, 488, 510, 503, 516, 492, 501, 550, 523, 489, 473, 505, 537, 534, 474, 504, 513, 512, 529, 535, 480, 490, 528, 490, 516, 529, 501, 507, 497, 511, 487, 514, 497, 550, 525, 467, 516, 535, 499, 518, 520, 525, 536, 531, 467, 526, 496, 495, 514, 520, 507, 516, 504, 503, 560, 531, 497, 522, 493, 513, 543, 499, 483, 497, 543, 485, 509, 524, 503, 504, 545, 501, 478, 509, 512, 536, 494, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[523, 452, 528, 510, 499, 490, 521, 524, 469, 464, 554, 526, 547, 495, 517, 477, 520, 497, 521, 525, 513, 515, 502, 473, 486, 468, 495, 510, 481, 517, 541, 494, 537, 507, 545, 510, 564, 516, 526, 504, 487, 517, 498, 513, 525, 486, 522, 532, 531, 509, 487, 499, 512, 500, 503, 511, 581, 514, 502, 511, 487, 515, 529, 522, 525, 500, 529, 517, 525, 509, 506, 505, 509, 553, 522, 492, 534, 519, 505, 515, 515, 550, 507, 533, 520, 545, 543, 530, 490, 477, 537, 501, 531, 504, 497, 523, 530, 510, 469, 480, 514, 502, 501, 474, 545, 482, 544, 511, 501, 558, 536, 515, 493, 502, 515, 534, 514, 522, 463, 512, 526, 501, 513, 485, 516, 506, 536, 502] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[524, 520, 511, 512, 540, 509, 505, 522, 511, 540, 513, 521, 478, 559, 500, 525, 507, 525, 521, 475, 533, 503, 474, 506, 512, 495, 522, 484, 488, 510, 501, 483, 489, 501, 513, 509, 524, 523, 509, 530, 499, 462, 524, 508, 530, 500, 513, 540, 515, 488, 499, 505, 502, 466, 541, 505, 525, 538, 523, 480, 540, 516, 525, 495, 528, 533, 488, 563, 515, 508, 505, 505, 482, 538, 497, 568, 474, 509, 512, 513, 544, 543, 492, 465, 506, 502, 488, 528, 485, 539, 512, 514, 528, 516, 515, 529, 509, 511, 528, 494, 521, 510, 525, 498, 498, 514, 528, 532, 533, 481, 515, 501, 537, 523, 494, 505, 526, 500, 474, 487, 515, 507, 543, 524, 513, 535, 514, 531] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[558, 546, 507, 491, 563, 489, 497, 495, 517, 508, 530, 492, 477, 533, 512, 502, 500, 523, 494, 527, 472, 505, 510, 521, 538, 515, 479, 488, 534, 509, 521, 518, 514, 451, 475, 513, 552, 537, 527, 496, 549, 512, 527, 490, 490, 470, 530, 488, 504, 532, 520, 511, 539, 539, 487, 464, 541, 525, 503, 538, 518, 503, 501, 491, 484, 526, 524, 521, 504, 478, 505, 519, 545, 512, 515, 518, 509, 511, 529, 513, 549, 493, 521, 534, 523, 529, 519, 521, 532, 528, 530, 516, 513, 544, 468, 508, 493, 503, 510, 510, 520, 511, 537, 553, 514, 502, 510, 504, 505, 506, 495, 523, 494, 495, 523, 458, 527, 527, 512, 524, 518, 500, 491, 474, 533, 474, 514, 527] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[510, 496, 502, 545, 487, 513, 433, 513, 500, 475, 531, 524, 504, 534, 529, 495, 481, 504, 536, 523, 506, 532, 553, 498, 500, 504, 506, 517, 528, 502, 519, 483, 500, 530, 533, 525, 512, 479, 533, 517, 538, 533, 495, 499, 515, 495, 524, 518, 494, 514, 504, 518, 555, 528, 551, 524, 492, 490, 455, 552, 519, 523, 536, 566, 552, 555, 496, 515, 521, 488, 490, 551, 488, 485, 480, 515, 512, 495, 500, 537, 500, 482, 513, 520, 537, 468, 518, 481, 499, 507, 518, 523, 529, 539, 482, 527, 473, 492, 524, 514, 535, 522, 545, 538, 486, 523, 476, 492, 527, 491, 530, 544, 519, 531, 492, 497, 494, 488, 512, 517, 467, 553, 485, 522, 509, 527, 504, 534] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[527, 542, 501, 545, 515, 504, 522, 517, 492, 500, 506, 509, 518, 533, 556, 511, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 527, 504, 453, 509, 508, 511, 555, 535, 471, 531, 486, 520, 518, 507, 508, 39, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 530, 514, 483, 491, 514, 508, 523, 514, 495, 560, 487, 505, 527, 545, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 503, 522, 488, 471, 503, 516, 492, 501, 550, 523, 489, 473, 505, 537, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 504, 513, 512, 529, 535, 480, 490, 528, 490, 516, 529, 501, 507, 497, 511, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 514, 497, 550, 525, 467, 516, 535, 499, 518, 520, 525, 536, 531, 467, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 495, 514, 520, 507, 516, 504, 503, 560, 531, 497, 522, 493, 513, 541, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 497, 543, 485, 509, 524, 503, 504, 545, 501, 478, 509, 512, 536, 494, 529, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8311, 8245, 8188, 8091, 8123, 8203, 8211, 8164] avg=8192.0000 max=8311 max_rank=0 min=8091 min_rank=3 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32701 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[524, 520, 511, 512, 540, 509, 505, 522, 511, 540, 513, 521, 478, 559, 500, 525, 507, 525, 521, 475, 533, 503, 474, 506, 512, 495, 522, 484, 488, 510, 501, 483, 489, 501, 513, 509, 524, 523, 509, 530, 499, 462, 524, 508, 530, 500, 513, 540, 515, 488, 499, 505, 502, 466, 541, 505, 525, 538, 523, 480, 540, 516, 525, 495, 528, 533, 488, 563, 515, 508, 505, 505, 482, 538, 497, 568, 474, 509, 512, 513, 544, 543, 492, 465, 506, 502, 488, 528, 485, 539, 512, 514, 528, 516, 515, 529, 509, 511, 528, 494, 521, 510, 525, 498, 498, 514, 528, 532, 533, 481, 515, 501, 537, 523, 494, 505, 526, 500, 474, 487, 515, 507, 543, 524, 513, 535, 514, 531] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[523, 452, 528, 510, 499, 490, 521, 524, 469, 464, 554, 526, 547, 495, 517, 477, 520, 497, 521, 525, 513, 515, 502, 473, 486, 468, 495, 510, 481, 517, 541, 494, 537, 507, 545, 510, 564, 516, 526, 504, 487, 517, 498, 513, 525, 486, 522, 532, 531, 509, 487, 499, 512, 500, 503, 511, 581, 514, 502, 511, 487, 515, 529, 522, 525, 500, 529, 517, 525, 509, 506, 505, 509, 553, 522, 492, 534, 519, 505, 515, 515, 550, 507, 533, 520, 545, 543, 530, 490, 477, 537, 501, 531, 504, 497, 523, 530, 510, 469, 480, 514, 502, 501, 474, 545, 482, 544, 511, 501, 558, 536, 515, 493, 502, 515, 534, 514, 522, 463, 512, 526, 501, 513, 485, 516, 506, 536, 502] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[510, 526, 528, 519, 559, 500, 500, 484, 529, 514, 487, 510, 511, 513, 480, 507, 532, 530, 532, 473, 516, 463, 511, 507, 528, 516, 506, 489, 523, 509, 485, 544, 485, 509, 482, 553, 557, 481, 526, 548, 514, 520, 553, 524, 496, 501, 560, 514, 490, 505, 510, 510, 546, 503, 476, 541, 516, 515, 476, 494, 533, 503, 556, 501, 516, 487, 550, 531, 526, 545, 530, 522, 510, 513, 546, 488, 489, 486, 526, 521, 528, 509, 500, 507, 497, 530, 485, 526, 511, 504, 502, 496, 476, 466, 529, 487, 496, 524, 498, 478, 489, 530, 503, 497, 497, 525, 516, 520, 536, 503, 533, 523, 542, 517, 526, 511, 503, 496, 505, 519, 510, 497, 476, 519, 562, 513, 507, 487] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[465, 541, 530, 517, 542, 521, 503, 512, 504, 515, 512, 513, 498, 509, 488, 500, 523, 482, 523, 500, 514, 512, 498, 538, 487, 523, 520, 459, 541, 523, 519, 508, 490, 536, 526, 510, 537, 504, 526, 576, 515, 506, 476, 508, 501, 531, 513, 529, 500, 494, 510, 528, 533, 550, 521, 495, 519, 511, 525, 508, 487, 534, 487, 557, 520, 533, 503, 495, 549, 501, 516, 508, 475, 485, 494, 529, 485, 450, 496, 514, 501, 515, 536, 503, 502, 491, 491, 516, 484, 529, 511, 544, 535, 541, 469, 538, 465, 473, 538, 502, 549, 521, 504, 515, 507, 510, 530, 505, 537, 479, 501, 507, 551, 496, 521, 510, 524, 528, 533, 517, 520, 541, 513, 500, 494, 511, 512, 480] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[527, 542, 501, 545, 515, 504, 522, 517, 492, 500, 506, 509, 518, 533, 556, 511, 551, 527, 504, 453, 509, 508, 511, 555, 535, 471, 531, 486, 520, 518, 507, 508, 494, 530, 514, 483, 523, 514, 508, 523, 514, 495, 560, 487, 505, 527, 545, 498, 484, 503, 522, 488, 510, 503, 516, 492, 501, 550, 523, 489, 473, 505, 537, 534, 474, 504, 513, 512, 529, 535, 480, 490, 528, 490, 516, 529, 501, 507, 497, 511, 487, 514, 497, 550, 525, 467, 516, 535, 499, 518, 520, 525, 536, 531, 467, 526, 496, 495, 514, 520, 507, 516, 504, 503, 560, 531, 497, 522, 493, 513, 543, 499, 483, 497, 543, 485, 509, 524, 503, 504, 545, 501, 478, 509, 512, 536, 494, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[523, 452, 528, 510, 499, 490, 521, 524, 469, 464, 554, 526, 547, 495, 517, 477, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 497, 521, 525, 513, 515, 502, 473, 486, 468, 495, 510, 481, 517, 541, 494, 44, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 507, 545, 510, 518, 516, 526, 504, 487, 517, 498, 513, 525, 486, 522, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 509, 487, 499, 468, 500, 503, 511, 581, 514, 502, 511, 487, 515, 529, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 500, 529, 517, 525, 509, 506, 505, 509, 553, 522, 492, 534, 519, 505, 515, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 550, 507, 533, 520, 545, 543, 530, 490, 477, 537, 501, 531, 504, 497, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 510, 469, 480, 514, 502, 501, 474, 545, 482, 544, 511, 501, 558, 530, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 502, 515, 534, 514, 522, 463, 512, 526, 501, 513, 485, 516, 506, 536, 502, 12, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8114, 8118, 8243, 8169, 8278, 8287, 8166, 8161] avg=8192.0000 max=8287 max_rank=5 min=8114 min_rank=0 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32892 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[510, 496, 502, 545, 487, 513, 433, 513, 500, 475, 531, 524, 504, 534, 529, 495, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 504, 536, 523, 506, 532, 553, 498, 500, 504, 506, 517, 528, 502, 519, 483, 42, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 530, 533, 525, 481, 479, 533, 517, 538, 533, 495, 499, 515, 495, 524, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 514, 504, 518, 513, 528, 551, 524, 492, 490, 455, 552, 519, 523, 536, 566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 555, 496, 515, 521, 488, 490, 551, 488, 485, 480, 515, 512, 495, 500, 537, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 482, 513, 520, 537, 468, 518, 481, 499, 507, 518, 523, 529, 539, 482, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 492, 524, 514, 535, 522, 545, 538, 486, 523, 476, 492, 527, 491, 528, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 531, 492, 497, 494, 488, 512, 517, 467, 553, 485, 522, 509, 527, 504, 534, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8104, 8245, 8215, 8279, 8187, 8133, 8210, 8163] avg=8192.0000 max=8279 max_rank=3 min=8104 min_rank=0 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32693 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[558, 546, 507, 491, 563, 489, 497, 495, 517, 508, 530, 492, 477, 533, 512, 502, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 523, 494, 527, 472, 505, 510, 521, 538, 515, 479, 488, 534, 509, 521, 518, 41, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 451, 475, 513, 518, 537, 527, 496, 549, 512, 527, 490, 490, 470, 530, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 532, 520, 511, 498, 539, 487, 464, 541, 525, 503, 538, 518, 503, 501, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 526, 524, 521, 504, 478, 505, 519, 545, 512, 515, 518, 509, 511, 529, 513, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 493, 521, 534, 523, 529, 519, 521, 532, 528, 530, 516, 513, 544, 468, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 503, 510, 510, 520, 511, 537, 553, 514, 502, 510, 504, 505, 506, 493, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 495, 523, 458, 527, 527, 512, 524, 518, 500, 491, 474, 533, 474, 514, 527, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8231, 8207, 8087, 8175, 8219, 8319, 8194, 8104] avg=8192.0000 max=8319 max_rank=5 min=8087 min_rank=2 imbalance_ratio=1.015503 max_load_reduction=57217 max_load_reduction_pct=0.873062 moved_to_echo=32836 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[516, 525, 501, 499, 509, 524, 464, 491, 511, 500, 518, 476, 499, 541, 504, 509, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 502, 502, 507, 471, 549, 515, 573, 472, 516, 465, 497, 505, 531, 556, 486, 41, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 533, 526, 528, 469, 514, 499, 505, 510, 519, 484, 513, 495, 521, 506, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 545, 512, 493, 498, 534, 505, 466, 548, 540, 511, 538, 565, 504, 512, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 524, 500, 487, 497, 547, 502, 507, 541, 519, 507, 531, 508, 496, 485, 509, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 466, 520, 540, 510, 486, 539, 509, 494, 512, 531, 491, 530, 515, 472, 524, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 504, 546, 538, 489, 552, 483, 498, 521, 503, 481, 525, 504, 555, 527, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 523, 491, 530, 476, 553, 502, 528, 501, 503, 501, 561, 530, 492, 527, 487, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8099, 8191, 8121, 8346, 8132, 8158, 8252, 8237] avg=8192.0000 max=8346 max_rank=3 min=8099 min_rank=0 imbalance_ratio=1.018799 max_load_reduction=57190 max_load_reduction_pct=0.872650 moved_to_echo=32779 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[465, 541, 530, 517, 542, 521, 503, 512, 504, 515, 512, 513, 498, 509, 488, 500, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 482, 523, 500, 514, 512, 498, 538, 487, 523, 520, 459, 541, 523, 519, 508, 41, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 536, 526, 510, 504, 504, 526, 576, 515, 506, 476, 508, 501, 531, 513, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 494, 510, 528, 492, 550, 521, 495, 519, 511, 525, 508, 487, 534, 487, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 533, 503, 495, 549, 501, 516, 508, 475, 485, 494, 529, 485, 450, 496, 514, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 515, 536, 503, 502, 491, 491, 516, 484, 529, 511, 544, 535, 541, 469, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 473, 538, 502, 549, 521, 504, 515, 507, 510, 530, 505, 537, 479, 499, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 496, 521, 510, 524, 528, 533, 517, 520, 541, 513, 500, 494, 511, 512, 480, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8223, 8251, 8218, 8060, 8196, 8141, 8264] avg=8192.0000 max=8264 max_rank=7 min=8060 min_rank=4 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32661 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[510, 526, 528, 519, 559, 500, 500, 484, 529, 514, 487, 510, 511, 513, 480, 507, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 530, 532, 473, 516, 463, 511, 507, 528, 516, 506, 489, 523, 509, 485, 544, 42, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 509, 482, 553, 523, 481, 526, 548, 514, 520, 553, 524, 496, 501, 560, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 505, 510, 510, 504, 503, 476, 541, 516, 515, 476, 494, 533, 503, 556, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 487, 550, 531, 526, 545, 530, 522, 510, 513, 546, 488, 489, 486, 526, 521, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 509, 500, 507, 497, 530, 485, 526, 511, 504, 502, 496, 476, 466, 529, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 524, 498, 478, 489, 530, 503, 497, 497, 525, 516, 520, 536, 503, 531, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 517, 526, 511, 503, 496, 505, 519, 510, 497, 476, 519, 562, 513, 507, 487, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8218, 8289, 8133, 8292, 8044, 8166, 8203] avg=8192.0000 max=8292 max_rank=4 min=8044 min_rank=5 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32705 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[524, 520, 511, 512, 540, 509, 505, 522, 511, 540, 513, 521, 478, 559, 500, 525, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 525, 521, 475, 533, 503, 474, 506, 512, 495, 522, 484, 488, 510, 501, 483, 38, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 501, 513, 509, 492, 523, 509, 530, 499, 462, 524, 508, 530, 500, 513, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 488, 499, 505, 464, 466, 541, 505, 525, 538, 523, 480, 540, 516, 525, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 533, 488, 563, 515, 508, 505, 505, 482, 538, 497, 568, 474, 509, 512, 513, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 543, 492, 465, 506, 502, 488, 528, 485, 539, 512, 514, 528, 516, 515, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 511, 528, 494, 521, 510, 525, 498, 498, 514, 528, 532, 533, 481, 513, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 523, 494, 505, 526, 500, 474, 487, 515, 507, 543, 524, 513, 535, 514, 531, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8303, 8089, 8142, 8125, 8245, 8196, 8196, 8240] avg=8192.0000 max=8303 max_rank=0 min=8089 min_rank=1 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32877 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[558, 546, 507, 491, 563, 489, 497, 495, 517, 508, 530, 492, 477, 533, 512, 502, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 523, 494, 527, 472, 505, 510, 521, 538, 515, 479, 488, 534, 509, 521, 518, 41, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 451, 475, 513, 518, 537, 527, 496, 549, 512, 527, 490, 490, 470, 530, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 532, 520, 511, 498, 539, 487, 464, 541, 525, 503, 538, 518, 503, 501, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 526, 524, 521, 504, 478, 505, 519, 545, 512, 515, 518, 509, 511, 529, 513, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 493, 521, 534, 523, 529, 519, 521, 532, 528, 530, 516, 513, 544, 468, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 503, 510, 510, 520, 511, 537, 553, 514, 502, 510, 504, 505, 506, 493, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 495, 523, 458, 527, 527, 512, 524, 518, 500, 491, 474, 533, 474, 514, 527, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8231, 8207, 8087, 8175, 8219, 8319, 8194, 8104] avg=8192.0000 max=8319 max_rank=5 min=8087 min_rank=2 imbalance_ratio=1.015503 max_load_reduction=57217 max_load_reduction_pct=0.873062 moved_to_echo=32836 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[510, 496, 502, 545, 487, 513, 433, 513, 500, 475, 531, 524, 504, 534, 529, 495, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 504, 536, 523, 506, 532, 553, 498, 500, 504, 506, 517, 528, 502, 519, 483, 42, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 530, 533, 525, 481, 479, 533, 517, 538, 533, 495, 499, 515, 495, 524, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 514, 504, 518, 513, 528, 551, 524, 492, 490, 455, 552, 519, 523, 536, 566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 555, 496, 515, 521, 488, 490, 551, 488, 485, 480, 515, 512, 495, 500, 537, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 482, 513, 520, 537, 468, 518, 481, 499, 507, 518, 523, 529, 539, 482, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 492, 524, 514, 535, 522, 545, 538, 486, 523, 476, 492, 527, 491, 528, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 531, 492, 497, 494, 488, 512, 517, 467, 553, 485, 522, 509, 527, 504, 534, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8104, 8245, 8215, 8279, 8187, 8133, 8210, 8163] avg=8192.0000 max=8279 max_rank=3 min=8104 min_rank=0 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32693 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[527, 542, 501, 545, 515, 504, 522, 517, 492, 500, 506, 509, 518, 533, 556, 511, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 527, 504, 453, 509, 508, 511, 555, 535, 471, 531, 486, 520, 518, 507, 508, 39, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 530, 514, 483, 491, 514, 508, 523, 514, 495, 560, 487, 505, 527, 545, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 503, 522, 488, 471, 503, 516, 492, 501, 550, 523, 489, 473, 505, 537, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 504, 513, 512, 529, 535, 480, 490, 528, 490, 516, 529, 501, 507, 497, 511, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 514, 497, 550, 525, 467, 516, 535, 499, 518, 520, 525, 536, 531, 467, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 495, 514, 520, 507, 516, 504, 503, 560, 531, 497, 522, 493, 513, 541, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 497, 543, 485, 509, 524, 503, 504, 545, 501, 478, 509, 512, 536, 494, 529, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8311, 8245, 8188, 8091, 8123, 8203, 8211, 8164] avg=8192.0000 max=8311 max_rank=0 min=8091 min_rank=3 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32701 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 3968, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 3968, 4224, 4224, 4224, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 3968, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 3968, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 3968, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 3968, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 3968, 4224, 4224, 4224, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[491, 509, 548, 522, 518, 509, 559, 524, 481, 459, 500, 507, 522, 547, 599, 481, 520, 499, 509, 506, 522, 522, 508, 485, 526, 518, 476, 487, 499, 527, 516, 504, 493, 503, 512, 485, 543, 505, 517, 474, 531, 542, 516, 478, 520, 539, 510, 513, 539, 504, 523, 502, 486, 541, 512, 488, 483, 505, 470, 464, 523, 531, 495, 513, 502, 532, 453, 547, 492, 517, 536, 506, 501, 505, 560, 498, 520, 526, 516, 506, 509, 522, 526, 484, 491, 527, 510, 528, 522, 540, 500, 534, 492, 525, 498, 514, 475, 507, 508, 476, 540, 511, 503, 502, 531, 518, 547, 506, 534, 472, 506, 494, 495, 506, 576, 507, 521, 516, 545, 503, 507, 466, 532, 540, 527, 517, 522, 527] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[536, 516, 520, 508, 507, 522, 502, 488, 543, 500, 498, 492, 518, 497, 511, 528, 494, 505, 510, 535, 501, 487, 501, 521, 512, 514, 473, 488, 519, 531, 536, 492, 543, 521, 482, 517, 519, 473, 490, 532, 509, 540, 537, 545, 477, 547, 522, 499, 491, 531, 464, 504, 502, 500, 513, 530, 503, 495, 561, 551, 522, 505, 509, 558, 521, 503, 525, 522, 499, 528, 489, 503, 465, 535, 509, 543, 501, 525, 531, 513, 513, 484, 481, 523, 561, 489, 527, 504, 523, 520, 524, 498, 498, 515, 522, 516, 494, 495, 516, 521, 506, 573, 511, 491, 548, 520, 492, 554, 547, 514, 510, 534, 475, 501, 519, 499, 484, 481, 488, 525, 464, 510, 470, 481, 532, 508, 521, 545] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[494, 487, 506, 544, 549, 499, 535, 508, 522, 524, 502, 511, 510, 476, 512, 502, 502, 521, 493, 565, 498, 495, 490, 506, 564, 522, 509, 527, 503, 508, 553, 534, 525, 529, 554, 529, 504, 493, 515, 541, 517, 509, 488, 512, 515, 464, 495, 519, 519, 492, 506, 517, 500, 525, 517, 548, 522, 505, 517, 500, 495, 494, 506, 540, 491, 545, 501, 510, 484, 542, 516, 533, 538, 526, 473, 549, 538, 500, 493, 528, 519, 500, 524, 516, 490, 483, 505, 534, 507, 476, 489, 562, 484, 487, 530, 528, 439, 537, 493, 498, 487, 522, 505, 513, 509, 514, 520, 529, 527, 495, 503, 542, 483, 489, 510, 494, 541, 535, 537, 506, 527, 488, 498, 508, 489, 506, 486, 522] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[508, 551, 511, 554, 491, 496, 488, 555, 522, 504, 513, 496, 538, 517, 512, 508, 514, 470, 508, 531, 550, 559, 529, 512, 525, 515, 491, 496, 526, 538, 536, 509, 522, 515, 531, 554, 487, 472, 507, 514, 514, 470, 514, 509, 524, 478, 579, 492, 515, 444, 511, 521, 529, 539, 508, 465, 529, 515, 544, 508, 504, 519, 518, 481, 502, 485, 529, 581, 504, 527, 499, 473, 487, 524, 503, 508, 513, 501, 478, 503, 550, 536, 520, 518, 548, 506, 511, 509, 506, 541, 485, 506, 520, 512, 539, 520, 512, 518, 522, 520, 533, 494, 509, 517, 458, 508, 480, 555, 522, 506, 491, 514, 495, 528, 520, 512, 475, 525, 479, 566, 470, 505, 472, 496, 510, 520, 509, 446] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[493, 485, 511, 502, 486, 496, 533, 475, 542, 507, 490, 518, 496, 494, 521, 535, 510, 487, 526, 497, 509, 514, 508, 545, 503, 506, 526, 489, 538, 535, 487, 532, 473, 539, 513, 525, 524, 511, 516, 512, 504, 504, 504, 502, 560, 512, 544, 506, 515, 526, 511, 530, 511, 502, 531, 492, 490, 518, 535, 508, 499, 509, 521, 510, 542, 498, 484, 497, 486, 495, 519, 517, 510, 501, 535, 537, 506, 517, 532, 532, 519, 485, 488, 526, 535, 517, 513, 533, 524, 535, 521, 517, 504, 516, 493, 528, 499, 487, 495, 489, 507, 528, 497, 507, 521, 487, 493, 525, 518, 522, 513, 514, 485, 519, 557, 537, 476, 530, 502, 484, 518, 539, 486, 537, 493, 509, 544, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[513, 483, 499, 483, 523, 520, 522, 513, 516, 525, 576, 471, 519, 517, 496, 478, 481, 499, 527, 496, 522, 495, 519, 513, 528, 553, 548, 510, 506, 498, 454, 510, 483, 483, 516, 500, 526, 495, 505, 544, 527, 479, 506, 499, 541, 460, 506, 514, 500, 502, 525, 551, 516, 533, 541, 535, 505, 494, 485, 511, 499, 509, 523, 506, 539, 500, 496, 516, 505, 536, 541, 503, 475, 517, 532, 494, 535, 520, 508, 534, 537, 526, 503, 522, 508, 480, 487, 485, 517, 523, 542, 520, 520, 505, 515, 504, 559, 529, 525, 493, 495, 541, 487, 520, 517, 507, 489, 526, 499, 488, 513, 537, 546, 491, 510, 497, 498, 528, 530, 524, 489, 509, 539, 496, 523, 513, 537, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[514, 512, 501, 573, 503, 533, 559, 509, 511, 489, 509, 543, 470, 506, 512, 500, 521, 498, 509, 492, 515, 551, 496, 520, 510, 536, 490, 500, 540, 502, 531, 513, 524, 529, 511, 461, 505, 525, 531, 470, 495, 491, 470, 507, 520, 540, 491, 512, 522, 531, 504, 523, 524, 506, 486, 532, 532, 535, 479, 524, 521, 542, 490, 543, 478, 515, 480, 490, 519, 522, 508, 486, 544, 526, 498, 494, 530, 504, 564, 476, 492, 521, 525, 499, 495, 566, 509, 506, 515, 531, 523, 513, 529, 502, 563, 496, 520, 482, 518, 496, 506, 504, 500, 519, 504, 538, 474, 516, 502, 519, 529, 504, 491, 541, 495, 510, 513, 521, 505, 510, 532, 451, 529, 509, 532, 514, 484, 505] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[536, 516, 520, 508, 507, 522, 502, 488, 543, 500, 498, 492, 518, 497, 511, 528, 494, 505, 510, 535, 501, 487, 501, 521, 512, 514, 473, 488, 519, 531, 536, 492, 543, 521, 482, 517, 519, 473, 490, 532, 509, 540, 537, 545, 477, 547, 522, 499, 491, 531, 464, 504, 502, 500, 513, 530, 503, 495, 561, 551, 522, 505, 509, 558, 521, 503, 525, 522, 499, 528, 489, 503, 465, 535, 509, 543, 501, 525, 531, 513, 513, 484, 481, 523, 561, 489, 527, 504, 523, 520, 524, 498, 498, 515, 522, 516, 494, 495, 516, 521, 506, 573, 511, 491, 548, 520, 492, 554, 547, 514, 510, 534, 475, 501, 519, 499, 484, 481, 488, 525, 464, 510, 470, 481, 532, 508, 521, 545] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[513, 483, 499, 483, 523, 520, 522, 513, 516, 525, 576, 471, 519, 517, 496, 478, 481, 499, 527, 496, 522, 495, 519, 513, 528, 553, 548, 510, 506, 498, 454, 510, 483, 483, 516, 500, 526, 495, 505, 544, 527, 479, 506, 499, 541, 460, 506, 514, 500, 502, 525, 551, 516, 533, 541, 535, 505, 494, 485, 511, 499, 509, 523, 506, 539, 500, 496, 516, 505, 536, 541, 503, 475, 517, 532, 494, 535, 520, 508, 534, 537, 526, 503, 522, 508, 480, 487, 485, 517, 523, 542, 520, 520, 505, 515, 504, 559, 529, 525, 493, 495, 541, 487, 520, 517, 507, 489, 526, 499, 488, 513, 537, 546, 491, 510, 497, 498, 528, 530, 524, 489, 509, 539, 496, 523, 513, 537, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[491, 509, 548, 522, 518, 509, 559, 524, 481, 459, 500, 507, 522, 547, 599, 481, 520, 499, 509, 506, 522, 522, 508, 485, 526, 518, 476, 487, 499, 527, 516, 504, 493, 503, 512, 485, 543, 505, 517, 474, 531, 542, 516, 478, 520, 539, 510, 513, 539, 504, 523, 502, 486, 541, 512, 488, 483, 505, 470, 464, 523, 531, 495, 513, 502, 532, 453, 547, 492, 517, 536, 506, 501, 505, 560, 498, 520, 526, 516, 506, 509, 522, 526, 484, 491, 527, 510, 528, 522, 540, 500, 534, 492, 525, 498, 514, 475, 507, 508, 476, 540, 511, 503, 502, 531, 518, 547, 506, 534, 472, 506, 494, 495, 506, 576, 507, 521, 516, 545, 503, 507, 466, 532, 540, 527, 517, 522, 527] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[508, 551, 511, 554, 491, 496, 488, 555, 522, 504, 513, 496, 538, 517, 512, 508, 514, 470, 508, 531, 550, 559, 529, 512, 525, 515, 491, 496, 526, 538, 536, 509, 522, 515, 531, 554, 487, 472, 507, 514, 514, 470, 514, 509, 524, 478, 579, 492, 515, 444, 511, 521, 529, 539, 508, 465, 529, 515, 544, 508, 504, 519, 518, 481, 502, 485, 529, 581, 504, 527, 499, 473, 487, 524, 503, 508, 513, 501, 478, 503, 550, 536, 520, 518, 548, 506, 511, 509, 506, 541, 485, 506, 520, 512, 539, 520, 512, 518, 522, 520, 533, 494, 509, 517, 458, 508, 480, 555, 522, 506, 491, 514, 495, 528, 520, 512, 475, 525, 479, 566, 470, 505, 472, 496, 510, 520, 509, 446] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[449, 508, 486, 546, 502, 514, 529, 492, 492, 500, 517, 533, 489, 554, 471, 523, 499, 534, 490, 517, 538, 480, 534, 524, 492, 543, 494, 532, 505, 494, 521, 506, 552, 511, 509, 533, 517, 496, 532, 503, 518, 484, 525, 515, 491, 505, 549, 536, 502, 530, 529, 495, 538, 538, 519, 497, 494, 518, 517, 524, 466, 501, 539, 512, 527, 537, 514, 492, 531, 480, 522, 505, 543, 495, 492, 519, 505, 487, 489, 510, 486, 502, 503, 518, 504, 516, 548, 515, 524, 508, 547, 508, 488, 506, 495, 523, 518, 512, 495, 511, 495, 509, 533, 547, 519, 508, 530, 550, 507, 544, 497, 480, 510, 520, 537, 486, 506, 526, 543, 502, 517, 487, 550, 463, 508, 534, 475, 475] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[508, 551, 511, 554, 491, 496, 488, 555, 522, 504, 513, 496, 538, 517, 512, 508, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 470, 508, 531, 550, 559, 529, 512, 525, 503, 491, 496, 526, 538, 536, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 515, 531, 554, 487, 472, 507, 514, 514, 470, 514, 509, 524, 478, 579, 492, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 444, 511, 521, 529, 528, 508, 465, 529, 515, 544, 508, 504, 519, 518, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 485, 529, 579, 504, 527, 499, 473, 487, 524, 503, 508, 513, 501, 478, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 536, 520, 518, 548, 506, 511, 509, 506, 507, 485, 506, 520, 512, 539, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 518, 522, 520, 533, 494, 509, 517, 458, 508, 480, 555, 522, 506, 491, 514, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 528, 520, 512, 475, 525, 479, 566, 470, 505, 472, 496, 510, 520, 509, 446, 34, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8269, 8297, 8184, 8139, 8115, 8293, 8168, 8071] avg=8192.0000 max=8297 max_rank=1 min=8071 min_rank=7 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32647 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[494, 487, 506, 544, 549, 499, 535, 508, 522, 524, 502, 511, 510, 476, 512, 502, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 521, 493, 565, 498, 495, 490, 506, 564, 510, 509, 527, 503, 508, 553, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 529, 554, 529, 504, 493, 515, 541, 517, 509, 488, 512, 515, 464, 495, 519, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 492, 506, 517, 500, 515, 517, 548, 522, 505, 517, 500, 495, 494, 506, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 545, 501, 508, 484, 542, 516, 533, 538, 526, 473, 549, 538, 500, 493, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 500, 524, 516, 490, 483, 505, 534, 507, 446, 489, 562, 484, 487, 530, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 439, 537, 493, 498, 487, 522, 505, 513, 509, 514, 520, 529, 527, 495, 503, 542, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 489, 510, 494, 541, 535, 537, 506, 527, 488, 498, 508, 489, 506, 486, 522, 30, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8278, 8211, 8193, 8265, 8104, 8141, 8158] avg=8192.0000 max=8278 max_rank=1 min=8104 min_rank=5 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32668 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[491, 509, 548, 522, 518, 509, 559, 524, 481, 459, 500, 507, 522, 547, 599, 481, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 499, 509, 506, 522, 522, 508, 485, 526, 506, 476, 487, 499, 527, 516, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 503, 512, 485, 543, 505, 517, 474, 531, 542, 516, 478, 520, 539, 510, 513, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 504, 523, 502, 486, 530, 512, 488, 483, 505, 470, 464, 523, 531, 495, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 532, 453, 545, 492, 517, 536, 506, 501, 505, 560, 498, 520, 526, 516, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 522, 526, 484, 491, 527, 510, 528, 522, 506, 500, 534, 492, 525, 498, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 507, 508, 476, 540, 511, 503, 502, 531, 518, 547, 506, 534, 472, 506, 494, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 506, 576, 507, 521, 516, 545, 503, 507, 466, 532, 540, 527, 517, 522, 527, 34, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8281, 8112, 8183, 8068, 8215, 8188, 8139, 8350] avg=8192.0000 max=8350 max_rank=7 min=8068 min_rank=3 imbalance_ratio=1.019287 max_load_reduction=57186 max_load_reduction_pct=0.872589 moved_to_echo=32892 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[513, 483, 499, 483, 523, 520, 522, 513, 516, 525, 576, 471, 519, 517, 496, 478, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 499, 527, 496, 522, 495, 519, 513, 528, 541, 548, 510, 506, 498, 454, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 483, 516, 500, 526, 495, 505, 544, 527, 479, 506, 499, 541, 460, 506, 514, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 502, 525, 551, 516, 523, 541, 535, 505, 494, 485, 511, 499, 509, 523, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 500, 496, 514, 505, 536, 541, 503, 475, 517, 532, 494, 535, 520, 508, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 526, 503, 522, 508, 480, 487, 485, 517, 490, 542, 520, 520, 505, 515, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 529, 525, 493, 495, 541, 487, 520, 517, 507, 489, 526, 499, 488, 513, 537, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 491, 510, 497, 498, 528, 530, 524, 489, 509, 539, 496, 523, 513, 537, 504, 33, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8159, 8147, 8086, 8225, 8249, 8161, 8233, 8276] avg=8192.0000 max=8276 max_rank=7 min=8086 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32919 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[536, 516, 520, 508, 507, 522, 502, 488, 543, 500, 498, 492, 518, 497, 511, 528, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 505, 510, 535, 501, 487, 501, 521, 512, 502, 473, 488, 519, 531, 536, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 521, 482, 517, 519, 473, 490, 532, 509, 540, 537, 545, 477, 547, 522, 499, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 531, 464, 504, 502, 490, 513, 530, 503, 495, 561, 551, 522, 505, 509, 558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 503, 525, 520, 499, 528, 489, 503, 465, 535, 509, 543, 501, 525, 531, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 484, 481, 523, 561, 489, 527, 504, 523, 487, 524, 498, 498, 515, 522, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 495, 516, 521, 506, 573, 511, 491, 548, 520, 492, 554, 547, 514, 510, 534, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 501, 519, 499, 484, 481, 488, 525, 464, 510, 470, 481, 532, 508, 521, 545, 33, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8107, 8255, 8229, 8210, 8165, 8334, 8045] avg=8192.0000 max=8334 max_rank=6 min=8045 min_rank=7 imbalance_ratio=1.017334 max_load_reduction=57202 max_load_reduction_pct=0.872833 moved_to_echo=32754 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[493, 485, 511, 502, 486, 496, 533, 475, 542, 507, 490, 518, 496, 494, 521, 535, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 487, 526, 497, 509, 514, 508, 545, 503, 494, 526, 489, 538, 535, 487, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 539, 513, 525, 524, 511, 516, 512, 504, 504, 504, 502, 560, 512, 544, 506, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 526, 511, 530, 511, 492, 531, 492, 490, 518, 535, 508, 499, 509, 521, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 498, 484, 495, 486, 495, 519, 517, 510, 501, 535, 537, 506, 517, 532, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 485, 488, 526, 535, 517, 513, 533, 524, 501, 521, 517, 504, 516, 493, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 487, 495, 489, 507, 528, 497, 507, 521, 487, 493, 525, 518, 522, 513, 514, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 519, 557, 537, 476, 530, 502, 484, 518, 539, 486, 537, 493, 509, 544, 503, 34, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8089, 8200, 8251, 8198, 8206, 8220, 8110, 8262] avg=8192.0000 max=8262 max_rank=7 min=8089 min_rank=0 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32798 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[514, 512, 501, 573, 503, 533, 559, 509, 511, 489, 509, 543, 470, 506, 512, 500, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 498, 509, 492, 515, 551, 496, 520, 510, 524, 490, 500, 540, 502, 531, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 529, 511, 461, 505, 525, 531, 470, 495, 491, 470, 507, 520, 540, 491, 512, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 531, 504, 523, 524, 496, 486, 532, 532, 535, 479, 524, 521, 542, 490, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 515, 480, 488, 519, 522, 508, 486, 544, 526, 498, 494, 530, 504, 564, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 521, 525, 499, 495, 566, 509, 506, 515, 497, 523, 513, 529, 502, 563, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 482, 518, 496, 506, 504, 500, 519, 504, 538, 474, 516, 502, 519, 529, 504, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 541, 495, 510, 513, 521, 505, 510, 532, 451, 529, 509, 532, 514, 484, 505, 34, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8249, 8212, 8084, 8284, 8132, 8251, 8139, 8185] avg=8192.0000 max=8284 max_rank=3 min=8084 min_rank=2 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32707 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[513, 483, 499, 483, 523, 520, 522, 513, 516, 525, 576, 471, 519, 517, 496, 478, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 499, 527, 496, 522, 495, 519, 513, 528, 541, 548, 510, 506, 498, 454, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 483, 516, 500, 526, 495, 505, 544, 527, 479, 506, 499, 541, 460, 506, 514, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 502, 525, 551, 516, 523, 541, 535, 505, 494, 485, 511, 499, 509, 523, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 500, 496, 514, 505, 536, 541, 503, 475, 517, 532, 494, 535, 520, 508, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 526, 503, 522, 508, 480, 487, 485, 517, 490, 542, 520, 520, 505, 515, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 529, 525, 493, 495, 541, 487, 520, 517, 507, 489, 526, 499, 488, 513, 537, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 491, 510, 497, 498, 528, 530, 524, 489, 509, 539, 496, 523, 513, 537, 504, 33, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8159, 8147, 8086, 8225, 8249, 8161, 8233, 8276] avg=8192.0000 max=8276 max_rank=7 min=8086 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32919 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[449, 508, 486, 546, 502, 514, 529, 492, 492, 500, 517, 533, 489, 554, 471, 523, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 534, 490, 517, 538, 480, 534, 524, 492, 523, 494, 532, 505, 494, 521, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 511, 509, 533, 517, 496, 532, 503, 518, 484, 525, 515, 491, 505, 549, 536, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 530, 529, 495, 538, 519, 519, 497, 494, 518, 517, 524, 466, 501, 539, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 537, 514, 488, 531, 480, 522, 505, 543, 495, 492, 519, 505, 487, 489, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 502, 503, 518, 504, 516, 548, 515, 524, 471, 547, 508, 488, 506, 495, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 512, 495, 511, 495, 509, 533, 547, 519, 508, 530, 550, 507, 544, 497, 480, 10, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 520, 537, 486, 506, 526, 543, 502, 517, 487, 550, 463, 508, 534, 475, 475, 37, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8112, 8183, 8282, 8200, 8144, 8154, 8272, 8189] avg=8192.0000 max=8282 max_rank=2 min=8112 min_rank=0 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32759 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[493, 485, 511, 502, 486, 496, 533, 475, 542, 507, 490, 518, 496, 494, 521, 535, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 487, 526, 497, 509, 514, 508, 545, 503, 494, 526, 489, 538, 535, 487, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 539, 513, 525, 524, 511, 516, 512, 504, 504, 504, 502, 560, 512, 544, 506, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 526, 511, 530, 511, 492, 531, 492, 490, 518, 535, 508, 499, 509, 521, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 498, 484, 495, 486, 495, 519, 517, 510, 501, 535, 537, 506, 517, 532, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 485, 488, 526, 535, 517, 513, 533, 524, 501, 521, 517, 504, 516, 493, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 487, 495, 489, 507, 528, 497, 507, 521, 487, 493, 525, 518, 522, 513, 514, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 519, 557, 537, 476, 530, 502, 484, 518, 539, 486, 537, 493, 509, 544, 503, 34, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8089, 8200, 8251, 8198, 8206, 8220, 8110, 8262] avg=8192.0000 max=8262 max_rank=7 min=8089 min_rank=0 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32798 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[536, 516, 520, 508, 507, 522, 502, 488, 543, 500, 498, 492, 518, 497, 511, 528, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 505, 510, 535, 501, 487, 501, 521, 512, 502, 473, 488, 519, 531, 536, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 521, 482, 517, 519, 473, 490, 532, 509, 540, 537, 545, 477, 547, 522, 499, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 531, 464, 504, 502, 490, 513, 530, 503, 495, 561, 551, 522, 505, 509, 558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 503, 525, 520, 499, 528, 489, 503, 465, 535, 509, 543, 501, 525, 531, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 484, 481, 523, 561, 489, 527, 504, 523, 487, 524, 498, 498, 515, 522, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 495, 516, 521, 506, 573, 511, 491, 548, 520, 492, 554, 547, 514, 510, 534, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 501, 519, 499, 484, 481, 488, 525, 464, 510, 470, 481, 532, 508, 521, 545, 33, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8107, 8255, 8229, 8210, 8165, 8334, 8045] avg=8192.0000 max=8334 max_rank=6 min=8045 min_rank=7 imbalance_ratio=1.017334 max_load_reduction=57202 max_load_reduction_pct=0.872833 moved_to_echo=32754 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[508, 551, 511, 554, 491, 496, 488, 555, 522, 504, 513, 496, 538, 517, 512, 508, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 470, 508, 531, 550, 559, 529, 512, 525, 503, 491, 496, 526, 538, 536, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 515, 531, 554, 487, 472, 507, 514, 514, 470, 514, 509, 524, 478, 579, 492, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 444, 511, 521, 529, 528, 508, 465, 529, 515, 544, 508, 504, 519, 518, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 485, 529, 579, 504, 527, 499, 473, 487, 524, 503, 508, 513, 501, 478, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 536, 520, 518, 548, 506, 511, 509, 506, 507, 485, 506, 520, 512, 539, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 518, 522, 520, 533, 494, 509, 517, 458, 508, 480, 555, 522, 506, 491, 514, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 528, 520, 512, 475, 525, 479, 566, 470, 505, 472, 496, 510, 520, 509, 446, 34, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8269, 8297, 8184, 8139, 8115, 8293, 8168, 8071] avg=8192.0000 max=8297 max_rank=1 min=8071 min_rank=7 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32647 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[491, 509, 548, 522, 518, 509, 559, 524, 481, 459, 500, 507, 522, 547, 599, 481, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 499, 509, 506, 522, 522, 508, 485, 526, 506, 476, 487, 499, 527, 516, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 503, 512, 485, 543, 505, 517, 474, 531, 542, 516, 478, 520, 539, 510, 513, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 504, 523, 502, 486, 530, 512, 488, 483, 505, 470, 464, 523, 531, 495, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 532, 453, 545, 492, 517, 536, 506, 501, 505, 560, 498, 520, 526, 516, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 522, 526, 484, 491, 527, 510, 528, 522, 506, 500, 534, 492, 525, 498, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 507, 508, 476, 540, 511, 503, 502, 531, 518, 547, 506, 534, 472, 506, 494, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 506, 576, 507, 521, 516, 545, 503, 507, 466, 532, 540, 527, 517, 522, 527, 34, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8281, 8112, 8183, 8068, 8215, 8188, 8139, 8350] avg=8192.0000 max=8350 max_rank=7 min=8068 min_rank=3 imbalance_ratio=1.019287 max_load_reduction=57186 max_load_reduction_pct=0.872589 moved_to_echo=32892 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4352, 4096, 4096, 4352, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 3968, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4352, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 3968, 4096, 4096, 4224, 4224, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 3968, 4096, 4096, 4224, 4224, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4352, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 3968, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[513, 507, 515, 479, 511, 512, 526, 468, 507, 482, 499, 493, 511, 507, 520, 509, 509, 509, 486, 520, 505, 500, 511, 509, 523, 511, 533, 553, 482, 522, 506, 511, 535, 532, 510, 543, 518, 580, 526, 528, 498, 517, 502, 518, 515, 469, 488, 522, 536, 514, 524, 517, 516, 531, 517, 509, 482, 515, 539, 521, 509, 481, 517, 522, 530, 532, 519, 528, 490, 520, 528, 505, 475, 493, 511, 512, 504, 490, 517, 511, 511, 492, 488, 523, 525, 529, 504, 503, 510, 477, 490, 538, 522, 548, 518, 475, 511, 490, 513, 538, 486, 537, 517, 543, 503, 541, 493, 516, 530, 496, 475, 516, 514, 499, 515, 534, 521, 506, 489, 494, 512, 481, 485, 559, 543, 482, 535, 544] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[507, 557, 523, 507, 525, 506, 500, 494, 510, 544, 482, 534, 496, 542, 525, 507, 512, 506, 527, 541, 525, 505, 475, 493, 512, 508, 538, 502, 525, 516, 494, 523, 511, 534, 472, 506, 482, 480, 541, 534, 521, 495, 543, 533, 503, 472, 516, 509, 509, 498, 523, 552, 502, 542, 524, 526, 498, 506, 543, 501, 498, 520, 511, 541, 524, 529, 522, 495, 521, 526, 531, 524, 504, 563, 524, 534, 527, 520, 519, 472, 508, 519, 467, 481, 511, 500, 475, 509, 516, 522, 510, 495, 535, 518, 495, 518, 541, 506, 522, 537, 486, 499, 488, 505, 528, 507, 471, 503, 486, 520, 474, 573, 506, 469, 522, 510, 514, 549, 496, 508, 487, 499, 510, 521, 490, 473, 526, 489] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[585, 481, 509, 496, 517, 536, 537, 513, 482, 533, 537, 511, 484, 502, 478, 488, 460, 513, 536, 525, 520, 541, 513, 534, 512, 495, 509, 488, 506, 505, 523, 518, 484, 549, 496, 523, 484, 561, 498, 570, 470, 505, 500, 568, 515, 530, 515, 509, 480, 490, 511, 511, 498, 489, 480, 517, 518, 478, 506, 533, 484, 496, 512, 548, 505, 576, 555, 478, 517, 495, 532, 511, 585, 508, 516, 521, 507, 511, 483, 496, 508, 508, 522, 519, 542, 479, 524, 550, 494, 515, 484, 510, 525, 518, 476, 502, 532, 489, 481, 527, 580, 509, 525, 467, 523, 487, 502, 538, 443, 519, 513, 547, 488, 530, 468, 516, 484, 534, 490, 475, 530, 528, 499, 537, 510, 549, 517, 512] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[517, 508, 514, 518, 471, 490, 503, 500, 538, 500, 513, 486, 535, 495, 527, 541, 519, 505, 488, 503, 500, 508, 529, 516, 508, 552, 504, 531, 486, 485, 525, 526, 510, 549, 528, 525, 513, 485, 511, 542, 495, 502, 466, 507, 516, 501, 561, 558, 477, 489, 512, 505, 510, 497, 510, 541, 520, 525, 494, 548, 514, 514, 543, 500, 517, 492, 541, 485, 529, 510, 509, 518, 516, 479, 505, 492, 496, 534, 499, 523, 509, 469, 497, 512, 532, 551, 483, 524, 505, 505, 505, 506, 508, 488, 522, 511, 545, 542, 521, 534, 512, 516, 524, 522, 491, 464, 542, 484, 510, 515, 530, 504, 484, 527, 518, 501, 495, 542, 542, 522, 475, 521, 510, 500, 524, 531, 508, 499] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[462, 555, 518, 471, 531, 525, 516, 517, 531, 502, 509, 519, 490, 492, 532, 539, 490, 542, 506, 532, 498, 520, 533, 529, 486, 481, 487, 541, 534, 501, 496, 506, 540, 546, 461, 548, 520, 536, 485, 531, 506, 517, 505, 513, 508, 525, 492, 504, 566, 505, 529, 494, 544, 522, 490, 495, 504, 505, 502, 508, 489, 531, 488, 495, 547, 508, 491, 558, 505, 507, 510, 531, 544, 479, 520, 524, 519, 505, 502, 492, 498, 538, 492, 532, 508, 467, 487, 493, 517, 515, 545, 549, 517, 517, 504, 485, 527, 535, 484, 524, 495, 537, 463, 530, 472, 531, 499, 532, 522, 529, 519, 480, 506, 528, 497, 544, 516, 484, 513, 562, 482, 542, 513, 487, 499, 525, 474, 484] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[525, 500, 498, 513, 527, 511, 523, 513, 515, 511, 491, 512, 516, 520, 520, 521, 552, 526, 512, 517, 518, 525, 544, 478, 533, 489, 505, 525, 456, 549, 499, 515, 478, 539, 493, 489, 538, 452, 493, 510, 542, 534, 533, 450, 518, 543, 526, 510, 526, 546, 510, 528, 519, 475, 470, 490, 522, 536, 538, 509, 521, 487, 521, 526, 510, 555, 439, 497, 533, 522, 498, 508, 479, 539, 526, 539, 530, 524, 512, 467, 508, 531, 506, 546, 483, 505, 496, 525, 538, 540, 502, 539, 478, 507, 507, 495, 522, 512, 500, 526, 527, 517, 529, 502, 510, 491, 513, 568, 474, 489, 516, 488, 518, 482, 445, 531, 524, 492, 505, 509, 481, 521, 530, 515, 512, 494, 566, 512] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[516, 492, 525, 538, 555, 518, 510, 497, 506, 519, 498, 503, 561, 542, 532, 518, 509, 493, 551, 486, 498, 528, 512, 522, 491, 527, 520, 544, 535, 503, 522, 503, 472, 514, 529, 500, 503, 474, 534, 520, 498, 542, 518, 499, 508, 510, 487, 514, 509, 533, 502, 481, 511, 502, 512, 500, 505, 471, 485, 515, 510, 525, 484, 476, 531, 540, 478, 525, 526, 543, 507, 524, 481, 485, 502, 504, 527, 489, 513, 515, 537, 505, 526, 544, 488, 498, 517, 510, 500, 488, 490, 526, 500, 506, 517, 509, 513, 519, 486, 568, 529, 492, 544, 502, 517, 519, 521, 547, 502, 467, 514, 502, 517, 515, 508, 489, 526, 551, 501, 526, 484, 519, 528, 493, 530, 525, 483, 531] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[513, 507, 515, 479, 511, 512, 526, 468, 507, 482, 499, 493, 511, 507, 520, 509, 509, 509, 486, 520, 505, 500, 511, 509, 523, 511, 533, 553, 482, 522, 506, 511, 535, 532, 510, 543, 518, 580, 526, 528, 498, 517, 502, 518, 515, 469, 488, 522, 536, 514, 524, 517, 516, 531, 517, 509, 482, 515, 539, 521, 509, 481, 517, 522, 530, 532, 519, 528, 490, 520, 528, 505, 475, 493, 511, 512, 504, 490, 517, 511, 511, 492, 488, 523, 525, 529, 504, 503, 510, 477, 490, 538, 522, 548, 518, 475, 511, 490, 513, 538, 486, 537, 517, 543, 503, 541, 493, 516, 530, 496, 475, 516, 514, 499, 515, 534, 521, 506, 489, 494, 512, 481, 485, 559, 543, 482, 535, 544] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[514, 475, 511, 529, 557, 509, 525, 489, 513, 501, 504, 464, 546, 546, 506, 514, 508, 508, 510, 455, 495, 493, 488, 511, 503, 506, 512, 557, 481, 493, 506, 523, 533, 507, 485, 534, 487, 492, 531, 542, 516, 499, 498, 526, 539, 519, 506, 497, 550, 519, 500, 496, 489, 493, 519, 558, 508, 487, 508, 552, 509, 513, 528, 517, 501, 525, 502, 496, 497, 529, 531, 520, 469, 495, 562, 483, 484, 509, 532, 505, 527, 501, 554, 491, 538, 449, 542, 534, 491, 481, 521, 511, 487, 530, 519, 530, 527, 485, 523, 489, 535, 516, 509, 561, 506, 511, 568, 528, 521, 541, 475, 510, 495, 479, 473, 527, 516, 546, 519, 490, 503, 501, 502, 492, 577, 524, 510, 522] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[462, 555, 518, 471, 531, 525, 516, 517, 531, 502, 509, 519, 490, 492, 532, 539, 490, 542, 506, 532, 498, 520, 533, 529, 486, 481, 487, 541, 534, 501, 496, 506, 540, 546, 461, 548, 520, 536, 485, 531, 506, 517, 505, 513, 508, 525, 492, 504, 566, 505, 529, 494, 544, 522, 490, 495, 504, 505, 502, 508, 489, 531, 488, 495, 547, 508, 491, 558, 505, 507, 510, 531, 544, 479, 520, 524, 519, 505, 502, 492, 498, 538, 492, 532, 508, 467, 487, 493, 517, 515, 545, 549, 517, 517, 504, 485, 527, 535, 484, 524, 495, 537, 463, 530, 472, 531, 499, 532, 522, 529, 519, 480, 506, 528, 497, 544, 516, 484, 513, 562, 482, 542, 513, 487, 499, 525, 474, 484] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[525, 500, 498, 513, 527, 511, 523, 513, 515, 511, 491, 512, 516, 520, 520, 521, 552, 526, 512, 517, 518, 525, 544, 478, 533, 489, 505, 525, 456, 549, 499, 515, 478, 539, 493, 489, 538, 452, 493, 510, 542, 534, 533, 450, 518, 543, 526, 510, 526, 546, 510, 528, 519, 475, 470, 490, 522, 536, 538, 509, 521, 487, 521, 526, 510, 555, 439, 497, 533, 522, 498, 508, 479, 539, 526, 539, 530, 524, 512, 467, 508, 531, 506, 546, 483, 505, 496, 525, 538, 540, 502, 539, 478, 507, 507, 495, 522, 512, 500, 526, 527, 517, 529, 502, 510, 491, 513, 568, 474, 489, 516, 488, 518, 482, 445, 531, 524, 492, 505, 509, 481, 521, 530, 515, 512, 494, 566, 512] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[517, 508, 514, 518, 471, 490, 503, 500, 538, 500, 513, 486, 535, 495, 527, 541, 519, 505, 488, 503, 500, 508, 529, 516, 508, 552, 504, 531, 486, 485, 525, 526, 510, 549, 528, 525, 513, 485, 511, 542, 495, 502, 466, 507, 516, 501, 561, 558, 477, 489, 512, 505, 510, 497, 510, 541, 520, 525, 494, 548, 514, 514, 543, 500, 517, 492, 541, 485, 529, 510, 509, 518, 516, 479, 505, 492, 496, 534, 499, 523, 509, 469, 497, 512, 532, 551, 483, 524, 505, 505, 505, 506, 508, 488, 522, 511, 545, 542, 521, 534, 512, 516, 524, 522, 491, 464, 542, 484, 510, 515, 530, 504, 484, 527, 518, 501, 495, 542, 542, 522, 475, 521, 510, 500, 524, 531, 508, 499] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[507, 557, 523, 507, 515, 506, 500, 494, 510, 544, 482, 534, 496, 542, 525, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 506, 527, 541, 525, 505, 475, 493, 512, 508, 538, 502, 525, 516, 494, 523, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 534, 472, 506, 482, 480, 541, 512, 521, 495, 543, 533, 503, 472, 516, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 498, 523, 552, 502, 542, 524, 526, 498, 506, 543, 501, 498, 520, 511, 541, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 510, 522, 495, 521, 526, 531, 524, 504, 563, 524, 534, 527, 520, 519, 472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 519, 467, 481, 511, 500, 475, 509, 516, 522, 510, 495, 535, 518, 495, 518, 22, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 506, 522, 517, 486, 499, 488, 505, 528, 507, 471, 503, 486, 520, 474, 573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 469, 522, 510, 514, 549, 496, 508, 487, 499, 510, 521, 490, 473, 526, 489, 14, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8249, 8207, 8130, 8304, 8316, 8111, 8126, 8093] avg=8192.0000 max=8316 max_rank=4 min=8093 min_rank=7 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32646 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[513, 507, 515, 479, 501, 512, 526, 468, 507, 482, 499, 493, 511, 507, 520, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 509, 486, 520, 505, 500, 511, 509, 523, 511, 533, 553, 482, 522, 506, 511, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 532, 510, 543, 518, 580, 526, 506, 498, 517, 502, 518, 515, 469, 488, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 514, 524, 517, 516, 531, 517, 509, 482, 515, 539, 521, 509, 481, 517, 522, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 513, 519, 528, 490, 520, 528, 505, 475, 493, 511, 512, 504, 490, 517, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 492, 488, 523, 525, 529, 504, 503, 510, 477, 490, 538, 522, 548, 518, 475, 22, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 490, 513, 518, 486, 537, 517, 543, 503, 541, 493, 516, 530, 496, 475, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 499, 515, 534, 521, 506, 489, 494, 512, 481, 485, 559, 543, 482, 535, 544, 14, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8049, 8195, 8279, 8260, 8146, 8185, 8185, 8237] avg=8192.0000 max=8279 max_rank=2 min=8049 min_rank=0 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32753 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[517, 508, 514, 518, 463, 490, 503, 500, 538, 500, 513, 486, 535, 495, 527, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 505, 488, 503, 500, 508, 529, 516, 508, 552, 504, 531, 486, 485, 525, 526, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 549, 528, 525, 513, 485, 511, 520, 495, 502, 466, 507, 516, 501, 561, 558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 489, 512, 505, 510, 497, 510, 541, 520, 525, 494, 548, 514, 514, 543, 500, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 475, 541, 485, 529, 510, 509, 518, 516, 479, 505, 492, 496, 534, 499, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 469, 497, 512, 532, 551, 483, 524, 505, 505, 505, 506, 508, 488, 522, 511, 22, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 542, 521, 514, 512, 516, 524, 522, 491, 464, 542, 484, 510, 515, 530, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 527, 518, 501, 495, 542, 542, 522, 475, 521, 510, 500, 524, 531, 508, 499, 13, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8148, 8189, 8247, 8207, 8128, 8159, 8236, 8222] avg=8192.0000 max=8247 max_rank=2 min=8128 min_rank=4 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32745 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[516, 492, 525, 538, 545, 518, 510, 497, 506, 519, 498, 503, 561, 542, 532, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 493, 551, 486, 498, 528, 512, 522, 491, 527, 520, 544, 535, 503, 522, 503, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 514, 529, 500, 503, 474, 534, 498, 498, 542, 518, 499, 508, 510, 487, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 533, 502, 481, 511, 502, 512, 500, 505, 471, 485, 515, 510, 525, 484, 476, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 521, 478, 525, 526, 543, 507, 524, 481, 485, 502, 504, 527, 489, 513, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 505, 526, 544, 488, 498, 517, 510, 500, 488, 490, 526, 500, 506, 517, 509, 22, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 519, 486, 547, 529, 492, 544, 502, 517, 519, 521, 547, 502, 467, 514, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 515, 508, 489, 526, 551, 501, 526, 484, 519, 528, 493, 530, 525, 483, 531, 14, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8320, 8249, 8100, 8031, 8171, 8194, 8221, 8250] avg=8192.0000 max=8320 max_rank=0 min=8031 min_rank=3 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32836 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[462, 555, 518, 471, 514, 525, 516, 517, 531, 502, 509, 519, 490, 492, 532, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 542, 506, 532, 498, 520, 533, 529, 486, 481, 487, 541, 534, 501, 496, 506, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 546, 461, 548, 520, 536, 485, 505, 506, 517, 505, 513, 508, 525, 492, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 566, 505, 529, 494, 544, 522, 490, 495, 504, 505, 502, 508, 489, 531, 488, 495, 8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 484, 491, 558, 505, 507, 510, 531, 544, 479, 520, 524, 519, 505, 502, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 538, 492, 532, 508, 467, 487, 493, 517, 515, 545, 549, 517, 517, 504, 485, 26, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 535, 484, 498, 495, 537, 463, 530, 472, 531, 499, 532, 522, 529, 519, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 528, 497, 544, 516, 484, 513, 562, 482, 542, 513, 487, 499, 525, 474, 484, 16, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8191, 8211, 8183, 8218, 8203, 8153, 8185] avg=8192.0000 max=8218 max_rank=4 min=8153 min_rank=6 imbalance_ratio=1.003174 max_load_reduction=57318 max_load_reduction_pct=0.874603 moved_to_echo=32759 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[507, 557, 523, 507, 515, 506, 500, 494, 510, 544, 482, 534, 496, 542, 525, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 506, 527, 541, 525, 505, 475, 493, 512, 508, 538, 502, 525, 516, 494, 523, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 534, 472, 506, 482, 480, 541, 512, 521, 495, 543, 533, 503, 472, 516, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 498, 523, 552, 502, 542, 524, 526, 498, 506, 543, 501, 498, 520, 511, 541, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 510, 522, 495, 521, 526, 531, 524, 504, 563, 524, 534, 527, 520, 519, 472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 519, 467, 481, 511, 500, 475, 509, 516, 522, 510, 495, 535, 518, 495, 518, 22, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 506, 522, 517, 486, 499, 488, 505, 528, 507, 471, 503, 486, 520, 474, 573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 469, 522, 510, 514, 549, 496, 508, 487, 499, 510, 521, 490, 473, 526, 489, 14, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8249, 8207, 8130, 8304, 8316, 8111, 8126, 8093] avg=8192.0000 max=8316 max_rank=4 min=8093 min_rank=7 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32646 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[585, 481, 509, 496, 507, 536, 537, 513, 482, 533, 537, 511, 484, 502, 478, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 460, 513, 536, 525, 520, 541, 513, 534, 512, 495, 509, 488, 506, 505, 523, 518, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 549, 496, 523, 484, 561, 498, 546, 470, 505, 500, 568, 515, 530, 515, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 490, 511, 511, 498, 489, 480, 517, 518, 478, 506, 533, 484, 496, 512, 548, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 556, 555, 478, 517, 495, 532, 511, 585, 508, 516, 521, 507, 511, 483, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 508, 522, 519, 542, 479, 524, 550, 494, 515, 484, 510, 525, 518, 476, 502, 24, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 489, 481, 508, 580, 509, 525, 467, 523, 487, 502, 538, 443, 519, 513, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 530, 468, 516, 484, 534, 490, 475, 530, 528, 499, 537, 510, 549, 517, 512, 15, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8203, 8253, 8061, 8276, 8210, 8163, 8191] avg=8192.0000 max=8276 max_rank=4 min=8061 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32840 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[525, 500, 498, 513, 517, 511, 523, 513, 515, 511, 491, 512, 516, 520, 520, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 526, 512, 517, 518, 525, 544, 478, 533, 489, 505, 525, 456, 549, 499, 515, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 539, 493, 489, 538, 452, 493, 489, 542, 534, 533, 450, 518, 543, 526, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 546, 510, 528, 519, 475, 470, 490, 522, 536, 538, 509, 521, 487, 521, 526, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 536, 439, 497, 533, 522, 498, 508, 479, 539, 526, 539, 530, 524, 512, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 531, 506, 546, 483, 505, 496, 525, 538, 540, 502, 539, 478, 507, 507, 495, 21, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 512, 500, 507, 527, 517, 529, 502, 510, 491, 513, 568, 474, 489, 516, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 482, 445, 531, 524, 492, 505, 509, 481, 521, 530, 515, 512, 494, 566, 512, 14, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8248, 8127, 8234, 8159, 8237, 8165, 8160] avg=8192.0000 max=8248 max_rank=1 min=8127 min_rank=2 imbalance_ratio=1.006836 max_load_reduction=57288 max_load_reduction_pct=0.874146 moved_to_echo=32721 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[516, 492, 525, 538, 545, 518, 510, 497, 506, 519, 498, 503, 561, 542, 532, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 493, 551, 486, 498, 528, 512, 522, 491, 527, 520, 544, 535, 503, 522, 503, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 514, 529, 500, 503, 474, 534, 498, 498, 542, 518, 499, 508, 510, 487, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 533, 502, 481, 511, 502, 512, 500, 505, 471, 485, 515, 510, 525, 484, 476, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 521, 478, 525, 526, 543, 507, 524, 481, 485, 502, 504, 527, 489, 513, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 505, 526, 544, 488, 498, 517, 510, 500, 488, 490, 526, 500, 506, 517, 509, 22, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 519, 486, 547, 529, 492, 544, 502, 517, 519, 521, 547, 502, 467, 514, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 515, 508, 489, 526, 551, 501, 526, 484, 519, 528, 493, 530, 525, 483, 531, 14, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8320, 8249, 8100, 8031, 8171, 8194, 8221, 8250] avg=8192.0000 max=8320 max_rank=0 min=8031 min_rank=3 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32836 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[514, 475, 511, 529, 547, 509, 525, 489, 513, 501, 504, 464, 546, 546, 506, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 508, 510, 455, 495, 493, 488, 511, 503, 506, 512, 557, 481, 493, 506, 523, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 507, 485, 534, 487, 492, 531, 520, 516, 499, 498, 526, 539, 519, 506, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 519, 500, 496, 489, 493, 519, 558, 508, 487, 508, 552, 509, 513, 528, 517, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 507, 502, 496, 497, 529, 531, 520, 469, 495, 562, 483, 484, 509, 532, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 501, 554, 491, 538, 449, 542, 534, 491, 481, 521, 511, 487, 530, 519, 530, 22, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 485, 523, 471, 535, 516, 509, 561, 506, 511, 568, 528, 521, 541, 475, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 479, 473, 527, 516, 546, 519, 490, 503, 501, 502, 492, 577, 524, 510, 522, 13, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8054, 8189, 8256, 8122, 8237, 8287, 8198] avg=8192.0000 max=8287 max_rank=6 min=8054 min_rank=1 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32844 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[525, 500, 498, 513, 517, 511, 523, 513, 515, 511, 491, 512, 516, 520, 520, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 526, 512, 517, 518, 525, 544, 478, 533, 489, 505, 525, 456, 549, 499, 515, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 539, 493, 489, 538, 452, 493, 489, 542, 534, 533, 450, 518, 543, 526, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 546, 510, 528, 519, 475, 470, 490, 522, 536, 538, 509, 521, 487, 521, 526, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 536, 439, 497, 533, 522, 498, 508, 479, 539, 526, 539, 530, 524, 512, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 531, 506, 546, 483, 505, 496, 525, 538, 540, 502, 539, 478, 507, 507, 495, 21, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 512, 500, 507, 527, 517, 529, 502, 510, 491, 513, 568, 474, 489, 516, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 482, 445, 531, 524, 492, 505, 509, 481, 521, 530, 515, 512, 494, 566, 512, 14, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8248, 8127, 8234, 8159, 8237, 8165, 8160] avg=8192.0000 max=8248 max_rank=1 min=8127 min_rank=2 imbalance_ratio=1.006836 max_load_reduction=57288 max_load_reduction_pct=0.874146 moved_to_echo=32721 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[517, 508, 514, 518, 463, 490, 503, 500, 538, 500, 513, 486, 535, 495, 527, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 505, 488, 503, 500, 508, 529, 516, 508, 552, 504, 531, 486, 485, 525, 526, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 549, 528, 525, 513, 485, 511, 520, 495, 502, 466, 507, 516, 501, 561, 558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 489, 512, 505, 510, 497, 510, 541, 520, 525, 494, 548, 514, 514, 543, 500, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 475, 541, 485, 529, 510, 509, 518, 516, 479, 505, 492, 496, 534, 499, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 469, 497, 512, 532, 551, 483, 524, 505, 505, 505, 506, 508, 488, 522, 511, 22, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 542, 521, 514, 512, 516, 524, 522, 491, 464, 542, 484, 510, 515, 530, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 527, 518, 501, 495, 542, 542, 522, 475, 521, 510, 500, 524, 531, 508, 499, 13, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8148, 8189, 8247, 8207, 8128, 8159, 8236, 8222] avg=8192.0000 max=8247 max_rank=2 min=8128 min_rank=4 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32745 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4352, 4096, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 3968, 4224, 4096, 4224, 4096, 4096, 3968, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4352, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[515, 466, 482, 518, 523, 519, 490, 534, 537, 509, 520, 473, 546, 530, 496, 515, 544, 527, 515, 492, 531, 544, 511, 500, 486, 515, 489, 486, 522, 539, 493, 501, 530, 517, 473, 514, 493, 503, 507, 535, 531, 513, 511, 453, 532, 494, 473, 497, 526, 515, 552, 513, 494, 518, 466, 549, 500, 537, 480, 459, 534, 477, 534, 542, 484, 516, 505, 544, 505, 510, 514, 517, 516, 514, 507, 503, 485, 496, 530, 507, 497, 521, 510, 538, 503, 487, 459, 524, 503, 549, 525, 499, 509, 554, 515, 544, 505, 520, 532, 532, 529, 519, 540, 505, 478, 519, 465, 508, 474, 481, 532, 540, 505, 503, 537, 538, 537, 575, 474, 511, 516, 511, 520, 512, 505, 541, 538, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[481, 540, 524, 521, 471, 540, 512, 517, 477, 482, 493, 512, 502, 569, 489, 516, 490, 461, 523, 520, 521, 491, 517, 528, 583, 487, 546, 488, 473, 523, 548, 530, 509, 491, 511, 526, 516, 512, 506, 508, 533, 490, 512, 525, 511, 507, 514, 509, 512, 527, 530, 516, 544, 557, 491, 525, 518, 520, 529, 488, 477, 514, 475, 482, 523, 528, 492, 502, 534, 511, 523, 485, 544, 572, 531, 527, 512, 507, 521, 478, 523, 519, 538, 496, 509, 482, 556, 501, 477, 482, 506, 501, 537, 508, 510, 522, 511, 493, 509, 525, 543, 490, 542, 505, 490, 491, 483, 495, 513, 518, 479, 505, 492, 554, 501, 512, 519, 546, 511, 527, 541, 537, 545, 502, 497, 494, 485, 464] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[470, 518, 500, 534, 470, 509, 501, 493, 515, 493, 501, 528, 529, 531, 498, 519, 537, 523, 510, 524, 474, 531, 526, 497, 516, 516, 525, 496, 523, 522, 527, 516, 512, 540, 532, 493, 518, 539, 571, 540, 471, 491, 495, 528, 512, 531, 512, 529, 477, 503, 516, 516, 489, 509, 549, 542, 467, 497, 527, 520, 540, 477, 496, 503, 505, 518, 488, 514, 509, 518, 519, 545, 518, 496, 499, 528, 488, 492, 525, 534, 501, 521, 498, 507, 486, 511, 485, 504, 518, 513, 471, 473, 555, 546, 499, 508, 494, 482, 529, 521, 538, 541, 552, 544, 496, 500, 504, 516, 551, 517, 493, 505, 494, 491, 518, 490, 515, 561, 504, 475, 511, 528, 534, 482, 542, 493, 509, 500] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[509, 465, 471, 519, 522, 524, 528, 516, 510, 494, 550, 525, 485, 514, 514, 497, 522, 508, 497, 523, 533, 500, 508, 531, 485, 532, 533, 522, 474, 534, 524, 507, 509, 516, 522, 533, 476, 509, 508, 533, 541, 452, 526, 489, 513, 500, 501, 515, 494, 512, 472, 547, 509, 481, 526, 508, 504, 511, 498, 521, 517, 547, 517, 496, 490, 451, 536, 506, 501, 496, 540, 517, 525, 534, 520, 536, 506, 535, 563, 508, 525, 536, 515, 454, 507, 534, 513, 544, 514, 520, 525, 506, 506, 520, 515, 523, 507, 534, 494, 523, 475, 548, 502, 482, 484, 502, 484, 507, 548, 514, 527, 495, 505, 511, 545, 467, 524, 507, 505, 531, 487, 534, 525, 539, 520, 505, 517, 488] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[506, 523, 514, 529, 555, 520, 541, 526, 514, 516, 492, 506, 487, 498, 531, 522, 498, 495, 500, 517, 526, 505, 503, 537, 518, 534, 502, 507, 582, 508, 462, 526, 500, 496, 519, 525, 506, 507, 492, 481, 487, 524, 492, 503, 500, 535, 454, 526, 537, 505, 479, 500, 531, 527, 533, 501, 509, 529, 479, 498, 557, 508, 489, 530, 526, 517, 531, 540, 530, 487, 538, 500, 539, 508, 517, 514, 489, 503, 513, 504, 509, 510, 520, 524, 482, 522, 501, 518, 548, 519, 499, 508, 476, 500, 526, 479, 511, 537, 522, 471, 562, 535, 491, 515, 498, 535, 530, 523, 509, 452, 468, 503, 537, 519, 518, 548, 519, 500, 493, 523, 502, 484, 533, 516, 508, 476, 528, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[527, 522, 510, 498, 509, 490, 531, 526, 509, 498, 518, 515, 498, 516, 533, 516, 512, 550, 487, 504, 525, 539, 486, 535, 511, 509, 490, 523, 540, 551, 514, 510, 493, 525, 495, 536, 524, 503, 536, 505, 474, 493, 460, 507, 508, 518, 537, 522, 466, 533, 519, 531, 553, 529, 529, 551, 523, 534, 469, 519, 518, 505, 496, 489, 469, 516, 518, 467, 536, 505, 552, 488, 498, 518, 539, 527, 554, 543, 516, 524, 491, 482, 520, 503, 539, 473, 528, 484, 498, 469, 520, 531, 513, 476, 509, 514, 490, 512, 527, 472, 516, 518, 531, 489, 504, 514, 501, 568, 494, 505, 498, 520, 512, 524, 493, 561, 470, 485, 498, 505, 558, 513, 526, 513, 506, 490, 506, 495] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[559, 509, 502, 510, 491, 539, 501, 497, 502, 531, 494, 525, 521, 525, 529, 498, 509, 516, 539, 510, 493, 567, 507, 557, 487, 513, 500, 504, 468, 522, 527, 501, 542, 523, 550, 542, 544, 547, 518, 489, 513, 513, 480, 527, 508, 547, 526, 491, 489, 499, 489, 490, 541, 465, 511, 558, 492, 483, 501, 503, 519, 524, 527, 496, 505, 509, 535, 518, 487, 525, 489, 509, 521, 502, 477, 516, 521, 523, 514, 498, 497, 496, 508, 534, 519, 520, 472, 529, 530, 496, 497, 532, 507, 503, 542, 495, 492, 524, 519, 483, 515, 510, 506, 512, 513, 487, 533, 488, 513, 476, 498, 507, 476, 508, 553, 518, 503, 508, 509, 502, 556, 525, 522, 494, 515, 505, 549, 491] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[522, 541, 477, 509, 539, 496, 526, 511, 498, 530, 514, 506, 550, 540, 526, 482, 497, 556, 505, 470, 529, 477, 517, 496, 497, 490, 490, 552, 480, 530, 533, 519, 535, 492, 524, 531, 496, 525, 495, 543, 496, 517, 520, 509, 509, 522, 514, 539, 535, 524, 516, 581, 504, 529, 501, 499, 527, 522, 492, 486, 468, 454, 502, 536, 526, 504, 527, 516, 513, 520, 511, 509, 523, 521, 505, 514, 537, 545, 540, 468, 493, 524, 476, 537, 538, 515, 529, 501, 509, 504, 519, 517, 487, 468, 493, 581, 543, 523, 553, 494, 490, 511, 525, 465, 519, 524, 490, 473, 484, 494, 575, 497, 508, 492, 494, 498, 484, 505, 507, 532, 500, 495, 507, 490, 522, 508, 517, 499] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[515, 466, 482, 518, 523, 519, 490, 534, 537, 509, 520, 473, 546, 527, 496, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 527, 515, 492, 531, 544, 511, 500, 486, 515, 489, 486, 522, 508, 493, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 517, 473, 514, 493, 503, 507, 535, 531, 513, 511, 453, 532, 494, 473, 497, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 515, 552, 513, 494, 518, 466, 549, 500, 537, 480, 459, 534, 477, 534, 542, 6, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 516, 505, 544, 505, 510, 514, 517, 516, 514, 507, 503, 485, 496, 491, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 521, 510, 538, 503, 487, 459, 524, 503, 549, 525, 499, 509, 554, 515, 544, 25, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 520, 532, 532, 529, 519, 540, 505, 478, 519, 465, 508, 474, 481, 532, 540, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 503, 537, 538, 537, 575, 474, 511, 516, 511, 516, 512, 505, 541, 538, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8170, 8164, 8077, 8208, 8114, 8264, 8216, 8323] avg=8192.0000 max=8323 max_rank=7 min=8077 min_rank=2 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32917 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[559, 509, 502, 510, 491, 539, 501, 497, 502, 531, 494, 525, 521, 525, 529, 498, 509, 516, 539, 510, 493, 567, 507, 557, 487, 513, 500, 504, 468, 522, 527, 501, 542, 523, 550, 542, 544, 547, 518, 489, 513, 513, 480, 527, 508, 547, 526, 491, 489, 499, 489, 490, 541, 465, 511, 558, 492, 483, 501, 503, 519, 524, 527, 496, 505, 509, 535, 518, 487, 525, 489, 509, 521, 502, 477, 516, 521, 523, 514, 498, 497, 496, 508, 534, 519, 520, 472, 529, 530, 496, 497, 532, 507, 503, 542, 495, 492, 524, 519, 483, 515, 510, 506, 512, 513, 487, 533, 488, 513, 476, 498, 507, 476, 508, 553, 518, 503, 508, 509, 502, 556, 525, 522, 494, 515, 505, 549, 491] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[506, 523, 514, 529, 555, 520, 541, 526, 514, 516, 492, 506, 487, 498, 531, 522, 498, 495, 500, 517, 526, 505, 503, 537, 518, 534, 502, 507, 582, 508, 462, 526, 500, 496, 519, 525, 506, 507, 492, 481, 487, 524, 492, 503, 500, 535, 454, 526, 537, 505, 479, 500, 531, 527, 533, 501, 509, 529, 479, 498, 557, 508, 489, 530, 526, 517, 531, 540, 530, 487, 538, 500, 539, 508, 517, 514, 489, 503, 513, 504, 509, 510, 520, 524, 482, 522, 501, 518, 548, 519, 499, 508, 476, 500, 526, 479, 511, 537, 522, 471, 562, 535, 491, 515, 498, 535, 530, 523, 509, 452, 468, 503, 537, 519, 518, 548, 519, 500, 493, 523, 502, 484, 533, 516, 508, 476, 528, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[509, 465, 471, 519, 522, 524, 528, 516, 510, 494, 550, 525, 485, 514, 514, 497, 522, 508, 497, 523, 533, 500, 508, 531, 485, 532, 533, 522, 474, 534, 524, 507, 509, 516, 522, 533, 476, 509, 508, 533, 541, 452, 526, 489, 513, 500, 501, 515, 494, 512, 472, 547, 509, 481, 526, 508, 504, 511, 498, 521, 517, 547, 517, 496, 490, 451, 536, 506, 501, 496, 540, 517, 525, 534, 520, 536, 506, 535, 563, 508, 525, 536, 515, 454, 507, 534, 513, 544, 514, 520, 525, 506, 506, 520, 515, 523, 507, 534, 494, 523, 475, 548, 502, 482, 484, 502, 484, 507, 548, 514, 527, 495, 505, 511, 545, 467, 524, 507, 505, 531, 487, 534, 525, 539, 520, 505, 517, 488] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[515, 466, 482, 518, 523, 519, 490, 534, 537, 509, 520, 473, 546, 530, 496, 515, 544, 527, 515, 492, 531, 544, 511, 500, 486, 515, 489, 486, 522, 539, 493, 501, 530, 517, 473, 514, 493, 503, 507, 535, 531, 513, 511, 453, 532, 494, 473, 497, 526, 515, 552, 513, 494, 518, 466, 549, 500, 537, 480, 459, 534, 477, 534, 542, 484, 516, 505, 544, 505, 510, 514, 517, 516, 514, 507, 503, 485, 496, 530, 507, 497, 521, 510, 538, 503, 487, 459, 524, 503, 549, 525, 499, 509, 554, 515, 544, 505, 520, 532, 532, 529, 519, 540, 505, 478, 519, 465, 508, 474, 481, 532, 540, 505, 503, 537, 538, 537, 575, 474, 511, 516, 511, 520, 512, 505, 541, 538, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[470, 518, 500, 534, 470, 509, 501, 493, 515, 493, 501, 528, 529, 531, 498, 519, 537, 523, 510, 524, 474, 531, 526, 497, 516, 516, 525, 496, 523, 522, 527, 516, 512, 540, 532, 493, 518, 539, 571, 540, 471, 491, 495, 528, 512, 531, 512, 529, 477, 503, 516, 516, 489, 509, 549, 542, 467, 497, 527, 520, 540, 477, 496, 503, 505, 518, 488, 514, 509, 518, 519, 545, 518, 496, 499, 528, 488, 492, 525, 534, 501, 521, 498, 507, 486, 511, 485, 504, 518, 513, 471, 473, 555, 546, 499, 508, 494, 482, 529, 521, 538, 541, 552, 544, 496, 500, 504, 516, 551, 517, 493, 505, 494, 491, 518, 490, 515, 561, 504, 475, 511, 528, 534, 482, 542, 493, 509, 500] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[481, 540, 524, 521, 471, 540, 512, 517, 477, 482, 493, 512, 502, 569, 489, 516, 490, 461, 523, 520, 521, 491, 517, 528, 583, 487, 546, 488, 473, 523, 548, 530, 509, 491, 511, 526, 516, 512, 506, 508, 533, 490, 512, 525, 511, 507, 514, 509, 512, 527, 530, 516, 544, 557, 491, 525, 518, 520, 529, 488, 477, 514, 475, 482, 523, 528, 492, 502, 534, 511, 523, 485, 544, 572, 531, 527, 512, 507, 521, 478, 523, 519, 538, 496, 509, 482, 556, 501, 477, 482, 506, 501, 537, 508, 510, 522, 511, 493, 509, 525, 543, 490, 542, 505, 490, 491, 483, 495, 513, 518, 479, 505, 492, 554, 501, 512, 519, 546, 511, 527, 541, 537, 545, 502, 497, 494, 485, 464] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[527, 522, 510, 498, 509, 490, 531, 526, 509, 498, 518, 515, 498, 513, 533, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 550, 487, 504, 525, 539, 486, 535, 511, 509, 490, 523, 540, 520, 514, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 525, 495, 536, 524, 503, 536, 505, 474, 493, 460, 507, 508, 518, 537, 522, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 466, 533, 519, 531, 553, 529, 529, 551, 523, 534, 469, 519, 518, 505, 496, 489, 6, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 516, 518, 467, 536, 505, 552, 488, 498, 518, 539, 527, 554, 543, 478, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 482, 520, 503, 539, 473, 528, 484, 498, 469, 520, 531, 513, 476, 509, 514, 25, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 512, 527, 472, 516, 518, 531, 489, 504, 514, 501, 568, 494, 505, 498, 520, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 524, 493, 561, 470, 485, 498, 505, 558, 513, 522, 513, 506, 490, 506, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8255, 8137, 8276, 8232, 8077, 8195, 8151] avg=8192.0000 max=8276 max_rank=3 min=8077 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32655 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[509, 465, 471, 519, 522, 524, 528, 516, 510, 494, 550, 525, 485, 511, 514, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 508, 497, 523, 533, 500, 508, 531, 485, 532, 533, 522, 474, 503, 524, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 516, 522, 533, 476, 509, 508, 533, 541, 452, 526, 489, 513, 500, 501, 515, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 512, 472, 547, 509, 481, 526, 508, 504, 511, 498, 521, 517, 547, 517, 496, 6, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 451, 536, 506, 501, 496, 540, 517, 525, 534, 520, 536, 506, 535, 522, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 536, 515, 454, 507, 534, 513, 544, 514, 520, 525, 506, 506, 520, 515, 523, 25, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 534, 494, 523, 475, 548, 502, 482, 484, 502, 484, 507, 548, 514, 527, 495, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 511, 545, 467, 524, 507, 505, 531, 487, 534, 521, 539, 520, 505, 517, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8140, 8202, 8144, 8172, 8223, 8284, 8165, 8206] avg=8192.0000 max=8284 max_rank=5 min=8140 min_rank=0 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32878 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[481, 540, 524, 521, 471, 540, 512, 517, 477, 482, 493, 512, 502, 559, 489, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 461, 523, 520, 521, 491, 517, 528, 583, 487, 546, 488, 473, 487, 548, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 491, 511, 526, 516, 512, 506, 508, 533, 490, 512, 525, 511, 507, 514, 509, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 527, 530, 516, 544, 557, 491, 525, 518, 520, 529, 488, 477, 514, 475, 482, 8, 10, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 528, 492, 502, 534, 511, 523, 485, 544, 572, 531, 527, 512, 507, 473, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 519, 538, 496, 509, 482, 556, 501, 477, 482, 506, 501, 537, 508, 510, 522, 28, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 493, 509, 525, 543, 490, 542, 505, 490, 491, 483, 495, 513, 518, 479, 505, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 554, 501, 512, 519, 546, 511, 527, 541, 537, 535, 502, 497, 494, 485, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8136, 8193, 8186, 8227, 8242, 8203, 8132, 8217] avg=8192.0000 max=8242 max_rank=4 min=8132 min_rank=6 imbalance_ratio=1.006104 max_load_reduction=57294 max_load_reduction_pct=0.874237 moved_to_echo=32794 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[527, 522, 510, 498, 509, 490, 531, 526, 509, 498, 518, 515, 498, 513, 533, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 550, 487, 504, 525, 539, 486, 535, 511, 509, 490, 523, 540, 520, 514, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 525, 495, 536, 524, 503, 536, 505, 474, 493, 460, 507, 508, 518, 537, 522, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 466, 533, 519, 531, 553, 529, 529, 551, 523, 534, 469, 519, 518, 505, 496, 489, 6, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 516, 518, 467, 536, 505, 552, 488, 498, 518, 539, 527, 554, 543, 478, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 482, 520, 503, 539, 473, 528, 484, 498, 469, 520, 531, 513, 476, 509, 514, 25, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 512, 527, 472, 516, 518, 531, 489, 504, 514, 501, 568, 494, 505, 498, 520, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 524, 493, 561, 470, 485, 498, 505, 558, 513, 522, 513, 506, 490, 506, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8255, 8137, 8276, 8232, 8077, 8195, 8151] avg=8192.0000 max=8276 max_rank=3 min=8077 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32655 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[470, 518, 500, 534, 470, 509, 501, 493, 515, 493, 501, 528, 529, 528, 498, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 523, 510, 524, 474, 531, 526, 497, 516, 516, 525, 496, 523, 492, 527, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 540, 532, 493, 518, 539, 571, 540, 471, 491, 495, 528, 512, 531, 512, 529, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 503, 516, 516, 489, 509, 549, 542, 467, 497, 527, 520, 540, 477, 496, 503, 6, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 518, 488, 514, 509, 518, 519, 545, 518, 496, 499, 528, 488, 492, 486, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 521, 498, 507, 486, 511, 485, 504, 518, 513, 471, 473, 555, 546, 499, 508, 24, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 482, 529, 521, 538, 541, 552, 544, 496, 500, 504, 516, 551, 517, 493, 505, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 491, 518, 490, 515, 561, 504, 475, 511, 528, 529, 482, 542, 493, 509, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8106, 8233, 8315, 8141, 8157, 8122, 8320, 8142] avg=8192.0000 max=8320 max_rank=6 min=8106 min_rank=0 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32741 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[506, 523, 514, 529, 555, 520, 541, 526, 514, 516, 492, 506, 487, 495, 531, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 495, 500, 517, 526, 505, 503, 537, 518, 534, 502, 507, 582, 479, 462, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 496, 519, 525, 506, 507, 492, 481, 487, 524, 492, 503, 500, 535, 454, 526, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 505, 479, 500, 531, 527, 533, 501, 509, 529, 479, 498, 557, 508, 489, 530, 6, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 517, 531, 540, 530, 487, 538, 500, 539, 508, 517, 514, 489, 503, 475, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 510, 520, 524, 482, 522, 501, 518, 548, 519, 499, 508, 476, 500, 526, 479, 23, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 537, 522, 471, 562, 535, 491, 515, 498, 535, 530, 523, 509, 452, 468, 503, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 519, 518, 548, 519, 500, 493, 523, 502, 484, 528, 516, 508, 476, 528, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8277, 8191, 8048, 8225, 8218, 8166, 8198, 8213] avg=8192.0000 max=8277 max_rank=0 min=8048 min_rank=2 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32795 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[522, 541, 477, 509, 539, 496, 526, 511, 498, 530, 514, 506, 550, 537, 526, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 556, 505, 470, 529, 477, 517, 496, 497, 490, 490, 552, 480, 500, 533, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 492, 524, 531, 496, 525, 495, 543, 496, 517, 520, 509, 509, 522, 514, 539, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 524, 516, 581, 504, 529, 501, 499, 527, 522, 492, 486, 468, 454, 502, 536, 6, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 504, 527, 516, 513, 520, 511, 509, 523, 521, 505, 514, 537, 545, 500, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 524, 476, 537, 538, 515, 529, 501, 509, 504, 519, 517, 487, 468, 493, 581, 24, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 523, 553, 494, 490, 511, 525, 465, 519, 524, 490, 473, 484, 494, 575, 497, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 492, 494, 498, 484, 505, 507, 532, 500, 495, 503, 490, 522, 508, 517, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8264, 8108, 8268, 8188, 8239, 8217, 8198, 8054] avg=8192.0000 max=8268 max_rank=2 min=8054 min_rank=7 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32708 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[559, 509, 502, 510, 491, 539, 501, 497, 502, 531, 494, 525, 521, 522, 529, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 516, 539, 510, 493, 567, 507, 557, 487, 513, 500, 504, 468, 492, 527, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 523, 550, 542, 544, 547, 518, 489, 513, 513, 480, 527, 508, 547, 526, 491, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 499, 489, 490, 541, 465, 511, 558, 492, 483, 501, 503, 519, 524, 527, 496, 6, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 509, 535, 518, 487, 525, 489, 509, 521, 502, 477, 516, 521, 523, 476, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 496, 508, 534, 519, 520, 472, 529, 530, 496, 497, 532, 507, 503, 542, 495, 24, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 524, 519, 483, 515, 510, 506, 512, 513, 487, 533, 488, 513, 476, 498, 507, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 508, 553, 518, 503, 508, 509, 502, 556, 525, 518, 494, 515, 505, 549, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 8190, 8361, 8099, 8111, 8203, 8112, 8230] avg=8192.0000 max=8361 max_rank=2 min=8099 min_rank=3 imbalance_ratio=1.020630 max_load_reduction=57175 max_load_reduction_pct=0.872421 moved_to_echo=32656 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[470, 518, 500, 534, 470, 509, 501, 493, 515, 493, 501, 528, 529, 528, 498, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 523, 510, 524, 474, 531, 526, 497, 516, 516, 525, 496, 523, 492, 527, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 540, 532, 493, 518, 539, 571, 540, 471, 491, 495, 528, 512, 531, 512, 529, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 503, 516, 516, 489, 509, 549, 542, 467, 497, 527, 520, 540, 477, 496, 503, 6, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 518, 488, 514, 509, 518, 519, 545, 518, 496, 499, 528, 488, 492, 486, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 521, 498, 507, 486, 511, 485, 504, 518, 513, 471, 473, 555, 546, 499, 508, 24, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 482, 529, 521, 538, 541, 552, 544, 496, 500, 504, 516, 551, 517, 493, 505, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 491, 518, 490, 515, 561, 504, 475, 511, 528, 529, 482, 542, 493, 509, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8106, 8233, 8315, 8141, 8157, 8122, 8320, 8142] avg=8192.0000 max=8320 max_rank=6 min=8106 min_rank=0 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32741 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[515, 466, 482, 518, 523, 519, 490, 534, 537, 509, 520, 473, 546, 527, 496, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 527, 515, 492, 531, 544, 511, 500, 486, 515, 489, 486, 522, 508, 493, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 517, 473, 514, 493, 503, 507, 535, 531, 513, 511, 453, 532, 494, 473, 497, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 515, 552, 513, 494, 518, 466, 549, 500, 537, 480, 459, 534, 477, 534, 542, 6, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 516, 505, 544, 505, 510, 514, 517, 516, 514, 507, 503, 485, 496, 491, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 521, 510, 538, 503, 487, 459, 524, 503, 549, 525, 499, 509, 554, 515, 544, 25, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 520, 532, 532, 529, 519, 540, 505, 478, 519, 465, 508, 474, 481, 532, 540, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 503, 537, 538, 537, 575, 474, 511, 516, 511, 516, 512, 505, 541, 538, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8170, 8164, 8077, 8208, 8114, 8264, 8216, 8323] avg=8192.0000 max=8323 max_rank=7 min=8077 min_rank=2 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32917 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[522, 541, 477, 509, 539, 496, 526, 511, 498, 530, 514, 506, 550, 537, 526, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 556, 505, 470, 529, 477, 517, 496, 497, 490, 490, 552, 480, 500, 533, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 492, 524, 531, 496, 525, 495, 543, 496, 517, 520, 509, 509, 522, 514, 539, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 524, 516, 581, 504, 529, 501, 499, 527, 522, 492, 486, 468, 454, 502, 536, 6, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 504, 527, 516, 513, 520, 511, 509, 523, 521, 505, 514, 537, 545, 500, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 524, 476, 537, 538, 515, 529, 501, 509, 504, 519, 517, 487, 468, 493, 581, 24, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 523, 553, 494, 490, 511, 525, 465, 519, 524, 490, 473, 484, 494, 575, 497, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 492, 494, 498, 484, 505, 507, 532, 500, 495, 503, 490, 522, 508, 517, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8264, 8108, 8268, 8188, 8239, 8217, 8198, 8054] avg=8192.0000 max=8268 max_rank=2 min=8054 min_rank=7 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32708 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 3968, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 3968, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4224, 4224, 4352, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 3968, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4224, 4224, 4352, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[518, 539, 441, 495, 478, 483, 456, 557, 513, 521, 532, 493, 513, 522, 546, 523, 524, 471, 487, 506, 484, 473, 501, 526, 491, 462, 521, 541, 545, 495, 482, 485, 494, 502, 529, 521, 489, 511, 522, 492, 490, 498, 522, 481, 515, 534, 547, 534, 542, 512, 534, 522, 528, 477, 494, 494, 524, 542, 502, 529, 517, 516, 531, 492, 526, 463, 459, 540, 513, 511, 497, 496, 543, 558, 504, 504, 512, 497, 531, 518, 534, 557, 503, 476, 527, 507, 512, 487, 539, 508, 530, 511, 489, 516, 495, 486, 532, 502, 534, 530, 537, 521, 520, 512, 534, 531, 533, 536, 507, 509, 508, 551, 531, 539, 548, 508, 502, 518, 485, 510, 502, 516, 515, 514, 538, 493, 490, 520] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[485, 490, 496, 537, 529, 568, 482, 537, 496, 493, 501, 488, 512, 503, 518, 490, 515, 521, 521, 522, 552, 531, 547, 547, 499, 530, 537, 496, 506, 513, 490, 460, 469, 530, 500, 499, 500, 507, 501, 553, 462, 543, 531, 519, 494, 517, 541, 477, 526, 507, 531, 515, 550, 534, 478, 519, 499, 510, 503, 468, 506, 511, 513, 510, 538, 521, 494, 506, 528, 511, 525, 508, 509, 528, 514, 515, 522, 515, 493, 515, 539, 494, 510, 541, 535, 482, 515, 515, 514, 494, 501, 497, 495, 532, 514, 490, 520, 507, 534, 520, 475, 488, 502, 496, 532, 551, 521, 483, 516, 501, 496, 489, 552, 515, 536, 512, 518, 508, 486, 489, 520, 530, 516, 479, 527, 518, 503, 551] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[490, 513, 552, 501, 519, 486, 494, 540, 481, 562, 512, 474, 526, 480, 517, 513, 490, 516, 507, 469, 543, 561, 527, 490, 493, 528, 523, 549, 535, 476, 502, 502, 509, 490, 506, 489, 476, 495, 513, 492, 495, 533, 526, 506, 534, 500, 481, 515, 525, 502, 478, 476, 516, 516, 534, 537, 524, 513, 497, 513, 509, 481, 520, 527, 509, 493, 511, 492, 500, 534, 570, 526, 523, 512, 524, 486, 533, 506, 537, 539, 518, 481, 530, 527, 461, 485, 484, 506, 530, 525, 549, 542, 515, 531, 534, 520, 519, 482, 508, 475, 484, 578, 505, 498, 506, 495, 512, 531, 523, 518, 498, 500, 565, 475, 537, 533, 528, 527, 488, 519, 492, 511, 498, 522, 517, 534, 473, 553] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[580, 509, 535, 510, 503, 491, 467, 518, 507, 503, 479, 518, 475, 536, 525, 508, 520, 503, 542, 521, 501, 509, 460, 529, 519, 507, 510, 519, 514, 491, 525, 499, 527, 519, 503, 541, 508, 506, 501, 520, 526, 493, 520, 482, 515, 514, 553, 475, 510, 491, 560, 506, 527, 517, 509, 508, 522, 532, 474, 530, 552, 495, 501, 508, 528, 487, 460, 527, 517, 489, 479, 521, 491, 463, 514, 539, 513, 511, 508, 572, 509, 536, 518, 538, 536, 532, 510, 518, 547, 556, 482, 534, 491, 478, 539, 534, 494, 505, 487, 531, 518, 517, 521, 517, 501, 496, 517, 468, 494, 536, 486, 502, 503, 518, 543, 523, 479, 525, 540, 515, 510, 530, 545, 477, 492, 510, 493, 488] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[510, 520, 507, 508, 459, 506, 479, 447, 524, 544, 522, 523, 486, 523, 491, 521, 507, 510, 539, 486, 526, 541, 501, 520, 505, 550, 506, 482, 512, 504, 499, 512, 519, 520, 526, 497, 490, 508, 514, 579, 529, 563, 512, 482, 524, 543, 509, 501, 502, 528, 500, 514, 481, 502, 576, 522, 475, 510, 496, 556, 524, 523, 517, 556, 557, 516, 506, 495, 538, 525, 553, 520, 545, 476, 521, 551, 534, 487, 483, 497, 492, 509, 536, 480, 491, 531, 516, 546, 510, 514, 517, 504, 498, 507, 523, 561, 516, 496, 467, 503, 492, 515, 540, 533, 486, 541, 500, 478, 491, 472, 522, 508, 505, 501, 537, 484, 492, 533, 512, 498, 469, 533, 525, 476, 508, 531, 466, 499] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[559, 527, 515, 521, 553, 554, 523, 506, 526, 529, 515, 490, 501, 500, 527, 491, 522, 525, 505, 532, 472, 516, 506, 517, 495, 551, 475, 504, 537, 547, 575, 527, 507, 519, 530, 529, 497, 519, 518, 455, 533, 507, 488, 497, 529, 523, 516, 524, 483, 515, 503, 509, 508, 517, 538, 538, 461, 475, 508, 518, 494, 478, 514, 525, 489, 533, 496, 514, 497, 486, 483, 492, 508, 486, 505, 523, 510, 495, 515, 525, 504, 527, 558, 496, 525, 507, 533, 533, 525, 496, 514, 512, 500, 509, 510, 494, 535, 506, 513, 487, 507, 490, 497, 507, 516, 532, 477, 511, 482, 515, 532, 508, 493, 487, 539, 507, 511, 527, 515, 482, 507, 499, 502, 529, 512, 531, 527, 535] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[490, 513, 552, 501, 519, 486, 494, 540, 481, 562, 512, 474, 526, 480, 517, 513, 490, 516, 507, 469, 543, 561, 527, 490, 493, 528, 523, 549, 535, 476, 502, 502, 509, 490, 506, 489, 476, 495, 513, 492, 495, 533, 526, 506, 534, 500, 481, 515, 525, 502, 478, 476, 516, 516, 534, 537, 524, 513, 497, 513, 509, 481, 520, 527, 509, 493, 511, 492, 500, 534, 570, 526, 523, 512, 524, 486, 533, 506, 537, 539, 518, 481, 530, 527, 461, 485, 484, 506, 530, 525, 549, 542, 515, 531, 534, 520, 519, 482, 508, 475, 484, 578, 505, 498, 506, 495, 512, 531, 523, 518, 498, 500, 565, 475, 537, 533, 528, 527, 488, 519, 492, 511, 498, 522, 517, 534, 473, 553] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[543, 532, 532, 500, 505, 522, 542, 486, 488, 495, 516, 497, 564, 521, 486, 517, 546, 521, 488, 501, 478, 527, 513, 485, 509, 529, 528, 522, 546, 544, 528, 554, 510, 511, 511, 535, 502, 507, 516, 512, 524, 518, 539, 498, 521, 471, 545, 521, 511, 520, 463, 464, 521, 536, 508, 518, 540, 505, 524, 474, 511, 482, 512, 544, 490, 493, 495, 487, 537, 498, 523, 538, 533, 540, 470, 538, 529, 533, 515, 496, 478, 505, 503, 486, 470, 502, 480, 498, 514, 470, 545, 575, 479, 524, 496, 529, 515, 532, 506, 486, 537, 517, 491, 544, 509, 519, 475, 479, 506, 490, 511, 513, 547, 523, 527, 492, 575, 538, 507, 466, 475, 471, 512, 500, 500, 508, 536, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[471, 500, 501, 498, 501, 505, 512, 506, 482, 515, 539, 488, 526, 523, 534, 525, 539, 504, 538, 503, 502, 522, 525, 510, 507, 514, 512, 500, 520, 491, 501, 486, 516, 505, 501, 496, 519, 499, 498, 459, 533, 526, 549, 518, 528, 487, 533, 519, 497, 483, 551, 529, 497, 505, 499, 505, 501, 537, 523, 504, 548, 489, 540, 512, 518, 480, 458, 512, 512, 464, 513, 524, 484, 497, 505, 542, 509, 529, 486, 540, 571, 483, 514, 532, 524, 483, 540, 498, 508, 504, 547, 548, 507, 517, 525, 484, 527, 513, 533, 538, 521, 556, 534, 506, 505, 503, 508, 523, 522, 528, 501, 539, 506, 503, 524, 527, 498, 491, 547, 463, 507, 498, 489, 525, 517, 484, 528, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[518, 539, 441, 495, 478, 483, 456, 557, 513, 521, 532, 493, 513, 522, 546, 523, 524, 471, 487, 506, 484, 473, 501, 526, 491, 462, 521, 541, 545, 495, 482, 485, 494, 502, 529, 521, 489, 511, 522, 492, 490, 498, 522, 481, 515, 534, 547, 534, 542, 512, 534, 522, 528, 477, 494, 494, 524, 542, 502, 529, 517, 516, 531, 492, 526, 463, 459, 540, 513, 511, 497, 496, 543, 558, 504, 504, 512, 497, 531, 518, 534, 557, 503, 476, 527, 507, 512, 487, 539, 508, 530, 511, 489, 516, 495, 486, 532, 502, 534, 530, 537, 521, 520, 512, 534, 531, 533, 536, 507, 509, 508, 551, 531, 539, 548, 508, 502, 518, 485, 510, 502, 516, 515, 514, 538, 493, 490, 520] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[580, 509, 535, 510, 503, 491, 467, 518, 507, 503, 479, 518, 475, 536, 525, 508, 520, 503, 542, 521, 501, 509, 460, 529, 519, 507, 510, 519, 514, 491, 525, 499, 527, 519, 503, 541, 508, 506, 501, 520, 526, 493, 520, 482, 515, 514, 553, 475, 510, 491, 560, 506, 527, 517, 509, 508, 522, 532, 474, 530, 552, 495, 501, 508, 528, 487, 460, 527, 517, 489, 479, 521, 491, 463, 514, 539, 513, 511, 508, 572, 509, 536, 518, 538, 536, 532, 510, 518, 547, 556, 482, 534, 491, 478, 539, 534, 494, 505, 487, 531, 518, 517, 521, 517, 501, 496, 517, 468, 494, 536, 486, 502, 503, 518, 543, 523, 479, 525, 540, 515, 510, 530, 545, 477, 492, 510, 493, 488] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[485, 490, 496, 537, 529, 568, 482, 537, 496, 493, 501, 488, 512, 503, 518, 490, 515, 521, 521, 522, 552, 531, 547, 547, 499, 530, 537, 496, 506, 513, 490, 460, 469, 530, 500, 499, 500, 507, 501, 553, 462, 543, 531, 519, 494, 517, 541, 477, 526, 507, 531, 515, 550, 534, 478, 519, 499, 510, 503, 468, 506, 511, 513, 510, 538, 521, 494, 506, 528, 511, 525, 508, 509, 528, 514, 515, 522, 515, 493, 515, 539, 494, 510, 541, 535, 482, 515, 515, 514, 494, 501, 497, 495, 532, 514, 490, 520, 507, 534, 520, 475, 488, 502, 496, 532, 551, 521, 483, 516, 501, 496, 489, 552, 515, 536, 512, 518, 508, 486, 489, 520, 530, 516, 479, 527, 518, 503, 551] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[471, 500, 501, 498, 501, 505, 512, 506, 482, 515, 539, 488, 526, 523, 534, 525, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 504, 538, 503, 502, 522, 525, 510, 507, 514, 512, 500, 506, 491, 501, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 505, 501, 496, 519, 499, 498, 459, 533, 526, 549, 518, 528, 487, 533, 519, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 483, 551, 529, 497, 505, 499, 505, 501, 537, 523, 504, 548, 489, 540, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 480, 458, 512, 512, 464, 513, 524, 484, 497, 505, 542, 509, 529, 486, 540, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 571, 483, 514, 532, 524, 483, 540, 498, 508, 504, 547, 520, 507, 517, 525, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 513, 533, 538, 521, 556, 534, 506, 505, 503, 508, 523, 522, 528, 501, 539, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 503, 524, 527, 498, 491, 547, 463, 507, 498, 489, 525, 517, 484, 528, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8149, 8160, 8187, 8218, 8079, 8257, 8371, 8115] avg=8192.0000 max=8371 max_rank=6 min=8079 min_rank=4 imbalance_ratio=1.021851 max_load_reduction=57165 max_load_reduction_pct=0.872269 moved_to_echo=32822 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[559, 527, 515, 521, 553, 554, 523, 506, 526, 529, 515, 490, 501, 500, 527, 491, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 525, 505, 532, 472, 516, 506, 517, 495, 551, 475, 504, 522, 547, 575, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 519, 530, 529, 497, 519, 518, 455, 533, 507, 488, 497, 529, 523, 516, 524, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 515, 503, 509, 508, 517, 538, 538, 461, 475, 508, 518, 494, 478, 514, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 533, 496, 514, 497, 486, 483, 492, 508, 486, 505, 523, 510, 495, 515, 525, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 527, 558, 496, 525, 507, 533, 533, 525, 496, 514, 486, 500, 509, 510, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 506, 513, 487, 507, 490, 497, 507, 516, 532, 477, 511, 482, 515, 532, 508, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 487, 539, 507, 511, 527, 515, 482, 507, 499, 502, 529, 512, 531, 527, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8358, 8291, 8192, 8082, 8063, 8217, 8130, 8203] avg=8192.0000 max=8358 max_rank=0 min=8063 min_rank=4 imbalance_ratio=1.020264 max_load_reduction=57178 max_load_reduction_pct=0.872467 moved_to_echo=32613 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[543, 532, 532, 500, 505, 522, 542, 486, 488, 495, 516, 497, 564, 521, 486, 517, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 521, 488, 501, 478, 527, 513, 485, 509, 529, 528, 522, 523, 544, 528, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 511, 511, 535, 502, 507, 516, 512, 524, 518, 539, 498, 521, 471, 545, 521, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 520, 463, 464, 521, 536, 508, 518, 540, 505, 524, 474, 511, 482, 512, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 493, 495, 487, 537, 498, 523, 538, 533, 540, 470, 538, 529, 533, 515, 496, 9, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 505, 503, 486, 470, 502, 480, 498, 514, 470, 545, 538, 479, 524, 496, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 532, 506, 486, 537, 517, 491, 544, 509, 519, 475, 479, 506, 490, 511, 513, 14, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 523, 526, 492, 575, 538, 507, 466, 475, 471, 512, 500, 500, 508, 536, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8274, 8296, 8249, 8118, 8232, 8017, 8153, 8197] avg=8192.0000 max=8296 max_rank=1 min=8017 min_rank=5 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32599 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[543, 532, 532, 500, 505, 522, 542, 486, 488, 495, 516, 497, 564, 521, 486, 517, 546, 521, 488, 501, 478, 527, 513, 485, 509, 529, 528, 522, 546, 544, 528, 554, 510, 511, 511, 535, 502, 507, 516, 512, 524, 518, 539, 498, 521, 471, 545, 521, 511, 520, 463, 464, 521, 536, 508, 518, 540, 505, 524, 474, 511, 482, 512, 544, 490, 493, 495, 487, 537, 498, 523, 538, 533, 540, 470, 538, 529, 533, 515, 496, 478, 505, 503, 486, 470, 502, 480, 498, 514, 470, 545, 575, 479, 524, 496, 529, 515, 532, 506, 486, 537, 517, 491, 544, 509, 519, 475, 479, 506, 490, 511, 513, 547, 523, 527, 492, 575, 538, 507, 466, 475, 471, 512, 500, 500, 508, 536, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[485, 490, 496, 537, 529, 568, 482, 537, 496, 493, 501, 488, 512, 503, 518, 490, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 521, 521, 522, 552, 531, 547, 547, 499, 530, 537, 496, 492, 513, 490, 460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 530, 500, 499, 500, 507, 501, 553, 462, 543, 531, 519, 494, 517, 541, 477, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 507, 531, 515, 550, 534, 478, 519, 499, 510, 503, 468, 506, 511, 513, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 521, 494, 506, 528, 511, 525, 508, 509, 528, 514, 515, 522, 515, 493, 515, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 494, 510, 541, 535, 482, 515, 515, 514, 494, 501, 472, 495, 532, 514, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 507, 534, 520, 475, 488, 502, 496, 532, 551, 521, 483, 516, 501, 496, 489, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 515, 536, 512, 518, 508, 486, 489, 520, 530, 516, 479, 527, 518, 503, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8145, 8273, 8144, 8178, 8248, 8143, 8145, 8260] avg=8192.0000 max=8273 max_rank=1 min=8143 min_rank=5 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32796 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[490, 513, 552, 501, 519, 486, 494, 540, 481, 562, 512, 474, 526, 480, 517, 513, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 516, 507, 469, 543, 561, 527, 490, 493, 528, 523, 549, 520, 476, 502, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 490, 506, 489, 476, 495, 513, 492, 495, 533, 526, 506, 534, 500, 481, 515, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 502, 478, 476, 516, 516, 534, 537, 524, 513, 497, 513, 509, 481, 520, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 493, 511, 492, 500, 534, 570, 526, 523, 512, 524, 486, 533, 506, 537, 539, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 481, 530, 527, 461, 485, 484, 506, 530, 525, 549, 515, 515, 531, 534, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 482, 508, 475, 484, 578, 505, 498, 506, 495, 512, 531, 523, 518, 498, 500, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 565, 475, 537, 533, 528, 527, 488, 519, 492, 511, 498, 522, 517, 534, 473, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8196, 8061, 8166, 8301, 8211, 8147, 8272] avg=8192.0000 max=8301 max_rank=4 min=8061 min_rank=2 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32931 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[518, 539, 441, 495, 478, 483, 456, 557, 513, 521, 532, 493, 513, 522, 546, 523, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 471, 487, 506, 484, 473, 501, 526, 491, 462, 521, 541, 530, 495, 482, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 502, 529, 521, 489, 511, 522, 492, 490, 498, 522, 481, 515, 534, 547, 534, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 512, 534, 522, 528, 477, 494, 494, 524, 542, 502, 529, 517, 516, 531, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 463, 459, 540, 513, 511, 497, 496, 543, 558, 504, 504, 512, 497, 531, 518, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 557, 503, 476, 527, 507, 512, 487, 539, 508, 530, 485, 489, 516, 495, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 502, 534, 530, 537, 521, 520, 512, 534, 531, 533, 536, 507, 509, 508, 551, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 539, 548, 508, 502, 518, 485, 510, 502, 516, 515, 514, 538, 493, 490, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 7979, 8182, 8254, 8178, 8151, 8412, 8229] avg=8192.0000 max=8412 max_rank=6 min=7979 min_rank=1 imbalance_ratio=1.026855 max_load_reduction=57124 max_load_reduction_pct=0.871643 moved_to_echo=32970 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[559, 527, 515, 521, 553, 554, 523, 506, 526, 529, 515, 490, 501, 500, 527, 491, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 525, 505, 532, 472, 516, 506, 517, 495, 551, 475, 504, 522, 547, 575, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 519, 530, 529, 497, 519, 518, 455, 533, 507, 488, 497, 529, 523, 516, 524, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 515, 503, 509, 508, 517, 538, 538, 461, 475, 508, 518, 494, 478, 514, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 533, 496, 514, 497, 486, 483, 492, 508, 486, 505, 523, 510, 495, 515, 525, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 527, 558, 496, 525, 507, 533, 533, 525, 496, 514, 486, 500, 509, 510, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 506, 513, 487, 507, 490, 497, 507, 516, 532, 477, 511, 482, 515, 532, 508, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 487, 539, 507, 511, 527, 515, 482, 507, 499, 502, 529, 512, 531, 527, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8358, 8291, 8192, 8082, 8063, 8217, 8130, 8203] avg=8192.0000 max=8358 max_rank=0 min=8063 min_rank=4 imbalance_ratio=1.020264 max_load_reduction=57178 max_load_reduction_pct=0.872467 moved_to_echo=32613 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[510, 520, 507, 508, 459, 506, 479, 447, 524, 544, 522, 523, 486, 523, 491, 521, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 510, 539, 486, 526, 541, 501, 520, 505, 550, 506, 482, 498, 504, 499, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 520, 526, 497, 490, 508, 514, 579, 529, 563, 512, 482, 524, 543, 509, 501, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 528, 500, 514, 481, 502, 576, 522, 475, 510, 496, 556, 524, 523, 517, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 557, 516, 506, 495, 538, 525, 553, 520, 545, 476, 521, 551, 534, 487, 483, 497, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 509, 536, 480, 491, 531, 516, 546, 510, 514, 517, 478, 498, 507, 523, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 496, 467, 503, 492, 515, 540, 533, 486, 541, 500, 478, 491, 472, 522, 508, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 501, 537, 484, 492, 533, 512, 498, 469, 533, 525, 476, 508, 531, 466, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8091, 8186, 8317, 8280, 8310, 8209, 8074, 8069] avg=8192.0000 max=8317 max_rank=2 min=8069 min_rank=7 imbalance_ratio=1.015259 max_load_reduction=57219 max_load_reduction_pct=0.873093 moved_to_echo=32662 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[543, 532, 532, 500, 505, 522, 542, 486, 488, 495, 516, 497, 564, 521, 486, 517, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 521, 488, 501, 478, 527, 513, 485, 509, 529, 528, 522, 523, 544, 528, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 511, 511, 535, 502, 507, 516, 512, 524, 518, 539, 498, 521, 471, 545, 521, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 520, 463, 464, 521, 536, 508, 518, 540, 505, 524, 474, 511, 482, 512, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 493, 495, 487, 537, 498, 523, 538, 533, 540, 470, 538, 529, 533, 515, 496, 9, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 505, 503, 486, 470, 502, 480, 498, 514, 470, 545, 538, 479, 524, 496, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 532, 506, 486, 537, 517, 491, 544, 509, 519, 475, 479, 506, 490, 511, 513, 14, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 523, 526, 492, 575, 538, 507, 466, 475, 471, 512, 500, 500, 508, 536, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8274, 8296, 8249, 8118, 8232, 8017, 8153, 8197] avg=8192.0000 max=8296 max_rank=1 min=8017 min_rank=5 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32599 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[580, 509, 535, 510, 503, 491, 467, 518, 507, 503, 479, 518, 475, 536, 525, 508, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 503, 542, 521, 501, 509, 460, 529, 519, 507, 510, 519, 500, 491, 525, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 519, 503, 541, 508, 506, 501, 520, 526, 493, 520, 482, 515, 514, 553, 475, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 491, 560, 506, 527, 517, 509, 508, 522, 532, 474, 530, 552, 495, 501, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 487, 460, 527, 517, 489, 479, 521, 491, 463, 514, 539, 513, 511, 508, 572, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 536, 518, 538, 536, 532, 510, 518, 547, 556, 482, 507, 491, 478, 539, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 505, 487, 531, 518, 517, 521, 517, 501, 496, 517, 468, 494, 536, 486, 502, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 518, 543, 523, 479, 525, 540, 515, 510, 530, 545, 477, 492, 510, 493, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8155, 8204, 8240, 8125, 8331, 8104, 8191] avg=8192.0000 max=8331 max_rank=5 min=8104 min_rank=6 imbalance_ratio=1.016968 max_load_reduction=57205 max_load_reduction_pct=0.872879 moved_to_echo=32751 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[518, 539, 441, 495, 478, 483, 456, 557, 513, 521, 532, 493, 513, 522, 546, 523, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 471, 487, 506, 484, 473, 501, 526, 491, 462, 521, 541, 530, 495, 482, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 502, 529, 521, 489, 511, 522, 492, 490, 498, 522, 481, 515, 534, 547, 534, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 512, 534, 522, 528, 477, 494, 494, 524, 542, 502, 529, 517, 516, 531, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 463, 459, 540, 513, 511, 497, 496, 543, 558, 504, 504, 512, 497, 531, 518, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 557, 503, 476, 527, 507, 512, 487, 539, 508, 530, 485, 489, 516, 495, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 502, 534, 530, 537, 521, 520, 512, 534, 531, 533, 536, 507, 509, 508, 551, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 539, 548, 508, 502, 518, 485, 510, 502, 516, 515, 514, 538, 493, 490, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 7979, 8182, 8254, 8178, 8151, 8412, 8229] avg=8192.0000 max=8412 max_rank=6 min=7979 min_rank=1 imbalance_ratio=1.026855 max_load_reduction=57124 max_load_reduction_pct=0.871643 moved_to_echo=32970 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[471, 500, 501, 498, 501, 505, 512, 506, 482, 515, 539, 488, 526, 523, 534, 525, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 504, 538, 503, 502, 522, 525, 510, 507, 514, 512, 500, 506, 491, 501, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 505, 501, 496, 519, 499, 498, 459, 533, 526, 549, 518, 528, 487, 533, 519, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 483, 551, 529, 497, 505, 499, 505, 501, 537, 523, 504, 548, 489, 540, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 480, 458, 512, 512, 464, 513, 524, 484, 497, 505, 542, 509, 529, 486, 540, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 571, 483, 514, 532, 524, 483, 540, 498, 508, 504, 547, 520, 507, 517, 525, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 513, 533, 538, 521, 556, 534, 506, 505, 503, 508, 523, 522, 528, 501, 539, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 503, 524, 527, 498, 491, 547, 463, 507, 498, 489, 525, 517, 484, 528, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8149, 8160, 8187, 8218, 8079, 8257, 8371, 8115] avg=8192.0000 max=8371 max_rank=6 min=8079 min_rank=4 imbalance_ratio=1.021851 max_load_reduction=57165 max_load_reduction_pct=0.872269 moved_to_echo=32822 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 3968, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4352, 4096, 4224, 4224, 4096, 3968, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 3968, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4352, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 3968, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4352, 4096, 4224, 4224, 4096, 3968, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[513, 471, 531, 479, 526, 518, 517, 546, 517, 483, 501, 473, 505, 481, 523, 536, 529, 498, 546, 516, 469, 494, 515, 526, 488, 499, 508, 527, 514, 517, 518, 552, 501, 506, 516, 550, 498, 542, 495, 479, 465, 536, 489, 515, 498, 490, 492, 499, 509, 507, 544, 518, 493, 484, 502, 487, 501, 535, 514, 519, 516, 479, 500, 497, 531, 553, 493, 522, 477, 516, 512, 524, 518, 505, 537, 529, 515, 530, 499, 527, 509, 544, 505, 507, 496, 551, 538, 529, 518, 527, 516, 578, 474, 487, 513, 486, 483, 494, 541, 524, 520, 510, 564, 504, 508, 491, 494, 545, 515, 509, 496, 527, 507, 533, 483, 530, 535, 529, 529, 481, 507, 507, 492, 524, 509, 498, 519, 550] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[543, 480, 493, 497, 511, 503, 547, 515, 539, 538, 554, 544, 507, 539, 443, 480, 478, 477, 505, 532, 499, 501, 514, 491, 513, 480, 473, 493, 542, 523, 493, 514, 571, 518, 491, 529, 534, 510, 501, 503, 501, 477, 517, 499, 562, 490, 468, 494, 537, 521, 534, 499, 559, 549, 513, 516, 552, 511, 554, 509, 454, 495, 505, 536, 489, 533, 544, 491, 506, 492, 491, 484, 489, 562, 531, 525, 518, 505, 507, 554, 496, 494, 486, 496, 531, 525, 453, 519, 487, 531, 552, 547, 524, 511, 533, 497, 517, 505, 495, 534, 521, 511, 529, 552, 507, 550, 482, 491, 501, 524, 489, 474, 515, 512, 513, 504, 535, 522, 524, 495, 494, 510, 514, 526, 510, 513, 494, 500] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[502, 503, 477, 519, 550, 537, 545, 507, 519, 497, 482, 513, 473, 488, 493, 531, 488, 521, 529, 520, 523, 490, 525, 505, 523, 506, 493, 517, 500, 527, 478, 502, 521, 536, 493, 499, 498, 516, 488, 501, 484, 507, 500, 525, 514, 509, 494, 498, 515, 517, 497, 516, 536, 510, 552, 529, 530, 486, 543, 509, 539, 522, 513, 510, 522, 538, 479, 541, 495, 532, 512, 508, 537, 517, 513, 478, 505, 478, 484, 471, 501, 557, 535, 518, 532, 518, 497, 479, 495, 516, 525, 551, 533, 521, 523, 506, 516, 516, 514, 526, 542, 507, 525, 522, 482, 521, 504, 517, 463, 503, 530, 522, 514, 531, 502, 533, 547, 508, 522, 517, 471, 477, 552, 510, 513, 507, 505, 510] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[510, 483, 492, 560, 467, 540, 550, 497, 508, 522, 509, 484, 531, 481, 542, 487, 483, 527, 530, 543, 472, 483, 554, 510, 539, 537, 510, 515, 516, 495, 519, 526, 517, 531, 537, 508, 464, 513, 527, 518, 498, 491, 512, 513, 506, 530, 482, 508, 501, 521, 512, 531, 488, 485, 521, 526, 520, 471, 514, 515, 551, 474, 545, 513, 511, 504, 510, 515, 521, 503, 528, 549, 496, 521, 482, 503, 529, 502, 497, 484, 519, 517, 498, 503, 516, 498, 523, 542, 504, 517, 507, 508, 518, 519, 520, 472, 511, 523, 512, 531, 536, 538, 517, 529, 488, 521, 511, 483, 532, 528, 472, 502, 511, 482, 512, 480, 519, 531, 513, 535, 486, 498, 506, 502, 562, 532, 517, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[556, 544, 543, 504, 515, 532, 538, 515, 505, 534, 528, 510, 499, 486, 465, 499, 532, 542, 554, 471, 490, 514, 487, 521, 543, 476, 549, 501, 527, 479, 504, 482, 506, 515, 548, 484, 522, 497, 521, 514, 531, 502, 493, 484, 517, 497, 497, 492, 521, 467, 499, 538, 512, 534, 518, 530, 541, 506, 542, 532, 508, 544, 506, 497, 438, 505, 527, 515, 492, 483, 515, 520, 487, 526, 520, 515, 534, 495, 508, 526, 498, 513, 478, 535, 526, 535, 522, 521, 495, 513, 508, 526, 502, 479, 495, 513, 504, 521, 504, 493, 509, 488, 529, 512, 572, 510, 536, 501, 478, 486, 521, 561, 496, 533, 486, 492, 526, 511, 508, 543, 513, 519, 545, 512, 502, 507, 499, 494] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[525, 520, 476, 511, 520, 507, 491, 504, 532, 538, 482, 500, 496, 507, 496, 498, 545, 551, 524, 558, 554, 512, 484, 542, 524, 500, 496, 495, 511, 533, 525, 518, 517, 507, 519, 513, 519, 490, 529, 551, 485, 496, 486, 535, 492, 481, 543, 500, 496, 522, 526, 509, 526, 523, 484, 516, 512, 509, 515, 501, 517, 536, 476, 491, 495, 471, 550, 519, 518, 544, 483, 531, 501, 519, 477, 521, 509, 464, 514, 499, 516, 522, 478, 534, 541, 518, 508, 493, 509, 518, 548, 551, 511, 533, 498, 498, 550, 495, 520, 525, 536, 488, 483, 506, 511, 499, 511, 487, 543, 489, 515, 496, 514, 524, 481, 530, 483, 533, 532, 532, 529, 510, 492, 505, 530, 503, 496, 500] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[516, 527, 495, 508, 511, 501, 530, 521, 510, 544, 492, 513, 531, 512, 515, 514, 460, 483, 511, 481, 474, 520, 501, 500, 529, 518, 510, 535, 527, 515, 538, 524, 523, 526, 483, 510, 508, 560, 534, 489, 521, 536, 492, 484, 495, 495, 511, 507, 514, 481, 496, 526, 502, 556, 495, 504, 469, 519, 536, 528, 564, 499, 515, 545, 520, 515, 491, 515, 495, 532, 564, 486, 543, 476, 503, 518, 473, 501, 541, 469, 496, 463, 519, 520, 511, 509, 489, 516, 516, 519, 543, 546, 535, 545, 513, 515, 533, 493, 525, 513, 502, 487, 512, 490, 522, 527, 489, 513, 516, 526, 479, 519, 550, 493, 540, 485, 517, 492, 463, 537, 514, 565, 487, 505, 527, 507, 526, 496] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[502, 503, 477, 519, 550, 537, 545, 507, 519, 497, 482, 513, 473, 488, 493, 531, 488, 521, 529, 520, 523, 490, 525, 505, 523, 506, 493, 517, 500, 527, 478, 502, 521, 536, 493, 499, 498, 516, 488, 501, 484, 507, 500, 525, 514, 509, 494, 498, 515, 517, 497, 516, 536, 510, 552, 529, 530, 486, 543, 509, 539, 522, 513, 510, 522, 538, 479, 541, 495, 532, 512, 508, 537, 517, 513, 478, 505, 478, 484, 471, 501, 557, 535, 518, 532, 518, 497, 479, 495, 516, 525, 551, 533, 521, 523, 506, 516, 516, 514, 526, 542, 507, 525, 522, 482, 521, 504, 517, 463, 503, 530, 522, 514, 531, 502, 533, 547, 508, 522, 517, 471, 477, 552, 510, 513, 507, 505, 510] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[489, 507, 471, 512, 511, 470, 475, 510, 538, 517, 548, 535, 529, 521, 499, 499, 572, 518, 511, 478, 522, 534, 512, 499, 520, 496, 506, 532, 528, 474, 507, 518, 506, 498, 495, 483, 496, 504, 478, 477, 558, 536, 551, 517, 510, 522, 507, 505, 498, 525, 532, 545, 520, 530, 502, 508, 503, 527, 496, 495, 546, 522, 518, 497, 538, 549, 509, 520, 525, 510, 525, 474, 504, 477, 498, 508, 506, 506, 482, 490, 534, 532, 503, 532, 525, 546, 503, 555, 505, 513, 495, 519, 504, 493, 538, 500, 500, 514, 494, 494, 504, 532, 544, 519, 504, 491, 534, 517, 503, 529, 528, 519, 481, 560, 509, 494, 493, 485, 451, 510, 519, 486, 532, 523, 531, 508, 493, 552] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[556, 544, 543, 504, 515, 532, 538, 515, 505, 534, 528, 510, 499, 486, 465, 499, 8, 6, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 542, 553, 471, 490, 514, 487, 521, 543, 476, 549, 501, 527, 479, 504, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 515, 548, 484, 522, 497, 521, 514, 531, 502, 493, 484, 517, 497, 497, 492, 48, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 467, 499, 538, 512, 534, 518, 530, 541, 506, 493, 532, 508, 544, 506, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 438, 505, 527, 515, 492, 483, 515, 520, 487, 526, 520, 515, 534, 495, 508, 526, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 513, 478, 535, 526, 535, 522, 521, 495, 513, 508, 478, 502, 479, 495, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 521, 504, 493, 509, 488, 521, 512, 572, 510, 536, 501, 478, 486, 521, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 533, 486, 492, 526, 511, 508, 543, 513, 519, 545, 512, 501, 507, 499, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8289, 8171, 8176, 8246, 8141, 8111, 8217, 8185] avg=8192.0000 max=8289 max_rank=0 min=8111 min_rank=5 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32654 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[556, 544, 543, 504, 515, 532, 538, 515, 505, 534, 528, 510, 499, 486, 465, 499, 532, 542, 554, 471, 490, 514, 487, 521, 543, 476, 549, 501, 527, 479, 504, 482, 506, 515, 548, 484, 522, 497, 521, 514, 531, 502, 493, 484, 517, 497, 497, 492, 521, 467, 499, 538, 512, 534, 518, 530, 541, 506, 542, 532, 508, 544, 506, 497, 438, 505, 527, 515, 492, 483, 515, 520, 487, 526, 520, 515, 534, 495, 508, 526, 498, 513, 478, 535, 526, 535, 522, 521, 495, 513, 508, 526, 502, 479, 495, 513, 504, 521, 504, 493, 509, 488, 529, 512, 572, 510, 536, 501, 478, 486, 521, 561, 496, 533, 486, 492, 526, 511, 508, 543, 513, 519, 545, 512, 502, 507, 499, 494] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[513, 471, 531, 479, 526, 518, 517, 546, 517, 483, 501, 473, 505, 481, 523, 536, 8, 6, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 498, 545, 516, 469, 494, 515, 526, 488, 499, 508, 527, 514, 517, 518, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 506, 516, 550, 498, 542, 495, 479, 465, 536, 489, 515, 498, 490, 492, 499, 53, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 507, 544, 518, 493, 484, 502, 487, 501, 535, 468, 519, 516, 479, 500, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 553, 493, 522, 477, 516, 512, 524, 518, 505, 537, 529, 515, 530, 499, 527, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 544, 505, 507, 496, 551, 538, 529, 518, 527, 516, 525, 474, 487, 513, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 494, 541, 524, 520, 510, 556, 504, 508, 491, 494, 545, 515, 509, 496, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 533, 483, 530, 535, 529, 529, 481, 507, 507, 492, 524, 508, 498, 519, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8136, 8215, 8131, 8059, 8321, 8225, 8217, 8232] avg=8192.0000 max=8321 max_rank=4 min=8059 min_rank=3 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32995 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[543, 480, 493, 497, 511, 503, 547, 515, 539, 538, 554, 544, 507, 539, 443, 480, 11, 9, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 477, 501, 532, 499, 501, 514, 491, 513, 480, 473, 493, 542, 523, 493, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 571, 518, 491, 529, 534, 510, 501, 503, 501, 477, 517, 499, 562, 490, 468, 494, 55, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 521, 534, 499, 559, 549, 513, 516, 552, 511, 492, 509, 454, 495, 505, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 533, 544, 491, 506, 492, 491, 484, 489, 562, 531, 525, 518, 505, 507, 554, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 494, 486, 496, 531, 525, 453, 519, 487, 531, 552, 492, 524, 511, 533, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 505, 495, 534, 521, 511, 518, 552, 507, 550, 482, 491, 501, 524, 489, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 512, 513, 504, 535, 522, 524, 495, 494, 510, 514, 526, 508, 513, 494, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8259, 8024, 8231, 8282, 8263, 8127, 8171, 8179] avg=8192.0000 max=8282 max_rank=3 min=8024 min_rank=1 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32740 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[525, 520, 476, 511, 520, 507, 491, 504, 532, 538, 482, 500, 496, 507, 496, 498, 7, 6, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 551, 523, 558, 554, 512, 484, 542, 524, 500, 496, 495, 511, 533, 525, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 507, 519, 513, 519, 490, 529, 551, 485, 496, 486, 535, 492, 481, 543, 500, 50, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 522, 526, 509, 526, 523, 484, 516, 512, 509, 469, 501, 517, 536, 476, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 471, 550, 519, 518, 544, 483, 531, 501, 519, 477, 521, 509, 464, 514, 499, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 522, 478, 534, 541, 518, 508, 493, 509, 518, 548, 501, 511, 533, 498, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 495, 520, 525, 536, 488, 476, 506, 511, 499, 511, 487, 543, 489, 515, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 524, 481, 530, 483, 533, 532, 532, 529, 510, 492, 505, 529, 503, 496, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8118, 8371, 8220, 8113, 8148, 8226, 8147, 8193] avg=8192.0000 max=8371 max_rank=1 min=8113 min_rank=3 imbalance_ratio=1.021851 max_load_reduction=57165 max_load_reduction_pct=0.872269 moved_to_echo=32714 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[510, 483, 492, 560, 467, 540, 550, 497, 508, 522, 509, 484, 531, 481, 542, 487, 8, 6, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 527, 529, 543, 472, 483, 554, 510, 539, 537, 510, 515, 516, 495, 519, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 531, 537, 508, 464, 513, 527, 518, 498, 491, 512, 513, 506, 530, 482, 508, 46, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 521, 512, 531, 488, 485, 521, 526, 520, 471, 468, 515, 551, 474, 545, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 504, 510, 515, 521, 503, 528, 549, 496, 521, 482, 503, 529, 502, 497, 484, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 517, 498, 503, 516, 498, 523, 542, 504, 517, 507, 462, 518, 519, 520, 472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 523, 512, 531, 536, 538, 509, 529, 488, 521, 511, 483, 532, 528, 472, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 482, 512, 480, 519, 531, 513, 535, 486, 498, 506, 502, 561, 532, 517, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8258, 8208, 8142, 8188, 8135, 8226, 8200] avg=8192.0000 max=8258 max_rank=1 min=8135 min_rank=5 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32749 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[516, 527, 495, 508, 511, 501, 530, 521, 510, 544, 492, 513, 531, 512, 515, 514, 8, 6, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 460, 483, 510, 481, 474, 520, 501, 500, 529, 518, 510, 535, 527, 515, 538, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 526, 483, 510, 508, 560, 534, 489, 521, 536, 492, 484, 495, 495, 511, 507, 50, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 481, 496, 526, 502, 556, 495, 504, 469, 519, 487, 528, 564, 499, 515, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 515, 491, 515, 495, 532, 564, 486, 543, 476, 503, 518, 473, 501, 541, 469, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 463, 519, 520, 511, 509, 489, 516, 516, 519, 543, 496, 535, 545, 513, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 493, 525, 513, 502, 487, 504, 490, 522, 527, 489, 513, 516, 526, 479, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 493, 540, 485, 517, 492, 463, 537, 514, 565, 487, 505, 526, 507, 526, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8256, 8125, 8232, 8200, 8177, 8205, 8138, 8203] avg=8192.0000 max=8256 max_rank=0 min=8125 min_rank=1 imbalance_ratio=1.007812 max_load_reduction=57280 max_load_reduction_pct=0.874023 moved_to_echo=32723 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[502, 503, 477, 519, 550, 537, 545, 507, 519, 497, 482, 513, 473, 488, 493, 531, 8, 6, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 521, 528, 520, 523, 490, 525, 505, 523, 506, 493, 517, 500, 527, 478, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 536, 493, 499, 498, 516, 488, 501, 484, 507, 500, 525, 514, 509, 494, 498, 50, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 517, 497, 516, 536, 510, 552, 529, 530, 486, 494, 509, 539, 522, 513, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 538, 479, 541, 495, 532, 512, 508, 537, 517, 513, 478, 505, 478, 484, 471, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 557, 535, 518, 532, 518, 497, 479, 495, 516, 525, 501, 533, 521, 523, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 516, 514, 526, 542, 507, 517, 522, 482, 521, 504, 517, 463, 503, 530, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 531, 502, 533, 547, 508, 522, 517, 471, 477, 552, 510, 512, 507, 505, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8152, 8146, 8141, 8275, 8145, 8257, 8202, 8218] avg=8192.0000 max=8275 max_rank=3 min=8141 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32822 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[489, 507, 471, 512, 511, 470, 475, 510, 538, 517, 548, 535, 529, 521, 499, 499, 8, 6, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 572, 518, 510, 478, 522, 534, 512, 499, 520, 496, 506, 532, 528, 474, 507, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 498, 495, 483, 496, 504, 478, 477, 558, 536, 551, 517, 510, 522, 507, 505, 47, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 525, 532, 545, 520, 530, 502, 508, 503, 527, 451, 495, 546, 522, 518, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 549, 509, 520, 525, 510, 525, 474, 504, 477, 498, 508, 506, 506, 482, 490, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 532, 503, 532, 525, 546, 503, 555, 505, 513, 495, 472, 504, 493, 538, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 514, 494, 494, 504, 532, 536, 519, 504, 491, 534, 517, 503, 529, 528, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 560, 509, 494, 493, 485, 451, 510, 519, 486, 532, 523, 530, 508, 493, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8147, 8226, 8197, 8219, 8153, 8250, 8218, 8126] avg=8192.0000 max=8250 max_rank=5 min=8126 min_rank=7 imbalance_ratio=1.007080 max_load_reduction=57286 max_load_reduction_pct=0.874115 moved_to_echo=32747 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[516, 527, 495, 508, 511, 501, 530, 521, 510, 544, 492, 513, 531, 512, 515, 514, 8, 6, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 460, 483, 510, 481, 474, 520, 501, 500, 529, 518, 510, 535, 527, 515, 538, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 526, 483, 510, 508, 560, 534, 489, 521, 536, 492, 484, 495, 495, 511, 507, 50, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 481, 496, 526, 502, 556, 495, 504, 469, 519, 487, 528, 564, 499, 515, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 515, 491, 515, 495, 532, 564, 486, 543, 476, 503, 518, 473, 501, 541, 469, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 463, 519, 520, 511, 509, 489, 516, 516, 519, 543, 496, 535, 545, 513, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 493, 525, 513, 502, 487, 504, 490, 522, 527, 489, 513, 516, 526, 479, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 493, 540, 485, 517, 492, 463, 537, 514, 565, 487, 505, 526, 507, 526, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8256, 8125, 8232, 8200, 8177, 8205, 8138, 8203] avg=8192.0000 max=8256 max_rank=0 min=8125 min_rank=1 imbalance_ratio=1.007812 max_load_reduction=57280 max_load_reduction_pct=0.874023 moved_to_echo=32723 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[556, 544, 543, 504, 515, 532, 538, 515, 505, 534, 528, 510, 499, 486, 465, 499, 8, 6, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 542, 553, 471, 490, 514, 487, 521, 543, 476, 549, 501, 527, 479, 504, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 515, 548, 484, 522, 497, 521, 514, 531, 502, 493, 484, 517, 497, 497, 492, 48, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 467, 499, 538, 512, 534, 518, 530, 541, 506, 493, 532, 508, 544, 506, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 438, 505, 527, 515, 492, 483, 515, 520, 487, 526, 520, 515, 534, 495, 508, 526, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 513, 478, 535, 526, 535, 522, 521, 495, 513, 508, 478, 502, 479, 495, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 521, 504, 493, 509, 488, 521, 512, 572, 510, 536, 501, 478, 486, 521, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 533, 486, 492, 526, 511, 508, 543, 513, 519, 545, 512, 501, 507, 499, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8289, 8171, 8176, 8246, 8141, 8111, 8217, 8185] avg=8192.0000 max=8289 max_rank=0 min=8111 min_rank=5 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32654 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[525, 520, 476, 511, 520, 507, 491, 504, 532, 538, 482, 500, 496, 507, 496, 498, 7, 6, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 551, 523, 558, 554, 512, 484, 542, 524, 500, 496, 495, 511, 533, 525, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 507, 519, 513, 519, 490, 529, 551, 485, 496, 486, 535, 492, 481, 543, 500, 50, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 522, 526, 509, 526, 523, 484, 516, 512, 509, 469, 501, 517, 536, 476, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 471, 550, 519, 518, 544, 483, 531, 501, 519, 477, 521, 509, 464, 514, 499, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 522, 478, 534, 541, 518, 508, 493, 509, 518, 548, 501, 511, 533, 498, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 495, 520, 525, 536, 488, 476, 506, 511, 499, 511, 487, 543, 489, 515, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 524, 481, 530, 483, 533, 532, 532, 529, 510, 492, 505, 529, 503, 496, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8118, 8371, 8220, 8113, 8148, 8226, 8147, 8193] avg=8192.0000 max=8371 max_rank=1 min=8113 min_rank=3 imbalance_ratio=1.021851 max_load_reduction=57165 max_load_reduction_pct=0.872269 moved_to_echo=32714 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 3968, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 3840, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 3968, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[518, 510, 509, 515, 467, 508, 479, 522, 479, 501, 492, 480, 527, 477, 459, 493, 547, 537, 504, 475, 519, 503, 520, 550, 508, 488, 501, 528, 525, 524, 499, 493, 503, 491, 499, 514, 523, 530, 505, 465, 482, 483, 514, 503, 544, 510, 501, 536, 551, 539, 520, 528, 488, 481, 529, 487, 495, 518, 513, 522, 516, 495, 521, 484, 465, 546, 499, 489, 541, 514, 525, 516, 477, 529, 493, 520, 522, 527, 516, 493, 527, 541, 532, 509, 540, 514, 520, 498, 499, 527, 542, 511, 546, 503, 510, 503, 537, 512, 578, 521, 512, 505, 508, 522, 477, 535, 511, 502, 536, 558, 515, 513, 518, 496, 526, 499, 543, 543, 514, 468, 525, 535, 513, 560, 464, 486, 546, 517] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[505, 517, 486, 519, 512, 522, 500, 504, 511, 544, 521, 529, 531, 546, 523, 493, 492, 515, 506, 497, 513, 497, 515, 519, 523, 490, 487, 522, 549, 505, 549, 489, 501, 535, 518, 541, 517, 510, 528, 526, 487, 510, 480, 504, 512, 551, 521, 531, 504, 522, 507, 523, 494, 508, 526, 511, 501, 537, 512, 490, 541, 514, 522, 519, 492, 544, 521, 537, 537, 494, 518, 517, 520, 545, 488, 510, 487, 519, 559, 460, 495, 545, 490, 497, 518, 532, 513, 507, 516, 547, 475, 494, 519, 495, 511, 504, 500, 506, 539, 492, 521, 548, 519, 499, 517, 489, 451, 518, 513, 469, 496, 506, 481, 491, 548, 509, 488, 519, 504, 507, 497, 515, 493, 537, 515, 506, 495, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[534, 525, 574, 550, 501, 479, 502, 499, 553, 560, 539, 525, 512, 502, 521, 539, 479, 472, 531, 474, 517, 508, 464, 508, 516, 489, 509, 531, 494, 494, 499, 545, 491, 483, 506, 536, 494, 510, 489, 501, 510, 521, 527, 514, 532, 511, 479, 515, 490, 519, 536, 521, 513, 535, 548, 524, 516, 540, 499, 513, 497, 566, 509, 491, 519, 481, 520, 494, 528, 513, 520, 519, 510, 507, 505, 547, 490, 541, 510, 536, 518, 496, 523, 508, 523, 488, 501, 503, 536, 500, 457, 509, 512, 523, 513, 510, 484, 522, 511, 534, 535, 523, 509, 522, 521, 522, 522, 539, 521, 491, 512, 510, 505, 487, 515, 514, 469, 489, 510, 504, 531, 495, 510, 507, 484, 522, 504, 471] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[505, 517, 486, 519, 512, 522, 500, 504, 511, 544, 521, 529, 531, 546, 523, 493, 492, 515, 506, 497, 513, 497, 515, 519, 523, 490, 487, 522, 549, 505, 549, 489, 501, 535, 518, 541, 517, 510, 528, 526, 487, 510, 480, 504, 512, 551, 521, 531, 504, 522, 507, 523, 494, 508, 526, 511, 501, 537, 512, 490, 541, 514, 522, 519, 492, 544, 521, 537, 537, 494, 518, 517, 520, 545, 488, 510, 487, 519, 559, 460, 495, 545, 490, 497, 518, 532, 513, 507, 516, 547, 475, 494, 519, 495, 511, 504, 500, 506, 539, 492, 521, 548, 519, 499, 517, 489, 451, 518, 513, 469, 496, 506, 481, 491, 548, 509, 488, 519, 504, 507, 497, 515, 493, 537, 515, 506, 495, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[543, 550, 504, 522, 483, 500, 560, 522, 517, 525, 527, 504, 484, 481, 493, 487, 472, 557, 505, 481, 529, 533, 528, 532, 517, 527, 511, 502, 511, 492, 492, 516, 502, 517, 522, 507, 496, 488, 522, 521, 487, 520, 529, 507, 520, 541, 537, 513, 513, 467, 538, 501, 520, 526, 508, 528, 526, 522, 561, 517, 530, 454, 502, 521, 507, 496, 508, 520, 530, 492, 495, 540, 517, 509, 492, 508, 506, 495, 572, 524, 532, 531, 502, 491, 498, 485, 482, 514, 505, 490, 515, 496, 535, 509, 496, 552, 505, 501, 498, 517, 489, 466, 504, 511, 510, 520, 506, 514, 513, 509, 510, 517, 521, 477, 530, 527, 498, 481, 522, 518, 514, 524, 537, 503, 517, 553, 515, 495] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[518, 510, 509, 515, 467, 508, 479, 522, 479, 501, 492, 480, 527, 477, 459, 493, 547, 537, 504, 475, 519, 503, 520, 550, 508, 488, 501, 528, 525, 524, 499, 493, 503, 491, 499, 514, 523, 530, 505, 465, 482, 483, 514, 503, 544, 510, 501, 536, 551, 539, 520, 528, 488, 481, 529, 487, 495, 518, 513, 522, 516, 495, 521, 484, 465, 546, 499, 489, 541, 514, 525, 516, 477, 529, 493, 520, 522, 527, 516, 493, 527, 541, 532, 509, 540, 514, 520, 498, 499, 527, 542, 511, 546, 503, 510, 503, 537, 512, 578, 521, 512, 505, 508, 522, 477, 535, 511, 502, 536, 558, 515, 513, 518, 496, 526, 499, 543, 543, 514, 468, 525, 535, 513, 560, 464, 486, 546, 517] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[557, 520, 500, 470, 514, 517, 519, 519, 500, 523, 508, 518, 475, 519, 523, 530, 491, 522, 506, 500, 511, 541, 503, 538, 542, 540, 501, 494, 478, 532, 531, 518, 506, 510, 494, 511, 525, 524, 506, 504, 477, 475, 506, 544, 518, 460, 488, 494, 479, 481, 468, 555, 464, 476, 477, 523, 499, 513, 546, 526, 477, 539, 523, 519, 487, 512, 512, 550, 534, 529, 511, 516, 508, 528, 516, 489, 504, 478, 519, 514, 508, 509, 509, 490, 531, 509, 529, 504, 513, 501, 504, 528, 540, 538, 508, 501, 523, 503, 518, 503, 547, 522, 520, 553, 522, 532, 500, 528, 539, 512, 499, 528, 532, 533, 511, 482, 498, 504, 518, 509, 550, 512, 507, 477, 493, 540, 504, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[538, 519, 517, 475, 526, 524, 500, 530, 534, 480, 540, 446, 533, 535, 547, 506, 508, 521, 499, 472, 511, 533, 508, 478, 510, 542, 500, 510, 520, 521, 506, 490, 523, 515, 535, 531, 505, 521, 502, 505, 491, 539, 514, 519, 527, 506, 509, 519, 535, 508, 520, 514, 517, 519, 538, 508, 519, 494, 527, 521, 462, 495, 518, 508, 481, 523, 533, 477, 504, 509, 508, 521, 518, 488, 513, 524, 510, 499, 489, 493, 490, 508, 517, 481, 494, 522, 497, 504, 532, 474, 485, 542, 520, 522, 507, 518, 537, 525, 498, 544, 513, 527, 546, 541, 531, 512, 495, 520, 532, 489, 507, 495, 534, 505, 493, 509, 488, 533, 523, 528, 529, 522, 489, 516, 505, 486, 493, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[474, 541, 543, 495, 508, 512, 481, 506, 493, 518, 471, 518, 543, 514, 536, 495, 505, 481, 555, 529, 550, 498, 501, 515, 516, 486, 507, 509, 502, 507, 520, 532, 525, 503, 550, 512, 508, 531, 520, 517, 508, 519, 521, 460, 501, 494, 489, 527, 498, 516, 507, 535, 519, 556, 532, 484, 533, 509, 517, 522, 521, 513, 523, 543, 521, 500, 533, 521, 509, 525, 514, 513, 557, 516, 487, 499, 541, 507, 542, 536, 504, 517, 505, 500, 510, 528, 533, 497, 481, 530, 505, 527, 480, 503, 499, 489, 481, 475, 517, 504, 491, 471, 550, 503, 499, 497, 493, 502, 524, 502, 513, 500, 509, 525, 526, 538, 490, 534, 505, 489, 500, 551, 499, 551, 480, 510, 494, 510] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[538, 519, 517, 475, 526, 524, 500, 530, 534, 480, 540, 446, 533, 535, 547, 506, 508, 521, 499, 472, 511, 533, 508, 478, 510, 542, 500, 510, 520, 521, 506, 490, 523, 515, 535, 531, 505, 521, 502, 505, 491, 539, 514, 519, 527, 506, 509, 519, 535, 508, 520, 514, 517, 519, 538, 508, 519, 494, 527, 521, 462, 495, 518, 508, 481, 523, 533, 477, 504, 509, 508, 521, 518, 488, 513, 524, 510, 499, 489, 493, 490, 508, 517, 481, 494, 522, 497, 504, 532, 474, 485, 542, 520, 522, 507, 518, 537, 525, 498, 544, 513, 527, 546, 541, 531, 512, 495, 520, 532, 489, 507, 495, 534, 505, 493, 509, 488, 533, 523, 528, 529, 522, 489, 516, 505, 486, 493, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[531, 532, 508, 492, 540, 516, 475, 506, 508, 545, 521, 517, 502, 541, 548, 491, 472, 493, 513, 554, 537, 518, 534, 469, 497, 552, 497, 536, 491, 528, 487, 532, 494, 515, 495, 493, 513, 505, 539, 511, 503, 510, 534, 478, 521, 537, 488, 524, 485, 505, 535, 499, 537, 549, 544, 522, 515, 480, 470, 496, 525, 528, 493, 488, 544, 508, 521, 506, 543, 503, 501, 526, 513, 487, 533, 527, 519, 503, 499, 541, 499, 529, 499, 476, 515, 495, 509, 491, 506, 518, 490, 503, 495, 488, 502, 523, 496, 550, 525, 519, 514, 540, 520, 498, 513, 489, 524, 527, 535, 530, 495, 524, 482, 474, 527, 541, 499, 497, 517, 499, 508, 477, 525, 489, 497, 547, 523, 509] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[557, 501, 500, 470, 514, 517, 519, 519, 500, 523, 508, 518, 475, 519, 523, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 522, 506, 500, 511, 541, 503, 538, 542, 540, 501, 494, 478, 532, 531, 518, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 510, 494, 511, 525, 524, 506, 504, 477, 475, 506, 544, 518, 460, 488, 494, 12, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 481, 468, 555, 464, 476, 456, 523, 499, 513, 546, 526, 477, 539, 523, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 512, 512, 550, 506, 529, 511, 516, 508, 528, 516, 489, 504, 478, 519, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 509, 509, 490, 531, 509, 529, 504, 513, 501, 504, 528, 540, 538, 508, 501, 30, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 503, 518, 503, 547, 522, 520, 553, 522, 532, 500, 528, 509, 512, 499, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 533, 511, 482, 498, 504, 518, 509, 550, 512, 507, 477, 493, 540, 504, 521, 18, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8261, 8060, 8044, 8179, 8262, 8319, 8218] avg=8192.0000 max=8319 max_rank=6 min=8044 min_rank=3 imbalance_ratio=1.015503 max_load_reduction=57217 max_load_reduction_pct=0.873062 moved_to_echo=32978 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[534, 498, 574, 550, 501, 479, 502, 499, 553, 560, 539, 525, 512, 502, 521, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 472, 531, 474, 517, 508, 464, 508, 516, 489, 509, 531, 494, 494, 499, 545, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 483, 506, 536, 494, 510, 489, 501, 510, 521, 527, 514, 532, 511, 479, 515, 19, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 519, 536, 521, 513, 535, 514, 524, 516, 540, 499, 513, 497, 566, 509, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 481, 520, 494, 493, 513, 520, 519, 510, 507, 505, 547, 490, 541, 510, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 496, 523, 508, 523, 488, 501, 503, 536, 500, 457, 509, 512, 523, 513, 510, 32, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 522, 511, 534, 535, 523, 509, 522, 521, 522, 522, 539, 489, 491, 512, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 487, 515, 514, 469, 489, 510, 504, 531, 495, 510, 507, 484, 522, 504, 471, 20, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8388, 8048, 8147, 8283, 8205, 8167, 8246, 8052] avg=8192.0000 max=8388 max_rank=0 min=8048 min_rank=1 imbalance_ratio=1.023926 max_load_reduction=57148 max_load_reduction_pct=0.872009 moved_to_echo=32670 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[538, 500, 517, 475, 526, 524, 500, 530, 534, 480, 540, 446, 533, 535, 547, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 521, 499, 472, 511, 533, 508, 478, 510, 542, 500, 510, 520, 521, 506, 490, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 515, 535, 531, 505, 521, 502, 505, 491, 539, 514, 519, 527, 506, 509, 519, 14, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 508, 520, 514, 517, 519, 513, 508, 519, 494, 527, 521, 462, 495, 518, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 523, 533, 477, 478, 509, 508, 521, 518, 488, 513, 524, 510, 499, 489, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 508, 517, 481, 494, 522, 497, 504, 532, 474, 485, 542, 520, 522, 507, 518, 30, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 525, 498, 544, 513, 527, 546, 541, 531, 512, 495, 520, 502, 489, 507, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 505, 493, 509, 488, 533, 523, 528, 529, 522, 489, 516, 505, 486, 493, 525, 17, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8231, 8142, 8281, 8178, 8064, 8152, 8282, 8206] avg=8192.0000 max=8282 max_rank=6 min=8064 min_rank=4 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32704 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[518, 491, 509, 515, 467, 508, 479, 522, 479, 501, 492, 480, 527, 477, 459, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 537, 504, 475, 519, 503, 520, 550, 508, 488, 501, 528, 525, 524, 499, 493, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 491, 499, 514, 523, 530, 505, 465, 482, 483, 514, 503, 544, 510, 501, 536, 14, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 539, 520, 528, 488, 481, 505, 487, 495, 518, 513, 522, 516, 495, 521, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 546, 499, 489, 513, 514, 525, 516, 477, 529, 493, 520, 522, 527, 516, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 541, 532, 509, 540, 514, 520, 498, 499, 527, 542, 511, 546, 503, 510, 503, 30, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 512, 578, 521, 512, 505, 508, 522, 477, 535, 511, 502, 506, 558, 515, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 496, 526, 499, 543, 543, 514, 468, 525, 535, 513, 560, 464, 486, 546, 517, 18, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7917, 8234, 8123, 8163, 8144, 8362, 8312, 8281] avg=8192.0000 max=8362 max_rank=5 min=7917 min_rank=0 imbalance_ratio=1.020752 max_load_reduction=57174 max_load_reduction_pct=0.872406 moved_to_echo=33099 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[505, 498, 486, 519, 512, 522, 500, 504, 511, 544, 521, 529, 531, 546, 523, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 515, 506, 497, 513, 497, 515, 519, 523, 490, 487, 522, 549, 505, 549, 489, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 535, 518, 541, 517, 510, 528, 526, 487, 510, 480, 504, 512, 551, 521, 531, 14, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 522, 507, 523, 494, 508, 502, 511, 501, 537, 512, 490, 541, 514, 522, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 544, 521, 537, 509, 494, 518, 517, 520, 545, 488, 510, 487, 519, 559, 460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 545, 490, 497, 518, 532, 513, 507, 516, 547, 475, 494, 519, 495, 511, 504, 29, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 506, 539, 492, 521, 548, 519, 499, 517, 489, 451, 518, 484, 469, 496, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 491, 548, 509, 488, 519, 504, 507, 497, 515, 493, 537, 515, 506, 495, 508, 18, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8244, 8181, 8292, 8207, 8220, 8197, 8054, 8141] avg=8192.0000 max=8292 max_rank=2 min=8054 min_rank=6 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32612 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[543, 530, 504, 522, 483, 500, 560, 522, 517, 525, 527, 504, 484, 481, 493, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 557, 505, 481, 529, 533, 528, 532, 517, 527, 511, 502, 511, 492, 492, 516, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 517, 522, 507, 496, 488, 522, 521, 487, 520, 529, 507, 520, 541, 537, 513, 13, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 467, 538, 501, 520, 526, 485, 528, 526, 522, 561, 517, 530, 454, 502, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 496, 508, 520, 502, 492, 495, 540, 517, 509, 492, 508, 506, 495, 572, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 531, 502, 491, 498, 485, 482, 514, 505, 490, 515, 496, 535, 509, 496, 552, 29, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 501, 498, 517, 489, 466, 504, 511, 510, 520, 506, 514, 484, 509, 510, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 477, 530, 527, 498, 481, 522, 518, 514, 524, 537, 503, 517, 553, 515, 495, 18, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8219, 8248, 8211, 8183, 8172, 8061, 8260] avg=8192.0000 max=8260 max_rank=7 min=8061 min_rank=6 imbalance_ratio=1.008301 max_load_reduction=57276 max_load_reduction_pct=0.873962 moved_to_echo=32676 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[505, 498, 486, 519, 512, 522, 500, 504, 511, 544, 521, 529, 531, 546, 523, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 515, 506, 497, 513, 497, 515, 519, 523, 490, 487, 522, 549, 505, 549, 489, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 535, 518, 541, 517, 510, 528, 526, 487, 510, 480, 504, 512, 551, 521, 531, 14, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 522, 507, 523, 494, 508, 502, 511, 501, 537, 512, 490, 541, 514, 522, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 544, 521, 537, 509, 494, 518, 517, 520, 545, 488, 510, 487, 519, 559, 460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 545, 490, 497, 518, 532, 513, 507, 516, 547, 475, 494, 519, 495, 511, 504, 29, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 506, 539, 492, 521, 548, 519, 499, 517, 489, 451, 518, 484, 469, 496, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 491, 548, 509, 488, 519, 504, 507, 497, 515, 493, 537, 515, 506, 495, 508, 18, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8244, 8181, 8292, 8207, 8220, 8197, 8054, 8141] avg=8192.0000 max=8292 max_rank=2 min=8054 min_rank=6 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32612 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[531, 512, 508, 492, 540, 516, 475, 506, 508, 545, 521, 517, 502, 541, 548, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 493, 513, 554, 537, 518, 534, 469, 497, 552, 497, 536, 491, 528, 487, 532, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 515, 495, 493, 513, 505, 539, 511, 503, 510, 534, 478, 521, 537, 488, 524, 14, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 505, 535, 499, 537, 549, 519, 522, 515, 480, 470, 496, 525, 528, 493, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 508, 521, 506, 515, 503, 501, 526, 513, 487, 533, 527, 519, 503, 499, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 529, 499, 476, 515, 495, 509, 491, 506, 518, 490, 503, 495, 488, 502, 523, 30, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 550, 525, 519, 514, 540, 520, 498, 513, 489, 524, 527, 505, 530, 495, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 474, 527, 541, 499, 497, 517, 499, 508, 477, 525, 489, 497, 547, 523, 509, 18, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8253, 8224, 8180, 8146, 8246, 8078, 8269, 8140] avg=8192.0000 max=8269 max_rank=6 min=8078 min_rank=5 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32733 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[474, 521, 543, 495, 508, 512, 481, 506, 493, 518, 471, 518, 543, 514, 536, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 481, 555, 529, 550, 498, 501, 515, 516, 486, 507, 509, 502, 507, 520, 532, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 503, 550, 512, 508, 531, 520, 517, 508, 519, 521, 460, 501, 494, 489, 527, 14, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 516, 507, 535, 519, 556, 508, 484, 533, 509, 517, 522, 521, 513, 523, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 500, 533, 521, 483, 525, 514, 513, 557, 516, 487, 499, 541, 507, 542, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 517, 505, 500, 510, 528, 533, 497, 481, 530, 505, 527, 480, 503, 499, 489, 29, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 475, 517, 504, 491, 471, 550, 503, 499, 497, 493, 502, 495, 502, 513, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 525, 526, 538, 490, 534, 505, 489, 500, 551, 499, 551, 480, 510, 494, 510, 17, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8128, 8227, 8205, 8304, 8295, 8146, 7993, 8238] avg=8192.0000 max=8304 max_rank=3 min=7993 min_rank=6 imbalance_ratio=1.013672 max_load_reduction=57232 max_load_reduction_pct=0.873291 moved_to_echo=32672 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[538, 500, 517, 475, 526, 524, 500, 530, 534, 480, 540, 446, 533, 535, 547, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 521, 499, 472, 511, 533, 508, 478, 510, 542, 500, 510, 520, 521, 506, 490, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 515, 535, 531, 505, 521, 502, 505, 491, 539, 514, 519, 527, 506, 509, 519, 14, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 508, 520, 514, 517, 519, 513, 508, 519, 494, 527, 521, 462, 495, 518, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 523, 533, 477, 478, 509, 508, 521, 518, 488, 513, 524, 510, 499, 489, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 508, 517, 481, 494, 522, 497, 504, 532, 474, 485, 542, 520, 522, 507, 518, 30, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 525, 498, 544, 513, 527, 546, 541, 531, 512, 495, 520, 502, 489, 507, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 505, 493, 509, 488, 533, 523, 528, 529, 522, 489, 516, 505, 486, 493, 525, 17, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8231, 8142, 8281, 8178, 8064, 8152, 8282, 8206] avg=8192.0000 max=8282 max_rank=6 min=8064 min_rank=4 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32704 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[557, 501, 500, 470, 514, 517, 519, 519, 500, 523, 508, 518, 475, 519, 523, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 522, 506, 500, 511, 541, 503, 538, 542, 540, 501, 494, 478, 532, 531, 518, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 510, 494, 511, 525, 524, 506, 504, 477, 475, 506, 544, 518, 460, 488, 494, 12, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 481, 468, 555, 464, 476, 456, 523, 499, 513, 546, 526, 477, 539, 523, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 512, 512, 550, 506, 529, 511, 516, 508, 528, 516, 489, 504, 478, 519, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 509, 509, 490, 531, 509, 529, 504, 513, 501, 504, 528, 540, 538, 508, 501, 30, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 503, 518, 503, 547, 522, 520, 553, 522, 532, 500, 528, 509, 512, 499, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 533, 511, 482, 498, 504, 518, 509, 550, 512, 507, 477, 493, 540, 504, 521, 18, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8261, 8060, 8044, 8179, 8262, 8319, 8218] avg=8192.0000 max=8319 max_rank=6 min=8044 min_rank=3 imbalance_ratio=1.015503 max_load_reduction=57217 max_load_reduction_pct=0.873062 moved_to_echo=32978 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[518, 491, 509, 515, 467, 508, 479, 522, 479, 501, 492, 480, 527, 477, 459, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 537, 504, 475, 519, 503, 520, 550, 508, 488, 501, 528, 525, 524, 499, 493, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 491, 499, 514, 523, 530, 505, 465, 482, 483, 514, 503, 544, 510, 501, 536, 14, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 539, 520, 528, 488, 481, 505, 487, 495, 518, 513, 522, 516, 495, 521, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 546, 499, 489, 513, 514, 525, 516, 477, 529, 493, 520, 522, 527, 516, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 541, 532, 509, 540, 514, 520, 498, 499, 527, 542, 511, 546, 503, 510, 503, 30, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 512, 578, 521, 512, 505, 508, 522, 477, 535, 511, 502, 506, 558, 515, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 496, 526, 499, 543, 543, 514, 468, 525, 535, 513, 560, 464, 486, 546, 517, 18, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7917, 8234, 8123, 8163, 8144, 8362, 8312, 8281] avg=8192.0000 max=8362 max_rank=5 min=7917 min_rank=0 imbalance_ratio=1.020752 max_load_reduction=57174 max_load_reduction_pct=0.872406 moved_to_echo=33099 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 3968, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[3968, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 3968, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 3968, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 3968, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 3968, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 3968, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[509, 517, 542, 475, 531, 509, 522, 511, 523, 504, 485, 525, 536, 523, 536, 490, 513, 541, 517, 478, 487, 507, 528, 486, 500, 456, 480, 501, 493, 491, 559, 505, 535, 519, 506, 465, 552, 497, 511, 506, 517, 539, 521, 545, 509, 501, 514, 501, 505, 489, 540, 449, 517, 507, 502, 508, 513, 503, 530, 544, 488, 536, 509, 508, 483, 476, 505, 506, 561, 531, 476, 533, 519, 533, 464, 500, 537, 509, 506, 512, 524, 510, 535, 494, 515, 520, 500, 488, 481, 492, 533, 568, 511, 485, 521, 552, 520, 484, 486, 517, 524, 530, 497, 501, 520, 518, 573, 511, 525, 491, 516, 523, 542, 521, 515, 494, 550, 512, 527, 531, 513, 501, 523, 494, 502, 517, 503, 509] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[515, 518, 517, 532, 531, 544, 502, 514, 527, 573, 506, 513, 532, 540, 521, 484, 529, 497, 494, 491, 518, 533, 470, 480, 513, 514, 494, 501, 497, 489, 491, 492, 533, 542, 520, 499, 493, 518, 505, 477, 531, 492, 517, 497, 488, 536, 503, 513, 497, 497, 486, 498, 506, 506, 484, 507, 528, 502, 516, 485, 515, 487, 540, 511, 501, 547, 540, 484, 501, 552, 509, 499, 495, 491, 509, 503, 488, 513, 495, 518, 554, 513, 570, 542, 484, 552, 472, 526, 514, 542, 540, 497, 491, 537, 505, 504, 560, 531, 485, 509, 544, 512, 528, 508, 505, 494, 530, 488, 528, 523, 519, 505, 517, 516, 516, 529, 490, 514, 503, 488, 547, 536, 494, 502, 568, 479, 494, 485] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[519, 533, 500, 526, 537, 474, 465, 486, 517, 526, 494, 522, 499, 479, 544, 497, 477, 499, 525, 463, 539, 507, 512, 506, 524, 519, 514, 495, 514, 518, 503, 529, 510, 511, 501, 540, 531, 526, 544, 520, 501, 540, 511, 537, 534, 499, 519, 508, 505, 514, 525, 578, 485, 528, 511, 513, 500, 531, 548, 550, 493, 518, 501, 531, 485, 536, 481, 469, 528, 504, 543, 521, 483, 509, 519, 542, 525, 502, 511, 494, 524, 523, 474, 495, 480, 518, 507, 526, 466, 519, 515, 510, 534, 520, 550, 495, 561, 524, 519, 541, 505, 480, 492, 501, 515, 505, 509, 491, 507, 561, 523, 505, 489, 515, 526, 512, 500, 470, 539, 475, 511, 517, 520, 497, 507, 509, 476, 501] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[513, 507, 517, 517, 513, 517, 514, 479, 546, 533, 516, 486, 500, 502, 523, 527, 542, 472, 495, 516, 532, 486, 500, 508, 501, 476, 512, 537, 476, 501, 442, 481, 503, 490, 474, 503, 531, 509, 528, 543, 488, 515, 503, 531, 507, 495, 490, 521, 537, 497, 512, 510, 522, 511, 529, 518, 474, 529, 529, 508, 513, 485, 504, 519, 510, 504, 466, 510, 552, 519, 506, 536, 488, 528, 503, 507, 534, 522, 520, 525, 520, 530, 539, 475, 504, 474, 533, 515, 528, 516, 541, 517, 521, 525, 524, 525, 482, 522, 561, 544, 539, 491, 507, 520, 490, 536, 487, 536, 482, 545, 535, 549, 514, 509, 489, 534, 533, 521, 510, 538, 519, 487, 509, 484, 508, 496, 514, 513] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[469, 526, 524, 453, 476, 483, 497, 521, 528, 557, 493, 502, 523, 562, 478, 499, 483, 493, 510, 531, 527, 551, 517, 524, 534, 460, 482, 491, 512, 509, 481, 487, 521, 511, 479, 503, 520, 512, 509, 475, 451, 496, 491, 546, 526, 541, 495, 491, 541, 514, 475, 490, 492, 497, 464, 512, 526, 528, 519, 498, 483, 521, 484, 510, 502, 558, 500, 547, 518, 532, 520, 526, 470, 503, 512, 553, 535, 535, 513, 508, 505, 550, 514, 535, 499, 462, 514, 475, 500, 518, 515, 494, 531, 511, 537, 504, 502, 527, 516, 569, 514, 526, 497, 521, 544, 526, 562, 506, 540, 543, 504, 547, 482, 530, 517, 482, 539, 523, 475, 573, 535, 497, 508, 559, 508, 521, 542, 501] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[513, 507, 517, 517, 513, 517, 514, 479, 546, 533, 516, 486, 500, 502, 523, 527, 542, 472, 495, 516, 532, 486, 500, 508, 501, 476, 512, 537, 476, 501, 442, 481, 503, 490, 474, 503, 531, 509, 528, 543, 488, 515, 503, 531, 507, 495, 490, 521, 537, 497, 512, 510, 522, 511, 529, 518, 474, 529, 529, 508, 513, 485, 504, 519, 510, 504, 466, 510, 552, 519, 506, 536, 488, 528, 503, 507, 534, 522, 520, 525, 520, 530, 539, 475, 504, 474, 533, 515, 528, 516, 541, 517, 521, 525, 524, 525, 482, 522, 561, 544, 539, 491, 507, 520, 490, 536, 487, 536, 482, 545, 535, 549, 514, 509, 489, 534, 533, 521, 510, 538, 519, 487, 509, 484, 508, 496, 514, 513] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[519, 533, 500, 526, 537, 474, 465, 486, 517, 526, 494, 522, 499, 479, 544, 497, 477, 499, 525, 463, 539, 507, 512, 506, 524, 519, 514, 495, 514, 518, 503, 529, 510, 511, 501, 540, 531, 526, 544, 520, 501, 540, 511, 537, 534, 499, 519, 508, 505, 514, 525, 578, 485, 528, 511, 513, 500, 531, 548, 550, 493, 518, 501, 531, 485, 536, 481, 469, 528, 504, 543, 521, 483, 509, 519, 542, 525, 502, 511, 494, 524, 523, 474, 495, 480, 518, 507, 526, 466, 519, 515, 510, 534, 520, 550, 495, 561, 524, 519, 541, 505, 480, 492, 501, 515, 505, 509, 491, 507, 561, 523, 505, 489, 515, 526, 512, 500, 470, 539, 475, 511, 517, 520, 497, 507, 509, 476, 501] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[511, 527, 529, 531, 520, 511, 511, 529, 534, 489, 517, 508, 467, 526, 536, 529, 504, 527, 570, 471, 505, 463, 571, 490, 488, 488, 545, 447, 512, 516, 536, 486, 501, 526, 541, 518, 528, 527, 495, 549, 515, 521, 492, 496, 490, 526, 485, 504, 475, 481, 542, 511, 501, 522, 494, 506, 486, 503, 544, 482, 517, 512, 487, 505, 510, 514, 492, 530, 510, 508, 562, 502, 479, 544, 521, 502, 498, 513, 484, 516, 547, 521, 528, 513, 495, 480, 522, 527, 492, 521, 590, 502, 519, 546, 503, 520, 525, 552, 501, 514, 513, 508, 489, 499, 492, 561, 522, 500, 518, 555, 503, 493, 498, 487, 544, 492, 546, 489, 519, 527, 488, 519, 479, 498, 501, 498, 497, 522] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[514, 544, 495, 503, 539, 491, 560, 548, 477, 510, 532, 531, 540, 530, 451, 506, 484, 500, 533, 529, 535, 504, 531, 498, 540, 548, 541, 523, 467, 515, 508, 505, 476, 490, 464, 490, 539, 488, 537, 470, 518, 547, 502, 519, 514, 488, 544, 494, 523, 530, 507, 517, 518, 534, 489, 545, 538, 508, 553, 512, 513, 504, 525, 463, 559, 501, 474, 528, 471, 474, 538, 534, 544, 528, 514, 499, 503, 493, 530, 541, 516, 518, 519, 532, 534, 534, 532, 506, 466, 481, 490, 509, 507, 516, 525, 497, 499, 497, 514, 513, 510, 526, 502, 536, 551, 518, 522, 503, 543, 499, 487, 480, 518, 458, 485, 513, 506, 531, 531, 484, 494, 486, 509, 508, 488, 513, 502, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[509, 517, 542, 475, 531, 509, 522, 511, 523, 504, 485, 525, 536, 523, 536, 490, 513, 541, 517, 478, 487, 507, 528, 486, 500, 456, 480, 501, 493, 491, 559, 505, 535, 519, 506, 465, 552, 497, 511, 506, 517, 539, 521, 545, 509, 501, 514, 501, 505, 489, 540, 449, 517, 507, 502, 508, 513, 503, 530, 544, 488, 536, 509, 508, 483, 476, 505, 506, 561, 531, 476, 533, 519, 533, 464, 500, 537, 509, 506, 512, 524, 510, 535, 494, 515, 520, 500, 488, 481, 492, 533, 568, 511, 485, 521, 552, 520, 484, 486, 517, 524, 530, 497, 501, 520, 518, 573, 511, 525, 491, 516, 523, 542, 521, 515, 494, 550, 512, 527, 531, 513, 501, 523, 494, 502, 517, 503, 509] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[530, 544, 569, 501, 494, 534, 539, 467, 524, 489, 456, 524, 520, 532, 522, 508, 496, 511, 515, 502, 544, 517, 532, 477, 460, 506, 510, 508, 508, 526, 497, 521, 545, 502, 491, 443, 490, 541, 483, 535, 518, 497, 478, 533, 524, 517, 556, 480, 514, 495, 558, 530, 532, 493, 469, 549, 473, 554, 513, 481, 533, 529, 541, 501, 493, 513, 513, 525, 519, 500, 485, 541, 517, 539, 497, 479, 500, 515, 547, 513, 520, 501, 499, 472, 525, 545, 524, 498, 514, 535, 503, 534, 526, 503, 532, 479, 519, 536, 529, 510, 521, 466, 522, 543, 505, 477, 526, 483, 499, 479, 537, 544, 551, 523, 510, 513, 486, 483, 519, 521, 476, 533, 536, 496, 479, 481, 508, 538] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[515, 484, 517, 532, 531, 544, 502, 514, 527, 573, 506, 513, 532, 540, 521, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 497, 494, 491, 518, 533, 470, 480, 513, 514, 494, 501, 497, 489, 491, 492, 77, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 542, 520, 499, 493, 518, 505, 477, 531, 492, 517, 497, 488, 536, 503, 513, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 497, 486, 498, 506, 506, 484, 507, 528, 502, 516, 485, 515, 487, 540, 511, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 547, 540, 484, 501, 552, 509, 489, 495, 491, 509, 503, 488, 513, 495, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 513, 570, 542, 484, 552, 472, 526, 514, 542, 494, 497, 491, 537, 505, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 560, 531, 485, 509, 544, 512, 528, 508, 505, 494, 453, 488, 528, 523, 519, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 516, 516, 529, 490, 514, 503, 488, 547, 536, 494, 502, 568, 479, 494, 485, 27, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8335, 8099, 8185, 8080, 8135, 8297, 8192, 8213] avg=8192.0000 max=8335 max_rank=0 min=8080 min_rank=3 imbalance_ratio=1.017456 max_load_reduction=57201 max_load_reduction_pct=0.872818 moved_to_echo=32837 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[509, 483, 542, 475, 531, 509, 522, 511, 523, 504, 485, 525, 536, 523, 536, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 541, 517, 478, 487, 507, 528, 486, 500, 456, 480, 501, 493, 491, 559, 505, 83, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 519, 506, 465, 552, 497, 511, 506, 517, 539, 521, 545, 509, 501, 514, 501, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 489, 540, 449, 517, 507, 502, 508, 513, 503, 530, 544, 488, 536, 509, 508, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 476, 505, 506, 561, 531, 476, 523, 519, 533, 464, 500, 537, 509, 506, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 510, 535, 494, 515, 520, 500, 488, 481, 492, 489, 568, 511, 485, 521, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 484, 486, 517, 524, 530, 497, 501, 520, 518, 490, 511, 525, 491, 516, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 521, 515, 494, 550, 512, 527, 531, 513, 501, 523, 494, 502, 517, 503, 509, 26, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8143, 8259, 8163, 8141, 8185, 8153, 8288] avg=8192.0000 max=8288 max_rank=7 min=8141 min_rank=4 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32767 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[515, 518, 517, 532, 531, 544, 502, 514, 527, 573, 506, 513, 532, 540, 521, 484, 529, 497, 494, 491, 518, 533, 470, 480, 513, 514, 494, 501, 497, 489, 491, 492, 533, 542, 520, 499, 493, 518, 505, 477, 531, 492, 517, 497, 488, 536, 503, 513, 497, 497, 486, 498, 506, 506, 484, 507, 528, 502, 516, 485, 515, 487, 540, 511, 501, 547, 540, 484, 501, 552, 509, 499, 495, 491, 509, 503, 488, 513, 495, 518, 554, 513, 570, 542, 484, 552, 472, 526, 514, 542, 540, 497, 491, 537, 505, 504, 560, 531, 485, 509, 544, 512, 528, 508, 505, 494, 530, 488, 528, 523, 519, 505, 517, 516, 516, 529, 490, 514, 503, 488, 547, 536, 494, 502, 568, 479, 494, 485] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[514, 544, 495, 503, 539, 491, 560, 548, 477, 510, 532, 531, 540, 530, 451, 506, 484, 500, 533, 529, 535, 504, 531, 498, 540, 548, 541, 523, 467, 515, 508, 505, 476, 490, 464, 490, 539, 488, 537, 470, 518, 547, 502, 519, 514, 488, 544, 494, 523, 530, 507, 517, 518, 534, 489, 545, 538, 508, 553, 512, 513, 504, 525, 463, 559, 501, 474, 528, 471, 474, 538, 534, 544, 528, 514, 499, 503, 493, 530, 541, 516, 518, 519, 532, 534, 534, 532, 506, 466, 481, 490, 509, 507, 516, 525, 497, 499, 497, 514, 513, 510, 526, 502, 536, 551, 518, 522, 503, 543, 499, 487, 480, 518, 458, 485, 513, 506, 531, 531, 484, 494, 486, 509, 508, 488, 513, 502, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[513, 461, 517, 517, 513, 517, 514, 479, 546, 533, 516, 486, 500, 502, 523, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 472, 495, 516, 532, 486, 500, 508, 501, 476, 512, 537, 476, 501, 442, 481, 75, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 490, 474, 503, 531, 509, 528, 543, 488, 515, 503, 531, 507, 495, 490, 521, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 497, 512, 510, 522, 511, 529, 518, 474, 529, 529, 508, 513, 485, 504, 519, 15, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 504, 466, 510, 552, 519, 506, 521, 488, 528, 503, 507, 534, 522, 520, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 530, 539, 475, 504, 474, 533, 515, 528, 516, 488, 517, 521, 525, 524, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 522, 561, 544, 539, 491, 507, 520, 490, 536, 412, 536, 482, 545, 535, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 509, 489, 534, 533, 521, 510, 538, 519, 487, 509, 484, 508, 496, 514, 513, 31, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8074, 8156, 8220, 8215, 8234, 8251, 8222] avg=8192.0000 max=8251 max_rank=6 min=8074 min_rank=1 imbalance_ratio=1.007202 max_load_reduction=57285 max_load_reduction_pct=0.874100 moved_to_echo=32922 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[511, 492, 529, 531, 520, 511, 511, 529, 534, 489, 517, 508, 467, 526, 536, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 527, 570, 471, 505, 463, 571, 490, 488, 488, 545, 447, 512, 516, 536, 486, 76, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 526, 541, 518, 528, 527, 495, 549, 515, 521, 492, 496, 490, 526, 485, 504, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 481, 542, 511, 501, 522, 494, 506, 486, 503, 544, 482, 517, 512, 487, 505, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 514, 492, 530, 510, 508, 562, 492, 479, 544, 521, 502, 498, 513, 484, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 521, 528, 513, 495, 480, 522, 527, 492, 521, 541, 502, 519, 546, 503, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 552, 501, 514, 513, 508, 489, 499, 492, 561, 446, 500, 518, 555, 503, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 487, 544, 492, 546, 489, 519, 527, 488, 519, 479, 498, 501, 498, 497, 522, 29, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8240, 8215, 8236, 8083, 8175, 8277, 8169, 8141] avg=8192.0000 max=8277 max_rank=5 min=8083 min_rank=3 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32762 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[519, 498, 500, 526, 537, 474, 465, 486, 517, 526, 494, 522, 499, 479, 544, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 499, 525, 463, 539, 507, 512, 506, 524, 519, 514, 495, 514, 518, 503, 529, 74, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 511, 501, 540, 531, 526, 544, 520, 501, 540, 511, 537, 534, 499, 519, 508, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 514, 525, 578, 485, 528, 511, 513, 500, 531, 548, 550, 493, 518, 501, 531, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 536, 481, 469, 528, 504, 543, 511, 483, 509, 519, 542, 525, 502, 511, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 523, 474, 495, 480, 518, 507, 526, 466, 519, 472, 510, 534, 520, 550, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 524, 519, 541, 505, 480, 492, 501, 515, 505, 435, 491, 507, 561, 523, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 515, 526, 512, 500, 470, 539, 475, 511, 517, 520, 497, 507, 509, 476, 501, 25, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8083, 8236, 8354, 8346, 8142, 8113, 8165, 8097] avg=8192.0000 max=8354 max_rank=2 min=8083 min_rank=0 imbalance_ratio=1.019775 max_load_reduction=57182 max_load_reduction_pct=0.872528 moved_to_echo=32517 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[514, 509, 495, 503, 539, 491, 560, 548, 477, 510, 532, 531, 540, 530, 451, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 500, 533, 529, 535, 504, 531, 498, 540, 548, 541, 523, 467, 515, 508, 505, 76, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 490, 464, 490, 539, 488, 537, 470, 518, 547, 502, 519, 514, 488, 544, 494, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 530, 507, 517, 518, 534, 489, 545, 538, 508, 553, 512, 513, 504, 525, 463, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 501, 474, 528, 471, 474, 538, 524, 544, 528, 514, 499, 503, 493, 530, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 518, 519, 532, 534, 534, 532, 506, 466, 481, 449, 509, 507, 516, 525, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 497, 514, 513, 510, 526, 502, 536, 551, 518, 446, 503, 543, 499, 487, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 458, 485, 513, 506, 531, 531, 484, 494, 486, 509, 508, 488, 513, 502, 506, 24, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8236, 8354, 8102, 8294, 8221, 8141, 8124, 8064] avg=8192.0000 max=8354 max_rank=1 min=8064 min_rank=7 imbalance_ratio=1.019775 max_load_reduction=57182 max_load_reduction_pct=0.872528 moved_to_echo=32550 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[469, 491, 524, 453, 476, 483, 497, 521, 528, 557, 493, 502, 523, 562, 478, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 493, 510, 531, 527, 551, 517, 524, 534, 460, 482, 491, 512, 509, 481, 487, 82, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 511, 479, 503, 520, 512, 509, 475, 451, 496, 491, 546, 526, 541, 495, 491, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 514, 475, 490, 492, 497, 464, 512, 526, 528, 519, 498, 483, 521, 484, 510, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 558, 500, 547, 518, 532, 520, 516, 470, 503, 512, 553, 535, 535, 513, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 550, 514, 535, 499, 462, 514, 475, 500, 518, 472, 494, 531, 511, 537, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 527, 516, 569, 514, 526, 497, 521, 544, 526, 480, 506, 540, 543, 504, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 530, 517, 482, 539, 523, 475, 573, 535, 497, 508, 559, 508, 521, 542, 501, 25, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8056, 8192, 8089, 8069, 8322, 8121, 8362, 8325] avg=8192.0000 max=8362 max_rank=6 min=8056 min_rank=0 imbalance_ratio=1.020752 max_load_reduction=57174 max_load_reduction_pct=0.872406 moved_to_echo=33130 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[509, 483, 542, 475, 531, 509, 522, 511, 523, 504, 485, 525, 536, 523, 536, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 541, 517, 478, 487, 507, 528, 486, 500, 456, 480, 501, 493, 491, 559, 505, 83, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 519, 506, 465, 552, 497, 511, 506, 517, 539, 521, 545, 509, 501, 514, 501, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 489, 540, 449, 517, 507, 502, 508, 513, 503, 530, 544, 488, 536, 509, 508, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 476, 505, 506, 561, 531, 476, 523, 519, 533, 464, 500, 537, 509, 506, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 510, 535, 494, 515, 520, 500, 488, 481, 492, 489, 568, 511, 485, 521, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 484, 486, 517, 524, 530, 497, 501, 520, 518, 490, 511, 525, 491, 516, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 521, 515, 494, 550, 512, 527, 531, 513, 501, 523, 494, 502, 517, 503, 509, 26, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8143, 8259, 8163, 8141, 8185, 8153, 8288] avg=8192.0000 max=8288 max_rank=7 min=8141 min_rank=4 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32767 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[511, 492, 529, 531, 520, 511, 511, 529, 534, 489, 517, 508, 467, 526, 536, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 527, 570, 471, 505, 463, 571, 490, 488, 488, 545, 447, 512, 516, 536, 486, 76, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 526, 541, 518, 528, 527, 495, 549, 515, 521, 492, 496, 490, 526, 485, 504, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 481, 542, 511, 501, 522, 494, 506, 486, 503, 544, 482, 517, 512, 487, 505, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 514, 492, 530, 510, 508, 562, 492, 479, 544, 521, 502, 498, 513, 484, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 521, 528, 513, 495, 480, 522, 527, 492, 521, 541, 502, 519, 546, 503, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 552, 501, 514, 513, 508, 489, 499, 492, 561, 446, 500, 518, 555, 503, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 487, 544, 492, 546, 489, 519, 527, 488, 519, 479, 498, 501, 498, 497, 522, 29, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8240, 8215, 8236, 8083, 8175, 8277, 8169, 8141] avg=8192.0000 max=8277 max_rank=5 min=8083 min_rank=3 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32762 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[530, 509, 569, 501, 494, 534, 539, 467, 524, 489, 456, 524, 520, 532, 522, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 511, 515, 502, 544, 517, 532, 477, 460, 506, 510, 508, 508, 526, 497, 521, 76, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 502, 491, 443, 490, 541, 483, 535, 518, 497, 478, 533, 524, 517, 556, 480, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 495, 558, 530, 532, 493, 469, 549, 473, 554, 513, 481, 533, 529, 541, 501, 11, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 513, 513, 525, 519, 500, 485, 530, 517, 539, 497, 479, 500, 515, 547, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 501, 499, 472, 525, 545, 524, 498, 514, 535, 461, 534, 526, 503, 532, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 536, 529, 510, 521, 466, 522, 543, 505, 477, 450, 483, 499, 479, 537, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 523, 510, 513, 486, 483, 519, 521, 476, 533, 536, 496, 479, 481, 508, 538, 25, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8218, 8223, 8155, 8281, 8185, 8168, 8120, 8186] avg=8192.0000 max=8281 max_rank=3 min=8120 min_rank=6 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32659 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[515, 484, 517, 532, 531, 544, 502, 514, 527, 573, 506, 513, 532, 540, 521, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 497, 494, 491, 518, 533, 470, 480, 513, 514, 494, 501, 497, 489, 491, 492, 77, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 542, 520, 499, 493, 518, 505, 477, 531, 492, 517, 497, 488, 536, 503, 513, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 497, 486, 498, 506, 506, 484, 507, 528, 502, 516, 485, 515, 487, 540, 511, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 547, 540, 484, 501, 552, 509, 489, 495, 491, 509, 503, 488, 513, 495, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 513, 570, 542, 484, 552, 472, 526, 514, 542, 494, 497, 491, 537, 505, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 560, 531, 485, 509, 544, 512, 528, 508, 505, 494, 453, 488, 528, 523, 519, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 516, 516, 529, 490, 514, 503, 488, 547, 536, 494, 502, 568, 479, 494, 485, 27, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8335, 8099, 8185, 8080, 8135, 8297, 8192, 8213] avg=8192.0000 max=8335 max_rank=0 min=8080 min_rank=3 imbalance_ratio=1.017456 max_load_reduction=57201 max_load_reduction_pct=0.872818 moved_to_echo=32837 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[514, 509, 495, 503, 539, 491, 560, 548, 477, 510, 532, 531, 540, 530, 451, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 500, 533, 529, 535, 504, 531, 498, 540, 548, 541, 523, 467, 515, 508, 505, 76, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 490, 464, 490, 539, 488, 537, 470, 518, 547, 502, 519, 514, 488, 544, 494, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 530, 507, 517, 518, 534, 489, 545, 538, 508, 553, 512, 513, 504, 525, 463, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 501, 474, 528, 471, 474, 538, 524, 544, 528, 514, 499, 503, 493, 530, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 518, 519, 532, 534, 534, 532, 506, 466, 481, 449, 509, 507, 516, 525, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 497, 514, 513, 510, 526, 502, 536, 551, 518, 446, 503, 543, 499, 487, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 458, 485, 513, 506, 531, 531, 484, 494, 486, 509, 508, 488, 513, 502, 506, 24, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8236, 8354, 8102, 8294, 8221, 8141, 8124, 8064] avg=8192.0000 max=8354 max_rank=1 min=8064 min_rank=7 imbalance_ratio=1.019775 max_load_reduction=57182 max_load_reduction_pct=0.872528 moved_to_echo=32550 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[519, 498, 500, 526, 537, 474, 465, 486, 517, 526, 494, 522, 499, 479, 544, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 499, 525, 463, 539, 507, 512, 506, 524, 519, 514, 495, 514, 518, 503, 529, 74, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 511, 501, 540, 531, 526, 544, 520, 501, 540, 511, 537, 534, 499, 519, 508, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 514, 525, 578, 485, 528, 511, 513, 500, 531, 548, 550, 493, 518, 501, 531, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 536, 481, 469, 528, 504, 543, 511, 483, 509, 519, 542, 525, 502, 511, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 523, 474, 495, 480, 518, 507, 526, 466, 519, 472, 510, 534, 520, 550, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 524, 519, 541, 505, 480, 492, 501, 515, 505, 435, 491, 507, 561, 523, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 515, 526, 512, 500, 470, 539, 475, 511, 517, 520, 497, 507, 509, 476, 501, 25, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8083, 8236, 8354, 8346, 8142, 8113, 8165, 8097] avg=8192.0000 max=8354 max_rank=2 min=8083 min_rank=0 imbalance_ratio=1.019775 max_load_reduction=57182 max_load_reduction_pct=0.872528 moved_to_echo=32517 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 3968, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 3968, 4096, 4096, 4096, 4096, 4096, 4096, 640, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 3968, 4224, 3968, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 3968, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 3712, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 3968, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 3968, 4224, 4096, 4224, 4352, 4096, 4096, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 3968, 4096, 4096, 4096, 4096, 4096, 4096, 640, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 3968, 4224, 3968, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[507, 521, 535, 538, 475, 540, 545, 538, 502, 506, 491, 498, 532, 540, 522, 514, 530, 547, 521, 500, 501, 503, 503, 468, 493, 528, 499, 497, 548, 527, 533, 469, 510, 517, 503, 534, 505, 495, 525, 507, 500, 478, 505, 492, 481, 511, 507, 533, 530, 528, 489, 494, 518, 534, 482, 510, 503, 525, 496, 545, 504, 501, 538, 534, 509, 490, 464, 489, 501, 520, 501, 512, 482, 508, 520, 505, 519, 524, 480, 499, 504, 509, 546, 553, 530, 505, 493, 500, 540, 510, 518, 482, 474, 549, 526, 479, 514, 480, 488, 527, 498, 544, 526, 536, 527, 541, 480, 515, 485, 535, 482, 504, 530, 524, 533, 484, 535, 536, 535, 547, 535, 526, 506, 518, 489, 500, 520, 490] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[533, 477, 512, 530, 537, 527, 506, 503, 466, 524, 498, 515, 460, 505, 528, 530, 511, 524, 499, 527, 527, 523, 548, 538, 536, 488, 525, 517, 513, 509, 479, 449, 479, 554, 500, 504, 503, 538, 513, 537, 519, 515, 483, 523, 528, 530, 531, 549, 551, 513, 500, 500, 513, 481, 475, 535, 466, 514, 545, 498, 557, 517, 488, 485, 543, 508, 500, 518, 501, 509, 521, 492, 516, 519, 505, 469, 504, 541, 502, 504, 515, 505, 476, 498, 520, 509, 522, 500, 539, 510, 483, 507, 530, 498, 533, 565, 485, 540, 542, 494, 499, 488, 559, 462, 488, 489, 532, 497, 535, 504, 533, 507, 514, 527, 516, 522, 519, 520, 535, 491, 484, 480, 507, 502, 545, 518, 542, 490] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[534, 499, 494, 519, 513, 508, 510, 504, 471, 530, 480, 529, 513, 524, 528, 500, 523, 491, 512, 504, 516, 498, 519, 519, 482, 492, 500, 498, 545, 504, 528, 533, 564, 470, 522, 526, 500, 505, 501, 538, 489, 522, 514, 509, 508, 502, 522, 508, 530, 528, 535, 522, 508, 504, 523, 517, 513, 503, 496, 515, 538, 550, 538, 546, 503, 504, 533, 532, 533, 509, 476, 497, 519, 521, 530, 516, 512, 500, 508, 514, 477, 469, 483, 510, 503, 522, 544, 495, 516, 545, 490, 517, 500, 500, 536, 527, 504, 508, 507, 532, 503, 519, 547, 546, 546, 496, 498, 509, 502, 487, 547, 538, 482, 494, 501, 473, 534, 491, 505, 500, 494, 508, 518, 491, 506, 490, 530, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[499, 504, 542, 483, 516, 532, 523, 538, 493, 522, 558, 459, 546, 520, 521, 525, 480, 489, 487, 510, 529, 569, 517, 525, 514, 515, 545, 476, 508, 522, 545, 529, 492, 481, 513, 508, 499, 486, 455, 485, 506, 551, 520, 508, 494, 513, 523, 532, 529, 531, 493, 508, 482, 505, 485, 516, 546, 536, 500, 554, 521, 523, 516, 509, 522, 476, 532, 499, 506, 526, 500, 499, 497, 485, 547, 526, 496, 505, 515, 483, 524, 503, 494, 491, 504, 507, 513, 535, 497, 496, 530, 508, 499, 511, 545, 524, 528, 467, 569, 525, 483, 503, 507, 499, 511, 500, 502, 519, 473, 509, 522, 534, 499, 500, 522, 479, 507, 492, 522, 556, 499, 528, 540, 512, 507, 511, 535, 520] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[499, 521, 517, 493, 547, 520, 524, 498, 519, 533, 550, 504, 528, 507, 494, 502, 503, 484, 536, 529, 482, 490, 506, 497, 461, 524, 552, 503, 530, 515, 535, 518, 512, 523, 497, 533, 520, 508, 520, 471, 503, 487, 511, 480, 505, 494, 517, 519, 511, 551, 494, 529, 543, 483, 530, 465, 537, 474, 504, 529, 513, 538, 513, 515, 488, 508, 531, 540, 537, 487, 525, 494, 487, 516, 529, 540, 541, 494, 511, 483, 545, 484, 500, 530, 486, 479, 492, 495, 480, 538, 513, 502, 498, 517, 516, 524, 506, 490, 501, 517, 478, 500, 542, 527, 509, 516, 478, 545, 485, 548, 548, 564, 528, 543, 448, 495, 515, 564, 503, 502, 527, 507, 505, 534, 487, 537, 526, 501] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[533, 520, 496, 499, 476, 527, 538, 489, 536, 490, 533, 497, 512, 538, 520, 499, 527, 504, 522, 501, 483, 505, 473, 502, 527, 502, 518, 486, 525, 487, 518, 499, 487, 522, 506, 533, 519, 472, 505, 501, 525, 502, 492, 524, 516, 488, 495, 501, 510, 486, 517, 494, 552, 504, 516, 514, 496, 480, 538, 507, 507, 487, 509, 501, 491, 504, 530, 530, 522, 525, 506, 514, 507, 512, 473, 556, 501, 530, 541, 504, 500, 565, 510, 496, 544, 507, 514, 536, 513, 563, 527, 520, 529, 497, 508, 505, 524, 510, 501, 500, 520, 521, 516, 528, 527, 491, 525, 501, 526, 501, 537, 505, 524, 531, 499, 488, 511, 505, 507, 525, 499, 549, 548, 512, 526, 521, 506, 484] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[523, 498, 520, 520, 529, 560, 515, 488, 504, 506, 520, 536, 532, 511, 520, 514, 509, 509, 535, 495, 524, 511, 498, 522, 489, 508, 492, 532, 512, 477, 511, 519, 528, 558, 493, 542, 477, 507, 506, 523, 485, 510, 522, 478, 547, 508, 508, 478, 478, 526, 504, 491, 491, 531, 513, 534, 524, 494, 499, 504, 499, 531, 524, 498, 524, 496, 545, 502, 475, 529, 507, 539, 537, 486, 516, 556, 531, 523, 546, 502, 517, 525, 551, 559, 520, 524, 521, 536, 537, 510, 522, 533, 484, 523, 579, 498, 499, 544, 472, 472, 484, 476, 507, 505, 509, 483, 508, 523, 490, 498, 492, 531, 498, 518, 508, 515, 524, 505, 439, 523, 475, 492, 515, 524, 496, 518, 500, 490] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[533, 477, 512, 530, 537, 527, 506, 503, 466, 524, 498, 515, 460, 505, 528, 530, 511, 524, 499, 527, 527, 523, 548, 538, 536, 488, 525, 517, 513, 509, 479, 449, 479, 554, 500, 504, 503, 538, 513, 537, 519, 515, 483, 523, 528, 530, 531, 549, 551, 513, 500, 500, 513, 481, 475, 535, 466, 514, 545, 498, 557, 517, 488, 485, 543, 508, 500, 518, 501, 509, 521, 492, 516, 519, 505, 469, 504, 541, 502, 504, 515, 505, 476, 498, 520, 509, 522, 500, 539, 510, 483, 507, 530, 498, 533, 565, 485, 540, 542, 494, 499, 488, 559, 462, 488, 489, 532, 497, 535, 504, 533, 507, 514, 527, 516, 522, 519, 520, 535, 491, 484, 480, 507, 502, 545, 518, 542, 490] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[499, 521, 517, 493, 547, 520, 524, 498, 519, 533, 550, 504, 528, 507, 494, 502, 503, 484, 536, 529, 482, 490, 506, 497, 461, 524, 552, 503, 530, 515, 535, 518, 512, 523, 497, 533, 520, 508, 520, 471, 503, 487, 511, 480, 505, 494, 517, 519, 511, 551, 494, 529, 543, 483, 530, 465, 537, 474, 504, 529, 513, 538, 513, 515, 488, 508, 531, 540, 537, 487, 525, 494, 487, 516, 529, 540, 541, 494, 511, 483, 545, 484, 500, 530, 486, 479, 492, 495, 480, 538, 513, 502, 498, 517, 516, 524, 506, 490, 501, 517, 478, 500, 542, 527, 509, 516, 478, 545, 485, 548, 548, 564, 528, 543, 448, 495, 515, 564, 503, 502, 527, 507, 505, 534, 487, 537, 526, 501] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[488, 524, 511, 542, 523, 514, 508, 476, 483, 525, 510, 507, 528, 508, 474, 518, 501, 475, 476, 535, 511, 507, 518, 507, 506, 496, 496, 506, 463, 508, 534, 539, 528, 493, 505, 552, 487, 510, 488, 506, 501, 502, 534, 509, 538, 518, 532, 529, 516, 540, 509, 507, 499, 536, 512, 506, 495, 517, 541, 530, 497, 547, 511, 550, 543, 505, 464, 517, 513, 516, 542, 509, 505, 504, 503, 467, 523, 512, 565, 519, 515, 512, 500, 514, 478, 484, 506, 506, 511, 481, 513, 551, 527, 573, 506, 556, 522, 487, 529, 528, 492, 520, 508, 511, 509, 533, 499, 525, 491, 491, 516, 502, 477, 511, 516, 477, 525, 524, 515, 530, 517, 559, 478, 544, 499, 530, 480, 489] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[534, 499, 494, 519, 513, 508, 510, 504, 471, 530, 480, 529, 513, 524, 528, 500, 523, 491, 512, 504, 516, 498, 519, 519, 482, 492, 500, 498, 545, 504, 528, 533, 564, 470, 522, 526, 500, 505, 501, 538, 489, 522, 514, 509, 508, 502, 522, 508, 530, 528, 535, 522, 508, 504, 523, 517, 513, 503, 496, 515, 538, 550, 538, 546, 503, 504, 533, 532, 533, 509, 476, 497, 519, 521, 530, 516, 512, 500, 508, 514, 477, 469, 483, 510, 503, 522, 544, 495, 516, 545, 490, 517, 500, 500, 536, 527, 504, 508, 507, 532, 503, 519, 547, 546, 546, 496, 498, 509, 502, 487, 547, 538, 482, 494, 501, 473, 534, 491, 505, 500, 494, 508, 518, 491, 506, 490, 530, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[507, 521, 535, 538, 475, 540, 545, 538, 502, 506, 491, 498, 532, 540, 522, 514, 530, 547, 521, 500, 501, 503, 503, 468, 493, 528, 499, 497, 548, 527, 533, 469, 510, 517, 503, 534, 505, 495, 525, 507, 500, 478, 505, 492, 481, 511, 507, 533, 530, 528, 489, 494, 518, 534, 482, 510, 503, 525, 496, 545, 504, 501, 538, 534, 509, 490, 464, 489, 501, 520, 501, 512, 482, 508, 520, 505, 519, 524, 480, 499, 504, 509, 546, 553, 530, 505, 493, 500, 540, 510, 518, 482, 474, 549, 526, 479, 514, 480, 488, 527, 498, 544, 526, 536, 527, 541, 480, 515, 485, 535, 482, 504, 530, 524, 533, 484, 535, 536, 535, 547, 535, 526, 506, 518, 489, 500, 520, 490] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[499, 521, 517, 493, 547, 491, 524, 498, 519, 533, 550, 504, 528, 507, 494, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 484, 536, 529, 482, 490, 506, 497, 461, 524, 552, 503, 530, 515, 535, 518, 29, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 523, 497, 533, 520, 508, 520, 471, 503, 487, 511, 480, 505, 494, 517, 519, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 519, 494, 529, 543, 483, 530, 465, 537, 474, 504, 529, 513, 538, 513, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 508, 531, 540, 537, 487, 525, 494, 487, 516, 529, 540, 541, 494, 511, 483, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 484, 500, 530, 486, 479, 492, 495, 480, 538, 513, 502, 498, 517, 480, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 490, 501, 517, 478, 500, 542, 527, 509, 516, 478, 545, 485, 548, 548, 564, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 543, 448, 495, 515, 564, 503, 502, 527, 507, 505, 534, 487, 537, 526, 501, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8227, 8198, 8132, 8197, 8218, 8063, 8267, 8234] avg=8192.0000 max=8267 max_rank=6 min=8063 min_rank=5 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32782 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[523, 498, 520, 520, 529, 560, 515, 488, 504, 506, 520, 536, 532, 511, 520, 514, 509, 509, 535, 495, 524, 511, 498, 522, 489, 508, 492, 532, 512, 477, 511, 519, 528, 558, 493, 542, 477, 507, 506, 523, 485, 510, 522, 478, 547, 508, 508, 478, 478, 526, 504, 491, 491, 531, 513, 534, 524, 494, 499, 504, 499, 531, 524, 498, 524, 496, 545, 502, 475, 529, 507, 539, 537, 486, 516, 556, 531, 523, 546, 502, 517, 525, 551, 559, 520, 524, 521, 536, 537, 510, 522, 533, 484, 523, 579, 498, 499, 544, 472, 472, 484, 476, 507, 505, 509, 483, 508, 523, 490, 498, 492, 531, 498, 518, 508, 515, 524, 505, 439, 523, 475, 492, 515, 524, 496, 518, 500, 490] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[507, 521, 535, 538, 475, 510, 545, 538, 502, 506, 491, 498, 532, 540, 522, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 547, 521, 500, 501, 503, 503, 468, 493, 528, 499, 497, 548, 527, 533, 469, 28, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 517, 503, 534, 505, 495, 525, 507, 500, 478, 505, 492, 481, 511, 507, 533, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 497, 489, 494, 518, 534, 482, 510, 503, 525, 496, 545, 504, 501, 538, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 490, 464, 489, 501, 520, 501, 512, 482, 508, 520, 505, 519, 524, 480, 499, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 509, 546, 553, 530, 505, 493, 500, 540, 510, 518, 482, 474, 549, 488, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 480, 488, 527, 498, 544, 526, 536, 527, 541, 480, 515, 485, 535, 482, 504, 11, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 524, 533, 484, 535, 536, 535, 547, 535, 526, 506, 518, 489, 500, 520, 490, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8274, 8200, 8136, 8200, 8030, 8180, 8196, 8320] avg=8192.0000 max=8320 max_rank=7 min=8030 min_rank=4 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32726 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[499, 504, 542, 483, 516, 490, 523, 538, 493, 522, 558, 459, 546, 520, 521, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 489, 487, 510, 529, 569, 517, 525, 514, 515, 545, 476, 508, 522, 545, 529, 31, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 481, 513, 508, 499, 486, 455, 485, 506, 551, 520, 508, 494, 513, 523, 532, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 493, 493, 508, 482, 505, 485, 516, 546, 536, 500, 554, 521, 523, 516, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 476, 532, 499, 506, 526, 500, 499, 497, 485, 547, 526, 496, 505, 515, 483, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 503, 494, 491, 504, 507, 513, 535, 497, 496, 530, 508, 499, 511, 497, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 467, 569, 525, 483, 503, 507, 499, 511, 500, 502, 519, 473, 509, 522, 534, 15, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 500, 522, 479, 507, 492, 522, 556, 499, 528, 540, 512, 507, 511, 535, 520, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8239, 8300, 8105, 8216, 8126, 8133, 8173, 8244] avg=8192.0000 max=8300 max_rank=1 min=8105 min_rank=2 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32676 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[533, 520, 496, 499, 476, 497, 538, 489, 536, 490, 533, 497, 512, 538, 520, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 504, 522, 501, 483, 505, 473, 502, 527, 502, 518, 486, 525, 487, 518, 499, 26, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 522, 506, 533, 519, 472, 505, 501, 525, 502, 492, 524, 516, 488, 495, 501, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 457, 517, 494, 552, 504, 516, 514, 496, 480, 538, 507, 507, 487, 509, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 504, 530, 530, 522, 525, 506, 514, 507, 512, 473, 556, 501, 530, 541, 504, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 565, 510, 496, 544, 507, 514, 536, 513, 563, 527, 520, 529, 497, 472, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 510, 501, 500, 520, 521, 516, 528, 527, 491, 525, 501, 526, 501, 537, 505, 11, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 531, 499, 488, 511, 505, 507, 525, 499, 549, 548, 512, 526, 521, 506, 484, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8173, 8109, 8120, 8089, 8253, 8298, 8247, 8247] avg=8192.0000 max=8298 max_rank=5 min=8089 min_rank=3 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=33045 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[523, 498, 520, 520, 529, 528, 515, 488, 504, 506, 520, 536, 532, 511, 520, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 509, 535, 495, 524, 511, 498, 522, 489, 508, 492, 532, 512, 477, 511, 519, 28, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 558, 493, 542, 477, 507, 506, 523, 485, 510, 522, 478, 547, 508, 508, 478, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 495, 504, 491, 491, 531, 513, 534, 524, 494, 499, 504, 499, 531, 524, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 496, 545, 502, 475, 529, 507, 539, 537, 486, 516, 556, 531, 523, 546, 502, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 525, 551, 559, 520, 524, 521, 536, 537, 510, 522, 533, 484, 523, 538, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 544, 472, 472, 484, 476, 507, 505, 509, 483, 508, 523, 490, 498, 492, 531, 11, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 518, 508, 515, 524, 505, 439, 523, 475, 492, 515, 524, 496, 518, 500, 490, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8264, 8176, 8206, 8110, 8322, 8398, 8007, 8053] avg=8192.0000 max=8398 max_rank=5 min=8007 min_rank=6 imbalance_ratio=1.025146 max_load_reduction=57138 max_load_reduction_pct=0.871857 moved_to_echo=32780 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[534, 499, 494, 519, 513, 480, 510, 504, 471, 530, 480, 529, 513, 524, 528, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 491, 512, 504, 516, 498, 519, 519, 482, 492, 500, 498, 545, 504, 528, 533, 28, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 564, 470, 522, 526, 500, 505, 501, 538, 489, 522, 514, 509, 508, 502, 522, 508, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 497, 535, 522, 508, 504, 523, 517, 513, 503, 496, 515, 538, 550, 538, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 504, 533, 532, 533, 509, 476, 497, 519, 521, 530, 516, 512, 500, 508, 514, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 469, 483, 510, 503, 522, 544, 495, 516, 545, 490, 517, 500, 500, 497, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 508, 507, 532, 503, 519, 547, 546, 546, 496, 498, 509, 502, 487, 547, 538, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 494, 501, 473, 534, 491, 505, 500, 494, 508, 518, 491, 506, 490, 530, 503, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8128, 8197, 8234, 8335, 8214, 8095, 8302, 8031] avg=8192.0000 max=8335 max_rank=3 min=8031 min_rank=7 imbalance_ratio=1.017456 max_load_reduction=57201 max_load_reduction_pct=0.872818 moved_to_echo=32642 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[533, 477, 512, 530, 537, 497, 506, 503, 466, 524, 498, 515, 460, 505, 528, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 524, 499, 527, 527, 523, 548, 538, 536, 488, 525, 517, 513, 509, 479, 449, 27, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 554, 500, 504, 503, 538, 513, 537, 519, 515, 483, 523, 528, 530, 531, 549, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 483, 500, 500, 513, 481, 475, 535, 466, 514, 545, 498, 557, 517, 488, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 508, 500, 518, 501, 509, 521, 492, 516, 519, 505, 469, 504, 541, 502, 504, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 505, 476, 498, 520, 509, 522, 500, 539, 510, 483, 507, 530, 498, 495, 565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 540, 542, 494, 499, 488, 559, 462, 488, 489, 532, 497, 535, 504, 533, 507, 11, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 527, 516, 522, 519, 520, 535, 491, 484, 480, 507, 502, 545, 518, 542, 490, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8121, 8245, 8339, 8108, 8159, 8172, 8168, 8224] avg=8192.0000 max=8339 max_rank=2 min=8108 min_rank=3 imbalance_ratio=1.017944 max_load_reduction=57197 max_load_reduction_pct=0.872757 moved_to_echo=32723 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[533, 520, 496, 499, 476, 497, 538, 489, 536, 490, 533, 497, 512, 538, 520, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 504, 522, 501, 483, 505, 473, 502, 527, 502, 518, 486, 525, 487, 518, 499, 26, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 522, 506, 533, 519, 472, 505, 501, 525, 502, 492, 524, 516, 488, 495, 501, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 457, 517, 494, 552, 504, 516, 514, 496, 480, 538, 507, 507, 487, 509, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 504, 530, 530, 522, 525, 506, 514, 507, 512, 473, 556, 501, 530, 541, 504, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 565, 510, 496, 544, 507, 514, 536, 513, 563, 527, 520, 529, 497, 472, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 510, 501, 500, 520, 521, 516, 528, 527, 491, 525, 501, 526, 501, 537, 505, 11, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 531, 499, 488, 511, 505, 507, 525, 499, 549, 548, 512, 526, 521, 506, 484, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8173, 8109, 8120, 8089, 8253, 8298, 8247, 8247] avg=8192.0000 max=8298 max_rank=5 min=8089 min_rank=3 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=33045 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[488, 524, 511, 542, 523, 485, 508, 476, 483, 525, 510, 507, 528, 508, 474, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 475, 476, 535, 511, 507, 518, 507, 506, 496, 496, 506, 463, 508, 534, 539, 29, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 493, 505, 552, 487, 510, 488, 506, 501, 502, 534, 509, 538, 518, 532, 529, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 508, 509, 507, 499, 536, 512, 506, 495, 517, 541, 530, 497, 547, 511, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 505, 464, 517, 513, 516, 542, 509, 505, 504, 503, 467, 523, 512, 565, 519, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 512, 500, 514, 478, 484, 506, 506, 511, 481, 513, 551, 527, 573, 470, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 487, 529, 528, 492, 520, 508, 511, 509, 533, 499, 525, 491, 491, 516, 502, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 511, 516, 477, 525, 524, 515, 530, 517, 559, 478, 544, 499, 530, 480, 489, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8110, 8111, 8264, 8281, 8214, 8197, 8176, 8183] avg=8192.0000 max=8281 max_rank=3 min=8110 min_rank=0 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32770 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[534, 499, 494, 519, 513, 480, 510, 504, 471, 530, 480, 529, 513, 524, 528, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 491, 512, 504, 516, 498, 519, 519, 482, 492, 500, 498, 545, 504, 528, 533, 28, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 564, 470, 522, 526, 500, 505, 501, 538, 489, 522, 514, 509, 508, 502, 522, 508, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 497, 535, 522, 508, 504, 523, 517, 513, 503, 496, 515, 538, 550, 538, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 504, 533, 532, 533, 509, 476, 497, 519, 521, 530, 516, 512, 500, 508, 514, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 469, 483, 510, 503, 522, 544, 495, 516, 545, 490, 517, 500, 500, 497, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 508, 507, 532, 503, 519, 547, 546, 546, 496, 498, 509, 502, 487, 547, 538, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 494, 501, 473, 534, 491, 505, 500, 494, 508, 518, 491, 506, 490, 530, 503, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8128, 8197, 8234, 8335, 8214, 8095, 8302, 8031] avg=8192.0000 max=8335 max_rank=3 min=8031 min_rank=7 imbalance_ratio=1.017456 max_load_reduction=57201 max_load_reduction_pct=0.872818 moved_to_echo=32642 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[507, 521, 535, 538, 475, 510, 545, 538, 502, 506, 491, 498, 532, 540, 522, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 547, 521, 500, 501, 503, 503, 468, 493, 528, 499, 497, 548, 527, 533, 469, 28, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 517, 503, 534, 505, 495, 525, 507, 500, 478, 505, 492, 481, 511, 507, 533, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 497, 489, 494, 518, 534, 482, 510, 503, 525, 496, 545, 504, 501, 538, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 490, 464, 489, 501, 520, 501, 512, 482, 508, 520, 505, 519, 524, 480, 499, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 509, 546, 553, 530, 505, 493, 500, 540, 510, 518, 482, 474, 549, 488, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 480, 488, 527, 498, 544, 526, 536, 527, 541, 480, 515, 485, 535, 482, 504, 11, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 524, 533, 484, 535, 536, 535, 547, 535, 526, 506, 518, 489, 500, 520, 490, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8274, 8200, 8136, 8200, 8030, 8180, 8196, 8320] avg=8192.0000 max=8320 max_rank=7 min=8030 min_rank=4 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32726 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[499, 521, 517, 493, 547, 491, 524, 498, 519, 533, 550, 504, 528, 507, 494, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 484, 536, 529, 482, 490, 506, 497, 461, 524, 552, 503, 530, 515, 535, 518, 29, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 523, 497, 533, 520, 508, 520, 471, 503, 487, 511, 480, 505, 494, 517, 519, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 519, 494, 529, 543, 483, 530, 465, 537, 474, 504, 529, 513, 538, 513, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 508, 531, 540, 537, 487, 525, 494, 487, 516, 529, 540, 541, 494, 511, 483, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 484, 500, 530, 486, 479, 492, 495, 480, 538, 513, 502, 498, 517, 480, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 490, 501, 517, 478, 500, 542, 527, 509, 516, 478, 545, 485, 548, 548, 564, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 543, 448, 495, 515, 564, 503, 502, 527, 507, 505, 534, 487, 537, 526, 501, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8227, 8198, 8132, 8197, 8218, 8063, 8267, 8234] avg=8192.0000 max=8267 max_rank=6 min=8063 min_rank=5 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32782 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[523, 498, 520, 520, 529, 528, 515, 488, 504, 506, 520, 536, 532, 511, 520, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 509, 535, 495, 524, 511, 498, 522, 489, 508, 492, 532, 512, 477, 511, 519, 28, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 558, 493, 542, 477, 507, 506, 523, 485, 510, 522, 478, 547, 508, 508, 478, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 495, 504, 491, 491, 531, 513, 534, 524, 494, 499, 504, 499, 531, 524, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 496, 545, 502, 475, 529, 507, 539, 537, 486, 516, 556, 531, 523, 546, 502, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 525, 551, 559, 520, 524, 521, 536, 537, 510, 522, 533, 484, 523, 538, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 544, 472, 472, 484, 476, 507, 505, 509, 483, 508, 523, 490, 498, 492, 531, 11, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 518, 508, 515, 524, 505, 439, 523, 475, 492, 515, 524, 496, 518, 500, 490, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8264, 8176, 8206, 8110, 8322, 8398, 8007, 8053] avg=8192.0000 max=8398 max_rank=5 min=8007 min_rank=6 imbalance_ratio=1.025146 max_load_reduction=57138 max_load_reduction_pct=0.871857 moved_to_echo=32780 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 3968, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 3968, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 3968, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 3968, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4352, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 3968, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 3968, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4352, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[525, 545, 526, 526, 496, 512, 491, 492, 544, 562, 505, 509, 503, 536, 520, 522, 462, 541, 492, 522, 551, 513, 528, 540, 529, 489, 494, 542, 508, 530, 515, 477, 492, 473, 478, 491, 514, 500, 476, 573, 521, 526, 488, 476, 526, 519, 469, 510, 537, 520, 506, 469, 525, 538, 493, 482, 506, 513, 566, 522, 500, 562, 474, 482, 515, 478, 512, 505, 542, 522, 530, 489, 541, 527, 500, 471, 489, 525, 543, 523, 524, 487, 496, 503, 475, 493, 536, 536, 513, 510, 527, 517, 535, 501, 501, 533, 581, 542, 499, 495, 505, 508, 496, 495, 476, 514, 530, 518, 462, 520, 538, 517, 503, 511, 517, 503, 518, 525, 476, 525, 507, 489, 491, 510, 515, 519, 544, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[522, 515, 549, 472, 529, 529, 485, 527, 499, 498, 548, 487, 505, 535, 481, 508, 534, 501, 532, 538, 505, 506, 502, 542, 514, 503, 473, 477, 488, 515, 531, 538, 540, 509, 492, 527, 512, 507, 492, 515, 498, 492, 496, 534, 486, 489, 556, 512, 534, 500, 516, 523, 485, 507, 460, 506, 470, 499, 522, 527, 520, 509, 504, 516, 511, 486, 462, 492, 503, 523, 534, 468, 507, 475, 539, 527, 492, 516, 540, 536, 513, 506, 490, 525, 506, 512, 499, 529, 529, 489, 520, 517, 547, 524, 510, 542, 476, 506, 559, 490, 462, 507, 545, 536, 499, 495, 480, 504, 542, 508, 521, 475, 526, 514, 504, 596, 543, 519, 534, 514, 524, 557, 550, 495, 514, 542, 470, 517] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[478, 479, 523, 505, 546, 519, 530, 553, 501, 522, 541, 521, 534, 508, 488, 501, 491, 480, 539, 553, 504, 479, 508, 512, 525, 484, 526, 500, 497, 503, 530, 534, 544, 526, 522, 449, 537, 538, 510, 541, 511, 546, 517, 504, 483, 527, 526, 503, 511, 495, 494, 496, 467, 538, 520, 485, 501, 509, 519, 519, 524, 516, 520, 518, 514, 503, 521, 513, 522, 523, 512, 516, 492, 502, 519, 517, 507, 463, 511, 501, 509, 494, 473, 523, 513, 529, 524, 505, 489, 535, 516, 530, 521, 520, 441, 537, 528, 529, 484, 543, 527, 547, 510, 555, 517, 515, 512, 488, 452, 509, 516, 525, 522, 516, 539, 522, 515, 504, 510, 495, 504, 462, 523, 503, 509, 505, 523, 502] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[497, 509, 492, 547, 501, 517, 439, 494, 493, 509, 528, 554, 476, 489, 528, 528, 507, 560, 521, 526, 548, 524, 531, 547, 476, 522, 504, 520, 492, 510, 495, 535, 485, 545, 501, 482, 531, 518, 489, 518, 496, 519, 484, 516, 514, 455, 568, 524, 484, 470, 499, 513, 524, 468, 509, 528, 556, 505, 482, 481, 501, 521, 548, 497, 508, 495, 511, 474, 492, 516, 517, 504, 529, 546, 507, 501, 520, 459, 508, 522, 534, 477, 494, 524, 507, 519, 560, 520, 525, 544, 509, 541, 491, 515, 531, 515, 501, 546, 464, 488, 527, 523, 502, 505, 509, 525, 530, 527, 514, 493, 513, 532, 476, 497, 505, 509, 521, 514, 516, 522, 507, 516, 527, 557, 526, 516, 511, 552] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[523, 476, 532, 533, 519, 515, 481, 523, 507, 491, 540, 500, 482, 529, 520, 542, 536, 536, 491, 543, 497, 519, 523, 506, 477, 491, 515, 494, 519, 507, 463, 497, 485, 497, 509, 522, 516, 524, 507, 459, 505, 538, 504, 511, 509, 480, 517, 538, 520, 485, 522, 506, 538, 505, 518, 536, 533, 536, 515, 493, 513, 539, 466, 537, 535, 510, 515, 518, 505, 486, 527, 515, 500, 507, 540, 496, 527, 499, 511, 488, 517, 522, 513, 521, 500, 560, 517, 506, 507, 537, 500, 506, 467, 488, 517, 573, 513, 540, 492, 502, 474, 531, 492, 491, 510, 547, 518, 538, 505, 530, 525, 523, 532, 483, 556, 486, 493, 500, 522, 560, 530, 504, 502, 491, 515, 496, 516, 479] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[522, 488, 505, 494, 506, 528, 510, 514, 560, 488, 517, 516, 527, 496, 547, 525, 516, 565, 489, 549, 491, 537, 506, 545, 542, 496, 484, 494, 534, 540, 511, 512, 473, 528, 534, 525, 524, 510, 520, 490, 503, 549, 492, 536, 522, 495, 489, 516, 517, 516, 520, 509, 515, 514, 495, 485, 500, 511, 525, 501, 530, 476, 525, 528, 509, 494, 518, 492, 475, 515, 525, 493, 488, 509, 512, 501, 526, 522, 524, 525, 484, 525, 534, 537, 467, 531, 523, 509, 492, 500, 503, 455, 468, 519, 517, 518, 512, 460, 514, 484, 490, 530, 512, 492, 540, 481, 519, 513, 541, 510, 539, 478, 515, 485, 542, 522, 489, 494, 492, 523, 539, 534, 544, 530, 497, 524, 515, 539] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[492, 508, 541, 528, 500, 506, 499, 549, 504, 537, 553, 479, 505, 500, 492, 521, 514, 509, 476, 514, 485, 505, 545, 526, 500, 490, 522, 473, 501, 542, 542, 506, 469, 498, 521, 501, 531, 503, 535, 479, 486, 554, 530, 471, 491, 505, 528, 524, 535, 516, 540, 519, 538, 529, 512, 516, 526, 465, 529, 462, 489, 507, 519, 529, 512, 538, 498, 509, 522, 549, 524, 527, 508, 477, 544, 483, 567, 525, 483, 512, 484, 486, 501, 519, 533, 528, 512, 504, 510, 529, 501, 513, 540, 525, 517, 477, 486, 564, 495, 550, 519, 484, 523, 487, 523, 537, 512, 526, 532, 521, 490, 495, 491, 518, 463, 540, 516, 505, 533, 494, 526, 511, 513, 512, 516, 498, 494, 484] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[478, 479, 523, 505, 546, 519, 530, 553, 501, 522, 541, 521, 534, 508, 488, 501, 491, 480, 539, 553, 504, 479, 508, 512, 525, 484, 526, 500, 497, 503, 530, 534, 544, 526, 522, 449, 537, 538, 510, 541, 511, 546, 517, 504, 483, 527, 526, 503, 511, 495, 494, 496, 467, 538, 520, 485, 501, 509, 519, 519, 524, 516, 520, 518, 514, 503, 521, 513, 522, 523, 512, 516, 492, 502, 519, 517, 507, 463, 511, 501, 509, 494, 473, 523, 513, 529, 524, 505, 489, 535, 516, 530, 521, 520, 441, 537, 528, 529, 484, 543, 527, 547, 510, 555, 517, 515, 512, 488, 452, 509, 516, 525, 522, 516, 539, 522, 515, 504, 510, 495, 504, 462, 523, 503, 509, 505, 523, 502] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[503, 540, 493, 488, 482, 495, 521, 497, 505, 512, 514, 529, 470, 525, 530, 503, 469, 497, 547, 536, 533, 497, 529, 516, 531, 484, 515, 459, 544, 481, 511, 475, 496, 516, 516, 549, 533, 475, 496, 514, 517, 511, 487, 538, 503, 492, 488, 476, 495, 486, 474, 512, 548, 513, 517, 493, 498, 499, 503, 496, 518, 496, 485, 520, 537, 529, 502, 502, 542, 513, 550, 521, 525, 484, 537, 526, 515, 537, 489, 531, 540, 475, 506, 506, 506, 490, 489, 553, 514, 525, 511, 546, 517, 511, 513, 545, 523, 531, 516, 496, 495, 489, 532, 501, 494, 520, 541, 548, 485, 552, 535, 508, 538, 527, 488, 569, 504, 518, 530, 496, 527, 533, 528, 497, 483, 523, 522, 509] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[525, 545, 526, 526, 496, 512, 491, 492, 544, 562, 494, 509, 503, 536, 520, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 541, 492, 513, 551, 513, 528, 540, 529, 489, 494, 542, 508, 530, 515, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 473, 478, 491, 514, 500, 476, 573, 521, 526, 488, 476, 526, 519, 469, 510, 39, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 520, 506, 469, 525, 538, 493, 482, 506, 513, 566, 522, 500, 562, 474, 482, 13, 11, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 478, 512, 505, 542, 522, 530, 489, 541, 527, 500, 471, 489, 525, 543, 523, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 487, 496, 503, 475, 493, 536, 536, 513, 510, 527, 517, 535, 501, 501, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 581, 533, 499, 495, 505, 508, 496, 495, 476, 514, 530, 518, 462, 520, 538, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 511, 517, 464, 518, 525, 476, 525, 507, 489, 491, 510, 515, 519, 544, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8303, 8224, 8074, 8232, 8217, 8171, 8187, 8128] avg=8192.0000 max=8303 max_rank=0 min=8074 min_rank=2 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32703 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[522, 515, 549, 472, 529, 529, 485, 527, 499, 498, 536, 487, 505, 535, 481, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 501, 532, 529, 505, 506, 502, 542, 514, 503, 473, 477, 488, 515, 531, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 509, 492, 527, 512, 507, 492, 515, 498, 492, 496, 534, 486, 489, 556, 512, 46, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 500, 516, 523, 485, 507, 460, 506, 470, 499, 522, 527, 520, 509, 504, 516, 13, 12, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 486, 462, 492, 503, 523, 534, 468, 507, 475, 539, 527, 492, 516, 540, 536, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 506, 490, 525, 506, 512, 499, 529, 529, 489, 520, 517, 547, 524, 510, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 497, 559, 490, 462, 507, 545, 536, 499, 495, 480, 504, 542, 508, 521, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 514, 504, 550, 543, 519, 534, 514, 524, 557, 550, 495, 514, 542, 470, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8190, 8206, 8136, 8116, 8242, 8096, 8373] avg=8192.0000 max=8373 max_rank=7 min=8096 min_rank=6 imbalance_ratio=1.022095 max_load_reduction=57163 max_load_reduction_pct=0.872238 moved_to_echo=32827 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[497, 509, 492, 547, 501, 517, 439, 494, 493, 509, 517, 554, 476, 489, 528, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 560, 521, 517, 548, 524, 531, 547, 476, 522, 504, 520, 492, 510, 495, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 545, 501, 482, 531, 518, 489, 518, 496, 519, 484, 516, 514, 455, 568, 524, 39, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 470, 499, 513, 524, 468, 509, 528, 556, 505, 482, 481, 501, 521, 548, 497, 12, 11, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 495, 511, 474, 492, 516, 517, 504, 529, 546, 507, 501, 520, 459, 508, 522, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 477, 494, 524, 507, 519, 560, 520, 525, 544, 509, 541, 491, 515, 531, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 537, 464, 488, 527, 523, 502, 505, 509, 525, 530, 527, 514, 493, 513, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 497, 505, 470, 521, 514, 516, 522, 507, 516, 527, 557, 526, 516, 511, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8090, 8309, 8187, 8122, 8114, 8291, 8190, 8233] avg=8192.0000 max=8309 max_rank=1 min=8090 min_rank=0 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32828 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[523, 476, 532, 533, 519, 515, 481, 523, 507, 491, 540, 500, 482, 529, 520, 542, 536, 536, 491, 543, 497, 519, 523, 506, 477, 491, 515, 494, 519, 507, 463, 497, 485, 497, 509, 522, 516, 524, 507, 459, 505, 538, 504, 511, 509, 480, 517, 538, 520, 485, 522, 506, 538, 505, 518, 536, 533, 536, 515, 493, 513, 539, 466, 537, 535, 510, 515, 518, 505, 486, 527, 515, 500, 507, 540, 496, 527, 499, 511, 488, 517, 522, 513, 521, 500, 560, 517, 506, 507, 537, 500, 506, 467, 488, 517, 573, 513, 540, 492, 502, 474, 531, 492, 491, 510, 547, 518, 538, 505, 530, 525, 523, 532, 483, 556, 486, 493, 500, 522, 560, 530, 504, 502, 491, 515, 496, 516, 479] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[522, 488, 505, 494, 506, 528, 510, 514, 560, 488, 517, 516, 527, 496, 547, 525, 516, 565, 489, 549, 491, 537, 506, 545, 542, 496, 484, 494, 534, 540, 511, 512, 473, 528, 534, 525, 524, 510, 520, 490, 503, 549, 492, 536, 522, 495, 489, 516, 517, 516, 520, 509, 515, 514, 495, 485, 500, 511, 525, 501, 530, 476, 525, 528, 509, 494, 518, 492, 475, 515, 525, 493, 488, 509, 512, 501, 526, 522, 524, 525, 484, 525, 534, 537, 467, 531, 523, 509, 492, 500, 503, 455, 468, 519, 517, 518, 512, 460, 514, 484, 490, 530, 512, 492, 540, 481, 519, 513, 541, 510, 539, 478, 515, 485, 542, 522, 489, 494, 492, 523, 539, 534, 544, 530, 497, 524, 515, 539] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[525, 545, 526, 526, 496, 512, 491, 492, 544, 562, 505, 509, 503, 536, 520, 522, 462, 541, 492, 522, 551, 513, 528, 540, 529, 489, 494, 542, 508, 530, 515, 477, 492, 473, 478, 491, 514, 500, 476, 573, 521, 526, 488, 476, 526, 519, 469, 510, 537, 520, 506, 469, 525, 538, 493, 482, 506, 513, 566, 522, 500, 562, 474, 482, 515, 478, 512, 505, 542, 522, 530, 489, 541, 527, 500, 471, 489, 525, 543, 523, 524, 487, 496, 503, 475, 493, 536, 536, 513, 510, 527, 517, 535, 501, 501, 533, 581, 542, 499, 495, 505, 508, 496, 495, 476, 514, 530, 518, 462, 520, 538, 517, 503, 511, 517, 503, 518, 525, 476, 525, 507, 489, 491, 510, 515, 519, 544, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[522, 515, 549, 472, 529, 529, 485, 527, 499, 498, 548, 487, 505, 535, 481, 508, 534, 501, 532, 538, 505, 506, 502, 542, 514, 503, 473, 477, 488, 515, 531, 538, 540, 509, 492, 527, 512, 507, 492, 515, 498, 492, 496, 534, 486, 489, 556, 512, 534, 500, 516, 523, 485, 507, 460, 506, 470, 499, 522, 527, 520, 509, 504, 516, 511, 486, 462, 492, 503, 523, 534, 468, 507, 475, 539, 527, 492, 516, 540, 536, 513, 506, 490, 525, 506, 512, 499, 529, 529, 489, 520, 517, 547, 524, 510, 542, 476, 506, 559, 490, 462, 507, 545, 536, 499, 495, 480, 504, 542, 508, 521, 475, 526, 514, 504, 596, 543, 519, 534, 514, 524, 557, 550, 495, 514, 542, 470, 517] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[478, 479, 523, 505, 546, 519, 530, 553, 501, 522, 526, 521, 534, 508, 488, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 480, 539, 539, 504, 479, 508, 512, 525, 484, 526, 500, 497, 503, 530, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 526, 522, 449, 537, 538, 510, 541, 511, 546, 517, 504, 483, 527, 526, 503, 44, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 495, 494, 496, 467, 538, 520, 485, 501, 509, 519, 519, 524, 516, 520, 518, 18, 15, 14, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 503, 521, 513, 522, 523, 512, 516, 492, 502, 519, 517, 507, 463, 511, 501, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 494, 473, 523, 513, 529, 524, 505, 489, 535, 516, 530, 521, 520, 441, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 513, 484, 543, 527, 547, 510, 555, 517, 515, 512, 488, 452, 509, 516, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 516, 539, 478, 515, 504, 510, 495, 504, 462, 523, 503, 509, 505, 523, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8234, 8151, 8334, 8186, 8145, 8135, 8241, 8110] avg=8192.0000 max=8334 max_rank=2 min=8110 min_rank=7 imbalance_ratio=1.017334 max_load_reduction=57202 max_load_reduction_pct=0.872833 moved_to_echo=32631 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[503, 540, 493, 488, 482, 495, 521, 497, 505, 512, 503, 529, 470, 525, 530, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 497, 547, 527, 533, 497, 529, 516, 531, 484, 515, 459, 544, 481, 511, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 516, 516, 549, 533, 475, 496, 514, 517, 511, 487, 538, 503, 492, 488, 476, 44, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 486, 474, 512, 548, 513, 517, 493, 498, 499, 503, 496, 518, 496, 485, 520, 13, 11, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 529, 502, 502, 542, 513, 550, 521, 525, 484, 537, 526, 515, 537, 489, 531, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 475, 506, 506, 506, 490, 489, 553, 514, 525, 511, 546, 517, 511, 513, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 522, 516, 496, 495, 489, 532, 501, 494, 520, 541, 548, 485, 552, 535, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 527, 488, 525, 504, 518, 530, 496, 527, 533, 528, 497, 483, 523, 522, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8096, 8115, 8154, 8090, 8345, 8231, 8257, 8248] avg=8192.0000 max=8345 max_rank=4 min=8090 min_rank=3 imbalance_ratio=1.018677 max_load_reduction=57191 max_load_reduction_pct=0.872665 moved_to_echo=33081 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[497, 509, 492, 547, 501, 517, 439, 494, 493, 509, 517, 554, 476, 489, 528, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 560, 521, 517, 548, 524, 531, 547, 476, 522, 504, 520, 492, 510, 495, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 545, 501, 482, 531, 518, 489, 518, 496, 519, 484, 516, 514, 455, 568, 524, 39, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 470, 499, 513, 524, 468, 509, 528, 556, 505, 482, 481, 501, 521, 548, 497, 12, 11, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 495, 511, 474, 492, 516, 517, 504, 529, 546, 507, 501, 520, 459, 508, 522, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 477, 494, 524, 507, 519, 560, 520, 525, 544, 509, 541, 491, 515, 531, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 537, 464, 488, 527, 523, 502, 505, 509, 525, 530, 527, 514, 493, 513, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 497, 505, 470, 521, 514, 516, 522, 507, 516, 527, 557, 526, 516, 511, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8090, 8309, 8187, 8122, 8114, 8291, 8190, 8233] avg=8192.0000 max=8309 max_rank=1 min=8090 min_rank=0 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32828 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[525, 545, 526, 526, 496, 512, 491, 492, 544, 562, 494, 509, 503, 536, 520, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 541, 492, 513, 551, 513, 528, 540, 529, 489, 494, 542, 508, 530, 515, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 473, 478, 491, 514, 500, 476, 573, 521, 526, 488, 476, 526, 519, 469, 510, 39, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 520, 506, 469, 525, 538, 493, 482, 506, 513, 566, 522, 500, 562, 474, 482, 13, 11, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 478, 512, 505, 542, 522, 530, 489, 541, 527, 500, 471, 489, 525, 543, 523, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 487, 496, 503, 475, 493, 536, 536, 513, 510, 527, 517, 535, 501, 501, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 581, 533, 499, 495, 505, 508, 496, 495, 476, 514, 530, 518, 462, 520, 538, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 511, 517, 464, 518, 525, 476, 525, 507, 489, 491, 510, 515, 519, 544, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8303, 8224, 8074, 8232, 8217, 8171, 8187, 8128] avg=8192.0000 max=8303 max_rank=0 min=8074 min_rank=2 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32703 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[523, 476, 532, 533, 519, 515, 481, 523, 507, 491, 529, 500, 482, 529, 520, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 536, 491, 534, 497, 519, 523, 506, 477, 491, 515, 494, 519, 507, 463, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 497, 509, 522, 516, 524, 507, 459, 505, 538, 504, 511, 509, 480, 517, 538, 37, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 485, 522, 506, 538, 505, 518, 536, 533, 536, 515, 493, 513, 539, 466, 537, 14, 11, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 510, 515, 518, 505, 486, 527, 515, 500, 507, 540, 496, 527, 499, 511, 488, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 522, 513, 521, 500, 560, 517, 506, 507, 537, 500, 506, 467, 488, 517, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 531, 492, 502, 474, 531, 492, 491, 510, 547, 518, 538, 505, 530, 525, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 483, 556, 449, 493, 500, 522, 560, 530, 504, 502, 491, 515, 496, 516, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8105, 8161, 8300, 8184, 8234, 8222, 8128] avg=8192.0000 max=8300 max_rank=3 min=8105 min_rank=1 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32768 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[492, 508, 541, 528, 500, 506, 499, 549, 504, 537, 541, 479, 505, 500, 492, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 509, 476, 505, 485, 505, 545, 526, 500, 490, 522, 473, 501, 542, 542, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 498, 521, 501, 531, 503, 535, 479, 486, 554, 530, 471, 491, 505, 528, 524, 42, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 516, 540, 519, 538, 529, 512, 516, 526, 465, 529, 462, 489, 507, 519, 529, 11, 12, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 538, 498, 509, 522, 549, 524, 527, 508, 477, 544, 483, 567, 525, 483, 512, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 486, 501, 519, 533, 528, 512, 504, 510, 529, 501, 513, 540, 525, 517, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 555, 495, 550, 519, 484, 523, 487, 523, 537, 512, 526, 532, 521, 490, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 518, 463, 498, 516, 505, 533, 494, 526, 511, 513, 512, 516, 498, 494, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8141, 8171, 8267, 8283, 8165, 8235, 8072] avg=8192.0000 max=8283 max_rank=4 min=8072 min_rank=7 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32755 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[522, 488, 505, 494, 506, 528, 510, 514, 560, 488, 506, 516, 527, 496, 547, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 565, 489, 539, 491, 537, 506, 545, 542, 496, 484, 494, 534, 540, 511, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 528, 534, 525, 524, 510, 520, 490, 503, 549, 492, 536, 522, 495, 489, 516, 40, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 516, 520, 509, 515, 514, 495, 485, 500, 511, 525, 501, 530, 476, 525, 528, 12, 11, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 494, 518, 492, 475, 515, 525, 493, 488, 509, 512, 501, 526, 522, 524, 525, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 525, 534, 537, 467, 531, 523, 509, 492, 500, 503, 455, 468, 519, 517, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 453, 514, 484, 490, 530, 512, 492, 540, 481, 519, 513, 541, 510, 539, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 485, 542, 482, 489, 494, 492, 523, 539, 534, 544, 530, 497, 524, 515, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8232, 8301, 8249, 8203, 8132, 8067, 8108, 8244] avg=8192.0000 max=8301 max_rank=1 min=8067 min_rank=5 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32551 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[503, 540, 493, 488, 482, 495, 521, 497, 505, 512, 503, 529, 470, 525, 530, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 497, 547, 527, 533, 497, 529, 516, 531, 484, 515, 459, 544, 481, 511, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 516, 516, 549, 533, 475, 496, 514, 517, 511, 487, 538, 503, 492, 488, 476, 44, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 486, 474, 512, 548, 513, 517, 493, 498, 499, 503, 496, 518, 496, 485, 520, 13, 11, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 529, 502, 502, 542, 513, 550, 521, 525, 484, 537, 526, 515, 537, 489, 531, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 475, 506, 506, 506, 490, 489, 553, 514, 525, 511, 546, 517, 511, 513, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 522, 516, 496, 495, 489, 532, 501, 494, 520, 541, 548, 485, 552, 535, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 527, 488, 525, 504, 518, 530, 496, 527, 533, 528, 497, 483, 523, 522, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8096, 8115, 8154, 8090, 8345, 8231, 8257, 8248] avg=8192.0000 max=8345 max_rank=4 min=8090 min_rank=3 imbalance_ratio=1.018677 max_load_reduction=57191 max_load_reduction_pct=0.872665 moved_to_echo=33081 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[492, 508, 541, 528, 500, 506, 499, 549, 504, 537, 541, 479, 505, 500, 492, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 509, 476, 505, 485, 505, 545, 526, 500, 490, 522, 473, 501, 542, 542, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 498, 521, 501, 531, 503, 535, 479, 486, 554, 530, 471, 491, 505, 528, 524, 42, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 516, 540, 519, 538, 529, 512, 516, 526, 465, 529, 462, 489, 507, 519, 529, 11, 12, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 538, 498, 509, 522, 549, 524, 527, 508, 477, 544, 483, 567, 525, 483, 512, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 486, 501, 519, 533, 528, 512, 504, 510, 529, 501, 513, 540, 525, 517, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 555, 495, 550, 519, 484, 523, 487, 523, 537, 512, 526, 532, 521, 490, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 518, 463, 498, 516, 505, 533, 494, 526, 511, 513, 512, 516, 498, 494, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8141, 8171, 8267, 8283, 8165, 8235, 8072] avg=8192.0000 max=8283 max_rank=4 min=8072 min_rank=7 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32755 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[522, 515, 549, 472, 529, 529, 485, 527, 499, 498, 536, 487, 505, 535, 481, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 501, 532, 529, 505, 506, 502, 542, 514, 503, 473, 477, 488, 515, 531, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 509, 492, 527, 512, 507, 492, 515, 498, 492, 496, 534, 486, 489, 556, 512, 46, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 500, 516, 523, 485, 507, 460, 506, 470, 499, 522, 527, 520, 509, 504, 516, 13, 12, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 486, 462, 492, 503, 523, 534, 468, 507, 475, 539, 527, 492, 516, 540, 536, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 506, 490, 525, 506, 512, 499, 529, 529, 489, 520, 517, 547, 524, 510, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 497, 559, 490, 462, 507, 545, 536, 499, 495, 480, 504, 542, 508, 521, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 514, 504, 550, 543, 519, 534, 514, 524, 557, 550, 495, 514, 542, 470, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8190, 8206, 8136, 8116, 8242, 8096, 8373] avg=8192.0000 max=8373 max_rank=7 min=8096 min_rank=6 imbalance_ratio=1.022095 max_load_reduction=57163 max_load_reduction_pct=0.872238 moved_to_echo=32827 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 3968, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4352, 4096, 3968, 4096, 3968, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 3968, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4352, 4096, 4096, 4096, 3968, 4224, 4224, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4352, 4096, 3968, 4096, 3968, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 3968, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 3968, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[532, 517, 484, 515, 480, 511, 547, 519, 542, 518, 531, 541, 556, 522, 514, 534, 528, 500, 474, 455, 522, 496, 508, 492, 505, 491, 482, 494, 562, 515, 510, 540, 527, 521, 488, 480, 522, 467, 488, 511, 526, 474, 478, 514, 514, 511, 517, 489, 498, 515, 489, 522, 496, 541, 553, 485, 512, 476, 498, 491, 536, 506, 498, 552, 487, 548, 539, 543, 513, 475, 480, 514, 507, 514, 505, 528, 513, 512, 515, 534, 511, 503, 511, 505, 503, 543, 522, 499, 543, 539, 481, 517, 498, 496, 528, 516, 492, 520, 531, 518, 511, 502, 510, 501, 525, 481, 543, 512, 506, 533, 552, 489, 512, 565, 516, 522, 537, 492, 526, 542, 504, 494, 536, 487, 516, 504, 508, 475] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[517, 527, 514, 503, 514, 478, 533, 503, 504, 497, 507, 527, 513, 498, 500, 511, 559, 520, 487, 530, 488, 498, 505, 516, 515, 509, 518, 530, 500, 505, 501, 528, 470, 534, 531, 504, 502, 470, 534, 521, 531, 512, 535, 503, 493, 461, 538, 518, 515, 516, 554, 531, 509, 540, 561, 507, 519, 516, 488, 533, 503, 511, 550, 529, 500, 495, 499, 534, 547, 546, 471, 543, 553, 499, 500, 504, 512, 508, 532, 525, 495, 496, 530, 517, 490, 536, 534, 524, 494, 482, 522, 512, 507, 493, 515, 520, 532, 460, 516, 510, 502, 518, 542, 497, 477, 509, 495, 494, 540, 510, 493, 489, 500, 502, 527, 482, 525, 496, 465, 496, 494, 489, 506, 562, 527, 518, 515, 519] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[508, 519, 474, 523, 505, 502, 538, 505, 527, 521, 478, 498, 500, 504, 496, 478, 521, 486, 536, 522, 522, 527, 537, 512, 540, 507, 516, 538, 510, 523, 497, 488, 506, 512, 467, 461, 536, 541, 523, 507, 505, 515, 479, 537, 544, 512, 508, 542, 476, 503, 522, 514, 505, 528, 480, 529, 469, 534, 548, 488, 532, 508, 526, 505, 514, 529, 536, 520, 491, 515, 515, 489, 486, 481, 549, 496, 525, 508, 517, 545, 512, 513, 536, 539, 508, 465, 464, 493, 485, 506, 489, 521, 520, 503, 517, 503, 525, 506, 528, 525, 516, 487, 500, 536, 526, 491, 524, 522, 517, 514, 502, 508, 532, 529, 550, 540, 522, 532, 528, 493, 535, 505, 510, 521, 497, 516, 485, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[515, 487, 550, 534, 530, 455, 526, 518, 503, 539, 520, 520, 494, 513, 483, 523, 518, 460, 497, 515, 492, 474, 514, 497, 460, 500, 518, 585, 512, 518, 554, 486, 482, 510, 535, 502, 551, 507, 524, 496, 551, 506, 507, 546, 503, 528, 500, 490, 535, 499, 501, 497, 541, 551, 493, 534, 519, 508, 517, 557, 527, 526, 480, 541, 505, 557, 515, 516, 479, 515, 499, 505, 493, 518, 549, 518, 495, 525, 516, 530, 507, 500, 518, 516, 542, 486, 527, 488, 511, 495, 492, 463, 488, 483, 471, 552, 539, 492, 491, 495, 539, 497, 485, 478, 579, 534, 477, 523, 489, 503, 524, 505, 499, 540, 500, 499, 501, 505, 508, 524, 535, 508, 509, 551, 538, 503, 516, 502] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[500, 495, 522, 499, 513, 519, 530, 529, 484, 546, 552, 521, 539, 487, 494, 472, 548, 507, 479, 475, 527, 489, 484, 527, 510, 516, 480, 543, 527, 482, 507, 499, 497, 520, 514, 552, 504, 535, 529, 491, 514, 553, 479, 560, 521, 502, 488, 548, 479, 544, 568, 517, 486, 506, 544, 478, 504, 506, 503, 504, 540, 468, 530, 501, 521, 510, 527, 542, 484, 484, 500, 521, 521, 529, 507, 515, 453, 517, 499, 486, 507, 515, 527, 544, 512, 472, 515, 527, 492, 517, 530, 482, 520, 519, 523, 468, 536, 577, 514, 511, 481, 553, 505, 503, 505, 518, 518, 477, 540, 504, 532, 535, 529, 507, 495, 521, 529, 499, 573, 529, 504, 478, 497, 497, 504, 506, 507, 479] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[517, 551, 492, 461, 474, 523, 523, 463, 512, 509, 469, 508, 527, 513, 523, 494, 520, 551, 521, 552, 498, 516, 536, 531, 550, 519, 476, 507, 503, 496, 530, 548, 510, 504, 496, 500, 567, 508, 493, 507, 487, 522, 569, 507, 512, 506, 528, 532, 536, 514, 495, 510, 509, 536, 532, 493, 483, 525, 477, 479, 521, 537, 522, 481, 529, 506, 494, 535, 505, 506, 515, 506, 535, 522, 537, 479, 509, 541, 540, 507, 475, 517, 540, 534, 542, 470, 468, 511, 532, 508, 506, 548, 506, 524, 496, 530, 496, 496, 554, 492, 540, 530, 507, 511, 507, 476, 486, 502, 493, 524, 523, 496, 495, 508, 501, 508, 539, 525, 507, 502, 536, 512, 496, 493, 459, 516, 502, 520] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[514, 484, 517, 527, 522, 524, 518, 523, 559, 528, 489, 490, 562, 527, 487, 534, 488, 505, 514, 513, 521, 525, 493, 484, 473, 483, 532, 482, 512, 495, 490, 523, 479, 488, 514, 522, 494, 553, 515, 477, 520, 540, 534, 473, 494, 504, 515, 489, 535, 508, 474, 526, 522, 520, 511, 540, 519, 555, 512, 487, 502, 490, 524, 501, 523, 525, 505, 518, 538, 502, 517, 499, 504, 525, 482, 501, 491, 483, 489, 519, 464, 518, 488, 551, 526, 495, 521, 500, 515, 493, 476, 539, 535, 549, 535, 503, 507, 557, 496, 515, 485, 530, 488, 543, 545, 543, 493, 536, 467, 511, 468, 517, 529, 484, 508, 525, 513, 521, 541, 559, 519, 536, 513, 493, 562, 508, 496, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[532, 517, 484, 515, 480, 511, 547, 519, 542, 518, 531, 541, 556, 522, 514, 534, 528, 500, 474, 455, 522, 496, 508, 492, 505, 491, 482, 494, 562, 515, 510, 540, 527, 521, 488, 480, 522, 467, 488, 511, 526, 474, 478, 514, 514, 511, 517, 489, 498, 515, 489, 522, 496, 541, 553, 485, 512, 476, 498, 491, 536, 506, 498, 552, 487, 548, 539, 543, 513, 475, 480, 514, 507, 514, 505, 528, 513, 512, 515, 534, 511, 503, 511, 505, 503, 543, 522, 499, 543, 539, 481, 517, 498, 496, 528, 516, 492, 520, 531, 518, 511, 502, 510, 501, 525, 481, 543, 512, 506, 533, 552, 489, 512, 565, 516, 522, 537, 492, 526, 542, 504, 494, 536, 487, 516, 504, 508, 475] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[517, 551, 492, 461, 474, 523, 523, 463, 512, 509, 469, 508, 527, 513, 523, 494, 520, 551, 521, 552, 498, 516, 536, 531, 550, 519, 476, 507, 503, 496, 530, 548, 510, 504, 496, 500, 567, 508, 493, 507, 487, 522, 569, 507, 512, 506, 528, 532, 536, 514, 495, 510, 509, 536, 532, 493, 483, 525, 477, 479, 521, 537, 522, 481, 529, 506, 494, 535, 505, 506, 515, 506, 535, 522, 537, 479, 509, 541, 540, 507, 475, 517, 540, 534, 542, 470, 468, 511, 532, 508, 506, 548, 506, 524, 496, 530, 496, 496, 554, 492, 540, 530, 507, 511, 507, 476, 486, 502, 493, 524, 523, 496, 495, 508, 501, 508, 539, 525, 507, 502, 536, 512, 496, 493, 459, 516, 502, 520] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[530, 504, 504, 548, 490, 513, 465, 522, 477, 501, 520, 489, 539, 531, 482, 540, 511, 491, 524, 511, 525, 523, 478, 528, 535, 519, 532, 497, 506, 511, 493, 485, 495, 531, 515, 522, 555, 497, 486, 515, 508, 494, 489, 510, 535, 501, 472, 498, 551, 471, 473, 513, 535, 526, 503, 519, 531, 515, 518, 521, 485, 541, 527, 515, 512, 532, 488, 508, 520, 516, 507, 497, 512, 493, 507, 526, 539, 522, 487, 543, 481, 550, 529, 491, 546, 547, 490, 501, 503, 529, 517, 502, 510, 482, 513, 472, 533, 509, 481, 486, 537, 541, 529, 512, 492, 531, 533, 530, 525, 480, 557, 467, 511, 551, 509, 519, 551, 502, 500, 506, 518, 501, 556, 522, 509, 519, 468, 488] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[517, 527, 514, 503, 514, 478, 533, 503, 504, 497, 507, 527, 513, 498, 500, 511, 559, 520, 487, 530, 488, 498, 505, 516, 515, 509, 518, 530, 500, 505, 501, 528, 470, 534, 531, 504, 502, 470, 534, 521, 531, 512, 535, 503, 493, 461, 538, 518, 515, 516, 554, 531, 509, 540, 561, 507, 519, 516, 488, 533, 503, 511, 550, 529, 500, 495, 499, 534, 547, 546, 471, 543, 553, 499, 500, 504, 512, 508, 532, 525, 495, 496, 530, 517, 490, 536, 534, 524, 494, 482, 522, 512, 507, 493, 515, 520, 532, 460, 516, 510, 502, 518, 542, 497, 477, 509, 495, 494, 540, 510, 493, 489, 500, 502, 527, 482, 525, 496, 465, 496, 494, 489, 506, 562, 527, 518, 515, 519] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[532, 517, 484, 515, 480, 511, 547, 519, 542, 518, 531, 541, 556, 522, 514, 534, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 500, 474, 455, 522, 496, 508, 492, 505, 491, 482, 494, 562, 515, 510, 540, 24, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 521, 488, 480, 522, 467, 488, 511, 526, 474, 478, 514, 514, 511, 517, 489, 14, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 515, 489, 522, 496, 503, 553, 485, 512, 476, 498, 491, 536, 506, 498, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 548, 539, 528, 513, 475, 480, 514, 507, 514, 505, 528, 513, 512, 515, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 503, 511, 505, 503, 543, 522, 499, 543, 539, 481, 517, 498, 496, 528, 516, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 520, 531, 518, 511, 502, 510, 501, 525, 481, 543, 512, 506, 533, 552, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 565, 516, 522, 513, 492, 526, 542, 504, 494, 536, 487, 516, 504, 508, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8365, 8100, 8043, 8130, 8212, 8252, 8222, 8212] avg=8192.0000 max=8365 max_rank=0 min=8043 min_rank=2 imbalance_ratio=1.021118 max_load_reduction=57171 max_load_reduction_pct=0.872360 moved_to_echo=32898 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[515, 487, 550, 534, 530, 455, 526, 518, 503, 539, 520, 520, 494, 513, 483, 523, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 460, 497, 515, 492, 474, 514, 497, 460, 500, 518, 585, 512, 518, 554, 486, 23, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 510, 535, 502, 551, 507, 524, 496, 551, 506, 507, 546, 503, 528, 500, 490, 13, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 499, 501, 497, 541, 512, 493, 534, 519, 508, 517, 557, 527, 526, 480, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 557, 515, 502, 479, 515, 499, 505, 493, 518, 549, 518, 495, 525, 516, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 500, 518, 516, 542, 486, 527, 488, 511, 495, 492, 463, 488, 483, 471, 552, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 492, 491, 495, 539, 497, 485, 478, 579, 534, 477, 523, 489, 503, 524, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 540, 500, 499, 478, 505, 508, 524, 535, 508, 509, 551, 538, 503, 516, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8125, 8253, 8287, 8221, 8077, 8146, 8215] avg=8192.0000 max=8287 max_rank=3 min=8077 min_rank=5 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32659 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[500, 495, 522, 499, 513, 519, 530, 529, 484, 546, 552, 521, 539, 487, 494, 472, 548, 507, 479, 475, 527, 489, 484, 527, 510, 516, 480, 543, 527, 482, 507, 499, 497, 520, 514, 552, 504, 535, 529, 491, 514, 553, 479, 560, 521, 502, 488, 548, 479, 544, 568, 517, 486, 506, 544, 478, 504, 506, 503, 504, 540, 468, 530, 501, 521, 510, 527, 542, 484, 484, 500, 521, 521, 529, 507, 515, 453, 517, 499, 486, 507, 515, 527, 544, 512, 472, 515, 527, 492, 517, 530, 482, 520, 519, 523, 468, 536, 577, 514, 511, 481, 553, 505, 503, 505, 518, 518, 477, 540, 504, 532, 535, 529, 507, 495, 521, 529, 499, 573, 529, 504, 478, 497, 497, 504, 506, 507, 479] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[515, 487, 550, 534, 530, 455, 526, 518, 503, 539, 520, 520, 494, 513, 483, 523, 518, 460, 497, 515, 492, 474, 514, 497, 460, 500, 518, 585, 512, 518, 554, 486, 482, 510, 535, 502, 551, 507, 524, 496, 551, 506, 507, 546, 503, 528, 500, 490, 535, 499, 501, 497, 541, 551, 493, 534, 519, 508, 517, 557, 527, 526, 480, 541, 505, 557, 515, 516, 479, 515, 499, 505, 493, 518, 549, 518, 495, 525, 516, 530, 507, 500, 518, 516, 542, 486, 527, 488, 511, 495, 492, 463, 488, 483, 471, 552, 539, 492, 491, 495, 539, 497, 485, 478, 579, 534, 477, 523, 489, 503, 524, 505, 499, 540, 500, 499, 501, 505, 508, 524, 535, 508, 509, 551, 538, 503, 516, 502] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[508, 519, 474, 523, 505, 502, 538, 505, 527, 521, 478, 498, 500, 504, 496, 478, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 486, 536, 522, 522, 527, 537, 512, 540, 507, 516, 538, 510, 523, 497, 488, 27, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 512, 467, 461, 536, 541, 523, 507, 505, 515, 479, 537, 544, 512, 508, 542, 18, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 503, 522, 514, 505, 483, 480, 529, 469, 534, 548, 488, 532, 508, 526, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 529, 536, 500, 491, 515, 515, 489, 486, 481, 549, 496, 525, 508, 517, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 513, 536, 539, 508, 465, 464, 493, 485, 506, 489, 521, 520, 503, 517, 503, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 506, 528, 525, 516, 487, 500, 536, 526, 491, 524, 522, 517, 514, 502, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 529, 550, 540, 495, 532, 528, 493, 535, 505, 510, 521, 497, 516, 485, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8082, 8316, 8216, 8122, 8196, 8114, 8218, 8272] avg=8192.0000 max=8316 max_rank=1 min=8082 min_rank=0 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32800 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[532, 517, 484, 515, 480, 511, 547, 519, 542, 518, 531, 541, 556, 522, 514, 534, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 500, 474, 455, 522, 496, 508, 492, 505, 491, 482, 494, 562, 515, 510, 540, 24, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 521, 488, 480, 522, 467, 488, 511, 526, 474, 478, 514, 514, 511, 517, 489, 14, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 515, 489, 522, 496, 503, 553, 485, 512, 476, 498, 491, 536, 506, 498, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 548, 539, 528, 513, 475, 480, 514, 507, 514, 505, 528, 513, 512, 515, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 503, 511, 505, 503, 543, 522, 499, 543, 539, 481, 517, 498, 496, 528, 516, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 520, 531, 518, 511, 502, 510, 501, 525, 481, 543, 512, 506, 533, 552, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 565, 516, 522, 513, 492, 526, 542, 504, 494, 536, 487, 516, 504, 508, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8365, 8100, 8043, 8130, 8212, 8252, 8222, 8212] avg=8192.0000 max=8365 max_rank=0 min=8043 min_rank=2 imbalance_ratio=1.021118 max_load_reduction=57171 max_load_reduction_pct=0.872360 moved_to_echo=32898 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[517, 527, 514, 503, 514, 478, 533, 503, 504, 497, 507, 527, 513, 498, 500, 511, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 520, 487, 530, 488, 498, 505, 516, 515, 509, 518, 530, 500, 505, 501, 528, 24, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 534, 531, 504, 502, 470, 534, 521, 531, 512, 535, 503, 493, 461, 538, 518, 14, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 516, 554, 531, 509, 502, 561, 507, 519, 516, 488, 533, 503, 511, 550, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 495, 499, 519, 547, 546, 471, 543, 553, 499, 500, 504, 512, 508, 532, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 496, 530, 517, 490, 536, 534, 524, 494, 482, 522, 512, 507, 493, 515, 520, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 460, 516, 510, 502, 518, 542, 497, 477, 509, 495, 494, 540, 510, 493, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 502, 527, 482, 501, 496, 465, 496, 494, 489, 506, 562, 527, 518, 515, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8148, 8235, 8173, 8344, 8253, 8204, 8080, 8099] avg=8192.0000 max=8344 max_rank=3 min=8080 min_rank=6 imbalance_ratio=1.018555 max_load_reduction=57192 max_load_reduction_pct=0.872681 moved_to_echo=32636 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[517, 551, 492, 461, 474, 523, 523, 463, 512, 509, 469, 508, 527, 513, 523, 494, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 551, 521, 552, 498, 516, 536, 531, 550, 519, 476, 507, 503, 496, 530, 548, 24, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 504, 496, 500, 567, 508, 493, 507, 487, 522, 569, 507, 512, 506, 528, 532, 14, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 514, 495, 510, 509, 499, 532, 493, 483, 525, 477, 479, 521, 537, 522, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 506, 494, 520, 505, 506, 515, 506, 535, 522, 537, 479, 509, 541, 540, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 517, 540, 534, 542, 470, 468, 511, 532, 508, 506, 548, 506, 524, 496, 530, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 496, 554, 492, 540, 530, 507, 511, 507, 476, 486, 502, 493, 524, 523, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 508, 501, 508, 515, 525, 507, 502, 536, 512, 496, 493, 459, 516, 502, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8061, 8380, 8264, 8113, 8251, 8243, 8129, 8095] avg=8192.0000 max=8380 max_rank=1 min=8061 min_rank=0 imbalance_ratio=1.022949 max_load_reduction=57156 max_load_reduction_pct=0.872131 moved_to_echo=32718 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[500, 495, 522, 499, 513, 519, 530, 529, 484, 546, 552, 521, 539, 487, 494, 472, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 507, 479, 475, 527, 489, 484, 527, 510, 516, 480, 543, 527, 482, 507, 499, 24, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 520, 514, 552, 504, 535, 529, 491, 514, 553, 479, 560, 521, 502, 488, 548, 14, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 544, 568, 517, 486, 471, 544, 478, 504, 506, 503, 504, 540, 468, 530, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 510, 527, 527, 484, 484, 500, 521, 521, 529, 507, 515, 453, 517, 499, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 515, 527, 544, 512, 472, 515, 527, 492, 517, 530, 482, 520, 519, 523, 468, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 577, 514, 511, 481, 553, 505, 503, 505, 518, 518, 477, 540, 504, 532, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 507, 495, 521, 505, 499, 573, 529, 504, 478, 497, 497, 504, 506, 507, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8126, 8323, 8143, 8101, 8204, 8305, 8130] avg=8192.0000 max=8323 max_rank=2 min=8101 min_rank=4 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32740 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[514, 484, 517, 527, 522, 524, 518, 523, 559, 528, 489, 490, 562, 527, 487, 534, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 505, 514, 513, 521, 525, 493, 484, 473, 483, 532, 482, 512, 495, 490, 523, 23, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 488, 514, 522, 494, 553, 515, 477, 520, 540, 534, 473, 494, 504, 515, 489, 13, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 508, 474, 526, 522, 484, 511, 540, 519, 555, 512, 487, 502, 490, 524, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 525, 505, 504, 538, 502, 517, 499, 504, 525, 482, 501, 491, 483, 489, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 464, 518, 488, 551, 526, 495, 521, 500, 515, 493, 476, 539, 535, 549, 535, 503, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 557, 496, 515, 485, 530, 488, 543, 545, 543, 493, 536, 467, 511, 468, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 484, 508, 525, 490, 521, 541, 559, 519, 536, 513, 493, 562, 508, 496, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8307, 8058, 8126, 8190, 8107, 8243, 8197, 8308] avg=8192.0000 max=8308 max_rank=7 min=8058 min_rank=1 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32855 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[530, 504, 504, 548, 490, 513, 465, 522, 477, 501, 520, 489, 539, 531, 482, 540, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 491, 524, 511, 525, 523, 478, 528, 535, 519, 532, 497, 506, 511, 493, 485, 25, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 531, 515, 522, 555, 497, 486, 515, 508, 494, 489, 510, 535, 501, 472, 498, 13, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 471, 473, 513, 535, 489, 503, 519, 531, 515, 518, 521, 485, 541, 527, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 532, 488, 494, 520, 516, 507, 497, 512, 493, 507, 526, 539, 522, 487, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 550, 529, 491, 546, 547, 490, 501, 503, 529, 517, 502, 510, 482, 513, 472, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 509, 481, 486, 537, 541, 529, 512, 492, 531, 533, 530, 525, 480, 557, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 551, 509, 519, 526, 502, 500, 506, 518, 501, 556, 522, 509, 519, 468, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8196, 8138, 8207, 8195, 8199, 8239, 8205] avg=8192.0000 max=8239 max_rank=6 min=8138 min_rank=2 imbalance_ratio=1.005737 max_load_reduction=57297 max_load_reduction_pct=0.874283 moved_to_echo=32838 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[517, 527, 514, 503, 514, 478, 533, 503, 504, 497, 507, 527, 513, 498, 500, 511, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 520, 487, 530, 488, 498, 505, 516, 515, 509, 518, 530, 500, 505, 501, 528, 24, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 534, 531, 504, 502, 470, 534, 521, 531, 512, 535, 503, 493, 461, 538, 518, 14, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 516, 554, 531, 509, 502, 561, 507, 519, 516, 488, 533, 503, 511, 550, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 495, 499, 519, 547, 546, 471, 543, 553, 499, 500, 504, 512, 508, 532, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 496, 530, 517, 490, 536, 534, 524, 494, 482, 522, 512, 507, 493, 515, 520, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 460, 516, 510, 502, 518, 542, 497, 477, 509, 495, 494, 540, 510, 493, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 502, 527, 482, 501, 496, 465, 496, 494, 489, 506, 562, 527, 518, 515, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8148, 8235, 8173, 8344, 8253, 8204, 8080, 8099] avg=8192.0000 max=8344 max_rank=3 min=8080 min_rank=6 imbalance_ratio=1.018555 max_load_reduction=57192 max_load_reduction_pct=0.872681 moved_to_echo=32636 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[514, 484, 517, 527, 522, 524, 518, 523, 559, 528, 489, 490, 562, 527, 487, 534, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 505, 514, 513, 521, 525, 493, 484, 473, 483, 532, 482, 512, 495, 490, 523, 23, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 488, 514, 522, 494, 553, 515, 477, 520, 540, 534, 473, 494, 504, 515, 489, 13, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 508, 474, 526, 522, 484, 511, 540, 519, 555, 512, 487, 502, 490, 524, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 525, 505, 504, 538, 502, 517, 499, 504, 525, 482, 501, 491, 483, 489, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 464, 518, 488, 551, 526, 495, 521, 500, 515, 493, 476, 539, 535, 549, 535, 503, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 557, 496, 515, 485, 530, 488, 543, 545, 543, 493, 536, 467, 511, 468, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 484, 508, 525, 490, 521, 541, 559, 519, 536, 513, 493, 562, 508, 496, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8307, 8058, 8126, 8190, 8107, 8243, 8197, 8308] avg=8192.0000 max=8308 max_rank=7 min=8058 min_rank=1 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32855 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[500, 495, 522, 499, 513, 519, 530, 529, 484, 546, 552, 521, 539, 487, 494, 472, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 507, 479, 475, 527, 489, 484, 527, 510, 516, 480, 543, 527, 482, 507, 499, 24, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 520, 514, 552, 504, 535, 529, 491, 514, 553, 479, 560, 521, 502, 488, 548, 14, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 544, 568, 517, 486, 471, 544, 478, 504, 506, 503, 504, 540, 468, 530, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 510, 527, 527, 484, 484, 500, 521, 521, 529, 507, 515, 453, 517, 499, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 515, 527, 544, 512, 472, 515, 527, 492, 517, 530, 482, 520, 519, 523, 468, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 577, 514, 511, 481, 553, 505, 503, 505, 518, 518, 477, 540, 504, 532, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 507, 495, 521, 505, 499, 573, 529, 504, 478, 497, 497, 504, 506, 507, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8126, 8323, 8143, 8101, 8204, 8305, 8130] avg=8192.0000 max=8323 max_rank=2 min=8101 min_rank=4 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32740 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[515, 487, 550, 534, 530, 455, 526, 518, 503, 539, 520, 520, 494, 513, 483, 523, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 460, 497, 515, 492, 474, 514, 497, 460, 500, 518, 585, 512, 518, 554, 486, 23, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 510, 535, 502, 551, 507, 524, 496, 551, 506, 507, 546, 503, 528, 500, 490, 13, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 499, 501, 497, 541, 512, 493, 534, 519, 508, 517, 557, 527, 526, 480, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 557, 515, 502, 479, 515, 499, 505, 493, 518, 549, 518, 495, 525, 516, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 500, 518, 516, 542, 486, 527, 488, 511, 495, 492, 463, 488, 483, 471, 552, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 492, 491, 495, 539, 497, 485, 478, 579, 534, 477, 523, 489, 503, 524, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 540, 500, 499, 478, 505, 508, 524, 535, 508, 509, 551, 538, 503, 516, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8125, 8253, 8287, 8221, 8077, 8146, 8215] avg=8192.0000 max=8287 max_rank=3 min=8077 min_rank=5 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32659 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[3968, 4224, 4096, 4096, 4352, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 3968, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4352, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[3968, 4224, 4096, 4096, 4352, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[3968, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 3968, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[508, 526, 513, 510, 498, 520, 516, 523, 510, 534, 536, 512, 544, 515, 497, 496, 516, 489, 529, 507, 513, 509, 510, 528, 526, 505, 490, 497, 511, 505, 485, 555, 498, 564, 507, 541, 502, 508, 527, 509, 531, 537, 483, 474, 516, 512, 482, 522, 483, 524, 507, 494, 507, 530, 531, 522, 504, 525, 494, 526, 535, 512, 551, 513, 484, 501, 472, 508, 513, 525, 471, 553, 521, 475, 545, 526, 528, 523, 533, 492, 501, 484, 554, 522, 553, 505, 514, 482, 474, 511, 510, 479, 562, 487, 479, 493, 489, 545, 494, 516, 527, 499, 543, 506, 506, 543, 518, 524, 493, 505, 507, 484, 462, 515, 504, 538, 517, 504, 500, 469, 477, 515, 509, 504, 544, 554, 512, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[455, 483, 541, 515, 535, 481, 532, 518, 502, 496, 495, 521, 552, 535, 516, 503, 513, 558, 518, 535, 477, 566, 512, 517, 518, 487, 504, 516, 497, 514, 476, 500, 521, 504, 497, 499, 511, 511, 556, 494, 449, 473, 492, 525, 486, 484, 525, 540, 556, 481, 501, 498, 502, 498, 522, 488, 489, 520, 534, 470, 506, 524, 497, 507, 491, 521, 499, 505, 483, 532, 538, 528, 519, 536, 533, 516, 509, 518, 540, 482, 536, 529, 501, 506, 504, 511, 517, 499, 527, 561, 534, 562, 503, 499, 511, 507, 507, 475, 499, 530, 565, 544, 483, 476, 538, 550, 464, 484, 533, 523, 485, 526, 517, 502, 477, 479, 472, 512, 608, 550, 500, 529, 519, 494, 496, 521, 547, 526] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[504, 559, 516, 552, 535, 473, 514, 507, 488, 506, 542, 489, 521, 558, 513, 463, 532, 517, 510, 485, 537, 507, 540, 472, 508, 527, 504, 537, 478, 446, 494, 519, 509, 509, 520, 524, 528, 507, 527, 511, 529, 479, 509, 468, 520, 518, 478, 522, 523, 511, 504, 495, 525, 486, 557, 525, 510, 511, 519, 551, 505, 522, 539, 523, 535, 473, 514, 533, 499, 506, 500, 469, 525, 533, 540, 503, 527, 531, 467, 504, 508, 505, 512, 475, 516, 494, 525, 504, 512, 519, 536, 549, 496, 523, 500, 498, 529, 491, 475, 526, 515, 514, 538, 517, 504, 509, 518, 506, 556, 496, 499, 481, 524, 484, 505, 507, 498, 543, 532, 509, 506, 528, 513, 509, 523, 519, 507, 507] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[484, 514, 525, 527, 494, 475, 496, 503, 529, 494, 475, 507, 468, 549, 505, 512, 531, 512, 525, 526, 500, 529, 541, 503, 539, 533, 519, 497, 543, 513, 499, 501, 495, 523, 510, 518, 524, 493, 508, 493, 492, 565, 498, 526, 530, 505, 493, 493, 497, 540, 521, 522, 495, 492, 532, 506, 546, 510, 536, 495, 526, 495, 502, 481, 502, 533, 491, 498, 500, 546, 504, 508, 513, 504, 520, 525, 471, 501, 480, 491, 532, 515, 533, 538, 479, 510, 501, 485, 520, 504, 533, 528, 472, 517, 522, 521, 553, 514, 521, 523, 509, 494, 502, 484, 500, 513, 494, 540, 500, 501, 482, 485, 558, 516, 564, 551, 497, 527, 524, 497, 510, 524, 587, 479, 525, 523, 523, 489] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[508, 526, 513, 510, 498, 520, 516, 523, 510, 534, 536, 512, 544, 515, 497, 496, 516, 489, 529, 507, 513, 509, 510, 528, 526, 505, 490, 497, 511, 505, 485, 555, 498, 564, 507, 541, 502, 508, 527, 509, 531, 537, 483, 474, 516, 512, 482, 522, 483, 524, 507, 494, 507, 530, 531, 522, 504, 525, 494, 526, 535, 512, 551, 513, 484, 501, 472, 508, 513, 525, 471, 553, 521, 475, 545, 526, 528, 523, 533, 492, 501, 484, 554, 522, 553, 505, 514, 482, 474, 511, 510, 479, 562, 487, 479, 493, 489, 545, 494, 516, 527, 499, 543, 506, 506, 543, 518, 524, 493, 505, 507, 484, 462, 515, 504, 538, 517, 504, 500, 469, 477, 515, 509, 504, 544, 554, 512, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[530, 525, 508, 508, 535, 532, 515, 512, 437, 488, 555, 531, 525, 561, 515, 539, 558, 523, 526, 476, 540, 538, 505, 546, 486, 510, 500, 527, 520, 500, 501, 501, 523, 511, 508, 510, 487, 500, 532, 495, 528, 507, 528, 507, 515, 516, 533, 547, 494, 511, 512, 524, 551, 541, 497, 514, 496, 480, 484, 546, 471, 479, 510, 508, 503, 516, 549, 503, 453, 491, 525, 483, 522, 516, 545, 516, 521, 514, 522, 509, 538, 490, 492, 472, 499, 499, 501, 518, 492, 510, 520, 514, 547, 490, 517, 531, 512, 529, 477, 521, 487, 472, 523, 547, 477, 495, 518, 501, 503, 508, 498, 549, 540, 497, 524, 515, 514, 525, 510, 520, 508, 495, 484, 544, 500, 480, 485, 522] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[543, 515, 528, 536, 509, 507, 517, 554, 521, 489, 495, 519, 520, 506, 495, 489, 517, 476, 490, 519, 521, 521, 485, 569, 561, 536, 508, 483, 470, 548, 517, 502, 495, 504, 539, 499, 515, 524, 506, 487, 509, 524, 552, 505, 530, 487, 520, 465, 505, 490, 513, 490, 536, 489, 482, 531, 513, 535, 514, 490, 476, 533, 534, 540, 527, 488, 505, 509, 510, 466, 539, 493, 528, 507, 514, 496, 543, 498, 542, 526, 503, 526, 515, 530, 502, 516, 503, 554, 490, 496, 535, 516, 522, 465, 546, 480, 478, 504, 522, 502, 492, 492, 530, 532, 525, 535, 496, 485, 526, 518, 542, 547, 513, 462, 538, 513, 480, 541, 479, 506, 535, 507, 506, 529, 489, 517, 522, 485] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[481, 532, 531, 494, 512, 529, 493, 493, 531, 523, 529, 516, 538, 533, 523, 534, 516, 484, 519, 495, 506, 480, 497, 495, 507, 515, 518, 506, 468, 516, 528, 486, 497, 523, 537, 537, 499, 499, 545, 544, 506, 517, 514, 555, 501, 523, 512, 515, 520, 523, 519, 504, 496, 523, 537, 503, 494, 544, 535, 532, 477, 485, 533, 487, 575, 504, 506, 549, 530, 527, 510, 499, 505, 501, 526, 501, 493, 467, 486, 497, 483, 497, 530, 501, 543, 466, 511, 561, 552, 516, 510, 503, 537, 517, 542, 494, 500, 547, 526, 550, 472, 499, 527, 521, 506, 455, 490, 525, 522, 504, 499, 452, 518, 523, 481, 510, 479, 489, 512, 512, 507, 522, 488, 528, 534, 489, 524, 522] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[520, 458, 516, 513, 523, 559, 548, 481, 480, 484, 526, 511, 503, 532, 498, 537, 513, 544, 516, 517, 508, 546, 536, 542, 528, 544, 517, 526, 512, 507, 478, 515, 536, 493, 529, 494, 510, 531, 511, 494, 498, 535, 509, 490, 538, 513, 532, 505, 513, 477, 442, 507, 504, 481, 525, 521, 532, 490, 495, 537, 532, 527, 512, 484, 545, 511, 526, 518, 498, 566, 518, 503, 503, 491, 547, 521, 532, 500, 475, 525, 519, 484, 495, 511, 487, 548, 488, 518, 516, 507, 501, 480, 555, 462, 492, 506, 481, 488, 550, 539, 536, 486, 469, 509, 488, 509, 490, 504, 513, 513, 542, 504, 508, 520, 484, 522, 490, 503, 521, 449, 530, 506, 555, 514, 541, 543, 543, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[455, 483, 541, 515, 535, 481, 532, 518, 502, 496, 495, 521, 552, 506, 516, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 558, 518, 535, 477, 548, 512, 517, 518, 487, 504, 516, 497, 514, 476, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 504, 497, 499, 511, 511, 554, 494, 449, 473, 492, 525, 486, 484, 525, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 481, 501, 498, 502, 498, 522, 488, 489, 520, 534, 470, 506, 524, 497, 507, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 521, 499, 505, 483, 532, 538, 528, 519, 536, 533, 516, 509, 518, 540, 482, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 529, 501, 506, 504, 511, 517, 499, 527, 561, 534, 562, 503, 499, 511, 507, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 475, 499, 530, 565, 544, 483, 476, 538, 550, 464, 484, 533, 523, 485, 526, 29, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 502, 477, 479, 472, 512, 590, 550, 500, 529, 519, 494, 496, 521, 547, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 8190, 8065, 8107, 8254, 8318, 8220, 8231] avg=8192.0000 max=8318 max_rank=5 min=8065 min_rank=2 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=33023 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[504, 559, 516, 552, 535, 473, 514, 507, 488, 506, 542, 489, 521, 527, 513, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 517, 510, 485, 537, 491, 540, 472, 508, 527, 504, 537, 478, 446, 494, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 509, 520, 524, 528, 507, 525, 511, 529, 479, 509, 468, 520, 518, 478, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 511, 504, 495, 525, 486, 557, 525, 510, 511, 519, 551, 505, 522, 539, 523, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 473, 514, 533, 499, 506, 500, 469, 525, 533, 540, 503, 527, 531, 467, 504, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 505, 512, 475, 516, 494, 525, 504, 512, 519, 536, 549, 496, 523, 500, 498, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 491, 475, 526, 515, 514, 538, 517, 504, 509, 518, 506, 556, 496, 499, 481, 31, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 484, 505, 507, 498, 543, 517, 509, 506, 528, 513, 509, 523, 519, 507, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8097, 8156, 8318, 8163, 8181, 8213, 8199] avg=8192.0000 max=8318 max_rank=3 min=8097 min_rank=1 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32756 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[481, 532, 531, 494, 512, 529, 493, 493, 531, 523, 529, 516, 538, 533, 523, 534, 516, 484, 519, 495, 506, 480, 497, 495, 507, 515, 518, 506, 468, 516, 528, 486, 497, 523, 537, 537, 499, 499, 545, 544, 506, 517, 514, 555, 501, 523, 512, 515, 520, 523, 519, 504, 496, 523, 537, 503, 494, 544, 535, 532, 477, 485, 533, 487, 575, 504, 506, 549, 530, 527, 510, 499, 505, 501, 526, 501, 493, 467, 486, 497, 483, 497, 530, 501, 543, 466, 511, 561, 552, 516, 510, 503, 537, 517, 542, 494, 500, 547, 526, 550, 472, 499, 527, 521, 506, 455, 490, 525, 522, 504, 499, 452, 518, 523, 481, 510, 479, 489, 512, 512, 507, 522, 488, 528, 534, 489, 524, 522] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[504, 559, 516, 552, 535, 473, 514, 507, 488, 506, 542, 489, 521, 558, 513, 463, 532, 517, 510, 485, 537, 507, 540, 472, 508, 527, 504, 537, 478, 446, 494, 519, 509, 509, 520, 524, 528, 507, 527, 511, 529, 479, 509, 468, 520, 518, 478, 522, 523, 511, 504, 495, 525, 486, 557, 525, 510, 511, 519, 551, 505, 522, 539, 523, 535, 473, 514, 533, 499, 506, 500, 469, 525, 533, 540, 503, 527, 531, 467, 504, 508, 505, 512, 475, 516, 494, 525, 504, 512, 519, 536, 549, 496, 523, 500, 498, 529, 491, 475, 526, 515, 514, 538, 517, 504, 509, 518, 506, 556, 496, 499, 481, 524, 484, 505, 507, 498, 543, 532, 509, 506, 528, 513, 509, 523, 519, 507, 507] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[484, 514, 525, 527, 494, 475, 496, 503, 529, 494, 475, 507, 468, 515, 505, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 512, 525, 526, 500, 505, 541, 503, 539, 533, 519, 497, 543, 513, 499, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 523, 510, 518, 524, 493, 504, 493, 492, 565, 498, 526, 530, 505, 493, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 540, 521, 522, 495, 492, 532, 506, 546, 510, 536, 495, 526, 495, 502, 481, 12, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 533, 491, 498, 500, 546, 504, 508, 513, 504, 520, 525, 471, 501, 480, 491, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 515, 533, 538, 479, 510, 501, 485, 520, 504, 533, 528, 472, 517, 522, 521, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 514, 521, 523, 509, 494, 502, 484, 500, 513, 494, 540, 500, 501, 482, 485, 34, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 558, 516, 564, 551, 497, 527, 502, 497, 510, 524, 587, 479, 525, 523, 523, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8023, 8287, 8162, 8214, 8095, 8222, 8161, 8372] avg=8192.0000 max=8372 max_rank=7 min=8023 min_rank=0 imbalance_ratio=1.021973 max_load_reduction=57164 max_load_reduction_pct=0.872253 moved_to_echo=32850 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[455, 483, 541, 515, 535, 481, 532, 518, 502, 496, 495, 521, 552, 506, 516, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 558, 518, 535, 477, 548, 512, 517, 518, 487, 504, 516, 497, 514, 476, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 504, 497, 499, 511, 511, 554, 494, 449, 473, 492, 525, 486, 484, 525, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 481, 501, 498, 502, 498, 522, 488, 489, 520, 534, 470, 506, 524, 497, 507, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 521, 499, 505, 483, 532, 538, 528, 519, 536, 533, 516, 509, 518, 540, 482, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 529, 501, 506, 504, 511, 517, 499, 527, 561, 534, 562, 503, 499, 511, 507, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 475, 499, 530, 565, 544, 483, 476, 538, 550, 464, 484, 533, 523, 485, 526, 29, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 502, 477, 479, 472, 512, 590, 550, 500, 529, 519, 494, 496, 521, 547, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 8190, 8065, 8107, 8254, 8318, 8220, 8231] avg=8192.0000 max=8318 max_rank=5 min=8065 min_rank=2 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=33023 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[508, 526, 513, 510, 498, 520, 516, 523, 510, 534, 536, 512, 544, 487, 497, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 489, 529, 507, 513, 493, 510, 528, 526, 505, 490, 497, 511, 505, 485, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 564, 507, 541, 502, 508, 525, 509, 531, 537, 483, 474, 516, 512, 482, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 524, 507, 494, 507, 530, 531, 522, 504, 525, 494, 526, 535, 512, 551, 513, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 501, 472, 508, 513, 525, 471, 553, 521, 475, 545, 526, 528, 523, 533, 492, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 484, 554, 522, 553, 505, 514, 482, 474, 511, 510, 479, 562, 487, 479, 493, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 545, 494, 516, 527, 499, 543, 506, 506, 543, 518, 524, 493, 505, 507, 484, 28, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 515, 504, 538, 517, 504, 485, 469, 477, 515, 509, 504, 544, 554, 512, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 8159, 8211, 8270, 8174, 8119, 8235, 8138] avg=8192.0000 max=8270 max_rank=3 min=8119 min_rank=5 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32666 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[530, 525, 508, 508, 535, 532, 515, 512, 437, 488, 555, 531, 525, 530, 515, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 558, 523, 526, 476, 540, 522, 505, 546, 486, 510, 500, 527, 520, 500, 501, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 511, 508, 510, 487, 500, 530, 495, 528, 507, 528, 507, 515, 516, 533, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 511, 512, 524, 551, 541, 497, 514, 496, 480, 484, 546, 471, 479, 510, 508, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 516, 549, 503, 453, 491, 525, 483, 522, 516, 545, 516, 521, 514, 522, 509, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 490, 492, 472, 499, 499, 501, 518, 492, 510, 520, 514, 547, 490, 517, 531, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 529, 477, 521, 487, 472, 523, 547, 477, 495, 518, 501, 503, 508, 498, 549, 31, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 497, 524, 515, 514, 525, 495, 520, 508, 495, 484, 544, 500, 480, 485, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8285, 8241, 8245, 8130, 8192, 8139, 8156, 8148] avg=8192.0000 max=8285 max_rank=0 min=8130 min_rank=3 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32635 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[481, 532, 531, 494, 512, 529, 493, 493, 531, 523, 529, 516, 538, 504, 523, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 484, 519, 495, 506, 466, 497, 495, 507, 515, 518, 506, 468, 516, 528, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 523, 537, 537, 499, 499, 543, 544, 506, 517, 514, 555, 501, 523, 512, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 523, 519, 504, 496, 523, 537, 503, 494, 544, 535, 532, 477, 485, 533, 487, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 575, 504, 506, 549, 530, 527, 510, 499, 505, 501, 526, 501, 493, 467, 486, 497, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 497, 530, 501, 543, 466, 511, 561, 552, 516, 510, 503, 537, 517, 542, 494, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 547, 526, 550, 472, 499, 527, 521, 506, 455, 490, 525, 522, 504, 499, 452, 29, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 523, 481, 510, 479, 489, 497, 512, 507, 522, 488, 528, 534, 489, 524, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8263, 8022, 8322, 8223, 8180, 8272, 8131, 8123] avg=8192.0000 max=8322 max_rank=2 min=8022 min_rank=1 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32706 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[543, 515, 528, 536, 509, 507, 517, 554, 521, 489, 495, 519, 520, 478, 495, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 476, 490, 519, 521, 505, 485, 569, 561, 536, 508, 483, 470, 548, 517, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 504, 539, 499, 515, 524, 504, 487, 509, 524, 552, 505, 530, 487, 520, 465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 490, 513, 490, 536, 489, 482, 531, 513, 535, 514, 490, 476, 533, 534, 540, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 488, 505, 509, 510, 466, 539, 493, 528, 507, 514, 496, 543, 498, 542, 526, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 526, 515, 530, 502, 516, 503, 554, 490, 496, 535, 516, 522, 465, 546, 480, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 504, 522, 502, 492, 492, 530, 532, 525, 535, 496, 485, 526, 518, 542, 547, 28, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 462, 538, 513, 480, 541, 465, 506, 535, 507, 506, 529, 489, 517, 522, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8215, 8207, 8159, 8183, 8195, 8207, 8262, 8108] avg=8192.0000 max=8262 max_rank=6 min=8108 min_rank=7 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32772 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[481, 532, 531, 494, 512, 529, 493, 493, 531, 523, 529, 516, 538, 504, 523, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 484, 519, 495, 506, 466, 497, 495, 507, 515, 518, 506, 468, 516, 528, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 523, 537, 537, 499, 499, 543, 544, 506, 517, 514, 555, 501, 523, 512, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 523, 519, 504, 496, 523, 537, 503, 494, 544, 535, 532, 477, 485, 533, 487, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 575, 504, 506, 549, 530, 527, 510, 499, 505, 501, 526, 501, 493, 467, 486, 497, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 497, 530, 501, 543, 466, 511, 561, 552, 516, 510, 503, 537, 517, 542, 494, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 547, 526, 550, 472, 499, 527, 521, 506, 455, 490, 525, 522, 504, 499, 452, 29, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 523, 481, 510, 479, 489, 497, 512, 507, 522, 488, 528, 534, 489, 524, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8263, 8022, 8322, 8223, 8180, 8272, 8131, 8123] avg=8192.0000 max=8322 max_rank=2 min=8022 min_rank=1 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32706 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[520, 458, 516, 513, 523, 559, 548, 481, 480, 484, 526, 511, 503, 503, 498, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 544, 516, 517, 508, 530, 536, 542, 528, 544, 517, 526, 512, 507, 478, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 493, 529, 494, 510, 531, 509, 494, 498, 535, 509, 490, 538, 513, 532, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 477, 442, 507, 504, 481, 525, 521, 532, 490, 495, 537, 532, 527, 512, 484, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 511, 526, 518, 498, 566, 518, 503, 503, 491, 547, 521, 532, 500, 475, 525, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 484, 495, 511, 487, 548, 488, 518, 516, 507, 501, 480, 555, 462, 492, 506, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 488, 550, 539, 536, 486, 469, 509, 488, 509, 490, 504, 513, 513, 542, 504, 29, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 520, 484, 522, 490, 503, 506, 449, 530, 506, 555, 514, 541, 543, 543, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8160, 8333, 8216, 8091, 8283, 8078, 8158, 8217] avg=8192.0000 max=8333 max_rank=1 min=8078 min_rank=5 imbalance_ratio=1.017212 max_load_reduction=57203 max_load_reduction_pct=0.872849 moved_to_echo=32736 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[504, 559, 516, 552, 535, 473, 514, 507, 488, 506, 542, 489, 521, 527, 513, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 517, 510, 485, 537, 491, 540, 472, 508, 527, 504, 537, 478, 446, 494, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 509, 520, 524, 528, 507, 525, 511, 529, 479, 509, 468, 520, 518, 478, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 511, 504, 495, 525, 486, 557, 525, 510, 511, 519, 551, 505, 522, 539, 523, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 473, 514, 533, 499, 506, 500, 469, 525, 533, 540, 503, 527, 531, 467, 504, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 505, 512, 475, 516, 494, 525, 504, 512, 519, 536, 549, 496, 523, 500, 498, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 491, 475, 526, 515, 514, 538, 517, 504, 509, 518, 506, 556, 496, 499, 481, 31, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 484, 505, 507, 498, 543, 517, 509, 506, 528, 513, 509, 523, 519, 507, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8097, 8156, 8318, 8163, 8181, 8213, 8199] avg=8192.0000 max=8318 max_rank=3 min=8097 min_rank=1 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32756 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 3968, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4224, 3968, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4352, 4224, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 3968, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4224, 3968, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True - [2026-08-26 14:49:30] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 418.7 | throughput per GPU (TFLOP/s/GPU): 431.9 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.522467E+00 | loss scale: 1.0 | grad norm: 5.546 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[508, 515, 528, 495, 518, 513, 496, 519, 527, 552, 513, 555, 516, 560, 487, 550, 526, 476, 504, 549, 531, 494, 519, 527, 508, 533, 488, 481, 511, 503, 523, 506, 517, 543, 473, 513, 516, 498, 513, 528, 546, 553, 498, 493, 499, 510, 533, 516, 528, 479, 505, 521, 443, 473, 512, 507, 539, 498, 493, 480, 498, 511, 510, 463, 514, 518, 492, 479, 509, 462, 481, 497, 512, 516, 509, 535, 526, 511, 499, 535, 558, 532, 489, 505, 542, 533, 495, 473, 524, 519, 502, 524, 538, 572, 520, 482, 486, 507, 486, 520, 549, 526, 508, 502, 504, 517, 505, 514, 518, 509, 544, 497, 521, 539, 518, 524, 504, 554, 554, 547, 457, 498, 496, 524, 486, 509, 478, 492] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[494, 511, 535, 507, 524, 522, 501, 517, 482, 498, 522, 508, 534, 522, 470, 527, 533, 493, 534, 521, 495, 487, 519, 526, 491, 525, 484, 480, 521, 484, 520, 475, 481, 491, 505, 537, 514, 536, 492, 529, 512, 518, 488, 507, 517, 490, 533, 511, 512, 515, 535, 531, 522, 510, 535, 546, 540, 531, 520, 507, 533, 496, 539, 526, 525, 479, 524, 486, 520, 508, 508, 517, 520, 494, 480, 502, 567, 529, 491, 522, 523, 495, 514, 506, 521, 492, 494, 512, 510, 478, 488, 487, 543, 512, 519, 516, 523, 540, 502, 512, 514, 531, 534, 486, 551, 503, 498, 515, 522, 535, 484, 492, 467, 484, 536, 491, 522, 508, 519, 529, 475, 526, 521, 534, 501, 559, 513, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[478, 532, 502, 534, 446, 495, 540, 515, 498, 519, 526, 490, 515, 536, 469, 519, 478, 492, 526, 513, 527, 510, 521, 521, 517, 526, 483, 525, 531, 481, 526, 500, 495, 478, 503, 542, 491, 507, 497, 508, 510, 541, 490, 524, 504, 536, 545, 488, 511, 533, 534, 531, 468, 497, 488, 544, 545, 541, 486, 541, 531, 509, 500, 510, 491, 527, 576, 534, 468, 544, 513, 514, 545, 436, 493, 497, 525, 491, 532, 510, 505, 481, 537, 517, 501, 511, 481, 513, 513, 513, 476, 518, 492, 521, 518, 491, 512, 497, 506, 493, 522, 534, 499, 522, 513, 540, 502, 503, 488, 553, 518, 471, 500, 539, 504, 508, 594, 540, 503, 543, 500, 520, 537, 513, 525, 531, 498, 505] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[514, 542, 481, 532, 502, 507, 504, 521, 481, 499, 543, 461, 504, 526, 543, 500, 494, 484, 491, 510, 514, 509, 514, 480, 567, 506, 546, 519, 554, 454, 557, 498, 544, 501, 525, 499, 502, 502, 534, 497, 515, 479, 523, 506, 508, 510, 487, 480, 505, 514, 521, 503, 545, 513, 528, 460, 558, 544, 521, 465, 526, 482, 507, 499, 486, 521, 484, 504, 553, 521, 491, 534, 517, 531, 504, 524, 505, 491, 513, 558, 512, 527, 503, 495, 486, 518, 523, 527, 527, 532, 540, 508, 498, 541, 495, 504, 501, 518, 554, 514, 543, 507, 495, 557, 557, 526, 525, 509, 531, 499, 474, 488, 495, 525, 490, 496, 527, 482, 505, 531, 540, 504, 486, 501, 476, 516, 490, 541] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[503, 530, 516, 508, 489, 485, 498, 499, 493, 536, 529, 489, 540, 458, 512, 491, 520, 528, 512, 484, 567, 527, 445, 520, 504, 543, 467, 513, 532, 502, 508, 538, 497, 526, 481, 489, 500, 502, 528, 490, 508, 530, 510, 543, 496, 490, 521, 512, 528, 525, 529, 465, 510, 562, 514, 523, 501, 527, 551, 498, 527, 496, 505, 550, 526, 502, 484, 531, 537, 521, 493, 515, 512, 515, 526, 515, 484, 572, 553, 552, 505, 521, 523, 507, 521, 482, 512, 539, 514, 499, 488, 536, 517, 512, 514, 468, 525, 475, 521, 523, 493, 476, 544, 489, 511, 529, 503, 511, 490, 504, 543, 544, 498, 535, 487, 509, 506, 531, 509, 498, 496, 482, 513, 511, 551, 495, 520, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True + [2026-08-26 15:25:33] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 416.2 | throughput per GPU (TFLOP/s/GPU): 434.5 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.532578E+00 | loss scale: 1.0 | grad norm: 5.547 | number of skipped iterations: 0 | number of nan iterations: 0 | WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[559, 494, 540, 541, 538, 461, 477, 524, 511, 516, 507, 491, 475, 514, 547, 551, 555, 503, 469, 532, 526, 515, 536, 481, 465, 526, 510, 531, 547, 522, 504, 527, 521, 508, 505, 516, 487, 542, 513, 527, 516, 482, 542, 552, 472, 516, 512, 468, 509, 550, 511, 539, 497, 498, 515, 513, 533, 513, 530, 485, 540, 491, 518, 526, 491, 512, 525, 519, 514, 527, 512, 493, 498, 510, 484, 504, 522, 513, 511, 486, 520, 506, 515, 487, 482, 499, 540, 518, 506, 508, 475, 511, 490, 522, 493, 535, 524, 547, 527, 497, 556, 496, 526, 480, 499, 529, 482, 500, 514, 564, 492, 483, 514, 513, 523, 511, 478, 519, 532, 492, 524, 495, 505, 529, 473, 506, 506, 530] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[503, 530, 516, 508, 489, 485, 498, 499, 493, 536, 529, 489, 540, 458, 512, 491, 520, 528, 512, 484, 567, 527, 445, 520, 504, 543, 467, 513, 532, 502, 508, 538, 497, 526, 481, 489, 500, 502, 528, 490, 508, 530, 510, 543, 496, 490, 521, 512, 528, 525, 529, 465, 510, 562, 514, 523, 501, 527, 551, 498, 527, 496, 505, 550, 526, 502, 484, 531, 537, 521, 493, 515, 512, 515, 526, 515, 484, 572, 553, 552, 505, 521, 523, 507, 521, 482, 512, 539, 514, 499, 488, 536, 517, 512, 514, 468, 525, 475, 521, 523, 493, 476, 544, 489, 511, 529, 503, 511, 490, 504, 543, 544, 498, 535, 487, 509, 506, 531, 509, 498, 496, 482, 513, 511, 551, 495, 520, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[509, 500, 531, 515, 500, 521, 521, 521, 486, 478, 514, 481, 508, 509, 527, 516, 502, 497, 520, 511, 520, 510, 496, 524, 527, 522, 551, 489, 513, 510, 509, 526, 498, 502, 504, 520, 506, 533, 524, 501, 515, 531, 552, 522, 499, 510, 470, 514, 519, 506, 485, 475, 515, 538, 510, 505, 512, 507, 496, 520, 517, 520, 528, 555, 500, 506, 505, 492, 505, 493, 538, 525, 497, 539, 508, 485, 508, 520, 480, 529, 494, 480, 484, 543, 513, 530, 505, 529, 507, 511, 507, 531, 509, 548, 477, 521, 537, 512, 499, 513, 498, 481, 508, 504, 538, 512, 495, 544, 489, 472, 513, 518, 544, 505, 532, 530, 536, 482, 511, 512, 524, 510, 536, 513, 514, 522, 525, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[514, 542, 481, 532, 502, 507, 504, 521, 481, 499, 543, 461, 504, 526, 543, 500, 494, 484, 491, 510, 514, 509, 514, 480, 567, 506, 546, 519, 554, 454, 557, 498, 544, 501, 525, 499, 502, 502, 534, 497, 515, 479, 523, 506, 508, 510, 487, 480, 505, 514, 521, 503, 545, 513, 528, 460, 558, 544, 521, 465, 526, 482, 507, 499, 486, 521, 484, 504, 553, 521, 491, 534, 517, 531, 504, 524, 505, 491, 513, 558, 512, 527, 503, 495, 486, 518, 523, 527, 527, 532, 540, 508, 498, 541, 495, 504, 501, 518, 554, 514, 543, 507, 495, 557, 557, 526, 525, 509, 531, 499, 474, 488, 495, 525, 490, 496, 527, 482, 505, 531, 540, 504, 486, 501, 476, 516, 490, 541] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[508, 515, 528, 495, 518, 513, 496, 519, 527, 552, 513, 555, 516, 560, 487, 550, 526, 476, 504, 549, 531, 494, 519, 527, 508, 533, 488, 481, 511, 503, 523, 506, 517, 543, 473, 513, 516, 498, 513, 528, 546, 553, 498, 493, 499, 510, 533, 516, 528, 479, 505, 521, 443, 473, 512, 507, 539, 498, 493, 480, 498, 511, 510, 463, 514, 518, 492, 479, 509, 462, 481, 497, 512, 516, 509, 535, 526, 511, 499, 535, 558, 532, 489, 505, 542, 533, 495, 473, 524, 519, 502, 524, 538, 572, 520, 482, 486, 507, 486, 520, 549, 526, 508, 502, 504, 517, 505, 514, 518, 509, 544, 497, 521, 539, 518, 524, 504, 554, 554, 547, 457, 498, 496, 524, 486, 509, 478, 492] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[526, 508, 509, 510, 549, 514, 504, 490, 520, 519, 555, 497, 444, 534, 515, 515, 535, 499, 509, 464, 512, 528, 473, 514, 489, 519, 495, 509, 493, 531, 533, 506, 471, 515, 503, 526, 537, 548, 544, 518, 519, 546, 533, 519, 511, 495, 521, 509, 511, 483, 519, 503, 530, 473, 489, 522, 531, 482, 494, 516, 510, 533, 511, 525, 508, 529, 571, 513, 508, 518, 502, 539, 540, 525, 479, 547, 531, 518, 537, 531, 514, 522, 495, 514, 493, 524, 534, 505, 514, 509, 518, 474, 491, 485, 512, 509, 487, 465, 510, 488, 504, 472, 527, 546, 495, 526, 512, 496, 541, 534, 492, 499, 496, 528, 513, 532, 493, 507, 525, 513, 495, 522, 492, 503, 552, 498, 473, 526] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[508, 515, 528, 495, 518, 513, 496, 519, 527, 552, 513, 555, 516, 560, 487, 550, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 476, 504, 549, 531, 494, 519, 527, 508, 533, 488, 481, 511, 503, 523, 506, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 543, 473, 513, 516, 498, 513, 528, 546, 553, 498, 493, 499, 510, 533, 516, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 479, 505, 521, 443, 473, 512, 507, 514, 498, 493, 480, 498, 511, 510, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 518, 492, 479, 509, 462, 481, 497, 512, 516, 509, 535, 526, 511, 499, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 558, 532, 489, 505, 542, 533, 495, 473, 524, 519, 502, 524, 538, 572, 520, 482, 25, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 507, 486, 520, 549, 526, 508, 502, 504, 517, 505, 514, 518, 509, 544, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 530, 518, 524, 504, 554, 554, 547, 457, 498, 496, 524, 486, 509, 478, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8359, 8191, 8253, 7935, 8078, 8336, 8192, 8192] avg=8192.0000 max=8359 max_rank=0 min=7935 min_rank=3 imbalance_ratio=1.020386 max_load_reduction=57177 max_load_reduction_pct=0.872452 moved_to_echo=32798 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[494, 511, 535, 507, 524, 522, 501, 517, 482, 498, 522, 508, 534, 522, 470, 527, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 493, 534, 521, 495, 487, 519, 526, 491, 525, 484, 480, 521, 484, 520, 475, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 491, 505, 537, 514, 536, 492, 529, 512, 518, 488, 507, 517, 490, 533, 511, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 515, 535, 531, 522, 510, 535, 546, 515, 531, 520, 507, 533, 496, 539, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 479, 524, 486, 520, 508, 508, 517, 520, 494, 480, 502, 567, 529, 491, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 495, 514, 506, 521, 492, 494, 512, 510, 478, 488, 487, 543, 512, 519, 516, 25, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 540, 502, 512, 514, 531, 534, 486, 551, 503, 498, 515, 522, 535, 484, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 475, 536, 491, 522, 508, 519, 529, 475, 526, 521, 534, 501, 559, 513, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8100, 8165, 8373, 8155, 8138, 8242, 8182] avg=8192.0000 max=8373 max_rank=3 min=8100 min_rank=1 imbalance_ratio=1.022095 max_load_reduction=57163 max_load_reduction_pct=0.872238 moved_to_echo=32717 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[514, 542, 481, 532, 502, 507, 504, 521, 481, 499, 543, 461, 504, 526, 543, 500, 10, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 484, 491, 510, 514, 509, 514, 480, 567, 506, 546, 519, 554, 454, 557, 498, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 501, 525, 499, 502, 502, 534, 497, 515, 479, 523, 506, 508, 510, 487, 480, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 514, 521, 503, 545, 513, 528, 460, 529, 544, 521, 465, 526, 482, 507, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 521, 484, 504, 553, 521, 491, 534, 517, 531, 504, 524, 505, 491, 513, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 527, 503, 495, 486, 518, 523, 527, 527, 532, 540, 508, 498, 541, 495, 504, 29, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 518, 554, 514, 543, 507, 495, 557, 557, 526, 525, 509, 531, 499, 474, 488, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 499, 490, 496, 527, 482, 505, 531, 540, 504, 486, 501, 476, 516, 490, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8213, 8121, 8162, 8207, 8271, 8305, 8079] avg=8192.0000 max=8305 max_rank=6 min=8079 min_rank=7 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32862 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[478, 532, 502, 534, 446, 495, 540, 515, 498, 519, 526, 490, 515, 536, 469, 519, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 492, 526, 513, 527, 510, 521, 521, 517, 526, 483, 525, 531, 481, 526, 500, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 478, 503, 542, 491, 507, 497, 508, 510, 541, 490, 524, 504, 536, 545, 488, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 533, 534, 531, 468, 497, 488, 544, 520, 541, 486, 541, 531, 509, 500, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 527, 576, 534, 468, 544, 513, 514, 545, 436, 493, 497, 525, 491, 532, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 481, 537, 517, 501, 511, 481, 513, 513, 513, 476, 518, 492, 521, 518, 491, 25, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 497, 506, 493, 522, 534, 499, 522, 513, 540, 502, 503, 488, 553, 518, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 530, 504, 508, 594, 540, 503, 543, 500, 520, 537, 513, 525, 531, 498, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8121, 8189, 8163, 8244, 8179, 8116, 8173, 8351] avg=8192.0000 max=8351 max_rank=7 min=8116 min_rank=5 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32819 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[494, 511, 535, 507, 524, 522, 501, 517, 482, 498, 522, 508, 534, 522, 470, 527, 533, 493, 534, 521, 495, 487, 519, 526, 491, 525, 484, 480, 521, 484, 520, 475, 481, 491, 505, 537, 514, 536, 492, 529, 512, 518, 488, 507, 517, 490, 533, 511, 512, 515, 535, 531, 522, 510, 535, 546, 540, 531, 520, 507, 533, 496, 539, 526, 525, 479, 524, 486, 520, 508, 508, 517, 520, 494, 480, 502, 567, 529, 491, 522, 523, 495, 514, 506, 521, 492, 494, 512, 510, 478, 488, 487, 543, 512, 519, 516, 523, 540, 502, 512, 514, 531, 534, 486, 551, 503, 498, 515, 522, 535, 484, 492, 467, 484, 536, 491, 522, 508, 519, 529, 475, 526, 521, 534, 501, 559, 513, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[478, 532, 502, 534, 446, 495, 540, 515, 498, 519, 526, 490, 515, 536, 469, 519, 478, 492, 526, 513, 527, 510, 521, 521, 517, 526, 483, 525, 531, 481, 526, 500, 495, 478, 503, 542, 491, 507, 497, 508, 510, 541, 490, 524, 504, 536, 545, 488, 511, 533, 534, 531, 468, 497, 488, 544, 545, 541, 486, 541, 531, 509, 500, 510, 491, 527, 576, 534, 468, 544, 513, 514, 545, 436, 493, 497, 525, 491, 532, 510, 505, 481, 537, 517, 501, 511, 481, 513, 513, 513, 476, 518, 492, 521, 518, 491, 512, 497, 506, 493, 522, 534, 499, 522, 513, 540, 502, 503, 488, 553, 518, 471, 500, 539, 504, 508, 594, 540, 503, 543, 500, 520, 537, 513, 525, 531, 498, 505] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[503, 530, 516, 508, 489, 485, 498, 499, 493, 536, 529, 489, 540, 458, 512, 491, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 528, 512, 484, 567, 527, 445, 520, 504, 543, 467, 513, 532, 502, 508, 538, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 526, 481, 489, 500, 502, 528, 490, 508, 530, 510, 543, 496, 490, 521, 512, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 525, 529, 465, 510, 562, 514, 523, 478, 527, 551, 498, 527, 496, 505, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 502, 484, 531, 537, 521, 493, 515, 512, 515, 526, 515, 484, 572, 553, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 521, 523, 507, 521, 482, 512, 539, 514, 499, 488, 536, 517, 512, 514, 468, 23, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 475, 521, 523, 493, 476, 544, 489, 511, 529, 503, 511, 490, 504, 543, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 526, 487, 509, 506, 531, 509, 498, 496, 482, 513, 511, 551, 495, 520, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8084, 8223, 8127, 8288, 8319, 8184, 8181, 8130] avg=8192.0000 max=8319 max_rank=4 min=8084 min_rank=0 imbalance_ratio=1.015503 max_load_reduction=57217 max_load_reduction_pct=0.873062 moved_to_echo=32814 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[509, 500, 531, 515, 500, 521, 521, 521, 486, 478, 514, 481, 508, 509, 527, 516, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 497, 520, 511, 520, 510, 496, 524, 527, 522, 551, 489, 513, 510, 509, 526, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 502, 504, 520, 506, 533, 524, 501, 515, 531, 552, 522, 499, 510, 470, 514, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 506, 485, 475, 515, 538, 510, 505, 488, 507, 496, 520, 517, 520, 528, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 506, 505, 492, 505, 493, 538, 525, 497, 539, 508, 485, 508, 520, 480, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 480, 484, 543, 513, 530, 505, 529, 507, 511, 507, 531, 509, 548, 477, 521, 24, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 512, 499, 513, 498, 481, 508, 504, 538, 512, 495, 544, 489, 472, 513, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 496, 532, 530, 536, 482, 511, 512, 524, 510, 536, 513, 514, 522, 525, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8144, 8239, 8205, 8184, 8113, 8216, 8133, 8302] avg=8192.0000 max=8302 max_rank=7 min=8113 min_rank=4 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32764 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[559, 494, 540, 541, 538, 461, 477, 524, 511, 516, 507, 491, 475, 514, 547, 551, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 503, 469, 532, 526, 515, 536, 481, 465, 526, 510, 531, 547, 522, 504, 527, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 508, 505, 516, 487, 542, 513, 527, 516, 482, 542, 552, 472, 516, 512, 468, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 550, 511, 539, 497, 498, 515, 513, 508, 513, 530, 485, 540, 491, 518, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 512, 525, 519, 514, 527, 512, 493, 498, 510, 484, 504, 522, 513, 511, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 506, 515, 487, 482, 499, 540, 518, 506, 508, 475, 511, 490, 522, 493, 535, 25, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 547, 527, 497, 556, 496, 526, 480, 499, 529, 482, 500, 514, 564, 492, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 504, 523, 511, 478, 519, 532, 492, 524, 495, 505, 529, 473, 506, 506, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8253, 8260, 8183, 8243, 8106, 8134, 8216, 8141] avg=8192.0000 max=8260 max_rank=1 min=8106 min_rank=4 imbalance_ratio=1.008301 max_load_reduction=57276 max_load_reduction_pct=0.873962 moved_to_echo=32597 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[509, 500, 531, 515, 500, 521, 521, 521, 486, 478, 514, 481, 508, 509, 527, 516, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 497, 520, 511, 520, 510, 496, 524, 527, 522, 551, 489, 513, 510, 509, 526, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 502, 504, 520, 506, 533, 524, 501, 515, 531, 552, 522, 499, 510, 470, 514, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 506, 485, 475, 515, 538, 510, 505, 488, 507, 496, 520, 517, 520, 528, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 506, 505, 492, 505, 493, 538, 525, 497, 539, 508, 485, 508, 520, 480, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 480, 484, 543, 513, 530, 505, 529, 507, 511, 507, 531, 509, 548, 477, 521, 24, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 512, 499, 513, 498, 481, 508, 504, 538, 512, 495, 544, 489, 472, 513, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 496, 532, 530, 536, 482, 511, 512, 524, 510, 536, 513, 514, 522, 525, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8144, 8239, 8205, 8184, 8113, 8216, 8133, 8302] avg=8192.0000 max=8302 max_rank=7 min=8113 min_rank=4 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32764 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[508, 515, 528, 495, 518, 513, 496, 519, 527, 552, 513, 555, 516, 560, 487, 550, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 476, 504, 549, 531, 494, 519, 527, 508, 533, 488, 481, 511, 503, 523, 506, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 543, 473, 513, 516, 498, 513, 528, 546, 553, 498, 493, 499, 510, 533, 516, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 479, 505, 521, 443, 473, 512, 507, 514, 498, 493, 480, 498, 511, 510, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 518, 492, 479, 509, 462, 481, 497, 512, 516, 509, 535, 526, 511, 499, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 558, 532, 489, 505, 542, 533, 495, 473, 524, 519, 502, 524, 538, 572, 520, 482, 25, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 507, 486, 520, 549, 526, 508, 502, 504, 517, 505, 514, 518, 509, 544, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 530, 518, 524, 504, 554, 554, 547, 457, 498, 496, 524, 486, 509, 478, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8359, 8191, 8253, 7935, 8078, 8336, 8192, 8192] avg=8192.0000 max=8359 max_rank=0 min=7935 min_rank=3 imbalance_ratio=1.020386 max_load_reduction=57177 max_load_reduction_pct=0.872452 moved_to_echo=32798 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[526, 508, 509, 510, 549, 514, 504, 490, 520, 519, 555, 497, 444, 534, 515, 515, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 499, 509, 464, 512, 528, 473, 514, 489, 519, 495, 509, 493, 531, 533, 506, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 515, 503, 526, 537, 548, 544, 518, 519, 546, 533, 519, 511, 495, 521, 509, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 483, 519, 503, 530, 473, 489, 522, 506, 482, 494, 516, 510, 533, 511, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 529, 571, 513, 508, 518, 502, 539, 540, 525, 479, 547, 531, 518, 537, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 522, 495, 514, 493, 524, 534, 505, 514, 509, 518, 474, 491, 485, 512, 509, 25, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 465, 510, 488, 504, 472, 527, 546, 495, 526, 512, 496, 541, 534, 492, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 519, 513, 532, 493, 507, 525, 513, 495, 522, 492, 503, 552, 498, 473, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8121, 8319, 8107, 8379, 8141, 8094, 8159] avg=8192.0000 max=8379 max_rank=4 min=8094 min_rank=6 imbalance_ratio=1.022827 max_load_reduction=57157 max_load_reduction_pct=0.872147 moved_to_echo=32773 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[494, 511, 535, 507, 524, 522, 501, 517, 482, 498, 522, 508, 534, 522, 470, 527, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 493, 534, 521, 495, 487, 519, 526, 491, 525, 484, 480, 521, 484, 520, 475, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 491, 505, 537, 514, 536, 492, 529, 512, 518, 488, 507, 517, 490, 533, 511, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 515, 535, 531, 522, 510, 535, 546, 515, 531, 520, 507, 533, 496, 539, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 479, 524, 486, 520, 508, 508, 517, 520, 494, 480, 502, 567, 529, 491, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 495, 514, 506, 521, 492, 494, 512, 510, 478, 488, 487, 543, 512, 519, 516, 25, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 540, 502, 512, 514, 531, 534, 486, 551, 503, 498, 515, 522, 535, 484, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 475, 536, 491, 522, 508, 519, 529, 475, 526, 521, 534, 501, 559, 513, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8100, 8165, 8373, 8155, 8138, 8242, 8182] avg=8192.0000 max=8373 max_rank=3 min=8100 min_rank=1 imbalance_ratio=1.022095 max_load_reduction=57163 max_load_reduction_pct=0.872238 moved_to_echo=32717 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[478, 532, 502, 534, 446, 495, 540, 515, 498, 519, 526, 490, 515, 536, 469, 519, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 492, 526, 513, 527, 510, 521, 521, 517, 526, 483, 525, 531, 481, 526, 500, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 478, 503, 542, 491, 507, 497, 508, 510, 541, 490, 524, 504, 536, 545, 488, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 533, 534, 531, 468, 497, 488, 544, 520, 541, 486, 541, 531, 509, 500, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 527, 576, 534, 468, 544, 513, 514, 545, 436, 493, 497, 525, 491, 532, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 481, 537, 517, 501, 511, 481, 513, 513, 513, 476, 518, 492, 521, 518, 491, 25, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 497, 506, 493, 522, 534, 499, 522, 513, 540, 502, 503, 488, 553, 518, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 530, 504, 508, 594, 540, 503, 543, 500, 520, 537, 513, 525, 531, 498, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8121, 8189, 8163, 8244, 8179, 8116, 8173, 8351] avg=8192.0000 max=8351 max_rank=7 min=8116 min_rank=5 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32819 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[514, 542, 481, 532, 502, 507, 504, 521, 481, 499, 543, 461, 504, 526, 543, 500, 10, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 484, 491, 510, 514, 509, 514, 480, 567, 506, 546, 519, 554, 454, 557, 498, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 501, 525, 499, 502, 502, 534, 497, 515, 479, 523, 506, 508, 510, 487, 480, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 514, 521, 503, 545, 513, 528, 460, 529, 544, 521, 465, 526, 482, 507, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 521, 484, 504, 553, 521, 491, 534, 517, 531, 504, 524, 505, 491, 513, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 527, 503, 495, 486, 518, 523, 527, 527, 532, 540, 508, 498, 541, 495, 504, 29, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 518, 554, 514, 543, 507, 495, 557, 557, 526, 525, 509, 531, 499, 474, 488, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 499, 490, 496, 527, 482, 505, 531, 540, 504, 486, 501, 476, 516, 490, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8213, 8121, 8162, 8207, 8271, 8305, 8079] avg=8192.0000 max=8305 max_rank=6 min=8079 min_rank=7 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32862 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[492, 517, 524, 508, 532, 533, 522, 486, 497, 504, 536, 529, 515, 501, 509, 509, 497, 485, 536, 529, 491, 522, 492, 519, 487, 509, 528, 508, 530, 511, 517, 491, 520, 512, 502, 472, 509, 588, 516, 504, 511, 520, 469, 491, 493, 518, 506, 468, 486, 470, 506, 507, 500, 552, 513, 465, 521, 523, 488, 514, 538, 506, 501, 530, 513, 511, 534, 462, 513, 498, 560, 502, 520, 475, 561, 510, 511, 532, 554, 509, 500, 497, 497, 509, 556, 514, 483, 493, 532, 520, 548, 508, 490, 532, 571, 544, 529, 480, 514, 532, 509, 499, 533, 564, 517, 526, 518, 557, 496, 549, 503, 503, 467, 522, 497, 507, 498, 514, 472, 489, 549, 485, 519, 522, 502, 475, 530, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[513, 548, 523, 495, 505, 524, 497, 503, 548, 496, 510, 488, 535, 476, 530, 523, 473, 529, 494, 489, 507, 524, 495, 549, 522, 495, 519, 544, 499, 525, 470, 511, 545, 507, 502, 493, 505, 502, 523, 488, 516, 487, 508, 518, 505, 528, 493, 521, 556, 477, 505, 551, 509, 489, 502, 505, 498, 466, 561, 485, 539, 527, 523, 538, 509, 532, 491, 517, 526, 517, 519, 486, 564, 555, 504, 544, 547, 490, 498, 494, 504, 516, 527, 473, 482, 499, 484, 508, 519, 524, 472, 510, 500, 531, 516, 527, 537, 518, 485, 502, 525, 519, 513, 511, 488, 517, 520, 531, 521, 517, 511, 542, 543, 470, 502, 538, 539, 491, 498, 512, 521, 482, 490, 487, 551, 509, 524, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[538, 516, 493, 530, 494, 531, 500, 531, 541, 509, 488, 521, 496, 482, 513, 511, 529, 505, 537, 505, 502, 478, 544, 509, 508, 538, 478, 541, 543, 530, 517, 499, 494, 521, 469, 491, 480, 522, 515, 509, 517, 506, 520, 510, 532, 529, 491, 532, 510, 524, 533, 465, 518, 490, 531, 528, 534, 518, 491, 529, 524, 515, 524, 512, 531, 514, 512, 490, 489, 495, 489, 521, 489, 511, 516, 567, 506, 495, 494, 495, 507, 509, 519, 520, 547, 494, 484, 476, 507, 500, 512, 518, 494, 524, 477, 516, 504, 513, 476, 547, 517, 524, 519, 437, 523, 514, 521, 514, 470, 486, 543, 527, 486, 548, 520, 521, 518, 490, 558, 518, 489, 536, 530, 506, 538, 512, 536, 536] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[515, 507, 526, 522, 510, 546, 518, 500, 514, 497, 545, 536, 504, 539, 500, 478, 486, 494, 486, 546, 503, 473, 550, 551, 525, 533, 522, 539, 486, 496, 518, 484, 529, 528, 495, 533, 533, 500, 521, 500, 489, 494, 513, 471, 519, 481, 544, 532, 557, 526, 498, 503, 547, 515, 497, 543, 531, 526, 515, 516, 502, 508, 494, 494, 517, 525, 497, 500, 497, 536, 480, 485, 504, 513, 495, 505, 491, 561, 535, 584, 477, 495, 556, 499, 518, 511, 476, 511, 541, 499, 483, 538, 503, 475, 524, 506, 531, 529, 526, 538, 478, 516, 422, 498, 509, 479, 499, 506, 505, 536, 555, 523, 518, 512, 509, 541, 488, 512, 489, 522, 521, 486, 525, 517, 516, 479, 484, 527] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[509, 533, 537, 527, 556, 512, 472, 516, 487, 522, 494, 488, 523, 526, 536, 492, 526, 507, 498, 550, 542, 540, 475, 491, 509, 458, 504, 500, 534, 558, 551, 504, 504, 506, 530, 535, 512, 501, 503, 466, 503, 485, 463, 515, 524, 525, 514, 481, 516, 501, 516, 526, 514, 506, 542, 500, 521, 549, 532, 491, 480, 521, 475, 511, 490, 482, 537, 498, 511, 501, 503, 511, 464, 511, 491, 501, 529, 485, 516, 513, 545, 524, 530, 553, 504, 547, 515, 554, 536, 484, 488, 507, 525, 499, 528, 553, 493, 513, 486, 466, 520, 530, 503, 501, 502, 474, 520, 506, 553, 518, 518, 486, 506, 509, 536, 528, 504, 542, 544, 511, 523, 530, 527, 523, 480, 500, 522, 482] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[508, 520, 542, 524, 514, 532, 512, 501, 486, 490, 521, 490, 511, 508, 491, 496, 535, 511, 511, 496, 475, 550, 542, 522, 533, 484, 540, 520, 467, 515, 482, 510, 532, 536, 533, 521, 522, 553, 526, 511, 485, 525, 542, 484, 504, 522, 551, 479, 500, 508, 486, 508, 501, 494, 520, 510, 512, 515, 541, 471, 497, 564, 510, 490, 496, 519, 523, 491, 528, 511, 496, 521, 560, 479, 515, 498, 500, 552, 530, 493, 530, 492, 498, 505, 505, 518, 477, 482, 503, 486, 510, 533, 520, 549, 539, 499, 524, 484, 473, 513, 509, 508, 521, 500, 559, 505, 546, 508, 538, 545, 497, 533, 517, 472, 526, 489, 524, 495, 512, 516, 513, 485, 511, 505, 534, 505, 519, 500] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[523, 533, 535, 497, 469, 476, 507, 507, 501, 536, 535, 516, 556, 490, 498, 481, 532, 565, 496, 532, 537, 500, 559, 510, 555, 506, 522, 481, 533, 493, 509, 478, 533, 485, 534, 535, 489, 508, 543, 518, 464, 478, 460, 446, 551, 462, 501, 478, 546, 522, 534, 515, 535, 554, 564, 470, 550, 542, 512, 460, 466, 514, 522, 518, 550, 518, 519, 505, 555, 487, 498, 485, 512, 534, 524, 476, 521, 482, 522, 481, 508, 542, 466, 477, 493, 527, 522, 524, 557, 474, 492, 507, 496, 514, 513, 494, 494, 494, 516, 541, 505, 505, 468, 521, 520, 528, 578, 511, 500, 529, 525, 547, 529, 502, 542, 506, 533, 486, 521, 528, 498, 517, 532, 500, 507, 488, 520, 493] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[509, 533, 537, 527, 556, 512, 472, 516, 487, 522, 494, 488, 523, 526, 536, 492, 526, 507, 498, 550, 542, 540, 475, 491, 509, 458, 504, 500, 534, 558, 551, 504, 504, 506, 530, 535, 512, 501, 503, 466, 503, 485, 463, 515, 524, 525, 514, 481, 516, 501, 516, 526, 514, 506, 542, 500, 521, 549, 532, 491, 480, 521, 475, 511, 490, 482, 537, 498, 511, 501, 503, 511, 464, 511, 491, 501, 529, 485, 516, 513, 545, 524, 530, 553, 504, 547, 515, 554, 536, 484, 488, 507, 525, 499, 528, 553, 493, 513, 486, 466, 520, 530, 503, 501, 502, 474, 520, 506, 553, 518, 518, 486, 506, 509, 536, 528, 504, 542, 544, 511, 523, 530, 527, 523, 480, 500, 522, 482] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[515, 507, 526, 522, 510, 546, 518, 500, 514, 497, 545, 536, 504, 539, 500, 478, 486, 494, 486, 546, 503, 473, 550, 551, 525, 533, 522, 539, 486, 496, 518, 484, 529, 528, 495, 533, 533, 500, 521, 500, 489, 494, 513, 471, 519, 481, 544, 532, 557, 526, 498, 503, 547, 515, 497, 543, 531, 526, 515, 516, 502, 508, 494, 494, 517, 525, 497, 500, 497, 536, 480, 485, 504, 513, 495, 505, 491, 561, 535, 584, 477, 495, 556, 499, 518, 511, 476, 511, 541, 499, 483, 538, 503, 475, 524, 506, 531, 529, 526, 538, 478, 516, 422, 498, 509, 479, 499, 506, 505, 536, 555, 523, 518, 512, 509, 541, 488, 512, 489, 522, 521, 486, 525, 517, 516, 479, 484, 527] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[473, 568, 509, 500, 530, 492, 537, 545, 530, 494, 534, 544, 521, 521, 542, 538, 511, 518, 520, 521, 514, 496, 535, 488, 498, 500, 514, 527, 513, 521, 514, 494, 521, 545, 503, 541, 490, 500, 518, 511, 544, 480, 516, 493, 541, 519, 506, 515, 500, 492, 515, 487, 524, 474, 544, 506, 508, 500, 504, 525, 487, 475, 520, 483, 527, 525, 496, 533, 557, 486, 514, 522, 503, 500, 497, 531, 518, 515, 488, 519, 535, 535, 492, 536, 542, 501, 493, 497, 518, 517, 518, 513, 518, 490, 482, 495, 506, 513, 519, 495, 503, 518, 551, 517, 503, 501, 504, 498, 482, 539, 497, 521, 518, 513, 507, 502, 506, 506, 490, 493, 544, 498, 505, 499, 492, 502, 522, 510] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[538, 516, 493, 530, 494, 531, 500, 531, 541, 509, 488, 521, 496, 482, 513, 511, 529, 505, 537, 505, 502, 478, 544, 509, 508, 538, 478, 541, 543, 530, 517, 499, 494, 521, 469, 491, 480, 522, 515, 509, 517, 506, 520, 510, 532, 529, 491, 532, 510, 524, 533, 465, 518, 490, 531, 528, 534, 518, 491, 529, 524, 515, 524, 512, 531, 514, 512, 490, 489, 495, 489, 521, 489, 511, 516, 567, 506, 495, 494, 495, 507, 509, 519, 520, 547, 494, 484, 476, 507, 500, 512, 518, 494, 524, 477, 516, 504, 513, 476, 547, 517, 524, 519, 437, 523, 514, 521, 514, 470, 486, 543, 527, 486, 548, 520, 521, 518, 490, 558, 518, 489, 536, 530, 506, 538, 512, 536, 536] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[523, 501, 535, 497, 469, 476, 507, 507, 501, 536, 535, 516, 556, 490, 498, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 565, 496, 532, 537, 500, 540, 510, 555, 506, 522, 481, 533, 493, 509, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 485, 534, 535, 489, 508, 543, 518, 464, 478, 460, 446, 551, 462, 501, 478, 32, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 522, 534, 515, 535, 554, 561, 470, 550, 542, 512, 460, 466, 514, 522, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 518, 519, 505, 553, 487, 498, 485, 512, 534, 524, 476, 521, 482, 522, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 542, 466, 477, 493, 527, 522, 524, 557, 474, 492, 507, 496, 514, 513, 494, 7, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 494, 516, 541, 505, 505, 468, 521, 520, 528, 578, 511, 500, 513, 525, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 502, 542, 506, 533, 486, 521, 528, 498, 517, 532, 500, 507, 488, 520, 493, 9, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8128, 8289, 8030, 8321, 8167, 8118, 8266, 8217] avg=8192.0000 max=8321 max_rank=3 min=8030 min_rank=2 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32768 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[513, 548, 523, 495, 505, 524, 497, 503, 548, 496, 510, 488, 535, 476, 530, 523, 473, 529, 494, 489, 507, 524, 495, 549, 522, 495, 519, 544, 499, 525, 470, 511, 545, 507, 502, 493, 505, 502, 523, 488, 516, 487, 508, 518, 505, 528, 493, 521, 556, 477, 505, 551, 509, 489, 502, 505, 498, 466, 561, 485, 539, 527, 523, 538, 509, 532, 491, 517, 526, 517, 519, 486, 564, 555, 504, 544, 547, 490, 498, 494, 504, 516, 527, 473, 482, 499, 484, 508, 519, 524, 472, 510, 500, 531, 516, 527, 537, 518, 485, 502, 525, 519, 513, 511, 488, 517, 520, 531, 521, 517, 511, 542, 543, 470, 502, 538, 539, 491, 498, 512, 521, 482, 490, 487, 551, 509, 524, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[492, 483, 524, 508, 532, 533, 522, 486, 497, 504, 536, 529, 515, 501, 509, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 485, 536, 529, 491, 522, 467, 519, 487, 509, 528, 508, 530, 511, 517, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 512, 502, 472, 509, 588, 516, 504, 511, 520, 469, 491, 493, 518, 506, 468, 34, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 470, 506, 507, 500, 552, 505, 465, 521, 523, 488, 514, 538, 506, 501, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 511, 534, 462, 508, 498, 560, 502, 520, 475, 561, 510, 511, 532, 554, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 497, 497, 509, 556, 514, 483, 493, 532, 520, 548, 508, 490, 532, 571, 544, 11, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 480, 514, 532, 509, 499, 533, 564, 517, 526, 518, 557, 496, 525, 503, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 522, 497, 507, 498, 514, 472, 489, 549, 485, 519, 522, 502, 475, 530, 515, 13, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8127, 8148, 8112, 8260, 8318, 8305, 8086] avg=8192.0000 max=8318 max_rank=5 min=8086 min_rank=7 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32969 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[513, 515, 523, 495, 505, 524, 497, 503, 548, 496, 510, 488, 535, 476, 530, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 529, 494, 489, 507, 524, 479, 549, 522, 495, 519, 544, 499, 525, 470, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 507, 502, 493, 505, 502, 523, 488, 516, 487, 508, 518, 505, 528, 493, 521, 33, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 477, 505, 551, 509, 489, 499, 505, 498, 466, 561, 485, 539, 527, 523, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 532, 491, 517, 524, 517, 519, 486, 564, 555, 504, 544, 547, 490, 498, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 516, 527, 473, 482, 499, 484, 508, 519, 524, 472, 510, 500, 531, 516, 527, 7, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 518, 485, 502, 525, 519, 513, 511, 488, 517, 520, 531, 521, 502, 511, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 470, 502, 538, 539, 491, 498, 512, 521, 482, 490, 487, 551, 509, 524, 506, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8129, 8185, 8228, 8291, 8104, 8242, 8176] avg=8192.0000 max=8291 max_rank=4 min=8104 min_rank=5 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32813 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[509, 501, 537, 527, 556, 512, 472, 516, 487, 522, 494, 488, 523, 526, 536, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 507, 498, 550, 542, 540, 459, 491, 509, 458, 504, 500, 534, 558, 551, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 506, 530, 535, 512, 501, 503, 466, 503, 485, 463, 515, 524, 525, 514, 481, 32, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 501, 516, 526, 514, 506, 539, 500, 521, 549, 532, 491, 480, 521, 475, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 482, 537, 498, 510, 501, 503, 511, 464, 511, 491, 501, 529, 485, 516, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 524, 530, 553, 504, 547, 515, 554, 536, 484, 488, 507, 525, 499, 528, 553, 7, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 513, 486, 466, 520, 530, 503, 501, 502, 474, 520, 506, 553, 503, 518, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 509, 536, 528, 504, 542, 544, 511, 523, 530, 527, 523, 480, 500, 522, 482, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8231, 8110, 8198, 8042, 8403, 8074, 8280] avg=8192.0000 max=8403 max_rank=5 min=8042 min_rank=4 imbalance_ratio=1.025757 max_load_reduction=57133 max_load_reduction_pct=0.871780 moved_to_echo=32799 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[508, 489, 542, 524, 514, 532, 512, 501, 486, 490, 521, 490, 511, 508, 491, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 511, 511, 496, 475, 550, 524, 522, 533, 484, 540, 520, 467, 515, 482, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 536, 533, 521, 522, 553, 526, 511, 485, 525, 542, 484, 504, 522, 551, 479, 31, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 508, 486, 508, 501, 494, 517, 510, 512, 515, 541, 471, 497, 564, 510, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 519, 523, 491, 526, 511, 496, 521, 560, 479, 515, 498, 500, 552, 530, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 492, 498, 505, 505, 518, 477, 482, 503, 486, 510, 533, 520, 549, 539, 499, 8, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 484, 473, 513, 509, 508, 521, 500, 559, 505, 546, 508, 538, 528, 497, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 472, 526, 489, 524, 495, 512, 516, 513, 485, 511, 505, 534, 505, 519, 500, 9, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8115, 8175, 8369, 8124, 8210, 8159, 8246, 8138] avg=8192.0000 max=8369 max_rank=2 min=8115 min_rank=0 imbalance_ratio=1.021606 max_load_reduction=57167 max_load_reduction_pct=0.872299 moved_to_echo=32753 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[515, 477, 526, 522, 510, 546, 518, 500, 514, 497, 545, 536, 504, 539, 500, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 494, 486, 546, 503, 473, 532, 551, 525, 533, 522, 539, 486, 496, 518, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 528, 495, 533, 533, 500, 521, 500, 489, 494, 513, 471, 519, 481, 544, 532, 30, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 557, 526, 498, 503, 547, 515, 494, 543, 531, 526, 515, 516, 502, 508, 494, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 525, 497, 500, 496, 536, 480, 485, 504, 513, 495, 505, 491, 561, 535, 584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 495, 556, 499, 518, 511, 476, 511, 541, 499, 483, 538, 503, 475, 524, 506, 8, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 529, 526, 538, 478, 516, 422, 498, 509, 479, 499, 506, 505, 519, 555, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 512, 509, 541, 488, 512, 489, 522, 521, 486, 525, 517, 516, 479, 484, 527, 9, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8227, 8174, 8224, 8269, 8224, 8124, 8133, 8161] avg=8192.0000 max=8269 max_rank=3 min=8124 min_rank=5 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32642 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[538, 485, 493, 530, 494, 531, 500, 531, 541, 509, 488, 521, 496, 482, 513, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 505, 537, 505, 502, 478, 526, 509, 508, 538, 478, 541, 543, 530, 517, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 521, 469, 491, 480, 522, 515, 509, 517, 506, 520, 510, 532, 529, 491, 532, 31, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 524, 533, 465, 518, 490, 528, 528, 534, 518, 491, 529, 524, 515, 524, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 514, 512, 490, 488, 495, 489, 521, 489, 511, 516, 567, 506, 495, 494, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 509, 519, 520, 547, 494, 484, 476, 507, 500, 512, 518, 494, 524, 477, 516, 7, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 513, 476, 547, 517, 524, 519, 437, 523, 514, 521, 514, 470, 471, 543, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 548, 520, 521, 518, 490, 558, 518, 489, 536, 530, 506, 538, 512, 536, 536, 8, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8163, 8245, 8181, 8243, 8113, 8115, 8120, 8356] avg=8192.0000 max=8356 max_rank=7 min=8113 min_rank=4 imbalance_ratio=1.020020 max_load_reduction=57180 max_load_reduction_pct=0.872498 moved_to_echo=32704 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[515, 477, 526, 522, 510, 546, 518, 500, 514, 497, 545, 536, 504, 539, 500, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 494, 486, 546, 503, 473, 532, 551, 525, 533, 522, 539, 486, 496, 518, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 528, 495, 533, 533, 500, 521, 500, 489, 494, 513, 471, 519, 481, 544, 532, 30, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 557, 526, 498, 503, 547, 515, 494, 543, 531, 526, 515, 516, 502, 508, 494, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 525, 497, 500, 496, 536, 480, 485, 504, 513, 495, 505, 491, 561, 535, 584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 495, 556, 499, 518, 511, 476, 511, 541, 499, 483, 538, 503, 475, 524, 506, 8, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 529, 526, 538, 478, 516, 422, 498, 509, 479, 499, 506, 505, 519, 555, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 512, 509, 541, 488, 512, 489, 522, 521, 486, 525, 517, 516, 479, 484, 527, 9, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8227, 8174, 8224, 8269, 8224, 8124, 8133, 8161] avg=8192.0000 max=8269 max_rank=3 min=8124 min_rank=5 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32642 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[473, 534, 509, 500, 530, 492, 537, 545, 530, 494, 534, 544, 521, 521, 542, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 518, 520, 521, 514, 496, 517, 488, 498, 500, 514, 527, 513, 521, 514, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 545, 503, 541, 490, 500, 518, 511, 544, 480, 516, 493, 541, 519, 506, 515, 34, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 492, 515, 487, 524, 474, 541, 506, 508, 500, 504, 525, 487, 475, 520, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 525, 496, 533, 555, 486, 514, 522, 503, 500, 497, 531, 518, 515, 488, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 535, 492, 536, 542, 501, 493, 497, 518, 517, 518, 513, 518, 490, 482, 495, 8, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 513, 519, 495, 503, 518, 551, 517, 503, 501, 504, 498, 482, 522, 497, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 513, 507, 502, 506, 506, 490, 493, 544, 498, 505, 499, 492, 502, 522, 510, 9, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8344, 8166, 8289, 8041, 8229, 8195, 8150, 8122] avg=8192.0000 max=8344 max_rank=0 min=8041 min_rank=3 imbalance_ratio=1.018555 max_load_reduction=57192 max_load_reduction_pct=0.872681 moved_to_echo=32696 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[508, 489, 542, 524, 514, 532, 512, 501, 486, 490, 521, 490, 511, 508, 491, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 511, 511, 496, 475, 550, 524, 522, 533, 484, 540, 520, 467, 515, 482, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 536, 533, 521, 522, 553, 526, 511, 485, 525, 542, 484, 504, 522, 551, 479, 31, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 508, 486, 508, 501, 494, 517, 510, 512, 515, 541, 471, 497, 564, 510, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 519, 523, 491, 526, 511, 496, 521, 560, 479, 515, 498, 500, 552, 530, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 492, 498, 505, 505, 518, 477, 482, 503, 486, 510, 533, 520, 549, 539, 499, 8, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 484, 473, 513, 509, 508, 521, 500, 559, 505, 546, 508, 538, 528, 497, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 472, 526, 489, 524, 495, 512, 516, 513, 485, 511, 505, 534, 505, 519, 500, 9, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8115, 8175, 8369, 8124, 8210, 8159, 8246, 8138] avg=8192.0000 max=8369 max_rank=2 min=8115 min_rank=0 imbalance_ratio=1.021606 max_load_reduction=57167 max_load_reduction_pct=0.872299 moved_to_echo=32753 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[509, 501, 537, 527, 556, 512, 472, 516, 487, 522, 494, 488, 523, 526, 536, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 507, 498, 550, 542, 540, 459, 491, 509, 458, 504, 500, 534, 558, 551, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 506, 530, 535, 512, 501, 503, 466, 503, 485, 463, 515, 524, 525, 514, 481, 32, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 501, 516, 526, 514, 506, 539, 500, 521, 549, 532, 491, 480, 521, 475, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 482, 537, 498, 510, 501, 503, 511, 464, 511, 491, 501, 529, 485, 516, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 524, 530, 553, 504, 547, 515, 554, 536, 484, 488, 507, 525, 499, 528, 553, 7, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 513, 486, 466, 520, 530, 503, 501, 502, 474, 520, 506, 553, 503, 518, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 509, 536, 528, 504, 542, 544, 511, 523, 530, 527, 523, 480, 500, 522, 482, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8231, 8110, 8198, 8042, 8403, 8074, 8280] avg=8192.0000 max=8403 max_rank=5 min=8042 min_rank=4 imbalance_ratio=1.025757 max_load_reduction=57133 max_load_reduction_pct=0.871780 moved_to_echo=32799 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[523, 501, 535, 497, 469, 476, 507, 507, 501, 536, 535, 516, 556, 490, 498, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 565, 496, 532, 537, 500, 540, 510, 555, 506, 522, 481, 533, 493, 509, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 485, 534, 535, 489, 508, 543, 518, 464, 478, 460, 446, 551, 462, 501, 478, 32, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 522, 534, 515, 535, 554, 561, 470, 550, 542, 512, 460, 466, 514, 522, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 518, 519, 505, 553, 487, 498, 485, 512, 534, 524, 476, 521, 482, 522, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 542, 466, 477, 493, 527, 522, 524, 557, 474, 492, 507, 496, 514, 513, 494, 7, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 494, 516, 541, 505, 505, 468, 521, 520, 528, 578, 511, 500, 513, 525, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 502, 542, 506, 533, 486, 521, 528, 498, 517, 532, 500, 507, 488, 520, 493, 9, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8128, 8289, 8030, 8321, 8167, 8118, 8266, 8217] avg=8192.0000 max=8321 max_rank=3 min=8030 min_rank=2 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32768 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[513, 515, 523, 495, 505, 524, 497, 503, 548, 496, 510, 488, 535, 476, 530, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 529, 494, 489, 507, 524, 479, 549, 522, 495, 519, 544, 499, 525, 470, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 507, 502, 493, 505, 502, 523, 488, 516, 487, 508, 518, 505, 528, 493, 521, 33, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 477, 505, 551, 509, 489, 499, 505, 498, 466, 561, 485, 539, 527, 523, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 532, 491, 517, 524, 517, 519, 486, 564, 555, 504, 544, 547, 490, 498, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 516, 527, 473, 482, 499, 484, 508, 519, 524, 472, 510, 500, 531, 516, 527, 7, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 518, 485, 502, 525, 519, 513, 511, 488, 517, 520, 531, 521, 502, 511, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 470, 502, 538, 539, 491, 498, 512, 521, 482, 490, 487, 551, 509, 524, 506, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8129, 8185, 8228, 8291, 8104, 8242, 8176] avg=8192.0000 max=8291 max_rank=4 min=8104 min_rank=5 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32813 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 3968, 4224, 4096, 4224, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 3968, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 3968, 4224, 4096, 4224, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[515, 518, 514, 513, 519, 504, 500, 528, 486, 508, 502, 525, 502, 501, 527, 505, 493, 511, 532, 487, 526, 490, 489, 508, 493, 534, 551, 551, 494, 457, 495, 485, 498, 546, 491, 518, 521, 526, 528, 533, 541, 520, 484, 480, 553, 493, 521, 527, 507, 528, 507, 475, 503, 497, 509, 520, 535, 513, 494, 505, 516, 475, 462, 508, 537, 502, 515, 513, 516, 472, 519, 541, 506, 514, 547, 532, 524, 525, 549, 551, 513, 503, 499, 505, 532, 525, 544, 486, 466, 480, 521, 539, 517, 538, 534, 477, 462, 503, 566, 497, 530, 500, 537, 496, 508, 531, 536, 491, 533, 562, 513, 510, 511, 527, 543, 495, 505, 500, 461, 526, 504, 504, 490, 530, 483, 521, 511, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[501, 488, 560, 519, 552, 533, 494, 523, 484, 513, 511, 546, 489, 536, 496, 541, 512, 540, 538, 505, 540, 505, 496, 491, 502, 483, 519, 523, 505, 495, 494, 533, 538, 472, 498, 513, 477, 529, 501, 523, 477, 490, 530, 484, 521, 505, 538, 532, 515, 500, 483, 512, 508, 533, 509, 516, 511, 505, 516, 524, 521, 512, 548, 505, 555, 503, 537, 511, 531, 495, 540, 538, 565, 491, 475, 484, 557, 507, 504, 506, 493, 491, 541, 478, 503, 474, 498, 471, 518, 494, 512, 538, 508, 527, 494, 510, 539, 539, 514, 493, 514, 460, 547, 529, 523, 495, 493, 496, 542, 503, 506, 478, 495, 515, 483, 506, 514, 519, 519, 504, 500, 503, 519, 535, 530, 506, 520, 535] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[500, 477, 520, 511, 505, 533, 528, 537, 506, 467, 512, 505, 478, 514, 518, 550, 539, 550, 525, 534, 506, 467, 524, 530, 496, 477, 530, 512, 482, 529, 546, 502, 483, 479, 535, 511, 534, 560, 519, 512, 522, 517, 477, 511, 522, 488, 507, 543, 484, 504, 492, 495, 499, 504, 504, 456, 492, 489, 534, 498, 489, 514, 525, 474, 548, 554, 502, 495, 519, 470, 540, 534, 534, 492, 534, 560, 494, 530, 503, 549, 528, 536, 480, 519, 487, 512, 502, 521, 500, 477, 514, 563, 537, 548, 492, 507, 534, 511, 576, 524, 495, 500, 527, 502, 493, 482, 509, 476, 527, 507, 528, 471, 520, 510, 499, 531, 476, 505, 533, 471, 514, 520, 544, 534, 505, 497, 534, 517] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[501, 488, 560, 519, 552, 533, 494, 523, 484, 513, 511, 546, 489, 536, 496, 541, 512, 540, 538, 505, 540, 505, 496, 491, 502, 483, 519, 523, 505, 495, 494, 533, 538, 472, 498, 513, 477, 529, 501, 523, 477, 490, 530, 484, 521, 505, 538, 532, 515, 500, 483, 512, 508, 533, 509, 516, 511, 505, 516, 524, 521, 512, 548, 505, 555, 503, 537, 511, 531, 495, 540, 538, 565, 491, 475, 484, 557, 507, 504, 506, 493, 491, 541, 478, 503, 474, 498, 471, 518, 494, 512, 538, 508, 527, 494, 510, 539, 539, 514, 493, 514, 460, 547, 529, 523, 495, 493, 496, 542, 503, 506, 478, 495, 515, 483, 506, 514, 519, 519, 504, 500, 503, 519, 535, 530, 506, 520, 535] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[499, 523, 544, 476, 523, 483, 502, 510, 462, 504, 473, 528, 557, 523, 535, 513, 512, 519, 519, 504, 501, 531, 542, 563, 512, 489, 482, 515, 521, 510, 535, 561, 546, 516, 505, 521, 518, 505, 483, 473, 476, 507, 551, 501, 530, 510, 508, 509, 541, 477, 512, 522, 525, 478, 511, 501, 496, 516, 512, 481, 488, 501, 472, 505, 508, 526, 543, 495, 550, 520, 490, 477, 523, 537, 491, 521, 504, 476, 530, 515, 527, 530, 500, 495, 570, 522, 488, 519, 519, 550, 524, 514, 460, 521, 519, 492, 492, 504, 488, 480, 499, 529, 496, 523, 513, 530, 483, 537, 553, 516, 529, 549, 511, 506, 498, 550, 529, 478, 534, 498, 514, 512, 502, 469, 538, 517, 510, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[484, 533, 537, 547, 499, 497, 547, 503, 489, 522, 487, 514, 467, 533, 500, 493, 546, 516, 529, 474, 537, 527, 497, 517, 508, 505, 496, 530, 476, 514, 498, 502, 508, 497, 513, 521, 524, 490, 531, 547, 515, 512, 508, 492, 505, 503, 485, 530, 560, 527, 521, 529, 495, 520, 499, 491, 529, 521, 530, 514, 525, 514, 512, 483, 517, 522, 521, 536, 483, 488, 478, 540, 530, 504, 499, 501, 489, 492, 496, 511, 503, 485, 517, 515, 540, 514, 519, 523, 529, 537, 512, 513, 538, 511, 505, 488, 516, 506, 503, 549, 519, 505, 507, 521, 483, 526, 525, 544, 523, 459, 502, 536, 564, 534, 488, 511, 504, 498, 487, 510, 492, 520, 544, 499, 493, 483, 515, 539] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[515, 518, 514, 513, 519, 504, 500, 528, 486, 508, 502, 525, 502, 501, 527, 505, 493, 511, 532, 487, 526, 490, 489, 508, 493, 534, 551, 551, 494, 457, 495, 485, 498, 546, 491, 518, 521, 526, 528, 533, 541, 520, 484, 480, 553, 493, 521, 527, 507, 528, 507, 475, 503, 497, 509, 520, 535, 513, 494, 505, 516, 475, 462, 508, 537, 502, 515, 513, 516, 472, 519, 541, 506, 514, 547, 532, 524, 525, 549, 551, 513, 503, 499, 505, 532, 525, 544, 486, 466, 480, 521, 539, 517, 538, 534, 477, 462, 503, 566, 497, 530, 500, 537, 496, 508, 531, 536, 491, 533, 562, 513, 510, 511, 527, 543, 495, 505, 500, 461, 526, 504, 504, 490, 530, 483, 521, 511, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[497, 498, 521, 534, 515, 554, 498, 520, 522, 488, 529, 541, 525, 516, 530, 499, 501, 473, 520, 509, 520, 494, 485, 523, 538, 515, 511, 488, 513, 540, 535, 505, 532, 491, 513, 500, 519, 537, 541, 500, 503, 548, 556, 501, 529, 518, 498, 523, 507, 474, 495, 516, 496, 506, 519, 493, 518, 515, 490, 465, 500, 497, 537, 539, 509, 483, 498, 517, 512, 543, 511, 491, 477, 497, 512, 518, 541, 490, 543, 470, 466, 470, 538, 556, 525, 534, 491, 551, 538, 522, 504, 515, 515, 539, 483, 499, 538, 520, 477, 520, 498, 486, 516, 514, 523, 547, 493, 475, 500, 516, 476, 528, 531, 496, 503, 510, 477, 524, 549, 525, 503, 543, 514, 516, 500, 520, 516, 491] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[548, 515, 481, 528, 518, 548, 490, 522, 468, 495, 484, 531, 492, 538, 508, 494, 486, 491, 498, 476, 523, 550, 503, 523, 525, 524, 515, 500, 524, 514, 515, 526, 523, 485, 494, 477, 518, 478, 545, 489, 511, 512, 505, 463, 520, 469, 514, 545, 539, 518, 510, 524, 480, 538, 520, 503, 518, 513, 547, 491, 539, 522, 524, 485, 479, 542, 509, 468, 516, 506, 507, 498, 527, 494, 547, 517, 500, 530, 477, 495, 534, 500, 525, 501, 488, 513, 510, 510, 539, 527, 492, 523, 542, 514, 522, 517, 485, 501, 512, 537, 519, 484, 549, 497, 517, 537, 490, 481, 526, 524, 502, 500, 536, 551, 537, 504, 499, 540, 512, 527, 514, 547, 513, 504, 506, 527, 502, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[499, 523, 544, 476, 523, 483, 502, 510, 462, 504, 473, 528, 557, 523, 535, 513, 512, 519, 519, 504, 501, 531, 542, 563, 512, 489, 482, 515, 521, 510, 535, 561, 546, 516, 505, 521, 518, 505, 483, 473, 476, 507, 551, 501, 530, 510, 508, 509, 541, 477, 512, 522, 525, 478, 511, 501, 496, 516, 512, 481, 488, 501, 472, 505, 508, 526, 543, 495, 550, 520, 490, 477, 523, 537, 491, 521, 504, 476, 530, 515, 527, 530, 500, 495, 570, 522, 488, 519, 519, 550, 524, 514, 460, 521, 519, 492, 492, 504, 488, 480, 499, 529, 496, 523, 513, 530, 483, 537, 553, 516, 529, 549, 511, 506, 498, 550, 529, 478, 534, 498, 514, 512, 502, 469, 538, 517, 510, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[484, 528, 513, 504, 550, 521, 486, 491, 528, 537, 555, 460, 500, 533, 468, 500, 512, 487, 498, 524, 503, 515, 504, 503, 542, 505, 527, 513, 514, 539, 472, 505, 522, 511, 497, 519, 501, 516, 496, 532, 523, 496, 522, 514, 500, 552, 490, 514, 498, 535, 511, 484, 475, 563, 521, 513, 510, 527, 536, 540, 487, 496, 511, 513, 530, 479, 538, 527, 519, 506, 514, 578, 550, 519, 480, 502, 563, 540, 493, 492, 547, 506, 540, 486, 511, 531, 492, 520, 533, 507, 462, 523, 495, 506, 496, 571, 499, 460, 533, 488, 475, 505, 503, 494, 485, 505, 508, 535, 501, 519, 477, 506, 518, 560, 473, 495, 497, 520, 539, 520, 532, 504, 517, 494, 504, 528, 517, 523] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[484, 533, 537, 547, 499, 497, 547, 503, 489, 522, 487, 514, 467, 533, 500, 493, 546, 516, 529, 474, 537, 527, 497, 517, 508, 505, 496, 530, 476, 514, 498, 502, 508, 497, 513, 521, 524, 490, 531, 547, 515, 512, 508, 492, 505, 503, 485, 530, 560, 527, 521, 529, 495, 520, 499, 491, 529, 521, 530, 514, 525, 514, 512, 483, 517, 522, 521, 536, 483, 488, 478, 540, 530, 504, 499, 501, 489, 492, 496, 511, 503, 485, 517, 515, 540, 514, 519, 523, 529, 537, 512, 513, 538, 511, 505, 488, 516, 506, 503, 549, 519, 505, 507, 521, 483, 526, 525, 544, 523, 459, 502, 536, 564, 534, 488, 511, 504, 498, 487, 510, 492, 520, 544, 499, 493, 483, 515, 539] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[497, 498, 521, 534, 515, 554, 498, 520, 522, 488, 529, 541, 525, 516, 530, 499, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 473, 520, 509, 520, 494, 485, 523, 538, 515, 511, 488, 513, 540, 535, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 491, 513, 500, 519, 537, 541, 500, 503, 548, 556, 501, 529, 518, 498, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 474, 495, 516, 496, 506, 519, 493, 518, 515, 490, 465, 500, 497, 537, 539, 39, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 483, 498, 517, 512, 543, 511, 491, 438, 497, 512, 518, 541, 490, 543, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 466, 470, 538, 556, 525, 534, 491, 551, 538, 522, 504, 498, 515, 539, 483, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 520, 477, 520, 498, 486, 516, 514, 523, 547, 493, 475, 500, 516, 476, 528, 16, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 477, 503, 510, 477, 524, 549, 525, 503, 543, 514, 516, 500, 520, 516, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8288, 8170, 8309, 8117, 8073, 8229, 8151, 8199] avg=8192.0000 max=8309 max_rank=2 min=8073 min_rank=4 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32652 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[515, 518, 514, 513, 519, 504, 500, 528, 486, 508, 502, 525, 502, 501, 527, 505, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 511, 532, 487, 526, 490, 489, 508, 493, 534, 551, 551, 494, 457, 495, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 546, 491, 518, 521, 526, 528, 533, 541, 520, 484, 480, 553, 493, 521, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 528, 507, 475, 503, 497, 509, 520, 535, 513, 494, 505, 516, 475, 462, 508, 42, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 502, 515, 513, 516, 472, 519, 541, 464, 514, 547, 532, 524, 525, 549, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 503, 499, 505, 532, 525, 544, 486, 466, 480, 521, 522, 517, 538, 534, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 503, 566, 497, 530, 500, 537, 496, 508, 531, 536, 491, 533, 562, 513, 510, 16, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 508, 543, 495, 505, 500, 461, 526, 504, 504, 490, 530, 483, 521, 511, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8168, 8096, 8280, 8107, 8321, 8162, 8299, 8103] avg=8192.0000 max=8321 max_rank=4 min=8096 min_rank=1 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32885 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[500, 477, 520, 511, 505, 533, 528, 537, 506, 467, 512, 505, 478, 514, 518, 550, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 550, 525, 534, 506, 467, 524, 530, 496, 477, 530, 512, 482, 529, 546, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 479, 535, 511, 534, 560, 519, 512, 522, 517, 477, 511, 522, 488, 507, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 504, 492, 495, 499, 504, 504, 456, 492, 489, 534, 498, 489, 514, 525, 474, 44, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 554, 502, 495, 519, 470, 540, 534, 490, 492, 534, 560, 494, 530, 503, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 536, 480, 519, 487, 512, 502, 521, 500, 477, 514, 545, 537, 548, 492, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 511, 576, 524, 495, 500, 527, 502, 493, 482, 509, 476, 527, 507, 528, 471, 17, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 491, 499, 531, 476, 505, 533, 471, 514, 520, 544, 534, 505, 497, 534, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8162, 8249, 8220, 8008, 8314, 8205, 8187, 8191] avg=8192.0000 max=8314 max_rank=4 min=8008 min_rank=3 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32897 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[499, 523, 544, 476, 523, 483, 502, 510, 462, 504, 473, 528, 557, 523, 535, 513, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 519, 515, 504, 501, 531, 542, 563, 512, 489, 482, 515, 521, 510, 535, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 516, 505, 521, 518, 505, 483, 473, 476, 507, 551, 501, 530, 510, 508, 509, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 477, 512, 522, 525, 478, 511, 501, 496, 516, 512, 481, 488, 501, 472, 505, 47, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 526, 543, 495, 550, 520, 490, 477, 476, 537, 491, 521, 504, 476, 530, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 530, 500, 495, 570, 522, 488, 519, 519, 550, 524, 490, 460, 521, 519, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 504, 488, 480, 499, 529, 496, 523, 513, 530, 483, 537, 553, 516, 529, 549, 19, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 479, 498, 550, 529, 478, 534, 498, 514, 512, 502, 469, 538, 517, 510, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8158, 8312, 8165, 8100, 8159, 8226, 8252, 8164] avg=8192.0000 max=8312 max_rank=1 min=8100 min_rank=3 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32801 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[500, 477, 520, 511, 505, 533, 528, 537, 506, 467, 512, 505, 478, 514, 518, 550, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 550, 525, 534, 506, 467, 524, 530, 496, 477, 530, 512, 482, 529, 546, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 479, 535, 511, 534, 560, 519, 512, 522, 517, 477, 511, 522, 488, 507, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 504, 492, 495, 499, 504, 504, 456, 492, 489, 534, 498, 489, 514, 525, 474, 44, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 554, 502, 495, 519, 470, 540, 534, 490, 492, 534, 560, 494, 530, 503, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 536, 480, 519, 487, 512, 502, 521, 500, 477, 514, 545, 537, 548, 492, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 511, 576, 524, 495, 500, 527, 502, 493, 482, 509, 476, 527, 507, 528, 471, 17, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 491, 499, 531, 476, 505, 533, 471, 514, 520, 544, 534, 505, 497, 534, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8162, 8249, 8220, 8008, 8314, 8205, 8187, 8191] avg=8192.0000 max=8314 max_rank=4 min=8008 min_rank=3 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32897 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[501, 488, 560, 519, 552, 533, 494, 523, 484, 513, 511, 546, 489, 536, 496, 541, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 540, 538, 505, 540, 505, 496, 491, 502, 483, 519, 523, 505, 495, 494, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 472, 498, 513, 477, 529, 501, 523, 477, 490, 530, 484, 521, 505, 538, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 500, 483, 512, 508, 533, 509, 516, 511, 505, 516, 524, 521, 512, 548, 505, 47, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 503, 537, 511, 531, 495, 540, 538, 518, 491, 475, 484, 557, 507, 504, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 491, 541, 478, 503, 474, 498, 471, 518, 494, 512, 521, 508, 527, 494, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 539, 514, 493, 514, 460, 547, 529, 523, 495, 493, 496, 542, 503, 506, 478, 16, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 496, 483, 506, 514, 519, 519, 504, 500, 503, 519, 535, 530, 506, 520, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8287, 8181, 8128, 8276, 8252, 8033, 8195, 8184] avg=8192.0000 max=8287 max_rank=0 min=8033 min_rank=5 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32664 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[515, 518, 514, 513, 519, 504, 500, 528, 486, 508, 502, 525, 502, 501, 527, 505, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 511, 532, 487, 526, 490, 489, 508, 493, 534, 551, 551, 494, 457, 495, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 546, 491, 518, 521, 526, 528, 533, 541, 520, 484, 480, 553, 493, 521, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 528, 507, 475, 503, 497, 509, 520, 535, 513, 494, 505, 516, 475, 462, 508, 42, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 502, 515, 513, 516, 472, 519, 541, 464, 514, 547, 532, 524, 525, 549, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 503, 499, 505, 532, 525, 544, 486, 466, 480, 521, 522, 517, 538, 534, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 503, 566, 497, 530, 500, 537, 496, 508, 531, 536, 491, 533, 562, 513, 510, 16, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 508, 543, 495, 505, 500, 461, 526, 504, 504, 490, 530, 483, 521, 511, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8168, 8096, 8280, 8107, 8321, 8162, 8299, 8103] avg=8192.0000 max=8321 max_rank=4 min=8096 min_rank=1 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32885 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[548, 515, 481, 528, 518, 548, 490, 522, 468, 495, 484, 531, 492, 538, 508, 494, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 491, 498, 476, 523, 550, 503, 523, 525, 524, 515, 500, 524, 514, 515, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 485, 494, 477, 518, 478, 545, 489, 511, 512, 505, 463, 520, 469, 514, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 518, 510, 524, 480, 538, 520, 503, 518, 513, 547, 491, 539, 522, 524, 485, 43, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 542, 509, 468, 516, 506, 507, 498, 484, 494, 547, 517, 500, 530, 477, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 500, 525, 501, 488, 513, 510, 510, 539, 527, 492, 506, 542, 514, 522, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 501, 512, 537, 519, 484, 549, 497, 517, 537, 490, 481, 526, 524, 502, 500, 16, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 530, 537, 504, 499, 540, 512, 527, 514, 547, 513, 504, 506, 527, 502, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8193, 8048, 8326, 8069, 8240, 8186, 8313] avg=8192.0000 max=8326 max_rank=3 min=8048 min_rank=2 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32808 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[484, 528, 513, 504, 550, 521, 486, 491, 528, 537, 555, 460, 500, 533, 468, 500, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 487, 498, 524, 503, 515, 504, 503, 542, 505, 527, 513, 514, 539, 472, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 511, 497, 519, 501, 516, 496, 532, 523, 496, 522, 514, 500, 552, 490, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 535, 511, 484, 475, 563, 521, 513, 510, 527, 536, 540, 487, 496, 511, 513, 45, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 479, 538, 527, 519, 506, 514, 578, 505, 519, 480, 502, 563, 540, 493, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 506, 540, 486, 511, 531, 492, 520, 533, 507, 462, 506, 495, 506, 496, 571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 460, 533, 488, 475, 505, 503, 494, 485, 505, 508, 535, 501, 519, 477, 506, 16, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 539, 473, 495, 497, 520, 539, 520, 532, 504, 517, 494, 504, 528, 517, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8159, 8163, 8205, 8277, 8285, 8209, 8018, 8220] avg=8192.0000 max=8285 max_rank=4 min=8018 min_rank=6 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32732 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[484, 533, 537, 547, 499, 497, 547, 503, 489, 522, 487, 514, 467, 533, 500, 493, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 516, 529, 474, 537, 527, 497, 517, 508, 505, 496, 530, 476, 514, 498, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 497, 513, 521, 524, 490, 531, 547, 515, 512, 508, 492, 505, 503, 485, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 560, 527, 521, 529, 495, 520, 499, 491, 529, 521, 530, 514, 525, 514, 512, 483, 44, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 522, 521, 536, 483, 488, 478, 540, 486, 504, 499, 501, 489, 492, 496, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 485, 517, 515, 540, 514, 519, 523, 529, 537, 512, 496, 538, 511, 505, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 506, 503, 549, 519, 505, 507, 521, 483, 526, 525, 544, 523, 459, 502, 536, 16, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 564, 515, 488, 511, 504, 498, 487, 510, 492, 520, 544, 499, 493, 483, 515, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8153, 8172, 8181, 8325, 8063, 8232, 8248, 8162] avg=8192.0000 max=8325 max_rank=3 min=8063 min_rank=4 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32705 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[497, 498, 521, 534, 515, 554, 498, 520, 522, 488, 529, 541, 525, 516, 530, 499, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 473, 520, 509, 520, 494, 485, 523, 538, 515, 511, 488, 513, 540, 535, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 491, 513, 500, 519, 537, 541, 500, 503, 548, 556, 501, 529, 518, 498, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 474, 495, 516, 496, 506, 519, 493, 518, 515, 490, 465, 500, 497, 537, 539, 39, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 483, 498, 517, 512, 543, 511, 491, 438, 497, 512, 518, 541, 490, 543, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 466, 470, 538, 556, 525, 534, 491, 551, 538, 522, 504, 498, 515, 539, 483, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 520, 477, 520, 498, 486, 516, 514, 523, 547, 493, 475, 500, 516, 476, 528, 16, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 477, 503, 510, 477, 524, 549, 525, 503, 543, 514, 516, 500, 520, 516, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8288, 8170, 8309, 8117, 8073, 8229, 8151, 8199] avg=8192.0000 max=8309 max_rank=2 min=8073 min_rank=4 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32652 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 3968, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 3968, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 3968, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 3968, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 3968, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[530, 537, 527, 499, 493, 536, 486, 477, 529, 537, 513, 515, 503, 531, 520, 513, 468, 544, 513, 559, 480, 490, 532, 510, 512, 491, 519, 514, 525, 555, 494, 527, 511, 506, 511, 529, 549, 528, 465, 472, 557, 530, 506, 502, 488, 529, 537, 511, 496, 471, 492, 515, 465, 520, 499, 535, 537, 515, 518, 569, 503, 480, 534, 490, 486, 483, 486, 544, 515, 515, 512, 488, 559, 486, 485, 502, 508, 482, 489, 501, 500, 530, 522, 512, 494, 529, 503, 511, 499, 541, 484, 533, 504, 485, 516, 518, 539, 505, 514, 524, 554, 503, 531, 509, 508, 527, 528, 519, 516, 498, 548, 501, 495, 497, 508, 538, 528, 518, 471, 503, 531, 498, 512, 499, 515, 502, 514, 512] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[510, 529, 486, 530, 497, 515, 510, 521, 521, 518, 543, 509, 510, 484, 521, 524, 508, 540, 490, 549, 472, 497, 537, 502, 520, 537, 508, 530, 563, 481, 526, 491, 527, 525, 548, 521, 532, 458, 483, 556, 486, 534, 524, 527, 472, 517, 494, 531, 519, 507, 451, 484, 539, 499, 542, 512, 530, 512, 496, 470, 548, 506, 501, 557, 499, 490, 503, 485, 517, 516, 529, 548, 536, 523, 491, 479, 478, 497, 544, 526, 460, 517, 501, 567, 498, 522, 517, 519, 524, 491, 534, 544, 544, 484, 471, 507, 507, 504, 514, 497, 534, 491, 495, 484, 485, 509, 495, 501, 530, 529, 530, 514, 527, 508, 501, 523, 515, 505, 516, 481, 491, 502, 481, 509, 567, 499, 526, 518] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[483, 539, 523, 500, 547, 526, 498, 540, 499, 498, 512, 521, 518, 520, 506, 491, 479, 482, 479, 500, 502, 542, 523, 526, 509, 514, 485, 514, 503, 535, 503, 488, 505, 486, 509, 519, 514, 536, 514, 479, 485, 491, 528, 496, 528, 475, 548, 533, 524, 481, 520, 518, 521, 559, 540, 515, 494, 497, 485, 505, 546, 482, 542, 516, 527, 503, 525, 488, 508, 515, 488, 501, 517, 527, 532, 523, 499, 507, 538, 466, 513, 493, 489, 475, 551, 498, 485, 516, 557, 535, 539, 536, 499, 503, 503, 526, 515, 506, 509, 533, 518, 558, 521, 486, 521, 523, 481, 532, 547, 516, 496, 511, 512, 509, 544, 517, 510, 536, 513, 511, 517, 500, 466, 544, 469, 531, 515, 491] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[528, 504, 486, 528, 539, 506, 450, 514, 432, 519, 518, 503, 535, 529, 508, 534, 556, 568, 509, 520, 488, 486, 533, 495, 500, 525, 499, 511, 536, 508, 495, 511, 478, 502, 545, 529, 500, 522, 509, 497, 499, 489, 509, 498, 541, 523, 509, 516, 500, 510, 520, 490, 485, 508, 509, 469, 535, 497, 505, 491, 480, 551, 477, 551, 515, 503, 522, 519, 535, 499, 491, 473, 511, 523, 513, 520, 539, 475, 515, 516, 503, 510, 504, 524, 519, 514, 485, 526, 491, 532, 518, 509, 563, 514, 495, 522, 467, 544, 524, 490, 499, 533, 532, 522, 502, 510, 529, 522, 507, 508, 541, 517, 526, 530, 498, 495, 530, 521, 540, 482, 531, 474, 536, 521, 529, 559, 513, 489] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[530, 537, 527, 499, 493, 536, 486, 477, 529, 537, 513, 515, 503, 531, 520, 513, 468, 544, 513, 559, 480, 490, 532, 510, 512, 491, 519, 514, 525, 555, 494, 527, 511, 506, 511, 529, 549, 528, 465, 472, 557, 530, 506, 502, 488, 529, 537, 511, 496, 471, 492, 515, 465, 520, 499, 535, 537, 515, 518, 569, 503, 480, 534, 490, 486, 483, 486, 544, 515, 515, 512, 488, 559, 486, 485, 502, 508, 482, 489, 501, 500, 530, 522, 512, 494, 529, 503, 511, 499, 541, 484, 533, 504, 485, 516, 518, 539, 505, 514, 524, 554, 503, 531, 509, 508, 527, 528, 519, 516, 498, 548, 501, 495, 497, 508, 538, 528, 518, 471, 503, 531, 498, 512, 499, 515, 502, 514, 512] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[497, 495, 539, 446, 530, 506, 534, 526, 546, 530, 543, 519, 532, 546, 523, 502, 536, 515, 515, 542, 496, 553, 540, 513, 472, 505, 497, 515, 533, 479, 488, 485, 534, 562, 509, 518, 495, 498, 514, 499, 486, 519, 516, 522, 512, 499, 502, 499, 550, 482, 505, 489, 545, 500, 492, 532, 532, 502, 523, 503, 545, 525, 504, 483, 473, 496, 507, 520, 463, 542, 492, 501, 503, 518, 527, 532, 504, 503, 520, 520, 539, 491, 504, 501, 502, 505, 517, 551, 494, 510, 513, 561, 510, 543, 492, 508, 509, 452, 494, 522, 519, 476, 487, 493, 495, 498, 510, 555, 520, 514, 529, 529, 501, 527, 523, 498, 501, 558, 511, 543, 521, 538, 515, 484, 495, 486, 484, 493] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[526, 526, 536, 528, 512, 521, 514, 494, 514, 505, 476, 512, 495, 483, 478, 537, 505, 520, 503, 523, 508, 513, 526, 514, 537, 524, 487, 515, 506, 540, 527, 505, 521, 516, 543, 506, 516, 518, 516, 545, 522, 507, 519, 531, 483, 519, 473, 489, 516, 507, 489, 496, 489, 510, 543, 463, 491, 499, 509, 480, 528, 508, 508, 499, 484, 521, 542, 497, 527, 527, 512, 520, 512, 488, 531, 544, 526, 500, 500, 469, 498, 508, 509, 492, 516, 467, 499, 523, 478, 516, 571, 501, 503, 553, 491, 516, 505, 556, 526, 486, 518, 520, 509, 502, 493, 556, 502, 534, 493, 486, 499, 508, 513, 538, 516, 538, 525, 550, 471, 543, 502, 510, 520, 505, 561, 491, 495, 555] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[518, 482, 505, 496, 493, 494, 509, 508, 526, 506, 515, 544, 527, 508, 512, 499, 473, 519, 470, 519, 519, 520, 523, 510, 478, 532, 496, 541, 511, 503, 480, 487, 515, 491, 514, 508, 537, 537, 479, 515, 543, 508, 531, 533, 530, 534, 497, 519, 519, 517, 523, 509, 515, 530, 543, 485, 487, 556, 519, 529, 493, 499, 510, 559, 452, 539, 518, 548, 493, 475, 518, 506, 515, 504, 536, 522, 495, 498, 502, 494, 526, 491, 513, 510, 539, 514, 491, 525, 493, 542, 550, 492, 535, 513, 503, 495, 531, 503, 468, 448, 517, 516, 482, 477, 519, 519, 478, 535, 477, 532, 520, 484, 509, 524, 525, 544, 508, 477, 536, 554, 502, 556, 525, 552, 477, 548, 500, 539] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[483, 539, 523, 500, 547, 526, 498, 540, 499, 498, 512, 521, 518, 520, 506, 491, 479, 482, 479, 500, 502, 542, 523, 526, 509, 514, 485, 514, 503, 535, 503, 488, 505, 486, 509, 519, 514, 536, 514, 479, 485, 491, 528, 496, 528, 475, 548, 533, 524, 481, 520, 518, 521, 559, 540, 515, 494, 497, 485, 505, 546, 482, 542, 516, 527, 503, 525, 488, 508, 515, 488, 501, 517, 527, 532, 523, 499, 507, 538, 466, 513, 493, 489, 475, 551, 498, 485, 516, 557, 535, 539, 536, 499, 503, 503, 526, 515, 506, 509, 533, 518, 558, 521, 486, 521, 523, 481, 532, 547, 516, 496, 511, 512, 509, 544, 517, 510, 536, 513, 511, 517, 500, 466, 544, 469, 531, 515, 491] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[500, 523, 475, 497, 533, 544, 468, 523, 524, 504, 553, 492, 532, 516, 508, 569, 493, 534, 506, 523, 538, 504, 516, 506, 523, 515, 506, 521, 505, 526, 496, 520, 522, 540, 500, 503, 519, 525, 507, 543, 507, 502, 495, 494, 519, 489, 529, 527, 527, 483, 513, 531, 528, 527, 511, 512, 484, 477, 503, 486, 506, 512, 508, 506, 534, 490, 523, 503, 495, 526, 515, 514, 500, 499, 487, 523, 490, 513, 515, 533, 499, 535, 524, 523, 522, 475, 500, 515, 515, 520, 519, 525, 508, 521, 539, 535, 503, 552, 505, 539, 533, 521, 519, 510, 485, 522, 514, 487, 544, 491, 504, 487, 461, 512, 545, 510, 492, 470, 545, 515, 475, 516, 489, 488, 546, 467, 512, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[528, 504, 486, 528, 539, 506, 450, 514, 432, 519, 499, 503, 535, 529, 508, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 568, 509, 518, 488, 486, 533, 495, 500, 525, 499, 511, 536, 508, 495, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 502, 525, 529, 500, 522, 509, 497, 499, 489, 509, 498, 541, 523, 509, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 510, 520, 490, 485, 508, 509, 469, 535, 497, 505, 491, 480, 551, 477, 551, 16, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 503, 522, 519, 535, 499, 491, 473, 511, 523, 513, 520, 539, 475, 515, 516, 22, 21, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 510, 504, 524, 519, 514, 485, 526, 491, 532, 496, 509, 563, 514, 495, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 544, 524, 490, 499, 533, 532, 522, 502, 510, 529, 522, 507, 508, 541, 517, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 530, 498, 474, 530, 521, 540, 482, 531, 474, 536, 521, 529, 559, 513, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8114, 8238, 8146, 8108, 8217, 8207, 8253, 8253] avg=8192.0000 max=8253 max_rank=6 min=8108 min_rank=3 imbalance_ratio=1.007446 max_load_reduction=57283 max_load_reduction_pct=0.874069 moved_to_echo=32930 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[530, 537, 527, 499, 493, 536, 486, 477, 529, 537, 494, 515, 503, 531, 520, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 544, 513, 557, 480, 490, 532, 510, 512, 491, 519, 514, 525, 555, 494, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 506, 492, 529, 549, 528, 465, 472, 557, 530, 506, 502, 488, 529, 537, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 471, 492, 515, 465, 520, 499, 535, 537, 515, 518, 569, 503, 480, 534, 490, 15, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 483, 486, 544, 515, 515, 512, 488, 559, 486, 485, 502, 508, 482, 489, 501, 20, 22, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 530, 522, 512, 494, 529, 503, 511, 499, 541, 464, 533, 504, 485, 516, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 505, 514, 524, 554, 503, 531, 509, 508, 527, 528, 519, 516, 498, 548, 501, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 497, 508, 516, 528, 518, 471, 503, 531, 498, 512, 499, 515, 502, 514, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8227, 8231, 8212, 8168, 8088, 8161, 8330, 8119] avg=8192.0000 max=8330 max_rank=6 min=8088 min_rank=4 imbalance_ratio=1.016846 max_load_reduction=57206 max_load_reduction_pct=0.872894 moved_to_echo=32698 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[510, 529, 486, 530, 497, 515, 510, 521, 521, 518, 543, 509, 510, 484, 521, 524, 508, 540, 490, 549, 472, 497, 537, 502, 520, 537, 508, 530, 563, 481, 526, 491, 527, 525, 548, 521, 532, 458, 483, 556, 486, 534, 524, 527, 472, 517, 494, 531, 519, 507, 451, 484, 539, 499, 542, 512, 530, 512, 496, 470, 548, 506, 501, 557, 499, 490, 503, 485, 517, 516, 529, 548, 536, 523, 491, 479, 478, 497, 544, 526, 460, 517, 501, 567, 498, 522, 517, 519, 524, 491, 534, 544, 544, 484, 471, 507, 507, 504, 514, 497, 534, 491, 495, 484, 485, 509, 495, 501, 530, 529, 530, 514, 527, 508, 501, 523, 515, 505, 516, 481, 491, 502, 481, 509, 567, 499, 526, 518] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[518, 482, 505, 496, 493, 494, 509, 508, 526, 506, 515, 544, 527, 508, 512, 499, 473, 519, 470, 519, 519, 520, 523, 510, 478, 532, 496, 541, 511, 503, 480, 487, 515, 491, 514, 508, 537, 537, 479, 515, 543, 508, 531, 533, 530, 534, 497, 519, 519, 517, 523, 509, 515, 530, 543, 485, 487, 556, 519, 529, 493, 499, 510, 559, 452, 539, 518, 548, 493, 475, 518, 506, 515, 504, 536, 522, 495, 498, 502, 494, 526, 491, 513, 510, 539, 514, 491, 525, 493, 542, 550, 492, 535, 513, 503, 495, 531, 503, 468, 448, 517, 516, 482, 477, 519, 519, 478, 535, 477, 532, 520, 484, 509, 524, 525, 544, 508, 477, 536, 554, 502, 556, 525, 552, 477, 548, 500, 539] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[497, 495, 539, 446, 530, 506, 534, 526, 546, 530, 513, 519, 532, 546, 523, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 515, 515, 534, 496, 553, 540, 513, 472, 505, 497, 515, 533, 479, 488, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 562, 483, 518, 495, 498, 514, 499, 486, 519, 516, 522, 512, 499, 502, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 482, 505, 489, 545, 500, 492, 532, 532, 502, 523, 503, 545, 525, 504, 483, 20, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 496, 507, 520, 463, 542, 492, 501, 503, 518, 527, 532, 504, 503, 520, 520, 25, 25, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 491, 504, 501, 502, 505, 517, 551, 494, 510, 488, 561, 510, 543, 492, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 452, 494, 522, 519, 476, 487, 493, 495, 498, 510, 555, 520, 514, 529, 529, 6, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 527, 523, 473, 501, 558, 511, 543, 521, 538, 515, 484, 495, 486, 484, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8284, 8176, 8158, 8252, 8181, 8216, 8116, 8153] avg=8192.0000 max=8284 max_rank=0 min=8116 min_rank=6 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32666 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[483, 539, 523, 500, 547, 526, 498, 540, 499, 498, 493, 521, 518, 520, 506, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 482, 479, 498, 502, 542, 523, 526, 509, 514, 485, 514, 503, 535, 503, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 486, 490, 519, 514, 536, 514, 479, 485, 491, 528, 496, 528, 475, 548, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 481, 520, 518, 521, 559, 540, 515, 494, 497, 485, 505, 546, 482, 542, 516, 15, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 503, 525, 488, 508, 515, 488, 501, 517, 527, 532, 523, 499, 507, 538, 466, 23, 21, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 493, 489, 475, 551, 498, 485, 516, 557, 535, 516, 536, 499, 503, 503, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 506, 509, 533, 518, 558, 521, 486, 521, 523, 481, 532, 547, 516, 496, 511, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 509, 544, 496, 510, 536, 513, 511, 517, 500, 466, 544, 469, 531, 515, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8082, 8127, 8274, 8213, 8195, 8279, 8164] avg=8192.0000 max=8279 max_rank=6 min=8082 min_rank=1 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32851 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[526, 526, 536, 528, 512, 521, 514, 494, 514, 505, 458, 512, 495, 483, 478, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 520, 503, 521, 508, 513, 526, 514, 537, 524, 487, 515, 506, 540, 527, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 516, 523, 506, 516, 518, 516, 545, 522, 507, 519, 531, 483, 519, 473, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 507, 489, 496, 489, 510, 543, 463, 491, 499, 509, 480, 528, 508, 508, 499, 16, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 521, 542, 497, 527, 527, 512, 520, 512, 488, 531, 544, 526, 500, 500, 469, 24, 22, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 508, 509, 492, 516, 467, 499, 523, 478, 516, 547, 501, 503, 553, 491, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 556, 526, 486, 518, 520, 509, 502, 493, 556, 502, 534, 493, 486, 499, 508, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 538, 516, 516, 525, 550, 471, 543, 502, 510, 520, 505, 561, 491, 495, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8139, 8251, 8204, 8064, 8251, 8117, 8199, 8311] avg=8192.0000 max=8311 max_rank=7 min=8064 min_rank=3 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32878 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[530, 537, 527, 499, 493, 536, 486, 477, 529, 537, 494, 515, 503, 531, 520, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 544, 513, 557, 480, 490, 532, 510, 512, 491, 519, 514, 525, 555, 494, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 506, 492, 529, 549, 528, 465, 472, 557, 530, 506, 502, 488, 529, 537, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 471, 492, 515, 465, 520, 499, 535, 537, 515, 518, 569, 503, 480, 534, 490, 15, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 483, 486, 544, 515, 515, 512, 488, 559, 486, 485, 502, 508, 482, 489, 501, 20, 22, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 530, 522, 512, 494, 529, 503, 511, 499, 541, 464, 533, 504, 485, 516, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 505, 514, 524, 554, 503, 531, 509, 508, 527, 528, 519, 516, 498, 548, 501, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 497, 508, 516, 528, 518, 471, 503, 531, 498, 512, 499, 515, 502, 514, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8227, 8231, 8212, 8168, 8088, 8161, 8330, 8119] avg=8192.0000 max=8330 max_rank=6 min=8088 min_rank=4 imbalance_ratio=1.016846 max_load_reduction=57206 max_load_reduction_pct=0.872894 moved_to_echo=32698 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[510, 529, 486, 530, 497, 515, 510, 521, 521, 518, 522, 509, 510, 484, 521, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 540, 490, 547, 472, 497, 537, 502, 520, 537, 508, 530, 563, 481, 526, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 525, 528, 521, 532, 458, 483, 556, 486, 534, 524, 527, 472, 517, 494, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 507, 451, 484, 539, 499, 542, 512, 530, 512, 496, 470, 548, 506, 501, 557, 16, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 490, 503, 485, 517, 516, 529, 548, 536, 523, 491, 479, 478, 497, 544, 526, 22, 22, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 460, 517, 501, 567, 498, 522, 517, 519, 524, 491, 512, 544, 544, 484, 471, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 504, 514, 497, 534, 491, 495, 484, 485, 509, 495, 501, 530, 529, 530, 514, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 508, 501, 501, 515, 505, 516, 481, 491, 502, 481, 509, 567, 499, 526, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8249, 8215, 8204, 8211, 8178, 8125, 8147] avg=8192.0000 max=8249 max_rank=1 min=8125 min_rank=6 imbalance_ratio=1.006958 max_load_reduction=57287 max_load_reduction_pct=0.874130 moved_to_echo=32661 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[518, 482, 505, 496, 493, 494, 509, 508, 526, 506, 496, 544, 527, 508, 512, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 519, 470, 517, 519, 520, 523, 510, 478, 532, 496, 541, 511, 503, 480, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 491, 495, 508, 537, 537, 479, 515, 543, 508, 531, 533, 530, 534, 497, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 517, 523, 509, 515, 530, 543, 485, 487, 556, 519, 529, 493, 499, 510, 559, 15, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 452, 539, 518, 548, 493, 475, 518, 506, 515, 504, 536, 522, 495, 498, 502, 494, 23, 23, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 491, 513, 510, 539, 514, 491, 525, 493, 542, 527, 492, 535, 513, 503, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 503, 468, 448, 517, 516, 482, 477, 519, 519, 478, 535, 477, 532, 520, 484, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 524, 525, 521, 508, 477, 536, 554, 502, 556, 525, 552, 477, 548, 500, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8123, 8079, 8272, 8322, 8166, 8209, 8012, 8353] avg=8192.0000 max=8353 max_rank=7 min=8012 min_rank=6 imbalance_ratio=1.019653 max_load_reduction=57183 max_load_reduction_pct=0.872543 moved_to_echo=32740 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[526, 526, 536, 528, 512, 521, 514, 494, 514, 505, 458, 512, 495, 483, 478, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 520, 503, 521, 508, 513, 526, 514, 537, 524, 487, 515, 506, 540, 527, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 516, 523, 506, 516, 518, 516, 545, 522, 507, 519, 531, 483, 519, 473, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 507, 489, 496, 489, 510, 543, 463, 491, 499, 509, 480, 528, 508, 508, 499, 16, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 521, 542, 497, 527, 527, 512, 520, 512, 488, 531, 544, 526, 500, 500, 469, 24, 22, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 508, 509, 492, 516, 467, 499, 523, 478, 516, 547, 501, 503, 553, 491, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 556, 526, 486, 518, 520, 509, 502, 493, 556, 502, 534, 493, 486, 499, 508, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 538, 516, 516, 525, 550, 471, 543, 502, 510, 520, 505, 561, 491, 495, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8139, 8251, 8204, 8064, 8251, 8117, 8199, 8311] avg=8192.0000 max=8311 max_rank=7 min=8064 min_rank=3 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32878 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[500, 523, 475, 497, 533, 544, 468, 523, 524, 504, 532, 492, 532, 516, 508, 569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 534, 506, 521, 538, 504, 516, 506, 523, 515, 506, 521, 505, 526, 496, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 540, 481, 503, 519, 525, 507, 543, 507, 502, 495, 494, 519, 489, 529, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 483, 513, 531, 528, 527, 511, 512, 484, 477, 503, 486, 506, 512, 508, 506, 15, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 490, 523, 503, 495, 526, 515, 514, 500, 499, 487, 523, 490, 513, 515, 533, 22, 21, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 535, 524, 523, 522, 475, 500, 515, 515, 520, 497, 525, 508, 521, 539, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 552, 505, 539, 533, 521, 519, 510, 485, 522, 514, 487, 544, 491, 504, 487, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 512, 545, 489, 492, 470, 545, 515, 475, 516, 489, 488, 546, 467, 512, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8240, 8230, 8202, 8144, 8209, 8253, 8222, 8036] avg=8192.0000 max=8253 max_rank=5 min=8036 min_rank=7 imbalance_ratio=1.007446 max_load_reduction=57283 max_load_reduction_pct=0.874069 moved_to_echo=32720 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[528, 504, 486, 528, 539, 506, 450, 514, 432, 519, 499, 503, 535, 529, 508, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 568, 509, 518, 488, 486, 533, 495, 500, 525, 499, 511, 536, 508, 495, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 502, 525, 529, 500, 522, 509, 497, 499, 489, 509, 498, 541, 523, 509, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 510, 520, 490, 485, 508, 509, 469, 535, 497, 505, 491, 480, 551, 477, 551, 16, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 503, 522, 519, 535, 499, 491, 473, 511, 523, 513, 520, 539, 475, 515, 516, 22, 21, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 510, 504, 524, 519, 514, 485, 526, 491, 532, 496, 509, 563, 514, 495, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 544, 524, 490, 499, 533, 532, 522, 502, 510, 529, 522, 507, 508, 541, 517, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 530, 498, 474, 530, 521, 540, 482, 531, 474, 536, 521, 529, 559, 513, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8114, 8238, 8146, 8108, 8217, 8207, 8253, 8253] avg=8192.0000 max=8253 max_rank=6 min=8108 min_rank=3 imbalance_ratio=1.007446 max_load_reduction=57283 max_load_reduction_pct=0.874069 moved_to_echo=32930 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[483, 539, 523, 500, 547, 526, 498, 540, 499, 498, 493, 521, 518, 520, 506, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 482, 479, 498, 502, 542, 523, 526, 509, 514, 485, 514, 503, 535, 503, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 486, 490, 519, 514, 536, 514, 479, 485, 491, 528, 496, 528, 475, 548, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 481, 520, 518, 521, 559, 540, 515, 494, 497, 485, 505, 546, 482, 542, 516, 15, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 503, 525, 488, 508, 515, 488, 501, 517, 527, 532, 523, 499, 507, 538, 466, 23, 21, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 493, 489, 475, 551, 498, 485, 516, 557, 535, 516, 536, 499, 503, 503, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 506, 509, 533, 518, 558, 521, 486, 521, 523, 481, 532, 547, 516, 496, 511, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 509, 544, 496, 510, 536, 513, 511, 517, 500, 466, 544, 469, 531, 515, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8082, 8127, 8274, 8213, 8195, 8279, 8164] avg=8192.0000 max=8279 max_rank=6 min=8082 min_rank=1 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32851 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 3968, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4352, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 3968, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 256, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[498, 533, 524, 540, 544, 495, 515, 516, 527, 523, 490, 492, 506, 530, 535, 497, 510, 498, 477, 532, 548, 488, 523, 463, 500, 499, 511, 535, 498, 520, 526, 498, 514, 480, 548, 530, 481, 502, 570, 519, 499, 515, 558, 506, 483, 513, 499, 477, 504, 484, 529, 526, 520, 530, 483, 535, 483, 504, 530, 515, 520, 506, 504, 487, 533, 491, 526, 485, 530, 537, 531, 489, 496, 478, 503, 532, 492, 497, 552, 481, 473, 474, 488, 529, 505, 475, 514, 493, 529, 509, 509, 534, 524, 513, 532, 541, 565, 479, 527, 502, 536, 487, 499, 521, 513, 523, 496, 541, 497, 549, 517, 494, 533, 532, 515, 490, 534, 512, 544, 490, 512, 501, 507, 505, 517, 535, 481, 542] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[518, 483, 528, 508, 531, 523, 502, 535, 532, 533, 539, 545, 484, 517, 510, 520, 505, 522, 524, 528, 503, 508, 511, 538, 485, 538, 515, 540, 503, 489, 521, 522, 547, 512, 516, 497, 533, 498, 526, 506, 493, 471, 508, 516, 448, 506, 497, 494, 503, 555, 532, 508, 499, 502, 517, 512, 527, 484, 511, 548, 508, 512, 510, 512, 523, 531, 514, 505, 517, 521, 477, 541, 500, 521, 511, 530, 518, 524, 493, 496, 515, 511, 473, 478, 537, 534, 517, 517, 505, 498, 535, 524, 510, 503, 510, 526, 533, 457, 472, 499, 556, 516, 486, 531, 489, 545, 493, 535, 513, 515, 498, 489, 523, 512, 505, 464, 520, 488, 512, 511, 478, 536, 498, 525, 525, 491, 521, 517] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[518, 504, 501, 471, 512, 512, 516, 507, 509, 547, 465, 486, 547, 511, 519, 518, 499, 493, 498, 516, 506, 538, 484, 521, 523, 537, 517, 529, 464, 497, 511, 525, 563, 519, 513, 515, 510, 536, 536, 518, 515, 516, 539, 502, 484, 508, 478, 513, 515, 533, 471, 466, 531, 518, 539, 492, 489, 468, 539, 475, 531, 531, 503, 520, 485, 503, 541, 522, 524, 511, 513, 507, 507, 487, 512, 509, 523, 492, 507, 499, 518, 535, 514, 549, 533, 502, 570, 466, 502, 516, 553, 515, 529, 544, 519, 477, 506, 535, 518, 547, 497, 506, 502, 506, 493, 510, 490, 520, 521, 490, 512, 519, 549, 542, 528, 494, 536, 495, 504, 502, 466, 474, 559, 508, 495, 522, 498, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[525, 477, 508, 522, 548, 505, 493, 500, 517, 503, 528, 539, 513, 512, 510, 556, 501, 503, 532, 504, 576, 543, 503, 519, 514, 500, 508, 459, 548, 511, 490, 514, 552, 518, 527, 487, 488, 524, 520, 491, 497, 535, 534, 468, 528, 504, 503, 507, 492, 530, 533, 488, 541, 518, 499, 529, 490, 507, 533, 489, 515, 516, 522, 557, 560, 549, 505, 511, 514, 496, 486, 498, 490, 478, 541, 523, 496, 475, 549, 498, 517, 463, 524, 520, 494, 555, 501, 495, 512, 478, 526, 477, 519, 476, 496, 496, 521, 502, 531, 503, 507, 517, 511, 523, 528, 515, 487, 500, 533, 486, 513, 529, 539, 496, 539, 560, 483, 504, 491, 520, 545, 482, 528, 502, 477, 522, 513, 488] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[481, 518, 507, 513, 487, 525, 512, 515, 501, 518, 497, 489, 517, 517, 497, 516, 521, 508, 499, 522, 491, 533, 477, 486, 520, 511, 520, 532, 500, 526, 534, 480, 532, 509, 505, 493, 533, 551, 531, 518, 508, 518, 513, 528, 521, 508, 493, 494, 524, 485, 477, 507, 500, 472, 496, 519, 553, 519, 511, 527, 527, 517, 513, 539, 535, 523, 514, 532, 518, 511, 488, 482, 506, 489, 526, 512, 559, 525, 513, 508, 501, 485, 506, 520, 493, 506, 517, 525, 521, 518, 515, 519, 516, 503, 510, 520, 523, 522, 531, 508, 524, 526, 480, 533, 519, 537, 505, 485, 560, 514, 515, 481, 500, 523, 518, 539, 523, 488, 474, 500, 518, 498, 509, 528, 513, 558, 454, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[523, 514, 555, 552, 493, 539, 521, 513, 496, 495, 506, 579, 541, 515, 502, 526, 525, 519, 535, 466, 523, 540, 535, 540, 525, 491, 497, 486, 542, 484, 484, 499, 468, 506, 528, 504, 512, 507, 539, 487, 512, 514, 512, 533, 543, 488, 515, 516, 533, 519, 489, 505, 523, 495, 507, 506, 540, 522, 550, 510, 508, 513, 531, 505, 520, 489, 513, 494, 542, 488, 499, 480, 511, 512, 566, 493, 494, 517, 500, 512, 533, 501, 515, 475, 562, 521, 539, 476, 496, 525, 516, 512, 484, 550, 516, 456, 528, 497, 519, 476, 550, 525, 521, 530, 534, 497, 495, 491, 513, 519, 507, 489, 510, 509, 488, 526, 487, 510, 504, 491, 507, 483, 466, 459, 520, 484, 551, 542] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[518, 483, 528, 508, 531, 523, 502, 535, 532, 533, 539, 545, 484, 517, 510, 520, 505, 522, 524, 528, 503, 508, 511, 538, 485, 538, 515, 540, 503, 489, 521, 522, 547, 512, 516, 497, 533, 498, 526, 506, 493, 471, 508, 516, 448, 506, 497, 494, 503, 555, 532, 508, 499, 502, 517, 512, 527, 484, 511, 548, 508, 512, 510, 512, 523, 531, 514, 505, 517, 521, 477, 541, 500, 521, 511, 530, 518, 524, 493, 496, 515, 511, 473, 478, 537, 534, 517, 517, 505, 498, 535, 524, 510, 503, 510, 526, 533, 457, 472, 499, 556, 516, 486, 531, 489, 545, 493, 535, 513, 515, 498, 489, 523, 512, 505, 464, 520, 488, 512, 511, 478, 536, 498, 525, 525, 491, 521, 517] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[498, 533, 524, 540, 544, 495, 515, 516, 527, 523, 490, 492, 506, 530, 535, 497, 510, 498, 477, 532, 548, 488, 523, 463, 500, 499, 511, 535, 498, 520, 526, 498, 514, 480, 548, 530, 481, 502, 570, 519, 499, 515, 558, 506, 483, 513, 499, 477, 504, 484, 529, 526, 520, 530, 483, 535, 483, 504, 530, 515, 520, 506, 504, 487, 533, 491, 526, 485, 530, 537, 531, 489, 496, 478, 503, 532, 492, 497, 552, 481, 473, 474, 488, 529, 505, 475, 514, 493, 529, 509, 509, 534, 524, 513, 532, 541, 565, 479, 527, 502, 536, 487, 499, 521, 513, 523, 496, 541, 497, 549, 517, 494, 533, 532, 515, 490, 534, 512, 544, 490, 512, 501, 507, 505, 517, 535, 481, 542] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[534, 523, 511, 508, 516, 475, 539, 514, 527, 464, 496, 497, 539, 485, 502, 519, 509, 529, 475, 544, 509, 547, 528, 540, 498, 507, 528, 550, 495, 531, 529, 465, 478, 528, 461, 509, 488, 507, 522, 514, 498, 469, 531, 539, 518, 525, 519, 502, 485, 557, 492, 545, 524, 513, 527, 542, 501, 509, 492, 513, 487, 488, 517, 570, 497, 498, 513, 494, 515, 515, 531, 538, 517, 500, 527, 540, 536, 503, 542, 506, 522, 504, 534, 510, 506, 501, 540, 509, 527, 508, 532, 484, 474, 505, 521, 515, 486, 522, 538, 504, 499, 488, 506, 499, 491, 521, 495, 541, 504, 506, 506, 485, 517, 497, 528, 518, 529, 512, 516, 505, 496, 486, 532, 536, 496, 475, 526, 509] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[518, 504, 501, 471, 512, 512, 516, 507, 509, 547, 465, 486, 547, 511, 519, 518, 499, 493, 498, 516, 506, 538, 484, 521, 523, 537, 517, 529, 464, 497, 511, 525, 563, 519, 513, 515, 510, 536, 536, 518, 515, 516, 539, 502, 484, 508, 478, 513, 515, 533, 471, 466, 531, 518, 539, 492, 489, 468, 539, 475, 531, 531, 503, 520, 485, 503, 541, 522, 524, 511, 513, 507, 507, 487, 512, 509, 523, 492, 507, 499, 518, 535, 514, 549, 533, 502, 570, 466, 502, 516, 553, 515, 529, 544, 519, 477, 506, 535, 518, 547, 497, 506, 502, 506, 493, 510, 490, 520, 521, 490, 512, 519, 549, 542, 528, 494, 536, 495, 504, 502, 466, 474, 559, 508, 495, 522, 498, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[481, 518, 507, 513, 487, 525, 512, 515, 501, 518, 497, 489, 517, 517, 497, 516, 521, 508, 499, 522, 491, 533, 477, 486, 520, 511, 520, 532, 500, 526, 534, 480, 532, 509, 505, 493, 533, 551, 531, 518, 508, 518, 513, 528, 521, 508, 493, 494, 524, 485, 477, 507, 500, 472, 496, 519, 553, 519, 511, 527, 527, 517, 513, 539, 535, 523, 514, 532, 518, 511, 488, 482, 506, 489, 526, 512, 559, 525, 513, 508, 501, 485, 506, 520, 493, 506, 517, 525, 521, 518, 515, 519, 516, 503, 510, 520, 523, 522, 531, 508, 524, 526, 480, 533, 519, 537, 505, 485, 560, 514, 515, 481, 500, 523, 518, 539, 523, 488, 474, 500, 518, 498, 509, 528, 513, 558, 454, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[536, 526, 529, 506, 473, 498, 502, 492, 506, 516, 498, 527, 557, 534, 505, 528, 508, 478, 498, 500, 490, 512, 523, 522, 513, 539, 506, 512, 509, 485, 506, 512, 484, 571, 528, 523, 519, 521, 480, 506, 527, 495, 525, 535, 518, 515, 503, 542, 513, 525, 482, 518, 527, 474, 527, 507, 496, 533, 480, 489, 503, 528, 491, 507, 513, 519, 484, 528, 517, 516, 510, 481, 505, 558, 507, 515, 494, 545, 515, 515, 554, 496, 532, 526, 511, 513, 488, 506, 515, 519, 533, 517, 524, 499, 498, 502, 552, 520, 544, 508, 480, 488, 512, 530, 512, 552, 502, 509, 503, 524, 493, 461, 486, 530, 524, 500, 512, 506, 496, 517, 471, 526, 557, 490, 485, 515, 534, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[498, 533, 524, 540, 544, 495, 515, 516, 527, 523, 490, 492, 471, 530, 535, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 498, 477, 532, 548, 488, 523, 463, 500, 499, 511, 535, 498, 520, 526, 498, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 480, 548, 530, 481, 502, 569, 519, 499, 515, 558, 506, 483, 513, 499, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 484, 529, 526, 520, 530, 483, 535, 483, 504, 530, 515, 520, 506, 504, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 491, 526, 485, 530, 537, 531, 489, 496, 478, 502, 532, 492, 497, 552, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 474, 488, 529, 505, 475, 514, 493, 529, 509, 509, 534, 524, 513, 532, 541, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 565, 479, 527, 502, 536, 487, 499, 521, 513, 523, 496, 541, 497, 549, 517, 494, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 532, 515, 490, 534, 512, 544, 490, 512, 501, 507, 505, 517, 535, 481, 542, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 8128, 8193, 8156, 8152, 8145, 8251, 8281] avg=8192.0000 max=8281 max_rank=7 min=8128 min_rank=1 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32829 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[523, 514, 555, 552, 493, 539, 521, 513, 496, 495, 506, 579, 541, 515, 502, 526, 525, 519, 535, 466, 523, 540, 535, 540, 525, 491, 497, 486, 542, 484, 484, 499, 468, 506, 528, 504, 512, 507, 539, 487, 512, 514, 512, 533, 543, 488, 515, 516, 533, 519, 489, 505, 523, 495, 507, 506, 540, 522, 550, 510, 508, 513, 531, 505, 520, 489, 513, 494, 542, 488, 499, 480, 511, 512, 566, 493, 494, 517, 500, 512, 533, 501, 515, 475, 562, 521, 539, 476, 496, 525, 516, 512, 484, 550, 516, 456, 528, 497, 519, 476, 550, 525, 521, 530, 534, 497, 495, 491, 513, 519, 507, 489, 510, 509, 488, 526, 487, 510, 504, 491, 507, 483, 466, 459, 520, 484, 551, 542] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[518, 504, 501, 471, 512, 512, 516, 507, 509, 547, 465, 486, 501, 511, 519, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 493, 498, 516, 506, 538, 484, 521, 523, 537, 517, 529, 464, 497, 511, 525, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 563, 519, 513, 515, 510, 536, 530, 518, 515, 516, 539, 502, 484, 508, 478, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 533, 471, 466, 531, 518, 539, 492, 489, 468, 539, 475, 531, 531, 503, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 503, 541, 522, 524, 511, 513, 507, 507, 487, 504, 509, 523, 492, 507, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 535, 514, 549, 533, 502, 570, 466, 502, 516, 553, 515, 529, 544, 519, 477, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 535, 518, 547, 497, 506, 502, 506, 493, 510, 490, 520, 521, 490, 512, 519, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 542, 528, 494, 536, 495, 504, 502, 466, 474, 559, 508, 495, 522, 498, 521, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8097, 8171, 8259, 8107, 8134, 8351, 8185, 8232] avg=8192.0000 max=8351 max_rank=5 min=8097 min_rank=0 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32902 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[525, 477, 508, 522, 548, 505, 493, 500, 517, 503, 528, 539, 477, 512, 510, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 503, 532, 504, 576, 543, 503, 519, 514, 500, 508, 459, 548, 511, 490, 514, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 518, 527, 487, 488, 524, 519, 491, 497, 535, 534, 468, 528, 504, 503, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 530, 533, 488, 541, 518, 499, 529, 490, 507, 533, 489, 515, 516, 522, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 560, 549, 505, 511, 514, 496, 486, 498, 490, 478, 540, 523, 496, 475, 549, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 463, 524, 520, 494, 555, 501, 495, 512, 478, 526, 477, 519, 476, 496, 496, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 502, 531, 503, 507, 517, 511, 523, 528, 515, 487, 500, 533, 486, 513, 529, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 496, 539, 560, 483, 504, 491, 520, 545, 482, 528, 502, 477, 522, 513, 488, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8220, 8227, 8182, 8253, 8168, 8053, 8212, 8221] avg=8192.0000 max=8253 max_rank=3 min=8053 min_rank=5 imbalance_ratio=1.007446 max_load_reduction=57283 max_load_reduction_pct=0.874069 moved_to_echo=32654 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[498, 533, 524, 540, 544, 495, 515, 516, 527, 523, 490, 492, 471, 530, 535, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 498, 477, 532, 548, 488, 523, 463, 500, 499, 511, 535, 498, 520, 526, 498, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 480, 548, 530, 481, 502, 569, 519, 499, 515, 558, 506, 483, 513, 499, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 484, 529, 526, 520, 530, 483, 535, 483, 504, 530, 515, 520, 506, 504, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 491, 526, 485, 530, 537, 531, 489, 496, 478, 502, 532, 492, 497, 552, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 474, 488, 529, 505, 475, 514, 493, 529, 509, 509, 534, 524, 513, 532, 541, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 565, 479, 527, 502, 536, 487, 499, 521, 513, 523, 496, 541, 497, 549, 517, 494, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 532, 515, 490, 534, 512, 544, 490, 512, 501, 507, 505, 517, 535, 481, 542, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 8128, 8193, 8156, 8152, 8145, 8251, 8281] avg=8192.0000 max=8281 max_rank=7 min=8128 min_rank=1 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32829 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[534, 523, 511, 508, 516, 475, 539, 514, 527, 464, 496, 497, 502, 485, 502, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 529, 475, 544, 509, 547, 528, 540, 498, 507, 528, 550, 495, 531, 529, 465, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 528, 461, 509, 488, 507, 521, 514, 498, 469, 531, 539, 518, 525, 519, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 557, 492, 545, 524, 513, 527, 542, 501, 509, 492, 513, 487, 488, 517, 564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 498, 513, 494, 515, 515, 531, 538, 517, 500, 526, 540, 536, 503, 542, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 504, 534, 510, 506, 501, 540, 509, 527, 508, 532, 484, 474, 505, 521, 515, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 522, 538, 504, 499, 488, 506, 499, 491, 521, 495, 541, 504, 506, 506, 485, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 497, 528, 518, 529, 512, 516, 505, 496, 486, 532, 536, 496, 475, 526, 509, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8112, 8286, 8107, 8256, 8271, 8196, 8097, 8211] avg=8192.0000 max=8286 max_rank=1 min=8097 min_rank=6 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32775 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[518, 483, 528, 508, 531, 523, 502, 535, 532, 533, 539, 545, 451, 517, 510, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 522, 524, 528, 503, 508, 511, 538, 485, 538, 515, 540, 503, 489, 521, 522, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 512, 516, 497, 533, 498, 525, 506, 493, 471, 508, 516, 448, 506, 497, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 555, 532, 508, 499, 502, 517, 512, 527, 484, 511, 548, 508, 512, 510, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 531, 514, 505, 517, 521, 477, 541, 500, 521, 510, 530, 518, 524, 493, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 511, 473, 478, 537, 534, 517, 517, 505, 498, 535, 524, 510, 503, 510, 526, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 457, 472, 499, 556, 516, 486, 531, 489, 545, 493, 535, 513, 515, 498, 489, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 512, 505, 464, 520, 488, 512, 511, 478, 536, 498, 525, 525, 491, 521, 517, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8275, 8254, 8067, 8236, 8221, 8196, 8131, 8156] avg=8192.0000 max=8275 max_rank=0 min=8067 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32704 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[518, 504, 501, 471, 512, 512, 516, 507, 509, 547, 465, 486, 501, 511, 519, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 493, 498, 516, 506, 538, 484, 521, 523, 537, 517, 529, 464, 497, 511, 525, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 563, 519, 513, 515, 510, 536, 530, 518, 515, 516, 539, 502, 484, 508, 478, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 533, 471, 466, 531, 518, 539, 492, 489, 468, 539, 475, 531, 531, 503, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 503, 541, 522, 524, 511, 513, 507, 507, 487, 504, 509, 523, 492, 507, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 535, 514, 549, 533, 502, 570, 466, 502, 516, 553, 515, 529, 544, 519, 477, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 535, 518, 547, 497, 506, 502, 506, 493, 510, 490, 520, 521, 490, 512, 519, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 542, 528, 494, 536, 495, 504, 502, 466, 474, 559, 508, 495, 522, 498, 521, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8097, 8171, 8259, 8107, 8134, 8351, 8185, 8232] avg=8192.0000 max=8351 max_rank=5 min=8097 min_rank=0 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32902 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[481, 518, 507, 513, 487, 525, 512, 515, 501, 518, 497, 489, 481, 517, 497, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 508, 499, 522, 491, 533, 477, 486, 520, 511, 520, 532, 500, 526, 534, 480, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 509, 505, 493, 533, 551, 530, 518, 508, 518, 513, 528, 521, 508, 493, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 485, 477, 507, 500, 472, 496, 519, 553, 519, 511, 527, 527, 517, 513, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 523, 514, 532, 518, 511, 488, 482, 506, 489, 525, 512, 559, 525, 513, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 485, 506, 520, 493, 506, 517, 525, 521, 518, 515, 519, 516, 503, 510, 520, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 522, 531, 508, 524, 526, 480, 533, 519, 537, 505, 485, 560, 514, 515, 481, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 523, 518, 539, 523, 488, 474, 500, 518, 498, 509, 528, 513, 558, 454, 503, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8074, 8162, 8254, 8180, 8240, 8179, 8269, 8178] avg=8192.0000 max=8269 max_rank=6 min=8074 min_rank=0 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32866 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[534, 523, 511, 508, 516, 475, 539, 514, 527, 464, 496, 497, 502, 485, 502, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 529, 475, 544, 509, 547, 528, 540, 498, 507, 528, 550, 495, 531, 529, 465, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 528, 461, 509, 488, 507, 521, 514, 498, 469, 531, 539, 518, 525, 519, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 557, 492, 545, 524, 513, 527, 542, 501, 509, 492, 513, 487, 488, 517, 564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 498, 513, 494, 515, 515, 531, 538, 517, 500, 526, 540, 536, 503, 542, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 504, 534, 510, 506, 501, 540, 509, 527, 508, 532, 484, 474, 505, 521, 515, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 522, 538, 504, 499, 488, 506, 499, 491, 521, 495, 541, 504, 506, 506, 485, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 497, 528, 518, 529, 512, 516, 505, 496, 486, 532, 536, 496, 475, 526, 509, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8112, 8286, 8107, 8256, 8271, 8196, 8097, 8211] avg=8192.0000 max=8286 max_rank=1 min=8097 min_rank=6 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32775 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[536, 526, 529, 506, 473, 498, 502, 492, 506, 516, 498, 527, 519, 534, 505, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 478, 498, 500, 490, 512, 523, 522, 513, 539, 506, 512, 509, 485, 506, 512, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 571, 528, 523, 519, 521, 479, 506, 527, 495, 525, 535, 518, 515, 503, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 525, 482, 518, 527, 474, 527, 507, 496, 533, 480, 489, 503, 528, 491, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 519, 484, 528, 517, 516, 510, 481, 505, 558, 506, 515, 494, 545, 515, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 496, 532, 526, 511, 513, 488, 506, 515, 519, 533, 517, 524, 499, 498, 502, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 520, 544, 508, 480, 488, 512, 530, 512, 552, 502, 509, 503, 524, 493, 461, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 530, 524, 500, 512, 506, 496, 517, 471, 526, 557, 490, 485, 515, 534, 504, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8115, 8291, 8096, 8221, 8236, 8195, 8187] avg=8192.0000 max=8291 max_rank=2 min=8096 min_rank=3 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32839 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[523, 514, 555, 552, 493, 539, 521, 513, 496, 495, 506, 579, 504, 515, 502, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 519, 535, 466, 523, 540, 535, 540, 525, 491, 497, 486, 542, 484, 484, 499, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 506, 528, 504, 512, 507, 538, 487, 512, 514, 512, 533, 543, 488, 515, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 519, 489, 505, 523, 495, 507, 506, 540, 522, 550, 510, 508, 513, 531, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 489, 513, 494, 542, 488, 499, 480, 511, 512, 565, 493, 494, 517, 500, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 501, 515, 475, 562, 521, 539, 476, 496, 525, 516, 512, 484, 550, 516, 456, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 497, 519, 476, 550, 525, 521, 530, 534, 497, 495, 491, 513, 519, 507, 489, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 509, 488, 526, 487, 510, 504, 491, 507, 483, 466, 459, 520, 484, 551, 542, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8333, 8193, 8183, 8252, 8129, 8180, 8196, 8070] avg=8192.0000 max=8333 max_rank=0 min=8070 min_rank=7 imbalance_ratio=1.017212 max_load_reduction=57203 max_load_reduction_pct=0.872849 moved_to_echo=32575 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 3968, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 3968, 4224, 4224, 4224, 4096, 3968, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 3968, 4224, 4224, 4224, 4096, 3968, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[521, 474, 519, 483, 472, 519, 576, 499, 548, 515, 524, 522, 487, 519, 506, 497, 491, 504, 491, 522, 531, 531, 513, 502, 533, 526, 499, 509, 502, 528, 512, 507, 509, 516, 504, 506, 497, 536, 499, 506, 542, 484, 512, 513, 542, 488, 535, 522, 529, 511, 495, 533, 485, 522, 469, 472, 536, 526, 501, 525, 507, 519, 531, 548, 522, 536, 538, 527, 514, 525, 514, 504, 499, 521, 468, 515, 528, 519, 521, 567, 502, 515, 521, 520, 527, 529, 509, 501, 502, 513, 515, 517, 469, 537, 522, 462, 512, 537, 511, 539, 498, 497, 509, 543, 503, 485, 513, 514, 480, 510, 496, 520, 509, 545, 517, 503, 495, 500, 494, 500, 473, 541, 503, 483, 500, 468, 539, 518] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[491, 504, 519, 481, 537, 507, 518, 498, 492, 522, 461, 491, 525, 546, 454, 530, 498, 504, 501, 482, 550, 547, 474, 466, 518, 509, 504, 479, 472, 485, 531, 500, 487, 464, 484, 565, 477, 508, 556, 501, 569, 513, 520, 503, 520, 518, 535, 544, 501, 512, 535, 541, 491, 515, 527, 550, 512, 533, 536, 480, 514, 508, 512, 526, 505, 491, 521, 529, 517, 499, 491, 543, 540, 521, 537, 503, 494, 510, 522, 503, 556, 501, 513, 570, 527, 529, 497, 507, 494, 487, 491, 533, 518, 534, 511, 488, 495, 557, 521, 519, 503, 493, 553, 550, 510, 479, 475, 509, 457, 490, 523, 504, 468, 513, 520, 503, 499, 501, 513, 518, 517, 516, 485, 523, 544, 572, 544, 527] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[567, 484, 494, 509, 496, 540, 505, 494, 520, 489, 524, 515, 504, 500, 507, 536, 513, 507, 497, 508, 496, 497, 552, 502, 509, 515, 491, 528, 495, 521, 520, 488, 514, 500, 522, 516, 528, 504, 487, 518, 495, 512, 484, 527, 483, 510, 539, 503, 506, 516, 505, 538, 483, 512, 497, 512, 502, 508, 495, 514, 477, 505, 506, 517, 489, 512, 515, 516, 487, 536, 529, 526, 528, 472, 511, 498, 514, 535, 495, 494, 537, 548, 476, 532, 546, 510, 538, 519, 578, 475, 530, 495, 482, 553, 512, 473, 529, 531, 489, 498, 516, 483, 492, 524, 503, 530, 505, 488, 472, 532, 500, 522, 487, 525, 520, 538, 527, 511, 532, 548, 516, 519, 495, 518, 536, 509, 588, 534] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[487, 514, 501, 499, 516, 510, 519, 498, 519, 496, 516, 564, 519, 509, 528, 518, 523, 506, 483, 502, 507, 524, 501, 537, 497, 515, 544, 530, 541, 495, 520, 537, 518, 525, 521, 500, 506, 530, 474, 513, 498, 520, 548, 539, 506, 494, 499, 510, 552, 506, 533, 508, 481, 492, 507, 499, 523, 525, 495, 481, 506, 519, 513, 479, 533, 517, 503, 534, 462, 529, 517, 494, 518, 535, 499, 529, 499, 483, 520, 531, 516, 516, 532, 504, 554, 575, 462, 526, 511, 510, 484, 506, 481, 501, 518, 516, 538, 502, 499, 504, 547, 531, 486, 499, 531, 553, 536, 520, 495, 515, 512, 488, 498, 543, 538, 499, 507, 518, 504, 498, 500, 486, 505, 519, 475, 483, 498, 499] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[529, 498, 523, 505, 510, 493, 464, 523, 527, 486, 514, 543, 535, 547, 515, 483, 498, 556, 478, 510, 489, 518, 478, 530, 545, 515, 497, 482, 517, 533, 541, 502, 531, 529, 522, 487, 540, 486, 476, 498, 523, 526, 511, 475, 537, 469, 502, 537, 538, 557, 495, 478, 510, 492, 550, 509, 473, 495, 541, 493, 500, 522, 515, 532, 570, 549, 516, 517, 521, 496, 476, 498, 516, 514, 567, 501, 539, 480, 484, 494, 545, 529, 473, 531, 533, 487, 487, 486, 540, 544, 506, 532, 489, 502, 483, 511, 477, 562, 470, 553, 478, 497, 532, 480, 529, 502, 532, 533, 493, 518, 539, 525, 519, 501, 501, 501, 548, 495, 530, 517, 511, 487, 510, 522, 484, 527, 461, 553] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[504, 545, 533, 495, 546, 519, 538, 537, 504, 539, 526, 481, 483, 492, 497, 450, 522, 468, 523, 489, 494, 514, 533, 494, 504, 482, 531, 554, 540, 511, 505, 520, 537, 537, 521, 521, 531, 517, 543, 504, 486, 512, 531, 542, 458, 498, 504, 474, 521, 531, 489, 487, 486, 506, 502, 549, 512, 516, 547, 493, 507, 520, 510, 513, 511, 553, 494, 522, 514, 556, 515, 530, 493, 500, 506, 520, 515, 501, 512, 514, 513, 510, 488, 542, 484, 524, 522, 487, 493, 522, 470, 514, 540, 450, 483, 499, 547, 528, 490, 508, 486, 518, 514, 506, 519, 539, 466, 516, 507, 544, 529, 494, 559, 519, 493, 516, 496, 512, 529, 535, 530, 512, 502, 507, 502, 503, 532, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[512, 498, 490, 518, 473, 513, 563, 503, 531, 466, 539, 523, 538, 554, 467, 503, 499, 514, 476, 511, 529, 515, 515, 482, 496, 519, 554, 486, 536, 509, 518, 518, 524, 545, 491, 559, 540, 553, 512, 502, 506, 500, 551, 486, 491, 500, 527, 477, 471, 510, 510, 567, 531, 529, 496, 539, 507, 537, 496, 540, 531, 501, 491, 499, 498, 526, 492, 512, 496, 537, 496, 544, 514, 500, 538, 508, 523, 509, 504, 520, 545, 509, 487, 510, 522, 527, 539, 497, 500, 514, 461, 510, 475, 472, 532, 540, 513, 520, 532, 491, 494, 532, 488, 498, 563, 471, 525, 491, 454, 513, 506, 525, 529, 504, 505, 534, 464, 531, 509, 503, 533, 511, 504, 489, 535, 497, 510, 518] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[491, 504, 519, 481, 537, 507, 518, 498, 492, 522, 461, 491, 525, 546, 454, 530, 498, 504, 501, 482, 550, 547, 474, 466, 518, 509, 504, 479, 472, 485, 531, 500, 487, 464, 484, 565, 477, 508, 556, 501, 569, 513, 520, 503, 520, 518, 535, 544, 501, 512, 535, 541, 491, 515, 527, 550, 512, 533, 536, 480, 514, 508, 512, 526, 505, 491, 521, 529, 517, 499, 491, 543, 540, 521, 537, 503, 494, 510, 522, 503, 556, 501, 513, 570, 527, 529, 497, 507, 494, 487, 491, 533, 518, 534, 511, 488, 495, 557, 521, 519, 503, 493, 553, 550, 510, 479, 475, 509, 457, 490, 523, 504, 468, 513, 520, 503, 499, 501, 513, 518, 517, 516, 485, 523, 544, 572, 544, 527] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[521, 474, 519, 483, 472, 519, 576, 499, 548, 515, 524, 522, 487, 519, 506, 497, 491, 504, 491, 522, 531, 531, 513, 502, 533, 526, 499, 509, 502, 528, 512, 507, 509, 516, 504, 506, 497, 536, 499, 506, 542, 484, 512, 513, 542, 488, 535, 522, 529, 511, 495, 533, 485, 522, 469, 472, 536, 526, 501, 525, 507, 519, 531, 548, 522, 536, 538, 527, 514, 525, 514, 504, 499, 521, 468, 515, 528, 519, 521, 567, 502, 515, 521, 520, 527, 529, 509, 501, 502, 513, 515, 517, 469, 537, 522, 462, 512, 537, 511, 539, 498, 497, 509, 543, 503, 485, 513, 514, 480, 510, 496, 520, 509, 545, 517, 503, 495, 500, 494, 500, 473, 541, 503, 483, 500, 468, 539, 518] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[549, 488, 531, 499, 517, 532, 517, 498, 533, 507, 508, 544, 501, 508, 500, 514, 513, 513, 533, 519, 504, 503, 491, 497, 507, 513, 544, 514, 538, 514, 482, 512, 537, 488, 508, 484, 496, 519, 502, 504, 492, 513, 508, 521, 525, 497, 494, 518, 495, 520, 509, 500, 564, 522, 530, 512, 491, 489, 506, 460, 520, 527, 514, 485, 506, 513, 448, 524, 557, 507, 529, 515, 556, 492, 491, 545, 499, 562, 476, 472, 534, 532, 571, 499, 509, 480, 506, 526, 497, 509, 513, 506, 483, 519, 507, 509, 506, 507, 513, 518, 519, 535, 500, 517, 512, 496, 513, 543, 556, 522, 528, 523, 499, 493, 497, 535, 519, 497, 483, 490, 542, 550, 514, 540, 506, 483, 497, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[504, 545, 533, 495, 546, 519, 538, 537, 504, 539, 526, 481, 483, 492, 497, 450, 522, 468, 523, 489, 494, 514, 533, 494, 504, 482, 531, 554, 540, 511, 505, 520, 537, 537, 521, 521, 531, 517, 543, 504, 486, 512, 531, 542, 458, 498, 504, 474, 521, 531, 489, 487, 486, 506, 502, 549, 512, 516, 547, 493, 507, 520, 510, 513, 511, 553, 494, 522, 514, 556, 515, 530, 493, 500, 506, 520, 515, 501, 512, 514, 513, 510, 488, 542, 484, 524, 522, 487, 493, 522, 470, 514, 540, 450, 483, 499, 547, 528, 490, 508, 486, 518, 514, 506, 519, 539, 466, 516, 507, 544, 529, 494, 559, 519, 493, 516, 496, 512, 529, 535, 530, 512, 502, 507, 502, 503, 532, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[567, 484, 494, 509, 496, 540, 505, 494, 520, 489, 524, 515, 504, 500, 507, 536, 513, 507, 497, 508, 496, 497, 552, 502, 509, 515, 491, 528, 495, 521, 520, 488, 514, 500, 522, 516, 528, 504, 487, 518, 495, 512, 484, 527, 483, 510, 539, 503, 506, 516, 505, 538, 483, 512, 497, 512, 502, 508, 495, 514, 477, 505, 506, 517, 489, 512, 515, 516, 487, 536, 529, 526, 528, 472, 511, 498, 514, 535, 495, 494, 537, 548, 476, 532, 546, 510, 538, 519, 578, 475, 530, 495, 482, 553, 512, 473, 529, 531, 489, 498, 516, 483, 492, 524, 503, 530, 505, 488, 472, 532, 500, 522, 487, 525, 520, 538, 527, 511, 532, 548, 516, 519, 495, 518, 536, 509, 588, 534] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[521, 474, 519, 483, 472, 519, 576, 499, 548, 515, 524, 522, 487, 519, 506, 497, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 504, 491, 522, 531, 531, 513, 502, 533, 526, 499, 509, 502, 528, 512, 507, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 516, 504, 506, 497, 536, 499, 506, 542, 484, 511, 513, 542, 488, 535, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 511, 495, 533, 485, 522, 469, 472, 536, 526, 501, 525, 507, 519, 531, 548, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 503, 538, 527, 514, 525, 514, 504, 499, 521, 468, 515, 528, 519, 521, 567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 515, 521, 520, 527, 529, 509, 501, 502, 513, 515, 517, 469, 537, 522, 462, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 537, 511, 539, 498, 497, 509, 543, 503, 485, 513, 514, 480, 510, 496, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 545, 517, 503, 495, 500, 494, 500, 473, 541, 503, 483, 500, 468, 537, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8222, 8210, 8212, 8285, 8166, 8167, 8086] avg=8192.0000 max=8285 max_rank=4 min=8086 min_rank=7 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32704 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[529, 498, 523, 505, 510, 493, 464, 523, 527, 486, 514, 543, 535, 547, 515, 483, 498, 556, 478, 510, 489, 518, 478, 530, 545, 515, 497, 482, 517, 533, 541, 502, 531, 529, 522, 487, 540, 486, 476, 498, 523, 526, 511, 475, 537, 469, 502, 537, 538, 557, 495, 478, 510, 492, 550, 509, 473, 495, 541, 493, 500, 522, 515, 532, 570, 549, 516, 517, 521, 496, 476, 498, 516, 514, 567, 501, 539, 480, 484, 494, 545, 529, 473, 531, 533, 487, 487, 486, 540, 544, 506, 532, 489, 502, 483, 511, 477, 562, 470, 553, 478, 497, 532, 480, 529, 502, 532, 533, 493, 518, 539, 525, 519, 501, 501, 501, 548, 495, 530, 517, 511, 487, 510, 522, 484, 527, 461, 553] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[567, 484, 494, 509, 496, 540, 505, 494, 520, 489, 524, 515, 504, 500, 507, 536, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 507, 497, 508, 496, 497, 552, 502, 509, 515, 491, 528, 495, 521, 520, 488, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 500, 522, 516, 528, 504, 487, 518, 495, 512, 483, 527, 483, 510, 539, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 516, 505, 538, 483, 512, 497, 512, 502, 508, 495, 514, 477, 505, 506, 517, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 480, 515, 516, 487, 536, 529, 526, 528, 472, 511, 498, 514, 535, 495, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 548, 476, 532, 546, 510, 538, 519, 578, 475, 530, 495, 482, 553, 512, 473, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 531, 489, 498, 516, 483, 492, 524, 503, 530, 505, 488, 472, 532, 500, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 525, 520, 538, 527, 511, 532, 548, 516, 519, 495, 518, 536, 509, 585, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8159, 8141, 8097, 8125, 8309, 8114, 8400] avg=8192.0000 max=8400 max_rank=7 min=8097 min_rank=3 imbalance_ratio=1.025391 max_load_reduction=57136 max_load_reduction_pct=0.871826 moved_to_echo=32948 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[491, 504, 519, 481, 537, 507, 518, 498, 492, 522, 461, 491, 525, 546, 454, 530, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 504, 501, 482, 550, 547, 474, 466, 518, 509, 504, 479, 472, 485, 531, 500, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 464, 484, 565, 477, 508, 556, 501, 569, 513, 510, 503, 520, 518, 535, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 512, 535, 541, 491, 515, 527, 550, 512, 533, 536, 480, 514, 508, 512, 526, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 449, 521, 529, 517, 499, 491, 543, 540, 521, 537, 503, 494, 510, 522, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 501, 513, 570, 527, 529, 497, 507, 494, 487, 491, 533, 518, 534, 511, 488, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 557, 521, 519, 503, 493, 553, 550, 510, 479, 475, 509, 457, 490, 523, 504, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 513, 520, 503, 499, 501, 513, 518, 517, 516, 485, 523, 544, 572, 535, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8087, 8043, 8254, 8305, 8184, 8265, 8144, 8254] avg=8192.0000 max=8305 max_rank=3 min=8043 min_rank=1 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32847 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[512, 498, 490, 518, 473, 513, 563, 503, 531, 466, 539, 523, 538, 554, 467, 503, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 514, 476, 511, 529, 515, 515, 482, 496, 519, 554, 486, 536, 509, 518, 518, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 545, 491, 559, 540, 553, 512, 502, 506, 500, 550, 486, 491, 500, 527, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 510, 510, 567, 531, 529, 496, 539, 507, 537, 496, 540, 531, 501, 491, 499, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 494, 492, 512, 496, 537, 496, 544, 514, 500, 538, 508, 523, 509, 504, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 509, 487, 510, 522, 527, 539, 497, 500, 514, 461, 510, 475, 472, 532, 540, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 520, 532, 491, 494, 532, 488, 498, 563, 471, 525, 491, 454, 513, 506, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 504, 505, 534, 464, 531, 509, 503, 533, 511, 504, 489, 535, 497, 508, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8197, 8263, 8258, 8185, 8145, 8116, 8174] avg=8192.0000 max=8263 max_rank=2 min=8116 min_rank=6 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32620 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[504, 545, 533, 495, 546, 519, 538, 537, 504, 539, 526, 481, 483, 492, 497, 450, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 468, 523, 489, 494, 514, 533, 494, 504, 482, 531, 554, 540, 511, 505, 520, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 537, 521, 521, 531, 517, 543, 504, 486, 512, 530, 542, 458, 498, 504, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 531, 489, 487, 486, 506, 502, 549, 512, 516, 547, 493, 507, 520, 510, 513, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 518, 494, 522, 514, 556, 515, 530, 493, 500, 506, 520, 515, 501, 512, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 510, 488, 542, 484, 524, 522, 487, 493, 522, 470, 514, 540, 450, 483, 499, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 528, 490, 508, 486, 518, 514, 506, 519, 539, 466, 516, 507, 544, 529, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 519, 493, 516, 496, 512, 529, 535, 530, 512, 502, 507, 502, 503, 530, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8206, 8215, 8192, 8221, 8046, 8211, 8248] avg=8192.0000 max=8248 max_rank=7 min=8046 min_rank=5 imbalance_ratio=1.006836 max_load_reduction=57288 max_load_reduction_pct=0.874146 moved_to_echo=32726 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[529, 498, 523, 505, 510, 493, 464, 523, 527, 486, 514, 543, 535, 547, 515, 483, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 556, 478, 510, 489, 518, 478, 530, 545, 515, 497, 482, 517, 533, 541, 502, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 529, 522, 487, 540, 486, 476, 498, 523, 526, 510, 475, 537, 469, 502, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 557, 495, 478, 510, 492, 550, 509, 473, 495, 541, 493, 500, 522, 515, 532, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 570, 516, 516, 517, 521, 496, 476, 498, 516, 514, 567, 501, 539, 480, 484, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 529, 473, 531, 533, 487, 487, 486, 540, 544, 506, 532, 489, 502, 483, 511, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 562, 470, 553, 478, 497, 532, 480, 529, 502, 532, 533, 493, 518, 539, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 501, 501, 501, 548, 495, 530, 517, 511, 487, 510, 522, 484, 527, 459, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8210, 8148, 8203, 8205, 8183, 8220, 8165] avg=8192.0000 max=8220 max_rank=6 min=8148 min_rank=2 imbalance_ratio=1.003418 max_load_reduction=57316 max_load_reduction_pct=0.874573 moved_to_echo=32773 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[549, 488, 531, 499, 517, 532, 517, 498, 533, 507, 508, 544, 501, 508, 500, 514, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 513, 533, 519, 504, 503, 491, 497, 507, 513, 544, 514, 538, 514, 482, 512, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 488, 508, 484, 496, 519, 502, 504, 492, 513, 507, 521, 525, 497, 494, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 520, 509, 500, 564, 522, 530, 512, 491, 489, 506, 460, 520, 527, 514, 485, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 481, 448, 524, 557, 507, 529, 515, 556, 492, 491, 545, 499, 562, 476, 472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 532, 571, 499, 509, 480, 506, 526, 497, 509, 513, 506, 483, 519, 507, 509, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 507, 513, 518, 519, 535, 500, 517, 512, 496, 513, 543, 556, 522, 528, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 493, 497, 535, 519, 497, 483, 490, 542, 550, 514, 540, 506, 483, 495, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8253, 8217, 8105, 8147, 8160, 8205, 8308, 8141] avg=8192.0000 max=8308 max_rank=6 min=8105 min_rank=2 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32814 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[504, 545, 533, 495, 546, 519, 538, 537, 504, 539, 526, 481, 483, 492, 497, 450, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 468, 523, 489, 494, 514, 533, 494, 504, 482, 531, 554, 540, 511, 505, 520, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 537, 521, 521, 531, 517, 543, 504, 486, 512, 530, 542, 458, 498, 504, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 531, 489, 487, 486, 506, 502, 549, 512, 516, 547, 493, 507, 520, 510, 513, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 518, 494, 522, 514, 556, 515, 530, 493, 500, 506, 520, 515, 501, 512, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 510, 488, 542, 484, 524, 522, 487, 493, 522, 470, 514, 540, 450, 483, 499, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 528, 490, 508, 486, 518, 514, 506, 519, 539, 466, 516, 507, 544, 529, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 519, 493, 516, 496, 512, 529, 535, 530, 512, 502, 507, 502, 503, 530, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8206, 8215, 8192, 8221, 8046, 8211, 8248] avg=8192.0000 max=8248 max_rank=7 min=8046 min_rank=5 imbalance_ratio=1.006836 max_load_reduction=57288 max_load_reduction_pct=0.874146 moved_to_echo=32726 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[567, 484, 494, 509, 496, 540, 505, 494, 520, 489, 524, 515, 504, 500, 507, 536, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 507, 497, 508, 496, 497, 552, 502, 509, 515, 491, 528, 495, 521, 520, 488, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 500, 522, 516, 528, 504, 487, 518, 495, 512, 483, 527, 483, 510, 539, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 516, 505, 538, 483, 512, 497, 512, 502, 508, 495, 514, 477, 505, 506, 517, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 480, 515, 516, 487, 536, 529, 526, 528, 472, 511, 498, 514, 535, 495, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 548, 476, 532, 546, 510, 538, 519, 578, 475, 530, 495, 482, 553, 512, 473, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 531, 489, 498, 516, 483, 492, 524, 503, 530, 505, 488, 472, 532, 500, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 525, 520, 538, 527, 511, 532, 548, 516, 519, 495, 518, 536, 509, 585, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8159, 8141, 8097, 8125, 8309, 8114, 8400] avg=8192.0000 max=8400 max_rank=7 min=8097 min_rank=3 imbalance_ratio=1.025391 max_load_reduction=57136 max_load_reduction_pct=0.871826 moved_to_echo=32948 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[491, 504, 519, 481, 537, 507, 518, 498, 492, 522, 461, 491, 525, 546, 454, 530, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 504, 501, 482, 550, 547, 474, 466, 518, 509, 504, 479, 472, 485, 531, 500, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 464, 484, 565, 477, 508, 556, 501, 569, 513, 510, 503, 520, 518, 535, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 512, 535, 541, 491, 515, 527, 550, 512, 533, 536, 480, 514, 508, 512, 526, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 449, 521, 529, 517, 499, 491, 543, 540, 521, 537, 503, 494, 510, 522, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 501, 513, 570, 527, 529, 497, 507, 494, 487, 491, 533, 518, 534, 511, 488, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 557, 521, 519, 503, 493, 553, 550, 510, 479, 475, 509, 457, 490, 523, 504, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 513, 520, 503, 499, 501, 513, 518, 517, 516, 485, 523, 544, 572, 535, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8087, 8043, 8254, 8305, 8184, 8265, 8144, 8254] avg=8192.0000 max=8305 max_rank=3 min=8043 min_rank=1 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32847 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[487, 514, 501, 499, 516, 510, 519, 498, 519, 496, 516, 564, 519, 509, 528, 518, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 506, 483, 502, 507, 524, 501, 537, 497, 515, 544, 530, 541, 495, 520, 537, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 525, 521, 500, 506, 530, 474, 513, 498, 520, 547, 539, 506, 494, 499, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 506, 533, 508, 481, 492, 507, 499, 523, 525, 495, 481, 506, 519, 513, 479, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 485, 503, 534, 462, 529, 517, 494, 518, 535, 499, 529, 499, 483, 520, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 516, 532, 504, 554, 575, 462, 526, 511, 510, 484, 506, 481, 501, 518, 516, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 502, 499, 504, 547, 531, 486, 499, 531, 553, 536, 520, 495, 515, 512, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 543, 538, 499, 507, 518, 504, 498, 500, 486, 505, 519, 475, 483, 496, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8220, 8282, 8200, 8122, 8171, 8217, 8256, 8068] avg=8192.0000 max=8282 max_rank=1 min=8068 min_rank=7 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32712 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[521, 474, 519, 483, 472, 519, 576, 499, 548, 515, 524, 522, 487, 519, 506, 497, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 504, 491, 522, 531, 531, 513, 502, 533, 526, 499, 509, 502, 528, 512, 507, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 516, 504, 506, 497, 536, 499, 506, 542, 484, 511, 513, 542, 488, 535, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 511, 495, 533, 485, 522, 469, 472, 536, 526, 501, 525, 507, 519, 531, 548, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 503, 538, 527, 514, 525, 514, 504, 499, 521, 468, 515, 528, 519, 521, 567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 515, 521, 520, 527, 529, 509, 501, 502, 513, 515, 517, 469, 537, 522, 462, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 537, 511, 539, 498, 497, 509, 543, 503, 485, 513, 514, 480, 510, 496, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 545, 517, 503, 495, 500, 494, 500, 473, 541, 503, 483, 500, 468, 537, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8222, 8210, 8212, 8285, 8166, 8167, 8086] avg=8192.0000 max=8285 max_rank=4 min=8086 min_rank=7 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32704 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4352, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 3968, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 3968, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 3968, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4352, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 3968, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4352, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 3968, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[496, 511, 509, 503, 489, 516, 529, 537, 518, 509, 540, 510, 524, 542, 507, 501, 519, 488, 517, 504, 497, 509, 539, 557, 511, 521, 496, 527, 530, 512, 516, 515, 516, 512, 516, 536, 513, 509, 489, 506, 504, 533, 537, 525, 540, 510, 529, 543, 479, 518, 519, 497, 479, 535, 498, 521, 541, 534, 511, 512, 537, 528, 519, 505, 483, 519, 512, 529, 459, 523, 507, 529, 541, 519, 522, 528, 513, 526, 467, 508, 516, 519, 504, 517, 488, 513, 540, 490, 504, 524, 492, 512, 504, 515, 496, 510, 518, 506, 488, 495, 520, 515, 534, 541, 495, 477, 480, 479, 542, 490, 518, 486, 515, 523, 470, 503, 487, 566, 491, 501, 487, 517, 491, 469, 545, 510, 529, 469] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[522, 502, 481, 516, 510, 464, 503, 517, 532, 512, 500, 523, 501, 470, 487, 520, 475, 516, 527, 483, 520, 521, 528, 552, 510, 495, 499, 500, 492, 504, 525, 518, 532, 554, 506, 522, 467, 504, 525, 490, 525, 531, 522, 508, 577, 476, 544, 499, 501, 518, 477, 504, 527, 457, 504, 560, 542, 496, 512, 505, 516, 497, 478, 477, 568, 501, 499, 549, 540, 522, 543, 516, 529, 478, 515, 456, 524, 515, 494, 498, 476, 534, 521, 538, 513, 519, 523, 499, 541, 535, 478, 494, 535, 482, 465, 500, 515, 524, 517, 538, 528, 526, 517, 551, 527, 507, 526, 511, 496, 531, 541, 515, 526, 507, 501, 493, 562, 498, 536, 506, 529, 541, 486, 533, 502, 510, 491, 467] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[520, 476, 547, 513, 510, 487, 487, 494, 495, 488, 514, 563, 490, 537, 483, 512, 515, 489, 495, 510, 527, 489, 534, 513, 517, 533, 524, 488, 491, 494, 514, 518, 545, 510, 516, 512, 495, 534, 470, 498, 600, 504, 479, 526, 530, 516, 526, 516, 534, 474, 518, 469, 486, 484, 545, 512, 530, 483, 517, 511, 491, 543, 495, 469, 570, 530, 501, 489, 485, 509, 504, 520, 501, 527, 495, 531, 511, 487, 536, 490, 518, 534, 467, 566, 520, 531, 491, 551, 496, 533, 534, 488, 506, 528, 554, 526, 523, 554, 492, 511, 490, 562, 502, 492, 517, 514, 474, 554, 537, 505, 483, 470, 509, 516, 550, 507, 494, 510, 514, 494, 519, 511, 514, 516, 479, 526, 534, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[516, 496, 512, 504, 492, 499, 526, 502, 541, 502, 554, 517, 527, 481, 511, 534, 469, 519, 500, 483, 498, 524, 506, 528, 551, 564, 487, 502, 492, 482, 517, 510, 491, 492, 527, 517, 519, 525, 482, 471, 518, 488, 536, 479, 509, 500, 543, 501, 555, 541, 509, 498, 497, 471, 540, 494, 471, 527, 508, 547, 522, 542, 487, 487, 507, 475, 523, 528, 508, 503, 488, 507, 485, 525, 502, 501, 516, 561, 519, 561, 478, 489, 529, 507, 485, 529, 513, 503, 545, 560, 511, 524, 546, 482, 513, 542, 529, 499, 518, 480, 470, 535, 506, 510, 526, 519, 513, 494, 514, 505, 503, 497, 519, 496, 482, 540, 549, 532, 506, 522, 533, 505, 522, 482, 509, 541, 528, 547] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[546, 492, 496, 527, 517, 473, 508, 523, 502, 512, 532, 496, 523, 497, 473, 516, 527, 534, 503, 538, 490, 501, 489, 509, 536, 516, 516, 499, 465, 530, 519, 496, 527, 500, 506, 549, 519, 529, 498, 538, 507, 496, 523, 522, 509, 494, 532, 514, 511, 486, 478, 564, 517, 537, 519, 525, 480, 501, 497, 500, 506, 526, 517, 516, 529, 522, 533, 468, 525, 488, 489, 511, 475, 513, 498, 502, 520, 486, 515, 497, 493, 538, 552, 513, 552, 489, 519, 472, 487, 489, 502, 515, 537, 508, 509, 492, 522, 564, 522, 506, 522, 489, 519, 511, 513, 502, 483, 545, 503, 508, 517, 509, 521, 526, 507, 504, 527, 553, 547, 506, 538, 502, 505, 500, 515, 516, 506, 546] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[520, 501, 530, 493, 551, 537, 563, 519, 485, 501, 553, 505, 471, 503, 513, 525, 516, 497, 481, 534, 492, 530, 502, 517, 521, 518, 547, 481, 514, 478, 472, 487, 508, 495, 542, 532, 482, 500, 509, 482, 492, 511, 505, 535, 510, 478, 503, 503, 524, 473, 496, 479, 513, 505, 508, 537, 550, 491, 512, 550, 495, 488, 525, 522, 526, 532, 523, 526, 540, 481, 524, 483, 498, 516, 529, 521, 478, 529, 483, 469, 515, 478, 512, 514, 522, 507, 519, 513, 522, 543, 537, 494, 518, 539, 515, 528, 521, 510, 483, 513, 531, 522, 523, 520, 537, 524, 528, 497, 532, 513, 542, 479, 500, 497, 518, 514, 526, 482, 508, 513, 553, 549, 527, 516, 497, 497, 501, 517] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[495, 533, 468, 534, 528, 486, 526, 517, 487, 514, 514, 522, 521, 512, 495, 479, 533, 526, 493, 502, 502, 505, 538, 490, 524, 504, 513, 442, 522, 465, 527, 509, 515, 511, 515, 464, 494, 529, 501, 526, 513, 514, 529, 497, 491, 504, 508, 469, 537, 486, 490, 527, 505, 515, 511, 519, 501, 483, 512, 498, 493, 493, 528, 538, 526, 520, 498, 536, 514, 523, 521, 533, 484, 522, 505, 514, 466, 501, 512, 514, 483, 552, 507, 525, 505, 530, 494, 495, 508, 496, 543, 480, 487, 489, 531, 523, 519, 504, 555, 525, 510, 488, 512, 508, 536, 503, 521, 543, 496, 529, 497, 561, 507, 530, 561, 521, 508, 501, 539, 569, 499, 528, 553, 503, 514, 565, 515, 537] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[496, 511, 509, 503, 489, 516, 529, 537, 518, 509, 540, 510, 524, 542, 507, 501, 519, 488, 517, 504, 497, 509, 539, 557, 511, 521, 496, 527, 530, 512, 516, 515, 516, 512, 516, 536, 513, 509, 489, 506, 504, 533, 537, 525, 540, 510, 529, 543, 479, 518, 519, 497, 479, 535, 498, 521, 541, 534, 511, 512, 537, 528, 519, 505, 483, 519, 512, 529, 459, 523, 507, 529, 541, 519, 522, 528, 513, 526, 467, 508, 516, 519, 504, 517, 488, 513, 540, 490, 504, 524, 492, 512, 504, 515, 496, 510, 518, 506, 488, 495, 520, 515, 534, 541, 495, 477, 480, 479, 542, 490, 518, 486, 515, 523, 470, 503, 487, 566, 491, 501, 487, 517, 491, 469, 545, 510, 529, 469] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[520, 501, 530, 493, 551, 537, 563, 519, 485, 501, 553, 505, 471, 503, 513, 525, 516, 497, 481, 534, 492, 530, 502, 517, 521, 518, 547, 481, 514, 478, 472, 487, 508, 495, 542, 532, 482, 500, 509, 482, 492, 511, 505, 535, 510, 478, 503, 503, 524, 473, 496, 479, 513, 505, 508, 537, 550, 491, 512, 550, 495, 488, 525, 522, 526, 532, 523, 526, 540, 481, 524, 483, 498, 516, 529, 521, 478, 529, 483, 469, 515, 478, 512, 514, 522, 507, 519, 513, 522, 543, 537, 494, 518, 539, 515, 528, 521, 510, 483, 513, 531, 522, 523, 520, 537, 524, 528, 497, 532, 513, 542, 479, 500, 497, 518, 514, 526, 482, 508, 513, 553, 549, 527, 516, 497, 497, 501, 517] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[483, 485, 507, 493, 508, 532, 503, 514, 477, 508, 496, 562, 488, 516, 558, 556, 490, 516, 513, 507, 485, 504, 556, 546, 519, 478, 505, 526, 511, 504, 495, 488, 536, 507, 504, 482, 529, 536, 549, 489, 510, 531, 500, 538, 518, 497, 521, 485, 523, 541, 539, 505, 517, 527, 513, 460, 504, 532, 555, 484, 494, 509, 508, 516, 477, 534, 508, 504, 537, 515, 527, 529, 515, 482, 525, 549, 494, 484, 504, 511, 529, 563, 485, 522, 513, 542, 510, 535, 501, 459, 488, 502, 526, 512, 477, 475, 540, 522, 534, 520, 543, 512, 502, 508, 475, 488, 506, 517, 505, 519, 510, 546, 514, 532, 514, 508, 506, 506, 464, 490, 516, 514, 532, 499, 521, 525, 500, 526] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[516, 496, 512, 504, 492, 499, 526, 502, 541, 502, 554, 517, 527, 481, 511, 534, 12, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 519, 500, 483, 498, 524, 506, 528, 551, 564, 487, 502, 492, 482, 517, 510, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 492, 527, 517, 519, 525, 482, 471, 518, 488, 536, 479, 509, 500, 531, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 541, 509, 498, 497, 471, 540, 494, 471, 527, 508, 547, 522, 542, 487, 487, 24, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 475, 523, 528, 508, 503, 488, 507, 485, 525, 502, 501, 516, 561, 519, 561, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 479, 529, 507, 485, 529, 513, 503, 545, 560, 511, 524, 546, 482, 513, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 499, 518, 480, 470, 535, 506, 510, 526, 519, 513, 494, 514, 505, 503, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 496, 482, 540, 549, 532, 506, 522, 533, 505, 522, 482, 509, 489, 528, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8237, 8175, 8086, 8229, 8219, 8246, 8083, 8261] avg=8192.0000 max=8261 max_rank=7 min=8083 min_rank=6 imbalance_ratio=1.008423 max_load_reduction=57275 max_load_reduction_pct=0.873947 moved_to_echo=32809 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[520, 476, 547, 513, 510, 487, 487, 494, 495, 488, 514, 563, 490, 537, 483, 512, 11, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 489, 495, 510, 527, 489, 534, 513, 517, 533, 524, 488, 491, 494, 514, 518, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 510, 516, 512, 495, 534, 470, 498, 600, 504, 479, 526, 530, 516, 515, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 474, 518, 469, 486, 484, 545, 512, 530, 483, 517, 511, 491, 543, 495, 469, 23, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 570, 530, 501, 489, 485, 509, 504, 520, 501, 527, 495, 531, 511, 487, 536, 490, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 523, 467, 566, 520, 531, 491, 551, 496, 533, 534, 488, 506, 528, 554, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 554, 492, 511, 490, 562, 502, 492, 517, 514, 474, 554, 537, 505, 483, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 516, 550, 507, 494, 510, 514, 494, 519, 511, 514, 516, 479, 475, 534, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8138, 8193, 8266, 8093, 8197, 8332, 8146, 8171] avg=8192.0000 max=8332 max_rank=5 min=8093 min_rank=3 imbalance_ratio=1.017090 max_load_reduction=57204 max_load_reduction_pct=0.872864 moved_to_echo=32846 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[522, 502, 481, 516, 510, 464, 503, 517, 532, 512, 500, 523, 501, 470, 487, 520, 475, 516, 527, 483, 520, 521, 528, 552, 510, 495, 499, 500, 492, 504, 525, 518, 532, 554, 506, 522, 467, 504, 525, 490, 525, 531, 522, 508, 577, 476, 544, 499, 501, 518, 477, 504, 527, 457, 504, 560, 542, 496, 512, 505, 516, 497, 478, 477, 568, 501, 499, 549, 540, 522, 543, 516, 529, 478, 515, 456, 524, 515, 494, 498, 476, 534, 521, 538, 513, 519, 523, 499, 541, 535, 478, 494, 535, 482, 465, 500, 515, 524, 517, 538, 528, 526, 517, 551, 527, 507, 526, 511, 496, 531, 541, 515, 526, 507, 501, 493, 562, 498, 536, 506, 529, 541, 486, 533, 502, 510, 491, 467] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[516, 496, 512, 504, 492, 499, 526, 502, 541, 502, 554, 517, 527, 481, 511, 534, 469, 519, 500, 483, 498, 524, 506, 528, 551, 564, 487, 502, 492, 482, 517, 510, 491, 492, 527, 517, 519, 525, 482, 471, 518, 488, 536, 479, 509, 500, 543, 501, 555, 541, 509, 498, 497, 471, 540, 494, 471, 527, 508, 547, 522, 542, 487, 487, 507, 475, 523, 528, 508, 503, 488, 507, 485, 525, 502, 501, 516, 561, 519, 561, 478, 489, 529, 507, 485, 529, 513, 503, 545, 560, 511, 524, 546, 482, 513, 542, 529, 499, 518, 480, 470, 535, 506, 510, 526, 519, 513, 494, 514, 505, 503, 497, 519, 496, 482, 540, 549, 532, 506, 522, 533, 505, 522, 482, 509, 541, 528, 547] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[520, 476, 547, 513, 510, 487, 487, 494, 495, 488, 514, 563, 490, 537, 483, 512, 515, 489, 495, 510, 527, 489, 534, 513, 517, 533, 524, 488, 491, 494, 514, 518, 545, 510, 516, 512, 495, 534, 470, 498, 600, 504, 479, 526, 530, 516, 526, 516, 534, 474, 518, 469, 486, 484, 545, 512, 530, 483, 517, 511, 491, 543, 495, 469, 570, 530, 501, 489, 485, 509, 504, 520, 501, 527, 495, 531, 511, 487, 536, 490, 518, 534, 467, 566, 520, 531, 491, 551, 496, 533, 534, 488, 506, 528, 554, 526, 523, 554, 492, 511, 490, 562, 502, 492, 517, 514, 474, 554, 537, 505, 483, 470, 509, 516, 550, 507, 494, 510, 514, 494, 519, 511, 514, 516, 479, 526, 534, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[546, 492, 496, 527, 517, 473, 508, 523, 502, 512, 532, 496, 523, 497, 473, 516, 12, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 534, 503, 538, 490, 501, 489, 509, 536, 516, 516, 499, 465, 530, 519, 496, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 500, 506, 549, 519, 529, 498, 538, 507, 496, 523, 522, 509, 494, 520, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 486, 478, 564, 517, 537, 519, 525, 480, 501, 497, 500, 506, 526, 517, 516, 23, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 522, 533, 468, 525, 488, 489, 511, 475, 513, 498, 502, 520, 486, 515, 497, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 527, 552, 513, 552, 489, 519, 472, 487, 489, 502, 515, 537, 508, 509, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 564, 522, 506, 522, 489, 519, 511, 513, 502, 483, 545, 503, 508, 517, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 526, 507, 504, 527, 553, 547, 506, 538, 502, 505, 500, 515, 466, 506, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8156, 8209, 8251, 8212, 8082, 8156, 8201, 8269] avg=8192.0000 max=8269 max_rank=7 min=8082 min_rank=4 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32708 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[496, 511, 509, 503, 489, 516, 529, 537, 518, 509, 540, 510, 524, 542, 507, 501, 11, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 488, 517, 504, 497, 509, 539, 557, 511, 521, 496, 527, 530, 512, 516, 515, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 512, 516, 536, 513, 509, 489, 506, 504, 533, 537, 525, 540, 510, 518, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 518, 519, 497, 479, 535, 498, 521, 541, 534, 511, 512, 537, 528, 519, 505, 23, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 519, 512, 529, 459, 523, 507, 529, 541, 519, 522, 528, 513, 526, 467, 508, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 508, 504, 517, 488, 513, 540, 490, 504, 524, 492, 512, 504, 515, 496, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 506, 488, 495, 520, 515, 534, 541, 495, 477, 480, 479, 542, 490, 518, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 523, 470, 503, 487, 566, 491, 501, 487, 517, 491, 469, 545, 460, 529, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8263, 8299, 8307, 8265, 8196, 8133, 8050, 8023] avg=8192.0000 max=8307 max_rank=2 min=8023 min_rank=7 imbalance_ratio=1.014038 max_load_reduction=57229 max_load_reduction_pct=0.873245 moved_to_echo=32402 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[495, 533, 468, 534, 528, 486, 526, 517, 487, 514, 514, 522, 521, 512, 495, 479, 15, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 526, 493, 502, 502, 505, 538, 490, 524, 504, 513, 442, 522, 465, 527, 509, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 511, 515, 464, 494, 529, 501, 526, 513, 514, 529, 497, 491, 504, 487, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 486, 490, 527, 505, 515, 511, 519, 501, 483, 512, 498, 493, 493, 528, 538, 27, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 520, 498, 536, 514, 523, 521, 533, 484, 522, 505, 514, 466, 501, 512, 514, 14, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 538, 507, 525, 505, 530, 494, 495, 508, 496, 543, 480, 487, 489, 531, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 504, 555, 525, 510, 488, 512, 508, 536, 503, 521, 543, 496, 529, 497, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 530, 561, 521, 508, 501, 539, 569, 499, 528, 553, 503, 514, 505, 515, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8159, 8143, 8059, 8175, 8209, 8134, 8267, 8390] avg=8192.0000 max=8390 max_rank=7 min=8059 min_rank=2 imbalance_ratio=1.024170 max_load_reduction=57146 max_load_reduction_pct=0.871979 moved_to_echo=33000 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[522, 502, 481, 516, 510, 464, 503, 517, 532, 512, 500, 523, 501, 470, 487, 520, 12, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 516, 527, 483, 520, 521, 528, 552, 510, 495, 499, 500, 492, 504, 525, 518, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 554, 506, 522, 467, 504, 525, 490, 525, 531, 522, 508, 577, 476, 532, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 518, 477, 504, 527, 457, 504, 560, 542, 496, 512, 505, 516, 497, 478, 477, 23, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 568, 501, 499, 549, 540, 522, 543, 516, 529, 478, 515, 456, 524, 515, 494, 498, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 523, 521, 538, 513, 519, 523, 499, 541, 535, 478, 494, 535, 482, 465, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 524, 517, 538, 528, 526, 517, 551, 527, 507, 526, 511, 496, 531, 541, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 507, 501, 493, 562, 498, 536, 506, 529, 541, 486, 533, 502, 460, 491, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8083, 8206, 8270, 8103, 8258, 8142, 8336, 8138] avg=8192.0000 max=8336 max_rank=6 min=8083 min_rank=0 imbalance_ratio=1.017578 max_load_reduction=57200 max_load_reduction_pct=0.872803 moved_to_echo=32874 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[496, 511, 509, 503, 489, 516, 529, 537, 518, 509, 540, 510, 524, 542, 507, 501, 11, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 488, 517, 504, 497, 509, 539, 557, 511, 521, 496, 527, 530, 512, 516, 515, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 512, 516, 536, 513, 509, 489, 506, 504, 533, 537, 525, 540, 510, 518, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 518, 519, 497, 479, 535, 498, 521, 541, 534, 511, 512, 537, 528, 519, 505, 23, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 519, 512, 529, 459, 523, 507, 529, 541, 519, 522, 528, 513, 526, 467, 508, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 508, 504, 517, 488, 513, 540, 490, 504, 524, 492, 512, 504, 515, 496, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 506, 488, 495, 520, 515, 534, 541, 495, 477, 480, 479, 542, 490, 518, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 523, 470, 503, 487, 566, 491, 501, 487, 517, 491, 469, 545, 460, 529, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8263, 8299, 8307, 8265, 8196, 8133, 8050, 8023] avg=8192.0000 max=8307 max_rank=2 min=8023 min_rank=7 imbalance_ratio=1.014038 max_load_reduction=57229 max_load_reduction_pct=0.873245 moved_to_echo=32402 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[520, 501, 530, 493, 551, 537, 563, 519, 485, 501, 553, 505, 471, 503, 513, 525, 11, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 497, 481, 534, 492, 530, 502, 517, 521, 518, 547, 481, 514, 478, 472, 487, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 495, 542, 532, 482, 500, 509, 482, 492, 511, 505, 535, 510, 478, 492, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 473, 496, 479, 513, 505, 508, 537, 550, 491, 512, 550, 495, 488, 525, 522, 23, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 532, 523, 526, 540, 481, 524, 483, 498, 516, 529, 521, 478, 529, 483, 469, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 468, 512, 514, 522, 507, 519, 513, 522, 543, 537, 494, 518, 539, 515, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 510, 483, 513, 531, 522, 523, 520, 537, 524, 528, 497, 532, 513, 542, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 497, 518, 514, 526, 482, 508, 513, 553, 549, 527, 516, 497, 450, 501, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8292, 8126, 8076, 8199, 8168, 8266, 8241, 8168] avg=8192.0000 max=8292 max_rank=0 min=8076 min_rank=2 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32843 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[522, 502, 481, 516, 510, 464, 503, 517, 532, 512, 500, 523, 501, 470, 487, 520, 12, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 516, 527, 483, 520, 521, 528, 552, 510, 495, 499, 500, 492, 504, 525, 518, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 554, 506, 522, 467, 504, 525, 490, 525, 531, 522, 508, 577, 476, 532, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 518, 477, 504, 527, 457, 504, 560, 542, 496, 512, 505, 516, 497, 478, 477, 23, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 568, 501, 499, 549, 540, 522, 543, 516, 529, 478, 515, 456, 524, 515, 494, 498, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 523, 521, 538, 513, 519, 523, 499, 541, 535, 478, 494, 535, 482, 465, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 524, 517, 538, 528, 526, 517, 551, 527, 507, 526, 511, 496, 531, 541, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 507, 501, 493, 562, 498, 536, 506, 529, 541, 486, 533, 502, 460, 491, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8083, 8206, 8270, 8103, 8258, 8142, 8336, 8138] avg=8192.0000 max=8336 max_rank=6 min=8083 min_rank=0 imbalance_ratio=1.017578 max_load_reduction=57200 max_load_reduction_pct=0.872803 moved_to_echo=32874 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[516, 496, 512, 504, 492, 499, 526, 502, 541, 502, 554, 517, 527, 481, 511, 534, 12, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 519, 500, 483, 498, 524, 506, 528, 551, 564, 487, 502, 492, 482, 517, 510, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 492, 527, 517, 519, 525, 482, 471, 518, 488, 536, 479, 509, 500, 531, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 541, 509, 498, 497, 471, 540, 494, 471, 527, 508, 547, 522, 542, 487, 487, 24, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 475, 523, 528, 508, 503, 488, 507, 485, 525, 502, 501, 516, 561, 519, 561, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 479, 529, 507, 485, 529, 513, 503, 545, 560, 511, 524, 546, 482, 513, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 499, 518, 480, 470, 535, 506, 510, 526, 519, 513, 494, 514, 505, 503, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 496, 482, 540, 549, 532, 506, 522, 533, 505, 522, 482, 509, 489, 528, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8237, 8175, 8086, 8229, 8219, 8246, 8083, 8261] avg=8192.0000 max=8261 max_rank=7 min=8083 min_rank=6 imbalance_ratio=1.008423 max_load_reduction=57275 max_load_reduction_pct=0.873947 moved_to_echo=32809 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[520, 476, 547, 513, 510, 487, 487, 494, 495, 488, 514, 563, 490, 537, 483, 512, 11, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 489, 495, 510, 527, 489, 534, 513, 517, 533, 524, 488, 491, 494, 514, 518, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 510, 516, 512, 495, 534, 470, 498, 600, 504, 479, 526, 530, 516, 515, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 474, 518, 469, 486, 484, 545, 512, 530, 483, 517, 511, 491, 543, 495, 469, 23, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 570, 530, 501, 489, 485, 509, 504, 520, 501, 527, 495, 531, 511, 487, 536, 490, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 523, 467, 566, 520, 531, 491, 551, 496, 533, 534, 488, 506, 528, 554, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 554, 492, 511, 490, 562, 502, 492, 517, 514, 474, 554, 537, 505, 483, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 516, 550, 507, 494, 510, 514, 494, 519, 511, 514, 516, 479, 475, 534, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8138, 8193, 8266, 8093, 8197, 8332, 8146, 8171] avg=8192.0000 max=8332 max_rank=5 min=8093 min_rank=3 imbalance_ratio=1.017090 max_load_reduction=57204 max_load_reduction_pct=0.872864 moved_to_echo=32846 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[483, 485, 507, 493, 508, 532, 503, 514, 477, 508, 496, 562, 488, 516, 558, 556, 11, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 516, 513, 507, 485, 504, 556, 546, 519, 478, 505, 526, 511, 504, 495, 488, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 507, 504, 482, 529, 536, 549, 489, 510, 531, 500, 538, 518, 497, 510, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 541, 539, 505, 517, 527, 513, 460, 504, 532, 555, 484, 494, 509, 508, 516, 24, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 534, 508, 504, 537, 515, 527, 529, 515, 482, 525, 549, 494, 484, 504, 511, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 551, 485, 522, 513, 542, 510, 535, 501, 459, 488, 502, 526, 512, 477, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 522, 534, 520, 543, 512, 502, 508, 475, 488, 506, 517, 505, 519, 510, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 532, 514, 508, 506, 506, 464, 490, 516, 514, 532, 499, 521, 474, 500, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8185, 8221, 8260, 8207, 8127, 8212, 8116] avg=8192.0000 max=8260 max_rank=3 min=8116 min_rank=7 imbalance_ratio=1.008301 max_load_reduction=57276 max_load_reduction_pct=0.873962 moved_to_echo=32662 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 3968, 4096, 4096, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 3840, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[3968, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 3840, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[537, 495, 524, 499, 530, 500, 523, 485, 505, 487, 532, 524, 527, 515, 494, 468, 499, 539, 535, 537, 482, 538, 519, 519, 513, 537, 537, 533, 504, 531, 508, 494, 500, 489, 533, 494, 497, 543, 515, 489, 514, 519, 513, 519, 494, 520, 483, 512, 511, 543, 515, 537, 509, 497, 531, 526, 516, 461, 483, 508, 506, 476, 495, 537, 527, 522, 517, 483, 501, 492, 510, 492, 514, 508, 509, 501, 493, 516, 518, 500, 542, 516, 485, 509, 517, 501, 534, 518, 494, 541, 519, 543, 507, 483, 521, 500, 508, 497, 546, 516, 507, 496, 518, 536, 550, 534, 555, 543, 488, 495, 469, 537, 487, 505, 493, 529, 489, 490, 487, 513, 515, 525, 549, 520, 506, 531, 504, 510] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[517, 512, 547, 510, 481, 511, 532, 525, 498, 543, 500, 532, 504, 525, 517, 544, 570, 504, 543, 522, 465, 492, 523, 535, 541, 519, 485, 505, 505, 556, 494, 515, 523, 549, 528, 538, 533, 518, 506, 488, 512, 527, 482, 519, 474, 508, 497, 509, 482, 545, 525, 530, 531, 488, 508, 521, 486, 496, 509, 541, 479, 506, 490, 485, 537, 542, 471, 503, 496, 541, 484, 484, 508, 554, 516, 495, 522, 509, 533, 499, 536, 519, 524, 512, 521, 467, 535, 521, 533, 521, 506, 526, 498, 516, 510, 512, 500, 481, 510, 537, 473, 496, 483, 526, 507, 475, 538, 535, 479, 502, 526, 476, 499, 534, 519, 511, 546, 480, 516, 479, 507, 495, 513, 534, 530, 479, 532, 462] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[516, 530, 501, 527, 481, 484, 511, 519, 480, 526, 511, 549, 505, 498, 499, 531, 543, 499, 522, 572, 517, 472, 513, 533, 521, 504, 507, 467, 512, 486, 485, 530, 549, 503, 479, 506, 520, 467, 471, 521, 570, 553, 476, 502, 508, 517, 493, 486, 504, 528, 539, 531, 501, 499, 513, 475, 504, 475, 523, 523, 529, 512, 522, 477, 510, 504, 526, 462, 518, 544, 483, 512, 487, 502, 505, 524, 509, 480, 530, 548, 502, 533, 506, 536, 535, 541, 511, 519, 549, 508, 532, 529, 483, 490, 558, 574, 520, 496, 528, 482, 538, 533, 503, 507, 513, 507, 475, 468, 538, 551, 529, 486, 492, 541, 480, 479, 529, 541, 524, 533, 494, 492, 491, 491, 523, 499, 523, 553] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[503, 525, 498, 495, 471, 519, 478, 524, 535, 505, 480, 530, 521, 512, 557, 528, 493, 481, 495, 509, 525, 530, 503, 536, 517, 498, 523, 509, 520, 501, 480, 549, 517, 533, 499, 530, 511, 508, 501, 518, 497, 538, 524, 525, 532, 482, 517, 510, 516, 533, 489, 473, 549, 481, 517, 517, 500, 503, 522, 510, 521, 511, 492, 508, 499, 532, 519, 526, 479, 507, 506, 497, 490, 493, 514, 538, 523, 491, 504, 550, 517, 517, 494, 495, 501, 509, 501, 521, 494, 481, 481, 536, 559, 517, 533, 494, 544, 521, 514, 481, 549, 511, 499, 495, 500, 542, 508, 498, 519, 522, 501, 494, 482, 521, 511, 486, 531, 576, 500, 510, 534, 508, 530, 533, 518, 509, 508, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[527, 532, 538, 502, 503, 504, 479, 523, 495, 536, 546, 542, 489, 525, 516, 492, 533, 538, 508, 509, 546, 529, 536, 524, 511, 498, 498, 511, 482, 513, 487, 515, 530, 524, 492, 481, 487, 509, 513, 543, 485, 530, 506, 521, 525, 507, 508, 537, 515, 521, 518, 540, 538, 555, 517, 547, 509, 472, 486, 471, 503, 539, 516, 522, 518, 521, 477, 469, 518, 558, 514, 529, 526, 502, 552, 505, 476, 495, 513, 534, 546, 496, 482, 530, 488, 510, 523, 518, 482, 514, 515, 565, 511, 492, 490, 470, 548, 496, 525, 514, 482, 558, 510, 481, 491, 495, 500, 488, 515, 488, 497, 494, 475, 469, 523, 510, 510, 540, 522, 493, 498, 519, 503, 523, 495, 544, 528, 509] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[537, 495, 524, 499, 530, 500, 523, 485, 505, 487, 532, 524, 527, 515, 494, 468, 499, 539, 535, 537, 482, 538, 519, 519, 513, 537, 537, 533, 504, 531, 508, 494, 500, 489, 533, 494, 497, 543, 515, 489, 514, 519, 513, 519, 494, 520, 483, 512, 511, 543, 515, 537, 509, 497, 531, 526, 516, 461, 483, 508, 506, 476, 495, 537, 527, 522, 517, 483, 501, 492, 510, 492, 514, 508, 509, 501, 493, 516, 518, 500, 542, 516, 485, 509, 517, 501, 534, 518, 494, 541, 519, 543, 507, 483, 521, 500, 508, 497, 546, 516, 507, 496, 518, 536, 550, 534, 555, 543, 488, 495, 469, 537, 487, 505, 493, 529, 489, 490, 487, 513, 515, 525, 549, 520, 506, 531, 504, 510] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[506, 516, 525, 521, 516, 513, 506, 505, 501, 480, 484, 508, 514, 514, 518, 463, 520, 558, 510, 529, 541, 570, 483, 505, 528, 518, 494, 518, 522, 531, 545, 492, 534, 481, 456, 525, 532, 538, 470, 547, 480, 534, 512, 471, 534, 513, 522, 500, 452, 499, 499, 529, 536, 488, 513, 561, 548, 520, 515, 535, 552, 497, 496, 512, 457, 493, 495, 515, 508, 499, 544, 529, 521, 545, 528, 530, 499, 531, 519, 510, 523, 545, 494, 512, 486, 506, 521, 497, 536, 497, 535, 505, 507, 506, 498, 500, 544, 502, 504, 463, 502, 499, 492, 451, 518, 512, 475, 487, 528, 551, 525, 521, 569, 495, 488, 573, 513, 499, 529, 499, 525, 505, 503, 488, 473, 501, 525, 531] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[587, 499, 561, 495, 500, 511, 536, 495, 495, 482, 509, 516, 529, 477, 519, 528, 481, 494, 496, 504, 541, 510, 469, 519, 505, 525, 544, 495, 462, 526, 542, 511, 491, 509, 539, 507, 491, 532, 541, 518, 511, 532, 562, 527, 493, 547, 537, 494, 473, 497, 526, 514, 520, 490, 493, 516, 511, 495, 514, 520, 504, 525, 547, 543, 518, 511, 532, 508, 483, 504, 487, 505, 465, 498, 522, 509, 545, 500, 521, 526, 516, 478, 525, 523, 494, 493, 505, 490, 523, 463, 501, 481, 513, 527, 496, 479, 524, 517, 465, 497, 486, 486, 523, 523, 529, 510, 540, 525, 530, 514, 486, 481, 503, 548, 521, 528, 533, 556, 529, 499, 561, 520, 505, 523, 490, 533, 502, 526] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[527, 532, 538, 502, 503, 504, 479, 523, 495, 536, 546, 542, 489, 525, 516, 492, 533, 538, 508, 509, 546, 529, 536, 524, 511, 498, 498, 511, 482, 513, 487, 515, 530, 524, 492, 481, 487, 509, 513, 543, 485, 530, 506, 521, 525, 507, 508, 537, 515, 521, 518, 540, 538, 555, 517, 547, 509, 472, 486, 471, 503, 539, 516, 522, 518, 521, 477, 469, 518, 558, 514, 529, 526, 502, 552, 505, 476, 495, 513, 534, 546, 496, 482, 530, 488, 510, 523, 518, 482, 514, 515, 565, 511, 492, 490, 470, 548, 496, 525, 514, 482, 558, 510, 481, 491, 495, 500, 488, 515, 488, 497, 494, 475, 469, 523, 510, 510, 540, 522, 493, 498, 519, 503, 523, 495, 544, 528, 509] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[503, 525, 498, 495, 471, 519, 478, 524, 535, 505, 480, 530, 521, 512, 557, 528, 493, 481, 495, 509, 525, 530, 503, 536, 517, 498, 523, 509, 520, 501, 480, 549, 517, 533, 499, 530, 511, 508, 501, 518, 497, 538, 524, 525, 532, 482, 517, 510, 516, 533, 489, 473, 549, 481, 517, 517, 500, 503, 522, 510, 521, 511, 492, 508, 499, 532, 519, 526, 479, 507, 506, 497, 490, 493, 514, 538, 523, 491, 504, 550, 517, 517, 494, 495, 501, 509, 501, 521, 494, 481, 481, 536, 559, 517, 533, 494, 544, 521, 514, 481, 549, 511, 499, 495, 500, 542, 508, 498, 519, 522, 501, 494, 482, 521, 511, 486, 531, 576, 500, 510, 534, 508, 530, 533, 518, 509, 508, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[541, 460, 542, 524, 552, 532, 522, 519, 524, 483, 512, 509, 536, 529, 503, 532, 462, 460, 532, 527, 525, 521, 524, 547, 492, 562, 506, 501, 467, 525, 535, 535, 508, 517, 477, 489, 532, 526, 512, 540, 521, 527, 532, 511, 473, 545, 492, 565, 517, 503, 527, 471, 502, 516, 500, 504, 529, 499, 514, 481, 493, 557, 526, 528, 555, 506, 509, 512, 507, 504, 524, 517, 491, 516, 493, 506, 505, 487, 473, 527, 519, 500, 506, 519, 516, 474, 494, 515, 482, 556, 505, 518, 542, 525, 531, 504, 526, 563, 514, 490, 514, 520, 522, 470, 518, 503, 463, 500, 487, 531, 478, 490, 541, 521, 537, 563, 510, 478, 475, 495, 512, 492, 478, 520, 481, 499, 509, 523] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[527, 532, 519, 502, 503, 504, 479, 523, 495, 536, 546, 542, 489, 525, 516, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 538, 508, 509, 546, 529, 536, 480, 511, 498, 498, 511, 482, 513, 487, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 524, 492, 481, 487, 509, 513, 543, 485, 516, 506, 521, 525, 507, 508, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 521, 518, 540, 538, 555, 517, 547, 509, 472, 486, 471, 503, 539, 516, 522, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 521, 477, 469, 518, 558, 514, 529, 526, 502, 552, 505, 476, 495, 513, 534, 14, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 496, 482, 530, 488, 510, 523, 518, 482, 514, 515, 563, 511, 492, 490, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 496, 525, 514, 482, 558, 510, 481, 491, 495, 500, 488, 515, 488, 497, 494, 44, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 469, 523, 497, 510, 540, 522, 493, 498, 519, 503, 523, 495, 544, 528, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 8194, 8184, 8279, 8235, 8130, 8136, 8148] avg=8192.0000 max=8279 max_rank=3 min=8130 min_rank=5 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32649 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[516, 530, 501, 527, 481, 484, 511, 519, 480, 526, 511, 549, 505, 498, 499, 531, 543, 499, 522, 572, 517, 472, 513, 533, 521, 504, 507, 467, 512, 486, 485, 530, 549, 503, 479, 506, 520, 467, 471, 521, 570, 553, 476, 502, 508, 517, 493, 486, 504, 528, 539, 531, 501, 499, 513, 475, 504, 475, 523, 523, 529, 512, 522, 477, 510, 504, 526, 462, 518, 544, 483, 512, 487, 502, 505, 524, 509, 480, 530, 548, 502, 533, 506, 536, 535, 541, 511, 519, 549, 508, 532, 529, 483, 490, 558, 574, 520, 496, 528, 482, 538, 533, 503, 507, 513, 507, 475, 468, 538, 551, 529, 486, 492, 541, 480, 479, 529, 541, 524, 533, 494, 492, 491, 491, 523, 499, 523, 553] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[517, 512, 520, 510, 481, 511, 532, 525, 498, 543, 500, 532, 504, 525, 517, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 570, 504, 543, 522, 465, 492, 523, 485, 541, 519, 485, 505, 505, 556, 494, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 549, 528, 538, 533, 518, 506, 488, 512, 509, 482, 519, 474, 508, 497, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 545, 525, 530, 531, 488, 508, 521, 486, 496, 509, 541, 479, 506, 490, 485, 12, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 542, 471, 503, 496, 541, 484, 484, 508, 554, 516, 495, 522, 509, 533, 499, 18, 13, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 519, 524, 512, 521, 467, 535, 521, 533, 521, 506, 520, 498, 516, 510, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 481, 510, 537, 473, 496, 483, 526, 507, 475, 538, 535, 479, 502, 526, 476, 50, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 534, 519, 490, 546, 480, 516, 479, 507, 495, 513, 534, 530, 479, 532, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8271, 8224, 8193, 8140, 8234, 8251, 8108, 8115] avg=8192.0000 max=8271 max_rank=0 min=8108 min_rank=6 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32708 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[537, 495, 506, 499, 530, 500, 523, 485, 505, 487, 532, 524, 527, 515, 494, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 539, 535, 537, 482, 538, 519, 475, 513, 537, 537, 533, 504, 531, 508, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 489, 533, 494, 497, 543, 515, 489, 514, 505, 513, 519, 494, 520, 483, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 543, 515, 537, 509, 497, 531, 526, 516, 461, 483, 508, 506, 476, 495, 537, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 522, 517, 483, 501, 492, 510, 492, 514, 508, 509, 501, 493, 516, 518, 500, 14, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 516, 485, 509, 517, 501, 534, 518, 494, 541, 519, 541, 507, 483, 521, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 497, 546, 516, 507, 496, 518, 536, 550, 534, 555, 543, 488, 495, 469, 537, 44, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 505, 493, 516, 489, 490, 487, 513, 515, 525, 549, 520, 506, 531, 504, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8127, 8281, 8120, 8161, 8130, 8228, 8349, 8140] avg=8192.0000 max=8349 max_rank=6 min=8120 min_rank=2 imbalance_ratio=1.019165 max_load_reduction=57187 max_load_reduction_pct=0.872604 moved_to_echo=32847 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[516, 530, 484, 527, 481, 484, 511, 519, 480, 526, 511, 549, 505, 498, 499, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 499, 522, 572, 517, 472, 513, 488, 521, 504, 507, 467, 512, 486, 485, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 503, 479, 506, 520, 467, 471, 521, 570, 538, 476, 502, 508, 517, 493, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 528, 539, 531, 501, 499, 513, 475, 504, 475, 523, 523, 529, 512, 522, 477, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 504, 526, 462, 518, 544, 483, 512, 487, 502, 505, 524, 509, 480, 530, 548, 15, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 533, 506, 536, 535, 541, 511, 519, 549, 508, 532, 527, 483, 490, 558, 574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 496, 528, 482, 538, 533, 503, 507, 513, 507, 475, 468, 538, 551, 529, 486, 45, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 541, 480, 467, 529, 541, 524, 533, 494, 492, 491, 491, 523, 499, 523, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 8138, 8106, 8165, 8171, 8404, 8228, 8173] avg=8192.0000 max=8404 max_rank=5 min=8106 min_rank=2 imbalance_ratio=1.025879 max_load_reduction=57132 max_load_reduction_pct=0.871765 moved_to_echo=32976 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[587, 499, 542, 495, 500, 511, 536, 495, 495, 482, 509, 516, 529, 477, 519, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 494, 496, 504, 541, 510, 469, 475, 505, 525, 544, 495, 462, 526, 542, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 509, 539, 507, 491, 532, 541, 518, 511, 518, 562, 527, 493, 547, 537, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 497, 526, 514, 520, 490, 493, 516, 511, 495, 514, 520, 504, 525, 547, 543, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 511, 532, 508, 483, 504, 487, 505, 465, 498, 522, 509, 545, 500, 521, 526, 14, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 478, 525, 523, 494, 493, 505, 490, 523, 463, 501, 479, 513, 527, 496, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 517, 465, 497, 486, 486, 523, 523, 529, 510, 540, 525, 530, 514, 486, 481, 44, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 548, 521, 515, 533, 556, 529, 499, 561, 520, 505, 523, 490, 533, 502, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8220, 8080, 8317, 8198, 8162, 8005, 8190, 8364] avg=8192.0000 max=8364 max_rank=7 min=8005 min_rank=5 imbalance_ratio=1.020996 max_load_reduction=57172 max_load_reduction_pct=0.872375 moved_to_echo=32721 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[541, 460, 523, 524, 552, 532, 522, 519, 524, 483, 512, 509, 536, 529, 503, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 460, 532, 527, 525, 521, 524, 501, 492, 562, 506, 501, 467, 525, 535, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 517, 477, 489, 532, 526, 512, 540, 521, 513, 532, 511, 473, 545, 492, 565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 503, 527, 471, 502, 516, 500, 504, 529, 499, 514, 481, 493, 557, 526, 528, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 506, 509, 512, 507, 504, 524, 517, 491, 516, 493, 506, 505, 487, 473, 527, 14, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 500, 506, 519, 516, 474, 494, 515, 482, 556, 505, 516, 542, 525, 531, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 563, 514, 490, 514, 520, 522, 470, 518, 503, 463, 500, 487, 531, 478, 490, 46, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 521, 537, 549, 510, 478, 475, 495, 512, 492, 478, 520, 481, 499, 509, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8301, 8175, 8253, 8178, 8160, 8204, 8145, 8120] avg=8192.0000 max=8301 max_rank=0 min=8120 min_rank=7 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32629 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[506, 516, 507, 521, 516, 513, 506, 505, 501, 480, 484, 508, 514, 514, 518, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 558, 510, 529, 541, 570, 483, 462, 528, 518, 494, 518, 522, 531, 545, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 481, 456, 525, 532, 538, 470, 547, 480, 520, 512, 471, 534, 513, 522, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 452, 499, 499, 529, 536, 488, 513, 561, 548, 520, 515, 535, 552, 497, 496, 512, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 457, 493, 495, 515, 508, 499, 544, 529, 521, 545, 528, 530, 499, 531, 519, 510, 14, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 545, 494, 512, 486, 506, 521, 497, 536, 497, 535, 503, 507, 506, 498, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 502, 504, 463, 502, 499, 492, 451, 518, 512, 475, 487, 528, 551, 525, 521, 43, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 569, 495, 488, 559, 513, 499, 529, 499, 525, 505, 503, 488, 473, 501, 525, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8072, 8321, 8135, 8263, 8250, 8166, 8127, 8202] avg=8192.0000 max=8321 max_rank=1 min=8072 min_rank=0 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32745 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[503, 525, 481, 495, 471, 519, 478, 524, 535, 505, 480, 530, 521, 512, 557, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 481, 495, 509, 525, 530, 503, 490, 517, 498, 523, 509, 520, 501, 480, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 533, 499, 530, 511, 508, 501, 518, 497, 524, 524, 525, 532, 482, 517, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 533, 489, 473, 549, 481, 517, 517, 500, 503, 522, 510, 521, 511, 492, 508, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 532, 519, 526, 479, 507, 506, 497, 490, 493, 514, 538, 523, 491, 504, 550, 14, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 517, 494, 495, 501, 509, 501, 521, 494, 481, 481, 534, 559, 517, 533, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 521, 514, 481, 549, 511, 499, 495, 500, 542, 508, 498, 519, 522, 501, 494, 46, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 521, 511, 474, 531, 576, 500, 510, 534, 508, 530, 533, 518, 509, 508, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8123, 8228, 8152, 8194, 8148, 8253, 8274] avg=8192.0000 max=8274 max_rank=7 min=8123 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32869 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[516, 530, 484, 527, 481, 484, 511, 519, 480, 526, 511, 549, 505, 498, 499, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 499, 522, 572, 517, 472, 513, 488, 521, 504, 507, 467, 512, 486, 485, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 503, 479, 506, 520, 467, 471, 521, 570, 538, 476, 502, 508, 517, 493, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 528, 539, 531, 501, 499, 513, 475, 504, 475, 523, 523, 529, 512, 522, 477, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 504, 526, 462, 518, 544, 483, 512, 487, 502, 505, 524, 509, 480, 530, 548, 15, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 533, 506, 536, 535, 541, 511, 519, 549, 508, 532, 527, 483, 490, 558, 574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 496, 528, 482, 538, 533, 503, 507, 513, 507, 475, 468, 538, 551, 529, 486, 45, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 541, 480, 467, 529, 541, 524, 533, 494, 492, 491, 491, 523, 499, 523, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 8138, 8106, 8165, 8171, 8404, 8228, 8173] avg=8192.0000 max=8404 max_rank=5 min=8106 min_rank=2 imbalance_ratio=1.025879 max_load_reduction=57132 max_load_reduction_pct=0.871765 moved_to_echo=32976 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[527, 532, 519, 502, 503, 504, 479, 523, 495, 536, 546, 542, 489, 525, 516, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 538, 508, 509, 546, 529, 536, 480, 511, 498, 498, 511, 482, 513, 487, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 524, 492, 481, 487, 509, 513, 543, 485, 516, 506, 521, 525, 507, 508, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 521, 518, 540, 538, 555, 517, 547, 509, 472, 486, 471, 503, 539, 516, 522, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 521, 477, 469, 518, 558, 514, 529, 526, 502, 552, 505, 476, 495, 513, 534, 14, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 496, 482, 530, 488, 510, 523, 518, 482, 514, 515, 563, 511, 492, 490, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 496, 525, 514, 482, 558, 510, 481, 491, 495, 500, 488, 515, 488, 497, 494, 44, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 469, 523, 497, 510, 540, 522, 493, 498, 519, 503, 523, 495, 544, 528, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 8194, 8184, 8279, 8235, 8130, 8136, 8148] avg=8192.0000 max=8279 max_rank=3 min=8130 min_rank=5 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32649 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[541, 460, 523, 524, 552, 532, 522, 519, 524, 483, 512, 509, 536, 529, 503, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 460, 532, 527, 525, 521, 524, 501, 492, 562, 506, 501, 467, 525, 535, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 517, 477, 489, 532, 526, 512, 540, 521, 513, 532, 511, 473, 545, 492, 565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 503, 527, 471, 502, 516, 500, 504, 529, 499, 514, 481, 493, 557, 526, 528, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 506, 509, 512, 507, 504, 524, 517, 491, 516, 493, 506, 505, 487, 473, 527, 14, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 500, 506, 519, 516, 474, 494, 515, 482, 556, 505, 516, 542, 525, 531, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 563, 514, 490, 514, 520, 522, 470, 518, 503, 463, 500, 487, 531, 478, 490, 46, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 521, 537, 549, 510, 478, 475, 495, 512, 492, 478, 520, 481, 499, 509, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8301, 8175, 8253, 8178, 8160, 8204, 8145, 8120] avg=8192.0000 max=8301 max_rank=0 min=8120 min_rank=7 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32629 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[587, 499, 542, 495, 500, 511, 536, 495, 495, 482, 509, 516, 529, 477, 519, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 494, 496, 504, 541, 510, 469, 475, 505, 525, 544, 495, 462, 526, 542, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 509, 539, 507, 491, 532, 541, 518, 511, 518, 562, 527, 493, 547, 537, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 497, 526, 514, 520, 490, 493, 516, 511, 495, 514, 520, 504, 525, 547, 543, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 511, 532, 508, 483, 504, 487, 505, 465, 498, 522, 509, 545, 500, 521, 526, 14, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 478, 525, 523, 494, 493, 505, 490, 523, 463, 501, 479, 513, 527, 496, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 517, 465, 497, 486, 486, 523, 523, 529, 510, 540, 525, 530, 514, 486, 481, 44, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 548, 521, 515, 533, 556, 529, 499, 561, 520, 505, 523, 490, 533, 502, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8220, 8080, 8317, 8198, 8162, 8005, 8190, 8364] avg=8192.0000 max=8364 max_rank=7 min=8005 min_rank=5 imbalance_ratio=1.020996 max_load_reduction=57172 max_load_reduction_pct=0.872375 moved_to_echo=32721 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4096, 3968, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4352, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 3968, 4096, 4096, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4096, 3968, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 3968, 4096, 4096, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[500, 487, 531, 527, 530, 506, 502, 481, 525, 507, 507, 490, 522, 468, 539, 507, 516, 520, 511, 479, 477, 551, 477, 537, 494, 511, 517, 500, 533, 490, 520, 519, 510, 501, 504, 501, 500, 522, 511, 500, 506, 525, 509, 497, 531, 506, 521, 534, 509, 517, 498, 508, 506, 529, 512, 513, 480, 493, 496, 503, 475, 541, 564, 446, 505, 503, 537, 518, 559, 529, 505, 524, 530, 529, 527, 507, 523, 478, 526, 556, 497, 512, 522, 514, 506, 487, 521, 467, 507, 509, 487, 498, 513, 532, 534, 541, 520, 505, 556, 515, 507, 553, 505, 529, 464, 547, 515, 541, 494, 499, 491, 503, 535, 530, 524, 544, 495, 499, 512, 514, 495, 510, 499, 528, 516, 546, 506, 487] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[528, 532, 512, 501, 524, 489, 500, 489, 478, 494, 514, 526, 537, 501, 480, 501, 521, 530, 484, 532, 490, 505, 516, 505, 500, 539, 491, 572, 499, 497, 513, 520, 519, 523, 501, 519, 521, 473, 530, 494, 528, 477, 514, 506, 509, 467, 533, 503, 505, 528, 510, 494, 552, 492, 490, 486, 494, 533, 497, 546, 524, 487, 524, 526, 529, 523, 483, 477, 528, 504, 519, 511, 473, 503, 481, 496, 572, 509, 549, 544, 537, 479, 530, 501, 515, 485, 503, 544, 509, 524, 577, 525, 488, 505, 516, 544, 564, 529, 518, 508, 471, 525, 498, 475, 503, 487, 496, 516, 505, 535, 527, 530, 530, 504, 528, 493, 549, 494, 528, 485, 497, 527, 547, 525, 519, 489, 506, 520] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[528, 526, 536, 518, 530, 502, 518, 536, 557, 501, 505, 508, 469, 526, 508, 507, 516, 473, 517, 497, 492, 531, 540, 526, 479, 529, 523, 498, 524, 507, 464, 495, 537, 538, 487, 506, 527, 499, 509, 493, 515, 524, 488, 495, 509, 529, 540, 523, 542, 510, 481, 512, 485, 513, 508, 494, 480, 502, 521, 534, 500, 508, 504, 511, 526, 544, 538, 512, 528, 532, 482, 498, 528, 483, 498, 494, 500, 482, 517, 515, 504, 525, 506, 530, 564, 507, 541, 519, 501, 505, 525, 471, 477, 514, 486, 529, 508, 526, 518, 522, 524, 517, 484, 530, 521, 519, 512, 513, 514, 507, 516, 539, 494, 515, 502, 507, 506, 555, 506, 556, 513, 493, 499, 507, 522, 486, 518, 496] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[525, 538, 492, 546, 489, 534, 512, 541, 520, 514, 515, 483, 500, 483, 517, 475, 534, 543, 508, 504, 496, 478, 527, 482, 536, 535, 529, 485, 525, 512, 519, 485, 490, 488, 527, 486, 516, 498, 494, 537, 528, 553, 485, 484, 499, 514, 533, 522, 503, 536, 493, 489, 515, 517, 525, 537, 518, 499, 543, 500, 528, 468, 494, 543, 475, 529, 476, 473, 513, 507, 519, 520, 563, 482, 506, 513, 467, 533, 494, 516, 534, 535, 526, 507, 518, 502, 548, 526, 489, 485, 536, 527, 477, 530, 531, 519, 537, 493, 475, 523, 501, 521, 581, 550, 493, 524, 518, 480, 508, 486, 493, 500, 518, 535, 537, 503, 519, 507, 482, 503, 497, 506, 548, 528, 521, 525, 471, 533] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[500, 537, 495, 483, 555, 467, 513, 532, 491, 516, 499, 554, 497, 527, 527, 506, 502, 509, 507, 535, 516, 529, 521, 476, 523, 501, 482, 504, 525, 512, 514, 509, 503, 500, 461, 524, 480, 513, 531, 511, 527, 511, 508, 493, 473, 511, 526, 521, 508, 536, 506, 535, 494, 509, 534, 508, 484, 522, 520, 497, 524, 497, 478, 519, 521, 519, 550, 528, 461, 480, 497, 516, 498, 518, 513, 471, 509, 513, 522, 511, 498, 536, 472, 504, 538, 521, 533, 533, 550, 549, 521, 524, 518, 519, 543, 502, 523, 528, 486, 513, 523, 535, 547, 525, 524, 517, 519, 519, 489, 509, 544, 526, 499, 480, 519, 515, 484, 524, 502, 482, 525, 478, 531, 469, 525, 509, 534, 517] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[519, 500, 528, 514, 509, 523, 494, 527, 485, 536, 499, 523, 498, 533, 494, 536, 510, 546, 504, 511, 493, 558, 513, 526, 506, 483, 515, 457, 494, 510, 524, 490, 498, 548, 515, 511, 531, 514, 527, 477, 481, 503, 481, 517, 545, 476, 531, 511, 514, 478, 521, 535, 551, 514, 493, 505, 485, 534, 498, 513, 513, 546, 503, 484, 524, 507, 482, 507, 457, 529, 480, 506, 500, 511, 512, 491, 534, 520, 483, 541, 518, 563, 481, 497, 503, 506, 549, 491, 466, 485, 529, 521, 568, 547, 518, 494, 526, 502, 525, 514, 475, 532, 507, 564, 501, 493, 493, 557, 513, 531, 488, 519, 485, 533, 566, 520, 508, 533, 500, 544, 504, 489, 551, 525, 487, 518, 488, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[528, 532, 512, 501, 524, 489, 500, 489, 478, 494, 514, 526, 537, 501, 480, 501, 521, 530, 484, 532, 490, 505, 516, 505, 500, 539, 491, 572, 499, 497, 513, 520, 519, 523, 501, 519, 521, 473, 530, 494, 528, 477, 514, 506, 509, 467, 533, 503, 505, 528, 510, 494, 552, 492, 490, 486, 494, 533, 497, 546, 524, 487, 524, 526, 529, 523, 483, 477, 528, 504, 519, 511, 473, 503, 481, 496, 572, 509, 549, 544, 537, 479, 530, 501, 515, 485, 503, 544, 509, 524, 577, 525, 488, 505, 516, 544, 564, 529, 518, 508, 471, 525, 498, 475, 503, 487, 496, 516, 505, 535, 527, 530, 530, 504, 528, 493, 549, 494, 528, 485, 497, 527, 547, 525, 519, 489, 506, 520] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[500, 487, 531, 527, 530, 506, 502, 481, 525, 507, 507, 490, 522, 468, 539, 507, 516, 520, 511, 479, 477, 551, 477, 537, 494, 511, 517, 500, 533, 490, 520, 519, 510, 501, 504, 501, 500, 522, 511, 500, 506, 525, 509, 497, 531, 506, 521, 534, 509, 517, 498, 508, 506, 529, 512, 513, 480, 493, 496, 503, 475, 541, 564, 446, 505, 503, 537, 518, 559, 529, 505, 524, 530, 529, 527, 507, 523, 478, 526, 556, 497, 512, 522, 514, 506, 487, 521, 467, 507, 509, 487, 498, 513, 532, 534, 541, 520, 505, 556, 515, 507, 553, 505, 529, 464, 547, 515, 541, 494, 499, 491, 503, 535, 530, 524, 544, 495, 499, 512, 514, 495, 510, 499, 528, 516, 546, 506, 487] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[500, 537, 495, 483, 555, 467, 513, 532, 491, 516, 499, 554, 497, 527, 527, 506, 502, 509, 507, 535, 516, 529, 521, 476, 523, 501, 482, 504, 525, 512, 514, 509, 503, 500, 461, 524, 480, 513, 531, 511, 527, 511, 508, 493, 473, 511, 526, 521, 508, 536, 506, 535, 494, 509, 534, 508, 484, 522, 520, 497, 524, 497, 478, 519, 521, 519, 550, 528, 461, 480, 497, 516, 498, 518, 513, 471, 509, 513, 522, 511, 498, 536, 472, 504, 538, 521, 533, 533, 550, 549, 521, 524, 518, 519, 543, 502, 523, 528, 486, 513, 523, 535, 547, 525, 524, 517, 519, 519, 489, 509, 544, 526, 499, 480, 519, 515, 484, 524, 502, 482, 525, 478, 531, 469, 525, 509, 534, 517] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[538, 505, 494, 561, 485, 492, 491, 534, 488, 549, 513, 522, 520, 518, 489, 508, 510, 504, 514, 510, 494, 516, 521, 524, 523, 526, 505, 516, 514, 514, 483, 523, 503, 527, 536, 478, 511, 547, 503, 539, 529, 500, 505, 500, 547, 504, 521, 533, 490, 509, 556, 514, 507, 489, 491, 510, 493, 476, 523, 526, 481, 479, 496, 501, 510, 518, 517, 488, 542, 535, 468, 510, 541, 514, 510, 526, 550, 527, 534, 573, 475, 489, 491, 522, 472, 493, 516, 541, 516, 495, 509, 483, 527, 526, 502, 497, 543, 544, 512, 543, 505, 503, 521, 510, 527, 550, 529, 486, 489, 535, 523, 495, 533, 503, 512, 523, 507, 487, 518, 511, 494, 491, 498, 532, 465, 491, 515, 496] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[528, 526, 536, 518, 530, 502, 518, 536, 557, 501, 505, 508, 469, 526, 508, 507, 516, 473, 517, 497, 492, 531, 540, 526, 479, 529, 523, 498, 524, 507, 464, 495, 537, 538, 487, 506, 527, 499, 509, 493, 515, 524, 488, 495, 509, 529, 540, 523, 542, 510, 481, 512, 485, 513, 508, 494, 480, 502, 521, 534, 500, 508, 504, 511, 526, 544, 538, 512, 528, 532, 482, 498, 528, 483, 498, 494, 500, 482, 517, 515, 504, 525, 506, 530, 564, 507, 541, 519, 501, 505, 525, 471, 477, 514, 486, 529, 508, 526, 518, 522, 524, 517, 484, 530, 521, 519, 512, 513, 514, 507, 516, 539, 494, 515, 502, 507, 506, 555, 506, 556, 513, 493, 499, 507, 522, 486, 518, 496] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[460, 488, 537, 515, 484, 530, 506, 519, 531, 525, 487, 550, 516, 530, 502, 513, 552, 512, 519, 558, 534, 532, 460, 487, 481, 527, 506, 529, 474, 513, 474, 534, 494, 511, 493, 505, 464, 504, 527, 511, 507, 547, 490, 513, 535, 502, 520, 488, 488, 547, 538, 528, 456, 518, 489, 509, 522, 510, 512, 552, 510, 489, 463, 490, 505, 503, 472, 496, 492, 505, 498, 505, 510, 510, 514, 492, 509, 542, 508, 504, 511, 490, 480, 546, 509, 525, 544, 505, 526, 498, 541, 492, 495, 517, 532, 544, 508, 497, 519, 569, 524, 511, 528, 498, 519, 525, 530, 522, 518, 462, 505, 524, 532, 498, 538, 494, 518, 509, 522, 524, 514, 536, 585, 512, 531, 509, 505, 513] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[525, 538, 492, 546, 489, 534, 512, 541, 520, 514, 515, 483, 500, 483, 517, 475, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 543, 508, 504, 496, 478, 527, 482, 536, 535, 529, 485, 525, 512, 519, 485, 18, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 488, 527, 486, 516, 498, 494, 537, 528, 553, 485, 484, 499, 514, 533, 522, 19, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 536, 493, 489, 515, 517, 525, 537, 518, 499, 543, 500, 528, 468, 494, 543, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 529, 476, 473, 513, 507, 519, 520, 563, 482, 506, 513, 467, 533, 494, 516, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 535, 526, 507, 518, 502, 511, 526, 489, 485, 536, 527, 477, 530, 531, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 493, 475, 523, 501, 521, 581, 550, 493, 524, 518, 480, 508, 486, 493, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 535, 537, 503, 519, 507, 482, 503, 497, 506, 533, 528, 521, 525, 471, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8219, 8181, 8261, 8095, 8253, 8122, 8218] avg=8192.0000 max=8261 max_rank=3 min=8095 min_rank=4 imbalance_ratio=1.008423 max_load_reduction=57275 max_load_reduction_pct=0.873947 moved_to_echo=32688 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[519, 500, 528, 514, 509, 523, 494, 527, 485, 536, 499, 523, 498, 533, 494, 536, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 546, 504, 511, 493, 558, 513, 526, 506, 483, 515, 457, 494, 510, 524, 490, 18, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 548, 515, 511, 531, 514, 527, 477, 481, 503, 481, 517, 545, 476, 531, 511, 19, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 478, 521, 535, 551, 514, 493, 505, 485, 534, 498, 513, 513, 546, 503, 484, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 507, 482, 507, 457, 529, 480, 506, 500, 511, 512, 491, 534, 520, 483, 541, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 563, 481, 497, 503, 506, 512, 491, 466, 485, 529, 521, 568, 547, 518, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 502, 525, 514, 475, 532, 507, 564, 501, 493, 493, 557, 513, 531, 488, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 533, 566, 520, 508, 533, 500, 544, 504, 489, 536, 525, 487, 518, 488, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8161, 8192, 8239, 8093, 8199, 8181, 8250] avg=8192.0000 max=8250 max_rank=7 min=8093 min_rank=4 imbalance_ratio=1.007080 max_load_reduction=57286 max_load_reduction_pct=0.874115 moved_to_echo=32723 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[500, 487, 531, 527, 530, 506, 502, 481, 525, 507, 507, 490, 522, 468, 539, 507, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 520, 511, 479, 477, 551, 477, 537, 494, 511, 517, 500, 533, 490, 520, 519, 17, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 501, 504, 501, 500, 522, 511, 500, 506, 525, 509, 497, 531, 506, 521, 534, 18, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 517, 498, 508, 506, 529, 512, 513, 480, 493, 496, 503, 475, 541, 564, 446, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 503, 537, 518, 559, 529, 505, 524, 530, 529, 527, 507, 523, 478, 526, 556, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 512, 522, 514, 506, 487, 486, 467, 507, 509, 487, 498, 513, 532, 534, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 505, 556, 515, 507, 553, 505, 529, 464, 547, 515, 541, 494, 499, 491, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 530, 524, 544, 495, 499, 512, 514, 495, 510, 486, 528, 516, 546, 506, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8131, 8171, 8203, 8142, 8365, 8112, 8185, 8227] avg=8192.0000 max=8365 max_rank=4 min=8112 min_rank=5 imbalance_ratio=1.021118 max_load_reduction=57171 max_load_reduction_pct=0.872360 moved_to_echo=32889 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[528, 532, 512, 501, 524, 489, 500, 489, 478, 494, 514, 526, 537, 501, 480, 501, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 530, 484, 532, 490, 505, 516, 505, 500, 539, 491, 572, 499, 497, 513, 520, 19, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 523, 501, 519, 521, 473, 530, 494, 528, 477, 514, 506, 509, 467, 533, 503, 21, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 528, 510, 494, 552, 492, 490, 486, 494, 533, 497, 546, 524, 487, 524, 526, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 523, 483, 477, 528, 504, 519, 511, 473, 503, 481, 496, 572, 509, 549, 544, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 479, 530, 501, 515, 485, 463, 544, 509, 524, 577, 525, 488, 505, 516, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 529, 518, 508, 471, 525, 498, 475, 503, 487, 496, 516, 505, 535, 527, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 504, 528, 493, 549, 494, 528, 485, 497, 527, 520, 525, 519, 489, 506, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8113, 8240, 8151, 8248, 8214, 8242, 8114, 8214] avg=8192.0000 max=8248 max_rank=3 min=8113 min_rank=0 imbalance_ratio=1.006836 max_load_reduction=57288 max_load_reduction_pct=0.874146 moved_to_echo=32784 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[525, 538, 492, 546, 489, 534, 512, 541, 520, 514, 515, 483, 500, 483, 517, 475, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 543, 508, 504, 496, 478, 527, 482, 536, 535, 529, 485, 525, 512, 519, 485, 18, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 488, 527, 486, 516, 498, 494, 537, 528, 553, 485, 484, 499, 514, 533, 522, 19, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 536, 493, 489, 515, 517, 525, 537, 518, 499, 543, 500, 528, 468, 494, 543, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 529, 476, 473, 513, 507, 519, 520, 563, 482, 506, 513, 467, 533, 494, 516, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 535, 526, 507, 518, 502, 511, 526, 489, 485, 536, 527, 477, 530, 531, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 493, 475, 523, 501, 521, 581, 550, 493, 524, 518, 480, 508, 486, 493, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 535, 537, 503, 519, 507, 482, 503, 497, 506, 533, 528, 521, 525, 471, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8219, 8181, 8261, 8095, 8253, 8122, 8218] avg=8192.0000 max=8261 max_rank=3 min=8095 min_rank=4 imbalance_ratio=1.008423 max_load_reduction=57275 max_load_reduction_pct=0.873947 moved_to_echo=32688 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[528, 526, 536, 518, 530, 502, 518, 536, 557, 501, 505, 508, 469, 526, 508, 507, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 473, 517, 497, 492, 531, 540, 526, 479, 529, 523, 498, 524, 507, 464, 495, 18, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 538, 487, 506, 527, 499, 509, 493, 515, 524, 488, 495, 509, 529, 540, 523, 19, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 510, 481, 512, 485, 513, 508, 494, 480, 502, 521, 534, 500, 508, 504, 511, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 544, 538, 512, 528, 532, 482, 498, 528, 483, 498, 494, 500, 482, 517, 515, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 525, 506, 530, 564, 507, 504, 519, 501, 505, 525, 471, 477, 514, 486, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 450, 526, 518, 522, 524, 517, 484, 530, 521, 519, 512, 513, 514, 507, 516, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 515, 502, 507, 506, 555, 506, 556, 513, 493, 486, 507, 522, 486, 518, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8277, 8131, 8245, 8156, 8186, 8167, 8212, 8162] avg=8192.0000 max=8277 max_rank=0 min=8131 min_rank=1 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32727 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[500, 537, 495, 483, 555, 467, 513, 532, 491, 516, 499, 554, 497, 527, 527, 506, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 509, 507, 535, 516, 529, 521, 476, 523, 501, 482, 504, 525, 512, 514, 509, 17, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 500, 461, 524, 480, 513, 531, 511, 527, 511, 508, 493, 473, 511, 526, 521, 18, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 536, 506, 535, 494, 509, 534, 508, 484, 522, 520, 497, 524, 497, 478, 519, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 519, 550, 528, 461, 480, 497, 516, 498, 518, 513, 471, 509, 513, 522, 511, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 536, 472, 504, 538, 521, 498, 533, 550, 549, 521, 524, 518, 519, 543, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 464, 528, 486, 513, 523, 535, 547, 525, 524, 517, 519, 519, 489, 509, 544, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 480, 519, 515, 484, 524, 502, 482, 525, 478, 516, 469, 525, 509, 534, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8185, 8118, 8223, 8136, 8326, 8268, 8078] avg=8192.0000 max=8326 max_rank=5 min=8078 min_rank=7 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32808 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[460, 488, 537, 515, 484, 530, 506, 519, 531, 525, 487, 550, 516, 530, 502, 513, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 512, 519, 558, 534, 532, 460, 487, 481, 527, 506, 529, 474, 513, 474, 534, 18, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 511, 493, 505, 464, 504, 527, 511, 507, 547, 490, 513, 535, 502, 520, 488, 19, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 547, 538, 528, 456, 518, 489, 509, 522, 510, 512, 552, 510, 489, 463, 490, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 503, 472, 496, 492, 505, 498, 505, 510, 510, 514, 492, 509, 542, 508, 504, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 490, 480, 546, 509, 525, 507, 505, 526, 498, 541, 492, 495, 517, 532, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 450, 497, 519, 569, 524, 511, 528, 498, 519, 525, 530, 522, 518, 462, 505, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 498, 538, 494, 518, 509, 522, 524, 514, 536, 569, 512, 531, 509, 505, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8213, 8137, 8172, 8075, 8218, 8201, 8324] avg=8192.0000 max=8324 max_rank=7 min=8075 min_rank=4 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32818 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[538, 505, 494, 561, 485, 492, 491, 534, 488, 549, 513, 522, 520, 518, 489, 508, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 504, 514, 510, 494, 516, 521, 524, 523, 526, 505, 516, 514, 514, 483, 523, 17, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 527, 536, 478, 511, 547, 503, 539, 529, 500, 505, 500, 547, 504, 521, 533, 18, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 509, 556, 514, 507, 489, 491, 510, 493, 476, 523, 526, 481, 479, 496, 501, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 518, 517, 488, 542, 535, 468, 510, 541, 514, 510, 526, 550, 527, 534, 573, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 489, 491, 522, 472, 493, 481, 541, 516, 495, 509, 483, 527, 526, 502, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 544, 512, 543, 505, 503, 521, 510, 527, 550, 529, 486, 489, 535, 523, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 503, 512, 523, 507, 487, 518, 511, 494, 491, 485, 532, 465, 491, 515, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8216, 8309, 8095, 8372, 8019, 8253, 8063] avg=8192.0000 max=8372 max_rank=4 min=8019 min_rank=5 imbalance_ratio=1.021973 max_load_reduction=57164 max_load_reduction_pct=0.872253 moved_to_echo=32707 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[500, 487, 531, 527, 530, 506, 502, 481, 525, 507, 507, 490, 522, 468, 539, 507, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 520, 511, 479, 477, 551, 477, 537, 494, 511, 517, 500, 533, 490, 520, 519, 17, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 501, 504, 501, 500, 522, 511, 500, 506, 525, 509, 497, 531, 506, 521, 534, 18, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 517, 498, 508, 506, 529, 512, 513, 480, 493, 496, 503, 475, 541, 564, 446, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 503, 537, 518, 559, 529, 505, 524, 530, 529, 527, 507, 523, 478, 526, 556, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 512, 522, 514, 506, 487, 486, 467, 507, 509, 487, 498, 513, 532, 534, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 505, 556, 515, 507, 553, 505, 529, 464, 547, 515, 541, 494, 499, 491, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 530, 524, 544, 495, 499, 512, 514, 495, 510, 486, 528, 516, 546, 506, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8131, 8171, 8203, 8142, 8365, 8112, 8185, 8227] avg=8192.0000 max=8365 max_rank=4 min=8112 min_rank=5 imbalance_ratio=1.021118 max_load_reduction=57171 max_load_reduction_pct=0.872360 moved_to_echo=32889 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[528, 526, 536, 518, 530, 502, 518, 536, 557, 501, 505, 508, 469, 526, 508, 507, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 473, 517, 497, 492, 531, 540, 526, 479, 529, 523, 498, 524, 507, 464, 495, 18, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 538, 487, 506, 527, 499, 509, 493, 515, 524, 488, 495, 509, 529, 540, 523, 19, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 510, 481, 512, 485, 513, 508, 494, 480, 502, 521, 534, 500, 508, 504, 511, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 544, 538, 512, 528, 532, 482, 498, 528, 483, 498, 494, 500, 482, 517, 515, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 525, 506, 530, 564, 507, 504, 519, 501, 505, 525, 471, 477, 514, 486, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 450, 526, 518, 522, 524, 517, 484, 530, 521, 519, 512, 513, 514, 507, 516, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 515, 502, 507, 506, 555, 506, 556, 513, 493, 486, 507, 522, 486, 518, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8277, 8131, 8245, 8156, 8186, 8167, 8212, 8162] avg=8192.0000 max=8277 max_rank=0 min=8131 min_rank=1 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32727 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[519, 500, 528, 514, 509, 523, 494, 527, 485, 536, 499, 523, 498, 533, 494, 536, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 546, 504, 511, 493, 558, 513, 526, 506, 483, 515, 457, 494, 510, 524, 490, 18, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 548, 515, 511, 531, 514, 527, 477, 481, 503, 481, 517, 545, 476, 531, 511, 19, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 478, 521, 535, 551, 514, 493, 505, 485, 534, 498, 513, 513, 546, 503, 484, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 507, 482, 507, 457, 529, 480, 506, 500, 511, 512, 491, 534, 520, 483, 541, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 563, 481, 497, 503, 506, 512, 491, 466, 485, 529, 521, 568, 547, 518, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 502, 525, 514, 475, 532, 507, 564, 501, 493, 493, 557, 513, 531, 488, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 533, 566, 520, 508, 533, 500, 544, 504, 489, 536, 525, 487, 518, 488, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8161, 8192, 8239, 8093, 8199, 8181, 8250] avg=8192.0000 max=8250 max_rank=7 min=8093 min_rank=4 imbalance_ratio=1.007080 max_load_reduction=57286 max_load_reduction_pct=0.874115 moved_to_echo=32723 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[460, 488, 537, 515, 484, 530, 506, 519, 531, 525, 487, 550, 516, 530, 502, 513, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 512, 519, 558, 534, 532, 460, 487, 481, 527, 506, 529, 474, 513, 474, 534, 18, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 511, 493, 505, 464, 504, 527, 511, 507, 547, 490, 513, 535, 502, 520, 488, 19, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 547, 538, 528, 456, 518, 489, 509, 522, 510, 512, 552, 510, 489, 463, 490, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 503, 472, 496, 492, 505, 498, 505, 510, 510, 514, 492, 509, 542, 508, 504, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 490, 480, 546, 509, 525, 507, 505, 526, 498, 541, 492, 495, 517, 532, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 450, 497, 519, 569, 524, 511, 528, 498, 519, 525, 530, 522, 518, 462, 505, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 498, 538, 494, 518, 509, 522, 524, 514, 536, 569, 512, 531, 509, 505, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8213, 8137, 8172, 8075, 8218, 8201, 8324] avg=8192.0000 max=8324 max_rank=7 min=8075 min_rank=4 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32818 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[3840, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4352, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 3968, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 3968, 4224, 4096, 4096, 4352, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 3968, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4352, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4352, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 3968, 4224, 4096, 4096, 4352, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 3968, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[3840, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[501, 492, 522, 518, 463, 522, 536, 517, 511, 504, 500, 541, 492, 479, 505, 494, 539, 475, 500, 485, 502, 516, 507, 531, 534, 533, 495, 508, 516, 492, 519, 520, 524, 537, 499, 503, 491, 492, 517, 498, 484, 509, 513, 501, 549, 470, 552, 528, 484, 487, 512, 539, 533, 517, 533, 511, 506, 493, 518, 540, 501, 548, 515, 526, 511, 532, 506, 493, 514, 515, 504, 541, 479, 568, 507, 532, 527, 505, 508, 508, 544, 521, 493, 535, 488, 516, 508, 478, 470, 509, 515, 487, 509, 516, 497, 495, 512, 524, 504, 488, 484, 555, 506, 496, 496, 530, 526, 532, 520, 533, 499, 510, 502, 501, 474, 547, 516, 585, 500, 504, 512, 556, 521, 486, 515, 525, 523, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[515, 499, 488, 528, 531, 528, 531, 476, 535, 516, 508, 525, 498, 510, 530, 521, 535, 496, 496, 517, 527, 521, 467, 491, 518, 465, 514, 516, 525, 486, 483, 537, 481, 539, 504, 524, 536, 479, 531, 521, 514, 469, 524, 523, 542, 494, 533, 524, 493, 502, 480, 491, 531, 534, 532, 553, 495, 538, 524, 496, 520, 527, 533, 537, 511, 497, 487, 524, 499, 505, 494, 554, 497, 535, 517, 521, 537, 514, 501, 542, 523, 511, 526, 525, 512, 517, 529, 522, 509, 507, 491, 506, 480, 479, 545, 489, 538, 508, 498, 500, 500, 533, 552, 479, 549, 507, 528, 515, 503, 502, 530, 497, 489, 494, 474, 472, 495, 541, 476, 522, 510, 526, 499, 516, 497, 503, 501, 519] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[515, 542, 498, 507, 510, 514, 493, 528, 534, 501, 505, 504, 504, 517, 461, 520, 509, 521, 517, 509, 522, 538, 516, 512, 519, 523, 490, 515, 514, 520, 501, 528, 503, 539, 512, 486, 506, 488, 502, 550, 497, 496, 519, 487, 536, 478, 528, 498, 535, 486, 488, 535, 507, 486, 494, 541, 476, 558, 490, 491, 477, 541, 490, 483, 526, 511, 521, 557, 505, 524, 517, 507, 541, 484, 521, 457, 545, 496, 509, 483, 515, 536, 541, 505, 548, 506, 553, 488, 498, 560, 537, 524, 525, 533, 509, 501, 543, 535, 500, 514, 496, 512, 500, 506, 516, 535, 471, 481, 497, 494, 488, 491, 485, 500, 488, 514, 519, 535, 567, 542, 521, 494, 480, 484, 517, 520, 546, 552] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[517, 490, 521, 530, 492, 550, 475, 512, 528, 505, 452, 508, 506, 505, 487, 499, 517, 535, 512, 559, 489, 496, 540, 492, 535, 504, 506, 515, 449, 494, 536, 500, 524, 504, 482, 533, 518, 514, 498, 495, 522, 490, 551, 523, 545, 514, 502, 548, 525, 488, 520, 507, 522, 502, 519, 525, 527, 552, 488, 478, 475, 501, 509, 481, 509, 520, 558, 514, 530, 507, 539, 481, 526, 510, 544, 516, 546, 544, 471, 501, 501, 528, 509, 506, 520, 481, 487, 487, 498, 537, 503, 526, 519, 517, 491, 504, 508, 516, 490, 487, 529, 512, 557, 502, 527, 539, 501, 502, 479, 532, 506, 504, 541, 539, 528, 541, 511, 503, 530, 505, 511, 511, 492, 497, 516, 527, 557, 468] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[500, 504, 506, 498, 527, 499, 530, 515, 499, 545, 528, 506, 526, 482, 524, 514, 513, 514, 518, 496, 501, 521, 494, 452, 489, 523, 519, 510, 524, 529, 489, 554, 525, 531, 534, 528, 511, 527, 533, 495, 497, 516, 508, 552, 527, 525, 592, 537, 521, 517, 524, 531, 553, 509, 521, 478, 525, 498, 511, 516, 486, 482, 526, 474, 517, 515, 473, 548, 480, 527, 512, 520, 517, 478, 517, 491, 491, 498, 527, 530, 520, 517, 492, 497, 476, 514, 481, 551, 499, 503, 517, 503, 521, 486, 514, 492, 501, 501, 507, 545, 502, 508, 504, 500, 517, 514, 474, 552, 486, 559, 528, 484, 544, 505, 496, 535, 492, 527, 531, 517, 515, 531, 508, 521, 437, 487, 496, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[530, 556, 517, 486, 532, 486, 495, 538, 499, 500, 479, 529, 513, 508, 515, 493, 528, 537, 528, 520, 553, 528, 473, 508, 549, 507, 533, 515, 545, 505, 487, 507, 543, 483, 498, 490, 512, 530, 505, 519, 496, 531, 537, 487, 445, 555, 482, 511, 512, 497, 480, 495, 537, 500, 504, 503, 526, 537, 515, 516, 505, 542, 509, 538, 505, 504, 485, 533, 516, 529, 489, 511, 498, 530, 543, 534, 574, 505, 476, 513, 555, 491, 519, 530, 567, 552, 548, 530, 508, 541, 478, 462, 460, 523, 495, 502, 504, 481, 516, 493, 474, 549, 493, 475, 501, 470, 491, 486, 500, 509, 532, 503, 522, 507, 548, 520, 522, 549, 494, 517, 510, 479, 504, 504, 513, 486, 544, 495] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[513, 519, 510, 513, 539, 492, 519, 529, 496, 558, 520, 551, 501, 494, 488, 484, 491, 529, 542, 494, 475, 528, 507, 542, 525, 503, 568, 535, 510, 522, 509, 494, 501, 520, 530, 494, 469, 532, 478, 463, 504, 499, 537, 485, 528, 498, 524, 491, 499, 535, 482, 505, 487, 481, 477, 523, 501, 540, 497, 557, 517, 541, 506, 483, 517, 505, 506, 536, 480, 524, 509, 531, 515, 537, 472, 530, 514, 529, 541, 510, 528, 472, 514, 519, 504, 489, 505, 475, 499, 498, 527, 465, 478, 546, 511, 549, 502, 507, 504, 526, 537, 516, 559, 526, 487, 541, 511, 524, 463, 522, 500, 543, 491, 494, 527, 494, 562, 558, 505, 518, 501, 541, 518, 533, 486, 518, 511, 492] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[500, 504, 506, 498, 527, 499, 530, 515, 499, 545, 528, 506, 526, 482, 524, 514, 513, 514, 518, 496, 501, 521, 494, 452, 489, 523, 519, 510, 524, 529, 489, 554, 525, 531, 534, 528, 511, 527, 533, 495, 497, 516, 508, 552, 527, 525, 592, 537, 521, 517, 524, 531, 553, 509, 521, 478, 525, 498, 511, 516, 486, 482, 526, 474, 517, 515, 473, 548, 480, 527, 512, 520, 517, 478, 517, 491, 491, 498, 527, 530, 520, 517, 492, 497, 476, 514, 481, 551, 499, 503, 517, 503, 521, 486, 514, 492, 501, 501, 507, 545, 502, 508, 504, 500, 517, 514, 474, 552, 486, 559, 528, 484, 544, 505, 496, 535, 492, 527, 531, 517, 515, 531, 508, 521, 437, 487, 496, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[517, 490, 521, 530, 492, 550, 475, 512, 528, 505, 452, 508, 506, 505, 487, 499, 517, 535, 512, 559, 489, 496, 540, 492, 535, 504, 506, 515, 449, 494, 536, 500, 524, 504, 482, 533, 518, 514, 498, 495, 522, 490, 551, 523, 545, 514, 502, 548, 525, 488, 520, 507, 522, 502, 519, 525, 527, 552, 488, 478, 475, 501, 509, 481, 509, 520, 558, 514, 530, 507, 539, 481, 526, 510, 544, 516, 546, 544, 471, 501, 501, 528, 509, 506, 520, 481, 487, 487, 498, 537, 503, 526, 519, 517, 491, 504, 508, 516, 490, 487, 529, 512, 557, 502, 527, 539, 501, 502, 479, 532, 506, 504, 541, 539, 528, 541, 511, 503, 530, 505, 511, 511, 492, 497, 516, 527, 557, 468] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[522, 459, 513, 527, 462, 502, 502, 526, 513, 517, 509, 531, 496, 518, 488, 518, 515, 551, 542, 519, 524, 507, 525, 484, 503, 532, 537, 521, 513, 492, 515, 527, 512, 514, 560, 539, 510, 518, 519, 530, 480, 533, 529, 483, 529, 509, 473, 512, 519, 506, 534, 568, 544, 480, 516, 491, 493, 483, 510, 507, 497, 523, 522, 542, 520, 498, 537, 496, 511, 539, 537, 504, 464, 492, 546, 488, 491, 531, 515, 479, 550, 530, 543, 523, 502, 544, 534, 519, 490, 473, 474, 536, 511, 517, 489, 500, 495, 506, 500, 509, 556, 475, 499, 497, 530, 495, 498, 508, 532, 468, 486, 538, 491, 535, 536, 454, 487, 549, 497, 536, 488, 516, 510, 524, 475, 564, 506, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[515, 542, 498, 507, 510, 514, 493, 528, 534, 501, 505, 504, 504, 517, 461, 520, 22, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 521, 517, 509, 522, 538, 516, 512, 493, 523, 490, 515, 514, 520, 501, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 539, 512, 486, 506, 488, 502, 550, 497, 496, 505, 487, 536, 478, 528, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 486, 488, 535, 507, 486, 494, 541, 476, 558, 490, 491, 477, 541, 490, 483, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 511, 521, 557, 505, 524, 517, 507, 541, 484, 521, 457, 512, 496, 509, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 536, 541, 505, 548, 506, 553, 488, 498, 560, 537, 524, 525, 533, 509, 501, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 535, 500, 514, 496, 512, 500, 506, 516, 535, 471, 481, 497, 494, 488, 491, 33, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 500, 488, 514, 519, 520, 567, 542, 521, 494, 480, 484, 517, 520, 546, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8228, 8111, 8082, 8171, 8394, 8116, 8249] avg=8192.0000 max=8394 max_rank=5 min=8082 min_rank=3 imbalance_ratio=1.024658 max_load_reduction=57142 max_load_reduction_pct=0.871918 moved_to_echo=32930 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[501, 492, 522, 518, 463, 522, 536, 517, 511, 504, 500, 541, 492, 479, 505, 494, 22, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 475, 500, 485, 502, 516, 507, 531, 508, 533, 495, 508, 516, 492, 519, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 537, 499, 503, 491, 492, 517, 498, 484, 509, 499, 501, 549, 470, 552, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 487, 512, 539, 533, 517, 533, 511, 506, 493, 518, 540, 501, 548, 515, 526, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 532, 506, 493, 514, 515, 504, 541, 479, 568, 507, 532, 495, 505, 508, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 521, 493, 535, 488, 516, 508, 478, 470, 509, 515, 487, 509, 516, 497, 495, 10, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 524, 504, 488, 484, 555, 506, 496, 496, 530, 526, 532, 520, 533, 499, 510, 32, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 501, 474, 547, 516, 568, 500, 504, 512, 556, 521, 486, 515, 525, 523, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8130, 8146, 8153, 8267, 8218, 8097, 8251, 8274] avg=8192.0000 max=8274 max_rank=7 min=8097 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32840 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[517, 490, 521, 530, 492, 550, 475, 512, 528, 505, 452, 508, 506, 505, 487, 499, 22, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 535, 512, 559, 489, 496, 540, 492, 509, 504, 506, 515, 449, 494, 536, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 504, 482, 533, 518, 514, 498, 495, 522, 490, 536, 523, 545, 514, 502, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 488, 520, 507, 522, 502, 519, 525, 527, 552, 488, 478, 475, 501, 509, 481, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 520, 558, 514, 530, 507, 539, 481, 526, 510, 544, 516, 513, 544, 471, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 528, 509, 506, 520, 481, 487, 487, 498, 537, 503, 526, 519, 517, 491, 504, 11, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 516, 490, 487, 529, 512, 557, 502, 527, 539, 501, 502, 479, 532, 506, 504, 33, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 539, 528, 541, 511, 489, 530, 505, 511, 511, 492, 497, 516, 527, 557, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8108, 8153, 8248, 8123, 8283, 8130, 8228, 8263] avg=8192.0000 max=8283 max_rank=4 min=8108 min_rank=0 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32904 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[530, 556, 517, 486, 532, 486, 495, 538, 499, 500, 479, 529, 513, 508, 515, 493, 528, 537, 528, 520, 553, 528, 473, 508, 549, 507, 533, 515, 545, 505, 487, 507, 543, 483, 498, 490, 512, 530, 505, 519, 496, 531, 537, 487, 445, 555, 482, 511, 512, 497, 480, 495, 537, 500, 504, 503, 526, 537, 515, 516, 505, 542, 509, 538, 505, 504, 485, 533, 516, 529, 489, 511, 498, 530, 543, 534, 574, 505, 476, 513, 555, 491, 519, 530, 567, 552, 548, 530, 508, 541, 478, 462, 460, 523, 495, 502, 504, 481, 516, 493, 474, 549, 493, 475, 501, 470, 491, 486, 500, 509, 532, 503, 522, 507, 548, 520, 522, 549, 494, 517, 510, 479, 504, 504, 513, 486, 544, 495] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[515, 542, 498, 507, 510, 514, 493, 528, 534, 501, 505, 504, 504, 517, 461, 520, 509, 521, 517, 509, 522, 538, 516, 512, 519, 523, 490, 515, 514, 520, 501, 528, 503, 539, 512, 486, 506, 488, 502, 550, 497, 496, 519, 487, 536, 478, 528, 498, 535, 486, 488, 535, 507, 486, 494, 541, 476, 558, 490, 491, 477, 541, 490, 483, 526, 511, 521, 557, 505, 524, 517, 507, 541, 484, 521, 457, 545, 496, 509, 483, 515, 536, 541, 505, 548, 506, 553, 488, 498, 560, 537, 524, 525, 533, 509, 501, 543, 535, 500, 514, 496, 512, 500, 506, 516, 535, 471, 481, 497, 494, 488, 491, 485, 500, 488, 514, 519, 535, 567, 542, 521, 494, 480, 484, 517, 520, 546, 552] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[515, 499, 488, 528, 531, 528, 531, 476, 535, 516, 508, 525, 498, 510, 530, 521, 535, 496, 496, 517, 527, 521, 467, 491, 518, 465, 514, 516, 525, 486, 483, 537, 481, 539, 504, 524, 536, 479, 531, 521, 514, 469, 524, 523, 542, 494, 533, 524, 493, 502, 480, 491, 531, 534, 532, 553, 495, 538, 524, 496, 520, 527, 533, 537, 511, 497, 487, 524, 499, 505, 494, 554, 497, 535, 517, 521, 537, 514, 501, 542, 523, 511, 526, 525, 512, 517, 529, 522, 509, 507, 491, 506, 480, 479, 545, 489, 538, 508, 498, 500, 500, 533, 552, 479, 549, 507, 528, 515, 503, 502, 530, 497, 489, 494, 474, 472, 495, 541, 476, 522, 510, 526, 499, 516, 497, 503, 501, 519] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[500, 504, 506, 498, 527, 499, 530, 515, 499, 545, 528, 506, 526, 482, 524, 514, 25, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 514, 518, 496, 501, 521, 494, 452, 458, 523, 519, 510, 524, 529, 489, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 531, 534, 528, 511, 527, 533, 495, 497, 516, 485, 552, 527, 525, 592, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 517, 524, 531, 553, 509, 521, 478, 525, 498, 511, 516, 486, 482, 526, 474, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 515, 473, 548, 480, 527, 512, 520, 517, 478, 517, 491, 458, 498, 527, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 517, 492, 497, 476, 514, 481, 551, 499, 503, 517, 503, 521, 486, 514, 492, 15, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 501, 507, 545, 502, 508, 504, 500, 517, 514, 474, 552, 486, 559, 528, 484, 33, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 505, 496, 535, 492, 504, 531, 517, 515, 531, 508, 521, 437, 487, 496, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8241, 8115, 8415, 8180, 8108, 8108, 8221, 8148] avg=8192.0000 max=8415 max_rank=2 min=8108 min_rank=4 imbalance_ratio=1.027222 max_load_reduction=57121 max_load_reduction_pct=0.871597 moved_to_echo=32585 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[515, 499, 488, 528, 531, 528, 531, 476, 535, 516, 508, 525, 498, 510, 530, 521, 22, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 496, 496, 517, 527, 521, 467, 491, 492, 465, 514, 516, 525, 486, 483, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 539, 504, 524, 536, 479, 531, 521, 514, 469, 510, 523, 542, 494, 533, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 502, 480, 491, 531, 534, 532, 553, 495, 538, 524, 496, 520, 527, 533, 537, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 497, 487, 524, 499, 505, 494, 554, 497, 535, 517, 521, 505, 514, 501, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 511, 526, 525, 512, 517, 529, 522, 509, 507, 491, 506, 480, 479, 545, 489, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 508, 498, 500, 500, 533, 552, 479, 549, 507, 528, 515, 503, 502, 530, 497, 32, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 494, 474, 472, 495, 526, 476, 522, 510, 526, 499, 516, 497, 503, 501, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8271, 8068, 8224, 8290, 8203, 8186, 8275, 8019] avg=8192.0000 max=8290 max_rank=3 min=8019 min_rank=7 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32683 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[530, 556, 517, 486, 532, 486, 495, 538, 499, 500, 479, 529, 513, 508, 515, 493, 23, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 537, 528, 520, 553, 528, 473, 508, 522, 507, 533, 515, 545, 505, 487, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 483, 498, 490, 512, 530, 505, 519, 496, 531, 523, 487, 445, 555, 482, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 497, 480, 495, 537, 500, 504, 503, 526, 537, 515, 516, 505, 542, 509, 538, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 504, 485, 533, 516, 529, 489, 511, 498, 530, 543, 534, 539, 505, 476, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 491, 519, 530, 567, 552, 548, 530, 508, 541, 478, 462, 460, 523, 495, 502, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 481, 516, 493, 474, 549, 493, 475, 501, 470, 491, 486, 500, 509, 532, 503, 35, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 507, 548, 520, 522, 534, 494, 517, 510, 479, 504, 504, 513, 486, 544, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8296, 8110, 8220, 8210, 8276, 8016, 8199] avg=8192.0000 max=8296 max_rank=1 min=8016 min_rank=6 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32701 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[513, 519, 510, 513, 539, 492, 519, 529, 496, 558, 520, 551, 501, 494, 488, 484, 22, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 529, 542, 494, 475, 528, 507, 542, 499, 503, 568, 535, 510, 522, 509, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 520, 530, 494, 469, 532, 478, 463, 504, 499, 523, 485, 528, 498, 524, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 535, 482, 505, 487, 481, 477, 523, 501, 540, 497, 557, 517, 541, 506, 483, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 505, 506, 536, 480, 524, 509, 531, 515, 537, 472, 530, 483, 529, 541, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 472, 514, 519, 504, 489, 505, 475, 499, 498, 527, 465, 478, 546, 511, 549, 10, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 507, 504, 526, 537, 516, 559, 526, 487, 541, 511, 524, 463, 522, 500, 543, 31, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 494, 527, 494, 562, 542, 505, 518, 501, 541, 518, 533, 486, 518, 511, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8258, 8248, 8039, 8135, 8225, 8095, 8303, 8233] avg=8192.0000 max=8303 max_rank=6 min=8039 min_rank=2 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32856 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[517, 490, 521, 530, 492, 550, 475, 512, 528, 505, 452, 508, 506, 505, 487, 499, 22, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 535, 512, 559, 489, 496, 540, 492, 509, 504, 506, 515, 449, 494, 536, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 504, 482, 533, 518, 514, 498, 495, 522, 490, 536, 523, 545, 514, 502, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 488, 520, 507, 522, 502, 519, 525, 527, 552, 488, 478, 475, 501, 509, 481, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 520, 558, 514, 530, 507, 539, 481, 526, 510, 544, 516, 513, 544, 471, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 528, 509, 506, 520, 481, 487, 487, 498, 537, 503, 526, 519, 517, 491, 504, 11, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 516, 490, 487, 529, 512, 557, 502, 527, 539, 501, 502, 479, 532, 506, 504, 33, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 539, 528, 541, 511, 489, 530, 505, 511, 511, 492, 497, 516, 527, 557, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8108, 8153, 8248, 8123, 8283, 8130, 8228, 8263] avg=8192.0000 max=8283 max_rank=4 min=8108 min_rank=0 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32904 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[501, 492, 522, 518, 463, 522, 536, 517, 511, 504, 500, 541, 492, 479, 505, 494, 22, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 475, 500, 485, 502, 516, 507, 531, 508, 533, 495, 508, 516, 492, 519, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 537, 499, 503, 491, 492, 517, 498, 484, 509, 499, 501, 549, 470, 552, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 487, 512, 539, 533, 517, 533, 511, 506, 493, 518, 540, 501, 548, 515, 526, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 532, 506, 493, 514, 515, 504, 541, 479, 568, 507, 532, 495, 505, 508, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 521, 493, 535, 488, 516, 508, 478, 470, 509, 515, 487, 509, 516, 497, 495, 10, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 524, 504, 488, 484, 555, 506, 496, 496, 530, 526, 532, 520, 533, 499, 510, 32, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 501, 474, 547, 516, 568, 500, 504, 512, 556, 521, 486, 515, 525, 523, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8130, 8146, 8153, 8267, 8218, 8097, 8251, 8274] avg=8192.0000 max=8274 max_rank=7 min=8097 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32840 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[522, 459, 513, 527, 462, 502, 502, 526, 513, 517, 509, 531, 496, 518, 488, 518, 21, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 551, 542, 519, 524, 507, 525, 484, 478, 532, 537, 521, 513, 492, 515, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 514, 560, 539, 510, 518, 519, 530, 480, 533, 515, 483, 529, 509, 473, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 506, 534, 568, 544, 480, 516, 491, 493, 483, 510, 507, 497, 523, 522, 542, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 498, 537, 496, 511, 539, 537, 504, 464, 492, 546, 488, 461, 531, 515, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 530, 543, 523, 502, 544, 534, 519, 490, 473, 474, 536, 511, 517, 489, 500, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 506, 500, 509, 556, 475, 499, 497, 530, 495, 498, 508, 532, 468, 486, 538, 30, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 535, 536, 454, 487, 534, 497, 536, 488, 516, 510, 524, 475, 564, 506, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8134, 8282, 8236, 8239, 8118, 8250, 8126, 8151] avg=8192.0000 max=8282 max_rank=1 min=8118 min_rank=4 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32645 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[530, 556, 517, 486, 532, 486, 495, 538, 499, 500, 479, 529, 513, 508, 515, 493, 23, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 537, 528, 520, 553, 528, 473, 508, 522, 507, 533, 515, 545, 505, 487, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 483, 498, 490, 512, 530, 505, 519, 496, 531, 523, 487, 445, 555, 482, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 497, 480, 495, 537, 500, 504, 503, 526, 537, 515, 516, 505, 542, 509, 538, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 504, 485, 533, 516, 529, 489, 511, 498, 530, 543, 534, 539, 505, 476, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 491, 519, 530, 567, 552, 548, 530, 508, 541, 478, 462, 460, 523, 495, 502, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 481, 516, 493, 474, 549, 493, 475, 501, 470, 491, 486, 500, 509, 532, 503, 35, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 507, 548, 520, 522, 534, 494, 517, 510, 479, 504, 504, 513, 486, 544, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8296, 8110, 8220, 8210, 8276, 8016, 8199] avg=8192.0000 max=8296 max_rank=1 min=8016 min_rank=6 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32701 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[515, 542, 498, 507, 510, 514, 493, 528, 534, 501, 505, 504, 504, 517, 461, 520, 22, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 521, 517, 509, 522, 538, 516, 512, 493, 523, 490, 515, 514, 520, 501, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 539, 512, 486, 506, 488, 502, 550, 497, 496, 505, 487, 536, 478, 528, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 486, 488, 535, 507, 486, 494, 541, 476, 558, 490, 491, 477, 541, 490, 483, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 511, 521, 557, 505, 524, 517, 507, 541, 484, 521, 457, 512, 496, 509, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 536, 541, 505, 548, 506, 553, 488, 498, 560, 537, 524, 525, 533, 509, 501, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 535, 500, 514, 496, 512, 500, 506, 516, 535, 471, 481, 497, 494, 488, 491, 33, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 500, 488, 514, 519, 520, 567, 542, 521, 494, 480, 484, 517, 520, 546, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8228, 8111, 8082, 8171, 8394, 8116, 8249] avg=8192.0000 max=8394 max_rank=5 min=8082 min_rank=3 imbalance_ratio=1.024658 max_load_reduction=57142 max_load_reduction_pct=0.871918 moved_to_echo=32930 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[515, 499, 488, 528, 531, 528, 531, 476, 535, 516, 508, 525, 498, 510, 530, 521, 22, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 496, 496, 517, 527, 521, 467, 491, 492, 465, 514, 516, 525, 486, 483, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 539, 504, 524, 536, 479, 531, 521, 514, 469, 510, 523, 542, 494, 533, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 502, 480, 491, 531, 534, 532, 553, 495, 538, 524, 496, 520, 527, 533, 537, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 497, 487, 524, 499, 505, 494, 554, 497, 535, 517, 521, 505, 514, 501, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 511, 526, 525, 512, 517, 529, 522, 509, 507, 491, 506, 480, 479, 545, 489, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 508, 498, 500, 500, 533, 552, 479, 549, 507, 528, 515, 503, 502, 530, 497, 32, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 494, 474, 472, 495, 526, 476, 522, 510, 526, 499, 516, 497, 503, 501, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8271, 8068, 8224, 8290, 8203, 8186, 8275, 8019] avg=8192.0000 max=8290 max_rank=3 min=8019 min_rank=7 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32683 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[4352, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 3968, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 3968, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 3968, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[4352, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 3968, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[519, 516, 530, 513, 493, 498, 525, 500, 493, 541, 483, 509, 519, 533, 509, 487, 491, 472, 518, 484, 510, 494, 526, 494, 506, 485, 488, 511, 472, 495, 485, 482, 503, 516, 498, 535, 521, 531, 505, 507, 512, 516, 452, 562, 537, 530, 500, 513, 497, 550, 529, 491, 536, 513, 543, 512, 481, 537, 522, 510, 531, 505, 500, 499, 527, 533, 530, 500, 496, 504, 517, 551, 513, 517, 515, 554, 496, 489, 501, 511, 530, 486, 525, 465, 532, 480, 508, 495, 541, 497, 556, 484, 545, 517, 537, 518, 501, 501, 520, 542, 473, 526, 533, 534, 525, 502, 491, 513, 556, 563, 499, 538, 526, 517, 527, 493, 461, 499, 552, 487, 512, 523, 512, 496, 533, 508, 504, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[489, 514, 516, 499, 511, 512, 502, 511, 519, 541, 484, 486, 505, 521, 517, 488, 535, 546, 542, 536, 451, 546, 466, 515, 514, 509, 532, 511, 514, 467, 556, 518, 528, 463, 499, 493, 509, 515, 540, 479, 541, 557, 516, 447, 506, 511, 528, 477, 513, 532, 519, 527, 559, 502, 530, 471, 505, 499, 474, 518, 515, 508, 523, 521, 501, 545, 518, 555, 507, 469, 500, 513, 521, 515, 509, 503, 493, 517, 506, 521, 500, 536, 555, 512, 533, 495, 523, 492, 488, 498, 496, 481, 507, 522, 505, 512, 468, 570, 538, 496, 510, 512, 517, 472, 563, 516, 542, 544, 477, 525, 525, 516, 497, 534, 500, 517, 512, 536, 478, 516, 528, 516, 494, 509, 505, 540, 535, 482] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[541, 548, 520, 494, 534, 515, 485, 488, 484, 511, 541, 501, 485, 557, 563, 523, 526, 491, 498, 563, 518, 504, 509, 551, 561, 529, 532, 533, 471, 564, 476, 481, 534, 525, 502, 542, 503, 519, 568, 512, 508, 551, 519, 480, 528, 480, 494, 520, 508, 490, 503, 519, 521, 513, 460, 496, 505, 464, 529, 476, 527, 503, 498, 484, 469, 490, 484, 512, 523, 516, 505, 509, 531, 481, 471, 482, 520, 493, 456, 514, 488, 494, 503, 529, 517, 522, 503, 529, 505, 468, 545, 520, 507, 503, 547, 529, 525, 546, 544, 533, 509, 507, 511, 516, 510, 513, 542, 499, 505, 492, 484, 510, 535, 519, 517, 525, 512, 500, 529, 487, 521, 570, 476, 528, 540, 502, 514, 472] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[479, 545, 506, 496, 479, 532, 471, 541, 502, 545, 517, 479, 539, 515, 510, 535, 475, 560, 496, 518, 498, 529, 505, 524, 564, 484, 536, 493, 498, 551, 534, 491, 492, 546, 497, 483, 505, 464, 506, 535, 498, 499, 509, 516, 530, 506, 512, 497, 470, 543, 519, 487, 537, 521, 499, 506, 517, 474, 537, 535, 509, 522, 544, 511, 496, 512, 495, 522, 493, 519, 539, 542, 486, 498, 512, 514, 550, 510, 467, 493, 547, 524, 493, 500, 526, 483, 513, 494, 533, 554, 500, 531, 500, 503, 484, 524, 504, 505, 501, 540, 517, 513, 520, 517, 502, 516, 490, 539, 527, 507, 480, 477, 510, 542, 524, 532, 512, 514, 537, 521, 502, 524, 507, 469, 517, 475, 552, 513] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[503, 504, 524, 515, 483, 548, 499, 493, 510, 573, 478, 554, 514, 507, 565, 522, 510, 490, 528, 499, 512, 511, 508, 485, 539, 504, 513, 519, 546, 503, 565, 515, 491, 498, 513, 490, 515, 511, 498, 482, 487, 513, 489, 507, 516, 536, 507, 544, 521, 494, 483, 527, 522, 484, 505, 475, 553, 482, 475, 504, 500, 548, 544, 515, 512, 535, 495, 512, 523, 494, 504, 497, 513, 497, 517, 505, 492, 506, 526, 531, 480, 513, 520, 520, 550, 500, 502, 517, 521, 516, 500, 529, 498, 513, 535, 500, 527, 510, 530, 519, 522, 537, 500, 510, 535, 488, 472, 464, 535, 547, 506, 536, 532, 474, 493, 549, 491, 523, 501, 523, 518, 493, 510, 517, 518, 482, 527, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[535, 550, 532, 508, 497, 558, 484, 501, 519, 505, 512, 503, 476, 495, 471, 509, 513, 510, 544, 521, 503, 550, 497, 524, 559, 467, 540, 499, 495, 486, 494, 541, 492, 545, 476, 478, 495, 512, 519, 503, 548, 539, 500, 507, 516, 481, 512, 514, 468, 538, 514, 534, 496, 515, 511, 496, 539, 484, 475, 517, 512, 527, 485, 516, 507, 500, 461, 475, 505, 504, 490, 519, 495, 496, 481, 538, 482, 522, 582, 541, 509, 531, 516, 493, 550, 530, 495, 507, 515, 493, 503, 525, 516, 532, 522, 496, 499, 522, 492, 498, 540, 547, 523, 512, 528, 514, 546, 512, 513, 532, 519, 514, 494, 549, 540, 548, 477, 525, 542, 527, 507, 517, 518, 492, 509, 489, 519, 479] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[532, 492, 522, 529, 532, 489, 529, 486, 508, 521, 493, 510, 477, 520, 499, 500, 498, 533, 499, 526, 530, 492, 488, 502, 518, 511, 502, 526, 534, 534, 497, 486, 539, 537, 540, 513, 513, 524, 477, 506, 515, 492, 477, 544, 536, 503, 518, 538, 516, 546, 514, 494, 516, 474, 509, 522, 541, 543, 521, 486, 525, 502, 513, 554, 524, 527, 520, 528, 508, 475, 563, 474, 508, 527, 502, 518, 523, 485, 546, 511, 529, 480, 509, 516, 488, 499, 552, 506, 516, 518, 487, 496, 499, 535, 515, 511, 543, 486, 523, 524, 509, 510, 515, 498, 504, 531, 529, 479, 478, 540, 503, 486, 484, 507, 499, 511, 521, 542, 526, 500, 543, 522, 480, 482, 492, 488, 478, 545] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[503, 504, 524, 515, 483, 548, 499, 493, 510, 573, 478, 554, 514, 507, 565, 522, 510, 490, 528, 499, 512, 511, 508, 485, 539, 504, 513, 519, 546, 503, 565, 515, 491, 498, 513, 490, 515, 511, 498, 482, 487, 513, 489, 507, 516, 536, 507, 544, 521, 494, 483, 527, 522, 484, 505, 475, 553, 482, 475, 504, 500, 548, 544, 515, 512, 535, 495, 512, 523, 494, 504, 497, 513, 497, 517, 505, 492, 506, 526, 531, 480, 513, 520, 520, 550, 500, 502, 517, 521, 516, 500, 529, 498, 513, 535, 500, 527, 510, 530, 519, 522, 537, 500, 510, 535, 488, 472, 464, 535, 547, 506, 536, 532, 474, 493, 549, 491, 523, 501, 523, 518, 493, 510, 517, 518, 482, 527, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[541, 548, 520, 494, 534, 515, 485, 488, 484, 511, 541, 501, 485, 557, 563, 523, 526, 491, 498, 563, 518, 504, 509, 551, 561, 529, 532, 533, 471, 564, 476, 481, 534, 525, 502, 542, 503, 519, 568, 512, 508, 551, 519, 480, 528, 480, 494, 520, 508, 490, 503, 519, 521, 513, 460, 496, 505, 464, 529, 476, 527, 503, 498, 484, 469, 490, 484, 512, 523, 516, 505, 509, 531, 481, 471, 482, 520, 493, 456, 514, 488, 494, 503, 529, 517, 522, 503, 529, 505, 468, 545, 520, 507, 503, 547, 529, 525, 546, 544, 533, 509, 507, 511, 516, 510, 513, 542, 499, 505, 492, 484, 510, 535, 519, 517, 525, 512, 500, 529, 487, 521, 570, 476, 528, 540, 502, 514, 472] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[548, 519, 480, 493, 532, 510, 543, 521, 517, 487, 526, 486, 521, 499, 518, 522, 530, 495, 527, 537, 506, 505, 482, 541, 529, 507, 509, 511, 499, 531, 525, 550, 513, 542, 501, 521, 557, 499, 547, 520, 536, 539, 485, 479, 526, 539, 477, 510, 517, 560, 486, 519, 496, 534, 533, 504, 516, 499, 521, 513, 477, 483, 518, 517, 469, 502, 505, 557, 537, 513, 479, 493, 520, 534, 524, 516, 475, 522, 526, 491, 477, 493, 546, 507, 513, 497, 526, 497, 477, 493, 528, 483, 473, 528, 560, 525, 519, 521, 444, 526, 520, 501, 500, 512, 503, 539, 504, 512, 517, 471, 523, 554, 524, 517, 474, 530, 521, 497, 500, 508, 525, 482, 491, 479, 500, 511, 537, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[519, 516, 530, 513, 493, 498, 525, 500, 493, 541, 483, 509, 519, 533, 509, 487, 491, 472, 518, 484, 510, 494, 526, 494, 506, 485, 488, 511, 472, 495, 485, 482, 503, 516, 498, 535, 521, 531, 505, 507, 512, 516, 452, 562, 537, 530, 500, 513, 497, 550, 529, 491, 536, 513, 543, 512, 481, 537, 522, 510, 531, 505, 500, 499, 527, 533, 530, 500, 496, 504, 517, 551, 513, 517, 515, 554, 496, 489, 501, 511, 530, 486, 525, 465, 532, 480, 508, 495, 541, 497, 556, 484, 545, 517, 537, 518, 501, 501, 520, 542, 473, 526, 533, 534, 525, 502, 491, 513, 556, 563, 499, 538, 526, 517, 527, 493, 461, 499, 552, 487, 512, 523, 512, 496, 533, 508, 504, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[489, 514, 516, 499, 511, 512, 502, 511, 519, 541, 484, 486, 505, 521, 517, 488, 535, 546, 542, 536, 451, 546, 466, 515, 514, 509, 532, 511, 514, 467, 556, 518, 528, 463, 499, 493, 509, 515, 540, 479, 541, 557, 516, 447, 506, 511, 528, 477, 513, 532, 519, 527, 559, 502, 530, 471, 505, 499, 474, 518, 515, 508, 523, 521, 501, 545, 518, 555, 507, 469, 500, 513, 521, 515, 509, 503, 493, 517, 506, 521, 500, 536, 555, 512, 533, 495, 523, 492, 488, 498, 496, 481, 507, 522, 505, 512, 468, 570, 538, 496, 510, 512, 517, 472, 563, 516, 542, 544, 477, 525, 525, 516, 497, 534, 500, 517, 512, 536, 478, 516, 528, 516, 494, 509, 505, 540, 535, 482] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[519, 516, 530, 513, 493, 498, 525, 500, 493, 538, 483, 509, 519, 533, 509, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 472, 518, 484, 510, 494, 526, 494, 490, 485, 488, 511, 472, 495, 485, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 516, 498, 535, 521, 531, 505, 507, 512, 516, 452, 562, 537, 530, 500, 513, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 550, 529, 491, 536, 513, 543, 512, 481, 537, 522, 510, 531, 505, 500, 499, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 533, 530, 500, 496, 504, 517, 551, 513, 517, 515, 554, 496, 489, 501, 511, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 486, 525, 465, 532, 480, 508, 495, 541, 497, 556, 484, 545, 517, 537, 518, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 501, 520, 498, 473, 526, 533, 534, 525, 502, 491, 513, 556, 563, 499, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 517, 527, 493, 461, 499, 552, 487, 512, 523, 512, 496, 533, 508, 504, 524, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8165, 7897, 8239, 8268, 8296, 8218, 8273, 8180] avg=8192.0000 max=8296 max_rank=4 min=7897 min_rank=1 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32967 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[489, 514, 516, 499, 511, 512, 502, 511, 519, 538, 484, 486, 505, 521, 517, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 546, 542, 536, 451, 546, 466, 515, 497, 509, 532, 511, 514, 467, 556, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 463, 499, 493, 509, 515, 540, 479, 541, 557, 516, 447, 506, 511, 528, 477, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 532, 519, 527, 559, 502, 530, 471, 505, 499, 474, 518, 515, 508, 523, 521, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 545, 518, 555, 507, 469, 500, 513, 521, 515, 509, 503, 493, 517, 506, 521, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 536, 555, 512, 533, 495, 523, 492, 488, 498, 496, 481, 507, 522, 505, 512, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 570, 538, 456, 510, 512, 517, 472, 563, 516, 542, 544, 477, 525, 525, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 534, 500, 517, 512, 536, 478, 516, 528, 516, 494, 509, 505, 540, 535, 482, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8112, 8241, 8110, 8228, 8231, 8157, 8251, 8206] avg=8192.0000 max=8251 max_rank=6 min=8110 min_rank=2 imbalance_ratio=1.007202 max_load_reduction=57285 max_load_reduction_pct=0.874100 moved_to_echo=32845 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[479, 545, 506, 496, 479, 532, 471, 541, 502, 542, 517, 479, 539, 515, 510, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 560, 496, 518, 498, 529, 505, 524, 546, 484, 536, 493, 498, 551, 534, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 546, 497, 483, 505, 464, 506, 535, 498, 499, 509, 516, 530, 506, 512, 497, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 543, 519, 487, 537, 521, 499, 506, 517, 474, 537, 535, 509, 522, 544, 511, 11, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 512, 495, 522, 493, 519, 539, 542, 486, 498, 512, 514, 550, 510, 467, 493, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 524, 493, 500, 526, 483, 513, 494, 533, 554, 500, 531, 500, 503, 484, 524, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 505, 501, 496, 517, 513, 520, 517, 502, 516, 490, 539, 527, 507, 480, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 542, 524, 532, 512, 514, 537, 521, 502, 524, 507, 469, 517, 475, 552, 513, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8238, 8096, 8244, 8190, 8211, 8111, 8258] avg=8192.0000 max=8258 max_rank=7 min=8096 min_rank=2 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32770 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[503, 504, 524, 515, 483, 548, 499, 493, 510, 570, 478, 554, 514, 507, 565, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 490, 528, 499, 512, 511, 508, 485, 521, 504, 513, 519, 546, 503, 565, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 498, 513, 490, 515, 511, 498, 482, 487, 513, 489, 507, 516, 536, 507, 544, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 494, 483, 527, 522, 484, 505, 475, 553, 482, 475, 504, 500, 548, 544, 515, 11, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 535, 495, 512, 523, 494, 504, 497, 513, 497, 517, 505, 492, 506, 526, 531, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 513, 520, 520, 550, 500, 502, 517, 521, 516, 500, 529, 498, 513, 535, 500, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 510, 530, 477, 522, 537, 500, 510, 535, 488, 472, 464, 535, 547, 506, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 474, 493, 549, 491, 523, 501, 523, 518, 493, 510, 517, 518, 482, 527, 506, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8289, 8229, 8098, 8145, 8199, 8216, 8196, 8164] avg=8192.0000 max=8289 max_rank=0 min=8098 min_rank=2 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32775 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[548, 519, 480, 493, 532, 510, 543, 521, 517, 472, 526, 486, 521, 499, 518, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 495, 527, 537, 506, 505, 482, 541, 503, 507, 509, 511, 499, 531, 525, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 542, 501, 521, 557, 499, 547, 520, 536, 539, 485, 479, 526, 539, 477, 510, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 560, 486, 519, 496, 534, 533, 504, 516, 499, 521, 513, 477, 483, 518, 517, 15, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 502, 505, 557, 537, 513, 479, 493, 520, 534, 524, 516, 475, 522, 526, 491, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 493, 546, 507, 513, 497, 526, 497, 477, 493, 528, 483, 473, 528, 560, 525, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 521, 444, 475, 520, 501, 500, 512, 503, 539, 504, 512, 517, 471, 523, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 517, 474, 530, 521, 497, 500, 508, 525, 482, 491, 479, 500, 511, 537, 498, 11, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8258, 8296, 8214, 8208, 8130, 8115, 8108] avg=8192.0000 max=8296 max_rank=2 min=8108 min_rank=7 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32561 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[532, 492, 522, 529, 532, 489, 529, 486, 508, 518, 493, 510, 477, 520, 499, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 533, 499, 526, 530, 492, 488, 502, 501, 511, 502, 526, 534, 534, 497, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 537, 540, 513, 513, 524, 477, 506, 515, 492, 477, 544, 536, 503, 518, 538, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 546, 514, 494, 516, 474, 509, 522, 541, 543, 521, 486, 525, 502, 513, 554, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 527, 520, 528, 508, 475, 563, 474, 508, 527, 502, 518, 523, 485, 546, 511, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 480, 509, 516, 488, 499, 552, 506, 516, 518, 487, 496, 499, 535, 515, 511, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 486, 523, 482, 509, 510, 515, 498, 504, 531, 529, 479, 478, 540, 503, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 507, 499, 511, 521, 542, 526, 500, 543, 522, 480, 482, 492, 488, 478, 545, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8136, 8159, 8273, 8288, 8279, 8158, 8116, 8127] avg=8192.0000 max=8288 max_rank=3 min=8116 min_rank=6 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32680 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[541, 548, 520, 494, 534, 515, 485, 488, 484, 508, 541, 501, 485, 557, 563, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 491, 498, 563, 518, 504, 509, 551, 543, 529, 532, 533, 471, 564, 476, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 525, 502, 542, 503, 519, 568, 512, 508, 551, 519, 480, 528, 480, 494, 520, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 490, 503, 519, 521, 513, 460, 496, 505, 464, 529, 476, 527, 503, 498, 484, 11, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 490, 484, 512, 523, 516, 505, 509, 531, 481, 471, 482, 520, 493, 456, 514, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 494, 503, 529, 517, 522, 503, 529, 505, 468, 545, 520, 507, 503, 547, 529, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 546, 544, 490, 509, 507, 511, 516, 510, 513, 542, 499, 505, 492, 484, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 519, 517, 525, 512, 500, 529, 487, 521, 570, 476, 528, 540, 502, 514, 472, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8287, 8289, 8286, 8009, 7997, 8211, 8203, 8254] avg=8192.0000 max=8289 max_rank=1 min=7997 min_rank=4 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32665 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[519, 516, 530, 513, 493, 498, 525, 500, 493, 538, 483, 509, 519, 533, 509, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 472, 518, 484, 510, 494, 526, 494, 490, 485, 488, 511, 472, 495, 485, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 516, 498, 535, 521, 531, 505, 507, 512, 516, 452, 562, 537, 530, 500, 513, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 550, 529, 491, 536, 513, 543, 512, 481, 537, 522, 510, 531, 505, 500, 499, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 533, 530, 500, 496, 504, 517, 551, 513, 517, 515, 554, 496, 489, 501, 511, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 486, 525, 465, 532, 480, 508, 495, 541, 497, 556, 484, 545, 517, 537, 518, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 501, 520, 498, 473, 526, 533, 534, 525, 502, 491, 513, 556, 563, 499, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 517, 527, 493, 461, 499, 552, 487, 512, 523, 512, 496, 533, 508, 504, 524, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8165, 7897, 8239, 8268, 8296, 8218, 8273, 8180] avg=8192.0000 max=8296 max_rank=4 min=7897 min_rank=1 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32967 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[489, 514, 516, 499, 511, 512, 502, 511, 519, 538, 484, 486, 505, 521, 517, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 546, 542, 536, 451, 546, 466, 515, 497, 509, 532, 511, 514, 467, 556, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 463, 499, 493, 509, 515, 540, 479, 541, 557, 516, 447, 506, 511, 528, 477, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 532, 519, 527, 559, 502, 530, 471, 505, 499, 474, 518, 515, 508, 523, 521, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 545, 518, 555, 507, 469, 500, 513, 521, 515, 509, 503, 493, 517, 506, 521, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 536, 555, 512, 533, 495, 523, 492, 488, 498, 496, 481, 507, 522, 505, 512, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 570, 538, 456, 510, 512, 517, 472, 563, 516, 542, 544, 477, 525, 525, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 534, 500, 517, 512, 536, 478, 516, 528, 516, 494, 509, 505, 540, 535, 482, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8112, 8241, 8110, 8228, 8231, 8157, 8251, 8206] avg=8192.0000 max=8251 max_rank=6 min=8110 min_rank=2 imbalance_ratio=1.007202 max_load_reduction=57285 max_load_reduction_pct=0.874100 moved_to_echo=32845 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[535, 550, 532, 508, 497, 558, 484, 501, 519, 502, 512, 503, 476, 495, 471, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 510, 544, 521, 503, 550, 497, 524, 541, 467, 540, 499, 495, 486, 494, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 545, 476, 478, 495, 512, 519, 503, 548, 539, 500, 507, 516, 481, 512, 514, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 538, 514, 534, 496, 515, 511, 496, 539, 484, 475, 517, 512, 527, 485, 516, 11, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 500, 461, 475, 505, 504, 490, 519, 495, 496, 481, 538, 482, 522, 582, 541, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 531, 516, 493, 550, 530, 495, 507, 515, 493, 503, 525, 516, 532, 522, 496, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 522, 492, 458, 540, 547, 523, 512, 528, 514, 546, 512, 513, 532, 519, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 549, 540, 548, 477, 525, 542, 527, 507, 517, 518, 492, 509, 489, 519, 479, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8152, 8225, 8138, 8140, 8136, 8235, 8271, 8239] avg=8192.0000 max=8271 max_rank=6 min=8136 min_rank=4 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32881 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[541, 548, 520, 494, 534, 515, 485, 488, 484, 508, 541, 501, 485, 557, 563, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 491, 498, 563, 518, 504, 509, 551, 543, 529, 532, 533, 471, 564, 476, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 525, 502, 542, 503, 519, 568, 512, 508, 551, 519, 480, 528, 480, 494, 520, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 490, 503, 519, 521, 513, 460, 496, 505, 464, 529, 476, 527, 503, 498, 484, 11, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 490, 484, 512, 523, 516, 505, 509, 531, 481, 471, 482, 520, 493, 456, 514, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 494, 503, 529, 517, 522, 503, 529, 505, 468, 545, 520, 507, 503, 547, 529, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 546, 544, 490, 509, 507, 511, 516, 510, 513, 542, 499, 505, 492, 484, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 519, 517, 525, 512, 500, 529, 487, 521, 570, 476, 528, 540, 502, 514, 472, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8287, 8289, 8286, 8009, 7997, 8211, 8203, 8254] avg=8192.0000 max=8289 max_rank=1 min=7997 min_rank=4 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32665 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 3968, 4096, 4224, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 3840, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4352, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 3968, 4096, 4224, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[534, 534, 502, 523, 514, 511, 462, 506, 535, 505, 524, 520, 506, 497, 516, 540, 489, 527, 490, 508, 541, 526, 472, 538, 526, 524, 505, 494, 514, 511, 514, 525, 500, 468, 520, 489, 543, 486, 544, 488, 505, 547, 491, 530, 547, 551, 498, 553, 511, 518, 496, 538, 539, 511, 496, 510, 528, 486, 542, 501, 536, 522, 490, 504, 498, 490, 512, 483, 541, 508, 487, 510, 510, 518, 475, 529, 523, 538, 520, 517, 526, 525, 477, 529, 525, 517, 489, 537, 527, 522, 533, 500, 560, 533, 473, 542, 512, 522, 517, 545, 484, 504, 520, 491, 513, 511, 529, 457, 473, 512, 556, 511, 501, 552, 523, 470, 455, 484, 489, 520, 482, 516, 526, 518, 487, 487, 477, 497] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[492, 508, 521, 518, 484, 527, 495, 491, 510, 473, 541, 464, 526, 507, 543, 551, 478, 523, 500, 449, 484, 510, 522, 502, 528, 501, 513, 535, 516, 537, 520, 486, 489, 506, 532, 485, 558, 499, 542, 501, 501, 509, 518, 509, 473, 453, 530, 507, 532, 529, 527, 524, 499, 473, 509, 533, 547, 513, 525, 472, 497, 514, 570, 538, 503, 516, 520, 512, 508, 535, 517, 502, 519, 529, 513, 505, 497, 533, 502, 521, 488, 479, 517, 528, 514, 481, 530, 543, 514, 492, 516, 498, 483, 525, 523, 496, 532, 485, 514, 524, 525, 510, 519, 515, 516, 532, 538, 482, 491, 557, 528, 519, 491, 517, 521, 517, 495, 514, 497, 525, 517, 492, 515, 494, 531, 501, 568, 526] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[542, 511, 510, 551, 510, 535, 537, 520, 533, 525, 490, 475, 500, 494, 533, 485, 496, 520, 494, 514, 546, 500, 528, 512, 520, 512, 532, 492, 537, 501, 518, 481, 519, 500, 530, 508, 510, 491, 527, 491, 503, 462, 493, 482, 525, 519, 507, 499, 494, 514, 505, 529, 511, 508, 559, 543, 556, 529, 438, 545, 529, 515, 532, 532, 509, 495, 510, 494, 522, 466, 531, 501, 552, 506, 479, 509, 489, 518, 511, 486, 515, 522, 524, 559, 509, 465, 479, 525, 543, 507, 528, 514, 497, 495, 502, 495, 530, 454, 529, 516, 523, 526, 539, 505, 494, 519, 508, 515, 519, 460, 487, 503, 509, 502, 573, 546, 507, 545, 512, 539, 510, 499, 525, 508, 524, 491, 475, 528] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[499, 468, 516, 528, 539, 496, 495, 525, 495, 547, 489, 455, 554, 527, 501, 501, 499, 532, 517, 522, 502, 510, 443, 520, 545, 518, 496, 493, 503, 522, 493, 563, 508, 514, 476, 547, 509, 488, 542, 539, 518, 525, 503, 521, 509, 554, 532, 495, 521, 529, 551, 487, 501, 544, 549, 549, 510, 486, 488, 491, 520, 521, 510, 531, 528, 532, 472, 523, 544, 516, 514, 517, 508, 447, 509, 520, 519, 497, 481, 530, 523, 488, 526, 495, 491, 547, 538, 502, 539, 476, 521, 536, 546, 503, 511, 504, 488, 488, 542, 484, 471, 480, 495, 517, 534, 508, 535, 509, 491, 501, 517, 530, 488, 507, 530, 572, 511, 493, 507, 497, 522, 512, 488, 491, 520, 488, 525, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[526, 529, 503, 513, 538, 498, 517, 505, 476, 533, 524, 508, 522, 528, 529, 529, 478, 521, 528, 499, 508, 481, 555, 538, 556, 510, 492, 487, 521, 542, 524, 537, 536, 502, 506, 538, 543, 491, 555, 483, 505, 526, 512, 529, 473, 479, 494, 514, 477, 485, 476, 529, 519, 521, 488, 503, 470, 485, 567, 519, 538, 504, 512, 501, 529, 522, 475, 524, 521, 494, 512, 507, 511, 475, 515, 540, 467, 529, 479, 522, 531, 504, 514, 493, 504, 507, 501, 528, 531, 510, 487, 482, 524, 556, 503, 517, 495, 518, 480, 518, 496, 520, 543, 531, 506, 558, 513, 515, 479, 511, 484, 484, 518, 563, 508, 535, 539, 517, 520, 506, 487, 512, 540, 499, 491, 508, 503, 490] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[502, 497, 522, 532, 520, 492, 515, 515, 524, 526, 484, 494, 498, 548, 510, 550, 465, 537, 517, 528, 550, 520, 523, 525, 519, 482, 542, 543, 500, 513, 519, 478, 541, 509, 529, 478, 541, 499, 466, 520, 482, 477, 520, 491, 501, 543, 498, 488, 495, 490, 539, 508, 507, 515, 511, 501, 494, 522, 507, 507, 518, 496, 516, 531, 523, 494, 550, 523, 503, 472, 530, 491, 502, 499, 522, 505, 525, 513, 505, 509, 492, 484, 524, 529, 518, 498, 555, 523, 533, 549, 490, 484, 507, 490, 502, 519, 478, 550, 505, 523, 536, 515, 525, 493, 466, 542, 527, 549, 528, 531, 542, 518, 481, 488, 527, 492, 487, 538, 499, 501, 516, 516, 492, 530, 522, 504, 516, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[532, 479, 528, 511, 500, 536, 494, 541, 519, 518, 514, 472, 522, 506, 506, 519, 506, 539, 488, 542, 518, 518, 494, 516, 519, 517, 501, 510, 492, 512, 514, 508, 529, 518, 480, 504, 539, 552, 537, 527, 522, 545, 495, 501, 532, 499, 498, 526, 527, 499, 532, 493, 539, 507, 515, 486, 494, 488, 520, 516, 521, 519, 494, 524, 518, 526, 534, 495, 504, 483, 496, 490, 484, 503, 545, 532, 531, 531, 516, 521, 494, 509, 516, 505, 495, 520, 513, 486, 507, 481, 527, 518, 516, 495, 504, 494, 516, 476, 549, 504, 500, 509, 487, 486, 506, 488, 524, 532, 536, 498, 540, 492, 545, 489, 498, 511, 540, 491, 484, 540, 509, 498, 519, 511, 483, 539, 517, 561] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[492, 508, 521, 518, 484, 527, 495, 491, 510, 473, 541, 464, 526, 507, 543, 551, 478, 523, 500, 449, 484, 510, 522, 502, 528, 501, 513, 535, 516, 537, 520, 486, 489, 506, 532, 485, 558, 499, 542, 501, 501, 509, 518, 509, 473, 453, 530, 507, 532, 529, 527, 524, 499, 473, 509, 533, 547, 513, 525, 472, 497, 514, 570, 538, 503, 516, 520, 512, 508, 535, 517, 502, 519, 529, 513, 505, 497, 533, 502, 521, 488, 479, 517, 528, 514, 481, 530, 543, 514, 492, 516, 498, 483, 525, 523, 496, 532, 485, 514, 524, 525, 510, 519, 515, 516, 532, 538, 482, 491, 557, 528, 519, 491, 517, 521, 517, 495, 514, 497, 525, 517, 492, 515, 494, 531, 501, 568, 526] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[502, 497, 522, 532, 520, 492, 515, 515, 524, 526, 484, 494, 498, 548, 510, 550, 465, 537, 517, 528, 550, 520, 523, 525, 519, 482, 542, 543, 500, 513, 519, 478, 541, 509, 529, 478, 541, 499, 466, 520, 482, 477, 520, 491, 501, 543, 498, 488, 495, 490, 539, 508, 507, 515, 511, 501, 494, 522, 507, 507, 518, 496, 516, 531, 523, 494, 550, 523, 503, 472, 530, 491, 502, 499, 522, 505, 525, 513, 505, 509, 492, 484, 524, 529, 518, 498, 555, 523, 533, 549, 490, 484, 507, 490, 502, 519, 478, 550, 505, 523, 536, 515, 525, 493, 466, 542, 527, 549, 528, 531, 542, 518, 481, 488, 527, 492, 487, 538, 499, 501, 516, 516, 492, 530, 522, 504, 516, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[487, 502, 501, 469, 491, 508, 559, 558, 546, 505, 525, 505, 483, 518, 509, 512, 522, 536, 528, 498, 512, 558, 543, 515, 531, 502, 502, 482, 502, 461, 517, 525, 517, 500, 509, 486, 504, 470, 519, 535, 501, 521, 562, 497, 471, 539, 529, 477, 561, 513, 480, 524, 524, 499, 501, 501, 485, 518, 527, 494, 504, 479, 528, 466, 493, 513, 505, 505, 514, 529, 527, 489, 520, 483, 533, 541, 528, 493, 512, 528, 505, 528, 535, 491, 503, 515, 498, 516, 526, 496, 509, 500, 511, 528, 555, 484, 495, 503, 499, 536, 553, 557, 554, 488, 523, 467, 498, 528, 514, 511, 482, 515, 504, 494, 549, 569, 502, 501, 545, 500, 518, 492, 476, 534, 529, 491, 513, 530] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[534, 534, 502, 523, 514, 511, 462, 506, 535, 505, 524, 520, 506, 497, 516, 540, 489, 527, 490, 508, 541, 526, 472, 538, 526, 524, 505, 494, 514, 511, 514, 525, 500, 468, 520, 489, 543, 486, 544, 488, 505, 547, 491, 530, 547, 551, 498, 553, 511, 518, 496, 538, 539, 511, 496, 510, 528, 486, 542, 501, 536, 522, 490, 504, 498, 490, 512, 483, 541, 508, 487, 510, 510, 518, 475, 529, 523, 538, 520, 517, 526, 525, 477, 529, 525, 517, 489, 537, 527, 522, 533, 500, 560, 533, 473, 542, 512, 522, 517, 545, 484, 504, 520, 491, 513, 511, 529, 457, 473, 512, 556, 511, 501, 552, 523, 470, 455, 484, 489, 520, 482, 516, 526, 518, 487, 487, 477, 497] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[526, 529, 503, 513, 538, 498, 517, 505, 476, 533, 524, 508, 522, 528, 529, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 521, 528, 499, 508, 481, 555, 538, 541, 510, 492, 487, 521, 542, 524, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 502, 506, 538, 543, 491, 555, 483, 505, 526, 512, 529, 473, 479, 494, 514, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 485, 476, 529, 519, 521, 488, 503, 470, 485, 567, 519, 519, 504, 512, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 522, 475, 524, 521, 494, 512, 507, 511, 475, 515, 540, 467, 529, 479, 522, 19, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 504, 514, 493, 504, 507, 501, 528, 531, 510, 487, 482, 524, 556, 503, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 518, 480, 518, 496, 520, 543, 531, 506, 558, 513, 515, 479, 511, 484, 484, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 563, 508, 535, 539, 517, 520, 506, 487, 512, 540, 499, 491, 508, 503, 490, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8266, 8262, 8199, 8075, 8147, 8192, 8154, 8241] avg=8192.0000 max=8266 max_rank=0 min=8075 min_rank=3 imbalance_ratio=1.009033 max_load_reduction=57270 max_load_reduction_pct=0.873871 moved_to_echo=32734 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[502, 497, 522, 532, 520, 492, 515, 515, 524, 526, 484, 494, 498, 548, 510, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 537, 517, 528, 550, 520, 523, 525, 505, 482, 542, 543, 500, 513, 519, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 509, 529, 478, 541, 499, 466, 520, 482, 477, 520, 491, 501, 543, 498, 488, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 490, 539, 508, 507, 515, 511, 501, 494, 522, 507, 507, 500, 496, 516, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 494, 550, 523, 503, 472, 530, 491, 502, 499, 522, 505, 525, 513, 505, 509, 18, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 484, 524, 529, 518, 498, 555, 523, 533, 549, 490, 484, 507, 490, 502, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 550, 505, 523, 536, 515, 525, 493, 466, 542, 527, 549, 528, 531, 542, 518, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 488, 527, 492, 487, 538, 499, 501, 516, 516, 492, 530, 522, 504, 516, 506, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8247, 8096, 8139, 8190, 8197, 8331, 8120] avg=8192.0000 max=8331 max_rank=6 min=8096 min_rank=2 imbalance_ratio=1.016968 max_load_reduction=57205 max_load_reduction_pct=0.872879 moved_to_echo=32838 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[526, 529, 503, 513, 538, 498, 517, 505, 476, 533, 524, 508, 522, 528, 529, 529, 478, 521, 528, 499, 508, 481, 555, 538, 556, 510, 492, 487, 521, 542, 524, 537, 536, 502, 506, 538, 543, 491, 555, 483, 505, 526, 512, 529, 473, 479, 494, 514, 477, 485, 476, 529, 519, 521, 488, 503, 470, 485, 567, 519, 538, 504, 512, 501, 529, 522, 475, 524, 521, 494, 512, 507, 511, 475, 515, 540, 467, 529, 479, 522, 531, 504, 514, 493, 504, 507, 501, 528, 531, 510, 487, 482, 524, 556, 503, 517, 495, 518, 480, 518, 496, 520, 543, 531, 506, 558, 513, 515, 479, 511, 484, 484, 518, 563, 508, 535, 539, 517, 520, 506, 487, 512, 540, 499, 491, 508, 503, 490] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[499, 468, 516, 528, 539, 496, 495, 525, 495, 547, 489, 455, 554, 527, 501, 501, 499, 532, 517, 522, 502, 510, 443, 520, 545, 518, 496, 493, 503, 522, 493, 563, 508, 514, 476, 547, 509, 488, 542, 539, 518, 525, 503, 521, 509, 554, 532, 495, 521, 529, 551, 487, 501, 544, 549, 549, 510, 486, 488, 491, 520, 521, 510, 531, 528, 532, 472, 523, 544, 516, 514, 517, 508, 447, 509, 520, 519, 497, 481, 530, 523, 488, 526, 495, 491, 547, 538, 502, 539, 476, 521, 536, 546, 503, 511, 504, 488, 488, 542, 484, 471, 480, 495, 517, 534, 508, 535, 509, 491, 501, 517, 530, 488, 507, 530, 572, 511, 493, 507, 497, 522, 512, 488, 491, 520, 488, 525, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[534, 534, 502, 523, 514, 511, 462, 506, 535, 505, 524, 520, 506, 497, 516, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 527, 490, 508, 541, 526, 472, 538, 512, 524, 505, 494, 514, 511, 514, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 468, 520, 489, 543, 486, 544, 488, 505, 547, 491, 530, 547, 551, 498, 553, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 518, 496, 538, 539, 511, 496, 510, 528, 486, 542, 501, 517, 522, 490, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 490, 512, 483, 541, 508, 487, 510, 510, 518, 475, 529, 523, 538, 520, 517, 19, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 525, 477, 529, 525, 517, 489, 537, 527, 522, 533, 500, 560, 533, 473, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 522, 517, 545, 484, 504, 520, 491, 513, 511, 529, 457, 473, 512, 556, 511, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 552, 523, 470, 455, 484, 489, 520, 482, 516, 526, 518, 487, 487, 477, 497, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8190, 8273, 8209, 8184, 8315, 8160, 7989] avg=8192.0000 max=8315 max_rank=5 min=7989 min_rank=7 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32648 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[542, 511, 510, 551, 510, 535, 537, 520, 533, 525, 490, 475, 500, 494, 533, 460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 520, 494, 514, 546, 500, 528, 512, 500, 512, 532, 492, 537, 501, 518, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 500, 530, 508, 510, 491, 527, 491, 503, 462, 493, 482, 525, 519, 507, 499, 12, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 514, 505, 529, 511, 508, 559, 543, 556, 529, 438, 545, 506, 515, 532, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 495, 510, 494, 522, 466, 531, 501, 552, 506, 479, 509, 489, 518, 511, 486, 23, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 522, 524, 559, 509, 465, 479, 525, 543, 507, 528, 514, 497, 495, 502, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 454, 529, 516, 523, 526, 539, 505, 494, 519, 508, 515, 519, 460, 487, 503, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 502, 573, 546, 507, 545, 512, 539, 510, 499, 525, 508, 524, 491, 475, 528, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 8183, 8086, 8316, 8109, 8179, 8136, 8301] avg=8192.0000 max=8316 max_rank=3 min=8086 min_rank=2 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32725 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[499, 468, 516, 528, 539, 496, 495, 525, 495, 547, 489, 455, 554, 527, 501, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 532, 517, 522, 502, 510, 443, 520, 531, 518, 496, 493, 503, 522, 493, 563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 514, 476, 547, 509, 488, 542, 539, 518, 525, 503, 521, 509, 554, 532, 495, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 529, 551, 487, 501, 544, 549, 549, 510, 486, 488, 491, 502, 521, 510, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 532, 472, 523, 544, 516, 514, 517, 508, 447, 509, 520, 519, 497, 481, 530, 18, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 488, 526, 495, 491, 547, 538, 502, 539, 476, 521, 536, 546, 503, 511, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 488, 542, 484, 471, 480, 495, 517, 534, 508, 535, 509, 491, 501, 517, 530, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 507, 530, 572, 511, 493, 507, 497, 522, 512, 488, 491, 520, 488, 525, 511, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8123, 8164, 8292, 8270, 8181, 8246, 8093, 8167] avg=8192.0000 max=8292 max_rank=2 min=8093 min_rank=6 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32687 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[492, 508, 521, 518, 484, 527, 495, 491, 510, 473, 541, 464, 526, 507, 543, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 523, 500, 449, 484, 510, 522, 502, 514, 501, 513, 535, 516, 537, 520, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 506, 532, 485, 558, 499, 542, 501, 501, 509, 518, 509, 473, 453, 530, 507, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 529, 527, 524, 499, 473, 509, 533, 547, 513, 525, 472, 480, 514, 570, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 516, 520, 512, 508, 535, 517, 502, 519, 529, 513, 505, 497, 533, 502, 521, 17, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 479, 517, 528, 514, 481, 530, 543, 514, 492, 516, 498, 483, 525, 523, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 485, 514, 524, 525, 510, 519, 515, 516, 532, 538, 482, 491, 557, 528, 519, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 517, 521, 517, 495, 514, 497, 525, 517, 492, 515, 494, 531, 501, 568, 526, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8138, 8090, 8125, 8285, 8255, 8127, 8290, 8226] avg=8192.0000 max=8290 max_rank=6 min=8090 min_rank=1 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32898 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[532, 479, 528, 511, 500, 536, 494, 541, 519, 518, 514, 472, 522, 506, 506, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 539, 488, 542, 518, 518, 494, 516, 505, 517, 501, 510, 492, 512, 514, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 518, 480, 504, 539, 552, 537, 527, 522, 545, 495, 501, 532, 499, 498, 526, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 499, 532, 493, 539, 507, 515, 486, 494, 488, 520, 516, 503, 519, 494, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 526, 534, 495, 504, 483, 496, 490, 484, 503, 545, 532, 531, 531, 516, 521, 18, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 509, 516, 505, 495, 520, 513, 486, 507, 481, 527, 518, 516, 495, 504, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 476, 549, 504, 500, 509, 487, 486, 506, 488, 524, 532, 536, 498, 540, 492, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 489, 498, 511, 540, 491, 484, 540, 509, 498, 519, 511, 483, 539, 517, 561, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8180, 8316, 8156, 8233, 8080, 8146, 8240] avg=8192.0000 max=8316 max_rank=2 min=8080 min_rank=5 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32699 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[534, 534, 502, 523, 514, 511, 462, 506, 535, 505, 524, 520, 506, 497, 516, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 527, 490, 508, 541, 526, 472, 538, 512, 524, 505, 494, 514, 511, 514, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 468, 520, 489, 543, 486, 544, 488, 505, 547, 491, 530, 547, 551, 498, 553, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 518, 496, 538, 539, 511, 496, 510, 528, 486, 542, 501, 517, 522, 490, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 490, 512, 483, 541, 508, 487, 510, 510, 518, 475, 529, 523, 538, 520, 517, 19, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 525, 477, 529, 525, 517, 489, 537, 527, 522, 533, 500, 560, 533, 473, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 522, 517, 545, 484, 504, 520, 491, 513, 511, 529, 457, 473, 512, 556, 511, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 552, 523, 470, 455, 484, 489, 520, 482, 516, 526, 518, 487, 487, 477, 497, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8190, 8273, 8209, 8184, 8315, 8160, 7989] avg=8192.0000 max=8315 max_rank=5 min=7989 min_rank=7 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32648 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[487, 502, 501, 469, 491, 508, 559, 558, 546, 505, 525, 505, 483, 518, 509, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 536, 528, 498, 512, 558, 543, 515, 517, 502, 502, 482, 502, 461, 517, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 500, 509, 486, 504, 470, 519, 535, 501, 521, 562, 497, 471, 539, 529, 477, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 513, 480, 524, 524, 499, 501, 501, 485, 518, 527, 494, 486, 479, 528, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 513, 505, 505, 514, 529, 527, 489, 520, 483, 533, 541, 528, 493, 512, 528, 18, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 528, 535, 491, 503, 515, 498, 516, 526, 496, 509, 500, 511, 528, 555, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 503, 499, 536, 553, 557, 554, 488, 523, 467, 498, 528, 514, 511, 482, 515, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 494, 549, 569, 502, 501, 545, 500, 518, 492, 476, 534, 529, 491, 513, 530, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8166, 8220, 8149, 8086, 8237, 8200, 8226, 8252] avg=8192.0000 max=8252 max_rank=7 min=8086 min_rank=3 imbalance_ratio=1.007324 max_load_reduction=57284 max_load_reduction_pct=0.874084 moved_to_echo=32915 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[499, 468, 516, 528, 539, 496, 495, 525, 495, 547, 489, 455, 554, 527, 501, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 532, 517, 522, 502, 510, 443, 520, 531, 518, 496, 493, 503, 522, 493, 563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 514, 476, 547, 509, 488, 542, 539, 518, 525, 503, 521, 509, 554, 532, 495, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 529, 551, 487, 501, 544, 549, 549, 510, 486, 488, 491, 502, 521, 510, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 532, 472, 523, 544, 516, 514, 517, 508, 447, 509, 520, 519, 497, 481, 530, 18, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 488, 526, 495, 491, 547, 538, 502, 539, 476, 521, 536, 546, 503, 511, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 488, 542, 484, 471, 480, 495, 517, 534, 508, 535, 509, 491, 501, 517, 530, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 507, 530, 572, 511, 493, 507, 497, 522, 512, 488, 491, 520, 488, 525, 511, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8123, 8164, 8292, 8270, 8181, 8246, 8093, 8167] avg=8192.0000 max=8292 max_rank=2 min=8093 min_rank=6 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32687 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[502, 497, 522, 532, 520, 492, 515, 515, 524, 526, 484, 494, 498, 548, 510, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 537, 517, 528, 550, 520, 523, 525, 505, 482, 542, 543, 500, 513, 519, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 509, 529, 478, 541, 499, 466, 520, 482, 477, 520, 491, 501, 543, 498, 488, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 490, 539, 508, 507, 515, 511, 501, 494, 522, 507, 507, 500, 496, 516, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 494, 550, 523, 503, 472, 530, 491, 502, 499, 522, 505, 525, 513, 505, 509, 18, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 484, 524, 529, 518, 498, 555, 523, 533, 549, 490, 484, 507, 490, 502, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 550, 505, 523, 536, 515, 525, 493, 466, 542, 527, 549, 528, 531, 542, 518, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 488, 527, 492, 487, 538, 499, 501, 516, 516, 492, 530, 522, 504, 516, 506, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8247, 8096, 8139, 8190, 8197, 8331, 8120] avg=8192.0000 max=8331 max_rank=6 min=8096 min_rank=2 imbalance_ratio=1.016968 max_load_reduction=57205 max_load_reduction_pct=0.872879 moved_to_echo=32838 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[492, 508, 521, 518, 484, 527, 495, 491, 510, 473, 541, 464, 526, 507, 543, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 523, 500, 449, 484, 510, 522, 502, 514, 501, 513, 535, 516, 537, 520, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 506, 532, 485, 558, 499, 542, 501, 501, 509, 518, 509, 473, 453, 530, 507, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 529, 527, 524, 499, 473, 509, 533, 547, 513, 525, 472, 480, 514, 570, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 516, 520, 512, 508, 535, 517, 502, 519, 529, 513, 505, 497, 533, 502, 521, 17, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 479, 517, 528, 514, 481, 530, 543, 514, 492, 516, 498, 483, 525, 523, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 485, 514, 524, 525, 510, 519, 515, 516, 532, 538, 482, 491, 557, 528, 519, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 517, 521, 517, 495, 514, 497, 525, 517, 492, 515, 494, 531, 501, 568, 526, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8138, 8090, 8125, 8285, 8255, 8127, 8290, 8226] avg=8192.0000 max=8290 max_rank=6 min=8090 min_rank=1 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32898 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[526, 529, 503, 513, 538, 498, 517, 505, 476, 533, 524, 508, 522, 528, 529, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 521, 528, 499, 508, 481, 555, 538, 541, 510, 492, 487, 521, 542, 524, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 502, 506, 538, 543, 491, 555, 483, 505, 526, 512, 529, 473, 479, 494, 514, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 485, 476, 529, 519, 521, 488, 503, 470, 485, 567, 519, 519, 504, 512, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 522, 475, 524, 521, 494, 512, 507, 511, 475, 515, 540, 467, 529, 479, 522, 19, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 504, 514, 493, 504, 507, 501, 528, 531, 510, 487, 482, 524, 556, 503, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 518, 480, 518, 496, 520, 543, 531, 506, 558, 513, 515, 479, 511, 484, 484, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 563, 508, 535, 539, 517, 520, 506, 487, 512, 540, 499, 491, 508, 503, 490, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8266, 8262, 8199, 8075, 8147, 8192, 8154, 8241] avg=8192.0000 max=8266 max_rank=0 min=8075 min_rank=3 imbalance_ratio=1.009033 max_load_reduction=57270 max_load_reduction_pct=0.873871 moved_to_echo=32734 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 3968, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[3968, 4352, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4096, 4352, 4096, 4352, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 3968, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[3968, 4352, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 3968, 4096, 4224, 4096, 4224, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4352, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 3968, 4096, 4224, 4096, 4224, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True - [2026-08-26 14:49:30] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 420.6 | throughput per GPU (TFLOP/s/GPU): 430.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.429702E+00 | loss scale: 1.0 | grad norm: 5.457 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True + [2026-08-26 15:25:34] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 414.2 | throughput per GPU (TFLOP/s/GPU): 436.5 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.438096E+00 | loss scale: 1.0 | grad norm: 5.451 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[482, 556, 503, 506, 492, 545, 491, 510, 538, 509, 515, 508, 511, 486, 511, 505, 564, 472, 536, 514, 522, 519, 512, 465, 538, 519, 508, 509, 513, 551, 502, 495, 550, 494, 517, 519, 500, 543, 492, 526, 526, 507, 511, 523, 556, 500, 535, 515, 482, 511, 508, 529, 478, 504, 502, 537, 492, 545, 501, 497, 501, 526, 500, 504, 499, 517, 499, 538, 492, 517, 512, 491, 503, 521, 545, 508, 485, 488, 535, 520, 524, 486, 494, 514, 504, 529, 529, 510, 506, 516, 516, 511, 515, 525, 504, 535, 458, 504, 553, 548, 532, 506, 513, 535, 502, 465, 485, 530, 503, 499, 457, 570, 529, 508, 513, 499, 492, 517, 502, 532, 501, 513, 492, 508, 503, 515, 491, 535] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[529, 502, 569, 504, 533, 493, 526, 517, 507, 552, 486, 480, 532, 496, 492, 528, 502, 495, 493, 510, 490, 501, 463, 531, 519, 486, 499, 468, 526, 562, 510, 537, 521, 479, 476, 499, 519, 493, 506, 468, 456, 512, 508, 532, 506, 544, 511, 499, 530, 467, 510, 520, 528, 483, 510, 515, 534, 519, 502, 517, 500, 572, 513, 544, 511, 497, 521, 509, 466, 525, 495, 505, 521, 502, 517, 527, 477, 532, 499, 523, 490, 533, 496, 488, 505, 554, 511, 491, 515, 565, 500, 547, 511, 537, 526, 530, 516, 506, 516, 519, 504, 507, 505, 548, 516, 544, 540, 544, 539, 525, 546, 528, 534, 524, 518, 498, 520, 523, 509, 474, 480, 511, 536, 467, 493, 526, 502, 461] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[507, 498, 560, 520, 534, 510, 527, 487, 533, 550, 512, 553, 521, 511, 494, 526, 506, 516, 482, 504, 562, 526, 543, 511, 476, 510, 497, 512, 510, 517, 478, 503, 477, 482, 499, 493, 505, 487, 511, 487, 492, 512, 506, 505, 574, 546, 527, 500, 510, 522, 503, 499, 503, 491, 511, 497, 553, 549, 472, 492, 483, 504, 530, 521, 515, 504, 516, 530, 512, 512, 504, 502, 530, 533, 501, 543, 491, 523, 503, 507, 545, 508, 504, 482, 535, 515, 533, 521, 502, 536, 490, 467, 546, 506, 519, 545, 495, 524, 523, 518, 525, 531, 489, 501, 499, 498, 548, 532, 499, 471, 492, 456, 535, 488, 527, 516, 524, 520, 469, 513, 537, 503, 523, 477, 532, 522, 523, 507] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[514, 501, 495, 480, 507, 486, 537, 496, 530, 499, 468, 521, 509, 505, 515, 507, 488, 518, 501, 497, 552, 526, 497, 541, 558, 520, 517, 519, 511, 507, 497, 525, 549, 519, 511, 559, 531, 525, 508, 532, 522, 531, 546, 489, 485, 515, 538, 517, 491, 554, 505, 539, 487, 511, 542, 504, 482, 503, 506, 475, 522, 503, 514, 498, 508, 500, 471, 533, 484, 481, 483, 519, 523, 550, 496, 525, 554, 555, 518, 522, 469, 510, 532, 525, 511, 559, 503, 515, 520, 489, 537, 487, 507, 517, 507, 504, 532, 473, 495, 530, 489, 476, 487, 485, 505, 498, 517, 501, 493, 509, 500, 526, 497, 514, 539, 525, 505, 537, 487, 522, 467, 509, 540, 535, 507, 506, 503, 556] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[494, 509, 502, 506, 500, 530, 472, 524, 526, 502, 500, 504, 506, 513, 514, 562, 512, 517, 533, 527, 526, 562, 549, 539, 504, 551, 521, 517, 526, 527, 540, 517, 496, 473, 509, 545, 513, 549, 528, 518, 526, 523, 489, 496, 489, 507, 490, 495, 499, 538, 493, 526, 500, 491, 496, 501, 504, 533, 502, 522, 504, 518, 508, 492, 499, 556, 517, 502, 505, 505, 522, 555, 529, 470, 498, 515, 526, 504, 498, 530, 540, 515, 538, 521, 460, 477, 502, 509, 514, 547, 484, 523, 500, 541, 472, 492, 493, 544, 483, 438, 511, 477, 494, 552, 504, 488, 527, 533, 516, 501, 497, 519, 533, 511, 518, 540, 525, 536, 517, 483, 532, 513, 515, 495, 509, 464, 516, 481] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[514, 507, 563, 484, 477, 472, 505, 537, 509, 530, 497, 509, 536, 503, 507, 513, 537, 526, 518, 513, 524, 518, 487, 538, 518, 515, 502, 519, 508, 511, 527, 490, 530, 515, 510, 527, 524, 545, 513, 503, 489, 504, 527, 501, 522, 565, 472, 552, 513, 458, 479, 522, 540, 513, 518, 522, 519, 503, 503, 519, 540, 495, 488, 502, 537, 530, 488, 508, 473, 525, 511, 540, 512, 501, 514, 547, 539, 488, 481, 498, 486, 549, 515, 524, 522, 488, 515, 501, 477, 531, 514, 555, 488, 488, 519, 533, 514, 553, 518, 457, 499, 493, 521, 479, 490, 481, 517, 575, 491, 562, 487, 517, 481, 504, 550, 460, 519, 504, 508, 516, 503, 501, 543, 550, 519, 502, 484, 494] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[482, 556, 503, 506, 492, 545, 491, 510, 538, 509, 515, 508, 511, 486, 511, 505, 564, 472, 536, 514, 522, 519, 512, 465, 538, 519, 508, 509, 513, 551, 502, 495, 550, 494, 517, 519, 500, 543, 492, 526, 526, 507, 511, 523, 556, 500, 535, 515, 482, 511, 508, 529, 478, 504, 502, 537, 492, 545, 501, 497, 501, 526, 500, 504, 499, 517, 499, 538, 492, 517, 512, 491, 503, 521, 545, 508, 485, 488, 535, 520, 524, 486, 494, 514, 504, 529, 529, 510, 506, 516, 516, 511, 515, 525, 504, 535, 458, 504, 553, 548, 532, 506, 513, 535, 502, 465, 485, 530, 503, 499, 457, 570, 529, 508, 513, 499, 492, 517, 502, 532, 501, 513, 492, 508, 503, 515, 491, 535] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[523, 472, 526, 503, 504, 533, 526, 528, 535, 528, 517, 506, 502, 516, 506, 504, 502, 502, 507, 469, 514, 513, 499, 533, 520, 475, 460, 488, 493, 516, 512, 522, 546, 510, 471, 529, 471, 521, 521, 525, 490, 496, 494, 478, 536, 521, 516, 499, 498, 524, 522, 509, 489, 513, 543, 506, 503, 501, 522, 535, 522, 486, 487, 506, 512, 528, 549, 509, 511, 538, 511, 535, 491, 515, 503, 464, 493, 538, 486, 503, 501, 463, 505, 474, 478, 518, 546, 493, 489, 517, 531, 525, 523, 522, 514, 526, 555, 549, 510, 508, 555, 552, 470, 548, 484, 497, 512, 515, 512, 500, 534, 498, 480, 488, 511, 551, 531, 553, 545, 514, 508, 533, 534, 522, 557, 521, 513, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[514, 501, 495, 480, 507, 486, 537, 496, 530, 499, 468, 521, 509, 505, 515, 507, 488, 518, 501, 497, 552, 526, 497, 541, 558, 520, 517, 519, 511, 507, 497, 525, 549, 519, 511, 559, 531, 525, 508, 532, 522, 531, 546, 489, 485, 515, 538, 517, 491, 554, 505, 539, 487, 511, 542, 504, 482, 503, 506, 475, 522, 503, 514, 498, 508, 500, 471, 533, 484, 481, 483, 519, 523, 550, 496, 525, 554, 555, 518, 522, 469, 510, 532, 525, 511, 559, 503, 515, 520, 489, 537, 487, 507, 517, 507, 504, 532, 473, 495, 530, 489, 476, 487, 485, 505, 498, 517, 501, 493, 509, 500, 526, 497, 514, 539, 525, 505, 537, 487, 522, 467, 509, 540, 535, 507, 506, 503, 556] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[529, 502, 569, 504, 533, 493, 526, 517, 507, 552, 486, 480, 532, 496, 492, 528, 502, 495, 493, 510, 490, 501, 463, 531, 519, 486, 499, 468, 526, 562, 510, 537, 521, 479, 476, 499, 519, 493, 506, 468, 456, 512, 508, 532, 506, 544, 511, 499, 530, 467, 510, 520, 528, 483, 510, 515, 534, 519, 502, 517, 500, 572, 513, 544, 511, 497, 521, 509, 466, 525, 495, 505, 521, 502, 517, 527, 477, 532, 499, 523, 490, 533, 496, 488, 505, 554, 511, 491, 515, 565, 500, 547, 511, 537, 526, 530, 516, 506, 516, 519, 504, 507, 505, 548, 516, 544, 540, 544, 539, 525, 546, 528, 534, 524, 518, 498, 520, 523, 509, 474, 480, 511, 536, 467, 493, 526, 502, 461] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[541, 533, 492, 518, 519, 492, 491, 490, 554, 514, 476, 515, 477, 550, 531, 493, 521, 515, 539, 559, 474, 515, 492, 454, 578, 492, 546, 561, 518, 531, 521, 484, 495, 507, 541, 510, 481, 542, 523, 513, 502, 507, 506, 499, 474, 504, 503, 541, 512, 471, 536, 503, 493, 520, 496, 497, 506, 531, 521, 492, 531, 495, 487, 549, 537, 503, 486, 531, 490, 500, 534, 541, 517, 516, 534, 489, 514, 484, 535, 493, 511, 460, 515, 485, 497, 483, 538, 546, 509, 471, 497, 527, 478, 511, 555, 523, 544, 497, 494, 514, 513, 515, 548, 520, 534, 493, 513, 527, 516, 511, 507, 516, 512, 521, 506, 511, 502, 492, 489, 567, 504, 483, 538, 506, 528, 510, 536, 485] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[507, 498, 560, 520, 534, 510, 527, 487, 529, 550, 512, 553, 521, 511, 494, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 516, 482, 504, 562, 526, 543, 511, 476, 510, 497, 512, 510, 485, 478, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 482, 499, 493, 505, 487, 511, 487, 492, 512, 506, 505, 574, 546, 527, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 522, 503, 499, 503, 491, 511, 497, 553, 549, 472, 492, 483, 504, 530, 521, 32, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 504, 516, 530, 512, 512, 504, 500, 530, 533, 501, 543, 491, 523, 503, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 508, 504, 482, 535, 515, 533, 521, 502, 536, 490, 467, 546, 506, 519, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 524, 523, 518, 525, 531, 489, 501, 499, 498, 548, 532, 499, 471, 492, 456, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 488, 527, 516, 524, 520, 469, 513, 537, 503, 518, 477, 532, 522, 523, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8339, 8121, 8103, 8177, 8224, 8254, 8107, 8211] avg=8192.0000 max=8339 max_rank=0 min=8103 min_rank=2 imbalance_ratio=1.017944 max_load_reduction=57197 max_load_reduction_pct=0.872757 moved_to_echo=32796 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[494, 509, 502, 506, 500, 530, 472, 524, 519, 502, 500, 504, 506, 513, 514, 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 517, 533, 527, 526, 562, 549, 539, 504, 551, 521, 517, 526, 490, 540, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 473, 509, 545, 513, 545, 528, 518, 526, 523, 489, 496, 489, 507, 490, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 538, 493, 526, 500, 491, 496, 501, 504, 533, 502, 522, 504, 518, 508, 492, 37, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 556, 517, 502, 505, 505, 522, 547, 529, 470, 498, 515, 526, 504, 498, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 515, 538, 521, 460, 477, 502, 509, 514, 547, 484, 523, 500, 541, 472, 492, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 544, 483, 438, 511, 477, 494, 552, 504, 488, 527, 533, 516, 501, 497, 519, 7, 8, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 511, 518, 540, 525, 536, 517, 483, 532, 513, 497, 495, 509, 464, 516, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8431, 8142, 8175, 8223, 8137, 8101, 8170] avg=8192.0000 max=8431 max_rank=1 min=8101 min_rank=6 imbalance_ratio=1.029175 max_load_reduction=57105 max_load_reduction_pct=0.871353 moved_to_echo=32631 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[482, 556, 503, 506, 492, 545, 491, 510, 534, 509, 515, 508, 511, 486, 511, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 564, 472, 536, 514, 522, 519, 512, 465, 538, 519, 508, 509, 513, 517, 502, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 494, 517, 519, 500, 543, 492, 526, 526, 507, 511, 523, 556, 500, 535, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 511, 508, 529, 478, 504, 502, 537, 492, 545, 501, 497, 501, 526, 500, 504, 34, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 517, 499, 538, 492, 517, 512, 489, 503, 521, 545, 508, 485, 488, 535, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 486, 494, 514, 504, 529, 529, 510, 506, 516, 516, 511, 515, 525, 504, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 458, 504, 553, 548, 532, 506, 513, 535, 502, 465, 485, 530, 503, 499, 457, 570, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 508, 513, 499, 492, 517, 502, 532, 501, 513, 487, 508, 503, 515, 491, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8205, 8314, 8156, 8168, 8218, 8166, 8145] avg=8192.0000 max=8314 max_rank=2 min=8145 min_rank=7 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32697 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[514, 507, 563, 484, 477, 472, 505, 537, 506, 530, 497, 509, 536, 503, 507, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 526, 518, 513, 524, 518, 487, 538, 518, 515, 502, 519, 508, 479, 527, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 515, 510, 527, 524, 545, 513, 503, 489, 504, 527, 501, 522, 565, 472, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 458, 479, 522, 540, 513, 518, 522, 519, 503, 503, 519, 540, 495, 488, 502, 32, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 530, 488, 508, 473, 525, 511, 538, 512, 501, 514, 547, 539, 488, 481, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 549, 515, 524, 522, 488, 515, 501, 477, 531, 514, 555, 488, 488, 519, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 553, 518, 457, 499, 493, 521, 479, 490, 481, 517, 575, 491, 562, 487, 517, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 504, 550, 460, 519, 504, 508, 516, 503, 501, 538, 550, 519, 502, 484, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8160, 8219, 8299, 8171, 8190, 8205, 8159, 8133] avg=8192.0000 max=8299 max_rank=2 min=8133 min_rank=7 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32687 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[529, 502, 569, 504, 533, 493, 526, 517, 504, 552, 486, 480, 532, 496, 492, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 495, 493, 510, 490, 501, 463, 531, 519, 486, 499, 468, 526, 527, 510, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 479, 476, 499, 519, 493, 506, 468, 456, 512, 508, 532, 506, 544, 511, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 467, 510, 520, 528, 483, 510, 515, 534, 519, 502, 517, 500, 572, 513, 544, 35, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 497, 521, 509, 466, 525, 495, 503, 521, 502, 517, 527, 477, 532, 499, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 533, 496, 488, 505, 554, 511, 491, 515, 565, 500, 547, 511, 537, 526, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 506, 516, 519, 504, 507, 505, 548, 516, 544, 540, 544, 539, 525, 546, 528, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 524, 518, 498, 520, 523, 509, 474, 480, 511, 531, 467, 493, 526, 502, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8243, 8057, 8029, 8304, 8125, 8299, 8408, 8071] avg=8192.0000 max=8408 max_rank=6 min=8029 min_rank=2 imbalance_ratio=1.026367 max_load_reduction=57128 max_load_reduction_pct=0.871704 moved_to_echo=32903 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[494, 509, 502, 506, 500, 530, 472, 524, 519, 502, 500, 504, 506, 513, 514, 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 517, 533, 527, 526, 562, 549, 539, 504, 551, 521, 517, 526, 490, 540, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 473, 509, 545, 513, 545, 528, 518, 526, 523, 489, 496, 489, 507, 490, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 538, 493, 526, 500, 491, 496, 501, 504, 533, 502, 522, 504, 518, 508, 492, 37, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 556, 517, 502, 505, 505, 522, 547, 529, 470, 498, 515, 526, 504, 498, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 515, 538, 521, 460, 477, 502, 509, 514, 547, 484, 523, 500, 541, 472, 492, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 544, 483, 438, 511, 477, 494, 552, 504, 488, 527, 533, 516, 501, 497, 519, 7, 8, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 511, 518, 540, 525, 536, 517, 483, 532, 513, 497, 495, 509, 464, 516, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8431, 8142, 8175, 8223, 8137, 8101, 8170] avg=8192.0000 max=8431 max_rank=1 min=8101 min_rank=6 imbalance_ratio=1.029175 max_load_reduction=57105 max_load_reduction_pct=0.871353 moved_to_echo=32631 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[514, 501, 495, 480, 507, 486, 537, 496, 526, 499, 468, 521, 509, 505, 515, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 518, 501, 497, 552, 526, 497, 541, 558, 520, 517, 519, 511, 476, 497, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 519, 511, 559, 531, 525, 508, 532, 522, 531, 546, 489, 485, 515, 538, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 554, 505, 539, 487, 511, 542, 504, 482, 503, 506, 475, 522, 503, 514, 498, 31, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 500, 471, 533, 484, 481, 483, 517, 523, 550, 496, 525, 554, 555, 518, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 510, 532, 525, 511, 559, 503, 515, 520, 489, 537, 487, 507, 517, 507, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 473, 495, 530, 489, 476, 487, 485, 505, 498, 517, 501, 493, 509, 500, 526, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 514, 539, 525, 505, 537, 487, 522, 467, 509, 535, 535, 507, 506, 503, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8066, 8243, 8377, 8172, 8220, 8192, 8022, 8244] avg=8192.0000 max=8377 max_rank=2 min=8022 min_rank=6 imbalance_ratio=1.022583 max_load_reduction=57159 max_load_reduction_pct=0.872177 moved_to_echo=32678 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[507, 498, 560, 520, 534, 510, 527, 487, 529, 550, 512, 553, 521, 511, 494, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 516, 482, 504, 562, 526, 543, 511, 476, 510, 497, 512, 510, 485, 478, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 482, 499, 493, 505, 487, 511, 487, 492, 512, 506, 505, 574, 546, 527, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 522, 503, 499, 503, 491, 511, 497, 553, 549, 472, 492, 483, 504, 530, 521, 32, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 504, 516, 530, 512, 512, 504, 500, 530, 533, 501, 543, 491, 523, 503, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 508, 504, 482, 535, 515, 533, 521, 502, 536, 490, 467, 546, 506, 519, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 524, 523, 518, 525, 531, 489, 501, 499, 498, 548, 532, 499, 471, 492, 456, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 488, 527, 516, 524, 520, 469, 513, 537, 503, 518, 477, 532, 522, 523, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8339, 8121, 8103, 8177, 8224, 8254, 8107, 8211] avg=8192.0000 max=8339 max_rank=0 min=8103 min_rank=2 imbalance_ratio=1.017944 max_load_reduction=57197 max_load_reduction_pct=0.872757 moved_to_echo=32796 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[523, 472, 526, 503, 504, 533, 526, 528, 531, 528, 517, 506, 502, 516, 506, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 502, 507, 469, 514, 513, 499, 533, 520, 475, 460, 488, 493, 484, 512, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 510, 471, 529, 471, 521, 521, 525, 490, 496, 494, 478, 536, 521, 516, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 524, 522, 509, 489, 513, 543, 506, 503, 501, 522, 535, 522, 486, 487, 506, 32, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 528, 549, 509, 511, 538, 511, 533, 491, 515, 503, 464, 493, 538, 486, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 463, 505, 474, 478, 518, 546, 493, 489, 517, 531, 525, 523, 522, 514, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 549, 510, 508, 555, 552, 470, 548, 484, 497, 512, 515, 512, 500, 534, 498, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 488, 511, 551, 531, 553, 545, 514, 508, 533, 529, 522, 557, 521, 513, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 7993, 8124, 8203, 8184, 8125, 8305, 8377] avg=8192.0000 max=8377 max_rank=7 min=7993 min_rank=1 imbalance_ratio=1.022583 max_load_reduction=57159 max_load_reduction_pct=0.872177 moved_to_echo=32991 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[529, 502, 569, 504, 533, 493, 526, 517, 504, 552, 486, 480, 532, 496, 492, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 495, 493, 510, 490, 501, 463, 531, 519, 486, 499, 468, 526, 527, 510, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 479, 476, 499, 519, 493, 506, 468, 456, 512, 508, 532, 506, 544, 511, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 467, 510, 520, 528, 483, 510, 515, 534, 519, 502, 517, 500, 572, 513, 544, 35, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 497, 521, 509, 466, 525, 495, 503, 521, 502, 517, 527, 477, 532, 499, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 533, 496, 488, 505, 554, 511, 491, 515, 565, 500, 547, 511, 537, 526, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 506, 516, 519, 504, 507, 505, 548, 516, 544, 540, 544, 539, 525, 546, 528, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 524, 518, 498, 520, 523, 509, 474, 480, 511, 531, 467, 493, 526, 502, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8243, 8057, 8029, 8304, 8125, 8299, 8408, 8071] avg=8192.0000 max=8408 max_rank=6 min=8029 min_rank=2 imbalance_ratio=1.026367 max_load_reduction=57128 max_load_reduction_pct=0.871704 moved_to_echo=32903 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[514, 507, 563, 484, 477, 472, 505, 537, 506, 530, 497, 509, 536, 503, 507, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 526, 518, 513, 524, 518, 487, 538, 518, 515, 502, 519, 508, 479, 527, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 515, 510, 527, 524, 545, 513, 503, 489, 504, 527, 501, 522, 565, 472, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 458, 479, 522, 540, 513, 518, 522, 519, 503, 503, 519, 540, 495, 488, 502, 32, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 530, 488, 508, 473, 525, 511, 538, 512, 501, 514, 547, 539, 488, 481, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 549, 515, 524, 522, 488, 515, 501, 477, 531, 514, 555, 488, 488, 519, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 553, 518, 457, 499, 493, 521, 479, 490, 481, 517, 575, 491, 562, 487, 517, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 504, 550, 460, 519, 504, 508, 516, 503, 501, 538, 550, 519, 502, 484, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8160, 8219, 8299, 8171, 8190, 8205, 8159, 8133] avg=8192.0000 max=8299 max_rank=2 min=8133 min_rank=7 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32687 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[541, 533, 492, 518, 519, 492, 491, 490, 550, 514, 476, 515, 477, 550, 531, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 515, 539, 559, 474, 515, 492, 454, 578, 492, 546, 561, 518, 498, 521, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 507, 541, 510, 481, 542, 523, 513, 502, 507, 506, 499, 474, 504, 503, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 471, 536, 503, 493, 520, 496, 497, 506, 531, 521, 492, 531, 495, 487, 549, 33, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 503, 486, 531, 490, 500, 534, 539, 517, 516, 534, 489, 514, 484, 535, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 460, 515, 485, 497, 483, 538, 546, 509, 471, 497, 527, 478, 511, 555, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 497, 494, 514, 513, 515, 548, 520, 534, 493, 513, 527, 516, 511, 507, 516, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 521, 506, 511, 502, 492, 489, 567, 504, 483, 533, 506, 528, 510, 536, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8267, 8148, 8178, 8202, 8106, 8268, 8185] avg=8192.0000 max=8268 max_rank=6 min=8106 min_rank=5 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32761 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 3968, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4224, 3968, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 3968, 4224, 4352, 4096, 4096, 4096, 4224, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 3968, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4224, 3968, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[468, 532, 548, 527, 476, 523, 473, 468, 541, 528, 541, 508, 507, 508, 503, 541, 476, 479, 516, 515, 496, 507, 527, 508, 560, 526, 523, 516, 466, 554, 522, 498, 519, 508, 512, 514, 519, 529, 522, 505, 527, 527, 490, 506, 527, 523, 468, 470, 512, 515, 542, 497, 472, 500, 527, 538, 512, 495, 523, 493, 500, 476, 513, 507, 543, 493, 504, 513, 461, 535, 514, 505, 481, 522, 494, 554, 530, 487, 495, 494, 495, 492, 505, 548, 541, 486, 485, 492, 512, 546, 512, 526, 520, 486, 520, 510, 524, 514, 521, 498, 524, 527, 461, 548, 509, 533, 525, 519, 529, 495, 495, 492, 477, 527, 536, 572, 494, 471, 533, 508, 575, 497, 522, 531, 536, 569, 487, 517] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[531, 545, 487, 508, 491, 544, 503, 531, 492, 539, 492, 516, 525, 529, 501, 506, 487, 508, 520, 523, 508, 491, 508, 529, 501, 495, 501, 487, 515, 481, 533, 485, 488, 474, 521, 507, 500, 536, 500, 536, 507, 492, 491, 525, 505, 502, 513, 566, 536, 540, 501, 514, 508, 547, 551, 516, 509, 495, 528, 536, 514, 532, 537, 468, 537, 510, 507, 502, 488, 507, 512, 479, 539, 486, 527, 514, 538, 579, 544, 501, 524, 471, 520, 480, 513, 498, 531, 531, 562, 532, 515, 455, 544, 518, 519, 548, 498, 488, 526, 474, 502, 550, 505, 481, 510, 488, 558, 499, 508, 512, 523, 538, 509, 494, 480, 487, 502, 549, 494, 505, 472, 515, 490, 505, 524, 511, 493, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[491, 500, 521, 527, 497, 514, 522, 504, 531, 526, 520, 482, 495, 497, 529, 514, 476, 539, 514, 498, 497, 503, 518, 505, 542, 492, 469, 511, 504, 499, 519, 525, 515, 484, 479, 514, 519, 503, 573, 549, 497, 507, 471, 499, 526, 506, 482, 525, 492, 514, 529, 494, 500, 525, 481, 475, 520, 498, 547, 519, 561, 544, 492, 510, 508, 523, 515, 505, 511, 536, 520, 535, 520, 507, 516, 519, 542, 508, 514, 493, 506, 548, 529, 505, 521, 519, 548, 505, 502, 537, 530, 491, 503, 532, 482, 476, 501, 522, 471, 541, 486, 510, 518, 539, 501, 526, 522, 548, 502, 528, 507, 548, 516, 493, 498, 509, 532, 510, 530, 501, 495, 543, 522, 522, 470, 512, 502, 474] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[540, 495, 526, 515, 518, 539, 467, 558, 530, 515, 506, 491, 533, 519, 500, 513, 481, 560, 486, 541, 514, 530, 511, 551, 460, 508, 513, 476, 493, 474, 490, 540, 510, 500, 524, 503, 521, 515, 537, 536, 520, 523, 488, 501, 545, 528, 493, 505, 492, 524, 513, 460, 467, 503, 548, 506, 540, 521, 535, 521, 492, 525, 502, 520, 512, 549, 583, 514, 516, 476, 512, 531, 508, 489, 509, 513, 512, 490, 529, 516, 515, 516, 525, 503, 511, 514, 495, 520, 511, 499, 505, 508, 562, 485, 513, 523, 563, 505, 502, 509, 528, 537, 530, 487, 525, 478, 502, 506, 531, 500, 466, 509, 472, 510, 464, 498, 500, 496, 486, 501, 557, 530, 516, 485, 537, 557, 473, 501] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[531, 545, 487, 508, 491, 544, 503, 531, 492, 539, 492, 516, 525, 529, 501, 506, 487, 508, 520, 523, 508, 491, 508, 529, 501, 495, 501, 487, 515, 481, 533, 485, 488, 474, 521, 507, 500, 536, 500, 536, 507, 492, 491, 525, 505, 502, 513, 566, 536, 540, 501, 514, 508, 547, 551, 516, 509, 495, 528, 536, 514, 532, 537, 468, 537, 510, 507, 502, 488, 507, 512, 479, 539, 486, 527, 514, 538, 579, 544, 501, 524, 471, 520, 480, 513, 498, 531, 531, 562, 532, 515, 455, 544, 518, 519, 548, 498, 488, 526, 474, 502, 550, 505, 481, 510, 488, 558, 499, 508, 512, 523, 538, 509, 494, 480, 487, 502, 549, 494, 505, 472, 515, 490, 505, 524, 511, 493, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[496, 540, 497, 494, 484, 512, 530, 492, 551, 529, 509, 474, 508, 519, 536, 517, 510, 548, 518, 499, 569, 494, 527, 519, 532, 474, 517, 515, 506, 533, 503, 530, 527, 525, 510, 488, 525, 483, 522, 474, 532, 484, 514, 504, 489, 540, 493, 577, 493, 488, 530, 501, 486, 486, 510, 522, 530, 528, 526, 547, 534, 515, 517, 482, 514, 517, 517, 529, 507, 497, 498, 498, 492, 548, 512, 469, 459, 518, 488, 529, 501, 489, 514, 495, 497, 506, 511, 548, 488, 501, 510, 525, 515, 516, 519, 491, 497, 502, 540, 535, 516, 529, 513, 529, 515, 524, 502, 485, 537, 499, 513, 552, 491, 515, 510, 532, 482, 483, 545, 546, 514, 517, 499, 495, 486, 525, 520, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[468, 532, 548, 527, 476, 523, 473, 468, 541, 528, 541, 508, 507, 508, 503, 541, 476, 479, 516, 515, 496, 507, 527, 508, 560, 526, 523, 516, 466, 554, 522, 498, 519, 508, 512, 514, 519, 529, 522, 505, 527, 527, 490, 506, 527, 523, 468, 470, 512, 515, 542, 497, 472, 500, 527, 538, 512, 495, 523, 493, 500, 476, 513, 507, 543, 493, 504, 513, 461, 535, 514, 505, 481, 522, 494, 554, 530, 487, 495, 494, 495, 492, 505, 548, 541, 486, 485, 492, 512, 546, 512, 526, 520, 486, 520, 510, 524, 514, 521, 498, 524, 527, 461, 548, 509, 533, 525, 519, 529, 495, 495, 492, 477, 527, 536, 572, 494, 471, 533, 508, 575, 497, 522, 531, 536, 569, 487, 517] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[491, 500, 521, 527, 497, 514, 522, 504, 531, 526, 520, 482, 495, 497, 529, 514, 476, 539, 514, 498, 497, 503, 518, 505, 542, 492, 469, 511, 504, 499, 519, 525, 515, 484, 479, 514, 519, 503, 573, 549, 497, 507, 471, 499, 526, 506, 482, 525, 492, 514, 529, 494, 500, 525, 481, 475, 520, 498, 547, 519, 561, 544, 492, 510, 508, 523, 515, 505, 511, 536, 520, 535, 520, 507, 516, 519, 542, 508, 514, 493, 506, 548, 529, 505, 521, 519, 548, 505, 502, 537, 530, 491, 503, 532, 482, 476, 501, 522, 471, 541, 486, 510, 518, 539, 501, 526, 522, 548, 502, 528, 507, 548, 516, 493, 498, 509, 532, 510, 530, 501, 495, 543, 522, 522, 470, 512, 502, 474] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[471, 504, 518, 505, 476, 554, 520, 540, 522, 532, 532, 523, 511, 506, 503, 525, 516, 499, 514, 539, 534, 522, 517, 507, 533, 541, 499, 475, 472, 506, 520, 510, 571, 502, 532, 506, 545, 474, 517, 514, 522, 511, 509, 510, 476, 510, 532, 543, 503, 545, 510, 511, 523, 486, 483, 491, 523, 502, 540, 488, 531, 492, 490, 504, 477, 521, 526, 512, 500, 517, 511, 529, 526, 465, 498, 518, 560, 529, 513, 504, 507, 503, 516, 531, 480, 532, 504, 525, 521, 476, 518, 489, 510, 496, 514, 539, 495, 496, 493, 509, 547, 536, 526, 483, 509, 504, 507, 524, 513, 521, 507, 542, 494, 528, 506, 523, 482, 515, 505, 527, 494, 514, 522, 508, 451, 541, 492, 513] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[476, 514, 568, 521, 534, 492, 516, 528, 503, 517, 508, 500, 503, 507, 533, 509, 535, 523, 514, 521, 499, 511, 497, 562, 532, 510, 526, 519, 533, 543, 502, 510, 503, 520, 511, 489, 509, 543, 522, 502, 515, 519, 519, 516, 550, 489, 494, 506, 511, 472, 503, 520, 482, 522, 512, 522, 510, 517, 529, 502, 488, 485, 496, 482, 497, 486, 510, 519, 539, 510, 503, 520, 482, 481, 515, 499, 543, 499, 511, 487, 509, 527, 506, 519, 537, 501, 498, 518, 525, 521, 489, 522, 525, 489, 518, 552, 529, 521, 488, 502, 502, 501, 538, 517, 521, 483, 547, 530, 512, 506, 515, 518, 474, 505, 542, 501, 510, 515, 495, 473, 528, 521, 491, 485, 480, 507, 545, 551] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[523, 473, 514, 488, 510, 506, 528, 524, 531, 520, 521, 507, 510, 520, 527, 558, 461, 541, 492, 546, 503, 544, 501, 488, 528, 463, 535, 525, 519, 502, 501, 535, 500, 462, 540, 479, 533, 495, 480, 507, 503, 516, 543, 511, 531, 469, 494, 541, 507, 535, 546, 523, 515, 505, 497, 536, 498, 510, 475, 501, 515, 464, 551, 523, 533, 492, 514, 487, 513, 489, 510, 522, 538, 492, 526, 532, 476, 496, 508, 524, 517, 530, 507, 535, 464, 524, 547, 523, 515, 470, 524, 503, 512, 494, 534, 490, 542, 517, 492, 485, 545, 492, 557, 510, 485, 513, 534, 529, 502, 551, 494, 506, 483, 521, 512, 523, 489, 534, 517, 505, 503, 498, 485, 546, 486, 491, 552, 547] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[468, 532, 548, 527, 476, 523, 473, 468, 541, 498, 541, 508, 507, 508, 503, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 479, 516, 515, 496, 507, 527, 508, 560, 526, 523, 516, 466, 554, 522, 498, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 508, 512, 514, 519, 529, 522, 505, 527, 527, 490, 506, 527, 523, 468, 470, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 515, 542, 497, 472, 500, 527, 538, 512, 495, 523, 493, 500, 476, 513, 507, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 493, 504, 513, 461, 535, 514, 505, 481, 522, 494, 554, 530, 487, 495, 494, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 492, 505, 548, 541, 486, 485, 492, 512, 546, 512, 526, 512, 486, 520, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 514, 521, 498, 524, 527, 461, 548, 509, 533, 525, 519, 529, 495, 495, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 527, 536, 572, 494, 471, 533, 508, 575, 497, 522, 531, 536, 569, 487, 517, 31, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8162, 8191, 8170, 8139, 8132, 8168, 8183, 8391] avg=8192.0000 max=8391 max_rank=7 min=8132 min_rank=4 imbalance_ratio=1.024292 max_load_reduction=57145 max_load_reduction_pct=0.871964 moved_to_echo=32874 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[531, 545, 487, 508, 491, 544, 503, 531, 492, 499, 492, 516, 525, 529, 501, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 508, 520, 523, 508, 491, 508, 529, 501, 495, 501, 487, 515, 481, 533, 485, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 474, 521, 507, 500, 536, 500, 536, 507, 492, 491, 525, 505, 502, 513, 566, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 540, 501, 514, 508, 547, 551, 516, 509, 495, 528, 536, 514, 532, 537, 468, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 510, 507, 502, 488, 507, 512, 479, 539, 486, 527, 514, 538, 579, 544, 501, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 471, 520, 480, 513, 498, 531, 531, 562, 532, 515, 455, 528, 518, 519, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 488, 526, 474, 502, 550, 505, 481, 510, 488, 558, 499, 508, 512, 523, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 494, 480, 487, 502, 549, 494, 505, 472, 515, 490, 505, 524, 511, 493, 508, 38, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8075, 8172, 8354, 8280, 8245, 8122, 8088] avg=8192.0000 max=8354 max_rank=3 min=8075 min_rank=1 imbalance_ratio=1.019775 max_load_reduction=57182 max_load_reduction_pct=0.872528 moved_to_echo=32735 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[471, 504, 518, 505, 476, 554, 520, 540, 522, 502, 532, 523, 511, 506, 503, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 499, 514, 539, 534, 522, 517, 507, 533, 541, 499, 475, 472, 506, 520, 510, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 571, 502, 532, 506, 545, 474, 517, 514, 522, 511, 509, 510, 476, 510, 532, 543, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 545, 510, 511, 523, 486, 483, 491, 523, 502, 540, 488, 531, 492, 490, 504, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 521, 526, 512, 500, 517, 511, 529, 526, 465, 498, 518, 560, 529, 513, 504, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 503, 516, 531, 480, 532, 504, 525, 521, 476, 518, 489, 502, 496, 514, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 496, 493, 509, 547, 536, 526, 483, 509, 504, 507, 524, 513, 521, 507, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 528, 506, 523, 482, 515, 505, 527, 494, 514, 522, 508, 451, 541, 492, 513, 34, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8206, 8278, 8139, 8213, 8153, 8178, 8157] avg=8192.0000 max=8278 max_rank=2 min=8139 min_rank=3 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32701 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[496, 540, 497, 494, 484, 512, 530, 492, 551, 499, 509, 474, 508, 519, 536, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 548, 518, 499, 569, 494, 527, 519, 532, 474, 517, 515, 506, 533, 503, 530, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 525, 510, 488, 525, 483, 522, 474, 532, 484, 514, 504, 489, 540, 493, 577, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 488, 530, 501, 486, 486, 510, 522, 530, 528, 526, 547, 534, 515, 517, 482, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 517, 517, 529, 507, 497, 498, 498, 492, 548, 512, 469, 459, 518, 488, 529, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 489, 514, 495, 497, 506, 511, 548, 488, 501, 510, 525, 507, 516, 519, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 502, 540, 535, 516, 529, 513, 529, 515, 524, 502, 485, 537, 499, 513, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 515, 510, 532, 482, 483, 545, 546, 514, 517, 499, 495, 486, 525, 520, 506, 35, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8158, 8296, 8191, 8212, 8099, 8118, 8253, 8209] avg=8192.0000 max=8296 max_rank=1 min=8099 min_rank=4 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32679 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[540, 495, 526, 515, 518, 539, 467, 558, 530, 485, 506, 491, 533, 519, 500, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 560, 486, 541, 514, 530, 511, 551, 460, 508, 513, 476, 493, 474, 490, 540, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 500, 524, 503, 521, 515, 537, 536, 520, 523, 488, 501, 545, 528, 493, 505, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 524, 513, 460, 467, 503, 548, 506, 540, 521, 535, 521, 492, 525, 502, 520, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 549, 583, 514, 516, 476, 512, 531, 508, 489, 509, 513, 512, 490, 529, 516, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 516, 525, 503, 511, 514, 495, 520, 511, 499, 505, 508, 554, 485, 513, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 563, 505, 502, 509, 528, 537, 530, 487, 525, 478, 502, 506, 531, 500, 466, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 510, 464, 498, 500, 496, 486, 501, 557, 530, 516, 485, 537, 557, 473, 501, 32, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8130, 8253, 8186, 8266, 8197, 8146, 8123] avg=8192.0000 max=8266 max_rank=4 min=8123 min_rank=7 imbalance_ratio=1.009033 max_load_reduction=57270 max_load_reduction_pct=0.873871 moved_to_echo=32732 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[491, 500, 521, 527, 497, 514, 522, 504, 531, 496, 520, 482, 495, 497, 529, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 539, 514, 498, 497, 503, 518, 505, 542, 492, 469, 511, 504, 499, 519, 525, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 484, 479, 514, 519, 503, 573, 549, 497, 507, 471, 499, 526, 506, 482, 525, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 514, 529, 494, 500, 525, 481, 475, 520, 498, 547, 519, 561, 544, 492, 510, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 523, 515, 505, 511, 536, 520, 535, 520, 507, 516, 519, 542, 508, 514, 493, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 548, 529, 505, 521, 519, 548, 505, 502, 537, 530, 491, 495, 532, 482, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 522, 471, 541, 486, 510, 518, 539, 501, 526, 522, 548, 502, 528, 507, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 493, 498, 509, 532, 510, 530, 501, 495, 543, 522, 522, 470, 512, 502, 474, 35, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8140, 8113, 8153, 8218, 8279, 8226, 8235, 8172] avg=8192.0000 max=8279 max_rank=4 min=8113 min_rank=1 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32912 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[468, 532, 548, 527, 476, 523, 473, 468, 541, 498, 541, 508, 507, 508, 503, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 479, 516, 515, 496, 507, 527, 508, 560, 526, 523, 516, 466, 554, 522, 498, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 508, 512, 514, 519, 529, 522, 505, 527, 527, 490, 506, 527, 523, 468, 470, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 515, 542, 497, 472, 500, 527, 538, 512, 495, 523, 493, 500, 476, 513, 507, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 493, 504, 513, 461, 535, 514, 505, 481, 522, 494, 554, 530, 487, 495, 494, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 492, 505, 548, 541, 486, 485, 492, 512, 546, 512, 526, 512, 486, 520, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 514, 521, 498, 524, 527, 461, 548, 509, 533, 525, 519, 529, 495, 495, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 527, 536, 572, 494, 471, 533, 508, 575, 497, 522, 531, 536, 569, 487, 517, 31, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8162, 8191, 8170, 8139, 8132, 8168, 8183, 8391] avg=8192.0000 max=8391 max_rank=7 min=8132 min_rank=4 imbalance_ratio=1.024292 max_load_reduction=57145 max_load_reduction_pct=0.871964 moved_to_echo=32874 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[496, 540, 497, 494, 484, 512, 530, 492, 551, 499, 509, 474, 508, 519, 536, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 548, 518, 499, 569, 494, 527, 519, 532, 474, 517, 515, 506, 533, 503, 530, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 525, 510, 488, 525, 483, 522, 474, 532, 484, 514, 504, 489, 540, 493, 577, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 488, 530, 501, 486, 486, 510, 522, 530, 528, 526, 547, 534, 515, 517, 482, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 517, 517, 529, 507, 497, 498, 498, 492, 548, 512, 469, 459, 518, 488, 529, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 489, 514, 495, 497, 506, 511, 548, 488, 501, 510, 525, 507, 516, 519, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 502, 540, 535, 516, 529, 513, 529, 515, 524, 502, 485, 537, 499, 513, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 515, 510, 532, 482, 483, 545, 546, 514, 517, 499, 495, 486, 525, 520, 506, 35, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8158, 8296, 8191, 8212, 8099, 8118, 8253, 8209] avg=8192.0000 max=8296 max_rank=1 min=8099 min_rank=4 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32679 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[523, 473, 514, 488, 510, 506, 528, 524, 531, 490, 521, 507, 510, 520, 527, 558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 541, 492, 546, 503, 544, 501, 488, 528, 463, 535, 525, 519, 502, 501, 535, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 462, 540, 479, 533, 495, 480, 507, 503, 516, 543, 511, 531, 469, 494, 541, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 535, 546, 523, 515, 505, 497, 536, 498, 510, 475, 501, 515, 464, 551, 523, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 492, 514, 487, 513, 489, 510, 522, 538, 492, 526, 532, 476, 496, 508, 524, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 530, 507, 535, 464, 524, 547, 523, 515, 470, 524, 503, 504, 494, 534, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 517, 492, 485, 545, 492, 557, 510, 485, 513, 534, 529, 502, 551, 494, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 521, 512, 523, 489, 534, 517, 505, 503, 498, 485, 546, 486, 491, 552, 547, 32, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 8186, 8108, 8218, 8159, 8181, 8222, 8232] avg=8192.0000 max=8232 max_rank=7 min=8108 min_rank=2 imbalance_ratio=1.004883 max_load_reduction=57304 max_load_reduction_pct=0.874390 moved_to_echo=32794 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[476, 514, 568, 521, 534, 492, 516, 528, 503, 487, 508, 500, 503, 507, 533, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 523, 514, 521, 499, 511, 497, 562, 532, 510, 526, 519, 533, 543, 502, 510, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 520, 511, 489, 509, 543, 522, 502, 515, 519, 519, 516, 550, 489, 494, 506, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 472, 503, 520, 482, 522, 512, 522, 510, 517, 529, 502, 488, 485, 496, 482, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 486, 510, 519, 539, 510, 503, 520, 482, 481, 515, 499, 543, 499, 511, 487, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 527, 506, 519, 537, 501, 498, 518, 525, 521, 489, 522, 517, 489, 518, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 521, 488, 502, 502, 501, 538, 517, 521, 483, 547, 530, 512, 506, 515, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 505, 542, 501, 510, 515, 495, 473, 528, 521, 491, 485, 480, 507, 545, 551, 33, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8339, 8211, 8070, 8108, 8248, 8197, 8164] avg=8192.0000 max=8339 max_rank=1 min=8070 min_rank=3 imbalance_ratio=1.017944 max_load_reduction=57197 max_load_reduction_pct=0.872757 moved_to_echo=32717 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[491, 500, 521, 527, 497, 514, 522, 504, 531, 496, 520, 482, 495, 497, 529, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 539, 514, 498, 497, 503, 518, 505, 542, 492, 469, 511, 504, 499, 519, 525, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 484, 479, 514, 519, 503, 573, 549, 497, 507, 471, 499, 526, 506, 482, 525, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 514, 529, 494, 500, 525, 481, 475, 520, 498, 547, 519, 561, 544, 492, 510, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 523, 515, 505, 511, 536, 520, 535, 520, 507, 516, 519, 542, 508, 514, 493, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 548, 529, 505, 521, 519, 548, 505, 502, 537, 530, 491, 495, 532, 482, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 522, 471, 541, 486, 510, 518, 539, 501, 526, 522, 548, 502, 528, 507, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 493, 498, 509, 532, 510, 530, 501, 495, 543, 522, 522, 470, 512, 502, 474, 35, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8140, 8113, 8153, 8218, 8279, 8226, 8235, 8172] avg=8192.0000 max=8279 max_rank=4 min=8113 min_rank=1 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32912 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[471, 504, 518, 505, 476, 554, 520, 540, 522, 502, 532, 523, 511, 506, 503, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 499, 514, 539, 534, 522, 517, 507, 533, 541, 499, 475, 472, 506, 520, 510, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 571, 502, 532, 506, 545, 474, 517, 514, 522, 511, 509, 510, 476, 510, 532, 543, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 545, 510, 511, 523, 486, 483, 491, 523, 502, 540, 488, 531, 492, 490, 504, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 521, 526, 512, 500, 517, 511, 529, 526, 465, 498, 518, 560, 529, 513, 504, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 503, 516, 531, 480, 532, 504, 525, 521, 476, 518, 489, 502, 496, 514, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 496, 493, 509, 547, 536, 526, 483, 509, 504, 507, 524, 513, 521, 507, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 528, 506, 523, 482, 515, 505, 527, 494, 514, 522, 508, 451, 541, 492, 513, 34, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8206, 8278, 8139, 8213, 8153, 8178, 8157] avg=8192.0000 max=8278 max_rank=2 min=8139 min_rank=3 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32701 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4096, 3968, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[3968, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4352, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 3968, 4224, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4096, 3968, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[3968, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[521, 520, 517, 522, 514, 479, 519, 474, 523, 521, 583, 498, 506, 502, 500, 548, 539, 494, 507, 523, 555, 513, 487, 522, 550, 488, 520, 496, 579, 535, 515, 532, 539, 498, 521, 476, 499, 494, 544, 499, 517, 514, 530, 498, 496, 489, 505, 490, 501, 486, 495, 520, 485, 541, 516, 497, 495, 464, 504, 509, 532, 513, 525, 502, 502, 501, 520, 504, 511, 555, 533, 486, 544, 505, 539, 539, 553, 526, 512, 494, 505, 492, 538, 462, 472, 519, 529, 506, 507, 522, 509, 497, 538, 486, 494, 545, 492, 522, 511, 494, 479, 530, 456, 525, 533, 504, 504, 506, 506, 529, 483, 494, 517, 511, 542, 545, 502, 504, 491, 504, 543, 476, 511, 470, 495, 541, 539, 536] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[511, 505, 515, 502, 508, 504, 529, 570, 490, 466, 535, 538, 493, 585, 516, 482, 509, 491, 491, 493, 529, 523, 518, 496, 499, 544, 504, 512, 528, 479, 526, 498, 514, 516, 497, 477, 488, 511, 489, 542, 512, 548, 506, 504, 537, 506, 531, 510, 508, 501, 535, 528, 503, 496, 501, 539, 476, 494, 518, 526, 549, 504, 518, 506, 527, 478, 510, 505, 505, 493, 517, 540, 521, 515, 548, 482, 549, 530, 495, 507, 483, 518, 528, 515, 491, 500, 488, 500, 479, 517, 513, 558, 523, 489, 507, 505, 528, 501, 503, 510, 505, 535, 489, 491, 506, 499, 523, 493, 523, 457, 539, 524, 460, 530, 535, 537, 532, 511, 526, 532, 532, 513, 491, 517, 518, 540, 511, 510] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[521, 495, 481, 489, 488, 496, 523, 502, 519, 493, 546, 512, 513, 536, 533, 528, 550, 547, 532, 540, 457, 505, 498, 484, 517, 498, 522, 572, 516, 490, 455, 543, 483, 546, 519, 510, 506, 496, 494, 520, 481, 494, 517, 534, 523, 542, 565, 512, 491, 543, 495, 469, 478, 522, 502, 484, 515, 502, 537, 502, 543, 521, 549, 502, 528, 489, 511, 518, 484, 530, 505, 515, 523, 548, 496, 486, 523, 530, 531, 502, 450, 517, 536, 512, 506, 506, 510, 490, 523, 526, 512, 512, 523, 531, 496, 496, 503, 533, 522, 484, 490, 535, 558, 509, 533, 532, 489, 482, 508, 512, 497, 460, 524, 510, 548, 527, 515, 548, 536, 501, 499, 492, 546, 503, 465, 487, 500, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[493, 497, 521, 498, 524, 546, 466, 494, 477, 535, 528, 484, 544, 497, 508, 541, 539, 525, 510, 480, 533, 501, 539, 540, 496, 498, 510, 509, 520, 482, 509, 482, 552, 505, 502, 484, 545, 535, 507, 513, 523, 521, 514, 518, 533, 521, 524, 499, 544, 513, 492, 512, 540, 489, 503, 509, 546, 524, 495, 531, 539, 504, 529, 490, 471, 539, 479, 489, 500, 508, 491, 470, 499, 535, 531, 497, 474, 517, 522, 480, 538, 525, 481, 516, 535, 570, 502, 531, 513, 482, 524, 508, 526, 499, 514, 518, 516, 526, 459, 492, 521, 509, 528, 498, 520, 534, 505, 532, 538, 544, 543, 491, 515, 497, 484, 504, 510, 530, 492, 508, 491, 523, 496, 520, 536, 522, 506, 480] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[521, 520, 517, 522, 514, 479, 519, 474, 523, 521, 583, 498, 506, 502, 500, 548, 539, 494, 507, 523, 555, 513, 487, 522, 550, 488, 520, 496, 579, 535, 515, 532, 539, 498, 521, 476, 499, 494, 544, 499, 517, 514, 530, 498, 496, 489, 505, 490, 501, 486, 495, 520, 485, 541, 516, 497, 495, 464, 504, 509, 532, 513, 525, 502, 502, 501, 520, 504, 511, 555, 533, 486, 544, 505, 539, 539, 553, 526, 512, 494, 505, 492, 538, 462, 472, 519, 529, 506, 507, 522, 509, 497, 538, 486, 494, 545, 492, 522, 511, 494, 479, 530, 456, 525, 533, 504, 504, 506, 506, 529, 483, 494, 517, 511, 542, 545, 502, 504, 491, 504, 543, 476, 511, 470, 495, 541, 539, 536] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[495, 507, 478, 531, 503, 513, 519, 532, 484, 509, 503, 529, 531, 523, 515, 494, 514, 529, 519, 519, 501, 496, 508, 492, 520, 465, 505, 523, 511, 535, 478, 483, 475, 526, 556, 529, 491, 508, 515, 550, 506, 502, 505, 513, 462, 509, 504, 543, 501, 503, 515, 532, 525, 494, 561, 509, 531, 503, 487, 545, 496, 518, 515, 511, 502, 503, 519, 484, 525, 512, 539, 531, 500, 525, 513, 472, 531, 514, 521, 492, 507, 525, 525, 548, 530, 541, 519, 521, 494, 519, 533, 508, 517, 481, 522, 511, 485, 528, 526, 521, 563, 521, 497, 480, 524, 508, 494, 500, 495, 489, 532, 504, 525, 512, 534, 494, 535, 538, 517, 491, 513, 517, 485, 478, 516, 509, 490, 527] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[558, 527, 512, 475, 516, 560, 489, 484, 524, 501, 497, 531, 506, 527, 489, 528, 540, 538, 476, 512, 496, 480, 506, 520, 525, 498, 532, 509, 538, 561, 559, 538, 487, 474, 512, 463, 523, 526, 504, 515, 444, 552, 574, 534, 470, 534, 479, 548, 536, 518, 557, 504, 534, 584, 465, 520, 519, 510, 506, 531, 535, 486, 489, 530, 448, 523, 474, 536, 519, 498, 511, 496, 496, 507, 514, 531, 529, 483, 491, 504, 559, 514, 503, 492, 521, 529, 491, 527, 491, 491, 464, 540, 515, 501, 520, 466, 509, 492, 536, 531, 540, 518, 515, 456, 503, 533, 501, 472, 505, 551, 549, 526, 490, 487, 531, 516, 499, 477, 535, 555, 458, 502, 494, 507, 502, 501, 562, 484] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[549, 495, 540, 499, 561, 534, 529, 500, 506, 547, 496, 497, 517, 527, 532, 535, 500, 508, 473, 476, 503, 503, 549, 513, 523, 503, 497, 520, 514, 537, 541, 467, 534, 508, 511, 504, 541, 492, 483, 513, 531, 497, 486, 487, 494, 542, 516, 503, 527, 481, 507, 509, 519, 531, 527, 517, 473, 506, 535, 522, 498, 511, 505, 479, 501, 507, 506, 470, 516, 533, 505, 483, 513, 543, 520, 500, 541, 560, 467, 506, 502, 483, 520, 524, 488, 500, 490, 492, 525, 526, 507, 487, 496, 483, 532, 517, 547, 522, 520, 504, 519, 546, 536, 500, 478, 514, 488, 511, 513, 519, 507, 548, 515, 501, 517, 506, 514, 521, 501, 549, 515, 536, 541, 520, 499, 510, 515, 481] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[511, 505, 515, 502, 508, 504, 529, 570, 490, 466, 535, 538, 493, 585, 516, 482, 509, 491, 491, 493, 529, 523, 518, 496, 499, 544, 504, 512, 528, 479, 526, 498, 514, 516, 497, 477, 488, 511, 489, 542, 512, 548, 506, 504, 537, 506, 531, 510, 508, 501, 535, 528, 503, 496, 501, 539, 476, 494, 518, 526, 549, 504, 518, 506, 527, 478, 510, 505, 505, 493, 517, 540, 521, 515, 548, 482, 549, 530, 495, 507, 483, 518, 528, 515, 491, 500, 488, 500, 479, 517, 513, 558, 523, 489, 507, 505, 528, 501, 503, 510, 505, 535, 489, 491, 506, 499, 523, 493, 523, 457, 539, 524, 460, 530, 535, 537, 532, 511, 526, 532, 532, 513, 491, 517, 518, 540, 511, 510] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[521, 495, 481, 489, 488, 496, 523, 502, 519, 493, 546, 512, 513, 536, 533, 528, 550, 547, 532, 540, 457, 505, 498, 484, 517, 498, 522, 572, 516, 490, 455, 543, 483, 546, 519, 510, 506, 496, 494, 520, 481, 494, 517, 534, 523, 542, 565, 512, 491, 543, 495, 469, 478, 522, 502, 484, 515, 502, 537, 502, 543, 521, 549, 502, 528, 489, 511, 518, 484, 530, 505, 515, 523, 548, 496, 486, 523, 530, 531, 502, 450, 517, 536, 512, 506, 506, 510, 490, 523, 526, 512, 512, 523, 531, 496, 496, 503, 533, 522, 484, 490, 535, 558, 509, 533, 532, 489, 482, 508, 512, 497, 460, 524, 510, 548, 527, 515, 548, 536, 501, 499, 492, 546, 503, 465, 487, 500, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[557, 510, 520, 473, 519, 563, 526, 501, 497, 511, 508, 502, 509, 545, 504, 490, 519, 499, 518, 494, 530, 483, 523, 494, 506, 521, 537, 544, 559, 498, 529, 491, 479, 515, 521, 507, 539, 490, 526, 504, 512, 473, 532, 489, 517, 501, 517, 544, 509, 475, 500, 532, 479, 539, 519, 496, 527, 504, 557, 503, 513, 504, 521, 497, 525, 517, 477, 530, 461, 503, 549, 526, 523, 503, 537, 499, 489, 547, 484, 541, 521, 509, 478, 502, 487, 463, 536, 522, 525, 497, 517, 493, 504, 536, 506, 508, 496, 518, 535, 497, 524, 550, 522, 533, 532, 505, 503, 472, 519, 500, 513, 496, 506, 529, 498, 530, 523, 541, 513, 531, 508, 495, 493, 495, 490, 511, 512, 510] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[558, 527, 512, 475, 516, 560, 489, 484, 524, 501, 497, 531, 506, 527, 489, 528, 540, 538, 476, 512, 496, 480, 506, 520, 525, 498, 532, 509, 538, 561, 559, 538, 487, 474, 512, 463, 523, 526, 504, 515, 444, 552, 574, 534, 470, 534, 479, 548, 536, 518, 557, 504, 534, 584, 465, 520, 519, 510, 506, 531, 535, 486, 489, 530, 448, 523, 474, 536, 519, 498, 511, 496, 496, 507, 514, 531, 529, 483, 491, 504, 559, 514, 503, 492, 521, 529, 491, 527, 491, 491, 464, 540, 515, 501, 520, 466, 509, 492, 536, 531, 540, 518, 515, 456, 503, 533, 501, 472, 505, 551, 549, 526, 490, 487, 531, 516, 499, 477, 535, 555, 458, 502, 494, 507, 502, 501, 562, 484] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[511, 505, 515, 502, 508, 504, 529, 570, 490, 466, 535, 538, 493, 548, 516, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 491, 491, 493, 529, 523, 518, 496, 499, 544, 504, 512, 509, 479, 526, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 516, 497, 477, 488, 511, 489, 542, 512, 548, 506, 504, 537, 506, 531, 510, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 501, 535, 528, 503, 496, 501, 539, 476, 494, 518, 526, 543, 504, 518, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 478, 510, 505, 505, 493, 517, 540, 521, 515, 548, 482, 549, 530, 495, 507, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 518, 528, 515, 491, 500, 488, 500, 479, 517, 513, 558, 523, 489, 507, 505, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 501, 503, 510, 505, 535, 489, 491, 506, 499, 523, 493, 523, 457, 539, 524, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 460, 530, 531, 537, 532, 511, 526, 532, 532, 513, 491, 517, 518, 540, 511, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8121, 8195, 8196, 8239, 8151, 8131, 8291] avg=8192.0000 max=8291 max_rank=7 min=8121 min_rank=1 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32812 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[521, 520, 517, 522, 514, 479, 519, 474, 523, 521, 583, 498, 506, 470, 500, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 494, 507, 523, 555, 513, 487, 522, 550, 488, 520, 496, 559, 535, 515, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 498, 521, 476, 499, 494, 544, 499, 517, 514, 530, 498, 496, 489, 505, 490, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 486, 495, 520, 485, 541, 516, 497, 495, 464, 504, 509, 526, 513, 525, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 501, 520, 504, 511, 555, 533, 486, 544, 505, 539, 539, 553, 526, 512, 494, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 492, 538, 462, 472, 519, 529, 506, 507, 522, 509, 497, 538, 486, 494, 545, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 522, 511, 494, 479, 530, 456, 525, 533, 504, 504, 506, 506, 529, 483, 494, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 511, 538, 545, 502, 504, 491, 504, 543, 476, 511, 470, 495, 541, 539, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8215, 8335, 8116, 8079, 8342, 8153, 8073, 8223] avg=8192.0000 max=8342 max_rank=4 min=8073 min_rank=6 imbalance_ratio=1.018311 max_load_reduction=57194 max_load_reduction_pct=0.872711 moved_to_echo=32791 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[493, 497, 521, 498, 524, 546, 466, 494, 477, 535, 528, 484, 544, 456, 508, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 525, 510, 480, 533, 501, 539, 540, 496, 498, 510, 509, 496, 482, 509, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 505, 502, 484, 545, 535, 507, 513, 523, 521, 514, 518, 533, 521, 524, 499, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 513, 492, 512, 540, 489, 503, 509, 546, 524, 495, 531, 523, 504, 529, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 539, 479, 489, 500, 508, 491, 470, 499, 535, 531, 497, 474, 517, 522, 480, 21, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 525, 481, 516, 535, 570, 502, 531, 513, 482, 524, 508, 526, 499, 514, 518, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 526, 459, 492, 521, 509, 528, 498, 520, 534, 505, 532, 538, 544, 543, 491, 6, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 497, 478, 504, 510, 530, 492, 508, 491, 523, 496, 520, 536, 522, 506, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8112, 8149, 8307, 8244, 8028, 8318, 8270, 8108] avg=8192.0000 max=8318 max_rank=5 min=8028 min_rank=4 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32724 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[521, 495, 481, 489, 488, 496, 523, 502, 519, 493, 546, 512, 513, 502, 533, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 547, 532, 540, 457, 505, 498, 484, 517, 498, 522, 572, 498, 490, 455, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 546, 519, 510, 506, 496, 494, 520, 481, 494, 517, 534, 523, 542, 565, 512, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 543, 495, 469, 478, 522, 502, 484, 515, 502, 537, 502, 537, 521, 549, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 489, 511, 518, 484, 530, 505, 515, 523, 548, 496, 486, 523, 530, 531, 502, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 450, 517, 536, 512, 506, 506, 510, 490, 523, 526, 512, 512, 523, 531, 496, 496, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 533, 522, 484, 490, 535, 558, 509, 533, 532, 489, 482, 508, 512, 497, 460, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 510, 544, 527, 515, 548, 536, 501, 499, 492, 546, 503, 465, 487, 500, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8141, 8208, 8249, 8149, 8235, 8180, 8152, 8222] avg=8192.0000 max=8249 max_rank=2 min=8141 min_rank=0 imbalance_ratio=1.006958 max_load_reduction=57287 max_load_reduction_pct=0.874130 moved_to_echo=32789 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[495, 507, 478, 531, 503, 513, 519, 532, 484, 509, 503, 529, 531, 490, 515, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 529, 519, 519, 501, 496, 508, 492, 520, 465, 505, 523, 493, 535, 478, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 526, 556, 529, 491, 508, 515, 550, 506, 502, 505, 513, 462, 509, 504, 543, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 503, 515, 532, 525, 494, 561, 509, 531, 503, 487, 545, 490, 518, 515, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 503, 519, 484, 525, 512, 539, 531, 500, 525, 513, 472, 531, 514, 521, 492, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 525, 525, 548, 530, 541, 519, 521, 494, 519, 533, 508, 517, 481, 522, 511, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 528, 526, 521, 563, 521, 497, 480, 524, 508, 494, 500, 495, 489, 532, 504, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 512, 530, 494, 535, 538, 517, 491, 513, 517, 485, 478, 516, 509, 490, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8133, 8080, 8201, 8240, 8199, 8334, 8172, 8177] avg=8192.0000 max=8334 max_rank=5 min=8080 min_rank=1 imbalance_ratio=1.017334 max_load_reduction=57202 max_load_reduction_pct=0.872833 moved_to_echo=32882 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[558, 527, 512, 475, 516, 560, 489, 484, 524, 501, 497, 531, 506, 494, 489, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 538, 476, 512, 496, 480, 506, 520, 525, 498, 532, 509, 519, 561, 559, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 474, 512, 463, 523, 526, 504, 515, 444, 552, 574, 534, 470, 534, 479, 548, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 518, 557, 504, 534, 584, 465, 520, 519, 510, 506, 531, 529, 486, 489, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 448, 523, 474, 536, 519, 498, 511, 496, 496, 507, 514, 531, 529, 483, 491, 504, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 514, 503, 492, 521, 529, 491, 527, 491, 491, 464, 540, 515, 501, 520, 466, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 492, 536, 531, 540, 518, 515, 456, 503, 533, 501, 472, 505, 551, 549, 526, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 487, 527, 516, 499, 477, 535, 555, 458, 502, 494, 507, 502, 501, 562, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8309, 8146, 8318, 8077, 8157, 8242, 8096] avg=8192.0000 max=8318 max_rank=3 min=8077 min_rank=4 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32572 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[521, 520, 517, 522, 514, 479, 519, 474, 523, 521, 583, 498, 506, 470, 500, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 494, 507, 523, 555, 513, 487, 522, 550, 488, 520, 496, 559, 535, 515, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 498, 521, 476, 499, 494, 544, 499, 517, 514, 530, 498, 496, 489, 505, 490, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 486, 495, 520, 485, 541, 516, 497, 495, 464, 504, 509, 526, 513, 525, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 501, 520, 504, 511, 555, 533, 486, 544, 505, 539, 539, 553, 526, 512, 494, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 492, 538, 462, 472, 519, 529, 506, 507, 522, 509, 497, 538, 486, 494, 545, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 522, 511, 494, 479, 530, 456, 525, 533, 504, 504, 506, 506, 529, 483, 494, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 511, 538, 545, 502, 504, 491, 504, 543, 476, 511, 470, 495, 541, 539, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8215, 8335, 8116, 8079, 8342, 8153, 8073, 8223] avg=8192.0000 max=8342 max_rank=4 min=8073 min_rank=6 imbalance_ratio=1.018311 max_load_reduction=57194 max_load_reduction_pct=0.872711 moved_to_echo=32791 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[549, 495, 540, 499, 561, 534, 529, 500, 506, 547, 496, 497, 517, 494, 532, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 508, 473, 476, 503, 503, 549, 513, 523, 503, 497, 520, 496, 537, 541, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 508, 511, 504, 541, 492, 483, 513, 531, 497, 486, 487, 494, 542, 516, 503, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 481, 507, 509, 519, 531, 527, 517, 473, 506, 535, 522, 492, 511, 505, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 507, 506, 470, 516, 533, 505, 483, 513, 543, 520, 500, 541, 560, 467, 506, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 483, 520, 524, 488, 500, 490, 492, 525, 526, 507, 487, 496, 483, 532, 517, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 522, 520, 504, 519, 546, 536, 500, 478, 514, 488, 511, 513, 519, 507, 548, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 501, 513, 506, 514, 521, 501, 549, 515, 536, 541, 520, 499, 510, 515, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8331, 8109, 8149, 8141, 8187, 8105, 8277, 8237] avg=8192.0000 max=8331 max_rank=0 min=8105 min_rank=5 imbalance_ratio=1.016968 max_load_reduction=57205 max_load_reduction_pct=0.872879 moved_to_echo=32806 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[521, 495, 481, 489, 488, 496, 523, 502, 519, 493, 546, 512, 513, 502, 533, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 547, 532, 540, 457, 505, 498, 484, 517, 498, 522, 572, 498, 490, 455, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 546, 519, 510, 506, 496, 494, 520, 481, 494, 517, 534, 523, 542, 565, 512, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 543, 495, 469, 478, 522, 502, 484, 515, 502, 537, 502, 537, 521, 549, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 489, 511, 518, 484, 530, 505, 515, 523, 548, 496, 486, 523, 530, 531, 502, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 450, 517, 536, 512, 506, 506, 510, 490, 523, 526, 512, 512, 523, 531, 496, 496, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 533, 522, 484, 490, 535, 558, 509, 533, 532, 489, 482, 508, 512, 497, 460, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 510, 544, 527, 515, 548, 536, 501, 499, 492, 546, 503, 465, 487, 500, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8141, 8208, 8249, 8149, 8235, 8180, 8152, 8222] avg=8192.0000 max=8249 max_rank=2 min=8141 min_rank=0 imbalance_ratio=1.006958 max_load_reduction=57287 max_load_reduction_pct=0.874130 moved_to_echo=32789 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[557, 510, 520, 473, 519, 563, 526, 501, 497, 511, 508, 502, 509, 511, 504, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 499, 518, 494, 530, 483, 523, 494, 506, 521, 537, 544, 539, 498, 529, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 515, 521, 507, 539, 490, 526, 504, 512, 473, 532, 489, 517, 501, 517, 544, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 475, 500, 532, 479, 539, 519, 496, 527, 504, 557, 503, 507, 504, 521, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 517, 477, 530, 461, 503, 549, 526, 523, 503, 537, 499, 489, 547, 484, 541, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 509, 478, 502, 487, 463, 536, 522, 525, 497, 517, 493, 504, 536, 506, 508, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 518, 535, 497, 524, 550, 522, 533, 532, 505, 503, 472, 519, 500, 513, 496, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 529, 495, 530, 523, 541, 513, 531, 508, 495, 493, 495, 490, 511, 512, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8225, 8173, 8169, 8229, 8138, 8219, 8182] avg=8192.0000 max=8229 max_rank=4 min=8138 min_rank=5 imbalance_ratio=1.004517 max_load_reduction=57307 max_load_reduction_pct=0.874435 moved_to_echo=32768 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[558, 527, 512, 475, 516, 560, 489, 484, 524, 501, 497, 531, 506, 494, 489, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 538, 476, 512, 496, 480, 506, 520, 525, 498, 532, 509, 519, 561, 559, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 474, 512, 463, 523, 526, 504, 515, 444, 552, 574, 534, 470, 534, 479, 548, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 518, 557, 504, 534, 584, 465, 520, 519, 510, 506, 531, 529, 486, 489, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 448, 523, 474, 536, 519, 498, 511, 496, 496, 507, 514, 531, 529, 483, 491, 504, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 514, 503, 492, 521, 529, 491, 527, 491, 491, 464, 540, 515, 501, 520, 466, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 492, 536, 531, 540, 518, 515, 456, 503, 533, 501, 472, 505, 551, 549, 526, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 487, 527, 516, 499, 477, 535, 555, 458, 502, 494, 507, 502, 501, 562, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8309, 8146, 8318, 8077, 8157, 8242, 8096] avg=8192.0000 max=8318 max_rank=3 min=8077 min_rank=4 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32572 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[511, 505, 515, 502, 508, 504, 529, 570, 490, 466, 535, 538, 493, 548, 516, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 491, 491, 493, 529, 523, 518, 496, 499, 544, 504, 512, 509, 479, 526, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 516, 497, 477, 488, 511, 489, 542, 512, 548, 506, 504, 537, 506, 531, 510, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 501, 535, 528, 503, 496, 501, 539, 476, 494, 518, 526, 543, 504, 518, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 478, 510, 505, 505, 493, 517, 540, 521, 515, 548, 482, 549, 530, 495, 507, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 518, 528, 515, 491, 500, 488, 500, 479, 517, 513, 558, 523, 489, 507, 505, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 501, 503, 510, 505, 535, 489, 491, 506, 499, 523, 493, 523, 457, 539, 524, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 460, 530, 531, 537, 532, 511, 526, 532, 532, 513, 491, 517, 518, 540, 511, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8121, 8195, 8196, 8239, 8151, 8131, 8291] avg=8192.0000 max=8291 max_rank=7 min=8121 min_rank=1 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32812 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 3968, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4352, 4224, 4096, 4224, 4224, 4096, 3968, 4224, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4352, 4224, 4096, 4224, 4224, 4096, 3968, 4224, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 3968, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[518, 521, 515, 497, 520, 523, 510, 526, 493, 560, 516, 522, 490, 534, 501, 485, 540, 518, 526, 525, 548, 512, 529, 517, 492, 492, 532, 505, 527, 497, 494, 521, 506, 535, 463, 534, 509, 530, 507, 525, 490, 526, 465, 512, 504, 543, 520, 510, 467, 481, 467, 533, 494, 507, 497, 497, 476, 510, 471, 513, 565, 528, 552, 520, 514, 503, 536, 521, 521, 530, 538, 513, 543, 520, 495, 517, 524, 492, 517, 512, 532, 501, 493, 506, 510, 492, 506, 504, 508, 520, 526, 487, 526, 501, 520, 524, 510, 529, 518, 498, 530, 528, 473, 491, 511, 583, 494, 546, 505, 493, 526, 483, 518, 490, 536, 489, 520, 487, 514, 507, 510, 502, 492, 510, 503, 484, 524, 517] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[506, 539, 488, 522, 517, 527, 501, 520, 546, 506, 510, 559, 509, 471, 564, 508, 519, 482, 531, 500, 515, 487, 517, 506, 511, 516, 510, 567, 512, 507, 499, 491, 531, 537, 525, 480, 471, 485, 494, 509, 498, 494, 515, 516, 477, 515, 502, 518, 489, 528, 492, 526, 528, 499, 500, 515, 505, 478, 459, 503, 502, 536, 518, 551, 493, 496, 548, 503, 541, 528, 493, 506, 467, 532, 515, 502, 502, 535, 506, 507, 500, 501, 490, 518, 535, 497, 519, 521, 510, 492, 486, 533, 537, 510, 501, 535, 539, 519, 510, 512, 536, 536, 524, 517, 487, 511, 517, 496, 486, 542, 536, 478, 544, 543, 553, 513, 483, 482, 497, 551, 544, 540, 495, 512, 514, 492, 507, 502] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[540, 498, 490, 486, 489, 553, 521, 503, 511, 496, 523, 527, 512, 507, 496, 515, 490, 471, 501, 502, 521, 492, 524, 523, 529, 495, 520, 514, 483, 485, 472, 501, 483, 504, 533, 529, 536, 539, 530, 540, 515, 550, 492, 509, 523, 532, 514, 529, 497, 514, 504, 545, 513, 525, 497, 494, 467, 480, 544, 557, 496, 540, 514, 524, 471, 509, 502, 533, 530, 524, 499, 490, 537, 517, 502, 492, 518, 530, 475, 505, 507, 505, 541, 490, 528, 508, 534, 501, 530, 489, 495, 516, 530, 497, 547, 525, 491, 515, 496, 512, 491, 555, 565, 499, 526, 458, 562, 537, 509, 496, 536, 513, 499, 527, 516, 459, 495, 541, 511, 496, 529, 564, 519, 556, 493, 492, 471, 471] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[520, 516, 502, 520, 486, 477, 482, 514, 507, 485, 531, 497, 556, 536, 529, 508, 508, 486, 484, 528, 497, 502, 520, 536, 503, 526, 530, 539, 506, 509, 498, 499, 475, 537, 523, 507, 484, 495, 500, 502, 519, 494, 540, 522, 495, 519, 509, 533, 509, 530, 519, 479, 512, 490, 473, 515, 537, 523, 482, 522, 499, 519, 559, 531, 503, 480, 527, 511, 514, 506, 508, 487, 526, 524, 505, 493, 542, 535, 494, 508, 543, 537, 530, 525, 501, 535, 502, 514, 482, 489, 514, 507, 483, 526, 506, 504, 523, 504, 524, 461, 552, 497, 503, 495, 516, 543, 542, 486, 470, 543, 508, 577, 526, 522, 520, 516, 520, 539, 499, 485, 524, 482, 506, 490, 527, 547, 557, 481] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[471, 485, 525, 533, 495, 489, 487, 518, 553, 537, 543, 501, 483, 487, 533, 477, 506, 560, 504, 491, 540, 512, 565, 523, 560, 497, 515, 521, 511, 551, 500, 508, 544, 494, 497, 531, 496, 489, 467, 543, 538, 488, 564, 565, 507, 507, 480, 531, 529, 515, 505, 510, 489, 536, 514, 531, 470, 524, 514, 492, 528, 505, 502, 515, 528, 473, 494, 534, 549, 493, 502, 482, 522, 546, 536, 487, 501, 497, 526, 494, 509, 532, 524, 459, 511, 484, 511, 506, 481, 479, 507, 527, 493, 512, 529, 510, 534, 485, 506, 470, 514, 490, 508, 491, 532, 489, 503, 498, 516, 572, 528, 494, 512, 490, 502, 540, 483, 534, 525, 546, 504, 528, 530, 520, 538, 515, 514, 486] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[506, 539, 488, 522, 517, 527, 501, 520, 546, 506, 510, 559, 509, 471, 564, 508, 519, 482, 531, 500, 515, 487, 517, 506, 511, 516, 510, 567, 512, 507, 499, 491, 531, 537, 525, 480, 471, 485, 494, 509, 498, 494, 515, 516, 477, 515, 502, 518, 489, 528, 492, 526, 528, 499, 500, 515, 505, 478, 459, 503, 502, 536, 518, 551, 493, 496, 548, 503, 541, 528, 493, 506, 467, 532, 515, 502, 502, 535, 506, 507, 500, 501, 490, 518, 535, 497, 519, 521, 510, 492, 486, 533, 537, 510, 501, 535, 539, 519, 510, 512, 536, 536, 524, 517, 487, 511, 517, 496, 486, 542, 536, 478, 544, 543, 553, 513, 483, 482, 497, 551, 544, 540, 495, 512, 514, 492, 507, 502] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[536, 542, 502, 509, 513, 513, 500, 556, 492, 480, 535, 504, 491, 483, 495, 529, 523, 503, 502, 512, 518, 537, 485, 485, 494, 515, 489, 532, 489, 502, 493, 534, 480, 508, 476, 505, 503, 477, 515, 516, 514, 515, 565, 500, 520, 541, 493, 481, 504, 490, 504, 505, 534, 539, 499, 529, 510, 506, 516, 524, 527, 515, 528, 512, 491, 522, 492, 539, 507, 517, 558, 519, 495, 494, 498, 514, 528, 529, 514, 563, 525, 518, 497, 448, 482, 521, 521, 552, 542, 515, 522, 503, 518, 485, 579, 504, 523, 486, 520, 494, 520, 520, 494, 464, 536, 518, 498, 540, 490, 552, 502, 529, 514, 526, 503, 526, 501, 516, 526, 513, 483, 505, 528, 512, 501, 522, 521, 497] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[540, 498, 490, 486, 489, 553, 521, 503, 511, 496, 523, 527, 512, 507, 496, 515, 490, 471, 501, 502, 521, 492, 524, 523, 529, 495, 520, 514, 483, 485, 472, 501, 483, 504, 533, 529, 536, 539, 530, 540, 515, 550, 492, 509, 523, 532, 514, 529, 497, 514, 504, 545, 513, 525, 497, 494, 467, 480, 544, 557, 496, 540, 514, 524, 471, 509, 502, 533, 530, 524, 499, 490, 537, 517, 502, 492, 518, 530, 475, 505, 507, 505, 541, 490, 528, 508, 534, 501, 530, 489, 495, 516, 530, 497, 547, 525, 491, 515, 496, 512, 491, 555, 565, 499, 526, 458, 562, 537, 509, 496, 536, 513, 499, 527, 516, 459, 495, 541, 511, 496, 529, 564, 519, 556, 493, 492, 471, 471] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[461, 473, 500, 490, 534, 487, 494, 518, 516, 553, 496, 515, 498, 517, 465, 548, 509, 488, 521, 508, 529, 523, 491, 497, 515, 472, 506, 580, 541, 512, 495, 516, 541, 522, 505, 535, 513, 500, 501, 532, 521, 538, 514, 513, 519, 487, 524, 501, 540, 533, 516, 501, 519, 499, 506, 533, 533, 501, 462, 517, 543, 490, 515, 502, 507, 521, 491, 500, 517, 512, 507, 553, 488, 532, 530, 473, 517, 534, 474, 504, 500, 505, 483, 519, 485, 478, 510, 465, 548, 505, 552, 523, 536, 546, 555, 514, 515, 509, 506, 494, 536, 480, 492, 494, 498, 517, 541, 520, 580, 495, 525, 527, 504, 544, 520, 529, 528, 509, 457, 506, 511, 502, 529, 513, 510, 506, 508, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[471, 485, 525, 533, 495, 489, 487, 518, 553, 537, 543, 501, 483, 487, 533, 477, 506, 560, 504, 491, 540, 512, 565, 523, 560, 497, 515, 521, 511, 551, 500, 508, 544, 494, 497, 531, 496, 489, 467, 543, 538, 488, 564, 565, 507, 507, 480, 531, 529, 515, 505, 510, 489, 536, 514, 531, 470, 524, 514, 492, 528, 505, 502, 515, 528, 473, 494, 534, 549, 493, 502, 482, 522, 546, 536, 487, 501, 497, 526, 494, 509, 532, 524, 459, 511, 484, 511, 506, 481, 479, 507, 527, 493, 512, 529, 510, 534, 485, 506, 470, 514, 490, 508, 491, 532, 489, 503, 498, 516, 572, 528, 494, 512, 490, 502, 540, 483, 534, 525, 546, 504, 528, 530, 520, 538, 515, 514, 486] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[504, 512, 475, 475, 462, 442, 527, 547, 552, 525, 538, 550, 511, 521, 521, 470, 496, 480, 509, 523, 498, 515, 517, 544, 506, 522, 522, 475, 486, 526, 497, 523, 537, 507, 516, 507, 530, 500, 553, 498, 474, 546, 551, 535, 507, 524, 502, 539, 524, 544, 527, 503, 491, 575, 503, 503, 545, 567, 493, 538, 498, 507, 498, 492, 512, 487, 503, 511, 516, 481, 518, 489, 538, 492, 514, 552, 519, 503, 504, 526, 502, 507, 525, 531, 526, 519, 504, 514, 537, 471, 503, 477, 488, 513, 509, 509, 488, 482, 531, 499, 520, 505, 497, 506, 497, 544, 459, 513, 538, 557, 524, 517, 507, 475, 545, 483, 503, 481, 525, 513, 535, 528, 491, 542, 484, 523, 524, 495] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[461, 473, 500, 490, 534, 487, 494, 518, 516, 553, 496, 515, 498, 517, 465, 548, 509, 488, 521, 508, 529, 523, 491, 497, 515, 472, 506, 580, 541, 512, 495, 516, 541, 522, 505, 535, 513, 500, 501, 532, 521, 538, 514, 513, 519, 487, 524, 501, 540, 533, 516, 501, 519, 499, 506, 533, 533, 501, 462, 517, 543, 490, 515, 502, 507, 521, 491, 500, 517, 512, 507, 553, 488, 532, 530, 473, 517, 534, 474, 504, 500, 505, 483, 519, 485, 478, 510, 465, 548, 505, 552, 523, 536, 546, 555, 514, 515, 509, 506, 494, 536, 480, 492, 494, 498, 517, 541, 520, 580, 495, 525, 527, 504, 544, 520, 529, 528, 509, 457, 506, 511, 502, 529, 513, 510, 506, 508, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[518, 521, 515, 497, 520, 523, 510, 526, 493, 560, 516, 522, 490, 534, 501, 485, 540, 518, 526, 525, 548, 512, 529, 517, 492, 492, 532, 505, 527, 497, 494, 521, 506, 535, 463, 534, 509, 530, 507, 525, 490, 526, 465, 512, 504, 543, 520, 510, 467, 481, 467, 533, 494, 507, 497, 497, 476, 510, 471, 513, 565, 528, 552, 520, 514, 503, 536, 521, 521, 530, 538, 513, 543, 520, 495, 517, 524, 492, 517, 512, 532, 501, 493, 506, 510, 492, 506, 504, 508, 520, 526, 487, 526, 501, 520, 524, 510, 529, 518, 498, 530, 528, 473, 491, 511, 583, 494, 546, 505, 493, 526, 483, 518, 490, 536, 489, 520, 487, 514, 507, 510, 502, 492, 510, 503, 484, 524, 517] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[506, 539, 488, 522, 517, 527, 501, 520, 546, 506, 510, 559, 509, 471, 564, 508, 20, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 482, 531, 500, 515, 487, 517, 506, 511, 516, 510, 567, 512, 507, 499, 491, 5, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 537, 525, 480, 471, 485, 494, 509, 498, 494, 495, 516, 477, 515, 502, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 528, 492, 526, 528, 499, 500, 515, 505, 478, 459, 503, 502, 536, 516, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 496, 548, 503, 541, 528, 493, 506, 467, 532, 515, 502, 502, 535, 506, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 501, 490, 518, 535, 497, 519, 521, 510, 492, 486, 533, 537, 510, 501, 535, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 519, 510, 512, 536, 536, 524, 517, 487, 511, 517, 496, 486, 524, 536, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 543, 552, 513, 483, 482, 497, 551, 544, 540, 495, 512, 514, 492, 507, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8315, 8178, 8047, 8127, 8174, 8196, 8228, 8271] avg=8192.0000 max=8315 max_rank=0 min=8047 min_rank=2 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32869 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[471, 485, 525, 533, 495, 489, 487, 518, 553, 537, 543, 501, 483, 487, 533, 477, 21, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 560, 504, 491, 540, 512, 565, 523, 560, 497, 515, 521, 511, 551, 500, 508, 6, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 494, 497, 531, 496, 489, 467, 543, 538, 488, 543, 565, 507, 507, 480, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 515, 505, 510, 489, 536, 514, 531, 470, 524, 514, 492, 528, 505, 500, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 473, 494, 534, 549, 493, 502, 482, 522, 546, 536, 487, 501, 497, 526, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 532, 524, 459, 511, 484, 511, 506, 481, 479, 507, 527, 493, 512, 529, 510, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 485, 506, 470, 514, 490, 508, 491, 532, 489, 503, 498, 516, 553, 528, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 490, 501, 540, 483, 534, 525, 546, 504, 528, 530, 520, 538, 515, 514, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8140, 8373, 8220, 8177, 8164, 8085, 8111, 8266] avg=8192.0000 max=8373 max_rank=1 min=8085 min_rank=5 imbalance_ratio=1.022095 max_load_reduction=57163 max_load_reduction_pct=0.872238 moved_to_echo=32626 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[518, 521, 515, 497, 520, 523, 510, 526, 493, 560, 516, 522, 490, 534, 501, 485, 18, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 518, 526, 525, 548, 512, 529, 517, 492, 492, 532, 505, 527, 497, 494, 521, 5, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 535, 463, 534, 509, 530, 507, 525, 490, 526, 447, 512, 504, 543, 520, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 481, 467, 533, 494, 507, 497, 497, 476, 510, 471, 513, 565, 528, 550, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 503, 536, 521, 521, 530, 538, 513, 543, 520, 495, 517, 524, 492, 517, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 501, 493, 506, 510, 492, 506, 504, 508, 520, 526, 487, 526, 501, 520, 524, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 529, 518, 498, 530, 528, 473, 491, 511, 583, 494, 546, 505, 476, 526, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 490, 535, 489, 520, 487, 514, 507, 510, 502, 492, 510, 503, 484, 524, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8251, 8283, 8161, 8076, 8296, 8166, 8201, 8102] avg=8192.0000 max=8296 max_rank=4 min=8076 min_rank=3 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32765 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[504, 512, 475, 475, 462, 442, 527, 547, 552, 525, 538, 550, 511, 521, 521, 470, 21, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 480, 509, 523, 498, 515, 517, 544, 506, 522, 522, 475, 486, 526, 497, 523, 5, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 507, 516, 507, 530, 500, 553, 498, 474, 546, 530, 535, 507, 524, 502, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 544, 527, 503, 491, 575, 503, 503, 545, 567, 493, 538, 498, 507, 496, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 487, 503, 511, 516, 481, 518, 489, 538, 492, 514, 552, 519, 503, 504, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 507, 525, 531, 526, 519, 504, 514, 537, 471, 503, 477, 488, 513, 509, 509, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 482, 531, 499, 520, 505, 497, 506, 497, 544, 459, 513, 538, 539, 524, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 475, 544, 483, 503, 481, 525, 513, 535, 528, 491, 542, 484, 523, 524, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8155, 8147, 8305, 8306, 8165, 8146, 8159, 8153] avg=8192.0000 max=8306 max_rank=3 min=8146 min_rank=5 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32623 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[536, 542, 502, 509, 513, 513, 500, 556, 492, 480, 535, 504, 491, 483, 495, 529, 24, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 503, 502, 512, 518, 537, 485, 485, 494, 515, 489, 532, 489, 502, 493, 534, 9, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 508, 476, 505, 503, 477, 515, 516, 514, 515, 541, 500, 520, 541, 493, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 490, 504, 505, 534, 539, 499, 529, 510, 506, 516, 524, 527, 515, 522, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 522, 492, 539, 507, 517, 558, 519, 495, 494, 498, 514, 528, 529, 514, 563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 518, 497, 448, 482, 521, 521, 552, 542, 515, 522, 503, 518, 485, 579, 504, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 486, 520, 494, 520, 520, 494, 464, 536, 518, 498, 540, 490, 521, 502, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 526, 497, 526, 501, 516, 526, 513, 483, 505, 528, 512, 501, 522, 521, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8211, 8134, 8085, 8236, 8280, 8247, 8155, 8188] avg=8192.0000 max=8280 max_rank=4 min=8085 min_rank=2 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32870 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[520, 516, 502, 520, 486, 477, 482, 514, 507, 485, 531, 497, 556, 536, 529, 508, 21, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 486, 484, 528, 497, 502, 520, 536, 503, 526, 530, 539, 506, 509, 498, 499, 5, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 537, 523, 507, 484, 495, 500, 502, 519, 494, 519, 522, 495, 519, 509, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 530, 519, 479, 512, 490, 473, 515, 537, 523, 482, 522, 499, 519, 557, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 480, 527, 511, 514, 506, 508, 487, 526, 524, 505, 493, 542, 535, 494, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 537, 530, 525, 501, 535, 502, 514, 482, 489, 514, 507, 483, 526, 506, 504, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 504, 524, 461, 552, 497, 503, 495, 516, 543, 542, 486, 470, 525, 508, 577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 522, 519, 516, 520, 539, 499, 485, 524, 482, 506, 490, 527, 547, 557, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8179, 8133, 8197, 8163, 8209, 8226, 8240] avg=8192.0000 max=8240 max_rank=7 min=8133 min_rank=2 imbalance_ratio=1.005859 max_load_reduction=57296 max_load_reduction_pct=0.874268 moved_to_echo=32838 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[540, 498, 490, 486, 489, 553, 521, 503, 511, 496, 523, 527, 512, 507, 496, 515, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 471, 501, 502, 521, 492, 524, 523, 529, 495, 520, 514, 483, 485, 472, 501, 5, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 504, 533, 529, 536, 539, 530, 540, 515, 550, 473, 509, 523, 532, 514, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 514, 504, 545, 513, 525, 497, 494, 467, 480, 544, 557, 496, 540, 512, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 509, 502, 533, 530, 524, 499, 490, 537, 517, 502, 492, 518, 530, 475, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 505, 541, 490, 528, 508, 534, 501, 530, 489, 495, 516, 530, 497, 547, 525, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 515, 496, 512, 491, 555, 565, 499, 526, 458, 562, 537, 509, 479, 536, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 527, 515, 459, 495, 541, 511, 496, 529, 564, 519, 556, 493, 492, 471, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8031, 8339, 8209, 8134, 8253, 8244, 8138] avg=8192.0000 max=8339 max_rank=2 min=8031 min_rank=1 imbalance_ratio=1.017944 max_load_reduction=57197 max_load_reduction_pct=0.872757 moved_to_echo=32769 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[461, 473, 500, 490, 534, 487, 494, 518, 516, 553, 496, 515, 498, 517, 465, 548, 20, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 488, 521, 508, 529, 523, 491, 497, 515, 472, 506, 580, 541, 512, 495, 516, 5, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 522, 505, 535, 513, 500, 501, 532, 521, 538, 494, 513, 519, 487, 524, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 533, 516, 501, 519, 499, 506, 533, 533, 501, 462, 517, 543, 490, 513, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 521, 491, 500, 517, 512, 507, 553, 488, 532, 530, 473, 517, 534, 474, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 505, 483, 519, 485, 478, 510, 465, 548, 505, 552, 523, 536, 546, 555, 514, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 509, 506, 494, 536, 480, 492, 494, 498, 517, 541, 520, 580, 478, 525, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 544, 519, 529, 528, 509, 457, 506, 511, 502, 529, 513, 510, 506, 508, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8087, 8211, 8246, 8208, 8160, 8234, 8212, 8178] avg=8192.0000 max=8246 max_rank=2 min=8087 min_rank=0 imbalance_ratio=1.006592 max_load_reduction=57290 max_load_reduction_pct=0.874176 moved_to_echo=32784 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[504, 512, 475, 475, 462, 442, 527, 547, 552, 525, 538, 550, 511, 521, 521, 470, 21, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 480, 509, 523, 498, 515, 517, 544, 506, 522, 522, 475, 486, 526, 497, 523, 5, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 507, 516, 507, 530, 500, 553, 498, 474, 546, 530, 535, 507, 524, 502, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 544, 527, 503, 491, 575, 503, 503, 545, 567, 493, 538, 498, 507, 496, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 487, 503, 511, 516, 481, 518, 489, 538, 492, 514, 552, 519, 503, 504, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 507, 525, 531, 526, 519, 504, 514, 537, 471, 503, 477, 488, 513, 509, 509, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 482, 531, 499, 520, 505, 497, 506, 497, 544, 459, 513, 538, 539, 524, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 475, 544, 483, 503, 481, 525, 513, 535, 528, 491, 542, 484, 523, 524, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8155, 8147, 8305, 8306, 8165, 8146, 8159, 8153] avg=8192.0000 max=8306 max_rank=3 min=8146 min_rank=5 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32623 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[518, 521, 515, 497, 520, 523, 510, 526, 493, 560, 516, 522, 490, 534, 501, 485, 18, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 518, 526, 525, 548, 512, 529, 517, 492, 492, 532, 505, 527, 497, 494, 521, 5, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 535, 463, 534, 509, 530, 507, 525, 490, 526, 447, 512, 504, 543, 520, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 481, 467, 533, 494, 507, 497, 497, 476, 510, 471, 513, 565, 528, 550, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 503, 536, 521, 521, 530, 538, 513, 543, 520, 495, 517, 524, 492, 517, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 501, 493, 506, 510, 492, 506, 504, 508, 520, 526, 487, 526, 501, 520, 524, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 529, 518, 498, 530, 528, 473, 491, 511, 583, 494, 546, 505, 476, 526, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 490, 535, 489, 520, 487, 514, 507, 510, 502, 492, 510, 503, 484, 524, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8251, 8283, 8161, 8076, 8296, 8166, 8201, 8102] avg=8192.0000 max=8296 max_rank=4 min=8076 min_rank=3 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32765 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[536, 542, 502, 509, 513, 513, 500, 556, 492, 480, 535, 504, 491, 483, 495, 529, 24, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 503, 502, 512, 518, 537, 485, 485, 494, 515, 489, 532, 489, 502, 493, 534, 9, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 508, 476, 505, 503, 477, 515, 516, 514, 515, 541, 500, 520, 541, 493, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 490, 504, 505, 534, 539, 499, 529, 510, 506, 516, 524, 527, 515, 522, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 522, 492, 539, 507, 517, 558, 519, 495, 494, 498, 514, 528, 529, 514, 563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 518, 497, 448, 482, 521, 521, 552, 542, 515, 522, 503, 518, 485, 579, 504, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 486, 520, 494, 520, 520, 494, 464, 536, 518, 498, 540, 490, 521, 502, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 526, 497, 526, 501, 516, 526, 513, 483, 505, 528, 512, 501, 522, 521, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8211, 8134, 8085, 8236, 8280, 8247, 8155, 8188] avg=8192.0000 max=8280 max_rank=4 min=8085 min_rank=2 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32870 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[471, 485, 525, 533, 495, 489, 487, 518, 553, 537, 543, 501, 483, 487, 533, 477, 21, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 560, 504, 491, 540, 512, 565, 523, 560, 497, 515, 521, 511, 551, 500, 508, 6, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 494, 497, 531, 496, 489, 467, 543, 538, 488, 543, 565, 507, 507, 480, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 515, 505, 510, 489, 536, 514, 531, 470, 524, 514, 492, 528, 505, 500, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 473, 494, 534, 549, 493, 502, 482, 522, 546, 536, 487, 501, 497, 526, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 532, 524, 459, 511, 484, 511, 506, 481, 479, 507, 527, 493, 512, 529, 510, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 485, 506, 470, 514, 490, 508, 491, 532, 489, 503, 498, 516, 553, 528, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 490, 501, 540, 483, 534, 525, 546, 504, 528, 530, 520, 538, 515, 514, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8140, 8373, 8220, 8177, 8164, 8085, 8111, 8266] avg=8192.0000 max=8373 max_rank=1 min=8085 min_rank=5 imbalance_ratio=1.022095 max_load_reduction=57163 max_load_reduction_pct=0.872238 moved_to_echo=32626 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4352, 4096, 4096, 3968, 4096, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 3968, 4224, 4224, 4096, 3968, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 3968, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4352, 4096, 4096, 3968, 4096, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 3968, 4224, 4096, 4224, 4096, 4352, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 3968, 4224, 4224, 4096, 3968, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[475, 501, 497, 536, 500, 461, 497, 508, 544, 524, 545, 544, 521, 518, 483, 556, 542, 518, 515, 547, 493, 502, 495, 499, 527, 521, 478, 508, 515, 475, 477, 464, 550, 531, 466, 492, 508, 498, 500, 557, 532, 523, 546, 506, 505, 501, 534, 522, 503, 530, 477, 485, 514, 478, 529, 518, 519, 502, 529, 521, 494, 544, 520, 558, 500, 545, 498, 519, 514, 508, 481, 505, 511, 514, 544, 484, 531, 510, 505, 528, 489, 519, 503, 501, 524, 502, 515, 530, 484, 544, 505, 566, 491, 520, 506, 500, 524, 525, 518, 475, 485, 547, 526, 498, 523, 511, 499, 511, 516, 480, 527, 484, 561, 487, 510, 514, 517, 529, 499, 516, 508, 496, 511, 530, 521, 466, 515, 533] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[496, 531, 537, 493, 514, 514, 511, 531, 495, 498, 510, 495, 521, 499, 540, 542, 498, 528, 489, 491, 510, 512, 497, 515, 487, 522, 520, 498, 516, 542, 499, 508, 507, 545, 516, 500, 553, 501, 496, 523, 504, 512, 524, 483, 510, 479, 511, 547, 528, 489, 495, 517, 542, 507, 489, 527, 526, 507, 480, 528, 476, 486, 553, 522, 521, 519, 478, 499, 483, 519, 467, 537, 526, 511, 571, 506, 529, 551, 516, 534, 516, 498, 480, 552, 526, 533, 512, 530, 520, 496, 543, 484, 524, 519, 494, 520, 503, 517, 518, 535, 505, 500, 517, 504, 517, 524, 530, 512, 502, 515, 502, 501, 519, 486, 483, 533, 481, 515, 524, 501, 466, 507, 512, 511, 515, 526, 514, 485] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[526, 533, 498, 520, 515, 507, 466, 510, 521, 496, 500, 531, 535, 499, 545, 536, 512, 457, 510, 528, 497, 496, 510, 489, 493, 532, 539, 526, 549, 474, 514, 518, 521, 528, 455, 530, 516, 538, 481, 520, 492, 496, 507, 517, 516, 516, 502, 529, 510, 514, 509, 566, 535, 519, 528, 495, 491, 513, 518, 491, 508, 492, 505, 512, 500, 518, 525, 534, 508, 521, 508, 543, 480, 535, 505, 505, 487, 510, 496, 475, 497, 527, 510, 519, 538, 506, 527, 520, 479, 480, 558, 527, 489, 507, 481, 552, 507, 505, 549, 531, 520, 501, 525, 477, 493, 488, 511, 521, 524, 518, 507, 531, 484, 491, 535, 526, 516, 518, 536, 524, 499, 512, 483, 517, 494, 510, 511, 553] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[497, 517, 498, 508, 550, 505, 525, 540, 499, 510, 487, 513, 534, 529, 522, 509, 496, 495, 487, 566, 519, 476, 504, 491, 511, 472, 493, 535, 501, 525, 537, 517, 522, 528, 483, 506, 495, 509, 513, 495, 507, 505, 490, 475, 497, 509, 474, 527, 515, 495, 498, 514, 519, 539, 483, 502, 547, 521, 495, 524, 507, 500, 534, 531, 502, 514, 516, 509, 504, 523, 553, 492, 507, 500, 525, 553, 538, 518, 505, 497, 527, 529, 541, 506, 530, 516, 486, 522, 520, 480, 503, 505, 556, 533, 485, 515, 507, 525, 530, 515, 531, 548, 532, 538, 541, 489, 507, 506, 519, 489, 505, 490, 561, 473, 499, 497, 499, 519, 532, 490, 477, 500, 511, 537, 519, 515, 488, 510] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[542, 493, 489, 502, 478, 548, 499, 484, 491, 519, 491, 540, 478, 506, 524, 507, 519, 535, 490, 504, 521, 518, 494, 508, 477, 523, 499, 492, 550, 444, 488, 490, 502, 526, 541, 507, 523, 528, 493, 568, 541, 522, 493, 510, 525, 488, 523, 518, 524, 532, 567, 518, 526, 506, 529, 472, 488, 537, 557, 487, 528, 535, 506, 476, 533, 502, 551, 537, 541, 486, 484, 536, 499, 554, 514, 483, 521, 511, 521, 488, 509, 527, 510, 504, 471, 504, 486, 537, 502, 520, 533, 505, 500, 538, 551, 528, 474, 500, 516, 473, 489, 500, 504, 516, 544, 516, 550, 525, 502, 521, 508, 553, 524, 517, 486, 509, 522, 510, 506, 511, 521, 504, 479, 494, 504, 522, 513, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[485, 471, 510, 500, 513, 498, 552, 511, 513, 514, 513, 548, 500, 476, 520, 497, 491, 511, 497, 481, 512, 531, 503, 509, 528, 496, 509, 498, 515, 512, 480, 497, 493, 529, 515, 478, 497, 513, 473, 544, 516, 534, 511, 499, 526, 517, 546, 521, 515, 553, 514, 522, 513, 508, 506, 497, 487, 527, 512, 544, 510, 516, 549, 488, 505, 504, 469, 509, 508, 528, 540, 484, 522, 503, 501, 518, 522, 564, 512, 539, 503, 523, 528, 534, 487, 500, 497, 525, 531, 516, 517, 532, 491, 475, 499, 552, 490, 538, 526, 536, 529, 496, 519, 503, 504, 524, 529, 505, 523, 522, 513, 499, 524, 475, 497, 499, 561, 505, 473, 498, 514, 524, 524, 518, 509, 532, 521, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[497, 519, 534, 522, 486, 504, 485, 492, 518, 510, 542, 527, 544, 474, 497, 518, 525, 521, 509, 550, 494, 505, 522, 508, 512, 525, 502, 554, 510, 537, 523, 517, 516, 505, 488, 497, 528, 495, 492, 521, 526, 515, 506, 504, 513, 494, 521, 518, 514, 532, 522, 522, 521, 528, 543, 505, 518, 532, 499, 498, 503, 510, 522, 501, 523, 516, 488, 521, 512, 528, 514, 459, 519, 518, 481, 514, 516, 547, 526, 523, 475, 504, 540, 463, 501, 480, 514, 512, 530, 498, 489, 530, 525, 478, 520, 483, 527, 508, 498, 521, 549, 493, 470, 490, 536, 533, 503, 532, 487, 483, 493, 498, 535, 513, 529, 523, 550, 539, 533, 464, 567, 525, 505, 480, 483, 464, 533, 533] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[497, 517, 498, 508, 550, 505, 525, 540, 499, 510, 487, 513, 534, 529, 522, 509, 496, 495, 487, 566, 519, 476, 504, 491, 511, 472, 493, 535, 501, 525, 537, 517, 522, 528, 483, 506, 495, 509, 513, 495, 507, 505, 490, 475, 497, 509, 474, 527, 515, 495, 498, 514, 519, 539, 483, 502, 547, 521, 495, 524, 507, 500, 534, 531, 502, 514, 516, 509, 504, 523, 553, 492, 507, 500, 525, 553, 538, 518, 505, 497, 527, 529, 541, 506, 530, 516, 486, 522, 520, 480, 503, 505, 556, 533, 485, 515, 507, 525, 530, 515, 531, 548, 532, 538, 541, 489, 507, 506, 519, 489, 505, 490, 561, 473, 499, 497, 499, 519, 532, 490, 477, 500, 511, 537, 519, 515, 488, 510] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[496, 531, 537, 493, 514, 514, 511, 531, 495, 498, 510, 495, 521, 499, 540, 542, 498, 528, 489, 491, 510, 512, 497, 515, 487, 522, 520, 498, 516, 542, 499, 508, 507, 545, 516, 500, 553, 501, 496, 523, 504, 512, 524, 483, 510, 479, 511, 547, 528, 489, 495, 517, 542, 507, 489, 527, 526, 507, 480, 528, 476, 486, 553, 522, 521, 519, 478, 499, 483, 519, 467, 537, 526, 511, 571, 506, 529, 551, 516, 534, 516, 498, 480, 552, 526, 533, 512, 530, 520, 496, 543, 484, 524, 519, 494, 520, 503, 517, 518, 535, 505, 500, 517, 504, 517, 524, 530, 512, 502, 515, 502, 501, 519, 486, 483, 533, 481, 515, 524, 501, 466, 507, 512, 511, 515, 526, 514, 485] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[475, 501, 497, 536, 500, 461, 497, 508, 544, 524, 545, 544, 521, 518, 483, 556, 542, 518, 515, 547, 493, 502, 495, 499, 527, 521, 478, 508, 515, 475, 477, 464, 550, 531, 466, 492, 508, 498, 500, 557, 532, 523, 546, 506, 505, 501, 534, 522, 503, 530, 477, 485, 514, 478, 529, 518, 519, 502, 529, 521, 494, 544, 520, 558, 500, 545, 498, 519, 514, 508, 481, 505, 511, 514, 544, 484, 531, 510, 505, 528, 489, 519, 503, 501, 524, 502, 515, 530, 484, 544, 505, 566, 491, 520, 506, 500, 524, 525, 518, 475, 485, 547, 526, 498, 523, 511, 499, 511, 516, 480, 527, 484, 561, 487, 510, 514, 517, 529, 499, 516, 508, 496, 511, 530, 521, 466, 515, 533] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[523, 493, 526, 504, 495, 518, 510, 516, 471, 506, 547, 495, 539, 533, 498, 512, 529, 472, 520, 526, 492, 541, 474, 470, 526, 506, 489, 539, 489, 500, 547, 490, 503, 505, 493, 525, 513, 514, 534, 502, 490, 491, 509, 558, 529, 544, 467, 510, 527, 516, 509, 535, 517, 498, 487, 562, 526, 533, 529, 485, 524, 490, 529, 489, 498, 483, 489, 497, 492, 530, 520, 516, 483, 519, 527, 509, 491, 511, 529, 493, 474, 511, 521, 486, 536, 518, 566, 511, 533, 512, 526, 524, 502, 543, 502, 535, 511, 516, 512, 454, 507, 504, 519, 556, 512, 487, 518, 517, 526, 528, 515, 509, 531, 507, 568, 528, 508, 506, 484, 517, 527, 515, 519, 510, 479, 480, 534, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[485, 471, 510, 500, 513, 498, 552, 511, 513, 514, 513, 548, 500, 476, 520, 497, 491, 511, 497, 481, 512, 531, 503, 509, 528, 496, 509, 498, 515, 512, 480, 497, 493, 529, 515, 478, 497, 513, 473, 544, 516, 534, 511, 499, 526, 517, 546, 521, 515, 553, 514, 522, 513, 508, 506, 497, 487, 527, 512, 544, 510, 516, 549, 488, 505, 504, 469, 509, 508, 528, 540, 484, 522, 503, 501, 518, 522, 564, 512, 539, 503, 523, 528, 534, 487, 500, 497, 525, 531, 516, 517, 532, 491, 475, 499, 552, 490, 538, 526, 536, 529, 496, 519, 503, 504, 524, 529, 505, 523, 522, 513, 499, 524, 475, 497, 499, 561, 505, 473, 498, 514, 524, 524, 518, 509, 532, 521, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[497, 517, 498, 508, 550, 505, 525, 540, 499, 510, 487, 513, 534, 529, 522, 509, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 495, 487, 566, 519, 476, 504, 491, 511, 472, 493, 535, 501, 525, 537, 517, 45, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 528, 483, 506, 495, 509, 513, 495, 507, 505, 490, 475, 497, 509, 474, 527, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 495, 498, 514, 519, 539, 483, 502, 547, 521, 495, 524, 507, 500, 489, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 514, 516, 509, 504, 523, 553, 492, 507, 500, 525, 553, 538, 505, 505, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 529, 541, 506, 530, 516, 486, 504, 520, 480, 503, 505, 556, 533, 485, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 525, 530, 515, 531, 548, 532, 538, 535, 489, 507, 506, 519, 489, 505, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 473, 499, 497, 499, 519, 532, 490, 477, 500, 511, 537, 519, 515, 488, 510, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8248, 8188, 8036, 8179, 8243, 8236, 8266, 8140] avg=8192.0000 max=8266 max_rank=6 min=8036 min_rank=2 imbalance_ratio=1.009033 max_load_reduction=57270 max_load_reduction_pct=0.873871 moved_to_echo=32885 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[496, 531, 537, 493, 514, 514, 511, 531, 495, 498, 510, 495, 521, 499, 540, 542, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 528, 489, 491, 510, 512, 497, 515, 487, 522, 520, 498, 516, 542, 499, 508, 47, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 545, 516, 500, 553, 501, 496, 523, 504, 512, 524, 483, 510, 479, 511, 547, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 489, 495, 517, 542, 507, 489, 527, 526, 507, 480, 528, 476, 486, 506, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 519, 478, 499, 483, 519, 467, 537, 526, 511, 571, 506, 529, 537, 516, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 498, 480, 552, 526, 533, 512, 511, 520, 496, 543, 484, 524, 519, 494, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 517, 518, 535, 505, 500, 517, 504, 511, 524, 530, 512, 502, 515, 502, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 486, 483, 533, 481, 515, 524, 501, 466, 507, 512, 511, 515, 526, 514, 485, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8232, 8198, 8212, 8125, 8253, 8228, 8196, 8092] avg=8192.0000 max=8253 max_rank=4 min=8092 min_rank=7 imbalance_ratio=1.007446 max_load_reduction=57283 max_load_reduction_pct=0.874069 moved_to_echo=32769 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[475, 501, 497, 536, 500, 461, 497, 508, 544, 524, 545, 544, 521, 518, 483, 556, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 518, 515, 547, 493, 502, 495, 499, 527, 521, 478, 508, 515, 475, 477, 464, 44, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 531, 466, 492, 508, 498, 500, 557, 532, 523, 546, 506, 505, 501, 534, 522, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 530, 477, 485, 514, 478, 529, 518, 519, 502, 529, 521, 494, 544, 476, 558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 545, 498, 519, 514, 508, 481, 505, 511, 514, 544, 484, 531, 497, 505, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 519, 503, 501, 524, 502, 515, 511, 484, 544, 505, 566, 491, 520, 506, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 525, 518, 475, 485, 547, 526, 498, 517, 511, 499, 511, 516, 480, 527, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 487, 510, 514, 517, 529, 499, 516, 508, 496, 511, 530, 521, 466, 515, 533, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8215, 8139, 8272, 8177, 8184, 8180, 8143, 8226] avg=8192.0000 max=8272 max_rank=2 min=8139 min_rank=1 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32733 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[526, 533, 498, 520, 515, 507, 466, 510, 521, 496, 500, 531, 535, 499, 545, 536, 10, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 457, 510, 528, 497, 496, 510, 489, 493, 532, 539, 526, 549, 474, 514, 518, 47, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 528, 455, 530, 516, 538, 481, 520, 492, 496, 507, 517, 516, 516, 502, 529, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 514, 509, 566, 535, 519, 528, 495, 491, 513, 518, 491, 508, 492, 458, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 518, 525, 534, 508, 521, 508, 543, 480, 535, 505, 505, 487, 489, 496, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 527, 510, 519, 538, 506, 527, 491, 479, 480, 558, 527, 489, 507, 481, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 505, 549, 531, 520, 501, 525, 477, 481, 488, 511, 521, 524, 518, 507, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 491, 535, 526, 516, 518, 536, 524, 499, 512, 483, 517, 494, 510, 511, 553, 15, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8254, 8213, 8166, 8159, 8129, 8188, 8196, 8231] avg=8192.0000 max=8254 max_rank=0 min=8129 min_rank=4 imbalance_ratio=1.007568 max_load_reduction=57282 max_load_reduction_pct=0.874054 moved_to_echo=32744 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[497, 517, 498, 508, 550, 505, 525, 540, 499, 510, 487, 513, 534, 529, 522, 509, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 495, 487, 566, 519, 476, 504, 491, 511, 472, 493, 535, 501, 525, 537, 517, 45, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 528, 483, 506, 495, 509, 513, 495, 507, 505, 490, 475, 497, 509, 474, 527, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 495, 498, 514, 519, 539, 483, 502, 547, 521, 495, 524, 507, 500, 489, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 514, 516, 509, 504, 523, 553, 492, 507, 500, 525, 553, 538, 505, 505, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 529, 541, 506, 530, 516, 486, 504, 520, 480, 503, 505, 556, 533, 485, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 525, 530, 515, 531, 548, 532, 538, 535, 489, 507, 506, 519, 489, 505, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 473, 499, 497, 499, 519, 532, 490, 477, 500, 511, 537, 519, 515, 488, 510, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8248, 8188, 8036, 8179, 8243, 8236, 8266, 8140] avg=8192.0000 max=8266 max_rank=6 min=8036 min_rank=2 imbalance_ratio=1.009033 max_load_reduction=57270 max_load_reduction_pct=0.873871 moved_to_echo=32885 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[542, 493, 489, 502, 478, 548, 499, 484, 491, 519, 491, 540, 478, 506, 524, 507, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 535, 490, 504, 521, 518, 494, 508, 477, 523, 499, 492, 550, 444, 488, 490, 43, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 526, 541, 507, 523, 528, 493, 568, 541, 522, 493, 510, 525, 488, 523, 518, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 532, 567, 518, 526, 506, 529, 472, 488, 537, 557, 487, 528, 535, 463, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 502, 551, 537, 541, 486, 484, 536, 499, 554, 514, 483, 521, 498, 521, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 527, 510, 504, 471, 504, 486, 518, 502, 520, 533, 505, 500, 538, 551, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 500, 516, 473, 489, 500, 504, 516, 538, 516, 550, 525, 502, 521, 508, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 517, 486, 509, 522, 510, 506, 511, 521, 504, 479, 494, 504, 522, 513, 498, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8096, 8114, 8309, 8245, 8248, 8206, 8185, 8133] avg=8192.0000 max=8309 max_rank=2 min=8096 min_rank=0 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32772 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[485, 471, 510, 500, 513, 498, 552, 511, 513, 514, 513, 548, 500, 476, 520, 497, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 511, 497, 481, 512, 531, 503, 509, 528, 496, 509, 498, 515, 512, 480, 497, 47, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 529, 515, 478, 497, 513, 473, 544, 516, 534, 511, 499, 526, 517, 546, 521, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 553, 514, 522, 513, 508, 506, 497, 487, 527, 512, 544, 510, 516, 502, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 504, 469, 509, 508, 528, 540, 484, 522, 503, 501, 518, 522, 550, 512, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 523, 528, 534, 487, 500, 497, 507, 531, 516, 517, 532, 491, 475, 499, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 538, 526, 536, 529, 496, 519, 503, 498, 524, 529, 505, 523, 522, 513, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 475, 497, 499, 561, 505, 473, 498, 514, 524, 524, 518, 509, 532, 521, 504, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8126, 8135, 8213, 8214, 8214, 8192, 8250, 8192] avg=8192.0000 max=8250 max_rank=6 min=8126 min_rank=0 imbalance_ratio=1.007080 max_load_reduction=57286 max_load_reduction_pct=0.874115 moved_to_echo=32848 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[523, 493, 526, 504, 495, 518, 510, 516, 471, 506, 547, 495, 539, 533, 498, 512, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 472, 520, 526, 492, 541, 474, 470, 526, 506, 489, 539, 489, 500, 547, 490, 45, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 505, 493, 525, 513, 514, 534, 502, 490, 491, 509, 558, 529, 544, 467, 510, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 516, 509, 535, 517, 498, 487, 562, 526, 533, 529, 485, 524, 490, 484, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 483, 489, 497, 492, 530, 520, 516, 483, 519, 527, 509, 491, 498, 529, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 511, 521, 486, 536, 518, 566, 493, 533, 512, 526, 524, 502, 543, 502, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 516, 512, 454, 507, 504, 519, 556, 506, 487, 518, 517, 526, 528, 515, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 507, 568, 528, 508, 506, 484, 517, 527, 515, 519, 510, 479, 480, 534, 506, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8173, 8188, 8211, 8074, 8282, 8185, 8232] avg=8192.0000 max=8282 max_rank=5 min=8074 min_rank=4 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32773 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[542, 493, 489, 502, 478, 548, 499, 484, 491, 519, 491, 540, 478, 506, 524, 507, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 535, 490, 504, 521, 518, 494, 508, 477, 523, 499, 492, 550, 444, 488, 490, 43, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 526, 541, 507, 523, 528, 493, 568, 541, 522, 493, 510, 525, 488, 523, 518, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 532, 567, 518, 526, 506, 529, 472, 488, 537, 557, 487, 528, 535, 463, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 502, 551, 537, 541, 486, 484, 536, 499, 554, 514, 483, 521, 498, 521, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 527, 510, 504, 471, 504, 486, 518, 502, 520, 533, 505, 500, 538, 551, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 500, 516, 473, 489, 500, 504, 516, 538, 516, 550, 525, 502, 521, 508, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 517, 486, 509, 522, 510, 506, 511, 521, 504, 479, 494, 504, 522, 513, 498, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8096, 8114, 8309, 8245, 8248, 8206, 8185, 8133] avg=8192.0000 max=8309 max_rank=2 min=8096 min_rank=0 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32772 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[497, 519, 534, 522, 486, 504, 485, 492, 518, 510, 542, 527, 544, 474, 497, 518, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 521, 509, 550, 494, 505, 522, 508, 512, 525, 502, 554, 510, 537, 523, 517, 44, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 505, 488, 497, 528, 495, 492, 521, 526, 515, 506, 504, 513, 494, 521, 518, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 532, 522, 522, 521, 528, 543, 505, 518, 532, 499, 498, 503, 510, 478, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 516, 488, 521, 512, 528, 514, 459, 519, 518, 481, 514, 516, 533, 526, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 504, 540, 463, 501, 480, 514, 494, 530, 498, 489, 530, 525, 478, 520, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 508, 498, 521, 549, 493, 470, 490, 530, 533, 503, 532, 487, 483, 493, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 513, 529, 523, 550, 539, 533, 464, 567, 525, 505, 480, 483, 464, 533, 533, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 8376, 8140, 8226, 8191, 8024, 8115, 8290] avg=8192.0000 max=8376 max_rank=1 min=8024 min_rank=5 imbalance_ratio=1.022461 max_load_reduction=57160 max_load_reduction_pct=0.872192 moved_to_echo=32620 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[485, 471, 510, 500, 513, 498, 552, 511, 513, 514, 513, 548, 500, 476, 520, 497, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 511, 497, 481, 512, 531, 503, 509, 528, 496, 509, 498, 515, 512, 480, 497, 47, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 529, 515, 478, 497, 513, 473, 544, 516, 534, 511, 499, 526, 517, 546, 521, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 553, 514, 522, 513, 508, 506, 497, 487, 527, 512, 544, 510, 516, 502, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 504, 469, 509, 508, 528, 540, 484, 522, 503, 501, 518, 522, 550, 512, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 523, 528, 534, 487, 500, 497, 507, 531, 516, 517, 532, 491, 475, 499, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 538, 526, 536, 529, 496, 519, 503, 498, 524, 529, 505, 523, 522, 513, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 475, 497, 499, 561, 505, 473, 498, 514, 524, 524, 518, 509, 532, 521, 504, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8126, 8135, 8213, 8214, 8214, 8192, 8250, 8192] avg=8192.0000 max=8250 max_rank=6 min=8126 min_rank=0 imbalance_ratio=1.007080 max_load_reduction=57286 max_load_reduction_pct=0.874115 moved_to_echo=32848 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[475, 501, 497, 536, 500, 461, 497, 508, 544, 524, 545, 544, 521, 518, 483, 556, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 518, 515, 547, 493, 502, 495, 499, 527, 521, 478, 508, 515, 475, 477, 464, 44, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 531, 466, 492, 508, 498, 500, 557, 532, 523, 546, 506, 505, 501, 534, 522, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 530, 477, 485, 514, 478, 529, 518, 519, 502, 529, 521, 494, 544, 476, 558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 545, 498, 519, 514, 508, 481, 505, 511, 514, 544, 484, 531, 497, 505, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 519, 503, 501, 524, 502, 515, 511, 484, 544, 505, 566, 491, 520, 506, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 525, 518, 475, 485, 547, 526, 498, 517, 511, 499, 511, 516, 480, 527, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 487, 510, 514, 517, 529, 499, 516, 508, 496, 511, 530, 521, 466, 515, 533, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8215, 8139, 8272, 8177, 8184, 8180, 8143, 8226] avg=8192.0000 max=8272 max_rank=2 min=8139 min_rank=1 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32733 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4352, 3968, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 3968, 4096, 4224, 4096, 4096, 4352, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 3968, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4352, 3968, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 384, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 3968, 4096, 4224, 4096, 4096, 4352, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[507, 503, 539, 504, 498, 534, 538, 511, 544, 515, 503, 514, 531, 548, 496, 535, 509, 525, 484, 515, 514, 511, 512, 526, 493, 557, 491, 478, 513, 576, 519, 527, 536, 483, 538, 526, 541, 513, 513, 503, 514, 506, 502, 483, 500, 482, 488, 488, 521, 529, 523, 537, 496, 505, 534, 538, 479, 500, 501, 494, 541, 475, 505, 478, 499, 532, 522, 469, 512, 522, 487, 527, 503, 507, 528, 515, 545, 531, 546, 462, 477, 507, 506, 503, 521, 529, 510, 487, 492, 486, 522, 509, 524, 542, 519, 514, 497, 559, 498, 515, 479, 502, 501, 496, 509, 493, 500, 493, 515, 510, 518, 500, 484, 524, 494, 508, 529, 496, 520, 501, 524, 531, 515, 531, 560, 506, 485, 546] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[562, 497, 519, 513, 493, 537, 530, 530, 485, 503, 534, 518, 540, 515, 487, 495, 497, 486, 498, 512, 497, 502, 529, 517, 516, 476, 500, 514, 536, 502, 556, 497, 547, 538, 517, 472, 515, 514, 509, 504, 508, 505, 520, 546, 486, 479, 488, 515, 492, 537, 515, 535, 501, 477, 514, 494, 500, 532, 521, 526, 499, 514, 551, 531, 526, 527, 547, 479, 524, 543, 510, 513, 497, 507, 530, 512, 544, 488, 497, 522, 492, 485, 486, 501, 534, 486, 541, 566, 544, 505, 557, 563, 496, 537, 492, 484, 538, 512, 472, 486, 489, 545, 541, 512, 501, 503, 495, 490, 467, 517, 556, 488, 500, 540, 525, 484, 512, 487, 524, 537, 522, 473, 456, 492, 515, 494, 545, 488] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[507, 503, 539, 504, 498, 534, 538, 511, 544, 515, 503, 514, 531, 548, 496, 535, 509, 525, 484, 515, 514, 511, 512, 526, 493, 557, 491, 478, 513, 576, 519, 527, 536, 483, 538, 526, 541, 513, 513, 503, 514, 506, 502, 483, 500, 482, 488, 488, 521, 529, 523, 537, 496, 505, 534, 538, 479, 500, 501, 494, 541, 475, 505, 478, 499, 532, 522, 469, 512, 522, 487, 527, 503, 507, 528, 515, 545, 531, 546, 462, 477, 507, 506, 503, 521, 529, 510, 487, 492, 486, 522, 509, 524, 542, 519, 514, 497, 559, 498, 515, 479, 502, 501, 496, 509, 493, 500, 493, 515, 510, 518, 500, 484, 524, 494, 508, 529, 496, 520, 501, 524, 531, 515, 531, 560, 506, 485, 546] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[515, 523, 520, 508, 530, 523, 493, 495, 531, 495, 510, 494, 499, 507, 522, 513, 497, 543, 557, 501, 535, 510, 536, 525, 510, 526, 496, 477, 535, 573, 531, 523, 505, 524, 568, 492, 502, 500, 545, 515, 507, 488, 492, 516, 482, 520, 505, 483, 507, 521, 518, 504, 498, 503, 501, 483, 512, 491, 566, 507, 509, 527, 494, 525, 509, 493, 493, 534, 503, 533, 526, 514, 473, 499, 487, 554, 520, 504, 549, 483, 481, 542, 560, 526, 484, 514, 537, 528, 502, 511, 515, 503, 496, 532, 521, 490, 488, 510, 460, 504, 489, 481, 487, 507, 498, 495, 498, 556, 505, 527, 536, 511, 509, 480, 505, 535, 519, 497, 512, 506, 501, 473, 506, 525, 532, 506, 539, 560] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[522, 510, 488, 539, 537, 509, 503, 472, 520, 533, 506, 509, 497, 497, 497, 529, 516, 536, 501, 511, 541, 502, 516, 512, 515, 564, 491, 509, 505, 527, 504, 490, 520, 490, 501, 561, 535, 508, 523, 518, 519, 535, 523, 534, 487, 497, 517, 512, 505, 536, 504, 543, 499, 505, 494, 487, 518, 501, 471, 484, 521, 527, 498, 483, 525, 524, 493, 502, 502, 530, 502, 487, 501, 507, 508, 517, 530, 460, 511, 525, 496, 512, 508, 511, 478, 538, 535, 521, 512, 520, 488, 528, 505, 518, 522, 458, 547, 520, 527, 504, 559, 526, 484, 519, 493, 454, 522, 553, 492, 501, 508, 517, 516, 480, 532, 538, 505, 520, 513, 518, 512, 493, 517, 528, 505, 538, 522, 535] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[524, 496, 525, 515, 489, 524, 495, 497, 502, 548, 508, 518, 527, 500, 524, 493, 528, 458, 516, 509, 515, 483, 533, 512, 485, 531, 515, 578, 483, 510, 473, 504, 490, 530, 497, 492, 534, 535, 492, 505, 530, 509, 518, 543, 521, 527, 515, 539, 503, 507, 524, 532, 530, 506, 512, 532, 513, 537, 537, 483, 535, 512, 523, 495, 525, 514, 539, 476, 536, 517, 521, 525, 524, 509, 521, 530, 486, 494, 467, 507, 529, 546, 544, 522, 517, 550, 487, 482, 514, 524, 518, 523, 513, 506, 500, 528, 521, 492, 505, 517, 531, 475, 503, 547, 520, 518, 510, 540, 544, 496, 478, 478, 505, 535, 511, 487, 481, 525, 491, 478, 510, 498, 499, 497, 484, 509, 495, 486] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[485, 483, 521, 505, 496, 521, 520, 514, 496, 486, 510, 538, 480, 525, 512, 517, 537, 521, 488, 514, 538, 487, 528, 473, 508, 513, 515, 520, 505, 517, 539, 520, 519, 484, 493, 493, 547, 525, 500, 534, 475, 550, 521, 530, 496, 531, 496, 492, 528, 547, 465, 489, 533, 472, 522, 526, 502, 514, 502, 532, 523, 497, 511, 512, 490, 539, 497, 511, 506, 519, 511, 522, 531, 463, 522, 528, 524, 500, 484, 487, 479, 516, 492, 470, 526, 511, 537, 543, 540, 536, 490, 508, 528, 527, 518, 481, 501, 497, 536, 512, 493, 502, 488, 509, 493, 508, 540, 539, 514, 479, 534, 512, 525, 512, 553, 513, 551, 524, 516, 489, 523, 503, 541, 538, 542, 493, 503, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[503, 506, 484, 517, 521, 522, 484, 472, 478, 442, 544, 495, 532, 473, 521, 522, 509, 493, 500, 536, 523, 500, 541, 484, 469, 500, 505, 541, 566, 523, 481, 502, 586, 531, 555, 518, 474, 519, 498, 531, 492, 516, 513, 526, 543, 485, 524, 491, 488, 524, 487, 507, 476, 511, 523, 551, 521, 533, 476, 515, 494, 520, 498, 485, 513, 526, 472, 540, 507, 555, 517, 486, 510, 485, 490, 503, 536, 518, 500, 524, 517, 499, 538, 500, 499, 513, 511, 525, 509, 471, 534, 536, 529, 556, 497, 547, 555, 533, 542, 511, 521, 554, 510, 493, 547, 468, 454, 552, 532, 516, 534, 508, 528, 507, 501, 505, 517, 510, 517, 507, 549, 526, 477, 447, 525, 502, 500, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[536, 503, 486, 507, 509, 529, 488, 525, 502, 504, 515, 527, 494, 512, 475, 481, 513, 521, 551, 520, 472, 518, 573, 510, 528, 518, 521, 479, 486, 520, 514, 526, 486, 506, 492, 528, 483, 525, 507, 531, 580, 491, 498, 502, 533, 512, 527, 507, 515, 529, 532, 499, 503, 495, 541, 493, 508, 496, 503, 510, 468, 503, 467, 524, 545, 579, 507, 508, 503, 516, 495, 490, 482, 524, 523, 562, 539, 502, 487, 541, 537, 481, 495, 556, 505, 510, 544, 531, 520, 528, 515, 456, 516, 503, 494, 513, 486, 525, 531, 540, 547, 524, 494, 485, 535, 516, 472, 543, 488, 560, 508, 484, 520, 515, 510, 473, 537, 491, 490, 491, 544, 527, 493, 525, 512, 511, 479, 516] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[485, 483, 521, 505, 496, 521, 520, 514, 496, 486, 510, 538, 480, 525, 512, 517, 537, 521, 488, 514, 538, 487, 528, 473, 508, 513, 515, 520, 505, 517, 539, 520, 519, 484, 493, 493, 547, 525, 500, 534, 475, 550, 521, 530, 496, 531, 496, 492, 528, 547, 465, 489, 533, 472, 522, 526, 502, 514, 502, 532, 523, 497, 511, 512, 490, 539, 497, 511, 506, 519, 511, 522, 531, 463, 522, 528, 524, 500, 484, 487, 479, 516, 492, 470, 526, 511, 537, 543, 540, 536, 490, 508, 528, 527, 518, 481, 501, 497, 536, 512, 493, 502, 488, 509, 493, 508, 540, 539, 514, 479, 534, 512, 525, 512, 553, 513, 551, 524, 516, 489, 523, 503, 541, 538, 542, 493, 503, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[507, 503, 539, 504, 498, 534, 538, 511, 544, 515, 503, 514, 531, 548, 496, 535, 24, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 525, 484, 515, 514, 511, 482, 526, 493, 557, 491, 478, 513, 576, 519, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 483, 538, 526, 541, 513, 513, 503, 514, 506, 502, 483, 500, 482, 488, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 529, 523, 537, 496, 505, 534, 538, 479, 500, 501, 494, 541, 475, 505, 478, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 532, 522, 469, 512, 517, 487, 527, 503, 507, 528, 515, 545, 531, 546, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 507, 506, 503, 521, 529, 510, 487, 492, 486, 522, 509, 524, 510, 519, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 559, 498, 515, 479, 502, 501, 496, 509, 493, 500, 493, 515, 510, 518, 500, 11, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 524, 494, 508, 529, 496, 520, 501, 524, 531, 515, 531, 560, 506, 485, 546, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8345, 8220, 8099, 8185, 8202, 8116, 8104, 8265] avg=8192.0000 max=8345 max_rank=0 min=8099 min_rank=2 imbalance_ratio=1.018677 max_load_reduction=57191 max_load_reduction_pct=0.872665 moved_to_echo=32687 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[522, 510, 488, 539, 537, 509, 503, 472, 520, 533, 506, 509, 497, 497, 497, 529, 516, 536, 501, 511, 541, 502, 516, 512, 515, 564, 491, 509, 505, 527, 504, 490, 520, 490, 501, 561, 535, 508, 523, 518, 519, 535, 523, 534, 487, 497, 517, 512, 505, 536, 504, 543, 499, 505, 494, 487, 518, 501, 471, 484, 521, 527, 498, 483, 525, 524, 493, 502, 502, 530, 502, 487, 501, 507, 508, 517, 530, 460, 511, 525, 496, 512, 508, 511, 478, 538, 535, 521, 512, 520, 488, 528, 505, 518, 522, 458, 547, 520, 527, 504, 559, 526, 484, 519, 493, 454, 522, 553, 492, 501, 508, 517, 516, 480, 532, 538, 505, 520, 513, 518, 512, 493, 517, 528, 505, 538, 522, 535] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[562, 497, 519, 513, 493, 537, 530, 530, 485, 503, 534, 518, 540, 515, 487, 495, 28, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 486, 498, 512, 497, 502, 487, 517, 516, 476, 500, 514, 536, 502, 556, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 538, 517, 472, 515, 514, 509, 504, 508, 505, 520, 546, 486, 479, 488, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 537, 515, 535, 501, 477, 514, 494, 500, 532, 521, 526, 499, 514, 551, 531, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 527, 547, 479, 524, 533, 510, 513, 497, 507, 530, 512, 544, 488, 497, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 485, 486, 501, 534, 486, 541, 566, 544, 505, 557, 563, 496, 497, 492, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 512, 472, 486, 489, 545, 541, 512, 501, 503, 495, 490, 467, 517, 556, 488, 16, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 540, 525, 484, 512, 487, 524, 537, 522, 473, 456, 492, 515, 494, 545, 488, 11, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8293, 8093, 8136, 8274, 8256, 8229, 8140, 8115] avg=8192.0000 max=8293 max_rank=0 min=8093 min_rank=1 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32740 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[515, 523, 520, 508, 530, 523, 493, 495, 531, 495, 510, 494, 499, 507, 522, 513, 24, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 543, 557, 501, 535, 510, 504, 525, 510, 526, 496, 477, 535, 573, 531, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 524, 568, 492, 502, 500, 545, 515, 507, 488, 492, 516, 482, 520, 505, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 521, 518, 504, 498, 503, 501, 483, 512, 491, 566, 507, 509, 527, 494, 525, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 493, 493, 534, 503, 528, 526, 514, 473, 499, 487, 554, 520, 504, 549, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 542, 560, 526, 484, 514, 537, 528, 502, 511, 515, 503, 496, 500, 521, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 510, 460, 504, 489, 481, 487, 507, 498, 495, 498, 556, 505, 527, 536, 511, 11, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 480, 505, 535, 519, 497, 512, 506, 501, 473, 506, 525, 532, 506, 539, 560, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8343, 8128, 8197, 8169, 8210, 8071, 8215] avg=8192.0000 max=8343 max_rank=1 min=8071 min_rank=6 imbalance_ratio=1.018433 max_load_reduction=57193 max_load_reduction_pct=0.872696 moved_to_echo=32665 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[503, 506, 484, 517, 521, 522, 484, 472, 478, 442, 544, 495, 532, 473, 521, 522, 25, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 493, 500, 536, 523, 500, 509, 484, 469, 500, 505, 541, 566, 523, 481, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 568, 531, 555, 518, 474, 519, 498, 531, 492, 516, 513, 526, 543, 485, 524, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 524, 487, 507, 476, 511, 523, 551, 521, 533, 476, 515, 494, 520, 498, 485, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 526, 472, 540, 507, 550, 517, 486, 510, 485, 490, 503, 536, 518, 500, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 499, 538, 500, 499, 513, 511, 525, 509, 471, 534, 536, 529, 522, 497, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 533, 542, 511, 521, 554, 510, 493, 547, 468, 454, 552, 532, 516, 534, 508, 12, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 507, 501, 505, 517, 510, 517, 507, 549, 526, 477, 447, 525, 502, 500, 525, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8042, 8141, 8284, 8140, 8177, 8247, 8351, 8154] avg=8192.0000 max=8351 max_rank=6 min=8042 min_rank=0 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32929 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[536, 503, 486, 507, 509, 529, 488, 525, 502, 504, 515, 527, 494, 512, 475, 481, 23, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 521, 551, 520, 472, 518, 538, 510, 528, 518, 521, 479, 486, 520, 514, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 506, 492, 528, 483, 525, 507, 531, 580, 491, 498, 502, 533, 512, 527, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 529, 532, 499, 503, 495, 541, 493, 508, 496, 503, 510, 468, 503, 467, 524, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 579, 507, 508, 503, 511, 495, 490, 482, 524, 523, 562, 539, 502, 487, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 481, 495, 556, 505, 510, 544, 531, 520, 528, 515, 456, 516, 472, 494, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 525, 531, 540, 547, 524, 494, 485, 535, 516, 472, 543, 488, 560, 508, 484, 10, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 515, 510, 473, 537, 491, 490, 491, 544, 527, 493, 525, 512, 511, 479, 516, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8118, 8235, 8193, 8119, 8298, 8173, 8256, 8144] avg=8192.0000 max=8298 max_rank=4 min=8118 min_rank=0 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32871 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[524, 496, 525, 515, 489, 524, 495, 497, 502, 548, 508, 518, 527, 500, 524, 493, 23, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 458, 516, 509, 515, 483, 502, 512, 485, 531, 515, 578, 483, 510, 473, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 530, 497, 492, 534, 535, 492, 505, 530, 509, 518, 543, 521, 527, 515, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 507, 524, 532, 530, 506, 512, 532, 513, 537, 537, 483, 535, 512, 523, 495, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 514, 539, 476, 536, 512, 521, 525, 524, 509, 521, 530, 486, 494, 467, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 546, 544, 522, 517, 550, 487, 482, 514, 524, 518, 523, 513, 475, 500, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 492, 505, 517, 531, 475, 503, 547, 520, 518, 510, 540, 544, 496, 478, 478, 10, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 535, 511, 487, 481, 525, 491, 478, 510, 498, 499, 497, 484, 509, 495, 486, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8102, 8262, 8311, 8186, 8272, 8193, 8001] avg=8192.0000 max=8311 max_rank=3 min=8001 min_rank=7 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32652 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[503, 506, 484, 517, 521, 522, 484, 472, 478, 442, 544, 495, 532, 473, 521, 522, 25, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 493, 500, 536, 523, 500, 509, 484, 469, 500, 505, 541, 566, 523, 481, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 568, 531, 555, 518, 474, 519, 498, 531, 492, 516, 513, 526, 543, 485, 524, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 524, 487, 507, 476, 511, 523, 551, 521, 533, 476, 515, 494, 520, 498, 485, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 526, 472, 540, 507, 550, 517, 486, 510, 485, 490, 503, 536, 518, 500, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 499, 538, 500, 499, 513, 511, 525, 509, 471, 534, 536, 529, 522, 497, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 533, 542, 511, 521, 554, 510, 493, 547, 468, 454, 552, 532, 516, 534, 508, 12, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 507, 501, 505, 517, 510, 517, 507, 549, 526, 477, 447, 525, 502, 500, 525, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8042, 8141, 8284, 8140, 8177, 8247, 8351, 8154] avg=8192.0000 max=8351 max_rank=6 min=8042 min_rank=0 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32929 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[522, 510, 488, 539, 537, 509, 503, 472, 520, 533, 506, 509, 497, 497, 497, 529, 23, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 536, 501, 511, 541, 502, 486, 512, 515, 564, 491, 509, 505, 527, 504, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 490, 501, 561, 535, 508, 523, 518, 519, 535, 523, 534, 487, 497, 517, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 536, 504, 543, 499, 505, 494, 487, 518, 501, 471, 484, 521, 527, 498, 483, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 524, 493, 502, 502, 525, 502, 487, 501, 507, 508, 517, 530, 460, 511, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 512, 508, 511, 478, 538, 535, 521, 512, 520, 488, 528, 505, 487, 522, 458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 520, 527, 504, 559, 526, 484, 519, 493, 454, 522, 553, 492, 501, 508, 517, 11, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 480, 532, 538, 505, 520, 513, 518, 512, 493, 517, 528, 505, 538, 522, 535, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8210, 8264, 8105, 8119, 8119, 8245, 8282] avg=8192.0000 max=8282 max_rank=7 min=8105 min_rank=3 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32765 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[507, 503, 539, 504, 498, 534, 538, 511, 544, 515, 503, 514, 531, 548, 496, 535, 24, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 525, 484, 515, 514, 511, 482, 526, 493, 557, 491, 478, 513, 576, 519, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 483, 538, 526, 541, 513, 513, 503, 514, 506, 502, 483, 500, 482, 488, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 529, 523, 537, 496, 505, 534, 538, 479, 500, 501, 494, 541, 475, 505, 478, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 532, 522, 469, 512, 517, 487, 527, 503, 507, 528, 515, 545, 531, 546, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 507, 506, 503, 521, 529, 510, 487, 492, 486, 522, 509, 524, 510, 519, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 559, 498, 515, 479, 502, 501, 496, 509, 493, 500, 493, 515, 510, 518, 500, 11, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 524, 494, 508, 529, 496, 520, 501, 524, 531, 515, 531, 560, 506, 485, 546, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8345, 8220, 8099, 8185, 8202, 8116, 8104, 8265] avg=8192.0000 max=8345 max_rank=0 min=8099 min_rank=2 imbalance_ratio=1.018677 max_load_reduction=57191 max_load_reduction_pct=0.872665 moved_to_echo=32687 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[485, 483, 521, 505, 496, 521, 520, 514, 496, 486, 510, 538, 480, 525, 512, 517, 24, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 521, 488, 514, 538, 487, 497, 473, 508, 513, 515, 520, 505, 517, 539, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 484, 493, 493, 547, 525, 500, 534, 475, 550, 521, 530, 496, 531, 496, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 547, 465, 489, 533, 472, 522, 526, 502, 514, 502, 532, 523, 497, 511, 512, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 539, 497, 511, 506, 514, 511, 522, 531, 463, 522, 528, 524, 500, 484, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 516, 492, 470, 526, 511, 537, 543, 540, 536, 490, 508, 528, 495, 518, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 497, 536, 512, 493, 502, 488, 509, 493, 508, 540, 539, 514, 479, 534, 512, 11, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 512, 553, 513, 551, 524, 516, 489, 523, 503, 541, 538, 542, 493, 503, 524, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8134, 8192, 8170, 8205, 8129, 8170, 8176, 8360] avg=8192.0000 max=8360 max_rank=7 min=8129 min_rank=4 imbalance_ratio=1.020508 max_load_reduction=57176 max_load_reduction_pct=0.872437 moved_to_echo=32835 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[536, 503, 486, 507, 509, 529, 488, 525, 502, 504, 515, 527, 494, 512, 475, 481, 23, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 521, 551, 520, 472, 518, 538, 510, 528, 518, 521, 479, 486, 520, 514, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 506, 492, 528, 483, 525, 507, 531, 580, 491, 498, 502, 533, 512, 527, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 529, 532, 499, 503, 495, 541, 493, 508, 496, 503, 510, 468, 503, 467, 524, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 579, 507, 508, 503, 511, 495, 490, 482, 524, 523, 562, 539, 502, 487, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 481, 495, 556, 505, 510, 544, 531, 520, 528, 515, 456, 516, 472, 494, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 525, 531, 540, 547, 524, 494, 485, 535, 516, 472, 543, 488, 560, 508, 484, 10, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 515, 510, 473, 537, 491, 490, 491, 544, 527, 493, 525, 512, 511, 479, 516, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8118, 8235, 8193, 8119, 8298, 8173, 8256, 8144] avg=8192.0000 max=8298 max_rank=4 min=8118 min_rank=0 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32871 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4352, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4352, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 3968, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4352, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4352, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 3968, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 3968, 4096, 4352, 4096, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[528, 532, 521, 567, 530, 522, 561, 540, 532, 493, 521, 509, 520, 510, 554, 515, 542, 511, 463, 492, 494, 493, 535, 505, 532, 534, 509, 520, 491, 491, 528, 515, 543, 488, 514, 508, 482, 498, 482, 550, 474, 503, 496, 506, 501, 473, 503, 476, 512, 539, 472, 476, 559, 546, 503, 490, 495, 506, 503, 500, 473, 520, 507, 483, 513, 513, 523, 523, 518, 499, 502, 495, 494, 480, 504, 479, 484, 533, 525, 496, 489, 513, 477, 561, 528, 513, 472, 530, 524, 536, 519, 519, 508, 560, 519, 516, 545, 550, 520, 533, 536, 513, 500, 509, 502, 493, 524, 529, 513, 498, 486, 516, 523, 506, 550, 518, 523, 487, 523, 496, 499, 518, 504, 502, 533, 494, 499, 538] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[543, 483, 478, 558, 516, 506, 501, 505, 481, 507, 494, 520, 529, 503, 478, 506, 534, 489, 503, 529, 529, 516, 539, 493, 508, 506, 538, 510, 511, 522, 548, 494, 475, 536, 498, 528, 492, 487, 545, 515, 551, 520, 498, 538, 467, 539, 497, 504, 491, 515, 488, 535, 503, 488, 523, 476, 508, 571, 499, 492, 522, 519, 502, 528, 499, 476, 524, 537, 500, 495, 538, 543, 511, 496, 552, 515, 514, 514, 508, 498, 507, 528, 533, 536, 530, 513, 503, 476, 553, 492, 507, 526, 517, 524, 476, 523, 531, 526, 474, 517, 495, 495, 520, 525, 548, 519, 534, 469, 545, 487, 491, 516, 459, 492, 565, 522, 517, 459, 511, 544, 525, 490, 480, 566, 497, 515, 508, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[544, 517, 507, 495, 552, 514, 504, 510, 510, 509, 505, 534, 492, 491, 504, 506, 507, 487, 502, 512, 500, 501, 511, 507, 510, 464, 500, 534, 509, 539, 501, 538, 523, 537, 571, 489, 506, 508, 482, 535, 526, 481, 555, 510, 511, 561, 530, 466, 524, 520, 536, 493, 504, 531, 501, 513, 513, 500, 480, 507, 476, 489, 522, 490, 537, 510, 510, 501, 482, 516, 522, 502, 498, 528, 531, 518, 543, 510, 496, 547, 513, 485, 503, 508, 514, 500, 543, 502, 528, 529, 485, 496, 533, 479, 522, 515, 532, 525, 505, 519, 479, 531, 485, 544, 477, 467, 482, 525, 510, 519, 513, 539, 522, 516, 520, 509, 512, 494, 527, 513, 519, 524, 526, 476, 555, 529, 518, 512] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[491, 532, 526, 512, 502, 541, 462, 556, 527, 485, 522, 522, 511, 537, 515, 478, 526, 538, 530, 539, 533, 492, 526, 516, 527, 490, 469, 498, 543, 504, 525, 521, 498, 487, 499, 509, 514, 582, 541, 499, 521, 501, 499, 497, 467, 471, 542, 548, 531, 527, 549, 508, 512, 477, 539, 542, 488, 530, 518, 513, 507, 488, 460, 495, 463, 518, 516, 538, 507, 509, 489, 521, 524, 526, 506, 491, 525, 490, 565, 532, 511, 519, 492, 520, 530, 562, 491, 524, 498, 513, 494, 496, 506, 475, 476, 488, 533, 531, 496, 512, 468, 482, 498, 525, 509, 512, 494, 507, 518, 501, 498, 498, 529, 541, 528, 508, 506, 526, 528, 569, 499, 525, 480, 493, 525, 509, 502, 516] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[534, 510, 487, 532, 521, 535, 497, 515, 483, 486, 467, 510, 513, 501, 498, 522, 542, 482, 525, 505, 530, 542, 500, 508, 513, 519, 530, 469, 535, 515, 512, 517, 475, 539, 509, 512, 547, 530, 495, 511, 499, 518, 500, 551, 506, 477, 501, 498, 509, 506, 523, 501, 506, 533, 511, 519, 504, 509, 535, 494, 519, 551, 508, 484, 481, 478, 502, 507, 472, 501, 490, 515, 537, 526, 538, 511, 473, 529, 539, 549, 548, 513, 518, 554, 516, 561, 578, 512, 520, 493, 489, 510, 510, 493, 532, 536, 499, 524, 523, 480, 502, 475, 508, 500, 515, 476, 507, 527, 530, 527, 496, 490, 495, 492, 535, 485, 517, 479, 512, 477, 514, 508, 516, 542, 524, 555, 515, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[521, 528, 494, 499, 541, 501, 509, 523, 514, 506, 523, 556, 476, 497, 518, 531, 570, 499, 530, 505, 493, 533, 516, 496, 504, 500, 479, 542, 526, 496, 491, 514, 503, 518, 524, 517, 527, 499, 499, 535, 480, 543, 472, 500, 506, 499, 483, 495, 530, 550, 519, 511, 543, 518, 519, 512, 543, 556, 463, 519, 566, 511, 545, 456, 518, 495, 552, 529, 534, 503, 483, 499, 497, 525, 485, 526, 500, 514, 540, 551, 526, 519, 494, 524, 538, 524, 490, 506, 512, 479, 510, 506, 529, 533, 524, 520, 498, 503, 492, 485, 510, 491, 486, 478, 507, 532, 511, 467, 534, 472, 510, 480, 499, 551, 497, 485, 521, 517, 534, 525, 508, 524, 490, 523, 489, 560, 489, 491] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[509, 518, 549, 476, 547, 493, 503, 530, 467, 491, 520, 533, 521, 508, 479, 495, 518, 525, 527, 483, 531, 474, 536, 541, 506, 540, 519, 530, 509, 524, 529, 494, 505, 481, 490, 498, 494, 543, 543, 525, 524, 500, 498, 515, 515, 501, 482, 542, 521, 503, 531, 500, 479, 537, 499, 511, 503, 491, 530, 520, 489, 506, 500, 498, 512, 511, 527, 474, 515, 487, 522, 520, 489, 520, 518, 495, 504, 528, 491, 528, 512, 532, 543, 495, 538, 532, 526, 493, 532, 513, 550, 496, 525, 522, 571, 496, 548, 519, 534, 511, 514, 542, 491, 514, 516, 516, 505, 461, 509, 496, 519, 494, 495, 506, 539, 520, 529, 485, 501, 515, 510, 496, 467, 516, 522, 511, 486, 533] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[544, 517, 507, 495, 552, 514, 504, 510, 510, 509, 505, 534, 492, 491, 504, 506, 507, 487, 502, 512, 500, 501, 511, 507, 510, 464, 500, 534, 509, 539, 501, 538, 523, 537, 571, 489, 506, 508, 482, 535, 526, 481, 555, 510, 511, 561, 530, 466, 524, 520, 536, 493, 504, 531, 501, 513, 513, 500, 480, 507, 476, 489, 522, 490, 537, 510, 510, 501, 482, 516, 522, 502, 498, 528, 531, 518, 543, 510, 496, 547, 513, 485, 503, 508, 514, 500, 543, 502, 528, 529, 485, 496, 533, 479, 522, 515, 532, 525, 505, 519, 479, 531, 485, 544, 477, 467, 482, 525, 510, 519, 513, 539, 522, 516, 520, 509, 512, 494, 527, 513, 519, 524, 526, 476, 555, 529, 518, 512] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[543, 483, 478, 558, 516, 506, 501, 505, 481, 507, 494, 520, 529, 503, 478, 506, 534, 489, 503, 529, 529, 516, 539, 493, 508, 506, 538, 510, 511, 522, 548, 494, 475, 536, 498, 528, 492, 487, 545, 515, 551, 520, 498, 538, 467, 539, 497, 504, 491, 515, 488, 535, 503, 488, 523, 476, 508, 571, 499, 492, 522, 519, 502, 528, 499, 476, 524, 537, 500, 495, 538, 543, 511, 496, 552, 515, 514, 514, 508, 498, 507, 528, 533, 536, 530, 513, 503, 476, 553, 492, 507, 526, 517, 524, 476, 523, 531, 526, 474, 517, 495, 495, 520, 525, 548, 519, 534, 469, 545, 487, 491, 516, 459, 492, 565, 522, 517, 459, 511, 544, 525, 490, 480, 566, 497, 515, 508, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[472, 513, 541, 514, 506, 519, 503, 525, 531, 508, 496, 512, 499, 522, 501, 510, 494, 499, 490, 495, 512, 518, 506, 500, 493, 518, 519, 525, 511, 522, 495, 507, 474, 500, 520, 556, 503, 518, 504, 478, 538, 482, 533, 499, 509, 547, 486, 506, 507, 509, 506, 506, 522, 539, 529, 511, 510, 507, 524, 545, 529, 544, 521, 516, 458, 500, 501, 549, 496, 528, 494, 551, 500, 526, 515, 520, 502, 531, 515, 535, 519, 487, 490, 541, 522, 532, 490, 494, 526, 494, 520, 501, 542, 504, 485, 502, 513, 530, 520, 524, 504, 525, 528, 529, 519, 489, 493, 545, 527, 513, 530, 500, 482, 524, 486, 550, 540, 514, 490, 525, 504, 538, 457, 453, 494, 534, 521, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[528, 532, 521, 567, 518, 522, 561, 540, 532, 493, 521, 509, 520, 510, 554, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 511, 463, 492, 494, 493, 535, 505, 532, 534, 509, 520, 491, 491, 528, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 488, 514, 508, 482, 498, 482, 550, 474, 503, 496, 506, 501, 473, 503, 476, 14, 12, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 539, 472, 476, 559, 546, 503, 490, 495, 506, 503, 500, 473, 520, 507, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 513, 523, 523, 518, 499, 502, 495, 494, 480, 504, 479, 484, 533, 525, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 513, 477, 511, 528, 513, 472, 530, 524, 536, 519, 519, 508, 560, 519, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 550, 520, 533, 536, 513, 500, 509, 502, 493, 524, 529, 513, 498, 486, 516, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 506, 546, 518, 523, 487, 523, 496, 499, 518, 504, 502, 533, 494, 499, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8443, 8141, 8034, 8084, 8081, 8234, 8310, 8209] avg=8192.0000 max=8443 max_rank=0 min=8034 min_rank=2 imbalance_ratio=1.030640 max_load_reduction=57093 max_load_reduction_pct=0.871170 moved_to_echo=32834 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[521, 528, 494, 499, 541, 501, 509, 523, 514, 506, 523, 556, 476, 497, 518, 531, 570, 499, 530, 505, 493, 533, 516, 496, 504, 500, 479, 542, 526, 496, 491, 514, 503, 518, 524, 517, 527, 499, 499, 535, 480, 543, 472, 500, 506, 499, 483, 495, 530, 550, 519, 511, 543, 518, 519, 512, 543, 556, 463, 519, 566, 511, 545, 456, 518, 495, 552, 529, 534, 503, 483, 499, 497, 525, 485, 526, 500, 514, 540, 551, 526, 519, 494, 524, 538, 524, 490, 506, 512, 479, 510, 506, 529, 533, 524, 520, 498, 503, 492, 485, 510, 491, 486, 478, 507, 532, 511, 467, 534, 472, 510, 480, 499, 551, 497, 485, 521, 517, 534, 525, 508, 524, 490, 523, 489, 560, 489, 491] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[534, 510, 487, 532, 521, 535, 497, 515, 483, 486, 467, 510, 513, 501, 498, 522, 542, 482, 525, 505, 530, 542, 500, 508, 513, 519, 530, 469, 535, 515, 512, 517, 475, 539, 509, 512, 547, 530, 495, 511, 499, 518, 500, 551, 506, 477, 501, 498, 509, 506, 523, 501, 506, 533, 511, 519, 504, 509, 535, 494, 519, 551, 508, 484, 481, 478, 502, 507, 472, 501, 490, 515, 537, 526, 538, 511, 473, 529, 539, 549, 548, 513, 518, 554, 516, 561, 578, 512, 520, 493, 489, 510, 510, 493, 532, 536, 499, 524, 523, 480, 502, 475, 508, 500, 515, 476, 507, 527, 530, 527, 496, 490, 495, 492, 535, 485, 517, 479, 512, 477, 514, 508, 516, 542, 524, 555, 515, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[528, 532, 521, 567, 530, 522, 561, 540, 532, 493, 521, 509, 520, 510, 554, 515, 542, 511, 463, 492, 494, 493, 535, 505, 532, 534, 509, 520, 491, 491, 528, 515, 543, 488, 514, 508, 482, 498, 482, 550, 474, 503, 496, 506, 501, 473, 503, 476, 512, 539, 472, 476, 559, 546, 503, 490, 495, 506, 503, 500, 473, 520, 507, 483, 513, 513, 523, 523, 518, 499, 502, 495, 494, 480, 504, 479, 484, 533, 525, 496, 489, 513, 477, 561, 528, 513, 472, 530, 524, 536, 519, 519, 508, 560, 519, 516, 545, 550, 520, 533, 536, 513, 500, 509, 502, 493, 524, 529, 513, 498, 486, 516, 523, 506, 550, 518, 523, 487, 523, 496, 499, 518, 504, 502, 533, 494, 499, 538] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[491, 532, 526, 512, 502, 541, 462, 556, 527, 485, 522, 522, 511, 537, 515, 478, 526, 538, 530, 539, 533, 492, 526, 516, 527, 490, 469, 498, 543, 504, 525, 521, 498, 487, 499, 509, 514, 582, 541, 499, 521, 501, 499, 497, 467, 471, 542, 548, 531, 527, 549, 508, 512, 477, 539, 542, 488, 530, 518, 513, 507, 488, 460, 495, 463, 518, 516, 538, 507, 509, 489, 521, 524, 526, 506, 491, 525, 490, 565, 532, 511, 519, 492, 520, 530, 562, 491, 524, 498, 513, 494, 496, 506, 475, 476, 488, 533, 531, 496, 512, 468, 482, 498, 525, 509, 512, 494, 507, 518, 501, 498, 498, 529, 541, 528, 508, 506, 526, 528, 569, 499, 525, 480, 493, 525, 509, 502, 516] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[544, 517, 507, 495, 535, 514, 504, 510, 510, 509, 505, 534, 492, 491, 504, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 487, 502, 512, 500, 501, 511, 507, 510, 464, 500, 534, 509, 539, 501, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 537, 571, 489, 506, 508, 482, 535, 526, 481, 555, 510, 511, 561, 530, 466, 17, 17, 10, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 520, 536, 493, 504, 531, 501, 513, 513, 493, 480, 507, 476, 489, 522, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 510, 510, 501, 482, 516, 522, 502, 498, 528, 531, 518, 543, 510, 496, 547, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 485, 503, 455, 514, 500, 543, 502, 528, 529, 485, 496, 533, 479, 522, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 525, 505, 519, 479, 531, 485, 544, 477, 467, 482, 525, 510, 519, 513, 539, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 516, 513, 509, 512, 494, 527, 513, 519, 524, 526, 476, 555, 529, 518, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8105, 8346, 8092, 8254, 8102, 8195, 8265] avg=8192.0000 max=8346 max_rank=2 min=8092 min_rank=3 imbalance_ratio=1.018799 max_load_reduction=57190 max_load_reduction_pct=0.872650 moved_to_echo=32816 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[534, 510, 487, 532, 509, 535, 497, 515, 483, 486, 467, 510, 513, 501, 498, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 482, 525, 505, 530, 542, 500, 508, 513, 519, 530, 469, 535, 515, 512, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 539, 509, 512, 547, 530, 495, 511, 499, 518, 500, 551, 506, 477, 501, 498, 14, 12, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 506, 523, 501, 506, 533, 511, 519, 504, 509, 535, 494, 519, 551, 508, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 478, 502, 507, 472, 501, 490, 515, 537, 526, 538, 511, 473, 529, 539, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 513, 518, 504, 516, 561, 578, 512, 520, 493, 489, 510, 510, 493, 532, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 524, 523, 480, 502, 475, 508, 500, 515, 476, 507, 527, 530, 527, 496, 490, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 492, 531, 485, 517, 479, 512, 477, 514, 508, 516, 542, 524, 555, 515, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8099, 8230, 8205, 8212, 8148, 8333, 8122, 8187] avg=8192.0000 max=8333 max_rank=5 min=8099 min_rank=0 imbalance_ratio=1.017212 max_load_reduction=57203 max_load_reduction_pct=0.872849 moved_to_echo=32790 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[472, 513, 541, 514, 495, 519, 503, 525, 531, 508, 496, 512, 499, 522, 501, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 499, 490, 495, 512, 518, 506, 500, 493, 518, 519, 525, 511, 522, 495, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 500, 520, 556, 503, 518, 504, 478, 538, 482, 533, 499, 509, 547, 486, 506, 13, 11, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 509, 506, 506, 522, 539, 529, 511, 510, 507, 524, 545, 529, 544, 521, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 458, 500, 501, 549, 496, 528, 494, 551, 500, 526, 515, 520, 502, 531, 515, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 487, 490, 493, 522, 532, 490, 494, 526, 494, 520, 501, 542, 504, 485, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 530, 520, 524, 504, 525, 528, 529, 519, 489, 493, 545, 527, 513, 530, 500, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 524, 483, 550, 540, 514, 490, 525, 504, 538, 457, 453, 494, 534, 521, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8091, 8186, 8325, 8221, 8101, 8331, 8120] avg=8192.0000 max=8331 max_rank=6 min=8091 min_rank=1 imbalance_ratio=1.016968 max_load_reduction=57205 max_load_reduction_pct=0.872879 moved_to_echo=32773 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[509, 518, 549, 476, 535, 493, 503, 530, 467, 491, 520, 533, 521, 508, 479, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 525, 527, 483, 531, 474, 536, 541, 506, 540, 519, 530, 509, 524, 529, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 481, 490, 498, 494, 543, 543, 525, 524, 500, 498, 515, 515, 501, 482, 542, 14, 12, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 503, 531, 500, 479, 537, 499, 511, 503, 491, 530, 520, 489, 506, 500, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 511, 527, 474, 515, 487, 522, 520, 489, 520, 518, 495, 504, 528, 491, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 532, 543, 451, 538, 532, 526, 493, 532, 513, 550, 496, 525, 522, 571, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 519, 534, 511, 514, 542, 491, 514, 516, 516, 505, 461, 509, 496, 519, 494, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 506, 535, 520, 529, 485, 501, 515, 510, 496, 467, 516, 522, 511, 486, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8127, 8272, 8192, 8118, 8141, 8332, 8227, 8127] avg=8192.0000 max=8332 max_rank=5 min=8118 min_rank=3 imbalance_ratio=1.017090 max_load_reduction=57204 max_load_reduction_pct=0.872864 moved_to_echo=32827 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[543, 483, 478, 558, 504, 506, 501, 505, 481, 507, 494, 520, 529, 503, 478, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 489, 503, 529, 529, 516, 539, 493, 508, 506, 538, 510, 511, 522, 548, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 536, 498, 528, 492, 487, 545, 515, 551, 520, 498, 538, 467, 539, 497, 504, 14, 12, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 515, 488, 535, 503, 488, 523, 476, 508, 571, 499, 492, 522, 519, 502, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 476, 524, 537, 500, 495, 538, 543, 511, 496, 552, 515, 514, 514, 508, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 528, 533, 489, 530, 513, 503, 476, 553, 492, 507, 526, 517, 524, 476, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 526, 474, 517, 495, 495, 520, 525, 548, 519, 534, 469, 545, 487, 491, 516, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 459, 492, 561, 522, 517, 459, 511, 544, 525, 490, 480, 566, 497, 515, 508, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8096, 8255, 8226, 8160, 8220, 8197, 8233, 8149] avg=8192.0000 max=8255 max_rank=1 min=8096 min_rank=0 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32799 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[491, 532, 526, 512, 491, 541, 462, 556, 527, 485, 522, 522, 511, 537, 515, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 538, 530, 539, 533, 492, 526, 516, 527, 490, 469, 498, 543, 504, 525, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 487, 499, 509, 514, 582, 541, 499, 521, 501, 499, 497, 467, 471, 542, 548, 14, 11, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 527, 549, 508, 512, 477, 539, 542, 488, 530, 518, 513, 507, 488, 460, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 463, 518, 516, 538, 507, 509, 489, 521, 524, 526, 506, 491, 525, 490, 565, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 519, 492, 474, 530, 562, 491, 524, 498, 513, 494, 496, 506, 475, 476, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 531, 496, 512, 468, 482, 498, 525, 509, 512, 494, 507, 518, 501, 498, 498, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 541, 524, 508, 506, 526, 528, 569, 499, 525, 480, 493, 525, 509, 502, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8263, 8210, 8184, 8220, 8049, 8122, 8280] avg=8192.0000 max=8280 max_rank=7 min=8049 min_rank=5 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32671 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[521, 528, 494, 499, 529, 501, 509, 523, 514, 506, 523, 556, 476, 497, 518, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 499, 530, 505, 493, 533, 516, 496, 504, 500, 479, 542, 526, 496, 491, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 518, 524, 517, 527, 499, 499, 535, 480, 543, 472, 500, 506, 499, 483, 495, 15, 12, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 550, 519, 511, 543, 518, 519, 512, 543, 556, 463, 519, 566, 511, 545, 456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 495, 552, 529, 534, 503, 483, 499, 497, 525, 485, 526, 500, 514, 540, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 519, 494, 478, 538, 524, 490, 506, 512, 479, 510, 506, 529, 533, 524, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 503, 492, 485, 510, 491, 486, 478, 507, 532, 511, 467, 534, 472, 510, 480, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 551, 493, 485, 521, 517, 534, 525, 508, 524, 490, 523, 489, 560, 489, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 8179, 8137, 8361, 8251, 8188, 7996, 8199] avg=8192.0000 max=8361 max_rank=3 min=7996 min_rank=6 imbalance_ratio=1.020630 max_load_reduction=57175 max_load_reduction_pct=0.872421 moved_to_echo=32634 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[509, 518, 549, 476, 535, 493, 503, 530, 467, 491, 520, 533, 521, 508, 479, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 525, 527, 483, 531, 474, 536, 541, 506, 540, 519, 530, 509, 524, 529, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 481, 490, 498, 494, 543, 543, 525, 524, 500, 498, 515, 515, 501, 482, 542, 14, 12, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 503, 531, 500, 479, 537, 499, 511, 503, 491, 530, 520, 489, 506, 500, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 511, 527, 474, 515, 487, 522, 520, 489, 520, 518, 495, 504, 528, 491, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 532, 543, 451, 538, 532, 526, 493, 532, 513, 550, 496, 525, 522, 571, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 519, 534, 511, 514, 542, 491, 514, 516, 516, 505, 461, 509, 496, 519, 494, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 506, 535, 520, 529, 485, 501, 515, 510, 496, 467, 516, 522, 511, 486, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8127, 8272, 8192, 8118, 8141, 8332, 8227, 8127] avg=8192.0000 max=8332 max_rank=5 min=8118 min_rank=3 imbalance_ratio=1.017090 max_load_reduction=57204 max_load_reduction_pct=0.872864 moved_to_echo=32827 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[472, 513, 541, 514, 495, 519, 503, 525, 531, 508, 496, 512, 499, 522, 501, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 499, 490, 495, 512, 518, 506, 500, 493, 518, 519, 525, 511, 522, 495, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 500, 520, 556, 503, 518, 504, 478, 538, 482, 533, 499, 509, 547, 486, 506, 13, 11, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 509, 506, 506, 522, 539, 529, 511, 510, 507, 524, 545, 529, 544, 521, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 458, 500, 501, 549, 496, 528, 494, 551, 500, 526, 515, 520, 502, 531, 515, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 487, 490, 493, 522, 532, 490, 494, 526, 494, 520, 501, 542, 504, 485, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 530, 520, 524, 504, 525, 528, 529, 519, 489, 493, 545, 527, 513, 530, 500, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 524, 483, 550, 540, 514, 490, 525, 504, 538, 457, 453, 494, 534, 521, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8091, 8186, 8325, 8221, 8101, 8331, 8120] avg=8192.0000 max=8331 max_rank=6 min=8091 min_rank=1 imbalance_ratio=1.016968 max_load_reduction=57205 max_load_reduction_pct=0.872879 moved_to_echo=32773 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[534, 510, 487, 532, 509, 535, 497, 515, 483, 486, 467, 510, 513, 501, 498, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 482, 525, 505, 530, 542, 500, 508, 513, 519, 530, 469, 535, 515, 512, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 539, 509, 512, 547, 530, 495, 511, 499, 518, 500, 551, 506, 477, 501, 498, 14, 12, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 506, 523, 501, 506, 533, 511, 519, 504, 509, 535, 494, 519, 551, 508, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 478, 502, 507, 472, 501, 490, 515, 537, 526, 538, 511, 473, 529, 539, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 513, 518, 504, 516, 561, 578, 512, 520, 493, 489, 510, 510, 493, 532, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 524, 523, 480, 502, 475, 508, 500, 515, 476, 507, 527, 530, 527, 496, 490, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 492, 531, 485, 517, 479, 512, 477, 514, 508, 516, 542, 524, 555, 515, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8099, 8230, 8205, 8212, 8148, 8333, 8122, 8187] avg=8192.0000 max=8333 max_rank=5 min=8099 min_rank=0 imbalance_ratio=1.017212 max_load_reduction=57203 max_load_reduction_pct=0.872849 moved_to_echo=32790 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[528, 532, 521, 567, 518, 522, 561, 540, 532, 493, 521, 509, 520, 510, 554, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 511, 463, 492, 494, 493, 535, 505, 532, 534, 509, 520, 491, 491, 528, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 488, 514, 508, 482, 498, 482, 550, 474, 503, 496, 506, 501, 473, 503, 476, 14, 12, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 539, 472, 476, 559, 546, 503, 490, 495, 506, 503, 500, 473, 520, 507, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 513, 523, 523, 518, 499, 502, 495, 494, 480, 504, 479, 484, 533, 525, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 513, 477, 511, 528, 513, 472, 530, 524, 536, 519, 519, 508, 560, 519, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 550, 520, 533, 536, 513, 500, 509, 502, 493, 524, 529, 513, 498, 486, 516, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 506, 546, 518, 523, 487, 523, 496, 499, 518, 504, 502, 533, 494, 499, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8443, 8141, 8034, 8084, 8081, 8234, 8310, 8209] avg=8192.0000 max=8443 max_rank=0 min=8034 min_rank=2 imbalance_ratio=1.030640 max_load_reduction=57093 max_load_reduction_pct=0.871170 moved_to_echo=32834 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[491, 532, 526, 512, 491, 541, 462, 556, 527, 485, 522, 522, 511, 537, 515, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 538, 530, 539, 533, 492, 526, 516, 527, 490, 469, 498, 543, 504, 525, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 487, 499, 509, 514, 582, 541, 499, 521, 501, 499, 497, 467, 471, 542, 548, 14, 11, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 527, 549, 508, 512, 477, 539, 542, 488, 530, 518, 513, 507, 488, 460, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 463, 518, 516, 538, 507, 509, 489, 521, 524, 526, 506, 491, 525, 490, 565, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 519, 492, 474, 530, 562, 491, 524, 498, 513, 494, 496, 506, 475, 476, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 531, 496, 512, 468, 482, 498, 525, 509, 512, 494, 507, 518, 501, 498, 498, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 541, 524, 508, 506, 526, 528, 569, 499, 525, 480, 493, 525, 509, 502, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8263, 8210, 8184, 8220, 8049, 8122, 8280] avg=8192.0000 max=8280 max_rank=7 min=8049 min_rank=5 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32671 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4352, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 3968, 4224, 4224, 4096, 4224, 3968, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 3968, 4224, 4352, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4352, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[515, 492, 516, 510, 514, 494, 511, 520, 486, 536, 525, 482, 478, 489, 523, 494, 498, 492, 511, 519, 523, 483, 495, 530, 529, 500, 534, 490, 475, 515, 494, 522, 533, 515, 563, 465, 505, 538, 534, 485, 519, 507, 571, 536, 539, 497, 573, 545, 530, 553, 518, 543, 491, 507, 494, 483, 529, 518, 507, 485, 531, 520, 506, 492, 531, 510, 499, 452, 537, 486, 489, 537, 543, 511, 528, 476, 524, 545, 496, 466, 527, 502, 478, 534, 538, 523, 521, 514, 525, 514, 506, 521, 495, 558, 511, 529, 498, 512, 514, 490, 527, 532, 528, 510, 492, 515, 539, 480, 492, 522, 521, 543, 488, 469, 518, 480, 523, 533, 489, 514, 517, 497, 475, 492, 511, 521, 521, 520] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[464, 530, 511, 537, 533, 540, 473, 553, 583, 533, 497, 503, 526, 537, 507, 527, 504, 511, 520, 504, 551, 516, 490, 527, 519, 529, 514, 474, 496, 470, 519, 514, 531, 504, 497, 492, 536, 527, 510, 457, 509, 529, 516, 490, 506, 504, 534, 552, 511, 481, 499, 529, 513, 553, 489, 532, 511, 495, 456, 466, 542, 506, 519, 541, 509, 496, 515, 503, 511, 499, 518, 491, 497, 536, 488, 498, 513, 516, 483, 501, 504, 505, 512, 487, 547, 515, 497, 506, 535, 495, 495, 511, 505, 537, 541, 512, 515, 504, 492, 467, 519, 481, 533, 507, 505, 505, 525, 531, 519, 529, 511, 507, 497, 530, 504, 487, 528, 517, 529, 516, 503, 527, 518, 499, 518, 536, 501, 549] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[488, 505, 527, 485, 526, 517, 536, 513, 490, 508, 533, 486, 517, 545, 485, 510, 495, 538, 541, 488, 488, 533, 559, 535, 527, 551, 501, 546, 470, 510, 479, 519, 531, 527, 566, 513, 504, 504, 528, 502, 460, 535, 501, 530, 486, 497, 537, 500, 511, 532, 455, 509, 530, 537, 524, 512, 492, 537, 502, 543, 518, 468, 461, 542, 513, 484, 493, 538, 516, 494, 514, 491, 503, 530, 517, 533, 501, 499, 548, 548, 523, 517, 509, 525, 494, 482, 470, 513, 515, 519, 502, 560, 515, 512, 511, 520, 499, 527, 514, 488, 489, 488, 508, 505, 496, 505, 518, 579, 529, 508, 457, 518, 508, 488, 525, 508, 508, 498, 522, 544, 507, 499, 453, 506, 517, 522, 522, 527] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[506, 529, 527, 532, 517, 488, 487, 568, 511, 520, 513, 540, 515, 527, 507, 485, 506, 510, 502, 470, 521, 520, 533, 514, 497, 530, 488, 488, 487, 529, 511, 451, 504, 517, 491, 508, 468, 541, 520, 562, 499, 534, 549, 480, 513, 506, 525, 488, 516, 507, 556, 497, 497, 527, 521, 474, 546, 517, 510, 480, 514, 507, 514, 537, 511, 541, 533, 482, 535, 487, 507, 519, 536, 526, 521, 499, 482, 507, 522, 477, 524, 484, 492, 495, 517, 510, 521, 558, 515, 502, 536, 521, 510, 502, 514, 527, 515, 466, 509, 496, 548, 533, 496, 509, 527, 518, 505, 489, 483, 521, 547, 508, 558, 526, 506, 514, 499, 506, 507, 521, 510, 533, 500, 494, 505, 529, 495, 496] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[527, 531, 493, 495, 476, 566, 554, 496, 523, 520, 511, 476, 518, 508, 498, 520, 513, 507, 498, 510, 509, 511, 515, 482, 542, 513, 510, 492, 520, 516, 521, 529, 494, 492, 563, 500, 500, 531, 521, 542, 498, 553, 522, 488, 542, 512, 495, 536, 544, 524, 487, 505, 503, 531, 533, 482, 506, 503, 521, 505, 502, 450, 507, 522, 492, 493, 528, 478, 518, 536, 522, 529, 495, 499, 512, 486, 519, 475, 523, 504, 508, 495, 540, 538, 539, 516, 530, 492, 518, 492, 524, 511, 489, 518, 527, 532, 473, 462, 487, 471, 509, 497, 496, 531, 497, 475, 520, 490, 557, 537, 505, 516, 542, 518, 520, 524, 518, 468, 521, 599, 539, 519, 534, 512, 521, 512, 486, 488] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[515, 492, 516, 510, 514, 494, 511, 520, 486, 536, 525, 482, 478, 489, 523, 494, 498, 492, 511, 519, 523, 483, 495, 530, 529, 500, 534, 490, 475, 515, 494, 522, 533, 515, 563, 465, 505, 538, 534, 485, 519, 507, 571, 536, 539, 497, 573, 545, 530, 553, 518, 543, 491, 507, 494, 483, 529, 518, 507, 485, 531, 520, 506, 492, 531, 510, 499, 452, 537, 486, 489, 537, 543, 511, 528, 476, 524, 545, 496, 466, 527, 502, 478, 534, 538, 523, 521, 514, 525, 514, 506, 521, 495, 558, 511, 529, 498, 512, 514, 490, 527, 532, 528, 510, 492, 515, 539, 480, 492, 522, 521, 543, 488, 469, 518, 480, 523, 533, 489, 514, 517, 497, 475, 492, 511, 521, 521, 520] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[540, 591, 502, 528, 521, 482, 548, 506, 486, 546, 529, 527, 507, 499, 535, 509, 543, 503, 521, 515, 540, 481, 570, 494, 541, 506, 486, 463, 493, 512, 480, 507, 543, 515, 493, 536, 505, 532, 495, 537, 485, 478, 505, 484, 496, 543, 501, 528, 537, 560, 494, 513, 471, 562, 530, 518, 490, 477, 517, 493, 471, 511, 530, 485, 509, 497, 488, 506, 496, 475, 508, 521, 507, 517, 476, 511, 536, 497, 503, 494, 523, 466, 491, 560, 506, 480, 507, 507, 527, 541, 548, 536, 546, 479, 498, 515, 498, 512, 515, 462, 519, 505, 502, 515, 525, 536, 534, 495, 537, 528, 507, 527, 537, 533, 486, 491, 528, 513, 500, 514, 503, 499, 526, 498, 475, 571, 498, 530] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[525, 526, 498, 549, 497, 534, 531, 490, 537, 517, 503, 532, 538, 492, 488, 534, 519, 510, 512, 465, 558, 517, 521, 519, 527, 501, 458, 508, 530, 510, 510, 514, 507, 496, 517, 517, 505, 522, 492, 519, 501, 547, 513, 534, 513, 495, 542, 516, 512, 514, 518, 541, 488, 523, 515, 497, 475, 521, 492, 513, 538, 492, 525, 510, 496, 487, 500, 525, 494, 541, 515, 523, 518, 468, 481, 491, 500, 502, 496, 497, 556, 537, 510, 528, 493, 520, 498, 469, 512, 534, 538, 490, 502, 499, 494, 510, 520, 494, 520, 518, 505, 532, 507, 509, 495, 495, 500, 529, 525, 534, 502, 532, 507, 524, 495, 492, 490, 529, 565, 490, 544, 499, 528, 502, 513, 539, 483, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[544, 516, 531, 510, 505, 509, 511, 519, 525, 489, 494, 501, 501, 533, 545, 515, 502, 513, 501, 496, 524, 523, 546, 539, 484, 509, 521, 496, 526, 496, 536, 471, 564, 534, 558, 525, 548, 548, 482, 521, 509, 538, 509, 491, 489, 471, 554, 525, 529, 493, 503, 514, 537, 518, 520, 517, 485, 513, 535, 547, 513, 523, 537, 524, 536, 535, 510, 509, 445, 518, 522, 486, 541, 550, 515, 504, 523, 496, 476, 497, 459, 478, 509, 483, 531, 490, 529, 468, 508, 513, 478, 538, 473, 472, 519, 511, 553, 513, 493, 515, 528, 498, 533, 504, 486, 551, 521, 498, 481, 497, 515, 506, 554, 520, 505, 480, 509, 483, 451, 535, 495, 537, 548, 467, 510, 544, 482, 497] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[540, 591, 502, 528, 521, 482, 548, 506, 486, 546, 529, 527, 507, 499, 535, 509, 543, 503, 521, 515, 540, 481, 570, 494, 541, 506, 486, 463, 493, 512, 480, 507, 543, 515, 493, 536, 505, 532, 495, 537, 485, 478, 505, 484, 496, 543, 501, 528, 537, 560, 494, 513, 471, 562, 530, 518, 490, 477, 517, 493, 471, 511, 530, 485, 509, 497, 488, 506, 496, 475, 508, 521, 507, 517, 476, 511, 536, 497, 503, 494, 523, 466, 491, 560, 506, 480, 507, 507, 527, 541, 548, 536, 546, 479, 498, 515, 498, 512, 515, 462, 519, 505, 502, 515, 525, 536, 534, 495, 537, 528, 507, 527, 537, 533, 486, 491, 528, 513, 500, 514, 503, 499, 526, 498, 475, 571, 498, 530] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[506, 529, 527, 532, 517, 488, 487, 568, 511, 520, 513, 540, 515, 527, 507, 485, 506, 510, 502, 470, 521, 520, 533, 514, 497, 530, 488, 488, 487, 529, 511, 451, 504, 517, 491, 508, 468, 541, 520, 562, 499, 534, 549, 480, 513, 506, 525, 488, 516, 507, 556, 497, 497, 527, 521, 474, 546, 517, 510, 480, 514, 507, 514, 537, 511, 541, 533, 482, 535, 487, 507, 519, 536, 526, 521, 499, 482, 507, 522, 477, 524, 484, 492, 495, 517, 510, 521, 558, 515, 502, 536, 521, 510, 502, 514, 527, 515, 466, 509, 496, 548, 533, 496, 509, 527, 518, 505, 489, 483, 521, 547, 508, 558, 526, 506, 514, 499, 506, 507, 521, 510, 533, 500, 494, 505, 529, 495, 496] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[488, 452, 527, 485, 526, 517, 536, 513, 490, 508, 533, 486, 517, 545, 485, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 538, 541, 488, 488, 533, 559, 535, 527, 551, 501, 546, 470, 510, 479, 519, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 527, 566, 513, 504, 504, 528, 502, 460, 535, 501, 530, 486, 497, 465, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 532, 455, 509, 530, 537, 524, 512, 492, 537, 502, 543, 518, 468, 461, 542, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 484, 493, 538, 516, 494, 514, 491, 503, 530, 517, 533, 501, 499, 548, 548, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 517, 509, 525, 494, 482, 470, 513, 515, 519, 502, 557, 515, 512, 511, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 527, 514, 488, 489, 488, 508, 505, 496, 505, 518, 579, 529, 508, 457, 518, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 488, 525, 508, 508, 498, 522, 544, 507, 499, 453, 506, 517, 522, 522, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8118, 8307, 8149, 8176, 8294, 8184, 8154, 8154] avg=8192.0000 max=8307 max_rank=1 min=8118 min_rank=0 imbalance_ratio=1.014038 max_load_reduction=57229 max_load_reduction_pct=0.873245 moved_to_echo=32786 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[464, 474, 511, 537, 533, 540, 473, 553, 583, 533, 497, 503, 526, 537, 507, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 511, 520, 504, 551, 516, 490, 527, 519, 529, 514, 474, 496, 470, 519, 514, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 504, 497, 492, 536, 527, 510, 457, 509, 529, 516, 490, 506, 504, 462, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 481, 499, 529, 513, 553, 489, 532, 511, 495, 456, 466, 542, 506, 519, 541, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 496, 515, 503, 511, 499, 518, 491, 497, 536, 488, 498, 513, 516, 483, 501, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 505, 512, 487, 547, 515, 497, 506, 535, 495, 495, 508, 505, 537, 541, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 504, 492, 467, 519, 481, 533, 507, 505, 505, 525, 531, 519, 529, 511, 507, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 530, 504, 487, 528, 517, 529, 516, 503, 527, 518, 499, 518, 536, 501, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8298, 8186, 8122, 8146, 8146, 8201, 8178, 8259] avg=8192.0000 max=8298 max_rank=0 min=8122 min_rank=2 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32784 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[488, 505, 527, 485, 526, 517, 536, 513, 490, 508, 533, 486, 517, 545, 485, 510, 495, 538, 541, 488, 488, 533, 559, 535, 527, 551, 501, 546, 470, 510, 479, 519, 531, 527, 566, 513, 504, 504, 528, 502, 460, 535, 501, 530, 486, 497, 537, 500, 511, 532, 455, 509, 530, 537, 524, 512, 492, 537, 502, 543, 518, 468, 461, 542, 513, 484, 493, 538, 516, 494, 514, 491, 503, 530, 517, 533, 501, 499, 548, 548, 523, 517, 509, 525, 494, 482, 470, 513, 515, 519, 502, 560, 515, 512, 511, 520, 499, 527, 514, 488, 489, 488, 508, 505, 496, 505, 518, 579, 529, 508, 457, 518, 508, 488, 525, 508, 508, 498, 522, 544, 507, 499, 453, 506, 517, 522, 522, 527] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[506, 473, 527, 532, 517, 488, 487, 568, 511, 520, 513, 540, 515, 527, 507, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 510, 502, 470, 521, 520, 533, 514, 497, 530, 488, 488, 487, 529, 511, 451, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 517, 491, 508, 468, 541, 520, 562, 499, 534, 549, 480, 513, 506, 455, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 507, 556, 497, 497, 527, 521, 474, 546, 517, 510, 480, 514, 507, 514, 537, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 541, 533, 482, 535, 487, 507, 519, 536, 526, 521, 499, 482, 507, 522, 477, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 484, 492, 495, 517, 510, 521, 558, 515, 502, 536, 518, 510, 502, 514, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 466, 509, 496, 548, 533, 496, 509, 527, 518, 505, 489, 483, 521, 547, 508, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 558, 526, 506, 514, 499, 506, 507, 521, 510, 533, 500, 494, 505, 529, 495, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8085, 8135, 8223, 8255, 8225, 8198, 8199] avg=8192.0000 max=8255 max_rank=4 min=8085 min_rank=1 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32877 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[515, 431, 516, 510, 514, 494, 511, 520, 486, 536, 525, 482, 478, 489, 523, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 492, 511, 519, 523, 483, 495, 530, 529, 500, 534, 490, 475, 515, 494, 522, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 515, 563, 465, 505, 538, 534, 485, 519, 507, 571, 536, 539, 497, 491, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 553, 518, 543, 491, 507, 494, 483, 529, 518, 507, 485, 531, 520, 506, 492, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 510, 499, 452, 537, 486, 489, 537, 543, 511, 528, 476, 524, 545, 496, 466, 82, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 502, 478, 534, 538, 523, 521, 514, 525, 514, 506, 515, 495, 558, 511, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 512, 514, 490, 527, 532, 528, 510, 492, 515, 539, 480, 492, 522, 521, 543, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 469, 518, 480, 523, 533, 489, 514, 517, 497, 475, 492, 511, 521, 521, 520, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8024, 8139, 8343, 8213, 8214, 8290, 8244, 8069] avg=8192.0000 max=8343 max_rank=2 min=8024 min_rank=0 imbalance_ratio=1.018433 max_load_reduction=57193 max_load_reduction_pct=0.872696 moved_to_echo=32817 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[540, 529, 502, 528, 521, 482, 548, 506, 486, 546, 529, 527, 507, 499, 535, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 503, 521, 515, 540, 481, 570, 494, 541, 506, 486, 463, 493, 512, 480, 507, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 515, 493, 536, 505, 532, 495, 537, 485, 478, 505, 484, 496, 543, 434, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 560, 494, 513, 471, 562, 530, 518, 490, 477, 517, 493, 471, 511, 530, 485, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 497, 488, 506, 496, 475, 508, 521, 507, 517, 476, 511, 536, 497, 503, 494, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 466, 491, 560, 506, 480, 507, 507, 527, 541, 548, 533, 546, 479, 498, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 512, 515, 462, 519, 505, 502, 515, 525, 536, 534, 495, 537, 528, 507, 527, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 533, 486, 491, 528, 513, 500, 514, 503, 499, 526, 498, 475, 571, 498, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8294, 8186, 8109, 8162, 8108, 8227, 8248, 8202] avg=8192.0000 max=8294 max_rank=0 min=8108 min_rank=4 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32785 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[527, 475, 493, 495, 476, 566, 554, 496, 523, 520, 511, 476, 518, 508, 498, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 507, 498, 510, 509, 511, 515, 482, 542, 513, 510, 492, 520, 516, 521, 529, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 492, 563, 500, 500, 531, 521, 542, 498, 553, 522, 488, 542, 512, 429, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 524, 487, 505, 503, 531, 533, 482, 506, 503, 521, 505, 502, 450, 507, 522, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 493, 528, 478, 518, 536, 522, 529, 495, 499, 512, 486, 519, 475, 523, 504, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 495, 540, 538, 539, 516, 530, 492, 518, 492, 524, 508, 489, 518, 527, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 462, 487, 471, 509, 497, 496, 531, 497, 475, 520, 490, 557, 537, 505, 516, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 518, 520, 524, 518, 468, 521, 599, 539, 519, 534, 512, 521, 512, 486, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8156, 8216, 8223, 8128, 8175, 8266, 8051, 8321] avg=8192.0000 max=8321 max_rank=7 min=8051 min_rank=6 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32813 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[464, 474, 511, 537, 533, 540, 473, 553, 583, 533, 497, 503, 526, 537, 507, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 511, 520, 504, 551, 516, 490, 527, 519, 529, 514, 474, 496, 470, 519, 514, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 504, 497, 492, 536, 527, 510, 457, 509, 529, 516, 490, 506, 504, 462, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 481, 499, 529, 513, 553, 489, 532, 511, 495, 456, 466, 542, 506, 519, 541, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 496, 515, 503, 511, 499, 518, 491, 497, 536, 488, 498, 513, 516, 483, 501, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 505, 512, 487, 547, 515, 497, 506, 535, 495, 495, 508, 505, 537, 541, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 504, 492, 467, 519, 481, 533, 507, 505, 505, 525, 531, 519, 529, 511, 507, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 530, 504, 487, 528, 517, 529, 516, 503, 527, 518, 499, 518, 536, 501, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8298, 8186, 8122, 8146, 8146, 8201, 8178, 8259] avg=8192.0000 max=8298 max_rank=0 min=8122 min_rank=2 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32784 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[540, 529, 502, 528, 521, 482, 548, 506, 486, 546, 529, 527, 507, 499, 535, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 503, 521, 515, 540, 481, 570, 494, 541, 506, 486, 463, 493, 512, 480, 507, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 515, 493, 536, 505, 532, 495, 537, 485, 478, 505, 484, 496, 543, 434, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 560, 494, 513, 471, 562, 530, 518, 490, 477, 517, 493, 471, 511, 530, 485, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 497, 488, 506, 496, 475, 508, 521, 507, 517, 476, 511, 536, 497, 503, 494, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 466, 491, 560, 506, 480, 507, 507, 527, 541, 548, 533, 546, 479, 498, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 512, 515, 462, 519, 505, 502, 515, 525, 536, 534, 495, 537, 528, 507, 527, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 533, 486, 491, 528, 513, 500, 514, 503, 499, 526, 498, 475, 571, 498, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8294, 8186, 8109, 8162, 8108, 8227, 8248, 8202] avg=8192.0000 max=8294 max_rank=0 min=8108 min_rank=4 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32785 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[544, 462, 531, 510, 505, 509, 511, 519, 525, 489, 494, 501, 501, 533, 545, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 513, 501, 496, 524, 523, 546, 539, 484, 509, 521, 496, 526, 496, 536, 471, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 564, 534, 558, 525, 548, 548, 482, 521, 509, 538, 509, 491, 489, 471, 480, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 493, 503, 514, 537, 518, 520, 517, 485, 513, 535, 547, 513, 523, 537, 524, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 535, 510, 509, 445, 518, 522, 486, 541, 550, 515, 504, 523, 496, 476, 497, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 459, 478, 509, 483, 531, 490, 529, 468, 508, 513, 478, 535, 473, 472, 519, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 513, 493, 515, 528, 498, 533, 504, 486, 551, 521, 498, 481, 497, 515, 506, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 520, 505, 480, 509, 483, 451, 535, 495, 537, 548, 467, 510, 544, 482, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8210, 8292, 8311, 8237, 7956, 8219, 8117] avg=8192.0000 max=8311 max_rank=3 min=7956 min_rank=5 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32529 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[488, 452, 527, 485, 526, 517, 536, 513, 490, 508, 533, 486, 517, 545, 485, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 538, 541, 488, 488, 533, 559, 535, 527, 551, 501, 546, 470, 510, 479, 519, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 527, 566, 513, 504, 504, 528, 502, 460, 535, 501, 530, 486, 497, 465, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 532, 455, 509, 530, 537, 524, 512, 492, 537, 502, 543, 518, 468, 461, 542, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 484, 493, 538, 516, 494, 514, 491, 503, 530, 517, 533, 501, 499, 548, 548, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 517, 509, 525, 494, 482, 470, 513, 515, 519, 502, 557, 515, 512, 511, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 527, 514, 488, 489, 488, 508, 505, 496, 505, 518, 579, 529, 508, 457, 518, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 488, 525, 508, 508, 498, 522, 544, 507, 499, 453, 506, 517, 522, 522, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8118, 8307, 8149, 8176, 8294, 8184, 8154, 8154] avg=8192.0000 max=8307 max_rank=1 min=8118 min_rank=0 imbalance_ratio=1.014038 max_load_reduction=57229 max_load_reduction_pct=0.873245 moved_to_echo=32786 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[525, 471, 498, 549, 497, 534, 531, 490, 537, 517, 503, 532, 538, 492, 488, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 510, 512, 465, 558, 517, 521, 519, 527, 501, 458, 508, 530, 510, 510, 514, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 496, 517, 517, 505, 522, 492, 519, 501, 547, 513, 534, 513, 495, 469, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 514, 518, 541, 488, 523, 515, 497, 475, 521, 492, 513, 538, 492, 525, 510, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 487, 500, 525, 494, 541, 515, 523, 518, 468, 481, 491, 500, 502, 496, 497, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 537, 510, 528, 493, 520, 498, 469, 512, 534, 538, 487, 502, 499, 494, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 494, 520, 518, 505, 532, 507, 509, 495, 495, 500, 529, 525, 534, 502, 532, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 524, 495, 492, 490, 529, 565, 490, 544, 499, 528, 502, 513, 539, 483, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8236, 8207, 8163, 8177, 8107, 8187, 8244, 8215] avg=8192.0000 max=8244 max_rank=6 min=8107 min_rank=4 imbalance_ratio=1.006348 max_load_reduction=57292 max_load_reduction_pct=0.874207 moved_to_echo=32753 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[506, 473, 527, 532, 517, 488, 487, 568, 511, 520, 513, 540, 515, 527, 507, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 510, 502, 470, 521, 520, 533, 514, 497, 530, 488, 488, 487, 529, 511, 451, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 517, 491, 508, 468, 541, 520, 562, 499, 534, 549, 480, 513, 506, 455, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 507, 556, 497, 497, 527, 521, 474, 546, 517, 510, 480, 514, 507, 514, 537, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 541, 533, 482, 535, 487, 507, 519, 536, 526, 521, 499, 482, 507, 522, 477, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 484, 492, 495, 517, 510, 521, 558, 515, 502, 536, 518, 510, 502, 514, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 466, 509, 496, 548, 533, 496, 509, 527, 518, 505, 489, 483, 521, 547, 508, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 558, 526, 506, 514, 499, 506, 507, 521, 510, 533, 500, 494, 505, 529, 495, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8085, 8135, 8223, 8255, 8225, 8198, 8199] avg=8192.0000 max=8255 max_rank=4 min=8085 min_rank=1 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32877 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 3968, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 3968, 4224, 4096, 4352, 4224, 4224, 4224, 4096, 3968, 4096, 4096, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4352, 4096, 4096, 4352, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 3712, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4352, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 3840, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4096, 4352, 4224, 4224, 4096, 4096, 4096, 4352, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4352, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 3968, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4352, 4096, 4096, 4352, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 3712, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[524, 538, 534, 535, 505, 507, 472, 523, 489, 467, 501, 509, 509, 519, 505, 513, 541, 512, 537, 542, 520, 511, 490, 499, 507, 547, 496, 549, 511, 540, 487, 524, 528, 494, 558, 515, 530, 525, 534, 509, 525, 526, 470, 500, 536, 498, 527, 480, 518, 534, 495, 536, 488, 494, 523, 545, 510, 505, 498, 515, 487, 528, 580, 534, 489, 520, 502, 515, 542, 521, 481, 562, 534, 514, 500, 481, 491, 507, 505, 505, 502, 514, 529, 536, 496, 502, 511, 488, 511, 462, 518, 506, 510, 504, 486, 497, 505, 497, 523, 503, 501, 498, 524, 523, 527, 533, 504, 504, 502, 511, 496, 519, 494, 506, 505, 517, 520, 522, 514, 523, 504, 488, 539, 530, 441, 497, 502, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[467, 519, 490, 496, 492, 504, 509, 518, 508, 488, 485, 522, 499, 545, 480, 531, 503, 510, 518, 526, 527, 504, 528, 549, 482, 517, 482, 514, 458, 479, 522, 493, 531, 519, 531, 508, 523, 480, 537, 527, 541, 462, 471, 494, 528, 465, 535, 458, 529, 530, 491, 507, 578, 513, 501, 501, 520, 533, 473, 485, 522, 497, 514, 521, 512, 542, 501, 560, 530, 501, 518, 522, 519, 512, 521, 541, 548, 538, 509, 506, 516, 511, 502, 516, 524, 477, 530, 518, 500, 514, 524, 499, 551, 554, 497, 509, 505, 518, 554, 506, 551, 524, 483, 512, 520, 527, 490, 510, 509, 543, 520, 515, 516, 502, 486, 498, 531, 489, 546, 492, 512, 510, 534, 468, 528, 472, 519, 534] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[499, 503, 517, 468, 535, 530, 543, 484, 496, 503, 527, 514, 535, 514, 504, 492, 537, 539, 494, 456, 519, 526, 498, 502, 524, 552, 524, 505, 531, 516, 497, 555, 496, 516, 513, 497, 480, 531, 534, 481, 496, 495, 516, 502, 530, 513, 517, 503, 518, 467, 525, 487, 505, 492, 529, 535, 500, 500, 494, 527, 490, 530, 501, 536, 496, 487, 478, 514, 535, 501, 506, 507, 493, 528, 561, 515, 481, 503, 519, 510, 474, 551, 496, 517, 501, 521, 500, 533, 483, 549, 513, 503, 528, 526, 515, 492, 518, 503, 481, 526, 503, 532, 507, 522, 510, 531, 491, 494, 556, 523, 491, 527, 523, 507, 498, 511, 508, 517, 486, 510, 500, 536, 527, 552, 492, 530, 533, 560] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[489, 477, 488, 512, 512, 529, 528, 528, 484, 506, 505, 513, 571, 501, 514, 529, 510, 530, 525, 500, 481, 514, 500, 559, 494, 540, 528, 509, 506, 539, 533, 550, 506, 516, 482, 513, 497, 523, 538, 511, 537, 501, 539, 479, 547, 520, 496, 530, 473, 494, 485, 498, 486, 480, 513, 550, 505, 480, 539, 536, 524, 522, 519, 511, 509, 481, 521, 515, 526, 474, 497, 562, 497, 509, 551, 544, 494, 513, 482, 483, 505, 469, 483, 557, 541, 512, 546, 493, 478, 566, 477, 520, 517, 537, 530, 479, 523, 502, 481, 512, 467, 505, 525, 538, 462, 543, 511, 522, 509, 523, 508, 468, 498, 524, 553, 540, 500, 520, 511, 522, 539, 501, 471, 489, 511, 488, 524, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[529, 535, 533, 545, 537, 524, 488, 485, 496, 539, 495, 541, 470, 510, 468, 551, 501, 533, 526, 537, 498, 536, 506, 477, 528, 467, 461, 496, 469, 482, 517, 544, 519, 506, 485, 509, 551, 508, 539, 491, 508, 509, 505, 490, 546, 494, 522, 474, 509, 487, 501, 505, 517, 523, 518, 515, 498, 512, 500, 519, 528, 509, 488, 540, 529, 545, 516, 473, 502, 482, 513, 515, 499, 510, 538, 523, 521, 503, 465, 542, 531, 503, 535, 495, 515, 546, 522, 510, 524, 486, 521, 475, 504, 530, 501, 529, 500, 508, 494, 523, 497, 518, 568, 495, 546, 500, 536, 563, 483, 511, 511, 527, 536, 466, 530, 498, 547, 547, 471, 515, 512, 516, 522, 484, 496, 491, 529, 544] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[553, 513, 510, 528, 492, 546, 492, 519, 489, 520, 492, 554, 496, 490, 524, 555, 475, 512, 510, 542, 512, 505, 534, 491, 526, 510, 529, 501, 492, 519, 525, 523, 538, 499, 500, 543, 505, 495, 501, 503, 544, 505, 492, 504, 506, 506, 542, 500, 526, 523, 508, 516, 501, 453, 528, 500, 516, 525, 534, 499, 496, 519, 446, 490, 522, 481, 480, 520, 504, 505, 528, 550, 543, 560, 513, 524, 533, 527, 515, 526, 515, 486, 534, 504, 496, 518, 467, 512, 539, 528, 500, 459, 449, 514, 507, 505, 529, 529, 539, 518, 489, 505, 528, 508, 508, 503, 523, 485, 524, 523, 488, 521, 501, 548, 531, 492, 505, 509, 512, 520, 559, 521, 502, 516, 514, 505, 488, 487] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[502, 516, 518, 538, 509, 518, 471, 529, 487, 510, 507, 539, 529, 490, 562, 534, 543, 525, 537, 495, 524, 525, 510, 519, 480, 532, 517, 531, 468, 495, 551, 524, 496, 545, 554, 540, 560, 503, 510, 492, 517, 557, 519, 511, 503, 529, 493, 495, 535, 512, 583, 496, 504, 526, 482, 530, 483, 526, 467, 504, 509, 522, 497, 462, 543, 514, 506, 528, 511, 544, 531, 513, 467, 530, 504, 479, 522, 523, 507, 561, 473, 516, 532, 512, 498, 529, 527, 506, 522, 495, 479, 520, 503, 474, 495, 527, 498, 547, 481, 494, 524, 538, 484, 510, 516, 503, 509, 481, 479, 467, 507, 527, 480, 506, 475, 517, 515, 485, 491, 509, 513, 522, 542, 490, 505, 519, 486, 528] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[537, 489, 518, 534, 507, 530, 538, 489, 510, 538, 544, 496, 527, 487, 476, 519, 529, 505, 479, 530, 528, 516, 548, 480, 479, 532, 490, 513, 528, 494, 549, 521, 551, 486, 519, 521, 495, 493, 529, 519, 520, 544, 504, 501, 491, 486, 511, 525, 515, 510, 515, 531, 484, 487, 519, 532, 512, 505, 507, 514, 538, 538, 532, 528, 500, 518, 482, 479, 498, 471, 522, 521, 499, 494, 496, 505, 520, 498, 515, 510, 503, 514, 525, 507, 512, 509, 570, 504, 484, 513, 506, 480, 512, 540, 486, 500, 507, 516, 563, 530, 511, 519, 503, 505, 500, 522, 490, 507, 503, 542, 534, 514, 497, 533, 524, 472, 520, 541, 498, 496, 492, 499, 516, 470, 556, 533, 503, 505] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[553, 513, 510, 528, 492, 546, 492, 519, 489, 520, 492, 554, 496, 490, 524, 555, 475, 512, 510, 542, 512, 505, 534, 491, 526, 510, 529, 501, 492, 519, 525, 523, 538, 499, 500, 543, 505, 495, 501, 503, 544, 505, 492, 504, 506, 506, 542, 500, 526, 523, 508, 516, 501, 453, 528, 500, 516, 525, 534, 499, 496, 519, 446, 490, 522, 481, 480, 520, 504, 505, 528, 550, 543, 560, 513, 524, 533, 527, 515, 526, 515, 486, 534, 504, 496, 518, 467, 512, 539, 528, 500, 459, 449, 514, 507, 505, 529, 529, 539, 518, 489, 505, 528, 508, 508, 503, 523, 485, 524, 523, 488, 521, 501, 548, 531, 492, 505, 509, 512, 520, 559, 521, 502, 516, 514, 505, 488, 487] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[524, 538, 534, 535, 505, 507, 472, 523, 489, 467, 501, 509, 509, 519, 505, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 512, 537, 542, 520, 511, 490, 499, 507, 547, 496, 549, 511, 540, 487, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 494, 558, 515, 530, 525, 529, 509, 525, 526, 470, 500, 536, 498, 527, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 534, 495, 536, 488, 494, 523, 545, 510, 505, 498, 515, 487, 528, 580, 534, 14, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 520, 502, 515, 542, 521, 481, 547, 534, 514, 500, 481, 491, 507, 505, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 514, 529, 536, 496, 502, 511, 488, 511, 462, 518, 506, 510, 504, 486, 497, 32, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 497, 523, 503, 501, 498, 524, 523, 527, 525, 504, 504, 502, 511, 496, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 506, 505, 517, 520, 522, 514, 523, 504, 488, 539, 530, 441, 497, 502, 515, 6, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8146, 8281, 8250, 8306, 8154, 8105, 8162, 8132] avg=8192.0000 max=8306 max_rank=3 min=8105 min_rank=5 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32553 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[467, 519, 490, 496, 492, 504, 509, 518, 508, 488, 485, 522, 499, 545, 480, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 510, 518, 526, 527, 504, 528, 549, 482, 517, 482, 514, 458, 479, 522, 459, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 519, 531, 508, 523, 480, 529, 527, 541, 462, 471, 494, 528, 465, 535, 458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 530, 491, 507, 578, 513, 501, 501, 520, 533, 473, 485, 522, 497, 514, 521, 16, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 542, 501, 560, 530, 501, 518, 499, 519, 512, 521, 541, 548, 538, 509, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 511, 502, 516, 524, 477, 530, 518, 500, 514, 524, 499, 551, 554, 497, 509, 34, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 518, 554, 506, 551, 524, 483, 512, 520, 514, 490, 510, 509, 543, 520, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 502, 486, 498, 531, 489, 546, 492, 512, 510, 534, 468, 528, 472, 519, 534, 10, 8, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8046, 8078, 8102, 8234, 8357, 8283, 8274, 8162] avg=8192.0000 max=8357 max_rank=4 min=8046 min_rank=0 imbalance_ratio=1.020142 max_load_reduction=57179 max_load_reduction_pct=0.872482 moved_to_echo=33076 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[467, 519, 490, 496, 492, 504, 509, 518, 508, 488, 485, 522, 499, 545, 480, 531, 503, 510, 518, 526, 527, 504, 528, 549, 482, 517, 482, 514, 458, 479, 522, 493, 531, 519, 531, 508, 523, 480, 537, 527, 541, 462, 471, 494, 528, 465, 535, 458, 529, 530, 491, 507, 578, 513, 501, 501, 520, 533, 473, 485, 522, 497, 514, 521, 512, 542, 501, 560, 530, 501, 518, 522, 519, 512, 521, 541, 548, 538, 509, 506, 516, 511, 502, 516, 524, 477, 530, 518, 500, 514, 524, 499, 551, 554, 497, 509, 505, 518, 554, 506, 551, 524, 483, 512, 520, 527, 490, 510, 509, 543, 520, 515, 516, 502, 486, 498, 531, 489, 546, 492, 512, 510, 534, 468, 528, 472, 519, 534] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[489, 477, 488, 512, 512, 529, 528, 528, 484, 506, 505, 513, 571, 501, 514, 529, 510, 530, 525, 500, 481, 514, 500, 559, 494, 540, 528, 509, 506, 539, 533, 550, 506, 516, 482, 513, 497, 523, 538, 511, 537, 501, 539, 479, 547, 520, 496, 530, 473, 494, 485, 498, 486, 480, 513, 550, 505, 480, 539, 536, 524, 522, 519, 511, 509, 481, 521, 515, 526, 474, 497, 562, 497, 509, 551, 544, 494, 513, 482, 483, 505, 469, 483, 557, 541, 512, 546, 493, 478, 566, 477, 520, 517, 537, 530, 479, 523, 502, 481, 512, 467, 505, 525, 538, 462, 543, 511, 522, 509, 523, 508, 468, 498, 524, 553, 540, 500, 520, 511, 522, 539, 501, 471, 489, 511, 488, 524, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[529, 535, 533, 545, 537, 524, 488, 485, 496, 539, 495, 541, 470, 510, 468, 551, 501, 533, 526, 537, 498, 536, 506, 477, 528, 467, 461, 496, 469, 482, 517, 544, 519, 506, 485, 509, 551, 508, 539, 491, 508, 509, 505, 490, 546, 494, 522, 474, 509, 487, 501, 505, 517, 523, 518, 515, 498, 512, 500, 519, 528, 509, 488, 540, 529, 545, 516, 473, 502, 482, 513, 515, 499, 510, 538, 523, 521, 503, 465, 542, 531, 503, 535, 495, 515, 546, 522, 510, 524, 486, 521, 475, 504, 530, 501, 529, 500, 508, 494, 523, 497, 518, 568, 495, 546, 500, 536, 563, 483, 511, 511, 527, 536, 466, 530, 498, 547, 547, 471, 515, 512, 516, 522, 484, 496, 491, 529, 544] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[524, 538, 534, 535, 505, 507, 472, 523, 489, 467, 501, 509, 509, 519, 505, 513, 541, 512, 537, 542, 520, 511, 490, 499, 507, 547, 496, 549, 511, 540, 487, 524, 528, 494, 558, 515, 530, 525, 534, 509, 525, 526, 470, 500, 536, 498, 527, 480, 518, 534, 495, 536, 488, 494, 523, 545, 510, 505, 498, 515, 487, 528, 580, 534, 489, 520, 502, 515, 542, 521, 481, 562, 534, 514, 500, 481, 491, 507, 505, 505, 502, 514, 529, 536, 496, 502, 511, 488, 511, 462, 518, 506, 510, 504, 486, 497, 505, 497, 523, 503, 501, 498, 524, 523, 527, 533, 504, 504, 502, 511, 496, 519, 494, 506, 505, 517, 520, 522, 514, 523, 504, 488, 539, 530, 441, 497, 502, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[553, 513, 510, 528, 492, 546, 492, 519, 489, 520, 492, 554, 496, 490, 524, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 512, 510, 542, 512, 505, 534, 491, 526, 510, 529, 501, 492, 519, 525, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 499, 500, 543, 505, 495, 497, 503, 544, 505, 492, 504, 506, 506, 542, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 523, 508, 516, 501, 453, 528, 500, 516, 525, 534, 499, 496, 519, 446, 490, 14, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 481, 480, 520, 504, 505, 528, 535, 543, 560, 513, 524, 533, 527, 515, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 486, 534, 504, 496, 518, 467, 512, 539, 528, 500, 459, 449, 514, 507, 505, 32, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 529, 539, 518, 489, 505, 528, 508, 508, 496, 523, 485, 524, 523, 488, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 548, 531, 492, 505, 509, 512, 520, 559, 521, 502, 516, 514, 505, 488, 487, 5, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8269, 8174, 8179, 8096, 8316, 8066, 8213, 8223] avg=8192.0000 max=8316 max_rank=4 min=8066 min_rank=5 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32818 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[499, 503, 517, 468, 535, 530, 543, 484, 496, 503, 527, 514, 535, 514, 504, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 539, 494, 456, 519, 526, 498, 502, 524, 552, 524, 505, 531, 516, 497, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 516, 513, 497, 480, 531, 529, 481, 496, 495, 516, 502, 530, 513, 517, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 467, 525, 487, 505, 492, 529, 535, 500, 500, 494, 527, 490, 530, 501, 536, 12, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 487, 478, 514, 535, 501, 506, 494, 493, 528, 561, 515, 481, 503, 519, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 551, 496, 517, 501, 521, 500, 533, 483, 549, 513, 503, 528, 526, 515, 492, 34, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 503, 481, 526, 503, 532, 507, 522, 510, 523, 491, 494, 556, 523, 491, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 507, 498, 511, 508, 517, 486, 510, 500, 536, 527, 552, 492, 530, 533, 560, 6, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8241, 8115, 8150, 8121, 8237, 8207, 8304] avg=8192.0000 max=8304 max_rank=7 min=8115 min_rank=2 imbalance_ratio=1.013672 max_load_reduction=57232 max_load_reduction_pct=0.873291 moved_to_echo=32869 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[489, 477, 488, 512, 512, 529, 528, 528, 484, 506, 505, 513, 571, 501, 514, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 530, 525, 500, 481, 514, 500, 559, 494, 540, 528, 509, 506, 539, 533, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 516, 482, 513, 497, 523, 533, 511, 537, 501, 539, 479, 547, 520, 496, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 494, 485, 498, 486, 480, 513, 550, 505, 480, 539, 536, 524, 522, 519, 511, 14, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 481, 521, 515, 526, 474, 497, 547, 497, 509, 551, 544, 494, 513, 482, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 469, 483, 557, 541, 512, 546, 493, 478, 566, 477, 520, 517, 537, 530, 479, 34, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 502, 481, 512, 467, 505, 525, 538, 462, 535, 511, 522, 509, 523, 508, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 524, 553, 540, 500, 520, 511, 522, 539, 501, 471, 489, 511, 488, 524, 524, 6, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8284, 8230, 8131, 8143, 8245, 8091, 8230] avg=8192.0000 max=8284 max_rank=1 min=8091 min_rank=6 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32709 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[529, 535, 533, 545, 537, 524, 488, 485, 496, 539, 495, 541, 470, 510, 468, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 533, 526, 537, 498, 536, 506, 477, 528, 467, 461, 496, 469, 482, 517, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 506, 485, 509, 551, 508, 534, 491, 508, 509, 505, 490, 546, 494, 522, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 487, 501, 505, 517, 523, 518, 515, 498, 512, 500, 519, 528, 509, 488, 540, 13, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 545, 516, 473, 502, 482, 513, 501, 499, 510, 538, 523, 521, 503, 465, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 503, 535, 495, 515, 546, 522, 510, 524, 486, 521, 475, 504, 530, 501, 529, 33, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 508, 494, 523, 497, 518, 568, 495, 546, 493, 536, 563, 483, 511, 511, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 466, 530, 498, 547, 547, 471, 515, 512, 516, 522, 484, 496, 491, 529, 544, 5, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8242, 8045, 8151, 8184, 8162, 8261, 8273, 8218] avg=8192.0000 max=8273 max_rank=6 min=8045 min_rank=1 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32914 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[467, 519, 490, 496, 492, 504, 509, 518, 508, 488, 485, 522, 499, 545, 480, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 510, 518, 526, 527, 504, 528, 549, 482, 517, 482, 514, 458, 479, 522, 459, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 519, 531, 508, 523, 480, 529, 527, 541, 462, 471, 494, 528, 465, 535, 458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 530, 491, 507, 578, 513, 501, 501, 520, 533, 473, 485, 522, 497, 514, 521, 16, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 542, 501, 560, 530, 501, 518, 499, 519, 512, 521, 541, 548, 538, 509, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 511, 502, 516, 524, 477, 530, 518, 500, 514, 524, 499, 551, 554, 497, 509, 34, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 518, 554, 506, 551, 524, 483, 512, 520, 514, 490, 510, 509, 543, 520, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 502, 486, 498, 531, 489, 546, 492, 512, 510, 534, 468, 528, 472, 519, 534, 10, 8, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8046, 8078, 8102, 8234, 8357, 8283, 8274, 8162] avg=8192.0000 max=8357 max_rank=4 min=8046 min_rank=0 imbalance_ratio=1.020142 max_load_reduction=57179 max_load_reduction_pct=0.872482 moved_to_echo=33076 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[502, 516, 518, 538, 509, 518, 471, 529, 487, 510, 507, 539, 529, 490, 562, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 525, 537, 495, 524, 525, 510, 519, 480, 532, 517, 531, 468, 495, 551, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 545, 554, 540, 560, 503, 505, 492, 517, 557, 519, 511, 503, 529, 493, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 512, 583, 496, 504, 526, 482, 530, 483, 526, 467, 504, 509, 522, 497, 462, 13, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 514, 506, 528, 511, 544, 531, 499, 467, 530, 504, 479, 522, 523, 507, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 516, 532, 512, 498, 529, 527, 506, 522, 495, 479, 520, 503, 474, 495, 527, 32, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 547, 481, 494, 524, 538, 484, 510, 516, 496, 509, 481, 479, 467, 507, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 506, 475, 517, 515, 485, 491, 509, 513, 522, 542, 490, 505, 519, 486, 528, 5, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8255, 8244, 8319, 8153, 8269, 8141, 8058, 8097] avg=8192.0000 max=8319 max_rank=2 min=8058 min_rank=6 imbalance_ratio=1.015503 max_load_reduction=57217 max_load_reduction_pct=0.873062 moved_to_echo=32565 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[537, 489, 518, 534, 507, 530, 538, 489, 510, 538, 544, 496, 527, 487, 476, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 505, 479, 530, 528, 516, 548, 480, 479, 532, 490, 513, 528, 494, 549, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 486, 519, 521, 495, 493, 524, 519, 520, 544, 504, 501, 491, 486, 511, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 510, 515, 531, 484, 487, 519, 532, 512, 505, 507, 514, 538, 538, 532, 528, 13, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 518, 482, 479, 498, 471, 522, 507, 499, 494, 496, 505, 520, 498, 515, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 514, 525, 507, 512, 509, 570, 504, 484, 513, 506, 480, 512, 540, 486, 500, 32, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 516, 563, 530, 511, 519, 503, 505, 500, 514, 490, 507, 503, 542, 534, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 533, 524, 472, 520, 541, 498, 496, 492, 499, 516, 470, 556, 533, 503, 505, 6, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8189, 8190, 8282, 8014, 8198, 8258, 8170] avg=8192.0000 max=8282 max_rank=3 min=8014 min_rank=4 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32640 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[553, 513, 510, 528, 492, 546, 492, 519, 489, 520, 492, 554, 496, 490, 524, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 512, 510, 542, 512, 505, 534, 491, 526, 510, 529, 501, 492, 519, 525, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 499, 500, 543, 505, 495, 497, 503, 544, 505, 492, 504, 506, 506, 542, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 523, 508, 516, 501, 453, 528, 500, 516, 525, 534, 499, 496, 519, 446, 490, 14, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 481, 480, 520, 504, 505, 528, 535, 543, 560, 513, 524, 533, 527, 515, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 486, 534, 504, 496, 518, 467, 512, 539, 528, 500, 459, 449, 514, 507, 505, 32, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 529, 539, 518, 489, 505, 528, 508, 508, 496, 523, 485, 524, 523, 488, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 548, 531, 492, 505, 509, 512, 520, 559, 521, 502, 516, 514, 505, 488, 487, 5, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8269, 8174, 8179, 8096, 8316, 8066, 8213, 8223] avg=8192.0000 max=8316 max_rank=4 min=8066 min_rank=5 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32818 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[489, 477, 488, 512, 512, 529, 528, 528, 484, 506, 505, 513, 571, 501, 514, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 530, 525, 500, 481, 514, 500, 559, 494, 540, 528, 509, 506, 539, 533, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 516, 482, 513, 497, 523, 533, 511, 537, 501, 539, 479, 547, 520, 496, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 494, 485, 498, 486, 480, 513, 550, 505, 480, 539, 536, 524, 522, 519, 511, 14, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 481, 521, 515, 526, 474, 497, 547, 497, 509, 551, 544, 494, 513, 482, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 469, 483, 557, 541, 512, 546, 493, 478, 566, 477, 520, 517, 537, 530, 479, 34, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 502, 481, 512, 467, 505, 525, 538, 462, 535, 511, 522, 509, 523, 508, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 524, 553, 540, 500, 520, 511, 522, 539, 501, 471, 489, 511, 488, 524, 524, 6, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8284, 8230, 8131, 8143, 8245, 8091, 8230] avg=8192.0000 max=8284 max_rank=1 min=8091 min_rank=6 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32709 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[524, 538, 534, 535, 505, 507, 472, 523, 489, 467, 501, 509, 509, 519, 505, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 512, 537, 542, 520, 511, 490, 499, 507, 547, 496, 549, 511, 540, 487, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 494, 558, 515, 530, 525, 529, 509, 525, 526, 470, 500, 536, 498, 527, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 534, 495, 536, 488, 494, 523, 545, 510, 505, 498, 515, 487, 528, 580, 534, 14, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 520, 502, 515, 542, 521, 481, 547, 534, 514, 500, 481, 491, 507, 505, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 514, 529, 536, 496, 502, 511, 488, 511, 462, 518, 506, 510, 504, 486, 497, 32, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 497, 523, 503, 501, 498, 524, 523, 527, 525, 504, 504, 502, 511, 496, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 506, 505, 517, 520, 522, 514, 523, 504, 488, 539, 530, 441, 497, 502, 515, 6, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8146, 8281, 8250, 8306, 8154, 8105, 8162, 8132] avg=8192.0000 max=8306 max_rank=3 min=8105 min_rank=5 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32553 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[529, 535, 533, 545, 537, 524, 488, 485, 496, 539, 495, 541, 470, 510, 468, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 533, 526, 537, 498, 536, 506, 477, 528, 467, 461, 496, 469, 482, 517, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 506, 485, 509, 551, 508, 534, 491, 508, 509, 505, 490, 546, 494, 522, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 487, 501, 505, 517, 523, 518, 515, 498, 512, 500, 519, 528, 509, 488, 540, 13, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 545, 516, 473, 502, 482, 513, 501, 499, 510, 538, 523, 521, 503, 465, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 503, 535, 495, 515, 546, 522, 510, 524, 486, 521, 475, 504, 530, 501, 529, 33, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 508, 494, 523, 497, 518, 568, 495, 546, 493, 536, 563, 483, 511, 511, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 466, 530, 498, 547, 547, 471, 515, 512, 516, 522, 484, 496, 491, 529, 544, 5, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8242, 8045, 8151, 8184, 8162, 8261, 8273, 8218] avg=8192.0000 max=8273 max_rank=6 min=8045 min_rank=1 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32914 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 3968, 4096, 4224, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 3968, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 3968, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 3968, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 3968, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 3968, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 3968, 4096, 4224, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[492, 533, 484, 555, 491, 481, 480, 479, 525, 501, 497, 554, 517, 536, 513, 533, 499, 495, 489, 530, 545, 540, 505, 503, 501, 509, 505, 537, 515, 526, 496, 527, 482, 459, 516, 498, 489, 544, 515, 514, 554, 500, 500, 527, 502, 504, 522, 513, 530, 501, 543, 535, 543, 520, 525, 527, 480, 493, 510, 546, 496, 471, 489, 504, 505, 477, 503, 538, 480, 516, 525, 568, 516, 510, 540, 535, 484, 529, 533, 503, 546, 523, 530, 511, 563, 464, 512, 568, 506, 544, 549, 508, 466, 501, 517, 487, 518, 534, 530, 509, 482, 488, 479, 485, 490, 470, 514, 517, 488, 507, 465, 486, 514, 525, 495, 518, 518, 479, 558, 523, 541, 527, 471, 472, 541, 538, 521, 531] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[527, 485, 544, 497, 534, 534, 517, 529, 506, 534, 553, 505, 488, 528, 479, 495, 517, 475, 544, 524, 548, 548, 529, 492, 543, 488, 500, 525, 497, 514, 544, 567, 494, 496, 500, 499, 517, 493, 513, 511, 475, 508, 448, 543, 528, 514, 537, 500, 536, 498, 509, 503, 545, 497, 513, 485, 514, 500, 519, 468, 490, 488, 508, 494, 520, 537, 522, 513, 480, 503, 477, 530, 519, 500, 490, 508, 506, 528, 494, 513, 519, 547, 547, 525, 491, 540, 501, 518, 506, 500, 498, 512, 502, 519, 476, 530, 488, 530, 529, 504, 498, 477, 512, 496, 528, 532, 544, 532, 520, 532, 533, 516, 480, 514, 521, 507, 539, 523, 521, 488, 544, 488, 519, 472, 520, 492, 518, 495] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[507, 503, 535, 480, 523, 541, 514, 487, 506, 500, 531, 497, 492, 496, 493, 492, 562, 476, 494, 542, 483, 536, 547, 523, 520, 503, 518, 556, 469, 505, 548, 489, 545, 489, 505, 540, 508, 567, 518, 505, 512, 480, 512, 486, 534, 541, 489, 503, 526, 542, 514, 498, 523, 508, 499, 541, 489, 540, 497, 488, 543, 540, 473, 516, 520, 522, 487, 508, 499, 527, 510, 488, 513, 517, 496, 480, 526, 540, 552, 528, 526, 550, 510, 512, 486, 491, 511, 483, 522, 507, 515, 521, 524, 508, 518, 532, 522, 518, 494, 492, 537, 470, 515, 515, 518, 512, 502, 490, 499, 538, 515, 551, 503, 502, 491, 503, 526, 503, 503, 495, 502, 499, 520, 516, 450, 514, 515, 538] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[512, 531, 505, 521, 492, 534, 506, 473, 490, 493, 546, 515, 537, 514, 536, 509, 510, 496, 528, 529, 517, 489, 474, 551, 535, 497, 536, 528, 479, 480, 524, 523, 498, 532, 553, 578, 462, 515, 523, 506, 523, 481, 532, 514, 496, 529, 485, 535, 537, 492, 513, 522, 518, 503, 486, 528, 498, 524, 527, 519, 532, 534, 508, 500, 470, 489, 529, 521, 502, 532, 522, 496, 513, 525, 499, 513, 514, 500, 502, 512, 491, 512, 502, 538, 498, 532, 522, 520, 481, 512, 474, 495, 500, 530, 504, 515, 529, 530, 482, 510, 484, 531, 521, 482, 490, 504, 479, 496, 524, 526, 500, 523, 512, 530, 476, 480, 559, 545, 473, 505, 547, 515, 508, 541, 550, 490, 509, 507] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[524, 523, 515, 506, 513, 497, 464, 501, 537, 554, 523, 529, 471, 502, 528, 535, 508, 529, 530, 529, 481, 536, 523, 496, 505, 513, 503, 510, 537, 493, 521, 522, 476, 500, 586, 483, 485, 474, 542, 517, 506, 459, 529, 486, 527, 537, 546, 490, 504, 503, 497, 503, 521, 487, 546, 504, 514, 559, 508, 492, 517, 506, 478, 536, 487, 494, 539, 522, 522, 524, 531, 528, 503, 483, 546, 494, 518, 535, 478, 522, 522, 514, 520, 521, 512, 538, 493, 543, 520, 553, 490, 502, 528, 504, 544, 498, 522, 508, 489, 493, 449, 509, 480, 482, 497, 503, 538, 509, 528, 491, 522, 489, 534, 519, 491, 530, 522, 521, 511, 515, 516, 510, 488, 515, 484, 516, 546, 505] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[509, 495, 506, 496, 536, 517, 533, 516, 507, 502, 489, 523, 496, 524, 534, 517, 494, 530, 524, 505, 500, 528, 502, 544, 503, 540, 502, 569, 538, 518, 540, 527, 530, 520, 515, 490, 528, 466, 511, 498, 540, 482, 541, 504, 503, 527, 516, 532, 529, 539, 511, 481, 525, 513, 500, 515, 508, 501, 495, 495, 499, 499, 525, 520, 504, 529, 491, 466, 508, 486, 502, 530, 536, 491, 461, 504, 540, 515, 496, 537, 493, 491, 546, 514, 480, 515, 536, 503, 500, 482, 482, 532, 499, 500, 520, 524, 487, 562, 517, 530, 479, 526, 529, 505, 531, 484, 524, 493, 507, 483, 495, 535, 473, 544, 514, 541, 514, 498, 499, 541, 481, 498, 518, 505, 565, 522, 496, 505] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[480, 540, 520, 517, 507, 530, 527, 491, 488, 517, 529, 480, 522, 533, 520, 538, 518, 520, 510, 526, 507, 517, 505, 552, 540, 526, 435, 522, 497, 542, 493, 540, 514, 491, 508, 510, 522, 505, 518, 504, 497, 515, 545, 534, 487, 470, 501, 499, 491, 498, 541, 495, 542, 509, 552, 499, 500, 476, 543, 552, 550, 500, 515, 513, 519, 505, 509, 559, 495, 490, 473, 504, 515, 522, 525, 532, 514, 499, 517, 510, 484, 499, 532, 532, 529, 532, 487, 520, 488, 469, 500, 510, 534, 497, 517, 510, 523, 501, 499, 538, 489, 513, 519, 534, 526, 520, 500, 538, 488, 487, 528, 518, 505, 515, 525, 518, 501, 518, 515, 495, 493, 568, 505, 458, 489, 487, 503, 507] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[524, 523, 515, 506, 513, 497, 464, 501, 537, 554, 523, 529, 471, 502, 528, 535, 508, 529, 530, 529, 481, 536, 523, 496, 505, 513, 503, 510, 537, 493, 521, 522, 476, 500, 586, 483, 485, 474, 542, 517, 506, 459, 529, 486, 527, 537, 546, 490, 504, 503, 497, 503, 521, 487, 546, 504, 514, 559, 508, 492, 517, 506, 478, 536, 487, 494, 539, 522, 522, 524, 531, 528, 503, 483, 546, 494, 518, 535, 478, 522, 522, 514, 520, 521, 512, 538, 493, 543, 520, 553, 490, 502, 528, 504, 544, 498, 522, 508, 489, 493, 449, 509, 480, 482, 497, 503, 538, 509, 528, 491, 522, 489, 534, 519, 491, 530, 522, 521, 511, 515, 516, 510, 488, 515, 484, 516, 546, 505] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[484, 538, 516, 564, 467, 537, 492, 519, 518, 451, 505, 530, 474, 472, 513, 519, 525, 501, 498, 509, 525, 463, 495, 487, 491, 530, 494, 514, 516, 503, 542, 499, 507, 515, 536, 538, 499, 497, 528, 503, 511, 499, 519, 524, 492, 495, 497, 490, 494, 533, 510, 521, 519, 523, 522, 495, 499, 537, 512, 543, 516, 522, 521, 531, 517, 541, 505, 536, 510, 493, 524, 502, 473, 497, 535, 539, 500, 499, 531, 489, 518, 544, 511, 509, 546, 504, 526, 517, 526, 461, 507, 502, 494, 514, 480, 494, 521, 544, 525, 510, 519, 496, 528, 552, 581, 532, 486, 502, 508, 474, 500, 529, 482, 499, 525, 528, 526, 545, 526, 517, 504, 503, 506, 537, 515, 520, 495, 518] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[492, 533, 484, 555, 491, 481, 480, 479, 525, 501, 497, 554, 517, 536, 513, 533, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 495, 489, 530, 545, 540, 505, 503, 501, 509, 505, 481, 515, 526, 496, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 459, 516, 498, 489, 544, 515, 514, 554, 500, 500, 527, 502, 504, 522, 513, 21, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 501, 543, 535, 529, 520, 525, 527, 480, 493, 510, 546, 496, 471, 489, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 477, 503, 538, 480, 516, 525, 568, 516, 510, 540, 535, 484, 529, 533, 503, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 523, 525, 511, 563, 464, 512, 568, 506, 544, 549, 508, 466, 501, 517, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 534, 530, 509, 482, 488, 479, 485, 490, 470, 514, 517, 488, 507, 465, 486, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 525, 495, 518, 518, 479, 558, 523, 541, 527, 471, 472, 541, 538, 521, 531, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8166, 8164, 8199, 8272, 8290, 7997, 8273] avg=8192.0000 max=8290 max_rank=5 min=7997 min_rank=6 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32832 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[507, 503, 535, 480, 523, 541, 514, 487, 506, 500, 531, 497, 492, 496, 493, 492, 562, 476, 494, 542, 483, 536, 547, 523, 520, 503, 518, 556, 469, 505, 548, 489, 545, 489, 505, 540, 508, 567, 518, 505, 512, 480, 512, 486, 534, 541, 489, 503, 526, 542, 514, 498, 523, 508, 499, 541, 489, 540, 497, 488, 543, 540, 473, 516, 520, 522, 487, 508, 499, 527, 510, 488, 513, 517, 496, 480, 526, 540, 552, 528, 526, 550, 510, 512, 486, 491, 511, 483, 522, 507, 515, 521, 524, 508, 518, 532, 522, 518, 494, 492, 537, 470, 515, 515, 518, 512, 502, 490, 499, 538, 515, 551, 503, 502, 491, 503, 526, 503, 503, 495, 502, 499, 520, 516, 450, 514, 515, 538] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[509, 495, 506, 496, 536, 517, 533, 516, 507, 502, 489, 523, 496, 524, 534, 517, 494, 530, 524, 505, 500, 528, 502, 544, 503, 540, 502, 569, 538, 518, 540, 527, 530, 520, 515, 490, 528, 466, 511, 498, 540, 482, 541, 504, 503, 527, 516, 532, 529, 539, 511, 481, 525, 513, 500, 515, 508, 501, 495, 495, 499, 499, 525, 520, 504, 529, 491, 466, 508, 486, 502, 530, 536, 491, 461, 504, 540, 515, 496, 537, 493, 491, 546, 514, 480, 515, 536, 503, 500, 482, 482, 532, 499, 500, 520, 524, 487, 562, 517, 530, 479, 526, 529, 505, 531, 484, 524, 493, 507, 483, 495, 535, 473, 544, 514, 541, 514, 498, 499, 541, 481, 498, 518, 505, 565, 522, 496, 505] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[492, 533, 484, 555, 491, 481, 480, 479, 525, 501, 497, 554, 517, 536, 513, 533, 499, 495, 489, 530, 545, 540, 505, 503, 501, 509, 505, 537, 515, 526, 496, 527, 482, 459, 516, 498, 489, 544, 515, 514, 554, 500, 500, 527, 502, 504, 522, 513, 530, 501, 543, 535, 543, 520, 525, 527, 480, 493, 510, 546, 496, 471, 489, 504, 505, 477, 503, 538, 480, 516, 525, 568, 516, 510, 540, 535, 484, 529, 533, 503, 546, 523, 530, 511, 563, 464, 512, 568, 506, 544, 549, 508, 466, 501, 517, 487, 518, 534, 530, 509, 482, 488, 479, 485, 490, 470, 514, 517, 488, 507, 465, 486, 514, 525, 495, 518, 518, 479, 558, 523, 541, 527, 471, 472, 541, 538, 521, 531] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[512, 531, 505, 521, 492, 534, 506, 473, 490, 493, 546, 515, 537, 514, 536, 509, 510, 496, 528, 529, 517, 489, 474, 551, 535, 497, 536, 528, 479, 480, 524, 523, 498, 532, 553, 578, 462, 515, 523, 506, 523, 481, 532, 514, 496, 529, 485, 535, 537, 492, 513, 522, 518, 503, 486, 528, 498, 524, 527, 519, 532, 534, 508, 500, 470, 489, 529, 521, 502, 532, 522, 496, 513, 525, 499, 513, 514, 500, 502, 512, 491, 512, 502, 538, 498, 532, 522, 520, 481, 512, 474, 495, 500, 530, 504, 515, 529, 530, 482, 510, 484, 531, 521, 482, 490, 504, 479, 496, 524, 526, 500, 523, 512, 530, 476, 480, 559, 545, 473, 505, 547, 515, 508, 541, 550, 490, 509, 507] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[527, 485, 544, 497, 534, 534, 517, 529, 506, 534, 553, 505, 488, 528, 479, 495, 517, 475, 544, 524, 548, 548, 529, 492, 543, 488, 500, 525, 497, 514, 544, 567, 494, 496, 500, 499, 517, 493, 513, 511, 475, 508, 448, 543, 528, 514, 537, 500, 536, 498, 509, 503, 545, 497, 513, 485, 514, 500, 519, 468, 490, 488, 508, 494, 520, 537, 522, 513, 480, 503, 477, 530, 519, 500, 490, 508, 506, 528, 494, 513, 519, 547, 547, 525, 491, 540, 501, 518, 506, 500, 498, 512, 502, 519, 476, 530, 488, 530, 529, 504, 498, 477, 512, 496, 528, 532, 544, 532, 520, 532, 533, 516, 480, 514, 521, 507, 539, 523, 521, 488, 544, 488, 519, 472, 520, 492, 518, 495] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[480, 540, 520, 517, 507, 530, 527, 491, 488, 517, 529, 480, 522, 533, 520, 538, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 520, 510, 526, 507, 517, 505, 552, 540, 526, 435, 468, 497, 542, 493, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 491, 508, 510, 522, 505, 518, 504, 497, 515, 545, 534, 487, 470, 501, 499, 20, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 498, 541, 495, 528, 509, 552, 499, 500, 476, 543, 552, 550, 500, 515, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 505, 509, 559, 495, 490, 473, 504, 515, 522, 525, 532, 514, 499, 517, 510, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 499, 527, 532, 529, 532, 487, 520, 488, 469, 500, 510, 534, 497, 517, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 501, 499, 538, 489, 513, 519, 534, 526, 520, 500, 538, 488, 487, 528, 518, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 515, 525, 518, 501, 518, 515, 495, 493, 568, 505, 458, 489, 487, 503, 507, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8243, 8196, 8144, 8262, 8198, 8135, 8255, 8103] avg=8192.0000 max=8262 max_rank=3 min=8103 min_rank=7 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32691 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[512, 531, 505, 521, 492, 534, 506, 473, 490, 493, 546, 515, 537, 514, 536, 509, 5, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 496, 528, 529, 517, 489, 474, 551, 535, 497, 536, 465, 479, 480, 524, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 532, 553, 578, 462, 515, 523, 506, 523, 481, 532, 514, 496, 529, 485, 535, 25, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 492, 513, 522, 489, 503, 486, 528, 498, 524, 527, 519, 532, 534, 508, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 489, 529, 521, 502, 532, 522, 496, 513, 525, 499, 513, 514, 500, 502, 512, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 512, 493, 538, 498, 532, 522, 520, 481, 512, 474, 495, 500, 530, 504, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 530, 482, 510, 484, 531, 521, 482, 490, 504, 479, 496, 524, 526, 500, 523, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 530, 476, 480, 559, 545, 473, 505, 547, 515, 508, 541, 550, 490, 509, 507, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 8133, 8297, 8212, 8152, 8117, 8149, 8251] avg=8192.0000 max=8297 max_rank=2 min=8117 min_rank=5 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32669 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[509, 495, 506, 496, 536, 517, 533, 516, 507, 502, 489, 523, 496, 524, 534, 517, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 530, 524, 505, 500, 528, 502, 544, 503, 540, 502, 510, 538, 518, 540, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 520, 515, 490, 528, 466, 511, 498, 540, 482, 541, 504, 503, 527, 516, 532, 22, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 539, 511, 481, 511, 513, 500, 515, 508, 501, 495, 495, 499, 499, 525, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 529, 491, 466, 508, 486, 502, 530, 536, 491, 461, 504, 540, 515, 496, 537, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 491, 541, 514, 480, 515, 536, 503, 500, 482, 482, 532, 499, 500, 520, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 562, 517, 530, 479, 526, 529, 505, 531, 484, 524, 493, 507, 483, 495, 535, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 544, 514, 541, 514, 498, 499, 541, 481, 498, 518, 505, 565, 522, 496, 505, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8305, 8229, 8141, 8106, 8112, 8224, 8215] avg=8192.0000 max=8305 max_rank=1 min=8106 min_rank=4 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32657 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[527, 485, 544, 497, 534, 534, 517, 529, 506, 534, 553, 505, 488, 528, 479, 495, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 475, 544, 524, 548, 548, 529, 492, 543, 488, 500, 471, 497, 514, 544, 567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 496, 500, 499, 517, 493, 513, 511, 475, 508, 448, 543, 528, 514, 537, 500, 20, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 498, 509, 503, 530, 497, 513, 485, 514, 500, 519, 468, 490, 488, 508, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 537, 522, 513, 480, 503, 477, 530, 519, 500, 490, 508, 506, 528, 494, 513, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 547, 542, 525, 491, 540, 501, 518, 506, 500, 498, 512, 502, 519, 476, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 530, 529, 504, 498, 477, 512, 496, 528, 532, 544, 532, 520, 532, 533, 516, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 514, 521, 507, 539, 523, 521, 488, 544, 488, 519, 472, 520, 492, 518, 495, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8259, 8301, 8101, 8052, 8150, 8226, 8305, 8142] avg=8192.0000 max=8305 max_rank=6 min=8052 min_rank=3 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32823 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[507, 503, 535, 480, 523, 541, 514, 487, 506, 500, 531, 497, 492, 496, 493, 492, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 562, 476, 494, 542, 483, 536, 547, 523, 520, 503, 518, 498, 469, 505, 548, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 489, 505, 540, 508, 567, 518, 505, 512, 480, 512, 486, 534, 541, 489, 503, 22, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 542, 514, 498, 509, 508, 499, 541, 489, 540, 497, 488, 543, 540, 473, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 522, 487, 508, 499, 527, 510, 488, 513, 517, 496, 480, 526, 540, 552, 528, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 550, 505, 512, 486, 491, 511, 483, 522, 507, 515, 521, 524, 508, 518, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 518, 494, 492, 537, 470, 515, 515, 518, 512, 502, 490, 499, 538, 515, 551, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 502, 491, 503, 526, 503, 503, 495, 502, 499, 520, 516, 450, 514, 515, 538, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8101, 8213, 8260, 8223, 8223, 8211, 8224, 8081] avg=8192.0000 max=8260 max_rank=2 min=8081 min_rank=7 imbalance_ratio=1.008301 max_load_reduction=57276 max_load_reduction_pct=0.873962 moved_to_echo=32739 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[480, 540, 520, 517, 507, 530, 527, 491, 488, 517, 529, 480, 522, 533, 520, 538, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 520, 510, 526, 507, 517, 505, 552, 540, 526, 435, 468, 497, 542, 493, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 491, 508, 510, 522, 505, 518, 504, 497, 515, 545, 534, 487, 470, 501, 499, 20, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 498, 541, 495, 528, 509, 552, 499, 500, 476, 543, 552, 550, 500, 515, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 505, 509, 559, 495, 490, 473, 504, 515, 522, 525, 532, 514, 499, 517, 510, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 499, 527, 532, 529, 532, 487, 520, 488, 469, 500, 510, 534, 497, 517, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 501, 499, 538, 489, 513, 519, 534, 526, 520, 500, 538, 488, 487, 528, 518, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 515, 525, 518, 501, 518, 515, 495, 493, 568, 505, 458, 489, 487, 503, 507, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8243, 8196, 8144, 8262, 8198, 8135, 8255, 8103] avg=8192.0000 max=8262 max_rank=3 min=8103 min_rank=7 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32691 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[484, 538, 516, 564, 467, 537, 492, 519, 518, 451, 505, 530, 474, 472, 513, 519, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 501, 498, 509, 525, 463, 495, 487, 491, 530, 494, 461, 516, 503, 542, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 515, 536, 538, 499, 497, 528, 503, 511, 499, 519, 524, 492, 495, 497, 490, 20, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 533, 510, 521, 506, 523, 522, 495, 499, 537, 512, 543, 516, 522, 521, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 541, 505, 536, 510, 493, 524, 502, 473, 497, 535, 539, 500, 499, 531, 489, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 544, 506, 509, 546, 504, 526, 517, 526, 461, 507, 502, 494, 514, 480, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 544, 525, 510, 519, 496, 528, 552, 581, 532, 486, 502, 508, 474, 500, 529, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 499, 525, 528, 526, 545, 526, 517, 504, 503, 506, 537, 515, 520, 495, 518, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8103, 8039, 8174, 8285, 8200, 8148, 8340, 8247] avg=8192.0000 max=8340 max_rank=6 min=8039 min_rank=1 imbalance_ratio=1.018066 max_load_reduction=57196 max_load_reduction_pct=0.872742 moved_to_echo=32935 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[492, 533, 484, 555, 491, 481, 480, 479, 525, 501, 497, 554, 517, 536, 513, 533, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 495, 489, 530, 545, 540, 505, 503, 501, 509, 505, 481, 515, 526, 496, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 459, 516, 498, 489, 544, 515, 514, 554, 500, 500, 527, 502, 504, 522, 513, 21, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 501, 543, 535, 529, 520, 525, 527, 480, 493, 510, 546, 496, 471, 489, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 477, 503, 538, 480, 516, 525, 568, 516, 510, 540, 535, 484, 529, 533, 503, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 523, 525, 511, 563, 464, 512, 568, 506, 544, 549, 508, 466, 501, 517, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 534, 530, 509, 482, 488, 479, 485, 490, 470, 514, 517, 488, 507, 465, 486, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 525, 495, 518, 518, 479, 558, 523, 541, 527, 471, 472, 541, 538, 521, 531, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8166, 8164, 8199, 8272, 8290, 7997, 8273] avg=8192.0000 max=8290 max_rank=5 min=7997 min_rank=6 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32832 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[509, 495, 506, 496, 536, 517, 533, 516, 507, 502, 489, 523, 496, 524, 534, 517, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 530, 524, 505, 500, 528, 502, 544, 503, 540, 502, 510, 538, 518, 540, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 520, 515, 490, 528, 466, 511, 498, 540, 482, 541, 504, 503, 527, 516, 532, 22, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 539, 511, 481, 511, 513, 500, 515, 508, 501, 495, 495, 499, 499, 525, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 529, 491, 466, 508, 486, 502, 530, 536, 491, 461, 504, 540, 515, 496, 537, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 491, 541, 514, 480, 515, 536, 503, 500, 482, 482, 532, 499, 500, 520, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 562, 517, 530, 479, 526, 529, 505, 531, 484, 524, 493, 507, 483, 495, 535, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 544, 514, 541, 514, 498, 499, 541, 481, 498, 518, 505, 565, 522, 496, 505, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8305, 8229, 8141, 8106, 8112, 8224, 8215] avg=8192.0000 max=8305 max_rank=1 min=8106 min_rank=4 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32657 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[527, 485, 544, 497, 534, 534, 517, 529, 506, 534, 553, 505, 488, 528, 479, 495, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 475, 544, 524, 548, 548, 529, 492, 543, 488, 500, 471, 497, 514, 544, 567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 496, 500, 499, 517, 493, 513, 511, 475, 508, 448, 543, 528, 514, 537, 500, 20, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 498, 509, 503, 530, 497, 513, 485, 514, 500, 519, 468, 490, 488, 508, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 537, 522, 513, 480, 503, 477, 530, 519, 500, 490, 508, 506, 528, 494, 513, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 547, 542, 525, 491, 540, 501, 518, 506, 500, 498, 512, 502, 519, 476, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 530, 529, 504, 498, 477, 512, 496, 528, 532, 544, 532, 520, 532, 533, 516, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 514, 521, 507, 539, 523, 521, 488, 544, 488, 519, 472, 520, 492, 518, 495, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8259, 8301, 8101, 8052, 8150, 8226, 8305, 8142] avg=8192.0000 max=8305 max_rank=6 min=8052 min_rank=3 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32823 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[524, 523, 515, 506, 513, 497, 464, 501, 537, 554, 523, 529, 471, 502, 528, 535, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 529, 530, 529, 481, 536, 523, 496, 505, 513, 503, 457, 537, 493, 521, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 500, 586, 483, 485, 474, 542, 517, 506, 459, 529, 486, 527, 537, 546, 490, 20, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 503, 497, 503, 508, 487, 546, 504, 514, 559, 508, 492, 517, 506, 478, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 494, 539, 522, 522, 524, 531, 528, 503, 483, 546, 494, 518, 535, 478, 522, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 514, 515, 521, 512, 538, 493, 543, 520, 553, 490, 502, 528, 504, 544, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 508, 489, 493, 449, 509, 480, 482, 497, 503, 538, 509, 528, 491, 522, 489, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 519, 491, 530, 522, 521, 511, 515, 516, 510, 488, 515, 484, 516, 546, 505, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 8183, 8167, 8162, 8235, 8297, 8042, 8224] avg=8192.0000 max=8297 max_rank=5 min=8042 min_rank=6 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32798 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 3968, 4224, 4224, 4096, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 3840, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4352, 4096, 4096, 3968, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 3968, 4224, 4224, 4096, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[526, 522, 498, 518, 510, 544, 502, 519, 503, 516, 527, 477, 504, 495, 513, 538, 516, 460, 502, 519, 536, 582, 520, 495, 490, 548, 491, 500, 506, 525, 515, 525, 503, 514, 519, 513, 498, 541, 499, 497, 518, 532, 529, 461, 529, 539, 520, 514, 489, 557, 468, 520, 502, 496, 504, 508, 486, 490, 530, 509, 491, 524, 515, 538, 562, 507, 509, 520, 536, 515, 485, 524, 503, 545, 502, 491, 503, 536, 482, 531, 489, 525, 495, 484, 535, 568, 503, 525, 536, 500, 510, 474, 554, 513, 505, 553, 512, 513, 460, 524, 500, 542, 505, 487, 530, 532, 526, 496, 467, 561, 475, 480, 539, 523, 516, 507, 485, 537, 536, 493, 514, 470, 492, 473, 506, 512, 499, 509] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[519, 527, 513, 495, 484, 503, 472, 483, 542, 514, 502, 515, 526, 494, 490, 509, 519, 503, 524, 530, 480, 539, 504, 539, 486, 484, 509, 538, 468, 486, 526, 480, 528, 505, 500, 499, 553, 515, 534, 520, 499, 509, 484, 529, 528, 541, 479, 488, 507, 529, 514, 522, 519, 484, 549, 521, 528, 521, 485, 504, 489, 516, 504, 521, 511, 550, 523, 537, 500, 487, 495, 489, 492, 497, 485, 506, 533, 509, 516, 511, 529, 542, 529, 503, 482, 524, 489, 498, 529, 512, 507, 489, 530, 541, 519, 499, 546, 518, 468, 553, 493, 514, 524, 518, 522, 524, 554, 498, 522, 551, 526, 488, 468, 483, 541, 535, 522, 553, 500, 508, 491, 516, 515, 506, 531, 512, 543, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[547, 512, 484, 527, 525, 540, 499, 497, 497, 505, 515, 476, 476, 518, 545, 517, 502, 525, 503, 504, 500, 553, 506, 496, 500, 519, 506, 551, 529, 549, 525, 503, 518, 522, 505, 516, 484, 557, 531, 512, 518, 465, 537, 488, 517, 480, 487, 499, 502, 530, 512, 540, 512, 531, 498, 512, 533, 499, 481, 539, 486, 524, 513, 494, 552, 502, 552, 561, 498, 523, 496, 512, 545, 507, 500, 495, 493, 510, 513, 461, 484, 497, 531, 483, 529, 475, 564, 582, 479, 531, 527, 470, 508, 533, 508, 497, 503, 525, 509, 499, 501, 494, 511, 475, 491, 465, 532, 531, 528, 494, 512, 498, 514, 519, 505, 528, 513, 495, 494, 489, 520, 526, 505, 533, 540, 551, 488, 537] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[500, 526, 533, 476, 517, 516, 488, 544, 498, 519, 528, 524, 493, 511, 498, 526, 475, 503, 534, 518, 527, 534, 495, 425, 499, 512, 544, 545, 527, 481, 521, 497, 504, 521, 503, 531, 527, 555, 512, 540, 510, 499, 506, 507, 509, 507, 503, 548, 498, 519, 555, 503, 480, 530, 495, 517, 510, 498, 531, 523, 467, 540, 510, 517, 481, 522, 551, 508, 485, 513, 495, 508, 521, 516, 513, 511, 522, 517, 528, 523, 532, 512, 476, 522, 521, 505, 512, 566, 528, 506, 531, 484, 533, 522, 485, 521, 455, 470, 496, 512, 508, 530, 513, 496, 542, 520, 523, 530, 583, 503, 472, 497, 517, 469, 573, 482, 457, 517, 455, 508, 516, 488, 495, 530, 526, 498, 539, 537] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[523, 492, 501, 529, 520, 498, 562, 532, 507, 500, 531, 503, 485, 494, 513, 508, 549, 539, 484, 493, 511, 515, 486, 495, 506, 517, 459, 551, 524, 534, 492, 502, 524, 539, 532, 472, 517, 538, 491, 507, 493, 526, 517, 526, 509, 499, 529, 505, 497, 520, 533, 507, 530, 527, 524, 480, 559, 545, 518, 489, 475, 526, 506, 494, 513, 531, 494, 521, 505, 515, 514, 509, 519, 537, 515, 508, 497, 554, 469, 512, 511, 502, 519, 514, 490, 508, 484, 518, 506, 540, 507, 492, 523, 518, 512, 541, 530, 499, 491, 497, 512, 530, 536, 489, 483, 507, 530, 561, 541, 495, 507, 496, 533, 503, 520, 508, 499, 464, 493, 521, 491, 526, 523, 533, 511, 492, 495, 513] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[503, 502, 521, 537, 505, 489, 562, 515, 462, 537, 490, 574, 499, 516, 497, 530, 517, 478, 532, 504, 486, 491, 527, 492, 508, 488, 503, 527, 518, 493, 488, 490, 540, 514, 512, 568, 513, 497, 506, 533, 543, 511, 496, 496, 505, 461, 540, 507, 565, 521, 516, 492, 512, 506, 506, 517, 488, 484, 503, 491, 540, 495, 538, 526, 487, 492, 517, 496, 535, 510, 509, 519, 488, 489, 529, 496, 490, 489, 496, 517, 530, 505, 505, 546, 556, 550, 541, 539, 478, 539, 494, 508, 488, 505, 483, 558, 544, 494, 521, 540, 497, 500, 515, 476, 499, 547, 529, 496, 501, 524, 525, 518, 479, 490, 562, 489, 524, 502, 547, 512, 504, 526, 523, 492, 505, 538, 520, 490] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[547, 512, 484, 527, 525, 540, 499, 497, 497, 505, 515, 476, 476, 518, 545, 517, 502, 525, 503, 504, 500, 553, 506, 496, 500, 519, 506, 551, 529, 549, 525, 503, 518, 522, 505, 516, 484, 557, 531, 512, 518, 465, 537, 488, 517, 480, 487, 499, 502, 530, 512, 540, 512, 531, 498, 512, 533, 499, 481, 539, 486, 524, 513, 494, 552, 502, 552, 561, 498, 523, 496, 512, 545, 507, 500, 495, 493, 510, 513, 461, 484, 497, 531, 483, 529, 475, 564, 582, 479, 531, 527, 470, 508, 533, 508, 497, 503, 525, 509, 499, 501, 494, 511, 475, 491, 465, 532, 531, 528, 494, 512, 498, 514, 519, 505, 528, 513, 495, 494, 489, 520, 526, 505, 533, 540, 551, 488, 537] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[553, 503, 510, 532, 531, 473, 518, 488, 545, 569, 493, 486, 500, 494, 522, 485, 561, 484, 463, 470, 520, 522, 534, 517, 501, 485, 518, 503, 496, 528, 503, 525, 535, 500, 475, 487, 520, 498, 563, 480, 541, 519, 520, 485, 535, 506, 500, 461, 487, 538, 511, 559, 536, 503, 508, 506, 488, 508, 528, 530, 489, 489, 507, 499, 532, 507, 529, 490, 455, 504, 505, 502, 538, 517, 526, 536, 516, 511, 503, 500, 520, 502, 510, 553, 510, 471, 483, 535, 502, 535, 502, 497, 537, 484, 493, 505, 510, 504, 534, 512, 535, 512, 507, 529, 549, 533, 531, 514, 507, 515, 506, 487, 545, 552, 518, 541, 504, 529, 506, 506, 513, 529, 536, 491, 475, 517, 511, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[495, 495, 478, 483, 537, 501, 530, 525, 509, 485, 517, 486, 521, 516, 533, 540, 485, 515, 498, 518, 453, 515, 494, 515, 512, 485, 502, 487, 501, 506, 520, 537, 543, 471, 513, 521, 517, 522, 544, 493, 508, 492, 529, 491, 555, 481, 521, 528, 549, 509, 503, 507, 521, 506, 525, 491, 488, 544, 512, 501, 529, 519, 501, 499, 510, 532, 490, 507, 486, 507, 510, 533, 486, 509, 517, 497, 528, 484, 521, 507, 509, 487, 545, 511, 509, 497, 518, 511, 511, 528, 530, 545, 519, 494, 511, 493, 529, 528, 502, 537, 549, 537, 541, 536, 495, 523, 514, 514, 536, 547, 509, 536, 546, 527, 552, 491, 522, 542, 499, 519, 529, 485, 497, 470, 521, 455, 514, 465] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[519, 527, 513, 495, 484, 503, 472, 483, 542, 514, 502, 515, 526, 494, 490, 509, 519, 503, 524, 530, 480, 539, 504, 539, 486, 484, 509, 538, 468, 486, 526, 480, 528, 505, 500, 499, 553, 515, 534, 520, 499, 509, 484, 529, 528, 541, 479, 488, 507, 529, 514, 522, 519, 484, 549, 521, 528, 521, 485, 504, 489, 516, 504, 521, 511, 550, 523, 537, 500, 487, 495, 489, 492, 497, 485, 506, 533, 509, 516, 511, 529, 542, 529, 503, 482, 524, 489, 498, 529, 512, 507, 489, 530, 541, 519, 499, 546, 518, 468, 553, 493, 514, 524, 518, 522, 524, 554, 498, 522, 551, 526, 488, 468, 483, 541, 535, 522, 553, 500, 508, 491, 516, 515, 506, 531, 512, 543, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[523, 492, 501, 529, 520, 498, 562, 532, 507, 500, 531, 503, 485, 494, 513, 508, 549, 539, 484, 493, 511, 515, 486, 495, 506, 517, 459, 551, 524, 534, 492, 502, 524, 539, 532, 472, 517, 538, 491, 507, 493, 526, 517, 526, 509, 499, 529, 505, 497, 520, 533, 507, 530, 527, 524, 480, 559, 545, 518, 489, 475, 526, 506, 494, 513, 531, 494, 521, 505, 515, 514, 509, 519, 537, 515, 508, 497, 554, 469, 512, 511, 502, 519, 514, 490, 508, 484, 518, 506, 540, 507, 492, 523, 518, 512, 541, 530, 499, 491, 497, 512, 530, 536, 489, 483, 507, 530, 561, 541, 495, 507, 496, 533, 503, 520, 508, 499, 464, 493, 521, 491, 526, 523, 533, 511, 492, 495, 513] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[500, 526, 533, 476, 517, 516, 488, 544, 498, 519, 528, 524, 493, 511, 498, 526, 475, 503, 534, 518, 527, 534, 495, 425, 499, 512, 544, 545, 527, 481, 521, 497, 504, 521, 503, 531, 527, 555, 512, 540, 510, 499, 506, 507, 509, 507, 503, 548, 498, 519, 555, 503, 480, 530, 495, 517, 510, 498, 531, 523, 467, 540, 510, 517, 481, 522, 551, 508, 485, 513, 495, 508, 521, 516, 513, 511, 522, 517, 528, 523, 532, 512, 476, 522, 521, 505, 512, 566, 528, 506, 531, 484, 533, 522, 485, 521, 455, 470, 496, 512, 508, 530, 513, 496, 542, 520, 523, 530, 583, 503, 472, 497, 517, 469, 573, 482, 457, 517, 455, 508, 516, 488, 495, 530, 526, 498, 539, 537] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[526, 522, 498, 518, 510, 544, 502, 519, 503, 516, 527, 477, 504, 495, 513, 538, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 460, 502, 519, 536, 582, 520, 495, 490, 548, 491, 500, 506, 525, 515, 525, 38, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 514, 519, 513, 498, 514, 499, 497, 518, 532, 529, 461, 529, 539, 520, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 548, 468, 520, 502, 496, 504, 508, 486, 490, 530, 509, 491, 524, 515, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 562, 507, 509, 520, 536, 515, 485, 524, 503, 545, 502, 491, 503, 536, 482, 531, 18, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 525, 495, 484, 535, 568, 503, 487, 536, 500, 510, 474, 554, 513, 505, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 513, 460, 524, 500, 542, 505, 487, 530, 532, 486, 496, 467, 561, 475, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 523, 516, 507, 485, 537, 536, 493, 514, 470, 492, 473, 506, 512, 499, 509, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 8290, 8199, 8118, 8274, 8231, 8070, 8124] avg=8192.0000 max=8290 max_rank=1 min=8070 min_rank=6 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32699 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[500, 526, 533, 476, 517, 516, 488, 544, 498, 519, 528, 524, 493, 511, 498, 526, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 503, 534, 518, 527, 534, 495, 425, 499, 512, 544, 545, 527, 481, 521, 497, 36, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 521, 503, 531, 527, 538, 512, 540, 510, 499, 506, 507, 509, 507, 503, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 517, 555, 503, 480, 530, 495, 517, 510, 498, 531, 523, 467, 540, 510, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 522, 551, 508, 485, 513, 495, 508, 521, 516, 513, 511, 522, 517, 528, 523, 15, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 512, 476, 522, 521, 505, 512, 530, 528, 506, 531, 484, 533, 522, 485, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 455, 470, 496, 512, 508, 530, 513, 496, 542, 520, 492, 530, 583, 503, 472, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 469, 573, 482, 457, 517, 455, 508, 516, 488, 495, 530, 526, 498, 539, 537, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8189, 8265, 8191, 8231, 8220, 8119, 8116] avg=8192.0000 max=8265 max_rank=2 min=8116 min_rank=7 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32686 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[553, 503, 510, 532, 531, 473, 518, 488, 545, 569, 493, 486, 500, 494, 522, 485, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 484, 463, 470, 520, 522, 534, 517, 501, 485, 518, 503, 496, 528, 503, 525, 34, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 500, 475, 487, 520, 483, 563, 480, 541, 519, 520, 485, 535, 506, 500, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 536, 511, 559, 536, 503, 508, 506, 488, 508, 528, 530, 489, 489, 507, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 507, 529, 490, 455, 504, 505, 502, 538, 517, 526, 536, 516, 511, 503, 500, 15, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 502, 510, 553, 510, 471, 483, 501, 502, 535, 502, 497, 537, 484, 493, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 504, 534, 512, 535, 512, 507, 529, 549, 533, 500, 514, 507, 515, 506, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 552, 518, 541, 504, 529, 506, 506, 513, 529, 536, 491, 475, 517, 511, 525, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8180, 8110, 8184, 8188, 8105, 8254, 8306] avg=8192.0000 max=8306 max_rank=7 min=8105 min_rank=5 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32853 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[495, 495, 478, 483, 537, 501, 530, 525, 509, 485, 517, 486, 521, 516, 533, 540, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 515, 498, 518, 453, 515, 494, 515, 512, 485, 502, 487, 501, 506, 520, 537, 32, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 471, 513, 521, 517, 506, 544, 493, 508, 492, 529, 491, 555, 481, 521, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 507, 503, 507, 521, 506, 525, 491, 488, 544, 512, 501, 529, 519, 501, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 532, 490, 507, 486, 507, 510, 533, 486, 509, 517, 497, 528, 484, 521, 507, 15, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 487, 545, 511, 509, 497, 518, 479, 511, 528, 530, 545, 519, 494, 511, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 528, 502, 537, 549, 537, 541, 536, 495, 523, 483, 514, 536, 547, 509, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 527, 552, 491, 522, 542, 499, 519, 529, 485, 497, 470, 521, 455, 514, 465, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8158, 8091, 8213, 8202, 8141, 8186, 8402, 8143] avg=8192.0000 max=8402 max_rank=6 min=8091 min_rank=1 imbalance_ratio=1.025635 max_load_reduction=57134 max_load_reduction_pct=0.871796 moved_to_echo=32872 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[519, 527, 513, 495, 484, 503, 472, 483, 542, 514, 502, 515, 526, 494, 490, 509, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 503, 524, 530, 480, 539, 504, 539, 486, 484, 509, 538, 468, 486, 526, 480, 32, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 505, 500, 499, 553, 499, 534, 520, 499, 509, 484, 529, 528, 541, 479, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 527, 514, 522, 519, 484, 549, 521, 528, 521, 485, 504, 489, 516, 504, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 550, 523, 537, 500, 487, 495, 489, 492, 497, 485, 506, 533, 509, 516, 511, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 542, 529, 503, 482, 524, 489, 466, 529, 512, 507, 489, 530, 541, 519, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 518, 468, 553, 493, 514, 524, 518, 522, 524, 521, 498, 522, 551, 526, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 483, 541, 535, 522, 553, 500, 508, 491, 516, 515, 506, 531, 512, 543, 503, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8095, 8164, 8195, 8211, 8159, 8190, 8286, 8236] avg=8192.0000 max=8286 max_rank=6 min=8095 min_rank=0 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32871 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[523, 492, 501, 529, 520, 498, 562, 532, 507, 500, 531, 503, 485, 494, 513, 508, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 539, 484, 493, 511, 515, 486, 495, 506, 517, 459, 551, 524, 534, 492, 502, 33, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 539, 532, 472, 517, 522, 491, 507, 493, 526, 517, 526, 509, 499, 529, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 518, 533, 507, 530, 527, 524, 480, 559, 545, 518, 489, 475, 526, 506, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 531, 494, 521, 505, 515, 514, 509, 519, 537, 515, 508, 497, 554, 469, 512, 15, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 502, 519, 514, 490, 508, 484, 485, 506, 540, 507, 492, 523, 518, 512, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 499, 491, 497, 512, 530, 536, 489, 483, 507, 499, 561, 541, 495, 507, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 503, 520, 508, 499, 464, 493, 521, 491, 526, 523, 533, 511, 492, 495, 513, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8206, 8208, 8228, 8230, 8152, 8173, 8134] avg=8192.0000 max=8230 max_rank=4 min=8134 min_rank=7 imbalance_ratio=1.004639 max_load_reduction=57306 max_load_reduction_pct=0.874420 moved_to_echo=32689 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[500, 526, 533, 476, 517, 516, 488, 544, 498, 519, 528, 524, 493, 511, 498, 526, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 503, 534, 518, 527, 534, 495, 425, 499, 512, 544, 545, 527, 481, 521, 497, 36, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 521, 503, 531, 527, 538, 512, 540, 510, 499, 506, 507, 509, 507, 503, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 517, 555, 503, 480, 530, 495, 517, 510, 498, 531, 523, 467, 540, 510, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 522, 551, 508, 485, 513, 495, 508, 521, 516, 513, 511, 522, 517, 528, 523, 15, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 512, 476, 522, 521, 505, 512, 530, 528, 506, 531, 484, 533, 522, 485, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 455, 470, 496, 512, 508, 530, 513, 496, 542, 520, 492, 530, 583, 503, 472, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 469, 573, 482, 457, 517, 455, 508, 516, 488, 495, 530, 526, 498, 539, 537, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8189, 8265, 8191, 8231, 8220, 8119, 8116] avg=8192.0000 max=8265 max_rank=2 min=8116 min_rank=7 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32686 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[503, 502, 521, 537, 505, 489, 562, 515, 462, 537, 490, 574, 499, 516, 497, 530, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 478, 532, 504, 486, 491, 527, 492, 508, 488, 503, 527, 518, 493, 488, 490, 34, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 514, 512, 568, 513, 482, 506, 533, 543, 511, 496, 496, 505, 461, 540, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 565, 519, 516, 492, 512, 506, 506, 517, 488, 484, 503, 491, 540, 495, 538, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 492, 517, 496, 535, 510, 509, 519, 488, 489, 529, 496, 490, 489, 496, 517, 15, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 505, 505, 546, 556, 550, 541, 505, 478, 539, 494, 508, 488, 505, 483, 558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 494, 521, 540, 497, 500, 515, 476, 499, 547, 498, 496, 501, 524, 525, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 490, 562, 489, 524, 502, 547, 512, 504, 526, 523, 492, 505, 538, 520, 490, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8246, 8092, 8227, 8198, 8076, 8291, 8195, 8211] avg=8192.0000 max=8291 max_rank=5 min=8076 min_rank=4 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32773 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[547, 512, 484, 527, 525, 540, 499, 497, 497, 505, 515, 476, 476, 518, 545, 517, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 525, 503, 504, 500, 553, 506, 496, 500, 519, 506, 551, 529, 549, 525, 503, 37, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 522, 505, 516, 484, 540, 531, 512, 518, 465, 537, 488, 517, 480, 487, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 528, 512, 540, 512, 531, 498, 512, 533, 499, 481, 539, 486, 524, 513, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 502, 552, 561, 498, 523, 496, 512, 545, 507, 500, 495, 493, 510, 513, 461, 15, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 497, 531, 483, 529, 475, 564, 545, 479, 531, 527, 470, 508, 533, 508, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 525, 509, 499, 501, 494, 511, 475, 491, 465, 501, 531, 528, 494, 512, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 519, 505, 528, 513, 495, 494, 489, 520, 526, 505, 533, 540, 551, 488, 537, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8324, 8119, 8204, 8237, 8161, 8037, 8266] avg=8192.0000 max=8324 max_rank=1 min=8037 min_rank=6 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32701 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[519, 527, 513, 495, 484, 503, 472, 483, 542, 514, 502, 515, 526, 494, 490, 509, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 503, 524, 530, 480, 539, 504, 539, 486, 484, 509, 538, 468, 486, 526, 480, 32, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 505, 500, 499, 553, 499, 534, 520, 499, 509, 484, 529, 528, 541, 479, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 527, 514, 522, 519, 484, 549, 521, 528, 521, 485, 504, 489, 516, 504, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 550, 523, 537, 500, 487, 495, 489, 492, 497, 485, 506, 533, 509, 516, 511, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 542, 529, 503, 482, 524, 489, 466, 529, 512, 507, 489, 530, 541, 519, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 518, 468, 553, 493, 514, 524, 518, 522, 524, 521, 498, 522, 551, 526, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 483, 541, 535, 522, 553, 500, 508, 491, 516, 515, 506, 531, 512, 543, 503, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8095, 8164, 8195, 8211, 8159, 8190, 8286, 8236] avg=8192.0000 max=8286 max_rank=6 min=8095 min_rank=0 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32871 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[495, 495, 478, 483, 537, 501, 530, 525, 509, 485, 517, 486, 521, 516, 533, 540, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 515, 498, 518, 453, 515, 494, 515, 512, 485, 502, 487, 501, 506, 520, 537, 32, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 471, 513, 521, 517, 506, 544, 493, 508, 492, 529, 491, 555, 481, 521, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 507, 503, 507, 521, 506, 525, 491, 488, 544, 512, 501, 529, 519, 501, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 532, 490, 507, 486, 507, 510, 533, 486, 509, 517, 497, 528, 484, 521, 507, 15, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 487, 545, 511, 509, 497, 518, 479, 511, 528, 530, 545, 519, 494, 511, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 528, 502, 537, 549, 537, 541, 536, 495, 523, 483, 514, 536, 547, 509, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 527, 552, 491, 522, 542, 499, 519, 529, 485, 497, 470, 521, 455, 514, 465, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8158, 8091, 8213, 8202, 8141, 8186, 8402, 8143] avg=8192.0000 max=8402 max_rank=6 min=8091 min_rank=1 imbalance_ratio=1.025635 max_load_reduction=57134 max_load_reduction_pct=0.871796 moved_to_echo=32872 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[553, 503, 510, 532, 531, 473, 518, 488, 545, 569, 493, 486, 500, 494, 522, 485, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 484, 463, 470, 520, 522, 534, 517, 501, 485, 518, 503, 496, 528, 503, 525, 34, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 500, 475, 487, 520, 483, 563, 480, 541, 519, 520, 485, 535, 506, 500, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 536, 511, 559, 536, 503, 508, 506, 488, 508, 528, 530, 489, 489, 507, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 507, 529, 490, 455, 504, 505, 502, 538, 517, 526, 536, 516, 511, 503, 500, 15, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 502, 510, 553, 510, 471, 483, 501, 502, 535, 502, 497, 537, 484, 493, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 504, 534, 512, 535, 512, 507, 529, 549, 533, 500, 514, 507, 515, 506, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 552, 518, 541, 504, 529, 506, 506, 513, 529, 536, 491, 475, 517, 511, 525, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8180, 8110, 8184, 8188, 8105, 8254, 8306] avg=8192.0000 max=8306 max_rank=7 min=8105 min_rank=5 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32853 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 3968, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 3968, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4352, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 384, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 3968, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 3968, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4352, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 384, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4352, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[535, 522, 499, 491, 521, 517, 540, 541, 476, 486, 461, 495, 541, 535, 477, 550, 530, 552, 520, 516, 515, 492, 487, 504, 503, 536, 486, 492, 499, 498, 512, 489, 482, 520, 527, 500, 519, 534, 511, 494, 553, 492, 530, 531, 497, 515, 496, 510, 549, 511, 493, 521, 458, 506, 518, 524, 533, 516, 509, 508, 482, 509, 527, 544, 489, 506, 514, 505, 549, 466, 525, 545, 537, 503, 522, 528, 499, 519, 492, 511, 522, 520, 529, 509, 472, 524, 509, 491, 518, 516, 531, 491, 521, 499, 520, 524, 525, 483, 501, 504, 525, 544, 530, 504, 491, 521, 528, 487, 492, 474, 568, 509, 516, 499, 500, 556, 511, 481, 493, 532, 545, 540, 503, 502, 517, 508, 479, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[530, 468, 512, 523, 537, 538, 483, 516, 498, 519, 458, 503, 524, 539, 474, 485, 498, 500, 523, 557, 515, 495, 492, 490, 525, 495, 522, 481, 511, 565, 522, 539, 497, 514, 543, 503, 523, 492, 558, 504, 493, 552, 521, 519, 509, 475, 495, 549, 535, 480, 500, 554, 596, 503, 517, 511, 525, 521, 506, 496, 485, 522, 517, 548, 493, 490, 536, 473, 489, 496, 503, 503, 489, 509, 566, 496, 532, 563, 487, 523, 511, 487, 536, 562, 463, 506, 508, 493, 541, 507, 522, 466, 521, 510, 467, 537, 518, 515, 531, 479, 527, 505, 519, 491, 496, 531, 481, 523, 528, 488, 514, 525, 498, 494, 478, 526, 519, 517, 504, 532, 509, 517, 514, 514, 519, 515, 479, 545] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[499, 499, 508, 522, 541, 531, 480, 489, 513, 488, 495, 535, 538, 530, 529, 504, 526, 505, 509, 497, 489, 487, 522, 536, 501, 529, 478, 538, 531, 475, 512, 515, 538, 472, 566, 523, 524, 521, 499, 514, 516, 494, 488, 500, 537, 543, 526, 514, 545, 499, 501, 522, 497, 491, 469, 487, 521, 483, 505, 479, 527, 507, 511, 539, 495, 490, 516, 554, 509, 502, 550, 551, 484, 454, 529, 510, 479, 543, 470, 502, 531, 485, 492, 511, 520, 530, 531, 499, 489, 527, 511, 537, 541, 537, 561, 491, 547, 521, 490, 541, 510, 522, 519, 501, 525, 497, 480, 542, 515, 504, 501, 501, 519, 527, 517, 554, 501, 492, 520, 496, 502, 523, 506, 477, 483, 548, 517, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[457, 518, 529, 548, 501, 540, 581, 505, 473, 517, 510, 516, 506, 518, 544, 526, 528, 446, 528, 519, 536, 509, 519, 485, 500, 516, 515, 519, 544, 477, 490, 535, 510, 495, 487, 512, 504, 482, 507, 538, 503, 515, 559, 540, 539, 481, 494, 525, 486, 530, 527, 508, 501, 527, 527, 502, 508, 488, 531, 490, 507, 525, 526, 467, 509, 523, 528, 502, 509, 530, 547, 538, 507, 501, 516, 494, 518, 493, 505, 488, 475, 521, 512, 525, 506, 513, 509, 516, 493, 562, 522, 470, 519, 522, 507, 500, 499, 491, 543, 528, 509, 480, 506, 499, 510, 505, 482, 535, 501, 474, 504, 506, 495, 544, 540, 534, 522, 499, 531, 516, 553, 501, 503, 525, 526, 515, 509, 475] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[535, 522, 499, 491, 521, 517, 540, 541, 476, 486, 461, 495, 541, 535, 477, 550, 530, 552, 520, 516, 515, 492, 487, 504, 503, 536, 486, 492, 499, 498, 512, 489, 482, 520, 527, 500, 519, 534, 511, 494, 553, 492, 530, 531, 497, 515, 496, 510, 549, 511, 493, 521, 458, 506, 518, 524, 533, 516, 509, 508, 482, 509, 527, 544, 489, 506, 514, 505, 549, 466, 525, 545, 537, 503, 522, 528, 499, 519, 492, 511, 522, 520, 529, 509, 472, 524, 509, 491, 518, 516, 531, 491, 521, 499, 520, 524, 525, 483, 501, 504, 525, 544, 530, 504, 491, 521, 528, 487, 492, 474, 568, 509, 516, 499, 500, 556, 511, 481, 493, 532, 545, 540, 503, 502, 517, 508, 479, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[462, 482, 523, 525, 517, 537, 544, 476, 484, 478, 513, 529, 517, 518, 528, 514, 514, 545, 535, 534, 530, 534, 491, 497, 533, 489, 482, 468, 575, 498, 500, 510, 529, 535, 556, 528, 487, 536, 546, 522, 522, 531, 532, 529, 506, 490, 527, 494, 533, 496, 534, 447, 501, 529, 492, 558, 511, 524, 533, 485, 582, 557, 477, 493, 529, 484, 526, 534, 558, 524, 518, 542, 500, 516, 466, 490, 492, 495, 472, 487, 528, 502, 520, 489, 484, 512, 532, 536, 509, 516, 501, 520, 517, 506, 492, 493, 533, 513, 494, 523, 541, 489, 500, 502, 509, 545, 485, 492, 533, 553, 501, 517, 529, 463, 521, 506, 510, 499, 517, 475, 480, 463, 490, 523, 515, 516, 490, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[523, 534, 509, 540, 531, 488, 508, 508, 483, 478, 490, 517, 503, 462, 478, 511, 533, 528, 530, 510, 532, 495, 489, 519, 512, 551, 536, 558, 513, 526, 490, 528, 488, 495, 512, 475, 479, 495, 535, 525, 530, 537, 481, 507, 523, 504, 513, 526, 499, 531, 541, 477, 468, 530, 511, 505, 516, 497, 483, 558, 527, 499, 487, 495, 514, 514, 511, 520, 495, 525, 506, 509, 499, 482, 505, 525, 514, 521, 517, 518, 497, 536, 528, 523, 517, 517, 492, 537, 534, 517, 503, 536, 537, 498, 505, 473, 507, 530, 508, 547, 468, 554, 545, 534, 540, 538, 509, 503, 507, 509, 505, 502, 530, 530, 521, 517, 493, 513, 480, 500, 474, 525, 548, 527, 471, 492, 526, 496] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[499, 499, 508, 522, 541, 531, 480, 489, 513, 488, 495, 535, 538, 530, 529, 504, 526, 505, 509, 497, 489, 487, 522, 536, 501, 529, 478, 538, 531, 475, 512, 515, 538, 472, 566, 523, 524, 521, 499, 514, 516, 494, 488, 500, 537, 543, 526, 514, 545, 499, 501, 522, 497, 491, 469, 487, 521, 483, 505, 479, 527, 507, 511, 539, 495, 490, 516, 554, 509, 502, 550, 551, 484, 454, 529, 510, 479, 543, 470, 502, 531, 485, 492, 511, 520, 530, 531, 499, 489, 527, 511, 537, 541, 537, 561, 491, 547, 521, 490, 541, 510, 522, 519, 501, 525, 497, 480, 542, 515, 504, 501, 501, 519, 527, 517, 554, 501, 492, 520, 496, 502, 523, 506, 477, 483, 548, 517, 498] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[555, 532, 532, 503, 485, 545, 489, 483, 557, 538, 522, 508, 557, 520, 526, 504, 518, 504, 532, 517, 515, 462, 505, 472, 483, 509, 536, 467, 523, 514, 500, 497, 513, 479, 507, 530, 501, 503, 476, 509, 498, 518, 521, 528, 491, 535, 494, 530, 513, 503, 509, 511, 482, 501, 511, 509, 532, 505, 508, 542, 505, 518, 507, 537, 553, 563, 495, 503, 498, 496, 533, 498, 558, 511, 519, 516, 552, 506, 521, 492, 523, 509, 505, 449, 504, 535, 517, 512, 524, 548, 463, 491, 503, 518, 536, 518, 503, 514, 505, 492, 515, 477, 482, 526, 542, 526, 515, 540, 539, 517, 492, 498, 541, 492, 493, 526, 489, 496, 500, 550, 512, 524, 495, 511, 514, 510, 495, 500] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[530, 468, 512, 523, 537, 538, 483, 516, 498, 519, 458, 503, 524, 539, 474, 485, 498, 500, 523, 557, 515, 495, 492, 490, 525, 495, 522, 481, 511, 565, 522, 539, 497, 514, 543, 503, 523, 492, 558, 504, 493, 552, 521, 519, 509, 475, 495, 549, 535, 480, 500, 554, 596, 503, 517, 511, 525, 521, 506, 496, 485, 522, 517, 548, 493, 490, 536, 473, 489, 496, 503, 503, 489, 509, 566, 496, 532, 563, 487, 523, 511, 487, 536, 562, 463, 506, 508, 493, 541, 507, 522, 466, 521, 510, 467, 537, 518, 515, 531, 479, 527, 505, 519, 491, 496, 531, 481, 523, 528, 488, 514, 525, 498, 494, 478, 526, 519, 517, 504, 532, 509, 517, 514, 514, 519, 515, 479, 545] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[519, 509, 483, 498, 485, 512, 504, 515, 493, 521, 526, 526, 518, 494, 508, 523, 506, 510, 523, 502, 515, 481, 520, 519, 557, 535, 523, 557, 560, 500, 507, 501, 520, 492, 516, 504, 458, 490, 508, 484, 506, 489, 469, 498, 512, 530, 530, 545, 515, 500, 515, 490, 515, 499, 455, 470, 482, 519, 519, 525, 496, 508, 520, 518, 520, 506, 553, 488, 471, 545, 519, 530, 514, 544, 512, 539, 498, 536, 473, 517, 521, 528, 508, 528, 513, 468, 529, 524, 531, 528, 501, 548, 515, 510, 522, 529, 448, 527, 529, 505, 490, 520, 485, 501, 529, 498, 507, 560, 560, 525, 510, 494, 525, 530, 497, 499, 525, 541, 503, 520, 501, 503, 547, 534, 472, 552, 504, 480] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[523, 534, 509, 540, 531, 488, 508, 508, 483, 478, 490, 517, 503, 462, 478, 511, 533, 528, 530, 510, 532, 495, 489, 519, 512, 551, 536, 558, 513, 526, 490, 528, 488, 495, 512, 475, 479, 495, 535, 525, 530, 537, 481, 507, 523, 504, 513, 526, 499, 531, 541, 477, 468, 530, 511, 505, 516, 497, 483, 558, 527, 499, 487, 495, 514, 514, 511, 520, 495, 525, 506, 509, 499, 482, 505, 525, 514, 521, 517, 518, 497, 536, 528, 523, 517, 517, 492, 537, 534, 517, 503, 536, 537, 498, 505, 473, 507, 530, 508, 547, 468, 554, 545, 534, 540, 538, 509, 503, 507, 509, 505, 502, 530, 530, 521, 517, 493, 513, 480, 500, 474, 525, 548, 527, 471, 492, 526, 496] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[535, 522, 499, 491, 521, 517, 540, 541, 476, 486, 461, 495, 541, 535, 477, 550, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 552, 520, 516, 515, 492, 487, 504, 503, 536, 486, 492, 489, 498, 512, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 520, 519, 500, 519, 534, 511, 494, 553, 492, 530, 531, 497, 515, 496, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 511, 493, 521, 458, 506, 518, 524, 533, 516, 509, 508, 482, 509, 527, 544, 15, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 506, 514, 505, 549, 466, 525, 539, 537, 503, 522, 528, 499, 519, 492, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 520, 529, 509, 472, 524, 509, 491, 518, 501, 531, 491, 521, 499, 520, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 483, 501, 504, 525, 544, 530, 504, 491, 521, 528, 486, 492, 474, 568, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 499, 500, 556, 511, 481, 493, 532, 545, 540, 503, 502, 517, 508, 479, 525, 8, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8121, 8203, 8227, 8204, 8181, 8185, 8223] avg=8192.0000 max=8227 max_rank=3 min=8121 min_rank=1 imbalance_ratio=1.004272 max_load_reduction=57309 max_load_reduction_pct=0.874466 moved_to_echo=32793 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[530, 468, 512, 523, 537, 538, 483, 516, 498, 519, 458, 503, 524, 539, 474, 485, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 500, 523, 557, 515, 495, 492, 490, 525, 495, 522, 481, 491, 565, 522, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 514, 531, 503, 523, 492, 558, 504, 493, 552, 521, 519, 509, 475, 495, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 480, 500, 554, 596, 503, 517, 511, 525, 521, 506, 496, 485, 522, 517, 548, 20, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 490, 536, 473, 489, 496, 503, 490, 489, 509, 566, 496, 532, 563, 487, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 487, 536, 562, 463, 506, 508, 493, 541, 487, 522, 466, 521, 510, 467, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 515, 531, 479, 527, 505, 519, 491, 496, 531, 481, 518, 528, 488, 514, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 494, 478, 526, 519, 517, 504, 532, 509, 517, 514, 514, 519, 515, 479, 545, 12, 11, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8120, 8210, 8235, 8345, 8135, 8117, 8166, 8208] avg=8192.0000 max=8345 max_rank=3 min=8117 min_rank=5 imbalance_ratio=1.018677 max_load_reduction=57191 max_load_reduction_pct=0.872665 moved_to_echo=32626 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[457, 518, 529, 548, 501, 540, 581, 505, 473, 517, 510, 516, 506, 518, 544, 526, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 446, 528, 519, 536, 509, 519, 485, 500, 516, 515, 519, 533, 477, 490, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 495, 480, 512, 504, 482, 507, 538, 503, 515, 559, 540, 539, 481, 494, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 530, 527, 508, 501, 527, 527, 502, 508, 488, 531, 490, 507, 525, 526, 467, 16, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 523, 528, 502, 509, 530, 547, 532, 507, 501, 516, 494, 518, 493, 505, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 521, 512, 525, 506, 513, 509, 516, 493, 546, 522, 470, 519, 522, 507, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 491, 543, 528, 509, 480, 506, 499, 510, 505, 482, 533, 501, 474, 504, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 544, 540, 534, 522, 499, 531, 516, 553, 501, 503, 525, 526, 515, 509, 475, 7, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8295, 8155, 8184, 8170, 8202, 8156, 8070, 8304] avg=8192.0000 max=8304 max_rank=7 min=8070 min_rank=6 imbalance_ratio=1.013672 max_load_reduction=57232 max_load_reduction_pct=0.873291 moved_to_echo=32732 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[535, 522, 499, 491, 521, 517, 540, 541, 476, 486, 461, 495, 541, 535, 477, 550, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 552, 520, 516, 515, 492, 487, 504, 503, 536, 486, 492, 489, 498, 512, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 520, 519, 500, 519, 534, 511, 494, 553, 492, 530, 531, 497, 515, 496, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 511, 493, 521, 458, 506, 518, 524, 533, 516, 509, 508, 482, 509, 527, 544, 15, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 506, 514, 505, 549, 466, 525, 539, 537, 503, 522, 528, 499, 519, 492, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 520, 529, 509, 472, 524, 509, 491, 518, 501, 531, 491, 521, 499, 520, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 483, 501, 504, 525, 544, 530, 504, 491, 521, 528, 486, 492, 474, 568, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 499, 500, 556, 511, 481, 493, 532, 545, 540, 503, 502, 517, 508, 479, 525, 8, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8121, 8203, 8227, 8204, 8181, 8185, 8223] avg=8192.0000 max=8227 max_rank=3 min=8121 min_rank=1 imbalance_ratio=1.004272 max_load_reduction=57309 max_load_reduction_pct=0.874466 moved_to_echo=32793 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[462, 482, 523, 525, 517, 537, 544, 476, 484, 478, 513, 529, 517, 518, 528, 514, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 545, 535, 534, 530, 534, 491, 497, 533, 489, 482, 468, 564, 498, 500, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 535, 548, 528, 487, 536, 546, 522, 522, 531, 532, 529, 506, 490, 527, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 496, 534, 447, 501, 529, 492, 558, 511, 524, 533, 485, 582, 557, 477, 493, 15, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 484, 526, 534, 558, 524, 518, 536, 500, 516, 466, 490, 492, 495, 472, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 502, 520, 489, 484, 512, 532, 536, 509, 501, 501, 520, 517, 506, 492, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 513, 494, 523, 541, 489, 500, 502, 509, 545, 485, 491, 533, 553, 501, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 463, 521, 506, 510, 499, 517, 475, 480, 463, 490, 523, 515, 516, 490, 515, 8, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8152, 8224, 8362, 8271, 8127, 8142, 8229, 8029] avg=8192.0000 max=8362 max_rank=2 min=8029 min_rank=7 imbalance_ratio=1.020752 max_load_reduction=57174 max_load_reduction_pct=0.872406 moved_to_echo=32527 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[499, 499, 508, 522, 541, 531, 480, 489, 513, 488, 495, 535, 538, 530, 529, 504, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 505, 509, 497, 489, 487, 522, 536, 501, 529, 478, 538, 520, 475, 512, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 472, 558, 523, 524, 521, 499, 514, 516, 494, 488, 500, 537, 543, 526, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 499, 501, 522, 497, 491, 469, 487, 521, 483, 505, 479, 527, 507, 511, 539, 15, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 490, 516, 554, 509, 502, 550, 545, 484, 454, 529, 510, 479, 543, 470, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 485, 492, 511, 520, 530, 531, 499, 489, 512, 511, 537, 541, 537, 561, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 521, 490, 541, 510, 522, 519, 501, 525, 497, 480, 540, 515, 504, 501, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 527, 517, 554, 501, 492, 520, 496, 502, 523, 506, 477, 483, 548, 517, 498, 8, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8139, 8267, 8102, 8132, 8278, 8214, 8197] avg=8192.0000 max=8278 max_rank=5 min=8102 min_rank=3 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32821 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[523, 534, 509, 540, 531, 488, 508, 508, 483, 478, 490, 517, 503, 462, 478, 511, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 528, 530, 510, 532, 495, 489, 519, 512, 551, 536, 558, 502, 526, 490, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 495, 504, 475, 479, 495, 535, 525, 530, 537, 481, 507, 523, 504, 513, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 531, 541, 477, 468, 530, 511, 505, 516, 497, 483, 558, 527, 499, 487, 495, 15, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 514, 511, 520, 495, 525, 506, 503, 499, 482, 505, 525, 514, 521, 517, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 536, 528, 523, 517, 517, 492, 537, 534, 502, 503, 536, 537, 498, 505, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 530, 508, 547, 468, 554, 545, 534, 540, 538, 509, 502, 507, 509, 505, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 530, 521, 517, 493, 513, 480, 500, 474, 525, 548, 527, 471, 492, 526, 496, 8, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8068, 8339, 8117, 8143, 8169, 8235, 8305, 8160] avg=8192.0000 max=8339 max_rank=1 min=8068 min_rank=0 imbalance_ratio=1.017944 max_load_reduction=57197 max_load_reduction_pct=0.872757 moved_to_echo=32869 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[555, 532, 532, 503, 485, 545, 489, 483, 557, 538, 522, 508, 557, 520, 526, 504, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 504, 532, 517, 515, 462, 505, 472, 483, 509, 536, 467, 512, 514, 500, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 479, 499, 530, 501, 503, 476, 509, 498, 518, 521, 528, 491, 535, 494, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 503, 509, 511, 482, 501, 511, 509, 532, 505, 508, 542, 505, 518, 507, 537, 16, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 563, 495, 503, 498, 496, 533, 492, 558, 511, 519, 516, 552, 506, 521, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 509, 505, 449, 504, 535, 517, 512, 524, 532, 463, 491, 503, 518, 536, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 514, 505, 492, 515, 477, 482, 526, 542, 526, 515, 538, 539, 517, 492, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 492, 493, 526, 489, 496, 500, 550, 512, 524, 495, 511, 514, 510, 495, 500, 8, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8362, 8043, 8125, 8213, 8308, 8139, 8181, 8165] avg=8192.0000 max=8362 max_rank=0 min=8043 min_rank=1 imbalance_ratio=1.020752 max_load_reduction=57174 max_load_reduction_pct=0.872406 moved_to_echo=32793 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[519, 509, 483, 498, 485, 512, 504, 515, 493, 521, 526, 526, 518, 494, 508, 523, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 510, 523, 502, 515, 481, 520, 519, 557, 535, 523, 557, 549, 500, 507, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 492, 508, 504, 458, 490, 508, 484, 506, 489, 469, 498, 512, 530, 530, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 500, 515, 490, 515, 499, 455, 470, 482, 519, 519, 525, 496, 508, 520, 518, 15, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 506, 553, 488, 471, 545, 519, 524, 514, 544, 512, 539, 498, 536, 473, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 528, 508, 528, 513, 468, 529, 524, 531, 513, 501, 548, 515, 510, 522, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 448, 527, 529, 505, 490, 520, 485, 501, 529, 498, 507, 558, 560, 525, 510, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 530, 497, 499, 525, 541, 503, 520, 501, 503, 547, 534, 472, 552, 504, 480, 8, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8140, 8305, 8043, 8065, 8259, 8288, 8186, 8250] avg=8192.0000 max=8305 max_rank=1 min=8043 min_rank=2 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32983 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[523, 534, 509, 540, 531, 488, 508, 508, 483, 478, 490, 517, 503, 462, 478, 511, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 528, 530, 510, 532, 495, 489, 519, 512, 551, 536, 558, 502, 526, 490, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 495, 504, 475, 479, 495, 535, 525, 530, 537, 481, 507, 523, 504, 513, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 531, 541, 477, 468, 530, 511, 505, 516, 497, 483, 558, 527, 499, 487, 495, 15, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 514, 511, 520, 495, 525, 506, 503, 499, 482, 505, 525, 514, 521, 517, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 536, 528, 523, 517, 517, 492, 537, 534, 502, 503, 536, 537, 498, 505, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 530, 508, 547, 468, 554, 545, 534, 540, 538, 509, 502, 507, 509, 505, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 530, 521, 517, 493, 513, 480, 500, 474, 525, 548, 527, 471, 492, 526, 496, 8, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8068, 8339, 8117, 8143, 8169, 8235, 8305, 8160] avg=8192.0000 max=8339 max_rank=1 min=8068 min_rank=0 imbalance_ratio=1.017944 max_load_reduction=57197 max_load_reduction_pct=0.872757 moved_to_echo=32869 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 3968, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 3968, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 3968, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True - [2026-08-26 14:49:31] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 416.4 | throughput per GPU (TFLOP/s/GPU): 434.3 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.297648E+00 | loss scale: 1.0 | grad norm: 5.375 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[518, 510, 533, 528, 554, 511, 514, 513, 492, 488, 503, 511, 489, 540, 526, 521, 520, 536, 517, 509, 502, 510, 513, 504, 509, 530, 522, 488, 488, 522, 493, 495, 498, 494, 493, 515, 490, 542, 518, 488, 487, 527, 511, 524, 525, 516, 514, 523, 523, 557, 525, 534, 511, 552, 475, 494, 513, 507, 514, 526, 498, 490, 486, 464, 521, 495, 542, 573, 464, 511, 516, 519, 539, 498, 476, 516, 497, 511, 499, 485, 521, 519, 540, 522, 547, 492, 509, 548, 502, 516, 502, 529, 495, 501, 502, 491, 476, 523, 496, 482, 498, 498, 548, 522, 485, 472, 457, 511, 505, 556, 529, 520, 520, 549, 532, 527, 489, 492, 484, 564, 553, 505, 489, 548, 513, 523, 517, 512] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[493, 489, 494, 529, 511, 516, 516, 531, 506, 511, 477, 491, 484, 516, 513, 521, 522, 538, 551, 508, 474, 512, 517, 510, 502, 541, 512, 534, 492, 520, 537, 540, 503, 471, 518, 538, 525, 562, 479, 526, 543, 552, 483, 484, 536, 509, 478, 493, 531, 524, 533, 524, 486, 546, 520, 520, 524, 528, 488, 478, 489, 512, 509, 497, 534, 502, 489, 464, 513, 494, 515, 526, 544, 561, 535, 532, 490, 472, 574, 550, 509, 498, 485, 541, 484, 542, 513, 487, 510, 538, 494, 510, 543, 532, 488, 495, 503, 491, 494, 525, 554, 486, 488, 485, 524, 532, 519, 507, 516, 505, 491, 474, 513, 531, 511, 523, 495, 492, 491, 554, 478, 513, 523, 511, 514, 539, 524, 449] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 + [2026-08-26 15:25:34] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 414.2 | throughput per GPU (TFLOP/s/GPU): 436.5 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.307316E+00 | loss scale: 1.0 | grad norm: 5.359 | number of skipped iterations: 0 | number of nan iterations: 0 | WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[490, 481, 521, 485, 552, 529, 469, 510, 523, 513, 488, 518, 492, 529, 534, 514, 526, 540, 510, 520, 500, 507, 504, 521, 454, 486, 526, 497, 523, 504, 518, 512, 507, 513, 552, 498, 518, 500, 530, 502, 475, 486, 553, 462, 523, 524, 501, 497, 469, 499, 507, 518, 493, 545, 526, 547, 493, 528, 505, 505, 476, 512, 518, 538, 518, 503, 476, 535, 535, 470, 523, 495, 490, 504, 514, 538, 523, 495, 540, 513, 542, 518, 504, 501, 537, 537, 551, 509, 540, 522, 514, 495, 545, 516, 530, 505, 539, 490, 504, 481, 507, 531, 505, 523, 498, 482, 510, 555, 532, 553, 486, 490, 498, 503, 509, 534, 495, 532, 527, 512, 518, 519, 529, 519, 514, 469, 496, 522] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[518, 510, 533, 528, 554, 511, 514, 513, 492, 488, 503, 511, 489, 540, 526, 521, 520, 536, 517, 509, 502, 510, 513, 504, 509, 530, 522, 488, 488, 522, 493, 495, 498, 494, 493, 515, 490, 542, 518, 488, 487, 527, 511, 524, 525, 516, 514, 523, 523, 557, 525, 534, 511, 552, 475, 494, 513, 507, 514, 526, 498, 490, 486, 464, 521, 495, 542, 573, 464, 511, 516, 519, 539, 498, 476, 516, 497, 511, 499, 485, 521, 519, 540, 522, 547, 492, 509, 548, 502, 516, 502, 529, 495, 501, 502, 491, 476, 523, 496, 482, 498, 498, 548, 522, 485, 472, 457, 511, 505, 556, 529, 520, 520, 549, 532, 527, 489, 492, 484, 564, 553, 505, 489, 548, 513, 523, 517, 512] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[522, 510, 504, 494, 482, 493, 530, 501, 524, 504, 521, 535, 589, 540, 483, 536, 531, 551, 485, 516, 517, 507, 556, 521, 497, 515, 496, 503, 496, 523, 480, 511, 518, 478, 473, 494, 529, 485, 512, 547, 511, 553, 515, 515, 495, 528, 476, 507, 513, 525, 487, 537, 489, 527, 517, 502, 543, 495, 491, 497, 526, 500, 523, 464, 525, 537, 497, 515, 487, 491, 548, 506, 500, 509, 512, 528, 502, 487, 503, 526, 518, 469, 513, 474, 526, 481, 505, 543, 551, 496, 528, 498, 540, 498, 530, 514, 510, 496, 533, 509, 481, 533, 545, 488, 529, 462, 557, 530, 525, 547, 469, 533, 494, 494, 521, 514, 497, 523, 512, 532, 549, 539, 502, 497, 489, 512, 491, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[450, 530, 488, 510, 508, 518, 521, 517, 516, 536, 491, 504, 517, 530, 519, 520, 500, 552, 498, 531, 529, 525, 493, 547, 516, 494, 530, 482, 525, 537, 505, 525, 516, 475, 523, 529, 521, 483, 530, 502, 503, 466, 496, 504, 483, 507, 515, 502, 510, 497, 509, 484, 554, 521, 517, 502, 523, 501, 500, 467, 505, 524, 523, 497, 530, 522, 517, 541, 550, 511, 533, 477, 499, 501, 528, 541, 533, 525, 522, 535, 538, 489, 507, 507, 538, 509, 528, 507, 504, 501, 521, 497, 509, 516, 518, 509, 533, 490, 524, 533, 479, 480, 510, 521, 509, 511, 492, 503, 516, 492, 503, 534, 525, 505, 468, 506, 549, 535, 485, 502, 510, 504, 513, 502, 527, 521, 508, 530] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[493, 489, 494, 529, 511, 516, 516, 531, 506, 511, 477, 491, 484, 516, 513, 521, 522, 538, 551, 508, 474, 512, 517, 510, 502, 541, 512, 534, 492, 520, 537, 540, 503, 471, 518, 538, 525, 562, 479, 526, 543, 552, 483, 484, 536, 509, 478, 493, 531, 524, 533, 524, 486, 546, 520, 520, 524, 528, 488, 478, 489, 512, 509, 497, 534, 502, 489, 464, 513, 494, 515, 526, 544, 561, 535, 532, 490, 472, 574, 550, 509, 498, 485, 541, 484, 542, 513, 487, 510, 538, 494, 510, 543, 532, 488, 495, 503, 491, 494, 525, 554, 486, 488, 485, 524, 532, 519, 507, 516, 505, 491, 474, 513, 531, 511, 523, 495, 492, 491, 554, 478, 513, 523, 511, 514, 539, 524, 449] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[523, 546, 511, 516, 504, 534, 544, 511, 497, 482, 532, 557, 528, 516, 539, 524, 506, 482, 473, 541, 488, 510, 517, 512, 530, 520, 499, 523, 508, 493, 513, 497, 507, 510, 478, 490, 512, 506, 502, 522, 558, 524, 513, 490, 535, 541, 500, 524, 495, 538, 490, 517, 526, 520, 524, 498, 504, 514, 519, 518, 550, 503, 502, 524, 514, 479, 498, 499, 538, 520, 463, 491, 513, 509, 481, 517, 496, 522, 529, 537, 500, 519, 482, 516, 523, 455, 524, 552, 497, 511, 489, 515, 547, 495, 504, 513, 554, 510, 503, 509, 513, 476, 518, 506, 505, 519, 532, 495, 502, 489, 482, 473, 564, 501, 527, 483, 562, 524, 501, 528, 502, 492, 553, 504, 489, 523, 494, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[545, 496, 526, 520, 506, 489, 501, 558, 529, 515, 522, 520, 491, 532, 511, 491, 540, 494, 465, 559, 476, 515, 521, 561, 516, 508, 499, 557, 519, 566, 490, 548, 501, 532, 523, 531, 524, 521, 516, 503, 522, 507, 517, 475, 542, 525, 510, 522, 519, 468, 506, 502, 529, 518, 433, 516, 502, 530, 473, 518, 506, 481, 496, 495, 483, 522, 529, 529, 486, 539, 524, 513, 500, 549, 528, 515, 519, 526, 502, 552, 541, 513, 522, 530, 529, 504, 504, 518, 468, 486, 550, 497, 493, 489, 476, 486, 501, 518, 487, 502, 506, 475, 524, 527, 505, 514, 509, 500, 541, 518, 500, 501, 495, 504, 527, 519, 524, 527, 482, 525, 510, 482, 515, 492, 521, 518, 463, 533] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[514, 511, 499, 524, 525, 508, 482, 472, 483, 514, 510, 513, 542, 520, 538, 548, 524, 538, 519, 515, 525, 509, 518, 506, 509, 529, 506, 537, 499, 512, 516, 500, 486, 511, 521, 491, 511, 503, 567, 474, 500, 524, 571, 529, 545, 495, 506, 540, 548, 542, 482, 499, 487, 490, 499, 493, 539, 521, 537, 495, 521, 509, 518, 500, 547, 508, 535, 515, 470, 515, 505, 509, 509, 526, 494, 503, 545, 499, 508, 538, 504, 509, 538, 468, 539, 501, 496, 516, 511, 505, 524, 510, 526, 498, 532, 503, 538, 540, 521, 495, 504, 512, 521, 494, 540, 490, 484, 486, 488, 503, 498, 482, 469, 469, 525, 540, 517, 513, 508, 489, 499, 505, 515, 515, 495, 540, 512, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[490, 481, 521, 485, 552, 529, 469, 510, 523, 513, 488, 518, 492, 502, 534, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 505, 510, 520, 500, 507, 504, 521, 454, 486, 526, 497, 523, 504, 518, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 513, 552, 498, 518, 500, 530, 502, 475, 486, 553, 462, 523, 524, 501, 497, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 499, 507, 518, 493, 545, 526, 547, 493, 528, 505, 505, 476, 512, 518, 538, 27, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 503, 476, 535, 535, 470, 523, 495, 490, 504, 514, 538, 523, 495, 540, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 518, 504, 501, 532, 537, 551, 509, 540, 522, 514, 495, 545, 516, 530, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 490, 504, 481, 507, 531, 505, 523, 498, 482, 510, 555, 532, 553, 486, 490, 35, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 503, 509, 534, 495, 532, 527, 508, 518, 519, 529, 519, 514, 469, 496, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8121, 8113, 8150, 8214, 8139, 8361, 8246, 8192] avg=8192.0000 max=8361 max_rank=5 min=8113 min_rank=1 imbalance_ratio=1.020630 max_load_reduction=57175 max_load_reduction_pct=0.872421 moved_to_echo=32938 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[518, 510, 533, 528, 554, 511, 514, 513, 492, 488, 503, 511, 489, 512, 526, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 501, 517, 509, 502, 510, 513, 504, 509, 530, 522, 488, 488, 522, 493, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 494, 493, 515, 490, 542, 518, 488, 487, 527, 511, 524, 525, 516, 514, 523, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 557, 525, 534, 511, 552, 475, 494, 513, 507, 514, 526, 498, 490, 486, 464, 28, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 495, 542, 573, 464, 511, 516, 519, 539, 498, 476, 516, 497, 511, 499, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 519, 540, 522, 542, 492, 509, 548, 502, 516, 502, 529, 495, 501, 502, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 523, 496, 482, 498, 498, 548, 522, 485, 472, 457, 511, 505, 556, 529, 520, 35, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 549, 532, 527, 489, 492, 484, 559, 553, 505, 489, 548, 513, 523, 517, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8223, 8123, 8175, 8205, 8131, 8231, 8136, 8312] avg=8192.0000 max=8312 max_rank=7 min=8123 min_rank=1 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32810 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[450, 530, 488, 510, 508, 518, 521, 517, 516, 536, 491, 504, 517, 499, 519, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 510, 498, 531, 529, 525, 493, 547, 516, 494, 530, 482, 525, 537, 505, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 475, 523, 529, 521, 483, 530, 502, 503, 466, 496, 504, 483, 507, 515, 502, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 497, 509, 484, 554, 521, 517, 502, 523, 501, 500, 467, 505, 524, 523, 497, 31, 12, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 522, 517, 541, 550, 511, 533, 477, 499, 501, 528, 541, 533, 525, 522, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 489, 507, 507, 527, 509, 528, 507, 504, 501, 521, 497, 509, 516, 518, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 490, 524, 533, 479, 480, 510, 521, 509, 511, 492, 503, 516, 492, 503, 534, 42, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 505, 468, 506, 549, 535, 485, 493, 510, 504, 513, 502, 527, 521, 508, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8144, 8247, 8073, 8179, 8323, 8187, 8202, 8181] avg=8192.0000 max=8323 max_rank=4 min=8073 min_rank=2 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32893 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[518, 510, 533, 528, 554, 511, 514, 513, 492, 488, 503, 511, 489, 512, 526, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 501, 517, 509, 502, 510, 513, 504, 509, 530, 522, 488, 488, 522, 493, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 494, 493, 515, 490, 542, 518, 488, 487, 527, 511, 524, 525, 516, 514, 523, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 557, 525, 534, 511, 552, 475, 494, 513, 507, 514, 526, 498, 490, 486, 464, 28, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 495, 542, 573, 464, 511, 516, 519, 539, 498, 476, 516, 497, 511, 499, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 519, 540, 522, 542, 492, 509, 548, 502, 516, 502, 529, 495, 501, 502, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 523, 496, 482, 498, 498, 548, 522, 485, 472, 457, 511, 505, 556, 529, 520, 35, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 549, 532, 527, 489, 492, 484, 559, 553, 505, 489, 548, 513, 523, 517, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8223, 8123, 8175, 8205, 8131, 8231, 8136, 8312] avg=8192.0000 max=8312 max_rank=7 min=8123 min_rank=1 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32810 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[493, 489, 494, 529, 511, 516, 516, 531, 506, 511, 477, 491, 484, 489, 513, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 503, 551, 508, 474, 512, 517, 510, 502, 541, 512, 534, 492, 520, 537, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 471, 518, 538, 525, 562, 479, 526, 543, 552, 483, 484, 536, 509, 478, 493, 4, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 524, 533, 524, 486, 546, 520, 520, 524, 528, 488, 478, 489, 512, 509, 497, 27, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 502, 489, 464, 513, 494, 515, 526, 544, 561, 535, 532, 490, 472, 574, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 498, 485, 541, 480, 542, 513, 487, 510, 538, 494, 510, 543, 532, 488, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 491, 494, 525, 554, 486, 488, 485, 524, 532, 519, 507, 516, 505, 491, 474, 35, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 531, 511, 523, 495, 492, 491, 549, 478, 513, 523, 511, 514, 539, 524, 449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8071, 8275, 8209, 8245, 8259, 8165, 8156, 8156] avg=8192.0000 max=8275 max_rank=1 min=8071 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32736 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[523, 546, 511, 516, 504, 534, 544, 511, 497, 482, 532, 557, 528, 489, 539, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 450, 473, 541, 488, 510, 517, 512, 530, 520, 499, 523, 508, 493, 513, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 510, 478, 490, 512, 506, 502, 522, 558, 524, 513, 490, 535, 541, 500, 524, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 538, 490, 517, 526, 520, 524, 498, 504, 514, 519, 518, 550, 503, 502, 524, 27, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 479, 498, 499, 538, 520, 463, 491, 513, 509, 481, 517, 496, 522, 529, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 519, 482, 516, 518, 455, 524, 552, 497, 511, 489, 515, 547, 495, 504, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 510, 503, 509, 513, 476, 518, 506, 505, 519, 532, 495, 502, 489, 482, 473, 32, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 564, 501, 527, 483, 562, 524, 501, 524, 502, 492, 553, 504, 489, 523, 494, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8337, 8080, 8221, 8277, 8072, 8137, 8144, 8268] avg=8192.0000 max=8337 max_rank=0 min=8072 min_rank=4 imbalance_ratio=1.017700 max_load_reduction=57199 max_load_reduction_pct=0.872787 moved_to_echo=32621 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[490, 481, 521, 485, 552, 529, 469, 510, 523, 513, 488, 518, 492, 502, 534, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 505, 510, 520, 500, 507, 504, 521, 454, 486, 526, 497, 523, 504, 518, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 513, 552, 498, 518, 500, 530, 502, 475, 486, 553, 462, 523, 524, 501, 497, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 499, 507, 518, 493, 545, 526, 547, 493, 528, 505, 505, 476, 512, 518, 538, 27, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 503, 476, 535, 535, 470, 523, 495, 490, 504, 514, 538, 523, 495, 540, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 518, 504, 501, 532, 537, 551, 509, 540, 522, 514, 495, 545, 516, 530, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 490, 504, 481, 507, 531, 505, 523, 498, 482, 510, 555, 532, 553, 486, 490, 35, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 503, 509, 534, 495, 532, 527, 508, 518, 519, 529, 519, 514, 469, 496, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8121, 8113, 8150, 8214, 8139, 8361, 8246, 8192] avg=8192.0000 max=8361 max_rank=5 min=8113 min_rank=1 imbalance_ratio=1.020630 max_load_reduction=57175 max_load_reduction_pct=0.872421 moved_to_echo=32938 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[522, 510, 504, 494, 482, 493, 530, 501, 524, 504, 521, 535, 589, 512, 483, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 515, 485, 516, 517, 507, 556, 521, 497, 515, 496, 503, 496, 523, 480, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 478, 473, 494, 529, 485, 512, 547, 511, 553, 515, 515, 495, 528, 476, 507, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 525, 487, 537, 489, 527, 517, 502, 543, 495, 491, 497, 526, 500, 523, 464, 28, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 537, 497, 515, 487, 491, 548, 506, 500, 509, 512, 528, 502, 487, 503, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 469, 513, 474, 521, 481, 505, 543, 551, 496, 528, 498, 540, 498, 530, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 496, 533, 509, 481, 533, 545, 488, 529, 462, 557, 530, 525, 547, 469, 533, 36, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 494, 521, 514, 497, 523, 512, 528, 549, 539, 502, 497, 489, 512, 491, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8240, 8169, 8145, 8172, 8140, 8179, 8308, 8183] avg=8192.0000 max=8308 max_rank=6 min=8140 min_rank=4 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32810 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[493, 489, 494, 529, 511, 516, 516, 531, 506, 511, 477, 491, 484, 489, 513, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 503, 551, 508, 474, 512, 517, 510, 502, 541, 512, 534, 492, 520, 537, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 471, 518, 538, 525, 562, 479, 526, 543, 552, 483, 484, 536, 509, 478, 493, 4, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 524, 533, 524, 486, 546, 520, 520, 524, 528, 488, 478, 489, 512, 509, 497, 27, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 502, 489, 464, 513, 494, 515, 526, 544, 561, 535, 532, 490, 472, 574, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 498, 485, 541, 480, 542, 513, 487, 510, 538, 494, 510, 543, 532, 488, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 491, 494, 525, 554, 486, 488, 485, 524, 532, 519, 507, 516, 505, 491, 474, 35, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 531, 511, 523, 495, 492, 491, 549, 478, 513, 523, 511, 514, 539, 524, 449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8071, 8275, 8209, 8245, 8259, 8165, 8156, 8156] avg=8192.0000 max=8275 max_rank=1 min=8071 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32736 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[545, 496, 526, 520, 506, 489, 501, 558, 529, 515, 522, 520, 491, 504, 511, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 462, 465, 559, 476, 515, 521, 561, 516, 508, 499, 557, 519, 566, 490, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 532, 523, 531, 524, 521, 516, 503, 522, 507, 517, 475, 542, 525, 510, 522, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 468, 506, 502, 529, 518, 433, 516, 502, 530, 473, 518, 506, 481, 496, 495, 28, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 522, 529, 529, 486, 539, 524, 513, 500, 549, 528, 515, 519, 526, 502, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 513, 522, 530, 524, 504, 504, 518, 468, 486, 550, 497, 493, 489, 476, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 518, 487, 502, 506, 475, 524, 527, 505, 514, 509, 500, 541, 518, 500, 501, 32, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 504, 527, 519, 524, 527, 482, 521, 510, 482, 515, 492, 521, 518, 463, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8224, 8302, 8280, 8029, 8280, 8101, 8187, 8133] avg=8192.0000 max=8302 max_rank=1 min=8029 min_rank=3 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32701 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[514, 511, 499, 524, 525, 508, 482, 472, 483, 514, 510, 513, 542, 493, 538, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 503, 519, 515, 525, 509, 518, 506, 509, 529, 506, 537, 499, 512, 516, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 511, 521, 491, 511, 503, 567, 474, 500, 524, 571, 529, 545, 495, 506, 540, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 542, 482, 499, 487, 490, 499, 493, 539, 521, 537, 495, 521, 509, 518, 500, 27, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 508, 535, 515, 470, 515, 505, 509, 509, 526, 494, 503, 545, 499, 508, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 509, 538, 468, 534, 501, 496, 516, 511, 505, 524, 510, 526, 498, 532, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 540, 521, 495, 504, 512, 521, 494, 540, 490, 484, 486, 488, 503, 498, 482, 35, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 469, 525, 540, 517, 513, 508, 485, 499, 505, 515, 515, 495, 540, 512, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8227, 8283, 8215, 8192, 8175, 8157, 8111] avg=8192.0000 max=8283 max_rank=2 min=8111 min_rank=7 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32635 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[545, 496, 526, 520, 506, 489, 501, 558, 529, 515, 522, 520, 491, 504, 511, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 462, 465, 559, 476, 515, 521, 561, 516, 508, 499, 557, 519, 566, 490, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 532, 523, 531, 524, 521, 516, 503, 522, 507, 517, 475, 542, 525, 510, 522, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 468, 506, 502, 529, 518, 433, 516, 502, 530, 473, 518, 506, 481, 496, 495, 28, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 522, 529, 529, 486, 539, 524, 513, 500, 549, 528, 515, 519, 526, 502, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 513, 522, 530, 524, 504, 504, 518, 468, 486, 550, 497, 493, 489, 476, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 518, 487, 502, 506, 475, 524, 527, 505, 514, 509, 500, 541, 518, 500, 501, 32, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 504, 527, 519, 524, 527, 482, 521, 510, 482, 515, 492, 521, 518, 463, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8224, 8302, 8280, 8029, 8280, 8101, 8187, 8133] avg=8192.0000 max=8302 max_rank=1 min=8029 min_rank=3 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32701 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 3968, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 3968, 4096, 384, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 3968, 4096, 384, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[502, 533, 546, 471, 509, 554, 494, 495, 559, 512, 493, 505, 539, 491, 506, 496, 503, 528, 486, 480, 496, 513, 504, 523, 509, 509, 489, 505, 509, 514, 519, 521, 549, 495, 531, 532, 502, 491, 490, 535, 513, 486, 514, 479, 508, 506, 546, 551, 528, 537, 524, 496, 570, 521, 497, 497, 523, 533, 505, 539, 529, 530, 526, 506, 516, 489, 506, 515, 529, 486, 535, 532, 494, 480, 470, 473, 561, 478, 510, 504, 542, 516, 499, 516, 523, 476, 521, 532, 528, 509, 569, 514, 534, 501, 504, 509, 534, 490, 485, 512, 515, 519, 506, 497, 502, 463, 498, 491, 498, 530, 496, 530, 505, 511, 522, 513, 498, 525, 527, 498, 518, 497, 490, 530, 518, 501, 516, 528] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[490, 528, 530, 544, 518, 517, 494, 520, 470, 541, 551, 516, 495, 521, 516, 472, 486, 529, 508, 499, 487, 505, 513, 543, 518, 530, 519, 511, 535, 534, 519, 519, 550, 515, 522, 514, 496, 539, 537, 517, 475, 511, 505, 526, 537, 500, 523, 494, 534, 487, 575, 506, 486, 496, 497, 508, 503, 449, 502, 512, 544, 512, 511, 497, 461, 539, 488, 475, 504, 520, 517, 549, 505, 527, 522, 501, 481, 519, 507, 518, 487, 503, 488, 520, 499, 528, 535, 541, 529, 509, 517, 540, 506, 495, 491, 539, 479, 502, 494, 476, 516, 484, 479, 510, 503, 484, 561, 535, 515, 505, 488, 532, 503, 497, 526, 461, 510, 517, 518, 560, 559, 549, 547, 526, 503, 502, 476, 501] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[497, 505, 501, 481, 552, 512, 511, 515, 551, 519, 525, 510, 528, 525, 485, 548, 536, 523, 516, 511, 506, 534, 528, 573, 466, 536, 543, 503, 565, 509, 473, 503, 494, 492, 475, 503, 548, 482, 515, 503, 526, 502, 508, 504, 440, 493, 502, 491, 516, 527, 502, 499, 485, 541, 547, 508, 508, 494, 522, 537, 551, 485, 498, 499, 504, 514, 532, 471, 498, 505, 537, 498, 560, 502, 517, 523, 517, 494, 528, 510, 522, 508, 524, 505, 513, 547, 522, 493, 487, 521, 506, 481, 514, 487, 473, 519, 521, 517, 520, 498, 502, 538, 514, 521, 471, 519, 519, 516, 504, 503, 513, 493, 541, 521, 512, 519, 504, 546, 508, 521, 493, 504, 477, 552, 518, 507, 493, 532] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[524, 493, 506, 489, 525, 529, 474, 513, 507, 532, 544, 511, 530, 493, 531, 513, 497, 517, 535, 522, 568, 482, 526, 551, 515, 512, 539, 501, 476, 525, 530, 530, 534, 503, 559, 505, 518, 501, 517, 538, 456, 513, 553, 526, 523, 529, 522, 522, 483, 522, 529, 503, 521, 508, 529, 504, 490, 489, 495, 537, 519, 481, 505, 456, 516, 494, 522, 517, 498, 519, 532, 500, 496, 521, 519, 512, 505, 530, 510, 515, 495, 515, 519, 500, 503, 455, 510, 487, 500, 538, 521, 489, 533, 483, 508, 500, 482, 479, 566, 476, 502, 508, 552, 529, 495, 494, 509, 528, 502, 520, 503, 536, 508, 489, 550, 515, 489, 484, 525, 520, 501, 557, 504, 507, 496, 528, 521, 469] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[539, 516, 527, 547, 541, 475, 552, 488, 496, 539, 530, 468, 479, 537, 505, 471, 570, 523, 488, 531, 509, 517, 512, 520, 513, 537, 544, 504, 539, 499, 542, 521, 521, 543, 480, 492, 497, 509, 502, 517, 540, 491, 507, 502, 515, 528, 494, 490, 525, 526, 504, 481, 532, 487, 482, 530, 500, 488, 514, 530, 490, 513, 531, 484, 512, 527, 505, 531, 495, 518, 502, 487, 511, 524, 506, 522, 503, 551, 539, 489, 509, 500, 542, 495, 516, 540, 496, 538, 512, 525, 494, 504, 492, 524, 492, 480, 471, 546, 486, 517, 537, 477, 496, 526, 505, 475, 512, 528, 528, 491, 498, 524, 505, 517, 524, 495, 508, 517, 522, 501, 509, 507, 513, 513, 532, 522, 493, 536] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[530, 514, 525, 541, 516, 489, 499, 506, 493, 484, 537, 512, 559, 502, 539, 506, 500, 523, 538, 530, 520, 513, 530, 501, 525, 486, 515, 471, 516, 511, 536, 522, 496, 512, 520, 484, 491, 517, 483, 519, 590, 528, 496, 485, 516, 459, 515, 530, 529, 550, 491, 498, 522, 530, 445, 515, 507, 490, 494, 509, 510, 490, 518, 507, 531, 508, 520, 509, 532, 531, 500, 505, 504, 483, 519, 516, 523, 490, 482, 544, 460, 561, 526, 531, 491, 532, 538, 540, 503, 529, 512, 532, 509, 555, 477, 525, 505, 523, 520, 481, 478, 503, 549, 504, 541, 514, 508, 519, 550, 502, 472, 498, 537, 503, 455, 534, 543, 536, 489, 498, 491, 523, 494, 504, 490, 528, 517, 474] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[519, 553, 480, 527, 485, 482, 521, 470, 515, 496, 466, 521, 504, 475, 544, 527, 513, 533, 512, 509, 523, 491, 546, 516, 551, 492, 532, 506, 504, 482, 472, 483, 468, 505, 530, 478, 552, 480, 529, 512, 548, 512, 478, 530, 499, 487, 514, 525, 509, 500, 512, 486, 538, 499, 519, 521, 517, 468, 514, 503, 512, 531, 509, 511, 504, 517, 500, 523, 480, 523, 510, 526, 502, 519, 530, 547, 513, 525, 506, 521, 474, 538, 530, 516, 466, 490, 510, 543, 535, 519, 528, 511, 505, 498, 539, 468, 494, 537, 516, 534, 505, 527, 523, 507, 487, 521, 498, 532, 530, 549, 537, 510, 533, 537, 497, 529, 498, 454, 500, 536, 493, 523, 487, 505, 534, 535, 565, 541] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[518, 508, 468, 503, 523, 498, 546, 518, 498, 510, 540, 532, 494, 532, 493, 539, 500, 485, 478, 509, 525, 501, 527, 492, 530, 505, 526, 536, 496, 502, 505, 492, 486, 519, 494, 499, 535, 505, 500, 516, 495, 509, 505, 488, 544, 503, 562, 508, 516, 519, 538, 504, 552, 505, 483, 540, 509, 553, 524, 509, 505, 564, 533, 506, 505, 529, 515, 491, 508, 488, 516, 519, 491, 540, 483, 507, 493, 505, 539, 502, 510, 509, 516, 516, 487, 480, 511, 497, 494, 502, 529, 569, 528, 486, 502, 498, 545, 533, 513, 499, 491, 506, 521, 554, 539, 477, 501, 549, 494, 487, 535, 497, 498, 496, 541, 496, 487, 527, 499, 550, 492, 530, 502, 487, 533, 513, 541, 481] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[502, 533, 546, 471, 509, 554, 494, 495, 559, 512, 478, 505, 539, 491, 506, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 528, 486, 480, 496, 513, 504, 480, 509, 509, 489, 505, 509, 514, 519, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 495, 531, 532, 502, 491, 490, 535, 513, 486, 514, 479, 508, 506, 546, 551, 16, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 537, 524, 496, 570, 521, 497, 497, 523, 533, 505, 539, 529, 530, 526, 506, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 489, 506, 515, 529, 486, 535, 532, 494, 480, 470, 473, 561, 478, 510, 504, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 516, 499, 516, 523, 476, 521, 532, 528, 509, 569, 514, 534, 501, 504, 509, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 490, 485, 512, 515, 519, 506, 497, 502, 463, 498, 491, 498, 530, 496, 530, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 511, 522, 513, 498, 525, 527, 498, 518, 487, 490, 530, 518, 501, 516, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8065, 8247, 8365, 8090, 8299, 8093, 8187] avg=8192.0000 max=8365 max_rank=3 min=8065 min_rank=1 imbalance_ratio=1.021118 max_load_reduction=57171 max_load_reduction_pct=0.872360 moved_to_echo=32669 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[490, 528, 530, 544, 518, 517, 494, 520, 470, 541, 527, 516, 495, 521, 516, 472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 529, 508, 499, 487, 505, 513, 490, 518, 530, 519, 511, 535, 534, 519, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 515, 522, 514, 496, 539, 537, 517, 475, 511, 505, 526, 537, 500, 523, 494, 20, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 487, 575, 506, 486, 496, 497, 508, 503, 449, 502, 512, 544, 512, 511, 497, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 539, 488, 475, 504, 520, 517, 549, 505, 527, 522, 501, 481, 519, 507, 518, 17, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 503, 488, 520, 499, 528, 535, 541, 529, 509, 517, 540, 506, 495, 491, 539, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 502, 494, 476, 516, 484, 479, 510, 503, 484, 561, 535, 515, 505, 488, 532, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 497, 526, 461, 510, 517, 518, 560, 559, 525, 547, 526, 503, 502, 476, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8202, 8288, 8126, 8156, 8238, 8096, 8231] avg=8192.0000 max=8288 max_rank=2 min=8096 min_rank=6 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32721 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[519, 553, 480, 527, 485, 482, 521, 470, 515, 496, 466, 521, 504, 475, 544, 527, 513, 533, 512, 509, 523, 491, 546, 516, 551, 492, 532, 506, 504, 482, 472, 483, 468, 505, 530, 478, 552, 480, 529, 512, 548, 512, 478, 530, 499, 487, 514, 525, 509, 500, 512, 486, 538, 499, 519, 521, 517, 468, 514, 503, 512, 531, 509, 511, 504, 517, 500, 523, 480, 523, 510, 526, 502, 519, 530, 547, 513, 525, 506, 521, 474, 538, 530, 516, 466, 490, 510, 543, 535, 519, 528, 511, 505, 498, 539, 468, 494, 537, 516, 534, 505, 527, 523, 507, 487, 521, 498, 532, 530, 549, 537, 510, 533, 537, 497, 529, 498, 454, 500, 536, 493, 523, 487, 505, 534, 535, 565, 541] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[524, 493, 506, 489, 525, 529, 474, 513, 507, 532, 544, 511, 530, 493, 531, 513, 497, 517, 535, 522, 568, 482, 526, 551, 515, 512, 539, 501, 476, 525, 530, 530, 534, 503, 559, 505, 518, 501, 517, 538, 456, 513, 553, 526, 523, 529, 522, 522, 483, 522, 529, 503, 521, 508, 529, 504, 490, 489, 495, 537, 519, 481, 505, 456, 516, 494, 522, 517, 498, 519, 532, 500, 496, 521, 519, 512, 505, 530, 510, 515, 495, 515, 519, 500, 503, 455, 510, 487, 500, 538, 521, 489, 533, 483, 508, 500, 482, 479, 566, 476, 502, 508, 552, 529, 495, 494, 509, 528, 502, 520, 503, 536, 508, 489, 550, 515, 489, 484, 525, 520, 501, 557, 504, 507, 496, 528, 521, 469] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[502, 533, 546, 471, 509, 554, 494, 495, 559, 512, 493, 505, 539, 491, 506, 496, 503, 528, 486, 480, 496, 513, 504, 523, 509, 509, 489, 505, 509, 514, 519, 521, 549, 495, 531, 532, 502, 491, 490, 535, 513, 486, 514, 479, 508, 506, 546, 551, 528, 537, 524, 496, 570, 521, 497, 497, 523, 533, 505, 539, 529, 530, 526, 506, 516, 489, 506, 515, 529, 486, 535, 532, 494, 480, 470, 473, 561, 478, 510, 504, 542, 516, 499, 516, 523, 476, 521, 532, 528, 509, 569, 514, 534, 501, 504, 509, 534, 490, 485, 512, 515, 519, 506, 497, 502, 463, 498, 491, 498, 530, 496, 530, 505, 511, 522, 513, 498, 525, 527, 498, 518, 497, 490, 530, 518, 501, 516, 528] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[497, 505, 501, 481, 552, 512, 511, 515, 551, 519, 525, 510, 528, 525, 485, 548, 536, 523, 516, 511, 506, 534, 528, 573, 466, 536, 543, 503, 565, 509, 473, 503, 494, 492, 475, 503, 548, 482, 515, 503, 526, 502, 508, 504, 440, 493, 502, 491, 516, 527, 502, 499, 485, 541, 547, 508, 508, 494, 522, 537, 551, 485, 498, 499, 504, 514, 532, 471, 498, 505, 537, 498, 560, 502, 517, 523, 517, 494, 528, 510, 522, 508, 524, 505, 513, 547, 522, 493, 487, 521, 506, 481, 514, 487, 473, 519, 521, 517, 520, 498, 502, 538, 514, 521, 471, 519, 519, 516, 504, 503, 513, 493, 541, 521, 512, 519, 504, 546, 508, 521, 493, 504, 477, 552, 518, 507, 493, 532] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[490, 528, 530, 544, 518, 517, 494, 520, 470, 541, 551, 516, 495, 521, 516, 472, 486, 529, 508, 499, 487, 505, 513, 543, 518, 530, 519, 511, 535, 534, 519, 519, 550, 515, 522, 514, 496, 539, 537, 517, 475, 511, 505, 526, 537, 500, 523, 494, 534, 487, 575, 506, 486, 496, 497, 508, 503, 449, 502, 512, 544, 512, 511, 497, 461, 539, 488, 475, 504, 520, 517, 549, 505, 527, 522, 501, 481, 519, 507, 518, 487, 503, 488, 520, 499, 528, 535, 541, 529, 509, 517, 540, 506, 495, 491, 539, 479, 502, 494, 476, 516, 484, 479, 510, 503, 484, 561, 535, 515, 505, 488, 532, 503, 497, 526, 461, 510, 517, 518, 560, 559, 549, 547, 526, 503, 502, 476, 501] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[539, 516, 527, 547, 541, 475, 552, 488, 496, 539, 513, 468, 479, 537, 505, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 570, 523, 488, 531, 509, 517, 512, 477, 513, 537, 544, 504, 539, 499, 542, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 543, 480, 492, 497, 509, 502, 517, 540, 491, 507, 502, 515, 528, 494, 490, 16, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 526, 504, 481, 532, 487, 482, 530, 500, 488, 514, 530, 490, 513, 531, 484, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 527, 505, 531, 495, 518, 502, 487, 511, 524, 506, 522, 503, 551, 539, 489, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 500, 542, 495, 516, 540, 496, 538, 512, 525, 494, 504, 492, 524, 492, 480, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 546, 486, 517, 537, 477, 496, 526, 505, 475, 512, 528, 528, 491, 498, 524, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 517, 524, 495, 508, 517, 522, 501, 509, 497, 513, 513, 532, 522, 493, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8326, 8148, 8121, 8235, 8165, 8144, 8204] avg=8192.0000 max=8326 max_rank=1 min=8121 min_rank=3 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32748 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[518, 508, 468, 503, 523, 498, 546, 518, 498, 510, 523, 532, 494, 532, 493, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 485, 478, 509, 525, 501, 527, 451, 530, 505, 526, 536, 496, 502, 505, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 519, 494, 499, 535, 505, 500, 516, 495, 509, 505, 488, 544, 503, 562, 508, 15, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 519, 538, 504, 552, 505, 483, 540, 509, 553, 524, 509, 505, 564, 533, 506, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 529, 515, 491, 508, 488, 516, 519, 491, 540, 483, 507, 493, 505, 539, 502, 13, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 509, 516, 516, 487, 480, 511, 497, 494, 502, 529, 569, 528, 486, 502, 498, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 533, 513, 499, 491, 506, 521, 554, 539, 477, 501, 549, 494, 487, 535, 497, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 496, 541, 496, 487, 527, 499, 550, 492, 519, 502, 487, 533, 513, 541, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8068, 8187, 8364, 8145, 8140, 8267, 8162] avg=8192.0000 max=8364 max_rank=3 min=8068 min_rank=1 imbalance_ratio=1.020996 max_load_reduction=57172 max_load_reduction_pct=0.872375 moved_to_echo=32714 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[530, 514, 525, 541, 516, 489, 499, 506, 493, 484, 520, 512, 559, 502, 539, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 523, 538, 530, 520, 513, 530, 460, 525, 486, 515, 471, 516, 511, 536, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 512, 520, 484, 491, 517, 483, 519, 590, 528, 496, 485, 516, 459, 515, 530, 15, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 550, 491, 498, 522, 530, 445, 515, 507, 490, 494, 509, 510, 490, 518, 507, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 508, 520, 509, 532, 531, 500, 505, 504, 483, 519, 516, 523, 490, 482, 544, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 460, 561, 526, 531, 491, 532, 538, 540, 503, 529, 512, 532, 509, 555, 477, 525, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 523, 520, 481, 478, 503, 549, 504, 541, 514, 508, 519, 550, 502, 472, 498, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 503, 455, 534, 543, 536, 489, 498, 491, 513, 494, 504, 490, 528, 517, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8196, 8160, 8109, 8210, 8327, 8193, 8106] avg=8192.0000 max=8327 max_rank=5 min=8106 min_rank=7 imbalance_ratio=1.016479 max_load_reduction=57209 max_load_reduction_pct=0.872940 moved_to_echo=32836 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[519, 553, 480, 527, 485, 482, 521, 470, 515, 496, 452, 521, 504, 475, 544, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 533, 512, 509, 523, 491, 546, 473, 551, 492, 532, 506, 504, 482, 472, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 505, 530, 478, 552, 480, 529, 512, 548, 512, 478, 530, 499, 487, 514, 525, 16, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 500, 512, 486, 538, 499, 519, 521, 517, 468, 514, 503, 512, 531, 509, 511, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 517, 500, 523, 480, 523, 510, 526, 502, 519, 530, 547, 513, 525, 506, 521, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 538, 530, 516, 466, 490, 510, 543, 535, 519, 528, 511, 505, 498, 539, 468, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 537, 516, 534, 505, 527, 523, 507, 487, 521, 498, 532, 530, 549, 537, 510, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 537, 497, 529, 498, 454, 500, 536, 493, 513, 487, 505, 534, 535, 565, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8071, 8122, 8166, 8153, 8257, 8176, 8334, 8257] avg=8192.0000 max=8334 max_rank=6 min=8071 min_rank=0 imbalance_ratio=1.017334 max_load_reduction=57202 max_load_reduction_pct=0.872833 moved_to_echo=33024 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[497, 505, 501, 481, 552, 512, 511, 515, 551, 519, 508, 510, 528, 525, 485, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 523, 516, 511, 506, 534, 528, 525, 466, 536, 543, 503, 565, 509, 473, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 492, 475, 503, 548, 482, 515, 503, 526, 502, 508, 504, 440, 493, 502, 491, 18, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 527, 502, 499, 485, 541, 547, 508, 508, 494, 522, 537, 551, 485, 498, 499, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 514, 532, 471, 498, 505, 537, 498, 560, 502, 517, 523, 517, 494, 528, 510, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 508, 524, 505, 513, 547, 522, 493, 487, 521, 506, 481, 514, 487, 473, 519, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 517, 520, 498, 502, 538, 514, 521, 471, 519, 519, 516, 504, 503, 513, 493, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 521, 512, 519, 504, 546, 508, 521, 493, 494, 477, 552, 518, 507, 493, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8248, 8277, 8000, 8223, 8223, 8128, 8199, 8238] avg=8192.0000 max=8277 max_rank=1 min=8000 min_rank=2 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32788 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[524, 493, 506, 489, 525, 529, 474, 513, 507, 532, 527, 511, 530, 493, 531, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 517, 535, 522, 568, 482, 526, 505, 515, 512, 539, 501, 476, 525, 530, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 503, 559, 505, 518, 501, 517, 538, 456, 513, 553, 526, 523, 529, 522, 522, 17, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 522, 529, 503, 521, 508, 529, 504, 490, 489, 495, 537, 519, 481, 505, 456, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 494, 522, 517, 498, 519, 532, 500, 496, 521, 519, 512, 505, 530, 510, 515, 13, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 515, 519, 500, 503, 455, 510, 487, 500, 538, 521, 489, 533, 483, 508, 500, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 479, 566, 476, 502, 508, 552, 529, 495, 494, 509, 528, 502, 520, 503, 536, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 489, 550, 515, 489, 484, 525, 520, 501, 545, 504, 507, 496, 528, 521, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8280, 8340, 8075, 8220, 8063, 8210, 8151] avg=8192.0000 max=8340 max_rank=2 min=8063 min_rank=5 imbalance_ratio=1.018066 max_load_reduction=57196 max_load_reduction_pct=0.872742 moved_to_echo=32644 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[539, 516, 527, 547, 541, 475, 552, 488, 496, 539, 513, 468, 479, 537, 505, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 570, 523, 488, 531, 509, 517, 512, 477, 513, 537, 544, 504, 539, 499, 542, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 543, 480, 492, 497, 509, 502, 517, 540, 491, 507, 502, 515, 528, 494, 490, 16, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 526, 504, 481, 532, 487, 482, 530, 500, 488, 514, 530, 490, 513, 531, 484, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 527, 505, 531, 495, 518, 502, 487, 511, 524, 506, 522, 503, 551, 539, 489, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 500, 542, 495, 516, 540, 496, 538, 512, 525, 494, 504, 492, 524, 492, 480, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 546, 486, 517, 537, 477, 496, 526, 505, 475, 512, 528, 528, 491, 498, 524, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 517, 524, 495, 508, 517, 522, 501, 509, 497, 513, 513, 532, 522, 493, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8326, 8148, 8121, 8235, 8165, 8144, 8204] avg=8192.0000 max=8326 max_rank=1 min=8121 min_rank=3 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32748 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[519, 553, 480, 527, 485, 482, 521, 470, 515, 496, 452, 521, 504, 475, 544, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 533, 512, 509, 523, 491, 546, 473, 551, 492, 532, 506, 504, 482, 472, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 505, 530, 478, 552, 480, 529, 512, 548, 512, 478, 530, 499, 487, 514, 525, 16, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 500, 512, 486, 538, 499, 519, 521, 517, 468, 514, 503, 512, 531, 509, 511, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 517, 500, 523, 480, 523, 510, 526, 502, 519, 530, 547, 513, 525, 506, 521, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 538, 530, 516, 466, 490, 510, 543, 535, 519, 528, 511, 505, 498, 539, 468, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 537, 516, 534, 505, 527, 523, 507, 487, 521, 498, 532, 530, 549, 537, 510, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 537, 497, 529, 498, 454, 500, 536, 493, 513, 487, 505, 534, 535, 565, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8071, 8122, 8166, 8153, 8257, 8176, 8334, 8257] avg=8192.0000 max=8334 max_rank=6 min=8071 min_rank=0 imbalance_ratio=1.017334 max_load_reduction=57202 max_load_reduction_pct=0.872833 moved_to_echo=33024 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[518, 508, 468, 503, 523, 498, 546, 518, 498, 510, 523, 532, 494, 532, 493, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 485, 478, 509, 525, 501, 527, 451, 530, 505, 526, 536, 496, 502, 505, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 519, 494, 499, 535, 505, 500, 516, 495, 509, 505, 488, 544, 503, 562, 508, 15, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 519, 538, 504, 552, 505, 483, 540, 509, 553, 524, 509, 505, 564, 533, 506, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 529, 515, 491, 508, 488, 516, 519, 491, 540, 483, 507, 493, 505, 539, 502, 13, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 509, 516, 516, 487, 480, 511, 497, 494, 502, 529, 569, 528, 486, 502, 498, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 533, 513, 499, 491, 506, 521, 554, 539, 477, 501, 549, 494, 487, 535, 497, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 496, 541, 496, 487, 527, 499, 550, 492, 519, 502, 487, 533, 513, 541, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8068, 8187, 8364, 8145, 8140, 8267, 8162] avg=8192.0000 max=8364 max_rank=3 min=8068 min_rank=1 imbalance_ratio=1.020996 max_load_reduction=57172 max_load_reduction_pct=0.872375 moved_to_echo=32714 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[502, 533, 546, 471, 509, 554, 494, 495, 559, 512, 478, 505, 539, 491, 506, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 528, 486, 480, 496, 513, 504, 480, 509, 509, 489, 505, 509, 514, 519, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 495, 531, 532, 502, 491, 490, 535, 513, 486, 514, 479, 508, 506, 546, 551, 16, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 537, 524, 496, 570, 521, 497, 497, 523, 533, 505, 539, 529, 530, 526, 506, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 489, 506, 515, 529, 486, 535, 532, 494, 480, 470, 473, 561, 478, 510, 504, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 516, 499, 516, 523, 476, 521, 532, 528, 509, 569, 514, 534, 501, 504, 509, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 490, 485, 512, 515, 519, 506, 497, 502, 463, 498, 491, 498, 530, 496, 530, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 511, 522, 513, 498, 525, 527, 498, 518, 487, 490, 530, 518, 501, 516, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8065, 8247, 8365, 8090, 8299, 8093, 8187] avg=8192.0000 max=8365 max_rank=3 min=8065 min_rank=1 imbalance_ratio=1.021118 max_load_reduction=57171 max_load_reduction_pct=0.872360 moved_to_echo=32669 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[490, 528, 530, 544, 518, 517, 494, 520, 470, 541, 527, 516, 495, 521, 516, 472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 529, 508, 499, 487, 505, 513, 490, 518, 530, 519, 511, 535, 534, 519, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 515, 522, 514, 496, 539, 537, 517, 475, 511, 505, 526, 537, 500, 523, 494, 20, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 487, 575, 506, 486, 496, 497, 508, 503, 449, 502, 512, 544, 512, 511, 497, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 539, 488, 475, 504, 520, 517, 549, 505, 527, 522, 501, 481, 519, 507, 518, 17, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 503, 488, 520, 499, 528, 535, 541, 529, 509, 517, 540, 506, 495, 491, 539, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 502, 494, 476, 516, 484, 479, 510, 503, 484, 561, 535, 515, 505, 488, 532, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 497, 526, 461, 510, 517, 518, 560, 559, 525, 547, 526, 503, 502, 476, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8202, 8288, 8126, 8156, 8238, 8096, 8231] avg=8192.0000 max=8288 max_rank=2 min=8096 min_rank=6 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32721 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4224, 3968, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 3968, 4096, 4224, 4224, 4224, 4224, 3968, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 3968, 4224, 4224, 4224, 4096, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4224, 3968, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 3968, 4224, 4224, 4224, 4096, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 3968, 4096, 4224, 4224, 4224, 4224, 3968, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[516, 523, 530, 480, 495, 523, 536, 506, 508, 521, 530, 491, 569, 525, 497, 503, 527, 482, 486, 479, 527, 514, 534, 504, 521, 497, 502, 516, 520, 504, 541, 519, 497, 500, 525, 503, 491, 476, 505, 543, 505, 466, 550, 519, 503, 511, 528, 522, 487, 527, 513, 477, 497, 514, 544, 525, 473, 499, 516, 517, 529, 519, 483, 476, 509, 507, 528, 487, 508, 519, 512, 528, 537, 528, 478, 502, 487, 535, 509, 526, 489, 528, 491, 520, 518, 514, 528, 535, 489, 511, 528, 495, 550, 551, 482, 533, 516, 488, 501, 550, 508, 481, 533, 484, 526, 495, 493, 536, 495, 491, 506, 528, 514, 501, 500, 518, 501, 518, 536, 546, 491, 537, 525, 527, 526, 516, 515, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[502, 511, 511, 518, 476, 513, 497, 520, 537, 540, 519, 526, 534, 498, 512, 518, 525, 468, 521, 519, 509, 508, 489, 493, 562, 504, 520, 462, 517, 520, 561, 508, 491, 558, 505, 477, 529, 524, 534, 497, 491, 527, 551, 516, 486, 503, 530, 526, 525, 502, 494, 554, 472, 542, 557, 511, 544, 534, 453, 487, 457, 482, 536, 481, 489, 522, 503, 509, 544, 480, 533, 532, 536, 465, 548, 507, 574, 482, 517, 508, 551, 510, 515, 498, 510, 521, 560, 514, 494, 488, 503, 544, 483, 502, 501, 506, 477, 521, 531, 509, 516, 499, 487, 488, 521, 492, 525, 478, 526, 524, 521, 462, 508, 526, 510, 524, 518, 504, 509, 562, 519, 519, 537, 457, 489, 491, 522, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[506, 478, 497, 517, 530, 501, 511, 452, 478, 502, 496, 514, 515, 493, 498, 526, 514, 499, 517, 535, 524, 559, 555, 477, 494, 512, 514, 511, 567, 482, 497, 513, 520, 456, 511, 488, 533, 513, 519, 547, 547, 521, 484, 525, 501, 537, 537, 502, 509, 534, 493, 496, 509, 557, 529, 492, 515, 510, 516, 479, 549, 500, 493, 514, 507, 477, 507, 501, 534, 519, 516, 483, 520, 525, 533, 549, 492, 509, 485, 480, 476, 532, 526, 515, 511, 498, 472, 551, 492, 516, 479, 528, 506, 486, 542, 492, 522, 510, 467, 516, 554, 551, 495, 557, 525, 500, 499, 489, 491, 499, 497, 501, 532, 511, 507, 529, 548, 526, 542, 564, 533, 508, 493, 511, 512, 498, 518, 552] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[532, 492, 500, 516, 536, 463, 548, 531, 500, 510, 465, 508, 551, 523, 523, 490, 514, 506, 494, 544, 516, 478, 477, 487, 508, 512, 517, 517, 491, 507, 515, 491, 484, 499, 501, 503, 492, 506, 537, 503, 506, 519, 490, 530, 503, 528, 497, 508, 553, 468, 519, 513, 481, 504, 508, 511, 497, 503, 559, 515, 518, 503, 512, 553, 537, 473, 528, 506, 515, 499, 543, 521, 537, 551, 531, 525, 527, 515, 496, 513, 511, 518, 539, 501, 513, 523, 507, 507, 475, 521, 523, 492, 507, 491, 546, 517, 504, 518, 498, 535, 520, 484, 527, 490, 526, 528, 511, 518, 503, 508, 529, 535, 477, 519, 543, 549, 516, 484, 511, 512, 498, 527, 528, 535, 479, 502, 530, 499] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[526, 490, 512, 564, 545, 476, 506, 472, 542, 490, 492, 487, 504, 491, 504, 546, 530, 491, 494, 465, 498, 487, 519, 517, 508, 504, 511, 524, 506, 497, 527, 531, 491, 557, 515, 534, 498, 516, 523, 472, 493, 535, 512, 499, 531, 521, 534, 493, 546, 505, 511, 541, 516, 578, 546, 528, 532, 515, 530, 513, 500, 523, 537, 485, 526, 525, 509, 524, 527, 504, 483, 524, 523, 521, 512, 524, 513, 493, 502, 520, 479, 515, 475, 535, 494, 505, 491, 505, 521, 529, 518, 490, 492, 504, 534, 529, 497, 509, 497, 510, 532, 486, 555, 511, 515, 533, 488, 520, 497, 473, 508, 526, 518, 525, 507, 544, 488, 510, 482, 484, 533, 519, 502, 468, 481, 514, 547, 525] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[510, 534, 464, 518, 541, 484, 551, 516, 517, 501, 516, 521, 470, 536, 530, 479, 516, 504, 524, 491, 494, 543, 485, 489, 512, 528, 510, 485, 526, 526, 509, 491, 514, 494, 491, 514, 516, 499, 515, 531, 505, 548, 500, 476, 562, 520, 501, 505, 522, 543, 512, 476, 507, 504, 538, 495, 480, 505, 518, 552, 530, 541, 486, 511, 496, 479, 496, 537, 534, 526, 511, 494, 506, 521, 521, 507, 560, 480, 515, 460, 505, 509, 475, 501, 495, 541, 508, 510, 492, 498, 502, 503, 486, 545, 514, 466, 516, 518, 517, 471, 551, 504, 547, 538, 511, 507, 524, 559, 545, 493, 544, 489, 501, 520, 504, 499, 522, 528, 524, 545, 513, 530, 485, 549, 497, 528, 508, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[502, 511, 511, 518, 476, 513, 497, 520, 537, 540, 519, 526, 534, 498, 512, 518, 525, 468, 521, 519, 509, 508, 489, 493, 562, 504, 520, 462, 517, 520, 561, 508, 491, 558, 505, 477, 529, 524, 534, 497, 491, 527, 551, 516, 486, 503, 530, 526, 525, 502, 494, 554, 472, 542, 557, 511, 544, 534, 453, 487, 457, 482, 536, 481, 489, 522, 503, 509, 544, 480, 533, 532, 536, 465, 548, 507, 574, 482, 517, 508, 551, 510, 515, 498, 510, 521, 560, 514, 494, 488, 503, 544, 483, 502, 501, 506, 477, 521, 531, 509, 516, 499, 487, 488, 521, 492, 525, 478, 526, 524, 521, 462, 508, 526, 510, 524, 518, 504, 509, 562, 519, 519, 537, 457, 489, 491, 522, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[478, 510, 485, 553, 561, 492, 515, 550, 544, 487, 507, 503, 489, 490, 494, 508, 561, 526, 496, 492, 543, 523, 530, 542, 526, 477, 515, 496, 495, 478, 466, 556, 544, 516, 539, 500, 516, 485, 525, 519, 510, 507, 517, 514, 493, 558, 512, 557, 498, 528, 461, 535, 499, 487, 500, 505, 532, 496, 517, 516, 501, 540, 500, 502, 495, 515, 509, 518, 537, 501, 496, 566, 451, 546, 528, 500, 467, 490, 518, 469, 504, 493, 540, 521, 515, 500, 540, 491, 522, 496, 513, 568, 539, 542, 544, 512, 487, 520, 490, 559, 475, 505, 516, 510, 482, 511, 507, 511, 500, 523, 519, 464, 557, 512, 527, 484, 540, 534, 517, 488, 483, 483, 497, 543, 539, 483, 536, 471] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[505, 541, 504, 525, 534, 537, 548, 532, 560, 520, 524, 501, 537, 525, 474, 502, 526, 477, 487, 489, 526, 512, 518, 521, 466, 517, 516, 537, 513, 529, 516, 478, 519, 498, 486, 523, 559, 514, 492, 490, 502, 531, 498, 534, 506, 498, 529, 526, 493, 536, 504, 519, 497, 495, 492, 526, 479, 495, 496, 549, 490, 528, 503, 518, 504, 507, 507, 502, 534, 543, 493, 511, 520, 496, 504, 534, 462, 483, 494, 492, 487, 505, 491, 472, 540, 510, 536, 506, 507, 493, 524, 507, 520, 512, 531, 512, 539, 540, 528, 495, 498, 477, 555, 498, 471, 530, 510, 518, 515, 547, 504, 552, 535, 509, 519, 516, 480, 464, 507, 518, 531, 514, 527, 520, 537, 514, 517, 490] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[502, 511, 511, 518, 474, 513, 497, 520, 537, 540, 519, 526, 534, 498, 512, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 468, 521, 519, 509, 508, 489, 493, 562, 504, 520, 462, 517, 520, 561, 508, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 558, 505, 477, 529, 524, 534, 497, 491, 527, 551, 516, 486, 488, 530, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 502, 494, 554, 472, 542, 557, 511, 544, 534, 453, 487, 457, 482, 536, 481, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 522, 503, 509, 544, 480, 533, 532, 536, 465, 548, 507, 574, 482, 517, 508, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 510, 515, 498, 510, 521, 560, 514, 494, 488, 503, 544, 483, 502, 501, 506, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 521, 531, 509, 516, 499, 487, 488, 521, 492, 525, 478, 526, 524, 521, 462, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 526, 510, 524, 518, 504, 509, 514, 519, 519, 537, 457, 489, 491, 522, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 8218, 8230, 8134, 8257, 8213, 8086, 8168] avg=8192.0000 max=8257 max_rank=4 min=8086 min_rank=6 imbalance_ratio=1.007935 max_load_reduction=57279 max_load_reduction_pct=0.874008 moved_to_echo=32724 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[506, 478, 497, 517, 528, 501, 511, 452, 478, 502, 496, 514, 515, 493, 498, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 499, 517, 535, 524, 559, 555, 477, 494, 512, 514, 511, 567, 482, 497, 513, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 456, 511, 488, 533, 513, 519, 547, 547, 521, 484, 525, 501, 522, 537, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 534, 493, 496, 509, 557, 529, 492, 515, 510, 516, 479, 549, 500, 493, 514, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 477, 507, 501, 534, 519, 516, 483, 520, 525, 533, 549, 492, 509, 485, 480, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 532, 526, 515, 511, 498, 472, 551, 492, 516, 479, 528, 506, 486, 542, 492, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 510, 467, 516, 554, 551, 495, 557, 525, 500, 499, 489, 491, 499, 497, 501, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 511, 507, 529, 548, 526, 542, 516, 533, 508, 493, 511, 512, 498, 518, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8012, 8302, 8226, 8198, 8145, 8135, 8182, 8336] avg=8192.0000 max=8336 max_rank=7 min=8012 min_rank=0 imbalance_ratio=1.017578 max_load_reduction=57200 max_load_reduction_pct=0.872803 moved_to_echo=32798 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[516, 523, 530, 480, 493, 523, 536, 506, 508, 521, 530, 491, 569, 525, 497, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 482, 486, 479, 527, 514, 534, 504, 521, 497, 502, 516, 520, 504, 541, 519, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 500, 525, 503, 491, 476, 505, 543, 505, 466, 550, 519, 503, 496, 528, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 527, 513, 477, 497, 514, 544, 525, 473, 499, 516, 517, 529, 519, 483, 476, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 507, 528, 487, 508, 519, 512, 528, 537, 528, 478, 502, 487, 535, 509, 526, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 528, 491, 520, 518, 514, 528, 535, 489, 511, 528, 495, 550, 551, 482, 533, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 488, 501, 550, 508, 481, 533, 484, 526, 495, 493, 536, 495, 491, 506, 528, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 501, 500, 518, 501, 518, 536, 500, 491, 537, 525, 527, 526, 516, 515, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8251, 8204, 8129, 8099, 8208, 8275, 8139, 8231] avg=8192.0000 max=8275 max_rank=5 min=8099 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32853 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[526, 490, 512, 564, 538, 476, 506, 472, 542, 490, 492, 487, 504, 491, 504, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 491, 494, 465, 498, 487, 519, 517, 508, 504, 511, 524, 506, 497, 527, 531, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 557, 515, 534, 498, 516, 523, 472, 493, 535, 512, 499, 531, 495, 534, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 505, 511, 541, 516, 578, 546, 528, 532, 515, 530, 513, 500, 523, 537, 485, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 525, 509, 524, 527, 504, 483, 524, 523, 521, 512, 524, 513, 493, 502, 520, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 515, 475, 535, 494, 505, 491, 505, 521, 529, 518, 490, 492, 504, 534, 529, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 509, 497, 510, 532, 486, 555, 511, 515, 533, 488, 520, 497, 473, 508, 526, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 525, 507, 544, 488, 510, 482, 432, 533, 519, 502, 468, 481, 514, 547, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8140, 8139, 8198, 8419, 8241, 8131, 8173, 8095] avg=8192.0000 max=8419 max_rank=3 min=8095 min_rank=7 imbalance_ratio=1.027710 max_load_reduction=57117 max_load_reduction_pct=0.871536 moved_to_echo=32640 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[478, 510, 485, 553, 559, 492, 515, 550, 544, 487, 507, 503, 489, 490, 494, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 526, 496, 492, 543, 523, 530, 542, 526, 477, 515, 496, 495, 478, 466, 556, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 516, 539, 500, 516, 485, 525, 519, 510, 507, 517, 514, 493, 542, 512, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 528, 461, 535, 499, 487, 500, 505, 532, 496, 517, 516, 501, 540, 500, 502, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 515, 509, 518, 537, 501, 496, 566, 451, 546, 528, 500, 467, 490, 518, 469, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 493, 540, 521, 515, 500, 540, 491, 522, 496, 513, 568, 539, 542, 544, 512, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 520, 490, 559, 475, 505, 516, 510, 482, 511, 507, 511, 500, 523, 519, 464, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 557, 512, 527, 484, 540, 534, 517, 448, 483, 483, 497, 543, 539, 483, 536, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8249, 8296, 8120, 8115, 8351, 8087, 8154] avg=8192.0000 max=8351 max_rank=5 min=8087 min_rank=6 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32707 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[532, 492, 500, 516, 534, 463, 548, 531, 500, 510, 465, 508, 551, 523, 523, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 506, 494, 544, 516, 478, 477, 487, 508, 512, 517, 517, 491, 507, 515, 491, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 499, 501, 503, 492, 506, 537, 503, 506, 519, 490, 530, 503, 513, 497, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 468, 519, 513, 481, 504, 508, 511, 497, 503, 559, 515, 518, 503, 512, 553, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 473, 528, 506, 515, 499, 543, 521, 537, 551, 531, 525, 527, 515, 496, 513, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 518, 539, 501, 513, 523, 507, 507, 475, 521, 523, 492, 507, 491, 546, 517, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 518, 498, 535, 520, 484, 527, 490, 526, 528, 511, 518, 503, 508, 529, 535, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 519, 543, 549, 516, 484, 511, 469, 498, 527, 528, 535, 479, 502, 530, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8103, 8091, 8220, 8325, 8203, 8242, 8166] avg=8192.0000 max=8325 max_rank=4 min=8091 min_rank=2 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32936 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[505, 541, 504, 525, 532, 537, 548, 532, 560, 520, 524, 501, 537, 525, 474, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 477, 487, 489, 526, 512, 518, 521, 466, 517, 516, 537, 513, 529, 516, 478, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 498, 486, 523, 559, 514, 492, 490, 502, 531, 498, 534, 506, 483, 529, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 536, 504, 519, 497, 495, 492, 526, 479, 495, 496, 549, 490, 528, 503, 518, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 507, 507, 502, 534, 543, 493, 511, 520, 496, 504, 534, 462, 483, 494, 492, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 505, 491, 472, 540, 510, 536, 506, 507, 493, 524, 507, 520, 512, 531, 512, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 540, 528, 495, 498, 477, 555, 498, 471, 530, 510, 518, 515, 547, 504, 552, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 509, 519, 516, 480, 464, 507, 475, 531, 514, 527, 520, 537, 514, 517, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8367, 8157, 8190, 8123, 8094, 8165, 8285, 8155] avg=8192.0000 max=8367 max_rank=0 min=8094 min_rank=4 imbalance_ratio=1.021362 max_load_reduction=57169 max_load_reduction_pct=0.872330 moved_to_echo=32699 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[510, 534, 464, 518, 539, 484, 551, 516, 517, 501, 516, 521, 470, 536, 530, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 504, 524, 491, 494, 543, 485, 489, 512, 528, 510, 485, 526, 526, 509, 491, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 494, 491, 514, 516, 499, 515, 531, 505, 548, 500, 476, 562, 505, 501, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 543, 512, 476, 507, 504, 538, 495, 480, 505, 518, 552, 530, 541, 486, 511, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 479, 496, 537, 534, 526, 511, 494, 506, 521, 521, 507, 560, 480, 515, 460, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 509, 475, 501, 495, 541, 508, 510, 492, 498, 502, 503, 486, 545, 514, 466, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 518, 517, 471, 551, 504, 547, 538, 511, 507, 524, 559, 545, 493, 544, 489, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 520, 504, 499, 522, 528, 524, 499, 513, 530, 485, 549, 497, 528, 508, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8164, 8176, 8223, 8151, 8063, 8342, 8231] avg=8192.0000 max=8342 max_rank=6 min=8063 min_rank=5 imbalance_ratio=1.018311 max_load_reduction=57194 max_load_reduction_pct=0.872711 moved_to_echo=32787 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[505, 541, 504, 525, 532, 537, 548, 532, 560, 520, 524, 501, 537, 525, 474, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 477, 487, 489, 526, 512, 518, 521, 466, 517, 516, 537, 513, 529, 516, 478, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 498, 486, 523, 559, 514, 492, 490, 502, 531, 498, 534, 506, 483, 529, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 536, 504, 519, 497, 495, 492, 526, 479, 495, 496, 549, 490, 528, 503, 518, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 507, 507, 502, 534, 543, 493, 511, 520, 496, 504, 534, 462, 483, 494, 492, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 505, 491, 472, 540, 510, 536, 506, 507, 493, 524, 507, 520, 512, 531, 512, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 540, 528, 495, 498, 477, 555, 498, 471, 530, 510, 518, 515, 547, 504, 552, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 509, 519, 516, 480, 464, 507, 475, 531, 514, 527, 520, 537, 514, 517, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8367, 8157, 8190, 8123, 8094, 8165, 8285, 8155] avg=8192.0000 max=8367 max_rank=0 min=8094 min_rank=4 imbalance_ratio=1.021362 max_load_reduction=57169 max_load_reduction_pct=0.872330 moved_to_echo=32699 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[526, 490, 512, 564, 538, 476, 506, 472, 542, 490, 492, 487, 504, 491, 504, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 491, 494, 465, 498, 487, 519, 517, 508, 504, 511, 524, 506, 497, 527, 531, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 557, 515, 534, 498, 516, 523, 472, 493, 535, 512, 499, 531, 495, 534, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 505, 511, 541, 516, 578, 546, 528, 532, 515, 530, 513, 500, 523, 537, 485, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 525, 509, 524, 527, 504, 483, 524, 523, 521, 512, 524, 513, 493, 502, 520, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 515, 475, 535, 494, 505, 491, 505, 521, 529, 518, 490, 492, 504, 534, 529, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 509, 497, 510, 532, 486, 555, 511, 515, 533, 488, 520, 497, 473, 508, 526, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 525, 507, 544, 488, 510, 482, 432, 533, 519, 502, 468, 481, 514, 547, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8140, 8139, 8198, 8419, 8241, 8131, 8173, 8095] avg=8192.0000 max=8419 max_rank=3 min=8095 min_rank=7 imbalance_ratio=1.027710 max_load_reduction=57117 max_load_reduction_pct=0.871536 moved_to_echo=32640 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[502, 511, 511, 518, 474, 513, 497, 520, 537, 540, 519, 526, 534, 498, 512, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 468, 521, 519, 509, 508, 489, 493, 562, 504, 520, 462, 517, 520, 561, 508, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 558, 505, 477, 529, 524, 534, 497, 491, 527, 551, 516, 486, 488, 530, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 502, 494, 554, 472, 542, 557, 511, 544, 534, 453, 487, 457, 482, 536, 481, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 522, 503, 509, 544, 480, 533, 532, 536, 465, 548, 507, 574, 482, 517, 508, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 510, 515, 498, 510, 521, 560, 514, 494, 488, 503, 544, 483, 502, 501, 506, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 521, 531, 509, 516, 499, 487, 488, 521, 492, 525, 478, 526, 524, 521, 462, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 526, 510, 524, 518, 504, 509, 514, 519, 519, 537, 457, 489, 491, 522, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 8218, 8230, 8134, 8257, 8213, 8086, 8168] avg=8192.0000 max=8257 max_rank=4 min=8086 min_rank=6 imbalance_ratio=1.007935 max_load_reduction=57279 max_load_reduction_pct=0.874008 moved_to_echo=32724 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[478, 510, 485, 553, 559, 492, 515, 550, 544, 487, 507, 503, 489, 490, 494, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 526, 496, 492, 543, 523, 530, 542, 526, 477, 515, 496, 495, 478, 466, 556, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 516, 539, 500, 516, 485, 525, 519, 510, 507, 517, 514, 493, 542, 512, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 528, 461, 535, 499, 487, 500, 505, 532, 496, 517, 516, 501, 540, 500, 502, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 515, 509, 518, 537, 501, 496, 566, 451, 546, 528, 500, 467, 490, 518, 469, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 493, 540, 521, 515, 500, 540, 491, 522, 496, 513, 568, 539, 542, 544, 512, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 520, 490, 559, 475, 505, 516, 510, 482, 511, 507, 511, 500, 523, 519, 464, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 557, 512, 527, 484, 540, 534, 517, 448, 483, 483, 497, 543, 539, 483, 536, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8249, 8296, 8120, 8115, 8351, 8087, 8154] avg=8192.0000 max=8351 max_rank=5 min=8087 min_rank=6 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32707 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4096, 4352, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 3968, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4224, 3968, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 3968, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 3968, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4096, 4352, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 3968, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[528, 487, 521, 550, 508, 516, 499, 557, 527, 552, 537, 520, 496, 528, 476, 503, 497, 498, 506, 486, 539, 520, 529, 516, 492, 484, 505, 547, 548, 507, 519, 516, 509, 493, 486, 517, 504, 515, 518, 513, 520, 535, 527, 534, 549, 519, 503, 517, 532, 525, 514, 502, 536, 487, 480, 550, 518, 498, 480, 498, 520, 517, 488, 519, 531, 508, 523, 483, 516, 496, 549, 492, 537, 506, 487, 495, 525, 502, 530, 483, 501, 509, 522, 470, 521, 520, 482, 508, 527, 534, 516, 506, 531, 515, 516, 522, 521, 484, 515, 487, 499, 505, 479, 524, 527, 497, 502, 499, 516, 508, 486, 535, 495, 515, 538, 554, 468, 494, 469, 465, 538, 502, 537, 505, 525, 508, 510, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[513, 506, 531, 521, 500, 540, 509, 487, 499, 526, 540, 493, 516, 515, 505, 531, 514, 484, 474, 502, 521, 505, 497, 512, 523, 555, 528, 472, 488, 509, 525, 505, 538, 539, 502, 495, 509, 476, 520, 500, 507, 529, 484, 526, 523, 477, 463, 540, 500, 490, 515, 514, 527, 500, 501, 520, 552, 506, 526, 507, 506, 531, 494, 533, 505, 534, 533, 521, 540, 505, 539, 534, 498, 513, 533, 486, 527, 524, 484, 478, 507, 518, 510, 527, 520, 494, 501, 489, 511, 500, 503, 537, 536, 529, 460, 494, 470, 525, 525, 506, 539, 520, 516, 489, 545, 510, 508, 518, 506, 579, 549, 509, 505, 500, 508, 530, 518, 495, 513, 551, 509, 503, 517, 452, 472, 509, 521, 533] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[527, 508, 563, 496, 464, 482, 528, 523, 538, 530, 516, 529, 489, 480, 480, 555, 497, 493, 506, 541, 501, 503, 484, 525, 560, 504, 548, 507, 528, 488, 503, 477, 530, 517, 510, 509, 505, 519, 523, 523, 492, 511, 490, 534, 519, 480, 511, 487, 517, 504, 478, 489, 538, 514, 537, 543, 509, 511, 486, 526, 500, 486, 516, 507, 494, 497, 533, 562, 495, 507, 522, 524, 524, 516, 478, 510, 498, 489, 536, 513, 497, 479, 577, 542, 456, 518, 449, 522, 519, 538, 485, 533, 552, 522, 549, 512, 492, 533, 565, 507, 502, 484, 533, 526, 491, 518, 555, 532, 509, 493, 530, 519, 513, 502, 529, 480, 514, 504, 508, 457, 511, 508, 489, 520, 475, 534, 540, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[511, 532, 500, 491, 545, 546, 529, 505, 509, 496, 536, 487, 538, 517, 514, 500, 538, 473, 540, 510, 521, 501, 491, 540, 521, 511, 509, 513, 537, 496, 542, 520, 487, 538, 508, 545, 523, 499, 493, 532, 459, 514, 537, 533, 511, 554, 479, 523, 560, 472, 529, 538, 544, 468, 509, 478, 519, 529, 506, 537, 495, 500, 486, 520, 464, 522, 503, 528, 527, 516, 528, 519, 483, 495, 512, 484, 539, 493, 507, 527, 478, 509, 510, 481, 496, 525, 529, 508, 479, 543, 502, 526, 496, 513, 529, 500, 496, 534, 522, 489, 541, 518, 485, 537, 538, 505, 481, 486, 496, 524, 508, 498, 489, 487, 513, 514, 518, 498, 521, 511, 513, 522, 505, 500, 502, 494, 559, 517] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[522, 504, 539, 540, 539, 489, 474, 501, 519, 523, 520, 503, 545, 502, 518, 514, 539, 478, 542, 515, 532, 497, 521, 495, 514, 512, 523, 558, 503, 550, 495, 507, 527, 481, 501, 532, 514, 514, 500, 501, 495, 502, 524, 477, 511, 520, 508, 534, 520, 494, 518, 501, 531, 520, 514, 482, 515, 500, 528, 497, 457, 530, 486, 527, 492, 480, 516, 484, 490, 519, 555, 508, 515, 523, 546, 538, 515, 509, 504, 536, 476, 518, 538, 510, 512, 546, 492, 521, 520, 504, 527, 529, 476, 476, 530, 525, 490, 520, 502, 521, 500, 515, 514, 509, 487, 496, 558, 501, 479, 485, 523, 492, 503, 519, 528, 505, 527, 525, 501, 525, 528, 494, 537, 498, 530, 533, 466, 501] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[536, 508, 509, 499, 518, 526, 479, 522, 523, 506, 536, 539, 460, 519, 506, 526, 539, 500, 500, 533, 487, 531, 511, 482, 515, 517, 505, 515, 486, 504, 538, 520, 500, 482, 486, 487, 524, 600, 499, 500, 468, 483, 545, 465, 500, 472, 513, 539, 489, 551, 552, 495, 533, 503, 506, 507, 484, 514, 499, 500, 525, 541, 506, 540, 489, 476, 499, 527, 524, 520, 496, 561, 514, 501, 516, 510, 506, 517, 501, 496, 540, 517, 498, 525, 536, 513, 525, 531, 518, 474, 476, 525, 490, 484, 502, 527, 505, 547, 481, 499, 534, 522, 518, 504, 518, 495, 523, 518, 532, 542, 507, 506, 538, 486, 522, 525, 500, 524, 505, 528, 524, 534, 514, 521, 532, 454, 540, 501] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[513, 506, 531, 521, 500, 540, 509, 487, 499, 526, 540, 493, 516, 515, 505, 531, 514, 484, 474, 502, 521, 505, 497, 512, 523, 555, 528, 472, 488, 509, 525, 505, 538, 539, 502, 495, 509, 476, 520, 500, 507, 529, 484, 526, 523, 477, 463, 540, 500, 490, 515, 514, 527, 500, 501, 520, 552, 506, 526, 507, 506, 531, 494, 533, 505, 534, 533, 521, 540, 505, 539, 534, 498, 513, 533, 486, 527, 524, 484, 478, 507, 518, 510, 527, 520, 494, 501, 489, 511, 500, 503, 537, 536, 529, 460, 494, 470, 525, 525, 506, 539, 520, 516, 489, 545, 510, 508, 518, 506, 579, 549, 509, 505, 500, 508, 530, 518, 495, 513, 551, 509, 503, 517, 452, 472, 509, 521, 533] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[484, 507, 510, 507, 483, 508, 537, 528, 538, 530, 501, 490, 476, 546, 548, 491, 499, 496, 493, 570, 491, 507, 486, 520, 502, 497, 505, 530, 521, 511, 516, 480, 528, 521, 501, 549, 500, 519, 512, 514, 540, 547, 520, 526, 511, 515, 513, 472, 525, 508, 508, 520, 492, 532, 505, 519, 519, 524, 488, 490, 506, 530, 508, 522, 514, 530, 550, 488, 527, 527, 492, 533, 490, 482, 536, 511, 523, 496, 482, 501, 490, 488, 529, 531, 530, 544, 518, 554, 525, 506, 478, 554, 526, 483, 536, 488, 510, 532, 505, 505, 492, 489, 516, 538, 525, 488, 534, 540, 521, 498, 503, 491, 504, 516, 517, 490, 494, 493, 534, 495, 502, 527, 535, 478, 499, 505, 501, 505] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[528, 487, 521, 550, 508, 516, 499, 557, 527, 552, 537, 520, 496, 528, 476, 503, 497, 498, 506, 486, 539, 520, 529, 516, 492, 484, 505, 547, 548, 507, 519, 516, 509, 493, 486, 517, 504, 515, 518, 513, 520, 535, 527, 534, 549, 519, 503, 517, 532, 525, 514, 502, 536, 487, 480, 550, 518, 498, 480, 498, 520, 517, 488, 519, 531, 508, 523, 483, 516, 496, 549, 492, 537, 506, 487, 495, 525, 502, 530, 483, 501, 509, 522, 470, 521, 520, 482, 508, 527, 534, 516, 506, 531, 515, 516, 522, 521, 484, 515, 487, 499, 505, 479, 524, 527, 497, 502, 499, 516, 508, 486, 535, 495, 515, 538, 554, 468, 494, 469, 465, 538, 502, 537, 505, 525, 508, 510, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[491, 544, 528, 515, 494, 538, 514, 499, 509, 506, 519, 493, 528, 503, 561, 495, 529, 515, 495, 542, 506, 501, 520, 468, 543, 470, 503, 502, 522, 523, 534, 523, 512, 500, 527, 504, 526, 493, 544, 501, 497, 531, 525, 537, 532, 486, 527, 538, 521, 520, 523, 478, 525, 510, 470, 506, 480, 512, 554, 514, 553, 502, 517, 478, 488, 502, 512, 509, 543, 514, 459, 519, 519, 519, 465, 513, 469, 522, 514, 495, 487, 503, 508, 525, 503, 490, 503, 514, 500, 519, 498, 524, 508, 527, 481, 519, 528, 527, 512, 514, 527, 520, 549, 511, 489, 532, 557, 482, 481, 525, 498, 518, 515, 484, 525, 504, 520, 515, 529, 499, 525, 526, 497, 516, 492, 492, 526, 554] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[527, 508, 563, 496, 464, 482, 528, 523, 538, 530, 516, 529, 489, 480, 480, 555, 497, 493, 506, 541, 501, 503, 484, 525, 560, 504, 548, 507, 528, 488, 503, 477, 530, 517, 510, 509, 505, 519, 523, 523, 492, 511, 490, 534, 519, 480, 511, 487, 517, 504, 478, 489, 538, 514, 537, 543, 509, 511, 486, 526, 500, 486, 516, 507, 494, 497, 533, 562, 495, 507, 522, 524, 524, 516, 478, 510, 498, 489, 536, 513, 497, 479, 577, 542, 456, 518, 449, 522, 519, 538, 485, 533, 552, 522, 549, 512, 492, 533, 565, 507, 502, 484, 533, 526, 491, 518, 555, 532, 509, 493, 530, 519, 513, 502, 529, 480, 514, 504, 508, 457, 511, 508, 489, 520, 475, 534, 540, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[522, 504, 539, 540, 539, 489, 474, 501, 519, 523, 520, 503, 545, 502, 518, 514, 539, 478, 542, 515, 532, 497, 521, 495, 514, 512, 523, 558, 503, 550, 495, 507, 527, 481, 501, 532, 514, 514, 500, 501, 495, 502, 524, 477, 511, 520, 508, 534, 520, 494, 518, 501, 531, 520, 514, 482, 515, 500, 528, 497, 457, 530, 486, 527, 492, 480, 516, 484, 490, 519, 555, 508, 515, 523, 546, 538, 515, 509, 504, 536, 476, 518, 538, 510, 512, 546, 492, 521, 520, 504, 527, 529, 476, 476, 530, 525, 490, 520, 502, 521, 500, 515, 514, 509, 487, 496, 558, 501, 479, 485, 523, 492, 503, 519, 528, 505, 527, 525, 501, 525, 528, 494, 537, 498, 530, 533, 466, 501] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[513, 506, 531, 521, 500, 540, 509, 487, 499, 526, 485, 493, 516, 515, 505, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 484, 474, 502, 521, 505, 497, 512, 523, 555, 528, 472, 488, 509, 525, 505, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 539, 502, 495, 509, 476, 520, 500, 507, 529, 484, 526, 519, 477, 463, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 490, 515, 514, 527, 500, 501, 520, 552, 506, 526, 507, 506, 531, 494, 533, 12, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 534, 533, 521, 540, 505, 539, 534, 498, 513, 533, 486, 527, 524, 484, 478, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 518, 510, 527, 520, 494, 501, 489, 511, 500, 503, 537, 536, 529, 460, 494, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 525, 525, 506, 539, 520, 516, 489, 545, 510, 488, 518, 506, 579, 549, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 500, 508, 530, 518, 495, 513, 551, 509, 503, 517, 452, 472, 509, 521, 533, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8115, 8124, 8239, 8276, 8147, 8294, 8164] avg=8192.0000 max=8294 max_rank=6 min=8115 min_rank=1 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32881 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[522, 504, 539, 540, 539, 489, 474, 501, 519, 523, 478, 503, 545, 502, 518, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 478, 542, 515, 532, 497, 521, 495, 514, 512, 523, 558, 503, 550, 495, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 481, 501, 532, 514, 514, 500, 501, 495, 502, 524, 477, 509, 520, 508, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 494, 518, 501, 531, 520, 514, 482, 515, 500, 528, 497, 457, 530, 486, 527, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 480, 516, 484, 490, 519, 555, 508, 515, 523, 546, 538, 515, 509, 504, 536, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 518, 538, 510, 512, 546, 492, 521, 520, 504, 527, 529, 476, 476, 530, 525, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 520, 502, 521, 500, 515, 514, 509, 487, 496, 544, 501, 479, 485, 523, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 519, 528, 505, 527, 525, 501, 525, 528, 494, 537, 498, 530, 533, 466, 501, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8281, 8139, 8129, 8248, 8207, 8078, 8244] avg=8192.0000 max=8281 max_rank=1 min=8078 min_rank=6 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32777 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[528, 487, 521, 550, 508, 516, 499, 557, 527, 552, 494, 520, 496, 528, 476, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 498, 506, 486, 539, 520, 529, 516, 492, 484, 505, 547, 548, 507, 519, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 493, 486, 517, 504, 515, 518, 513, 520, 535, 527, 534, 547, 519, 503, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 525, 514, 502, 536, 487, 480, 550, 518, 498, 480, 498, 520, 517, 488, 519, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 508, 523, 483, 516, 496, 549, 492, 537, 506, 487, 495, 525, 502, 530, 483, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 509, 522, 470, 521, 520, 482, 508, 527, 534, 516, 506, 531, 515, 516, 522, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 484, 515, 487, 499, 505, 479, 524, 527, 497, 490, 499, 516, 508, 486, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 515, 538, 554, 468, 494, 469, 465, 538, 502, 537, 505, 525, 508, 510, 529, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8262, 8209, 8257, 8172, 8181, 8206, 8072, 8177] avg=8192.0000 max=8262 max_rank=0 min=8072 min_rank=6 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32636 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[527, 508, 563, 496, 464, 482, 528, 523, 538, 530, 474, 529, 489, 480, 480, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 493, 506, 541, 501, 503, 484, 525, 560, 504, 548, 507, 528, 488, 503, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 517, 510, 509, 505, 519, 523, 523, 492, 511, 490, 534, 517, 480, 511, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 504, 478, 489, 538, 514, 537, 543, 509, 511, 486, 526, 500, 486, 516, 507, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 497, 533, 562, 495, 507, 522, 524, 524, 516, 478, 510, 498, 489, 536, 513, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 479, 577, 542, 456, 518, 449, 522, 519, 538, 485, 533, 552, 522, 549, 512, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 533, 565, 507, 502, 484, 533, 526, 491, 518, 541, 532, 509, 493, 530, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 502, 529, 480, 514, 504, 508, 457, 511, 508, 489, 520, 475, 534, 540, 521, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8166, 8165, 8158, 8170, 8216, 8257, 8275, 8129] avg=8192.0000 max=8275 max_rank=6 min=8129 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32877 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[511, 532, 500, 491, 545, 546, 529, 505, 509, 496, 493, 487, 538, 517, 514, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 473, 540, 510, 521, 501, 491, 540, 521, 511, 509, 513, 537, 496, 542, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 538, 508, 545, 523, 499, 493, 532, 459, 514, 537, 533, 509, 554, 479, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 560, 472, 529, 538, 544, 468, 509, 478, 519, 529, 506, 537, 495, 500, 486, 520, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 464, 522, 503, 528, 527, 516, 528, 519, 483, 495, 512, 484, 539, 493, 507, 527, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 509, 510, 481, 496, 525, 529, 508, 479, 543, 502, 526, 496, 513, 529, 500, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 534, 522, 489, 541, 518, 485, 537, 538, 505, 470, 486, 496, 524, 508, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 487, 513, 514, 518, 498, 521, 511, 513, 522, 505, 500, 502, 494, 559, 517, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8263, 8233, 8197, 8165, 8130, 8147, 8188] avg=8192.0000 max=8263 max_rank=1 min=8130 min_rank=5 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32630 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[484, 507, 510, 507, 483, 508, 537, 528, 538, 530, 461, 490, 476, 546, 548, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 496, 493, 570, 491, 507, 486, 520, 502, 497, 505, 530, 521, 511, 516, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 521, 501, 549, 500, 519, 512, 514, 540, 547, 520, 526, 509, 515, 513, 472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 508, 508, 520, 492, 532, 505, 519, 519, 524, 488, 490, 506, 530, 508, 522, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 530, 550, 488, 527, 527, 492, 533, 490, 482, 536, 511, 523, 496, 482, 501, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 488, 529, 531, 530, 544, 518, 554, 525, 506, 478, 554, 526, 483, 536, 488, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 532, 505, 505, 492, 489, 516, 538, 525, 488, 522, 540, 521, 498, 503, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 516, 517, 490, 494, 493, 534, 495, 502, 527, 535, 478, 499, 505, 501, 505, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8144, 8124, 8286, 8204, 8199, 8286, 8175, 8118] avg=8192.0000 max=8286 max_rank=2 min=8118 min_rank=7 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32778 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[536, 508, 509, 499, 518, 526, 479, 522, 523, 506, 493, 539, 460, 519, 506, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 500, 500, 533, 487, 531, 511, 482, 515, 517, 505, 515, 486, 504, 538, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 482, 486, 487, 524, 600, 499, 500, 468, 483, 545, 465, 498, 472, 513, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 551, 552, 495, 533, 503, 506, 507, 484, 514, 499, 500, 525, 541, 506, 540, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 476, 499, 527, 524, 520, 496, 561, 514, 501, 516, 510, 506, 517, 501, 496, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 517, 498, 525, 536, 513, 525, 531, 518, 474, 476, 525, 490, 484, 502, 527, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 547, 481, 499, 534, 522, 518, 504, 518, 495, 511, 518, 532, 542, 507, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 486, 522, 525, 500, 524, 505, 528, 524, 534, 514, 521, 532, 454, 540, 501, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8169, 8183, 8061, 8253, 8171, 8187, 8239, 8273] avg=8192.0000 max=8273 max_rank=7 min=8061 min_rank=2 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32870 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[513, 506, 531, 521, 500, 540, 509, 487, 499, 526, 485, 493, 516, 515, 505, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 484, 474, 502, 521, 505, 497, 512, 523, 555, 528, 472, 488, 509, 525, 505, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 539, 502, 495, 509, 476, 520, 500, 507, 529, 484, 526, 519, 477, 463, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 490, 515, 514, 527, 500, 501, 520, 552, 506, 526, 507, 506, 531, 494, 533, 12, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 534, 533, 521, 540, 505, 539, 534, 498, 513, 533, 486, 527, 524, 484, 478, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 518, 510, 527, 520, 494, 501, 489, 511, 500, 503, 537, 536, 529, 460, 494, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 525, 525, 506, 539, 520, 516, 489, 545, 510, 488, 518, 506, 579, 549, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 500, 508, 530, 518, 495, 513, 551, 509, 503, 517, 452, 472, 509, 521, 533, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8115, 8124, 8239, 8276, 8147, 8294, 8164] avg=8192.0000 max=8294 max_rank=6 min=8115 min_rank=1 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32881 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[522, 504, 539, 540, 539, 489, 474, 501, 519, 523, 478, 503, 545, 502, 518, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 478, 542, 515, 532, 497, 521, 495, 514, 512, 523, 558, 503, 550, 495, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 481, 501, 532, 514, 514, 500, 501, 495, 502, 524, 477, 509, 520, 508, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 494, 518, 501, 531, 520, 514, 482, 515, 500, 528, 497, 457, 530, 486, 527, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 480, 516, 484, 490, 519, 555, 508, 515, 523, 546, 538, 515, 509, 504, 536, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 518, 538, 510, 512, 546, 492, 521, 520, 504, 527, 529, 476, 476, 530, 525, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 520, 502, 521, 500, 515, 514, 509, 487, 496, 544, 501, 479, 485, 523, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 519, 528, 505, 527, 525, 501, 525, 528, 494, 537, 498, 530, 533, 466, 501, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8281, 8139, 8129, 8248, 8207, 8078, 8244] avg=8192.0000 max=8281 max_rank=1 min=8078 min_rank=6 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32777 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[491, 544, 528, 515, 494, 538, 514, 499, 509, 506, 477, 493, 528, 503, 561, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 515, 495, 542, 506, 501, 520, 468, 543, 470, 503, 502, 522, 523, 534, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 500, 527, 504, 526, 493, 544, 501, 497, 531, 525, 537, 530, 486, 527, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 520, 523, 478, 525, 510, 470, 506, 480, 512, 554, 514, 553, 502, 517, 478, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 502, 512, 509, 543, 514, 459, 519, 519, 519, 465, 513, 469, 522, 514, 495, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 503, 508, 525, 503, 490, 503, 514, 500, 519, 498, 524, 508, 527, 481, 519, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 527, 512, 514, 527, 520, 549, 511, 489, 532, 543, 482, 481, 525, 498, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 484, 525, 504, 520, 515, 529, 499, 525, 526, 497, 516, 492, 492, 526, 554, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8196, 8278, 8172, 8080, 8116, 8256, 8243] avg=8192.0000 max=8278 max_rank=2 min=8080 min_rank=4 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32695 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[484, 507, 510, 507, 483, 508, 537, 528, 538, 530, 461, 490, 476, 546, 548, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 496, 493, 570, 491, 507, 486, 520, 502, 497, 505, 530, 521, 511, 516, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 521, 501, 549, 500, 519, 512, 514, 540, 547, 520, 526, 509, 515, 513, 472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 508, 508, 520, 492, 532, 505, 519, 519, 524, 488, 490, 506, 530, 508, 522, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 530, 550, 488, 527, 527, 492, 533, 490, 482, 536, 511, 523, 496, 482, 501, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 488, 529, 531, 530, 544, 518, 554, 525, 506, 478, 554, 526, 483, 536, 488, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 532, 505, 505, 492, 489, 516, 538, 525, 488, 522, 540, 521, 498, 503, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 516, 517, 490, 494, 493, 534, 495, 502, 527, 535, 478, 499, 505, 501, 505, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8144, 8124, 8286, 8204, 8199, 8286, 8175, 8118] avg=8192.0000 max=8286 max_rank=2 min=8118 min_rank=7 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32778 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[527, 508, 563, 496, 464, 482, 528, 523, 538, 530, 474, 529, 489, 480, 480, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 493, 506, 541, 501, 503, 484, 525, 560, 504, 548, 507, 528, 488, 503, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 517, 510, 509, 505, 519, 523, 523, 492, 511, 490, 534, 517, 480, 511, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 504, 478, 489, 538, 514, 537, 543, 509, 511, 486, 526, 500, 486, 516, 507, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 497, 533, 562, 495, 507, 522, 524, 524, 516, 478, 510, 498, 489, 536, 513, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 479, 577, 542, 456, 518, 449, 522, 519, 538, 485, 533, 552, 522, 549, 512, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 533, 565, 507, 502, 484, 533, 526, 491, 518, 541, 532, 509, 493, 530, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 502, 529, 480, 514, 504, 508, 457, 511, 508, 489, 520, 475, 534, 540, 521, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8166, 8165, 8158, 8170, 8216, 8257, 8275, 8129] avg=8192.0000 max=8275 max_rank=6 min=8129 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32877 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 3968, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 3968, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4352, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4224, 4096, 4352, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4352, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[530, 544, 477, 504, 494, 496, 536, 518, 544, 521, 505, 507, 505, 526, 480, 564, 455, 517, 535, 520, 515, 488, 536, 544, 519, 505, 543, 503, 543, 505, 491, 502, 493, 519, 518, 506, 497, 523, 500, 503, 527, 495, 513, 505, 486, 518, 550, 493, 531, 530, 475, 504, 541, 520, 526, 529, 512, 506, 536, 482, 503, 512, 541, 507, 514, 516, 491, 559, 505, 526, 507, 494, 492, 551, 498, 496, 499, 491, 497, 517, 538, 519, 516, 532, 523, 501, 533, 514, 512, 543, 547, 529, 507, 490, 509, 516, 511, 515, 488, 491, 511, 458, 541, 521, 499, 501, 528, 506, 497, 523, 499, 544, 517, 542, 494, 482, 520, 529, 497, 535, 504, 492, 510, 487, 525, 516, 442, 456] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[507, 507, 518, 522, 526, 471, 470, 545, 526, 489, 510, 536, 511, 503, 508, 545, 473, 526, 559, 536, 527, 495, 493, 509, 545, 506, 518, 539, 522, 473, 562, 532, 493, 492, 502, 576, 522, 521, 482, 535, 524, 502, 542, 511, 503, 537, 539, 530, 523, 502, 507, 496, 516, 487, 504, 460, 528, 460, 513, 511, 508, 544, 514, 507, 522, 532, 517, 570, 499, 508, 494, 510, 484, 487, 528, 571, 495, 540, 512, 482, 483, 530, 481, 512, 543, 503, 462, 471, 495, 495, 502, 525, 499, 529, 515, 487, 512, 528, 520, 503, 482, 490, 481, 495, 498, 493, 562, 513, 482, 547, 554, 518, 520, 544, 523, 567, 513, 530, 480, 489, 529, 508, 499, 511, 456, 518, 488, 500] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[509, 500, 495, 521, 508, 485, 538, 528, 493, 525, 538, 533, 504, 475, 554, 511, 550, 475, 490, 503, 510, 546, 552, 526, 499, 510, 511, 495, 500, 549, 498, 523, 531, 536, 490, 473, 493, 547, 562, 491, 504, 504, 556, 504, 514, 458, 490, 510, 519, 534, 513, 515, 482, 507, 511, 485, 534, 508, 495, 544, 502, 500, 523, 489, 505, 518, 468, 497, 552, 505, 534, 553, 543, 520, 508, 542, 506, 518, 524, 512, 495, 509, 521, 529, 496, 547, 512, 533, 506, 514, 490, 508, 483, 502, 514, 526, 525, 527, 528, 519, 531, 543, 537, 507, 548, 506, 495, 505, 490, 530, 475, 489, 512, 499, 473, 491, 518, 472, 492, 509, 498, 499, 500, 472, 528, 487, 539, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[528, 501, 504, 477, 532, 514, 519, 533, 502, 538, 518, 500, 521, 507, 528, 531, 496, 523, 536, 474, 462, 489, 517, 513, 494, 563, 517, 495, 553, 481, 513, 531, 535, 524, 532, 533, 497, 522, 512, 475, 496, 495, 505, 518, 524, 523, 535, 515, 490, 496, 496, 484, 544, 524, 545, 514, 484, 505, 505, 484, 478, 516, 534, 527, 501, 518, 499, 504, 495, 529, 463, 533, 537, 533, 518, 536, 480, 501, 507, 502, 491, 487, 503, 561, 564, 506, 523, 516, 501, 512, 523, 491, 531, 509, 525, 494, 477, 492, 473, 503, 501, 514, 485, 520, 496, 539, 540, 485, 529, 543, 479, 553, 512, 530, 537, 523, 526, 516, 516, 536, 472, 504, 517, 507, 536, 483, 541, 481] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[524, 495, 520, 492, 519, 505, 527, 497, 543, 477, 491, 479, 505, 481, 517, 500, 505, 509, 497, 472, 559, 489, 495, 517, 560, 486, 513, 519, 514, 481, 541, 519, 492, 507, 489, 522, 546, 471, 530, 488, 502, 495, 515, 498, 536, 520, 513, 450, 550, 535, 523, 512, 522, 519, 513, 489, 507, 489, 493, 528, 494, 555, 528, 501, 542, 537, 511, 505, 517, 512, 483, 526, 516, 531, 479, 535, 517, 531, 507, 504, 528, 507, 524, 505, 526, 516, 497, 473, 505, 529, 541, 511, 552, 529, 490, 471, 528, 521, 532, 552, 495, 504, 506, 491, 509, 493, 502, 513, 530, 525, 509, 497, 496, 523, 517, 523, 556, 525, 468, 534, 471, 515, 537, 531, 517, 526, 536, 517] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[529, 486, 527, 498, 555, 523, 523, 529, 484, 490, 561, 532, 525, 490, 541, 477, 529, 516, 505, 538, 511, 530, 505, 485, 471, 523, 500, 517, 481, 490, 519, 553, 537, 544, 517, 515, 516, 515, 528, 472, 488, 547, 536, 539, 531, 516, 515, 510, 515, 533, 525, 492, 508, 516, 481, 506, 525, 520, 538, 533, 495, 536, 527, 506, 540, 512, 501, 509, 496, 532, 483, 519, 536, 473, 511, 529, 497, 515, 510, 495, 520, 487, 499, 490, 499, 513, 495, 503, 542, 504, 537, 488, 516, 533, 500, 543, 534, 507, 528, 480, 498, 520, 543, 506, 491, 516, 503, 541, 487, 501, 523, 472, 489, 496, 497, 468, 515, 474, 497, 524, 525, 523, 487, 519, 492, 518, 505, 505] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[533, 533, 517, 467, 508, 505, 508, 506, 529, 552, 484, 539, 542, 476, 537, 506, 520, 489, 532, 511, 527, 535, 503, 499, 517, 476, 495, 507, 540, 479, 523, 502, 487, 531, 505, 534, 539, 561, 548, 533, 519, 512, 514, 520, 479, 497, 525, 525, 510, 508, 508, 512, 503, 528, 502, 479, 521, 532, 518, 487, 514, 521, 530, 482, 489, 515, 563, 495, 546, 523, 524, 508, 510, 525, 491, 535, 504, 523, 528, 541, 491, 485, 471, 543, 529, 487, 456, 489, 511, 525, 486, 498, 507, 478, 529, 520, 511, 532, 501, 528, 510, 515, 551, 528, 535, 508, 502, 506, 495, 500, 474, 506, 481, 496, 514, 509, 533, 508, 513, 518, 503, 518, 508, 517, 491, 463, 527, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[509, 500, 495, 521, 508, 485, 538, 528, 493, 525, 538, 533, 504, 475, 554, 511, 550, 475, 490, 503, 510, 546, 552, 526, 499, 510, 511, 495, 500, 549, 498, 523, 531, 536, 490, 473, 493, 547, 562, 491, 504, 504, 556, 504, 514, 458, 490, 510, 519, 534, 513, 515, 482, 507, 511, 485, 534, 508, 495, 544, 502, 500, 523, 489, 505, 518, 468, 497, 552, 505, 534, 553, 543, 520, 508, 542, 506, 518, 524, 512, 495, 509, 521, 529, 496, 547, 512, 533, 506, 514, 490, 508, 483, 502, 514, 526, 525, 527, 528, 519, 531, 543, 537, 507, 548, 506, 495, 505, 490, 530, 475, 489, 512, 499, 473, 491, 518, 472, 492, 509, 498, 499, 500, 472, 528, 487, 539, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[530, 544, 477, 504, 494, 496, 536, 518, 544, 521, 505, 507, 505, 526, 480, 564, 455, 517, 535, 520, 515, 488, 536, 544, 519, 505, 543, 503, 543, 505, 491, 502, 493, 519, 518, 506, 497, 523, 500, 503, 527, 495, 513, 505, 486, 518, 550, 493, 531, 530, 475, 504, 541, 520, 526, 529, 512, 506, 536, 482, 503, 512, 541, 507, 514, 516, 491, 559, 505, 526, 507, 494, 492, 551, 498, 496, 499, 491, 497, 517, 538, 519, 516, 532, 523, 501, 533, 514, 512, 543, 547, 529, 507, 490, 509, 516, 511, 515, 488, 491, 511, 458, 541, 521, 499, 501, 528, 506, 497, 523, 499, 544, 517, 542, 494, 482, 520, 529, 497, 535, 504, 492, 510, 487, 525, 516, 442, 456] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[506, 578, 481, 507, 527, 544, 535, 517, 502, 541, 539, 502, 496, 489, 516, 523, 472, 506, 539, 494, 527, 508, 567, 508, 510, 501, 537, 496, 541, 522, 544, 486, 496, 494, 530, 493, 509, 493, 516, 479, 521, 506, 512, 474, 479, 483, 496, 506, 516, 506, 513, 489, 535, 504, 525, 455, 494, 523, 539, 495, 466, 513, 528, 504, 486, 517, 525, 474, 522, 477, 483, 541, 544, 524, 517, 502, 558, 529, 507, 512, 512, 524, 469, 520, 542, 504, 532, 514, 528, 477, 525, 535, 502, 503, 499, 478, 539, 482, 486, 543, 484, 564, 514, 505, 493, 546, 483, 515, 485, 529, 490, 517, 520, 517, 508, 508, 539, 557, 533, 477, 508, 497, 551, 545, 540, 510, 515, 501] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[507, 507, 518, 522, 526, 471, 470, 545, 526, 489, 510, 536, 511, 503, 477, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 526, 559, 536, 527, 495, 493, 509, 545, 506, 518, 539, 503, 473, 562, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 492, 502, 576, 522, 521, 482, 535, 524, 502, 537, 511, 503, 537, 539, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 502, 507, 496, 516, 487, 504, 460, 528, 460, 513, 511, 508, 544, 514, 507, 11, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 532, 517, 570, 499, 508, 494, 510, 484, 487, 528, 534, 495, 540, 512, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 530, 481, 512, 543, 503, 462, 471, 495, 495, 502, 525, 499, 529, 515, 487, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 528, 520, 503, 482, 490, 481, 495, 498, 493, 562, 513, 482, 547, 554, 518, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 544, 523, 567, 513, 530, 480, 489, 529, 508, 499, 511, 456, 518, 488, 500, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8163, 8296, 8306, 8097, 8214, 8057, 8191, 8212] avg=8192.0000 max=8306 max_rank=2 min=8057 min_rank=5 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32674 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[533, 533, 517, 467, 508, 505, 508, 506, 529, 552, 484, 539, 542, 476, 537, 506, 520, 489, 532, 511, 527, 535, 503, 499, 517, 476, 495, 507, 540, 479, 523, 502, 487, 531, 505, 534, 539, 561, 548, 533, 519, 512, 514, 520, 479, 497, 525, 525, 510, 508, 508, 512, 503, 528, 502, 479, 521, 532, 518, 487, 514, 521, 530, 482, 489, 515, 563, 495, 546, 523, 524, 508, 510, 525, 491, 535, 504, 523, 528, 541, 491, 485, 471, 543, 529, 487, 456, 489, 511, 525, 486, 498, 507, 478, 529, 520, 511, 532, 501, 528, 510, 515, 551, 528, 535, 508, 502, 506, 495, 500, 474, 506, 481, 496, 514, 509, 533, 508, 513, 518, 503, 518, 508, 517, 491, 463, 527, 529] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[528, 501, 504, 477, 532, 514, 519, 533, 502, 538, 518, 500, 521, 507, 528, 531, 496, 523, 536, 474, 462, 489, 517, 513, 494, 563, 517, 495, 553, 481, 513, 531, 535, 524, 532, 533, 497, 522, 512, 475, 496, 495, 505, 518, 524, 523, 535, 515, 490, 496, 496, 484, 544, 524, 545, 514, 484, 505, 505, 484, 478, 516, 534, 527, 501, 518, 499, 504, 495, 529, 463, 533, 537, 533, 518, 536, 480, 501, 507, 502, 491, 487, 503, 561, 564, 506, 523, 516, 501, 512, 523, 491, 531, 509, 525, 494, 477, 492, 473, 503, 501, 514, 485, 520, 496, 539, 540, 485, 529, 543, 479, 553, 512, 530, 537, 523, 526, 516, 516, 536, 472, 504, 517, 507, 536, 483, 541, 481] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[507, 507, 518, 522, 526, 471, 470, 545, 526, 489, 510, 536, 511, 503, 508, 545, 473, 526, 559, 536, 527, 495, 493, 509, 545, 506, 518, 539, 522, 473, 562, 532, 493, 492, 502, 576, 522, 521, 482, 535, 524, 502, 542, 511, 503, 537, 539, 530, 523, 502, 507, 496, 516, 487, 504, 460, 528, 460, 513, 511, 508, 544, 514, 507, 522, 532, 517, 570, 499, 508, 494, 510, 484, 487, 528, 571, 495, 540, 512, 482, 483, 530, 481, 512, 543, 503, 462, 471, 495, 495, 502, 525, 499, 529, 515, 487, 512, 528, 520, 503, 482, 490, 481, 495, 498, 493, 562, 513, 482, 547, 554, 518, 520, 544, 523, 567, 513, 530, 480, 489, 529, 508, 499, 511, 456, 518, 488, 500] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[509, 500, 495, 521, 508, 485, 538, 528, 493, 525, 538, 533, 504, 475, 521, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 475, 490, 503, 510, 546, 552, 526, 499, 510, 511, 495, 483, 549, 498, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 536, 490, 473, 493, 547, 562, 491, 504, 504, 551, 504, 514, 458, 490, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 534, 513, 515, 482, 507, 511, 485, 534, 508, 495, 544, 502, 500, 523, 489, 10, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 518, 468, 497, 552, 505, 534, 553, 543, 520, 508, 507, 506, 518, 524, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 509, 521, 529, 496, 547, 512, 533, 506, 514, 490, 508, 483, 502, 514, 526, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 527, 528, 519, 531, 543, 537, 507, 548, 506, 495, 505, 490, 530, 475, 489, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 499, 473, 491, 518, 472, 492, 509, 498, 499, 500, 472, 528, 487, 539, 524, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8220, 8158, 8177, 8270, 8212, 8267, 8048] avg=8192.0000 max=8270 max_rank=4 min=8048 min_rank=7 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32797 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[528, 501, 504, 477, 532, 514, 519, 533, 502, 538, 518, 500, 521, 507, 496, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 523, 536, 474, 462, 489, 517, 513, 494, 563, 517, 495, 534, 481, 513, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 524, 532, 533, 497, 522, 512, 475, 496, 495, 501, 518, 524, 523, 535, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 496, 496, 484, 544, 524, 545, 514, 484, 505, 505, 484, 478, 516, 534, 527, 11, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 518, 499, 504, 495, 529, 463, 533, 537, 533, 518, 501, 480, 501, 507, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 487, 503, 561, 564, 506, 523, 516, 501, 512, 523, 491, 531, 509, 525, 494, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 492, 473, 503, 501, 514, 485, 520, 496, 539, 540, 485, 529, 543, 479, 553, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 530, 537, 523, 526, 516, 516, 536, 472, 504, 517, 507, 536, 483, 541, 481, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8138, 8237, 8143, 8121, 8263, 8141, 8272] avg=8192.0000 max=8272 max_rank=7 min=8121 min_rank=4 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32797 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[530, 544, 477, 504, 494, 496, 536, 518, 544, 521, 505, 507, 505, 526, 451, 564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 455, 517, 535, 520, 515, 488, 536, 544, 519, 505, 543, 503, 524, 505, 491, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 519, 518, 506, 497, 523, 500, 503, 527, 495, 508, 505, 486, 518, 550, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 530, 475, 504, 541, 520, 526, 529, 512, 506, 536, 482, 503, 512, 541, 507, 11, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 516, 491, 559, 505, 526, 507, 494, 492, 551, 498, 464, 499, 491, 497, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 519, 516, 532, 523, 501, 533, 514, 512, 543, 547, 529, 507, 490, 509, 516, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 515, 488, 491, 511, 458, 541, 521, 499, 501, 528, 506, 497, 523, 499, 544, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 542, 494, 482, 520, 529, 497, 535, 504, 492, 510, 487, 525, 516, 442, 456, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8202, 8141, 8271, 8121, 8353, 8146, 8080] avg=8192.0000 max=8353 max_rank=5 min=8080 min_rank=7 imbalance_ratio=1.019653 max_load_reduction=57183 max_load_reduction_pct=0.872543 moved_to_echo=32700 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[506, 578, 481, 507, 527, 544, 535, 517, 502, 541, 539, 502, 496, 489, 473, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 506, 539, 494, 527, 508, 567, 508, 510, 501, 537, 496, 514, 522, 544, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 494, 530, 493, 509, 493, 516, 479, 521, 506, 505, 474, 479, 483, 496, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 506, 513, 489, 535, 504, 525, 455, 494, 523, 539, 495, 466, 513, 528, 504, 15, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 517, 525, 474, 522, 477, 483, 541, 544, 524, 517, 467, 558, 529, 507, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 524, 469, 520, 542, 504, 532, 514, 528, 477, 525, 535, 502, 503, 499, 478, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 482, 486, 543, 484, 564, 514, 505, 493, 546, 483, 515, 485, 529, 490, 517, 12, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 517, 508, 508, 539, 557, 533, 477, 508, 497, 551, 545, 540, 510, 515, 501, 35, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8260, 8231, 7980, 8129, 8183, 8193, 8194, 8366] avg=8192.0000 max=8366 max_rank=7 min=7980 min_rank=2 imbalance_ratio=1.021240 max_load_reduction=57170 max_load_reduction_pct=0.872345 moved_to_echo=32936 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[529, 486, 527, 498, 555, 523, 523, 529, 484, 490, 561, 532, 525, 490, 508, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 516, 505, 538, 511, 530, 505, 485, 471, 523, 500, 517, 464, 490, 519, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 544, 517, 515, 516, 515, 528, 472, 488, 547, 531, 539, 531, 516, 515, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 533, 525, 492, 508, 516, 481, 506, 525, 520, 538, 533, 495, 536, 527, 506, 10, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 512, 501, 509, 496, 532, 483, 519, 536, 473, 511, 495, 497, 515, 510, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 487, 499, 490, 499, 513, 495, 503, 542, 504, 537, 488, 516, 533, 500, 543, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 507, 528, 480, 498, 520, 543, 506, 491, 516, 503, 541, 487, 501, 523, 472, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 496, 497, 468, 515, 474, 497, 524, 525, 523, 487, 519, 492, 518, 505, 505, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8237, 8156, 8321, 8272, 8124, 8196, 8162, 8068] avg=8192.0000 max=8321 max_rank=2 min=8068 min_rank=7 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32550 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[524, 495, 520, 492, 519, 505, 527, 497, 543, 477, 491, 479, 505, 481, 485, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 509, 497, 472, 559, 489, 495, 517, 560, 486, 513, 519, 495, 481, 541, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 507, 489, 522, 546, 471, 530, 488, 502, 495, 510, 498, 536, 520, 513, 450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 535, 523, 512, 522, 519, 513, 489, 507, 489, 493, 528, 494, 555, 528, 501, 11, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 537, 511, 505, 517, 512, 483, 526, 516, 531, 479, 500, 517, 531, 507, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 507, 524, 505, 526, 516, 497, 473, 505, 529, 541, 511, 552, 529, 490, 471, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 521, 532, 552, 495, 504, 506, 491, 509, 493, 502, 513, 530, 525, 509, 497, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 523, 517, 523, 556, 525, 468, 534, 471, 515, 537, 531, 517, 526, 536, 517, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8040, 8157, 8069, 8275, 8218, 8230, 8220, 8327] avg=8192.0000 max=8327 max_rank=7 min=8040 min_rank=0 imbalance_ratio=1.016479 max_load_reduction=57209 max_load_reduction_pct=0.872940 moved_to_echo=32995 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[506, 578, 481, 507, 527, 544, 535, 517, 502, 541, 539, 502, 496, 489, 473, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 506, 539, 494, 527, 508, 567, 508, 510, 501, 537, 496, 514, 522, 544, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 494, 530, 493, 509, 493, 516, 479, 521, 506, 505, 474, 479, 483, 496, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 506, 513, 489, 535, 504, 525, 455, 494, 523, 539, 495, 466, 513, 528, 504, 15, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 517, 525, 474, 522, 477, 483, 541, 544, 524, 517, 467, 558, 529, 507, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 524, 469, 520, 542, 504, 532, 514, 528, 477, 525, 535, 502, 503, 499, 478, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 482, 486, 543, 484, 564, 514, 505, 493, 546, 483, 515, 485, 529, 490, 517, 12, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 517, 508, 508, 539, 557, 533, 477, 508, 497, 551, 545, 540, 510, 515, 501, 35, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8260, 8231, 7980, 8129, 8183, 8193, 8194, 8366] avg=8192.0000 max=8366 max_rank=7 min=7980 min_rank=2 imbalance_ratio=1.021240 max_load_reduction=57170 max_load_reduction_pct=0.872345 moved_to_echo=32936 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[533, 533, 517, 467, 508, 505, 508, 506, 529, 552, 484, 539, 542, 476, 504, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 489, 532, 511, 527, 535, 503, 499, 517, 476, 495, 507, 521, 479, 523, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 531, 505, 534, 539, 561, 548, 533, 519, 512, 509, 520, 479, 497, 525, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 508, 508, 512, 503, 528, 502, 479, 521, 532, 518, 487, 514, 521, 530, 482, 11, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 515, 563, 495, 546, 523, 524, 508, 510, 525, 491, 500, 504, 523, 528, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 485, 471, 543, 529, 487, 456, 489, 511, 525, 486, 498, 507, 478, 529, 520, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 532, 501, 528, 510, 515, 551, 528, 535, 508, 502, 506, 495, 500, 474, 506, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 496, 514, 509, 533, 508, 513, 518, 503, 518, 508, 517, 491, 463, 527, 529, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8136, 8324, 8172, 8285, 8032, 8215, 8163] avg=8192.0000 max=8324 max_rank=2 min=8032 min_rank=5 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32695 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[528, 501, 504, 477, 532, 514, 519, 533, 502, 538, 518, 500, 521, 507, 496, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 523, 536, 474, 462, 489, 517, 513, 494, 563, 517, 495, 534, 481, 513, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 524, 532, 533, 497, 522, 512, 475, 496, 495, 501, 518, 524, 523, 535, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 496, 496, 484, 544, 524, 545, 514, 484, 505, 505, 484, 478, 516, 534, 527, 11, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 518, 499, 504, 495, 529, 463, 533, 537, 533, 518, 501, 480, 501, 507, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 487, 503, 561, 564, 506, 523, 516, 501, 512, 523, 491, 531, 509, 525, 494, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 492, 473, 503, 501, 514, 485, 520, 496, 539, 540, 485, 529, 543, 479, 553, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 530, 537, 523, 526, 516, 516, 536, 472, 504, 517, 507, 536, 483, 541, 481, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8138, 8237, 8143, 8121, 8263, 8141, 8272] avg=8192.0000 max=8272 max_rank=7 min=8121 min_rank=4 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32797 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[529, 486, 527, 498, 555, 523, 523, 529, 484, 490, 561, 532, 525, 490, 508, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 516, 505, 538, 511, 530, 505, 485, 471, 523, 500, 517, 464, 490, 519, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 544, 517, 515, 516, 515, 528, 472, 488, 547, 531, 539, 531, 516, 515, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 533, 525, 492, 508, 516, 481, 506, 525, 520, 538, 533, 495, 536, 527, 506, 10, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 512, 501, 509, 496, 532, 483, 519, 536, 473, 511, 495, 497, 515, 510, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 487, 499, 490, 499, 513, 495, 503, 542, 504, 537, 488, 516, 533, 500, 543, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 507, 528, 480, 498, 520, 543, 506, 491, 516, 503, 541, 487, 501, 523, 472, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 496, 497, 468, 515, 474, 497, 524, 525, 523, 487, 519, 492, 518, 505, 505, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8237, 8156, 8321, 8272, 8124, 8196, 8162, 8068] avg=8192.0000 max=8321 max_rank=2 min=8068 min_rank=7 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32550 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[530, 544, 477, 504, 494, 496, 536, 518, 544, 521, 505, 507, 505, 526, 451, 564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 455, 517, 535, 520, 515, 488, 536, 544, 519, 505, 543, 503, 524, 505, 491, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 519, 518, 506, 497, 523, 500, 503, 527, 495, 508, 505, 486, 518, 550, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 530, 475, 504, 541, 520, 526, 529, 512, 506, 536, 482, 503, 512, 541, 507, 11, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 516, 491, 559, 505, 526, 507, 494, 492, 551, 498, 464, 499, 491, 497, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 519, 516, 532, 523, 501, 533, 514, 512, 543, 547, 529, 507, 490, 509, 516, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 515, 488, 491, 511, 458, 541, 521, 499, 501, 528, 506, 497, 523, 499, 544, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 542, 494, 482, 520, 529, 497, 535, 504, 492, 510, 487, 525, 516, 442, 456, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8202, 8141, 8271, 8121, 8353, 8146, 8080] avg=8192.0000 max=8353 max_rank=5 min=8080 min_rank=7 imbalance_ratio=1.019653 max_load_reduction=57183 max_load_reduction_pct=0.872543 moved_to_echo=32700 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[507, 507, 518, 522, 526, 471, 470, 545, 526, 489, 510, 536, 511, 503, 477, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 526, 559, 536, 527, 495, 493, 509, 545, 506, 518, 539, 503, 473, 562, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 492, 502, 576, 522, 521, 482, 535, 524, 502, 537, 511, 503, 537, 539, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 502, 507, 496, 516, 487, 504, 460, 528, 460, 513, 511, 508, 544, 514, 507, 11, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 532, 517, 570, 499, 508, 494, 510, 484, 487, 528, 534, 495, 540, 512, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 530, 481, 512, 543, 503, 462, 471, 495, 495, 502, 525, 499, 529, 515, 487, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 528, 520, 503, 482, 490, 481, 495, 498, 493, 562, 513, 482, 547, 554, 518, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 544, 523, 567, 513, 530, 480, 489, 529, 508, 499, 511, 456, 518, 488, 500, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8163, 8296, 8306, 8097, 8214, 8057, 8191, 8212] avg=8192.0000 max=8306 max_rank=2 min=8057 min_rank=5 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32674 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4224, 3968, 4224, 4096, 4224, 4096, 3968, 4224, 4352, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4224, 3968, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4224, 3968, 4224, 4096, 4224, 4096, 3968, 4224, 4352, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 3968, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 3968, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[503, 513, 508, 508, 527, 505, 512, 574, 519, 512, 491, 508, 563, 505, 483, 490, 497, 526, 499, 513, 506, 534, 513, 539, 511, 527, 557, 516, 509, 574, 514, 511, 487, 502, 484, 505, 466, 544, 538, 521, 505, 542, 511, 492, 507, 478, 526, 541, 470, 504, 523, 520, 557, 476, 519, 523, 497, 542, 505, 489, 519, 511, 508, 520, 506, 506, 540, 495, 517, 495, 513, 544, 497, 499, 508, 525, 508, 481, 506, 526, 479, 508, 488, 502, 507, 521, 504, 522, 510, 527, 458, 524, 501, 516, 517, 517, 549, 497, 532, 547, 512, 495, 473, 554, 476, 490, 532, 510, 507, 549, 548, 524, 490, 529, 496, 517, 506, 560, 456, 484, 522, 484, 499, 515, 533, 488, 500, 496] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[482, 531, 470, 509, 498, 513, 520, 527, 482, 476, 496, 504, 471, 514, 493, 528, 531, 510, 516, 511, 540, 538, 511, 547, 470, 523, 517, 509, 530, 507, 517, 521, 506, 502, 465, 533, 497, 489, 532, 519, 509, 526, 522, 512, 497, 532, 575, 477, 506, 528, 539, 529, 474, 486, 519, 541, 457, 559, 533, 522, 513, 552, 529, 532, 546, 546, 565, 508, 503, 502, 538, 509, 485, 514, 515, 487, 478, 515, 493, 550, 521, 525, 558, 500, 471, 484, 570, 543, 490, 484, 513, 492, 502, 505, 482, 520, 538, 470, 553, 490, 496, 538, 538, 462, 549, 529, 510, 491, 507, 501, 504, 505, 503, 526, 520, 496, 490, 543, 496, 488, 518, 448, 542, 469, 498, 500, 527, 553] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[487, 508, 470, 530, 498, 459, 554, 486, 552, 508, 493, 500, 462, 546, 542, 495, 511, 497, 546, 536, 479, 530, 539, 530, 498, 493, 508, 517, 538, 496, 503, 508, 508, 494, 500, 537, 542, 521, 529, 513, 501, 505, 514, 527, 529, 515, 526, 534, 472, 514, 539, 513, 486, 555, 526, 479, 510, 503, 469, 539, 506, 518, 503, 531, 517, 511, 541, 490, 497, 489, 481, 513, 549, 509, 477, 508, 518, 468, 522, 482, 546, 548, 494, 536, 505, 520, 526, 523, 516, 519, 537, 500, 515, 544, 530, 461, 499, 526, 495, 515, 507, 533, 485, 496, 494, 496, 557, 539, 568, 517, 523, 523, 509, 537, 520, 468, 534, 504, 529, 520, 496, 480, 476, 476, 513, 523, 487, 522] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[479, 514, 510, 533, 525, 497, 512, 513, 539, 509, 473, 532, 548, 488, 520, 531, 501, 533, 466, 511, 523, 501, 534, 491, 544, 491, 486, 537, 509, 475, 514, 488, 494, 510, 517, 500, 535, 527, 480, 486, 493, 515, 530, 552, 490, 516, 527, 520, 508, 466, 549, 542, 513, 493, 494, 499, 516, 539, 515, 537, 476, 498, 509, 478, 520, 549, 508, 472, 532, 513, 510, 517, 507, 510, 472, 535, 514, 491, 527, 488, 548, 516, 513, 558, 491, 531, 493, 523, 525, 486, 528, 501, 525, 486, 443, 503, 520, 502, 468, 525, 488, 515, 535, 531, 543, 494, 513, 513, 576, 497, 514, 496, 494, 521, 516, 529, 494, 544, 487, 550, 516, 507, 510, 583, 555, 493, 520, 501] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[513, 496, 507, 526, 535, 496, 502, 482, 520, 522, 514, 510, 535, 509, 528, 527, 502, 515, 453, 515, 466, 542, 492, 528, 488, 526, 510, 493, 528, 551, 521, 502, 479, 519, 547, 519, 514, 527, 511, 524, 495, 504, 518, 521, 545, 529, 532, 531, 475, 525, 519, 498, 518, 525, 498, 498, 477, 533, 502, 490, 506, 497, 510, 506, 507, 526, 512, 519, 477, 484, 530, 523, 480, 523, 517, 471, 481, 537, 517, 534, 522, 538, 536, 523, 508, 490, 489, 503, 506, 494, 504, 518, 482, 507, 516, 522, 546, 534, 478, 503, 545, 523, 528, 522, 554, 530, 523, 506, 519, 491, 559, 513, 505, 475, 497, 515, 510, 511, 496, 511, 525, 500, 515, 492, 522, 499, 536, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[500, 534, 499, 482, 496, 524, 542, 504, 497, 476, 521, 534, 558, 529, 492, 480, 523, 493, 529, 481, 521, 482, 554, 479, 479, 523, 510, 526, 503, 454, 499, 510, 531, 508, 531, 510, 517, 531, 551, 488, 534, 531, 509, 507, 534, 515, 521, 511, 497, 498, 505, 541, 520, 516, 508, 514, 532, 506, 493, 474, 490, 511, 487, 477, 516, 513, 508, 503, 514, 526, 538, 495, 502, 511, 502, 527, 527, 523, 543, 520, 480, 515, 540, 534, 528, 525, 516, 528, 510, 544, 468, 515, 538, 495, 528, 487, 484, 492, 539, 518, 506, 540, 514, 481, 524, 502, 510, 528, 546, 507, 538, 490, 478, 475, 487, 472, 495, 548, 567, 519, 517, 528, 530, 470, 528, 557, 473, 522] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[482, 531, 470, 509, 498, 513, 520, 527, 482, 476, 496, 504, 471, 514, 493, 528, 531, 510, 516, 511, 540, 538, 511, 547, 470, 523, 517, 509, 530, 507, 517, 521, 506, 502, 465, 533, 497, 489, 532, 519, 509, 526, 522, 512, 497, 532, 575, 477, 506, 528, 539, 529, 474, 486, 519, 541, 457, 559, 533, 522, 513, 552, 529, 532, 546, 546, 565, 508, 503, 502, 538, 509, 485, 514, 515, 487, 478, 515, 493, 550, 521, 525, 558, 500, 471, 484, 570, 543, 490, 484, 513, 492, 502, 505, 482, 520, 538, 470, 553, 490, 496, 538, 538, 462, 549, 529, 510, 491, 507, 501, 504, 505, 503, 526, 520, 496, 490, 543, 496, 488, 518, 448, 542, 469, 498, 500, 527, 553] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[489, 506, 523, 543, 546, 518, 510, 502, 509, 511, 537, 528, 433, 509, 539, 516, 538, 482, 512, 526, 494, 461, 502, 505, 541, 515, 518, 520, 525, 541, 547, 515, 536, 516, 506, 482, 525, 497, 510, 528, 536, 519, 536, 514, 517, 483, 535, 509, 530, 547, 518, 528, 528, 504, 501, 488, 484, 495, 493, 489, 497, 485, 485, 532, 503, 518, 514, 503, 509, 509, 517, 531, 472, 556, 523, 507, 528, 478, 554, 477, 496, 495, 477, 469, 514, 547, 486, 545, 524, 534, 528, 492, 500, 503, 552, 529, 499, 513, 483, 510, 486, 545, 522, 543, 517, 482, 501, 550, 529, 485, 514, 546, 498, 527, 503, 513, 520, 508, 542, 547, 519, 455, 488, 493, 495, 526, 484, 489] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[503, 513, 508, 508, 527, 505, 512, 574, 519, 512, 491, 508, 563, 505, 483, 490, 497, 526, 499, 513, 506, 534, 513, 539, 511, 527, 557, 516, 509, 574, 514, 511, 487, 502, 484, 505, 466, 544, 538, 521, 505, 542, 511, 492, 507, 478, 526, 541, 470, 504, 523, 520, 557, 476, 519, 523, 497, 542, 505, 489, 519, 511, 508, 520, 506, 506, 540, 495, 517, 495, 513, 544, 497, 499, 508, 525, 508, 481, 506, 526, 479, 508, 488, 502, 507, 521, 504, 522, 510, 527, 458, 524, 501, 516, 517, 517, 549, 497, 532, 547, 512, 495, 473, 554, 476, 490, 532, 510, 507, 549, 548, 524, 490, 529, 496, 517, 506, 560, 456, 484, 522, 484, 499, 515, 533, 488, 500, 496] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[530, 518, 501, 519, 525, 527, 550, 520, 497, 498, 508, 503, 513, 514, 535, 524, 461, 516, 502, 564, 508, 565, 528, 524, 520, 521, 493, 515, 467, 511, 532, 494, 506, 505, 498, 544, 546, 565, 532, 508, 495, 496, 501, 476, 531, 495, 509, 494, 505, 487, 507, 514, 472, 481, 544, 496, 509, 496, 506, 533, 503, 535, 543, 486, 495, 516, 521, 481, 494, 531, 511, 539, 507, 497, 546, 491, 521, 512, 549, 497, 486, 535, 528, 485, 497, 507, 517, 529, 505, 484, 499, 537, 504, 546, 480, 527, 560, 503, 502, 468, 520, 518, 505, 538, 523, 489, 512, 511, 505, 498, 481, 542, 505, 505, 514, 491, 529, 504, 492, 539, 533, 506, 482, 505, 481, 497, 515, 568] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[479, 514, 510, 533, 525, 497, 512, 513, 539, 509, 473, 532, 548, 488, 520, 531, 501, 533, 466, 511, 523, 501, 534, 491, 544, 491, 486, 537, 509, 475, 514, 488, 494, 510, 517, 500, 535, 527, 480, 486, 493, 515, 530, 552, 490, 516, 527, 520, 508, 466, 549, 542, 513, 493, 494, 499, 516, 539, 515, 537, 476, 498, 509, 478, 520, 549, 508, 472, 532, 513, 510, 517, 507, 510, 472, 535, 514, 491, 527, 488, 548, 516, 513, 558, 491, 531, 493, 523, 525, 486, 528, 501, 525, 486, 443, 503, 520, 502, 468, 525, 488, 515, 535, 531, 543, 494, 513, 513, 576, 497, 514, 496, 494, 521, 516, 529, 494, 544, 487, 550, 516, 507, 510, 583, 555, 493, 520, 501] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[487, 508, 470, 530, 498, 459, 554, 486, 552, 508, 493, 500, 462, 546, 542, 495, 511, 497, 546, 536, 479, 530, 539, 530, 498, 493, 508, 517, 538, 496, 503, 508, 508, 494, 500, 537, 542, 521, 529, 513, 501, 505, 514, 527, 529, 515, 526, 534, 472, 514, 539, 513, 486, 555, 526, 479, 510, 503, 469, 539, 506, 518, 503, 531, 517, 511, 541, 490, 497, 489, 481, 513, 549, 509, 477, 508, 518, 468, 522, 482, 546, 548, 494, 536, 505, 520, 526, 523, 516, 519, 537, 500, 515, 544, 530, 461, 499, 526, 495, 515, 507, 533, 485, 496, 494, 496, 557, 539, 568, 517, 523, 523, 509, 537, 520, 468, 534, 504, 529, 520, 496, 480, 476, 476, 513, 523, 487, 522] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[500, 534, 499, 482, 496, 524, 542, 504, 497, 476, 521, 534, 558, 529, 492, 480, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 493, 529, 481, 521, 482, 542, 479, 479, 523, 510, 526, 503, 454, 499, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 508, 531, 510, 517, 531, 551, 488, 534, 531, 509, 507, 534, 515, 475, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 498, 505, 541, 520, 516, 508, 514, 532, 506, 493, 474, 490, 511, 487, 477, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 513, 508, 503, 514, 526, 538, 495, 502, 511, 502, 527, 527, 523, 543, 520, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 515, 540, 534, 528, 525, 516, 528, 510, 544, 468, 515, 538, 495, 528, 487, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 492, 539, 518, 506, 540, 514, 481, 524, 502, 510, 528, 491, 507, 538, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 475, 487, 472, 495, 548, 567, 519, 517, 528, 530, 470, 528, 557, 473, 522, 37, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8054, 8283, 8116, 8276, 8253, 8164, 8211] avg=8192.0000 max=8283 max_rank=2 min=8054 min_rank=1 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32904 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[482, 531, 470, 509, 498, 513, 520, 527, 482, 476, 496, 504, 471, 514, 493, 528, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 510, 516, 511, 540, 538, 499, 547, 470, 523, 517, 509, 530, 507, 517, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 502, 465, 533, 497, 489, 532, 519, 509, 526, 522, 512, 497, 532, 524, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 528, 539, 529, 474, 486, 519, 541, 457, 559, 533, 522, 513, 552, 529, 532, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 546, 565, 508, 503, 502, 538, 509, 485, 514, 515, 487, 478, 515, 493, 550, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 525, 558, 500, 471, 484, 570, 543, 490, 484, 513, 492, 502, 505, 482, 520, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 470, 553, 490, 496, 538, 538, 462, 549, 529, 510, 491, 456, 501, 504, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 526, 520, 496, 490, 543, 496, 488, 518, 448, 542, 469, 498, 500, 527, 553, 41, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8026, 8286, 8142, 8363, 8262, 8162, 8130, 8165] avg=8192.0000 max=8363 max_rank=3 min=8026 min_rank=0 imbalance_ratio=1.020874 max_load_reduction=57173 max_load_reduction_pct=0.872391 moved_to_echo=32719 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[503, 513, 508, 508, 527, 505, 512, 574, 519, 512, 491, 508, 563, 505, 483, 490, 14, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 526, 499, 513, 506, 534, 493, 539, 511, 527, 557, 516, 509, 574, 514, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 502, 484, 505, 466, 544, 538, 521, 505, 542, 511, 492, 507, 478, 470, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 504, 523, 520, 557, 476, 519, 523, 497, 542, 505, 489, 519, 511, 508, 520, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 506, 540, 495, 517, 495, 513, 544, 497, 499, 508, 525, 508, 481, 506, 526, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 508, 488, 502, 507, 521, 504, 522, 510, 527, 458, 524, 501, 516, 517, 517, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 497, 532, 547, 512, 495, 473, 554, 476, 490, 532, 510, 448, 549, 548, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 529, 496, 517, 506, 560, 456, 484, 522, 484, 499, 515, 533, 488, 500, 496, 42, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8326, 8093, 8230, 8178, 8106, 8236, 8129] avg=8192.0000 max=8326 max_rank=1 min=8093 min_rank=2 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32649 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[513, 496, 507, 526, 535, 496, 502, 482, 520, 522, 514, 510, 535, 509, 528, 527, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 515, 453, 515, 466, 542, 481, 528, 488, 526, 510, 493, 528, 551, 521, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 519, 547, 519, 514, 527, 511, 524, 495, 504, 518, 521, 545, 529, 484, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 525, 519, 498, 518, 525, 498, 498, 477, 533, 502, 490, 506, 497, 510, 506, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 526, 512, 519, 477, 484, 530, 523, 480, 523, 517, 471, 481, 537, 517, 534, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 538, 536, 523, 508, 490, 489, 503, 506, 494, 504, 518, 482, 507, 516, 522, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 534, 478, 503, 545, 523, 528, 522, 554, 530, 523, 506, 467, 491, 559, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 475, 497, 515, 510, 511, 496, 511, 525, 500, 515, 492, 522, 499, 536, 511, 38, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8234, 8121, 8267, 8122, 8145, 8160, 8322, 8165] avg=8192.0000 max=8322 max_rank=6 min=8121 min_rank=1 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32792 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[487, 508, 470, 530, 498, 459, 554, 486, 552, 508, 493, 500, 462, 546, 542, 495, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 497, 546, 536, 479, 530, 527, 530, 498, 493, 508, 517, 538, 496, 503, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 494, 500, 537, 542, 521, 529, 513, 501, 505, 514, 527, 529, 515, 480, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 514, 539, 513, 486, 555, 526, 479, 510, 503, 469, 539, 506, 518, 503, 531, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 511, 541, 490, 497, 489, 481, 513, 549, 509, 477, 508, 518, 468, 522, 482, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 548, 494, 536, 505, 520, 526, 523, 516, 519, 537, 500, 515, 544, 530, 461, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 526, 495, 515, 507, 533, 485, 496, 494, 496, 557, 539, 511, 517, 523, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 537, 520, 468, 534, 504, 529, 520, 496, 480, 476, 476, 513, 523, 487, 522, 37, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8101, 8217, 8249, 8212, 8080, 8322, 8216, 8139] avg=8192.0000 max=8322 max_rank=5 min=8080 min_rank=4 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32757 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[513, 496, 507, 526, 535, 496, 502, 482, 520, 522, 514, 510, 535, 509, 528, 527, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 515, 453, 515, 466, 542, 481, 528, 488, 526, 510, 493, 528, 551, 521, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 519, 547, 519, 514, 527, 511, 524, 495, 504, 518, 521, 545, 529, 484, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 525, 519, 498, 518, 525, 498, 498, 477, 533, 502, 490, 506, 497, 510, 506, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 526, 512, 519, 477, 484, 530, 523, 480, 523, 517, 471, 481, 537, 517, 534, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 538, 536, 523, 508, 490, 489, 503, 506, 494, 504, 518, 482, 507, 516, 522, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 534, 478, 503, 545, 523, 528, 522, 554, 530, 523, 506, 467, 491, 559, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 475, 497, 515, 510, 511, 496, 511, 525, 500, 515, 492, 522, 499, 536, 511, 38, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8234, 8121, 8267, 8122, 8145, 8160, 8322, 8165] avg=8192.0000 max=8322 max_rank=6 min=8121 min_rank=1 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32792 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[503, 513, 508, 508, 527, 505, 512, 574, 519, 512, 491, 508, 563, 505, 483, 490, 14, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 526, 499, 513, 506, 534, 493, 539, 511, 527, 557, 516, 509, 574, 514, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 502, 484, 505, 466, 544, 538, 521, 505, 542, 511, 492, 507, 478, 470, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 504, 523, 520, 557, 476, 519, 523, 497, 542, 505, 489, 519, 511, 508, 520, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 506, 540, 495, 517, 495, 513, 544, 497, 499, 508, 525, 508, 481, 506, 526, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 508, 488, 502, 507, 521, 504, 522, 510, 527, 458, 524, 501, 516, 517, 517, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 497, 532, 547, 512, 495, 473, 554, 476, 490, 532, 510, 448, 549, 548, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 529, 496, 517, 506, 560, 456, 484, 522, 484, 499, 515, 533, 488, 500, 496, 42, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8326, 8093, 8230, 8178, 8106, 8236, 8129] avg=8192.0000 max=8326 max_rank=1 min=8093 min_rank=2 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32649 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[489, 506, 523, 543, 546, 518, 510, 502, 509, 511, 537, 528, 433, 509, 539, 516, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 482, 512, 526, 494, 461, 491, 505, 541, 515, 518, 520, 525, 541, 547, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 516, 506, 482, 525, 497, 510, 528, 536, 519, 536, 514, 517, 483, 487, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 547, 518, 528, 528, 504, 501, 488, 484, 495, 493, 489, 497, 485, 485, 532, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 518, 514, 503, 509, 509, 517, 531, 472, 556, 523, 507, 528, 478, 554, 477, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 495, 477, 469, 514, 547, 486, 545, 524, 534, 528, 492, 500, 503, 552, 529, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 513, 483, 510, 486, 545, 522, 543, 517, 482, 501, 550, 476, 485, 514, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 527, 503, 513, 520, 508, 542, 547, 519, 455, 488, 493, 495, 526, 484, 489, 38, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8231, 8231, 8201, 8150, 8206, 8193, 8172, 8152] avg=8192.0000 max=8231 max_rank=0 min=8150 min_rank=3 imbalance_ratio=1.004761 max_load_reduction=57305 max_load_reduction_pct=0.874405 moved_to_echo=32723 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[482, 531, 470, 509, 498, 513, 520, 527, 482, 476, 496, 504, 471, 514, 493, 528, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 510, 516, 511, 540, 538, 499, 547, 470, 523, 517, 509, 530, 507, 517, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 502, 465, 533, 497, 489, 532, 519, 509, 526, 522, 512, 497, 532, 524, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 528, 539, 529, 474, 486, 519, 541, 457, 559, 533, 522, 513, 552, 529, 532, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 546, 565, 508, 503, 502, 538, 509, 485, 514, 515, 487, 478, 515, 493, 550, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 525, 558, 500, 471, 484, 570, 543, 490, 484, 513, 492, 502, 505, 482, 520, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 470, 553, 490, 496, 538, 538, 462, 549, 529, 510, 491, 456, 501, 504, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 526, 520, 496, 490, 543, 496, 488, 518, 448, 542, 469, 498, 500, 527, 553, 41, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8026, 8286, 8142, 8363, 8262, 8162, 8130, 8165] avg=8192.0000 max=8363 max_rank=3 min=8026 min_rank=0 imbalance_ratio=1.020874 max_load_reduction=57173 max_load_reduction_pct=0.872391 moved_to_echo=32719 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[479, 514, 510, 533, 525, 497, 512, 513, 539, 509, 473, 532, 548, 488, 520, 531, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 533, 466, 511, 523, 501, 522, 491, 544, 491, 486, 537, 509, 475, 514, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 510, 517, 500, 535, 527, 480, 486, 493, 515, 530, 552, 490, 516, 480, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 466, 549, 542, 513, 493, 494, 499, 516, 539, 515, 537, 476, 498, 509, 478, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 549, 508, 472, 532, 513, 510, 517, 507, 510, 472, 535, 514, 491, 527, 488, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 516, 513, 558, 491, 531, 493, 523, 525, 486, 528, 501, 525, 486, 443, 503, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 502, 468, 525, 488, 515, 535, 531, 543, 494, 513, 513, 518, 497, 514, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 521, 516, 529, 494, 544, 487, 550, 516, 507, 510, 583, 555, 493, 520, 501, 38, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8234, 8092, 8145, 8182, 8173, 8172, 8172, 8366] avg=8192.0000 max=8366 max_rank=7 min=8092 min_rank=1 imbalance_ratio=1.021240 max_load_reduction=57170 max_load_reduction_pct=0.872345 moved_to_echo=32883 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[530, 518, 501, 519, 525, 527, 550, 520, 497, 498, 508, 503, 513, 514, 535, 524, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 516, 502, 564, 508, 565, 516, 524, 520, 521, 493, 515, 467, 511, 532, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 505, 498, 544, 546, 565, 532, 508, 495, 496, 501, 476, 531, 495, 464, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 487, 507, 514, 472, 481, 544, 496, 509, 496, 506, 533, 503, 535, 543, 486, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 516, 521, 481, 494, 531, 511, 539, 507, 497, 546, 491, 521, 512, 549, 497, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 535, 528, 485, 497, 507, 517, 529, 505, 484, 499, 537, 504, 546, 480, 527, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 560, 503, 502, 468, 520, 518, 505, 538, 523, 489, 512, 511, 454, 498, 481, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 505, 514, 491, 529, 504, 492, 539, 533, 506, 482, 505, 481, 497, 515, 568, 36, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8293, 8209, 8156, 8161, 8216, 8168, 8124, 8209] avg=8192.0000 max=8293 max_rank=0 min=8124 min_rank=6 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32717 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[500, 534, 499, 482, 496, 524, 542, 504, 497, 476, 521, 534, 558, 529, 492, 480, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 493, 529, 481, 521, 482, 542, 479, 479, 523, 510, 526, 503, 454, 499, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 508, 531, 510, 517, 531, 551, 488, 534, 531, 509, 507, 534, 515, 475, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 498, 505, 541, 520, 516, 508, 514, 532, 506, 493, 474, 490, 511, 487, 477, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 513, 508, 503, 514, 526, 538, 495, 502, 511, 502, 527, 527, 523, 543, 520, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 515, 540, 534, 528, 525, 516, 528, 510, 544, 468, 515, 538, 495, 528, 487, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 492, 539, 518, 506, 540, 514, 481, 524, 502, 510, 528, 491, 507, 538, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 475, 487, 472, 495, 548, 567, 519, 517, 528, 530, 470, 528, 557, 473, 522, 37, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8054, 8283, 8116, 8276, 8253, 8164, 8211] avg=8192.0000 max=8283 max_rank=2 min=8054 min_rank=1 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32904 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 3968, 4096, 4096, 4224, 4096, 4096, 4224, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 3840, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[3968, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 3840, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[542, 496, 548, 511, 511, 490, 523, 526, 535, 509, 505, 533, 497, 504, 513, 521, 491, 519, 555, 507, 488, 510, 525, 503, 525, 517, 521, 533, 499, 487, 517, 485, 495, 523, 522, 510, 547, 524, 535, 519, 529, 521, 538, 474, 490, 536, 508, 512, 471, 519, 467, 487, 506, 507, 482, 527, 501, 486, 518, 504, 491, 478, 503, 476, 505, 483, 531, 525, 533, 512, 519, 487, 526, 496, 481, 497, 545, 525, 481, 498, 544, 531, 515, 507, 545, 530, 516, 512, 508, 503, 523, 491, 498, 482, 516, 487, 502, 528, 496, 492, 547, 504, 510, 520, 501, 543, 478, 511, 506, 540, 486, 546, 547, 506, 513, 550, 496, 532, 518, 538, 529, 495, 534, 510, 491, 507, 547, 509] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[524, 524, 515, 509, 519, 511, 507, 510, 477, 510, 508, 516, 470, 525, 529, 527, 496, 522, 483, 494, 533, 524, 489, 551, 478, 540, 474, 465, 518, 507, 524, 518, 507, 525, 510, 526, 488, 513, 530, 480, 526, 535, 492, 524, 490, 513, 486, 509, 543, 492, 491, 528, 482, 478, 530, 512, 515, 519, 486, 521, 527, 512, 520, 512, 506, 550, 501, 506, 502, 507, 489, 531, 509, 511, 501, 492, 538, 521, 510, 472, 542, 489, 525, 518, 483, 537, 518, 500, 496, 545, 489, 526, 532, 507, 517, 521, 493, 493, 493, 534, 523, 552, 490, 525, 535, 560, 479, 505, 505, 544, 562, 544, 497, 565, 536, 530, 501, 525, 509, 489, 540, 509, 533, 503, 474, 505, 472, 501] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[482, 467, 514, 506, 548, 472, 519, 565, 497, 494, 512, 518, 492, 532, 529, 511, 488, 491, 505, 509, 476, 514, 524, 499, 460, 511, 506, 491, 511, 497, 514, 540, 550, 519, 527, 548, 517, 525, 525, 495, 496, 530, 572, 548, 494, 486, 554, 510, 483, 537, 509, 524, 493, 536, 542, 511, 487, 513, 517, 487, 534, 516, 496, 515, 544, 522, 518, 516, 522, 492, 501, 511, 514, 494, 501, 497, 490, 516, 526, 519, 520, 509, 500, 501, 550, 481, 535, 498, 508, 504, 577, 519, 495, 498, 535, 499, 513, 509, 491, 500, 526, 525, 500, 507, 535, 493, 509, 502, 491, 532, 499, 552, 478, 509, 514, 512, 517, 485, 510, 509, 575, 546, 513, 496, 477, 504, 502, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[498, 487, 490, 523, 519, 524, 519, 526, 528, 529, 521, 531, 502, 500, 521, 490, 511, 524, 517, 463, 540, 504, 530, 504, 551, 517, 511, 500, 489, 475, 510, 492, 507, 501, 498, 516, 498, 490, 498, 479, 497, 549, 545, 487, 467, 570, 526, 502, 516, 532, 542, 516, 562, 497, 563, 479, 510, 547, 547, 532, 505, 515, 507, 543, 565, 523, 519, 496, 505, 523, 493, 480, 509, 520, 542, 515, 526, 520, 539, 517, 510, 514, 516, 514, 522, 520, 553, 505, 475, 501, 520, 493, 498, 482, 483, 521, 523, 475, 502, 504, 492, 510, 531, 538, 500, 501, 518, 496, 535, 500, 484, 497, 492, 496, 516, 476, 517, 489, 467, 512, 518, 523, 529, 565, 498, 488, 512, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[473, 488, 517, 559, 532, 541, 492, 542, 523, 502, 505, 465, 525, 513, 531, 502, 551, 491, 516, 483, 534, 532, 522, 502, 500, 488, 507, 499, 515, 508, 476, 495, 524, 510, 508, 485, 487, 482, 548, 520, 561, 532, 470, 466, 505, 509, 542, 502, 535, 457, 520, 547, 487, 533, 495, 515, 554, 499, 494, 501, 511, 555, 520, 503, 502, 511, 495, 505, 503, 531, 490, 524, 470, 489, 513, 489, 469, 538, 491, 546, 470, 515, 541, 558, 493, 516, 530, 473, 541, 485, 516, 537, 549, 495, 514, 536, 533, 540, 503, 509, 543, 503, 496, 510, 477, 525, 531, 527, 464, 505, 532, 495, 503, 547, 477, 519, 543, 521, 550, 485, 483, 536, 515, 527, 558, 529, 520, 489] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[524, 524, 515, 509, 519, 511, 507, 510, 477, 510, 508, 516, 470, 525, 529, 527, 496, 522, 483, 494, 533, 524, 489, 551, 478, 540, 474, 465, 518, 507, 524, 518, 507, 525, 510, 526, 488, 513, 530, 480, 526, 535, 492, 524, 490, 513, 486, 509, 543, 492, 491, 528, 482, 478, 530, 512, 515, 519, 486, 521, 527, 512, 520, 512, 506, 550, 501, 506, 502, 507, 489, 531, 509, 511, 501, 492, 538, 521, 510, 472, 542, 489, 525, 518, 483, 537, 518, 500, 496, 545, 489, 526, 532, 507, 517, 521, 493, 493, 493, 534, 523, 552, 490, 525, 535, 560, 479, 505, 505, 544, 562, 544, 497, 565, 536, 530, 501, 525, 509, 489, 540, 509, 533, 503, 474, 505, 472, 501] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[518, 475, 532, 473, 496, 515, 513, 537, 522, 460, 495, 487, 526, 542, 516, 536, 522, 515, 520, 528, 502, 519, 514, 492, 506, 519, 504, 512, 518, 493, 531, 481, 534, 480, 504, 509, 524, 514, 492, 503, 505, 516, 507, 523, 517, 485, 578, 533, 499, 498, 512, 567, 496, 525, 531, 521, 482, 538, 508, 483, 501, 485, 503, 529, 534, 476, 498, 502, 511, 513, 484, 511, 503, 517, 509, 539, 523, 520, 503, 506, 554, 478, 532, 473, 493, 505, 517, 475, 494, 495, 541, 519, 508, 525, 474, 518, 488, 548, 538, 551, 484, 485, 525, 523, 552, 553, 513, 515, 531, 494, 514, 500, 537, 509, 519, 516, 514, 520, 512, 528, 514, 510, 508, 529, 481, 477, 550, 527] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[542, 496, 548, 511, 511, 490, 523, 526, 535, 509, 505, 533, 497, 504, 513, 521, 491, 519, 555, 507, 488, 510, 525, 503, 525, 517, 521, 533, 499, 487, 517, 485, 495, 523, 522, 510, 547, 524, 535, 519, 529, 521, 538, 474, 490, 536, 508, 512, 471, 519, 467, 487, 506, 507, 482, 527, 501, 486, 518, 504, 491, 478, 503, 476, 505, 483, 531, 525, 533, 512, 519, 487, 526, 496, 481, 497, 545, 525, 481, 498, 544, 531, 515, 507, 545, 530, 516, 512, 508, 503, 523, 491, 498, 482, 516, 487, 502, 528, 496, 492, 547, 504, 510, 520, 501, 543, 478, 511, 506, 540, 486, 546, 547, 506, 513, 550, 496, 532, 518, 538, 529, 495, 534, 510, 491, 507, 547, 509] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[516, 502, 474, 540, 508, 521, 551, 536, 495, 483, 472, 511, 555, 514, 491, 494, 478, 520, 541, 526, 509, 515, 498, 510, 515, 508, 542, 521, 478, 501, 479, 519, 503, 511, 465, 507, 517, 528, 514, 541, 557, 505, 518, 508, 505, 478, 513, 508, 519, 519, 519, 523, 525, 495, 547, 510, 488, 509, 514, 518, 510, 474, 502, 542, 506, 517, 531, 513, 512, 493, 509, 480, 510, 532, 512, 510, 527, 483, 522, 512, 500, 488, 520, 482, 489, 496, 500, 540, 525, 491, 496, 510, 511, 544, 500, 511, 494, 520, 524, 540, 493, 535, 521, 497, 534, 560, 532, 508, 517, 539, 524, 495, 488, 489, 505, 529, 527, 513, 491, 507, 528, 540, 507, 519, 522, 490, 512, 549] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[473, 488, 517, 559, 532, 541, 492, 542, 523, 502, 505, 465, 525, 513, 531, 502, 551, 491, 516, 483, 534, 532, 522, 502, 500, 488, 507, 499, 515, 508, 476, 495, 524, 510, 508, 485, 487, 482, 548, 520, 561, 532, 470, 466, 505, 509, 542, 502, 535, 457, 520, 547, 487, 533, 495, 515, 554, 499, 494, 501, 511, 555, 520, 503, 502, 511, 495, 505, 503, 531, 490, 524, 470, 489, 513, 489, 469, 538, 491, 546, 470, 515, 541, 558, 493, 516, 530, 473, 541, 485, 516, 537, 549, 495, 514, 536, 533, 540, 503, 509, 543, 503, 496, 510, 477, 525, 531, 527, 464, 505, 532, 495, 503, 547, 477, 519, 543, 521, 550, 485, 483, 536, 515, 527, 558, 529, 520, 489] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[505, 507, 517, 548, 497, 532, 519, 501, 507, 482, 500, 495, 534, 532, 509, 486, 461, 518, 483, 501, 530, 568, 524, 501, 510, 516, 524, 527, 506, 524, 491, 498, 521, 559, 495, 562, 536, 507, 490, 502, 515, 477, 529, 497, 544, 516, 513, 474, 492, 523, 521, 487, 511, 545, 546, 518, 500, 539, 502, 515, 521, 502, 490, 543, 518, 489, 531, 488, 493, 517, 503, 542, 522, 506, 526, 454, 535, 525, 484, 542, 532, 528, 485, 531, 511, 487, 510, 465, 531, 503, 541, 538, 541, 498, 496, 501, 488, 531, 550, 487, 557, 533, 486, 528, 543, 501, 510, 504, 475, 483, 517, 488, 476, 525, 529, 498, 505, 488, 462, 505, 506, 476, 519, 527, 497, 560, 532, 532] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[518, 475, 532, 473, 496, 515, 513, 537, 522, 460, 495, 487, 526, 542, 516, 536, 522, 515, 520, 528, 502, 519, 514, 492, 506, 519, 504, 512, 518, 493, 531, 481, 534, 480, 504, 509, 524, 514, 492, 503, 505, 516, 507, 523, 517, 485, 578, 533, 499, 498, 512, 567, 496, 525, 531, 521, 482, 538, 508, 483, 501, 485, 503, 529, 534, 476, 498, 502, 511, 513, 484, 511, 503, 517, 509, 539, 523, 520, 503, 506, 554, 478, 532, 473, 493, 505, 517, 475, 494, 495, 541, 519, 508, 525, 474, 518, 488, 548, 538, 551, 484, 485, 525, 523, 552, 553, 513, 515, 531, 494, 514, 500, 537, 509, 519, 516, 514, 520, 512, 528, 514, 510, 508, 529, 481, 477, 550, 527] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[482, 467, 514, 506, 548, 472, 519, 565, 497, 494, 512, 518, 492, 532, 529, 511, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 491, 505, 509, 476, 514, 524, 499, 460, 511, 506, 491, 511, 497, 514, 540, 37, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 519, 527, 548, 517, 525, 525, 495, 496, 530, 572, 548, 494, 486, 527, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 537, 509, 524, 493, 536, 538, 511, 487, 513, 517, 487, 534, 516, 496, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 522, 518, 516, 522, 492, 501, 511, 514, 494, 501, 497, 490, 516, 526, 519, 14, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 509, 500, 501, 550, 481, 535, 498, 508, 504, 577, 519, 495, 498, 535, 499, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 509, 491, 500, 526, 525, 500, 507, 535, 456, 509, 502, 491, 532, 499, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 509, 514, 512, 517, 485, 510, 509, 555, 546, 513, 496, 477, 504, 502, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8162, 8086, 8369, 8196, 8210, 8236, 8147, 8130] avg=8192.0000 max=8369 max_rank=2 min=8086 min_rank=1 imbalance_ratio=1.021606 max_load_reduction=57167 max_load_reduction_pct=0.872299 moved_to_echo=32723 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[516, 502, 474, 540, 508, 521, 551, 536, 495, 483, 472, 511, 555, 514, 491, 494, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 520, 541, 526, 509, 515, 498, 510, 515, 508, 542, 521, 478, 501, 479, 519, 42, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 511, 465, 507, 517, 528, 514, 541, 557, 505, 518, 508, 505, 478, 488, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 519, 519, 523, 525, 495, 543, 510, 488, 509, 514, 518, 510, 474, 502, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 517, 531, 513, 512, 493, 509, 480, 510, 532, 512, 510, 527, 483, 522, 512, 13, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 488, 520, 482, 489, 496, 500, 540, 525, 491, 496, 510, 511, 544, 500, 511, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 520, 524, 540, 493, 535, 521, 497, 534, 518, 532, 508, 517, 539, 524, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 489, 505, 529, 527, 513, 491, 507, 510, 540, 507, 519, 522, 490, 512, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8214, 8153, 8210, 8194, 8109, 8291, 8198] avg=8192.0000 max=8291 max_rank=6 min=8109 min_rank=5 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32792 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[542, 496, 548, 511, 511, 490, 523, 526, 535, 509, 505, 533, 497, 504, 513, 521, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 519, 555, 507, 488, 510, 525, 503, 525, 517, 521, 533, 499, 487, 517, 485, 41, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 523, 522, 510, 547, 524, 535, 519, 529, 521, 538, 474, 490, 536, 483, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 519, 467, 487, 506, 507, 478, 527, 501, 486, 518, 504, 491, 478, 503, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 483, 531, 525, 533, 512, 519, 487, 526, 496, 481, 497, 545, 525, 481, 498, 13, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 531, 515, 507, 545, 530, 516, 512, 508, 503, 523, 491, 498, 482, 516, 487, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 528, 496, 492, 547, 504, 510, 520, 501, 502, 478, 511, 506, 540, 486, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 506, 513, 550, 496, 532, 518, 538, 511, 495, 534, 510, 491, 507, 547, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8268, 8235, 8258, 7919, 8169, 8214, 8169, 8304] avg=8192.0000 max=8304 max_rank=7 min=7919 min_rank=3 imbalance_ratio=1.013672 max_load_reduction=57232 max_load_reduction_pct=0.873291 moved_to_echo=32856 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[524, 524, 515, 509, 519, 511, 507, 510, 477, 510, 508, 516, 470, 525, 529, 527, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 522, 483, 494, 533, 524, 489, 551, 478, 540, 474, 465, 518, 507, 524, 518, 45, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 525, 510, 526, 488, 513, 530, 480, 526, 535, 492, 524, 490, 513, 453, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 492, 491, 528, 482, 478, 518, 512, 515, 519, 486, 521, 527, 512, 520, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 550, 501, 506, 502, 507, 489, 531, 509, 511, 501, 492, 538, 521, 510, 472, 17, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 489, 525, 518, 483, 537, 518, 500, 496, 545, 489, 526, 532, 507, 517, 521, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 493, 493, 534, 523, 552, 490, 525, 535, 515, 479, 505, 505, 544, 562, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 565, 536, 530, 501, 525, 509, 489, 514, 509, 533, 503, 474, 505, 472, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8177, 8121, 8156, 8179, 8258, 8292, 8163] avg=8192.0000 max=8292 max_rank=6 min=8121 min_rank=2 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32892 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[473, 488, 517, 559, 532, 541, 492, 542, 523, 502, 505, 465, 525, 513, 531, 502, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 491, 516, 483, 534, 532, 522, 502, 500, 488, 507, 499, 515, 508, 476, 495, 39, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 510, 508, 485, 487, 482, 548, 520, 561, 532, 470, 466, 505, 509, 515, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 457, 520, 547, 487, 533, 491, 515, 554, 499, 494, 501, 511, 555, 520, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 511, 495, 505, 503, 531, 490, 524, 470, 489, 513, 489, 469, 538, 491, 546, 14, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 515, 541, 558, 493, 516, 530, 473, 541, 485, 516, 537, 549, 495, 514, 536, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 540, 503, 509, 543, 503, 496, 510, 477, 486, 531, 527, 464, 505, 532, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 547, 477, 519, 543, 521, 550, 485, 466, 536, 515, 527, 558, 529, 520, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8214, 8171, 8124, 8222, 8091, 8275, 8154, 8285] avg=8192.0000 max=8285 max_rank=7 min=8091 min_rank=4 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32805 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[542, 496, 548, 511, 511, 490, 523, 526, 535, 509, 505, 533, 497, 504, 513, 521, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 519, 555, 507, 488, 510, 525, 503, 525, 517, 521, 533, 499, 487, 517, 485, 41, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 523, 522, 510, 547, 524, 535, 519, 529, 521, 538, 474, 490, 536, 483, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 519, 467, 487, 506, 507, 478, 527, 501, 486, 518, 504, 491, 478, 503, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 483, 531, 525, 533, 512, 519, 487, 526, 496, 481, 497, 545, 525, 481, 498, 13, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 531, 515, 507, 545, 530, 516, 512, 508, 503, 523, 491, 498, 482, 516, 487, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 528, 496, 492, 547, 504, 510, 520, 501, 502, 478, 511, 506, 540, 486, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 506, 513, 550, 496, 532, 518, 538, 511, 495, 534, 510, 491, 507, 547, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8268, 8235, 8258, 7919, 8169, 8214, 8169, 8304] avg=8192.0000 max=8304 max_rank=7 min=7919 min_rank=3 imbalance_ratio=1.013672 max_load_reduction=57232 max_load_reduction_pct=0.873291 moved_to_echo=32856 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[498, 487, 490, 523, 519, 524, 519, 526, 528, 529, 521, 531, 502, 500, 521, 490, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 524, 517, 463, 540, 504, 530, 504, 551, 517, 511, 500, 489, 475, 510, 492, 38, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 501, 498, 516, 498, 490, 498, 479, 497, 549, 545, 487, 467, 570, 500, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 532, 542, 516, 562, 497, 558, 479, 510, 547, 547, 532, 505, 515, 507, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 565, 523, 519, 496, 505, 523, 493, 480, 509, 520, 542, 515, 526, 520, 539, 517, 13, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 514, 516, 514, 522, 520, 553, 505, 475, 501, 520, 493, 498, 482, 483, 521, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 475, 502, 504, 492, 510, 531, 538, 500, 463, 518, 496, 535, 500, 484, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 496, 516, 476, 517, 489, 467, 512, 500, 523, 529, 565, 498, 488, 512, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8189, 8104, 8408, 8317, 8133, 8068, 8104] avg=8192.0000 max=8408 max_rank=3 min=8068 min_rank=6 imbalance_ratio=1.026367 max_load_reduction=57128 max_load_reduction_pct=0.871704 moved_to_echo=32622 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[516, 502, 474, 540, 508, 521, 551, 536, 495, 483, 472, 511, 555, 514, 491, 494, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 520, 541, 526, 509, 515, 498, 510, 515, 508, 542, 521, 478, 501, 479, 519, 42, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 511, 465, 507, 517, 528, 514, 541, 557, 505, 518, 508, 505, 478, 488, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 519, 519, 523, 525, 495, 543, 510, 488, 509, 514, 518, 510, 474, 502, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 517, 531, 513, 512, 493, 509, 480, 510, 532, 512, 510, 527, 483, 522, 512, 13, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 488, 520, 482, 489, 496, 500, 540, 525, 491, 496, 510, 511, 544, 500, 511, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 520, 524, 540, 493, 535, 521, 497, 534, 518, 532, 508, 517, 539, 524, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 489, 505, 529, 527, 513, 491, 507, 510, 540, 507, 519, 522, 490, 512, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8214, 8153, 8210, 8194, 8109, 8291, 8198] avg=8192.0000 max=8291 max_rank=6 min=8109 min_rank=5 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32792 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[518, 475, 532, 473, 496, 515, 513, 537, 522, 460, 495, 487, 526, 542, 516, 536, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 515, 520, 528, 502, 519, 514, 492, 506, 519, 504, 512, 518, 493, 531, 481, 42, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 480, 504, 509, 524, 514, 492, 503, 505, 516, 507, 523, 517, 485, 549, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 498, 512, 567, 496, 525, 527, 521, 482, 538, 508, 483, 501, 485, 503, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 476, 498, 502, 511, 513, 484, 511, 503, 517, 509, 539, 523, 520, 503, 506, 15, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 478, 532, 473, 493, 505, 517, 475, 494, 495, 541, 519, 508, 525, 474, 518, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 548, 538, 551, 484, 485, 525, 523, 552, 511, 513, 515, 531, 494, 514, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 509, 519, 516, 514, 520, 512, 528, 496, 510, 508, 529, 481, 477, 550, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8147, 8232, 8195, 8174, 8176, 8107, 8272, 8233] avg=8192.0000 max=8272 max_rank=6 min=8107 min_rank=5 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32788 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[505, 507, 517, 548, 497, 532, 519, 501, 507, 482, 500, 495, 534, 532, 509, 486, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 518, 483, 501, 530, 568, 524, 501, 510, 516, 524, 527, 506, 524, 491, 498, 38, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 559, 495, 562, 536, 507, 490, 502, 515, 477, 529, 497, 544, 516, 488, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 523, 521, 487, 511, 545, 542, 518, 500, 539, 502, 515, 521, 502, 490, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 489, 531, 488, 493, 517, 503, 542, 522, 506, 526, 454, 535, 525, 484, 542, 13, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 528, 485, 531, 511, 487, 510, 465, 531, 503, 541, 538, 541, 498, 496, 501, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 531, 550, 487, 557, 533, 486, 528, 543, 463, 510, 504, 475, 483, 517, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 525, 529, 498, 505, 488, 462, 505, 488, 476, 519, 527, 497, 560, 532, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8232, 8212, 8251, 8200, 8204, 8143, 8119] avg=8192.0000 max=8251 max_rank=3 min=8119 min_rank=7 imbalance_ratio=1.007202 max_load_reduction=57285 max_load_reduction_pct=0.874100 moved_to_echo=32666 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[473, 488, 517, 559, 532, 541, 492, 542, 523, 502, 505, 465, 525, 513, 531, 502, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 491, 516, 483, 534, 532, 522, 502, 500, 488, 507, 499, 515, 508, 476, 495, 39, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 510, 508, 485, 487, 482, 548, 520, 561, 532, 470, 466, 505, 509, 515, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 457, 520, 547, 487, 533, 491, 515, 554, 499, 494, 501, 511, 555, 520, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 511, 495, 505, 503, 531, 490, 524, 470, 489, 513, 489, 469, 538, 491, 546, 14, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 515, 541, 558, 493, 516, 530, 473, 541, 485, 516, 537, 549, 495, 514, 536, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 540, 503, 509, 543, 503, 496, 510, 477, 486, 531, 527, 464, 505, 532, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 547, 477, 519, 543, 521, 550, 485, 466, 536, 515, 527, 558, 529, 520, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8214, 8171, 8124, 8222, 8091, 8275, 8154, 8285] avg=8192.0000 max=8285 max_rank=7 min=8091 min_rank=4 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32805 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 3968, 4224, 4224, 4224, 4224, 4224, 4352, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 3968, 4096, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 3968, 4224, 4224, 4224, 4224, 4224, 4352, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 3968, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[511, 493, 475, 529, 508, 514, 531, 520, 521, 525, 507, 521, 578, 532, 544, 463, 596, 498, 484, 527, 514, 481, 558, 498, 508, 531, 499, 509, 540, 529, 506, 551, 508, 474, 525, 493, 498, 508, 502, 545, 523, 464, 488, 538, 471, 512, 514, 511, 493, 510, 504, 512, 532, 517, 509, 490, 548, 529, 526, 518, 544, 492, 469, 502, 581, 505, 507, 486, 532, 515, 524, 534, 528, 485, 543, 520, 495, 494, 504, 501, 529, 502, 505, 522, 496, 530, 506, 481, 523, 526, 498, 484, 506, 484, 511, 490, 500, 527, 517, 499, 542, 512, 530, 485, 501, 538, 521, 508, 503, 528, 489, 507, 501, 512, 487, 515, 494, 523, 509, 501, 497, 500, 481, 534, 524, 500, 533, 501] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[536, 540, 545, 515, 466, 535, 484, 498, 524, 509, 510, 530, 521, 521, 522, 545, 526, 520, 444, 474, 494, 501, 543, 503, 553, 551, 504, 526, 489, 527, 530, 505, 518, 543, 502, 502, 508, 499, 501, 548, 524, 507, 489, 523, 548, 502, 495, 475, 502, 528, 499, 523, 513, 484, 513, 543, 507, 526, 510, 536, 506, 493, 484, 507, 518, 479, 521, 495, 530, 490, 501, 500, 544, 508, 505, 487, 567, 518, 518, 483, 530, 531, 533, 502, 542, 523, 493, 466, 519, 496, 486, 509, 534, 487, 518, 558, 461, 498, 521, 503, 509, 490, 501, 532, 542, 517, 522, 505, 507, 508, 514, 486, 551, 515, 556, 500, 529, 464, 488, 487, 520, 543, 519, 485, 516, 495, 535, 477] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[525, 520, 512, 484, 515, 499, 476, 521, 541, 528, 553, 507, 486, 482, 510, 529, 526, 506, 517, 511, 476, 519, 489, 518, 538, 537, 519, 482, 474, 500, 511, 507, 482, 518, 512, 516, 530, 513, 524, 513, 479, 517, 486, 535, 515, 518, 542, 491, 481, 511, 507, 538, 550, 515, 529, 526, 470, 506, 526, 534, 513, 512, 505, 530, 547, 538, 537, 511, 506, 515, 512, 523, 527, 495, 523, 519, 503, 510, 515, 507, 519, 504, 492, 492, 500, 539, 517, 524, 534, 546, 527, 505, 504, 462, 519, 511, 553, 497, 518, 479, 523, 534, 555, 487, 492, 506, 497, 492, 488, 521, 478, 538, 523, 484, 449, 529, 518, 529, 529, 501, 480, 472, 556, 493, 509, 521, 538, 502] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[551, 503, 500, 522, 518, 517, 508, 517, 497, 490, 453, 491, 507, 479, 533, 531, 510, 508, 503, 507, 504, 499, 515, 497, 511, 511, 510, 525, 485, 517, 568, 513, 546, 519, 515, 492, 511, 488, 487, 532, 525, 514, 539, 519, 528, 535, 487, 543, 535, 530, 498, 507, 496, 523, 544, 503, 528, 528, 476, 501, 537, 466, 497, 486, 521, 499, 484, 501, 495, 507, 520, 486, 542, 514, 509, 496, 508, 501, 493, 521, 487, 507, 515, 540, 516, 515, 519, 499, 504, 511, 551, 519, 505, 495, 509, 535, 539, 494, 559, 520, 493, 484, 527, 516, 484, 529, 492, 533, 516, 493, 518, 488, 542, 509, 509, 507, 549, 508, 486, 523, 509, 493, 494, 513, 576, 518, 535, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[500, 502, 525, 518, 511, 519, 500, 553, 528, 512, 489, 524, 552, 527, 505, 507, 491, 517, 537, 547, 511, 497, 511, 542, 522, 531, 507, 524, 455, 534, 476, 494, 490, 498, 522, 505, 524, 533, 493, 534, 523, 515, 504, 516, 539, 497, 515, 500, 490, 509, 511, 562, 507, 502, 513, 494, 520, 526, 506, 479, 476, 511, 508, 495, 473, 504, 545, 523, 508, 537, 551, 499, 507, 498, 543, 509, 504, 495, 529, 487, 477, 510, 495, 517, 526, 524, 542, 528, 498, 471, 492, 550, 491, 493, 523, 498, 481, 532, 528, 514, 532, 505, 535, 527, 538, 555, 474, 540, 520, 535, 519, 530, 481, 457, 530, 492, 521, 502, 519, 497, 522, 508, 493, 468, 510, 490, 515, 534] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[516, 474, 513, 479, 525, 515, 520, 504, 475, 533, 512, 521, 500, 497, 540, 499, 538, 563, 498, 496, 495, 537, 507, 544, 505, 477, 496, 513, 528, 530, 511, 469, 494, 498, 514, 494, 501, 527, 494, 481, 536, 517, 552, 506, 512, 487, 521, 476, 504, 505, 522, 517, 523, 504, 520, 506, 540, 491, 482, 524, 536, 510, 504, 558, 507, 514, 527, 516, 478, 520, 526, 493, 548, 559, 521, 528, 478, 542, 522, 538, 536, 462, 533, 527, 511, 501, 510, 479, 527, 554, 497, 473, 477, 523, 494, 515, 512, 523, 514, 542, 488, 550, 532, 526, 494, 517, 490, 512, 517, 526, 506, 506, 494, 521, 501, 543, 521, 569, 487, 524, 503, 498, 479, 516, 505, 533, 477, 488] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[536, 540, 545, 515, 466, 535, 484, 498, 524, 509, 510, 530, 521, 521, 522, 545, 526, 520, 444, 474, 494, 501, 543, 503, 553, 551, 504, 526, 489, 527, 530, 505, 518, 543, 502, 502, 508, 499, 501, 548, 524, 507, 489, 523, 548, 502, 495, 475, 502, 528, 499, 523, 513, 484, 513, 543, 507, 526, 510, 536, 506, 493, 484, 507, 518, 479, 521, 495, 530, 490, 501, 500, 544, 508, 505, 487, 567, 518, 518, 483, 530, 531, 533, 502, 542, 523, 493, 466, 519, 496, 486, 509, 534, 487, 518, 558, 461, 498, 521, 503, 509, 490, 501, 532, 542, 517, 522, 505, 507, 508, 514, 486, 551, 515, 556, 500, 529, 464, 488, 487, 520, 543, 519, 485, 516, 495, 535, 477] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[496, 502, 536, 496, 495, 515, 519, 490, 523, 517, 536, 495, 503, 531, 507, 495, 545, 492, 477, 501, 504, 500, 543, 487, 538, 505, 475, 527, 533, 487, 464, 495, 500, 509, 536, 512, 529, 521, 490, 536, 523, 538, 517, 531, 505, 510, 530, 512, 531, 481, 504, 555, 464, 507, 490, 499, 511, 534, 498, 504, 510, 532, 486, 508, 480, 490, 488, 522, 563, 517, 521, 515, 525, 531, 544, 534, 500, 522, 499, 474, 534, 506, 544, 525, 483, 468, 528, 505, 505, 548, 519, 534, 507, 546, 494, 557, 504, 512, 499, 533, 485, 514, 512, 483, 485, 506, 534, 519, 490, 488, 525, 506, 539, 464, 465, 524, 512, 522, 484, 525, 549, 506, 563, 511, 515, 519, 530, 543] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[511, 493, 475, 529, 508, 514, 531, 520, 521, 525, 507, 521, 578, 532, 544, 463, 596, 498, 484, 527, 514, 481, 558, 498, 508, 531, 499, 509, 540, 529, 506, 551, 508, 474, 525, 493, 498, 508, 502, 545, 523, 464, 488, 538, 471, 512, 514, 511, 493, 510, 504, 512, 532, 517, 509, 490, 548, 529, 526, 518, 544, 492, 469, 502, 581, 505, 507, 486, 532, 515, 524, 534, 528, 485, 543, 520, 495, 494, 504, 501, 529, 502, 505, 522, 496, 530, 506, 481, 523, 526, 498, 484, 506, 484, 511, 490, 500, 527, 517, 499, 542, 512, 530, 485, 501, 538, 521, 508, 503, 528, 489, 507, 501, 512, 487, 515, 494, 523, 509, 501, 497, 500, 481, 534, 524, 500, 533, 501] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[551, 503, 500, 522, 518, 517, 508, 517, 497, 490, 453, 491, 507, 479, 533, 531, 510, 508, 503, 507, 504, 499, 515, 497, 511, 511, 510, 525, 485, 517, 568, 513, 546, 519, 515, 492, 511, 488, 487, 532, 525, 514, 539, 519, 528, 535, 487, 543, 535, 530, 498, 507, 496, 523, 544, 503, 528, 528, 476, 501, 537, 466, 497, 486, 521, 499, 484, 501, 495, 507, 520, 486, 542, 514, 509, 496, 508, 501, 493, 521, 487, 507, 515, 540, 516, 515, 519, 499, 504, 511, 551, 519, 505, 495, 509, 535, 539, 494, 559, 520, 493, 484, 527, 516, 484, 529, 492, 533, 516, 493, 518, 488, 542, 509, 509, 507, 549, 508, 486, 523, 509, 493, 494, 513, 576, 518, 535, 521] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[557, 489, 523, 523, 511, 511, 525, 492, 503, 510, 485, 514, 519, 523, 520, 480, 514, 537, 515, 545, 476, 495, 525, 525, 561, 545, 516, 520, 482, 528, 465, 496, 494, 545, 549, 555, 525, 526, 498, 512, 482, 493, 530, 505, 554, 490, 496, 530, 519, 508, 526, 526, 518, 504, 523, 481, 526, 478, 477, 509, 500, 457, 501, 516, 501, 518, 523, 531, 473, 542, 507, 512, 518, 501, 518, 480, 503, 499, 514, 541, 516, 543, 501, 487, 545, 546, 486, 498, 520, 509, 520, 528, 503, 490, 506, 509, 504, 515, 499, 513, 504, 525, 538, 498, 533, 496, 518, 528, 494, 496, 497, 529, 512, 513, 513, 523, 544, 459, 507, 506, 531, 546, 499, 506, 496, 500, 483, 540] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[502, 493, 475, 529, 508, 514, 531, 520, 521, 525, 507, 521, 578, 532, 544, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 594, 498, 484, 527, 514, 481, 558, 498, 508, 531, 499, 509, 540, 529, 506, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 474, 525, 493, 498, 508, 502, 534, 523, 464, 488, 538, 471, 512, 514, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 510, 504, 512, 532, 517, 509, 490, 548, 529, 526, 518, 544, 492, 469, 502, 25, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 581, 505, 507, 486, 532, 515, 524, 534, 503, 485, 543, 520, 495, 494, 504, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 502, 505, 522, 496, 530, 506, 481, 523, 526, 498, 484, 506, 484, 511, 490, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 527, 517, 499, 542, 512, 527, 485, 501, 538, 521, 508, 503, 528, 489, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 512, 487, 515, 494, 523, 509, 501, 497, 500, 481, 534, 524, 500, 533, 501, 9, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8263, 8327, 8063, 8222, 8229, 8096, 8204, 8132] avg=8192.0000 max=8327 max_rank=1 min=8063 min_rank=2 imbalance_ratio=1.016479 max_load_reduction=57209 max_load_reduction_pct=0.872940 moved_to_echo=32661 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[500, 502, 525, 518, 511, 519, 500, 553, 528, 512, 489, 524, 552, 527, 505, 507, 491, 517, 537, 547, 511, 497, 511, 542, 522, 531, 507, 524, 455, 534, 476, 494, 490, 498, 522, 505, 524, 533, 493, 534, 523, 515, 504, 516, 539, 497, 515, 500, 490, 509, 511, 562, 507, 502, 513, 494, 520, 526, 506, 479, 476, 511, 508, 495, 473, 504, 545, 523, 508, 537, 551, 499, 507, 498, 543, 509, 504, 495, 529, 487, 477, 510, 495, 517, 526, 524, 542, 528, 498, 471, 492, 550, 491, 493, 523, 498, 481, 532, 528, 514, 532, 505, 535, 527, 538, 555, 474, 540, 520, 535, 519, 530, 481, 457, 530, 492, 521, 502, 519, 497, 522, 508, 493, 468, 510, 490, 515, 534] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[525, 520, 512, 484, 515, 499, 476, 521, 541, 528, 553, 507, 486, 482, 510, 529, 526, 506, 517, 511, 476, 519, 489, 518, 538, 537, 519, 482, 474, 500, 511, 507, 482, 518, 512, 516, 530, 513, 524, 513, 479, 517, 486, 535, 515, 518, 542, 491, 481, 511, 507, 538, 550, 515, 529, 526, 470, 506, 526, 534, 513, 512, 505, 530, 547, 538, 537, 511, 506, 515, 512, 523, 527, 495, 523, 519, 503, 510, 515, 507, 519, 504, 492, 492, 500, 539, 517, 524, 534, 546, 527, 505, 504, 462, 519, 511, 553, 497, 518, 479, 523, 534, 555, 487, 492, 506, 497, 492, 488, 521, 478, 538, 523, 484, 449, 529, 518, 529, 529, 501, 480, 472, 556, 493, 509, 521, 538, 502] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[523, 540, 545, 515, 466, 535, 484, 498, 524, 509, 510, 530, 521, 521, 522, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 520, 444, 474, 494, 501, 543, 503, 553, 551, 504, 526, 489, 527, 530, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 543, 502, 502, 508, 499, 501, 530, 524, 507, 489, 523, 548, 502, 495, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 528, 499, 523, 513, 484, 513, 543, 507, 526, 510, 536, 506, 493, 484, 507, 28, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 479, 521, 495, 530, 490, 501, 500, 516, 508, 505, 487, 567, 518, 518, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 531, 533, 502, 542, 523, 493, 466, 519, 496, 486, 509, 534, 487, 518, 558, 4, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 498, 521, 503, 509, 490, 490, 532, 542, 517, 522, 505, 507, 508, 514, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 515, 556, 500, 529, 464, 488, 487, 520, 543, 519, 485, 516, 495, 535, 477, 13, 13, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8288, 8186, 8166, 8207, 8136, 8237, 8105, 8211] avg=8192.0000 max=8288 max_rank=0 min=8105 min_rank=6 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32689 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[541, 503, 500, 522, 518, 517, 508, 517, 497, 490, 453, 491, 507, 479, 533, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 508, 503, 507, 504, 499, 515, 497, 511, 511, 510, 525, 485, 517, 568, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 519, 515, 492, 511, 488, 487, 521, 525, 514, 539, 519, 528, 535, 487, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 530, 498, 507, 496, 523, 544, 503, 528, 528, 476, 501, 537, 466, 497, 486, 25, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 499, 484, 501, 495, 507, 520, 486, 517, 514, 509, 496, 508, 501, 493, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 507, 515, 540, 516, 515, 519, 499, 504, 511, 551, 519, 505, 495, 509, 535, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 494, 559, 520, 493, 484, 524, 516, 484, 529, 492, 533, 516, 493, 518, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 509, 509, 507, 549, 508, 486, 523, 509, 493, 494, 513, 576, 518, 535, 521, 10, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8107, 8181, 8269, 8182, 8072, 8230, 8182, 8313] avg=8192.0000 max=8313 max_rank=7 min=8072 min_rank=4 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32797 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[547, 489, 523, 523, 511, 511, 525, 492, 503, 510, 485, 514, 519, 523, 520, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 537, 515, 545, 476, 495, 525, 525, 561, 545, 516, 520, 482, 528, 465, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 545, 549, 555, 525, 526, 498, 501, 482, 493, 530, 505, 554, 490, 496, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 508, 526, 526, 518, 504, 523, 481, 526, 478, 477, 509, 500, 457, 501, 516, 24, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 518, 523, 531, 473, 542, 507, 512, 494, 501, 518, 480, 503, 499, 514, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 543, 501, 487, 545, 546, 486, 498, 520, 509, 520, 528, 503, 490, 506, 509, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 515, 499, 513, 504, 525, 535, 498, 533, 496, 518, 528, 494, 496, 497, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 513, 513, 523, 544, 459, 507, 506, 531, 546, 499, 506, 496, 500, 483, 540, 10, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8243, 8273, 8095, 8157, 8210, 8184, 8199] avg=8192.0000 max=8273 max_rank=2 min=8095 min_rank=3 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32750 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[491, 502, 525, 518, 511, 519, 500, 553, 528, 512, 489, 524, 552, 527, 505, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 517, 537, 547, 511, 497, 511, 542, 522, 531, 507, 524, 455, 534, 476, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 498, 522, 505, 524, 533, 493, 523, 523, 515, 504, 516, 539, 497, 515, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 509, 511, 562, 507, 502, 513, 494, 520, 526, 506, 479, 476, 511, 508, 495, 24, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 504, 545, 523, 508, 537, 551, 499, 483, 498, 543, 509, 504, 495, 529, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 510, 495, 517, 526, 524, 542, 528, 498, 471, 492, 550, 491, 493, 523, 498, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 532, 528, 514, 532, 505, 532, 527, 538, 555, 474, 540, 520, 535, 519, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 457, 530, 492, 521, 502, 519, 497, 522, 508, 493, 468, 510, 490, 515, 534, 9, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8263, 8195, 8197, 8135, 8188, 8137, 8362, 8059] avg=8192.0000 max=8362 max_rank=6 min=8059 min_rank=7 imbalance_ratio=1.020752 max_load_reduction=57174 max_load_reduction_pct=0.872406 moved_to_echo=32746 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[516, 520, 512, 484, 515, 499, 476, 521, 541, 528, 553, 507, 486, 482, 510, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 506, 517, 511, 476, 519, 489, 518, 538, 537, 519, 482, 474, 500, 511, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 518, 512, 516, 530, 513, 524, 502, 479, 517, 486, 535, 515, 518, 542, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 511, 507, 538, 550, 515, 529, 526, 470, 506, 526, 534, 513, 512, 505, 530, 25, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 538, 537, 511, 506, 515, 512, 523, 502, 495, 523, 519, 503, 510, 515, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 504, 492, 492, 500, 539, 517, 524, 534, 546, 527, 505, 504, 462, 519, 511, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 497, 518, 479, 523, 534, 552, 487, 492, 506, 497, 492, 488, 521, 478, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 484, 449, 529, 518, 529, 529, 501, 480, 472, 556, 493, 509, 521, 538, 502, 9, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8128, 8180, 8280, 8263, 8198, 8155, 8153] avg=8192.0000 max=8280 max_rank=3 min=8128 min_rank=1 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32769 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[487, 502, 536, 496, 495, 515, 519, 490, 523, 517, 536, 495, 503, 531, 507, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 492, 477, 501, 504, 500, 543, 487, 538, 505, 475, 527, 533, 487, 464, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 509, 536, 512, 529, 521, 490, 525, 523, 538, 517, 531, 505, 510, 530, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 481, 504, 555, 464, 507, 490, 499, 511, 534, 498, 504, 510, 532, 486, 508, 25, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 490, 488, 522, 563, 517, 521, 515, 500, 531, 544, 534, 500, 522, 499, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 506, 544, 525, 483, 468, 528, 505, 505, 548, 519, 534, 507, 546, 494, 557, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 512, 499, 533, 485, 514, 509, 483, 485, 506, 534, 519, 490, 488, 525, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 464, 465, 524, 512, 522, 484, 525, 549, 506, 563, 511, 515, 519, 530, 543, 9, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8147, 8071, 8288, 8141, 8200, 8306, 8092, 8291] avg=8192.0000 max=8306 max_rank=5 min=8071 min_rank=1 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32889 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[507, 474, 513, 479, 525, 515, 520, 504, 475, 533, 512, 521, 500, 497, 540, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 563, 498, 496, 495, 537, 507, 544, 505, 477, 496, 513, 528, 530, 511, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 498, 514, 494, 501, 527, 494, 471, 536, 517, 552, 506, 512, 487, 521, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 505, 522, 517, 523, 504, 520, 506, 540, 491, 482, 524, 536, 510, 504, 558, 26, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 514, 527, 516, 478, 520, 526, 493, 522, 559, 521, 528, 478, 542, 522, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 462, 533, 527, 511, 501, 510, 479, 527, 554, 497, 473, 477, 523, 494, 515, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 523, 514, 542, 488, 550, 529, 526, 494, 517, 490, 512, 517, 526, 506, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 521, 501, 543, 521, 569, 487, 524, 503, 498, 479, 516, 505, 533, 477, 488, 9, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8114, 8205, 8100, 8274, 8291, 8122, 8252, 8178] avg=8192.0000 max=8291 max_rank=4 min=8100 min_rank=2 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32843 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[502, 493, 475, 529, 508, 514, 531, 520, 521, 525, 507, 521, 578, 532, 544, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 594, 498, 484, 527, 514, 481, 558, 498, 508, 531, 499, 509, 540, 529, 506, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 474, 525, 493, 498, 508, 502, 534, 523, 464, 488, 538, 471, 512, 514, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 510, 504, 512, 532, 517, 509, 490, 548, 529, 526, 518, 544, 492, 469, 502, 25, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 581, 505, 507, 486, 532, 515, 524, 534, 503, 485, 543, 520, 495, 494, 504, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 502, 505, 522, 496, 530, 506, 481, 523, 526, 498, 484, 506, 484, 511, 490, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 527, 517, 499, 542, 512, 527, 485, 501, 538, 521, 508, 503, 528, 489, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 512, 487, 515, 494, 523, 509, 501, 497, 500, 481, 534, 524, 500, 533, 501, 9, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8263, 8327, 8063, 8222, 8229, 8096, 8204, 8132] avg=8192.0000 max=8327 max_rank=1 min=8063 min_rank=2 imbalance_ratio=1.016479 max_load_reduction=57209 max_load_reduction_pct=0.872940 moved_to_echo=32661 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[547, 489, 523, 523, 511, 511, 525, 492, 503, 510, 485, 514, 519, 523, 520, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 537, 515, 545, 476, 495, 525, 525, 561, 545, 516, 520, 482, 528, 465, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 545, 549, 555, 525, 526, 498, 501, 482, 493, 530, 505, 554, 490, 496, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 508, 526, 526, 518, 504, 523, 481, 526, 478, 477, 509, 500, 457, 501, 516, 24, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 518, 523, 531, 473, 542, 507, 512, 494, 501, 518, 480, 503, 499, 514, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 543, 501, 487, 545, 546, 486, 498, 520, 509, 520, 528, 503, 490, 506, 509, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 515, 499, 513, 504, 525, 535, 498, 533, 496, 518, 528, 494, 496, 497, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 513, 513, 523, 544, 459, 507, 506, 531, 546, 499, 506, 496, 500, 483, 540, 10, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8243, 8273, 8095, 8157, 8210, 8184, 8199] avg=8192.0000 max=8273 max_rank=2 min=8095 min_rank=3 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32750 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[541, 503, 500, 522, 518, 517, 508, 517, 497, 490, 453, 491, 507, 479, 533, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 508, 503, 507, 504, 499, 515, 497, 511, 511, 510, 525, 485, 517, 568, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 519, 515, 492, 511, 488, 487, 521, 525, 514, 539, 519, 528, 535, 487, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 530, 498, 507, 496, 523, 544, 503, 528, 528, 476, 501, 537, 466, 497, 486, 25, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 499, 484, 501, 495, 507, 520, 486, 517, 514, 509, 496, 508, 501, 493, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 507, 515, 540, 516, 515, 519, 499, 504, 511, 551, 519, 505, 495, 509, 535, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 494, 559, 520, 493, 484, 524, 516, 484, 529, 492, 533, 516, 493, 518, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 509, 509, 507, 549, 508, 486, 523, 509, 493, 494, 513, 576, 518, 535, 521, 10, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8107, 8181, 8269, 8182, 8072, 8230, 8182, 8313] avg=8192.0000 max=8313 max_rank=7 min=8072 min_rank=4 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32797 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[516, 520, 512, 484, 515, 499, 476, 521, 541, 528, 553, 507, 486, 482, 510, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 506, 517, 511, 476, 519, 489, 518, 538, 537, 519, 482, 474, 500, 511, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 518, 512, 516, 530, 513, 524, 502, 479, 517, 486, 535, 515, 518, 542, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 511, 507, 538, 550, 515, 529, 526, 470, 506, 526, 534, 513, 512, 505, 530, 25, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 538, 537, 511, 506, 515, 512, 523, 502, 495, 523, 519, 503, 510, 515, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 504, 492, 492, 500, 539, 517, 524, 534, 546, 527, 505, 504, 462, 519, 511, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 497, 518, 479, 523, 534, 552, 487, 492, 506, 497, 492, 488, 521, 478, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 484, 449, 529, 518, 529, 529, 501, 480, 472, 556, 493, 509, 521, 538, 502, 9, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8128, 8180, 8280, 8263, 8198, 8155, 8153] avg=8192.0000 max=8280 max_rank=3 min=8128 min_rank=1 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32769 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[491, 502, 525, 518, 511, 519, 500, 553, 528, 512, 489, 524, 552, 527, 505, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 517, 537, 547, 511, 497, 511, 542, 522, 531, 507, 524, 455, 534, 476, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 498, 522, 505, 524, 533, 493, 523, 523, 515, 504, 516, 539, 497, 515, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 509, 511, 562, 507, 502, 513, 494, 520, 526, 506, 479, 476, 511, 508, 495, 24, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 504, 545, 523, 508, 537, 551, 499, 483, 498, 543, 509, 504, 495, 529, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 510, 495, 517, 526, 524, 542, 528, 498, 471, 492, 550, 491, 493, 523, 498, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 532, 528, 514, 532, 505, 532, 527, 538, 555, 474, 540, 520, 535, 519, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 457, 530, 492, 521, 502, 519, 497, 522, 508, 493, 468, 510, 490, 515, 534, 9, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8263, 8195, 8197, 8135, 8188, 8137, 8362, 8059] avg=8192.0000 max=8362 max_rank=6 min=8059 min_rank=7 imbalance_ratio=1.020752 max_load_reduction=57174 max_load_reduction_pct=0.872406 moved_to_echo=32746 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=67072 observed_loss=-1536 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4352, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4352, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4096, 4096, 4352, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 3968, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=67072 slack_slots=64000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[506, 520, 573, 503, 505, 497, 514, 524, 528, 481, 509, 510, 542, 538, 524, 502, 545, 544, 506, 533, 498, 518, 516, 507, 548, 522, 555, 487, 512, 481, 535, 491, 504, 514, 481, 518, 477, 488, 506, 512, 506, 555, 543, 524, 539, 536, 520, 490, 506, 522, 479, 510, 520, 563, 508, 502, 518, 481, 532, 487, 490, 499, 521, 478, 520, 503, 503, 542, 541, 500, 494, 526, 499, 541, 503, 480, 536, 477, 519, 526, 493, 502, 504, 496, 488, 500, 523, 485, 513, 507, 533, 520, 501, 510, 525, 536, 521, 486, 510, 504, 559, 536, 506, 502, 464, 513, 502, 505, 514, 518, 488, 529, 510, 522, 531, 557, 545, 472, 493, 502, 478, 487, 504, 515, 507, 525, 496, 486] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[488, 517, 485, 515, 520, 545, 526, 531, 516, 445, 523, 526, 505, 528, 512, 474, 518, 516, 518, 499, 527, 560, 528, 518, 519, 541, 501, 553, 501, 489, 525, 535, 523, 523, 542, 502, 499, 515, 476, 500, 513, 533, 502, 485, 519, 540, 530, 495, 500, 500, 524, 521, 544, 540, 480, 566, 498, 491, 512, 515, 512, 506, 472, 482, 507, 536, 520, 536, 514, 504, 518, 526, 511, 497, 488, 503, 498, 519, 506, 529, 517, 516, 544, 517, 512, 514, 487, 490, 499, 524, 526, 481, 536, 553, 537, 502, 482, 557, 488, 470, 534, 499, 503, 502, 507, 462, 473, 523, 574, 473, 516, 527, 487, 472, 473, 504, 500, 578, 483, 512, 528, 547, 527, 540, 468, 482, 528, 486] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[531, 535, 503, 515, 535, 537, 538, 515, 542, 495, 471, 505, 531, 516, 507, 539, 522, 492, 503, 512, 506, 505, 513, 510, 530, 532, 479, 526, 545, 493, 484, 527, 513, 486, 545, 518, 542, 526, 501, 552, 500, 494, 476, 486, 536, 528, 499, 502, 499, 537, 510, 496, 498, 535, 533, 512, 502, 523, 508, 494, 468, 528, 489, 517, 484, 489, 513, 522, 536, 534, 494, 556, 544, 477, 532, 506, 483, 517, 515, 527, 541, 461, 508, 525, 539, 509, 500, 496, 518, 532, 519, 496, 512, 493, 508, 494, 514, 521, 523, 494, 556, 507, 486, 488, 470, 522, 493, 545, 533, 499, 499, 546, 499, 501, 525, 519, 484, 512, 494, 491, 529, 491, 515, 533, 491, 498, 516, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[470, 490, 470, 574, 479, 537, 493, 500, 505, 506, 499, 562, 518, 499, 478, 512, 515, 514, 491, 520, 533, 552, 504, 511, 505, 532, 473, 508, 530, 536, 499, 499, 525, 513, 524, 517, 509, 517, 510, 519, 548, 485, 497, 520, 492, 509, 513, 536, 489, 509, 489, 513, 528, 519, 497, 502, 533, 502, 482, 505, 524, 533, 502, 554, 496, 486, 535, 501, 509, 498, 507, 494, 521, 526, 524, 519, 501, 489, 494, 548, 527, 527, 519, 550, 551, 515, 487, 534, 522, 515, 517, 520, 498, 532, 521, 533, 560, 490, 536, 529, 500, 510, 525, 507, 498, 509, 528, 491, 532, 503, 531, 470, 522, 498, 491, 528, 505, 475, 484, 520, 506, 503, 474, 463, 461, 514, 551, 577] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[550, 507, 551, 492, 533, 474, 475, 528, 500, 538, 530, 463, 484, 526, 520, 505, 541, 523, 510, 537, 475, 544, 527, 519, 481, 504, 494, 541, 503, 502, 518, 540, 489, 494, 495, 519, 507, 545, 522, 505, 512, 518, 505, 501, 496, 511, 502, 522, 511, 502, 474, 516, 539, 536, 505, 519, 531, 511, 540, 543, 501, 465, 517, 504, 537, 516, 520, 538, 493, 515, 510, 516, 490, 529, 492, 515, 524, 531, 548, 496, 478, 538, 528, 522, 470, 509, 538, 534, 543, 491, 505, 528, 503, 518, 510, 522, 541, 477, 518, 494, 478, 463, 503, 509, 483, 491, 465, 524, 552, 500, 510, 511, 519, 506, 533, 536, 522, 514, 520, 524, 492, 521, 523, 524, 487, 518, 489, 490] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[531, 476, 529, 510, 510, 521, 535, 509, 528, 565, 499, 498, 494, 510, 519, 513, 505, 490, 489, 497, 511, 539, 494, 543, 511, 490, 528, 515, 509, 494, 520, 501, 492, 534, 491, 534, 503, 508, 490, 522, 540, 508, 476, 511, 484, 457, 465, 510, 458, 525, 460, 509, 528, 519, 550, 517, 511, 552, 560, 468, 558, 489, 516, 533, 506, 523, 513, 491, 530, 509, 519, 545, 483, 521, 537, 537, 477, 519, 478, 484, 526, 536, 547, 497, 501, 514, 497, 494, 507, 537, 498, 501, 484, 485, 509, 548, 507, 495, 569, 502, 496, 515, 537, 533, 521, 482, 515, 514, 528, 526, 515, 494, 511, 515, 477, 531, 511, 540, 507, 505, 526, 524, 514, 527, 498, 510, 563, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[470, 490, 470, 574, 479, 537, 493, 500, 505, 506, 499, 562, 518, 499, 478, 512, 515, 514, 491, 520, 533, 552, 504, 511, 505, 532, 473, 508, 530, 536, 499, 499, 525, 513, 524, 517, 509, 517, 510, 519, 548, 485, 497, 520, 492, 509, 513, 536, 489, 509, 489, 513, 528, 519, 497, 502, 533, 502, 482, 505, 524, 533, 502, 554, 496, 486, 535, 501, 509, 498, 507, 494, 521, 526, 524, 519, 501, 489, 494, 548, 527, 527, 519, 550, 551, 515, 487, 534, 522, 515, 517, 520, 498, 532, 521, 533, 560, 490, 536, 529, 500, 510, 525, 507, 498, 509, 528, 491, 532, 503, 531, 470, 522, 498, 491, 528, 505, 475, 484, 520, 506, 503, 474, 463, 461, 514, 551, 577] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[488, 517, 485, 515, 520, 545, 526, 531, 516, 445, 523, 526, 505, 528, 512, 474, 518, 516, 518, 499, 527, 560, 528, 518, 519, 541, 501, 553, 501, 489, 525, 535, 523, 523, 542, 502, 499, 515, 476, 500, 513, 533, 502, 485, 519, 540, 530, 495, 500, 500, 524, 521, 544, 540, 480, 566, 498, 491, 512, 515, 512, 506, 472, 482, 507, 536, 520, 536, 514, 504, 518, 526, 511, 497, 488, 503, 498, 519, 506, 529, 517, 516, 544, 517, 512, 514, 487, 490, 499, 524, 526, 481, 536, 553, 537, 502, 482, 557, 488, 470, 534, 499, 503, 502, 507, 462, 473, 523, 574, 473, 516, 527, 487, 472, 473, 504, 500, 578, 483, 512, 528, 547, 527, 540, 468, 482, 528, 486] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[461, 444, 505, 498, 513, 506, 549, 492, 525, 495, 497, 518, 517, 509, 503, 502, 533, 536, 519, 518, 502, 461, 474, 513, 539, 515, 509, 504, 502, 495, 515, 521, 515, 525, 525, 558, 518, 508, 532, 492, 517, 469, 482, 524, 511, 497, 531, 519, 513, 486, 487, 550, 487, 494, 522, 492, 501, 500, 534, 501, 516, 518, 523, 513, 502, 539, 501, 470, 516, 500, 524, 519, 526, 493, 528, 505, 537, 529, 527, 537, 487, 516, 505, 517, 503, 529, 501, 540, 545, 534, 480, 524, 519, 524, 486, 522, 535, 499, 494, 510, 503, 524, 525, 531, 514, 530, 462, 498, 508, 550, 529, 498, 507, 549, 548, 524, 512, 490, 514, 516, 515, 550, 501, 540, 507, 522, 488, 508] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[506, 520, 573, 503, 505, 497, 514, 524, 528, 481, 509, 510, 542, 538, 524, 502, 545, 544, 506, 533, 498, 518, 516, 507, 548, 522, 555, 487, 512, 481, 535, 491, 504, 514, 481, 518, 477, 488, 506, 512, 506, 555, 543, 524, 539, 536, 520, 490, 506, 522, 479, 510, 520, 563, 508, 502, 518, 481, 532, 487, 490, 499, 521, 478, 520, 503, 503, 542, 541, 500, 494, 526, 499, 541, 503, 480, 536, 477, 519, 526, 493, 502, 504, 496, 488, 500, 523, 485, 513, 507, 533, 520, 501, 510, 525, 536, 521, 486, 510, 504, 559, 536, 506, 502, 464, 513, 502, 505, 514, 518, 488, 529, 510, 522, 531, 557, 545, 472, 493, 502, 478, 487, 504, 515, 507, 525, 496, 486] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[537, 562, 507, 518, 536, 517, 497, 537, 548, 537, 467, 530, 578, 520, 536, 535, 480, 511, 484, 500, 509, 478, 543, 516, 517, 504, 533, 531, 530, 512, 468, 525, 506, 528, 471, 526, 532, 512, 496, 508, 515, 512, 508, 474, 535, 494, 524, 471, 474, 499, 525, 503, 498, 483, 477, 507, 484, 476, 545, 532, 456, 526, 520, 497, 504, 541, 498, 532, 499, 481, 490, 487, 502, 538, 505, 563, 544, 526, 521, 503, 504, 501, 527, 558, 472, 517, 536, 517, 532, 512, 490, 508, 486, 487, 516, 480, 534, 510, 548, 526, 496, 516, 480, 529, 549, 536, 511, 538, 508, 484, 476, 496, 520, 527, 519, 553, 505, 535, 470, 475, 510, 510, 542, 485, 502, 513, 544, 495] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[506, 520, 573, 503, 505, 497, 514, 524, 501, 481, 509, 510, 542, 538, 524, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 544, 506, 533, 498, 518, 516, 507, 548, 522, 555, 464, 512, 481, 535, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 514, 481, 518, 477, 488, 506, 512, 506, 555, 543, 524, 539, 536, 520, 490, 12, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 522, 479, 510, 520, 563, 508, 502, 518, 481, 532, 487, 490, 499, 521, 478, 27, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 503, 503, 542, 541, 500, 494, 502, 499, 541, 503, 480, 536, 477, 519, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 502, 504, 477, 488, 500, 523, 485, 513, 507, 533, 520, 501, 510, 525, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 486, 510, 504, 559, 536, 506, 502, 464, 513, 502, 505, 514, 518, 488, 529, 14, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 522, 531, 557, 545, 472, 493, 502, 478, 487, 504, 515, 507, 525, 496, 486, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8249, 8275, 8235, 8154, 8186, 8117, 8176, 8144] avg=8192.0000 max=8275 max_rank=1 min=8117 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32623 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[531, 535, 503, 515, 535, 537, 538, 515, 514, 495, 471, 505, 531, 516, 507, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 492, 503, 512, 506, 505, 513, 510, 530, 532, 479, 501, 545, 493, 484, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 486, 545, 518, 542, 526, 501, 552, 500, 494, 476, 486, 536, 528, 499, 502, 13, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 537, 510, 496, 498, 535, 533, 512, 502, 523, 508, 494, 468, 528, 489, 517, 28, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 489, 513, 522, 536, 534, 494, 532, 544, 477, 532, 506, 483, 517, 515, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 461, 508, 505, 539, 509, 500, 496, 518, 532, 519, 496, 512, 493, 508, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 521, 523, 494, 556, 507, 486, 488, 470, 522, 493, 545, 533, 499, 499, 546, 14, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 501, 525, 519, 484, 512, 494, 491, 529, 491, 515, 533, 491, 498, 516, 515, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8287, 8154, 8227, 8189, 8205, 8131, 8216, 8127] avg=8192.0000 max=8287 max_rank=0 min=8127 min_rank=7 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32679 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[531, 535, 503, 515, 535, 537, 538, 515, 542, 495, 471, 505, 531, 516, 507, 539, 522, 492, 503, 512, 506, 505, 513, 510, 530, 532, 479, 526, 545, 493, 484, 527, 513, 486, 545, 518, 542, 526, 501, 552, 500, 494, 476, 486, 536, 528, 499, 502, 499, 537, 510, 496, 498, 535, 533, 512, 502, 523, 508, 494, 468, 528, 489, 517, 484, 489, 513, 522, 536, 534, 494, 556, 544, 477, 532, 506, 483, 517, 515, 527, 541, 461, 508, 525, 539, 509, 500, 496, 518, 532, 519, 496, 512, 493, 508, 494, 514, 521, 523, 494, 556, 507, 486, 488, 470, 522, 493, 545, 533, 499, 499, 546, 499, 501, 525, 519, 484, 512, 494, 491, 529, 491, 515, 533, 491, 498, 516, 515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[470, 490, 470, 574, 479, 537, 493, 500, 479, 506, 499, 562, 518, 499, 478, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 514, 491, 520, 533, 552, 504, 511, 505, 532, 473, 484, 530, 536, 499, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 513, 524, 517, 509, 517, 510, 519, 548, 485, 497, 520, 492, 509, 513, 536, 12, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 509, 489, 513, 528, 519, 497, 502, 533, 502, 482, 505, 524, 533, 502, 554, 26, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 486, 535, 501, 509, 498, 507, 472, 521, 526, 524, 519, 501, 489, 494, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 527, 519, 529, 551, 515, 487, 534, 522, 515, 517, 520, 498, 532, 521, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 560, 490, 536, 529, 500, 510, 525, 507, 498, 509, 528, 491, 532, 503, 531, 470, 13, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 498, 491, 528, 505, 475, 484, 520, 506, 503, 474, 463, 461, 514, 551, 577, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8066, 8198, 8255, 8219, 8126, 8347, 8238, 8087] avg=8192.0000 max=8347 max_rank=5 min=8066 min_rank=0 imbalance_ratio=1.018921 max_load_reduction=57189 max_load_reduction_pct=0.872635 moved_to_echo=32798 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[531, 476, 529, 510, 510, 521, 535, 509, 496, 565, 499, 498, 494, 510, 519, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 490, 489, 497, 511, 539, 494, 543, 511, 490, 528, 483, 509, 494, 520, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 534, 491, 534, 503, 508, 490, 522, 540, 508, 476, 511, 484, 457, 465, 510, 16, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 458, 525, 460, 509, 528, 519, 550, 517, 511, 552, 560, 468, 558, 489, 516, 533, 32, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 523, 513, 491, 530, 509, 519, 514, 483, 521, 537, 537, 477, 519, 478, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 536, 547, 468, 501, 514, 497, 494, 507, 537, 498, 501, 484, 485, 509, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 495, 569, 502, 496, 515, 537, 533, 521, 482, 515, 514, 528, 526, 515, 494, 18, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 515, 477, 531, 511, 540, 507, 505, 526, 524, 514, 527, 498, 510, 563, 514, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8215, 8104, 8054, 8301, 8141, 8152, 8277, 8292] avg=8192.0000 max=8301 max_rank=3 min=8054 min_rank=2 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32862 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[550, 507, 551, 492, 533, 474, 475, 528, 474, 538, 530, 463, 484, 526, 520, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 523, 510, 537, 475, 544, 527, 519, 481, 504, 494, 516, 503, 502, 518, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 494, 495, 519, 507, 545, 522, 505, 512, 518, 505, 501, 496, 511, 502, 522, 13, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 502, 474, 516, 539, 536, 505, 519, 531, 511, 540, 543, 501, 465, 517, 504, 26, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 516, 520, 538, 493, 515, 510, 493, 490, 529, 492, 515, 524, 531, 548, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 538, 528, 503, 470, 509, 538, 534, 543, 491, 505, 528, 503, 518, 510, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 477, 518, 494, 478, 463, 503, 509, 483, 491, 465, 524, 552, 500, 510, 511, 13, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 506, 533, 536, 522, 514, 520, 524, 492, 521, 523, 524, 487, 518, 489, 490, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8150, 8234, 8166, 8252, 8247, 8218, 8037, 8232] avg=8192.0000 max=8252 max_rank=3 min=8037 min_rank=6 imbalance_ratio=1.007324 max_load_reduction=57284 max_load_reduction_pct=0.874084 moved_to_echo=32734 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[488, 517, 485, 515, 520, 545, 526, 531, 489, 445, 523, 526, 505, 528, 512, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 516, 518, 499, 527, 560, 528, 518, 519, 541, 501, 527, 501, 489, 525, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 523, 542, 502, 499, 515, 476, 500, 513, 533, 502, 485, 519, 540, 530, 495, 13, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 500, 524, 521, 544, 540, 480, 566, 498, 491, 512, 515, 512, 506, 472, 482, 27, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 536, 520, 536, 514, 504, 518, 502, 511, 497, 488, 503, 498, 519, 506, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 516, 544, 498, 512, 514, 487, 490, 499, 524, 526, 481, 536, 553, 537, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 557, 488, 470, 534, 499, 503, 502, 507, 462, 473, 523, 574, 473, 516, 527, 14, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 472, 473, 504, 500, 578, 483, 512, 528, 547, 527, 540, 468, 482, 528, 486, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8129, 8322, 8220, 8203, 8188, 8236, 8109, 8129] avg=8192.0000 max=8322 max_rank=1 min=8109 min_rank=6 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32662 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[531, 476, 529, 510, 510, 521, 535, 509, 496, 565, 499, 498, 494, 510, 519, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 490, 489, 497, 511, 539, 494, 543, 511, 490, 528, 483, 509, 494, 520, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 534, 491, 534, 503, 508, 490, 522, 540, 508, 476, 511, 484, 457, 465, 510, 16, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 458, 525, 460, 509, 528, 519, 550, 517, 511, 552, 560, 468, 558, 489, 516, 533, 32, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 523, 513, 491, 530, 509, 519, 514, 483, 521, 537, 537, 477, 519, 478, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 536, 547, 468, 501, 514, 497, 494, 507, 537, 498, 501, 484, 485, 509, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 495, 569, 502, 496, 515, 537, 533, 521, 482, 515, 514, 528, 526, 515, 494, 18, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 515, 477, 531, 511, 540, 507, 505, 526, 524, 514, 527, 498, 510, 563, 514, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8215, 8104, 8054, 8301, 8141, 8152, 8277, 8292] avg=8192.0000 max=8301 max_rank=3 min=8054 min_rank=2 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32862 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[537, 562, 507, 518, 536, 517, 497, 537, 519, 537, 467, 530, 578, 520, 536, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 511, 484, 500, 509, 478, 543, 516, 517, 504, 533, 506, 530, 512, 468, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 528, 471, 526, 532, 512, 496, 508, 515, 512, 508, 474, 535, 494, 524, 471, 13, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 499, 525, 503, 498, 483, 477, 507, 484, 476, 545, 532, 456, 526, 520, 497, 29, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 541, 498, 532, 499, 481, 490, 466, 502, 538, 505, 563, 544, 526, 521, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 501, 527, 537, 472, 517, 536, 517, 532, 512, 490, 508, 486, 487, 516, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 510, 548, 526, 496, 516, 480, 529, 549, 536, 511, 538, 508, 484, 476, 496, 12, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 527, 519, 553, 505, 535, 470, 475, 510, 510, 542, 485, 502, 513, 544, 495, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8433, 8116, 8134, 8043, 8213, 8122, 8255, 8220] avg=8192.0000 max=8433 max_rank=0 min=8043 min_rank=3 imbalance_ratio=1.029419 max_load_reduction=57103 max_load_reduction_pct=0.871323 moved_to_echo=32810 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[470, 490, 470, 574, 479, 537, 493, 500, 479, 506, 499, 562, 518, 499, 478, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 514, 491, 520, 533, 552, 504, 511, 505, 532, 473, 484, 530, 536, 499, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 513, 524, 517, 509, 517, 510, 519, 548, 485, 497, 520, 492, 509, 513, 536, 12, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 509, 489, 513, 528, 519, 497, 502, 533, 502, 482, 505, 524, 533, 502, 554, 26, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 486, 535, 501, 509, 498, 507, 472, 521, 526, 524, 519, 501, 489, 494, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 527, 519, 529, 551, 515, 487, 534, 522, 515, 517, 520, 498, 532, 521, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 560, 490, 536, 529, 500, 510, 525, 507, 498, 509, 528, 491, 532, 503, 531, 470, 13, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 498, 491, 528, 505, 475, 484, 520, 506, 503, 474, 463, 461, 514, 551, 577, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8066, 8198, 8255, 8219, 8126, 8347, 8238, 8087] avg=8192.0000 max=8347 max_rank=5 min=8066 min_rank=0 imbalance_ratio=1.018921 max_load_reduction=57189 max_load_reduction_pct=0.872635 moved_to_echo=32798 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[506, 520, 573, 503, 505, 497, 514, 524, 501, 481, 509, 510, 542, 538, 524, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 544, 506, 533, 498, 518, 516, 507, 548, 522, 555, 464, 512, 481, 535, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 514, 481, 518, 477, 488, 506, 512, 506, 555, 543, 524, 539, 536, 520, 490, 12, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 522, 479, 510, 520, 563, 508, 502, 518, 481, 532, 487, 490, 499, 521, 478, 27, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 503, 503, 542, 541, 500, 494, 502, 499, 541, 503, 480, 536, 477, 519, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 502, 504, 477, 488, 500, 523, 485, 513, 507, 533, 520, 501, 510, 525, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 486, 510, 504, 559, 536, 506, 502, 464, 513, 502, 505, 514, 518, 488, 529, 14, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 522, 531, 557, 545, 472, 493, 502, 478, 487, 504, 515, 507, 525, 496, 486, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8249, 8275, 8235, 8154, 8186, 8117, 8176, 8144] avg=8192.0000 max=8275 max_rank=1 min=8117 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32623 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[461, 444, 505, 498, 513, 506, 549, 492, 498, 495, 497, 518, 517, 509, 503, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 536, 519, 518, 502, 461, 474, 513, 539, 515, 509, 481, 502, 495, 515, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 525, 525, 558, 518, 508, 532, 492, 517, 469, 482, 524, 511, 497, 531, 519, 12, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 486, 487, 550, 487, 494, 522, 492, 501, 500, 534, 501, 516, 518, 523, 513, 27, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 539, 501, 470, 516, 500, 524, 496, 526, 493, 528, 505, 537, 529, 527, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 516, 505, 498, 503, 529, 501, 540, 545, 534, 480, 524, 519, 524, 486, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 499, 494, 510, 503, 524, 525, 531, 514, 530, 462, 498, 508, 550, 529, 498, 13, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 549, 548, 524, 512, 490, 514, 516, 515, 550, 501, 540, 507, 522, 488, 508, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8007, 8133, 8245, 8175, 8230, 8213, 8228, 8305] avg=8192.0000 max=8305 max_rank=7 min=8007 min_rank=0 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32976 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[531, 535, 503, 515, 535, 537, 538, 515, 514, 495, 471, 505, 531, 516, 507, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 492, 503, 512, 506, 505, 513, 510, 530, 532, 479, 501, 545, 493, 484, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 486, 545, 518, 542, 526, 501, 552, 500, 494, 476, 486, 536, 528, 499, 502, 13, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 537, 510, 496, 498, 535, 533, 512, 502, 523, 508, 494, 468, 528, 489, 517, 28, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 489, 513, 522, 536, 534, 494, 532, 544, 477, 532, 506, 483, 517, 515, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 461, 508, 505, 539, 509, 500, 496, 518, 532, 519, 496, 512, 493, 508, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 521, 523, 494, 556, 507, 486, 488, 470, 522, 493, 545, 533, 499, 499, 546, 14, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 501, 525, 519, 484, 512, 494, 491, 529, 491, 515, 533, 491, 498, 516, 515, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8287, 8154, 8227, 8189, 8205, 8131, 8216, 8127] avg=8192.0000 max=8287 max_rank=0 min=8127 min_rank=7 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32679 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[537, 562, 507, 518, 536, 517, 497, 537, 519, 537, 467, 530, 578, 520, 536, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 511, 484, 500, 509, 478, 543, 516, 517, 504, 533, 506, 530, 512, 468, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 528, 471, 526, 532, 512, 496, 508, 515, 512, 508, 474, 535, 494, 524, 471, 13, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 499, 525, 503, 498, 483, 477, 507, 484, 476, 545, 532, 456, 526, 520, 497, 29, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 541, 498, 532, 499, 481, 490, 466, 502, 538, 505, 563, 544, 526, 521, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 501, 527, 537, 472, 517, 536, 517, 532, 512, 490, 508, 486, 487, 516, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 510, 548, 526, 496, 516, 480, 529, 549, 536, 511, 538, 508, 484, 476, 496, 12, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 527, 519, 553, 505, 535, 470, 475, 510, 510, 542, 485, 502, 513, 544, 495, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8433, 8116, 8134, 8043, 8213, 8122, 8255, 8220] avg=8192.0000 max=8433 max_rank=0 min=8043 min_rank=3 imbalance_ratio=1.029419 max_load_reduction=57103 max_load_reduction_pct=0.871323 moved_to_echo=32810 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 3968, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[3968, 4096, 3968, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 3968, 4224, 4352, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 3968, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4352, 4096, 4224, 3968, 4096, 4096, 4224, 4224, 4224, 3968, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[3968, 4096, 3968, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[512, 524, 495, 562, 493, 533, 542, 513, 493, 524, 524, 536, 504, 528, 496, 535, 482, 527, 489, 538, 515, 490, 522, 535, 533, 485, 538, 511, 525, 491, 506, 504, 498, 502, 511, 528, 533, 509, 554, 523, 487, 494, 466, 558, 504, 523, 475, 505, 532, 497, 501, 483, 503, 521, 492, 498, 489, 508, 515, 521, 505, 483, 495, 515, 513, 541, 528, 516, 524, 495, 534, 522, 500, 543, 520, 515, 491, 506, 527, 532, 466, 521, 525, 493, 471, 502, 533, 506, 506, 518, 512, 519, 565, 534, 540, 476, 442, 528, 535, 525, 495, 490, 533, 533, 547, 521, 535, 505, 502, 508, 479, 494, 502, 526, 473, 518, 523, 492, 509, 501, 509, 516, 523, 487, 483, 540, 518, 517] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[493, 533, 516, 498, 499, 495, 501, 535, 483, 501, 504, 480, 486, 528, 516, 524, 540, 517, 538, 529, 491, 536, 470, 518, 537, 506, 485, 493, 523, 508, 524, 527, 502, 495, 483, 515, 511, 519, 492, 503, 518, 512, 538, 495, 525, 534, 511, 477, 514, 507, 562, 466, 490, 533, 483, 498, 514, 553, 500, 509, 496, 496, 517, 489, 521, 534, 504, 557, 486, 504, 507, 519, 487, 531, 531, 536, 525, 503, 509, 486, 518, 503, 516, 530, 531, 535, 531, 512, 496, 512, 489, 523, 525, 524, 539, 504, 484, 516, 489, 508, 511, 490, 522, 521, 520, 476, 545, 529, 548, 512, 533, 563, 463, 534, 511, 508, 501, 531, 517, 499, 530, 492, 523, 515, 526, 483, 506, 511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[515, 495, 555, 517, 519, 519, 500, 470, 522, 556, 537, 537, 521, 487, 463, 499, 515, 529, 492, 516, 506, 521, 529, 504, 469, 502, 502, 503, 527, 540, 508, 555, 512, 495, 506, 462, 460, 499, 515, 520, 530, 453, 480, 464, 494, 521, 521, 491, 506, 479, 534, 515, 514, 541, 484, 497, 518, 506, 544, 506, 528, 535, 515, 533, 519, 531, 541, 520, 531, 506, 508, 494, 525, 522, 527, 531, 502, 497, 484, 540, 501, 526, 503, 515, 514, 492, 508, 519, 511, 544, 544, 540, 540, 505, 526, 468, 556, 540, 481, 483, 502, 566, 523, 530, 523, 534, 476, 543, 546, 504, 480, 467, 543, 517, 539, 471, 493, 495, 529, 482, 534, 510, 505, 489, 501, 544, 464, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[562, 521, 536, 478, 469, 492, 506, 505, 514, 506, 505, 528, 471, 523, 504, 506, 537, 517, 500, 521, 499, 556, 509, 519, 529, 531, 513, 507, 503, 502, 485, 515, 514, 482, 505, 513, 544, 497, 474, 541, 486, 538, 477, 507, 482, 526, 519, 514, 495, 491, 502, 509, 480, 511, 517, 489, 525, 497, 514, 527, 527, 525, 502, 477, 496, 511, 530, 522, 537, 539, 531, 515, 502, 517, 526, 521, 524, 496, 517, 537, 494, 533, 518, 476, 495, 560, 520, 546, 508, 470, 519, 551, 491, 489, 528, 533, 526, 470, 518, 549, 512, 513, 493, 525, 519, 509, 492, 501, 476, 534, 516, 530, 488, 532, 519, 519, 507, 504, 526, 543, 505, 517, 548, 497, 519, 467, 521, 513] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[492, 531, 538, 585, 494, 528, 524, 539, 508, 513, 496, 493, 489, 560, 532, 519, 481, 481, 526, 527, 504, 545, 559, 531, 507, 480, 497, 478, 509, 517, 488, 482, 483, 494, 495, 516, 532, 500, 521, 507, 516, 512, 524, 561, 497, 462, 527, 491, 493, 528, 498, 508, 480, 506, 533, 468, 501, 517, 547, 524, 513, 491, 540, 538, 522, 495, 506, 550, 516, 529, 514, 539, 525, 491, 485, 467, 524, 483, 521, 530, 486, 536, 531, 526, 523, 523, 498, 528, 537, 503, 515, 503, 520, 541, 510, 515, 542, 462, 530, 539, 538, 490, 508, 524, 484, 528, 525, 477, 497, 501, 530, 511, 489, 509, 533, 505, 515, 513, 488, 534, 510, 473, 494, 486, 503, 491, 533, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[481, 544, 522, 495, 545, 500, 512, 489, 527, 490, 502, 529, 498, 494, 476, 507, 483, 520, 484, 471, 491, 531, 498, 499, 512, 535, 538, 504, 533, 548, 489, 512, 535, 531, 525, 506, 512, 510, 526, 527, 535, 541, 450, 532, 524, 518, 507, 485, 543, 505, 501, 529, 508, 501, 497, 495, 495, 505, 518, 507, 460, 485, 545, 491, 532, 493, 512, 503, 515, 504, 530, 538, 470, 487, 521, 519, 542, 498, 477, 567, 526, 503, 511, 532, 510, 504, 531, 524, 533, 488, 518, 517, 518, 493, 500, 551, 510, 531, 525, 533, 511, 523, 478, 519, 499, 521, 490, 515, 550, 536, 464, 522, 513, 531, 510, 485, 533, 495, 547, 547, 529, 483, 535, 485, 540, 493, 509, 499] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[507, 515, 502, 523, 526, 506, 502, 506, 482, 534, 559, 513, 498, 529, 498, 505, 491, 474, 513, 523, 488, 506, 477, 528, 513, 512, 505, 526, 508, 506, 533, 496, 490, 506, 501, 517, 531, 492, 517, 484, 530, 536, 537, 523, 548, 494, 493, 514, 551, 510, 529, 481, 505, 522, 535, 503, 513, 499, 487, 496, 492, 522, 495, 521, 481, 503, 458, 535, 549, 535, 475, 481, 516, 517, 527, 512, 510, 516, 537, 508, 524, 528, 556, 488, 495, 462, 512, 493, 494, 553, 513, 499, 519, 515, 519, 502, 539, 507, 500, 490, 516, 563, 481, 534, 489, 506, 523, 535, 502, 540, 533, 512, 525, 562, 522, 529, 482, 533, 491, 498, 515, 519, 536, 476, 509, 540, 456, 563] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[515, 495, 555, 517, 519, 519, 500, 470, 522, 556, 537, 537, 521, 487, 463, 499, 515, 529, 492, 516, 506, 521, 529, 504, 469, 502, 502, 503, 527, 540, 508, 555, 512, 495, 506, 462, 460, 499, 515, 520, 530, 453, 480, 464, 494, 521, 521, 491, 506, 479, 534, 515, 514, 541, 484, 497, 518, 506, 544, 506, 528, 535, 515, 533, 519, 531, 541, 520, 531, 506, 508, 494, 525, 522, 527, 531, 502, 497, 484, 540, 501, 526, 503, 515, 514, 492, 508, 519, 511, 544, 544, 540, 540, 505, 526, 468, 556, 540, 481, 483, 502, 566, 523, 530, 523, 534, 476, 543, 546, 504, 480, 467, 543, 517, 539, 471, 493, 495, 529, 482, 534, 510, 505, 489, 501, 544, 464, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[512, 524, 495, 562, 493, 533, 542, 513, 493, 524, 524, 536, 504, 528, 496, 535, 482, 527, 489, 538, 515, 490, 522, 535, 533, 485, 538, 511, 525, 491, 506, 504, 498, 502, 511, 528, 533, 509, 554, 523, 487, 494, 466, 558, 504, 523, 475, 505, 532, 497, 501, 483, 503, 521, 492, 498, 489, 508, 515, 521, 505, 483, 495, 515, 513, 541, 528, 516, 524, 495, 534, 522, 500, 543, 520, 515, 491, 506, 527, 532, 466, 521, 525, 493, 471, 502, 533, 506, 506, 518, 512, 519, 565, 534, 540, 476, 442, 528, 535, 525, 495, 490, 533, 533, 547, 521, 535, 505, 502, 508, 479, 494, 502, 526, 473, 518, 523, 492, 509, 501, 509, 516, 523, 487, 483, 540, 518, 517] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[509, 539, 495, 474, 509, 501, 509, 472, 495, 531, 528, 496, 528, 490, 514, 485, 478, 459, 542, 519, 536, 590, 541, 499, 524, 521, 511, 514, 522, 544, 519, 535, 497, 516, 520, 511, 517, 514, 505, 478, 523, 528, 530, 525, 511, 510, 497, 509, 513, 493, 496, 475, 499, 499, 543, 502, 522, 517, 511, 502, 515, 504, 502, 510, 528, 537, 531, 517, 518, 500, 509, 549, 509, 524, 549, 503, 497, 488, 540, 505, 512, 488, 497, 493, 536, 495, 545, 512, 495, 485, 521, 552, 520, 518, 517, 525, 495, 489, 457, 570, 541, 541, 497, 502, 509, 492, 520, 527, 506, 484, 535, 532, 524, 506, 509, 461, 545, 511, 549, 502, 500, 488, 476, 496, 487, 514, 524, 509] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[512, 524, 495, 562, 493, 533, 542, 513, 493, 524, 524, 536, 504, 528, 496, 535, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 527, 489, 538, 515, 483, 522, 535, 533, 485, 538, 511, 525, 491, 506, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 502, 511, 528, 533, 509, 554, 523, 487, 494, 466, 558, 504, 523, 475, 505, 44, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 497, 501, 483, 503, 521, 492, 498, 489, 508, 515, 521, 505, 483, 495, 515, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 541, 528, 459, 524, 495, 534, 522, 500, 543, 520, 515, 491, 506, 527, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 466, 521, 525, 493, 471, 502, 533, 506, 506, 518, 512, 475, 565, 534, 540, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 442, 528, 535, 498, 495, 490, 533, 533, 547, 521, 535, 505, 502, 508, 479, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 526, 473, 518, 523, 492, 509, 501, 509, 516, 523, 487, 483, 540, 518, 517, 25, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8320, 8184, 8216, 8115, 8250, 8143, 8145, 8163] avg=8192.0000 max=8320 max_rank=0 min=8115 min_rank=3 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32701 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[562, 521, 536, 478, 469, 492, 506, 505, 514, 506, 505, 528, 471, 523, 504, 506, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 517, 500, 521, 499, 541, 509, 519, 529, 531, 513, 507, 503, 502, 485, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 482, 505, 513, 544, 497, 474, 541, 486, 538, 477, 507, 482, 526, 519, 514, 51, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 491, 502, 509, 480, 511, 517, 489, 525, 497, 514, 527, 527, 525, 502, 477, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 511, 530, 455, 537, 539, 531, 515, 502, 517, 526, 521, 524, 496, 517, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 533, 518, 476, 495, 560, 520, 546, 508, 470, 519, 500, 491, 489, 528, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 470, 518, 515, 512, 513, 493, 525, 519, 509, 492, 501, 476, 534, 516, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 532, 519, 519, 507, 504, 526, 543, 505, 517, 548, 497, 519, 467, 521, 513, 28, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8137, 8228, 8181, 8150, 8254, 8180, 8149, 8257] avg=8192.0000 max=8257 max_rank=7 min=8137 min_rank=0 imbalance_ratio=1.007935 max_load_reduction=57279 max_load_reduction_pct=0.874008 moved_to_echo=32840 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[493, 533, 516, 498, 499, 495, 501, 535, 483, 501, 504, 480, 486, 528, 516, 524, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 517, 538, 529, 491, 528, 470, 518, 537, 506, 485, 493, 523, 508, 524, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 495, 483, 515, 511, 519, 492, 503, 518, 512, 538, 495, 525, 534, 511, 477, 45, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 507, 562, 466, 490, 533, 483, 498, 514, 553, 500, 509, 496, 496, 517, 489, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 534, 504, 495, 486, 504, 507, 519, 487, 531, 531, 536, 525, 503, 509, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 503, 516, 530, 531, 535, 531, 512, 496, 512, 489, 478, 525, 524, 539, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 516, 489, 482, 511, 490, 522, 521, 520, 476, 545, 529, 548, 512, 533, 563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 463, 534, 511, 508, 501, 531, 517, 499, 530, 492, 523, 515, 526, 483, 506, 511, 24, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8099, 8234, 8177, 8189, 8178, 8243, 8241, 8175] avg=8192.0000 max=8243 max_rank=5 min=8099 min_rank=0 imbalance_ratio=1.006226 max_load_reduction=57293 max_load_reduction_pct=0.874222 moved_to_echo=32837 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[562, 521, 536, 478, 469, 492, 506, 505, 514, 506, 505, 528, 471, 523, 504, 506, 537, 517, 500, 521, 499, 556, 509, 519, 529, 531, 513, 507, 503, 502, 485, 515, 514, 482, 505, 513, 544, 497, 474, 541, 486, 538, 477, 507, 482, 526, 519, 514, 495, 491, 502, 509, 480, 511, 517, 489, 525, 497, 514, 527, 527, 525, 502, 477, 496, 511, 530, 522, 537, 539, 531, 515, 502, 517, 526, 521, 524, 496, 517, 537, 494, 533, 518, 476, 495, 560, 520, 546, 508, 470, 519, 551, 491, 489, 528, 533, 526, 470, 518, 549, 512, 513, 493, 525, 519, 509, 492, 501, 476, 534, 516, 530, 488, 532, 519, 519, 507, 504, 526, 543, 505, 517, 548, 497, 519, 467, 521, 513] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[492, 531, 538, 585, 494, 528, 524, 539, 508, 513, 496, 493, 489, 560, 532, 519, 481, 481, 526, 527, 504, 545, 559, 531, 507, 480, 497, 478, 509, 517, 488, 482, 483, 494, 495, 516, 532, 500, 521, 507, 516, 512, 524, 561, 497, 462, 527, 491, 493, 528, 498, 508, 480, 506, 533, 468, 501, 517, 547, 524, 513, 491, 540, 538, 522, 495, 506, 550, 516, 529, 514, 539, 525, 491, 485, 467, 524, 483, 521, 530, 486, 536, 531, 526, 523, 523, 498, 528, 537, 503, 515, 503, 520, 541, 510, 515, 542, 462, 530, 539, 538, 490, 508, 524, 484, 528, 525, 477, 497, 501, 530, 511, 489, 509, 533, 505, 515, 513, 488, 534, 510, 473, 494, 486, 503, 491, 533, 506] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[507, 515, 502, 523, 526, 506, 502, 506, 482, 534, 559, 513, 498, 529, 498, 505, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 474, 513, 523, 488, 498, 477, 528, 513, 512, 505, 526, 508, 506, 533, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 506, 501, 517, 531, 492, 517, 484, 530, 536, 537, 523, 548, 494, 493, 514, 43, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 510, 529, 481, 505, 522, 535, 503, 513, 499, 487, 496, 492, 522, 495, 521, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 503, 458, 475, 549, 535, 475, 481, 516, 517, 527, 512, 510, 516, 537, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 528, 556, 488, 495, 462, 512, 493, 494, 553, 513, 456, 519, 515, 519, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 507, 500, 465, 516, 563, 481, 534, 489, 506, 523, 535, 502, 540, 533, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 562, 522, 529, 482, 533, 491, 498, 515, 519, 536, 476, 509, 540, 456, 563, 23, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8091, 8258, 8221, 8100, 8129, 8245, 8280] avg=8192.0000 max=8280 max_rank=7 min=8091 min_rank=1 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32754 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[492, 531, 538, 585, 494, 528, 524, 539, 508, 513, 496, 493, 489, 560, 532, 519, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 481, 526, 527, 504, 537, 559, 531, 507, 480, 497, 478, 509, 517, 488, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 494, 495, 516, 532, 500, 521, 507, 516, 512, 524, 561, 497, 462, 527, 491, 43, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 528, 498, 508, 480, 506, 533, 468, 501, 517, 547, 524, 513, 491, 540, 538, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 495, 506, 489, 516, 529, 514, 539, 525, 491, 485, 467, 524, 483, 521, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 536, 531, 526, 523, 523, 498, 528, 537, 503, 515, 460, 520, 541, 510, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 462, 530, 512, 538, 490, 508, 524, 484, 528, 525, 477, 497, 501, 530, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 509, 533, 505, 515, 513, 488, 534, 510, 473, 494, 486, 503, 491, 533, 506, 25, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8348, 8104, 8183, 8246, 8136, 8252, 8159, 8108] avg=8192.0000 max=8348 max_rank=0 min=8104 min_rank=1 imbalance_ratio=1.019043 max_load_reduction=57188 max_load_reduction_pct=0.872620 moved_to_echo=32655 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[515, 495, 555, 517, 519, 519, 500, 470, 522, 556, 537, 537, 521, 487, 463, 499, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 529, 492, 516, 506, 513, 529, 504, 469, 502, 502, 503, 527, 540, 508, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 495, 506, 462, 460, 499, 515, 520, 530, 453, 480, 464, 494, 521, 521, 491, 46, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 479, 534, 515, 514, 541, 484, 497, 518, 506, 544, 506, 528, 535, 515, 533, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 531, 541, 462, 531, 506, 508, 494, 525, 522, 527, 531, 502, 497, 484, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 526, 503, 515, 514, 492, 508, 519, 511, 544, 544, 494, 540, 505, 526, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 540, 481, 458, 502, 566, 523, 530, 523, 534, 476, 543, 546, 504, 480, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 517, 539, 471, 493, 495, 529, 482, 534, 510, 505, 489, 501, 544, 464, 524, 23, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 8210, 7971, 8313, 8220, 8210, 8229, 8164] avg=8192.0000 max=8313 max_rank=3 min=7971 min_rank=2 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32823 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[481, 544, 522, 495, 545, 500, 512, 489, 527, 490, 502, 529, 498, 494, 476, 507, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 520, 484, 471, 491, 523, 498, 499, 512, 535, 538, 504, 533, 548, 489, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 531, 525, 506, 512, 510, 526, 527, 535, 541, 450, 532, 524, 518, 507, 485, 44, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 505, 501, 529, 508, 501, 497, 495, 495, 505, 518, 507, 460, 485, 545, 491, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 493, 512, 447, 515, 504, 530, 538, 470, 487, 521, 519, 542, 498, 477, 567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 503, 511, 532, 510, 504, 531, 524, 533, 488, 518, 473, 518, 493, 500, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 531, 525, 506, 511, 523, 478, 519, 499, 521, 490, 515, 550, 536, 464, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 531, 510, 485, 533, 495, 547, 547, 529, 483, 535, 485, 540, 493, 509, 499, 25, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8118, 8140, 8310, 8141, 8152, 8215, 8200, 8260] avg=8192.0000 max=8310 max_rank=2 min=8118 min_rank=0 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32827 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[562, 521, 536, 478, 469, 492, 506, 505, 514, 506, 505, 528, 471, 523, 504, 506, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 517, 500, 521, 499, 541, 509, 519, 529, 531, 513, 507, 503, 502, 485, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 482, 505, 513, 544, 497, 474, 541, 486, 538, 477, 507, 482, 526, 519, 514, 51, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 491, 502, 509, 480, 511, 517, 489, 525, 497, 514, 527, 527, 525, 502, 477, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 511, 530, 455, 537, 539, 531, 515, 502, 517, 526, 521, 524, 496, 517, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 533, 518, 476, 495, 560, 520, 546, 508, 470, 519, 500, 491, 489, 528, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 470, 518, 515, 512, 513, 493, 525, 519, 509, 492, 501, 476, 534, 516, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 532, 519, 519, 507, 504, 526, 543, 505, 517, 548, 497, 519, 467, 521, 513, 28, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8137, 8228, 8181, 8150, 8254, 8180, 8149, 8257] avg=8192.0000 max=8257 max_rank=7 min=8137 min_rank=0 imbalance_ratio=1.007935 max_load_reduction=57279 max_load_reduction_pct=0.874008 moved_to_echo=32840 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[515, 495, 555, 517, 519, 519, 500, 470, 522, 556, 537, 537, 521, 487, 463, 499, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 529, 492, 516, 506, 513, 529, 504, 469, 502, 502, 503, 527, 540, 508, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 495, 506, 462, 460, 499, 515, 520, 530, 453, 480, 464, 494, 521, 521, 491, 46, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 479, 534, 515, 514, 541, 484, 497, 518, 506, 544, 506, 528, 535, 515, 533, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 531, 541, 462, 531, 506, 508, 494, 525, 522, 527, 531, 502, 497, 484, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 526, 503, 515, 514, 492, 508, 519, 511, 544, 544, 494, 540, 505, 526, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 540, 481, 458, 502, 566, 523, 530, 523, 534, 476, 543, 546, 504, 480, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 517, 539, 471, 493, 495, 529, 482, 534, 510, 505, 489, 501, 544, 464, 524, 23, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 8210, 7971, 8313, 8220, 8210, 8229, 8164] avg=8192.0000 max=8313 max_rank=3 min=7971 min_rank=2 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32823 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[493, 533, 516, 498, 499, 495, 501, 535, 483, 501, 504, 480, 486, 528, 516, 524, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 517, 538, 529, 491, 528, 470, 518, 537, 506, 485, 493, 523, 508, 524, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 495, 483, 515, 511, 519, 492, 503, 518, 512, 538, 495, 525, 534, 511, 477, 45, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 507, 562, 466, 490, 533, 483, 498, 514, 553, 500, 509, 496, 496, 517, 489, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 534, 504, 495, 486, 504, 507, 519, 487, 531, 531, 536, 525, 503, 509, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 503, 516, 530, 531, 535, 531, 512, 496, 512, 489, 478, 525, 524, 539, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 516, 489, 482, 511, 490, 522, 521, 520, 476, 545, 529, 548, 512, 533, 563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 463, 534, 511, 508, 501, 531, 517, 499, 530, 492, 523, 515, 526, 483, 506, 511, 24, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8099, 8234, 8177, 8189, 8178, 8243, 8241, 8175] avg=8192.0000 max=8243 max_rank=5 min=8099 min_rank=0 imbalance_ratio=1.006226 max_load_reduction=57293 max_load_reduction_pct=0.874222 moved_to_echo=32837 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[509, 539, 495, 474, 509, 501, 509, 472, 495, 531, 528, 496, 528, 490, 514, 485, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 459, 542, 519, 536, 581, 541, 499, 524, 521, 511, 514, 522, 544, 519, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 516, 520, 511, 517, 514, 505, 478, 523, 528, 530, 525, 511, 510, 497, 509, 47, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 493, 496, 475, 499, 499, 543, 502, 522, 517, 511, 502, 515, 504, 502, 510, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 537, 531, 459, 518, 500, 509, 549, 509, 524, 549, 503, 497, 488, 540, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 488, 497, 493, 536, 495, 545, 512, 495, 485, 521, 505, 520, 518, 517, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 489, 457, 541, 541, 541, 497, 502, 509, 492, 520, 527, 506, 484, 535, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 506, 509, 461, 545, 511, 549, 502, 500, 488, 476, 496, 487, 514, 524, 509, 27, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8083, 8345, 8240, 8161, 8246, 8164, 8168, 8129] avg=8192.0000 max=8345 max_rank=1 min=8083 min_rank=0 imbalance_ratio=1.018677 max_load_reduction=57191 max_load_reduction_pct=0.872665 moved_to_echo=32707 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[512, 524, 495, 562, 493, 533, 542, 513, 493, 524, 524, 536, 504, 528, 496, 535, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 527, 489, 538, 515, 483, 522, 535, 533, 485, 538, 511, 525, 491, 506, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 502, 511, 528, 533, 509, 554, 523, 487, 494, 466, 558, 504, 523, 475, 505, 44, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 497, 501, 483, 503, 521, 492, 498, 489, 508, 515, 521, 505, 483, 495, 515, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 541, 528, 459, 524, 495, 534, 522, 500, 543, 520, 515, 491, 506, 527, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 466, 521, 525, 493, 471, 502, 533, 506, 506, 518, 512, 475, 565, 534, 540, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 442, 528, 535, 498, 495, 490, 533, 533, 547, 521, 535, 505, 502, 508, 479, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 526, 473, 518, 523, 492, 509, 501, 509, 516, 523, 487, 483, 540, 518, 517, 25, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8320, 8184, 8216, 8115, 8250, 8143, 8145, 8163] avg=8192.0000 max=8320 max_rank=0 min=8115 min_rank=3 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32701 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[492, 531, 538, 585, 494, 528, 524, 539, 508, 513, 496, 493, 489, 560, 532, 519, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 481, 526, 527, 504, 537, 559, 531, 507, 480, 497, 478, 509, 517, 488, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 494, 495, 516, 532, 500, 521, 507, 516, 512, 524, 561, 497, 462, 527, 491, 43, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 528, 498, 508, 480, 506, 533, 468, 501, 517, 547, 524, 513, 491, 540, 538, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 495, 506, 489, 516, 529, 514, 539, 525, 491, 485, 467, 524, 483, 521, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 536, 531, 526, 523, 523, 498, 528, 537, 503, 515, 460, 520, 541, 510, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 462, 530, 512, 538, 490, 508, 524, 484, 528, 525, 477, 497, 501, 530, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 509, 533, 505, 515, 513, 488, 534, 510, 473, 494, 486, 503, 491, 533, 506, 25, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8348, 8104, 8183, 8246, 8136, 8252, 8159, 8108] avg=8192.0000 max=8348 max_rank=0 min=8104 min_rank=1 imbalance_ratio=1.019043 max_load_reduction=57188 max_load_reduction_pct=0.872620 moved_to_echo=32655 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4224, 4224, 3840, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 3968, 4096, 4224, 4096, 3968, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 384, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 3968, 4096, 4224, 4096, 3968, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 3968, 4096, 4096, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 3968, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4224, 4224, 4224, 3840, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[483, 514, 552, 517, 516, 503, 540, 482, 508, 500, 525, 510, 533, 526, 512, 530, 504, 498, 504, 479, 521, 492, 509, 537, 526, 519, 539, 511, 488, 523, 492, 518, 499, 519, 480, 515, 540, 511, 555, 566, 525, 499, 493, 485, 525, 505, 527, 496, 481, 495, 530, 475, 549, 503, 542, 548, 541, 508, 534, 487, 510, 537, 472, 523, 522, 537, 511, 487, 525, 489, 504, 503, 515, 532, 479, 507, 507, 508, 518, 525, 551, 518, 518, 529, 531, 464, 494, 529, 488, 474, 505, 504, 504, 532, 511, 523, 481, 501, 520, 494, 518, 529, 480, 504, 531, 471, 535, 515, 520, 533, 503, 523, 527, 480, 491, 486, 507, 496, 550, 494, 496, 530, 487, 512, 494, 541, 547, 510] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[509, 502, 470, 547, 524, 529, 528, 551, 517, 517, 537, 525, 502, 498, 541, 521, 505, 525, 548, 453, 491, 514, 508, 494, 517, 492, 530, 513, 487, 530, 523, 506, 498, 531, 537, 520, 530, 519, 516, 494, 541, 495, 507, 507, 517, 529, 510, 510, 473, 512, 516, 523, 564, 523, 511, 473, 493, 546, 498, 499, 537, 468, 518, 542, 513, 475, 515, 486, 514, 488, 481, 502, 507, 482, 508, 504, 483, 506, 519, 524, 501, 521, 519, 519, 508, 500, 531, 503, 510, 525, 508, 516, 511, 539, 478, 533, 498, 510, 509, 508, 497, 499, 479, 503, 543, 535, 515, 527, 505, 518, 547, 517, 532, 484, 531, 537, 525, 504, 515, 504, 518, 468, 483, 503, 547, 513, 508, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[524, 512, 511, 470, 531, 524, 513, 493, 551, 509, 474, 508, 556, 491, 550, 493, 503, 495, 492, 515, 530, 531, 517, 501, 525, 493, 488, 515, 508, 527, 519, 514, 530, 519, 498, 474, 484, 503, 513, 547, 503, 529, 540, 511, 494, 508, 495, 499, 548, 512, 520, 508, 569, 516, 504, 501, 528, 506, 528, 491, 513, 492, 494, 508, 517, 522, 487, 546, 538, 548, 541, 506, 508, 505, 559, 545, 501, 483, 563, 520, 484, 533, 509, 519, 523, 463, 493, 534, 515, 483, 516, 509, 499, 529, 538, 506, 547, 490, 505, 461, 489, 506, 467, 487, 491, 522, 540, 508, 493, 498, 545, 498, 536, 498, 502, 499, 517, 491, 559, 502, 521, 527, 493, 507, 531, 534, 505, 457] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[488, 516, 516, 482, 519, 521, 541, 503, 493, 535, 555, 503, 530, 518, 496, 520, 486, 526, 539, 499, 494, 512, 519, 485, 525, 497, 536, 487, 499, 496, 521, 528, 538, 522, 458, 517, 492, 539, 499, 515, 504, 570, 506, 528, 508, 511, 516, 533, 492, 492, 500, 520, 522, 530, 475, 516, 527, 471, 488, 515, 552, 530, 537, 515, 531, 551, 538, 468, 518, 529, 500, 506, 534, 471, 497, 495, 532, 495, 509, 500, 496, 518, 539, 506, 518, 483, 534, 554, 485, 550, 515, 511, 487, 513, 488, 493, 511, 504, 533, 517, 479, 506, 471, 500, 512, 521, 480, 551, 490, 499, 543, 507, 530, 536, 498, 520, 538, 506, 516, 559, 541, 503, 525, 505, 481, 509, 493, 465] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[489, 500, 510, 514, 538, 536, 541, 493, 501, 542, 530, 579, 482, 495, 521, 499, 508, 499, 518, 524, 479, 525, 546, 497, 535, 468, 531, 503, 531, 512, 501, 508, 536, 517, 519, 526, 516, 493, 520, 522, 500, 503, 500, 519, 552, 461, 505, 497, 452, 513, 497, 489, 502, 503, 531, 481, 505, 490, 507, 512, 489, 555, 500, 519, 540, 514, 500, 514, 490, 537, 505, 497, 504, 522, 546, 519, 485, 492, 501, 543, 474, 504, 524, 526, 502, 525, 487, 498, 500, 564, 533, 503, 522, 516, 525, 523, 507, 561, 533, 554, 506, 539, 501, 510, 518, 570, 508, 527, 488, 481, 507, 493, 520, 524, 508, 499, 478, 491, 519, 528, 499, 486, 501, 561, 489, 506, 531, 472] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[521, 466, 518, 526, 528, 530, 480, 503, 509, 528, 517, 528, 513, 486, 539, 485, 502, 541, 524, 504, 549, 490, 543, 503, 508, 523, 488, 508, 523, 545, 505, 479, 488, 471, 532, 496, 522, 512, 514, 488, 520, 493, 553, 557, 521, 501, 503, 526, 507, 512, 508, 501, 556, 529, 535, 525, 497, 518, 481, 534, 496, 570, 514, 496, 542, 488, 537, 525, 476, 530, 489, 511, 499, 479, 485, 520, 532, 454, 527, 498, 524, 522, 536, 511, 488, 525, 500, 494, 551, 529, 504, 488, 501, 530, 525, 507, 513, 546, 528, 519, 528, 523, 498, 530, 497, 556, 500, 478, 521, 483, 490, 509, 502, 483, 469, 523, 531, 536, 515, 529, 475, 517, 521, 489, 492, 535, 489, 496] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[502, 540, 510, 508, 571, 557, 541, 513, 497, 518, 447, 468, 525, 507, 493, 504, 553, 540, 530, 496, 506, 522, 503, 535, 496, 495, 544, 499, 502, 516, 557, 527, 532, 515, 499, 530, 552, 515, 505, 484, 515, 487, 537, 494, 469, 514, 549, 533, 517, 451, 477, 509, 477, 514, 524, 507, 541, 503, 520, 504, 512, 492, 522, 498, 476, 533, 489, 559, 491, 498, 481, 478, 515, 499, 559, 537, 520, 506, 537, 497, 493, 503, 531, 502, 528, 507, 505, 502, 503, 514, 519, 564, 521, 510, 509, 497, 532, 526, 497, 505, 530, 486, 504, 509, 494, 498, 505, 524, 510, 465, 483, 501, 508, 501, 526, 514, 488, 524, 523, 487, 524, 573, 492, 518, 491, 555, 517, 523] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[524, 512, 511, 470, 531, 524, 513, 493, 551, 509, 474, 508, 556, 491, 550, 493, 503, 495, 492, 515, 530, 531, 517, 501, 525, 493, 488, 515, 508, 527, 519, 514, 530, 519, 498, 474, 484, 503, 513, 547, 503, 529, 540, 511, 494, 508, 495, 499, 548, 512, 520, 508, 569, 516, 504, 501, 528, 506, 528, 491, 513, 492, 494, 508, 517, 522, 487, 546, 538, 548, 541, 506, 508, 505, 559, 545, 501, 483, 563, 520, 484, 533, 509, 519, 523, 463, 493, 534, 515, 483, 516, 509, 499, 529, 538, 506, 547, 490, 505, 461, 489, 506, 467, 487, 491, 522, 540, 508, 493, 498, 545, 498, 536, 498, 502, 499, 517, 491, 559, 502, 521, 527, 493, 507, 531, 534, 505, 457] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[509, 502, 470, 547, 524, 529, 528, 551, 517, 517, 537, 525, 502, 498, 541, 521, 505, 525, 548, 453, 491, 514, 508, 494, 517, 492, 530, 513, 487, 530, 523, 506, 498, 531, 537, 520, 530, 519, 516, 494, 541, 495, 507, 507, 517, 529, 510, 510, 473, 512, 516, 523, 564, 523, 511, 473, 493, 546, 498, 499, 537, 468, 518, 542, 513, 475, 515, 486, 514, 488, 481, 502, 507, 482, 508, 504, 483, 506, 519, 524, 501, 521, 519, 519, 508, 500, 531, 503, 510, 525, 508, 516, 511, 539, 478, 533, 498, 510, 509, 508, 497, 499, 479, 503, 543, 535, 515, 527, 505, 518, 547, 517, 532, 484, 531, 537, 525, 504, 515, 504, 518, 468, 483, 503, 547, 513, 508, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[489, 500, 510, 514, 538, 536, 541, 493, 501, 542, 530, 579, 482, 495, 521, 499, 508, 499, 518, 524, 479, 525, 546, 497, 535, 468, 531, 503, 531, 512, 501, 508, 536, 517, 519, 526, 516, 493, 520, 522, 500, 503, 500, 519, 552, 461, 505, 497, 452, 513, 497, 489, 502, 503, 531, 481, 505, 490, 507, 512, 489, 555, 500, 519, 540, 514, 500, 514, 490, 537, 505, 497, 504, 522, 546, 519, 485, 492, 501, 543, 474, 504, 524, 526, 502, 525, 487, 498, 500, 564, 533, 503, 522, 516, 525, 523, 507, 561, 533, 554, 506, 539, 501, 510, 518, 570, 508, 527, 488, 481, 507, 493, 520, 524, 508, 499, 478, 491, 519, 528, 499, 486, 501, 561, 489, 506, 531, 472] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[520, 472, 503, 522, 552, 564, 495, 474, 494, 497, 493, 551, 530, 475, 515, 490, 489, 526, 492, 529, 485, 510, 498, 505, 506, 519, 463, 507, 510, 500, 534, 553, 550, 464, 543, 516, 493, 518, 521, 522, 489, 523, 494, 494, 510, 534, 523, 485, 538, 524, 533, 558, 474, 502, 503, 534, 515, 520, 503, 539, 494, 511, 493, 498, 544, 490, 505, 531, 516, 511, 476, 500, 498, 513, 542, 534, 519, 520, 523, 524, 503, 494, 522, 546, 516, 537, 507, 483, 530, 478, 520, 523, 482, 497, 517, 492, 519, 527, 516, 492, 495, 554, 551, 513, 498, 551, 491, 513, 501, 468, 499, 537, 527, 497, 522, 512, 491, 479, 483, 504, 622, 498, 510, 499, 470, 557, 519, 537] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[521, 466, 518, 526, 528, 530, 480, 503, 509, 528, 517, 528, 513, 486, 539, 485, 502, 541, 524, 504, 549, 490, 543, 503, 508, 523, 488, 508, 523, 545, 505, 479, 488, 471, 532, 496, 522, 512, 514, 488, 520, 493, 553, 557, 521, 501, 503, 526, 507, 512, 508, 501, 556, 529, 535, 525, 497, 518, 481, 534, 496, 570, 514, 496, 542, 488, 537, 525, 476, 530, 489, 511, 499, 479, 485, 520, 532, 454, 527, 498, 524, 522, 536, 511, 488, 525, 500, 494, 551, 529, 504, 488, 501, 530, 525, 507, 513, 546, 528, 519, 528, 523, 498, 530, 497, 556, 500, 478, 521, 483, 490, 509, 502, 483, 469, 523, 531, 536, 515, 529, 475, 517, 521, 489, 492, 535, 489, 496] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[483, 514, 552, 517, 476, 503, 540, 482, 508, 500, 525, 510, 533, 526, 512, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 498, 504, 479, 521, 492, 509, 537, 526, 519, 539, 511, 488, 523, 492, 518, 9, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 519, 480, 515, 540, 511, 555, 566, 525, 499, 493, 485, 525, 505, 527, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 495, 530, 475, 549, 503, 542, 548, 541, 508, 534, 487, 510, 537, 472, 523, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 537, 511, 487, 525, 489, 504, 503, 515, 532, 479, 507, 507, 508, 518, 525, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 518, 512, 529, 531, 464, 494, 529, 488, 474, 505, 504, 504, 532, 511, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 501, 520, 494, 518, 529, 480, 504, 531, 471, 535, 515, 520, 533, 503, 523, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 480, 491, 486, 507, 496, 550, 494, 496, 530, 487, 512, 494, 541, 547, 510, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8211, 8175, 8206, 8246, 8183, 8169, 8183, 8163] avg=8192.0000 max=8246 max_rank=3 min=8163 min_rank=7 imbalance_ratio=1.006592 max_load_reduction=57290 max_load_reduction_pct=0.874176 moved_to_echo=32698 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[524, 512, 511, 470, 498, 524, 513, 493, 551, 509, 474, 508, 556, 491, 550, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 495, 492, 515, 530, 531, 517, 501, 525, 493, 488, 515, 508, 527, 519, 514, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 519, 498, 474, 484, 503, 513, 547, 503, 529, 540, 511, 494, 508, 495, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 512, 520, 508, 569, 516, 504, 501, 528, 506, 528, 491, 513, 492, 494, 508, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 522, 487, 546, 538, 548, 541, 506, 508, 505, 559, 545, 501, 483, 563, 520, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 533, 507, 519, 523, 463, 493, 534, 515, 483, 516, 509, 499, 529, 538, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 490, 505, 461, 489, 506, 467, 487, 491, 522, 540, 508, 493, 498, 545, 498, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 498, 502, 499, 517, 491, 559, 502, 521, 527, 493, 507, 531, 534, 505, 457, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8178, 8129, 8244, 8398, 8151, 8069, 8190] avg=8192.0000 max=8398 max_rank=4 min=8069 min_rank=6 imbalance_ratio=1.025146 max_load_reduction=57138 max_load_reduction_pct=0.871857 moved_to_echo=32808 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[502, 540, 510, 508, 535, 557, 541, 513, 497, 518, 447, 468, 525, 507, 493, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 540, 530, 496, 506, 522, 503, 535, 496, 495, 544, 499, 502, 516, 557, 527, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 515, 499, 530, 552, 515, 505, 484, 515, 487, 537, 494, 469, 514, 549, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 451, 477, 509, 477, 514, 524, 507, 541, 503, 520, 504, 512, 492, 522, 498, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 533, 489, 559, 491, 498, 481, 478, 515, 499, 559, 537, 520, 506, 537, 497, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 503, 529, 502, 528, 507, 505, 502, 503, 514, 519, 564, 521, 510, 509, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 526, 497, 505, 530, 486, 504, 509, 494, 498, 505, 524, 510, 465, 483, 501, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 501, 526, 514, 488, 524, 523, 487, 524, 573, 492, 518, 491, 555, 517, 523, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8165, 8326, 8212, 8074, 8184, 8206, 8093, 8276] avg=8192.0000 max=8326 max_rank=1 min=8074 min_rank=3 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32759 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[509, 502, 470, 547, 491, 529, 528, 551, 517, 517, 537, 525, 502, 498, 541, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 525, 548, 453, 491, 514, 508, 494, 517, 492, 530, 513, 487, 530, 523, 506, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 531, 537, 520, 530, 519, 516, 494, 541, 495, 507, 507, 517, 529, 510, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 512, 516, 523, 564, 523, 511, 473, 493, 546, 498, 499, 537, 468, 518, 542, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 475, 515, 486, 514, 488, 481, 502, 507, 482, 508, 504, 483, 506, 519, 524, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 521, 517, 519, 508, 500, 531, 503, 510, 525, 508, 516, 511, 539, 478, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 510, 509, 508, 497, 499, 479, 503, 543, 535, 515, 527, 505, 518, 547, 517, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 484, 531, 537, 525, 504, 515, 504, 518, 468, 483, 503, 547, 513, 508, 514, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8285, 8141, 8244, 8202, 8015, 8220, 8232, 8197] avg=8192.0000 max=8285 max_rank=0 min=8015 min_rank=4 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32664 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[488, 516, 516, 482, 487, 521, 541, 503, 493, 535, 555, 503, 530, 518, 496, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 526, 539, 499, 494, 512, 519, 485, 525, 497, 536, 487, 499, 496, 521, 528, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 522, 458, 517, 492, 539, 499, 515, 504, 570, 506, 528, 508, 511, 516, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 492, 500, 520, 522, 530, 475, 516, 527, 471, 488, 515, 552, 530, 537, 515, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 551, 538, 468, 518, 529, 500, 506, 534, 471, 497, 495, 532, 495, 509, 500, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 518, 537, 506, 518, 483, 534, 554, 485, 550, 515, 511, 487, 513, 488, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 504, 533, 517, 479, 506, 471, 500, 512, 521, 480, 551, 490, 499, 543, 507, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 536, 498, 520, 538, 506, 516, 559, 541, 503, 525, 505, 481, 509, 493, 465, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8154, 8238, 8188, 8183, 8188, 8145, 8236] avg=8192.0000 max=8238 max_rank=2 min=8145 min_rank=6 imbalance_ratio=1.005615 max_load_reduction=57298 max_load_reduction_pct=0.874298 moved_to_echo=32752 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[521, 466, 518, 526, 495, 530, 480, 503, 509, 528, 517, 528, 513, 486, 539, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 541, 524, 504, 549, 490, 543, 503, 508, 523, 488, 508, 523, 545, 505, 479, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 471, 532, 496, 522, 512, 514, 488, 520, 493, 553, 557, 521, 501, 503, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 512, 508, 501, 556, 529, 535, 525, 497, 518, 481, 534, 496, 570, 514, 496, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 488, 537, 525, 476, 530, 489, 511, 499, 479, 485, 520, 532, 454, 527, 498, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 522, 534, 511, 488, 525, 500, 494, 551, 529, 504, 488, 501, 530, 525, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 546, 528, 519, 528, 523, 498, 530, 497, 556, 500, 478, 521, 483, 490, 509, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 483, 469, 523, 531, 536, 515, 529, 475, 517, 521, 489, 492, 535, 489, 496, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8144, 8240, 8180, 8285, 8100, 8233, 8241, 8113] avg=8192.0000 max=8285 max_rank=3 min=8100 min_rank=4 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32687 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[489, 500, 510, 514, 505, 536, 541, 493, 501, 542, 530, 579, 482, 495, 521, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 499, 518, 524, 479, 525, 546, 497, 535, 468, 531, 503, 531, 512, 501, 508, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 517, 519, 526, 516, 493, 520, 522, 500, 503, 500, 519, 552, 461, 505, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 452, 513, 497, 489, 502, 503, 531, 481, 505, 490, 507, 512, 489, 555, 500, 519, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 514, 500, 514, 490, 537, 505, 497, 504, 522, 546, 519, 485, 492, 501, 543, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 504, 522, 526, 502, 525, 487, 498, 500, 564, 533, 503, 522, 516, 525, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 561, 533, 554, 506, 539, 501, 510, 518, 570, 508, 527, 488, 481, 507, 493, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 524, 508, 499, 478, 491, 519, 528, 499, 486, 501, 561, 489, 506, 531, 472, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8237, 8190, 8168, 8051, 8218, 8224, 8325, 8123] avg=8192.0000 max=8325 max_rank=6 min=8051 min_rank=3 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32890 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[520, 472, 503, 522, 518, 564, 495, 474, 494, 497, 493, 551, 530, 475, 515, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 526, 492, 529, 485, 510, 498, 505, 506, 519, 463, 507, 510, 500, 534, 553, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 464, 543, 516, 493, 518, 521, 522, 489, 523, 494, 494, 510, 534, 523, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 524, 533, 558, 474, 502, 503, 534, 515, 520, 503, 539, 494, 511, 493, 498, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 490, 505, 531, 516, 511, 476, 500, 498, 513, 542, 534, 519, 520, 523, 524, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 494, 520, 546, 516, 537, 507, 483, 530, 478, 520, 523, 482, 497, 517, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 527, 516, 492, 495, 554, 551, 513, 498, 551, 491, 513, 501, 468, 499, 537, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 497, 522, 512, 491, 479, 483, 504, 622, 498, 510, 499, 470, 557, 519, 537, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8113, 8132, 8159, 8246, 8255, 8145, 8248, 8238] avg=8192.0000 max=8255 max_rank=4 min=8113 min_rank=0 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32886 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[488, 516, 516, 482, 487, 521, 541, 503, 493, 535, 555, 503, 530, 518, 496, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 526, 539, 499, 494, 512, 519, 485, 525, 497, 536, 487, 499, 496, 521, 528, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 522, 458, 517, 492, 539, 499, 515, 504, 570, 506, 528, 508, 511, 516, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 492, 500, 520, 522, 530, 475, 516, 527, 471, 488, 515, 552, 530, 537, 515, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 551, 538, 468, 518, 529, 500, 506, 534, 471, 497, 495, 532, 495, 509, 500, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 518, 537, 506, 518, 483, 534, 554, 485, 550, 515, 511, 487, 513, 488, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 504, 533, 517, 479, 506, 471, 500, 512, 521, 480, 551, 490, 499, 543, 507, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 536, 498, 520, 538, 506, 516, 559, 541, 503, 525, 505, 481, 509, 493, 465, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8154, 8238, 8188, 8183, 8188, 8145, 8236] avg=8192.0000 max=8238 max_rank=2 min=8145 min_rank=6 imbalance_ratio=1.005615 max_load_reduction=57298 max_load_reduction_pct=0.874298 moved_to_echo=32752 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[502, 540, 510, 508, 535, 557, 541, 513, 497, 518, 447, 468, 525, 507, 493, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 540, 530, 496, 506, 522, 503, 535, 496, 495, 544, 499, 502, 516, 557, 527, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 515, 499, 530, 552, 515, 505, 484, 515, 487, 537, 494, 469, 514, 549, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 451, 477, 509, 477, 514, 524, 507, 541, 503, 520, 504, 512, 492, 522, 498, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 533, 489, 559, 491, 498, 481, 478, 515, 499, 559, 537, 520, 506, 537, 497, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 503, 529, 502, 528, 507, 505, 502, 503, 514, 519, 564, 521, 510, 509, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 526, 497, 505, 530, 486, 504, 509, 494, 498, 505, 524, 510, 465, 483, 501, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 501, 526, 514, 488, 524, 523, 487, 524, 573, 492, 518, 491, 555, 517, 523, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8165, 8326, 8212, 8074, 8184, 8206, 8093, 8276] avg=8192.0000 max=8326 max_rank=1 min=8074 min_rank=3 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32759 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[524, 512, 511, 470, 498, 524, 513, 493, 551, 509, 474, 508, 556, 491, 550, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 495, 492, 515, 530, 531, 517, 501, 525, 493, 488, 515, 508, 527, 519, 514, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 519, 498, 474, 484, 503, 513, 547, 503, 529, 540, 511, 494, 508, 495, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 512, 520, 508, 569, 516, 504, 501, 528, 506, 528, 491, 513, 492, 494, 508, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 522, 487, 546, 538, 548, 541, 506, 508, 505, 559, 545, 501, 483, 563, 520, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 533, 507, 519, 523, 463, 493, 534, 515, 483, 516, 509, 499, 529, 538, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 490, 505, 461, 489, 506, 467, 487, 491, 522, 540, 508, 493, 498, 545, 498, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 498, 502, 499, 517, 491, 559, 502, 521, 527, 493, 507, 531, 534, 505, 457, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8178, 8129, 8244, 8398, 8151, 8069, 8190] avg=8192.0000 max=8398 max_rank=4 min=8069 min_rank=6 imbalance_ratio=1.025146 max_load_reduction=57138 max_load_reduction_pct=0.871857 moved_to_echo=32808 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4352, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 3968, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4352, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 3968, 4096, 4096, 4224, 4096, 4224, 4096, 3968, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 4352, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[484, 469, 519, 521, 494, 509, 495, 481, 499, 502, 520, 516, 513, 520, 523, 534, 500, 526, 521, 543, 487, 537, 502, 473, 543, 488, 542, 508, 513, 513, 527, 502, 461, 531, 472, 543, 545, 494, 549, 523, 560, 510, 483, 478, 534, 560, 480, 528, 517, 496, 511, 513, 505, 549, 498, 498, 512, 512, 508, 473, 507, 524, 500, 508, 509, 495, 478, 483, 543, 479, 512, 490, 538, 531, 498, 517, 561, 559, 510, 512, 491, 492, 508, 535, 528, 525, 502, 504, 539, 527, 505, 461, 539, 513, 524, 513, 549, 523, 533, 507, 508, 517, 489, 497, 492, 504, 518, 533, 503, 521, 537, 498, 513, 526, 520, 519, 547, 534, 486, 499, 502, 466, 515, 511, 510, 479, 549, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[550, 522, 506, 499, 510, 488, 473, 501, 497, 540, 479, 555, 522, 538, 476, 528, 487, 466, 518, 498, 495, 534, 533, 523, 510, 548, 535, 465, 491, 516, 497, 479, 482, 505, 519, 526, 493, 506, 530, 485, 510, 501, 497, 532, 547, 515, 506, 494, 543, 478, 511, 524, 552, 523, 530, 514, 493, 500, 506, 520, 515, 526, 493, 492, 514, 514, 514, 515, 526, 502, 534, 522, 520, 517, 515, 501, 516, 518, 505, 489, 559, 549, 508, 529, 521, 524, 502, 491, 551, 551, 520, 528, 512, 531, 491, 511, 501, 526, 483, 529, 497, 531, 511, 485, 524, 495, 506, 565, 500, 494, 508, 494, 453, 490, 500, 475, 493, 496, 518, 507, 560, 480, 521, 532, 554, 490, 526, 545] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[497, 479, 500, 499, 523, 522, 532, 492, 527, 500, 561, 478, 541, 494, 534, 496, 501, 521, 479, 528, 523, 508, 521, 523, 514, 513, 520, 516, 513, 491, 489, 486, 535, 521, 533, 526, 527, 513, 509, 522, 511, 564, 496, 514, 513, 509, 543, 478, 524, 512, 530, 456, 516, 527, 538, 513, 490, 500, 506, 521, 483, 495, 511, 514, 501, 476, 469, 560, 498, 511, 516, 523, 495, 481, 536, 504, 523, 497, 543, 539, 512, 480, 546, 520, 496, 544, 506, 535, 536, 508, 520, 467, 487, 488, 462, 503, 495, 519, 525, 489, 505, 513, 544, 493, 512, 514, 538, 520, 510, 547, 473, 508, 528, 528, 525, 564, 492, 491, 495, 505, 509, 529, 514, 526, 531, 517, 522, 502] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[492, 464, 499, 507, 497, 510, 484, 518, 552, 515, 495, 544, 545, 527, 526, 553, 509, 487, 526, 540, 549, 496, 489, 474, 510, 539, 506, 515, 491, 517, 523, 540, 514, 497, 507, 530, 555, 493, 519, 508, 533, 539, 515, 521, 528, 525, 523, 511, 507, 501, 518, 495, 516, 503, 529, 557, 524, 526, 464, 469, 491, 518, 582, 479, 493, 526, 550, 491, 504, 502, 525, 552, 511, 526, 495, 537, 513, 514, 518, 530, 507, 474, 491, 528, 526, 468, 521, 481, 527, 492, 484, 534, 513, 504, 497, 513, 480, 470, 524, 495, 486, 519, 491, 504, 481, 533, 521, 492, 517, 545, 512, 522, 514, 508, 506, 494, 499, 542, 479, 489, 544, 529, 529, 507, 506, 527, 484, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[484, 469, 519, 521, 494, 509, 495, 481, 499, 502, 520, 516, 513, 520, 523, 534, 500, 526, 521, 543, 487, 537, 502, 473, 543, 488, 542, 508, 513, 513, 527, 502, 461, 531, 472, 543, 545, 494, 549, 523, 560, 510, 483, 478, 534, 560, 480, 528, 517, 496, 511, 513, 505, 549, 498, 498, 512, 512, 508, 473, 507, 524, 500, 508, 509, 495, 478, 483, 543, 479, 512, 490, 538, 531, 498, 517, 561, 559, 510, 512, 491, 492, 508, 535, 528, 525, 502, 504, 539, 527, 505, 461, 539, 513, 524, 513, 549, 523, 533, 507, 508, 517, 489, 497, 492, 504, 518, 533, 503, 521, 537, 498, 513, 526, 520, 519, 547, 534, 486, 499, 502, 466, 515, 511, 510, 479, 549, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[537, 517, 490, 502, 509, 516, 499, 532, 499, 478, 510, 507, 520, 541, 504, 503, 496, 511, 476, 506, 488, 537, 487, 513, 513, 515, 531, 504, 491, 506, 532, 500, 487, 532, 530, 523, 519, 539, 482, 519, 496, 519, 505, 451, 497, 518, 483, 525, 510, 530, 501, 549, 517, 500, 533, 514, 538, 534, 511, 522, 517, 496, 496, 547, 518, 524, 526, 529, 543, 529, 487, 493, 542, 503, 492, 513, 525, 492, 502, 576, 472, 505, 474, 499, 524, 540, 521, 482, 482, 511, 496, 498, 550, 541, 498, 475, 511, 506, 479, 491, 511, 502, 501, 523, 548, 533, 553, 556, 511, 487, 460, 543, 500, 478, 533, 504, 510, 530, 518, 541, 527, 512, 539, 519, 519, 507, 505, 507] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[514, 480, 525, 563, 485, 529, 525, 499, 519, 518, 534, 510, 512, 505, 529, 502, 520, 521, 518, 534, 524, 549, 535, 497, 506, 529, 496, 537, 483, 531, 509, 502, 504, 511, 515, 512, 540, 519, 505, 494, 542, 500, 494, 536, 481, 498, 467, 524, 492, 504, 503, 501, 532, 510, 486, 522, 490, 500, 507, 554, 531, 516, 529, 550, 489, 477, 511, 503, 519, 468, 524, 511, 481, 524, 546, 524, 521, 554, 536, 479, 526, 528, 490, 536, 527, 501, 529, 516, 511, 512, 535, 477, 509, 502, 494, 516, 488, 524, 465, 516, 490, 509, 500, 527, 531, 498, 522, 490, 526, 519, 516, 531, 496, 486, 504, 487, 475, 526, 488, 525, 525, 463, 537, 519, 515, 510, 529, 514] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[492, 464, 499, 507, 497, 510, 484, 518, 552, 515, 495, 544, 545, 527, 526, 553, 509, 487, 526, 540, 549, 496, 489, 474, 510, 539, 506, 515, 491, 517, 523, 540, 514, 497, 507, 530, 555, 493, 519, 508, 533, 539, 515, 521, 528, 525, 523, 511, 507, 501, 518, 495, 516, 503, 529, 557, 524, 526, 464, 469, 491, 518, 582, 479, 493, 526, 550, 491, 504, 502, 525, 552, 511, 526, 495, 537, 513, 514, 518, 530, 507, 474, 491, 528, 526, 468, 521, 481, 527, 492, 484, 534, 513, 504, 497, 513, 480, 470, 524, 495, 486, 519, 491, 504, 481, 533, 521, 492, 517, 545, 512, 522, 514, 508, 506, 494, 499, 542, 479, 489, 544, 529, 529, 507, 506, 527, 484, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[489, 461, 530, 517, 506, 481, 491, 535, 516, 474, 528, 512, 508, 533, 519, 522, 465, 515, 520, 512, 525, 498, 540, 532, 504, 491, 507, 533, 535, 512, 473, 527, 486, 480, 530, 516, 530, 482, 514, 479, 510, 498, 555, 526, 491, 515, 502, 514, 532, 527, 512, 494, 501, 528, 521, 511, 485, 496, 505, 541, 514, 532, 526, 525, 520, 476, 537, 525, 530, 484, 499, 493, 535, 509, 508, 498, 502, 525, 529, 536, 507, 487, 525, 487, 507, 481, 511, 519, 526, 527, 554, 495, 539, 505, 500, 522, 524, 525, 514, 476, 480, 535, 505, 504, 479, 520, 564, 555, 487, 492, 517, 510, 506, 530, 507, 532, 520, 522, 487, 492, 481, 533, 533, 539, 534, 528, 515, 503] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[521, 471, 534, 488, 486, 504, 496, 522, 508, 522, 486, 527, 541, 483, 519, 490, 517, 515, 496, 511, 507, 517, 540, 488, 502, 515, 532, 541, 480, 518, 476, 516, 524, 542, 505, 473, 544, 475, 554, 474, 542, 525, 492, 495, 475, 516, 500, 499, 484, 508, 514, 510, 529, 557, 544, 543, 460, 502, 541, 518, 535, 468, 489, 540, 452, 530, 513, 517, 532, 520, 501, 548, 549, 470, 494, 506, 519, 504, 504, 526, 505, 487, 492, 524, 513, 478, 497, 524, 530, 532, 524, 521, 538, 488, 512, 520, 500, 547, 502, 524, 521, 499, 561, 478, 496, 525, 525, 490, 459, 517, 548, 551, 501, 492, 528, 556, 513, 515, 523, 537, 495, 531, 544, 494, 496, 503, 555, 494] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[537, 517, 490, 502, 509, 516, 499, 532, 499, 478, 510, 507, 520, 541, 504, 503, 496, 511, 476, 506, 488, 537, 487, 513, 513, 515, 531, 504, 491, 506, 532, 500, 487, 532, 530, 523, 519, 539, 482, 519, 496, 519, 505, 451, 497, 518, 483, 525, 510, 530, 501, 549, 517, 500, 533, 514, 538, 534, 511, 522, 517, 496, 496, 547, 518, 524, 526, 529, 543, 529, 487, 493, 542, 503, 492, 513, 525, 492, 502, 576, 472, 505, 474, 499, 524, 540, 521, 482, 482, 511, 496, 498, 550, 541, 498, 475, 511, 506, 479, 491, 511, 502, 501, 523, 548, 533, 553, 556, 511, 487, 460, 543, 500, 478, 533, 504, 510, 530, 518, 541, 527, 512, 539, 519, 519, 507, 505, 507] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[550, 522, 506, 499, 510, 488, 473, 501, 497, 540, 479, 555, 522, 538, 476, 528, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 466, 518, 498, 495, 534, 533, 523, 510, 548, 535, 465, 491, 516, 497, 479, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 505, 519, 526, 491, 506, 530, 485, 510, 501, 497, 532, 547, 515, 506, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 478, 511, 524, 552, 505, 530, 514, 493, 500, 506, 520, 515, 526, 493, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 514, 514, 515, 500, 502, 534, 522, 520, 517, 515, 501, 516, 518, 505, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 549, 508, 529, 521, 524, 502, 491, 551, 551, 520, 528, 512, 531, 491, 511, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 526, 483, 529, 497, 531, 511, 485, 524, 495, 506, 565, 500, 494, 508, 494, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 453, 490, 500, 475, 493, 496, 518, 507, 560, 480, 509, 532, 554, 490, 526, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8106, 8146, 8202, 8196, 8393, 8155, 8128] avg=8192.0000 max=8393 max_rank=5 min=8106 min_rank=1 imbalance_ratio=1.024536 max_load_reduction=57143 max_load_reduction_pct=0.871933 moved_to_echo=32872 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[492, 464, 499, 507, 497, 510, 484, 518, 552, 515, 495, 544, 545, 527, 526, 553, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 487, 526, 540, 549, 496, 489, 474, 510, 539, 506, 515, 491, 517, 523, 540, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 497, 507, 530, 552, 493, 519, 508, 533, 539, 515, 521, 528, 525, 523, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 501, 518, 495, 516, 485, 529, 557, 524, 526, 464, 469, 491, 518, 582, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 526, 550, 491, 479, 502, 525, 552, 511, 526, 495, 537, 513, 514, 518, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 474, 491, 528, 526, 468, 521, 481, 527, 492, 484, 534, 513, 504, 497, 513, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 470, 524, 495, 486, 519, 491, 504, 481, 533, 521, 492, 517, 545, 512, 522, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 508, 506, 494, 499, 542, 479, 489, 544, 529, 517, 507, 506, 527, 484, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8253, 8222, 8315, 8161, 8262, 8075, 8099, 8149] avg=8192.0000 max=8315 max_rank=2 min=8075 min_rank=5 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32585 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[537, 517, 490, 502, 509, 516, 499, 532, 499, 478, 510, 507, 520, 541, 504, 503, 31, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 511, 476, 506, 488, 537, 487, 513, 513, 515, 531, 504, 491, 506, 532, 500, 13, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 532, 530, 523, 514, 539, 482, 519, 496, 519, 505, 451, 497, 518, 483, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 530, 501, 549, 517, 468, 533, 514, 538, 534, 511, 522, 517, 496, 496, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 524, 526, 529, 512, 529, 487, 493, 542, 503, 492, 513, 525, 492, 502, 576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 505, 474, 499, 524, 540, 521, 482, 482, 511, 496, 498, 550, 541, 498, 475, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 506, 479, 491, 511, 502, 501, 523, 548, 533, 553, 556, 511, 487, 460, 543, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 478, 533, 504, 510, 530, 518, 541, 527, 512, 516, 519, 519, 507, 505, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8127, 8120, 8283, 8263, 8087, 8230, 8226] avg=8192.0000 max=8283 max_rank=3 min=8087 min_rank=5 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32806 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[514, 480, 525, 563, 485, 529, 525, 499, 519, 518, 534, 510, 512, 505, 529, 502, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 521, 518, 534, 524, 549, 535, 497, 506, 529, 496, 537, 483, 531, 509, 502, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 511, 515, 512, 537, 519, 505, 494, 542, 500, 494, 536, 481, 498, 467, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 504, 503, 501, 532, 492, 486, 522, 490, 500, 507, 554, 531, 516, 529, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 477, 511, 503, 493, 468, 524, 511, 481, 524, 546, 524, 521, 554, 536, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 528, 490, 536, 527, 501, 529, 516, 511, 512, 535, 477, 509, 502, 494, 516, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 524, 465, 516, 490, 509, 500, 527, 531, 498, 522, 490, 526, 519, 516, 531, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 486, 504, 487, 475, 526, 488, 525, 525, 463, 524, 519, 515, 510, 529, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8275, 8303, 8139, 8209, 8141, 8224, 8159, 8086] avg=8192.0000 max=8303 max_rank=1 min=8086 min_rank=7 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32610 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[497, 479, 500, 499, 523, 522, 532, 492, 527, 500, 561, 478, 541, 494, 534, 496, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 521, 479, 528, 523, 508, 521, 523, 514, 513, 520, 516, 513, 491, 489, 486, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 521, 533, 526, 524, 513, 509, 522, 511, 564, 496, 514, 513, 509, 543, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 512, 530, 456, 516, 508, 538, 513, 490, 500, 506, 521, 483, 495, 511, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 476, 469, 560, 473, 511, 516, 523, 495, 481, 536, 504, 523, 497, 543, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 480, 546, 520, 496, 544, 506, 535, 536, 508, 520, 467, 487, 488, 462, 503, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 519, 525, 489, 505, 513, 544, 493, 512, 514, 538, 520, 510, 547, 473, 508, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 528, 525, 564, 492, 491, 495, 505, 509, 529, 502, 526, 531, 517, 522, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8157, 8311, 8117, 8147, 8126, 8212, 8266] avg=8192.0000 max=8311 max_rank=2 min=8117 min_rank=3 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32751 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[489, 461, 530, 517, 506, 481, 491, 535, 516, 474, 528, 512, 508, 533, 519, 522, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 515, 520, 512, 525, 498, 540, 532, 504, 491, 507, 533, 535, 512, 473, 527, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 480, 530, 516, 527, 482, 514, 479, 510, 498, 555, 526, 491, 515, 502, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 527, 512, 494, 501, 509, 521, 511, 485, 496, 505, 541, 514, 532, 526, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 476, 537, 525, 504, 484, 499, 493, 535, 509, 508, 498, 502, 525, 529, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 487, 525, 487, 507, 481, 511, 519, 526, 527, 554, 495, 539, 505, 500, 522, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 525, 514, 476, 480, 535, 505, 504, 479, 520, 564, 555, 487, 492, 517, 510, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 530, 507, 532, 520, 522, 487, 492, 481, 533, 521, 539, 534, 528, 515, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8148, 8200, 8125, 8231, 8180, 8208, 8194, 8250] avg=8192.0000 max=8250 max_rank=7 min=8125 min_rank=2 imbalance_ratio=1.007080 max_load_reduction=57286 max_load_reduction_pct=0.874115 moved_to_echo=32832 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[484, 469, 519, 521, 494, 509, 495, 481, 499, 502, 520, 516, 513, 520, 523, 534, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 526, 521, 543, 487, 537, 502, 473, 543, 488, 542, 508, 513, 513, 527, 502, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 531, 472, 543, 542, 494, 549, 523, 560, 510, 483, 478, 534, 560, 480, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 496, 511, 513, 505, 529, 498, 498, 512, 512, 508, 473, 507, 524, 500, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 495, 478, 483, 516, 479, 512, 490, 538, 531, 498, 517, 561, 559, 510, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 492, 508, 535, 528, 525, 502, 504, 539, 527, 505, 461, 539, 513, 524, 513, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 523, 533, 507, 508, 517, 489, 497, 492, 504, 518, 533, 503, 521, 537, 498, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 526, 520, 519, 547, 534, 486, 499, 502, 466, 503, 511, 510, 479, 549, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8126, 8237, 8248, 8111, 8188, 8222, 8236, 8168] avg=8192.0000 max=8248 max_rank=2 min=8111 min_rank=3 imbalance_ratio=1.006836 max_load_reduction=57288 max_load_reduction_pct=0.874146 moved_to_echo=32814 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 -WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[514, 480, 525, 563, 485, 529, 525, 499, 519, 518, 534, 510, 512, 505, 529, 502, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 521, 518, 534, 524, 549, 535, 497, 506, 529, 496, 537, 483, 531, 509, 502, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 511, 515, 512, 537, 519, 505, 494, 542, 500, 494, 536, 481, 498, 467, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 504, 503, 501, 532, 492, 486, 522, 490, 500, 507, 554, 531, 516, 529, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 477, 511, 503, 493, 468, 524, 511, 481, 524, 546, 524, 521, 554, 536, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 528, 490, 536, 527, 501, 529, 516, 511, 512, 535, 477, 509, 502, 494, 516, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 524, 465, 516, 490, 509, 500, 527, 531, 498, 522, 490, 526, 519, 516, 531, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 486, 504, 487, 475, 526, 488, 525, 525, 463, 524, 519, 515, 510, 529, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8275, 8303, 8139, 8209, 8141, 8224, 8159, 8086] avg=8192.0000 max=8303 max_rank=1 min=8086 min_rank=7 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32610 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[489, 461, 530, 517, 506, 481, 491, 535, 516, 474, 528, 512, 508, 533, 519, 522, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 515, 520, 512, 525, 498, 540, 532, 504, 491, 507, 533, 535, 512, 473, 527, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 480, 530, 516, 527, 482, 514, 479, 510, 498, 555, 526, 491, 515, 502, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 527, 512, 494, 501, 509, 521, 511, 485, 496, 505, 541, 514, 532, 526, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 476, 537, 525, 504, 484, 499, 493, 535, 509, 508, 498, 502, 525, 529, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 487, 525, 487, 507, 481, 511, 519, 526, 527, 554, 495, 539, 505, 500, 522, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 525, 514, 476, 480, 535, 505, 504, 479, 520, 564, 555, 487, 492, 517, 510, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 530, 507, 532, 520, 522, 487, 492, 481, 533, 521, 539, 534, 528, 515, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8148, 8200, 8125, 8231, 8180, 8208, 8194, 8250] avg=8192.0000 max=8250 max_rank=7 min=8125 min_rank=2 imbalance_ratio=1.007080 max_load_reduction=57286 max_load_reduction_pct=0.874115 moved_to_echo=32832 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[521, 471, 534, 488, 486, 504, 496, 522, 508, 522, 486, 527, 541, 483, 519, 490, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 515, 496, 511, 507, 517, 540, 488, 502, 515, 532, 541, 480, 518, 476, 516, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 542, 505, 473, 541, 475, 554, 474, 542, 525, 492, 495, 475, 516, 500, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 508, 514, 510, 529, 537, 544, 543, 460, 502, 541, 518, 535, 468, 489, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 452, 530, 513, 517, 506, 520, 501, 548, 549, 470, 494, 506, 519, 504, 504, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 487, 492, 524, 513, 478, 497, 524, 530, 532, 524, 521, 538, 488, 512, 520, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 547, 502, 524, 521, 499, 561, 478, 496, 525, 525, 490, 459, 517, 548, 551, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 492, 528, 556, 513, 515, 523, 537, 495, 531, 530, 494, 496, 503, 555, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8124, 8184, 8132, 8222, 8159, 8201, 8251, 8263] avg=8192.0000 max=8263 max_rank=7 min=8124 min_rank=0 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32874 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[492, 464, 499, 507, 497, 510, 484, 518, 552, 515, 495, 544, 545, 527, 526, 553, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 487, 526, 540, 549, 496, 489, 474, 510, 539, 506, 515, 491, 517, 523, 540, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 497, 507, 530, 552, 493, 519, 508, 533, 539, 515, 521, 528, 525, 523, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 501, 518, 495, 516, 485, 529, 557, 524, 526, 464, 469, 491, 518, 582, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 526, 550, 491, 479, 502, 525, 552, 511, 526, 495, 537, 513, 514, 518, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 474, 491, 528, 526, 468, 521, 481, 527, 492, 484, 534, 513, 504, 497, 513, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 470, 524, 495, 486, 519, 491, 504, 481, 533, 521, 492, 517, 545, 512, 522, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 508, 506, 494, 499, 542, 479, 489, 544, 529, 517, 507, 506, 527, 484, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8253, 8222, 8315, 8161, 8262, 8075, 8099, 8149] avg=8192.0000 max=8315 max_rank=2 min=8075 min_rank=5 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32585 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 3968, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=8 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 3968, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66944 observed_loss=-1408 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 4352, 4224, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4224, 4352, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4224, 4224, 4352, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66944 slack_slots=64128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True -WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True - [2026-08-26 14:49:31] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 421.4 | throughput per GPU (TFLOP/s/GPU): 429.1 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.167898E+00 | loss scale: 1.0 | grad norm: 5.244 | number of skipped iterations: 0 | number of nan iterations: 0 | -[after training is done] datetime: 2026-08-26 14:49:31 -W0826 14:49:32.518000 16295 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers -W0826 14:49:32.519000 16295 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 16389 closing signal SIGTERM -W0826 14:49:32.519000 16295 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 16390 closing signal SIGTERM -W0826 14:49:32.520000 16295 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 16391 closing signal SIGTERM -W0826 14:49:32.521000 16295 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 16392 closing signal SIGTERM -W0826 14:49:32.522000 16295 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 16393 closing signal SIGTERM -W0826 14:49:32.523000 16295 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 16394 closing signal SIGTERM -W0826 14:49:32.523000 16295 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 16395 closing signal SIGTERM -W0826 14:49:32.524000 16295 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 16396 closing signal SIGTERM + [2026-08-26 15:25:34] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 416.9 | throughput per GPU (TFLOP/s/GPU): 433.8 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.177896E+00 | loss scale: 1.0 | grad norm: 5.214 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-26 15:25:34 +W0826 15:25:35.351000 83333 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 15:25:35.352000 83333 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 83427 closing signal SIGTERM +W0826 15:25:35.353000 83333 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 83428 closing signal SIGTERM +W0826 15:25:35.354000 83333 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 83429 closing signal SIGTERM +W0826 15:25:35.356000 83333 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 83430 closing signal SIGTERM +W0826 15:25:35.357000 83333 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 83431 closing signal SIGTERM +W0826 15:25:35.357000 83333 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 83432 closing signal SIGTERM +W0826 15:25:35.358000 83333 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 83433 closing signal SIGTERM +W0826 15:25:35.360000 83333 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 83434 closing signal SIGTERM Traceback (most recent call last): File "/usr/local/bin/torchrun", line 8, in sys.exit(main()) @@ -7179,4 +7179,4 @@ Traceback (most recent call last): time.sleep(monitor_interval) File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 16295 got signal: 15 +torch.distributed.elastic.multiprocessing.api.SignalException: Process 83333 got signal: 15 diff --git a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1/mem_samples.csv b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1/mem_samples.csv index 2a596625cfbc2a4bb7dbf2e63ec73fefed1f556b..7e94139ef74b7cd6d335d4fd3c2cee08ebfb3833 100644 --- a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1/mem_samples.csv +++ b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1/mem_samples.csv @@ -14,7 +14,7 @@ 5, 0 6, 0 7, 0 -0, 1 +0, 2 1, 1 2, 1 3, 1 @@ -49,7 +49,7 @@ 0, 821 1, 821 2, 821 -3, 819 +3, 821 4, 821 5, 821 6, 821 @@ -62,222 +62,206 @@ 5, 1537 6, 1537 7, 1537 -0, 2135 -1, 2135 -2, 2135 -3, 2135 -4, 2135 -5, 2135 -6, 2135 -7, 2135 -0, 38993 -1, 38993 -2, 38993 -3, 38993 -4, 38993 -5, 38993 -6, 38993 -7, 38993 -0, 44871 -1, 44881 -2, 44859 -3, 44829 -4, 44833 -5, 44879 -6, 44849 -7, 44879 -0, 45505 -1, 45509 -2, 45521 -3, 45481 -4, 45495 -5, 45511 -6, 45515 -7, 45511 -0, 47045 -1, 47047 -2, 47057 -3, 47019 -4, 47043 -5, 47057 -6, 47047 -7, 47057 -0, 47141 -1, 47143 -2, 47153 -3, 47115 -4, 47139 -5, 47153 -6, 47143 -7, 47153 -0, 47189 -1, 47191 -2, 47201 -3, 47115 -4, 47139 -5, 47153 -6, 47191 -7, 47201 -0, 48005 -1, 48007 -2, 48017 -3, 47979 -4, 48003 -5, 48017 -6, 48007 -7, 48017 -0, 48005 -1, 48007 -2, 48017 -3, 47979 -4, 48003 -5, 48017 -6, 48007 -7, 48017 -0, 48261 -1, 48263 -2, 48273 -3, 48235 -4, 48259 -5, 48273 -6, 48263 -7, 48273 -0, 49163 -1, 49183 -2, 49177 -3, 49157 -4, 49159 -5, 49177 -6, 49149 -7, 49177 -0, 59521 -1, 59555 -2, 59549 -3, 59547 -4, 59539 -5, 59529 -6, 59549 -7, 59549 -0, 100847 -1, 100833 -2, 100823 -3, 100867 -4, 100819 -5, 100811 -6, 100801 -7, 100833 -0, 101359 -1, 101373 -2, 101365 -3, 101365 -4, 101377 -5, 101365 -6, 101365 -7, 101365 -0, 101361 -1, 101375 -2, 101367 -3, 101367 -4, 101379 -5, 101367 -6, 101367 -7, 101367 -0, 101361 -1, 101375 -2, 101367 -3, 101367 -4, 101379 -5, 101367 -6, 101367 -7, 101367 -0, 101361 -1, 101375 -2, 101367 -3, 101367 -4, 101379 -5, 101367 -6, 101367 -7, 101367 -0, 101995 -1, 101945 -2, 101969 -3, 102055 -4, 101937 -5, 101977 -6, 102063 -7, 101985 -0, 102733 -1, 102513 -2, 102713 -3, 103041 -4, 102253 -5, 102247 -6, 103053 -7, 102537 -0, 103593 -1, 103419 -2, 103599 -3, 103885 -4, 103129 -5, 103117 -6, 103911 -7, 103415 -0, 133217 -1, 133057 -2, 133243 -3, 133531 -4, 132805 -5, 132763 -6, 133511 -7, 133051 -0, 133217 -1, 133057 -2, 133243 -3, 133531 -4, 132805 -5, 132763 -6, 133511 -7, 133051 -0, 133217 -1, 133057 -2, 133243 -3, 133531 -4, 132805 -5, 132763 -6, 133511 -7, 133051 -0, 133217 -1, 133057 -2, 133243 -3, 133531 -4, 132805 -5, 132763 -6, 133511 -7, 133051 -0, 133217 -1, 133057 -2, 133243 -3, 133531 -4, 132805 -5, 132763 -6, 133511 -7, 133051 -0, 133217 -1, 133057 -2, 133243 -3, 133531 -4, 132805 -5, 132763 -6, 133511 -7, 133051 -0, 2442 -1, 132241 -2, 2442 -3, 132715 -4, 131989 -5, 131947 -6, 132695 -7, 2186 +0, 1981 +1, 1981 +2, 1981 +3, 1981 +4, 1981 +5, 1981 +6, 1981 +7, 1981 +0, 44781 +1, 44781 +2, 44791 +3, 44771 +4, 44825 +5, 44825 +6, 44801 +7, 44779 +0, 45895 +1, 45897 +2, 45907 +3, 45869 +4, 45893 +5, 45907 +6, 45897 +7, 45907 +0, 46917 +1, 46919 +2, 46929 +3, 46891 +4, 46915 +5, 46929 +6, 46919 +7, 46929 +0, 47013 +1, 47015 +2, 47025 +3, 46987 +4, 47011 +5, 47025 +6, 47015 +7, 47025 +0, 47155 +1, 47169 +2, 47177 +3, 47135 +4, 47175 +5, 47185 +6, 47193 +7, 47183 +0, 47877 +1, 47879 +2, 47889 +3, 47851 +4, 47875 +5, 47889 +6, 47879 +7, 47889 +0, 48133 +1, 48135 +2, 48145 +3, 48107 +4, 48131 +5, 48145 +6, 48135 +7, 48145 +0, 48777 +1, 48799 +2, 48793 +3, 48771 +4, 48775 +5, 48793 +6, 48763 +7, 48793 +0, 49131 +1, 49151 +2, 49145 +3, 49125 +4, 49127 +5, 49145 +6, 49117 +7, 49145 +0, 99847 +1, 99869 +2, 99853 +3, 99861 +4, 100651 +5, 99853 +6, 99853 +7, 99853 +0, 101347 +1, 101369 +2, 101353 +3, 101361 +4, 101365 +5, 101353 +6, 101353 +7, 101353 +0, 101347 +1, 101369 +2, 101353 +3, 101361 +4, 101365 +5, 101353 +6, 101353 +7, 101353 +0, 101349 +1, 101371 +2, 101355 +3, 101363 +4, 101367 +5, 101355 +6, 101355 +7, 101355 +0, 101349 +1, 101371 +2, 101355 +3, 101363 +4, 101367 +5, 101355 +6, 101355 +7, 101355 +0, 101901 +1, 101921 +2, 101875 +3, 101933 +4, 101909 +5, 101933 +6, 101933 +7, 101963 +0, 102901 +1, 103019 +2, 103003 +3, 103011 +4, 102919 +5, 103003 +6, 102715 +7, 103003 +0, 103459 +1, 103637 +2, 103573 +3, 103647 +4, 103505 +5, 103567 +6, 103295 +7, 103555 +0, 132949 +1, 133053 +2, 133059 +3, 133047 +4, 132993 +5, 133059 +6, 132751 +7, 133059 +0, 132949 +1, 133053 +2, 133059 +3, 133047 +4, 132993 +5, 133059 +6, 132751 +7, 133059 +0, 132949 +1, 133053 +2, 133059 +3, 133047 +4, 132993 +5, 133059 +6, 132751 +7, 133059 +0, 132949 +1, 133053 +2, 133059 +3, 133047 +4, 132993 +5, 133059 +6, 132751 +7, 133059 +0, 132949 +1, 133053 +2, 133059 +3, 133047 +4, 132993 +5, 133059 +6, 132751 +7, 133059 +0, 132949 +1, 133053 +2, 133059 +3, 133047 +4, 132993 +5, 133059 +6, 132751 +7, 133059 +0, 2186 +1, 906 +2, 906 +3, 906 +4, 906 +5, 906 +6, 1162 +7, 132243 0, 0 1, 0 2, 0 diff --git a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1/train.log b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1/train.log index 70836691988f3a5c05a3371c2f963bafdfe30a01..6ff450516aa9bd85567e7db6df72150aaa0640b9 100644 --- a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1/train.log +++ b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1/train.log @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:72de395920622e828fb1cbfbff6311c5a16622a7080dfaaac31c06a44b16cfd8 -size 13018324 +oid sha256:7f7e0b6dc40caf86b4c035ca7a7c67cb5d4cf44da64e840fa8612ea38296857b +size 13019657 diff --git a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1/mem_samples.csv b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1/mem_samples.csv index f3b2009a18bf57f566d1067630cca93bc5ba5cd2..51b0de22b30151f6bffc364a06f70b7a9f07848d 100644 --- a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1/mem_samples.csv +++ b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1/mem_samples.csv @@ -39,21 +39,21 @@ 6, 4 7, 4 0, 4 -1, 4 -2, 4 -3, 4 -4, 45 -5, 4 -6, 4 -7, 4 -0, 821 -1, 821 -2, 821 -3, 821 -4, 821 -5, 821 -6, 821 -7, 821 +1, 629 +2, 629 +3, 629 +4, 629 +5, 629 +6, 629 +7, 629 +0, 929 +1, 937 +2, 917 +3, 935 +4, 931 +5, 935 +6, 929 +7, 931 0, 1831 1, 1831 2, 1831 @@ -64,24 +64,24 @@ 7, 1831 0, 39169 1, 38995 -2, 39197 -3, 38995 +2, 39209 +3, 41603 4, 38995 5, 39197 6, 38995 7, 39197 -0, 44953 -1, 44955 -2, 44965 -3, 44927 -4, 44951 -5, 44965 -6, 44955 -7, 44965 +0, 44989 +1, 44991 +2, 45001 +3, 44963 +4, 44987 +5, 45001 +6, 44991 +7, 45001 0, 45895 1, 45897 2, 45907 -3, 45871 +3, 45869 4, 45893 5, 45907 6, 45897 @@ -102,14 +102,14 @@ 5, 47025 6, 47015 7, 47025 -0, 47199 -1, 47199 -2, 47197 -3, 47159 -4, 47163 -5, 47223 -6, 47193 -7, 47193 +0, 47235 +1, 47243 +2, 47251 +3, 47215 +4, 47231 +5, 47251 +6, 47229 +7, 47237 0, 47877 1, 47879 2, 47889 @@ -142,14 +142,14 @@ 5, 49145 6, 49117 7, 49145 -0, 100633 -1, 100655 +0, 100699 +1, 100719 2, 100639 -3, 100647 -4, 100651 -5, 100639 -6, 100639 -7, 100639 +3, 100713 +4, 100715 +5, 100705 +6, 100703 +7, 100703 0, 101347 1, 101369 2, 101353 @@ -182,83 +182,83 @@ 5, 101355 6, 101355 7, 101355 -0, 101937 -1, 101989 -2, 101987 -3, 101949 -4, 101933 -5, 101903 -6, 101951 -7, 101957 -0, 102901 -1, 102731 -2, 102715 -3, 102723 -4, 102727 -5, 102907 -6, 102907 +0, 101947 +1, 101939 +2, 101939 +3, 101927 +4, 101931 +5, 101895 +6, 101997 +7, 101907 +0, 103477 +1, 103499 +2, 103483 +3, 103491 +4, 103495 +5, 102715 +6, 103483 7, 102715 -0, 103519 -1, 103311 -2, 103321 -3, 103339 -4, 103301 -5, 103527 -6, 103519 -7, 103343 -0, 132949 -1, 132765 -2, 132771 -3, 132759 -4, 132801 -5, 132963 -6, 132943 +0, 104123 +1, 104139 +2, 104131 +3, 104199 +4, 104171 +5, 103365 +6, 104125 +7, 103345 +0, 133525 +1, 133533 +2, 133539 +3, 133527 +4, 133569 +5, 132771 +6, 133519 7, 132771 -0, 132949 -1, 132765 -2, 132771 -3, 132759 -4, 132801 -5, 132963 -6, 132943 +0, 133525 +1, 133533 +2, 133539 +3, 133527 +4, 133569 +5, 132771 +6, 133519 7, 132771 -0, 132949 -1, 132765 -2, 132771 -3, 132759 -4, 132801 -5, 132963 -6, 132943 +0, 133525 +1, 133533 +2, 133539 +3, 133527 +4, 133569 +5, 132771 +6, 133519 7, 132771 -0, 132949 -1, 132765 -2, 132771 -3, 132759 -4, 132801 -5, 132963 -6, 132943 +0, 133525 +1, 133533 +2, 133539 +3, 133527 +4, 133569 +5, 132771 +6, 133519 7, 132771 -0, 132949 -1, 132765 -2, 132771 -3, 132759 -4, 132801 -5, 132963 -6, 132943 +0, 133525 +1, 133533 +2, 133539 +3, 133527 +4, 133569 +5, 132771 +6, 133519 7, 132771 -0, 132453 -1, 131949 -2, 131955 -3, 132007 -4, 132631 -5, 132467 -6, 132191 +0, 133355 +1, 133363 +2, 133369 +3, 133357 +4, 133399 +5, 132601 +6, 133349 7, 132601 -0, 2186 -1, 906 -2, 906 -3, 906 +0, 906 +1, 1162 +2, 132723 +3, 2186 4, 906 5, 906 -6, 1162 -7, 131955 +6, 906 +7, 906 diff --git a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1/train.log b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1/train.log index 7bd5c247594ade17d459261e851dbe54ce2a480b..ac444a89a5d5585898466bd1854c78e7f50a92df 100644 --- a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1/train.log +++ b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1/train.log @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ef09afeb18cd633009a4746f61bd2eacf3b7254d528f518008ca8edc3468bc83 -size 15163277 +oid sha256:f9f46b4b8021d619b97955997079ccf2617f88e4f8a1a8c42f3da1c8ec280ea3 +size 15163531 diff --git a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv index cab80b3e0b505667f8e39074574c08e9812d9317..642ab923bd86201a32093d409b42653941ff9dec 100644 --- a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv +++ b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv @@ -1,9 +1,5 @@ model backend purpose capacity status peak_mib zero_drop drop_rate_pct required_capacity run_dir QWEN3_235B echo natural 2.00 OK 141943 1 0.0 1.0 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1 -QWEN3_235B echo forced 2.00 OK 141555 1 0.0 1.0 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00__forcebal/rep1 -QWEN3_30B echo natural 2.00 OK 133531 1 0.0 1.0 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1 -QWEN3_30B echo forced 2.00 OK 132963 1 0.0 1.0 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1 -DSV2_LITE_L24 echo natural 2.00 OK 140961 1 0.0 1.0 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1 -DSV2_LITE_L24 echo forced 2.00 OK 140541 1 0.0 1.0 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00__forcebal/rep1 -QWEN2_57B echo natural 2.00 OK 108303 1 0.0 1.0 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap2p00/rep1 -QWEN2_57B echo forced 2.00 OK 108287 1 0.0 1.0 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1 +QWEN3_235B echo forced 2.00 OK 141939 1 0.0 1.0 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00__forcebal/rep1 +QWEN3_30B echo natural 2.00 OK 133059 1 0.0 1.0 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1 +QWEN3_30B echo forced 2.00 OK 133569 1 0.0 1.0 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1 diff --git a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv index 46b5e16a387e211a42ba27c065f15e2015290151..c2c482e2160cc7cad2db3d799061e2175326f9af 100644 --- a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv +++ b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv @@ -1,5 +1,3 @@ model backend status selected_capacity required_capacity drop_rate_pct peak_mib run_dir QWEN3_235B echo SELECTED 2.00 1.0 0.0 141943 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1 -QWEN3_30B echo SELECTED 2.00 1.0 0.0 133531 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1 -DSV2_LITE_L24 echo SELECTED 2.00 1.0 0.0 140961 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1 -QWEN2_57B echo SELECTED 2.00 1.0 0.0 108303 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap2p00/rep1 +QWEN3_30B echo SELECTED 2.00 1.0 0.0 133059 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1 diff --git a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/results.tsv b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/results.tsv index fc867ced8c824d45d248c4efaf53575df0629615..ffd439dc551d1262dd9ce0f1234ce2948cd7635d 100644 --- a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/results.tsv +++ b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/results.tsv @@ -1,9 +1,5 @@ config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir -QWEN3_235B echo matched 16 8192 8 2.00 none 1 OK 437.21 436.00 141943 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1 -QWEN3_235B echo matched 16 8192 8 2.00 none+forcebal 1 OK 419.87 416.40 141555 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00__forcebal/rep1 -QWEN3_30B echo matched 16 4096 8 2.00 none 1 OK 990.49 942.90 133531 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1 -QWEN3_30B echo matched 16 4096 8 2.00 none+forcebal 1 OK 954.76 910.80 132963 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1 -DSV2_LITE_L24 echo matched 8 8192 8 2.00 none 1 OK 826.93 818.00 140961 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1 -DSV2_LITE_L24 echo matched 8 8192 8 2.00 none+forcebal 1 OK 817.73 813.10 140541 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00__forcebal/rep1 -QWEN2_57B echo matched 8 4096 8 2.00 none 1 OK 281.00 277.80 108303 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap2p00/rep1 -QWEN2_57B echo matched 8 4096 8 2.00 none+forcebal 1 OK 281.80 277.00 108287 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1 +QWEN3_235B echo matched 16 8192 8 2.00 none 1 OK 448.09 440.90 141943 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1 +QWEN3_235B echo matched 16 8192 8 2.00 none+forcebal 1 OK 418.59 413.80 141939 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00__forcebal/rep1 +QWEN3_30B echo matched 16 4096 8 2.00 none 1 OK 985.37 944.00 133059 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1 +QWEN3_30B echo matched 16 4096 8 2.00 none+forcebal 1 OK 971.26 930.30 133569 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1 diff --git a/tierb_study_20260826_144644/hf_watcher.log b/tierb_study_20260826_144644/hf_watcher.log index 9b0234e16604817cbbf549dad01fd0c343df2297..9cf11ac4d26752088bfeab37afa28c33db472b66 100644 --- a/tierb_study_20260826_144644/hf_watcher.log +++ b/tierb_study_20260826_144644/hf_watcher.log @@ -12,3 +12,8 @@ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main Already logged in to Hugging Face as: AbhishekGhosh1998 Watching /workspace/tierb_study_20260826_144644 -> HF every 600s (kill to stop) [2026-08-26T15:22:49+00:00] backup pass + uploaded: /workspace/tierb_study_20260826_144644 -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tierb_study_20260826_144644 +https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main +Already logged in to Hugging Face as: AbhishekGhosh1998 +Watching /workspace/tierb_study_20260826_144644 -> HF every 600s (kill to stop) +[2026-08-26T15:32:33+00:00] backup pass diff --git a/tierb_study_20260826_144644/hf_watcher.pid b/tierb_study_20260826_144644/hf_watcher.pid index 3b4c068b9000506b27d823e51bcadee894a9f0f4..fc665fbeaa06937a8aa2dcc02d1a073bbe4882fc 100644 --- a/tierb_study_20260826_144644/hf_watcher.pid +++ b/tierb_study_20260826_144644/hf_watcher.pid @@ -1 +1 @@ -79542 +99772 diff --git a/tierb_study_20260826_144644/study.log b/tierb_study_20260826_144644/study.log index f42a0a7e38a22cdc331d37f9980a38cbbea4b0d8..1be6a512b10b7289b8f4967892449b7e5e977493 100644 --- a/tierb_study_20260826_144644/study.log +++ b/tierb_study_20260826_144644/study.log @@ -426,3 +426,41 @@ driver_repo: /workspace/blackwell_run_scripts @ 7ec1185 === STAGE 7 -- Tier B probe and zero-drop capacity selection === STEP echo__tierbprobe__gbs8 +============================================================== + CAMPAIGN: echo backend=echo phase=tierbprobe + Megatron: /workspace/Megatron-MoE-Echo @ token-drop-analysis + GBS=8 seq=4096 repeats=3 nsys=1 + results -> /workspace/tierb_study_20260826_144644/echo/tierbprobe +============================================================== +-------------------------------------------------------------------------- +[15:23:04] CELL QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00 (slots=16) + [guard] iter 10/12 (70s) + [guard] iter 12/12 (80s) +-------------------------------------------------------------------------- +[15:24:25] CELL QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00__forcebal (slots=16) + [guard] iter 12/12 (70s) + [guard] tearing down process group (outcome=DONE) +-------------------------------------------------------------------------- +[15:25:41] CELL QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00 (slots=16) + [guard] iter 5/12 (70s) + [guard] iter 12/12 (80s) + [guard] tearing down process group (outcome=DONE) +-------------------------------------------------------------------------- +[15:27:06] CELL QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00__forcebal (slots=16) + [guard] iter 5/12 (70s) + [guard] iter 12/12 (80s) + [guard] tearing down process group (outcome=DONE) +-------------------------------------------------------------------------- +[15:28:32] CELL DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00 (slots=8) +study_root: /workspace/tierb_study_20260826_144644 +gbs: 8 +stages: 7,4,8,9 +nsys: 1 +hf_repo: MoE-CUDA-Graph-Analysis/tierb-b200-gbs8 +hf_remote_root: tierb_study_20260826_144644 +started: 2026-08-26T15:32:33+00:00 +driver_repo: /workspace/blackwell_run_scripts @ d11391f + +=== STAGE 7 -- Tier B probe and zero-drop capacity selection + +=== STEP echo__tierbprobe__gbs8